From c4a1f2ca030c10c05e9fc9f6764cae13b3eed2e2 Mon Sep 17 00:00:00 2001 From: Matt Burke Date: Tue, 30 May 2017 12:32:32 -0400 Subject: [PATCH] mochilo v1 == bert v3, mochilo v2 == bert v4 --- Gemfile.lock | 2 +- bench/bench.rb | 6 +++++- ext/bert/c/decode.c | 26 ++++++++++++++++++++++---- lib/bert/bert.rb | 15 +++++++++++++++ lib/bert/decode.rb | 16 +++++----------- lib/bert/encode.rb | 41 +++++++++++++++++++++++++++++++++-------- lib/bert/encoder.rb | 2 +- lib/bert/types.rb | 1 + test/bert_test.rb | 35 +++++++++++++++++++++++++++++++---- 9 files changed, 114 insertions(+), 30 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 60841dc..0c485c2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/spraints/mochilo - revision: 1bf415c4e81edbe72c0aab5404598dec9bef41b9 + revision: 73fcaa99f01c42ad6017b201b0aeeca5f25a7a53 ref: symbols-and-regexps-and-time-oh-my specs: mochilo (2.0) diff --git a/bench/bench.rb b/bench/bench.rb index e8c7950..d9c17db 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -16,7 +16,11 @@ complex = [42, {:foo => 'bac' * 100}, t[(1..100).to_a]] * 10 long_array = {:a => ["a", :a, Time.now, /a/]*1000} Benchmark.bm(30) do |bench| - [:v1, :v2, :v3].each do |v| + [:v1, :v2, :v3, :v4].each do |v| + unless BERT.supports?(v) + puts "SKIP #{v} (unsupported)" + next + end BERT::Encode.version = v bench.report("BERT #{v} tiny") {ITER.times {BERT.decode(BERT.encode(tiny))}} bench.report("BERT #{v} small") {ITER.times {BERT.decode(BERT.encode(small))}} diff --git a/ext/bert/c/decode.c b/ext/bert/c/decode.c index d3424f7..01a032a 100644 --- a/ext/bert/c/decode.c +++ b/ext/bert/c/decode.c @@ -24,7 +24,8 @@ /* Protocol version constants. */ #define ERL_VERSION 131 #define ERL_VERSION2 132 -#define MSGPACK_VERSION 133 +#define MOCHILO_VERSION1 133 +#define MOCHILO_VERSION2 134 #define BERT_VALID_TYPE(t) ((t) >= ERL_SMALL_INT && (t) <= ERLEXT_UNICODE_STRING) #define BERT_TYPE_OFFSET (ERL_SMALL_INT) @@ -33,6 +34,7 @@ static VALUE rb_mBERT; static VALUE rb_cDecode; static VALUE rb_cTuple; static VALUE rb_cMochilo; +static VALUE id_unpack_unsafe; static VALUE id_unpack; struct bert_buf { @@ -510,6 +512,11 @@ static VALUE bert_read_invalid(struct bert_buf *buf) return Qnil; } +static int supports(const char *version) +{ + return RTEST(rb_funcall(rb_mBERT, rb_intern("supports?"), 1, ID2SYM(rb_intern(version)))); +} + static VALUE rb_bert_decode(VALUE klass, VALUE rb_string) { struct bert_buf buf; @@ -531,10 +538,20 @@ static VALUE rb_bert_decode(VALUE klass, VALUE rb_string) proto_version = bert_buf_read8(&buf); if (proto_version == ERL_VERSION || proto_version == ERL_VERSION2) { return bert_read(&buf); - } else if (proto_version == MSGPACK_VERSION) { - return rb_funcall(rb_cMochilo, id_unpack, 1, rb_str_new(str + 1, size - 1)); + } else if (proto_version == MOCHILO_VERSION1) { + if (supports("v3")) { + return rb_funcall(rb_cMochilo, id_unpack_unsafe, 1, rb_str_new(str + 1, size - 1)); + } else { + rb_raise(rb_eTypeError, "v3 stream cannot be decoded"); + } + } else if (proto_version == MOCHILO_VERSION2) { + if (supports("v4")) { + return rb_funcall(rb_cMochilo, id_unpack, 1, rb_str_new(str + 1, size - 1)); + } else { + rb_raise(rb_eTypeError, "v4 stream cannot be decoded"); + } } else { - rb_raise(rb_eTypeError, "Invalid magic value for BERT string"); + rb_raise(rb_eTypeError, "Invalid magic value (%d) for BERT string", proto_version); } } @@ -550,6 +567,7 @@ void Init_decode() rb_require("mochilo"); rb_cMochilo = rb_const_get(rb_cObject, rb_intern("Mochilo")); + id_unpack_unsafe = rb_intern("unpack_unsafe"); id_unpack = rb_intern("unpack"); rb_cDecode = rb_define_class_under(rb_mBERT, "Decode", rb_cObject); diff --git a/lib/bert/bert.rb b/lib/bert/bert.rb index 9c860bc..5661287 100644 --- a/lib/bert/bert.rb +++ b/lib/bert/bert.rb @@ -1,4 +1,19 @@ +require "mochilo/version" + module BERT + def self.supports?(v) + case v + when :v1, :v2 + true + when :v3 + Mochilo.respond_to?(:pack_unsafe) + when :v4 + !Mochilo.respond_to?(:pack_unsafe) + else + false + end + end + def self.encode(ruby) Encoder.encode(ruby) end diff --git a/lib/bert/decode.rb b/lib/bert/decode.rb index b1cec74..d7fb340 100644 --- a/lib/bert/decode.rb +++ b/lib/bert/decode.rb @@ -14,8 +14,12 @@ module BERT io.set_encoding('binary') if io.respond_to?(:set_encoding) header = io.getbyte case header + when VERSION_4 + raise "v4 stream cannot be decoded" unless BERT.supports?(:v4) + Mochilo.unpack(io.read) when VERSION_3 - V3.new(io).read_any + raise "v3 stream cannot be decoded" unless BERT.supports?(:v3) + Mochilo.unpack_unsafe(io.read) when MAGIC, VERSION_2 new(io).read_any else @@ -23,16 +27,6 @@ module BERT end end - class V3 - def initialize(ins) - @ins = ins - end - - def read_any - Mochilo.unpack(@ins.read) - end - end - def initialize(ins) @in = ins @peeked = "" diff --git a/lib/bert/encode.rb b/lib/bert/encode.rb index 5c50845..7aad954 100644 --- a/lib/bert/encode.rb +++ b/lib/bert/encode.rb @@ -49,7 +49,7 @@ module BERT def write_any(obj) out.write(version_header.chr) - out.write(Mochilo.pack(obj)) + out.write(Mochilo.pack_unsafe(obj)) end private @@ -59,6 +59,25 @@ module BERT end end + class V4 + def initialize(out) + @out = out + end + + attr_reader :out + + def write_any(obj) + out.write(version_header.chr) + out.write(Mochilo.pack(obj)) + end + + private + + def version_header + BERT::Encode::VERSION_4 + end + end + class << self attr_accessor :version end @@ -103,13 +122,19 @@ module BERT end def self.encode_data(data, io) - if version == :v3 - Encode::V3.new(io).write_any(data) - elsif version == :v2 - Encode::V2.new(io).write_any(data) - else - new(io).write_any(data) - end + fail "Cannot encode with requested version (#{version})" unless BERT.supports?(version) + encoder = + case version + when :v4 + V4.new(io) + when :v3 + V3.new(io) + when :v2 + V2.new(io) + else + new(io) + end + encoder.write_any(data) end def write_any obj diff --git a/lib/bert/encoder.rb b/lib/bert/encoder.rb index 3e9bd0b..dbbde72 100644 --- a/lib/bert/encoder.rb +++ b/lib/bert/encoder.rb @@ -19,7 +19,7 @@ module BERT # # Returns the converted Ruby object def self.convert(item) - return item if Encode.version == :v3 + return item if Encode.version == :v3 || Encode.version == :v4 case item when Hash pairs = [] diff --git a/lib/bert/types.rb b/lib/bert/types.rb index c1d0120..04de952 100644 --- a/lib/bert/types.rb +++ b/lib/bert/types.rb @@ -18,6 +18,7 @@ module BERT MAGIC = 131 VERSION_2 = 132 VERSION_3 = 133 + VERSION_4 = 134 MAX_INT = (1 << 27) -1 MIN_INT = -(1 << 27) end diff --git a/test/bert_test.rb b/test/bert_test.rb index a6aa55a..a73d255 100644 --- a/test/bert_test.rb +++ b/test/bert_test.rb @@ -4,21 +4,27 @@ require 'test_helper' class BertTest < Test::Unit::TestCase context "BERT" do setup do - time = Time.at(1254976067) + time = Time.at(1254976067).utc @ruby = t[:user, {:name => 'TPW'}, [/cat/i, 9.9], time, nil, true, false, :true, :false] @bert_v1 = "\203h\td\000\004userh\003d\000\004bertd\000\004dictl\000\000\000\001h\002d\000\004namem\000\000\000\003TPWjl\000\000\000\002h\004d\000\004bertd\000\005regexm\000\000\000\003catl\000\000\000\001d\000\bcaselessjc9.900000000000000e+00\000\000\000\000\000\000\000\000\000\000jh\005d\000\004bertd\000\004timeb\000\000\004\346b\000\016\344\303a\000h\002d\000\004bertd\000\003nilh\002d\000\004bertd\000\004trueh\002d\000\004bertd\000\005falsed\000\004trued\000\005false".b @ebin_v1 = "<<131,104,9,100,0,4,117,115,101,114,104,3,100,0,4,98,101,114,116,100,0,4,100,105,99,116,108,0,0,0,1,104,2,100,0,4,110,97,109,101,109,0,0,0,3,84,80,87,106,108,0,0,0,2,104,4,100,0,4,98,101,114,116,100,0,5,114,101,103,101,120,109,0,0,0,3,99,97,116,108,0,0,0,1,100,0,8,99,97,115,101,108,101,115,115,106,99,57,46,57,48,48,48,48,48,48,48,48,48,48,48,48,48,48,101,43,48,48,0,0,0,0,0,0,0,0,0,0,106,104,5,100,0,4,98,101,114,116,100,0,4,116,105,109,101,98,0,0,4,230,98,0,14,228,195,97,0,104,2,100,0,4,98,101,114,116,100,0,3,110,105,108,104,2,100,0,4,98,101,114,116,100,0,4,116,114,117,101,104,2,100,0,4,98,101,114,116,100,0,5,102,97,108,115,101,100,0,4,116,114,117,101,100,0,5,102,97,108,115,101>>" @berts = { :v2 => "\x84h\td\x00\x04userh\x03d\x00\x04bertd\x00\x04dictl\x00\x00\x00\x01h\x02d\x00\x04nameq\x00\x00\x00\x03TPWjl\x00\x00\x00\x02h\x04d\x00\x04bertd\x00\x05regexq\x00\x00\x00\x03catl\x00\x00\x00\x01d\x00\bcaselessjc9.900000000000000e+00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00jh\x05d\x00\x04bertd\x00\x04timeb\x00\x00\x04\xE6b\x00\x0E\xE4\xC3a\x00h\x02d\x00\x04bertd\x00\x03nilh\x02d\x00\x04bertd\x00\x04trueh\x02d\x00\x04bertd\x00\x05falsed\x00\x04trued\x00\x05false".b, - :v3 => "\x85\x99\xC7\x05\xFF\x00user\x81\xC7\x05\xFF\x00name\xA3TPW\x92\xC7\x09\xFF\x01\x00\x00\x00\x01\x01cat\xCB@#\xCC\xCC\xCC\xCC\xCC\xCD\xC7\x11\xFF\x02\x00\x00\x00\x00\x4A\xCD\x6A\x43\x00\x00\x00\x00\x00\x00\x00\x00\xC0\xC3\xC2\xC7\x05\xFF\x00true\xC7\x06\xFF\x00false".b, + :v3 => "\x85\x99\xD4\x04user\x81\xD4\x04name\xD8\x00\x03\x00TPW\x92\xD5\x00\x03\x00\x00\x00\x01\x01cat\xCB@#\xCC\xCC\xCC\xCC\xCC\xCD\xD6\x00\x00\x00\x00\x4A\xCD\x6A\x43\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xC0\xC3\xC2\xD4\x04true\xD4\x05false".b, + :v4 => "\x86\x99\xC7\x05\xFF\x00user\x81\xC7\x05\xFF\x00name\xA3TPW\x92\xC7\x09\xFF\x01\x00\x00\x00\x01\x01cat\xCB@#\xCC\xCC\xCC\xCC\xCC\xCD\xC7\x15\xFF\x02\x00\x00\x00\x00\x4A\xCD\x6A\x43\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xC0\xC3\xC2\xC7\x05\xFF\x00true\xC7\x06\xFF\x00false".b, } @ebins = { :v2 => "<<132,104,9,100,0,4,117,115,101,114,104,3,100,0,4,98,101,114,116,100,0,4,100,105,99,116,108,0,0,0,1,104,2,100,0,4,110,97,109,101,113,0,0,0,3,84,80,87,106,108,0,0,0,2,104,4,100,0,4,98,101,114,116,100,0,5,114,101,103,101,120,113,0,0,0,3,99,97,116,108,0,0,0,1,100,0,8,99,97,115,101,108,101,115,115,106,99,57,46,57,48,48,48,48,48,48,48,48,48,48,48,48,48,48,101,43,48,48,0,0,0,0,0,0,0,0,0,0,106,104,5,100,0,4,98,101,114,116,100,0,4,116,105,109,101,98,0,0,4,230,98,0,14,228,195,97,0,104,2,100,0,4,98,101,114,116,100,0,3,110,105,108,104,2,100,0,4,98,101,114,116,100,0,4,116,114,117,101,104,2,100,0,4,98,101,114,116,100,0,5,102,97,108,115,101,100,0,4,116,114,117,101,100,0,5,102,97,108,115,101>>", - :v3 => "<<133,153,199,5,255,0,117,115,101,114,129,199,5,255,0,110,97,109,101,163,84,80,87,146,199,9,255,1,0,0,0,1,1,99,97,116,203,64,35,204,204,204,204,204,205,199,17,255,2,0,0,0,0,74,205,106,67,0,0,0,0,0,0,0,0,192,195,194,199,5,255,0,116,114,117,101,199,6,255,0,102,97,108,115,101>>", + :v3 => "<<133,153,212,4,117,115,101,114,129,212,4,110,97,109,101,216,0,3,0,84,80,87,146,213,0,3,0,0,0,1,1,99,97,116,203,64,35,204,204,204,204,204,205,214,0,0,0,0,74,205,106,67,0,0,0,0,0,0,0,0,0,0,0,0,192,195,194,212,4,116,114,117,101,212,5,102,97,108,115,101>>", + :v4 => "<<134,153,199,5,255,0,117,115,101,114,129,199,5,255,0,110,97,109,101,163,84,80,87,146,199,9,255,1,0,0,0,1,1,99,97,116,203,64,35,204,204,204,204,204,205,199,21,255,2,0,0,0,0,74,205,106,67,0,0,0,0,0,0,0,0,0,0,0,0,192,195,194,199,5,255,0,116,114,117,101,199,6,255,0,102,97,108,115,101>>", } end - [:v2, :v3].each do |v| + should "support either v3 or v4" do + assert BERT.supports?(:v3) || BERT.supports?(:v4) + end + + [:v2, :v3, :v4].each do |v| context "#{v} encoder" do setup do @old_version = BERT::Encode.version @@ -32,10 +38,12 @@ class BertTest < Test::Unit::TestCase end should "decode new format" do + pend unless BERT.supports?(v) assert_equal @ruby, BERT.decode(@bert) end should "roundtrip string and maintain encoding" do + pend unless BERT.supports?(v) str = "日本語".encode 'EUC-JP' round = BERT.decode(BERT.encode(str)) assert_equal str, round @@ -43,6 +51,7 @@ class BertTest < Test::Unit::TestCase end should "roundtrip binary string" do + pend unless BERT.supports?(v) str = "日本語".b round = BERT.decode(BERT.encode(str)) assert_equal str, round @@ -50,14 +59,17 @@ class BertTest < Test::Unit::TestCase end should "encode" do + pend unless BERT.supports?(v) assert_equal @bert, BERT.encode(@ruby) end should "roundtrip obj" do + pend unless BERT.supports?(v) assert_equal @ruby, BERT.decode(BERT.encode(@ruby)) end should "encode with buffer" do + pend unless BERT.supports?(v) buf = BERT.encode_to_buffer(@ruby) io = StringIO.new io.set_encoding 'binary' @@ -66,8 +78,23 @@ class BertTest < Test::Unit::TestCase end should "ebin" do + pend unless BERT.supports?(v) assert_equal @ebin, BERT.ebin(@bert) end + + should "raise on encode when unsupported" do + pend if BERT.supports?(v) + assert_raises do + BERT.encode_to_buffer(@ruby) + end + end + + should "raise on decode when unsupported" do + pend if BERT.supports?(v) + assert_raises do + BERT.decode(@bert) + end + end end end