remove reliance on erlectricity

This commit is contained in:
Tom Preston-Werner 2009-10-24 00:42:12 -07:00
Родитель 1ba84ebd73
Коммит 4150346547
15 изменённых файлов: 808 добавлений и 59 удалений

3
.gitignore поставляемый
Просмотреть файл

@ -3,3 +3,6 @@
coverage
rdoc
pkg
ext/bert/c/Makefile
ext/bert/c/*.bundle
ext/bert/c/*.o

Просмотреть файл

@ -1,3 +1,7 @@
=
* Major Changes
* Remove reliance on Erlectricity.
= 1.0.0 / 2009-10-19
* No changes. Production ready!

Просмотреть файл

@ -12,6 +12,9 @@ begin
gem.authors = ["Tom Preston-Werner"]
gem.add_dependency('erlectricity', '>= 1.1.0')
gem.add_development_dependency("thoughtbot-shoulda")
gem.require_paths = ["lib", "ext"]
gem.files.include("ext")
gem.extensions << 'ext/bert/c/extconf.rb'
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
end
rescue LoadError
@ -19,12 +22,32 @@ rescue LoadError
end
require 'rake/testtask'
Rake::TestTask.new(:test) do |test|
Rake::TestTask.new(:runtests) do |test|
test.libs << 'lib' << 'test'
test.pattern = 'test/**/*_test.rb'
test.verbose = true
end
task :test => :check_dependencies do
require 'fileutils'
puts "\nCleaning extension build files and running all specs in native ruby mode..."
['rm -f ext/bert/c/*.bundle', 'rm -f ext/bert/c/*.o'].each do |cmd|
`#{cmd}` && puts(cmd)
end
pid = fork do
exec 'rake runtests'
end
Process.waitpid(pid)
puts "\nRunning `make` to build extensions and rerunning decoder specs..."
Dir.chdir('ext/bert/c') { `make` }
pid = fork do
exec 'rake runtests'
end
Process.waitpid(pid)
end
begin
require 'rcov/rcovtask'
Rcov::RcovTask.new do |test|
@ -38,8 +61,6 @@ rescue LoadError
end
end
task :test => :check_dependencies
task :default => :test
require 'rake/rdoctask'

352
ext/bert/c/decode.c Normal file
Просмотреть файл

@ -0,0 +1,352 @@
#include "ruby.h"
#include <string.h>
#define ERL_VERSION 131
#define ERL_SMALL_INT 97
#define ERL_INT 98
#define ERL_SMALL_BIGNUM 110
#define ERL_LARGE_BIGNUM 111
#define ERL_FLOAT 99
#define ERL_ATOM 100
#define ERL_SMALL_TUPLE 104
#define ERL_LARGE_TUPLE 105
#define ERL_NIL 106
#define ERL_STRING 107
#define ERL_LIST 108
#define ERL_BIN 109
static VALUE mBERT;
static VALUE cDecode;
void Init_decode();
VALUE method_decode(VALUE klass, VALUE rString);
VALUE read_any_raw(unsigned char **pData);
// checkers
void check_int(int num) {
char buf[17];
sprintf(buf, "%u", num);
rb_raise(rb_eStandardError, buf);
}
void check_str(char *str) {
rb_raise(rb_eStandardError, str);
}
// string peekers/readers
unsigned int peek_1(unsigned char **pData) {
return (unsigned int) **pData;
}
unsigned int peek_2(unsigned char **pData) {
return (unsigned int) ((**pData << 8) + *(*pData + 1));
}
unsigned int peek_4(unsigned char **pData) {
return (unsigned int) ((**pData << 24) + (*(*pData + 1) << 16) + (*(*pData + 2) << 8) + *(*pData + 3));
}
unsigned int read_1(unsigned char **pData) {
unsigned int val = peek_1(pData);
*pData += 1;
return val;
}
unsigned int read_2(unsigned char **pData) {
unsigned int val = peek_2(pData);
*pData += 2;
return val;
}
unsigned int read_4(unsigned char **pData) {
unsigned int val = peek_4(pData);
*pData += 4;
return val;
}
// tuples, lists
VALUE read_small_tuple(unsigned char **pData) {
if(read_1(pData) != ERL_SMALL_TUPLE) {
rb_raise(rb_eStandardError, "Invalid Type, not a small tuple");
}
int arity = read_1(pData);
VALUE array = rb_ary_new2(arity);
int i;
for(i = 0; i < arity; ++i) {
rb_ary_store(array, i, read_any_raw(pData));
}
return array;
}
VALUE read_large_tuple(unsigned char **pData) {
if(read_1(pData) != ERL_LARGE_TUPLE) {
rb_raise(rb_eStandardError, "Invalid Type, not a large tuple");
}
int arity = read_4(pData);
VALUE array = rb_ary_new2(arity);
int i;
for(i = 0; i < arity; ++i) {
rb_ary_store(array, i, read_any_raw(pData));
}
return array;
}
VALUE read_list(unsigned char **pData) {
if(read_1(pData) != ERL_LIST) {
rb_raise(rb_eStandardError, "Invalid Type, not an erlang list");
}
int size = read_4(pData);
VALUE list_class = rb_const_get(mBERT, rb_intern("List"));
VALUE array = rb_funcall(list_class, rb_intern("new"), 1, INT2NUM(size));
int i;
for(i = 0; i < size; ++i) {
rb_ary_store(array, i, read_any_raw(pData));
}
read_1(pData);
return array;
}
// primitives
void read_string_raw(unsigned char *dest, unsigned char **pData, int length) {
memcpy((char *) dest, (char *) *pData, length);
*(dest + length) = (unsigned char) 0;
*pData += length;
}
VALUE read_bin(unsigned char **pData) {
if(read_1(pData) != ERL_BIN) {
rb_raise(rb_eStandardError, "Invalid Type, not an erlang binary");
}
int length = read_4(pData);
unsigned char buf[length + 1];
read_string_raw(buf, pData, length);
return rb_str_new((char *) buf, length);
}
VALUE read_string(unsigned char **pData) {
if(read_1(pData) != ERL_STRING) {
rb_raise(rb_eStandardError, "Invalid Type, not an erlang string");
}
int length = read_2(pData);
unsigned char buf[length + 1];
read_string_raw(buf, pData, length);
VALUE list_class = rb_const_get(mBERT, rb_intern("List"));
VALUE array = rb_funcall(list_class, rb_intern("new"), 1, INT2NUM(length));
int i = 0;
for(i; i < length; ++i) {
rb_ary_store(array, i, INT2NUM(*(buf + i)));
}
return array;
}
VALUE read_atom(unsigned char **pData) {
if(read_1(pData) != ERL_ATOM) {
rb_raise(rb_eStandardError, "Invalid Type, not an atom");
}
int length = read_2(pData);
unsigned char buf[length + 1];
read_string_raw(buf, pData, length);
return ID2SYM(rb_intern((char *) buf));
}
VALUE read_small_int(unsigned char **pData) {
if(read_1(pData) != ERL_SMALL_INT) {
rb_raise(rb_eStandardError, "Invalid Type, not a small int");
}
int value = read_1(pData);
return INT2FIX(value);
}
VALUE read_int(unsigned char **pData) {
if(read_1(pData) != ERL_INT) {
rb_raise(rb_eStandardError, "Invalid Type, not an int");
}
long long value = read_4(pData);
long long negative = ((value >> 31) & 0x1 == 1);
if(negative) {
value = (value - ((long long) 1 << 32));
}
return INT2FIX(value);
}
VALUE read_small_bignum(unsigned char **pData) {
if(read_1(pData) != ERL_SMALL_BIGNUM) {
rb_raise(rb_eStandardError, "Invalid Type, not a small bignum");
}
unsigned int size = read_1(pData);
unsigned int sign = read_1(pData);
VALUE num = INT2NUM(0);
VALUE tmp;
unsigned char buf[size + 1];
read_string_raw(buf, pData, size);
int i;
for(i = 0; i < size; ++i) {
tmp = INT2FIX(*(buf + i));
tmp = rb_funcall(tmp, rb_intern("<<"), 1, INT2NUM(i * 8));
num = rb_funcall(num, rb_intern("+"), 1, tmp);
}
if(sign) {
num = rb_funcall(num, rb_intern("*"), 1, INT2NUM(-1));
}
return num;
}
VALUE read_large_bignum(unsigned char **pData) {
if(read_1(pData) != ERL_LARGE_BIGNUM) {
rb_raise(rb_eStandardError, "Invalid Type, not a small bignum");
}
unsigned int size = read_4(pData);
unsigned int sign = read_1(pData);
VALUE num = INT2NUM(0);
VALUE tmp;
unsigned char buf[size + 1];
read_string_raw(buf, pData, size);
int i;
for(i = 0; i < size; ++i) {
tmp = INT2FIX(*(buf + i));
tmp = rb_funcall(tmp, rb_intern("<<"), 1, INT2NUM(i * 8));
num = rb_funcall(num, rb_intern("+"), 1, tmp);
}
if(sign) {
num = rb_funcall(num, rb_intern("*"), 1, INT2NUM(-1));
}
return num;
}
VALUE read_float(unsigned char **pData) {
if(read_1(pData) != ERL_FLOAT) {
rb_raise(rb_eStandardError, "Invalid Type, not a float");
}
unsigned char buf[32];
read_string_raw(buf, pData, 31);
VALUE rString = rb_str_new2((char *) buf);
return rb_funcall(rString, rb_intern("to_f"), 0);
}
VALUE read_nil(unsigned char **pData) {
if(read_1(pData) != ERL_NIL) {
rb_raise(rb_eStandardError, "Invalid Type, not a nil list");
}
VALUE list_class = rb_const_get(mBERT, rb_intern("List"));
return rb_funcall(list_class, rb_intern("new"), 0);
}
// read_any_raw
VALUE read_any_raw(unsigned char **pData) {
switch(peek_1(pData)) {
case ERL_SMALL_INT:
return read_small_int(pData);
break;
case ERL_INT:
return read_int(pData);
break;
case ERL_FLOAT:
return read_float(pData);
break;
case ERL_ATOM:
return read_atom(pData);
break;
case ERL_SMALL_TUPLE:
return read_small_tuple(pData);
break;
case ERL_LARGE_TUPLE:
return read_large_tuple(pData);
break;
case ERL_NIL:
return read_nil(pData);
break;
case ERL_STRING:
return read_string(pData);
break;
case ERL_LIST:
return read_list(pData);
break;
case ERL_BIN:
return read_bin(pData);
break;
case ERL_SMALL_BIGNUM:
return read_small_bignum(pData);
break;
case ERL_LARGE_BIGNUM:
return read_large_bignum(pData);
break;
}
return Qnil;
}
VALUE method_decode(VALUE klass, VALUE rString) {
unsigned char *data = (unsigned char *) StringValuePtr(rString);
unsigned char **pData = &data;
// check protocol version
if(read_1(pData) != ERL_VERSION) {
rb_raise(rb_eStandardError, "Bad Magic");
}
return read_any_raw(pData);
}
VALUE method_impl(VALUE klass) {
return rb_str_new("C", 2);
}
void Init_decode() {
mBERT = rb_const_get(rb_cObject, rb_intern("BERT"));
cDecode = rb_define_class_under(mBERT, "Decode", rb_cObject);
rb_define_singleton_method(cDecode, "decode", method_decode, 1);
rb_define_singleton_method(cDecode, "impl", method_impl, 0);
}

11
ext/bert/c/extconf.rb Normal file
Просмотреть файл

@ -0,0 +1,11 @@
# Loads mkmf which is used to make makefiles for Ruby extensions
require 'mkmf'
# Give it a name
extension_name = 'decode'
# The destination
dir_config(extension_name)
# Do the work
create_makefile(extension_name)

Просмотреть файл

@ -1,33 +1,24 @@
require 'rubygems'
require 'erlectricity'
$:.unshift File.join(File.dirname(__FILE__), *%w[.. ext])
require 'bert/bert'
require 'bert/types'
begin
# try to load the C extension
require 'bert/c/decode'
rescue LoadError
# fall back on the pure ruby version
require 'bert/decode'
end
require 'bert/encode'
require 'bert/encoder'
require 'bert/decoder'
module BERT
def self.encode(ruby)
Encoder.encode(ruby)
end
def self.decode(bert)
Decoder.decode(bert)
end
def self.ebin(str)
bytes = []
str.each_byte { |b| bytes << b.to_s }
"<<" + bytes.join(',') + ">>"
end
end
module BERT
class Tuple < Array
def inspect
"t#{super}"
end
end
end
# Global method for specifying that an array should be encoded as a tuple.
def t
BERT::Tuple
end

27
lib/bert/bert.rb Normal file
Просмотреть файл

@ -0,0 +1,27 @@
module BERT
def self.encode(ruby)
Encoder.encode(ruby)
end
def self.decode(bert)
Decoder.decode(bert)
end
def self.ebin(str)
bytes = []
str.each_byte { |b| bytes << b.to_s }
"<<" + bytes.join(',') + ">>"
end
class List < Array
def inspect
"l#{super}"
end
end
class Tuple < Array
def inspect
"t#{super}"
end
end
end

192
lib/bert/decode.rb Normal file
Просмотреть файл

@ -0,0 +1,192 @@
module BERT
class Decode
attr_accessor :in
include Types
def self.impl
'Ruby'
end
def self.decode(string)
new(StringIO.new(string)).read_any
end
def initialize(ins)
@in = ins
@peeked = ""
end
def read_any
fail("Bad Magic") unless read_1 == MAGIC
read_any_raw
end
def read_any_raw
case peek_1
when ATOM then read_atom
when SMALL_INT then read_small_int
when INT then read_int
when SMALL_BIGNUM then read_small_bignum
when LARGE_BIGNUM then read_large_bignum
when FLOAT then read_float
when SMALL_TUPLE then read_small_tuple
when LARGE_TUPLE then read_large_tuple
when NIL then read_nil
when STRING then read_erl_string
when LIST then read_list
when BIN then read_bin
else
fail("Unknown term tag: #{peek_1}")
end
end
def read(length)
if length < @peeked.length
result = @peeked[0...length]
@peeked = @peeked[length..-1]
length = 0
else
result = @peeked
@peeked = ''
length -= result.length
end
if length > 0
result << @in.read(length)
end
result
end
def peek(length)
if length <= @peeked.length
@peeked[0...length]
else
read_bytes = @in.read(length - @peeked.length)
@peeked << read_bytes if read_bytes
@peeked
end
end
def peek_1
peek(1).unpack("C").first
end
def peek_2
peek(2).unpack("n").first
end
def read_1
read(1).unpack("C").first
end
def read_2
read(2).unpack("n").first
end
def read_4
read(4).unpack("N").first
end
def read_string(length)
read(length)
end
def read_atom
fail("Invalid Type, not an atom") unless read_1 == ATOM
length = read_2
a = read_string(length)
case a
when ""
Marshal.load("\004\b:\005") # Workaround for inability to do ''.to_sym
else
a.to_sym
end
end
def read_small_int
fail("Invalid Type, not a small int") unless read_1 == SMALL_INT
read_1
end
def read_int
fail("Invalid Type, not an int") unless read_1 == INT
value = read_4
negative = (value >> 31)[0] == 1
value = (value - (1 << 32)) if negative
value = Fixnum.induced_from(value)
end
def read_small_bignum
fail("Invalid Type, not a small bignum") unless read_1 == SMALL_BIGNUM
size = read_1
sign = read_1
bytes = read_string(size).unpack("C" * size)
added = bytes.zip((0..bytes.length).to_a).inject(0) do |result, byte_index|
byte, index = *byte_index
value = (byte * (256 ** index))
sign != 0 ? (result - value) : (result + value)
end
Bignum.induced_from(added)
end
def read_large_bignum
fail("Invalid Type, not a large bignum") unless read_1 == LARGE_BIGNUM
size = read_4
sign = read_1
bytes = read_string(size).unpack("C" * size)
added = bytes.zip((0..bytes.length).to_a).inject(0) do |result, byte_index|
byte, index = *byte_index
value = (byte * (256 ** index))
sign != 0 ? (result - value) : (result + value)
end
Bignum.induced_from(added)
end
def read_float
fail("Invalid Type, not a float") unless read_1 == FLOAT
string_value = read_string(31)
result = string_value.to_f
end
def read_small_tuple
fail("Invalid Type, not a small tuple") unless read_1 == SMALL_TUPLE
arity = read_1
(0...arity).map { |i| read_any_raw }
end
def read_large_tuple
fail("Invalid Type, not a small tuple") unless read_1 == LARGE_TUPLE
arity = read_4
(0...arity).map { |i| read_any_raw }
end
def read_nil
fail("Invalid Type, not a nil list") unless read_1 == NIL
List.new([])
end
def read_erl_string
fail("Invalid Type, not an erlang string") unless read_1 == STRING
length = read_2
List.new(read_string(length).unpack('C' * length))
end
def read_list
fail("Invalid Type, not an erlang list") unless read_1 == LIST
length = read_4
list = (0...length).map { |i| read_any_raw }
read_1
List.new(list)
end
def read_bin
fail("Invalid Type, not an erlang binary") unless read_1 == BIN
length = read_4
read_string(length)
end
def fail(str)
raise str
end
end
end

Просмотреть файл

@ -5,20 +5,17 @@ module BERT
#
# Returns a Ruby object
def self.decode(bert)
simple_ruby = Erlectricity::Decoder.decode(bert)
simple_ruby = Decode.decode(bert)
convert(simple_ruby)
end
# Convert Erlectricity representation of BERT complex types into
# corresponding Ruby types.
# Convert simple Ruby form into complex Ruby form.
# +item+ is the Ruby object to convert
#
# Returns the converted Ruby object
def self.convert(item)
case item
when TrueClass, FalseClass
item.to_s.to_sym
when Erl::List
when List
item.map { |x| convert(x) }
when Array
if item[0] == :bert
@ -43,9 +40,9 @@ module BERT
item[2].inject({}) do |acc, x|
acc[convert(x[0])] = convert(x[1]); acc
end
when TrueClass
when :true
true
when FalseClass
when :false
false
when :time
Time.at(item[2] * 1_000_000 + item[3], item[4])

142
lib/bert/encode.rb Normal file
Просмотреть файл

@ -0,0 +1,142 @@
module BERT
class Encode
include Types
attr_accessor :out
def initialize(out)
self.out = out
end
def self.encode(data)
io = StringIO.new
self.new(io).write_any(data)
io.string
end
def write_any obj
write_1 MAGIC
write_any_raw obj
end
def write_any_raw obj
case obj
when Symbol then write_symbol(obj)
when Fixnum, Bignum then write_fixnum(obj)
when Float then write_float(obj)
when BERT::List then write_list(obj)
when Array then write_tuple(obj)
when String then write_binary(obj)
else
fail(obj)
end
end
def write_1(byte)
out.write([byte].pack("C"))
end
def write_2(short)
out.write([short].pack("n"))
end
def write_4(long)
out.write([long].pack("N"))
end
def write_string(string)
out.write(string)
end
def write_boolean(bool)
write_symbol(bool.to_s.to_sym)
end
def write_symbol(sym)
fail(sym) unless sym.is_a?(Symbol)
data = sym.to_s
write_1 ATOM
write_2 data.length
write_string data
end
def write_fixnum(num)
if num >= 0 && num < 256
write_1 SMALL_INT
write_1 num
elsif num <= MAX_INT && num >= MIN_INT
write_1 INT
write_4 num
else
write_bignum num
end
end
def write_float(float)
write_1 FLOAT
write_string format("%15.15e", float).ljust(31, "\000")
end
def write_bignum(num)
if num.is_a?(Bignum)
n = num.size
else
n = (num.to_s(2).size / 8.0).ceil
end
if n <= 256
write_1 SMALL_BIGNUM
write_1 n
write_bignum_guts(num)
else
write_1 LARGE_BIGNUM
write_4 n
write_bignum_guts(num)
end
end
def write_bignum_guts(num)
write_1 (num >= 0 ? 0 : 1)
num = num.abs
i = 0
while (rem = (num >> i * 8) % (256)) != 0
write_1 rem
i += 1
end
end
def write_tuple(data)
fail(data) unless data.is_a? Array
if data.length < 256
write_1 SMALL_TUPLE
write_1 data.length
else
write_1 LARGE_TUPLE
write_4 data.length
end
data.each { |e| write_any_raw e }
end
def write_list(data)
fail(data) unless data.is_a? Array
write_1 NIL and return if data.empty?
write_1 LIST
write_4 data.length
data.each{|e| write_any_raw e }
write_1 NIL
end
def write_binary(data)
write_1 BIN
write_4 data.length
write_string data
end
private
def fail(obj)
raise "Cannot encode to erlang external format: #{obj.inspect}"
end
end
end

Просмотреть файл

@ -6,24 +6,23 @@ module BERT
# Returns a BERT
def self.encode(ruby)
complex_ruby = convert(ruby)
Erlectricity::Encoder.encode(complex_ruby)
Encode.encode(complex_ruby)
end
# Convert Ruby types into corresponding Erlectricity representation
# of BERT complex types.
# Convert complex Ruby form in simple Ruby form.
# +item+ is the Ruby object to convert
#
# Returns the converted Ruby object
def self.convert(item)
case item
when Hash
pairs = Erl::List[]
pairs = List[]
item.each_pair { |k, v| pairs << [convert(k), convert(v)] }
[:bert, :dict, pairs]
when Tuple
item.map { |x| convert(x) }
when Array
Erl::List.new(item.map { |x| convert(x) })
List.new(item.map { |x| convert(x) })
when nil
[:bert, :nil]
when TrueClass
@ -33,7 +32,7 @@ module BERT
when Time
[:bert, :time, item.to_i / 1_000_000, item.to_i % 1_000_000, item.usec]
when Regexp
options = Erl::List[]
options = List[]
options << :caseless if item.options & Regexp::IGNORECASE > 0
options << :extended if item.options & Regexp::EXTENDED > 0
options << :multiline if item.options & Regexp::MULTILINE > 0

21
lib/bert/types.rb Normal file
Просмотреть файл

@ -0,0 +1,21 @@
module BERT
module Types
SMALL_INT = 97
INT = 98
SMALL_BIGNUM = 110
LARGE_BIGNUM = 111
FLOAT = 99
ATOM = 100
SMALL_TUPLE = 104
LARGE_TUPLE = 105
NIL = 106
STRING = 107
LIST = 108
BIN = 109
FUN = 117
NEW_FUN = 112
MAGIC = 131
MAX_INT = (1 << 27) -1
MIN_INT = -(1 << 27)
end
end

Просмотреть файл

@ -33,13 +33,13 @@ class DecoderTest < Test::Unit::TestCase
end
should "convert true" do
before = [:bert, true]
before = [:bert, :true]
after = true
assert_equal after, BERT::Decoder.convert(before)
end
should "convert false" do
before = [:bert, false]
before = [:bert, :false]
after = false
assert_equal after, BERT::Decoder.convert(before)
end

Просмотреть файл

@ -27,7 +27,7 @@ class EncoderTest < Test::Unit::TestCase
should "convert hash to tuple with array of tuples" do
arr = BERT::Encoder.convert({:foo => 'bar'})
assert arr.is_a?(Array)
assert arr[2].is_a?(Erl::List)
assert arr[2].is_a?(BERT::List)
assert arr[2][0].is_a?(Array)
end
@ -38,13 +38,13 @@ class EncoderTest < Test::Unit::TestCase
should "convert array to erl list" do
list = BERT::Encoder.convert([1, 2])
assert list.is_a?(Erl::List)
assert list.is_a?(BERT::List)
end
should "convert an array in a tuple" do
arrtup = BERT::Encoder.convert(t[:foo, [1, 2]])
assert arrtup.is_a?(Array)
assert arrtup[1].is_a?(Erl::List)
assert arrtup[1].is_a?(BERT::List)
end
should "convert true" do

Просмотреть файл

@ -5,15 +5,4 @@ require 'shoulda'
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
$LOAD_PATH.unshift(File.dirname(__FILE__))
require 'bert'
class Test::Unit::TestCase
end
# So I can easily see which arrays are Erl::List
module Erl
class List
def inspect
"l#{super}"
end
end
end
puts "Using #{BERT::Decode.impl} implementation."