bert/bench/bench.rb

50 строки
2.2 KiB
Ruby
Исходник Обычный вид История

2009-10-24 11:49:44 +04:00
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
require 'rubygems'
require 'bert'
require 'json'
require 'yajl'
require 'benchmark'
ITER = 1_000
Serialize symbol, time, regexp in the benchmark, too user system total real BERT v1 tiny 0.010000 0.000000 0.010000 ( 0.013280) BERT v1 small 0.110000 0.000000 0.110000 ( 0.116110) BERT v1 large 0.630000 0.010000 0.640000 ( 0.646742) BERT v1 complex 2.710000 0.000000 2.710000 ( 2.738346) BERT v1 long array 5.360000 0.000000 5.360000 ( 5.391206) BERT v2 tiny 0.020000 0.000000 0.020000 ( 0.018202) BERT v2 small 0.140000 0.000000 0.140000 ( 0.133662) BERT v2 large 0.640000 0.010000 0.650000 ( 0.660656) BERT v2 complex 2.520000 0.010000 2.530000 ( 2.534638) BERT v2 long array 5.460000 0.010000 5.470000 ( 5.520119) BERT v3 tiny 0.010000 0.000000 0.010000 ( 0.010693) BERT v3 small 0.010000 0.000000 0.010000 ( 0.011390) BERT v3 large 0.220000 0.030000 0.250000 ( 0.246068) BERT v3 complex 0.060000 0.000000 0.060000 ( 0.055431) BERT v3 long array 0.520000 0.000000 0.520000 ( 0.522735) JSON tiny 0.000000 0.000000 0.000000 ( 0.005991) JSON small 0.020000 0.000000 0.020000 ( 0.010092) JSON large 2.200000 0.020000 2.220000 ( 2.252311) JSON complex 0.160000 0.000000 0.160000 ( 0.159742) JSON long array 0.450000 0.010000 0.460000 ( 0.454021) YAJL tiny 0.010000 0.000000 0.010000 ( 0.006357) YAJL small 0.020000 0.000000 0.020000 ( 0.023327) YAJL large 0.960000 0.020000 0.980000 ( 1.002778) YAJL complex 0.280000 0.030000 0.310000 ( 0.317845) YAJL long array 0.750000 0.000000 0.750000 ( 0.747592) Ruby tiny 0.000000 0.000000 0.000000 ( 0.003547) Ruby small 0.010000 0.000000 0.010000 ( 0.007403) Ruby large 0.010000 0.000000 0.010000 ( 0.012739) Ruby complex 0.020000 0.000000 0.020000 ( 0.015415) Ruby long array 0.030000 0.000000 0.030000 ( 0.036519)
2017-05-26 21:27:43 +03:00
LONG_ITER = 100
2009-10-24 11:49:44 +04:00
tiny = t[:ok, :awesome]
small = t[:ok, :answers, [42] * 42]
large = ["abc" * 1000] * 100
complex = [42, {:foo => 'bac' * 100}, t[(1..100).to_a]] * 10
Serialize symbol, time, regexp in the benchmark, too user system total real BERT v1 tiny 0.010000 0.000000 0.010000 ( 0.013280) BERT v1 small 0.110000 0.000000 0.110000 ( 0.116110) BERT v1 large 0.630000 0.010000 0.640000 ( 0.646742) BERT v1 complex 2.710000 0.000000 2.710000 ( 2.738346) BERT v1 long array 5.360000 0.000000 5.360000 ( 5.391206) BERT v2 tiny 0.020000 0.000000 0.020000 ( 0.018202) BERT v2 small 0.140000 0.000000 0.140000 ( 0.133662) BERT v2 large 0.640000 0.010000 0.650000 ( 0.660656) BERT v2 complex 2.520000 0.010000 2.530000 ( 2.534638) BERT v2 long array 5.460000 0.010000 5.470000 ( 5.520119) BERT v3 tiny 0.010000 0.000000 0.010000 ( 0.010693) BERT v3 small 0.010000 0.000000 0.010000 ( 0.011390) BERT v3 large 0.220000 0.030000 0.250000 ( 0.246068) BERT v3 complex 0.060000 0.000000 0.060000 ( 0.055431) BERT v3 long array 0.520000 0.000000 0.520000 ( 0.522735) JSON tiny 0.000000 0.000000 0.000000 ( 0.005991) JSON small 0.020000 0.000000 0.020000 ( 0.010092) JSON large 2.200000 0.020000 2.220000 ( 2.252311) JSON complex 0.160000 0.000000 0.160000 ( 0.159742) JSON long array 0.450000 0.010000 0.460000 ( 0.454021) YAJL tiny 0.010000 0.000000 0.010000 ( 0.006357) YAJL small 0.020000 0.000000 0.020000 ( 0.023327) YAJL large 0.960000 0.020000 0.980000 ( 1.002778) YAJL complex 0.280000 0.030000 0.310000 ( 0.317845) YAJL long array 0.750000 0.000000 0.750000 ( 0.747592) Ruby tiny 0.000000 0.000000 0.000000 ( 0.003547) Ruby small 0.010000 0.000000 0.010000 ( 0.007403) Ruby large 0.010000 0.000000 0.010000 ( 0.012739) Ruby complex 0.020000 0.000000 0.020000 ( 0.015415) Ruby long array 0.030000 0.000000 0.030000 ( 0.036519)
2017-05-26 21:27:43 +03:00
long_array = {:a => ["a", :a, Time.now, /a/]*1000}
2009-10-24 11:49:44 +04:00
2017-04-21 16:03:53 +03:00
Benchmark.bm(30) do |bench|
[:v1, :v2, :v3, :v4].each do |v|
unless BERT.supports?(v)
puts "SKIP #{v} (unsupported)"
next
end
benchmark the new BERT encoder It's slightly slower than plain msgpack, but still much better than older BERT. user system total real BERT v1 tiny 0.020000 0.000000 0.020000 ( 0.019957) BERT v1 small 0.150000 0.000000 0.150000 ( 0.141066) BERT v1 large 0.710000 0.000000 0.710000 ( 0.723820) BERT v1 complex 2.950000 0.000000 2.950000 ( 2.972712) BERT v1 long array 3.870000 0.000000 3.870000 ( 3.898899) BERT v2 tiny 0.010000 0.000000 0.010000 ( 0.017890) BERT v2 small 0.160000 0.000000 0.160000 ( 0.159581) BERT v2 large 0.720000 0.000000 0.720000 ( 0.733793) BERT v2 complex 2.960000 0.010000 2.970000 ( 2.975499) BERT v2 long array 4.080000 0.000000 4.080000 ( 4.106795) BERT v3 tiny 0.020000 0.000000 0.020000 ( 0.024472) BERT v3 small 0.020000 0.000000 0.020000 ( 0.025297) BERT v3 large 0.320000 0.020000 0.340000 ( 0.337157) BERT v3 complex 0.090000 0.000000 0.090000 ( 0.085884) BERT v3 long array 0.100000 0.000000 0.100000 ( 0.104439) JSON tiny 0.010000 0.000000 0.010000 ( 0.007604) JSON small 0.010000 0.000000 0.010000 ( 0.014245) JSON large 2.450000 0.010000 2.460000 ( 2.473644) JSON complex 0.170000 0.000000 0.170000 ( 0.174540) JSON long array 0.350000 0.000000 0.350000 ( 0.357081) YAJL tiny 0.010000 0.000000 0.010000 ( 0.005566) YAJL small 0.020000 0.000000 0.020000 ( 0.019177) YAJL large 1.060000 0.030000 1.090000 ( 1.094342) YAJL complex 0.300000 0.000000 0.300000 ( 0.307467) YAJL long array 0.140000 0.000000 0.140000 ( 0.136261) Ruby tiny 0.000000 0.000000 0.000000 ( 0.002919) Ruby small 0.010000 0.000000 0.010000 ( 0.008211) Ruby large 0.000000 0.010000 0.010000 ( 0.009826) Ruby complex 0.020000 0.000000 0.020000 ( 0.013560) Ruby long array 0.080000 0.000000 0.080000 ( 0.082170) Msgpack tiny 0.000000 0.000000 0.000000 ( 0.002561) Msgpack small 0.000000 0.000000 0.000000 ( 0.003771) Msgpack large 0.190000 0.000000 0.190000 ( 0.195700) Msgpack complex 0.040000 0.010000 0.050000 ( 0.044514) Msgpack long array 0.070000 0.000000 0.070000 ( 0.071905)
2017-04-21 17:36:27 +03:00
BERT::Encode.version = v
bench.report("BERT #{v} tiny") {ITER.times {BERT.decode(BERT.encode(tiny))}}
bench.report("BERT #{v} small") {ITER.times {BERT.decode(BERT.encode(small))}}
bench.report("BERT #{v} large") {ITER.times {BERT.decode(BERT.encode(large))}}
bench.report("BERT #{v} complex") {ITER.times {BERT.decode(BERT.encode(complex))}}
Serialize symbol, time, regexp in the benchmark, too user system total real BERT v1 tiny 0.010000 0.000000 0.010000 ( 0.013280) BERT v1 small 0.110000 0.000000 0.110000 ( 0.116110) BERT v1 large 0.630000 0.010000 0.640000 ( 0.646742) BERT v1 complex 2.710000 0.000000 2.710000 ( 2.738346) BERT v1 long array 5.360000 0.000000 5.360000 ( 5.391206) BERT v2 tiny 0.020000 0.000000 0.020000 ( 0.018202) BERT v2 small 0.140000 0.000000 0.140000 ( 0.133662) BERT v2 large 0.640000 0.010000 0.650000 ( 0.660656) BERT v2 complex 2.520000 0.010000 2.530000 ( 2.534638) BERT v2 long array 5.460000 0.010000 5.470000 ( 5.520119) BERT v3 tiny 0.010000 0.000000 0.010000 ( 0.010693) BERT v3 small 0.010000 0.000000 0.010000 ( 0.011390) BERT v3 large 0.220000 0.030000 0.250000 ( 0.246068) BERT v3 complex 0.060000 0.000000 0.060000 ( 0.055431) BERT v3 long array 0.520000 0.000000 0.520000 ( 0.522735) JSON tiny 0.000000 0.000000 0.000000 ( 0.005991) JSON small 0.020000 0.000000 0.020000 ( 0.010092) JSON large 2.200000 0.020000 2.220000 ( 2.252311) JSON complex 0.160000 0.000000 0.160000 ( 0.159742) JSON long array 0.450000 0.010000 0.460000 ( 0.454021) YAJL tiny 0.010000 0.000000 0.010000 ( 0.006357) YAJL small 0.020000 0.000000 0.020000 ( 0.023327) YAJL large 0.960000 0.020000 0.980000 ( 1.002778) YAJL complex 0.280000 0.030000 0.310000 ( 0.317845) YAJL long array 0.750000 0.000000 0.750000 ( 0.747592) Ruby tiny 0.000000 0.000000 0.000000 ( 0.003547) Ruby small 0.010000 0.000000 0.010000 ( 0.007403) Ruby large 0.010000 0.000000 0.010000 ( 0.012739) Ruby complex 0.020000 0.000000 0.020000 ( 0.015415) Ruby long array 0.030000 0.000000 0.030000 ( 0.036519)
2017-05-26 21:27:43 +03:00
bench.report("BERT #{v} long array") {LONG_ITER.times {BERT.decode(BERT.encode(long_array))}}
benchmark the new BERT encoder It's slightly slower than plain msgpack, but still much better than older BERT. user system total real BERT v1 tiny 0.020000 0.000000 0.020000 ( 0.019957) BERT v1 small 0.150000 0.000000 0.150000 ( 0.141066) BERT v1 large 0.710000 0.000000 0.710000 ( 0.723820) BERT v1 complex 2.950000 0.000000 2.950000 ( 2.972712) BERT v1 long array 3.870000 0.000000 3.870000 ( 3.898899) BERT v2 tiny 0.010000 0.000000 0.010000 ( 0.017890) BERT v2 small 0.160000 0.000000 0.160000 ( 0.159581) BERT v2 large 0.720000 0.000000 0.720000 ( 0.733793) BERT v2 complex 2.960000 0.010000 2.970000 ( 2.975499) BERT v2 long array 4.080000 0.000000 4.080000 ( 4.106795) BERT v3 tiny 0.020000 0.000000 0.020000 ( 0.024472) BERT v3 small 0.020000 0.000000 0.020000 ( 0.025297) BERT v3 large 0.320000 0.020000 0.340000 ( 0.337157) BERT v3 complex 0.090000 0.000000 0.090000 ( 0.085884) BERT v3 long array 0.100000 0.000000 0.100000 ( 0.104439) JSON tiny 0.010000 0.000000 0.010000 ( 0.007604) JSON small 0.010000 0.000000 0.010000 ( 0.014245) JSON large 2.450000 0.010000 2.460000 ( 2.473644) JSON complex 0.170000 0.000000 0.170000 ( 0.174540) JSON long array 0.350000 0.000000 0.350000 ( 0.357081) YAJL tiny 0.010000 0.000000 0.010000 ( 0.005566) YAJL small 0.020000 0.000000 0.020000 ( 0.019177) YAJL large 1.060000 0.030000 1.090000 ( 1.094342) YAJL complex 0.300000 0.000000 0.300000 ( 0.307467) YAJL long array 0.140000 0.000000 0.140000 ( 0.136261) Ruby tiny 0.000000 0.000000 0.000000 ( 0.002919) Ruby small 0.010000 0.000000 0.010000 ( 0.008211) Ruby large 0.000000 0.010000 0.010000 ( 0.009826) Ruby complex 0.020000 0.000000 0.020000 ( 0.013560) Ruby long array 0.080000 0.000000 0.080000 ( 0.082170) Msgpack tiny 0.000000 0.000000 0.000000 ( 0.002561) Msgpack small 0.000000 0.000000 0.000000 ( 0.003771) Msgpack large 0.190000 0.000000 0.190000 ( 0.195700) Msgpack complex 0.040000 0.010000 0.050000 ( 0.044514) Msgpack long array 0.070000 0.000000 0.070000 ( 0.071905)
2017-04-21 17:36:27 +03:00
end
2009-10-24 11:49:44 +04:00
bench.report("JSON tiny") {ITER.times {JSON.load(JSON.dump(tiny))}}
bench.report("JSON small") {ITER.times {JSON.load(JSON.dump(small))}}
bench.report("JSON large") {ITER.times {JSON.load(JSON.dump(large))}}
bench.report("JSON complex") {ITER.times {JSON.load(JSON.dump(complex))}}
Serialize symbol, time, regexp in the benchmark, too user system total real BERT v1 tiny 0.010000 0.000000 0.010000 ( 0.013280) BERT v1 small 0.110000 0.000000 0.110000 ( 0.116110) BERT v1 large 0.630000 0.010000 0.640000 ( 0.646742) BERT v1 complex 2.710000 0.000000 2.710000 ( 2.738346) BERT v1 long array 5.360000 0.000000 5.360000 ( 5.391206) BERT v2 tiny 0.020000 0.000000 0.020000 ( 0.018202) BERT v2 small 0.140000 0.000000 0.140000 ( 0.133662) BERT v2 large 0.640000 0.010000 0.650000 ( 0.660656) BERT v2 complex 2.520000 0.010000 2.530000 ( 2.534638) BERT v2 long array 5.460000 0.010000 5.470000 ( 5.520119) BERT v3 tiny 0.010000 0.000000 0.010000 ( 0.010693) BERT v3 small 0.010000 0.000000 0.010000 ( 0.011390) BERT v3 large 0.220000 0.030000 0.250000 ( 0.246068) BERT v3 complex 0.060000 0.000000 0.060000 ( 0.055431) BERT v3 long array 0.520000 0.000000 0.520000 ( 0.522735) JSON tiny 0.000000 0.000000 0.000000 ( 0.005991) JSON small 0.020000 0.000000 0.020000 ( 0.010092) JSON large 2.200000 0.020000 2.220000 ( 2.252311) JSON complex 0.160000 0.000000 0.160000 ( 0.159742) JSON long array 0.450000 0.010000 0.460000 ( 0.454021) YAJL tiny 0.010000 0.000000 0.010000 ( 0.006357) YAJL small 0.020000 0.000000 0.020000 ( 0.023327) YAJL large 0.960000 0.020000 0.980000 ( 1.002778) YAJL complex 0.280000 0.030000 0.310000 ( 0.317845) YAJL long array 0.750000 0.000000 0.750000 ( 0.747592) Ruby tiny 0.000000 0.000000 0.000000 ( 0.003547) Ruby small 0.010000 0.000000 0.010000 ( 0.007403) Ruby large 0.010000 0.000000 0.010000 ( 0.012739) Ruby complex 0.020000 0.000000 0.020000 ( 0.015415) Ruby long array 0.030000 0.000000 0.030000 ( 0.036519)
2017-05-26 21:27:43 +03:00
bench.report("JSON long array") {LONG_ITER.times {JSON.load(JSON.dump(long_array))}}
2009-10-24 11:49:44 +04:00
bench.report("YAJL tiny") {ITER.times {Yajl::Parser.parse(Yajl::Encoder.encode(tiny))}}
bench.report("YAJL small") {ITER.times {Yajl::Parser.parse(Yajl::Encoder.encode(small))}}
bench.report("YAJL large") {ITER.times {Yajl::Parser.parse(Yajl::Encoder.encode(large))}}
bench.report("YAJL complex") {ITER.times {Yajl::Parser.parse(Yajl::Encoder.encode(complex))}}
Serialize symbol, time, regexp in the benchmark, too user system total real BERT v1 tiny 0.010000 0.000000 0.010000 ( 0.013280) BERT v1 small 0.110000 0.000000 0.110000 ( 0.116110) BERT v1 large 0.630000 0.010000 0.640000 ( 0.646742) BERT v1 complex 2.710000 0.000000 2.710000 ( 2.738346) BERT v1 long array 5.360000 0.000000 5.360000 ( 5.391206) BERT v2 tiny 0.020000 0.000000 0.020000 ( 0.018202) BERT v2 small 0.140000 0.000000 0.140000 ( 0.133662) BERT v2 large 0.640000 0.010000 0.650000 ( 0.660656) BERT v2 complex 2.520000 0.010000 2.530000 ( 2.534638) BERT v2 long array 5.460000 0.010000 5.470000 ( 5.520119) BERT v3 tiny 0.010000 0.000000 0.010000 ( 0.010693) BERT v3 small 0.010000 0.000000 0.010000 ( 0.011390) BERT v3 large 0.220000 0.030000 0.250000 ( 0.246068) BERT v3 complex 0.060000 0.000000 0.060000 ( 0.055431) BERT v3 long array 0.520000 0.000000 0.520000 ( 0.522735) JSON tiny 0.000000 0.000000 0.000000 ( 0.005991) JSON small 0.020000 0.000000 0.020000 ( 0.010092) JSON large 2.200000 0.020000 2.220000 ( 2.252311) JSON complex 0.160000 0.000000 0.160000 ( 0.159742) JSON long array 0.450000 0.010000 0.460000 ( 0.454021) YAJL tiny 0.010000 0.000000 0.010000 ( 0.006357) YAJL small 0.020000 0.000000 0.020000 ( 0.023327) YAJL large 0.960000 0.020000 0.980000 ( 1.002778) YAJL complex 0.280000 0.030000 0.310000 ( 0.317845) YAJL long array 0.750000 0.000000 0.750000 ( 0.747592) Ruby tiny 0.000000 0.000000 0.000000 ( 0.003547) Ruby small 0.010000 0.000000 0.010000 ( 0.007403) Ruby large 0.010000 0.000000 0.010000 ( 0.012739) Ruby complex 0.020000 0.000000 0.020000 ( 0.015415) Ruby long array 0.030000 0.000000 0.030000 ( 0.036519)
2017-05-26 21:27:43 +03:00
bench.report("YAJL long array") {LONG_ITER.times {Yajl::Parser.parse(Yajl::Encoder.encode(long_array))}}
2009-10-24 11:49:44 +04:00
bench.report("Ruby tiny") {ITER.times {Marshal.load(Marshal.dump(tiny))}}
bench.report("Ruby small") {ITER.times {Marshal.load(Marshal.dump(small))}}
bench.report("Ruby large") {ITER.times {Marshal.load(Marshal.dump(large))}}
bench.report("Ruby complex") {ITER.times {Marshal.load(Marshal.dump(complex))}}
Serialize symbol, time, regexp in the benchmark, too user system total real BERT v1 tiny 0.010000 0.000000 0.010000 ( 0.013280) BERT v1 small 0.110000 0.000000 0.110000 ( 0.116110) BERT v1 large 0.630000 0.010000 0.640000 ( 0.646742) BERT v1 complex 2.710000 0.000000 2.710000 ( 2.738346) BERT v1 long array 5.360000 0.000000 5.360000 ( 5.391206) BERT v2 tiny 0.020000 0.000000 0.020000 ( 0.018202) BERT v2 small 0.140000 0.000000 0.140000 ( 0.133662) BERT v2 large 0.640000 0.010000 0.650000 ( 0.660656) BERT v2 complex 2.520000 0.010000 2.530000 ( 2.534638) BERT v2 long array 5.460000 0.010000 5.470000 ( 5.520119) BERT v3 tiny 0.010000 0.000000 0.010000 ( 0.010693) BERT v3 small 0.010000 0.000000 0.010000 ( 0.011390) BERT v3 large 0.220000 0.030000 0.250000 ( 0.246068) BERT v3 complex 0.060000 0.000000 0.060000 ( 0.055431) BERT v3 long array 0.520000 0.000000 0.520000 ( 0.522735) JSON tiny 0.000000 0.000000 0.000000 ( 0.005991) JSON small 0.020000 0.000000 0.020000 ( 0.010092) JSON large 2.200000 0.020000 2.220000 ( 2.252311) JSON complex 0.160000 0.000000 0.160000 ( 0.159742) JSON long array 0.450000 0.010000 0.460000 ( 0.454021) YAJL tiny 0.010000 0.000000 0.010000 ( 0.006357) YAJL small 0.020000 0.000000 0.020000 ( 0.023327) YAJL large 0.960000 0.020000 0.980000 ( 1.002778) YAJL complex 0.280000 0.030000 0.310000 ( 0.317845) YAJL long array 0.750000 0.000000 0.750000 ( 0.747592) Ruby tiny 0.000000 0.000000 0.000000 ( 0.003547) Ruby small 0.010000 0.000000 0.010000 ( 0.007403) Ruby large 0.010000 0.000000 0.010000 ( 0.012739) Ruby complex 0.020000 0.000000 0.020000 ( 0.015415) Ruby long array 0.030000 0.000000 0.030000 ( 0.036519)
2017-05-26 21:27:43 +03:00
bench.report("Ruby long array") {LONG_ITER.times {Marshal.load(Marshal.dump(long_array))}}
2017-04-21 16:03:53 +03:00
end