diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..c006df8c1 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,54 @@ +# Rhino Benchmarks + +This directory contains a collection of various benchmarks that have been added to the Rhino +project over the years. They have all been collected here and run using the JMH framework. + +## Running the Benchmarks + +To run all the benchmarks that exist, simply run, from the top-level directory: + + ./gradlew jmh + +In addition, the environment variable BENCHMARK may be used to restrict which benchmarks are +run -- it is a regular expression that matches the names of the benchmarks. + +For example, to run only the SunSpider and V8 benchmarks, you can run: + + BENCHMARK=V8|SunSpider ./gradlew jmh + +Running all the benchmarks takes about half an hour, so this is a valuable thing to do! + +## How the Benchmarks work + +Java, with its just-in-time compilation pipeline, bytecode generation in Rhino, and +Java's super-complex garbage collector, is sensitive to warm up time and as a result, Rhino has +more variation between runs than other JavaScript engines. To get a repeatable result, we use +the JMH framework, which runs each benchmark many times and does other "black hole" protection +operations to try and get an accurate result. For this reason, these benchmarks take a lot longer +to run than your favorite on-line JavaScript benchmarking web site. + +The purpose of these benchmarks has historically been to make Rhino perform better in server-side +environments, so they all run at the maximum optimization level (9). Since so many people also +use Rhino in interpreted mode, if there's an interest in benchmarking that too then we can +always adjust these tests. + +## Benchmark Notes + +Here are a few notes on the specific benchmarks: + +* **SunSpiderBenchmark**: These are the venerable JavaScript benchmarks that have been used +for a long time. They test both low-level operations like math operations, as well as higher-level +tasks. +* **V8Benchmark**: These are Google's V8 benchmarks, which may have been created to show how +efficient V8 is. They are still a good way to show how far we have to go. +* **SlotMapBenchmark**: This is a low-level benchmark of the various SlotMap types in Rhino. +Tiny changes in SlotMap performance affect property access in Rhino and translate into big +changes in the other benchmarks. +* **PropertyBenchmark**: This is a micro-benchmark that uses JavaScript code to test the efficiency +of object property access. +* **ObjectBenchmark**: These serve a similar purpose to PropertyBenchmark and perhaps we should +have deleted these by now. +* **BuiltinBenchmark**: This tries to measure the relative performance of the various ways to create +native JavaScript objects in Java -- the reflection-based method, the IdScriptableObject that is used +for many internal objects, and lambda functions. +* **MathBenchmark**: This is a vehicle for quickly testing low-level math operations. \ No newline at end of file diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index db9cd8ffb..47dbfec1f 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -9,13 +9,13 @@ dependencies { } jmh { - // use this to include only some - //includes = ['SlotMap'] + if (System.getenv('BENCHMARK') != null) { + includes = [System.getenv('BENCHMARK')] + } benchmarkMode = ['avgt'] fork = 1 - //timeUnit = 'ns' - iterations = 3 - timeOnIteration = '5s' + iterations = 5 + timeOnIteration = '2s' warmupIterations = 3 warmup = '5s' } diff --git a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/BuiltinBenchmark.java b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/BuiltinBenchmark.java index cd866dc2a..2e943bb44 100644 --- a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/BuiltinBenchmark.java +++ b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/BuiltinBenchmark.java @@ -1,6 +1,7 @@ package org.mozilla.javascript.benchmarks; import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.TimeUnit; import org.mozilla.javascript.Context; import org.mozilla.javascript.IdFunctionObject; import org.mozilla.javascript.IdScriptableObject; @@ -13,8 +14,8 @@ import org.mozilla.javascript.Undefined; import org.mozilla.javascript.annotations.*; import org.openjdk.jmh.annotations.*; +@OutputTimeUnit(TimeUnit.MICROSECONDS) public class BuiltinBenchmark { - @State(Scope.Thread) public static class AbstractClassState { diff --git a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/MathBenchmark.java b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/MathBenchmark.java index 47992dcf4..427e5a989 100644 --- a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/MathBenchmark.java +++ b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/MathBenchmark.java @@ -2,6 +2,7 @@ package org.mozilla.javascript.benchmarks; import java.io.FileReader; import java.io.IOException; +import java.util.concurrent.TimeUnit; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptRuntime; @@ -9,6 +10,7 @@ import org.mozilla.javascript.Scriptable; import org.mozilla.javascript.ScriptableObject; import org.openjdk.jmh.annotations.*; +@OutputTimeUnit(TimeUnit.NANOSECONDS) public class MathBenchmark { @State(Scope.Thread) public static class MathState { diff --git a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/ObjectBenchmark.java b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/ObjectBenchmark.java index 54cb0e25b..53ed05195 100644 --- a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/ObjectBenchmark.java +++ b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/ObjectBenchmark.java @@ -3,6 +3,7 @@ package org.mozilla.javascript.benchmarks; import java.io.FileReader; import java.io.IOException; import java.util.Random; +import java.util.concurrent.TimeUnit; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.Scriptable; @@ -10,6 +11,7 @@ import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.tools.shell.Global; import org.openjdk.jmh.annotations.*; +@OutputTimeUnit(TimeUnit.MICROSECONDS) public class ObjectBenchmark { static final Random rand = new Random(); diff --git a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/PropertyBenchmark.java b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/PropertyBenchmark.java index 03431cdde..329e39a15 100644 --- a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/PropertyBenchmark.java +++ b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/PropertyBenchmark.java @@ -2,6 +2,7 @@ package org.mozilla.javascript.benchmarks; import java.io.FileReader; import java.io.IOException; +import java.util.concurrent.TimeUnit; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptRuntime; @@ -9,6 +10,7 @@ import org.mozilla.javascript.Scriptable; import org.mozilla.javascript.ScriptableObject; import org.openjdk.jmh.annotations.*; +@OutputTimeUnit(TimeUnit.NANOSECONDS) public class PropertyBenchmark { @State(Scope.Thread) public static class PropertyState { diff --git a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SlotMapBenchmark.java b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SlotMapBenchmark.java index 61085c8b6..a841daf13 100644 --- a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SlotMapBenchmark.java +++ b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SlotMapBenchmark.java @@ -1,12 +1,14 @@ package org.mozilla.javascript.benchmarks; import java.util.Random; +import java.util.concurrent.TimeUnit; import org.mozilla.javascript.EmbeddedSlotMap; import org.mozilla.javascript.HashSlotMap; import org.mozilla.javascript.Slot; import org.mozilla.javascript.SlotMap; import org.openjdk.jmh.annotations.*; +@OutputTimeUnit(TimeUnit.NANOSECONDS) public class SlotMapBenchmark { // Fixed seed for repeatability private static final Random rand = new Random(0); diff --git a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SunSpiderBenchmark.java b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SunSpiderBenchmark.java index b89880a97..55a181b75 100644 --- a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SunSpiderBenchmark.java +++ b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SunSpiderBenchmark.java @@ -11,10 +11,7 @@ import org.openjdk.jmh.annotations.*; public class SunSpiderBenchmark { private static final String TEST_BASE = "testsrc/benchmarks/sunspider-1.0/"; - private static final int WARMUP_RUNS = 3; - private static final int MEASUREMENT_RUNS = 3; - private static final int DURATION_SECONDS = 5; - + @OutputTimeUnit(TimeUnit.MICROSECONDS) abstract static class AbstractState { Context cx; Scriptable scope; @@ -56,12 +53,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object threeDCube(ThreeDCubeState state) { return state.run(); } @@ -74,12 +65,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object threeDMorph(ThreeDMorphState state) { return state.run(); } @@ -92,12 +77,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object threeDRayTrace(ThreeDRayState state) { return state.run(); } @@ -110,12 +89,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object accessBinaryTrees(AccessBinaryTreesState state) { return state.run(); } @@ -128,12 +101,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object accessFannkuch(AccessFannkuchState state) { return state.run(); } @@ -146,12 +113,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object accessNBody(AccessNBodyState state) { return state.run(); } @@ -164,12 +125,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object accessNsieve(AccessFannAccessNsieveState state) { return state.run(); } @@ -182,12 +137,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object bitops3BitBitsInByte(Bitops3BitState state) { return state.run(); } @@ -200,12 +149,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object bitopsBitsInByte(BitopsBitsState state) { return state.run(); } @@ -218,12 +161,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object bitopsBitwiseAnd(BitopsAndState state) { return state.run(); } @@ -236,12 +173,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object bitopsNsieveBits(BitopsNsieveState state) { return state.run(); } @@ -254,12 +185,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object controlflowRecursive(RecursiveState state) { return state.run(); } @@ -272,12 +197,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object cryptoAes(CryptoAesState state) { return state.run(); } @@ -290,12 +209,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object cryptoMd5(CryptoMd5State state) { return state.run(); } @@ -308,12 +221,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object cryptoSha1(CryptoShaState state) { return state.run(); } @@ -326,12 +233,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object dateFormatToFte(DateFormatToFteState state) { return state.run(); } @@ -344,12 +245,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object dateFormatXparb(DateFormatXparbState state) { return state.run(); } @@ -362,12 +257,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object mathCordic(MathCordicState state) { return state.run(); } @@ -380,12 +269,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object mathPartialSums(MathPartialState state) { return state.run(); } @@ -398,12 +281,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object mathSpectralNorm(MathSpectralNormState state) { return state.run(); } @@ -416,12 +293,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object regexpDna(RegexpState state) { return state.run(); } @@ -434,12 +305,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object stringBase64(StringBase64State state) { return state.run(); } @@ -452,12 +317,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object stringFasta(StringFastaState state) { return state.run(); } @@ -470,12 +329,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object stringTagcloud(StringTagcloudState state) { return state.run(); } @@ -488,12 +341,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object stringUnpackCode(StringUnpackState state) { return state.run(); } @@ -506,12 +353,6 @@ public class SunSpiderBenchmark { } @Benchmark - @OutputTimeUnit(TimeUnit.MILLISECONDS) - @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS) - @Measurement( - iterations = MEASUREMENT_RUNS, - time = DURATION_SECONDS, - timeUnit = TimeUnit.SECONDS) public Object stringValidateInput(StringValidateState state) { return state.run(); } diff --git a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/V8Benchmark.java b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/V8Benchmark.java index 00596b79a..5b7f92a48 100644 --- a/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/V8Benchmark.java +++ b/benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/V8Benchmark.java @@ -2,12 +2,14 @@ package org.mozilla.javascript.benchmarks; import java.io.FileReader; import java.io.IOException; +import java.util.concurrent.TimeUnit; import org.mozilla.javascript.Callable; import org.mozilla.javascript.Context; import org.mozilla.javascript.Scriptable; import org.mozilla.javascript.ScriptableObject; import org.openjdk.jmh.annotations.*; +@OutputTimeUnit(TimeUnit.MICROSECONDS) public class V8Benchmark { static Object[] emptyArgs = new Object[] {};