Rationalize benchmark setup and time units

* Make sure that all the benchmarks run using "gradle jmh"
* Make sure that each has a reasonable default time unit
* Add a way to change which benchmarks run via the command-line
* Add a README for the benchmarks

benchmarks/README.md

@@ -0,0 +1,54 @@
+# Rhino Benchmarks
+
+This directory contains a collection of various benchmarks that have been added to the Rhino
+project over the years. They have all been collected here and run using the JMH framework.
+
+## Running the Benchmarks
+
+To run all the benchmarks that exist, simply run, from the top-level directory:
+
+    ./gradlew jmh
+
+In addition, the environment variable BENCHMARK may be used to restrict which benchmarks are
+run -- it is a regular expression that matches the names of the benchmarks. 
+
+For example, to run only the SunSpider and V8 benchmarks, you can run:
+
+    BENCHMARK=V8|SunSpider ./gradlew jmh
+
+Running all the benchmarks takes about half an hour, so this is a valuable thing to do!
+
+## How the Benchmarks work
+
+Java, with its just-in-time compilation pipeline, bytecode generation in Rhino, and
+Java's super-complex garbage collector, is sensitive to warm up time and as a result, Rhino has 
+more variation between runs than other JavaScript engines. To get a repeatable result, we use
+the JMH framework, which runs each benchmark many times and does other "black hole" protection
+operations to try and get an accurate result. For this reason, these benchmarks take a lot longer
+to run than your favorite on-line JavaScript benchmarking web site.
+
+The purpose of these benchmarks has historically been to make Rhino perform better in server-side
+environments, so they all run at the maximum optimization level (9). Since so many people also
+use Rhino in interpreted mode, if there's an interest in benchmarking that too then we can
+always adjust these tests.
+
+## Benchmark Notes
+
+Here are a few notes on the specific benchmarks:
+
+* **SunSpiderBenchmark**: These are the venerable JavaScript benchmarks that have been used
+for a long time. They test both low-level operations like math operations, as well as higher-level
+tasks.
+* **V8Benchmark**: These are Google's V8 benchmarks, which may have been created to show how 
+efficient V8 is. They are still a good way to show how far we have to go.
+* **SlotMapBenchmark**: This is a low-level benchmark of the various SlotMap types in Rhino.
+Tiny changes in SlotMap performance affect property access in Rhino and translate into big
+changes in the other benchmarks.
+* **PropertyBenchmark**: This is a micro-benchmark that uses JavaScript code to test the efficiency
+of object property access.
+* **ObjectBenchmark**: These serve a similar purpose to PropertyBenchmark and perhaps we should
+have deleted these by now.
+* **BuiltinBenchmark**: This tries to measure the relative performance of the various ways to create
+native JavaScript objects in Java -- the reflection-based method, the IdScriptableObject that is used
+for many internal objects, and lambda functions.
+* **MathBenchmark**: This is a vehicle for quickly testing low-level math operations.

benchmarks/build.gradle

@@ -9,13 +9,13 @@ dependencies {
 }
 
 jmh {
-    // use this to include only some
-    //includes = ['SlotMap']
+    if (System.getenv('BENCHMARK') != null) {
+      includes = [System.getenv('BENCHMARK')]
+    }
     benchmarkMode = ['avgt']
     fork = 1
-    //timeUnit = 'ns'
-    iterations = 3
-    timeOnIteration = '5s'
+    iterations = 5
+    timeOnIteration = '2s'
     warmupIterations = 3
     warmup = '5s'
 }

benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/BuiltinBenchmark.java

@@ -1,6 +1,7 @@
 package org.mozilla.javascript.benchmarks;
 
 import java.lang.reflect.InvocationTargetException;
+import java.util.concurrent.TimeUnit;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.IdFunctionObject;
 import org.mozilla.javascript.IdScriptableObject;
@@ -13,8 +14,8 @@ import org.mozilla.javascript.Undefined;
 import org.mozilla.javascript.annotations.*;
 import org.openjdk.jmh.annotations.*;
 
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
 public class BuiltinBenchmark {
-
     @State(Scope.Thread)
     public static class AbstractClassState {
 

benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/MathBenchmark.java

@@ -2,6 +2,7 @@ package org.mozilla.javascript.benchmarks;
 
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptRuntime;
@@ -9,6 +10,7 @@ import org.mozilla.javascript.Scriptable;
 import org.mozilla.javascript.ScriptableObject;
 import org.openjdk.jmh.annotations.*;
 
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
 public class MathBenchmark {
     @State(Scope.Thread)
     public static class MathState {

benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/ObjectBenchmark.java

@@ -3,6 +3,7 @@ package org.mozilla.javascript.benchmarks;
 import java.io.FileReader;
 import java.io.IOException;
 import java.util.Random;
+import java.util.concurrent.TimeUnit;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.Scriptable;
@@ -10,6 +11,7 @@ import org.mozilla.javascript.ScriptableObject;
 import org.mozilla.javascript.tools.shell.Global;
 import org.openjdk.jmh.annotations.*;
 
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
 public class ObjectBenchmark {
     static final Random rand = new Random();
 

benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/PropertyBenchmark.java

@@ -2,6 +2,7 @@ package org.mozilla.javascript.benchmarks;
 
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptRuntime;
@@ -9,6 +10,7 @@ import org.mozilla.javascript.Scriptable;
 import org.mozilla.javascript.ScriptableObject;
 import org.openjdk.jmh.annotations.*;
 
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
 public class PropertyBenchmark {
     @State(Scope.Thread)
     public static class PropertyState {

benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SlotMapBenchmark.java

@@ -1,12 +1,14 @@
 package org.mozilla.javascript.benchmarks;
 
 import java.util.Random;
+import java.util.concurrent.TimeUnit;
 import org.mozilla.javascript.EmbeddedSlotMap;
 import org.mozilla.javascript.HashSlotMap;
 import org.mozilla.javascript.Slot;
 import org.mozilla.javascript.SlotMap;
 import org.openjdk.jmh.annotations.*;
 
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
 public class SlotMapBenchmark {
     // Fixed seed for repeatability
     private static final Random rand = new Random(0);

benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/SunSpiderBenchmark.java

@@ -11,10 +11,7 @@ import org.openjdk.jmh.annotations.*;
 public class SunSpiderBenchmark {
     private static final String TEST_BASE = "testsrc/benchmarks/sunspider-1.0/";
 
-    private static final int WARMUP_RUNS = 3;
-    private static final int MEASUREMENT_RUNS = 3;
-    private static final int DURATION_SECONDS = 5;
-
+    @OutputTimeUnit(TimeUnit.MICROSECONDS)
     abstract static class AbstractState {
         Context cx;
         Scriptable scope;
@@ -56,12 +53,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object threeDCube(ThreeDCubeState state) {
             return state.run();
         }
@@ -74,12 +65,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object threeDMorph(ThreeDMorphState state) {
             return state.run();
         }
@@ -92,12 +77,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object threeDRayTrace(ThreeDRayState state) {
             return state.run();
         }
@@ -110,12 +89,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object accessBinaryTrees(AccessBinaryTreesState state) {
             return state.run();
         }
@@ -128,12 +101,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object accessFannkuch(AccessFannkuchState state) {
             return state.run();
         }
@@ -146,12 +113,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object accessNBody(AccessNBodyState state) {
             return state.run();
         }
@@ -164,12 +125,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object accessNsieve(AccessFannAccessNsieveState state) {
             return state.run();
         }
@@ -182,12 +137,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object bitops3BitBitsInByte(Bitops3BitState state) {
             return state.run();
         }
@@ -200,12 +149,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object bitopsBitsInByte(BitopsBitsState state) {
             return state.run();
         }
@@ -218,12 +161,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object bitopsBitwiseAnd(BitopsAndState state) {
             return state.run();
         }
@@ -236,12 +173,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object bitopsNsieveBits(BitopsNsieveState state) {
             return state.run();
         }
@@ -254,12 +185,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object controlflowRecursive(RecursiveState state) {
             return state.run();
         }
@@ -272,12 +197,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object cryptoAes(CryptoAesState state) {
             return state.run();
         }
@@ -290,12 +209,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object cryptoMd5(CryptoMd5State state) {
             return state.run();
         }
@@ -308,12 +221,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object cryptoSha1(CryptoShaState state) {
             return state.run();
         }
@@ -326,12 +233,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object dateFormatToFte(DateFormatToFteState state) {
             return state.run();
         }
@@ -344,12 +245,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object dateFormatXparb(DateFormatXparbState state) {
             return state.run();
         }
@@ -362,12 +257,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object mathCordic(MathCordicState state) {
             return state.run();
         }
@@ -380,12 +269,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object mathPartialSums(MathPartialState state) {
             return state.run();
         }
@@ -398,12 +281,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object mathSpectralNorm(MathSpectralNormState state) {
             return state.run();
         }
@@ -416,12 +293,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object regexpDna(RegexpState state) {
             return state.run();
         }
@@ -434,12 +305,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object stringBase64(StringBase64State state) {
             return state.run();
         }
@@ -452,12 +317,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object stringFasta(StringFastaState state) {
             return state.run();
         }
@@ -470,12 +329,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object stringTagcloud(StringTagcloudState state) {
             return state.run();
         }
@@ -488,12 +341,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object stringUnpackCode(StringUnpackState state) {
             return state.run();
         }
@@ -506,12 +353,6 @@ public class SunSpiderBenchmark {
         }
 
         @Benchmark
-        @OutputTimeUnit(TimeUnit.MILLISECONDS)
-        @Warmup(iterations = WARMUP_RUNS, time = DURATION_SECONDS, timeUnit = TimeUnit.SECONDS)
-        @Measurement(
-                iterations = MEASUREMENT_RUNS,
-                time = DURATION_SECONDS,
-                timeUnit = TimeUnit.SECONDS)
         public Object stringValidateInput(StringValidateState state) {
             return state.run();
         }

benchmarks/src/jmh/java/org/mozilla/javascript/benchmarks/V8Benchmark.java

@@ -2,12 +2,14 @@ package org.mozilla.javascript.benchmarks;
 
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 import org.mozilla.javascript.Callable;
 import org.mozilla.javascript.Context;
 import org.mozilla.javascript.Scriptable;
 import org.mozilla.javascript.ScriptableObject;
 import org.openjdk.jmh.annotations.*;
 
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
 public class V8Benchmark {
     static Object[] emptyArgs = new Object[] {};