Bug 998206. Add --shuffle option to runreftests.py and 'mach reftest'. r=dbaron

--HG--
extra : rebase_source : 8270c601ad8d5afe4b598e78ed8d84a3c1385f1f

layout/tools/reftest/mach_commands.py

@@ -206,8 +206,8 @@ class ReftestRunner(MozbuildObject):
         return reftest.run_remote_reftests(parser, options, args)
 
     def run_desktop_test(self, test_file=None, filter=None, suite=None,
-            debugger=None, parallel=False, e10s=False, this_chunk=None,
-            total_chunks=None):
+            debugger=None, parallel=False, shuffle=False,
+            e10s=False, this_chunk=None, total_chunks=None):
         """Runs a reftest.
 
         test_file is a path to a test file. It can be a relative path from the
@@ -224,6 +224,8 @@ class ReftestRunner(MozbuildObject):
         debugger to run.
 
         parallel indicates whether tests should be run in parallel or not.
+
+        shuffle indicates whether to run tests in random order.
         """
 
         if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc'):
@@ -249,6 +251,9 @@ class ReftestRunner(MozbuildObject):
         if parallel:
             extra_args.append('--run-tests-in-parallel')
 
+        if shuffle:
+            extra_args.append('--shuffle')
+
         if e10s:
             extra_args.append('--e10s')
 
@@ -289,6 +294,10 @@ def ReftestCommand(func):
         help='Run tests in parallel.')
     func = parallel(func)
 
+    shuffle = CommandArgument('--shuffle', action='store_true',
+        help='Run tests in random order.')
+    func = shuffle(func)
+
     e10s = CommandArgument('--e10s', action='store_true',
         help='Use content processes.')
     func = e10s(func)

layout/tools/reftest/reftest.js

@@ -41,6 +41,7 @@ var gLoadTimeout = 0;
 var gTimeoutHook = null;
 var gRemote = false;
 var gIgnoreWindowSize = false;
+var gShuffle = false;
 var gTotalChunks = 0;
 var gThisChunk = 0;
 var gContainingWindow = null;
@@ -432,6 +433,17 @@ function StartHTTPServer()
     gHttpServerPort = gServer.identity.primaryPort;
 }
 
+// Perform a Fisher-Yates shuffle of the array.
+function Shuffle(array)
+{
+    for (var i = array.length - 1; i > 0; i--) {
+        var j = Math.floor(Math.random() * (i + 1));
+        var temp = array[i];
+        array[i] = array[j];
+        array[j] = temp;
+    }
+}
+
 function StartTests()
 {
     var uri;
@@ -450,6 +462,12 @@ function StartTests()
         gNoCanvasCache = false;
     }
 
+    try {
+      gShuffle = prefs.getBoolPref("reftest.shuffle");
+    } catch (e) {
+      gShuffle = false;
+    }
+
     try {
         gRunSlowTests = prefs.getIntPref("reftest.skipslowtests");
     } catch(e) {
@@ -484,6 +502,11 @@ function StartTests()
         DoneTests();
     }
 #endif
+
+    if (gShuffle) {
+        gNoCanvasCache = true;
+    }
+
     try {
         ReadTopManifest(uri);
         BuildUseCounts();
@@ -522,6 +545,11 @@ function StartTests()
             gDumpLog("REFTEST INFO | Running chunk " + gThisChunk + " out of " + gTotalChunks + " chunks.  ");
             gDumpLog("tests " + (start+1) + "-" + end + "/" + gURLs.length + "\n");
         }
+
+        if (gShuffle) {
+            Shuffle(gURLs);
+        }
+
         gTotalTests = gURLs.length;
 
         if (!gTotalTests)

layout/tools/reftest/runreftest.py

@@ -161,6 +161,8 @@ class RefTest(object):
       prefs['reftest.ignoreWindowSize'] = True
     if options.filter:
       prefs['reftest.filter'] = options.filter
+    if options.shuffle:
+      prefs['reftest.shuffle'] = True
     prefs['reftest.focusFilterMode'] = options.focusFilterMode
 
     if options.e10s:
@@ -441,6 +443,11 @@ class ReftestOptions(OptionParser):
                            "only test items that have a matching test URL will be run.")
     defaults["filter"] = None
 
+    self.add_option("--shuffle",
+                    action = "store_true", dest = "shuffle",
+                    help = "run reftests in random order")
+    defaults["shuffle"] = False
+
     self.add_option("--focus-filter-mode",
                     action = "store", type = "string", dest = "focusFilterMode",
                     help = "filters tests to run by whether they require focus. "