Bug 1196831 - Add 'run-until-failure' and 'repeat' flags to reftest. r=jmaher

--HG--
extra : commitid : Ig2JmBnAcrw
extra : rebase_source : ee3026bd82263dd77583dfd3d576eb622fa582bc

layout/tools/reftest/reftest.js

@@ -46,6 +46,8 @@ var gTimeoutHook = null;
 var gRemote = false;
 var gIgnoreWindowSize = false;
 var gShuffle = false;
+var gRepeat = null;
+var gRunUntilFailure = false;
 var gTotalChunks = 0;
 var gThisChunk = 0;
 var gContainingWindow = null;
@@ -169,6 +171,16 @@ var gVerbose = false;
 // manifest URL (yet!).
 var gDumpedConditionSandbox = false;
 
+function HasUnexpectedResult()
+{
+    return gTestResults.Exception > 0 ||
+           gTestResults.FailedLoad > 0 ||
+           gTestResults.UnexpectedFail > 0 ||
+           gTestResults.UnexpectedPass > 0 ||
+           gTestResults.AssertionUnexpected > 0 ||
+           gTestResults.AssertionUnexpectedFixed > 0;
+}
+
 function LogWarning(str)
 {
     gDumpLog("REFTEST INFO | " + str + "\n");
@@ -468,6 +480,22 @@ function StartTests()
       gShuffle = false;
     }
 
+    try {
+      gRunUntilFailure = prefs.getBoolPref("reftest.runUntilFailure");
+    } catch (e) {
+      gRunUntilFailure = false;
+    }
+
+    // When we repeat this function is called again, so really only want to set
+    // gRepeat once.
+    if (gRepeat == null) {
+      try {
+        gRepeat = prefs.getIntPref("reftest.repeat");
+      } catch (e) {
+        gRepeat = 0;
+      }
+    }
+
     try {
         gRunSlowTests = prefs.getIntPref("reftest.skipslowtests");
     } catch(e) {
@@ -479,6 +507,7 @@ function StartTests()
     }
 
     gURLs = [];
+    gManifestsLoaded = {};
 
     try {
         var manifests = JSON.parse(prefs.getCharPref("reftest.manifests"));
@@ -1226,11 +1255,15 @@ function StartCurrentTest()
         }
     }
 
-    if (gURLs.length == 0) {
+    if ((gURLs.length == 0 && gRepeat == 0) ||
+        (gRunUntilFailure && HasUnexpectedResult())) {
         RestoreChangedPreferences();
         DoneTests();
-    }
-    else {
+    } else if (gURLs.length == 0 && gRepeat > 0) {
+        // Repeat
+        gRepeat--;
+        StartTests();
+    } else {
         gDumpLog("REFTEST TEST-START | " + gURLs[0].prettyPath + "\n");
         if (gURLs[0].chaosMode) {
             gWindowUtils.enterChaosMode();

layout/tools/reftest/reftestcommandline.py

@@ -150,6 +150,20 @@ class ReftestArgumentsParser(argparse.ArgumentParser):
                           dest="shuffle",
                           help="run reftests in random order")
 
+        self.add_argument("--run-until-failure",
+                          action="store_true",
+                          default=False,
+                          dest="runUntilFailure",
+                          help="stop running on the first failure. Useful for RR recordings.")
+
+        self.add_argument("--repeat",
+                          action="store",
+                          type=int,
+                          default=0,
+                          dest="repeat",
+                          help="number of times the select test(s) will be executed. Useful for "
+                          "finding intermittent failures.")
+
         self.add_argument("--focus-filter-mode",
                           action="store",
                           type=str,

layout/tools/reftest/runreftest.py

@@ -250,6 +250,10 @@ class RefTest(object):
             prefs['reftest.ignoreWindowSize'] = True
         if options.shuffle:
             prefs['reftest.shuffle'] = True
+        if options.repeat:
+            prefs['reftest.repeat'] = options.repeat
+        if options.runUntilFailure:
+            prefs['reftest.runUntilFailure'] = True
         prefs['reftest.focusFilterMode'] = options.focusFilterMode
         prefs['reftest.manifests'] = json.dumps(manifests)