Bug 916797 - --run-until-failure mochitest option should support running multiple tests. r=jmaher

testing/mochitest/browser-test.js

@@ -185,9 +185,6 @@ Tester.prototype = {
     var failCount = this.tests.reduce(function(a, f) a + f.failCount, 0);
     var todoCount = this.tests.reduce(function(a, f) a + f.todoCount, 0);
 
-    if (failCount > 0 && this.runUntilFailure)
-      this.repeat = 0;
-
     if (this.repeat > 0) {
       --this.repeat;
       this.currentTestIndex = -1;
@@ -222,6 +219,12 @@ Tester.prototype = {
     }
   },
 
+  haltTests: function Tester_haltTests() {
+    // Do not run any further tests
+    this.currentTestIndex = this.tests.length - 1;
+    this.repeat = 0;
+  },
+
   observe: function Tester_observe(aSubject, aTopic, aData) {
     if (!aTopic) {
       this.onConsoleMessage(aSubject);
@@ -354,6 +357,10 @@ Tester.prototype = {
       this.dumper.dump("INFO TEST-END | " + this.currentTest.path + " | finished in " + time + "ms\n");
       this.currentTest.setDuration(time);
 
+      if (this.runUntilFailure && this.currentTest.failCount > 0) {
+        this.haltTests();
+      }
+
       testScope.destroy();
       this.currentTest.scope = null;
     }

testing/mochitest/mach_commands.py

@@ -392,9 +392,9 @@ def MochitestCommand(func):
     func = repeat(func)
 
     runUntilFailure = CommandArgument("--run-until-failure", action='store_true',
-        help='Run a test repeatedly and stops on the first time the test fails. ' \
-             'Only available when running a single test. Default cap is 30 runs, ' \
-             'which can be overwritten with the --repeat parameter.')
+        help='Run tests repeatedly and stops on the first time a test fails. ' \
+             'Default cap is 30 runs, which can be overwritten ' \
+             'with the --repeat parameter.')
     func = runUntilFailure(func)
 
     slow = CommandArgument('--slow', action='store_true',

testing/mochitest/mochitest_options.py

@@ -274,9 +274,8 @@ class MochitestOptions(optparse.OptionParser):
         [["--run-until-failure"],
         { "action": "store_true",
           "dest": "runUntilFailure",
-          "help": "Run a test repeatedly and stops on the first time the test fails. "
-                "Only available when running a single test. Default cap is 30 runs, "
-                "which can be overwritten with the --repeat parameter.",
+          "help": "Run tests repeatedly and stops on the first time a test fails. "
+                "Default cap is 30 runs, which can be overwritten with the --repeat parameter.",
           "default": False,
         }],
         [["--run-only-tests"],
@@ -495,8 +494,6 @@ class MochitestOptions(optparse.OptionParser):
                            mochitest.immersiveHelperPath)
 
         if options.runUntilFailure:
-            if not os.path.isfile(os.path.join(mochitest.oldcwd, os.path.dirname(__file__), mochitest.getTestRoot(options), options.testPath)):
-                self.error("--run-until-failure can only be used together with --test-path specifying a single test.")
             if not options.repeat:
                 options.repeat = 29
 

testing/mochitest/tests/SimpleTest/TestRunner.js

@@ -214,6 +214,10 @@ TestRunner.error = function(msg) {
         dump(msg + "\n");
     }
 
+    if (TestRunner.runUntilFailure) {
+      TestRunner._haltTests = true;
+    }
+
     if (TestRunner.debugOnFailure) {
       // You've hit this line because you requested to break into the
       // debugger upon a testcase failure on your test run.
@@ -366,10 +370,7 @@ TestRunner.runNextTest = function() {
              TestRunner.onComplete();
          }
 
-        var failCount = parseInt($("fail-count").innerHTML);
-        var stopLooping = failCount > 0 && TestRunner.runUntilFailure;
-
-        if (TestRunner._currentLoop <= TestRunner.repeat && !stopLooping) {
+        if (TestRunner._currentLoop <= TestRunner.repeat && !TestRunner._haltTests) {
           TestRunner._currentLoop++;
           TestRunner.resetTests(TestRunner._urls);
           TestRunner._loopIsRestarting = true;