Bug 1405141 - Add support for rerunning web-platform-tests without restarting, r=gbrown

This adds a --rerun=N flag which runs each selected test N times in a row.

MozReview-Commit-ID: 4dGuPDkjWeZ

--HG--
extra : rebase_source : 0acd1393665af9a95023a547f869a57b48249931

testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py

@@ -248,7 +248,7 @@ RunnerManagerState = _RunnerManagerState()
 
 class TestRunnerManager(threading.Thread):
     def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs,
-                 executor_cls, executor_kwargs, stop_flag, pause_after_test=False,
+                 executor_cls, executor_kwargs, stop_flag, rerun=1, pause_after_test=False,
                  pause_on_unexpected=False, restart_on_unexpected=True, debug_info=None):
         """Thread that owns a single TestRunner process and any processes required
         by the TestRunner (e.g. the Firefox binary).
@@ -279,6 +279,8 @@ class TestRunnerManager(threading.Thread):
         self.parent_stop_flag = stop_flag
         self.child_stop_flag = multiprocessing.Event()
 
+        self.rerun = rerun
+        self.run_count = 0
         self.pause_after_test = pause_after_test
         self.pause_on_unexpected = pause_on_unexpected
         self.restart_on_unexpected = restart_on_unexpected
@@ -502,10 +504,14 @@ class TestRunnerManager(threading.Thread):
                     self.logger.info("No more tests")
                     return None, None, None
             test = test_group.popleft()
+        self.run_count = 0
         return test, test_group, group_metadata
 
 
     def run_test(self):
+        self.run_count += 1
+        if self.rerun > 1:
+            self.logger.info("Run %d/%d" % (self.run_count, self.rerun))
         assert isinstance(self.state, RunnerManagerState.running)
         assert self.state.test is not None
 
@@ -579,7 +585,7 @@ class TestRunnerManager(threading.Thread):
             self.logger.info("Pausing until the browser exits")
             self.send_message("wait")
         else:
-            return self.after_test_end(restart_before_next)
+            return self.after_test_end(test, restart_before_next)
 
     def wait_finished(self):
         assert isinstance(self.state, RunnerManagerState.running)
@@ -589,14 +595,19 @@ class TestRunnerManager(threading.Thread):
 
         return self.after_test_end(True)
 
-    def after_test_end(self, restart):
+    def after_test_end(self, test, restart):
         assert isinstance(self.state, RunnerManagerState.running)
-        test, test_group, group_metadata = self.get_next_test()
-        if test is None:
-            return RunnerManagerState.stop()
-        if test_group != self.state.test_group:
-            # We are starting a new group of tests, so force a restart
-            restart = True
+        if self.run_count == self.rerun:
+            test, test_group, group_metadata = self.get_next_test()
+            if test is None:
+                return RunnerManagerState.stop()
+            if test_group != self.state.test_group:
+                # We are starting a new group of tests, so force a restart
+                restart = True
+        else:
+            test = test
+            test_group = self.state.test_group
+            group_metadata = self.state.group_metadata
         if restart:
             return RunnerManagerState.restarting(test, test_group, group_metadata)
         else:
@@ -685,6 +696,7 @@ class ManagerGroup(object):
     def __init__(self, suite_name, size, test_source_cls, test_source_kwargs,
                  browser_cls, browser_kwargs,
                  executor_cls, executor_kwargs,
+                 rerun=1,
                  pause_after_test=False,
                  pause_on_unexpected=False,
                  restart_on_unexpected=True,
@@ -702,6 +714,7 @@ class ManagerGroup(object):
         self.pause_on_unexpected = pause_on_unexpected
         self.restart_on_unexpected = restart_on_unexpected
         self.debug_info = debug_info
+        self.rerun = rerun
 
         self.pool = set()
         # Event that is polled by threads so that they can gracefully exit in the face
@@ -734,6 +747,7 @@ class ManagerGroup(object):
                                         self.executor_cls,
                                         self.executor_kwargs,
                                         self.stop_flag,
+                                        self.rerun,
                                         self.pause_after_test,
                                         self.pause_on_unexpected,
                                         self.restart_on_unexpected,

testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py

@@ -93,8 +93,10 @@ scheme host and port.""")
     debugging_group.add_argument('--debugger', const="__default__", nargs="?",
                                  help="run under a debugger, e.g. gdb or valgrind")
     debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
+    debugging_group.add_argument("--rerun", action="store", type=int, default=1,
+                                 help="Number of times to re run each test without restarts")
     debugging_group.add_argument("--repeat", action="store", type=int, default=1,
-                                 help="Number of times to run the tests")
+                                 help="Number of times to run the tests, restarting between each run")
     debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
                                  help="Run tests in a loop until one returns an unexpected result")
     debugging_group.add_argument('--pause-after-test', action="store_true", default=None,

testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py

@@ -241,6 +241,7 @@ def run_tests(config, test_paths, product, **kwargs):
                                       browser_kwargs,
                                       executor_cls,
                                       executor_kwargs,
+                                      kwargs["rerun"],
                                       kwargs["pause_after_test"],
                                       kwargs["pause_on_unexpected"],
                                       kwargs["restart_on_unexpected"],