diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py index c4655b7fae3f..c980ca7349c6 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py @@ -248,7 +248,7 @@ RunnerManagerState = _RunnerManagerState() class TestRunnerManager(threading.Thread): def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs, - executor_cls, executor_kwargs, stop_flag, pause_after_test=False, + executor_cls, executor_kwargs, stop_flag, rerun=1, pause_after_test=False, pause_on_unexpected=False, restart_on_unexpected=True, debug_info=None): """Thread that owns a single TestRunner process and any processes required by the TestRunner (e.g. the Firefox binary). @@ -279,6 +279,8 @@ class TestRunnerManager(threading.Thread): self.parent_stop_flag = stop_flag self.child_stop_flag = multiprocessing.Event() + self.rerun = rerun + self.run_count = 0 self.pause_after_test = pause_after_test self.pause_on_unexpected = pause_on_unexpected self.restart_on_unexpected = restart_on_unexpected @@ -502,10 +504,14 @@ class TestRunnerManager(threading.Thread): self.logger.info("No more tests") return None, None, None test = test_group.popleft() + self.run_count = 0 return test, test_group, group_metadata def run_test(self): + self.run_count += 1 + if self.rerun > 1: + self.logger.info("Run %d/%d" % (self.run_count, self.rerun)) assert isinstance(self.state, RunnerManagerState.running) assert self.state.test is not None @@ -579,7 +585,7 @@ class TestRunnerManager(threading.Thread): self.logger.info("Pausing until the browser exits") self.send_message("wait") else: - return self.after_test_end(restart_before_next) + return self.after_test_end(test, restart_before_next) def wait_finished(self): assert isinstance(self.state, RunnerManagerState.running) @@ -589,14 +595,19 @@ class TestRunnerManager(threading.Thread): return self.after_test_end(True) - def after_test_end(self, restart): + def after_test_end(self, test, restart): assert isinstance(self.state, RunnerManagerState.running) - test, test_group, group_metadata = self.get_next_test() - if test is None: - return RunnerManagerState.stop() - if test_group != self.state.test_group: - # We are starting a new group of tests, so force a restart - restart = True + if self.run_count == self.rerun: + test, test_group, group_metadata = self.get_next_test() + if test is None: + return RunnerManagerState.stop() + if test_group != self.state.test_group: + # We are starting a new group of tests, so force a restart + restart = True + else: + test = test + test_group = self.state.test_group + group_metadata = self.state.group_metadata if restart: return RunnerManagerState.restarting(test, test_group, group_metadata) else: @@ -685,6 +696,7 @@ class ManagerGroup(object): def __init__(self, suite_name, size, test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, + rerun=1, pause_after_test=False, pause_on_unexpected=False, restart_on_unexpected=True, @@ -702,6 +714,7 @@ class ManagerGroup(object): self.pause_on_unexpected = pause_on_unexpected self.restart_on_unexpected = restart_on_unexpected self.debug_info = debug_info + self.rerun = rerun self.pool = set() # Event that is polled by threads so that they can gracefully exit in the face @@ -734,6 +747,7 @@ class ManagerGroup(object): self.executor_cls, self.executor_kwargs, self.stop_flag, + self.rerun, self.pause_after_test, self.pause_on_unexpected, self.restart_on_unexpected, diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py index 0016f4ed0169..3529f345f202 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py @@ -93,8 +93,10 @@ scheme host and port.""") debugging_group.add_argument('--debugger', const="__default__", nargs="?", help="run under a debugger, e.g. gdb or valgrind") debugging_group.add_argument('--debugger-args', help="arguments to the debugger") + debugging_group.add_argument("--rerun", action="store", type=int, default=1, + help="Number of times to re run each test without restarts") debugging_group.add_argument("--repeat", action="store", type=int, default=1, - help="Number of times to run the tests") + help="Number of times to run the tests, restarting between each run") debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None, help="Run tests in a loop until one returns an unexpected result") debugging_group.add_argument('--pause-after-test', action="store_true", default=None, diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py index 9855ecc672c9..b137b7217a25 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py @@ -241,6 +241,7 @@ def run_tests(config, test_paths, product, **kwargs): browser_kwargs, executor_cls, executor_kwargs, + kwargs["rerun"], kwargs["pause_after_test"], kwargs["pause_on_unexpected"], kwargs["restart_on_unexpected"],