diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/base.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/base.py index 583ca0d6cded..52527599ffbd 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/base.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/base.py @@ -83,8 +83,11 @@ class Browser(object): """Used for browser-specific setup that happens at the start of a test run""" pass + def settings(self, test): + return {} + @abstractmethod - def start(self): + def start(self, **kwargs): """Launch the browser object and get it into a state where is is ready to run tests""" pass @@ -126,7 +129,7 @@ class NullBrowser(Browser): def __init__(self, logger, **kwargs): super(NullBrowser, self).__init__(logger) - def start(self): + def start(self, **kwargs): """No-op browser to use in scenarios where the TestRunnerManager shouldn't actually own the browser process (e.g. Servo where we start one browser per test)""" diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py index 3bd227b32f55..1c08c27922b6 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/chrome.py @@ -20,7 +20,7 @@ def check_args(**kwargs): require_arg(kwargs, "webdriver_binary") -def browser_kwargs(**kwargs): +def browser_kwargs(test_type, run_info_data, **kwargs): return {"binary": kwargs["binary"], "webdriver_binary": kwargs["webdriver_binary"], "webdriver_args": kwargs.get("webdriver_args")} @@ -72,7 +72,7 @@ class ChromeBrowser(Browser): binary=webdriver_binary, args=webdriver_args) - def start(self): + def start(self, **kwargs): self.server.start(block=False) def stop(self, force=False): diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/edge.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/edge.py index 354edb3bab12..32f3e582c5b0 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/edge.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/edge.py @@ -51,7 +51,7 @@ class EdgeBrowser(Browser): self.webdriver_host = "localhost" self.webdriver_port = self.server.port - def start(self): + def start(self, **kwargs): print self.server.url self.server.start() diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py index 7152fd7c1c20..2d45a69dbe75 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py @@ -5,6 +5,7 @@ import subprocess import sys import mozinfo +import mozleak from mozprocess import ProcessHandler from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations @@ -41,13 +42,26 @@ __wptrunner__ = {"product": "firefox", "update_properties": "update_properties"} +def get_timeout_multiplier(test_type, run_info_data, **kwargs): + if kwargs["timeout_multiplier"] is not None: + return kwargs["timeout_multiplier"] + if test_type == "reftest": + if run_info_data["debug"] or run_info_data.get("asan"): + return 4 + else: + return 2 + elif run_info_data["debug"] or run_info_data.get("asan"): + return 3 + return 1 + + def check_args(**kwargs): require_arg(kwargs, "binary") if kwargs["ssl_type"] != "none": require_arg(kwargs, "certutil_binary") -def browser_kwargs(**kwargs): +def browser_kwargs(test_type, run_info_data, **kwargs): return {"binary": kwargs["binary"], "prefs_root": kwargs["prefs_root"], "extra_prefs": kwargs["extra_prefs"], @@ -58,7 +72,11 @@ def browser_kwargs(**kwargs): "ca_certificate_path": kwargs["ssl_env"].ca_cert_path(), "e10s": kwargs["gecko_e10s"], "stackfix_dir": kwargs["stackfix_dir"], - "binary_args": kwargs["binary_args"]} + "binary_args": kwargs["binary_args"], + "timeout_multiplier": get_timeout_multiplier(test_type, + run_info_data, + **kwargs), + "leak_check": kwargs["leak_check"]} def executor_kwargs(test_type, server_config, cache_manager, run_info_data, @@ -66,14 +84,9 @@ def executor_kwargs(test_type, server_config, cache_manager, run_info_data, executor_kwargs = base_executor_kwargs(test_type, server_config, cache_manager, **kwargs) executor_kwargs["close_after_done"] = test_type != "reftest" - if kwargs["timeout_multiplier"] is None: - if test_type == "reftest": - if run_info_data["debug"] or run_info_data.get("asan"): - executor_kwargs["timeout_multiplier"] = 4 - else: - executor_kwargs["timeout_multiplier"] = 2 - elif run_info_data["debug"] or run_info_data.get("asan"): - executor_kwargs["timeout_multiplier"] = 3 + executor_kwargs["timeout_multiplier"] = get_timeout_multiplier(test_type, + run_info_data, + **kwargs) if test_type == "wdspec": executor_kwargs["binary"] = kwargs["binary"] executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary") @@ -119,7 +132,7 @@ class FirefoxBrowser(Browser): def __init__(self, logger, binary, prefs_root, extra_prefs=None, debug_info=None, symbols_path=None, stackwalk_binary=None, certutil_binary=None, ca_certificate_path=None, e10s=False, stackfix_dir=None, - binary_args=None): + binary_args=None, timeout_multiplier=None, leak_check=False): Browser.__init__(self, logger) self.binary = binary self.prefs_root = prefs_root @@ -140,9 +153,19 @@ class FirefoxBrowser(Browser): else: self.stack_fixer = None - def start(self): - self.marionette_port = get_free_port(2828, exclude=self.used_ports) - self.used_ports.add(self.marionette_port) + if timeout_multiplier: + self.init_timeout = self.init_timeout * timeout_multiplier + + self.leak_report_file = None + self.leak_check = leak_check + + def settings(self, test): + return {"check_leaks": self.leak_check and not test.leaks} + + def start(self, **kwargs): + if self.marionette_port is None: + self.marionette_port = get_free_port(2828, exclude=self.used_ports) + self.used_ports.add(self.marionette_port) env = os.environ.copy() env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" @@ -153,8 +176,7 @@ class FirefoxBrowser(Browser): self.profile = FirefoxProfile(locations=locations, preferences=preferences) - self.profile.set_preferences({"marionette.enabled": True, - "marionette.port": self.marionette_port, + self.profile.set_preferences({"marionette.port": self.marionette_port, "dom.disable_open_during_load": False, "network.dns.localDomains": ",".join(hostnames), "network.proxy.type": 0, @@ -162,6 +184,14 @@ class FirefoxBrowser(Browser): if self.e10s: self.profile.set_preferences({"browser.tabs.remote.autostart": True}) + if self.leak_check and kwargs.get("check_leaks", True): + self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log") + if os.path.exists(self.leak_report_file): + os.remove(self.leak_report_file) + env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file + else: + self.leak_report_file = None + # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): @@ -217,6 +247,24 @@ class FirefoxBrowser(Browser): except OSError: # This can happen on Windows if the process is already dead pass + self.logger.debug("stopped") + + def process_leaks(self): + self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file) + if self.leak_report_file is None: + return + mozleak.process_leak_log( + self.leak_report_file, + leak_thresholds={ + "default": 0, + "tab": 10000, # See dependencies of bug 1051230. + # GMP rarely gets a log, but when it does, it leaks a little. + "geckomediaplugin": 20000, + }, + ignore_missing_leaks=["geckomediaplugin"], + log=self.logger, + stack_fixer=self.stack_fixer + ) def pid(self): if self.runner.process_handler is None: @@ -243,6 +291,7 @@ class FirefoxBrowser(Browser): def cleanup(self): self.stop() + self.process_leaks() def executor_browser(self): assert self.marionette_port is not None diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/sauce.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/sauce.py index 7300b6afefb8..a2f29a4c54f3 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/sauce.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/sauce.py @@ -92,7 +92,7 @@ def check_args(**kwargs): require_arg(kwargs, "sauce_key") -def browser_kwargs(**kwargs): +def browser_kwargs(test_type, run_info_data, **kwargs): sauce_config = get_sauce_config(**kwargs) return {"sauce_config": sauce_config} diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py index e24e4dfd9dfe..89d69ab9e543 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py @@ -27,7 +27,7 @@ def check_args(**kwargs): require_arg(kwargs, "binary") -def browser_kwargs(**kwargs): +def browser_kwargs(test_type, run_info_data, **kwargs): return { "binary": kwargs["binary"], "debug_info": kwargs["debug_info"], diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py index 9575e0fbff9d..c251de84d163 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py @@ -39,7 +39,7 @@ def check_args(**kwargs): require_arg(kwargs, "binary") -def browser_kwargs(**kwargs): +def browser_kwargs(test_type, run_info_data, **kwargs): return { "binary": kwargs["binary"], "debug_info": kwargs["debug_info"], @@ -91,7 +91,7 @@ class ServoWebDriverBrowser(Browser): self.command = None self.user_stylesheets = user_stylesheets if user_stylesheets else [] - def start(self): + def start(self, **kwargs): self.webdriver_port = get_free_port(4444, exclude=self.used_ports) self.used_ports.add(self.webdriver_port) diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py index 21e6509af167..51b9f30f470f 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py @@ -48,12 +48,13 @@ def do_delayed_imports(): class MarionetteProtocol(Protocol): - def __init__(self, executor, browser): + def __init__(self, executor, browser, timeout_multiplier=1): do_delayed_imports() Protocol.__init__(self, executor, browser) self.marionette = None self.marionette_port = browser.marionette_port + self.timeout_multiplier = timeout_multiplier self.timeout = None self.runner_handle = None @@ -62,14 +63,16 @@ class MarionetteProtocol(Protocol): Protocol.setup(self, runner) self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port) + startup_timeout = marionette.Marionette.DEFAULT_STARTUP_TIMEOUT * self.timeout_multiplier self.marionette = marionette.Marionette(host='localhost', port=self.marionette_port, - socket_timeout=None) + socket_timeout=None, + startup_timeout=startup_timeout) # XXX Move this timeout somewhere self.logger.debug("Waiting for Marionette connection") while True: - success = self.marionette.wait_for_port(60) + success = self.marionette.wait_for_port(60 * self.timeout_multiplier) #When running in a debugger wait indefinitely for firefox to start if success or self.executor.debug_info is None: break @@ -407,7 +410,7 @@ class MarionetteTestharnessExecutor(TestharnessExecutor): timeout_multiplier=timeout_multiplier, debug_info=debug_info) - self.protocol = MarionetteProtocol(self, browser) + self.protocol = MarionetteProtocol(self, browser, timeout_multiplier) self.script = open(os.path.join(here, "testharness_marionette.js")).read() self.close_after_done = close_after_done self.window_id = str(uuid.uuid4()) diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py index fb5c1b1e7239..5d57b70e517c 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py @@ -111,6 +111,10 @@ class ExpectedManifest(ManifestItem): def restart_after(self): return bool_prop("restart-after", self) + @property + def leaks(self): + return bool_prop("leaks", self) + @property def tags(self): return tags(self) @@ -129,6 +133,10 @@ class DirectoryManifest(ManifestItem): def restart_after(self): return bool_prop("restart-after", self) + @property + def leaks(self): + return bool_prop("leaks", self) + @property def tags(self): return tags(self) @@ -174,6 +182,10 @@ class TestNode(ManifestItem): def restart_after(self): return bool_prop("restart-after", self) + @property + def leaks(self): + return bool_prop("leaks", self) + @property def tags(self): return tags(self) diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py index 74dce4aafe33..2b2033f3d213 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py @@ -92,8 +92,8 @@ class EqualTimeChunker(TestChunker): by_dir[test_dir] = PathData(test_dir) data = by_dir[test_dir] - time = sum(wpttest.DEFAULT_TIMEOUT if test.timeout != - "long" else wpttest.LONG_TIMEOUT for test in tests) + time = sum(test.default_timeout if test.timeout != + "long" else test.long_timeout for test in tests) data.time += time total_time += time data.tests.append((test_type, test_path, tests)) @@ -627,7 +627,6 @@ class PathGroupedSource(TestSource): self.current_queue.put(item) except Empty: return None - return self.current_queue def requeue_test(self, test): diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py index e422cfefec61..d494010a9f76 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py @@ -153,11 +153,20 @@ class BrowserManager(object): self.logger = logger self.browser = browser self.no_timeout = no_timeout + self.browser_settings = None self.started = False self.init_timer = None + def update_settings(self, test): + browser_settings = self.browser.settings(test) + restart_required = ((self.browser_settings is not None and + self.browser_settings != browser_settings) or + test.expected() == "CRASH") + self.browser_settings = browser_settings + return restart_required + def init(self): """Launch the browser that is being tested, and the TestRunner process that will run the tests.""" @@ -178,7 +187,8 @@ class BrowserManager(object): try: if self.init_timer is not None: self.init_timer.start() - self.browser.start() + self.logger.debug("Starting browser with settings %r" % self.browser_settings) + self.browser.start(**self.browser_settings) self.browser_pid = self.browser.pid() except: self.logger.warning("Failure during init %s" % traceback.format_exc()) @@ -437,6 +447,8 @@ class TestRunnerManager(threading.Thread): self.logger.error("Max restarts exceeded") return RunnerManagerState.error() + self.browser.update_settings(self.state.test) + result = self.browser.init() if result is Stop: return RunnerManagerState.error() @@ -495,15 +507,21 @@ class TestRunnerManager(threading.Thread): return None, None try: # Need to block here just to allow for contention with other processes - test = test_queue.get(block=True, timeout=1) + test = test_queue.get(block=True, timeout=2) except Empty: - pass + if test_queue.empty(): + test_queue = None return test, test_queue def run_test(self): assert isinstance(self.state, RunnerManagerState.running) assert self.state.test is not None + if self.browser.update_settings(self.state.test): + self.logger.info("Restarting browser for new test environment") + return RunnerManagerState.restarting(self.state.test, + self.state.test_queue) + self.logger.test_start(self.state.test.id) self.send_message("run_test", self.state.test) diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_chunker.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_chunker.py index 4968940feb8b..062b687fab9a 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_chunker.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_chunker.py @@ -12,6 +12,8 @@ from wptrunner.testloader import EqualTimeChunker structured.set_default_logger(structured.structuredlog.StructuredLogger("TestChunker")) class MockTest(object): + default_timeout = 10 + def __init__(self, id, timeout=10): self.id = id self.item_type = "testharness" diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/sync.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/sync.py index 3f6092a046a0..fb7948874c48 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/sync.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/sync.py @@ -1,4 +1,6 @@ +import fnmatch import os +import re import shutil import sys import uuid @@ -42,7 +44,7 @@ POSSIBILITY OF SUCH DAMAGE. """ -def copy_wpt_tree(tree, dest): +def copy_wpt_tree(tree, dest, excludes=None, includes=None): """Copy the working copy of a Tree to a destination directory. :param tree: The Tree to copy. @@ -51,9 +53,24 @@ def copy_wpt_tree(tree, dest): assert os.path.isdir(dest) shutil.rmtree(dest) + os.mkdir(dest) + if excludes is None: + excludes = [] + + excludes = [re.compile(fnmatch.translate(item)) for item in excludes] + + if includes is None: + includes = [] + + includes = [re.compile(fnmatch.translate(item)) for item in includes] + for tree_path in tree.paths(): + if (any(item.match(tree_path) for item in excludes) and + not any(item.match(tree_path) for item in includes)): + continue + source_path = os.path.join(tree.root, tree_path) dest_path = os.path.join(dest, tree_path) @@ -79,6 +96,7 @@ def add_license(dest): with open(os.path.join(dest, "LICENSE"), "w") as f: f.write(bsd_license) + class UpdateCheckout(Step): """Pull changes from upstream into the local sync tree.""" @@ -142,7 +160,9 @@ class CopyWorkTree(Step): def create(self, state): copy_wpt_tree(state.sync_tree, - state.tests_path) + state.tests_path, + excludes=state.path_excludes, + includes=state.path_includes) class CreateSyncPatch(Step): diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/tree.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/tree.py index a15df907aa41..01df0b4f94a3 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/tree.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/tree.py @@ -312,9 +312,8 @@ class GitTree(object): for repo_path in repo_paths: paths = vcs.git("ls-tree", "-r", "--name-only", "HEAD", repo=repo_path).split("\n") - rel_path = os.path.relpath(repo_path, self.root) - rv.extend(os.path.join(rel_path, item.strip()) for item in paths if item.strip()) - + rv.extend(os.path.relpath(os.path.join(repo_path, item), self.root) for item in paths + if item.strip()) return rv def submodules(self): diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py index 4a5e7ab73d4b..4c0099e63bb1 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py @@ -73,6 +73,8 @@ class SyncFromUpstream(Step): state.target_rev = kwargs["rev"] state.no_patch = kwargs["no_patch"] state.suite_name = kwargs["suite_name"] + state.path_excludes = kwargs["exclude"] + state.path_includes = kwargs["include"] runner = SyncFromUpstreamRunner(self.logger, state) runner.run() diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py index 01d3ab9fb52c..5d44df2fad9b 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py @@ -177,6 +177,8 @@ scheme host and port.""") gecko_group.add_argument("--setpref", dest="extra_prefs", action='append', default=[], metavar="PREF=VALUE", help="Defines an extra user preference (overrides those in prefs_root)") + gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true", + help="Enable leak checking") servo_group = parser.add_argument_group("Servo-specific") servo_group.add_argument("--user-stylesheet", @@ -424,6 +426,10 @@ def create_parser_update(product_choices=None): parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.") parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script") parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script") + parser.add_argument("--exclude", action="store", nargs="*", + help="List of glob-style paths to exclude when syncing tests") + parser.add_argument("--include", action="store", nargs="*", + help="List of glob-style paths to include which would otherwise be excluded when syncing tests") # Should make this required iff run=logfile parser.add_argument("run_log", nargs="*", type=abs_path, help="Log file from run of tests") diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py index bf9b74e4398f..65e5d7f7e0c1 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py @@ -158,8 +158,6 @@ def run_tests(config, test_paths, product, **kwargs): logger.critical("Error starting test environment: %s" % e.message) raise - browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs) - repeat = kwargs["repeat"] repeat_count = 0 repeat_until_unexpected = kwargs["repeat_until_unexpected"] @@ -186,9 +184,11 @@ def run_tests(config, test_paths, product, **kwargs): else: browser_cls = target_browser_cls - for test in test_loader.disabled_tests[test_type]: - logger.test_start(test.id) - logger.test_end(test.id, status="SKIP") + browser_kwargs = get_browser_kwargs(test_type, + run_info, + ssl_env=ssl_env, + **kwargs) + executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs(test_type, @@ -202,6 +202,9 @@ def run_tests(config, test_paths, product, **kwargs): (test_type, product)) continue + for test in test_loader.disabled_tests[test_type]: + logger.test_start(test.id) + logger.test_end(test.id, status="SKIP") with ManagerGroup("web-platform-tests", kwargs["processes"], diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py index d144cd66ea78..12701f611179 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py @@ -1,6 +1,3 @@ -DEFAULT_TIMEOUT = 10 # seconds -LONG_TIMEOUT = 60 # seconds - import os import mozinfo @@ -95,17 +92,21 @@ class RunInfo(dict): class Test(object): + result_cls = None subtest_result_cls = None test_type = None + default_timeout = 10 # seconds + long_timeout = 60 # seconds + def __init__(self, tests_root, url, inherit_metadata, test_metadata, - timeout=DEFAULT_TIMEOUT, path=None, protocol="http"): + timeout=None, path=None, protocol="http"): self.tests_root = tests_root self.url = url self._inherit_metadata = inherit_metadata self._test_metadata = test_metadata - self.timeout = timeout + self.timeout = timeout if timeout is not None else self.default_timeout self.path = path self.environment = {"protocol": protocol, "prefs": self.prefs} @@ -114,7 +115,7 @@ class Test(object): @classmethod def from_manifest(cls, manifest_item, inherit_metadata, test_metadata): - timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT + timeout = cls.long_timeout if manifest_item.timeout == "long" else cls.default_timeout protocol = "https" if hasattr(manifest_item, "https") and manifest_item.https else "http" return cls(manifest_item.source_file.tests_root, manifest_item.url, @@ -168,6 +169,14 @@ class Test(object): return True return False + @property + def leaks(self): + for meta in self.itermeta(None): + leaks = meta.leaks + if leaks is not None: + return leaks + return False + @property def tags(self): tags = set() @@ -237,8 +246,7 @@ class ReftestTest(Test): test_type = "reftest" def __init__(self, tests_root, url, inherit_metadata, test_metadata, references, - timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None, - dpi=None, protocol="http"): + timeout=None, path=None, viewport_size=None, dpi=None, protocol="http"): Test.__init__(self, tests_root, url, inherit_metadata, test_metadata, timeout, path, protocol) @@ -258,7 +266,7 @@ class ReftestTest(Test): nodes=None, references_seen=None): - timeout = LONG_TIMEOUT if manifest_test.timeout == "long" else DEFAULT_TIMEOUT + timeout = cls.long_timeout if manifest_test.timeout == "long" else cls.default_timeout if nodes is None: nodes = {} @@ -322,10 +330,14 @@ class ReftestTest(Test): class WdspecTest(Test): + result_cls = WdspecResult subtest_result_cls = WdspecSubtestResult test_type = "wdspec" + default_timeout = 25 + long_timeout = 120 + manifest_test_cls = {"reftest": ReftestTest, "testharness": TestharnessTest,