Bug 1415674 - Allow skipping web-paltform-tests that timeout, r=ato

For local runs it can be helpful to skip tests are expected to time
out. This adds a --skip-timeout flag that will skip tests that are
expected to time out on the current platform.

It doesn't solve the problem that the metadata might be over-specific
in the case that the expectation data isn't universal across all platforms.

MozReview-Commit-ID: 7S4EsVKv33P

--HG--
extra : rebase_source : 7e18bbcefc7350465bae799c27454a13da64a32e

testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py

@@ -448,7 +448,8 @@ class TestLoader(object):
                  chunk_type="none",
                  total_chunks=1,
                  chunk_number=1,
-                 include_https=True):
+                 include_https=True,
+                 skip_timeout=False):
 
         self.test_types = test_types
         self.run_info = run_info
@@ -460,6 +461,7 @@ class TestLoader(object):
         self.tests = None
         self.disabled_tests = None
         self.include_https = include_https
+        self.skip_timeout = skip_timeout
 
         self.chunk_type = chunk_type
         self.total_chunks = total_chunks
@@ -545,6 +547,8 @@ class TestLoader(object):
             enabled = not test.disabled()
             if not self.include_https and test.environment["protocol"] == "https":
                 enabled = False
+            if self.skip_timeout and test.expected() == "TIMEOUT":
+                enabled = False
             key = "enabled" if enabled else "disabled"
             tests[key][test_type].append(test)
 

testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py

@@ -89,6 +89,8 @@ scheme host and port.""")
                                       help="URL prefix to exclude")
     test_selection_group.add_argument("--include-manifest", type=abs_path,
                                       help="Path to manifest listing tests to include")
+    test_selection_group.add_argument("--skip-timeout", action="store_true",
+                                      help="Skip tests that are expected to time out")
     test_selection_group.add_argument("--tag", action="append", dest="tags",
                                       help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
 

testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py

@@ -38,6 +38,7 @@ def setup_logging(*args, **kwargs):
     global logger
     logger = wptlogging.setup(*args, **kwargs)
 
+
 def get_loader(test_paths, product, ssl_env, debug=None, run_info_extras=None, **kwargs):
     if run_info_extras is None:
         run_info_extras = {}
@@ -66,9 +67,11 @@ def get_loader(test_paths, product, ssl_env, debug=None, run_info_extras=None, *
                                         chunk_type=kwargs["chunk_type"],
                                         total_chunks=kwargs["total_chunks"],
                                         chunk_number=kwargs["this_chunk"],
-                                        include_https=ssl_env.ssl_enabled)
+                                        include_https=ssl_env.ssl_enabled,
+                                        skip_timeout=kwargs["skip_timeout"])
     return run_info, test_loader
 
+
 def list_test_groups(test_paths, product, **kwargs):
     env.do_delayed_imports(logger, test_paths)
 
@@ -216,7 +219,6 @@ def run_tests(config, test_paths, product, **kwargs):
                                                         ssl_env=ssl_env,
                                                         **kwargs)
 
-
                     executor_cls = executor_classes.get(test_type)
                     executor_kwargs = get_executor_kwargs(test_type,
                                                           test_environment.external_config,