Bug 1595515 - Add support for implementation-status in wpt metadata, r=twisniewski

This adds an `implementation-status` field to the wpt metadata with 3
possible values; "not-implementing", "backlog" and "implmenting" (the
latter being the same as the default value). It also adds a
`--skip-implementation-status` command line argument to wptrunner that
can be used to skip running tests with the specified implementation
statuses. This is primarilly to allow gecko to avoid running tests (or
run tests less frequently) for things where we don't implement the
spec and don't plan to either ever or in the current roadmap.

Differential Revision: https://phabricator.services.mozilla.com/D65765

--HG--
extra : moz-landing-system : lando

testing/web-platform/tests/tools/wptrunner/wptrunner/manifestexpected.py

@@ -17,6 +17,7 @@ Each TestNode has zero or more SubtestNode children, one for each
 known subtest of the test.
 """
 
+
 def data_cls_getter(output_node, visited_node):
     # visited_node is intentionally unused
     if output_node is None:
@@ -55,6 +56,16 @@ def list_prop(name, node):
         return []
 
 
+def str_prop(name, node):
+    try:
+        prop = node.get(name)
+        if not isinstance(prop, string_types):
+            raise ValueError
+        return prop
+    except KeyError:
+        return None
+
+
 def tags(node):
     """Set of tags that have been applied to the test"""
     try:
@@ -309,6 +320,10 @@ class ExpectedManifest(ManifestItem):
     def known_intermittent(self):
         return list_prop("expected", self)[1:]
 
+    @property
+    def implementation_status(self):
+        return str_prop("implementation-status", self)
+
 
 class DirectoryManifest(ManifestItem):
     @property
@@ -359,6 +374,10 @@ class DirectoryManifest(ManifestItem):
     def fuzzy(self):
         return fuzzy_prop(self)
 
+    @property
+    def implementation_status(self):
+        return str_prop("implementation-status", self)
+
 
 class TestNode(ManifestItem):
     def __init__(self, node, **kwargs):
@@ -444,6 +463,10 @@ class TestNode(ManifestItem):
     def known_intermittent(self):
         return list_prop("expected", self)[1:]
 
+    @property
+    def implementation_status(self):
+        return str_prop("implementation-status", self)
+
     def append(self, node):
         """Add a subtest to the current test
 

testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py

@@ -17,6 +17,7 @@ manifest = None
 manifest_update = None
 download_from_github = None
 
+
 def do_delayed_imports():
     # This relies on an already loaded module having set the sys.path correctly :(
     global manifest, manifest_update, download_from_github
@@ -168,6 +169,7 @@ class TestLoader(object):
                  chunk_number=1,
                  include_https=True,
                  skip_timeout=False,
+                 skip_implementation_status=None,
                  chunker_kwargs=None):
 
         self.test_types = test_types
@@ -180,6 +182,7 @@ class TestLoader(object):
         self.disabled_tests = None
         self.include_https = include_https
         self.skip_timeout = skip_timeout
+        self.skip_implementation_status = skip_implementation_status
 
         self.chunk_type = chunk_type
         self.total_chunks = total_chunks
@@ -266,6 +269,8 @@ class TestLoader(object):
                 enabled = False
             if self.skip_timeout and test.expected() == "TIMEOUT":
                 enabled = False
+            if self.skip_implementation_status and test.implementation_status() in self.skip_implementation_status:
+                enabled = False
             key = "enabled" if enabled else "disabled"
             tests[key][test_type].append(test)
 

testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py

@@ -137,6 +137,10 @@ scheme host and port.""")
                                       help="Path to manifest listing tests to include")
     test_selection_group.add_argument("--skip-timeout", action="store_true",
                                       help="Skip tests that are expected to time out")
+    test_selection_group.add_argument("--skip-implementation-status",
+                                      action="append",
+                                      choices=["not-implementing", "backlog", "implementing"],
+                                      help="Skip tests that have the given implementation status")
     test_selection_group.add_argument("--tag", action="append", dest="tags",
                                       help="Labels applied to tests to include in the run. "
                                            "Labels starting dir: are equivalent to top-level directories.")

testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py

@@ -79,6 +79,7 @@ def get_loader(test_paths, product, debug=None, run_info_extras=None, chunker_kw
                                         chunk_number=kwargs["this_chunk"],
                                         include_https=ssl_enabled,
                                         skip_timeout=kwargs["skip_timeout"],
+                                        skip_implementation_status=kwargs["skip_implementation_status"],
                                         chunker_kwargs=chunker_kwargs)
     return run_info, test_loader
 

testing/web-platform/tests/tools/wptrunner/wptrunner/wpttest.py

@@ -342,6 +342,13 @@ class Test(object):
         except KeyError:
             return default
 
+    def implementation_status(self):
+        implementation_status = None
+        for meta in self.itermeta():
+            implementation_status = meta.implementation_status
+            if implementation_status:
+                return implementation_status
+
     def known_intermittent(self, subtest=None):
         metadata = self._get_metadata(subtest)
         if metadata is None: