Bug 1634554 - Add support for --test-groups to wpt r=ahal,egao

This adds a --test-groups command line argument which points at a JSON file containing lists of tests divided into explicit groups like:

{"/dom": ["/dom/historical.html", ...],
 "/dom/events/": [...]}

This is for situations where the division of tests into groups is
performed by an external process (in the case of gecko: by the
decision task).

Group names must be a path prefix, as this metadata is reused as the
test "scope" which is passed down into the output and can be used by
automatic metadata update for per-group properties like the LSAN allow
list.

--test-groups is incompatible with --run-by-dir but composes with
passing an explicit include list by running the intersection of the
supplied tests.

Differential Revision: https://phabricator.services.mozilla.com/D75175

testing/web-platform/tests/tools/wptrunner/wptrunner/testloader.py

@@ -1,4 +1,5 @@
 import hashlib
+import json
 import os
 from six.moves.urllib.parse import urlsplit
 from abc import ABCMeta, abstractmethod
@@ -26,6 +27,45 @@ def do_delayed_imports():
     from manifest.download import download_from_github
 
 
+class TestGroupsFile(object):
+    """
+    Mapping object representing {group name: [test ids]}
+    """
+
+    def __init__(self, logger, path):
+        try:
+            with open(path) as f:
+                self._data = json.load(f)
+        except ValueError:
+            logger.critical("test groups file %s not valid json" % path)
+            raise
+
+        self.group_by_test = {}
+        for group, test_ids in iteritems(self._data):
+            for test_id in test_ids:
+                self.group_by_test[test_id] = group
+
+    def __contains__(self, key):
+        return key in self._data
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+
+def update_include_for_groups(test_groups, include):
+    if include is None:
+        # We're just running everything
+        return
+
+    new_include = []
+    for item in include:
+        if item in test_groups:
+            new_include.extend(test_groups[item])
+        else:
+            new_include.append(item)
+    return new_include
+
+
 class TestChunker(object):
     def __init__(self, total_chunks, chunk_number, **kwargs):
         self.total_chunks = total_chunks
@@ -292,6 +332,23 @@ class TestLoader(object):
         return groups
 
 
+def get_test_src(**kwargs):
+    test_source_kwargs = {"processes": kwargs["processes"],
+                          "logger": kwargs["logger"]}
+    chunker_kwargs = {}
+    if kwargs["run_by_dir"] is not False:
+        # A value of None indicates infinite depth
+        test_source_cls = PathGroupedSource
+        test_source_kwargs["depth"] = kwargs["run_by_dir"]
+        chunker_kwargs["depth"] = kwargs["run_by_dir"]
+    elif kwargs["test_groups"]:
+        test_source_cls = GroupFileTestSource
+        test_source_kwargs["test_groups"] = kwargs["test_groups"]
+    else:
+        test_source_cls = SingleTestSource
+    return test_source_cls, test_source_kwargs, chunker_kwargs
+
+
 class TestSource(object):
     __metaclass__ = ABCMeta
 
@@ -397,3 +454,39 @@ class PathGroupedSource(GroupedSource):
     @classmethod
     def group_metadata(cls, state):
         return {"scope": "/%s" % "/".join(state["prev_path"])}
+
+
+class GroupFileTestSource(TestSource):
+    @classmethod
+    def make_queue(cls, tests, **kwargs):
+        tests_by_group = cls.tests_by_group(tests, **kwargs)
+
+        test_queue = Queue()
+
+        for group_name, tests in iteritems(tests_by_group):
+            group_metadata = {"scope": group_name}
+            group = deque()
+
+            for test in tests:
+                group.append(test)
+                test.update_metadata(group_metadata)
+
+            test_queue.put((group, group_metadata))
+
+        return test_queue
+
+    @classmethod
+    def tests_by_group(cls, tests, **kwargs):
+        logger = kwargs["logger"]
+        test_groups = kwargs["test_groups"]
+
+        tests_by_group = defaultdict(list)
+        for test in tests:
+            try:
+                group = test_groups.group_by_test[test.id]
+            except KeyError:
+                logger.error("%s is missing from test groups file" % test.id)
+                raise
+            tests_by_group[group].append(test)
+
+        return tests_by_group

testing/web-platform/tests/tools/wptrunner/wptrunner/testrunner.py

@@ -700,8 +700,9 @@ class TestRunnerManager(threading.Thread):
             test, test_group, group_metadata = self.get_next_test()
             if test is None:
                 return RunnerManagerState.stop()
-            if test_group != self.state.test_group:
+            if test_group is not self.state.test_group:
                 # We are starting a new group of tests, so force a restart
+                self.logger.info("Restarting browser for new test group")
                 restart = True
         else:
             test_group = self.state.test_group

testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py

@@ -135,6 +135,8 @@ scheme host and port.""")
                                       help="URL prefix to exclude")
     test_selection_group.add_argument("--include-manifest", type=abs_path,
                                       help="Path to manifest listing tests to include")
+    test_selection_group.add_argument("--test-groups", dest="test_groups_file", type=abs_path,
+                                      help="Path to json file containing a mapping {group_name: [test_ids]}")
     test_selection_group.add_argument("--skip-timeout", action="store_true",
                                       help="Skip tests that are expected to time out")
     test_selection_group.add_argument("--skip-implementation-status",
@@ -504,6 +506,14 @@ def check_args(kwargs):
         else:
             kwargs["chunk_type"] = "none"
 
+    if kwargs["test_groups_file"] is not None:
+        if kwargs["run_by_dir"] is not False:
+            print("Can't pass --test-groups and --run-by-dir")
+            sys.exit(1)
+        if not os.path.exists(kwargs["test_groups_file"]):
+            print("--test-groups file %s not found" % kwargs["test_groups_file"])
+            sys.exit(1)
+
     if kwargs["processes"] is None:
         kwargs["processes"] = 1
 

testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py

@@ -45,7 +45,8 @@ def setup_logging(*args, **kwargs):
     return logger
 
 
-def get_loader(test_paths, product, debug=None, run_info_extras=None, chunker_kwargs=None, **kwargs):
+def get_loader(test_paths, product, debug=None, run_info_extras=None, chunker_kwargs=None,
+               test_groups=None, **kwargs):
     if run_info_extras is None:
         run_info_extras = {}
 
@@ -62,8 +63,12 @@ def get_loader(test_paths, product, debug=None, run_info_extras=None, chunker_kw
 
     manifest_filters = []
 
-    if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"] or kwargs["default_exclude"]:
-        manifest_filters.append(testloader.TestFilter(include=kwargs["include"],
+    include = kwargs["include"]
+    if test_groups:
+        include = testloader.update_include_for_groups(test_groups, include)
+
+    if include or kwargs["exclude"] or kwargs["include_manifest"] or kwargs["default_exclude"]:
+        manifest_filters.append(testloader.TestFilter(include=include,
                                                       exclude=kwargs["exclude"],
                                                       manifest_path=kwargs["include_manifest"],
                                                       test_manifests=test_manifests,
@@ -166,23 +171,21 @@ def run_tests(config, test_paths, product, **kwargs):
 
         recording.set(["startup", "load_tests"])
 
-        test_source_kwargs = {"processes": kwargs["processes"]}
-        chunker_kwargs = {}
-        if kwargs["run_by_dir"] is False:
-            test_source_cls = testloader.SingleTestSource
-        else:
-            # A value of None indicates infinite depth
-            test_source_cls = testloader.PathGroupedSource
-            test_source_kwargs["depth"] = kwargs["run_by_dir"]
-            chunker_kwargs["depth"] = kwargs["run_by_dir"]
+        test_groups = (testloader.TestGroupsFile(logger, kwargs["test_groups_file"])
+                       if kwargs["test_groups_file"] else None)
 
+        (test_source_cls,
+         test_source_kwargs,
+         chunker_kwargs) = testloader.get_test_src(logger=logger,
+                                                   test_groups=test_groups,
+                                                   **kwargs)
         run_info, test_loader = get_loader(test_paths,
                                            product.name,
                                            run_info_extras=product.run_info_extras(**kwargs),
                                            chunker_kwargs=chunker_kwargs,
+                                           test_groups=test_groups,
                                            **kwargs)
 
-
         logger.info("Using %i client processes" % kwargs["processes"])
 
         skipped_tests = 0
@@ -203,7 +206,9 @@ def run_tests(config, test_paths, product, **kwargs):
                                        "host_cert_path": kwargs["host_cert_path"],
                                        "ca_cert_path": kwargs["ca_cert_path"]}}
 
-        testharness_timeout_multipler = product.get_timeout_multiplier("testharness", run_info, **kwargs)
+        testharness_timeout_multipler = product.get_timeout_multiplier("testharness",
+                                                                       run_info,
+                                                                       **kwargs)
 
         recording.set(["startup", "start_environment"])
         with env.TestEnvironment(test_paths,
@@ -241,7 +246,13 @@ def run_tests(config, test_paths, product, **kwargs):
                 for test_type in test_loader.test_types:
                     tests.extend(test_loader.tests[test_type])
 
-                logger.suite_start(test_source_cls.tests_by_group(tests, **test_source_kwargs),
+                try:
+                    test_groups = test_source_cls.tests_by_group(tests, **test_source_kwargs)
+                except Exception:
+                    logger.critical("Loading tests failed")
+                    return False
+
+                logger.suite_start(test_groups,
                                    name='web-platform-test',
                                    run_info=run_info,
                                    extra={"run_by_dir": kwargs["run_by_dir"]})