Bug 1904817 - Allow test selection --tag to support multiple tags. r=ahal,taskgraph-reviewers

Differential Revision: https://phabricator.services.mozilla.com/D214965

taskcluster/gecko_taskgraph/test/test_util_chunking.py

@@ -143,7 +143,7 @@ def mock_mozinfo():
         fission=False,
         headless=False,
         tsan=False,
-        tag="",
+        tag="[]",
     ):
         return {
             "os": os,

taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py

@@ -15,6 +15,7 @@ from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_u
 from gecko_taskgraph.transforms.job.common import get_expiration, support_vcs_checkout
 from gecko_taskgraph.transforms.test import normpath, test_description_schema
 from gecko_taskgraph.util.attributes import is_try
+from gecko_taskgraph.util.chunking import get_test_tags
 from gecko_taskgraph.util.perftest import is_external_browser
 
 VARIANTS = [
@@ -229,12 +230,11 @@ def mozharness_test_on_docker(config, job, taskdesc):
         env["MOZHARNESS_TEST_PATHS"] = json.dumps(
             {test["suite"]: test["test-manifests"]}, sort_keys=True
         )
-    elif "MOZHARNESS_TEST_TAG" in config.params["try_task_config"].get("env", {}):
+
-        command.append(
+    test_tags = get_test_tags(config, env)
-            "--tag={}".format(
+    if test_tags:
-                config.params["try_task_config"]["env"]["MOZHARNESS_TEST_TAG"]
+        env["MOZHARNESS_TEST_TAG"] = json.dumps(test_tags)
-            )
+        command.extend(["--tag={}".format(x) for x in test_tags])
-        )
 
     # TODO: remove the need for run['chunked']
     elif mozharness.get("chunked") or test["chunks"] > 1:
@@ -445,12 +445,11 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
         env["MOZHARNESS_TEST_PATHS"] = json.dumps(
             {test["suite"]: test["test-manifests"]}, sort_keys=True
         )
-    elif "MOZHARNESS_TEST_TAG" in config.params["try_task_config"].get("env", {}):
+
-        mh_command.append(
+    test_tags = get_test_tags(config, env)
-            "--tag={}".format(
+    if test_tags:
-                config.params["try_task_config"]["env"]["MOZHARNESS_TEST_TAG"]
+        env["MOZHARNESS_TEST_TAG"] = json.dumps(test_tags)
-            )
+        mh_command.extend(["--tag={}".format(x) for x in test_tags])
-        )
 
     # TODO: remove the need for run['chunked']
     elif mozharness.get("chunked") or test["chunks"] > 1:

taskcluster/gecko_taskgraph/transforms/test/chunk.py

@@ -17,6 +17,7 @@ from gecko_taskgraph.util.chunking import (
     chunk_manifests,
     get_manifest_loader,
     get_runtimes,
+    get_test_tags,
     guess_mozinfo_from_task,
 )
 from gecko_taskgraph.util.perfile import perfile_number_of_chunks
@@ -94,7 +95,7 @@ def set_test_manifests(config, tasks):
         mozinfo = guess_mozinfo_from_task(
             task,
             config.params.get("head_repository", ""),
-            config.params.get("try_task_config", {}).get("env", {}),
+            get_test_tags(config, task.get("worker", {}).get("env", {})),
         )
 
         loader_name = task.pop(
@@ -169,9 +170,7 @@ def set_test_manifests(config, tasks):
             # this could be the test suite doesn't support test paths
             continue
         elif (
-            config.params.get("try_task_config", {})
+            get_test_tags(config, task.get("worker", {}).get("env", {}))
-            .get("env", {})
-            .get("MOZHARNESS_TEST_TAG", "")
             and not task["test-manifests"]["active"]
         ):
             # no MH_TEST_PATHS, but MH_TEST_TAG or other filters

taskcluster/gecko_taskgraph/util/chunking.py

@@ -35,7 +35,22 @@ WPT_SUBSUITES = {
 }
 
 
-def guess_mozinfo_from_task(task, repo="", env={}):
+def get_test_tags(config, env):
+    test_tags = []
+    try_config = json.loads(
+        config.params["try_task_config"].get("env", {}).get("MOZHARNESS_TEST_TAG", "[]")
+    )
+    env_tags = env.get("MOZHARNESS_TEST_TAG", [])
+    if env_tags:
+        if try_config:
+            env_tags.extend(try_config)
+        test_tags = list(set(env_tags))
+    elif try_config:
+        test_tags = try_config
+    return test_tags
+
+
+def guess_mozinfo_from_task(task, repo="", test_tags=[]):
     """Attempt to build a mozinfo dict from a task definition.
 
     This won't be perfect and many values used in the manifests will be missing. But
@@ -137,7 +152,9 @@ def guess_mozinfo_from_task(task, repo="", env={}):
         else:
             info[tag] = False
 
-    info["tag"] = env.get("MOZHARNESS_TEST_TAG", "")
+    # NOTE: as we are using an array here, frozenset() cannot work with a 'list'
+    # this is cast to a string
+    info["tag"] = json.dumps(test_tags)
 
     info["automation"] = True
     return info
@@ -273,11 +290,8 @@ class DefaultLoader(BaseManifestLoader):
         manifests = {chunk_by_runtime.get_manifest(t) for t in tests}
 
         filters = []
-        if mozinfo["condprof"]:
+        if json.loads(mozinfo["tag"]):
-            filters.extend([tags(["condprof"])])
+            filters.extend([tags([x]) for x in json.loads(mozinfo["tag"])])
-
-        if mozinfo["tag"]:
-            filters.extend([tags([mozinfo["tag"]])])
 
         # Compute  the active tests.
         m = TestManifest()

taskcluster/kinds/test/variants.yml

@@ -184,6 +184,10 @@ conditioned_profile:
                     || "xpcshell" == task["try-name"]
                 )'
     merge:
+        worker:
+            env:
+                MOZHARNESS_TEST_TAG:
+                    - "condprof"
         mozharness:
             extra-options:
                 - "--conditioned-profile"
@@ -329,9 +333,12 @@ wmf-media-engine:
     replace:
         tier: 2
     merge:
+        worker:
+            env:
+                MOZHARNESS_TEST_TAG:
+                    - "media-engine-compatible"
         mozharness:
             extra-options:
-                - "--tag=media-engine-compatible"
                 - "--setpref=media.wmf.media-engine.enabled=1"
                 - "--setpref=media.wmf.media-engine.channel-decoder.enabled=true"
                 - "--setpref=media.eme.wmf.clearkey.enabled=true"
@@ -350,10 +357,13 @@ media-gpu:
                     || "macosx" in task["test-platform"]
                 '
     merge:
+        worker:
+            env:
+                MOZHARNESS_TEST_TAG:
+                    - "media-gpu"
         virtualization: virtual-with-gpu
         mozharness:
             extra-options:
-                - "--tag=media-gpu"
                 - "--setpref=media.hardware-video-decoding.force-enabled=true"
 
 msix:

testing/mochitest/runtests.py

@@ -1726,12 +1726,6 @@ toolbar#nav-bar {
                 subsuite(options.subsuite),
             ]
 
-            # Allow for only running tests/manifests which match this tag
-            if options.conditionedProfile:
-                if not options.test_tags:
-                    options.test_tags = []
-                options.test_tags.append("condprof")
-
             if options.test_tags:
                 filters.append(tags(options.test_tags))
 

testing/mozharness/scripts/android_emulator_unittest.py

@@ -152,6 +152,16 @@ class AndroidEmulatorTest(
                     "help": "Extra user prefs.",
                 },
             ],
+            [
+                ["--tag"],
+                {
+                    "action": "append",
+                    "default": [],
+                    "dest": "test_tags",
+                    "help": "Filter out tests that don't have the given tag. Can be used multiple "
+                    "times in which case the test must contain at least one of the given tags.",
+                },
+            ],
         ]
         + copy.deepcopy(testing_config_options)
         + copy.deepcopy(code_coverage_config_options)
@@ -199,6 +209,7 @@ class AndroidEmulatorTest(
         self.disable_fission = c.get("disable_fission")
         self.web_content_isolation_strategy = c.get("web_content_isolation_strategy")
         self.extra_prefs = c.get("extra_prefs")
+        self.test_tags = c.get("test_tags")
 
     def query_abs_dirs(self):
         if self.abs_dirs:
@@ -346,8 +357,11 @@ class AndroidEmulatorTest(
         cmd.extend(["--setpref={}".format(p) for p in self.extra_prefs])
 
         if not (self.verify_enabled or self.per_test_coverage):
+            if user_paths or self.test_tags:
                 if user_paths:
                     cmd.extend(user_paths)
+                if self.test_tags:
+                    cmd.extend(["--tag={}".format(t) for t in self.test_tags])
             else:
                 if self.this_chunk is not None:
                     cmd.extend(["--this-chunk", self.this_chunk])

testing/xpcshell/runxpcshelltests.py

@@ -1112,7 +1112,7 @@ class XPCShellTests(object):
 
         filters = []
         if test_tags:
-            filters.append(tags(test_tags))
+            filters.extend([tags(x) for x in test_tags])
 
         path_filter = None
         if test_paths:
@@ -1806,9 +1806,6 @@ class XPCShellTests(object):
                 "full", self.appPath
             )
             options["self_test"] = False
-            if not options["test_tags"]:
-                options["test_tags"] = []
-            options["test_tags"].append("condprof")
 
         self.setAbsPath()
 

tools/tryselect/task_config.py

@@ -234,7 +234,7 @@ class Tag(TryConfig):
 
         return {
             "env": {
-                "MOZHARNESS_TEST_TAG": tag[0],
+                "MOZHARNESS_TEST_TAG": json.dumps(tag),
             }
         }