Bug 1696995 - [taskgraph] Drop requirement single match requirement in 'resolve_keyed_by' for test kind, r=taskgraph-reviewers,aki

At least in the 'test' kind, this is causing the keyed_by dicts to get very
complex as we need to ensure that each task is only matched by a single regex.
Instead, this makes them act more like case / match statements where the first
arm that matches the task will be used.

Differential Revision: https://phabricator.services.mozilla.com/D111727

taskcluster/taskgraph/test/test_util_schema.py

@@ -176,6 +176,16 @@ class TestResolveKeyedBy(unittest.TestCase):
             "n",
         )
 
+        self.assertEqual(
+            resolve_keyed_by(
+                {"f": "hats", "x": {"by-f": {"hat.*": "head", "ha.*": "hair"}}},
+                "x",
+                "n",
+                enforce_single_match=False,
+            ),
+            {"f": "hats", "x": "head"},
+        )
+
     def test_no_key_no_default(self):
         """
         When the key referenced in `by-*` doesn't exist, and there is not default value,
@@ -196,7 +206,9 @@ class TestResolveKeyedBy(unittest.TestCase):
         """
         self.assertEqual(
             resolve_keyed_by(
-                {"x": {"by-f": {"hat": "head", "default": "anywhere"}}}, "x", "n"
+                {"x": {"by-f": {"hat": "head", "default": "anywhere"}}},
+                "x",
+                "n",
             ),
             {"x": "anywhere"},
         )

taskcluster/taskgraph/test/test_util_yaml.py

@@ -5,23 +5,47 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
+from textwrap import dedent
 
 from taskgraph.util import yaml
 from mozunit import main, MockedOpen
 
-FOO_YML = """\
-prop:
-    - val1
-"""
-
 
 class TestYaml(unittest.TestCase):
     def test_load(self):
-        with MockedOpen({"/dir1/dir2/foo.yml": FOO_YML}):
+        with MockedOpen(
+            {
+                "/dir1/dir2/foo.yml": dedent(
+                    """\
+                    prop:
+                        - val1
+                    """
+                )
+            }
+        ):
+
             self.assertEqual(
                 yaml.load_yaml("/dir1/dir2", "foo.yml"), {"prop": ["val1"]}
             )
 
+    def test_key_order(self):
+        with MockedOpen(
+            {
+                "/dir1/dir2/foo.yml": dedent(
+                    """\
+                    job:
+                        foo: 1
+                        bar: 2
+                        xyz: 3
+                    """
+                )
+            }
+        ):
+            self.assertEqual(
+                list(yaml.load_yaml("/dir1/dir2", "foo.yml")["job"].keys()),
+                ["foo", "bar", "xyz"],
+            )
+
 
 if __name__ == "__main__":
     main()

taskcluster/taskgraph/transforms/tests.py

@@ -631,7 +631,9 @@ def handle_keyed_by_mozharness(config, tasks):
     ]
     for task in tasks:
         for field in fields:
-            resolve_keyed_by(task, field, item_name=task["test-name"])
+            resolve_keyed_by(
+                task, field, item_name=task["test-name"], enforce_single_match=False
+            )
         yield task
 
 
@@ -702,6 +704,7 @@ def resolve_keys(config, tasks):
             task,
             "require-signed-extensions",
             item_name=task["test-name"],
+            enforce_single_match=False,
             **{
                 "release-type": config.params["release_type"],
             }
@@ -837,7 +840,9 @@ def set_target(config, tasks):
         build_platform = task["build-platform"]
         target = None
         if "target" in task:
-            resolve_keyed_by(task, "target", item_name=task["test-name"])
+            resolve_keyed_by(
+                task, "target", item_name=task["test-name"], enforce_single_match=False
+            )
             target = task["target"]
         if not target:
             if build_platform.startswith("macosx"):
@@ -965,6 +970,7 @@ def handle_keyed_by(config, tasks):
                 field,
                 item_name=task["test-name"],
                 defer=["variant"],
+                enforce_single_match=False,
                 project=config.params["project"],
             )
         yield task
@@ -1232,6 +1238,7 @@ def handle_keyed_by_variant(config, tasks):
                 task,
                 field,
                 item_name=task["test-name"],
+                enforce_single_match=False,
                 variant=task["attributes"].get("unittest_variant"),
             )
         yield task
@@ -1337,7 +1344,9 @@ def handle_tier(config, tasks):
     specify a tier otherwise."""
     for task in tasks:
         if "tier" in task:
-            resolve_keyed_by(task, "tier", item_name=task["test-name"])
+            resolve_keyed_by(
+                task, "tier", item_name=task["test-name"], enforce_single_match=False
+            )
 
         # only override if not set for the test
         if "tier" not in task or task["tier"] == "default":

taskcluster/taskgraph/util/keyed_by.py

@@ -7,7 +7,9 @@ from __future__ import absolute_import, print_function, unicode_literals
 from .attributes import keymatch
 
 
-def evaluate_keyed_by(value, item_name, attributes, defer=None):
+def evaluate_keyed_by(
+    value, item_name, attributes, defer=None, enforce_single_match=True
+):
     """
     For values which can either accept a literal value, or be keyed by some
     attributes, perform that lookup and return the result.
@@ -22,7 +24,6 @@ def evaluate_keyed_by(value, item_name, attributes, defer=None):
     a call to `evaluate_keyed_by(item, 'thing-name', {'test-platform': 'linux96')`
     would return `12`.
 
-    The `item_name` parameter is used to generate useful error messages.
     Items can be nested as deeply as desired::
 
         by-test-platform:
@@ -33,11 +34,19 @@ def evaluate_keyed_by(value, item_name, attributes, defer=None):
             linux: 13
             default: 12
 
-    The `defer` parameter allows evaluating a by-* entry at a later time. In the
-    example above it's possible that the project attribute hasn't been set
-    yet, in which case we'd want to stop before resolving that subkey and then
-    call this function again later. This can be accomplished by setting
-    `defer=["project"]` in this example.
+    Args:
+        value (str): Name of the value to perform evaluation on.
+        item_name (str): Used to generate useful error messages.
+        attributes (dict): Dictionary of attributes used to lookup 'by-<key>' with.
+        defer (list):
+            Allows evaluating a by-* entry at a later time. In the example
+            above it's possible that the project attribute hasn't been set yet,
+            in which case we'd want to stop before resolving that subkey and
+            then call this function again later. This can be accomplished by
+            setting `defer=["project"]` in this example.
+        enforce_single_match (bool):
+            If True (default), each task may only match a single arm of the
+            evaluation.
     """
     while True:
         if not isinstance(value, dict) or len(value) != 1:
@@ -73,7 +82,7 @@ def evaluate_keyed_by(value, item_name, attributes, defer=None):
                 )
 
         matches = keymatch(alternatives, key)
-        if len(matches) > 1:
+        if enforce_single_match and len(matches) > 1:
             raise Exception(
                 "Multiple matching values for {} {!r} found while "
                 "determining item {}".format(keyed_by, key, item_name)

taskcluster/taskgraph/util/schema.py

@@ -65,7 +65,9 @@ def optionally_keyed_by(*arguments):
     return validator
 
 
-def resolve_keyed_by(item, field, item_name, defer=None, **extra_values):
+def resolve_keyed_by(
+    item, field, item_name, defer=None, enforce_single_match=True, **extra_values
+):
     """
     For values which can either accept a literal value, or be keyed by some
     other attribute of the item, perform that lookup and replacement in-place
@@ -89,11 +91,6 @@ def resolve_keyed_by(item, field, item_name, defer=None, **extra_values):
             test-platform: linux128
             chunks: 12
 
-    The `item_name` parameter is used to generate useful error messages.
-
-    If extra_values are supplied, they represent additional values available
-    for reference from by-<field>.
-
     Items can be nested as deeply as the schema will allow::
 
         chunks:
@@ -105,11 +102,25 @@ def resolve_keyed_by(item, field, item_name, defer=None, **extra_values):
                 linux: 13
                 default: 12
 
-    The `defer` parameter allows evaluating a by-* entry at a later time. In the
-    example above it's possible that the project attribute hasn't been set
-    yet, in which case we'd want to stop before resolving that subkey and then
-    call this function again later. This can be accomplished by setting
-    `defer=["project"]` in this example.
+    Args:
+        item (dict): Object being evaluated.
+        field (str): Name of the key to perform evaluation on.
+        item_name (str): Used to generate useful error messages.
+        defer (list):
+            Allows evaluating a by-* entry at a later time. In the example
+            above it's possible that the project attribute hasn't been set yet,
+            in which case we'd want to stop before resolving that subkey and
+            then call this function again later. This can be accomplished by
+            setting `defer=["project"]` in this example.
+        enforce_single_match (bool):
+            If True (default), each task may only match a single arm of the
+            evaluation.
+        extra_values (kwargs):
+            If supplied, represent additional values available
+            for reference from by-<field>.
+
+    Returns:
+        dict: item which has also been modified in-place.
     """
     # find the field, returning the item unchanged if anything goes wrong
     container, subfield = item, field
@@ -128,6 +139,7 @@ def resolve_keyed_by(item, field, item_name, defer=None, **extra_values):
         value=container[subfield],
         item_name="`{}` in `{}`".format(field, item_name),
         defer=defer,
+        enforce_single_match=enforce_single_match,
         attributes=dict(item, **extra_values),
     )