Bug 1632870 - [ci] Create a schema for test settings and associated test to validate it, r=jmaher

This also adds way to bypass the 'check_schema' call since this schema's keys
contains strings from `variants.yml` which sometimes have underscoes in them.

Differential Revision: https://phabricator.services.mozilla.com/D131282

taskcluster/docs/kinds/test.rst

@@ -125,3 +125,6 @@ Test ``settings`` are available in the ``task.extra.test-setting`` object in
 all test tasks. They are defined by the
 :py:func:`~gecko_taskgraph.transforms.tests.set_test_setting` transform
 function.
+
+The full schema is defined in the
+:py:data:`~gecko_taskgraph.transforms.tests.test_setting_description_schema`.

taskcluster/gecko_taskgraph/transforms/tests.py

@@ -1441,6 +1441,39 @@ def split_e10s(config, tasks):
             yield task
 
 
+test_setting_description_schema = Schema(
+    {
+        "platform": {
+            Required("arch"): Any("32", "64", "aarch64", "arm7", "x86_64"),
+            Required("os"): {
+                Required("name"): Any("android", "linux", "macosx", "windows"),
+                Required("version"): str,
+                Optional("build"): str,
+            },
+            Optional("device"): str,
+            Optional("machine"): Any("ref-hw-2017"),
+        },
+        "build": {
+            Required("type"): Any("opt", "debug", "debug-isolated-process"),
+            Any(
+                "asan",
+                "ccov",
+                "clang-trunk",
+                "devedition",
+                "lite",
+                "mingwclang",
+                "shippable",
+                "tsan",
+            ): bool,
+        },
+        "runtime": {Any(*list(TEST_VARIANTS.keys()) + ["1proc"]): bool},
+    },
+    check=False,
+)
+"""Schema test settings must conform to. Validated by
+:py:func:`~test.test_mozilla_central.test_test_setting`"""
+
+
 @transforms.add
 def set_test_setting(config, tasks):
     """A test ``setting`` is the set of configuration that uniquely

taskcluster/gecko_taskgraph/util/schema.py

@@ -209,14 +209,18 @@ class Schema(voluptuous.Schema):
     in the process.
     """
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, check=True, **kwargs):
         super().__init__(*args, **kwargs)
-        if not gecko_taskgraph.fast:
+
+        self.check = check
+        if not gecko_taskgraph.fast and self.check:
             check_schema(self)
 
     def extend(self, *args, **kwargs):
         schema = super().extend(*args, **kwargs)
-        check_schema(schema)
+
+        if self.check:
+            check_schema(schema)
         # We want twice extend schema to be checked too.
         schema.__class__ = Schema
         return schema

taskcluster/test/test_mozilla_central.py

@@ -35,5 +35,37 @@ def test_tasks_are_scheduled(optimized_task_graph, filter_tasks, func, min_expec
     assert len(tasks) >= min_expected
 
 
+def test_test_setting(full_task_graph, filter_tasks):
+    """Verify that all test tasks' ``test-setting`` object conforms to the schema."""
+    from gecko_taskgraph.transforms.tests import test_setting_description_schema
+    from gecko_taskgraph.util.schema import validate_schema
+
+    tasks = filter_tasks(full_task_graph, lambda t: t.kind == "test")
+
+    failures = []
+    for task in tasks:
+        try:
+            validate_schema(
+                test_setting_description_schema,
+                task.task["extra"]["test-setting"],
+                task.label,
+            )
+        except Exception as e:
+            failures.append(e)
+
+    if failures:
+        more = None
+        # Only display a few failures at once.
+        if len(failures) > 10:
+            more = len(failures) - 10
+            failures = failures[:10]
+
+        failstr = "\n\n" + "\n\n".join(str(e) for e in failures)
+        if more:
+            failstr += "\n\n" + f"... and {more} more"
+
+        pytest.fail(f"Task validation errors:{failstr}")
+
+
 if __name__ == "__main__":
     main()