Bug 1766316 - [taskgraph] Merge 'util/schema.py' between the two taskgraphs, r=releng-reviewers,aki

Differential Revision: https://phabricator.services.mozilla.com/D150414

taskcluster/gecko_taskgraph/__init__.py

@@ -4,7 +4,7 @@
 
 
 import os
-from taskgraph.util import taskcluster as tc_util
+from taskgraph.util import taskcluster as tc_util, schema
 
 GECKO = os.path.normpath(os.path.realpath(os.path.join(__file__, "..", "..", "..")))
 
@@ -18,6 +18,21 @@ MAX_DEPENDENCIES = 99
 # to the production Taskcluster deployment used for CI.
 tc_util.PRODUCTION_TASKCLUSTER_ROOT_URL = "https://firefox-ci-tc.services.mozilla.com"
 
+# Schemas for YAML files should use dashed identifiers by default.  If there are
+# components of the schema for which there is a good reason to use another format,
+# they can be whitelisted here.
+schema.WHITELISTED_SCHEMA_IDENTIFIERS.extend(
+    [
+        # upstream-artifacts are handed directly to scriptWorker, which expects interCaps
+        lambda path: "[{!r}]".format("upstream-artifacts") in path,
+        lambda path: (
+            "[{!r}]".format("test_name") in path
+            or "[{!r}]".format("json_location") in path
+            or "[{!r}]".format("video_location") in path
+        ),
+    ]
+)
+
 
 def register(graph_config):
     """Used to register Gecko specific extensions.

taskcluster/gecko_taskgraph/config.py

@@ -7,11 +7,10 @@ import os
 import logging
 
 from taskgraph.config import GraphConfig
+from taskgraph.util.schema import Schema, optionally_keyed_by, validate_schema
 from taskgraph.util.yaml import load_yaml
 from voluptuous import Required, Optional, Any
 
-from .util.schema import validate_schema, Schema, optionally_keyed_by
-
 logger = logging.getLogger(__name__)
 
 graph_config_schema = Schema(

taskcluster/gecko_taskgraph/decision.py

@@ -16,6 +16,7 @@ from redo import retry
 from taskgraph.parameters import Parameters
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.util.python_path import find_object
+from taskgraph.util.schema import Schema, validate_schema
 from taskgraph.util.taskcluster import get_artifact
 from taskgraph.util.yaml import load_yaml
 from voluptuous import Required, Optional, Any
@@ -31,7 +32,6 @@ from .util.bugbug import push_schedules
 from .util.chunking import resolver
 from .util.hg import get_hg_revision_branch, get_hg_commit_message
 from .util.partials import populate_release_history
-from .util.schema import validate_schema, Schema
 from .util.taskcluster import insert_index
 from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds
 

taskcluster/gecko_taskgraph/loader/multi_dep.py

@@ -6,10 +6,10 @@
 import copy
 
 from taskgraph.task import Task
+from taskgraph.util.schema import Schema
 from voluptuous import Required
 
 from ..util.attributes import sorted_unique_list
-from ..util.schema import Schema
 
 schema = Schema(
     {

taskcluster/gecko_taskgraph/loader/single_dep.py

@@ -6,10 +6,9 @@
 import copy
 
 from taskgraph.task import Task
+from taskgraph.util.schema import Schema
 from voluptuous import Required
 
-from ..util.schema import Schema
-
 schema = Schema(
     {
         Required("primary-dependency", "primary dependency task"): Task,

taskcluster/gecko_taskgraph/test/python.ini

@@ -23,6 +23,5 @@ subsuite = taskgraph
 [test_util_docker.py]
 [test_util_partials.py]
 [test_util_runnable_jobs.py]
-[test_util_schema.py]
 [test_util_templates.py]
 [test_util_verify.py]

taskcluster/gecko_taskgraph/test/test_transforms_job.py

@@ -12,6 +12,7 @@ from copy import deepcopy
 
 import pytest
 from mozunit import main
+from taskgraph.util.schema import Schema, validate_schema
 
 from gecko_taskgraph import GECKO
 from gecko_taskgraph.config import load_graph_config
@@ -20,7 +21,6 @@ from gecko_taskgraph.transforms.job import run_task  # noqa: F401
 from gecko_taskgraph.transforms.base import TransformConfig
 from gecko_taskgraph.transforms.job.common import add_cache
 from gecko_taskgraph.transforms.task import payload_builders
-from gecko_taskgraph.util.schema import Schema, validate_schema
 
 from conftest import FakeParameters
 

taskcluster/gecko_taskgraph/test/test_util_schema.py

@@ -1,216 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-import unittest
-from mozunit import main
-from gecko_taskgraph.util.schema import (
-    validate_schema,
-    resolve_keyed_by,
-    Schema,
-)
-
-schema = Schema(
-    {
-        "x": int,
-        "y": str,
-    }
-)
-
-
-class TestValidateSchema(unittest.TestCase):
-    def test_valid(self):
-        validate_schema(schema, {"x": 10, "y": "foo"}, "pfx")
-
-    def test_invalid(self):
-        try:
-            validate_schema(schema, {"x": "not-int"}, "pfx")
-            self.fail("no exception raised")
-        except Exception as e:
-            self.assertTrue(str(e).startswith("pfx\n"))
-
-
-class TestCheckSchema(unittest.TestCase):
-    def test_schema(self):
-        "Creating a schema applies taskgraph checks."
-        with self.assertRaises(Exception):
-            Schema({"camelCase": int})
-
-    def test_extend_schema(self):
-        "Extending a schema applies taskgraph checks."
-        with self.assertRaises(Exception):
-            Schema({"kebab-case": int}).extend({"camelCase": int})
-
-    def test_extend_schema_twice(self):
-        "Extending a schema twice applies taskgraph checks."
-        with self.assertRaises(Exception):
-            Schema({"kebab-case": int}).extend({"more-kebab": int}).extend(
-                {"camelCase": int}
-            )
-
-
-class TestResolveKeyedBy(unittest.TestCase):
-    def test_no_by(self):
-        self.assertEqual(resolve_keyed_by({"x": 10}, "z", "n"), {"x": 10})
-
-    def test_no_by_dotted(self):
-        self.assertEqual(
-            resolve_keyed_by({"x": {"y": 10}}, "x.z", "n"), {"x": {"y": 10}}
-        )
-
-    def test_no_by_not_dict(self):
-        self.assertEqual(resolve_keyed_by({"x": 10}, "x.y", "n"), {"x": 10})
-
-    def test_no_by_not_by(self):
-        self.assertEqual(resolve_keyed_by({"x": {"a": 10}}, "x", "n"), {"x": {"a": 10}})
-
-    def test_nested(self):
-        x = {
-            "by-foo": {
-                "F1": {
-                    "by-bar": {
-                        "B1": 11,
-                        "B2": 12,
-                    },
-                },
-                "F2": 20,
-                "default": 0,
-            },
-        }
-        self.assertEqual(
-            resolve_keyed_by({"x": x}, "x", "x", foo="F1", bar="B1"), {"x": 11}
-        )
-        self.assertEqual(
-            resolve_keyed_by({"x": x}, "x", "x", foo="F1", bar="B2"), {"x": 12}
-        )
-        self.assertEqual(resolve_keyed_by({"x": x}, "x", "x", foo="F2"), {"x": 20})
-        self.assertEqual(
-            resolve_keyed_by({"x": x}, "x", "x", foo="F99", bar="B1"), {"x": 0}
-        )
-
-        # bar is deferred
-        self.assertEqual(
-            resolve_keyed_by({"x": x}, "x", "x", defer=["bar"], foo="F1", bar="B1"),
-            {"x": {"by-bar": {"B1": 11, "B2": 12}}},
-        )
-
-    def test_no_by_empty_dict(self):
-        self.assertEqual(resolve_keyed_by({"x": {}}, "x", "n"), {"x": {}})
-
-    def test_no_by_not_only_by(self):
-        self.assertEqual(
-            resolve_keyed_by({"x": {"by-y": True, "a": 10}}, "x", "n"),
-            {"x": {"by-y": True, "a": 10}},
-        )
-
-    def test_match_nested_exact(self):
-        self.assertEqual(
-            resolve_keyed_by(
-                {
-                    "f": "shoes",
-                    "x": {"y": {"by-f": {"shoes": "feet", "gloves": "hands"}}},
-                },
-                "x.y",
-                "n",
-            ),
-            {"f": "shoes", "x": {"y": "feet"}},
-        )
-
-    def test_match_regexp(self):
-        self.assertEqual(
-            resolve_keyed_by(
-                {
-                    "f": "shoes",
-                    "x": {"by-f": {"s?[hH]oes?": "feet", "gloves": "hands"}},
-                },
-                "x",
-                "n",
-            ),
-            {"f": "shoes", "x": "feet"},
-        )
-
-    def test_match_partial_regexp(self):
-        self.assertEqual(
-            resolve_keyed_by(
-                {"f": "shoes", "x": {"by-f": {"sh": "feet", "default": "hands"}}},
-                "x",
-                "n",
-            ),
-            {"f": "shoes", "x": "hands"},
-        )
-
-    def test_match_default(self):
-        self.assertEqual(
-            resolve_keyed_by(
-                {"f": "shoes", "x": {"by-f": {"hat": "head", "default": "anywhere"}}},
-                "x",
-                "n",
-            ),
-            {"f": "shoes", "x": "anywhere"},
-        )
-
-    def test_match_extra_value(self):
-        self.assertEqual(
-            resolve_keyed_by({"f": {"by-foo": {"x": 10, "y": 20}}}, "f", "n", foo="y"),
-            {"f": 20},
-        )
-
-    def test_no_match(self):
-        self.assertRaises(
-            Exception,
-            resolve_keyed_by,
-            {"f": "shoes", "x": {"by-f": {"hat": "head"}}},
-            "x",
-            "n",
-        )
-
-    def test_multiple_matches(self):
-        self.assertRaises(
-            Exception,
-            resolve_keyed_by,
-            {"f": "hats", "x": {"by-f": {"hat.*": "head", "ha.*": "hair"}}},
-            "x",
-            "n",
-        )
-
-        self.assertEqual(
-            resolve_keyed_by(
-                {"f": "hats", "x": {"by-f": {"hat.*": "head", "ha.*": "hair"}}},
-                "x",
-                "n",
-                enforce_single_match=False,
-            ),
-            {"f": "hats", "x": "head"},
-        )
-
-    def test_no_key_no_default(self):
-        """
-        When the key referenced in `by-*` doesn't exist, and there is not default value,
-        an exception is raised.
-        """
-        self.assertRaises(
-            Exception,
-            resolve_keyed_by,
-            {"x": {"by-f": {"hat.*": "head", "ha.*": "hair"}}},
-            "x",
-            "n",
-        )
-
-    def test_no_key(self):
-        """
-        When the key referenced in `by-*` doesn't exist, and there is a default value,
-        that value is used as the result.
-        """
-        self.assertEqual(
-            resolve_keyed_by(
-                {"x": {"by-f": {"hat": "head", "default": "anywhere"}}},
-                "x",
-                "n",
-            ),
-            {"x": "anywhere"},
-        )
-
-
-if __name__ == "__main__":
-    main()

taskcluster/gecko_taskgraph/transforms/balrog_submit.py

@@ -5,14 +5,14 @@
 Transform the per-locale balrog task into an actual task description.
 """
 
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
+from taskgraph.util.treeherder import replace_group
+from voluptuous import Optional
 
 from gecko_taskgraph.loader.single_dep import schema
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.task import task_description_schema
 from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
-from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
-from taskgraph.util.treeherder import replace_group
-from voluptuous import Optional
 
 balrog_description_schema = schema.extend(
     {

taskcluster/gecko_taskgraph/transforms/base.py

@@ -6,8 +6,7 @@
 import attr
 from taskgraph.config import GraphConfig
 from taskgraph.parameters import Parameters
-
-from ..util.schema import Schema, validate_schema
+from taskgraph.util.schema import Schema, validate_schema
 
 
 @attr.s(frozen=True)

taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py

@@ -8,6 +8,9 @@ Transform the beetmover task into an actual task description.
 
 from copy import deepcopy
 
+from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
+from voluptuous import Required, Optional
+
 from gecko_taskgraph.loader.single_dep import schema
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.beetmover import (
@@ -22,9 +25,7 @@ from gecko_taskgraph.util.declarative_artifacts import (
     get_geckoview_upstream_artifacts,
     get_geckoview_artifact_id,
 )
-from gecko_taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
 from gecko_taskgraph.transforms.task import task_description_schema
-from voluptuous import Required, Optional
 
 
 beetmover_description_schema = schema.extend(

taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py

@@ -5,19 +5,15 @@
 Transform the beetmover-push-to-release task into a task description.
 """
 
+from taskgraph.util.schema import Schema, taskref_or_string
+from voluptuous import Required, Optional
 
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import (
-    Schema,
-    taskref_or_string,
-)
 from gecko_taskgraph.util.scriptworker import (
     get_beetmover_bucket_scope,
     add_scope_prefix,
 )
 from gecko_taskgraph.transforms.task import task_description_schema
-from voluptuous import Required, Optional
-
 
 beetmover_push_to_release_description_schema = Schema(
     {

taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py

@@ -8,6 +8,7 @@ Transform the beetmover task into an actual task description.
 import logging
 from copy import deepcopy
 
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
 from taskgraph.util.taskcluster import get_artifact_prefix
 from voluptuous import Any, Required, Optional
 
@@ -22,10 +23,6 @@ from gecko_taskgraph.util.partners import (
     get_ftp_platform,
     get_partner_config_by_kind,
 )
-from gecko_taskgraph.util.schema import (
-    optionally_keyed_by,
-    resolve_keyed_by,
-)
 from gecko_taskgraph.util.scriptworker import (
     add_scope_prefix,
     get_beetmover_bucket_scope,

taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py

@@ -8,6 +8,8 @@ Add from parameters.yml into bouncer submission tasks.
 
 import logging
 
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.bouncer_submission import craft_bouncer_product_name
 from gecko_taskgraph.transforms.bouncer_submission_partners import (
@@ -15,7 +17,6 @@ from gecko_taskgraph.transforms.bouncer_submission_partners import (
 )
 from gecko_taskgraph.util.attributes import release_level
 from gecko_taskgraph.util.partners import get_partners_to_be_published
-from gecko_taskgraph.util.schema import resolve_keyed_by
 from gecko_taskgraph.util.scriptworker import get_release_config
 
 logger = logging.getLogger(__name__)

taskcluster/gecko_taskgraph/transforms/bouncer_check.py

@@ -5,10 +5,11 @@
 import json
 from pipes import quote as shell_quote
 
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import release_level
 from gecko_taskgraph.util.scriptworker import get_release_config
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 import logging
 

taskcluster/gecko_taskgraph/transforms/bouncer_locations.py

@@ -4,8 +4,9 @@
 
 import logging
 
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 logger = logging.getLogger(__name__)
 

taskcluster/gecko_taskgraph/transforms/bouncer_submission.py

@@ -10,11 +10,11 @@ import copy
 import logging
 
 import attr
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.l10n import parse_locales_file
 from gecko_taskgraph.util.attributes import release_level
-from gecko_taskgraph.util.schema import resolve_keyed_by
 from gecko_taskgraph.util.scriptworker import get_release_config
 
 logger = logging.getLogger(__name__)

taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py

@@ -8,6 +8,8 @@ Add from parameters.yml into bouncer submission tasks.
 
 import logging
 
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.bouncer_submission import (
     FTP_PLATFORMS_PER_BOUNCER_PLATFORM,
@@ -20,7 +22,6 @@ from gecko_taskgraph.util.partners import (
     check_if_partners_enabled,
     get_partners_to_be_published,
 )
-from gecko_taskgraph.util.schema import resolve_keyed_by
 from gecko_taskgraph.util.scriptworker import get_release_config
 
 logger = logging.getLogger(__name__)

taskcluster/gecko_taskgraph/transforms/build.py

@@ -5,16 +5,15 @@
 Apply some defaults and minor modifications to the jobs defined in the build
 kind.
 """
-
-
 import logging
 
+from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.treeherder import add_suffix
+
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import RELEASE_PROJECTS, is_try, release_level
-from gecko_taskgraph.util.schema import resolve_keyed_by
 from gecko_taskgraph.util.workertypes import worker_type_implementation
-from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS
-from taskgraph.util.treeherder import add_suffix
 
 logger = logging.getLogger(__name__)
 

taskcluster/gecko_taskgraph/transforms/condprof.py

@@ -8,6 +8,7 @@ the condprof/kind.yml file
 
 from copy import deepcopy
 
+from taskgraph.util.schema import Schema
 from voluptuous import (
     Optional,
 )
@@ -15,7 +16,6 @@ from voluptuous import (
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.job import job_description_schema
 from gecko_taskgraph.transforms.task import task_description_schema
-from gecko_taskgraph.util.schema import Schema
 
 diff_description_schema = Schema(
     {

taskcluster/gecko_taskgraph/transforms/cross_channel.py

@@ -8,8 +8,9 @@ Build a command to run `mach l10n-cross-channel`.
 
 from pipes import quote as shell_quote
 
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/diffoscope.py

@@ -7,6 +7,7 @@ defined in kind.yml
 """
 
 from taskgraph.util.taskcluster import get_artifact_path
+from taskgraph.util.schema import Schema
 from voluptuous import (
     Any,
     Optional,
@@ -15,7 +16,6 @@ from voluptuous import (
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.task import task_description_schema
-from gecko_taskgraph.util.schema import Schema
 
 index_or_string = Any(
     str,

taskcluster/gecko_taskgraph/transforms/docker_image.py

@@ -10,6 +10,7 @@ import json
 
 import mozpack.path as mozpath
 import taskgraph
+from taskgraph.util.schema import Schema
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from .. import GECKO
@@ -18,7 +19,6 @@ from gecko_taskgraph.util.docker import (
     generate_context_hash,
     image_path,
 )
-from gecko_taskgraph.util.schema import Schema
 from voluptuous import (
     Optional,
     Required,

taskcluster/gecko_taskgraph/transforms/fetch.py

@@ -13,13 +13,13 @@ import attr
 import taskgraph
 from mozbuild.shellutil import quote as shell_quote
 from mozpack import path as mozpath
+from taskgraph.util.schema import Schema, validate_schema
 from taskgraph.util.treeherder import join_symbol
 from voluptuous import Any, Extra, Optional, Required
 
 import gecko_taskgraph
 from .base import TransformSequence
 from ..util.cached_tasks import add_optimization
-from ..util.schema import Schema, validate_schema
 
 CACHE_TYPE = "content.v1"
 

taskcluster/gecko_taskgraph/transforms/job/__init__.py

@@ -17,6 +17,7 @@ import json
 import mozpack.path as mozpath
 from taskgraph.util.python_path import import_sibling_modules
 from taskgraph.util.taskcluster import get_artifact_prefix
+from taskgraph.util.schema import Schema, validate_schema
 from voluptuous import (
     Extra,
     Optional,
@@ -26,10 +27,6 @@ from voluptuous import (
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.cached_tasks import order_tasks
-from gecko_taskgraph.util.schema import (
-    validate_schema,
-    Schema,
-)
 from gecko_taskgraph.util.workertypes import worker_type_implementation
 from gecko_taskgraph.transforms.task import task_description_schema
 

taskcluster/gecko_taskgraph/transforms/job/distro_package.py

@@ -10,6 +10,7 @@ import os
 import re
 
 import taskgraph
+from taskgraph.util.schema import Schema
 from taskgraph.util.taskcluster import get_root_url
 from voluptuous import Any, Optional, Required
 
@@ -17,7 +18,6 @@ from gecko_taskgraph.transforms.job import run_job_using
 from gecko_taskgraph.transforms.job.common import add_artifacts
 
 from gecko_taskgraph.util.hash import hash_path
-from gecko_taskgraph.util.schema import Schema
 from gecko_taskgraph import GECKO
 
 DSC_PACKAGE_RE = re.compile(".*(?=_)")

taskcluster/gecko_taskgraph/transforms/job/hazard.py

@@ -6,7 +6,7 @@ Support for running hazard jobs via dedicated scripts
 """
 
 
-from gecko_taskgraph.util.schema import Schema
+from taskgraph.util.schema import Schema
 from voluptuous import Required, Optional, Any
 
 from gecko_taskgraph.transforms.job import (

taskcluster/gecko_taskgraph/transforms/job/mach.py

@@ -5,13 +5,10 @@
 Support for running mach tasks (via run-task)
 """
 
+from taskgraph.util.schema import Schema, taskref_or_string
+from voluptuous import Required, Optional, Any
 
 from gecko_taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
-from gecko_taskgraph.util.schema import (
-    Schema,
-    taskref_or_string,
-)
-from voluptuous import Required, Optional, Any
 
 mach_schema = Schema(
     {

taskcluster/gecko_taskgraph/transforms/job/mozharness.py

@@ -12,7 +12,7 @@ import json
 
 from textwrap import dedent
 
-from gecko_taskgraph.util.schema import Schema
+from taskgraph.util.schema import Schema
 from voluptuous import Required, Optional, Any
 from voluptuous.validators import Match
 

taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py

@@ -7,6 +7,7 @@ import json
 import os
 import re
 
+from taskgraph.util.schema import Schema
 from taskgraph.util.taskcluster import get_artifact_path, get_artifact_url
 from voluptuous import Extra, Optional, Required
 
@@ -15,7 +16,6 @@ from gecko_taskgraph.transforms.job import (
     run_job_using,
 )
 from gecko_taskgraph.util.attributes import is_try
-from gecko_taskgraph.util.schema import Schema
 from gecko_taskgraph.transforms.test import test_description_schema, normpath
 from gecko_taskgraph.transforms.job.common import support_vcs_checkout
 

taskcluster/gecko_taskgraph/transforms/job/python_test.py

@@ -6,10 +6,11 @@ Support for running mach python-test tasks (via run-task)
 """
 
 
-from gecko_taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
-from gecko_taskgraph.util.schema import Schema
+from taskgraph.util.schema import Schema
 from voluptuous import Required, Optional
 
+from gecko_taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
+
 python_test_schema = Schema(
     {
         Required("using"): "python-test",

taskcluster/gecko_taskgraph/transforms/job/run_task.py

@@ -9,10 +9,10 @@ Support for running jobs that are invoked via the `run-task` script.
 import os
 
 from mozpack import path
+from taskgraph.util.schema import Schema
 
 from gecko_taskgraph.transforms.task import taskref_or_string
 from gecko_taskgraph.transforms.job import run_job_using
-from gecko_taskgraph.util.schema import Schema
 from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout
 from voluptuous import Any, Optional, Required
 

taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py

@@ -6,7 +6,7 @@ Support for running spidermonkey jobs via dedicated scripts
 """
 
 
-from gecko_taskgraph.util.schema import Schema
+from taskgraph.util.schema import Schema
 from voluptuous import Required, Any, Optional
 
 from gecko_taskgraph.transforms.job import (

taskcluster/gecko_taskgraph/transforms/job/toolchain.py

@@ -9,7 +9,7 @@ Support for running toolchain-building jobs via dedicated scripts
 import taskgraph
 from mozbuild.shellutil import quote as shell_quote
 
-from gecko_taskgraph.util.schema import Schema
+from taskgraph.util.schema import Schema
 from voluptuous import Optional, Required, Any
 
 from gecko_taskgraph.transforms.job import (

taskcluster/gecko_taskgraph/transforms/l10n.py

@@ -9,6 +9,20 @@ Do transforms specific to l10n kind
 import copy
 import json
 
+from mozbuild.chunkify import chunkify
+from taskgraph.util.schema import (
+    optionally_keyed_by,
+    resolve_keyed_by,
+    taskref_or_string,
+)
+from taskgraph.util.taskcluster import get_artifact_prefix
+from taskgraph.util.treeherder import add_suffix
+from voluptuous import (
+    Any,
+    Optional,
+    Required,
+)
+
 from gecko_taskgraph.loader.multi_dep import schema
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.job import job_description_schema
@@ -17,15 +31,6 @@ from gecko_taskgraph.util.attributes import (
     copy_attributes_from_dependent_job,
     task_name,
 )
-from gecko_taskgraph.util.schema import (
-    optionally_keyed_by,
-    resolve_keyed_by,
-    taskref_or_string,
-)
-from mozbuild.chunkify import chunkify
-from taskgraph.util.taskcluster import get_artifact_prefix
-from taskgraph.util.treeherder import add_suffix
-from voluptuous import Any, Optional, Required
 
 
 def _by_platform(arg):

taskcluster/gecko_taskgraph/transforms/maybe_release.py

@@ -2,10 +2,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import release_level
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/merge_automation.py

@@ -5,9 +5,9 @@
 Transform the update generation task into an actual task description.
 """
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py

@@ -9,6 +9,7 @@ from collections import defaultdict
 from copy import deepcopy
 
 from taskgraph.util.taskcluster import get_artifact_prefix
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
 from voluptuous import Any, Required, Optional
 
 from gecko_taskgraph.loader.single_dep import schema
@@ -22,10 +23,6 @@ from gecko_taskgraph.util.partners import (
     get_partner_config_by_kind,
     apply_partner_priority,
 )
-from gecko_taskgraph.util.schema import (
-    optionally_keyed_by,
-    resolve_keyed_by,
-)
 from gecko_taskgraph.util.scriptworker import (
     add_scope_prefix,
     get_beetmover_bucket_scope,

taskcluster/gecko_taskgraph/transforms/partner_repack.py

@@ -5,10 +5,10 @@
 Transform the partner repack task into an actual task description.
 """
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import release_level
-from gecko_taskgraph.util.schema import resolve_keyed_by
 from gecko_taskgraph.util.scriptworker import get_release_config
 from gecko_taskgraph.util.partners import (
     check_if_partners_enabled,

taskcluster/gecko_taskgraph/transforms/perftest.py

@@ -9,10 +9,15 @@ import json
 from copy import deepcopy
 from datetime import date, timedelta
 
-from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
+from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
 from taskgraph.util.treeherder import join_symbol, split_symbol
-from voluptuous import Any, Extra, Optional
+from voluptuous import (
+    Any,
+    Optional,
+    Extra,
+)
+
+from gecko_taskgraph.transforms.base import TransformSequence
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/python_update.py

@@ -5,9 +5,9 @@
 Transform the repo-update task into an actual task description.
 """
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py

@@ -5,10 +5,13 @@
 Transform the beetmover task into an actual task description.
 """
 
-
 import copy
 import logging
 
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
+from taskgraph.util.treeherder import inherit_treeherder_from_dep
+from voluptuous import Required, Optional
+
 from gecko_taskgraph.loader.single_dep import schema
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.beetmover import craft_release_properties
@@ -17,15 +20,12 @@ from gecko_taskgraph.util.attributes import (
     copy_attributes_from_dependent_job,
     release_level,
 )
-from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
 from gecko_taskgraph.util.scriptworker import (
     generate_beetmover_artifact_map,
     generate_beetmover_upstream_artifacts,
     get_beetmover_action_scope,
     get_beetmover_bucket_scope,
 )
-from taskgraph.util.treeherder import inherit_treeherder_from_dep
-from voluptuous import Optional, Required
 
 logger = logging.getLogger(__name__)
 

taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py

@@ -5,14 +5,14 @@
 Transform the release-flatpak-push kind into an actual task description.
 """
 
+from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
+from voluptuous import Optional, Required
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.task import task_description_schema
 from gecko_taskgraph.util.attributes import release_level
-from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
 from gecko_taskgraph.util.scriptworker import add_scope_prefix
 
-from voluptuous import Optional, Required
 
 push_flatpak_description_schema = Schema(
     {

taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py

@@ -2,11 +2,11 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import release_level
 from gecko_taskgraph.util.scriptworker import get_release_config
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 
 transforms = TransformSequence()

taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py

@@ -7,10 +7,11 @@ Transform the checksums task into an actual task description.
 
 import copy
 
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import release_level
 from gecko_taskgraph.util.scriptworker import get_release_config
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 import logging
 

taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py

@@ -2,10 +2,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import release_level
-from gecko_taskgraph.util.schema import resolve_keyed_by
 from gecko_taskgraph.util.scriptworker import get_release_config
 
 transforms = TransformSequence()

taskcluster/gecko_taskgraph/transforms/release_msix_push.py

@@ -5,15 +5,14 @@
 Transform the release-msix-push kind into an actual task description.
 """
 
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
+from voluptuous import Optional, Required
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.task import task_description_schema
 from gecko_taskgraph.util.attributes import release_level
-from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
 from gecko_taskgraph.util.scriptworker import add_scope_prefix
 
-from voluptuous import Optional, Required
-
 push_msix_description_schema = Schema(
     {
         Required("name"): str,

taskcluster/gecko_taskgraph/transforms/release_notifications.py

@@ -4,12 +4,12 @@
 """
 Add notifications via taskcluster-notify for release tasks
 """
-
-
 from string import Formatter
+
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.scriptworker import get_release_config
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 
 transforms = TransformSequence()

taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py

@@ -5,6 +5,9 @@
 Transform the release-sign-and-push task into an actual task description.
 """
 
+from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
+from taskgraph.util.treeherder import inherit_treeherder_from_dep
+from voluptuous import Any, Required
 
 from gecko_taskgraph.loader.single_dep import schema
 from gecko_taskgraph.transforms.base import TransformSequence
@@ -13,9 +16,6 @@ from gecko_taskgraph.util.attributes import (
     copy_attributes_from_dependent_job,
     release_level,
 )
-from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
-from taskgraph.util.treeherder import inherit_treeherder_from_dep
-from voluptuous import Any, Required
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py

@@ -2,11 +2,11 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import release_level
 from gecko_taskgraph.util.scriptworker import get_release_config
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 
 transforms = TransformSequence()

taskcluster/gecko_taskgraph/transforms/release_started.py

@@ -4,12 +4,11 @@
 """
 Add notifications via taskcluster-notify for release tasks
 """
-
-
 from pipes import quote as shell_quote
 
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 
 transforms = TransformSequence()

taskcluster/gecko_taskgraph/transforms/release_version_bump.py

@@ -5,9 +5,9 @@
 Transform the update generation task into an actual task description.
 """
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/repackage.py

@@ -9,15 +9,12 @@ Transform the repackage task into an actual task description.
 import copy
 
 from taskgraph.util.taskcluster import get_artifact_prefix
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
 from voluptuous import Required, Optional, Extra
 
 from gecko_taskgraph.loader.single_dep import schema
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
-from gecko_taskgraph.util.schema import (
-    optionally_keyed_by,
-    resolve_keyed_by,
-)
 from gecko_taskgraph.util.platforms import archive_format, architecture
 from gecko_taskgraph.util.workertypes import worker_type_implementation
 from gecko_taskgraph.transforms.job import job_description_schema

taskcluster/gecko_taskgraph/transforms/repackage_partner.py

@@ -8,16 +8,13 @@ Transform the repackage task into an actual task description.
 
 import copy
 
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
 from taskgraph.util.taskcluster import get_artifact_prefix
 from voluptuous import Required, Optional
 
 from gecko_taskgraph.loader.single_dep import schema
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
-from gecko_taskgraph.util.schema import (
-    optionally_keyed_by,
-    resolve_keyed_by,
-)
 from gecko_taskgraph.util.partners import get_partner_config_by_kind
 from gecko_taskgraph.util.platforms import archive_format, executable_extension
 from gecko_taskgraph.util.workertypes import worker_type_implementation

taskcluster/gecko_taskgraph/transforms/repo_update.py

@@ -5,9 +5,9 @@
 Transform the repo-update task into an actual task description.
 """
 
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/signing.py

@@ -6,6 +6,8 @@ Transform the signing task into an actual task description.
 """
 
 from taskgraph.util.keyed_by import evaluate_keyed_by
+from taskgraph.util.schema import taskref_or_string
+from voluptuous import Required, Optional
 
 from gecko_taskgraph.loader.single_dep import schema
 from gecko_taskgraph.transforms.base import TransformSequence
@@ -13,11 +15,8 @@ from gecko_taskgraph.util.attributes import (
     copy_attributes_from_dependent_job,
     release_level,
 )
-from gecko_taskgraph.util.schema import taskref_or_string
 from gecko_taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
 from gecko_taskgraph.transforms.task import task_description_schema
-from voluptuous import Required, Optional
-
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/source_test.py

@@ -12,6 +12,7 @@ import os
 
 import taskgraph
 from taskgraph.util.attributes import keymatch
+from taskgraph.util.schema import Schema, resolve_keyed_by, optionally_keyed_by
 from taskgraph.util.treeherder import join_symbol, split_symbol
 from voluptuous import (
     Any,
@@ -23,7 +24,6 @@ from voluptuous import (
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.job import job_description_schema
 from gecko_taskgraph.util.hg import get_json_automationrelevance
-from gecko_taskgraph.util.schema import Schema, resolve_keyed_by, optionally_keyed_by
 
 source_test_description_schema = Schema(
     {

taskcluster/gecko_taskgraph/transforms/spidermonkey.py

@@ -1,10 +1,11 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import copy
+
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import resolve_keyed_by
-import copy
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/task.py

@@ -16,30 +16,29 @@ import time
 from copy import deepcopy
 
 import attr
-from gecko_taskgraph import GECKO, MAX_DEPENDENCIES
-from gecko_taskgraph.optimize.schema import OptimizationSchema
-from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.attributes import TRUNK_PROJECTS, is_try, release_level
-from gecko_taskgraph.util.hash import hash_path
-from gecko_taskgraph.util.partners import get_partners_to_be_published
-from gecko_taskgraph.util.schema import (
+from mozbuild.util import memoize
+from taskgraph.util.keyed_by import evaluate_keyed_by
+from taskgraph.util.schema import (
     Schema,
     optionally_keyed_by,
     resolve_keyed_by,
     taskref_or_string,
     validate_schema,
 )
-from gecko_taskgraph.util.scriptworker import BALROG_ACTIONS, get_release_config
-from gecko_taskgraph.util.signed_artifacts import get_signed_artifacts
-from gecko_taskgraph.util.workertypes import worker_type_implementation
-from mozbuild.util import memoize
-from taskgraph.util.keyed_by import evaluate_keyed_by
 from taskgraph.util.time import value_of
 from taskgraph.util.treeherder import split_symbol
-from voluptuous import All, Any, Extra, Match, NotIn, Optional, Required
+from voluptuous import Any, Required, Optional, Extra, Match, All, NotIn
 
-from ..util import docker as dockerutil
-from ..util.workertypes import get_worker_type
+from gecko_taskgraph import GECKO, MAX_DEPENDENCIES
+from gecko_taskgraph.optimize.schema import OptimizationSchema
+from gecko_taskgraph.transforms.base import TransformSequence
+from gecko_taskgraph.util import docker as dockerutil
+from gecko_taskgraph.util.attributes import TRUNK_PROJECTS, is_try, release_level
+from gecko_taskgraph.util.hash import hash_path
+from gecko_taskgraph.util.partners import get_partners_to_be_published
+from gecko_taskgraph.util.scriptworker import BALROG_ACTIONS, get_release_config
+from gecko_taskgraph.util.signed_artifacts import get_signed_artifacts
+from gecko_taskgraph.util.workertypes import get_worker_type, worker_type_implementation
 
 RUN_TASK = os.path.join(GECKO, "taskcluster", "scripts", "run-task")
 

taskcluster/gecko_taskgraph/transforms/test/__init__.py

@@ -22,6 +22,7 @@ import logging
 from importlib import import_module
 
 from mozbuild.schedules import INCLUSIVE_COMPONENTS
+from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
 from voluptuous import (
     Any,
     Optional,
@@ -32,11 +33,6 @@ from voluptuous import (
 from gecko_taskgraph.optimize.schema import OptimizationSchema
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.test.other import get_mobile_project
-from gecko_taskgraph.util.schema import (
-    optionally_keyed_by,
-    resolve_keyed_by,
-    Schema,
-)
 from gecko_taskgraph.util.chunking import manifest_loaders
 
 

taskcluster/gecko_taskgraph/transforms/test/other.py

@@ -6,16 +6,21 @@ import hashlib
 import json
 import re
 
-from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS
-from gecko_taskgraph.util.platforms import platform_family
-from gecko_taskgraph.util.schema import Schema, resolve_keyed_by
 from mozbuild.schedules import INCLUSIVE_COMPONENTS
 from mozbuild.util import ReadOnlyDict
 from taskgraph.util.attributes import keymatch
 from taskgraph.util.keyed_by import evaluate_keyed_by
+from taskgraph.util.schema import Schema, resolve_keyed_by
 from taskgraph.util.taskcluster import get_artifact_path, get_index_url
-from voluptuous import Any, Optional, Required
+from voluptuous import (
+    Any,
+    Optional,
+    Required,
+)
+
+from gecko_taskgraph.transforms.base import TransformSequence
+from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS
+from gecko_taskgraph.util.platforms import platform_family
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/test/raptor.py

@@ -5,11 +5,16 @@
 
 from copy import deepcopy
 
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
+from taskgraph.util.treeherder import join_symbol, split_symbol
+from voluptuous import (
+    Optional,
+    Required,
+    Extra,
+)
+
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.transforms.test import test_description_schema
-from gecko_taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
-from taskgraph.util.treeherder import join_symbol, split_symbol
-from voluptuous import Extra, Optional, Required
 
 transforms = TransformSequence()
 task_transforms = TransformSequence()

taskcluster/gecko_taskgraph/transforms/test/variant.py

@@ -3,14 +3,19 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import copy
 
-import gecko_taskgraph
 import jsone
-from gecko_taskgraph.transforms.base import TransformSequence
-from gecko_taskgraph.util.schema import Schema, validate_schema
-from gecko_taskgraph.util.templates import merge
+from taskgraph.util.schema import Schema, validate_schema
 from taskgraph.util.treeherder import join_symbol, split_symbol
 from taskgraph.util.yaml import load_yaml
-from voluptuous import Any, Optional, Required
+from voluptuous import (
+    Any,
+    Optional,
+    Required,
+)
+
+import gecko_taskgraph
+from gecko_taskgraph.transforms.base import TransformSequence
+from gecko_taskgraph.util.templates import merge
 
 transforms = TransformSequence()
 

taskcluster/gecko_taskgraph/transforms/update_verify_config.py

@@ -7,9 +7,10 @@ Transform the beetmover task into an actual task description.
 
 from urllib.parse import urlsplit
 
+from taskgraph.util.schema import resolve_keyed_by
+
 from gecko_taskgraph.transforms.base import TransformSequence
 from gecko_taskgraph.util.attributes import release_level
-from gecko_taskgraph.util.schema import resolve_keyed_by
 from gecko_taskgraph.util.scriptworker import get_release_config
 from gecko_taskgraph.transforms.task import (
     get_branch_repo,

taskcluster/gecko_taskgraph/util/partners.py

@@ -13,9 +13,9 @@ from urllib.parse import urlencode
 
 import yaml
 from redo import retry
+from taskgraph.util.schema import resolve_keyed_by
 
 from gecko_taskgraph.util.attributes import release_level
-from gecko_taskgraph.util.schema import resolve_keyed_by
 
 # Suppress chatty requests logging
 logging.getLogger("requests").setLevel(logging.WARNING)

taskcluster/gecko_taskgraph/util/schema.py

@@ -1,241 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-
-import re
-import pprint
-import collections
-import collections.abc
-
-import taskgraph
-import voluptuous
-from taskgraph.util.keyed_by import evaluate_keyed_by
-
-
-def validate_schema(schema, obj, msg_prefix):
-    """
-    Validate that object satisfies schema.  If not, generate a useful exception
-    beginning with msg_prefix.
-    """
-    if taskgraph.fast:
-        return
-    try:
-        schema(obj)
-    except voluptuous.MultipleInvalid as exc:
-        msg = [msg_prefix]
-        for error in exc.errors:
-            msg.append(str(error))
-        raise Exception("\n".join(msg) + "\n" + pprint.pformat(obj))
-
-
-def optionally_keyed_by(*arguments):
-    """
-    Mark a schema value as optionally keyed by any of a number of fields.  The
-    schema is the last argument, and the remaining fields are taken to be the
-    field names.  For example:
-
-        'some-value': optionally_keyed_by(
-            'test-platform', 'build-platform',
-            Any('a', 'b', 'c'))
-
-    The resulting schema will allow nesting of `by-test-platform` and
-    `by-build-platform` in either order.
-    """
-    schema = arguments[-1]
-    fields = arguments[:-1]
-
-    def validator(obj):
-        if isinstance(obj, dict) and len(obj) == 1:
-            k, v = list(obj.items())[0]
-            if k.startswith("by-") and k[len("by-") :] in fields:
-                res = {}
-                for kk, vv in v.items():
-                    try:
-                        res[kk] = validator(vv)
-                    except voluptuous.Invalid as e:
-                        e.prepend([k, kk])
-                        raise
-                return res
-        return Schema(schema)(obj)
-
-    return validator
-
-
-def resolve_keyed_by(
-    item, field, item_name, defer=None, enforce_single_match=True, **extra_values
-):
-    """
-    For values which can either accept a literal value, or be keyed by some
-    other attribute of the item, perform that lookup and replacement in-place
-    (modifying `item` directly).  The field is specified using dotted notation
-    to traverse dictionaries.
-
-    For example, given item::
-
-        job:
-            test-platform: linux128
-            chunks:
-                by-test-platform:
-                    macosx-10.11/debug: 13
-                    win.*: 6
-                    default: 12
-
-    a call to `resolve_keyed_by(item, 'job.chunks', item['thing-name'])`
-    would mutate item in-place to::
-
-        job:
-            test-platform: linux128
-            chunks: 12
-
-    Items can be nested as deeply as the schema will allow::
-
-        chunks:
-            by-test-platform:
-                win.*:
-                    by-project:
-                        ash: ..
-                        cedar: ..
-                linux: 13
-                default: 12
-
-    Args:
-        item (dict): Object being evaluated.
-        field (str): Name of the key to perform evaluation on.
-        item_name (str): Used to generate useful error messages.
-        defer (list):
-            Allows evaluating a by-* entry at a later time. In the example
-            above it's possible that the project attribute hasn't been set yet,
-            in which case we'd want to stop before resolving that subkey and
-            then call this function again later. This can be accomplished by
-            setting `defer=["project"]` in this example.
-        enforce_single_match (bool):
-            If True (default), each task may only match a single arm of the
-            evaluation.
-        extra_values (kwargs):
-            If supplied, represent additional values available
-            for reference from by-<field>.
-
-    Returns:
-        dict: item which has also been modified in-place.
-    """
-    # find the field, returning the item unchanged if anything goes wrong
-    container, subfield = item, field
-    while "." in subfield:
-        f, subfield = subfield.split(".", 1)
-        if f not in container:
-            return item
-        container = container[f]
-        if not isinstance(container, dict):
-            return item
-
-    if subfield not in container:
-        return item
-
-    container[subfield] = evaluate_keyed_by(
-        value=container[subfield],
-        item_name=f"`{field}` in `{item_name}`",
-        defer=defer,
-        enforce_single_match=enforce_single_match,
-        attributes=dict(item, **extra_values),
-    )
-
-    return item
-
-
-# Schemas for YAML files should use dashed identifiers by default.  If there are
-# components of the schema for which there is a good reason to use another format,
-# they can be whitelisted here.
-WHITELISTED_SCHEMA_IDENTIFIERS = [
-    # upstream-artifacts are handed directly to scriptWorker, which expects interCaps
-    lambda path: "[{!r}]".format("upstream-artifacts") in path,
-    lambda path: (
-        "[{!r}]".format("test_name") in path
-        or "[{!r}]".format("json_location") in path
-        or "[{!r}]".format("video_location") in path
-    ),
-]
-
-
-def check_schema(schema):
-    identifier_re = re.compile(r"^\$?[a-z][a-z0-9-]*$")
-
-    def whitelisted(path):
-        return any(f(path) for f in WHITELISTED_SCHEMA_IDENTIFIERS)
-
-    def iter(path, sch):
-        def check_identifier(path, k):
-            if k in (str, str, voluptuous.Extra):
-                pass
-            elif isinstance(k, voluptuous.NotIn):
-                pass
-            elif isinstance(k, str):
-                if not identifier_re.match(k) and not whitelisted(path):
-                    raise RuntimeError(
-                        "YAML schemas should use dashed lower-case identifiers, "
-                        "not {!r} @ {}".format(k, path)
-                    )
-            elif isinstance(k, (voluptuous.Optional, voluptuous.Required)):
-                check_identifier(path, k.schema)
-            elif isinstance(k, (voluptuous.Any, voluptuous.All)):
-                for v in k.validators:
-                    check_identifier(path, v)
-            elif not whitelisted(path):
-                raise RuntimeError(
-                    "Unexpected type in YAML schema: {} @ {}".format(
-                        type(k).__name__, path
-                    )
-                )
-
-        if isinstance(sch, collections.abc.Mapping):
-            for k, v in sch.items():
-                child = f"{path}[{k!r}]"
-                check_identifier(child, k)
-                iter(child, v)
-        elif isinstance(sch, (list, tuple)):
-            for i, v in enumerate(sch):
-                iter(f"{path}[{i}]", v)
-        elif isinstance(sch, voluptuous.Any):
-            for v in sch.validators:
-                iter(path, v)
-
-    iter("schema", schema.schema)
-
-
-class Schema(voluptuous.Schema):
-    """
-    Operates identically to voluptuous.Schema, but applying some taskgraph-specific checks
-    in the process.
-    """
-
-    def __init__(self, *args, check=True, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        self.check = check
-        if not taskgraph.fast and self.check:
-            check_schema(self)
-
-    def extend(self, *args, **kwargs):
-        schema = super().extend(*args, **kwargs)
-
-        if self.check:
-            check_schema(schema)
-        # We want twice extend schema to be checked too.
-        schema.__class__ = Schema
-        return schema
-
-    def _compile(self, schema):
-        if taskgraph.fast:
-            return
-        return super()._compile(schema)
-
-    def __getitem__(self, item):
-        return self.schema[item]
-
-
-# shortcut for a string where task references are allowed
-taskref_or_string = voluptuous.Any(
-    str,
-    {voluptuous.Required("task-reference"): str},
-    {voluptuous.Required("artifact-reference"): str},
-)

taskcluster/gecko_taskgraph/util/scriptworker.py

@@ -25,11 +25,10 @@ from datetime import datetime
 import jsone
 
 from mozbuild.util import memoize
+from taskgraph.util.schema import resolve_keyed_by
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.yaml import load_yaml
 
-from .schema import resolve_keyed_by
-
 # constants {{{1
 """Map signing scope aliases to sets of projects.
 

taskcluster/test/test_mozilla_central.py

@@ -38,7 +38,7 @@ def test_tasks_are_scheduled(optimized_task_graph, filter_tasks, func, min_expec
 def test_test_setting(full_task_graph, filter_tasks):
     """Verify that all test tasks' ``test-setting`` object conforms to the schema."""
     from gecko_taskgraph.transforms.test.other import test_setting_description_schema
-    from gecko_taskgraph.util.schema import validate_schema
+    from taskgraph.util.schema import validate_schema
 
     tasks = filter_tasks(full_task_graph, lambda t: t.kind == "test")
 

tools/lint/rejected-words.yml

@@ -297,7 +297,7 @@ avoid-blacklist-and-whitelist:
         - taskcluster/gecko_taskgraph/target_tasks.py
         - taskcluster/gecko_taskgraph/transforms/test/other.py
         - taskcluster/gecko_taskgraph/try_option_syntax.py
-        - taskcluster/gecko_taskgraph/util/schema.py
+        - taskcluster/gecko_taskgraph/__init__.py
         - taskcluster/test/test_mach_try_auto.py
         - testing/condprofile/condprof/client.py
         - testing/condprofile/condprof/tests/profile/prefs.js