diff --git a/taskcluster/gecko_taskgraph/__init__.py b/taskcluster/gecko_taskgraph/__init__.py index af822add27b1..e48b6848365e 100644 --- a/taskcluster/gecko_taskgraph/__init__.py +++ b/taskcluster/gecko_taskgraph/__init__.py @@ -4,7 +4,7 @@ import os -from taskgraph.util import taskcluster as tc_util +from taskgraph.util import taskcluster as tc_util, schema GECKO = os.path.normpath(os.path.realpath(os.path.join(__file__, "..", "..", ".."))) @@ -18,6 +18,21 @@ MAX_DEPENDENCIES = 99 # to the production Taskcluster deployment used for CI. tc_util.PRODUCTION_TASKCLUSTER_ROOT_URL = "https://firefox-ci-tc.services.mozilla.com" +# Schemas for YAML files should use dashed identifiers by default. If there are +# components of the schema for which there is a good reason to use another format, +# they can be whitelisted here. +schema.WHITELISTED_SCHEMA_IDENTIFIERS.extend( + [ + # upstream-artifacts are handed directly to scriptWorker, which expects interCaps + lambda path: "[{!r}]".format("upstream-artifacts") in path, + lambda path: ( + "[{!r}]".format("test_name") in path + or "[{!r}]".format("json_location") in path + or "[{!r}]".format("video_location") in path + ), + ] +) + def register(graph_config): """Used to register Gecko specific extensions. diff --git a/taskcluster/gecko_taskgraph/config.py b/taskcluster/gecko_taskgraph/config.py index 60ede0367cc2..bb4af2bb8649 100644 --- a/taskcluster/gecko_taskgraph/config.py +++ b/taskcluster/gecko_taskgraph/config.py @@ -7,11 +7,10 @@ import os import logging from taskgraph.config import GraphConfig +from taskgraph.util.schema import Schema, optionally_keyed_by, validate_schema from taskgraph.util.yaml import load_yaml from voluptuous import Required, Optional, Any -from .util.schema import validate_schema, Schema, optionally_keyed_by - logger = logging.getLogger(__name__) graph_config_schema = Schema( diff --git a/taskcluster/gecko_taskgraph/decision.py b/taskcluster/gecko_taskgraph/decision.py index 7bd6e96397de..e9036815395b 100644 --- a/taskcluster/gecko_taskgraph/decision.py +++ b/taskcluster/gecko_taskgraph/decision.py @@ -16,6 +16,7 @@ from redo import retry from taskgraph.parameters import Parameters from taskgraph.taskgraph import TaskGraph from taskgraph.util.python_path import find_object +from taskgraph.util.schema import Schema, validate_schema from taskgraph.util.taskcluster import get_artifact from taskgraph.util.yaml import load_yaml from voluptuous import Required, Optional, Any @@ -31,7 +32,6 @@ from .util.bugbug import push_schedules from .util.chunking import resolver from .util.hg import get_hg_revision_branch, get_hg_commit_message from .util.partials import populate_release_history -from .util.schema import validate_schema, Schema from .util.taskcluster import insert_index from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds diff --git a/taskcluster/gecko_taskgraph/loader/multi_dep.py b/taskcluster/gecko_taskgraph/loader/multi_dep.py index 0a5fbc9bd2a1..771eecb786c0 100644 --- a/taskcluster/gecko_taskgraph/loader/multi_dep.py +++ b/taskcluster/gecko_taskgraph/loader/multi_dep.py @@ -6,10 +6,10 @@ import copy from taskgraph.task import Task +from taskgraph.util.schema import Schema from voluptuous import Required from ..util.attributes import sorted_unique_list -from ..util.schema import Schema schema = Schema( { diff --git a/taskcluster/gecko_taskgraph/loader/single_dep.py b/taskcluster/gecko_taskgraph/loader/single_dep.py index 7b646a75aa47..24588a42078e 100644 --- a/taskcluster/gecko_taskgraph/loader/single_dep.py +++ b/taskcluster/gecko_taskgraph/loader/single_dep.py @@ -6,10 +6,9 @@ import copy from taskgraph.task import Task +from taskgraph.util.schema import Schema from voluptuous import Required -from ..util.schema import Schema - schema = Schema( { Required("primary-dependency", "primary dependency task"): Task, diff --git a/taskcluster/gecko_taskgraph/test/python.ini b/taskcluster/gecko_taskgraph/test/python.ini index e60877133905..eff9a7a07b23 100644 --- a/taskcluster/gecko_taskgraph/test/python.ini +++ b/taskcluster/gecko_taskgraph/test/python.ini @@ -23,6 +23,5 @@ subsuite = taskgraph [test_util_docker.py] [test_util_partials.py] [test_util_runnable_jobs.py] -[test_util_schema.py] [test_util_templates.py] [test_util_verify.py] diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py index 44f2a6a9e179..c0981f83bdee 100644 --- a/taskcluster/gecko_taskgraph/test/test_transforms_job.py +++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py @@ -12,6 +12,7 @@ from copy import deepcopy import pytest from mozunit import main +from taskgraph.util.schema import Schema, validate_schema from gecko_taskgraph import GECKO from gecko_taskgraph.config import load_graph_config @@ -20,7 +21,6 @@ from gecko_taskgraph.transforms.job import run_task # noqa: F401 from gecko_taskgraph.transforms.base import TransformConfig from gecko_taskgraph.transforms.job.common import add_cache from gecko_taskgraph.transforms.task import payload_builders -from gecko_taskgraph.util.schema import Schema, validate_schema from conftest import FakeParameters diff --git a/taskcluster/gecko_taskgraph/test/test_util_schema.py b/taskcluster/gecko_taskgraph/test/test_util_schema.py deleted file mode 100644 index 968c85326b40..000000000000 --- a/taskcluster/gecko_taskgraph/test/test_util_schema.py +++ /dev/null @@ -1,216 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - - -import unittest -from mozunit import main -from gecko_taskgraph.util.schema import ( - validate_schema, - resolve_keyed_by, - Schema, -) - -schema = Schema( - { - "x": int, - "y": str, - } -) - - -class TestValidateSchema(unittest.TestCase): - def test_valid(self): - validate_schema(schema, {"x": 10, "y": "foo"}, "pfx") - - def test_invalid(self): - try: - validate_schema(schema, {"x": "not-int"}, "pfx") - self.fail("no exception raised") - except Exception as e: - self.assertTrue(str(e).startswith("pfx\n")) - - -class TestCheckSchema(unittest.TestCase): - def test_schema(self): - "Creating a schema applies taskgraph checks." - with self.assertRaises(Exception): - Schema({"camelCase": int}) - - def test_extend_schema(self): - "Extending a schema applies taskgraph checks." - with self.assertRaises(Exception): - Schema({"kebab-case": int}).extend({"camelCase": int}) - - def test_extend_schema_twice(self): - "Extending a schema twice applies taskgraph checks." - with self.assertRaises(Exception): - Schema({"kebab-case": int}).extend({"more-kebab": int}).extend( - {"camelCase": int} - ) - - -class TestResolveKeyedBy(unittest.TestCase): - def test_no_by(self): - self.assertEqual(resolve_keyed_by({"x": 10}, "z", "n"), {"x": 10}) - - def test_no_by_dotted(self): - self.assertEqual( - resolve_keyed_by({"x": {"y": 10}}, "x.z", "n"), {"x": {"y": 10}} - ) - - def test_no_by_not_dict(self): - self.assertEqual(resolve_keyed_by({"x": 10}, "x.y", "n"), {"x": 10}) - - def test_no_by_not_by(self): - self.assertEqual(resolve_keyed_by({"x": {"a": 10}}, "x", "n"), {"x": {"a": 10}}) - - def test_nested(self): - x = { - "by-foo": { - "F1": { - "by-bar": { - "B1": 11, - "B2": 12, - }, - }, - "F2": 20, - "default": 0, - }, - } - self.assertEqual( - resolve_keyed_by({"x": x}, "x", "x", foo="F1", bar="B1"), {"x": 11} - ) - self.assertEqual( - resolve_keyed_by({"x": x}, "x", "x", foo="F1", bar="B2"), {"x": 12} - ) - self.assertEqual(resolve_keyed_by({"x": x}, "x", "x", foo="F2"), {"x": 20}) - self.assertEqual( - resolve_keyed_by({"x": x}, "x", "x", foo="F99", bar="B1"), {"x": 0} - ) - - # bar is deferred - self.assertEqual( - resolve_keyed_by({"x": x}, "x", "x", defer=["bar"], foo="F1", bar="B1"), - {"x": {"by-bar": {"B1": 11, "B2": 12}}}, - ) - - def test_no_by_empty_dict(self): - self.assertEqual(resolve_keyed_by({"x": {}}, "x", "n"), {"x": {}}) - - def test_no_by_not_only_by(self): - self.assertEqual( - resolve_keyed_by({"x": {"by-y": True, "a": 10}}, "x", "n"), - {"x": {"by-y": True, "a": 10}}, - ) - - def test_match_nested_exact(self): - self.assertEqual( - resolve_keyed_by( - { - "f": "shoes", - "x": {"y": {"by-f": {"shoes": "feet", "gloves": "hands"}}}, - }, - "x.y", - "n", - ), - {"f": "shoes", "x": {"y": "feet"}}, - ) - - def test_match_regexp(self): - self.assertEqual( - resolve_keyed_by( - { - "f": "shoes", - "x": {"by-f": {"s?[hH]oes?": "feet", "gloves": "hands"}}, - }, - "x", - "n", - ), - {"f": "shoes", "x": "feet"}, - ) - - def test_match_partial_regexp(self): - self.assertEqual( - resolve_keyed_by( - {"f": "shoes", "x": {"by-f": {"sh": "feet", "default": "hands"}}}, - "x", - "n", - ), - {"f": "shoes", "x": "hands"}, - ) - - def test_match_default(self): - self.assertEqual( - resolve_keyed_by( - {"f": "shoes", "x": {"by-f": {"hat": "head", "default": "anywhere"}}}, - "x", - "n", - ), - {"f": "shoes", "x": "anywhere"}, - ) - - def test_match_extra_value(self): - self.assertEqual( - resolve_keyed_by({"f": {"by-foo": {"x": 10, "y": 20}}}, "f", "n", foo="y"), - {"f": 20}, - ) - - def test_no_match(self): - self.assertRaises( - Exception, - resolve_keyed_by, - {"f": "shoes", "x": {"by-f": {"hat": "head"}}}, - "x", - "n", - ) - - def test_multiple_matches(self): - self.assertRaises( - Exception, - resolve_keyed_by, - {"f": "hats", "x": {"by-f": {"hat.*": "head", "ha.*": "hair"}}}, - "x", - "n", - ) - - self.assertEqual( - resolve_keyed_by( - {"f": "hats", "x": {"by-f": {"hat.*": "head", "ha.*": "hair"}}}, - "x", - "n", - enforce_single_match=False, - ), - {"f": "hats", "x": "head"}, - ) - - def test_no_key_no_default(self): - """ - When the key referenced in `by-*` doesn't exist, and there is not default value, - an exception is raised. - """ - self.assertRaises( - Exception, - resolve_keyed_by, - {"x": {"by-f": {"hat.*": "head", "ha.*": "hair"}}}, - "x", - "n", - ) - - def test_no_key(self): - """ - When the key referenced in `by-*` doesn't exist, and there is a default value, - that value is used as the result. - """ - self.assertEqual( - resolve_keyed_by( - {"x": {"by-f": {"hat": "head", "default": "anywhere"}}}, - "x", - "n", - ), - {"x": "anywhere"}, - ) - - -if __name__ == "__main__": - main() diff --git a/taskcluster/gecko_taskgraph/transforms/balrog_submit.py b/taskcluster/gecko_taskgraph/transforms/balrog_submit.py index d59fd8d824be..80251dbef96d 100644 --- a/taskcluster/gecko_taskgraph/transforms/balrog_submit.py +++ b/taskcluster/gecko_taskgraph/transforms/balrog_submit.py @@ -5,14 +5,14 @@ Transform the per-locale balrog task into an actual task description. """ +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import replace_group +from voluptuous import Optional from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job -from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by -from taskgraph.util.treeherder import replace_group -from voluptuous import Optional balrog_description_schema = schema.extend( { diff --git a/taskcluster/gecko_taskgraph/transforms/base.py b/taskcluster/gecko_taskgraph/transforms/base.py index b9ec6afc34e2..0f087550c0e7 100644 --- a/taskcluster/gecko_taskgraph/transforms/base.py +++ b/taskcluster/gecko_taskgraph/transforms/base.py @@ -6,8 +6,7 @@ import attr from taskgraph.config import GraphConfig from taskgraph.parameters import Parameters - -from ..util.schema import Schema, validate_schema +from taskgraph.util.schema import Schema, validate_schema @attr.s(frozen=True) diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py b/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py index 4e4e09adc7e1..c1b83372e28f 100644 --- a/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_geckoview.py @@ -8,6 +8,9 @@ Transform the beetmover task into an actual task description. from copy import deepcopy +from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by +from voluptuous import Required, Optional + from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.beetmover import ( @@ -22,9 +25,7 @@ from gecko_taskgraph.util.declarative_artifacts import ( get_geckoview_upstream_artifacts, get_geckoview_artifact_id, ) -from gecko_taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by from gecko_taskgraph.transforms.task import task_description_schema -from voluptuous import Required, Optional beetmover_description_schema = schema.extend( diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py b/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py index 2b1d79b17b87..dee6c37d8fb2 100644 --- a/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_push_to_release.py @@ -5,19 +5,15 @@ Transform the beetmover-push-to-release task into a task description. """ +from taskgraph.util.schema import Schema, taskref_or_string +from voluptuous import Required, Optional from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import ( - Schema, - taskref_or_string, -) from gecko_taskgraph.util.scriptworker import ( get_beetmover_bucket_scope, add_scope_prefix, ) from gecko_taskgraph.transforms.task import task_description_schema -from voluptuous import Required, Optional - beetmover_push_to_release_description_schema = Schema( { diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py index 8589aa7d2b44..a93e62d970b9 100644 --- a/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py +++ b/taskcluster/gecko_taskgraph/transforms/beetmover_repackage_partner.py @@ -8,6 +8,7 @@ Transform the beetmover task into an actual task description. import logging from copy import deepcopy +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by from taskgraph.util.taskcluster import get_artifact_prefix from voluptuous import Any, Required, Optional @@ -22,10 +23,6 @@ from gecko_taskgraph.util.partners import ( get_ftp_platform, get_partner_config_by_kind, ) -from gecko_taskgraph.util.schema import ( - optionally_keyed_by, - resolve_keyed_by, -) from gecko_taskgraph.util.scriptworker import ( add_scope_prefix, get_beetmover_bucket_scope, diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py b/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py index e8bb48b32404..b79574e69d7e 100644 --- a/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_aliases.py @@ -8,6 +8,8 @@ Add from parameters.yml into bouncer submission tasks. import logging +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.bouncer_submission import craft_bouncer_product_name from gecko_taskgraph.transforms.bouncer_submission_partners import ( @@ -15,7 +17,6 @@ from gecko_taskgraph.transforms.bouncer_submission_partners import ( ) from gecko_taskgraph.util.attributes import release_level from gecko_taskgraph.util.partners import get_partners_to_be_published -from gecko_taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.util.scriptworker import get_release_config logger = logging.getLogger(__name__) diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_check.py b/taskcluster/gecko_taskgraph/transforms/bouncer_check.py index 4ca66274673a..9c054f1d8d21 100644 --- a/taskcluster/gecko_taskgraph/transforms/bouncer_check.py +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_check.py @@ -5,10 +5,11 @@ import json from pipes import quote as shell_quote +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import release_level from gecko_taskgraph.util.scriptworker import get_release_config -from gecko_taskgraph.util.schema import resolve_keyed_by import logging diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_locations.py b/taskcluster/gecko_taskgraph/transforms/bouncer_locations.py index 99094af92159..bf9acbe5fc6e 100644 --- a/taskcluster/gecko_taskgraph/transforms/bouncer_locations.py +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_locations.py @@ -4,8 +4,9 @@ import logging +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import resolve_keyed_by logger = logging.getLogger(__name__) diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py index 2b79bdb67ea7..e8f15275052e 100644 --- a/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_submission.py @@ -10,11 +10,11 @@ import copy import logging import attr +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.l10n import parse_locales_file from gecko_taskgraph.util.attributes import release_level -from gecko_taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.util.scriptworker import get_release_config logger = logging.getLogger(__name__) diff --git a/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py b/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py index c0e0f4c718e0..d05411a670bf 100644 --- a/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py +++ b/taskcluster/gecko_taskgraph/transforms/bouncer_submission_partners.py @@ -8,6 +8,8 @@ Add from parameters.yml into bouncer submission tasks. import logging +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.bouncer_submission import ( FTP_PLATFORMS_PER_BOUNCER_PLATFORM, @@ -20,7 +22,6 @@ from gecko_taskgraph.util.partners import ( check_if_partners_enabled, get_partners_to_be_published, ) -from gecko_taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.util.scriptworker import get_release_config logger = logging.getLogger(__name__) diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py index ca22c53310e2..8c722937bb54 100644 --- a/taskcluster/gecko_taskgraph/transforms/build.py +++ b/taskcluster/gecko_taskgraph/transforms/build.py @@ -5,16 +5,15 @@ Apply some defaults and minor modifications to the jobs defined in the build kind. """ - - import logging +from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.treeherder import add_suffix + from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import RELEASE_PROJECTS, is_try, release_level -from gecko_taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.util.workertypes import worker_type_implementation -from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS -from taskgraph.util.treeherder import add_suffix logger = logging.getLogger(__name__) diff --git a/taskcluster/gecko_taskgraph/transforms/condprof.py b/taskcluster/gecko_taskgraph/transforms/condprof.py index 2ee90006ddca..4f8e54021811 100644 --- a/taskcluster/gecko_taskgraph/transforms/condprof.py +++ b/taskcluster/gecko_taskgraph/transforms/condprof.py @@ -8,6 +8,7 @@ the condprof/kind.yml file from copy import deepcopy +from taskgraph.util.schema import Schema from voluptuous import ( Optional, ) @@ -15,7 +16,6 @@ from voluptuous import ( from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.job import job_description_schema from gecko_taskgraph.transforms.task import task_description_schema -from gecko_taskgraph.util.schema import Schema diff_description_schema = Schema( { diff --git a/taskcluster/gecko_taskgraph/transforms/cross_channel.py b/taskcluster/gecko_taskgraph/transforms/cross_channel.py index 15d0979cc0e5..3914303deb26 100644 --- a/taskcluster/gecko_taskgraph/transforms/cross_channel.py +++ b/taskcluster/gecko_taskgraph/transforms/cross_channel.py @@ -8,8 +8,9 @@ Build a command to run `mach l10n-cross-channel`. from pipes import quote as shell_quote +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/diffoscope.py b/taskcluster/gecko_taskgraph/transforms/diffoscope.py index 6c1f861ac2ad..9e9a8f5c8797 100644 --- a/taskcluster/gecko_taskgraph/transforms/diffoscope.py +++ b/taskcluster/gecko_taskgraph/transforms/diffoscope.py @@ -7,6 +7,7 @@ defined in kind.yml """ from taskgraph.util.taskcluster import get_artifact_path +from taskgraph.util.schema import Schema from voluptuous import ( Any, Optional, @@ -15,7 +16,6 @@ from voluptuous import ( from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.task import task_description_schema -from gecko_taskgraph.util.schema import Schema index_or_string = Any( str, diff --git a/taskcluster/gecko_taskgraph/transforms/docker_image.py b/taskcluster/gecko_taskgraph/transforms/docker_image.py index 8ce3348fa47c..eeaa9b0bc06d 100644 --- a/taskcluster/gecko_taskgraph/transforms/docker_image.py +++ b/taskcluster/gecko_taskgraph/transforms/docker_image.py @@ -10,6 +10,7 @@ import json import mozpack.path as mozpath import taskgraph +from taskgraph.util.schema import Schema from gecko_taskgraph.transforms.base import TransformSequence from .. import GECKO @@ -18,7 +19,6 @@ from gecko_taskgraph.util.docker import ( generate_context_hash, image_path, ) -from gecko_taskgraph.util.schema import Schema from voluptuous import ( Optional, Required, diff --git a/taskcluster/gecko_taskgraph/transforms/fetch.py b/taskcluster/gecko_taskgraph/transforms/fetch.py index 8d11de4056f4..32ba5c213279 100644 --- a/taskcluster/gecko_taskgraph/transforms/fetch.py +++ b/taskcluster/gecko_taskgraph/transforms/fetch.py @@ -13,13 +13,13 @@ import attr import taskgraph from mozbuild.shellutil import quote as shell_quote from mozpack import path as mozpath +from taskgraph.util.schema import Schema, validate_schema from taskgraph.util.treeherder import join_symbol from voluptuous import Any, Extra, Optional, Required import gecko_taskgraph from .base import TransformSequence from ..util.cached_tasks import add_optimization -from ..util.schema import Schema, validate_schema CACHE_TYPE = "content.v1" diff --git a/taskcluster/gecko_taskgraph/transforms/job/__init__.py b/taskcluster/gecko_taskgraph/transforms/job/__init__.py index 4b7316a99d0e..131a8b65337b 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/__init__.py +++ b/taskcluster/gecko_taskgraph/transforms/job/__init__.py @@ -17,6 +17,7 @@ import json import mozpack.path as mozpath from taskgraph.util.python_path import import_sibling_modules from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.schema import Schema, validate_schema from voluptuous import ( Extra, Optional, @@ -26,10 +27,6 @@ from voluptuous import ( from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.cached_tasks import order_tasks -from gecko_taskgraph.util.schema import ( - validate_schema, - Schema, -) from gecko_taskgraph.util.workertypes import worker_type_implementation from gecko_taskgraph.transforms.task import task_description_schema diff --git a/taskcluster/gecko_taskgraph/transforms/job/distro_package.py b/taskcluster/gecko_taskgraph/transforms/job/distro_package.py index 357d5a58677d..72135f54175d 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/distro_package.py +++ b/taskcluster/gecko_taskgraph/transforms/job/distro_package.py @@ -10,6 +10,7 @@ import os import re import taskgraph +from taskgraph.util.schema import Schema from taskgraph.util.taskcluster import get_root_url from voluptuous import Any, Optional, Required @@ -17,7 +18,6 @@ from gecko_taskgraph.transforms.job import run_job_using from gecko_taskgraph.transforms.job.common import add_artifacts from gecko_taskgraph.util.hash import hash_path -from gecko_taskgraph.util.schema import Schema from gecko_taskgraph import GECKO DSC_PACKAGE_RE = re.compile(".*(?=_)") diff --git a/taskcluster/gecko_taskgraph/transforms/job/hazard.py b/taskcluster/gecko_taskgraph/transforms/job/hazard.py index 5bc8d92a77ec..e65177e6a28c 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/hazard.py +++ b/taskcluster/gecko_taskgraph/transforms/job/hazard.py @@ -6,7 +6,7 @@ Support for running hazard jobs via dedicated scripts """ -from gecko_taskgraph.util.schema import Schema +from taskgraph.util.schema import Schema from voluptuous import Required, Optional, Any from gecko_taskgraph.transforms.job import ( diff --git a/taskcluster/gecko_taskgraph/transforms/job/mach.py b/taskcluster/gecko_taskgraph/transforms/job/mach.py index 959abd6908a8..a3cd42e7d3d9 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mach.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mach.py @@ -5,13 +5,10 @@ Support for running mach tasks (via run-task) """ +from taskgraph.util.schema import Schema, taskref_or_string +from voluptuous import Required, Optional, Any from gecko_taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run -from gecko_taskgraph.util.schema import ( - Schema, - taskref_or_string, -) -from voluptuous import Required, Optional, Any mach_schema = Schema( { diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py index 1950dbcfb108..6ecdc2aec531 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py @@ -12,7 +12,7 @@ import json from textwrap import dedent -from gecko_taskgraph.util.schema import Schema +from taskgraph.util.schema import Schema from voluptuous import Required, Optional, Any from voluptuous.validators import Match diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py index cea70255f45a..f881f6ba2b21 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness_test.py @@ -7,6 +7,7 @@ import json import os import re +from taskgraph.util.schema import Schema from taskgraph.util.taskcluster import get_artifact_path, get_artifact_url from voluptuous import Extra, Optional, Required @@ -15,7 +16,6 @@ from gecko_taskgraph.transforms.job import ( run_job_using, ) from gecko_taskgraph.util.attributes import is_try -from gecko_taskgraph.util.schema import Schema from gecko_taskgraph.transforms.test import test_description_schema, normpath from gecko_taskgraph.transforms.job.common import support_vcs_checkout diff --git a/taskcluster/gecko_taskgraph/transforms/job/python_test.py b/taskcluster/gecko_taskgraph/transforms/job/python_test.py index 29879483f549..8fbd5e2441fb 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/python_test.py +++ b/taskcluster/gecko_taskgraph/transforms/job/python_test.py @@ -6,10 +6,11 @@ Support for running mach python-test tasks (via run-task) """ -from gecko_taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run -from gecko_taskgraph.util.schema import Schema +from taskgraph.util.schema import Schema from voluptuous import Required, Optional +from gecko_taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run + python_test_schema = Schema( { Required("using"): "python-test", diff --git a/taskcluster/gecko_taskgraph/transforms/job/run_task.py b/taskcluster/gecko_taskgraph/transforms/job/run_task.py index 973ac6fc8913..cf33dbc2f8a4 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/run_task.py +++ b/taskcluster/gecko_taskgraph/transforms/job/run_task.py @@ -9,10 +9,10 @@ Support for running jobs that are invoked via the `run-task` script. import os from mozpack import path +from taskgraph.util.schema import Schema from gecko_taskgraph.transforms.task import taskref_or_string from gecko_taskgraph.transforms.job import run_job_using -from gecko_taskgraph.util.schema import Schema from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout from voluptuous import Any, Optional, Required diff --git a/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py b/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py index bac260307297..79b187be82cd 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py +++ b/taskcluster/gecko_taskgraph/transforms/job/spidermonkey.py @@ -6,7 +6,7 @@ Support for running spidermonkey jobs via dedicated scripts """ -from gecko_taskgraph.util.schema import Schema +from taskgraph.util.schema import Schema from voluptuous import Required, Any, Optional from gecko_taskgraph.transforms.job import ( diff --git a/taskcluster/gecko_taskgraph/transforms/job/toolchain.py b/taskcluster/gecko_taskgraph/transforms/job/toolchain.py index 616a0208b90e..1412b9ce5e6f 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/toolchain.py +++ b/taskcluster/gecko_taskgraph/transforms/job/toolchain.py @@ -9,7 +9,7 @@ Support for running toolchain-building jobs via dedicated scripts import taskgraph from mozbuild.shellutil import quote as shell_quote -from gecko_taskgraph.util.schema import Schema +from taskgraph.util.schema import Schema from voluptuous import Optional, Required, Any from gecko_taskgraph.transforms.job import ( diff --git a/taskcluster/gecko_taskgraph/transforms/l10n.py b/taskcluster/gecko_taskgraph/transforms/l10n.py index 30a73be192e5..500f94e8a065 100644 --- a/taskcluster/gecko_taskgraph/transforms/l10n.py +++ b/taskcluster/gecko_taskgraph/transforms/l10n.py @@ -9,6 +9,20 @@ Do transforms specific to l10n kind import copy import json +from mozbuild.chunkify import chunkify +from taskgraph.util.schema import ( + optionally_keyed_by, + resolve_keyed_by, + taskref_or_string, +) +from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.treeherder import add_suffix +from voluptuous import ( + Any, + Optional, + Required, +) + from gecko_taskgraph.loader.multi_dep import schema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.job import job_description_schema @@ -17,15 +31,6 @@ from gecko_taskgraph.util.attributes import ( copy_attributes_from_dependent_job, task_name, ) -from gecko_taskgraph.util.schema import ( - optionally_keyed_by, - resolve_keyed_by, - taskref_or_string, -) -from mozbuild.chunkify import chunkify -from taskgraph.util.taskcluster import get_artifact_prefix -from taskgraph.util.treeherder import add_suffix -from voluptuous import Any, Optional, Required def _by_platform(arg): diff --git a/taskcluster/gecko_taskgraph/transforms/maybe_release.py b/taskcluster/gecko_taskgraph/transforms/maybe_release.py index b70b4db8dafe..c2ff6432ab36 100644 --- a/taskcluster/gecko_taskgraph/transforms/maybe_release.py +++ b/taskcluster/gecko_taskgraph/transforms/maybe_release.py @@ -2,10 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import release_level -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/merge_automation.py b/taskcluster/gecko_taskgraph/transforms/merge_automation.py index 7786f47eea75..60d39c0bf4aa 100644 --- a/taskcluster/gecko_taskgraph/transforms/merge_automation.py +++ b/taskcluster/gecko_taskgraph/transforms/merge_automation.py @@ -5,9 +5,9 @@ Transform the update generation task into an actual task description. """ +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py b/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py index 5a72efefcd72..78b1e090577a 100644 --- a/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py +++ b/taskcluster/gecko_taskgraph/transforms/partner_attribution_beetmover.py @@ -9,6 +9,7 @@ from collections import defaultdict from copy import deepcopy from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by from voluptuous import Any, Required, Optional from gecko_taskgraph.loader.single_dep import schema @@ -22,10 +23,6 @@ from gecko_taskgraph.util.partners import ( get_partner_config_by_kind, apply_partner_priority, ) -from gecko_taskgraph.util.schema import ( - optionally_keyed_by, - resolve_keyed_by, -) from gecko_taskgraph.util.scriptworker import ( add_scope_prefix, get_beetmover_bucket_scope, diff --git a/taskcluster/gecko_taskgraph/transforms/partner_repack.py b/taskcluster/gecko_taskgraph/transforms/partner_repack.py index 1edb9ce3967f..8f569a05b987 100644 --- a/taskcluster/gecko_taskgraph/transforms/partner_repack.py +++ b/taskcluster/gecko_taskgraph/transforms/partner_repack.py @@ -5,10 +5,10 @@ Transform the partner repack task into an actual task description. """ +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import release_level -from gecko_taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.util.scriptworker import get_release_config from gecko_taskgraph.util.partners import ( check_if_partners_enabled, diff --git a/taskcluster/gecko_taskgraph/transforms/perftest.py b/taskcluster/gecko_taskgraph/transforms/perftest.py index 95ece7551aaa..c84e35755ffa 100644 --- a/taskcluster/gecko_taskgraph/transforms/perftest.py +++ b/taskcluster/gecko_taskgraph/transforms/perftest.py @@ -9,10 +9,15 @@ import json from copy import deepcopy from datetime import date, timedelta -from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by from taskgraph.util.treeherder import join_symbol, split_symbol -from voluptuous import Any, Extra, Optional +from voluptuous import ( + Any, + Optional, + Extra, +) + +from gecko_taskgraph.transforms.base import TransformSequence transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/python_update.py b/taskcluster/gecko_taskgraph/transforms/python_update.py index 0ca7de8f136c..e4efa1f9b350 100644 --- a/taskcluster/gecko_taskgraph/transforms/python_update.py +++ b/taskcluster/gecko_taskgraph/transforms/python_update.py @@ -5,9 +5,9 @@ Transform the repo-update task into an actual task description. """ +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py b/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py index 147b2b8cb255..2594d5ecb1f9 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py +++ b/taskcluster/gecko_taskgraph/transforms/release_beetmover_signed_addons.py @@ -5,10 +5,13 @@ Transform the beetmover task into an actual task description. """ - import copy import logging +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Required, Optional + from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.beetmover import craft_release_properties @@ -17,15 +20,12 @@ from gecko_taskgraph.util.attributes import ( copy_attributes_from_dependent_job, release_level, ) -from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by from gecko_taskgraph.util.scriptworker import ( generate_beetmover_artifact_map, generate_beetmover_upstream_artifacts, get_beetmover_action_scope, get_beetmover_bucket_scope, ) -from taskgraph.util.treeherder import inherit_treeherder_from_dep -from voluptuous import Optional, Required logger = logging.getLogger(__name__) diff --git a/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py b/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py index 54046fbd1ca0..35aa31a59ace 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py +++ b/taskcluster/gecko_taskgraph/transforms/release_flatpak_push.py @@ -5,14 +5,14 @@ Transform the release-flatpak-push kind into an actual task description. """ +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from voluptuous import Optional, Required from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import release_level -from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema from gecko_taskgraph.util.scriptworker import add_scope_prefix -from voluptuous import Optional, Required push_flatpak_description_schema = Schema( { diff --git a/taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py b/taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py index d5b3f2653098..202faafff182 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py +++ b/taskcluster/gecko_taskgraph/transforms/release_flatpak_repackage.py @@ -2,11 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import release_level from gecko_taskgraph.util.scriptworker import get_release_config -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py index 494c79029a89..6055ca307216 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py +++ b/taskcluster/gecko_taskgraph/transforms/release_generate_checksums.py @@ -7,10 +7,11 @@ Transform the checksums task into an actual task description. import copy +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import release_level from gecko_taskgraph.util.scriptworker import get_release_config -from gecko_taskgraph.util.schema import resolve_keyed_by import logging diff --git a/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py b/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py index ecd05bd76270..0bc5ee119d84 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py +++ b/taskcluster/gecko_taskgraph/transforms/release_mark_as_shipped.py @@ -2,10 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import release_level -from gecko_taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.util.scriptworker import get_release_config transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/release_msix_push.py b/taskcluster/gecko_taskgraph/transforms/release_msix_push.py index f8210b73847d..12b5ee820eb8 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_msix_push.py +++ b/taskcluster/gecko_taskgraph/transforms/release_msix_push.py @@ -5,15 +5,14 @@ Transform the release-msix-push kind into an actual task description. """ +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema +from voluptuous import Optional, Required from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import release_level -from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema from gecko_taskgraph.util.scriptworker import add_scope_prefix -from voluptuous import Optional, Required - push_msix_description_schema = Schema( { Required("name"): str, diff --git a/taskcluster/gecko_taskgraph/transforms/release_notifications.py b/taskcluster/gecko_taskgraph/transforms/release_notifications.py index b4e745928da9..57df6eb1b0bd 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_notifications.py +++ b/taskcluster/gecko_taskgraph/transforms/release_notifications.py @@ -4,12 +4,12 @@ """ Add notifications via taskcluster-notify for release tasks """ - - from string import Formatter + +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.scriptworker import get_release_config -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py b/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py index f9682ad6296e..82858ca3581e 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py +++ b/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py @@ -5,6 +5,9 @@ Transform the release-sign-and-push task into an actual task description. """ +from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from voluptuous import Any, Required from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence @@ -13,9 +16,6 @@ from gecko_taskgraph.util.attributes import ( copy_attributes_from_dependent_job, release_level, ) -from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by -from taskgraph.util.treeherder import inherit_treeherder_from_dep -from voluptuous import Any, Required transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py b/taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py index 7f7abb215267..f0f01cfe440d 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py +++ b/taskcluster/gecko_taskgraph/transforms/release_snap_repackage.py @@ -2,11 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import release_level from gecko_taskgraph.util.scriptworker import get_release_config -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/release_started.py b/taskcluster/gecko_taskgraph/transforms/release_started.py index 4ab86ee587a3..a6daf1eac3a0 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_started.py +++ b/taskcluster/gecko_taskgraph/transforms/release_started.py @@ -4,12 +4,11 @@ """ Add notifications via taskcluster-notify for release tasks """ - - from pipes import quote as shell_quote +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/release_version_bump.py b/taskcluster/gecko_taskgraph/transforms/release_version_bump.py index eae0b0aa65a1..1a502440604f 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_version_bump.py +++ b/taskcluster/gecko_taskgraph/transforms/release_version_bump.py @@ -5,9 +5,9 @@ Transform the update generation task into an actual task description. """ +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/repackage.py b/taskcluster/gecko_taskgraph/transforms/repackage.py index 33fc467f788e..cf53a7cfc827 100644 --- a/taskcluster/gecko_taskgraph/transforms/repackage.py +++ b/taskcluster/gecko_taskgraph/transforms/repackage.py @@ -9,15 +9,12 @@ Transform the repackage task into an actual task description. import copy from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by from voluptuous import Required, Optional, Extra from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job -from gecko_taskgraph.util.schema import ( - optionally_keyed_by, - resolve_keyed_by, -) from gecko_taskgraph.util.platforms import archive_format, architecture from gecko_taskgraph.util.workertypes import worker_type_implementation from gecko_taskgraph.transforms.job import job_description_schema diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py index d241001c894e..e9976789241f 100644 --- a/taskcluster/gecko_taskgraph/transforms/repackage_partner.py +++ b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py @@ -8,16 +8,13 @@ Transform the repackage task into an actual task description. import copy +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by from taskgraph.util.taskcluster import get_artifact_prefix from voluptuous import Required, Optional from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job -from gecko_taskgraph.util.schema import ( - optionally_keyed_by, - resolve_keyed_by, -) from gecko_taskgraph.util.partners import get_partner_config_by_kind from gecko_taskgraph.util.platforms import archive_format, executable_extension from gecko_taskgraph.util.workertypes import worker_type_implementation diff --git a/taskcluster/gecko_taskgraph/transforms/repo_update.py b/taskcluster/gecko_taskgraph/transforms/repo_update.py index 0ca7de8f136c..e4efa1f9b350 100644 --- a/taskcluster/gecko_taskgraph/transforms/repo_update.py +++ b/taskcluster/gecko_taskgraph/transforms/repo_update.py @@ -5,9 +5,9 @@ Transform the repo-update task into an actual task description. """ +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import resolve_keyed_by transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/signing.py b/taskcluster/gecko_taskgraph/transforms/signing.py index 4a2c0fa609d4..3dfe6702bfbb 100644 --- a/taskcluster/gecko_taskgraph/transforms/signing.py +++ b/taskcluster/gecko_taskgraph/transforms/signing.py @@ -6,6 +6,8 @@ Transform the signing task into an actual task description. """ from taskgraph.util.keyed_by import evaluate_keyed_by +from taskgraph.util.schema import taskref_or_string +from voluptuous import Required, Optional from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence @@ -13,11 +15,8 @@ from gecko_taskgraph.util.attributes import ( copy_attributes_from_dependent_job, release_level, ) -from gecko_taskgraph.util.schema import taskref_or_string from gecko_taskgraph.util.scriptworker import get_signing_cert_scope_per_platform from gecko_taskgraph.transforms.task import task_description_schema -from voluptuous import Required, Optional - transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/source_test.py b/taskcluster/gecko_taskgraph/transforms/source_test.py index 29de3de97fa8..eaab2b9ad651 100644 --- a/taskcluster/gecko_taskgraph/transforms/source_test.py +++ b/taskcluster/gecko_taskgraph/transforms/source_test.py @@ -12,6 +12,7 @@ import os import taskgraph from taskgraph.util.attributes import keymatch +from taskgraph.util.schema import Schema, resolve_keyed_by, optionally_keyed_by from taskgraph.util.treeherder import join_symbol, split_symbol from voluptuous import ( Any, @@ -23,7 +24,6 @@ from voluptuous import ( from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.job import job_description_schema from gecko_taskgraph.util.hg import get_json_automationrelevance -from gecko_taskgraph.util.schema import Schema, resolve_keyed_by, optionally_keyed_by source_test_description_schema = Schema( { diff --git a/taskcluster/gecko_taskgraph/transforms/spidermonkey.py b/taskcluster/gecko_taskgraph/transforms/spidermonkey.py index 2ac3cb1a77c5..9ada14a4b8e2 100644 --- a/taskcluster/gecko_taskgraph/transforms/spidermonkey.py +++ b/taskcluster/gecko_taskgraph/transforms/spidermonkey.py @@ -1,10 +1,11 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import copy + +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import resolve_keyed_by -import copy transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py index c0c576af5dd6..1d4032541a22 100644 --- a/taskcluster/gecko_taskgraph/transforms/task.py +++ b/taskcluster/gecko_taskgraph/transforms/task.py @@ -16,30 +16,29 @@ import time from copy import deepcopy import attr -from gecko_taskgraph import GECKO, MAX_DEPENDENCIES -from gecko_taskgraph.optimize.schema import OptimizationSchema -from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.attributes import TRUNK_PROJECTS, is_try, release_level -from gecko_taskgraph.util.hash import hash_path -from gecko_taskgraph.util.partners import get_partners_to_be_published -from gecko_taskgraph.util.schema import ( +from mozbuild.util import memoize +from taskgraph.util.keyed_by import evaluate_keyed_by +from taskgraph.util.schema import ( Schema, optionally_keyed_by, resolve_keyed_by, taskref_or_string, validate_schema, ) -from gecko_taskgraph.util.scriptworker import BALROG_ACTIONS, get_release_config -from gecko_taskgraph.util.signed_artifacts import get_signed_artifacts -from gecko_taskgraph.util.workertypes import worker_type_implementation -from mozbuild.util import memoize -from taskgraph.util.keyed_by import evaluate_keyed_by from taskgraph.util.time import value_of from taskgraph.util.treeherder import split_symbol -from voluptuous import All, Any, Extra, Match, NotIn, Optional, Required +from voluptuous import Any, Required, Optional, Extra, Match, All, NotIn -from ..util import docker as dockerutil -from ..util.workertypes import get_worker_type +from gecko_taskgraph import GECKO, MAX_DEPENDENCIES +from gecko_taskgraph.optimize.schema import OptimizationSchema +from gecko_taskgraph.transforms.base import TransformSequence +from gecko_taskgraph.util import docker as dockerutil +from gecko_taskgraph.util.attributes import TRUNK_PROJECTS, is_try, release_level +from gecko_taskgraph.util.hash import hash_path +from gecko_taskgraph.util.partners import get_partners_to_be_published +from gecko_taskgraph.util.scriptworker import BALROG_ACTIONS, get_release_config +from gecko_taskgraph.util.signed_artifacts import get_signed_artifacts +from gecko_taskgraph.util.workertypes import get_worker_type, worker_type_implementation RUN_TASK = os.path.join(GECKO, "taskcluster", "scripts", "run-task") diff --git a/taskcluster/gecko_taskgraph/transforms/test/__init__.py b/taskcluster/gecko_taskgraph/transforms/test/__init__.py index 365031db65cb..b8102be8089a 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/__init__.py +++ b/taskcluster/gecko_taskgraph/transforms/test/__init__.py @@ -22,6 +22,7 @@ import logging from importlib import import_module from mozbuild.schedules import INCLUSIVE_COMPONENTS +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by from voluptuous import ( Any, Optional, @@ -32,11 +33,6 @@ from voluptuous import ( from gecko_taskgraph.optimize.schema import OptimizationSchema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.test.other import get_mobile_project -from gecko_taskgraph.util.schema import ( - optionally_keyed_by, - resolve_keyed_by, - Schema, -) from gecko_taskgraph.util.chunking import manifest_loaders diff --git a/taskcluster/gecko_taskgraph/transforms/test/other.py b/taskcluster/gecko_taskgraph/transforms/test/other.py index 2dc864caf562..c5330cc06ede 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/other.py +++ b/taskcluster/gecko_taskgraph/transforms/test/other.py @@ -6,16 +6,21 @@ import hashlib import json import re -from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS -from gecko_taskgraph.util.platforms import platform_family -from gecko_taskgraph.util.schema import Schema, resolve_keyed_by from mozbuild.schedules import INCLUSIVE_COMPONENTS from mozbuild.util import ReadOnlyDict from taskgraph.util.attributes import keymatch from taskgraph.util.keyed_by import evaluate_keyed_by +from taskgraph.util.schema import Schema, resolve_keyed_by from taskgraph.util.taskcluster import get_artifact_path, get_index_url -from voluptuous import Any, Optional, Required +from voluptuous import ( + Any, + Optional, + Required, +) + +from gecko_taskgraph.transforms.base import TransformSequence +from gecko_taskgraph.transforms.test.variant import TEST_VARIANTS +from gecko_taskgraph.util.platforms import platform_family transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py index 0a1d9b5aa3b7..dbbbc3585ccd 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py +++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py @@ -5,11 +5,16 @@ from copy import deepcopy +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema +from taskgraph.util.treeherder import join_symbol, split_symbol +from voluptuous import ( + Optional, + Required, + Extra, +) + from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.test import test_description_schema -from gecko_taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by -from taskgraph.util.treeherder import join_symbol, split_symbol -from voluptuous import Extra, Optional, Required transforms = TransformSequence() task_transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/test/variant.py b/taskcluster/gecko_taskgraph/transforms/test/variant.py index e23574dae7b0..493a8e282632 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/variant.py +++ b/taskcluster/gecko_taskgraph/transforms/test/variant.py @@ -3,14 +3,19 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import copy -import gecko_taskgraph import jsone -from gecko_taskgraph.transforms.base import TransformSequence -from gecko_taskgraph.util.schema import Schema, validate_schema -from gecko_taskgraph.util.templates import merge +from taskgraph.util.schema import Schema, validate_schema from taskgraph.util.treeherder import join_symbol, split_symbol from taskgraph.util.yaml import load_yaml -from voluptuous import Any, Optional, Required +from voluptuous import ( + Any, + Optional, + Required, +) + +import gecko_taskgraph +from gecko_taskgraph.transforms.base import TransformSequence +from gecko_taskgraph.util.templates import merge transforms = TransformSequence() diff --git a/taskcluster/gecko_taskgraph/transforms/update_verify_config.py b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py index ea5b7bb4a0ec..e88ee24b579b 100644 --- a/taskcluster/gecko_taskgraph/transforms/update_verify_config.py +++ b/taskcluster/gecko_taskgraph/transforms/update_verify_config.py @@ -7,9 +7,10 @@ Transform the beetmover task into an actual task description. from urllib.parse import urlsplit +from taskgraph.util.schema import resolve_keyed_by + from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import release_level -from gecko_taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.util.scriptworker import get_release_config from gecko_taskgraph.transforms.task import ( get_branch_repo, diff --git a/taskcluster/gecko_taskgraph/util/partners.py b/taskcluster/gecko_taskgraph/util/partners.py index 37fca50ac4c3..23cda51d6590 100644 --- a/taskcluster/gecko_taskgraph/util/partners.py +++ b/taskcluster/gecko_taskgraph/util/partners.py @@ -13,9 +13,9 @@ from urllib.parse import urlencode import yaml from redo import retry +from taskgraph.util.schema import resolve_keyed_by from gecko_taskgraph.util.attributes import release_level -from gecko_taskgraph.util.schema import resolve_keyed_by # Suppress chatty requests logging logging.getLogger("requests").setLevel(logging.WARNING) diff --git a/taskcluster/gecko_taskgraph/util/schema.py b/taskcluster/gecko_taskgraph/util/schema.py deleted file mode 100644 index 0b4e64bfc559..000000000000 --- a/taskcluster/gecko_taskgraph/util/schema.py +++ /dev/null @@ -1,241 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - - -import re -import pprint -import collections -import collections.abc - -import taskgraph -import voluptuous -from taskgraph.util.keyed_by import evaluate_keyed_by - - -def validate_schema(schema, obj, msg_prefix): - """ - Validate that object satisfies schema. If not, generate a useful exception - beginning with msg_prefix. - """ - if taskgraph.fast: - return - try: - schema(obj) - except voluptuous.MultipleInvalid as exc: - msg = [msg_prefix] - for error in exc.errors: - msg.append(str(error)) - raise Exception("\n".join(msg) + "\n" + pprint.pformat(obj)) - - -def optionally_keyed_by(*arguments): - """ - Mark a schema value as optionally keyed by any of a number of fields. The - schema is the last argument, and the remaining fields are taken to be the - field names. For example: - - 'some-value': optionally_keyed_by( - 'test-platform', 'build-platform', - Any('a', 'b', 'c')) - - The resulting schema will allow nesting of `by-test-platform` and - `by-build-platform` in either order. - """ - schema = arguments[-1] - fields = arguments[:-1] - - def validator(obj): - if isinstance(obj, dict) and len(obj) == 1: - k, v = list(obj.items())[0] - if k.startswith("by-") and k[len("by-") :] in fields: - res = {} - for kk, vv in v.items(): - try: - res[kk] = validator(vv) - except voluptuous.Invalid as e: - e.prepend([k, kk]) - raise - return res - return Schema(schema)(obj) - - return validator - - -def resolve_keyed_by( - item, field, item_name, defer=None, enforce_single_match=True, **extra_values -): - """ - For values which can either accept a literal value, or be keyed by some - other attribute of the item, perform that lookup and replacement in-place - (modifying `item` directly). The field is specified using dotted notation - to traverse dictionaries. - - For example, given item:: - - job: - test-platform: linux128 - chunks: - by-test-platform: - macosx-10.11/debug: 13 - win.*: 6 - default: 12 - - a call to `resolve_keyed_by(item, 'job.chunks', item['thing-name'])` - would mutate item in-place to:: - - job: - test-platform: linux128 - chunks: 12 - - Items can be nested as deeply as the schema will allow:: - - chunks: - by-test-platform: - win.*: - by-project: - ash: .. - cedar: .. - linux: 13 - default: 12 - - Args: - item (dict): Object being evaluated. - field (str): Name of the key to perform evaluation on. - item_name (str): Used to generate useful error messages. - defer (list): - Allows evaluating a by-* entry at a later time. In the example - above it's possible that the project attribute hasn't been set yet, - in which case we'd want to stop before resolving that subkey and - then call this function again later. This can be accomplished by - setting `defer=["project"]` in this example. - enforce_single_match (bool): - If True (default), each task may only match a single arm of the - evaluation. - extra_values (kwargs): - If supplied, represent additional values available - for reference from by-. - - Returns: - dict: item which has also been modified in-place. - """ - # find the field, returning the item unchanged if anything goes wrong - container, subfield = item, field - while "." in subfield: - f, subfield = subfield.split(".", 1) - if f not in container: - return item - container = container[f] - if not isinstance(container, dict): - return item - - if subfield not in container: - return item - - container[subfield] = evaluate_keyed_by( - value=container[subfield], - item_name=f"`{field}` in `{item_name}`", - defer=defer, - enforce_single_match=enforce_single_match, - attributes=dict(item, **extra_values), - ) - - return item - - -# Schemas for YAML files should use dashed identifiers by default. If there are -# components of the schema for which there is a good reason to use another format, -# they can be whitelisted here. -WHITELISTED_SCHEMA_IDENTIFIERS = [ - # upstream-artifacts are handed directly to scriptWorker, which expects interCaps - lambda path: "[{!r}]".format("upstream-artifacts") in path, - lambda path: ( - "[{!r}]".format("test_name") in path - or "[{!r}]".format("json_location") in path - or "[{!r}]".format("video_location") in path - ), -] - - -def check_schema(schema): - identifier_re = re.compile(r"^\$?[a-z][a-z0-9-]*$") - - def whitelisted(path): - return any(f(path) for f in WHITELISTED_SCHEMA_IDENTIFIERS) - - def iter(path, sch): - def check_identifier(path, k): - if k in (str, str, voluptuous.Extra): - pass - elif isinstance(k, voluptuous.NotIn): - pass - elif isinstance(k, str): - if not identifier_re.match(k) and not whitelisted(path): - raise RuntimeError( - "YAML schemas should use dashed lower-case identifiers, " - "not {!r} @ {}".format(k, path) - ) - elif isinstance(k, (voluptuous.Optional, voluptuous.Required)): - check_identifier(path, k.schema) - elif isinstance(k, (voluptuous.Any, voluptuous.All)): - for v in k.validators: - check_identifier(path, v) - elif not whitelisted(path): - raise RuntimeError( - "Unexpected type in YAML schema: {} @ {}".format( - type(k).__name__, path - ) - ) - - if isinstance(sch, collections.abc.Mapping): - for k, v in sch.items(): - child = f"{path}[{k!r}]" - check_identifier(child, k) - iter(child, v) - elif isinstance(sch, (list, tuple)): - for i, v in enumerate(sch): - iter(f"{path}[{i}]", v) - elif isinstance(sch, voluptuous.Any): - for v in sch.validators: - iter(path, v) - - iter("schema", schema.schema) - - -class Schema(voluptuous.Schema): - """ - Operates identically to voluptuous.Schema, but applying some taskgraph-specific checks - in the process. - """ - - def __init__(self, *args, check=True, **kwargs): - super().__init__(*args, **kwargs) - - self.check = check - if not taskgraph.fast and self.check: - check_schema(self) - - def extend(self, *args, **kwargs): - schema = super().extend(*args, **kwargs) - - if self.check: - check_schema(schema) - # We want twice extend schema to be checked too. - schema.__class__ = Schema - return schema - - def _compile(self, schema): - if taskgraph.fast: - return - return super()._compile(schema) - - def __getitem__(self, item): - return self.schema[item] - - -# shortcut for a string where task references are allowed -taskref_or_string = voluptuous.Any( - str, - {voluptuous.Required("task-reference"): str}, - {voluptuous.Required("artifact-reference"): str}, -) diff --git a/taskcluster/gecko_taskgraph/util/scriptworker.py b/taskcluster/gecko_taskgraph/util/scriptworker.py index 272dea406365..3e15056c9a08 100644 --- a/taskcluster/gecko_taskgraph/util/scriptworker.py +++ b/taskcluster/gecko_taskgraph/util/scriptworker.py @@ -25,11 +25,10 @@ from datetime import datetime import jsone from mozbuild.util import memoize +from taskgraph.util.schema import resolve_keyed_by from taskgraph.util.taskcluster import get_artifact_prefix from taskgraph.util.yaml import load_yaml -from .schema import resolve_keyed_by - # constants {{{1 """Map signing scope aliases to sets of projects. diff --git a/taskcluster/test/test_mozilla_central.py b/taskcluster/test/test_mozilla_central.py index 729d94f81775..042b14a18a57 100644 --- a/taskcluster/test/test_mozilla_central.py +++ b/taskcluster/test/test_mozilla_central.py @@ -38,7 +38,7 @@ def test_tasks_are_scheduled(optimized_task_graph, filter_tasks, func, min_expec def test_test_setting(full_task_graph, filter_tasks): """Verify that all test tasks' ``test-setting`` object conforms to the schema.""" from gecko_taskgraph.transforms.test.other import test_setting_description_schema - from gecko_taskgraph.util.schema import validate_schema + from taskgraph.util.schema import validate_schema tasks = filter_tasks(full_task_graph, lambda t: t.kind == "test") diff --git a/tools/lint/rejected-words.yml b/tools/lint/rejected-words.yml index d13507e28792..adff41350787 100644 --- a/tools/lint/rejected-words.yml +++ b/tools/lint/rejected-words.yml @@ -297,7 +297,7 @@ avoid-blacklist-and-whitelist: - taskcluster/gecko_taskgraph/target_tasks.py - taskcluster/gecko_taskgraph/transforms/test/other.py - taskcluster/gecko_taskgraph/try_option_syntax.py - - taskcluster/gecko_taskgraph/util/schema.py + - taskcluster/gecko_taskgraph/__init__.py - taskcluster/test/test_mach_try_auto.py - testing/condprofile/condprof/client.py - testing/condprofile/condprof/tests/profile/prefs.js