From e18820c69be49f70d20ce5fb287609818aa4445e Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Tue, 14 Nov 2023 20:32:55 +0000 Subject: [PATCH] Bug 1839891 - [ci] Convert 'release-push-langpacks' tasks to 'from_deps' transforms, r=releng-reviewers,taskgraph-reviewers,bhearsum Differential Revision: https://phabricator.services.mozilla.com/D193076 --- .../ci/release-push-langpacks/kind.yml | 68 ++++++++++--------- .../release_sign_and_push_langpacks.py | 50 ++++++++------ 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/taskcluster/ci/release-push-langpacks/kind.yml b/taskcluster/ci/release-push-langpacks/kind.yml index cd46683bbb7b..fbc5bcc8a934 100644 --- a/taskcluster/ci/release-push-langpacks/kind.yml +++ b/taskcluster/ci/release-push-langpacks/kind.yml @@ -2,49 +2,51 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. --- -loader: gecko_taskgraph.loader.single_dep:loader +loader: gecko_taskgraph.loader.transform:loader transforms: - - gecko_taskgraph.transforms.release_sign_and_push_langpacks:transforms - - gecko_taskgraph.transforms.task:transforms + - taskgraph.transforms.from_deps + - gecko_taskgraph.transforms.release_sign_and_push_langpacks + - gecko_taskgraph.transforms.task kind-dependencies: - build - shippable-l10n - only-for-build-platforms: - linux64-shippable/opt # addons.mozilla.org only support 1 platform per locale. That's why we use linux64 - linux64-devedition/opt - macosx64-shippable/opt # Although, we need the special locale "ja-JP-Mac" from this platform - macosx64-devedition/opt - -job-template: - description: Sends {locales} XPIs for platform to addons.mozilla.org - worker-type: addon - worker: - channel: - by-project: - mozilla-release: - by-platform: - # ja-JP-mac is only langpack on mac, and is unlisted always - macosx64.*: unlisted - default: listed - mozilla-beta: - by-platform: - # ja-JP-mac is only langpack on mac, and is unlisted always - macosx64-shippable.*: unlisted - macosx64-devedition.*: unlisted - linux.*-devedition.*: unlisted - default: listed - default: unlisted - upstream-artifacts: # See transforms - run-on-projects: [] - scopes: - by-release-level: - production: - - project:releng:addons.mozilla.org:server:production - staging: - - project:releng:addons.mozilla.org:server:staging - shipping-phase: promote +jobs: + release-push-langpacks: + description: Sends {locales} XPIs for platform to addons.mozilla.org + from-deps: + group-by: single-with-filters + worker-type: addon + worker: + channel: + by-project: + mozilla-release: + by-platform: + # ja-JP-mac is only langpack on mac, and is unlisted always + macosx64.*: unlisted + default: listed + mozilla-beta: + by-platform: + # ja-JP-mac is only langpack on mac, and is unlisted always + macosx64-shippable.*: unlisted + macosx64-devedition.*: unlisted + linux.*-devedition.*: unlisted + default: listed + default: unlisted + upstream-artifacts: # See transforms + run-on-projects: [] + scopes: + by-release-level: + production: + - project:releng:addons.mozilla.org:server:production + staging: + - project:releng:addons.mozilla.org:server:staging + shipping-phase: promote diff --git a/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py b/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py index 17e4d37fb38e..6761c76416a4 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py +++ b/taskcluster/gecko_taskgraph/transforms/release_sign_and_push_langpacks.py @@ -6,11 +6,11 @@ Transform the release-sign-and-push task into an actual task description. """ from taskgraph.transforms.base import TransformSequence -from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by +from taskgraph.util.dependencies import get_primary_dependency +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by from taskgraph.util.treeherder import inherit_treeherder_from_dep -from voluptuous import Any, Required +from voluptuous import Any, Optional, Required -from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.task import task_description_schema from gecko_taskgraph.util.attributes import ( copy_attributes_from_dependent_job, @@ -19,7 +19,7 @@ from gecko_taskgraph.util.attributes import ( transforms = TransformSequence() -langpack_sign_push_description_schema = schema.extend( +langpack_sign_push_description_schema = Schema( { Required("label"): str, Required("description"): str, @@ -33,7 +33,9 @@ langpack_sign_push_description_schema = schema.extend( Required("run-on-projects"): [], Required("scopes"): optionally_keyed_by("release-level", [str]), Required("shipping-phase"): task_description_schema["shipping-phase"], - Required("shipping-product"): task_description_schema["shipping-product"], + Optional("job-from"): task_description_schema["job-from"], + Optional("attributes"): task_description_schema["attributes"], + Optional("dependencies"): task_description_schema["dependencies"], } ) @@ -41,8 +43,13 @@ langpack_sign_push_description_schema = schema.extend( @transforms.add def set_label(config, jobs): for job in jobs: - label = "push-langpacks-{}".format(job["primary-dependency"].label) - job["label"] = label + dep_job = get_primary_dependency(config, job) + assert dep_job + + job["label"] = f"push-langpacks-{dep_job.label}" + + if "name" in job: + del job["name"] yield job @@ -53,6 +60,9 @@ transforms.add_validate(langpack_sign_push_description_schema) @transforms.add def resolve_keys(config, jobs): for job in jobs: + dep_job = get_primary_dependency(config, job) + assert dep_job + resolve_keyed_by( job, "worker-type", @@ -70,7 +80,7 @@ def resolve_keys(config, jobs): "worker.channel", item_name=job["label"], project=config.params["project"], - platform=job["primary-dependency"].attributes["build_platform"], + platform=dep_job.attributes["build_platform"], ) yield job @@ -79,8 +89,12 @@ def resolve_keys(config, jobs): @transforms.add def copy_attributes(config, jobs): for job in jobs: - dep_job = job["primary-dependency"] - job["attributes"] = copy_attributes_from_dependent_job(dep_job) + dep_job = get_primary_dependency(config, job) + assert dep_job + + job.setdefault("attributes", {}).update( + copy_attributes_from_dependent_job(dep_job) + ) job["attributes"]["chunk_locales"] = dep_job.attributes.get( "chunk_locales", ["en-US"] ) @@ -91,7 +105,10 @@ def copy_attributes(config, jobs): @transforms.add def filter_out_macos_jobs_but_mac_only_locales(config, jobs): for job in jobs: - build_platform = job["primary-dependency"].attributes.get("build_platform") + dep_job = get_primary_dependency(config, job) + assert dep_job + + build_platform = dep_job.attributes.get("build_platform") if build_platform in ("linux64-devedition", "linux64-shippable"): yield job @@ -112,7 +129,8 @@ def filter_out_macos_jobs_but_mac_only_locales(config, jobs): @transforms.add def make_task_description(config, jobs): for job in jobs: - dep_job = job["primary-dependency"] + dep_job = get_primary_dependency(config, job) + assert dep_job treeherder = inherit_treeherder_from_dep(job, dep_job) treeherder.setdefault( @@ -170,11 +188,3 @@ def get_upstream_task_ref(job, expected_kinds): raise Exception("Only one dependency expected") return f"<{upstream_tasks[0]}>" - - -@transforms.add -def strip_unused_data(config, jobs): - for job in jobs: - del job["primary-dependency"] - - yield job