diff --git a/.cron.yml b/.cron.yml index 33e1af2c4f1d..04422d22c789 100644 --- a/.cron.yml +++ b/.cron.yml @@ -29,8 +29,6 @@ jobs: - mozilla-central when: by-project: - # `l10n-bumper` job should have enough time to finish before this - # job runs mozilla-central: [{hour: 10, minute: 0}, {hour: 22, minute: 0}] # No default @@ -277,12 +275,14 @@ jobs: treeherder-symbol: l10n-bump target-tasks-method: l10n_bump run-on-projects: + - autoland - mozilla-central - mozilla-beta when: by-project: # 3h15m before launch of `nightly-desktop` - mozilla-central: [{hour: 6, minute: 45}, {hour: 18, minute: 45}] + autoland: [{hour: 6, minute: 45}, {hour: 18, minute: 45}] + mozilla-central: [] # 3h15m before launch of `daily-releases` mozilla-beta: [{hour: 18, minute: 45}] # No default diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index fc610d46bd25..28f6ebe7c6ba 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -555,7 +555,10 @@ workers: worker-type: by-release-level: production: '{trust-domain}-3-tree' - staging: '{trust-domain}-1-tree' + staging: + by-project: + autoland: '{trust-domain}-3-tree' + default: '{trust-domain}-1-tree' tree-dev: provisioner: scriptworker-k8s implementation: treescript diff --git a/taskcluster/ci/l10n-bump/kind.yml b/taskcluster/ci/l10n-bump/kind.yml index 396e1ee51e9e..2dd21e40689d 100644 --- a/taskcluster/ci/l10n-bump/kind.yml +++ b/taskcluster/ci/l10n-bump/kind.yml @@ -8,7 +8,6 @@ transforms: - gecko_taskgraph.transforms.release_version_bump:transforms - gecko_taskgraph.transforms.task:transforms - job-defaults: worker-type: tree worker: @@ -19,15 +18,20 @@ job-defaults: by-project: mozilla-beta: false default: true - ignore-closed-tree: false + ignore-closed-tree: + by-project: + autoland: true + default: false push: by-project: mozilla-(central|beta): true + autoland: true default: false source-repo: by-release-type: beta: https://hg.mozilla.org/releases/mozilla-beta - default: https://hg.mozilla.org/mozilla-central + nightly: https://hg.mozilla.org/mozilla-central + default: https://hg.mozilla.org/integration/autoland l10n-bump-info: by-release-type: # XXX whenever the `beta` config changes, make sure to make @@ -52,23 +56,27 @@ job-defaults: - win64-devedition - win64-aarch64 - win64-aarch64-devedition - platform-configs: [{ - "platforms": [ - "linux", - "linux-devedition", - "linux64", - "linux64-devedition", - "macosx64", - "macosx64-devedition", - "win32", - "win32-devedition", - "win64", - "win64-devedition", - "win64-aarch64", - "win64-aarch64-devedition", - ], - "path": "browser/locales/shipped-locales", - }] + platform-configs: + [ + { + "platforms": + [ + "linux", + "linux-devedition", + "linux64", + "linux64-devedition", + "macosx64", + "macosx64-devedition", + "win32", + "win32-devedition", + "win64", + "win64-devedition", + "win64-aarch64", + "win64-aarch64-devedition", + ], + "path": "browser/locales/shipped-locales", + }, + ] default: name: Firefox l10n changesets path: browser/locales/l10n-changesets.json @@ -89,23 +97,27 @@ job-defaults: - win64-devedition - win64-aarch64 - win64-aarch64-devedition - platform-configs: [{ - "platforms": [ - "linux", - "linux-devedition", - "linux64", - "linux64-devedition", - "macosx64", - "macosx64-devedition", - "win32", - "win32-devedition", - "win64", - "win64-devedition", - "win64-aarch64", - "win64-aarch64-devedition", - ], - "path": "browser/locales/all-locales", - }] + platform-configs: + [ + { + "platforms": + [ + "linux", + "linux-devedition", + "linux64", + "linux64-devedition", + "macosx64", + "macosx64-devedition", + "win32", + "win32-devedition", + "win64", + "win64-devedition", + "win64-aarch64", + "win64-aarch64-devedition", + ], + "path": "browser/locales/all-locales", + }, + ] jobs: l10n-bumper: diff --git a/taskcluster/gecko_taskgraph/config.py b/taskcluster/gecko_taskgraph/config.py index bb4af2bb8649..86bbae1a4d10 100644 --- a/taskcluster/gecko_taskgraph/config.py +++ b/taskcluster/gecko_taskgraph/config.py @@ -107,7 +107,7 @@ graph_config_schema = Schema( Required("implementation"): str, Required("os"): str, Required("worker-type"): optionally_keyed_by( - "level", "release-level", str + "level", "release-level", "project", str ), } }, diff --git a/taskcluster/gecko_taskgraph/morph.py b/taskcluster/gecko_taskgraph/morph.py index 566ccd00aaf7..d3abc1711f66 100644 --- a/taskcluster/gecko_taskgraph/morph.py +++ b/taskcluster/gecko_taskgraph/morph.py @@ -29,7 +29,6 @@ from taskgraph.graph import Graph from taskgraph.task import Task from taskgraph.taskgraph import TaskGraph -from .util.attributes import release_level from .util.workertypes import get_worker_type here = os.path.abspath(os.path.dirname(__file__)) @@ -71,7 +70,9 @@ def derive_misc_task( image_taskid = label_to_taskid["docker-image-" + image] provisioner_id, worker_type = get_worker_type( - graph_config, "misc", parameters["level"], release_level(parameters["project"]) + graph_config, + parameters, + "misc", ) deps = copy.copy(dependencies) diff --git a/taskcluster/gecko_taskgraph/test/test_transforms_job.py b/taskcluster/gecko_taskgraph/test/test_transforms_job.py index 43bfef1bf4fa..ef7181b466e6 100644 --- a/taskcluster/gecko_taskgraph/test/test_transforms_job.py +++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py @@ -45,6 +45,7 @@ def config(): "head_repository": "http://hg.example.com", "head_rev": "abcdef", "level": 1, + "project": "example", } ) return TransformConfig( diff --git a/taskcluster/gecko_taskgraph/transforms/build.py b/taskcluster/gecko_taskgraph/transforms/build.py index dad5e6ec81e8..553709879e29 100644 --- a/taskcluster/gecko_taskgraph/transforms/build.py +++ b/taskcluster/gecko_taskgraph/transforms/build.py @@ -27,7 +27,7 @@ def set_defaults(config, jobs): job["treeherder"].setdefault("kind", "build") job["treeherder"].setdefault("tier", 1) _, worker_os = worker_type_implementation( - config.graph_config, job["worker-type"] + config.graph_config, config.params, job["worker-type"] ) worker = job.setdefault("worker", {}) worker.setdefault("env", {}) @@ -172,7 +172,7 @@ def use_profile_data(config, jobs): job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"}) _, worker_os = worker_type_implementation( - config.graph_config, job["worker-type"] + config.graph_config, config.params, job["worker-type"] ) if worker_os == "linux": # LTO linkage needs more open files than the default from run-task. diff --git a/taskcluster/gecko_taskgraph/transforms/job/__init__.py b/taskcluster/gecko_taskgraph/transforms/job/__init__.py index b2a6e0752475..0f27b6b73b8b 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/__init__.py +++ b/taskcluster/gecko_taskgraph/transforms/job/__init__.py @@ -133,7 +133,9 @@ def rewrite_when_to_optimization(config, jobs): @transforms.add def set_implementation(config, jobs): for job in jobs: - impl, os = worker_type_implementation(config.graph_config, job["worker-type"]) + impl, os = worker_type_implementation( + config.graph_config, config.params, job["worker-type"] + ) if os: job.setdefault("tags", {})["os"] = os if impl: @@ -163,7 +165,7 @@ def add_resource_monitor(config, jobs): for job in jobs: if job.get("attributes", {}).get("resource-monitor"): worker_implementation, worker_os = worker_type_implementation( - config.graph_config, job["worker-type"] + config.graph_config, config.params, job["worker-type"] ) # Normalise worker os so that linux-bitbar and similar use linux tools. worker_os = worker_os.split("-")[0] diff --git a/taskcluster/gecko_taskgraph/transforms/release_version_bump.py b/taskcluster/gecko_taskgraph/transforms/release_version_bump.py index 40bcd237cf8f..a0f3f69d05eb 100644 --- a/taskcluster/gecko_taskgraph/transforms/release_version_bump.py +++ b/taskcluster/gecko_taskgraph/transforms/release_version_bump.py @@ -21,7 +21,12 @@ def handle_keyed_by(config, tasks): ] for task in tasks: fields = default_fields[:] - for additional_field in ("l10n-bump-info", "source-repo", "dontbuild"): + for additional_field in ( + "l10n-bump-info", + "source-repo", + "dontbuild", + "ignore-closed-tree", + ): if additional_field in task["worker"]: fields.append(f"worker.{additional_field}") for field in fields: diff --git a/taskcluster/gecko_taskgraph/transforms/repackage.py b/taskcluster/gecko_taskgraph/transforms/repackage.py index 9d68882d1ad3..a72559accec7 100644 --- a/taskcluster/gecko_taskgraph/transforms/repackage.py +++ b/taskcluster/gecko_taskgraph/transforms/repackage.py @@ -475,7 +475,7 @@ def make_job_description(config, jobs): worker["artifacts"] = _generate_task_output_files( dep_job, - worker_type_implementation(config.graph_config, worker_type), + worker_type_implementation(config.graph_config, config.params, worker_type), repackage_config=repackage_config, locale=locale, ) diff --git a/taskcluster/gecko_taskgraph/transforms/repackage_partner.py b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py index d5f7b98923a6..f71b24b5f9b4 100644 --- a/taskcluster/gecko_taskgraph/transforms/repackage_partner.py +++ b/taskcluster/gecko_taskgraph/transforms/repackage_partner.py @@ -177,7 +177,7 @@ def make_job_description(config, jobs): worker["artifacts"] = _generate_task_output_files( dep_job, - worker_type_implementation(config.graph_config, worker_type), + worker_type_implementation(config.graph_config, config.params, worker_type), repackage_config, partner=repack_id, ) diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py index e45b2f70d6d5..8181992989ff 100644 --- a/taskcluster/gecko_taskgraph/transforms/task.py +++ b/taskcluster/gecko_taskgraph/transforms/task.py @@ -1431,7 +1431,9 @@ def set_implementation(config, tasks): yield task continue - impl, os = worker_type_implementation(config.graph_config, task["worker-type"]) + impl, os = worker_type_implementation( + config.graph_config, config.params, task["worker-type"] + ) tags = task.setdefault("tags", {}) tags["worker-implementation"] = impl @@ -1820,9 +1822,8 @@ def build_task(config, tasks): else: provisioner_id, worker_type = get_worker_type( config.graph_config, + config.params, task["worker-type"], - level=level, - release_level=release_level(config.params["project"]), ) task["worker-type"] = "/".join([provisioner_id, worker_type]) project = config.params["project"] diff --git a/taskcluster/gecko_taskgraph/util/workertypes.py b/taskcluster/gecko_taskgraph/util/workertypes.py index 0b970c425d55..aafa4fe56b03 100644 --- a/taskcluster/gecko_taskgraph/util/workertypes.py +++ b/taskcluster/gecko_taskgraph/util/workertypes.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - +from gecko_taskgraph.util.attributes import release_level as _release_level from mozbuild.util import memoize from taskgraph.util.attributes import keymatch from taskgraph.util.keyed_by import evaluate_keyed_by @@ -20,7 +20,7 @@ WORKER_TYPES = { @memoize -def _get(graph_config, alias, level, release_level): +def _get(graph_config, alias, level, release_level, project): """Get the configuration for this worker_type alias: {provisioner, worker-type, implementation, os}""" level = str(level) @@ -61,10 +61,13 @@ def _get(graph_config, alias, level, release_level): "alias": alias, } ) + attrs = {"level": level, "release-level": release_level} + if project: + attrs["project"] = project worker_config["worker-type"] = evaluate_keyed_by( worker_config["worker-type"], f"worker-type alias {alias} field worker-type", - {"level": level, "release-level": release_level}, + attrs, ).format( **{ "trust-domain": graph_config["trust-domain"], @@ -76,18 +79,26 @@ def _get(graph_config, alias, level, release_level): return worker_config -def worker_type_implementation(graph_config, worker_type): +def worker_type_implementation(graph_config, parameters, worker_type): """Get the worker implementation and OS for the given workerType, where the OS represents the host system, not the target OS, in the case of cross-compiles.""" - worker_config = _get(graph_config, worker_type, "1", "staging") + worker_config = _get( + graph_config, worker_type, "1", "staging", parameters["project"] + ) return worker_config["implementation"], worker_config.get("os") -def get_worker_type(graph_config, worker_type, level, release_level): +def get_worker_type(graph_config, parameters, worker_type): """ Get the worker type provisioner and worker-type, optionally evaluating aliases from the graph config. """ - worker_config = _get(graph_config, worker_type, level, release_level) + worker_config = _get( + graph_config, + worker_type, + parameters["level"], + _release_level(parameters.get("project")), + parameters.get("project"), + ) return worker_config["provisioner"], worker_config["worker-type"] diff --git a/tools/tryselect/task_config.py b/tools/tryselect/task_config.py index e2e0c4d05d79..c9b6e8781119 100644 --- a/tools/tryselect/task_config.py +++ b/tools/tryselect/task_config.py @@ -505,10 +505,7 @@ class WorkerOverrides(TryConfig): ) sys.exit(1) provisioner, worker_type = get_worker_type( - graph_config, - alias, - level="1", - release_level="staging", + graph_config, worker_type=alias, parameters={"level": "1"} ) overrides[alias] = "{provisioner}/{worker_type}{suffix}".format( provisioner=provisioner, worker_type=worker_type, suffix=suffix