From bf83843fa9bb348382c54b536bd8ad4966cb61c4 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Fri, 22 Nov 2019 23:57:25 +0000 Subject: [PATCH] Bug 1481916 - add l10n-bumper task. r=Callek,tomprince Differential Revision: https://phabricator.services.mozilla.com/D53116 --HG-- extra : moz-landing-system : lando --- .cron.yml | 12 ++ taskcluster/ci/config.yml | 1 + taskcluster/ci/l10n-bump/kind.yml | 127 ++++++++++++++++++ taskcluster/docs/kinds.rst | 4 + taskcluster/taskgraph/target_tasks.py | 10 ++ .../transforms/release_version_bump.py | 10 +- taskcluster/taskgraph/transforms/task.py | 40 +++++- 7 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 taskcluster/ci/l10n-bump/kind.yml diff --git a/.cron.yml b/.cron.yml index a4a32087ebdf..052d041c042e 100644 --- a/.cron.yml +++ b/.cron.yml @@ -218,3 +218,15 @@ jobs: - mozilla-central when: - {hour: 16, minute: 00} + + - name: l10n-bumper + job: + type: decision-task + treeherder-symbol: l10n-bump + target-tasks-method: l10n_bump + run-on-projects: + - mozilla-central + - mozilla-beta + - mozilla-esr68 + when: + - {minute: 45} diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 710cfdaca55c..8df873861f98 100755 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -34,6 +34,7 @@ treeherder: 'Fxfn-r': 'Firefox functional tests (remote)' 'Fxfn-r-1proc': 'Firefox functional tests (remote) without e10s' 'iris': 'Iris testing suite' + 'l10n-bump': 'L10n Bumper' 'M': 'Mochitests' 'M-1proc': 'Mochitests without e10s' 'M-fis': 'Mochitests with fission enabled' diff --git a/taskcluster/ci/l10n-bump/kind.yml b/taskcluster/ci/l10n-bump/kind.yml new file mode 100644 index 000000000000..99a7d7673235 --- /dev/null +++ b/taskcluster/ci/l10n-bump/kind.yml @@ -0,0 +1,127 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.release_version_bump:transforms + - taskgraph.transforms.task:transforms + + +job-defaults: + worker-type: + by-project: + mozilla-(beta|central|esr.*): scriptworker-k8s/gecko-3-tree + default: scriptworker-k8s/gecko-1-tree + worker: + implementation: treescript + tags: [] + bump: false + dontbuild: true + ignore-closed-tree: false + push: + by-project: + mozilla-(central|beta|esr.*): true + default: false + source-repo: + by-release-type: + esr68: https://hg.mozilla.org/releases/mozilla-esr68 + beta: https://hg.mozilla.org/releases/mozilla-beta + default: https://hg.mozilla.org/mozilla-central + l10n-bump-info: + by-release-type: + esr.*: + name: Fennec l10n changesets + path: mobile/locales/l10n-changesets.json + version-path: mobile/android/config/version-files/release/version.txt + revision-url: "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(COMBINED_MAJOR_VERSION)s" + platform-configs: [{ + "platforms": ["android-multilocale"], + "path": "mobile/android/locales/maemo-locales", + }] + beta: + name: Firefox l10n changesets + path: browser/locales/l10n-changesets.json + version-path: browser/config/version.txt + revision-url: "https://l10n.mozilla.org/shipping/l10n-changesets?av=fx%(MAJOR_VERSION)s" + ignore-config: + ja: + - macosx64 + - macosx64-devedition + ja-JP-mac: + - linux + - linux-devedition + - linux64 + - linux64-devedition + - win32 + - win32-devedition + - win64 + - win64-devedition + - win64-aarch64 + - win64-aarch64-devedition + platform-configs: [{ + "platforms": [ + "linux", + "linux-devedition", + "linux64", + "linux64-devedition", + "macosx64", + "macosx64-devedition", + "win32", + "win32-devedition", + "win64", + "win64-devedition", + "win64-aarch64", + "win64-aarch64-devedition", + ], + "path": "browser/locales/shipped-locales", + "format": "shipped-locales", + }] + default: + name: Firefox l10n changesets + path: browser/locales/l10n-changesets.json + version-path: browser/config/version.txt + ignore-config: + ja: + - macosx64 + - macosx64-devedition + ja-JP-mac: + - linux + - linux-devedition + - linux64 + - linux64-devedition + - win32 + - win32-devedition + - win64 + - win64-devedition + - win64-aarch64 + - win64-aarch64-devedition + platform-configs: [{ + "platforms": [ + "linux", + "linux-devedition", + "linux64", + "linux64-devedition", + "macosx64", + "macosx64-devedition", + "win32", + "win32-devedition", + "win64", + "win64-devedition", + "win64-aarch64", + "win64-aarch64-devedition", + ], + "path": "browser/locales/all-locales", + }] + +jobs: + l10n-bumper: + name: l10n_bumper + description: l10n changesets bumper + run-on-projects: [] + treeherder: + kind: build + platform: linux64/opt + symbol: l10n-bump(lb) + tier: 1 diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index 07ad731f57ef..2e2281766eac 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -616,3 +616,7 @@ by another task that produces a jobs.json artifact iris ---- Iris testing suite + +l10n-bump +--------- +Cron-driven tasks that bump l10n-changesets files in-tree, using data from the l10n dashboard. diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index 82b76503345d..7a90a0b04951 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -698,6 +698,16 @@ def target_tasks_file_update(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)] +@_target_task('l10n_bump') +def target_tasks_l10n_bump(full_task_graph, parameters, graph_config): + """Select the set of tasks required to perform l10n bumping. + """ + def filter(task): + # For now any task in the repo-update kind is ok + return task.kind in ['l10n-bump'] + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)] + + @_target_task('cron_bouncer_check') def target_tasks_bouncer_check(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform bouncer version verification. diff --git a/taskcluster/taskgraph/transforms/release_version_bump.py b/taskcluster/taskgraph/transforms/release_version_bump.py index e9d9684cdf0d..05bebb668f59 100644 --- a/taskcluster/taskgraph/transforms/release_version_bump.py +++ b/taskcluster/taskgraph/transforms/release_version_bump.py @@ -16,13 +16,19 @@ transforms = TransformSequence() @transforms.add def handle_keyed_by(config, tasks): """Resolve fields that can be keyed by platform, etc.""" - fields = [ + default_fields = [ 'worker.push', 'worker.bump-files', 'worker-type', ] for task in tasks: + fields = default_fields[:] + for additional_field in ("l10n-bump-info", "source-repo"): + if additional_field in task["worker"]: + fields.append('worker.{}'.format(additional_field)) for field in fields: resolve_keyed_by(task, field, item_name=task['name'], - project=config.params['project']) + **{'project': config.params['project'], + 'release-type': config.params['release_type']} + ) yield task diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index 304e6ffbb046..738f9b6c3f57 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -38,7 +38,6 @@ from taskgraph.util.partners import get_partners_to_be_published from taskgraph.util.scriptworker import ( BALROG_ACTIONS, get_release_config, - add_scope_prefix, ) from taskgraph.util.signed_artifacts import get_signed_artifacts from taskgraph.util.workertypes import worker_type_implementation @@ -1244,15 +1243,29 @@ def build_push_addons_payload(config, task, task_def): Optional('bump-files'): [basestring], Optional('repo-param-prefix'): basestring, Optional('dontbuild'): bool, + Optional('ignore-closed-tree'): bool, Required('force-dry-run', default=True): bool, - Required('push', default=False): bool + Required('push', default=False): bool, + Optional('source-repo'): basestring, + Optional('l10n-bump-info'): { + Required('name'): basestring, + Required('path'): basestring, + Required('version-path'): basestring, + Optional('revision-url'): basestring, + Optional('ignore-config'): object, + Required('platform-configs'): [{ + Required('platforms'): [basestring], + Required('path'): basestring, + Optional('format'): basestring, + }], + }, }) def build_treescript_payload(config, task, task_def): worker = task['worker'] release_config = get_release_config(config) - task_def['payload'] = {} - task_def.setdefault('scopes', []) + task_def['payload'] = {'actions': []} + actions = task_def['payload']['actions'] if worker['tags']: tag_names = [] product = task['shipping-product'].upper() @@ -1271,7 +1284,7 @@ def build_treescript_payload(config, task, task_def): 'revision': config.params['{}head_rev'.format(worker.get('repo-param-prefix', ''))], } task_def['payload']['tag_info'] = tag_info - task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:tagging')) + actions.append('tag') if worker['bump']: if not worker['bump-files']: @@ -1281,10 +1294,17 @@ def build_treescript_payload(config, task, task_def): bump_info['next_version'] = release_config['next_version'] bump_info['files'] = worker['bump-files'] task_def['payload']['version_bump_info'] = bump_info - task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:version_bump')) + actions.append('version_bump') + + if worker.get('l10n-bump-info'): + l10n_bump_info = {} + for k, v in worker['l10n-bump-info'].items(): + l10n_bump_info[k.replace('-', '_')] = worker['l10n-bump-info'][k] + task_def['payload']['l10n_bump_info'] = [l10n_bump_info] + actions.append('l10n_bump') if worker['push']: - task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:push')) + actions.append('push') if worker.get('force-dry-run'): task_def['payload']['dry_run'] = True @@ -1292,6 +1312,12 @@ def build_treescript_payload(config, task, task_def): if worker.get('dontbuild'): task_def['payload']['dontbuild'] = True + if worker.get('ignore-closed-tree') is not None: + task_def['payload']['ignore_closed_tree'] = worker['ignore-closed-tree'] + + if worker.get('source-repo'): + task_def['payload']['source_repo'] = worker['source-repo'] + @payload_builder('invalid', schema={ # an invalid task is one which should never actually be created; this is used in