Bug 1481916 - add l10n-bumper task. r=Callek,tomprince

Differential Revision: https://phabricator.services.mozilla.com/D53116

--HG--
extra : moz-landing-system : lando

.cron.yml

@@ -218,3 +218,15 @@ jobs:
           - mozilla-central
       when:
           - {hour: 16, minute: 00}
+
+    - name: l10n-bumper
+      job:
+          type: decision-task
+          treeherder-symbol: l10n-bump
+          target-tasks-method: l10n_bump
+      run-on-projects:
+          - mozilla-central
+          - mozilla-beta
+          - mozilla-esr68
+      when:
+          - {minute: 45}

taskcluster/ci/config.yml

@@ -34,6 +34,7 @@ treeherder:
         'Fxfn-r': 'Firefox functional tests (remote)'
         'Fxfn-r-1proc': 'Firefox functional tests (remote) without e10s'
         'iris': 'Iris testing suite'
+        'l10n-bump': 'L10n Bumper'
         'M': 'Mochitests'
         'M-1proc': 'Mochitests without e10s'
         'M-fis': 'Mochitests with fission enabled'

taskcluster/ci/l10n-bump/kind.yml

@@ -0,0 +1,127 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.release_version_bump:transforms
+    - taskgraph.transforms.task:transforms
+
+
+job-defaults:
+    worker-type:
+        by-project:
+            mozilla-(beta|central|esr.*): scriptworker-k8s/gecko-3-tree
+            default: scriptworker-k8s/gecko-1-tree
+    worker:
+        implementation: treescript
+        tags: []
+        bump: false
+        dontbuild: true
+        ignore-closed-tree: false
+        push:
+            by-project:
+                mozilla-(central|beta|esr.*): true
+                default: false
+        source-repo:
+            by-release-type:
+                esr68: https://hg.mozilla.org/releases/mozilla-esr68
+                beta: https://hg.mozilla.org/releases/mozilla-beta
+                default: https://hg.mozilla.org/mozilla-central
+        l10n-bump-info:
+            by-release-type:
+                esr.*:
+                    name: Fennec l10n changesets
+                    path: mobile/locales/l10n-changesets.json
+                    version-path: mobile/android/config/version-files/release/version.txt
+                    revision-url: "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(COMBINED_MAJOR_VERSION)s"
+                    platform-configs: [{
+                        "platforms": ["android-multilocale"],
+                        "path": "mobile/android/locales/maemo-locales",
+                    }]
+                beta:
+                    name: Firefox l10n changesets
+                    path: browser/locales/l10n-changesets.json
+                    version-path: browser/config/version.txt
+                    revision-url: "https://l10n.mozilla.org/shipping/l10n-changesets?av=fx%(MAJOR_VERSION)s"
+                    ignore-config:
+                        ja:
+                            - macosx64
+                            - macosx64-devedition
+                        ja-JP-mac:
+                            - linux
+                            - linux-devedition
+                            - linux64
+                            - linux64-devedition
+                            - win32
+                            - win32-devedition
+                            - win64
+                            - win64-devedition
+                            - win64-aarch64
+                            - win64-aarch64-devedition
+                    platform-configs: [{
+                        "platforms": [
+                            "linux",
+                            "linux-devedition",
+                            "linux64",
+                            "linux64-devedition",
+                            "macosx64",
+                            "macosx64-devedition",
+                            "win32",
+                            "win32-devedition",
+                            "win64",
+                            "win64-devedition",
+                            "win64-aarch64",
+                            "win64-aarch64-devedition",
+                        ],
+                        "path": "browser/locales/shipped-locales",
+                        "format": "shipped-locales",
+                    }]
+                default:
+                    name: Firefox l10n changesets
+                    path: browser/locales/l10n-changesets.json
+                    version-path: browser/config/version.txt
+                    ignore-config:
+                        ja:
+                            - macosx64
+                            - macosx64-devedition
+                        ja-JP-mac:
+                            - linux
+                            - linux-devedition
+                            - linux64
+                            - linux64-devedition
+                            - win32
+                            - win32-devedition
+                            - win64
+                            - win64-devedition
+                            - win64-aarch64
+                            - win64-aarch64-devedition
+                    platform-configs: [{
+                        "platforms": [
+                            "linux",
+                            "linux-devedition",
+                            "linux64",
+                            "linux64-devedition",
+                            "macosx64",
+                            "macosx64-devedition",
+                            "win32",
+                            "win32-devedition",
+                            "win64",
+                            "win64-devedition",
+                            "win64-aarch64",
+                            "win64-aarch64-devedition",
+                        ],
+                        "path": "browser/locales/all-locales",
+                    }]
+
+jobs:
+    l10n-bumper:
+        name: l10n_bumper
+        description: l10n changesets bumper
+        run-on-projects: []
+        treeherder:
+            kind: build
+            platform: linux64/opt
+            symbol: l10n-bump(lb)
+            tier: 1

taskcluster/docs/kinds.rst

@@ -616,3 +616,7 @@ by another task that produces a jobs.json artifact
 iris
 ----
 Iris testing suite
+
+l10n-bump
+---------
+Cron-driven tasks that bump l10n-changesets files in-tree, using data from the l10n dashboard.

taskcluster/taskgraph/target_tasks.py

@@ -698,6 +698,16 @@ def target_tasks_file_update(full_task_graph, parameters, graph_config):
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
+@_target_task('l10n_bump')
+def target_tasks_l10n_bump(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required to perform l10n bumping.
+    """
+    def filter(task):
+        # For now any task in the repo-update kind is ok
+        return task.kind in ['l10n-bump']
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
 @_target_task('cron_bouncer_check')
 def target_tasks_bouncer_check(full_task_graph, parameters, graph_config):
     """Select the set of tasks required to perform bouncer version verification.

taskcluster/taskgraph/transforms/release_version_bump.py

@@ -16,13 +16,19 @@ transforms = TransformSequence()
 @transforms.add
 def handle_keyed_by(config, tasks):
     """Resolve fields that can be keyed by platform, etc."""
-    fields = [
+    default_fields = [
         'worker.push',
         'worker.bump-files',
         'worker-type',
     ]
     for task in tasks:
+        fields = default_fields[:]
+        for additional_field in ("l10n-bump-info", "source-repo"):
+            if additional_field in task["worker"]:
+                fields.append('worker.{}'.format(additional_field))
         for field in fields:
             resolve_keyed_by(task, field, item_name=task['name'],
-                             project=config.params['project'])
+                             **{'project': config.params['project'],
+                                'release-type': config.params['release_type']}
+                             )
         yield task

taskcluster/taskgraph/transforms/task.py

@@ -38,7 +38,6 @@ from taskgraph.util.partners import get_partners_to_be_published
 from taskgraph.util.scriptworker import (
     BALROG_ACTIONS,
     get_release_config,
-    add_scope_prefix,
 )
 from taskgraph.util.signed_artifacts import get_signed_artifacts
 from taskgraph.util.workertypes import worker_type_implementation
@@ -1244,15 +1243,29 @@ def build_push_addons_payload(config, task, task_def):
     Optional('bump-files'): [basestring],
     Optional('repo-param-prefix'): basestring,
     Optional('dontbuild'): bool,
+    Optional('ignore-closed-tree'): bool,
     Required('force-dry-run', default=True): bool,
-    Required('push', default=False): bool
+    Required('push', default=False): bool,
+    Optional('source-repo'): basestring,
+    Optional('l10n-bump-info'): {
+        Required('name'): basestring,
+        Required('path'): basestring,
+        Required('version-path'): basestring,
+        Optional('revision-url'): basestring,
+        Optional('ignore-config'): object,
+        Required('platform-configs'): [{
+            Required('platforms'): [basestring],
+            Required('path'): basestring,
+            Optional('format'): basestring,
+        }],
+    },
 })
 def build_treescript_payload(config, task, task_def):
     worker = task['worker']
     release_config = get_release_config(config)
 
-    task_def['payload'] = {}
+    task_def['payload'] = {'actions': []}
-    task_def.setdefault('scopes', [])
+    actions = task_def['payload']['actions']
     if worker['tags']:
         tag_names = []
         product = task['shipping-product'].upper()
@@ -1271,7 +1284,7 @@ def build_treescript_payload(config, task, task_def):
             'revision': config.params['{}head_rev'.format(worker.get('repo-param-prefix', ''))],
         }
         task_def['payload']['tag_info'] = tag_info
-        task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:tagging'))
+        actions.append('tag')
 
     if worker['bump']:
         if not worker['bump-files']:
@@ -1281,10 +1294,17 @@ def build_treescript_payload(config, task, task_def):
         bump_info['next_version'] = release_config['next_version']
         bump_info['files'] = worker['bump-files']
         task_def['payload']['version_bump_info'] = bump_info
-        task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:version_bump'))
+        actions.append('version_bump')
+
+    if worker.get('l10n-bump-info'):
+        l10n_bump_info = {}
+        for k, v in worker['l10n-bump-info'].items():
+            l10n_bump_info[k.replace('-', '_')] = worker['l10n-bump-info'][k]
+        task_def['payload']['l10n_bump_info'] = [l10n_bump_info]
+        actions.append('l10n_bump')
 
     if worker['push']:
-        task_def['scopes'].append(add_scope_prefix(config, 'treescript:action:push'))
+        actions.append('push')
 
     if worker.get('force-dry-run'):
         task_def['payload']['dry_run'] = True
@@ -1292,6 +1312,12 @@ def build_treescript_payload(config, task, task_def):
     if worker.get('dontbuild'):
         task_def['payload']['dontbuild'] = True
 
+    if worker.get('ignore-closed-tree') is not None:
+        task_def['payload']['ignore_closed_tree'] = worker['ignore-closed-tree']
+
+    if worker.get('source-repo'):
+        task_def['payload']['source_repo'] = worker['source-repo']
+
 
 @payload_builder('invalid', schema={
     # an invalid task is one which should never actually be created; this is used in