Bug 1385401 - Fetch Google Play listings outside of the push-apk task r=aki

MozReview-Commit-ID: HVFVqjliqor

--HG--
extra : rebase_source : a1f4ac27254b42766661677132ccae4427a3c494

taskcluster/ci/docker-image/kind.yml

@@ -35,6 +35,8 @@ jobs:
     symbol: I(idx)
   funsize-update-generator:
     symbol: I(pg)
+  google-play-strings:
+    symbol: I(gps)
   funsize-balrog-submitter:
     symbol: I(fbs)
   beet-mover:

taskcluster/ci/google-play-strings/kind.yml

@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.google_play_strings:transforms
+   - taskgraph.transforms.task:transforms
+
+jobs:
+   google-play-strings:
+      description: Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/
+      attributes:
+         build_type: google_play_strings
+         build_platform: android-nightly
+         nightly: true
+      shipping-phase: promote
+      shipping-product: fennec
+      worker-type: aws-provisioner-v1/gecko-{level}-b-android
+      worker:
+         implementation: docker-worker
+         os: linux
+         docker-image: {in-tree: google-play-strings}
+         chain-of-trust: true
+         max-run-time: 600
+         artifacts:
+            - name: 'public/google_play_strings.json'
+              # XXX The folder depends on the one defined in the Dockerfile
+              path: /builds/worker/google_play_strings.json
+              type: 'file'
+         env:
+            # TODO Use the branch name instead of the android package name
+            PACKAGE_NAME:
+               by-project:
+                  mozilla-central: org.mozilla.fennec_aurora
+                  mozilla-beta: org.mozilla.firefox_beta
+                  mozilla-release: org.mozilla.firefox_beta
+                  default: org.mozilla.fennec_aurora  # Fetches strings for mozilla-central
+            # XXX The folder depends on the one defined in the Dockerfile
+            GOOGLE_PLAY_STRING_FILE: /builds/worker/google_play_strings.json
+         command:
+            - bash
+            - -cx
+            - >
+              python3 ./mozapkpublisher/get_l10n_strings.py
+              --package-name "${PACKAGE_NAME}"
+              --output-file "${GOOGLE_PLAY_STRING_FILE}"
+      treeherder:
+         symbol: pub(gps)
+         platform: Android/opt
+         tier: 2
+         kind: other
+      run-on-projects: ['maple', 'mozilla-central', 'mozilla-beta', 'mozilla-release']

taskcluster/ci/push-apk/kind.yml

@@ -5,39 +5,39 @@
 loader: taskgraph.loader.push_apk:loader
 
 transforms:
-    - taskgraph.transforms.push_apk:transforms
-    - taskgraph.transforms.task:transforms
+   - taskgraph.transforms.push_apk:transforms
+   - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-    - build-signing
-    - push-apk-breakpoint
+   - build-signing
+   - google-play-strings
+   - push-apk-breakpoint
 
 jobs:
-    push-apk/opt:
-        description: Publishes APK onto Google Play Store
-        attributes:
-            build_platform: android-nightly
-            nightly: true
-        shipping-phase: ship
-        shipping-product: fennec
-        worker-type:
-            by-project:
-                mozilla-central: scriptworker-prov-v1/pushapk-v1
-                mozilla-beta: scriptworker-prov-v1/pushapk-v1
-                mozilla-release: scriptworker-prov-v1/pushapk-v1
-                default: scriptworker-prov-v1/dep-pushapk
-        worker:
-            upstream-artifacts:  # see transforms
-            google-play-track:  # see transforms
-            implementation: push-apk
-            commit:  # see transforms
-        scopes:  # see transforms
-        treeherder:
-            symbol: pub(gp)
-            platform: Android/opt
-            tier: 2
-            kind: other
-        run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
-        deadline-after: 5 days
-        extra:
-            product: fennec
+   push-apk/opt:
+      description: Publishes APK onto Google Play Store
+      attributes:
+         build_platform: android-nightly
+         nightly: true
+      shipping-phase: ship
+      shipping-product: fennec
+      worker-type:
+         by-project:
+            mozilla-central: scriptworker-prov-v1/pushapk-v1
+            mozilla-beta: scriptworker-prov-v1/pushapk-v1
+            mozilla-release: scriptworker-prov-v1/pushapk-v1
+            default: scriptworker-prov-v1/dep-pushapk
+      worker:
+         upstream-artifacts:  # see transforms
+         google-play-track:  # see transforms
+         implementation: push-apk
+         commit:  # see transforms
+      requires: all-resolved
+      scopes:  # see transforms
+      treeherder:
+         symbol: pub(gp)
+         platform: Android/opt
+         tier: 2
+         kind: other
+      run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release', 'maple']
+      deadline-after: 5 days

taskcluster/docker/google-play-strings/Dockerfile

@@ -0,0 +1,19 @@
+FROM          ubuntu:16.04
+MAINTAINER    Johan Lorenzo <jlorenzo+tc@mozilla.com>
+
+RUN mkdir /builds
+RUN groupadd -g 500 worker
+RUN useradd -u 500 -g 500 -d /builds/worker -s /bin/bash -m worker
+
+RUN apt-get update
+RUN apt-get install --yes git python3-setuptools build-essential libssl-dev libffi-dev python3-dev
+
+WORKDIR /builds/worker/
+RUN git clone https://github.com/mozilla-releng/mozapkpublisher
+WORKDIR /builds/worker/mozapkpublisher
+RUN python3 setup.py develop
+
+RUN chown -R worker:worker /builds/worker
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]

taskcluster/docs/kinds.rst

@@ -217,6 +217,11 @@ Beetmover, takes specific artifact checksums and pushes it to a location outside
 of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
 process determines the final location and "pretty" names it (version product name)
 
+google-play-strings
+-------------------
+Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/.
+Artifact is then used by push-apk.
+
 push-apk-breakpoint
 -------------------
 Decides whether or not APKs should be published onto Google Play Store. Jobs of this

taskcluster/taskgraph/transforms/google_play_strings.py

@@ -0,0 +1,64 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.task import task_description_schema
+from taskgraph.util.schema import resolve_keyed_by, Schema
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial
+
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+google_play_description_schema = Schema({
+    Required('name'): basestring,
+    Required('label'): task_description_schema['label'],
+    Required('description'): task_description_schema['description'],
+    Required('job-from'): task_description_schema['job-from'],
+    Required('attributes'): task_description_schema['attributes'],
+    Required('treeherder'): task_description_schema['treeherder'],
+    Required('run-on-projects'): task_description_schema['run-on-projects'],
+    Required('shipping-phase'): task_description_schema['shipping-phase'],
+    Required('shipping-product'): task_description_schema['shipping-product'],
+    Required('worker-type'): task_description_schema['worker-type'],
+    Required('worker'): object,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    google_play_description_schema,
+    'GooglePlayStrings'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+
+
+@transforms.add
+def set_worker_data(config, jobs):
+    for job in jobs:
+        worker = job['worker']
+
+        env = worker.setdefault('env', {})
+        resolve_keyed_by(
+            env, 'PACKAGE_NAME', item_name=job['name'],
+            project=config.params['project']
+        )
+
+        cot = job.setdefault('extra', {}).setdefault('chainOfTrust', {})
+        cot.setdefault('inputs', {})['docker-image'] = {'task-reference': '<docker-image>'}
+
+        yield job

taskcluster/taskgraph/transforms/push_apk.py

@@ -28,22 +28,22 @@ task_description_schema = {str(k): v for k, v in task_description_schema.schema.
 
 
 push_apk_description_schema = Schema({
-    # the dependent task (object) for this beetmover job, used to inform beetmover.
     Required('dependent-tasks'): object,
     Required('name'): basestring,
-    Required('label'): basestring,
-    Required('description'): basestring,
-    Required('job-from'): basestring,
-    Required('attributes'): object,
-    Required('treeherder'): object,
-    Required('run-on-projects'): list,
+    Required('label'): task_description_schema['label'],
+    Required('description'): task_description_schema['description'],
+    Required('job-from'): task_description_schema['job-from'],
+    Required('attributes'): task_description_schema['attributes'],
+    Required('treeherder'): task_description_schema['treeherder'],
+    Required('run-on-projects'): task_description_schema['run-on-projects'],
     Required('worker-type'): optionally_keyed_by('project', basestring),
     Required('worker'): object,
     Required('scopes'): None,
+    Required('requires'): task_description_schema['requires'],
     Required('deadline-after'): basestring,
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     Required('shipping-product'): task_description_schema['shipping-product'],
-    Optional('extra'): object,
+    Optional('extra'): task_description_schema['extra'],
 })
 
 validate_jobs_schema_transform = functools.partial(
@@ -83,8 +83,21 @@ transforms.add(delete_non_required_fields_transform)
 
 
 def generate_upstream_artifacts(dependencies):
-    return [{
+    apks = [{
         'taskId': {'task-reference': '<{}>'.format(task_kind)},
         'taskType': 'signing',
         'paths': ['public/build/target.apk'],
-    } for task_kind in dependencies.keys() if 'breakpoint' not in task_kind]
+    } for task_kind in dependencies.keys()
+      if task_kind not in ('push-apk-breakpoint', 'google-play-strings')
+    ]
+
+    google_play_strings = [{
+        'taskId': {'task-reference': '<{}>'.format(task_kind)},
+        'taskType': 'build',
+        'paths': ['public/google_play_strings.json'],
+        'optional': True,
+    } for task_kind in dependencies.keys()
+      if 'google-play-strings' in task_kind
+    ]
+
+    return apks + google_play_strings

taskcluster/taskgraph/transforms/task.py

@@ -93,6 +93,8 @@ task_description_schema = Schema({
     # method.
     Optional('dependencies'): {basestring: object},
 
+    Optional('requires'): Any('all-completed', 'all-resolved'),
+
     # expiration and deadline times, relative to task creation, with units
     # (e.g., "14 days").  Defaults are set based on the project.
     Optional('expires-after'): basestring,
@@ -571,6 +573,9 @@ task_description_schema = Schema({
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
+
+            # Artifact is optional to run the task
+            Optional('optional', default=False): bool,
         }],
 
         # "Invalid" is a noop for try and other non-supported branches
@@ -1471,6 +1476,9 @@ def build_task(config, tasks):
             'priority': task['priority'],
         }
 
+        if task.get('requires', None):
+            task_def['requires'] = task['requires']
+
         if task_th:
             # link back to treeherder in description
             th_push_link = 'https://treeherder.mozilla.org/#/jobs?repo={}&revision={}'.format(