Bug 1317783 - Put PushApk tasks in-tree r=aki

MozReview-Commit-ID: 8uGIuj7OXwZ

--HG--
extra : rebase_source : d9ff99b54a7cffb258cdfe390ab9f733cdffb4a3

taskcluster/ci/push-apk-breakpoint/kind.yml

@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+   - taskgraph.transforms.push_apk_breakpoint:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+
+jobs:
+    android-push-apk-breakpoint/opt:
+        description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store
+        attributes:
+            build_platform: android-nightly
+            nightly: true
+        worker-type: # see transforms
+        worker:
+            implementation: push-apk-breakpoint
+        treeherder:
+            symbol: pub(Br)
+            platform: Android/opt
+            tier: 2
+            kind: other
+        run-on-projects:
+            - mozilla-aurora
+            - mozilla-beta
+            - mozilla-release
+        deadline-after: 5 days

taskcluster/ci/push-apk/kind.yml

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.push_apk:loader
+
+transforms:
+   - taskgraph.transforms.push_apk:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build-signing
+  - push-apk-breakpoint
+
+jobs:
+    push-apk/opt:
+        description: Publishes APK onto Google Play Store
+        attributes:
+            build_platform: android-nightly
+            nightly: true
+        worker-type: scriptworker-prov-v1/pushapk-v1
+        worker:
+            upstream-artifacts: # see transforms
+            google-play-track: # see transforms
+            implementation: push-apk
+            # TODO unhardcode that line
+            dry-run: true
+        scopes: # see transforms
+        treeherder:
+            symbol: pub(gp)
+            platform: Android/opt
+            tier: 2
+            kind: other
+        run-on-projects:
+            - mozilla-aurora
+            - mozilla-beta
+            - mozilla-release
+        deadline-after: 5 days

taskcluster/docs/kinds.rst

@@ -195,3 +195,15 @@ beetmover-checksums
 Beetmover, takes specific artifact checksums and pushes it to a location outside
 of Taskcluster's task artifacts (archive.mozilla.org as one place) and in the
 process determines the final location and "pretty" names it (version product name)
+
+push-apk-breakpoint
+-------------------
+Decides whether or not APKs should be published onto Google Play Store. Jobs of this
+kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
+in order to make the decision.
+
+push-apk
+--------
+PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
+all the signed multi-locales (aka "multi") APKs for a given release and upload them
+all at once. They also depend on the breakpoint.

taskcluster/taskgraph/loader/push_apk.py

@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from .transform import loader as base_loader
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Generate inputs implementing PushApk jobs. These depend on signed multi-locales nightly builds.
+    """
+    jobs = base_loader(kind, path, config, params, loaded_tasks)
+
+    for job in jobs:
+        job['dependent-tasks'] = get_dependent_loaded_tasks(config, loaded_tasks)
+        yield job
+
+
+def get_dependent_loaded_tasks(config, loaded_tasks):
+    nightly_tasks = (
+        task for task in loaded_tasks if task.attributes.get('nightly')
+    )
+    tasks_with_matching_kind = (
+        task for task in nightly_tasks if task.kind in config.get('kind-dependencies')
+    )
+    android_tasks = [
+        task for task in tasks_with_matching_kind if 'android' in task.label
+    ]
+
+    return android_tasks

taskcluster/taskgraph/target_tasks.py

@@ -184,13 +184,13 @@ def target_tasks_code_coverage(full_task_graph, parameters):
 
 
 @_target_task('nightly_fennec')
-def target_tasks_nightly(full_task_graph, parameters):
+def target_tasks_nightly_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a nightly build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     def filter(task):
         platform = task.attributes.get('build_platform')
-        if platform in ('android-api-15-nightly', 'android-x86-nightly'):
+        if platform in ('android-api-15-nightly', 'android-x86-nightly', 'android-nightly'):
             return task.attributes.get('nightly', False)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
@@ -228,6 +228,7 @@ def target_tasks_mozilla_beta(full_task_graph, parameters):
         if task.kind in [
             'balrog', 'beetmover', 'beetmover-checksums', 'beetmover-l10n',
             'checksums-signing', 'nightly-l10n', 'nightly-l10n-signing',
+            'push-apk', 'push-apk-breakpoint',
         ]:
             return False
         return True
@@ -248,7 +249,7 @@ def target_tasks_candidates_fennec(full_task_graph, parameters):
     """Select the set of tasks required for a candidates build of fennec. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
-    filtered_for_project = target_tasks_nightly(full_task_graph, parameters)
+    filtered_for_project = target_tasks_nightly_fennec(full_task_graph, parameters)
 
     def filter(task):
         if task.kind not in ['balrog']:

taskcluster/taskgraph/transforms/push_apk.py

@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.scriptworker import get_push_apk_scope, get_push_apk_track
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+push_apk_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('attributes'): object,
+    Required('treeherder'): object,
+    Required('run-on-projects'): list,
+    Required('worker-type'): basestring,
+    Required('worker'): object,
+    Required('scopes'): None,
+    Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    push_apk_description_schema,
+    'PushApk'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+        job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
+        job['worker']['google-play-track'] = get_push_apk_track(config)
+        job['scopes'] = [get_push_apk_scope(config)]
+
+        yield job
+
+
+transforms.add(delete_non_required_fields_transform)
+
+
+def generate_upstream_artifacts(dependencies):
+    return [{
+        'taskId': {'task-reference': '<{}>'.format(task_kind)},
+        'taskType': 'signing',
+        'paths': ['public/build/target.apk'],
+    } for task_kind in dependencies.keys() if 'breakpoint' not in task_kind]

taskcluster/taskgraph/transforms/push_apk_breakpoint.py

@@ -0,0 +1,68 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the push-apk-breakpoint kind into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import functools
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.scriptworker import get_push_apk_breakpoint_worker_type
+from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
+    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
+from voluptuous import Required
+
+
+transforms = TransformSequence()
+
+push_apk_breakpoint_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-tasks'): object,
+    Required('name'): basestring,
+    Required('label'): basestring,
+    Required('description'): basestring,
+    Required('attributes'): object,
+    Required('worker-type'): None,
+    Required('worker'): object,
+    Required('treeherder'): object,
+    Required('run-on-projects'): list,
+    Required('deadline-after'): basestring,
+})
+
+validate_jobs_schema_transform = functools.partial(
+    validate_jobs_schema_transform_partial,
+    push_apk_breakpoint_description_schema,
+    'PushApkBreakpoint'
+)
+
+transforms.add(fill_labels_tranform)
+transforms.add(validate_jobs_schema_transform)
+transforms.add(validate_dependent_tasks_transform)
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
+
+        worker_type = get_push_apk_breakpoint_worker_type(config)
+        job['worker-type'] = worker_type
+
+        job['worker']['payload'] = {} if 'human' in worker_type else {
+                'image': 'ubuntu:16.10',
+                'command': [
+                    '/bin/bash',
+                    '-c',
+                    'echo "Dummy task while while bug 1351664 is implemented"'
+                ],
+                'maxRunTime': 600,
+            }
+
+        yield job
+
+
+transforms.add(delete_non_required_fields_transform)

taskcluster/taskgraph/transforms/task.py

@@ -344,6 +344,28 @@ task_description_schema = Schema({
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
+    }, {
+        Required('implementation'): 'push-apk-breakpoint',
+        Required('payload'): object,
+
+    }, {
+        Required('implementation'): 'push-apk',
+
+        # list of artifact URLs for the artifacts that should be beetmoved
+        Required('upstream-artifacts'): [{
+            # taskId of the task with the artifact
+            Required('taskId'): taskref_or_string,
+
+            # type of signing task (for CoT)
+            Required('taskType'): basestring,
+
+            # Paths to the artifacts to sign
+            Required('paths'): [basestring],
+        }],
+
+        # "Invalid" is a noop for try and other non-supported branches
+        Required('google-play-track'): Any('production', 'beta', 'alpha', 'invalid'),
+        Required('dry-run', default=True): bool,
     }),
 })
 
@@ -381,6 +403,7 @@ GROUP_NAMES = {
     'TW32': 'Toolchain builds for Windows 32-bits',
     'TW64': 'Toolchain builds for Windows 64-bits',
     'SM-tc': 'Spidermonkey builds',
+    'pub': 'APK publishing',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
@@ -597,6 +620,22 @@ def build_balrog_payload(config, task, task_def):
     }
 
 
+@payload_builder('push-apk')
+def build_push_apk_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'dry_run': worker['dry-run'],
+        'upstreamArtifacts':  worker['upstream-artifacts'],
+        'google_play_track': worker['google-play-track'],
+    }
+
+
+@payload_builder('push-apk-breakpoint')
+def build_push_apk_breakpoint_payload(config, task, task_def):
+    task_def['payload'] = task['worker']['payload']
+
+
 @payload_builder('native-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']

taskcluster/taskgraph/util/push_apk.py

@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Common functions for both push-apk and push-apk-breakpoint.
+"""
+
+import re
+
+from taskgraph.util.schema import validate_schema
+
+REQUIRED_ARCHITECTURES = ('android-x86', 'android-api-15')
+PLATFORM_REGEX = re.compile(r'signing-android-(\S+)-nightly')
+
+
+def fill_labels_tranform(_, jobs):
+    for job in jobs:
+        job['label'] = job['name']
+
+        yield job
+
+
+def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
+    for job in jobs:
+        label = job.get('label', '?no-label?')
+        yield validate_schema(
+            description_schema, job,
+            "In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
+        )
+
+
+def validate_dependent_tasks_transform(_, jobs):
+    for job in jobs:
+        check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
+        yield job
+
+
+def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
+    dependencies_labels = [task.label for task in dependent_tasks]
+
+    is_this_required_architecture_present = {
+        architecture: any(architecture in label for label in dependencies_labels)
+        for architecture in REQUIRED_ARCHITECTURES
+    }
+    are_all_required_achitectures_present = all(is_this_required_architecture_present.values())
+
+    if not are_all_required_achitectures_present:
+        raise Exception('''One or many required architectures are missing.
+
+Required architectures: {}.
+Given dependencies: {}.
+'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
+        )
+
+
+def delete_non_required_fields_transform(_, jobs):
+    for job in jobs:
+        del job['name']
+        del job['dependent-tasks']
+
+        yield job
+
+
+def generate_dependencies(dependent_tasks):
+    # Because we depend on several tasks that have the same kind, we introduce the platform
+    dependencies = {}
+    for task in dependent_tasks:
+        platform_match = PLATFORM_REGEX.match(task.label)
+        # platform_match is None when the breakpoint task is given
+        task_kind = task.kind if platform_match is None else \
+            '{}-{}'.format(task.kind, platform_match.group(1))
+        dependencies[task_kind] = task.label
+    return dependencies

taskcluster/taskgraph/util/scriptworker.py

@@ -144,6 +144,44 @@ BALROG_SERVER_SCOPES = {
 }
 
 
+PUSH_APK_SCOPE_ALIAS_TO_PROJECT = [[
+    'aurora', set([
+        'mozilla-aurora',
+    ])
+], [
+    'beta', set([
+        'mozilla-beta',
+    ])
+], [
+    'release', set([
+        'mozilla-release',
+    ])
+]]
+
+
+PUSH_APK_SCOPES = {
+    'aurora': 'project:releng:googleplay:aurora',
+    'beta': 'project:releng:googleplay:beta',
+    'release': 'project:releng:googleplay:release',
+    'default': 'project:releng:googleplay:invalid',
+}
+
+# See https://github.com/mozilla-releng/pushapkscript#aurora-beta-release-vs-alpha-beta-production
+PUSH_APK_GOOGLE_PLAY_TRACT = {
+    'aurora': 'beta',
+    'beta': 'production',
+    'release': 'production',
+    'default': 'invalid',
+}
+
+PUSH_APK_BREAKPOINT_WORKER_TYPE = {
+    'aurora': 'aws-provisioner-v1/taskcluster-generic',
+    'beta': 'null-provisioner/human-breakpoint',
+    'release': 'null-provisioner/human-breakpoint',
+    'default': 'invalid/invalid',
+}
+
+
 # scope functions {{{1
 def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
     """Determine the restricted scope from `config.params['project']`.
@@ -236,6 +274,24 @@ get_balrog_server_scope = functools.partial(
     BALROG_SERVER_SCOPES
 )
 
+get_push_apk_scope = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_SCOPES
+)
+
+get_push_apk_track = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_GOOGLE_PLAY_TRACT
+)
+
+get_push_apk_breakpoint_worker_type = functools.partial(
+    get_scope_from_project,
+    PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
+    PUSH_APK_BREAKPOINT_WORKER_TYPE
+)
+
 
 # release_config {{{1
 def get_release_config(config):