Bug 1432817 - [in-tree relpro] Delete push-apk-breakpoint task r=aki

MozReview-Commit-ID: G5nvGfNIVNN

--HG--
extra : rebase_source : 4554c1b7ce44edd9f6cef3dd9e2990d7ff2e5b05

taskcluster/ci/push-apk-breakpoint/kind.yml

@@ -1,38 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-loader: taskgraph.loader.push_apk:loader
-
-transforms:
-    - taskgraph.transforms.push_apk_breakpoint:transforms
-    - taskgraph.transforms.release_notifications:transforms
-    - taskgraph.transforms.task:transforms
-
-kind-dependencies:
-    - build-signing
-
-jobs:
-    android-push-apk-breakpoint/opt:
-        description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store
-        attributes:
-            build_platform: android-nightly
-            nightly: true
-        shipping-phase: ship
-        shipping-product: fennec
-        worker-type:  # see transforms
-            by-project:
-                mozilla-central: aws-provisioner-v1/taskcluster-generic
-                mozilla-beta: null-provisioner/human-breakpoint
-                mozilla-release: null-provisioner/human-breakpoint
-                maple: aws-provisioner-v1/taskcluster-generic
-                default: invalid/invalid
-        worker:
-            implementation: push-apk-breakpoint
-        treeherder:
-            symbol: pub(Br)
-            platform: Android/opt
-            tier: 2
-            kind: other
-        run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
-        deadline-after: 5 days

taskcluster/ci/push-apk/kind.yml

@@ -12,7 +12,6 @@ transforms:
 kind-dependencies:
    - build-signing
    - google-play-strings
-   - push-apk-breakpoint
    - beetmover-checksums
 
 jobs:

taskcluster/docs/kinds.rst

@@ -222,17 +222,11 @@ google-play-strings
 Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/.
 Artifact is then used by push-apk.
 
-push-apk-breakpoint
--------------------
-Decides whether or not APKs should be published onto Google Play Store. Jobs of this
-kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
-in order to make the decision.
-
 push-apk
 --------
 PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
 all the signed multi-locales (aka "multi") APKs for a given release and upload them
-all at once. They also depend on the breakpoint.
+all at once.
 
 release-balrog-submit-toplevel
 ----------------------

taskcluster/taskgraph/loader/push_apk.py

@@ -21,6 +21,8 @@ def loader(kind, path, config, params, loaded_tasks):
             continue
 
         job['dependent-tasks'] = dependent_tasks
+        job['label'] = job['name']
+
         yield job
 
 

taskcluster/taskgraph/target_tasks.py

@@ -52,8 +52,7 @@ def filter_beta_release_tasks(task, parameters, ignore_kinds=None, allow_l10n=Fa
             'beetmover-repackage', 'beetmover-repackage-signing',
             'checksums-signing',
             'nightly-l10n', 'nightly-l10n-signing',
-            'push-apk', 'push-apk-breakpoint',
+            'push-apk', 'repackage-l10n',
-            'repackage-l10n',
         ]
     platform = task.attributes.get('build_platform')
     if platform in (
@@ -486,7 +485,7 @@ def target_tasks_promote_fennec(full_task_graph, parameters, graph_config):
             # are using run-on-projects properly here.
             if 'old-id' in task.label:
                 return False
-            if task.kind not in ('balrog', 'push-apk', 'push-apk-breakpoint'):
+            if task.kind not in ('balrog', 'push-apk'):
                 if task.attributes.get('nightly'):
                     return True
         if task.attributes.get('shipping_product') == 'fennec' and \
@@ -512,12 +511,9 @@ def target_tasks_ship_fennec(full_task_graph, parameters, graph_config):
         if task.attributes.get('shipping_product') != 'fennec' or \
                 task.attributes.get('shipping_phase') not in ('ship', 'push'):
             return False
-        # We always run push-apk* during ship
+        # We always run push-apk during ship
-        if task.kind in (
+        if task.kind == 'push-apk':
-            'push-apk',
+            return True
-            'push-apk-breakpoint',
-        ):
-                return True
         # secondary-notify-ship is only for RC
         if task.kind in (
             'release-secondary-notify-ship',

taskcluster/taskgraph/transforms/google_play_strings.py

@@ -7,12 +7,9 @@ Transform the push-apk kind into an actual task description.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import functools
-
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.task import task_description_schema
-from taskgraph.util.schema import resolve_keyed_by, Schema
+from taskgraph.util.schema import resolve_keyed_by, Schema, validate_schema
-from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial
 
 from voluptuous import Required
 
@@ -37,14 +34,16 @@ google_play_description_schema = Schema({
     Required('worker'): object,
 })
 
-validate_jobs_schema_transform = functools.partial(
-    validate_jobs_schema_transform_partial,
-    google_play_description_schema,
-    'GooglePlayStrings'
-)
 
-transforms.add(fill_labels_tranform)
+@transforms.add
-transforms.add(validate_jobs_schema_transform)
+def validate_jobs_schema(config, jobs):
+    for job in jobs:
+        job['label'] = job['name']
+        validate_schema(
+            google_play_description_schema, job,
+            "In GooglePlayStrings ({!r} kind) task for {!r}:".format(config.kind, job['label'])
+        )
+        yield job
 
 
 @transforms.add

taskcluster/taskgraph/transforms/push_apk.py

@@ -7,14 +7,12 @@ Transform the push-apk kind into an actual task description.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import functools
+import re
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.task import task_description_schema
-from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema, validate_schema
 from taskgraph.util.scriptworker import get_push_apk_scope
-from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
-    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
 
 from voluptuous import Optional, Required
 
@@ -45,15 +43,42 @@ push_apk_description_schema = Schema({
     Optional('extra'): task_description_schema['extra'],
 })
 
-validate_jobs_schema_transform = functools.partial(
-    validate_jobs_schema_transform_partial,
-    push_apk_description_schema,
-    'PushApk'
-)
 
-transforms.add(fill_labels_tranform)
+REQUIRED_ARCHITECTURES = {
-transforms.add(validate_jobs_schema_transform)
+    'android-x86-nightly',
-transforms.add(validate_dependent_tasks_transform)
+    'android-api-16-nightly',
+}
+PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
+
+
+@transforms.add
+def validate_jobs_schema_transform_partial(config, jobs):
+    for job in jobs:
+        label = job.get('label', '?no-label?')
+        validate_schema(
+            push_apk_description_schema, job,
+            "In PushApk ({!r} kind) task for {!r}:".format(config.kind, label)
+        )
+        yield job
+
+
+@transforms.add
+def validate_dependent_tasks(_, jobs):
+    for job in jobs:
+        check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
+        yield job
+
+
+def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
+    dep_platforms = set(t.attributes.get('build_platform') for t in dependent_tasks)
+    missed_architectures = REQUIRED_ARCHITECTURES - dep_platforms
+    if missed_architectures:
+        raise Exception('''One or many required architectures are missing.
+
+Required architectures: {}.
+Given dependencies: {}.
+'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
+        )
 
 
 @transforms.add
@@ -66,8 +91,6 @@ def make_task_description(config, jobs):
         if config.params['release_type'] == 'rc':
             job['worker']['google-play-track'] = job['worker']['rc-google-play-track']
             job['worker']['rollout-percentage'] = job['worker']['rc-rollout-percentage']
-        del(job['worker']['rc-google-play-track'])
-        del(job['worker']['rc-rollout-percentage'])
 
         resolve_keyed_by(
             job, 'worker.google-play-track', item_name=job['name'],
@@ -93,7 +116,16 @@ def make_task_description(config, jobs):
         yield job
 
 
-transforms.add(delete_non_required_fields_transform)
+def generate_dependencies(dependent_tasks):
+    # Because we depend on several tasks that have the same kind, we introduce the platform
+    dependencies = {}
+    for task in dependent_tasks:
+        platform_match = PLATFORM_REGEX.match(task.label)
+        # platform_match is None when the google-play-string task is given, for instance
+        task_kind = task.kind if platform_match is None else \
+            '{}-{}'.format(task.kind, platform_match.group(1))
+        dependencies[task_kind] = task.label
+    return dependencies
 
 
 def generate_upstream_artifacts(dependencies):
@@ -102,7 +134,7 @@ def generate_upstream_artifacts(dependencies):
         'taskType': 'signing',
         'paths': ['public/build/target.apk'],
     } for task_kind in dependencies.keys()
-      if task_kind not in ('push-apk-breakpoint', 'google-play-strings', 'beetmover-checksums')
+      if task_kind not in ('google-play-strings', 'beetmover-checksums')
     ]
 
     google_play_strings = [{
@@ -115,3 +147,15 @@ def generate_upstream_artifacts(dependencies):
     ]
 
     return apks + google_play_strings
+
+
+@transforms.add
+def delete_non_required_fields(_, jobs):
+    for job in jobs:
+        del job['name']
+        del job['dependent-tasks']
+
+        del(job['worker']['rc-google-play-track'])
+        del(job['worker']['rc-rollout-percentage'])
+
+        yield job

taskcluster/taskgraph/transforms/push_apk_breakpoint.py

@@ -1,77 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-Transform the push-apk-breakpoint kind into an actual task description.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import functools
-
-from taskgraph.transforms.base import TransformSequence
-from taskgraph.transforms.task import task_description_schema
-from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
-from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
-    validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
-from voluptuous import Required
-
-
-transforms = TransformSequence()
-
-# Voluptuous uses marker objects as dictionary *keys*, but they are not
-# comparable, so we cast all of the keys back to regular strings
-task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
-
-push_apk_breakpoint_description_schema = Schema({
-    # the dependent task (object) for this beetmover job, used to inform beetmover.
-    Required('dependent-tasks'): object,
-    Required('name'): basestring,
-    Required('label'): basestring,
-    Required('description'): basestring,
-    Required('job-from'): basestring,
-    Required('attributes'): object,
-    Required('worker-type'): optionally_keyed_by('project', basestring),
-    Required('worker'): object,
-    Required('treeherder'): object,
-    Required('run-on-projects'): list,
-    Required('deadline-after'): basestring,
-    Required('shipping-phase'): task_description_schema['shipping-phase'],
-    Required('shipping-product'): task_description_schema['shipping-product'],
-})
-
-validate_jobs_schema_transform = functools.partial(
-    validate_jobs_schema_transform_partial,
-    push_apk_breakpoint_description_schema,
-    'PushApkBreakpoint'
-)
-
-transforms.add(fill_labels_tranform)
-transforms.add(validate_jobs_schema_transform)
-transforms.add(validate_dependent_tasks_transform)
-
-
-@transforms.add
-def make_task_description(config, jobs):
-    for job in jobs:
-        job['dependencies'] = generate_dependencies(job['dependent-tasks'])
-
-        resolve_keyed_by(
-            job, 'worker-type', item_name=job['name'],
-            project=config.params['project']
-        )
-
-        job['worker']['payload'] = {} if 'human' in job['worker-type'] else {
-                'image': 'ubuntu:16.10',
-                'command': [
-                    '/bin/bash',
-                    '-c',
-                    'echo "Dummy task while while bug 1351664 is implemented"'
-                ],
-                'maxRunTime': 600,
-            }
-
-        yield job
-
-
-transforms.add(delete_non_required_fields_transform)

taskcluster/taskgraph/transforms/task.py

@@ -558,10 +558,6 @@ task_description_schema = Schema({
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
-    }, {
-        Required('implementation'): 'push-apk-breakpoint',
-        Required('payload'): object,
-
     }, {
         Required('implementation'): 'invalid',
         # an invalid task is one which should never actually be created; this is used in
@@ -1119,11 +1115,6 @@ def build_push_apk_payload(config, task, task_def):
         task_def['payload']['rollout_percentage'] = worker['rollout-percentage']
 
 
-@payload_builder('push-apk-breakpoint')
-def build_push_apk_breakpoint_payload(config, task, task_def):
-    task_def['payload'] = task['worker']['payload']
-
-
 @payload_builder('shipit')
 def build_ship_it_payload(config, task, task_def):
     worker = task['worker']

taskcluster/taskgraph/util/push_apk.py

@@ -1,72 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-Common functions for both push-apk and push-apk-breakpoint.
-"""
-from __future__ import absolute_import, print_function, unicode_literals
-
-import re
-
-from taskgraph.util.schema import validate_schema
-
-REQUIRED_ARCHITECTURES = {
-    'android-x86-nightly',
-    'android-api-16-nightly',
-}
-PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
-
-
-def fill_labels_tranform(_, jobs):
-    for job in jobs:
-        job['label'] = job['name']
-
-        yield job
-
-
-def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
-    for job in jobs:
-        label = job.get('label', '?no-label?')
-        validate_schema(
-            description_schema, job,
-            "In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
-        )
-        yield job
-
-
-def validate_dependent_tasks_transform(_, jobs):
-    for job in jobs:
-        check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
-        yield job
-
-
-def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
-    dep_platforms = set(t.attributes.get('build_platform') for t in dependent_tasks)
-    missed_architectures = REQUIRED_ARCHITECTURES - dep_platforms
-    if missed_architectures:
-        raise Exception('''One or many required architectures are missing.
-
-Required architectures: {}.
-Given dependencies: {}.
-'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
-        )
-
-
-def delete_non_required_fields_transform(_, jobs):
-    for job in jobs:
-        del job['name']
-        del job['dependent-tasks']
-
-        yield job
-
-
-def generate_dependencies(dependent_tasks):
-    # Because we depend on several tasks that have the same kind, we introduce the platform
-    dependencies = {}
-    for task in dependent_tasks:
-        platform_match = PLATFORM_REGEX.match(task.label)
-        # platform_match is None when the breakpoint task is given
-        task_kind = task.kind if platform_match is None else \
-            '{}-{}'.format(task.kind, platform_match.group(1))
-        dependencies[task_kind] = task.label
-    return dependencies

taskcluster/taskgraph/util/workertypes.py

@@ -40,7 +40,6 @@ WORKER_TYPES = {
     'buildbot-bridge/buildbot-bridge': ('buildbot-bridge', None),
     'invalid/invalid': ('invalid', None),
     'invalid/always-optimized': ('always-optimized', None),
-    'null-provisioner/human-breakpoint': ('push-apk-breakpoint', None),
     'releng-hardware/gecko-t-linux-talos': ('native-engine', 'linux'),
     'scriptworker-prov-v1/balrog-dev': ('balrog', None),
     'scriptworker-prov-v1/balrogworker-v1': ('balrog', None),