From 77a4f5d36b9fb70f5b076004bc7f0ab940208f70 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Thu, 22 Feb 2018 17:10:29 +0000 Subject: [PATCH] Bug 1432817 - [in-tree relpro] Delete push-apk-breakpoint task r=aki MozReview-Commit-ID: G5nvGfNIVNN --HG-- extra : rebase_source : 4554c1b7ce44edd9f6cef3dd9e2990d7ff2e5b05 --- taskcluster/ci/push-apk-breakpoint/kind.yml | 38 --------- taskcluster/ci/push-apk/kind.yml | 1 - taskcluster/docs/kinds.rst | 8 +- taskcluster/taskgraph/loader/push_apk.py | 2 + taskcluster/taskgraph/target_tasks.py | 14 ++-- .../transforms/google_play_strings.py | 21 +++-- taskcluster/taskgraph/transforms/push_apk.py | 76 ++++++++++++++---- .../transforms/push_apk_breakpoint.py | 77 ------------------- taskcluster/taskgraph/transforms/task.py | 9 --- taskcluster/taskgraph/util/push_apk.py | 72 ----------------- taskcluster/taskgraph/util/workertypes.py | 1 - 11 files changed, 78 insertions(+), 241 deletions(-) delete mode 100644 taskcluster/ci/push-apk-breakpoint/kind.yml delete mode 100644 taskcluster/taskgraph/transforms/push_apk_breakpoint.py delete mode 100644 taskcluster/taskgraph/util/push_apk.py diff --git a/taskcluster/ci/push-apk-breakpoint/kind.yml b/taskcluster/ci/push-apk-breakpoint/kind.yml deleted file mode 100644 index 7b24bdebb1af..000000000000 --- a/taskcluster/ci/push-apk-breakpoint/kind.yml +++ /dev/null @@ -1,38 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -loader: taskgraph.loader.push_apk:loader - -transforms: - - taskgraph.transforms.push_apk_breakpoint:transforms - - taskgraph.transforms.release_notifications:transforms - - taskgraph.transforms.task:transforms - -kind-dependencies: - - build-signing - -jobs: - android-push-apk-breakpoint/opt: - description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store - attributes: - build_platform: android-nightly - nightly: true - shipping-phase: ship - shipping-product: fennec - worker-type: # see transforms - by-project: - mozilla-central: aws-provisioner-v1/taskcluster-generic - mozilla-beta: null-provisioner/human-breakpoint - mozilla-release: null-provisioner/human-breakpoint - maple: aws-provisioner-v1/taskcluster-generic - default: invalid/invalid - worker: - implementation: push-apk-breakpoint - treeherder: - symbol: pub(Br) - platform: Android/opt - tier: 2 - kind: other - run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release'] - deadline-after: 5 days diff --git a/taskcluster/ci/push-apk/kind.yml b/taskcluster/ci/push-apk/kind.yml index 9d0de39c6d97..56d544a6a3d2 100644 --- a/taskcluster/ci/push-apk/kind.yml +++ b/taskcluster/ci/push-apk/kind.yml @@ -12,7 +12,6 @@ transforms: kind-dependencies: - build-signing - google-play-strings - - push-apk-breakpoint - beetmover-checksums jobs: diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index 99409a04adfa..be42634c3abe 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -222,17 +222,11 @@ google-play-strings Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/. Artifact is then used by push-apk. -push-apk-breakpoint -------------------- -Decides whether or not APKs should be published onto Google Play Store. Jobs of this -kind depend on all the signed multi-locales (aka "multi") APKs for a given release, -in order to make the decision. - push-apk -------- PushApk publishes Android packages onto Google Play Store. Jobs of this kind take all the signed multi-locales (aka "multi") APKs for a given release and upload them -all at once. They also depend on the breakpoint. +all at once. release-balrog-submit-toplevel ---------------------- diff --git a/taskcluster/taskgraph/loader/push_apk.py b/taskcluster/taskgraph/loader/push_apk.py index 070f493e6186..559e7ab5add9 100644 --- a/taskcluster/taskgraph/loader/push_apk.py +++ b/taskcluster/taskgraph/loader/push_apk.py @@ -21,6 +21,8 @@ def loader(kind, path, config, params, loaded_tasks): continue job['dependent-tasks'] = dependent_tasks + job['label'] = job['name'] + yield job diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index 6594140591ea..1884d08125e3 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -52,8 +52,7 @@ def filter_beta_release_tasks(task, parameters, ignore_kinds=None, allow_l10n=Fa 'beetmover-repackage', 'beetmover-repackage-signing', 'checksums-signing', 'nightly-l10n', 'nightly-l10n-signing', - 'push-apk', 'push-apk-breakpoint', - 'repackage-l10n', + 'push-apk', 'repackage-l10n', ] platform = task.attributes.get('build_platform') if platform in ( @@ -486,7 +485,7 @@ def target_tasks_promote_fennec(full_task_graph, parameters, graph_config): # are using run-on-projects properly here. if 'old-id' in task.label: return False - if task.kind not in ('balrog', 'push-apk', 'push-apk-breakpoint'): + if task.kind not in ('balrog', 'push-apk'): if task.attributes.get('nightly'): return True if task.attributes.get('shipping_product') == 'fennec' and \ @@ -512,12 +511,9 @@ def target_tasks_ship_fennec(full_task_graph, parameters, graph_config): if task.attributes.get('shipping_product') != 'fennec' or \ task.attributes.get('shipping_phase') not in ('ship', 'push'): return False - # We always run push-apk* during ship - if task.kind in ( - 'push-apk', - 'push-apk-breakpoint', - ): - return True + # We always run push-apk during ship + if task.kind == 'push-apk': + return True # secondary-notify-ship is only for RC if task.kind in ( 'release-secondary-notify-ship', diff --git a/taskcluster/taskgraph/transforms/google_play_strings.py b/taskcluster/taskgraph/transforms/google_play_strings.py index ba867dc885d3..ea10a18d4f4f 100644 --- a/taskcluster/taskgraph/transforms/google_play_strings.py +++ b/taskcluster/taskgraph/transforms/google_play_strings.py @@ -7,12 +7,9 @@ Transform the push-apk kind into an actual task description. from __future__ import absolute_import, print_function, unicode_literals -import functools - from taskgraph.transforms.base import TransformSequence from taskgraph.transforms.task import task_description_schema -from taskgraph.util.schema import resolve_keyed_by, Schema -from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial +from taskgraph.util.schema import resolve_keyed_by, Schema, validate_schema from voluptuous import Required @@ -37,14 +34,16 @@ google_play_description_schema = Schema({ Required('worker'): object, }) -validate_jobs_schema_transform = functools.partial( - validate_jobs_schema_transform_partial, - google_play_description_schema, - 'GooglePlayStrings' -) -transforms.add(fill_labels_tranform) -transforms.add(validate_jobs_schema_transform) +@transforms.add +def validate_jobs_schema(config, jobs): + for job in jobs: + job['label'] = job['name'] + validate_schema( + google_play_description_schema, job, + "In GooglePlayStrings ({!r} kind) task for {!r}:".format(config.kind, job['label']) + ) + yield job @transforms.add diff --git a/taskcluster/taskgraph/transforms/push_apk.py b/taskcluster/taskgraph/transforms/push_apk.py index d463dd31275f..186f43c69c67 100644 --- a/taskcluster/taskgraph/transforms/push_apk.py +++ b/taskcluster/taskgraph/transforms/push_apk.py @@ -7,14 +7,12 @@ Transform the push-apk kind into an actual task description. from __future__ import absolute_import, print_function, unicode_literals -import functools +import re from taskgraph.transforms.base import TransformSequence from taskgraph.transforms.task import task_description_schema -from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema +from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema, validate_schema from taskgraph.util.scriptworker import get_push_apk_scope -from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \ - validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies from voluptuous import Optional, Required @@ -45,15 +43,42 @@ push_apk_description_schema = Schema({ Optional('extra'): task_description_schema['extra'], }) -validate_jobs_schema_transform = functools.partial( - validate_jobs_schema_transform_partial, - push_apk_description_schema, - 'PushApk' -) -transforms.add(fill_labels_tranform) -transforms.add(validate_jobs_schema_transform) -transforms.add(validate_dependent_tasks_transform) +REQUIRED_ARCHITECTURES = { + 'android-x86-nightly', + 'android-api-16-nightly', +} +PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly') + + +@transforms.add +def validate_jobs_schema_transform_partial(config, jobs): + for job in jobs: + label = job.get('label', '?no-label?') + validate_schema( + push_apk_description_schema, job, + "In PushApk ({!r} kind) task for {!r}:".format(config.kind, label) + ) + yield job + + +@transforms.add +def validate_dependent_tasks(_, jobs): + for job in jobs: + check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks']) + yield job + + +def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks): + dep_platforms = set(t.attributes.get('build_platform') for t in dependent_tasks) + missed_architectures = REQUIRED_ARCHITECTURES - dep_platforms + if missed_architectures: + raise Exception('''One or many required architectures are missing. + +Required architectures: {}. +Given dependencies: {}. +'''.format(REQUIRED_ARCHITECTURES, dependent_tasks) + ) @transforms.add @@ -66,8 +91,6 @@ def make_task_description(config, jobs): if config.params['release_type'] == 'rc': job['worker']['google-play-track'] = job['worker']['rc-google-play-track'] job['worker']['rollout-percentage'] = job['worker']['rc-rollout-percentage'] - del(job['worker']['rc-google-play-track']) - del(job['worker']['rc-rollout-percentage']) resolve_keyed_by( job, 'worker.google-play-track', item_name=job['name'], @@ -93,7 +116,16 @@ def make_task_description(config, jobs): yield job -transforms.add(delete_non_required_fields_transform) +def generate_dependencies(dependent_tasks): + # Because we depend on several tasks that have the same kind, we introduce the platform + dependencies = {} + for task in dependent_tasks: + platform_match = PLATFORM_REGEX.match(task.label) + # platform_match is None when the google-play-string task is given, for instance + task_kind = task.kind if platform_match is None else \ + '{}-{}'.format(task.kind, platform_match.group(1)) + dependencies[task_kind] = task.label + return dependencies def generate_upstream_artifacts(dependencies): @@ -102,7 +134,7 @@ def generate_upstream_artifacts(dependencies): 'taskType': 'signing', 'paths': ['public/build/target.apk'], } for task_kind in dependencies.keys() - if task_kind not in ('push-apk-breakpoint', 'google-play-strings', 'beetmover-checksums') + if task_kind not in ('google-play-strings', 'beetmover-checksums') ] google_play_strings = [{ @@ -115,3 +147,15 @@ def generate_upstream_artifacts(dependencies): ] return apks + google_play_strings + + +@transforms.add +def delete_non_required_fields(_, jobs): + for job in jobs: + del job['name'] + del job['dependent-tasks'] + + del(job['worker']['rc-google-play-track']) + del(job['worker']['rc-rollout-percentage']) + + yield job diff --git a/taskcluster/taskgraph/transforms/push_apk_breakpoint.py b/taskcluster/taskgraph/transforms/push_apk_breakpoint.py deleted file mode 100644 index c4c70caa2680..000000000000 --- a/taskcluster/taskgraph/transforms/push_apk_breakpoint.py +++ /dev/null @@ -1,77 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Transform the push-apk-breakpoint kind into an actual task description. -""" - -from __future__ import absolute_import, print_function, unicode_literals - -import functools - -from taskgraph.transforms.base import TransformSequence -from taskgraph.transforms.task import task_description_schema -from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema -from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \ - validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies -from voluptuous import Required - - -transforms = TransformSequence() - -# Voluptuous uses marker objects as dictionary *keys*, but they are not -# comparable, so we cast all of the keys back to regular strings -task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} - -push_apk_breakpoint_description_schema = Schema({ - # the dependent task (object) for this beetmover job, used to inform beetmover. - Required('dependent-tasks'): object, - Required('name'): basestring, - Required('label'): basestring, - Required('description'): basestring, - Required('job-from'): basestring, - Required('attributes'): object, - Required('worker-type'): optionally_keyed_by('project', basestring), - Required('worker'): object, - Required('treeherder'): object, - Required('run-on-projects'): list, - Required('deadline-after'): basestring, - Required('shipping-phase'): task_description_schema['shipping-phase'], - Required('shipping-product'): task_description_schema['shipping-product'], -}) - -validate_jobs_schema_transform = functools.partial( - validate_jobs_schema_transform_partial, - push_apk_breakpoint_description_schema, - 'PushApkBreakpoint' -) - -transforms.add(fill_labels_tranform) -transforms.add(validate_jobs_schema_transform) -transforms.add(validate_dependent_tasks_transform) - - -@transforms.add -def make_task_description(config, jobs): - for job in jobs: - job['dependencies'] = generate_dependencies(job['dependent-tasks']) - - resolve_keyed_by( - job, 'worker-type', item_name=job['name'], - project=config.params['project'] - ) - - job['worker']['payload'] = {} if 'human' in job['worker-type'] else { - 'image': 'ubuntu:16.10', - 'command': [ - '/bin/bash', - '-c', - 'echo "Dummy task while while bug 1351664 is implemented"' - ], - 'maxRunTime': 600, - } - - yield job - - -transforms.add(delete_non_required_fields_transform) diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index 76a1491488c2..9bc660be2cab 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -558,10 +558,6 @@ task_description_schema = Schema({ # Paths to the artifacts to sign Required('paths'): [basestring], }], - }, { - Required('implementation'): 'push-apk-breakpoint', - Required('payload'): object, - }, { Required('implementation'): 'invalid', # an invalid task is one which should never actually be created; this is used in @@ -1119,11 +1115,6 @@ def build_push_apk_payload(config, task, task_def): task_def['payload']['rollout_percentage'] = worker['rollout-percentage'] -@payload_builder('push-apk-breakpoint') -def build_push_apk_breakpoint_payload(config, task, task_def): - task_def['payload'] = task['worker']['payload'] - - @payload_builder('shipit') def build_ship_it_payload(config, task, task_def): worker = task['worker'] diff --git a/taskcluster/taskgraph/util/push_apk.py b/taskcluster/taskgraph/util/push_apk.py deleted file mode 100644 index 92c876d768b1..000000000000 --- a/taskcluster/taskgraph/util/push_apk.py +++ /dev/null @@ -1,72 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Common functions for both push-apk and push-apk-breakpoint. -""" -from __future__ import absolute_import, print_function, unicode_literals - -import re - -from taskgraph.util.schema import validate_schema - -REQUIRED_ARCHITECTURES = { - 'android-x86-nightly', - 'android-api-16-nightly', -} -PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly') - - -def fill_labels_tranform(_, jobs): - for job in jobs: - job['label'] = job['name'] - - yield job - - -def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs): - for job in jobs: - label = job.get('label', '?no-label?') - validate_schema( - description_schema, job, - "In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label) - ) - yield job - - -def validate_dependent_tasks_transform(_, jobs): - for job in jobs: - check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks']) - yield job - - -def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks): - dep_platforms = set(t.attributes.get('build_platform') for t in dependent_tasks) - missed_architectures = REQUIRED_ARCHITECTURES - dep_platforms - if missed_architectures: - raise Exception('''One or many required architectures are missing. - -Required architectures: {}. -Given dependencies: {}. -'''.format(REQUIRED_ARCHITECTURES, dependent_tasks) - ) - - -def delete_non_required_fields_transform(_, jobs): - for job in jobs: - del job['name'] - del job['dependent-tasks'] - - yield job - - -def generate_dependencies(dependent_tasks): - # Because we depend on several tasks that have the same kind, we introduce the platform - dependencies = {} - for task in dependent_tasks: - platform_match = PLATFORM_REGEX.match(task.label) - # platform_match is None when the breakpoint task is given - task_kind = task.kind if platform_match is None else \ - '{}-{}'.format(task.kind, platform_match.group(1)) - dependencies[task_kind] = task.label - return dependencies diff --git a/taskcluster/taskgraph/util/workertypes.py b/taskcluster/taskgraph/util/workertypes.py index fa87b1c98072..4639d4e342de 100644 --- a/taskcluster/taskgraph/util/workertypes.py +++ b/taskcluster/taskgraph/util/workertypes.py @@ -40,7 +40,6 @@ WORKER_TYPES = { 'buildbot-bridge/buildbot-bridge': ('buildbot-bridge', None), 'invalid/invalid': ('invalid', None), 'invalid/always-optimized': ('always-optimized', None), - 'null-provisioner/human-breakpoint': ('push-apk-breakpoint', None), 'releng-hardware/gecko-t-linux-talos': ('native-engine', 'linux'), 'scriptworker-prov-v1/balrog-dev': ('balrog', None), 'scriptworker-prov-v1/balrogworker-v1': ('balrog', None),