mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1432817 - [in-tree relpro] Delete push-apk-breakpoint task r=aki 

MozReview-Commit-ID: G5nvGfNIVNN

--HG--
extra : rebase_source : 4554c1b7ce44edd9f6cef3dd9e2990d7ff2e5b05
This commit is contained in:
Johan Lorenzo 2018-02-22 17:10:29 +00:00
Родитель f603468239
Коммит 77a4f5d36b
11 изменённых файлов: 78 добавлений и 241 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

38
taskcluster/ci/push-apk-breakpoint/kind.yml
Просмотреть файл

@ -1,38 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.push_apk:loader
transforms:
- taskgraph.transforms.push_apk_breakpoint:transforms
- taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build-signing
jobs:
android-push-apk-breakpoint/opt:
description: PushApk breakpoint. Decides whether APK should be published onto Google Play Store
attributes:
build_platform: android-nightly
nightly: true
shipping-phase: ship
shipping-product: fennec
worker-type: # see transforms
by-project:
mozilla-central: aws-provisioner-v1/taskcluster-generic
mozilla-beta: null-provisioner/human-breakpoint
mozilla-release: null-provisioner/human-breakpoint
maple: aws-provisioner-v1/taskcluster-generic
default: invalid/invalid
worker:
implementation: push-apk-breakpoint
treeherder:
symbol: pub(Br)
platform: Android/opt
tier: 2
kind: other
run-on-projects: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
deadline-after: 5 days

1
taskcluster/ci/push-apk/kind.yml
Просмотреть файл

@ -12,7 +12,6 @@ transforms:
kind-dependencies:
- build-signing
- google-play-strings
- push-apk-breakpoint
- beetmover-checksums
jobs:

8
taskcluster/docs/kinds.rst
Просмотреть файл

@ -222,17 +222,11 @@ google-play-strings
Download strings to display on Google Play from https://l10n.mozilla-community.org/stores_l10n/.
Artifact is then used by push-apk.
push-apk-breakpoint
-------------------
Decides whether or not APKs should be published onto Google Play Store. Jobs of this
kind depend on all the signed multi-locales (aka "multi") APKs for a given release,
in order to make the decision.
push-apk
--------
PushApk publishes Android packages onto Google Play Store. Jobs of this kind take
all the signed multi-locales (aka "multi") APKs for a given release and upload them
all at once. They also depend on the breakpoint.
all at once.
release-balrog-submit-toplevel
----------------------

2
taskcluster/taskgraph/loader/push_apk.py
Просмотреть файл

@ -21,6 +21,8 @@ def loader(kind, path, config, params, loaded_tasks):
continue
job['dependent-tasks'] = dependent_tasks
job['label'] = job['name']
yield job

14
taskcluster/taskgraph/target_tasks.py
Просмотреть файл

@ -52,8 +52,7 @@ def filter_beta_release_tasks(task, parameters, ignore_kinds=None, allow_l10n=Fa
'beetmover-repackage', 'beetmover-repackage-signing',
'checksums-signing',
'nightly-l10n', 'nightly-l10n-signing',
'push-apk', 'push-apk-breakpoint',
'repackage-l10n',
'push-apk', 'repackage-l10n',
]
platform = task.attributes.get('build_platform')
if platform in (
@ -486,7 +485,7 @@ def target_tasks_promote_fennec(full_task_graph, parameters, graph_config):
# are using run-on-projects properly here.
if 'old-id' in task.label:
return False
if task.kind not in ('balrog', 'push-apk', 'push-apk-breakpoint'):
if task.kind not in ('balrog', 'push-apk'):
if task.attributes.get('nightly'):
return True
if task.attributes.get('shipping_product') == 'fennec' and \
@ -512,12 +511,9 @@ def target_tasks_ship_fennec(full_task_graph, parameters, graph_config):
if task.attributes.get('shipping_product') != 'fennec' or \
task.attributes.get('shipping_phase') not in ('ship', 'push'):
return False
# We always run push-apk* during ship
if task.kind in (
'push-apk',
'push-apk-breakpoint',
):
return True
# We always run push-apk during ship
if task.kind == 'push-apk':
return True
# secondary-notify-ship is only for RC
if task.kind in (
'release-secondary-notify-ship',

21
taskcluster/taskgraph/transforms/google_play_strings.py
Просмотреть файл

@ -7,12 +7,9 @@ Transform the push-apk kind into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
import functools
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.schema import resolve_keyed_by, Schema
from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial
from taskgraph.util.schema import resolve_keyed_by, Schema, validate_schema
from voluptuous import Required
@ -37,14 +34,16 @@ google_play_description_schema = Schema({
Required('worker'): object,
})
validate_jobs_schema_transform = functools.partial(
validate_jobs_schema_transform_partial,
google_play_description_schema,
'GooglePlayStrings'
)
transforms.add(fill_labels_tranform)
transforms.add(validate_jobs_schema_transform)
@transforms.add
def validate_jobs_schema(config, jobs):
for job in jobs:
job['label'] = job['name']
validate_schema(
google_play_description_schema, job,
"In GooglePlayStrings ({!r} kind) task for {!r}:".format(config.kind, job['label'])
)
yield job
@transforms.add

76
taskcluster/taskgraph/transforms/push_apk.py
Просмотреть файл

@ -7,14 +7,12 @@ Transform the push-apk kind into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
import functools
import re
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema, validate_schema
from taskgraph.util.scriptworker import get_push_apk_scope
from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
from voluptuous import Optional, Required
@ -45,15 +43,42 @@ push_apk_description_schema = Schema({
Optional('extra'): task_description_schema['extra'],
})
validate_jobs_schema_transform = functools.partial(
validate_jobs_schema_transform_partial,
push_apk_description_schema,
'PushApk'
)
transforms.add(fill_labels_tranform)
transforms.add(validate_jobs_schema_transform)
transforms.add(validate_dependent_tasks_transform)
REQUIRED_ARCHITECTURES = {
'android-x86-nightly',
'android-api-16-nightly',
}
PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
@transforms.add
def validate_jobs_schema_transform_partial(config, jobs):
for job in jobs:
label = job.get('label', '?no-label?')
validate_schema(
push_apk_description_schema, job,
"In PushApk ({!r} kind) task for {!r}:".format(config.kind, label)
)
yield job
@transforms.add
def validate_dependent_tasks(_, jobs):
for job in jobs:
check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
yield job
def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
dep_platforms = set(t.attributes.get('build_platform') for t in dependent_tasks)
missed_architectures = REQUIRED_ARCHITECTURES - dep_platforms
if missed_architectures:
raise Exception('''One or many required architectures are missing.
Required architectures: {}.
Given dependencies: {}.
'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
)
@transforms.add
@ -66,8 +91,6 @@ def make_task_description(config, jobs):
if config.params['release_type'] == 'rc':
job['worker']['google-play-track'] = job['worker']['rc-google-play-track']
job['worker']['rollout-percentage'] = job['worker']['rc-rollout-percentage']
del(job['worker']['rc-google-play-track'])
del(job['worker']['rc-rollout-percentage'])
resolve_keyed_by(
job, 'worker.google-play-track', item_name=job['name'],
@ -93,7 +116,16 @@ def make_task_description(config, jobs):
yield job
transforms.add(delete_non_required_fields_transform)
def generate_dependencies(dependent_tasks):
# Because we depend on several tasks that have the same kind, we introduce the platform
dependencies = {}
for task in dependent_tasks:
platform_match = PLATFORM_REGEX.match(task.label)
# platform_match is None when the google-play-string task is given, for instance
task_kind = task.kind if platform_match is None else \
'{}-{}'.format(task.kind, platform_match.group(1))
dependencies[task_kind] = task.label
return dependencies
def generate_upstream_artifacts(dependencies):
@ -102,7 +134,7 @@ def generate_upstream_artifacts(dependencies):
'taskType': 'signing',
'paths': ['public/build/target.apk'],
} for task_kind in dependencies.keys()
if task_kind not in ('push-apk-breakpoint', 'google-play-strings', 'beetmover-checksums')
if task_kind not in ('google-play-strings', 'beetmover-checksums')
]
google_play_strings = [{
@ -115,3 +147,15 @@ def generate_upstream_artifacts(dependencies):
]
return apks + google_play_strings
@transforms.add
def delete_non_required_fields(_, jobs):
for job in jobs:
del job['name']
del job['dependent-tasks']
del(job['worker']['rc-google-play-track'])
del(job['worker']['rc-rollout-percentage'])
yield job

77
taskcluster/taskgraph/transforms/push_apk_breakpoint.py
Просмотреть файл

@ -1,77 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the push-apk-breakpoint kind into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
import functools
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
from voluptuous import Required
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
push_apk_breakpoint_description_schema = Schema({
# the dependent task (object) for this beetmover job, used to inform beetmover.
Required('dependent-tasks'): object,
Required('name'): basestring,
Required('label'): basestring,
Required('description'): basestring,
Required('job-from'): basestring,
Required('attributes'): object,
Required('worker-type'): optionally_keyed_by('project', basestring),
Required('worker'): object,
Required('treeherder'): object,
Required('run-on-projects'): list,
Required('deadline-after'): basestring,
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
})
validate_jobs_schema_transform = functools.partial(
validate_jobs_schema_transform_partial,
push_apk_breakpoint_description_schema,
'PushApkBreakpoint'
)
transforms.add(fill_labels_tranform)
transforms.add(validate_jobs_schema_transform)
transforms.add(validate_dependent_tasks_transform)
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
job['dependencies'] = generate_dependencies(job['dependent-tasks'])
resolve_keyed_by(
job, 'worker-type', item_name=job['name'],
project=config.params['project']
)
job['worker']['payload'] = {} if 'human' in job['worker-type'] else {
'image': 'ubuntu:16.10',
'command': [
'/bin/bash',
'-c',
'echo "Dummy task while while bug 1351664 is implemented"'
],
'maxRunTime': 600,
}
yield job
transforms.add(delete_non_required_fields_transform)

9
taskcluster/taskgraph/transforms/task.py
Просмотреть файл

@ -558,10 +558,6 @@ task_description_schema = Schema({
# Paths to the artifacts to sign
Required('paths'): [basestring],
}],
}, {
Required('implementation'): 'push-apk-breakpoint',
Required('payload'): object,
}, {
Required('implementation'): 'invalid',
# an invalid task is one which should never actually be created; this is used in
@ -1119,11 +1115,6 @@ def build_push_apk_payload(config, task, task_def):
task_def['payload']['rollout_percentage'] = worker['rollout-percentage']
@payload_builder('push-apk-breakpoint')
def build_push_apk_breakpoint_payload(config, task, task_def):
task_def['payload'] = task['worker']['payload']
@payload_builder('shipit')
def build_ship_it_payload(config, task, task_def):
worker = task['worker']

72
taskcluster/taskgraph/util/push_apk.py
Просмотреть файл

@ -1,72 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Common functions for both push-apk and push-apk-breakpoint.
"""
from __future__ import absolute_import, print_function, unicode_literals
import re
from taskgraph.util.schema import validate_schema
REQUIRED_ARCHITECTURES = {
'android-x86-nightly',
'android-api-16-nightly',
}
PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
def fill_labels_tranform(_, jobs):
for job in jobs:
job['label'] = job['name']
yield job
def validate_jobs_schema_transform_partial(description_schema, transform_type, config, jobs):
for job in jobs:
label = job.get('label', '?no-label?')
validate_schema(
description_schema, job,
"In {} ({!r} kind) task for {!r}:".format(transform_type, config.kind, label)
)
yield job
def validate_dependent_tasks_transform(_, jobs):
for job in jobs:
check_every_architecture_is_present_in_dependent_tasks(job['dependent-tasks'])
yield job
def check_every_architecture_is_present_in_dependent_tasks(dependent_tasks):
dep_platforms = set(t.attributes.get('build_platform') for t in dependent_tasks)
missed_architectures = REQUIRED_ARCHITECTURES - dep_platforms
if missed_architectures:
raise Exception('''One or many required architectures are missing.
Required architectures: {}.
Given dependencies: {}.
'''.format(REQUIRED_ARCHITECTURES, dependent_tasks)
)
def delete_non_required_fields_transform(_, jobs):
for job in jobs:
del job['name']
del job['dependent-tasks']
yield job
def generate_dependencies(dependent_tasks):
# Because we depend on several tasks that have the same kind, we introduce the platform
dependencies = {}
for task in dependent_tasks:
platform_match = PLATFORM_REGEX.match(task.label)
# platform_match is None when the breakpoint task is given
task_kind = task.kind if platform_match is None else \
'{}-{}'.format(task.kind, platform_match.group(1))
dependencies[task_kind] = task.label
return dependencies

1
taskcluster/taskgraph/util/workertypes.py
Просмотреть файл

@ -40,7 +40,6 @@ WORKER_TYPES = {
'buildbot-bridge/buildbot-bridge': ('buildbot-bridge', None),
'invalid/invalid': ('invalid', None),
'invalid/always-optimized': ('always-optimized', None),
'null-provisioner/human-breakpoint': ('push-apk-breakpoint', None),
'releng-hardware/gecko-t-linux-talos': ('native-engine', 'linux'),
'scriptworker-prov-v1/balrog-dev': ('balrog', None),
'scriptworker-prov-v1/balrogworker-v1': ('balrog', None),