зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1433459 - part 1: Move bouncer submission tasks to scriptworker r=mtabara,rail
MozReview-Commit-ID: 6SKhjf1ywoH --HG-- extra : rebase_source : dc2206f232f3552f3871972abc8c1f2db3beb56a
This commit is contained in:
Johan Lorenzo
Родитель
8dccbf98a3
Коммит
084a9fc1fe
|
|
@ -5,37 +5,54 @@
|
|||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.bouncer_submission:transforms
|
||||
- taskgraph.transforms.release_notifications:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
job-defaults:
|
||||
description: release bouncer submission job
|
||||
worker-type: buildbot-bridge/buildbot-bridge
|
||||
worker-type:
|
||||
by-project:
|
||||
mozilla-central: scriptworker-prov-v1/bouncer-v1
|
||||
mozilla-beta: scriptworker-prov-v1/bouncer-v1
|
||||
mozilla-release: scriptworker-prov-v1/bouncer-v1
|
||||
default: scriptworker-prov-v1/bouncer-dev
|
||||
worker:
|
||||
implementation: bouncer-submission
|
||||
scopes:
|
||||
by-project:
|
||||
mozilla-beta:
|
||||
- project:releng:bouncer:action:submission
|
||||
- project:releng:bouncer:server:production
|
||||
mozilla-release:
|
||||
- project:releng:bouncer:action:submission
|
||||
- project:releng:bouncer:server:production
|
||||
default:
|
||||
- project:releng:bouncer:action:submission
|
||||
- project:releng:bouncer:server:staging
|
||||
run-on-projects: []
|
||||
shipping-phase: promote
|
||||
run:
|
||||
using: buildbot
|
||||
release-promotion: true
|
||||
shipping-product: firefox
|
||||
locales-file: browser/locales/l10n-changesets.json
|
||||
|
||||
jobs:
|
||||
devedition:
|
||||
bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
|
||||
bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
|
||||
shipping-product: devedition
|
||||
|
||||
fennec:
|
||||
name: fennec_release_bouncer_sub
|
||||
bouncer-platforms: ['android', 'android-x86']
|
||||
bouncer-products: ['apk']
|
||||
shipping-product: fennec
|
||||
run:
|
||||
product: fennec
|
||||
buildername: release-{branch}-fennec_bncr_sub
|
||||
locales-file: mobile/locales/l10n-changesets.json
|
||||
|
||||
firefox:
|
||||
name: firefox_release_bouncer_sub
|
||||
bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
|
||||
bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
|
||||
shipping-product: firefox
|
||||
run:
|
||||
product: firefox
|
||||
buildername: release-{branch}_firefox_bncr_sub
|
||||
|
||||
devedition:
|
||||
name: devedition_release_bouncer_sub
|
||||
shipping-product: devedition
|
||||
run:
|
||||
product: devedition
|
||||
buildername: release-{branch}_devedition_bncr_sub
|
||||
firefox-rc:
|
||||
bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
|
||||
bouncer-products: ['partial-mar-candidates']
|
||||
shipping-product: firefox
|
||||
|
|
|
|||
|
|
@ -0,0 +1,253 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Add from parameters.yml into bouncer submission tasks.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import logging
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.transforms.l10n import parse_locales_file
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
from taskgraph.util.scriptworker import get_release_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
FTP_PLATFORMS_PER_BOUNCER_PLATFORM = {
|
||||
'android': 'android-api-16',
|
||||
'android-x86': 'android-x86',
|
||||
'linux': 'linux-i686',
|
||||
'linux64': 'linux-x86_64',
|
||||
'osx': 'mac',
|
||||
'win': 'win32',
|
||||
'win64': 'win64',
|
||||
}
|
||||
|
||||
# :lang is interpolated by bouncer at runtime
|
||||
CANDIDATES_PATH_TEMPLATE = '/{product}/candidates/{version}-candidates/build{build_number}/\
|
||||
{update_folder}{ftp_platform}/:lang/{file}'
|
||||
RELEASES_PATH_TEMPLATE = '/{product}/releases/{version}/{update_folder}{ftp_platform}/:lang/{file}'
|
||||
|
||||
|
||||
CONFIG_PER_BOUNCER_PRODUCT = {
|
||||
'apk': {
|
||||
'path_template': RELEASES_PATH_TEMPLATE,
|
||||
'file_names': {
|
||||
'android': 'fennec-{version}.:lang.android-arm.apk',
|
||||
'android-x86': 'fennec-{version}.:lang.android-i386.apk',
|
||||
},
|
||||
},
|
||||
'complete-mar': {
|
||||
'path_template': RELEASES_PATH_TEMPLATE,
|
||||
'file_names': {
|
||||
'default': 'firefox-{version}.complete.mar',
|
||||
},
|
||||
},
|
||||
'installer': {
|
||||
'path_template': RELEASES_PATH_TEMPLATE,
|
||||
'file_names': {
|
||||
'linux': 'firefox-{version}.tar.bz2',
|
||||
'linux64': 'firefox-{version}.tar.bz2',
|
||||
'osx': 'Firefox%20{version}.dmg',
|
||||
'win': 'Firefox%20Setup%20{version}.exe',
|
||||
'win64': 'Firefox%20Setup%20{version}.exe',
|
||||
},
|
||||
},
|
||||
'partial-mar': {
|
||||
'path_template': RELEASES_PATH_TEMPLATE,
|
||||
'file_names': {
|
||||
'default': 'firefox-{previous_version}-{version}.partial.mar',
|
||||
},
|
||||
},
|
||||
'partial-mar-candidates': {
|
||||
'path_template': CANDIDATES_PATH_TEMPLATE,
|
||||
'file_names': {
|
||||
'default': 'firefox-{previous_version}-{version}.partial.mar',
|
||||
},
|
||||
},
|
||||
'stub-installer': {
|
||||
'path_template': RELEASES_PATH_TEMPLATE,
|
||||
'file_names': {
|
||||
'win': 'Firefox%20Installer.exe',
|
||||
'win64': 'Firefox%20Installer.exe',
|
||||
},
|
||||
},
|
||||
}
|
||||
CONFIG_PER_BOUNCER_PRODUCT['installer-ssl'] = CONFIG_PER_BOUNCER_PRODUCT['installer']
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def make_task_worker(config, jobs):
|
||||
for job in jobs:
|
||||
resolve_keyed_by(
|
||||
job, 'worker-type', item_name=job['name'], project=config.params['project']
|
||||
)
|
||||
resolve_keyed_by(
|
||||
job, 'scopes', item_name=job['name'], project=config.params['project']
|
||||
)
|
||||
|
||||
# No need to filter out ja-JP-mac, we need to upload both
|
||||
all_locales = list(sorted(parse_locales_file(job['locales-file']).keys()))
|
||||
job['worker']['locales'] = all_locales
|
||||
job['worker']['entries'] = craft_bouncer_entries(config, job)
|
||||
|
||||
del job['locales-file']
|
||||
del job['bouncer-platforms']
|
||||
del job['bouncer-products']
|
||||
|
||||
if job['worker']['entries']:
|
||||
# XXX Because rc jobs are defined within the same kind, we need to delete the
|
||||
# firefox-rc job at this stage, if we're not building an RC. Otherwise, even if
|
||||
# target_tasks.py filters out the rc job, it gets resurected by any kind that depends
|
||||
# on the release-bouncer-sub one (release-notify-promote as of time of this writing).
|
||||
if config.params['release_type'] == 'rc' or job['name'] != 'firefox-rc':
|
||||
yield job
|
||||
else:
|
||||
logger.warn('No bouncer entries defined in bouncer submission task for "{}". \
|
||||
Job deleted.'.format(job['name']))
|
||||
|
||||
|
||||
def craft_bouncer_entries(config, job):
|
||||
release_config = get_release_config(config)
|
||||
|
||||
product = job['shipping-product']
|
||||
bouncer_platforms = job['bouncer-platforms']
|
||||
|
||||
current_version = release_config['version']
|
||||
current_build_number = release_config['build_number']
|
||||
|
||||
bouncer_products = job['bouncer-products']
|
||||
previous_versions_string = release_config.get('partial_versions', None)
|
||||
if previous_versions_string:
|
||||
previous_versions = previous_versions_string.split(', ')
|
||||
else:
|
||||
logger.warn('No partials defined! Bouncer submission task won\'t send any \
|
||||
partial-related entry for "{}"'.format(job['name']))
|
||||
bouncer_products = [
|
||||
bouncer_product
|
||||
for bouncer_product in bouncer_products
|
||||
if 'partial' not in bouncer_product
|
||||
]
|
||||
previous_versions = [None]
|
||||
|
||||
project = config.params['project']
|
||||
|
||||
return {
|
||||
craft_bouncer_product_name(
|
||||
product, bouncer_product, current_version, current_build_number, previous_version
|
||||
): {
|
||||
'options': {
|
||||
'add_locales': craft_add_locales(product),
|
||||
'check_uptake': craft_check_uptake(bouncer_product),
|
||||
'ssl_only': craft_ssl_only(bouncer_product, project),
|
||||
},
|
||||
'paths_per_bouncer_platform': craft_paths_per_bouncer_platform(
|
||||
product, bouncer_product, bouncer_platforms, current_version,
|
||||
current_build_number, previous_version
|
||||
),
|
||||
}
|
||||
for bouncer_product in bouncer_products
|
||||
for previous_version in previous_versions
|
||||
}
|
||||
|
||||
|
||||
def craft_paths_per_bouncer_platform(product, bouncer_product, bouncer_platforms, current_version,
|
||||
current_build_number, previous_version=None):
|
||||
paths_per_bouncer_platform = {}
|
||||
for bouncer_platform in bouncer_platforms:
|
||||
ftp_platform = FTP_PLATFORMS_PER_BOUNCER_PLATFORM[bouncer_platform]
|
||||
|
||||
file_names_per_platform = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['file_names']
|
||||
file_name_template = file_names_per_platform.get(
|
||||
bouncer_platform, file_names_per_platform.get('default', None)
|
||||
)
|
||||
if not file_name_template:
|
||||
# Some bouncer product like stub-installer are only meant to be on Windows.
|
||||
# Thus no default value is defined there
|
||||
continue
|
||||
|
||||
file_name = file_name_template.format(
|
||||
version=current_version, previous_version=strip_build_data(previous_version)
|
||||
)
|
||||
|
||||
path_template = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['path_template']
|
||||
file_relative_location = path_template.format(
|
||||
product=product.lower(),
|
||||
version=current_version,
|
||||
build_number=current_build_number,
|
||||
update_folder='updates/' if '-mar' in bouncer_product else '',
|
||||
ftp_platform=ftp_platform,
|
||||
file=file_name,
|
||||
)
|
||||
|
||||
paths_per_bouncer_platform[bouncer_platform] = file_relative_location
|
||||
|
||||
return paths_per_bouncer_platform
|
||||
|
||||
|
||||
def craft_bouncer_product_name(product, bouncer_product, current_version,
|
||||
current_build_number=None, previous_version=None):
|
||||
if '-ssl' in bouncer_product:
|
||||
postfix = '-SSL'
|
||||
elif 'stub-' in bouncer_product:
|
||||
postfix = '-stub'
|
||||
elif 'complete-' in bouncer_product:
|
||||
postfix = '-Complete'
|
||||
elif 'partial-' in bouncer_product:
|
||||
if not previous_version:
|
||||
raise Exception('Partial is being processed, but no previous version defined.')
|
||||
|
||||
if '-candidates' in bouncer_product:
|
||||
if not current_build_number:
|
||||
raise Exception('Partial in candidates directory is being processed, \
|
||||
but no current build number defined.')
|
||||
|
||||
postfix = 'build{build_number}-Partial-{previous_version_with_build_number}'.format(
|
||||
build_number=current_build_number,
|
||||
previous_version_with_build_number=previous_version,
|
||||
)
|
||||
else:
|
||||
postfix = '-Partial-{previous_version}'.format(
|
||||
previous_version=strip_build_data(previous_version)
|
||||
)
|
||||
|
||||
elif 'sha1-' in bouncer_product:
|
||||
postfix = '-sha1'
|
||||
else:
|
||||
postfix = ''
|
||||
|
||||
return '{product}-{version}{postfix}'.format(
|
||||
product=product.capitalize(), version=current_version, postfix=postfix
|
||||
)
|
||||
|
||||
|
||||
def craft_check_uptake(bouncer_product):
|
||||
return bouncer_product != 'complete-mar-candidates'
|
||||
|
||||
|
||||
def craft_ssl_only(bouncer_product, project):
|
||||
# XXX ESR is the only channel where we force serve the installer over SSL
|
||||
if '-esr' in project and bouncer_product == 'installer':
|
||||
return True
|
||||
|
||||
return bouncer_product not in (
|
||||
'complete-mar',
|
||||
'installer',
|
||||
'partial-mar',
|
||||
'partial-mar-candidates',
|
||||
)
|
||||
|
||||
|
||||
def craft_add_locales(product):
|
||||
# Do not add locales on Fennec in order to let "multi" work
|
||||
return product != 'fennec'
|
||||
|
||||
|
||||
def strip_build_data(version):
|
||||
return version.split('build')[0] if version and 'build' in version else version
|
||||
|
|
@ -163,7 +163,7 @@ l10n_description_schema = Schema({
|
|||
transforms = TransformSequence()
|
||||
|
||||
|
||||
def _parse_locales_file(locales_file, platform):
|
||||
def parse_locales_file(locales_file, platform=None):
|
||||
""" Parse the passed locales file for a list of locales.
|
||||
"""
|
||||
locales = []
|
||||
|
|
@ -175,7 +175,7 @@ def _parse_locales_file(locales_file, platform):
|
|||
locales = {
|
||||
locale: data['revision']
|
||||
for locale, data in all_locales.items()
|
||||
if platform in data['platforms']
|
||||
if platform is None or platform in data['platforms']
|
||||
}
|
||||
else:
|
||||
all_locales = f.read().split()
|
||||
|
|
@ -299,8 +299,8 @@ def handle_keyed_by(config, jobs):
|
|||
def all_locales_attribute(config, jobs):
|
||||
for job in jobs:
|
||||
locales_platform = job['attributes']['build_platform'].replace("-nightly", "")
|
||||
locales_with_changesets = _parse_locales_file(job["locales-file"],
|
||||
platform=locales_platform)
|
||||
locales_with_changesets = parse_locales_file(job["locales-file"],
|
||||
platform=locales_platform)
|
||||
locales_with_changesets = _remove_locales(locales_with_changesets,
|
||||
to_remove=job['ignore-locales'])
|
||||
|
||||
|
|
|
|||
|
|
@ -558,10 +558,13 @@ task_description_schema = Schema({
|
|||
# Paths to the artifacts to sign
|
||||
Required('paths'): [basestring],
|
||||
}],
|
||||
}, {
|
||||
Required('implementation'): 'bouncer-submission',
|
||||
Required('locales'): [basestring],
|
||||
Required('entries'): object,
|
||||
}, {
|
||||
Required('implementation'): 'push-apk-breakpoint',
|
||||
Required('payload'): object,
|
||||
|
||||
}, {
|
||||
Required('implementation'): 'invalid',
|
||||
# an invalid task is one which should never actually be created; this is used in
|
||||
|
|
@ -1112,6 +1115,16 @@ def build_balrog_payload(config, task, task_def):
|
|||
})
|
||||
|
||||
|
||||
@payload_builder('bouncer-submission')
|
||||
def build_bouncer_submission_payload(config, task, task_def):
|
||||
worker = task['worker']
|
||||
|
||||
task_def['payload'] = {
|
||||
'locales': worker['locales'],
|
||||
'submission_entries': worker['entries']
|
||||
}
|
||||
|
||||
|
||||
@payload_builder('push-apk')
|
||||
def build_push_apk_payload(config, task, task_def):
|
||||
worker = task['worker']
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче