mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge m-c to autoland, a=merge 

MozReview-Commit-ID: DihMPQQtOlw
This commit is contained in:
Wes Kocher 2017-08-15 23:17:09 -07:00
Родитель fa58525e28 9d3cf9697f
Коммит 6fc856c286
16 изменённых файлов: 171 добавлений и 190 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
python/mozbuild/mozbuild/repackaging/dmg.py
Просмотреть файл

@ -2,6 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import errno
import os
import tempfile
import tarfile

8
taskcluster/ci/beetmover-l10n/kind.yml
Просмотреть файл

@ -17,6 +17,8 @@ only-for-attributes:
- nightly
not-for-build-platforms:
- macosx64-nightly/opt
- win32-nightly/opt
- win64-nightly/opt
- linux-nightly/opt
- linux64-nightly/opt
- macosx64-nightly/opt
- win32-nightly/opt
- win64-nightly/opt

2
taskcluster/ci/beetmover-repackage/kind.yml
Просмотреть файл

@ -14,6 +14,8 @@ kind-dependencies:
- repackage-signing
only-for-build-platforms:
- linux-nightly/opt
- linux64-nightly/opt
- macosx64-nightly/opt
- win32-nightly/opt
- win64-nightly/opt

8
taskcluster/ci/beetmover/kind.yml
Просмотреть файл

@ -16,6 +16,8 @@ only-for-attributes:
- nightly
not-for-build-platforms:
- macosx64-nightly/opt
- win32-nightly/opt
- win64-nightly/opt
- linux-nightly/opt
- linux64-nightly/opt
- macosx64-nightly/opt
- win32-nightly/opt
- win64-nightly/opt

2
taskcluster/ci/repackage-l10n/kind.yml
Просмотреть файл

@ -15,6 +15,8 @@ kind-dependencies:
- nightly-l10n-signing
only-for-build-platforms:
- linux-nightly/opt
- linux64-nightly/opt
- macosx64-nightly/opt
- win32-nightly/opt
- win64-nightly/opt

4
taskcluster/ci/repackage-signing/kind.yml
Просмотреть файл

@ -15,6 +15,10 @@ kind-dependencies:
- repackage-l10n
only-for-build-platforms:
- linux-nightly/opt
- linux-devedition-nightly/opt
- linux64-nightly/opt
- linux64-devedition-nightly/opt
- macosx64-nightly/opt
- macosx64-devedition-nightly/opt
- win32-nightly/opt

4
taskcluster/ci/repackage/kind.yml
Просмотреть файл

@ -15,6 +15,10 @@ kind-dependencies:
- build-signing
only-for-build-platforms:
- linux-nightly/opt
- linux-devedition-nightly/opt
- linux64-nightly/opt
- linux64-devedition-nightly/opt
- macosx64-nightly/opt
- macosx64-devedition-nightly/opt
- win32-nightly/opt

85
taskcluster/taskgraph/transforms/beetmover_repackage.py
Просмотреть файл

@ -16,6 +16,8 @@ from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
import logging
import re
logger = logging.getLogger(__name__)
@ -63,21 +65,15 @@ _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
"host/bin/mar",
"host/bin/mbsdiff",
r'^(linux(|64)|macosx64)-nightly$': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
'host/bin/mar',
'host/bin/mbsdiff',
],
'macosx64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'win64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
r'^win(32|64)-nightly$': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
"host/bin/mar.exe",
"host/bin/mbsdiff.exe",
],
'win64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
'win32-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
"host/bin/mar.exe",
"host/bin/mbsdiff.exe",
],
'win32-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
r'^(linux(|64)|macosx64|win(32|64))-nightly-l10n$': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
}
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
@ -85,10 +81,8 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
UPSTREAM_ARTIFACT_SIGNED_PATHS = {
'win64-nightly': ['target.zip'],
'win64-nightly-l10n': ['target.zip'],
'win32-nightly': ['target.zip'],
'win32-nightly-l10n': ['target.zip'],
r'^linux(|64)-nightly(|-l10n)$': ['target.tar.bz2', 'target.tar.bz2.asc'],
r'^win(32|64)-nightly(|-l10n)$': ['target.zip'],
}
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
@ -96,30 +90,32 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
'macosx64-nightly': ['target.dmg'],
'macosx64-nightly-l10n': ['target.dmg'],
r'^macosx64-nightly(|-l10n)$': ['target.dmg'],
}
# Until bug 1331141 is fixed, if you are adding any new artifacts here that
# need to be transfered to S3, please be aware you also need to follow-up
# with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
# See example in bug 1348286
UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = {
'macosx64-nightly': ['target.complete.mar'],
'macosx64-nightly-l10n': ['target.complete.mar'],
'win64-nightly': ['target.complete.mar', 'target.installer.exe'],
'win64-nightly-l10n': ['target.complete.mar', 'target.installer.exe'],
'win32-nightly': [
'target.complete.mar',
'target.installer.exe',
'target.stub-installer.exe'
],
'win32-nightly-l10n': [
r'^(linux(|64)|macosx64)-nightly(|-l10n)$': ['target.complete.mar'],
r'^win64-nightly(|-l10n)$': ['target.complete.mar', 'target.installer.exe'],
r'^win32-nightly(|-l10n)$': [
'target.complete.mar',
'target.installer.exe',
'target.stub-installer.exe'
],
}
# Compile every regex once at import time
for dict_ in (
UPSTREAM_ARTIFACT_UNSIGNED_PATHS, UPSTREAM_ARTIFACT_SIGNED_PATHS,
UPSTREAM_ARTIFACT_REPACKAGE_PATHS, UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS,
):
for uncompiled_regex, value in dict_.iteritems():
compiled_regex = re.compile(uncompiled_regex)
del dict_[uncompiled_regex]
dict_[compiled_regex] = value
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
@ -271,18 +267,37 @@ def generate_upstream_artifacts(build_task_ref, build_signing_task_ref,
]
for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping):
if platform in mapping:
upstream_artifacts.append({
"taskId": {"task-reference": ref},
"taskType": tasktype,
"paths": ["{}/{}".format(artifact_prefix, p)
for p in mapping[platform]],
"locale": locale or "en-US",
})
plarform_was_previously_matched_by_regex = None
for platform_regex, paths in mapping.iteritems():
if platform_regex.match(platform) is not None:
_check_platform_matched_only_one_regex(
tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
)
upstream_artifacts.append({
"taskId": {"task-reference": ref},
"taskType": tasktype,
"paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
"locale": locale or "en-US",
})
plarform_was_previously_matched_by_regex = platform_regex
return upstream_artifacts
def _check_platform_matched_only_one_regex(
task_type, platform, plarform_was_previously_matched_by_regex, platform_regex
):
if plarform_was_previously_matched_by_regex is not None:
raise Exception('In task type "{task_type}", platform "{platform}" matches at \
least 2 regular expressions. First matched: "{first_matched}". Second matched: \
"{second_matched}"'.format(
task_type=task_type, platform=platform,
first_matched=plarform_was_previously_matched_by_regex.pattern,
second_matched=platform_regex.pattern
))
def is_valid_beetmover_job(job):
# windows builds don't have docker-image, so fewer dependencies
if any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):

76
taskcluster/taskgraph/transforms/build_signing.py
Просмотреть файл

@ -8,6 +8,8 @@ Transform the signing task into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.signed_artifacts import generate_specifications_of_artifacts_to_sign
transforms = TransformSequence()
@ -34,66 +36,26 @@ def add_signed_routes(config, jobs):
@transforms.add
def make_signing_description(config, jobs):
def define_upstream_artifacts(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
build_platform = dep_job.attributes.get('build_platform')
job['upstream-artifacts'] = _generate_upstream_artifacts(
dep_job.attributes.get('build_platform'),
dep_job.attributes.get('nightly')
artifacts_specifications = generate_specifications_of_artifacts_to_sign(
build_platform,
dep_job.attributes.get('nightly'),
keep_locale_template=False
)
if 'android' in build_platform:
# We're in the job that creates both multilocale and en-US APKs
artifacts_specifications[0]['artifacts'].append('public/build/en-US/target.apk')
job['upstream-artifacts'] = [{
'taskId': {'task-reference': '<build>'},
'taskType': 'build',
'paths': spec['artifacts'],
'formats': spec['formats'],
} for spec in artifacts_specifications]
yield job
def _generate_upstream_artifacts(build_platform, is_nightly=False):
if 'android' in build_platform:
artifacts_specificities = [{
'artifacts': [
'public/build/target.apk',
'public/build/en-US/target.apk'
],
'format': 'jar',
}]
# XXX: Mac and Windows don't sign mars because internal aren't signed at
# this stage of the release
elif 'macosx' in build_platform:
artifacts_specificities = [{
'artifacts': ['public/build/target.dmg'],
'format': 'macapp',
}]
elif 'win64' in build_platform:
artifacts_specificities = [{
'artifacts': [
'public/build/target.zip',
'public/build/setup.exe'
],
'format': 'sha2signcode',
}]
elif 'win32' in build_platform:
artifacts_specificities = [{
'artifacts': [
'public/build/target.zip',
'public/build/setup.exe',
],
'format': 'sha2signcode',
}]
if is_nightly:
artifacts_specificities[0]['artifacts'] += ['public/build/setup-stub.exe']
elif 'linux' in build_platform:
artifacts_specificities = [{
'artifacts': ['public/build/target.tar.bz2'],
'format': 'gpg',
}, {
'artifacts': ['public/build/update/target.complete.mar'],
'format': 'mar_sha384',
}]
else:
raise Exception("Platform not implemented for signing")
return [{
'taskId': {'task-reference': '<build>'},
'taskType': 'build',
'paths': specificity['artifacts'],
'formats': [specificity['format']],
} for specificity in artifacts_specificities]

8
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
Просмотреть файл

@ -31,6 +31,10 @@ JOB_NAME_WHITELIST = set([
'browser-haz-debug',
'linux-debug',
'linux-devedition',
'linux-devedition-nightly-repackage',
'linux-devedition-nightly-repackage-signing',
'linux-nightly-repackage',
'linux-nightly-repackage-signing',
'linux-opt',
'linux-pgo',
'linux64-add-on-devel',
@ -44,7 +48,11 @@ JOB_NAME_WHITELIST = set([
'linux64-clang-tidy',
'linux64-debug',
'linux64-devedition',
'linux64-devedition-nightly-repackage',
'linux64-devedition-nightly-repackage-signing',
'linux64-jsdcov-opt',
'linux64-nightly-repackage',
'linux64-nightly-repackage-signing',
'linux64-noopt-debug',
'linux64-opt',
'linux64-pgo',

98
taskcluster/taskgraph/transforms/nightly_l10n_signing.py
Просмотреть файл

@ -8,6 +8,7 @@ Transform the signing task into an actual task description.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.signed_artifacts import generate_specifications_of_artifacts_to_sign
from taskgraph.util.treeherder import join_symbol
transforms = TransformSequence()
@ -19,72 +20,6 @@ def make_signing_description(config, jobs):
job['depname'] = 'unsigned-repack'
dep_job = job['dependent-task']
dep_platform = dep_job.attributes.get('build_platform')
job['upstream-artifacts'] = []
if 'android' in dep_platform:
job_specs = [
{
'artifacts': ['public/build/{locale}/target.apk'],
'format': 'jar',
},
]
elif 'macosx' in dep_platform:
job_specs = [
{
'artifacts': ['public/build/{locale}/target.dmg'],
'format': 'macapp',
}
]
elif 'win32' in dep_platform:
job_specs = [
{
'artifacts': [
'public/build/{locale}/target.zip',
'public/build/{locale}/setup.exe',
'public/build/{locale}/setup-stub.exe'
],
'format': 'sha2signcode',
}
]
elif 'win64' in dep_platform:
job_specs = [
{
'artifacts': [
'public/build/{locale}/target.zip',
'public/build/{locale}/setup.exe',
],
'format': 'sha2signcode',
}
]
elif 'linux' in dep_platform:
job_specs = [
{
'artifacts': ['public/build/{locale}/target.tar.bz2'],
'format': 'gpg',
}, {
'artifacts': ['public/build/{locale}/target.complete.mar'],
'format': 'mar_sha384',
}
]
else:
raise Exception("Platform not implemented for signing")
upstream_artifacts = []
for spec in job_specs:
fmt = spec['format']
upstream_artifacts.append({
"taskId": {"task-reference": "<unsigned-repack>"},
"taskType": "l10n",
# Set paths based on artifacts in the specs (above) one per
# locale present in the chunk this is signing stuff for.
"paths": [f.format(locale=l)
for l in dep_job.attributes.get('chunk_locales', [])
for f in spec['artifacts']],
"formats": [fmt]
})
job['upstream-artifacts'] = upstream_artifacts
# add the chunk number to the TH symbol
symbol = 'Ns{}'.format(dep_job.attributes.get('l10n_chunk'))
@ -95,3 +30,34 @@ def make_signing_description(config, jobs):
}
yield job
@transforms.add
def define_upstream_artifacts(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
locale_specifications = generate_specifications_of_artifacts_to_sign(
dep_job.attributes.get('build_platform'),
is_nightly=True,
keep_locale_template=True
)
upstream_artifacts = []
for spec in locale_specifications:
upstream_artifacts.append({
'taskId': {'task-reference': '<unsigned-repack>'},
'taskType': 'l10n',
# Set paths based on artifacts in the specs (above) one per
# locale present in the chunk this is signing stuff for.
'paths': [
path_template.format(locale=locale)
for locale in dep_job.attributes.get('chunk_locales', [])
for path_template in spec['artifacts']
],
'formats': spec['formats']
})
job['upstream-artifacts'] = upstream_artifacts
yield job

56
taskcluster/taskgraph/transforms/repackage.py
Просмотреть файл

@ -148,8 +148,15 @@ def make_job_description(config, jobs):
if build_platform.startswith('win'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
run['use-magic-mh-args'] = False
elif build_platform.startswith('macosx'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
else:
if build_platform.startswith('macosx'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-macosx64' % level
elif build_platform.startswith('linux'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
else:
raise NotImplementedError(
'Unsupported build_platform: "{}"'.format(build_platform)
)
run['tooltool-downloads'] = 'internal'
worker['docker-image'] = {"in-tree": "desktop-build"}
@ -185,20 +192,26 @@ def make_job_description(config, jobs):
def _generate_task_mozharness_config(build_platform):
if build_platform.startswith('macosx'):
return ['repackage/osx_signed.py']
elif build_platform.startswith('win'):
return ['repackage/win32_signed.py'] if '32' in build_platform \
else ['repackage/win64_signed.py']
else:
raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
bits = 32 if '32' in build_platform else 64
if build_platform.startswith('linux'):
return ['repackage/linux{}_signed.py'.format(bits)]
elif build_platform.startswith('win'):
return ['repackage/win{}_signed.py'.format(bits)]
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None)
signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale)
if build_platform.startswith('macosx'):
if build_platform.startswith('linux') or build_platform.startswith('macosx'):
tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
return {
'SIGNED_INPUT': {'task-reference': '{}target.tar.gz'.format(signed_prefix)},
'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format(
signed_prefix, tarball_extension
)},
'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
}
elif build_platform.startswith('win'):
@ -215,8 +228,7 @@ def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=
}
return task_env
else:
raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
@ -228,18 +240,23 @@ def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
def _generate_task_output_files(build_platform, locale=None):
locale_output_path = '{}/'.format(locale) if locale else ''
if build_platform.startswith('macosx'):
return [{
'type': 'file',
'path': '/home/worker/workspace/build/artifacts/{}target.dmg'
.format(locale_output_path),
'name': 'public/build/{}target.dmg'.format(locale_output_path),
}, {
if build_platform.startswith('linux') or build_platform.startswith('macosx'):
output_files = [{
'type': 'file',
'path': '/home/worker/workspace/build/artifacts/{}target.complete.mar'
.format(locale_output_path),
'name': 'public/build/{}target.complete.mar'.format(locale_output_path),
}]
if build_platform.startswith('macosx'):
output_files.append({
'type': 'file',
'path': '/home/worker/workspace/build/artifacts/{}target.dmg'
.format(locale_output_path),
'name': 'public/build/{}target.dmg'.format(locale_output_path),
})
elif build_platform.startswith('win'):
output_files = [{
'type': 'file',
@ -259,6 +276,7 @@ def _generate_task_output_files(build_platform, locale=None):
'name': 'public/build/{}target.stub-installer.exe'.format(locale_output_path),
})
if output_files:
return output_files
else:
raise NotImplemented('Unsupported build_platform: "{}"'.format(build_platform))
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))

2
taskcluster/taskgraph/transforms/repackage_signing.py
Просмотреть файл

@ -130,6 +130,8 @@ def make_repackage_signing_description(config, jobs):
}
funsize_platforms = [
'linux-nightly',
'linux64-nightly',
'macosx64-nightly',
'win32-nightly',
'win64-nightly'

5
taskcluster/taskgraph/transforms/signing.py
Просмотреть файл

@ -136,11 +136,6 @@ def make_task_description(config, jobs):
'routes': job.get('routes', []),
}
if 'linux' in dep_job.attributes.get('build_platform') and \
dep_job.attributes.get('nightly'):
task['routes'].append("project.releng.funsize.level-{level}.{project}".format(
project=config.params['project'], level=config.params['level']))
yield task

1
testing/mozharness/configs/repackage/win32_signed.py
Просмотреть файл

@ -32,7 +32,6 @@ repackage_config = [[
config = {
"input_home": "{abs_work_dir}\\inputs",
"output_home": "{base_work_dir}\\public\\build{locale}",
"src_mozconfig": "browser/config/mozconfigs/{}/repack".format(platform),
"locale": os.environ.get("LOCALE"),

1
testing/mozharness/configs/repackage/win64_signed.py
Просмотреть файл

@ -25,7 +25,6 @@ repackage_config = [[
config = {
"input_home": "{abs_work_dir}\\inputs",
"output_home": "{base_work_dir}\\public\\build{locale}",
"src_mozconfig": "browser/config/mozconfigs/{}/repack".format(platform),
"locale": os.environ.get("LOCALE"),