mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1342392 Migrate partial update generation in-tree r=rail 

MozReview-Commit-ID: G6EFXY0UzSa

--HG--
extra : rebase_source : dbea33ec8d96cc9a0e016e4828f9f989fca7131b
This commit is contained in:
Simon Fraser 2017-09-18 13:36:36 +01:00
Родитель 1c62403696
Коммит 8ad84443f2
20 изменённых файлов: 636 добавлений и 35 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
taskcluster/ci/beetmover-repackage/kind.yml
Просмотреть файл

@ -12,6 +12,7 @@ transforms:
kind-dependencies:
- repackage-signing
- partials-signing
only-for-build-platforms:
- linux-nightly/opt

2
taskcluster/ci/docker-image/kind.yml
Просмотреть файл

@ -26,3 +26,5 @@ jobs:
symbol: I(agb)
index-task:
symbol: I(idx)
funsize-update-generator:
symbol: I(pg)

13
taskcluster/ci/partials-signing/kind.yml Normal file
Просмотреть файл

@ -0,0 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.single_dep:loader
transforms:
- taskgraph.transforms.name_sanity:transforms
- taskgraph.transforms.partials_signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- partials

23
taskcluster/ci/partials/kind.yml Normal file
Просмотреть файл

@ -0,0 +1,23 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
loader: taskgraph.loader.single_dep:loader
transforms:
- taskgraph.transforms.name_sanity:transforms
- taskgraph.transforms.partials:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- repackage-signing
only-for-attributes:
- nightly
only-for-build-platforms:
- macosx64-nightly/opt
- win32-nightly/opt
- win64-nightly/opt
- linux-nightly/opt
- linux64-nightly/opt

3
taskcluster/docker/funsize-update-generator/Dockerfile
Просмотреть файл

@ -25,6 +25,7 @@ RUN pip install -r /tmp/requirements.txt
# scripts
RUN mkdir /home/worker/bin
COPY scripts/* /home/worker/bin/
COPY runme.sh /runme.sh
COPY recompress.sh /recompress.sh
RUN chmod 755 /home/worker/bin/* /*.sh
@ -35,3 +36,5 @@ ENV HOME /home/worker
ENV SHELL /bin/bash
ENV USER worker
ENV LOGNAME worker
CMD ["/runme.sh"]

13
taskcluster/docker/funsize-update-generator/scripts/funsize.py
Просмотреть файл

@ -1,5 +1,9 @@
#!/usr/bin/env python
from __future__ import absolute_import, print_function
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import ConfigParser
import argparse
@ -27,6 +31,7 @@ ALLOWED_URL_PREFIXES = [
"http://ftp.mozilla.org/",
"http://download.mozilla.org/",
"https://archive.mozilla.org/",
"https://queue.taskcluster.net/v1/task/",
]
DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
@ -286,7 +291,11 @@ def main():
# if branch not set explicitly use repo-name
mar_data["branch"] = e.get("branch",
mar_data["repo"].rstrip("/").split("/")[-1])
mar_name = args.filename_template.format(**mar_data)
if 'dest_mar' in e:
mar_name = e['dest_mar']
else:
# default to formatted name if not specified
mar_name = args.filename_template.format(**mar_data)
mar_data["mar"] = mar_name
dest_mar = os.path.join(work_env.workdir, mar_name)
# TODO: download these once

10
taskcluster/docs/kinds.rst
Просмотреть файл

@ -233,3 +233,13 @@ repackage-signing
-----------------
Repackage-signing take the repackaged installers (windows) and update packaging (with
the signed internal bits) and signs them.
partials
--------
Partials takes the complete.mar files produced in previous tasks and generates partial
updates between previous nightly releases and the new one. Requires a release_history
in the parameters. See ``mach release-history`` if doing this manually.
partials-signing
----------------
Partials-signing takes the partial updates produced in Partials and signs them.

6
taskcluster/docs/parameters.rst
Просмотреть файл

@ -107,6 +107,12 @@ syntax or reading a project-specific configuration file).
``include_nightly``
If true, then nightly tasks are eligible for optimization.
``release_history``
History of recent releases by platform and locale, used when generating
partial updates for nightly releases.
Suitable contents can be generated with ``mach release-history``,
which will print to the console by default.
Morphed Set
-----------

20
taskcluster/mach_commands.py
Просмотреть файл

@ -504,3 +504,23 @@ class TaskClusterImagesProvider(object):
except Exception:
traceback.print_exc()
sys.exit(1)
@CommandProvider
class TaskClusterPartialsData(object):
@Command('release-history', category="ci",
description="Query balrog for release history used by enable partials generation")
@CommandArgument('-b', '--branch',
help="The gecko project branch used in balrog, such as "
"mozilla-central, release, date")
@CommandArgument('--product', default='Firefox',
help="The product identifier, such as 'Firefox'")
def generate_partials_builds(self, product, branch):
from taskgraph.util.partials import populate_release_history
try:
import yaml
release_history = {'release_history': populate_release_history(product, branch)}
print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
except Exception:
traceback.print_exc()
sys.exit(1)

9
taskcluster/taskgraph/decision.py
Просмотреть файл

@ -18,6 +18,7 @@ from .create import create_tasks
from .parameters import Parameters
from .taskgraph import TaskGraph
from .actions import render_actions_json
from taskgraph.util.partials import populate_release_history
from . import GECKO
from taskgraph.util.templates import Templates
@ -107,6 +108,7 @@ def taskgraph_decision(options):
"""
parameters = get_decision_parameters(options)
# create a TaskGraphGenerator instance
tgg = TaskGraphGenerator(
root_dir=options['root'],
@ -202,6 +204,13 @@ def get_decision_parameters(options):
if options.get('target_tasks_method'):
parameters['target_tasks_method'] = options['target_tasks_method']
# If the target method is nightly, we should build partials. This means
# knowing what has been released previously.
# An empty release_history is fine, it just means no partials will be built
parameters.setdefault('release_history', dict())
if 'nightly' in parameters.get('target_tasks_method', ''):
parameters['release_history'] = populate_release_history('Firefox', project)
return Parameters(parameters)

1
taskcluster/taskgraph/parameters.py
Просмотреть файл

@ -28,6 +28,7 @@ PARAMETER_NAMES = set([
'project',
'pushdate',
'pushlog_id',
'release_history',
'target_task_labels',
'target_tasks_method',
])

8
taskcluster/taskgraph/transforms/balrog.py
Просмотреть файл

@ -56,7 +56,7 @@ def make_task_description(config, jobs):
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'tc-Up(N)')
treeherder.setdefault('symbol', 'c-Up(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
@ -66,11 +66,14 @@ def make_task_description(config, jobs):
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder_job_symbol = dep_job.attributes.get('locale', 'N')
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
treeherder['symbol'] = 'c-Up({})'.format(treeherder_job_symbol)
attributes['locale'] = dep_job.attributes.get('locale')
label = job['label']
description = (
"Balrog submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
@ -94,7 +97,6 @@ def make_task_description(config, jobs):
task = {
'label': label,
'description': description,
# do we have to define worker type somewhere?
'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
'worker': {
'implementation': 'balrog',

92
taskcluster/taskgraph/transforms/beetmover_repackage.py
Просмотреть файл

@ -9,6 +9,9 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partials import (get_balrog_platform_name,
get_partials_artifacts,
get_partials_artifact_map)
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
get_beetmover_action_scope)
@ -215,6 +218,14 @@ def make_task_description(config, jobs):
}
dependencies.update(repackage_dependencies)
# If this isn't a direct dependency, it won't be in there.
if 'repackage-signing' not in dependencies:
repackage_signing_name = "repackage-signing"
repackage_signing_deps = {"repackage-signing":
dep_job.dependencies[repackage_signing_name]
}
dependencies.update(repackage_signing_deps)
attributes = copy_attributes_from_dependent_job(dep_job)
if job.get('locale'):
attributes['locale'] = job['locale']
@ -273,7 +284,6 @@ def generate_upstream_artifacts(build_task_ref, build_signing_task_ref,
_check_platform_matched_only_one_regex(
tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
)
upstream_artifacts.append({
"taskId": {"task-reference": ref},
"taskType": tasktype,
@ -285,6 +295,23 @@ def generate_upstream_artifacts(build_task_ref, build_signing_task_ref,
return upstream_artifacts
def generate_partials_upstream_artifacts(artifacts, platform, locale=None):
if not locale or locale == 'en-US':
artifact_prefix = 'public/build'
else:
artifact_prefix = 'public/build/{}'.format(locale)
upstream_artifacts = [{
'taskId': {'task-reference': '<partials-signing>'},
'taskType': 'signing',
'paths': ["{}/{}".format(artifact_prefix, p)
for p in artifacts],
'locale': locale or 'en-US',
}]
return upstream_artifacts
def _check_platform_matched_only_one_regex(
task_type, platform, plarform_was_previously_matched_by_regex, platform_regex
):
@ -299,8 +326,12 @@ least 2 regular expressions. First matched: "{first_matched}". Second matched: \
def is_valid_beetmover_job(job):
# windows builds don't have docker-image, so fewer dependencies
if any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):
# beetmover after partials-signing should have six dependencies.
# windows builds w/o partials don't have docker-image, so fewer
# dependencies
if 'partials-signing' in job['dependencies'].keys():
expected_dep_count = 6
elif any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):
expected_dep_count = 4
else:
expected_dep_count = 5
@ -321,6 +352,7 @@ def make_task_worker(config, jobs):
build_signing_task = None
repackage_task = None
repackage_signing_task = None
for dependency in job["dependencies"].keys():
if 'repackage-signing' in dependency:
repackage_signing_task = dependency
@ -348,3 +380,57 @@ def make_task_worker(config, jobs):
job["worker"] = worker
yield job
@transforms.add
def make_partials_artifacts(config, jobs):
for job in jobs:
locale = job["attributes"].get("locale")
if not locale:
locale = 'en-US'
# Remove when proved reliable
# job['treeherder']['tier'] = 3
platform = job["attributes"]["build_platform"]
balrog_platform = get_balrog_platform_name(platform)
artifacts = get_partials_artifacts(config.params.get('release_history'),
balrog_platform, locale)
# Dependency: | repackage-signing | partials-signing
# Partials artifacts | Skip | Populate & yield
# No partials | Yield | continue
if len(artifacts) == 0:
if 'partials-signing' in job['dependencies']:
continue
else:
yield job
continue
else:
if 'partials-signing' not in job['dependencies']:
continue
upstream_artifacts = generate_partials_upstream_artifacts(
artifacts, balrog_platform, locale
)
job['worker']['upstream-artifacts'].extend(upstream_artifacts)
extra = list()
artifact_map = get_partials_artifact_map(
config.params.get('release_history'), balrog_platform, locale)
for artifact in artifact_map:
extra.append({
'locale': locale,
'artifact_name': artifact,
'buildid': artifact_map[artifact],
'platform': balrog_platform,
})
job.setdefault('extra', {})
job['extra']['partials'] = extra
yield job

137
taskcluster/taskgraph/transforms/partials.py Normal file
Просмотреть файл

@ -0,0 +1,137 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the partials task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partials import get_balrog_platform_name, get_builds
from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
import logging
logger = logging.getLogger(__name__)
transforms = TransformSequence()
def _generate_task_output_files(filenames, locale=None):
locale_output_path = '{}/'.format(locale) if locale else ''
data = list()
for filename in filenames:
data.append({
'type': 'file',
'path': '/home/worker/artifacts/{}'.format(filename),
'name': 'public/build/{}{}'.format(locale_output_path, filename)
})
data.append({
'type': 'file',
'path': '/home/worker/artifacts/manifest.json',
'name': 'public/build/{}manifest.json'.format(locale_output_path)
})
return data
@transforms.add
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
build_locale = locale or 'en-US'
builds = get_builds(config.params['release_history'], dep_th_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
signing_task = None
for dependency in sorted(dependencies.keys()):
if 'repackage-signing' in dependency:
signing_task = dependency
break
signing_task_ref = '<{}>'.format(signing_task)
extra = {'funsize': {'partials': list()}}
update_number = 1
artifact_path = "{}{}".format(
get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
'target.complete.mar'
)
for build in builds:
extra['funsize']['partials'].append({
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {'task-reference': artifact_path},
'platform': get_balrog_platform_name(dep_th_platform),
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
})
update_number += 1
cot = extra.setdefault('chainOfTrust', {})
cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
worker = {
'artifacts': _generate_task_output_files(builds.keys(), locale),
'implementation': 'docker-worker',
'docker-image': {'in-tree': 'funsize-update-generator'},
'os': 'linux',
'max-run-time': 3600,
'chain-of-trust': True,
'env': {
'SHA1_SIGNING_CERT': 'nightly_sha1',
'SHA384_SIGNING_CERT': 'nightly_sha384'
}
}
level = config.params['level']
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
'worker': worker,
}
yield task

96
taskcluster/taskgraph/transforms/partials_signing.py Normal file
Просмотреть файл

@ -0,0 +1,96 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the partials task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts
import logging
logger = logging.getLogger(__name__)
transforms = TransformSequence()
def generate_upstream_artifacts(release_history, platform, locale=None):
artifact_prefix = 'public/build'
if locale:
artifact_prefix = 'public/build/{}'.format(locale)
else:
locale = 'en-US'
artifacts = get_partials_artifacts(release_history, platform, locale)
upstream_artifacts = [{
"taskId": {"task-reference": '<partials>'},
"taskType": 'partials',
"paths": ["{}/{}".format(artifact_prefix, p)
for p in artifacts],
"formats": ["mar_sha384"],
}]
return upstream_artifacts
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'ps(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
label = job.get('label', "partials-signing-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependent_kind = str(dep_job.kind)
dependencies = {dependent_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = 'ps({})'.format(locale)
balrog_platform = get_balrog_platform_name(dep_th_platform)
upstream_artifacts = generate_upstream_artifacts(
config.params['release_history'], balrog_platform, locale)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384']
task = {
'label': label,
'description': "{} Partials".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'dependencies': dependencies,
'attributes': attributes,
'scopes': scopes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task

15
taskcluster/taskgraph/transforms/repackage.py
Просмотреть файл

@ -10,12 +10,10 @@ from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import validate_schema, Schema
from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
_TC_ARTIFACT_LOCATION = \
'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
@ -203,8 +201,8 @@ def _generate_task_mozharness_config(build_platform):
def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None)
signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale)
mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None)
signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale)
if build_platform.startswith('linux') or build_platform.startswith('macosx'):
tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
@ -231,13 +229,6 @@ def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
if locale:
postfix = '{}/{}'.format(locale, postfix)
return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
def _generate_task_output_files(build_platform, locale=None):
locale_output_path = '{}/'.format(locale) if locale else ''

14
taskcluster/taskgraph/transforms/repackage_signing.py
Просмотреть файл

@ -129,20 +129,6 @@ def make_repackage_signing_description(config, jobs):
'treeherder': treeherder,
}
funsize_platforms = [
'linux-nightly',
'linux64-nightly',
'macosx64-nightly',
'win32-nightly',
'win64-nightly'
]
if build_platform in funsize_platforms and is_nightly:
route_template = "project.releng.funsize.level-{level}.{project}"
task['routes'] = [
route_template.format(project=config.params['project'],
level=config.params['level'])
]
yield task

5
taskcluster/taskgraph/transforms/task.py
Просмотреть файл

@ -515,7 +515,7 @@ GROUP_NAMES = {
'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster',
'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
'c-Up': 'Balrog submission of complete updates',
'tc-cs': 'Checksum signing executed by Taskcluster',
'tc-rs': 'Repackage signing executed by Taskcluster',
'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
@ -528,7 +528,10 @@ GROUP_NAMES = {
'TW64': 'Toolchain builds for Windows 64-bits',
'SM-tc': 'Spidermonkey builds',
'pub': 'APK publishing',
'p': 'Partial generation',
'ps': 'Partials signing',
}
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
V2_ROUTE_TEMPLATES = [

193
taskcluster/taskgraph/util/partials.py Normal file
Просмотреть файл

@ -0,0 +1,193 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import requests
import redo
import logging
logger = logging.getLogger(__name__)
BALROG_API_ROOT = 'https://aus5.mozilla.org/api/v1'
PLATFORM_RENAMES = {
'windows2012-32': 'win32',
'windows2012-64': 'win64',
'osx-cross': 'macosx64',
}
BALROG_PLATFORM_MAP = {
"linux": [
"Linux_x86-gcc3"
],
"linux64": [
"Linux_x86_64-gcc3"
],
"macosx64": [
"Darwin_x86_64-gcc3-u-i386-x86_64",
"Darwin_x86-gcc3-u-i386-x86_64",
"Darwin_x86-gcc3",
"Darwin_x86_64-gcc3"
],
"win32": [
"WINNT_x86-msvc",
"WINNT_x86-msvc-x86",
"WINNT_x86-msvc-x64"
],
"win64": [
"WINNT_x86_64-msvc",
"WINNT_x86_64-msvc-x64"
]
}
def get_balrog_platform_name(platform):
"""Convert build platform names into balrog platform names"""
if '-nightly' in platform:
platform = platform.replace('-nightly', '')
if '-devedition' in platform:
platform = platform.replace('-devedition', '')
return PLATFORM_RENAMES.get(platform, platform)
def _sanitize_platform(platform):
platform = get_balrog_platform_name(platform)
if platform not in BALROG_PLATFORM_MAP:
return platform
return BALROG_PLATFORM_MAP[platform][0]
def get_builds(release_history, platform, locale):
"""Examine cached balrog release history and return the list of
builds we need to generate diffs from"""
platform = _sanitize_platform(platform)
return release_history.get(platform, {}).get(locale, {})
def get_partials_artifacts(release_history, platform, locale):
platform = _sanitize_platform(platform)
return release_history.get(platform, {}).get(locale, {}).keys()
def get_partials_artifact_map(release_history, platform, locale):
platform = _sanitize_platform(platform)
return {k: release_history[platform][locale][k]['buildid']
for k in release_history.get(platform, {}).get(locale, {})}
def _retry_on_http_errors(url, verify, params, errors):
if params:
params_str = "&".join("=".join([k, str(v)])
for k, v in params.iteritems())
else:
params_str = ''
logger.info("Connecting to %s?%s", url, params_str)
for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
try:
req = requests.get(url, verify=verify, params=params, timeout=4)
req.raise_for_status()
return req
except requests.HTTPError as e:
if e.response.status_code in errors:
logger.exception("Got HTTP %s trying to reach %s",
e.response.status_code, url)
else:
raise
else:
raise
def get_sorted_releases(product, branch):
"""Returns a list of release names from Balrog.
:param product: product name, AKA appName
:param branch: branch name, e.g. mozilla-central
:return: a sorted list of release names, most recent first.
"""
url = "{}/releases".format(BALROG_API_ROOT)
params = {
"product": product,
# Adding -nightly-2 (2 stands for the beginning of build ID
# based on date) should filter out release and latest blobs.
# This should be changed to -nightly-3 in 3000 ;)
"name_prefix": "{}-{}-nightly-2".format(product, branch),
"names_only": True
}
req = _retry_on_http_errors(
url=url, verify=True, params=params,
errors=[500])
releases = req.json()["names"]
releases = sorted(releases, reverse=True)
return releases
def get_release_builds(release):
url = "{}/releases/{}".format(BALROG_API_ROOT, release)
req = _retry_on_http_errors(
url=url, verify=True, params=None,
errors=[500])
return req.json()
def populate_release_history(product, branch, maxbuilds=4, maxsearch=10):
"""Find relevant releases in Balrog
Not all releases have all platforms and locales, due
to Taskcluster migration.
Args:
product (str): capitalized product name, AKA appName, e.g. Firefox
branch (str): branch name (mozilla-central)
maxbuilds (int): Maximum number of historical releases to populate
maxsearch(int): Traverse at most this many releases, to avoid
working through the entire history.
Returns:
json object based on data from balrog api
results = {
'platform1': {
'locale1': {
'buildid1': mar_url,
'buildid2': mar_url,
'buildid3': mar_url,
},
'locale2': {
'target.partial-1.mar': {'buildid1': 'mar_url'},
}
},
'platform2': {
}
}
"""
last_releases = get_sorted_releases(product, branch)
partial_mar_tmpl = 'target.partial-{}.mar'
builds = dict()
for release in last_releases[:maxsearch]:
# maxbuilds in all categories, don't make any more queries
full = len(builds) > 0 and all(
len(builds[platform][locale]) >= maxbuilds
for platform in builds for locale in builds[platform])
if full:
break
history = get_release_builds(release)
for platform in history['platforms']:
if 'alias' in history['platforms'][platform]:
continue
if platform not in builds:
builds[platform] = dict()
for locale in history['platforms'][platform]['locales']:
if locale not in builds[platform]:
builds[platform][locale] = dict()
if len(builds[platform][locale]) >= maxbuilds:
continue
buildid = history['platforms'][platform]['locales'][locale]['buildID']
url = history['platforms'][platform]['locales'][locale]['completes'][0]['fileUrl']
nextkey = len(builds[platform][locale]) + 1
builds[platform][locale][partial_mar_tmpl.format(nextkey)] = {
'buildid': buildid,
'mar_url': url,
}
return builds

10
taskcluster/taskgraph/util/taskcluster.py
Просмотреть файл

@ -13,6 +13,9 @@ from mozbuild.util import memoize
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
_TC_ARTIFACT_LOCATION = \
'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
@memoize
def get_session():
@ -101,3 +104,10 @@ def get_task_url(task_id, use_proxy=False):
def get_task_definition(task_id, use_proxy=False):
response = _do_request(get_task_url(task_id, use_proxy))
return response.json()
def get_taskcluster_artifact_prefix(task_id, postfix='', locale=None):
if locale:
postfix = '{}/{}'.format(locale, postfix)
return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)