зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1342392 Move partial update generation in-tree r=dustin,jlorenzo
This commit is contained in:
Simon Fraser
Родитель
10aea37400
Коммит
71165165c4
|
|
@ -13,6 +13,7 @@ kind-dependencies:
|
|||
- beetmover
|
||||
- beetmover-l10n
|
||||
- beetmover-repackage
|
||||
- beetmover-partials
|
||||
|
||||
only-for-attributes:
|
||||
- nightly
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
loader: taskgraph.loader.single_dep:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.name_sanity:transforms
|
||||
- taskgraph.transforms.beetmover_repackage_l10n:transforms
|
||||
- taskgraph.transforms.beetmover_repackage:transforms
|
||||
- taskgraph.transforms.beetmover_partials:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- partials-signing
|
||||
|
|
@ -26,3 +26,5 @@ jobs:
|
|||
symbol: I(agb)
|
||||
index-task:
|
||||
symbol: I(idx)
|
||||
partial-update-generator:
|
||||
symbol: I(pg)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
loader: taskgraph.loader.single_dep:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.name_sanity:transforms
|
||||
- taskgraph.transforms.partials_signing:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- partials
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
loader: taskgraph.loader.single_dep:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.name_sanity:transforms
|
||||
- taskgraph.transforms.partials:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- repackage-signing
|
||||
|
||||
only-for-attributes:
|
||||
- nightly
|
||||
|
||||
only-for-build-platforms:
|
||||
- macosx64-nightly/opt
|
||||
- win32-nightly/opt
|
||||
- win64-nightly/opt
|
||||
- linux-nightly/opt
|
||||
- linux64-nightly/opt
|
||||
|
|
@ -25,6 +25,7 @@ RUN pip install -r /tmp/requirements.txt
|
|||
# scripts
|
||||
RUN mkdir /home/worker/bin
|
||||
COPY scripts/* /home/worker/bin/
|
||||
|
||||
COPY runme.sh /runme.sh
|
||||
RUN chmod 755 /home/worker/bin/* /runme.sh
|
||||
RUN mkdir /home/worker/keys
|
||||
|
|
@ -34,3 +35,5 @@ ENV HOME /home/worker
|
|||
ENV SHELL /bin/bash
|
||||
ENV USER worker
|
||||
ENV LOGNAME worker
|
||||
|
||||
CMD ["/runme.sh"]
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ ALLOWED_URL_PREFIXES = [
|
|||
"http://ftp.mozilla.org/",
|
||||
"http://download.mozilla.org/",
|
||||
"https://archive.mozilla.org/",
|
||||
"https://queue.taskcluster.net/v1/task/",
|
||||
]
|
||||
|
||||
DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
|
||||
|
|
@ -286,7 +287,11 @@ def main():
|
|||
# if branch not set explicitly use repo-name
|
||||
mar_data["branch"] = e.get("branch",
|
||||
mar_data["repo"].rstrip("/").split("/")[-1])
|
||||
mar_name = args.filename_template.format(**mar_data)
|
||||
if 'dest_mar' in e:
|
||||
mar_name = e['dest_mar']
|
||||
else:
|
||||
# default to formatted name if not specified
|
||||
mar_name = args.filename_template.format(**mar_data)
|
||||
mar_data["mar"] = mar_name
|
||||
dest_mar = os.path.join(work_env.workdir, mar_name)
|
||||
# TODO: download these once
|
||||
|
|
|
|||
|
|
@ -0,0 +1,311 @@
|
|||
#!/usr/bin/env python
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import ConfigParser
|
||||
import argparse
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import requests
|
||||
import sh
|
||||
|
||||
import redo
|
||||
from mardor.reader import MarReader
|
||||
from mardor.signing import get_keysize
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
ALLOWED_URL_PREFIXES = [
|
||||
"http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/",
|
||||
"http://download.cdn.mozilla.net/pub/firefox/nightly/",
|
||||
"https://mozilla-nightly-updates.s3.amazonaws.com",
|
||||
"https://queue.taskcluster.net/",
|
||||
"http://ftp.mozilla.org/",
|
||||
"http://download.mozilla.org/",
|
||||
"https://archive.mozilla.org/",
|
||||
]
|
||||
|
||||
DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
|
||||
"{locale}-{from_buildid}-{to_buildid}.partial.mar"
|
||||
|
||||
|
||||
def verify_signature(mar, certs):
|
||||
log.info("Checking %s signature", mar)
|
||||
with open(mar, 'rb') as mar_fh:
|
||||
m = MarReader(mar_fh)
|
||||
m.verify(verify_key=certs.get(m.signature_type))
|
||||
|
||||
|
||||
def is_lzma_compressed_mar(mar):
|
||||
log.info("Checking %s for lzma compression", mar)
|
||||
result = MarReader(open(mar, 'rb')).compression_type == 'xz'
|
||||
if result:
|
||||
log.info("%s is lzma compressed", mar)
|
||||
else:
|
||||
log.info("%s is not lzma compressed", mar)
|
||||
return result
|
||||
|
||||
|
||||
@redo.retriable()
|
||||
def download(url, dest, mode=None):
|
||||
log.debug("Downloading %s to %s", url, dest)
|
||||
r = requests.get(url)
|
||||
r.raise_for_status()
|
||||
|
||||
bytes_downloaded = 0
|
||||
with open(dest, 'wb') as fd:
|
||||
for chunk in r.iter_content(4096):
|
||||
fd.write(chunk)
|
||||
bytes_downloaded += len(chunk)
|
||||
|
||||
log.debug('Downloaded %s bytes', bytes_downloaded)
|
||||
if 'content-length' in r.headers:
|
||||
log.debug('Content-Length: %s bytes', r.headers['content-length'])
|
||||
if bytes_downloaded != int(r.headers['content-length']):
|
||||
raise IOError('Unexpected number of bytes downloaded')
|
||||
|
||||
if mode:
|
||||
log.debug("chmod %o %s", mode, dest)
|
||||
os.chmod(dest, mode)
|
||||
|
||||
|
||||
def unpack(work_env, mar, dest_dir):
|
||||
os.mkdir(dest_dir)
|
||||
unwrap_cmd = sh.Command(os.path.join(work_env.workdir,
|
||||
"unwrap_full_update.pl"))
|
||||
log.debug("Unwrapping %s", mar)
|
||||
env = work_env.env
|
||||
if not is_lzma_compressed_mar(mar):
|
||||
env['MAR_OLD_FORMAT'] = '1'
|
||||
elif 'MAR_OLD_FORMAT' in env:
|
||||
del env['MAR_OLD_FORMAT']
|
||||
out = unwrap_cmd(mar, _cwd=dest_dir, _env=env, _timeout=240,
|
||||
_err_to_out=True)
|
||||
if out:
|
||||
log.debug(out)
|
||||
|
||||
|
||||
def find_file(directory, filename):
|
||||
log.debug("Searching for %s in %s", filename, directory)
|
||||
for root, dirs, files in os.walk(directory):
|
||||
if filename in files:
|
||||
f = os.path.join(root, filename)
|
||||
log.debug("Found %s", f)
|
||||
return f
|
||||
|
||||
|
||||
def get_option(directory, filename, section, option):
|
||||
log.debug("Exctracting [%s]: %s from %s/**/%s", section, option, directory,
|
||||
filename)
|
||||
f = find_file(directory, filename)
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.read(f)
|
||||
rv = config.get(section, option)
|
||||
log.debug("Found %s", rv)
|
||||
return rv
|
||||
|
||||
|
||||
def generate_partial(work_env, from_dir, to_dir, dest_mar, channel_ids,
|
||||
version, use_old_format):
|
||||
log.debug("Generating partial %s", dest_mar)
|
||||
env = work_env.env
|
||||
env["MOZ_PRODUCT_VERSION"] = version
|
||||
env["MOZ_CHANNEL_ID"] = channel_ids
|
||||
if use_old_format:
|
||||
env['MAR_OLD_FORMAT'] = '1'
|
||||
elif 'MAR_OLD_FORMAT' in env:
|
||||
del env['MAR_OLD_FORMAT']
|
||||
make_incremental_update = os.path.join(work_env.workdir,
|
||||
"make_incremental_update.sh")
|
||||
out = sh.bash(make_incremental_update, dest_mar, from_dir, to_dir,
|
||||
_cwd=work_env.workdir, _env=env, _timeout=900,
|
||||
_err_to_out=True)
|
||||
if out:
|
||||
log.debug(out)
|
||||
|
||||
|
||||
def get_hash(path, hash_type="sha512"):
|
||||
h = hashlib.new(hash_type)
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(functools.partial(f.read, 4096), ''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
class WorkEnv(object):
|
||||
|
||||
def __init__(self):
|
||||
self.workdir = tempfile.mkdtemp()
|
||||
|
||||
def setup(self):
|
||||
self.download_unwrap()
|
||||
self.download_martools()
|
||||
|
||||
def download_unwrap(self):
|
||||
# unwrap_full_update.pl is not too sensitive to the revision
|
||||
url = "https://hg.mozilla.org/mozilla-central/raw-file/default/" \
|
||||
"tools/update-packaging/unwrap_full_update.pl"
|
||||
download(url, dest=os.path.join(self.workdir, "unwrap_full_update.pl"),
|
||||
mode=0o755)
|
||||
|
||||
def download_buildsystem_bits(self, repo, revision):
|
||||
prefix = "{repo}/raw-file/{revision}/tools/update-packaging"
|
||||
prefix = prefix.format(repo=repo, revision=revision)
|
||||
for f in ("make_incremental_update.sh", "common.sh"):
|
||||
url = "{prefix}/{f}".format(prefix=prefix, f=f)
|
||||
download(url, dest=os.path.join(self.workdir, f), mode=0o755)
|
||||
|
||||
def download_martools(self):
|
||||
# TODO: check if the tools have to be branch specific
|
||||
prefix = "https://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/" \
|
||||
"latest-mozilla-central/mar-tools/linux64"
|
||||
for f in ("mar", "mbsdiff"):
|
||||
url = "{prefix}/{f}".format(prefix=prefix, f=f)
|
||||
download(url, dest=os.path.join(self.workdir, f), mode=0o755)
|
||||
|
||||
def cleanup(self):
|
||||
shutil.rmtree(self.workdir)
|
||||
|
||||
@property
|
||||
def env(self):
|
||||
my_env = os.environ.copy()
|
||||
my_env['LC_ALL'] = 'C'
|
||||
my_env['MAR'] = os.path.join(self.workdir, "mar")
|
||||
my_env['MBSDIFF'] = os.path.join(self.workdir, "mbsdiff")
|
||||
return my_env
|
||||
|
||||
|
||||
def verify_allowed_url(mar):
|
||||
if not any(mar.startswith(prefix) for prefix in ALLOWED_URL_PREFIXES):
|
||||
raise ValueError("{mar} is not in allowed URL prefixes: {p}".format(
|
||||
mar=mar, p=ALLOWED_URL_PREFIXES
|
||||
))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--artifacts-dir", required=True)
|
||||
parser.add_argument("--sha1-signing-cert", required=True)
|
||||
parser.add_argument("--sha384-signing-cert", required=True)
|
||||
parser.add_argument("--task-definition", required=True,
|
||||
type=argparse.FileType('r'))
|
||||
parser.add_argument("--filename-template",
|
||||
default=DEFAULT_FILENAME_TEMPLATE)
|
||||
parser.add_argument("--no-freshclam", action="store_true", default=False,
|
||||
help="Do not refresh ClamAV DB")
|
||||
parser.add_argument("-q", "--quiet", dest="log_level",
|
||||
action="store_const", const=logging.WARNING,
|
||||
default=logging.DEBUG)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
level=args.log_level)
|
||||
task = json.load(args.task_definition)
|
||||
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
|
||||
|
||||
signing_certs = {
|
||||
'sha1': open(args.sha1_signing_cert, 'rb').read(),
|
||||
'sha384': open(args.sha384_signing_cert, 'rb').read(),
|
||||
}
|
||||
|
||||
assert(get_keysize(signing_certs['sha1']) == 2048)
|
||||
assert(get_keysize(signing_certs['sha384']) == 4096)
|
||||
|
||||
if args.no_freshclam:
|
||||
log.info("Skipping freshclam")
|
||||
else:
|
||||
log.info("Refreshing clamav db...")
|
||||
try:
|
||||
redo.retry(lambda: sh.freshclam("--stdout", "--verbose",
|
||||
_timeout=300, _err_to_out=True))
|
||||
log.info("Done.")
|
||||
except sh.ErrorReturnCode:
|
||||
log.warning("Freshclam failed, skipping DB update")
|
||||
manifest = []
|
||||
for e in task["extra"]["funsize"]["partials"]:
|
||||
for mar in (e["from_mar"], e["to_mar"]):
|
||||
verify_allowed_url(mar)
|
||||
|
||||
work_env = WorkEnv()
|
||||
# TODO: run setup once
|
||||
work_env.setup()
|
||||
complete_mars = {}
|
||||
use_old_format = False
|
||||
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
|
||||
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
|
||||
unpack_dir = os.path.join(work_env.workdir, mar_type)
|
||||
download(f, dest)
|
||||
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
|
||||
verify_signature(dest, signing_certs)
|
||||
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
|
||||
complete_mars["%s_hash" % mar_type] = get_hash(dest)
|
||||
if mar_type == 'to' and not is_lzma_compressed_mar(dest):
|
||||
use_old_format = True
|
||||
unpack(work_env, dest, unpack_dir)
|
||||
log.info("AV-scanning %s ...", unpack_dir)
|
||||
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
|
||||
log.info("Done.")
|
||||
|
||||
path = os.path.join(work_env.workdir, "to")
|
||||
from_path = os.path.join(work_env.workdir, "from")
|
||||
mar_data = {
|
||||
"ACCEPTED_MAR_CHANNEL_IDS": get_option(
|
||||
path, filename="update-settings.ini", section="Settings",
|
||||
option="ACCEPTED_MAR_CHANNEL_IDS"),
|
||||
"version": get_option(path, filename="application.ini",
|
||||
section="App", option="Version"),
|
||||
"to_buildid": get_option(path, filename="application.ini",
|
||||
section="App", option="BuildID"),
|
||||
"from_buildid": get_option(from_path, filename="application.ini",
|
||||
section="App", option="BuildID"),
|
||||
"appName": get_option(from_path, filename="application.ini",
|
||||
section="App", option="Name"),
|
||||
# Use Gecko repo and rev from platform.ini, not application.ini
|
||||
"repo": get_option(path, filename="platform.ini", section="Build",
|
||||
option="SourceRepository"),
|
||||
"revision": get_option(path, filename="platform.ini",
|
||||
section="Build", option="SourceStamp"),
|
||||
"from_mar": e["from_mar"],
|
||||
"to_mar": e["to_mar"],
|
||||
"platform": e["platform"],
|
||||
"locale": e["locale"],
|
||||
}
|
||||
# Override ACCEPTED_MAR_CHANNEL_IDS if needed
|
||||
if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ:
|
||||
mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ["ACCEPTED_MAR_CHANNEL_IDS"]
|
||||
for field in ("update_number", "previousVersion",
|
||||
"previousBuildNumber", "toVersion",
|
||||
"toBuildNumber"):
|
||||
if field in e:
|
||||
mar_data[field] = e[field]
|
||||
mar_data.update(complete_mars)
|
||||
# if branch not set explicitly use repo-name
|
||||
mar_data["branch"] = e.get("branch",
|
||||
mar_data["repo"].rstrip("/").split("/")[-1])
|
||||
mar_name = args.filename_template.format(**mar_data)
|
||||
mar_data["mar"] = mar_name
|
||||
dest_mar = os.path.join(work_env.workdir, mar_name)
|
||||
# TODO: download these once
|
||||
work_env.download_buildsystem_bits(repo=mar_data["repo"],
|
||||
revision=mar_data["revision"])
|
||||
generate_partial(work_env, from_path, path, dest_mar,
|
||||
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
|
||||
mar_data["version"],
|
||||
use_old_format)
|
||||
mar_data["size"] = os.path.getsize(dest_mar)
|
||||
mar_data["hash"] = get_hash(dest_mar)
|
||||
|
||||
shutil.copy(dest_mar, args.artifacts_dir)
|
||||
work_env.cleanup()
|
||||
manifest.append(mar_data)
|
||||
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
|
||||
with open(manifest_file, "w") as fp:
|
||||
json.dump(manifest, fp, indent=2, sort_keys=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -193,6 +193,14 @@ Beetmover-repackage is beetmover but for tasks that need an intermediate step
|
|||
between signing and packaging, such as OSX. For more details see the definitions
|
||||
of the Beetmover kind above and the repackage kind below.
|
||||
|
||||
beetmover-partials
|
||||
------------------
|
||||
|
||||
Beetmover-partials is beetmover but for the partial updates that have been
|
||||
generated. Not every build produces partial updates, and so these are kept
|
||||
separate from the regular beetmover jobs to avoid situations where the completes
|
||||
are not uploaded.
|
||||
|
||||
checksums-signing
|
||||
-----------------
|
||||
Checksums-signing take as input the checksums file generated by beetmover tasks
|
||||
|
|
@ -233,3 +241,13 @@ repackage-signing
|
|||
-----------------
|
||||
Repackage-signing take the repackaged installers (windows) and update packaging (with
|
||||
the signed internal bits) and signs them.
|
||||
|
||||
partials
|
||||
--------
|
||||
Partials takes the complete.mar files produced in previous tasks and generates partial
|
||||
updates between previous nightly releases and the new one. Requires a release_history
|
||||
in the parameters. See ``mach release-history`` if doing this manually.
|
||||
|
||||
partials-signing
|
||||
----------------
|
||||
Partials-signing takes the partial updates produced in Partials and signs them.
|
||||
|
|
|
|||
|
|
@ -107,6 +107,12 @@ syntax or reading a project-specific configuration file).
|
|||
``include_nightly``
|
||||
If true, then nightly tasks are eligible for optimization.
|
||||
|
||||
``release_history``
|
||||
History of recent releases by platform and locale, used when generating
|
||||
partial updates for nightly releases.
|
||||
Suitable contents can be generated with ``mach release-history``,
|
||||
which will print to the console by default.
|
||||
|
||||
Morphed Set
|
||||
-----------
|
||||
|
||||
|
|
|
|||
|
|
@ -504,3 +504,22 @@ class TaskClusterImagesProvider(object):
|
|||
except Exception:
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
@CommandProvider
|
||||
class TaskClusterPartialsData(object):
|
||||
@Command('release-history', category="ci",
|
||||
description="Query balrog for release history used by enable partials generation")
|
||||
@CommandArgument('-b', '--branch',
|
||||
help="The gecko project branch used in balrog, such as "
|
||||
"mozilla-central, release, date")
|
||||
@CommandArgument('--product', default='Firefox',
|
||||
help="The product identifier, such as 'Firefox'")
|
||||
def generate_partials_builds(self, product, branch):
|
||||
from taskgraph.util.partials import populate_release_history
|
||||
try:
|
||||
import yaml
|
||||
release_history = {'release_history': populate_release_history(product, branch)}
|
||||
print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from .create import create_tasks
|
|||
from .parameters import Parameters
|
||||
from .taskgraph import TaskGraph
|
||||
from .actions import render_actions_json
|
||||
from taskgraph.util.partials import populate_release_history
|
||||
from . import GECKO
|
||||
|
||||
from taskgraph.util.templates import Templates
|
||||
|
|
@ -107,6 +108,7 @@ def taskgraph_decision(options):
|
|||
"""
|
||||
|
||||
parameters = get_decision_parameters(options)
|
||||
|
||||
# create a TaskGraphGenerator instance
|
||||
tgg = TaskGraphGenerator(
|
||||
root_dir=options['root'],
|
||||
|
|
@ -202,6 +204,13 @@ def get_decision_parameters(options):
|
|||
if options.get('target_tasks_method'):
|
||||
parameters['target_tasks_method'] = options['target_tasks_method']
|
||||
|
||||
# If the target method is nightly, we should build partials. This means
|
||||
# knowing what has been released previously.
|
||||
# An empty release_history is fine, it just means no partials will be built
|
||||
parameters.setdefault('release_history', dict())
|
||||
if 'nightly' in parameters.get('target_tasks_method', ''):
|
||||
parameters['release_history'] = populate_release_history('Firefox', project)
|
||||
|
||||
return Parameters(parameters)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ PARAMETER_NAMES = set([
|
|||
'project',
|
||||
'pushdate',
|
||||
'pushlog_id',
|
||||
'release_history',
|
||||
'target_task_labels',
|
||||
'target_tasks_method',
|
||||
])
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ def make_task_description(config, jobs):
|
|||
dep_job = job['dependent-task']
|
||||
|
||||
treeherder = job.get('treeherder', {})
|
||||
treeherder.setdefault('symbol', 'tc-Up(N)')
|
||||
treeherder.setdefault('symbol', 'c-Up(N)')
|
||||
dep_th_platform = dep_job.task.get('extra', {}).get(
|
||||
'treeherder', {}).get('machine', {}).get('platform', '')
|
||||
treeherder.setdefault('platform',
|
||||
|
|
@ -66,11 +66,18 @@ def make_task_description(config, jobs):
|
|||
|
||||
attributes = copy_attributes_from_dependent_job(dep_job)
|
||||
|
||||
treeherder_job_symbol = dep_job.attributes.get('locale', 'N')
|
||||
|
||||
if dep_job.attributes.get('locale'):
|
||||
treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
|
||||
treeherder['symbol'] = 'c-Up({})'.format(treeherder_job_symbol)
|
||||
attributes['locale'] = dep_job.attributes.get('locale')
|
||||
|
||||
label = job['label']
|
||||
if 'partials' in dep_job.kind:
|
||||
label = "partials-{}".format(label)
|
||||
treeherder['symbol'] = 'cp-Up({})'.format(treeherder_job_symbol)
|
||||
treeherder['tier'] = 3 # remove once proven stable
|
||||
|
||||
description = (
|
||||
"Balrog submission for locale '{locale}' for build '"
|
||||
"{build_platform}/{build_type}'".format(
|
||||
|
|
@ -94,7 +101,6 @@ def make_task_description(config, jobs):
|
|||
task = {
|
||||
'label': label,
|
||||
'description': description,
|
||||
# do we have to define worker type somewhere?
|
||||
'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
|
||||
'worker': {
|
||||
'implementation': 'balrog',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Add partial update artifacts to a beetmover task.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.partials import (get_balrog_platform_name,
|
||||
get_partials_artifacts,
|
||||
get_partials_artifact_map)
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
def generate_upstream_artifacts(release_history, platform, locale=None):
|
||||
if not locale or locale == 'en-US':
|
||||
artifact_prefix = 'public/build'
|
||||
else:
|
||||
artifact_prefix = 'public/build/{}'.format(locale)
|
||||
|
||||
artifacts = get_partials_artifacts(release_history, platform, locale)
|
||||
|
||||
upstream_artifacts = [{
|
||||
'taskId': {'task-reference': '<partials-signing>'},
|
||||
'taskType': 'signing',
|
||||
'paths': ["{}/{}".format(artifact_prefix, p)
|
||||
for p in artifacts],
|
||||
'locale': locale or 'en-US',
|
||||
}]
|
||||
|
||||
return upstream_artifacts
|
||||
|
||||
|
||||
@transforms.add
|
||||
def make_partials_artifacts(config, jobs):
|
||||
for job in jobs:
|
||||
locale = job["attributes"].get("locale")
|
||||
if locale:
|
||||
job['treeherder']['symbol'] = 'pBM({})'.format(locale)
|
||||
else:
|
||||
locale = 'en-US'
|
||||
job['treeherder']['symbol'] = 'pBM(N)'
|
||||
|
||||
# Remove when proved reliable
|
||||
job['treeherder']['tier'] = 3
|
||||
|
||||
platform = job["attributes"]["build_platform"]
|
||||
|
||||
platform = get_balrog_platform_name(platform)
|
||||
upstream_artifacts = generate_upstream_artifacts(
|
||||
config.params.get('release_history'), platform, locale
|
||||
)
|
||||
|
||||
job['worker']['upstream-artifacts'].extend(upstream_artifacts)
|
||||
|
||||
extra = list()
|
||||
|
||||
artifact_map = get_partials_artifact_map(
|
||||
config.params.get('release_history'), platform, locale)
|
||||
for artifact in artifact_map:
|
||||
extra.append({
|
||||
'locale': locale,
|
||||
'artifact_name': artifact,
|
||||
'buildid': artifact_map[artifact],
|
||||
'platform': platform,
|
||||
})
|
||||
|
||||
job.setdefault('extra', {})
|
||||
job['extra']['partials'] = extra
|
||||
|
||||
yield job
|
||||
|
|
@ -215,6 +215,14 @@ def make_task_description(config, jobs):
|
|||
}
|
||||
dependencies.update(repackage_dependencies)
|
||||
|
||||
# If this isn't a direct dependency, it won't be in there.
|
||||
if 'repackage-signing' not in dependencies:
|
||||
repackage_signing_name = "repackage-signing"
|
||||
repackage_signing_deps = {"repackage-signing":
|
||||
dep_job.dependencies[repackage_signing_name]
|
||||
}
|
||||
dependencies.update(repackage_signing_deps)
|
||||
|
||||
attributes = copy_attributes_from_dependent_job(dep_job)
|
||||
if job.get('locale'):
|
||||
attributes['locale'] = job['locale']
|
||||
|
|
@ -273,7 +281,6 @@ def generate_upstream_artifacts(build_task_ref, build_signing_task_ref,
|
|||
_check_platform_matched_only_one_regex(
|
||||
tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
|
||||
)
|
||||
|
||||
upstream_artifacts.append({
|
||||
"taskId": {"task-reference": ref},
|
||||
"taskType": tasktype,
|
||||
|
|
@ -299,8 +306,12 @@ least 2 regular expressions. First matched: "{first_matched}". Second matched: \
|
|||
|
||||
|
||||
def is_valid_beetmover_job(job):
|
||||
# windows builds don't have docker-image, so fewer dependencies
|
||||
if any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):
|
||||
# beetmover after partials-signing should have six dependencies.
|
||||
# windows builds w/o partials don't have docker-image, so fewer
|
||||
# dependencies
|
||||
if 'partials-signing' in job['dependencies'].keys():
|
||||
expected_dep_count = 6
|
||||
elif any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):
|
||||
expected_dep_count = 4
|
||||
else:
|
||||
expected_dep_count = 5
|
||||
|
|
@ -321,6 +332,7 @@ def make_task_worker(config, jobs):
|
|||
build_signing_task = None
|
||||
repackage_task = None
|
||||
repackage_signing_task = None
|
||||
|
||||
for dependency in job["dependencies"].keys():
|
||||
if 'repackage-signing' in dependency:
|
||||
repackage_signing_task = dependency
|
||||
|
|
|
|||
|
|
@ -0,0 +1,133 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Transform the partials task into an actual task description.
|
||||
"""
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.attributes import copy_attributes_from_dependent_job
|
||||
from taskgraph.util.partials import get_balrog_platform_name, get_builds
|
||||
from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
def _generate_task_output_files(filenames, locale=None):
|
||||
locale_output_path = '{}/'.format(locale) if locale else ''
|
||||
|
||||
data = list()
|
||||
for filename in filenames:
|
||||
data.append({
|
||||
'type': 'file',
|
||||
'path': '/home/worker/artifacts/{}'.format(filename),
|
||||
'name': 'public/build/{}{}'.format(locale_output_path, filename)
|
||||
})
|
||||
data.append({
|
||||
'type': 'file',
|
||||
'path': '/home/worker/artifacts/manifest.json',
|
||||
'name': 'public/build/{}manifest.json'.format(locale_output_path)
|
||||
})
|
||||
return data
|
||||
|
||||
|
||||
@transforms.add
|
||||
def make_task_description(config, jobs):
|
||||
# If no balrog release history, then don't generate partials
|
||||
if not config.params.get('release_history'):
|
||||
return
|
||||
for job in jobs:
|
||||
dep_job = job['dependent-task']
|
||||
|
||||
treeherder = job.get('treeherder', {})
|
||||
treeherder.setdefault('symbol', 'p(N)')
|
||||
|
||||
label = job.get('label', "partials-{}".format(dep_job.label))
|
||||
dep_th_platform = dep_job.task.get('extra', {}).get(
|
||||
'treeherder', {}).get('machine', {}).get('platform', '')
|
||||
|
||||
treeherder.setdefault('platform',
|
||||
"{}/opt".format(dep_th_platform))
|
||||
treeherder.setdefault('kind', 'build')
|
||||
treeherder.setdefault('tier', 3)
|
||||
|
||||
dependent_kind = str(dep_job.kind)
|
||||
dependencies = {dependent_kind: dep_job.label}
|
||||
signing_dependencies = dep_job.dependencies
|
||||
# This is so we get the build task etc in our dependencies to
|
||||
# have better beetmover support.
|
||||
dependencies.update(signing_dependencies)
|
||||
|
||||
attributes = copy_attributes_from_dependent_job(dep_job)
|
||||
locale = dep_job.attributes.get('locale')
|
||||
if locale:
|
||||
attributes['locale'] = locale
|
||||
treeherder['symbol'] = "p({})".format(locale)
|
||||
|
||||
build_locale = locale or 'en-US'
|
||||
|
||||
builds = get_builds(config.params['release_history'], dep_th_platform,
|
||||
build_locale)
|
||||
|
||||
# If the list is empty there's no available history for this platform
|
||||
# and locale combination, so we can't build any partials.
|
||||
if not builds:
|
||||
continue
|
||||
|
||||
signing_task = None
|
||||
for dependency in dependencies.keys():
|
||||
if 'repackage-signing' in dependency:
|
||||
signing_task = dependency
|
||||
signing_task_ref = '<{}>'.format(signing_task)
|
||||
|
||||
extra = {'funsize': {'partials': list()}}
|
||||
update_number = 1
|
||||
artifact_path = "{}{}".format(get_taskcluster_artifact_prefix(signing_task_ref, locale=locale), 'target.complete.mar')
|
||||
for build in builds:
|
||||
extra['funsize']['partials'].append({
|
||||
'locale': build_locale,
|
||||
'from_mar': builds[build]['mar_url'],
|
||||
'to_mar': {'task-reference': artifact_path},
|
||||
'platform': get_balrog_platform_name(dep_th_platform),
|
||||
'branch': config.params['project'],
|
||||
'update_number': update_number,
|
||||
'dest_mar': build,
|
||||
})
|
||||
update_number += 1
|
||||
|
||||
cot = extra.setdefault('chainOfTrust', {})
|
||||
cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
|
||||
|
||||
worker = {
|
||||
'artifacts': _generate_task_output_files(builds.keys(), locale),
|
||||
'implementation': 'docker-worker',
|
||||
'docker-image': {'in-tree': 'partial-update-generator'},
|
||||
'os': 'linux',
|
||||
'max-run-time': 3600,
|
||||
'chain-of-trust': True,
|
||||
'env': {
|
||||
'SHA1_SIGNING_CERT': 'nightly_sha1',
|
||||
'SHA384_SIGNING_CERT': 'nightly_sha384'
|
||||
}
|
||||
}
|
||||
|
||||
level = config.params['level']
|
||||
|
||||
task = {
|
||||
'label': label,
|
||||
'description': "{} Partials".format(
|
||||
dep_job.task["metadata"]["description"]),
|
||||
'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
|
||||
'dependencies': dependencies,
|
||||
'attributes': attributes,
|
||||
'run-on-projects': dep_job.attributes.get('run_on_projects'),
|
||||
'treeherder': treeherder,
|
||||
'extra': extra,
|
||||
'worker': worker,
|
||||
}
|
||||
|
||||
yield task
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Transform the partials task into an actual task description.
|
||||
"""
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.attributes import copy_attributes_from_dependent_job
|
||||
from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
|
||||
from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
def generate_upstream_artifacts(release_history, platform, locale=None):
|
||||
artifact_prefix = 'public/build'
|
||||
if locale:
|
||||
artifact_prefix = 'public/build/{}'.format(locale)
|
||||
else:
|
||||
locale = 'en-US'
|
||||
|
||||
artifacts = get_partials_artifacts(release_history, platform, locale)
|
||||
|
||||
upstream_artifacts = [{
|
||||
"taskId": {"task-reference": '<partials>'},
|
||||
"taskType": 'partials',
|
||||
"paths": ["{}/{}".format(artifact_prefix, p)
|
||||
for p in artifacts],
|
||||
"formats": ["mar_sha384"],
|
||||
}]
|
||||
|
||||
return upstream_artifacts
|
||||
|
||||
|
||||
@transforms.add
|
||||
def make_task_description(config, jobs):
|
||||
for job in jobs:
|
||||
dep_job = job['dependent-task']
|
||||
|
||||
treeherder = job.get('treeherder', {})
|
||||
treeherder.setdefault('symbol', 'ps(N)')
|
||||
|
||||
dep_th_platform = dep_job.task.get('extra', {}).get(
|
||||
'treeherder', {}).get('machine', {}).get('platform', '')
|
||||
label = job.get('label', "partials-signing-{}".format(dep_job.label))
|
||||
dep_th_platform = dep_job.task.get('extra', {}).get(
|
||||
'treeherder', {}).get('machine', {}).get('platform', '')
|
||||
treeherder.setdefault('platform',
|
||||
"{}/opt".format(dep_th_platform))
|
||||
treeherder.setdefault('kind', 'build')
|
||||
treeherder.setdefault('tier', 3)
|
||||
|
||||
dependent_kind = str(dep_job.kind)
|
||||
dependencies = {dependent_kind: dep_job.label}
|
||||
signing_dependencies = dep_job.dependencies
|
||||
# This is so we get the build task etc in our dependencies to
|
||||
# have better beetmover support.
|
||||
dependencies.update(signing_dependencies)
|
||||
|
||||
attributes = copy_attributes_from_dependent_job(dep_job)
|
||||
locale = dep_job.attributes.get('locale')
|
||||
if locale:
|
||||
attributes['locale'] = locale
|
||||
treeherder['symbol'] = 'ps({})'.format(locale)
|
||||
|
||||
balrog_platform = get_balrog_platform_name(dep_th_platform)
|
||||
upstream_artifacts = generate_upstream_artifacts(config.params['release_history'], balrog_platform, locale)
|
||||
|
||||
build_platform = dep_job.attributes.get('build_platform')
|
||||
is_nightly = dep_job.attributes.get('nightly')
|
||||
signing_cert_scope = get_signing_cert_scope_per_platform(
|
||||
build_platform, is_nightly, config
|
||||
)
|
||||
scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384']
|
||||
task = {
|
||||
'label': label,
|
||||
'description': "{} Partials".format(
|
||||
dep_job.task["metadata"]["description"]),
|
||||
'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
|
||||
'worker': {'implementation': 'scriptworker-signing',
|
||||
'upstream-artifacts': upstream_artifacts,
|
||||
'max-run-time': 3600},
|
||||
'dependencies': dependencies,
|
||||
'attributes': attributes,
|
||||
'scopes': scopes,
|
||||
'run-on-projects': dep_job.attributes.get('run_on_projects'),
|
||||
'treeherder': treeherder,
|
||||
}
|
||||
|
||||
yield task
|
||||
|
|
@ -10,12 +10,10 @@ from __future__ import absolute_import, print_function, unicode_literals
|
|||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.attributes import copy_attributes_from_dependent_job
|
||||
from taskgraph.util.schema import validate_schema, Schema
|
||||
from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
|
||||
from taskgraph.transforms.task import task_description_schema
|
||||
from voluptuous import Any, Required, Optional
|
||||
|
||||
_TC_ARTIFACT_LOCATION = \
|
||||
'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
# Voluptuous uses marker objects as dictionary *keys*, but they are not
|
||||
|
|
@ -203,8 +201,8 @@ def _generate_task_mozharness_config(build_platform):
|
|||
|
||||
|
||||
def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
|
||||
mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None)
|
||||
signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale)
|
||||
mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None)
|
||||
signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale)
|
||||
|
||||
if build_platform.startswith('linux') or build_platform.startswith('macosx'):
|
||||
tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
|
||||
|
|
@ -231,13 +229,6 @@ def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=
|
|||
raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
|
||||
|
||||
|
||||
def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
|
||||
if locale:
|
||||
postfix = '{}/{}'.format(locale, postfix)
|
||||
|
||||
return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
|
||||
|
||||
|
||||
def _generate_task_output_files(build_platform, locale=None):
|
||||
locale_output_path = '{}/'.format(locale) if locale else ''
|
||||
|
||||
|
|
|
|||
|
|
@ -129,20 +129,6 @@ def make_repackage_signing_description(config, jobs):
|
|||
'treeherder': treeherder,
|
||||
}
|
||||
|
||||
funsize_platforms = [
|
||||
'linux-nightly',
|
||||
'linux64-nightly',
|
||||
'macosx64-nightly',
|
||||
'win32-nightly',
|
||||
'win64-nightly'
|
||||
]
|
||||
if build_platform in funsize_platforms and is_nightly:
|
||||
route_template = "project.releng.funsize.level-{level}.{project}"
|
||||
task['routes'] = [
|
||||
route_template.format(project=config.params['project'],
|
||||
level=config.params['level'])
|
||||
]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -499,7 +499,7 @@ GROUP_NAMES = {
|
|||
'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
|
||||
'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
|
||||
'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster',
|
||||
'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
|
||||
'c-Up': 'Balrog submission of complete updates',
|
||||
'tc-cs': 'Checksum signing executed by Taskcluster',
|
||||
'tc-rs': 'Repackage signing executed by Taskcluster',
|
||||
'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
|
||||
|
|
@ -512,6 +512,10 @@ GROUP_NAMES = {
|
|||
'TW64': 'Toolchain builds for Windows 64-bits',
|
||||
'SM-tc': 'Spidermonkey builds',
|
||||
'pub': 'APK publishing',
|
||||
'p': 'Partial generation',
|
||||
'ps': 'Partials signing',
|
||||
'pBM': 'Beetmover for partials',
|
||||
'cp-Up': 'Balrog submission of updates, completes and partials',
|
||||
}
|
||||
UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
|
||||
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,193 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
import json
|
||||
import os
|
||||
|
||||
import requests
|
||||
import redo
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BALROG_API_ROOT = 'https://aus5.mozilla.org/api/v1'
|
||||
|
||||
PLATFORM_RENAMES = {
|
||||
'windows2012-32': 'win32',
|
||||
'windows2012-64': 'win64',
|
||||
'osx-cross': 'macosx64',
|
||||
}
|
||||
|
||||
BALROG_PLATFORM_MAP = {
|
||||
"linux": [
|
||||
"Linux_x86-gcc3"
|
||||
],
|
||||
"linux64": [
|
||||
"Linux_x86_64-gcc3"
|
||||
],
|
||||
"macosx64": [
|
||||
"Darwin_x86_64-gcc3-u-i386-x86_64",
|
||||
"Darwin_x86-gcc3-u-i386-x86_64",
|
||||
"Darwin_x86-gcc3",
|
||||
"Darwin_x86_64-gcc3"
|
||||
],
|
||||
"win32": [
|
||||
"WINNT_x86-msvc",
|
||||
"WINNT_x86-msvc-x86",
|
||||
"WINNT_x86-msvc-x64"
|
||||
],
|
||||
"win64": [
|
||||
"WINNT_x86_64-msvc",
|
||||
"WINNT_x86_64-msvc-x64"
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def get_balrog_platform_name(platform):
|
||||
"""Convert build platform names into balrog platform names"""
|
||||
if '-nightly' in platform:
|
||||
platform = platform.replace('-nightly', '')
|
||||
if '-devedition' in platform:
|
||||
platform = platform.replace('-devedition', '')
|
||||
return PLATFORM_RENAMES.get(platform, platform)
|
||||
|
||||
|
||||
def _sanitize_platform(platform):
|
||||
platform = get_balrog_platform_name(platform)
|
||||
if platform not in BALROG_PLATFORM_MAP:
|
||||
return platform
|
||||
return BALROG_PLATFORM_MAP[platform][0]
|
||||
|
||||
|
||||
def get_builds(release_history, platform, locale):
|
||||
"""Examine cached balrog release history and return the list of
|
||||
builds we need to generate diffs from"""
|
||||
platform = _sanitize_platform(platform)
|
||||
return release_history.get(platform, {}).get(locale, {})
|
||||
|
||||
|
||||
def get_partials_artifacts(release_history, platform, locale):
|
||||
platform = _sanitize_platform(platform)
|
||||
return release_history.get(platform, {}).get(locale, {}).keys()
|
||||
|
||||
|
||||
def get_partials_artifact_map(release_history, platform, locale):
|
||||
platform = _sanitize_platform(platform)
|
||||
return {k: release_history[platform][locale][k]['buildid'] for k in release_history.get(platform, {}).get(locale, {})}
|
||||
|
||||
|
||||
def _retry_on_http_errors(url, verify, params, errors):
|
||||
if params:
|
||||
params_str = "&".join("=".join([k, str(v)])
|
||||
for k, v in params.iteritems())
|
||||
else:
|
||||
params_str = ''
|
||||
logger.info("Connecting to %s?%s", url, params_str)
|
||||
for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
|
||||
try:
|
||||
req = requests.get(url, verify=verify, params=params)
|
||||
req.raise_for_status()
|
||||
return req
|
||||
except requests.HTTPError as e:
|
||||
if e.response.status_code in errors:
|
||||
logger.exception("Got HTTP %s trying to reach %s",
|
||||
e.response.status_code, url)
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def get_sorted_releases(product, branch):
|
||||
"""Returns a list of release names from Balrog.
|
||||
:param product: product name, AKA appName
|
||||
:param branch: branch name, e.g. mozilla-central
|
||||
:return: a sorted list of release names, most recent first.
|
||||
"""
|
||||
url = "{}/releases".format(BALROG_API_ROOT)
|
||||
params = {
|
||||
"product": product,
|
||||
# Adding -nightly-2 (2 stands for the beginning of build ID
|
||||
# based on date) should filter out release and latest blobs.
|
||||
# This should be changed to -nightly-3 in 3000 ;)
|
||||
"name_prefix": "{}-{}-nightly-2".format(product, branch),
|
||||
"names_only": True
|
||||
}
|
||||
req = _retry_on_http_errors(
|
||||
url=url, verify=True, params=params,
|
||||
errors=[500])
|
||||
releases = req.json()["names"]
|
||||
releases = sorted(releases, reverse=True)
|
||||
return releases
|
||||
|
||||
|
||||
def get_release_builds(release):
|
||||
url = "{}/releases/{}".format(BALROG_API_ROOT, release)
|
||||
req = _retry_on_http_errors(
|
||||
url=url, verify=True, params=None,
|
||||
errors=[500])
|
||||
return req.json()
|
||||
|
||||
|
||||
def populate_release_history(product, branch, maxbuilds=4, maxsearch=10):
|
||||
"""Find relevant releases in Balrog
|
||||
Not all releases have all platforms and locales, due
|
||||
to Taskcluster migration.
|
||||
|
||||
Args:
|
||||
product (str): capitalized product name, AKA appName, e.g. Firefox
|
||||
branch (str): branch name (mozilla-central)
|
||||
maxbuilds (int): Maximum number of historical releases to populate
|
||||
maxsearch(int): Traverse at most this many releases, to avoid
|
||||
working through the entire history.
|
||||
Returns:
|
||||
json object based on data from balrog api
|
||||
|
||||
results = {
|
||||
'platform1': {
|
||||
'locale1': {
|
||||
'buildid1': mar_url,
|
||||
'buildid2': mar_url,
|
||||
'buildid3': mar_url,
|
||||
},
|
||||
'locale2': {
|
||||
'target.partial-1.mar': ('buildid1': 'mar_url'),
|
||||
}
|
||||
},
|
||||
'platform2': {
|
||||
}
|
||||
}
|
||||
"""
|
||||
last_releases = get_sorted_releases(product, branch)
|
||||
|
||||
partial_mar_tmpl = 'target.partial-{}.mar'
|
||||
|
||||
builds = dict()
|
||||
for release in last_releases[:maxsearch]:
|
||||
# maxbuilds in all categories, don't make any more queries
|
||||
full = len(builds) > 0 and all(
|
||||
len(builds[platform][locale]) >= maxbuilds for platform in builds for locale in builds[platform])
|
||||
if full:
|
||||
break
|
||||
history = get_release_builds(release)
|
||||
|
||||
for platform in history['platforms']:
|
||||
if 'alias' in history['platforms'][platform]:
|
||||
continue
|
||||
if platform not in builds:
|
||||
builds[platform] = dict()
|
||||
for locale in history['platforms'][platform]['locales']:
|
||||
if locale not in builds[platform]:
|
||||
builds[platform][locale] = dict()
|
||||
if len(builds[platform][locale]) >= maxbuilds:
|
||||
continue
|
||||
buildid = history['platforms'][platform]['locales'][locale]['buildID']
|
||||
url = history['platforms'][platform]['locales'][locale]['completes'][0]['fileUrl']
|
||||
nextkey = len(builds[platform][locale]) + 1
|
||||
builds[platform][locale][partial_mar_tmpl.format(nextkey)] = {
|
||||
'buildid': buildid,
|
||||
'mar_url': url,
|
||||
}
|
||||
return builds
|
||||
|
|
@ -13,6 +13,9 @@ from mozbuild.util import memoize
|
|||
from requests.packages.urllib3.util.retry import Retry
|
||||
from requests.adapters import HTTPAdapter
|
||||
|
||||
_TC_ARTIFACT_LOCATION = \
|
||||
'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
|
||||
|
||||
|
||||
@memoize
|
||||
def get_session():
|
||||
|
|
@ -101,3 +104,10 @@ def get_task_url(task_id, use_proxy=False):
|
|||
def get_task_definition(task_id, use_proxy=False):
|
||||
response = _do_request(get_task_url(task_id, use_proxy))
|
||||
return response.json()
|
||||
|
||||
|
||||
def get_taskcluster_artifact_prefix(task_id, postfix='', locale=None):
|
||||
if locale:
|
||||
postfix = '{}/{}'.format(locale, postfix)
|
||||
|
||||
return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче