зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1182073 Add metrics to partial mar generation r=rail
MozReview-Commit-ID: 6MC3CAz6PXz --HG-- extra : rebase_source : 19d43a32fb0dc89bda6e6315b416bc9f9644d5e7
This commit is contained in:
Simon Fraser
Родитель
db4aa2e56a
Коммит
e5e73e1e45
|
|
@ -1,3 +1,4 @@
|
|||
mar==2.1.2
|
||||
backports.lzma==0.0.8
|
||||
datadog==0.17.0
|
||||
redo
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
|
||||
set -xe
|
||||
|
||||
test $TASK_ID
|
||||
test $SHA1_SIGNING_CERT
|
||||
test $SHA384_SIGNING_CERT
|
||||
test "$TASK_ID"
|
||||
test "$SHA1_SIGNING_CERT"
|
||||
test "$SHA384_SIGNING_CERT"
|
||||
|
||||
ARTIFACTS_DIR="/home/worker/artifacts"
|
||||
mkdir -p "$ARTIFACTS_DIR"
|
||||
|
|
@ -15,7 +15,7 @@ curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json \
|
|||
# enable locale cache
|
||||
export MBSDIFF_HOOK="/home/worker/bin/mbsdiff_hook.sh -c /tmp/fs-cache"
|
||||
|
||||
if [ ! -z $FILENAME_TEMPLATE ]; then
|
||||
if [ ! -z "$FILENAME_TEMPLATE" ]; then
|
||||
EXTRA_PARAMS="--filename-template $FILENAME_TEMPLATE $EXTRA_PARAMS"
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import logging
|
|||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import requests
|
||||
import sh
|
||||
|
||||
|
|
@ -20,6 +21,8 @@ import redo
|
|||
from mardor.reader import MarReader
|
||||
from mardor.signing import get_keysize
|
||||
|
||||
from datadog import initialize, ThreadStats
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
ALLOWED_URL_PREFIXES = [
|
||||
|
|
@ -55,6 +58,18 @@ def is_lzma_compressed_mar(mar):
|
|||
return result
|
||||
|
||||
|
||||
@redo.retriable()
|
||||
def get_secret(secret_name):
|
||||
secrets_url = "http://taskcluster/secrets/v1/secret/{}"
|
||||
log.debug("Fetching {}".format(secret_name))
|
||||
r = requests.get(secrets_url.format(secret_name))
|
||||
# 403: If unauthorized, just give up.
|
||||
if r.status_code == 403:
|
||||
return {}
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
@redo.retriable()
|
||||
def download(url, dest, mode=None):
|
||||
log.debug("Downloading %s to %s", url, dest)
|
||||
|
|
@ -191,6 +206,9 @@ def verify_allowed_url(mar):
|
|||
|
||||
|
||||
def main():
|
||||
|
||||
start = time.time()
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--artifacts-dir", required=True)
|
||||
parser.add_argument("--sha1-signing-cert", required=True)
|
||||
|
|
@ -219,6 +237,26 @@ def main():
|
|||
assert(get_keysize(signing_certs['sha1']) == 2048)
|
||||
assert(get_keysize(signing_certs['sha384']) == 4096)
|
||||
|
||||
# Intended for local testing.
|
||||
dd_api_key = os.environ.get('DATADOG_API_KEY')
|
||||
# Intended for Taskcluster.
|
||||
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
|
||||
dd_api_key = get_secret(os.environ.get('DATADOG_API_SECRET')).get('key')
|
||||
|
||||
# Create this even when not sending metrics, so the context manager
|
||||
# statements work.
|
||||
ddstats = ThreadStats(namespace='releng.releases.partials')
|
||||
|
||||
if dd_api_key:
|
||||
dd_options = {
|
||||
'api_key': dd_api_key,
|
||||
}
|
||||
log.info("Starting metric collection")
|
||||
initialize(**dd_options)
|
||||
ddstats.start(flush_interval=1)
|
||||
else:
|
||||
log.info("No metric collection")
|
||||
|
||||
if args.no_freshclam:
|
||||
log.info("Skipping freshclam")
|
||||
else:
|
||||
|
|
@ -229,6 +267,7 @@ def main():
|
|||
log.info("Done.")
|
||||
except sh.ErrorReturnCode:
|
||||
log.warning("Freshclam failed, skipping DB update")
|
||||
|
||||
manifest = []
|
||||
for e in task["extra"]["funsize"]["partials"]:
|
||||
for mar in (e["from_mar"], e["to_mar"]):
|
||||
|
|
@ -242,12 +281,14 @@ def main():
|
|||
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
|
||||
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
|
||||
unpack_dir = os.path.join(work_env.workdir, mar_type)
|
||||
download(f, dest)
|
||||
with ddstats.timer('mar.download.time'):
|
||||
download(f, dest)
|
||||
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
|
||||
verify_signature(dest, signing_certs)
|
||||
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
|
||||
complete_mars["%s_hash" % mar_type] = get_hash(dest)
|
||||
unpack(work_env, dest, unpack_dir)
|
||||
with ddstats.timer('mar.unpack.time'):
|
||||
unpack(work_env, dest, unpack_dir)
|
||||
if mar_type == 'from':
|
||||
version = get_option(unpack_dir, filename="application.ini",
|
||||
section="App", option="Version")
|
||||
|
|
@ -258,7 +299,11 @@ def main():
|
|||
use_old_format = True
|
||||
log.info("Forcing BZ2 compression for %s", f)
|
||||
log.info("AV-scanning %s ...", unpack_dir)
|
||||
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
|
||||
metric_tags = [
|
||||
"platform:{}".format(e['platform']),
|
||||
]
|
||||
with ddstats.timer('mar.clamscan.time', tags=metric_tags):
|
||||
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
|
||||
log.info("Done.")
|
||||
|
||||
path = os.path.join(work_env.workdir, "to")
|
||||
|
|
@ -307,20 +352,50 @@ def main():
|
|||
# TODO: download these once
|
||||
work_env.download_buildsystem_bits(repo=mar_data["repo"],
|
||||
revision=mar_data["revision"])
|
||||
generate_partial(work_env, from_path, path, dest_mar,
|
||||
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
|
||||
mar_data["version"],
|
||||
use_old_format)
|
||||
|
||||
metric_tags = [
|
||||
"branch:{}".format(mar_data['branch']),
|
||||
"platform:{}".format(mar_data['platform']),
|
||||
# If required. Shouldn't add much useful info, but increases
|
||||
# cardinality of metrics substantially, so avoided.
|
||||
# "locale:{}".format(mar_data['locale']),
|
||||
]
|
||||
|
||||
with ddstats.timer('generate_partial.time', tags=metric_tags):
|
||||
generate_partial(work_env, from_path, path, dest_mar,
|
||||
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
|
||||
mar_data["version"],
|
||||
use_old_format)
|
||||
|
||||
mar_data["size"] = os.path.getsize(dest_mar)
|
||||
metric_tags.append("unit:bytes")
|
||||
# Allows us to find out how many releases there were between the two,
|
||||
# making buckets of the file sizes easier.
|
||||
metric_tags.append("update_number:{}".format(mar_data.get('update_number', 0)))
|
||||
ddstats.gauge('partial_mar_size', mar_data['size'], tags=metric_tags)
|
||||
|
||||
mar_data["hash"] = get_hash(dest_mar)
|
||||
|
||||
shutil.copy(dest_mar, args.artifacts_dir)
|
||||
work_env.cleanup()
|
||||
manifest.append(mar_data)
|
||||
|
||||
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
|
||||
with open(manifest_file, "w") as fp:
|
||||
json.dump(manifest, fp, indent=2, sort_keys=True)
|
||||
|
||||
# Warning: Assumption that one partials task will always be for one branch.
|
||||
metric_tags = [
|
||||
"branch:{}".format(mar_data['branch']),
|
||||
]
|
||||
|
||||
ddstats.timing('task_duration', time.time() - start,
|
||||
start, tags=metric_tags)
|
||||
# Wait for all the metrics to flush. If the program ends before
|
||||
# they've been sent, they'll be dropped.
|
||||
# Should be more than the flush_interval for the ThreadStats object
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ def make_task_description(config, jobs):
|
|||
get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
|
||||
'target.complete.mar'
|
||||
)
|
||||
for build in builds:
|
||||
for build in sorted(builds):
|
||||
extra['funsize']['partials'].append({
|
||||
'locale': build_locale,
|
||||
'from_mar': builds[build]['mar_url'],
|
||||
|
|
@ -113,6 +113,7 @@ def make_task_description(config, jobs):
|
|||
'os': 'linux',
|
||||
'max-run-time': 3600,
|
||||
'chain-of-trust': True,
|
||||
'taskcluster-proxy': True,
|
||||
'env': {
|
||||
'SHA1_SIGNING_CERT': 'nightly_sha1',
|
||||
'SHA384_SIGNING_CERT': 'nightly_sha384'
|
||||
|
|
@ -127,6 +128,7 @@ def make_task_description(config, jobs):
|
|||
dep_job.task["metadata"]["description"]),
|
||||
'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
|
||||
'dependencies': dependencies,
|
||||
'scopes': ['secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level],
|
||||
'attributes': attributes,
|
||||
'run-on-projects': dep_job.attributes.get('run_on_projects'),
|
||||
'treeherder': treeherder,
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче