From 720b949436069c1b66b33037c8ed58dff31bc73b Mon Sep 17 00:00:00 2001 From: Justin Wood Date: Wed, 2 Jan 2019 21:53:25 +0000 Subject: [PATCH] Bug 1485861 - Define many treeherder symbols based on the parent task. r=aki This changes a bunch of symbols, notably: * Group name BMR-L10n(*) ==> BMR(*) * Group name BM-L10n(*) ==> BM(*) * Groups the BMR symbol itself for en-US Nightly [so BMR ==> BMR(N)] * Groups the BM symbol itself for en-US Nightly [so BM ==> BM(N)] * Changes Source Signing fron symbol Bs to Srcs * Groups Source signing beetmoving as BM(Srcs) instead of symbol BM-S * Balrog complete update submission changes from c-Up(N) to c-Up(Ns) [Fennec Only] * Beetmover en-US changes from symbol BM-S to BM(Ns) [used in Fennec] * Beetmover Checksums (en-US) changes from BMcs(N) => BMcs(Ns) [fennec] * Checksums signing (en-US, fennec) changes from cs(N) => cs(Ns) * [Asan] Balrog changed [c-UP(N) ==> c-UP(BoR)] * [Asan] signing changed from Ns to BoRs * [Asan] checksums signing changed from cs(N) to cs(BoR) * [Asan] Change generated source uploading from Ugs ==> UgsBoR Differential Revision: https://phabricator.services.mozilla.com/D4178 --HG-- extra : moz-landing-system : lando --- taskcluster/ci/config.yml | 4 ++-- taskcluster/taskgraph/transforms/balrog_submit.py | 5 +++-- taskcluster/taskgraph/transforms/beetmover.py | 6 +++++- .../taskgraph/transforms/beetmover_checksums.py | 6 +++++- taskcluster/taskgraph/transforms/beetmover_l10n.py | 2 +- .../taskgraph/transforms/beetmover_repackage.py | 9 ++++++++- .../transforms/beetmover_repackage_l10n.py | 2 +- .../taskgraph/transforms/checksums_signing.py | 6 +++++- taskcluster/taskgraph/transforms/signing.py | 14 ++++++-------- .../transforms/upload_generated_sources.py | 2 ++ taskcluster/taskgraph/util/treeherder.py | 10 ++++++++-- 11 files changed, 46 insertions(+), 20 deletions(-) diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 79186fb3ddcd..0cbbc2893a53 100755 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -54,8 +54,8 @@ treeherder: 'X-sw': 'Xpcshell tests with serviceworker redesign' 'L10n': 'Localised Repacks' 'L10n-Rpk': 'Localized Repackaged Repacks' - 'BM-L10n': 'Beetmover for locales' - 'BMR-L10n': 'Beetmover repackages for locales' + 'BM': 'Beetmover' + 'BMR': 'Beetmover repackages' 'c-Up': 'Balrog submission of complete updates' 'cs': 'Checksum signing' 'css': 'Checksum signing for source' diff --git a/taskcluster/taskgraph/transforms/balrog_submit.py b/taskcluster/taskgraph/transforms/balrog_submit.py index a263098ccc7e..c69c4f34fdba 100644 --- a/taskcluster/taskgraph/transforms/balrog_submit.py +++ b/taskcluster/taskgraph/transforms/balrog_submit.py @@ -16,6 +16,7 @@ from taskgraph.util.schema import ( from taskgraph.util.scriptworker import ( get_balrog_server_scope, get_worker_type_for_scope ) +from taskgraph.util.treeherder import replace_group from taskgraph.transforms.task import task_description_schema from voluptuous import Optional @@ -89,10 +90,10 @@ def make_task_description(config, jobs): attributes = copy_attributes_from_dependent_job(dep_job) - treeherder_job_symbol = dep_job.attributes.get('locale', 'N') + treeherder_job_symbol = dep_job.task['extra']['treeherder']['symbol'] + treeherder['symbol'] = replace_group(treeherder_job_symbol, 'c-Up') if dep_job.attributes.get('locale'): - treeherder['symbol'] = 'c-Up({})'.format(treeherder_job_symbol) attributes['locale'] = dep_job.attributes.get('locale') label = job['label'] diff --git a/taskcluster/taskgraph/transforms/beetmover.py b/taskcluster/taskgraph/transforms/beetmover.py index a8e94e2c1c67..af48397be2e0 100644 --- a/taskcluster/taskgraph/transforms/beetmover.py +++ b/taskcluster/taskgraph/transforms/beetmover.py @@ -20,6 +20,7 @@ from taskgraph.util.scriptworker import (generate_beetmover_artifact_map, get_worker_type_for_scope, should_use_artifact_map) from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.treeherder import replace_group # Until bug 1331141 is fixed, if you are adding any new artifacts here that # need to be transfered to S3, please be aware you also need to follow-up @@ -150,7 +151,10 @@ def make_task_description(config, jobs): attributes = dep_job.attributes treeherder = job.get('treeherder', {}) - treeherder.setdefault('symbol', 'BM-S') + treeherder.setdefault( + 'symbol', + replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BM') + ) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', diff --git a/taskcluster/taskgraph/transforms/beetmover_checksums.py b/taskcluster/taskgraph/transforms/beetmover_checksums.py index d38a97fe5c6b..686fbf37dc84 100644 --- a/taskcluster/taskgraph/transforms/beetmover_checksums.py +++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py @@ -19,6 +19,7 @@ from taskgraph.util.scriptworker import (generate_beetmover_artifact_map, get_worker_type_for_scope, should_use_artifact_map) from voluptuous import Optional, Required +from taskgraph.util.treeherder import replace_group from taskgraph.transforms.task import task_description_schema # Voluptuous uses marker objects as dictionary *keys*, but they are not @@ -46,7 +47,10 @@ def make_beetmover_checksums_description(config, jobs): attributes = dep_job.attributes treeherder = job.get('treeherder', {}) - treeherder.setdefault('symbol', 'BMcs(N)') + treeherder.setdefault( + 'symbol', + replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BMcs') + ) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', diff --git a/taskcluster/taskgraph/transforms/beetmover_l10n.py b/taskcluster/taskgraph/transforms/beetmover_l10n.py index 5ad0d69e384f..5a714364fa11 100644 --- a/taskcluster/taskgraph/transforms/beetmover_l10n.py +++ b/taskcluster/taskgraph/transforms/beetmover_l10n.py @@ -19,7 +19,7 @@ def make_beetmover_description(config, jobs): dep_job = job['primary-dependency'] for locale in dep_job.attributes.get('chunk_locales', []): - group = 'BM-L10n' + group = 'BM' # add the locale code symbol = locale diff --git a/taskcluster/taskgraph/transforms/beetmover_repackage.py b/taskcluster/taskgraph/transforms/beetmover_repackage.py index 460be5a6539d..83c969e9efa8 100644 --- a/taskcluster/taskgraph/transforms/beetmover_repackage.py +++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py @@ -18,6 +18,7 @@ from taskgraph.util.scriptworker import (get_beetmover_bucket_scope, get_beetmover_action_scope, get_worker_type_for_scope) from taskgraph.util.taskcluster import get_artifact_prefix +from taskgraph.util.treeherder import replace_group from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional @@ -171,7 +172,13 @@ def make_task_description(config, jobs): attributes = dep_job.attributes treeherder = job.get('treeherder', {}) - treeherder.setdefault('symbol', 'BM-R') + upstream_symbol = dep_job.task['extra']['treeherder']['symbol'] + if 'build' in job['dependent-tasks']: + upstream_symbol = job['dependent-tasks']['build'].task['extra']['treeherder']['symbol'] + treeherder.setdefault( + 'symbol', + replace_group(upstream_symbol, 'BMR') + ) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', diff --git a/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py b/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py index 4731bfc8023f..c22acffe89fc 100644 --- a/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py +++ b/taskcluster/taskgraph/transforms/beetmover_repackage_l10n.py @@ -23,7 +23,7 @@ def make_beetmover_description(config, jobs): yield job continue - group = 'BMR-L10n' + group = 'BMR' # add the locale code symbol = locale diff --git a/taskcluster/taskgraph/transforms/checksums_signing.py b/taskcluster/taskgraph/transforms/checksums_signing.py index 907d72316539..33a674784160 100644 --- a/taskcluster/taskgraph/transforms/checksums_signing.py +++ b/taskcluster/taskgraph/transforms/checksums_signing.py @@ -15,6 +15,7 @@ from taskgraph.util.scriptworker import ( get_worker_type_for_scope, add_scope_prefix, ) +from taskgraph.util.treeherder import replace_group from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional @@ -41,7 +42,10 @@ def make_checksums_signing_description(config, jobs): attributes = dep_job.attributes treeherder = job.get('treeherder', {}) - treeherder.setdefault('symbol', 'cs(N)') + treeherder.setdefault( + 'symbol', + replace_group(dep_job.task['extra']['treeherder']['symbol'], 'cs') + ) dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', diff --git a/taskcluster/taskgraph/transforms/signing.py b/taskcluster/taskgraph/transforms/signing.py index bc2d2c81fe1e..442c5d7cebdc 100644 --- a/taskcluster/taskgraph/transforms/signing.py +++ b/taskcluster/taskgraph/transforms/signing.py @@ -104,9 +104,6 @@ def make_task_description(config, jobs): treeherder.setdefault('platform', _generate_treeherder_platform( dep_th_platform, build_platform, build_type )) - treeherder.setdefault('symbol', _generate_treeherder_symbol( - is_nightly, build_platform - )) # ccov builds are tier 2, so they cannot have tier 1 tasks # depending on them. @@ -114,6 +111,9 @@ def make_task_description(config, jobs): 'tier', dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1) ) + treeherder.setdefault('symbol', _generate_treeherder_symbol( + dep_job.task.get('extra', {}).get('treeherder', {}).get('symbol') + )) treeherder.setdefault('kind', 'build') label = job['label'] @@ -171,8 +171,6 @@ def _generate_treeherder_platform(dep_th_platform, build_platform, build_type): return '{}/{}'.format(dep_th_platform, actual_build_type) -def _generate_treeherder_symbol(is_nightly, build_platform): - if is_nightly: - return 'Ns' - else: - return 'Bs' +def _generate_treeherder_symbol(build_symbol): + symbol = build_symbol + 's' + return symbol diff --git a/taskcluster/taskgraph/transforms/upload_generated_sources.py b/taskcluster/taskgraph/transforms/upload_generated_sources.py index 3d755358a99e..263a56425e50 100644 --- a/taskcluster/taskgraph/transforms/upload_generated_sources.py +++ b/taskcluster/taskgraph/transforms/upload_generated_sources.py @@ -32,6 +32,8 @@ def add_task_info(config, jobs): plat = '{}/{}'.format(dep_th['machine']['platform'], dep_task.attributes.get('build_type')) job['treeherder']['platform'] = plat job['treeherder']['tier'] = dep_th['tier'] + if dep_th['symbol'] != "N": + job['treeherder']['symbol'] = "Ugs{}".format(dep_th['symbol']) # Add an environment variable pointing at the artifact from the build. artifact_url = get_artifact_url('', 'public/build/target.generated-files.tar.gz') diff --git a/taskcluster/taskgraph/util/treeherder.py b/taskcluster/taskgraph/util/treeherder.py index 245462910ab8..e9d87ec6b5a9 100644 --- a/taskcluster/taskgraph/util/treeherder.py +++ b/taskcluster/taskgraph/util/treeherder.py @@ -24,8 +24,14 @@ def join_symbol(group, symbol): return '{}({})'.format(group, symbol) -def add_suffix(treeherder_symbol, chunk): +def add_suffix(treeherder_symbol, suffix): """Add a suffix to a treeherder symbol that may contain a group.""" group, symbol = split_symbol(treeherder_symbol) - symbol += str(chunk) + symbol += str(suffix) return join_symbol(group, symbol) + + +def replace_group(treeherder_symbol, new_group): + """Add a suffix to a treeherder symbol that may contain a group.""" + _, symbol = split_symbol(treeherder_symbol) + return join_symbol(new_group, symbol)