Bug 1617481 - Remove most of nightly attribute from taskcluster/taskgraph. r=aki

We leave the nightly index in place for now in order to support Geckodriver until Bug 1577110 sticks.

Differential Revision: https://phabricator.services.mozilla.com/D74510
This commit is contained in:
Justin Wood 2020-05-21 16:31:04 +00:00
Родитель 3f24c1f0f0
Коммит 6c368aa658
24 изменённых файлов: 45 добавлений и 100 удалений

Просмотреть файл

@ -51,8 +51,6 @@ def loader(kind, path, config, params, loaded_tasks):
test['build-attributes'] = test_platform['build-attributes']
test['test-name'] = test_name
if test_platform.get('nightly'):
test.setdefault('attributes', {})['nightly'] = True
if test_platform.get('shippable'):
test.setdefault('attributes', {})['shippable'] = True
test['attributes']['shipping_product'] = test_platform['shipping_product']
@ -99,9 +97,6 @@ def get_test_platforms(test_platforms_cfg, builds_by_platform, signed_builds_by_
'build-attributes': builds_by_platform[build_platform].attributes,
}
if builds_by_platform[build_platform].attributes.get('nightly'):
test_platforms[test_platform]['nightly'] = \
builds_by_platform[build_platform].attributes['nightly']
if builds_by_platform[build_platform].attributes.get('shippable'):
test_platforms[test_platform]['shippable'] = \
builds_by_platform[build_platform].attributes['shippable']

Просмотреть файл

@ -53,8 +53,9 @@ def get_method(method):
return _target_task_methods[method]
def filter_out_nightly(task, parameters):
def filter_out_shipping_phase(task, parameters):
return (
# nightly still here because of geckodriver
not task.attributes.get('nightly') and
task.attributes.get('shipping_phase') in (None, 'build')
)
@ -232,7 +233,7 @@ def target_tasks_try_auto(full_task_graph, parameters, graph_config):
parameters = Parameters(**params)
return [l for l, t in six.iteritems(full_task_graph.tasks)
if standard_filter(t, parameters)
and filter_out_nightly(t, parameters)
and filter_out_shipping_phase(t, parameters)
and filter_by_uncommon_try_tasks(t.label)]
@ -242,7 +243,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config):
via the `run_on_projects` attributes."""
return [l for l, t in six.iteritems(full_task_graph.tasks)
if standard_filter(t, parameters)
and filter_out_nightly(t, parameters)
and filter_out_shipping_phase(t, parameters)
and filter_out_devedition(t, parameters)]
@ -433,8 +434,8 @@ def target_tasks_pine(full_task_graph, parameters, graph_config):
# disable asan
if platform == 'linux64-asan':
return False
# disable non-pine and nightly tasks
if standard_filter(task, parameters) or filter_out_nightly(task, parameters):
# disable non-pine and tasks with a shipping phase
if standard_filter(task, parameters) or filter_out_shipping_phase(task, parameters):
return True
return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t)]
@ -572,10 +573,7 @@ def make_desktop_nightly_filter(platforms):
return all([
filter_on_platforms(task, platforms),
filter_for_project(task, parameters),
any([
task.attributes.get('nightly', False),
task.attributes.get('shippable', False),
]),
task.attributes.get('shippable', False),
# Tests and nightly only builds don't have `shipping_product` set
task.attributes.get('shipping_product') in {None, "firefox", "thunderbird"},
task.kind not in {'l10n'}, # no on-change l10n
@ -589,7 +587,7 @@ def target_tasks_nightly_linux(full_task_graph, parameters, graph_config):
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
filter = make_desktop_nightly_filter({
'linux64-nightly', 'linux-nightly', 'linux64-shippable', 'linux-shippable'
'linux64-shippable', 'linux-shippable'
})
return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t, parameters)]
@ -599,7 +597,7 @@ def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of macosx. The
nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
filter = make_desktop_nightly_filter({'macosx64-nightly', 'macosx64-shippable'})
filter = make_desktop_nightly_filter({'macosx64-shippable'})
return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t, parameters)]
@ -608,7 +606,7 @@ def target_tasks_nightly_win32(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
filter = make_desktop_nightly_filter({'win32-nightly', 'win32-shippable'})
filter = make_desktop_nightly_filter({'win32-shippable'})
return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t, parameters)]
@ -617,7 +615,7 @@ def target_tasks_nightly_win64(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
filter = make_desktop_nightly_filter({'win64-nightly', 'win64-shippable'})
filter = make_desktop_nightly_filter({'win64-shippable'})
return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t, parameters)]
@ -626,7 +624,7 @@ def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config
"""Select the set of tasks required for a nightly build of win32 and win64.
The nightly build process involves a pipeline of builds, signing,
and, eventually, uploading the tasks to balrog."""
filter = make_desktop_nightly_filter({'win64-aarch64-nightly', 'win64-aarch64-shippable'})
filter = make_desktop_nightly_filter({'win64-aarch64-shippable'})
return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t, parameters)]

Просмотреть файл

@ -114,7 +114,6 @@ def make_task_description(config, jobs):
def craft_release_properties(config, job):
params = config.params
build_platform = job['attributes']['build_platform']
build_platform = build_platform.replace('-nightly', '')
build_platform = build_platform.replace('-shippable', '')
if build_platform.endswith("-source"):
build_platform = build_platform.replace('-source', '-release')

Просмотреть файл

@ -171,7 +171,6 @@ def use_artifact(config, jobs):
use_artifact = False
for job in jobs:
if (config.kind == 'build' and use_artifact and
not job.get('attributes', {}).get('nightly', False) and
job.get('index', {}).get('job-name') in ARTIFACT_JOBS):
job['treeherder']['symbol'] += 'a'
job['worker']['env']['USE_ARTIFACT'] = '1'

Просмотреть файл

@ -39,7 +39,6 @@ def check_mozharness_perfherder_options(config, jobs):
platform = job['treeherder']['platform']
primary_config = job['run']['config'][0]
options = worker.get('env', {}).get('PERFHERDER_EXTRA_OPTIONS')
nightly = job.get('attributes', {}).get('nightly', False)
shippable = job.get('attributes', {}).get('shippable', False)
# This isn't strictly necessary. But the Perfherder code looking at the
@ -47,7 +46,7 @@ def check_mozharness_perfherder_options(config, jobs):
# sense to run this linter elsewhere.
assert primary_config.startswith('builds/')
key = (platform, primary_config, (nightly or shippable), options)
key = (platform, primary_config, shippable, options)
if key in SEEN_CONFIGS:
raise Exception(

Просмотреть файл

@ -25,15 +25,7 @@ def add_signed_routes(config, jobs):
enable_signing_routes = job.pop('enable-signing-routes', True)
job['routes'] = []
if dep_job.attributes.get('nightly') and enable_signing_routes:
for dep_route in dep_job.task.get('routes', []):
if not dep_route.startswith('index.gecko.v2'):
continue
branch = dep_route.split(".")[3]
rest = ".".join(dep_route.split(".")[4:])
job['routes'].append(
'index.gecko.v2.{}.signed-nightly.{}'.format(branch, rest))
elif dep_job.attributes.get('shippable') and enable_signing_routes:
if dep_job.attributes.get('shippable') and enable_signing_routes:
for dep_route in dep_job.task.get('routes', []):
if not dep_route.startswith('index.gecko.v2'):
continue

Просмотреть файл

@ -55,9 +55,9 @@ def make_signing_description(config, jobs):
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly', dep_job.attributes.get('shippable'))
is_shippable = dep_job.attributes.get('shippable')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
build_platform, is_shippable, config
)
upstream_artifacts = _craft_upstream_artifacts(dep_job, dep_job.kind, build_platform)

Просмотреть файл

@ -27,7 +27,6 @@ from taskgraph.transforms.job.common import (
)
VARIANTS = [
'nightly',
'shippable',
'devedition',
'pgo',
@ -57,10 +56,10 @@ def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
artifact_url = get_artifact_url('<build>', get_artifact_path(taskdesc,
'target.test_packages.json'))
# for android nightly we need to add 'en-US' to the artifact url
# for android shippable we need to add 'en-US' to the artifact url
test = taskdesc['run']['test']
if 'android' in test['test-platform'] and (
get_variant(test['test-platform']) in ("nightly", 'shippable')):
get_variant(test['test-platform']) in ('shippable', )):
head, tail = os.path.split(artifact_url)
artifact_url = os.path.join(head, 'en-US', tail)
return artifact_url

Просмотреть файл

@ -137,7 +137,7 @@ l10n_description_schema = schema.extend({
Optional('locales-per-chunk'): _by_platform(int),
# Task deps to chain this task with, added in transforms from primary-dependency
# if this is a nightly
# if this is a shippable-style build
Optional('dependencies'): {text_type: text_type},
# Run the task when the listed files change (if present).
@ -211,7 +211,7 @@ transforms.add_validate(l10n_description_schema)
@transforms.add
def setup_nightly_dependency(config, jobs):
def setup_shippable_dependency(config, jobs):
""" Sets up a task dependency to the signing job this relates to """
for job in jobs:
job['dependencies'] = {'build': job['dependent-tasks']['build'].label}
@ -282,7 +282,6 @@ def handle_artifact_prefix(config, jobs):
def all_locales_attribute(config, jobs):
for job in jobs:
locales_platform = job['attributes']['build_platform'].replace("-shippable", "")
locales_platform = locales_platform.replace("-nightly", "")
locales_platform = locales_platform.replace("-pgo", "")
locales_with_changesets = parse_locales_file(job["locales-file"],
platform=locales_platform)

Просмотреть файл

@ -128,8 +128,7 @@ def make_task_description(config, jobs):
is_shippable = job.get(
'shippable', # First check current job
dep_job.attributes.get(
'shippable', # Then dep job for 'shippable'
dep_job.attributes.get('nightly'))) # lastly dep job for 'nightly'
'shippable')) # Then dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)

Просмотреть файл

@ -68,15 +68,13 @@ def add_command_arguments(config, tasks):
all_locales.update(sub_partner.get('locales', []))
for task in tasks:
# add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64
if not task['attributes']['build_platform'].endswith('-shippable') and \
not task['attributes']['build_platform'].endswith('-nightly'):
if not task['attributes']['build_platform'].endswith('-shippable'):
raise Exception(
"Unexpected partner repack platform: {}".format(
task['attributes']['build_platform'],
),
)
platform = task['attributes']['build_platform'].partition('-shippable')[0]
platform = platform.partition('-nightly')[0]
task['run']['options'] = [
'version={}'.format(release_config['version']),
'build-number={}'.format(release_config['build_number']),

Просмотреть файл

@ -84,11 +84,9 @@ def filter_out_macos_jobs_but_mac_only_locales(config, jobs):
for job in jobs:
build_platform = job['primary-dependency'].attributes.get('build_platform')
if build_platform in (
'linux64-nightly', 'linux64-devedition', 'linux64-shippable'):
if build_platform in ('linux64-devedition', 'linux64-shippable'):
yield job
elif build_platform in (
'macosx64-nightly', 'macosx64-devedition', 'macosx64-shippable') and \
elif build_platform in ('macosx64-devedition', 'macosx64-shippable') and \
'ja-JP-mac' in job['attributes']['chunk_locales']:
# Other locales of the same job shouldn't be processed
job['attributes']['chunk_locales'] = ['ja-JP-mac']

Просмотреть файл

@ -205,10 +205,7 @@ def make_job_description(config, jobs):
attributes['locale'] = locale
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'Nr')
else:
treeherder.setdefault('symbol', 'Rpk')
treeherder.setdefault('symbol', 'Rpk')
dep_th_platform = dep_job.task.get('extra', {}).get('treeherder-platform')
treeherder.setdefault('platform', dep_th_platform)
treeherder.setdefault('tier', 1)

Просмотреть файл

@ -23,12 +23,8 @@ def add_indexes(config, jobs):
index_type = 'generic'
if job['attributes'].get('shippable') and job['attributes'].get('locale'):
index_type = 'shippable-l10n'
if job['attributes'].get('nightly') and job['attributes'].get('locale'):
index_type = 'nightly-l10n'
if job['attributes'].get('shippable'):
index_type = 'shippable'
if job['attributes'].get('nightly'):
index_type = 'nightly'
if job['attributes'].get('locale'):
index_type = 'l10n'
job['index'] = {

Просмотреть файл

@ -46,10 +46,7 @@ def make_repackage_signing_description(config, jobs):
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'rs(N)')
else:
treeherder.setdefault('symbol', 'rs(B)')
treeherder.setdefault('symbol', 'rs(B)')
dep_th_platform = dep_job.task.get('extra', {}).get('treeherder-platform')
treeherder.setdefault('platform', dep_th_platform)
treeherder.setdefault(
@ -88,9 +85,9 @@ def make_repackage_signing_description(config, jobs):
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly', dep_job.attributes.get('shippable'))
is_shippable = dep_job.attributes.get('shippable')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
build_platform, is_shippable, config
)
scopes = [signing_cert_scope]

Просмотреть файл

@ -41,7 +41,7 @@ def make_repackage_signing_description(config, jobs):
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly', dep_job.attributes.get('shippable'))
is_shippable = dep_job.attributes.get('shippable')
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
@ -70,7 +70,7 @@ def make_repackage_signing_description(config, jobs):
attributes['repackage_type'] = 'repackage-signing'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
build_platform, is_shippable, config
)
scopes = [signing_cert_scope]

Просмотреть файл

@ -114,8 +114,7 @@ def make_task_description(config, jobs):
for f in artifacts['formats']:
formats.add(f) # Add each format only once
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable', False))
is_shippable = dep_job.attributes.get('shippable', False)
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
@ -158,9 +157,9 @@ def make_task_description(config, jobs):
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
build_platform, is_shippable, config
)
worker_type_alias = 'linux-signing' if is_nightly else 'linux-depsigning'
worker_type_alias = 'linux-signing' if is_shippable else 'linux-depsigning'
mac_behavior = None
task = {
'label': label,
@ -182,9 +181,6 @@ def make_task_description(config, jobs):
shippable = "false"
if "shippable" in attributes and attributes["shippable"]:
shippable = "true"
# remove the nightly check once nightly is gone as an attribute
if "nightly" in attributes and attributes["nightly"]:
shippable = "true"
mac_behavior = evaluate_keyed_by(
config.graph_config['mac-notarization']['mac-behavior'],
'mac behavior',

Просмотреть файл

@ -860,7 +860,6 @@ def set_tier(config, tasks):
if 'tier' not in task or task['tier'] == 'default':
if task['test-platform'] in [
'linux64/opt',
'linux64-nightly/opt',
'linux64/debug',
'linux64-pgo/opt',
'linux64-shippable/opt',
@ -882,7 +881,6 @@ def set_tier(config, tasks):
'windows7-32/opt',
'windows7-32-pgo/opt',
'windows7-32-devedition/opt',
'windows7-32-nightly/opt',
'windows7-32-shippable/opt',
'windows10-aarch64/opt',
'windows10-64/debug',
@ -890,7 +888,6 @@ def set_tier(config, tasks):
'windows10-64-pgo/opt',
'windows10-64-shippable/opt',
'windows10-64-devedition/opt',
'windows10-64-nightly/opt',
'windows10-64-asan/opt',
'windows10-64-qr/opt',
'windows10-64-qr/debug',
@ -898,7 +895,6 @@ def set_tier(config, tasks):
'windows10-64-shippable-qr/opt',
'macosx1014-64/opt',
'macosx1014-64/debug',
'macosx1014-64-nightly/opt',
'macosx1014-64-shippable/opt',
'macosx1014-64-devedition/opt',
'macosx1014-64-qr/opt',

Просмотреть файл

@ -28,8 +28,6 @@ def add_task_info(config, jobs):
dep_th = dep_task.task['extra']['treeherder']
job.setdefault('attributes', {})
job['attributes']['build_platform'] = dep_task.attributes.get('build_platform')
if dep_task.attributes.get('nightly'):
job['attributes']['nightly'] = True
if dep_task.attributes.get('shippable'):
job['attributes']['shippable'] = True
plat = '{}/{}'.format(dep_th['machine']['platform'], dep_task.attributes.get('build_type'))

Просмотреть файл

@ -21,7 +21,7 @@ transforms = TransformSequence()
@transforms.add
def check_nightlies(config, tasks):
"""Ensure that we upload symbols for all nightly builds, so that crash-stats can
"""Ensure that we upload symbols for all shippable builds, so that crash-stats can
resolve any reports sent to it. Try may enable full symbols but not upload them.
Putting this check here (instead of the transforms for the build kind) lets us
@ -29,9 +29,9 @@ def check_nightlies(config, tasks):
for task in tasks:
dep = task['primary-dependency']
if config.params['project'] in RELEASE_PROJECTS and \
dep.attributes.get('nightly', dep.attributes.get('shippable')) and \
dep.attributes.get('shippable') and \
not dep.attributes.get('enable-full-crashsymbols'):
raise Exception('Nightly job %s should have enable-full-crashsymbols attribute '
raise Exception('Shippable job %s should have enable-full-crashsymbols attribute '
'set to true to enable symbol upload to crash-stats' % dep.label)
yield task
@ -71,10 +71,7 @@ def fill_template(config, tasks):
)
task['treeherder'] = treeherder
if attributes.get('nightly'):
# For nightly builds, we want to run these tasks if the build is run.
task['run-on-projects'] = dep.attributes.get('run_on_projects')
elif attributes.get('shippable'):
if attributes.get('shippable'):
# For shippable builds, we want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task['run-on-projects'] = dep.attributes.get('run_on_projects')

Просмотреть файл

@ -557,8 +557,6 @@ class TryOptionSyntax(object):
attr = task.attributes.get
def check_run_on_projects():
if attr('nightly') and not self.include_nightly:
return False
return set(['try', 'all']) & set(attr('run_on_projects', []))
# Don't schedule fission tests when try option syntax is used

Просмотреть файл

@ -83,7 +83,7 @@ def get_balrog_platform_name(platform):
Remove known values instead to catch aarch64 and other platforms
that may be added.
"""
removals = ["-devedition", "-nightly", "-shippable"]
removals = ["-devedition", "-shippable"]
for remove in removals:
platform = platform.replace(remove, '')
return PLATFORM_RENAMES.get(platform, platform)

Просмотреть файл

@ -336,10 +336,10 @@ def get_release_config(config):
return release_config
def get_signing_cert_scope_per_platform(build_platform, is_nightly, config):
def get_signing_cert_scope_per_platform(build_platform, is_shippable, config):
if 'devedition' in build_platform:
return get_devedition_signing_cert_scope(config)
elif is_nightly:
elif is_shippable:
return get_signing_cert_scope(config)
else:
return add_scope_prefix(config, 'signing:cert:dep-signing')

Просмотреть файл

@ -290,11 +290,11 @@ def verify_always_optimized(task, taskgraph, scratch_pad, graph_config):
@verifications.add('full_task_graph')
def verify_nightly_no_sccache(task, taskgraph, scratch_pad, graph_config):
if task and any([task.attributes.get('nightly'), task.attributes.get('shippable')]):
def verify_shippable_no_sccache(task, taskgraph, scratch_pad, graph_config):
if task and task.attributes.get('shippable'):
if task.task.get('payload', {}).get('env', {}).get('USE_SCCACHE'):
raise Exception(
'Nightly job {} cannot use sccache'.format(task.label))
'Shippable job {} cannot use sccache'.format(task.label))
@verifications.add('full_task_graph')
@ -306,7 +306,6 @@ def verify_test_packaging(task, taskgraph, scratch_pad, graph_config):
build_env = task.task.get('payload', {}).get('env', {})
package_tests = build_env.get('MOZ_AUTOMATION_PACKAGE_TESTS')
shippable = task.attributes.get('shippable', False)
nightly = task.attributes.get('nightly', False)
build_has_tests = scratch_pad.get(task.label)
if package_tests != '1':
@ -315,10 +314,6 @@ def verify_test_packaging(task, taskgraph, scratch_pad, graph_config):
exceptions.append('Build job {} is shippable and does not specify '
'MOZ_AUTOMATION_PACKAGE_TESTS=1 in the '
'environment.'.format(task.label))
if nightly:
exceptions.append('Build job {} is nightly and does not specify '
'MOZ_AUTOMATION_PACKAGE_TESTS=1 in the '
'environment.'.format(task.label))
# Build tasks in the scratch pad have tests dependent on
# them, so we need to package tests during build.
@ -329,9 +324,9 @@ def verify_test_packaging(task, taskgraph, scratch_pad, graph_config):
else:
# Build tasks that aren't in the scratch pad have no
# dependent tests, so we shouldn't package tests.
# With the caveat that we expect shippable and nightly jobs to always
# With the caveat that we expect shippable jobs to always
# produce tests.
if not build_has_tests and not any([shippable, nightly]):
if not build_has_tests and not shippable:
exceptions.append(
'Build job {} has no tests, but specifies '
'MOZ_AUTOMATION_PACKAGE_TESTS={} in the environment. '