From 33c417e948cc442d077134f634c3231f853f0b3b Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Mon, 29 May 2017 15:34:54 -0400 Subject: [PATCH] Bug 1048446 - [taskcluster] Move 'require-build' out of run_task and into source_test, r=dustin The 'platform' key was recently moved out of 'job' tasks and into the 'source-test' kind. Since the concept of requiring a build depends on this key, let's move that back to source-test as well. MozReview-Commit-ID: 4bs8G4wN5OH --HG-- extra : rebase_source : 96b7c4631bfce2828fc496241bd29e839702c59f --- .../taskgraph/transforms/job/common.py | 28 -------------- taskcluster/taskgraph/transforms/job/mach.py | 6 --- .../taskgraph/transforms/job/run_task.py | 14 +------ .../taskgraph/transforms/source_test.py | 38 +++++++++++++++++++ 4 files changed, 39 insertions(+), 47 deletions(-) diff --git a/taskcluster/taskgraph/transforms/job/common.py b/taskcluster/taskgraph/transforms/job/common.py index 9a2bf51a38b7..8d9ddbd78bee 100644 --- a/taskcluster/taskgraph/transforms/job/common.py +++ b/taskcluster/taskgraph/transforms/job/common.py @@ -9,10 +9,6 @@ consistency. from __future__ import absolute_import, print_function, unicode_literals -from taskgraph.util.attributes import keymatch - - -ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}' @@ -61,30 +57,6 @@ def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc): }) -def add_build_dependency(config, job, taskdesc): - """Add build dependency to the task description and installer_url to env.""" - key = job['platform'] - build_labels = config.config.get('dependent-build-platforms', {}) - matches = keymatch(build_labels, key) - if not matches: - raise Exception("No build platform found for '{}'. " - "Define 'dependent-build-platforms' in the kind config.".format(key)) - - if len(matches) > 1: - raise Exception("More than one build platform found for '{}'.".format(key)) - - label = matches[0]['label'] - target = matches[0]['target-name'] - deps = taskdesc.setdefault('dependencies', {}) - deps.update({'build': label}) - - build_artifact = 'public/build/{}'.format(target) - installer_url = ARTIFACT_URL.format('', build_artifact) - - env = taskdesc['worker'].setdefault('env', {}) - env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}}) - - def support_vcs_checkout(config, job, taskdesc): """Update a job/task with parameters to enable a VCS checkout. diff --git a/taskcluster/taskgraph/transforms/job/mach.py b/taskcluster/taskgraph/transforms/job/mach.py index e8637e0b089f..2df8fe2ae840 100644 --- a/taskcluster/taskgraph/transforms/job/mach.py +++ b/taskcluster/taskgraph/transforms/job/mach.py @@ -20,12 +20,6 @@ mach_schema = Schema({ # The mach command (omitting `./mach`) to run Required('mach'): basestring, - - # Whether the job requires a build artifact or not. If True, the task - # will depend on a build task and run-task will download and set up the - # installer. Build labels are determined by the `dependent-build-platforms` - # config in kind.yml. - Required('requires-build', default=False): bool, }) diff --git a/taskcluster/taskgraph/transforms/job/run_task.py b/taskcluster/taskgraph/transforms/job/run_task.py index 87a77cf18d8f..cc53107e8ff8 100644 --- a/taskcluster/taskgraph/transforms/job/run_task.py +++ b/taskcluster/taskgraph/transforms/job/run_task.py @@ -9,10 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals from taskgraph.transforms.job import run_job_using from taskgraph.util.schema import Schema -from taskgraph.transforms.job.common import ( - add_build_dependency, - support_vcs_checkout, -) +from taskgraph.transforms.job.common import support_vcs_checkout from voluptuous import Required, Any run_task_schema = Schema({ @@ -29,12 +26,6 @@ run_task_schema = Schema({ # checkout arguments. If a list, it will be passed directly; otherwise # it will be included in a single argument to `bash -cx`. Required('command'): Any([basestring], basestring), - - # Whether the job requires a build artifact or not. If True, the task - # will depend on a build task and run-task will download and set up the - # installer. Build labels are determined by the `dependent-build-platforms` - # config in kind.yml. - Required('requires-build', default=False): bool, }) @@ -43,9 +34,6 @@ def common_setup(config, job, taskdesc): if run['checkout']: support_vcs_checkout(config, job, taskdesc) - if run['requires-build']: - add_build_dependency(config, job, taskdesc) - @run_job_using("docker-worker", "run-task", schema=run_task_schema) def docker_worker_run_task(config, job, taskdesc): diff --git a/taskcluster/taskgraph/transforms/source_test.py b/taskcluster/taskgraph/transforms/source_test.py index 5537681f48f7..81eb50f45864 100644 --- a/taskcluster/taskgraph/transforms/source_test.py +++ b/taskcluster/taskgraph/transforms/source_test.py @@ -13,6 +13,7 @@ import copy from taskgraph.transforms.base import TransformSequence from taskgraph.transforms.job import job_description_schema +from taskgraph.util.attributes import keymatch from taskgraph.util.schema import ( validate_schema, resolve_keyed_by, @@ -24,6 +25,8 @@ from voluptuous import ( Schema, ) +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' + job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()} source_test_description_schema = Schema({ @@ -36,6 +39,12 @@ source_test_description_schema = Schema({ # the job will be "split" into multiple tasks, one with each platform. Required('platform'): Any(basestring, [basestring]), + # Whether the job requires a build artifact or not. If True, the task will + # depend on a build task and the installer url will be saved to the + # GECKO_INSTALLER_URL environment variable. Build labels are determined by the + # `dependent-build-platforms` config in kind.yml. + Required('require-build', default=False): bool, + # These fields can be keyed by "platform", and are otherwise identical to # job descriptions. Required('worker-type'): Any( @@ -83,6 +92,32 @@ def expand_platforms(config, jobs): yield pjob +def add_build_dependency(config, job): + """ + Add build dependency to the job and installer_url to env. + """ + key = job['platform'] + build_labels = config.config.get('dependent-build-platforms', {}) + matches = keymatch(build_labels, key) + if not matches: + raise Exception("No build platform found for '{}'. " + "Define 'dependent-build-platforms' in the kind config.".format(key)) + + if len(matches) > 1: + raise Exception("More than one build platform found for '{}'.".format(key)) + + label = matches[0]['label'] + target = matches[0]['target-name'] + deps = job.setdefault('dependencies', {}) + deps.update({'build': label}) + + build_artifact = 'public/build/{}'.format(target) + installer_url = ARTIFACT_URL.format('', build_artifact) + + env = job['worker'].setdefault('env', {}) + env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}}) + + @transforms.add def handle_platform(config, jobs): """ @@ -103,5 +138,8 @@ def handle_platform(config, jobs): if 'treeherder' in job: job['treeherder']['platform'] = platform + if job.pop('require-build'): + add_build_dependency(config, job) + del job['platform'] yield job