diff --git a/taskcluster/ci/source-test/doc.yml b/taskcluster/ci/source-test/doc.yml index 0ab91c51819c..c8b75d54f201 100644 --- a/taskcluster/ci/source-test/doc.yml +++ b/taskcluster/ci/source-test/doc.yml @@ -1,10 +1,10 @@ -sphinx/opt: +sphinx: description: Generate the Sphinx documentation + platform: lint/opt treeherder: symbol: tc(Doc) kind: test tier: 1 - platform: lint/opt worker-type: aws-provisioner-v1/b2gtest worker: implementation: docker-worker diff --git a/taskcluster/ci/source-test/kind.yml b/taskcluster/ci/source-test/kind.yml index 04bc09b472da..688facde0c83 100644 --- a/taskcluster/ci/source-test/kind.yml +++ b/taskcluster/ci/source-test/kind.yml @@ -5,7 +5,7 @@ implementation: taskgraph.task.transform:TransformTask transforms: - - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.source_test:transforms - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms diff --git a/taskcluster/ci/source-test/mozlint.yml b/taskcluster/ci/source-test/mozlint.yml index b24ea0ccc48a..b43e210031fe 100644 --- a/taskcluster/ci/source-test/mozlint.yml +++ b/taskcluster/ci/source-test/mozlint.yml @@ -1,10 +1,10 @@ -mozlint-eslint/opt: +mozlint-eslint: description: JS lint check + platform: lint/opt treeherder: symbol: ES kind: test tier: 1 - platform: lint/opt worker-type: aws-provisioner-v1/b2gtest worker: implementation: docker-worker @@ -41,13 +41,13 @@ mozlint-eslint/opt: - 'python/mozlint/**' - 'tools/lint/**' -mozlint-flake8/opt: +mozlint-flake8: description: flake8 run over the gecko codebase + platform: lint/opt treeherder: symbol: f8 kind: test tier: 1 - platform: lint/opt worker-type: aws-provisioner-v1/b2gtest worker: implementation: docker-worker @@ -66,13 +66,13 @@ mozlint-flake8/opt: - 'python/mozlint/**' - 'tools/lint/**' -wptlint-gecko/opt: +wptlint-gecko: description: web-platform-tests linter + platform: lint/opt treeherder: symbol: W kind: test tier: 1 - platform: lint/opt worker-type: aws-provisioner-v1/b2gtest worker: implementation: docker-worker diff --git a/taskcluster/ci/source-test/python-tests.yml b/taskcluster/ci/source-test/python-tests.yml index da86fad29000..c913f4383e67 100644 --- a/taskcluster/ci/source-test/python-tests.yml +++ b/taskcluster/ci/source-test/python-tests.yml @@ -1,10 +1,10 @@ -taskgraph-tests/opt: +taskgraph-tests: description: taskcluster/taskgraph unit tests + platform: linux64/opt treeherder: symbol: py(tg) kind: test tier: 2 - platform: linux64/opt worker-type: aws-provisioner-v1/b2gtest worker: implementation: docker-worker @@ -22,10 +22,9 @@ taskgraph-tests/opt: - 'config/mozunit.py' - 'python/mach/**/*.py' -marionette-harness/opt: +marionette-harness: description: testing/marionette/harness unit tests - platforms: - - linux64/opt + platform: linux64/opt treeherder: symbol: py(mnh) kind: test @@ -51,9 +50,9 @@ marionette-harness/opt: - 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**' - 'python/mach_commands.py' -mozbase/opt: +mozbase: description: testing/mozbase unit tests - platforms: + platform: - linux64/opt treeherder: symbol: py(mb) @@ -80,13 +79,13 @@ mozbase/opt: - 'config/mozunit.py' - 'python/mach_commands.py' -mozharness/opt: +mozharness: description: mozharness integration tests + platform: lint/opt treeherder: symbol: py(MH) kind: test tier: 2 - platform: lint/opt worker-type: aws-provisioner-v1/b2gtest worker: implementation: docker-worker @@ -106,10 +105,9 @@ mozharness/opt: files-changed: - 'testing/mozharness/**' -mozlint/opt: +mozlint: description: python/mozlint unit tests - platforms: - - linux64/opt + platform: linux64/opt treeherder: symbol: py(ml) kind: test diff --git a/taskcluster/ci/source-test/webidl.yml b/taskcluster/ci/source-test/webidl.yml index 22460a471917..52b93cd7e107 100644 --- a/taskcluster/ci/source-test/webidl.yml +++ b/taskcluster/ci/source-test/webidl.yml @@ -1,10 +1,10 @@ -webidl-test/opt: +webidl-test: description: WebIDL parser tests + platform: lint/opt treeherder: symbol: Wp kind: test tier: 1 - platform: lint/opt worker-type: aws-provisioner-v1/b2gtest worker: implementation: docker-worker diff --git a/taskcluster/taskgraph/transforms/job/__init__.py b/taskcluster/taskgraph/transforms/job/__init__.py index 7c6d98f81507..46bf63a870b9 100644 --- a/taskcluster/taskgraph/transforms/job/__init__.py +++ b/taskcluster/taskgraph/transforms/job/__init__.py @@ -16,10 +16,7 @@ import logging import os from taskgraph.transforms.base import TransformSequence -from taskgraph.util.schema import ( - validate_schema, - resolve_keyed_by, -) +from taskgraph.util.schema import validate_schema from taskgraph.transforms.task import task_description_schema from voluptuous import ( Any, @@ -70,7 +67,7 @@ job_description_schema = Schema({ # own schema. Extra: object, }, - Optional('platforms'): [basestring], + Required('worker-type'): Any( task_description_schema['worker-type'], {'by-platform': {basestring: task_description_schema['worker-type']}}, @@ -91,36 +88,6 @@ def validate(config, jobs): "In job {!r}:".format(job['name'])) -@transforms.add -def expand_platforms(config, jobs): - for job in jobs: - if 'platforms' not in job: - yield job - continue - - for platform in job['platforms']: - pjob = copy.deepcopy(job) - pjob['platform'] = platform - del pjob['platforms'] - - platform, buildtype = platform.rsplit('/', 1) - pjob['name'] = '{}-{}-{}'.format(pjob['name'], platform, buildtype) - yield pjob - - -@transforms.add -def handle_keyed_by(config, jobs): - fields = [ - 'worker-type', - 'worker', - ] - - for job in jobs: - for field in fields: - resolve_keyed_by(job, field, item_name=job['name']) - yield job - - @transforms.add def make_task_description(config, jobs): """Given a build description, create a task description""" @@ -147,11 +114,6 @@ def make_task_description(config, jobs): configure_taskdesc_for_run(config, job, taskdesc) del taskdesc['run'] - if 'platform' in taskdesc: - if 'treeherder' in taskdesc: - taskdesc['treeherder']['platform'] = taskdesc['platform'] - del taskdesc['platform'] - # yield only the task description, discarding the job description yield taskdesc diff --git a/taskcluster/taskgraph/transforms/source_test.py b/taskcluster/taskgraph/transforms/source_test.py new file mode 100644 index 000000000000..d1c69ffc20f0 --- /dev/null +++ b/taskcluster/taskgraph/transforms/source_test.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Source-test jobs can run on multiple platforms. These transforms allow jobs +with either `platform` or a list of `platforms`, and set the appropriate +treeherder configuration and attributes for that platform. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import copy + +from taskgraph.transforms.base import TransformSequence +from taskgraph.transforms.job import job_description_schema +from taskgraph.util.schema import ( + validate_schema, + resolve_keyed_by, +) +from voluptuous import ( + Any, + Extra, + Required, + Schema, +) + +job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()} + +source_test_description_schema = Schema({ + # most fields are passed directly through as job fields, and are not + # repeated here + Extra: object, + + # The platform on which this task runs. This will be used to set up attributes + # (for try selection) and treeherder metadata (for display). If given as a list, + # the job will be "split" into multiple tasks, one with each platform. + Required('platform'): Any(basestring, [basestring]), + + # These fields can be keyed by "platform", and are otherwise identical to + # job descriptions. + Required('worker-type'): Any( + job_description_schema['worker-type'], + {'by-platform': {basestring: job_description_schema['worker-type']}}, + ), + Required('worker'): Any( + job_description_schema['worker'], + {'by-platform': {basestring: job_description_schema['worker']}}, + ), +}) + +transforms = TransformSequence() + + +@transforms.add +def validate(config, jobs): + for job in jobs: + yield validate_schema(source_test_description_schema, job, + "In job {!r}:".format(job['name'])) + + +@transforms.add +def expand_platforms(config, jobs): + for job in jobs: + if isinstance(job['platform'], basestring): + yield job + continue + + for platform in job['platform']: + pjob = copy.deepcopy(job) + pjob['platform'] = platform + + if 'name' in pjob: + pjob['name'] = '{}-{}'.format(pjob['name'], platform) + else: + pjob['label'] = '{}-{}'.format(pjob['label'], platform) + yield pjob + + +@transforms.add +def handle_platform(config, jobs): + """ + Handle the 'platform' property, setting up treeherder context as well as + try-related attributes. + """ + fields = [ + 'worker-type', + 'worker', + ] + + for job in jobs: + platform = job['platform'] + + for field in fields: + resolve_keyed_by(job, field, item_name=job['name']) + + if 'treeherder' in job: + job['treeherder']['platform'] = platform + + build_platform, build_type = platform.split('/') + + attributes = job.setdefault('attributes', {}) + attributes.update({ + 'build_platform': build_platform, + 'build_type': build_type, + }) + + del job['platform'] + yield job