Bug 1337903: handle platforms for source-test distinctly; r=ahal

This clears up some confusion and undocumented behavior around platforms in
source-tests (and available to any job).

With this change, the attributes come out like this:

  "source-test-mozbase-linux64/opt": {
    "attributes": {
      "build_platform": "linux64",
      "build_type": "opt",
      "kind": "source-test",
      "run_on_projects": [
        "integration",
        "release"
      ]
    },

MozReview-Commit-ID: HN1Zi8YUf0

--HG--
extra : rebase_source : 552bffc4646a3eec46e7edb508d8eb4d2a8e2e03

taskcluster/ci/source-test/doc.yml

@@ -1,10 +1,10 @@
-sphinx/opt:
+sphinx:
     description: Generate the Sphinx documentation
+    platform: lint/opt
     treeherder:
         symbol: tc(Doc)
         kind: test
         tier: 1
-        platform: lint/opt
     worker-type: aws-provisioner-v1/b2gtest
     worker:
         implementation: docker-worker

taskcluster/ci/source-test/kind.yml

@@ -5,7 +5,7 @@
 implementation: taskgraph.task.transform:TransformTask
 
 transforms:
-   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.source_test:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 

taskcluster/ci/source-test/mozlint.yml

@@ -1,10 +1,10 @@
-mozlint-eslint/opt:
+mozlint-eslint:
     description: JS lint check
+    platform: lint/opt
     treeherder:
         symbol: ES
         kind: test
         tier: 1
-        platform: lint/opt
     worker-type: aws-provisioner-v1/b2gtest
     worker:
         implementation: docker-worker
@@ -41,13 +41,13 @@ mozlint-eslint/opt:
             - 'python/mozlint/**'
             - 'tools/lint/**'
 
-mozlint-flake8/opt:
+mozlint-flake8:
     description: flake8 run over the gecko codebase
+    platform: lint/opt
     treeherder:
         symbol: f8
         kind: test
         tier: 1
-        platform: lint/opt
     worker-type: aws-provisioner-v1/b2gtest
     worker:
         implementation: docker-worker
@@ -66,13 +66,13 @@ mozlint-flake8/opt:
             - 'python/mozlint/**'
             - 'tools/lint/**'
 
-wptlint-gecko/opt:
+wptlint-gecko:
     description: web-platform-tests linter
+    platform: lint/opt
     treeherder:
         symbol: W
         kind: test
         tier: 1
-        platform: lint/opt
     worker-type: aws-provisioner-v1/b2gtest
     worker:
         implementation: docker-worker

taskcluster/ci/source-test/python-tests.yml

@@ -1,10 +1,10 @@
-taskgraph-tests/opt:
+taskgraph-tests:
     description: taskcluster/taskgraph unit tests
+    platform: linux64/opt
     treeherder:
         symbol: py(tg)
         kind: test
         tier: 2
-        platform: linux64/opt
     worker-type: aws-provisioner-v1/b2gtest
     worker:
         implementation: docker-worker
@@ -22,10 +22,9 @@ taskgraph-tests/opt:
             - 'config/mozunit.py'
             - 'python/mach/**/*.py'
 
-marionette-harness/opt:
+marionette-harness:
     description: testing/marionette/harness unit tests
-    platforms:
-        - linux64/opt
+    platform: linux64/opt
     treeherder:
         symbol: py(mnh)
         kind: test
@@ -51,9 +50,9 @@ marionette-harness/opt:
           - 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**'
           - 'python/mach_commands.py'
 
-mozbase/opt:
+mozbase:
     description: testing/mozbase unit tests
-    platforms:
+    platform:
         - linux64/opt
     treeherder:
         symbol: py(mb)
@@ -80,13 +79,13 @@ mozbase/opt:
             - 'config/mozunit.py'
             - 'python/mach_commands.py'
 
-mozharness/opt:
+mozharness:
     description: mozharness integration tests
+    platform: lint/opt
     treeherder:
         symbol: py(MH)
         kind: test
         tier: 2
-        platform: lint/opt
     worker-type: aws-provisioner-v1/b2gtest
     worker:
         implementation: docker-worker
@@ -106,10 +105,9 @@ mozharness/opt:
         files-changed:
             - 'testing/mozharness/**'
 
-mozlint/opt:
+mozlint:
     description: python/mozlint unit tests
-    platforms:
-        - linux64/opt
+    platform: linux64/opt
     treeherder:
         symbol: py(ml)
         kind: test

taskcluster/ci/source-test/webidl.yml

@@ -1,10 +1,10 @@
-webidl-test/opt:
+webidl-test:
     description: WebIDL parser tests
+    platform: lint/opt
     treeherder:
         symbol: Wp
         kind: test
         tier: 1
-        platform: lint/opt
     worker-type: aws-provisioner-v1/b2gtest
     worker:
         implementation: docker-worker

taskcluster/taskgraph/transforms/job/__init__.py

@@ -16,10 +16,7 @@ import logging
 import os
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import (
-    validate_schema,
-    resolve_keyed_by,
-)
+from taskgraph.util.schema import validate_schema
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import (
     Any,
@@ -70,7 +67,7 @@ job_description_schema = Schema({
         # own schema.
         Extra: object,
     },
-    Optional('platforms'): [basestring],
+
     Required('worker-type'): Any(
         task_description_schema['worker-type'],
         {'by-platform': {basestring: task_description_schema['worker-type']}},
@@ -91,36 +88,6 @@ def validate(config, jobs):
                               "In job {!r}:".format(job['name']))
 
 
-@transforms.add
-def expand_platforms(config, jobs):
-    for job in jobs:
-        if 'platforms' not in job:
-            yield job
-            continue
-
-        for platform in job['platforms']:
-            pjob = copy.deepcopy(job)
-            pjob['platform'] = platform
-            del pjob['platforms']
-
-            platform, buildtype = platform.rsplit('/', 1)
-            pjob['name'] = '{}-{}-{}'.format(pjob['name'], platform, buildtype)
-            yield pjob
-
-
-@transforms.add
-def handle_keyed_by(config, jobs):
-    fields = [
-        'worker-type',
-        'worker',
-    ]
-
-    for job in jobs:
-        for field in fields:
-            resolve_keyed_by(job, field, item_name=job['name'])
-        yield job
-
-
 @transforms.add
 def make_task_description(config, jobs):
     """Given a build description, create a task description"""
@@ -147,11 +114,6 @@ def make_task_description(config, jobs):
         configure_taskdesc_for_run(config, job, taskdesc)
         del taskdesc['run']
 
-        if 'platform' in taskdesc:
-            if 'treeherder' in taskdesc:
-                taskdesc['treeherder']['platform'] = taskdesc['platform']
-            del taskdesc['platform']
-
         # yield only the task description, discarding the job description
         yield taskdesc
 

taskcluster/taskgraph/transforms/source_test.py

@@ -0,0 +1,108 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Source-test jobs can run on multiple platforms.  These transforms allow jobs
+with either `platform` or a list of `platforms`, and set the appropriate
+treeherder configuration and attributes for that platform.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.job import job_description_schema
+from taskgraph.util.schema import (
+    validate_schema,
+    resolve_keyed_by,
+)
+from voluptuous import (
+    Any,
+    Extra,
+    Required,
+    Schema,
+)
+
+job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
+
+source_test_description_schema = Schema({
+    # most fields are passed directly through as job fields, and are not
+    # repeated here
+    Extra: object,
+
+    # The platform on which this task runs.  This will be used to set up attributes
+    # (for try selection) and treeherder metadata (for display).  If given as a list,
+    # the job will be "split" into multiple tasks, one with each platform.
+    Required('platform'): Any(basestring, [basestring]),
+
+    # These fields can be keyed by "platform", and are otherwise identical to
+    # job descriptions.
+    Required('worker-type'): Any(
+        job_description_schema['worker-type'],
+        {'by-platform': {basestring: job_description_schema['worker-type']}},
+    ),
+    Required('worker'): Any(
+        job_description_schema['worker'],
+        {'by-platform': {basestring: job_description_schema['worker']}},
+    ),
+})
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        yield validate_schema(source_test_description_schema, job,
+                              "In job {!r}:".format(job['name']))
+
+
+@transforms.add
+def expand_platforms(config, jobs):
+    for job in jobs:
+        if isinstance(job['platform'], basestring):
+            yield job
+            continue
+
+        for platform in job['platform']:
+            pjob = copy.deepcopy(job)
+            pjob['platform'] = platform
+
+            if 'name' in pjob:
+                pjob['name'] = '{}-{}'.format(pjob['name'], platform)
+            else:
+                pjob['label'] = '{}-{}'.format(pjob['label'], platform)
+            yield pjob
+
+
+@transforms.add
+def handle_platform(config, jobs):
+    """
+    Handle the 'platform' property, setting up treeherder context as well as
+    try-related attributes.
+    """
+    fields = [
+        'worker-type',
+        'worker',
+    ]
+
+    for job in jobs:
+        platform = job['platform']
+
+        for field in fields:
+            resolve_keyed_by(job, field, item_name=job['name'])
+
+        if 'treeherder' in job:
+            job['treeherder']['platform'] = platform
+
+        build_platform, build_type = platform.split('/')
+
+        attributes = job.setdefault('attributes', {})
+        attributes.update({
+            'build_platform': build_platform,
+            'build_type': build_type,
+        })
+
+        del job['platform']
+        yield job