Bug 1286075: add source-check kind; r=ahal

MozReview-Commit-ID: EDdPraYxNlH

--HG--
extra : rebase_source : aab182e3dfda2bbe831f8aef6a06a0d946384b8f

taskcluster/ci/source-check/doc.yml

@@ -0,0 +1,36 @@
+sphinx/opt:
+    description: Generate the Sphinx documentation
+    index:
+        product: lint
+        job-name:
+            gecko-v2: sphinx-opt
+    treeherder:
+        symbol: tc(Doc)
+        kind: test
+        tier: 1
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+        artifacts:
+            - type: file
+              name: public/docs.tar.gz
+              path: /home/worker/checkouts/gecko/docs.tar.gz
+    run:
+        using: run-task
+        command: >
+            cd /home/worker/checkouts/gecko &&
+            ./mach doc --outdir docs-out --no-open &&
+            rm -rf docs-out/html/Mozilla_Source_Tree_Docs/_venv &&
+            mv docs-out/html/Mozilla_Source_Tree_Docs docs &&
+            tar -czf docs.tar.gz docs
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - '**/*.py'
+            - '**/*.rst'
+            - 'tools/docs/**'

taskcluster/ci/source-check/kind.yml

@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+jobs-from:
+    - python-tests.yml
+    - mozlint.yml
+    - doc.yml

taskcluster/ci/source-check/mozlint.yml

@@ -0,0 +1,107 @@
+mozlint-eslint/opt:
+    description: JS lint check
+    index:
+        product: lint
+        job-name:
+            gecko-v2: eslint-gecko-opt
+    treeherder:
+        symbol: ES
+        kind: test
+        tier: 1
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: run-task
+        command: >
+            cd /home/worker/checkouts/gecko/tools/lint/eslint &&
+            /build/tooltool.py fetch -m manifest.tt &&
+            tar xvfz eslint.tar.gz &&
+            rm eslint.tar.gz &&
+            ln -s ../eslint-plugin-mozilla node_modules &&
+            cd ../../.. &&
+            ./mach lint -l eslint -f treeherder --quiet
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            # Files that are likely audited.
+            - '**/*.js'
+            - '**/*.jsm'
+            - '**/*.jsx'
+            - '**/*.html'
+            - '**/*.xhtml'
+            - '**/*.xml'
+            # Run when eslint policies change.
+            - '**/.eslintignore'
+            - '**/*eslintrc*'
+            # The plugin implementing custom checks.
+            - 'tools/lint/eslint/eslint-plugin-mozilla/**'
+            # Other misc lint related files.
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
+            - 'testing/docker/lint/**'
+
+mozlint-flake8/opt:
+    description: flake8 run over the gecko codebase
+    index:
+        product: lint
+        job-name:
+            gecko-v2: mozlint-flake8-opt
+    treeherder:
+        symbol: f8
+        kind: test
+        tier: 2
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: lint -l flake8 -f treeherder
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - '**/*.py'
+            - '**/.flake8'
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
+            - 'testing/docker/lint/**'
+
+wptlint-gecko/opt:
+    description: web-platform-tests linter
+    index:
+        product: lint
+        job-name:
+            gecko-v2: wpt-lint-gecko-opt
+    treeherder:
+        symbol: W
+        kind: test
+        tier: 2
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: lint -l wpt -f treeherder
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'testing/web-platform/tests/**'
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
+            - 'testing/docker/lint/**'
+

taskcluster/ci/source-check/python-tests.yml

@@ -0,0 +1,57 @@
+taskgraph-tests/opt:
+    description: taskcluster/taskgraph unit tests
+    index:
+        product: lint
+        job-name:
+            gecko-v2: taskgraph-tests-opt
+    treeherder:
+        symbol: tg
+        kind: test
+        tier: 2
+        platform: linux64/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: taskgraph python-tests
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'taskcluster/**/*.py'
+            - 'config/mozunit.py'
+            - 'python/mach/**/*.py'
+
+mozharness/opt:
+    description: mozharness integration tests
+    index:
+        product: lint
+        job-name:
+            gecko-v2: mozharness-tox-opt
+    treeherder:
+        symbol: MH
+        kind: test
+        tier: 2
+        platform: lint/opt
+    worker-type: aws-provisioner-v1/b2gtest
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: run-task
+        cache-dotcache: true
+        command: >
+            cd /home/worker/checkouts/gecko/testing/mozharness &&
+            /usr/bin/pip2 install tox &&
+            /home/worker/.local/bin/tox -e py27-hg3.7
+    run-on-projects:
+        - integration
+        - release
+    when:
+        files-changed:
+            - 'testing/mozharness/**'

taskcluster/docs/kinds.rst

@@ -30,6 +30,14 @@ l10n
 
 TBD (Callek)
 
+source-check
+------------
+
+Source-checks are tasks that look at the Gecko source directly to check
+correctness.  This can include linting, Python unit tests, source-code
+analysis, or measurement work -- basically anything that does not require a
+build.
+
 Tests
 -----
 

taskcluster/taskgraph/transforms/job/mach.py

@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running mach tasks (via run-task)
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.run_task import docker_worker_run_task
+from voluptuous import Schema, Required
+
+mach_schema = Schema({
+    Required('using'): 'mach',
+
+    # The mach command (omitting `./mach`) to run
+    Required('mach'): basestring,
+})
+
+
+@run_job_using("docker-worker", "mach", schema=mach_schema)
+def docker_worker_mach(config, job, taskdesc):
+    run = job['run']
+
+    # defer to the run_task implementation
+    run['command'] = 'cd /home/worker/checkouts/gecko && ./mach ' + run['mach']
+    run['checkout'] = True
+    del run['mach']
+    docker_worker_run_task(config, job, taskdesc)

taskcluster/taskgraph/transforms/task.py

@@ -83,7 +83,8 @@ task_description_schema = Schema({
     # if omitted, the build will not be indexed.
     Optional('index'): {
         # the name of the product this build produces
-        'product': Any('firefox', 'mobile', 'b2g'),
+        # TODO: remove lint when not used
+        'product': Any('firefox', 'mobile', 'b2g', 'lint'),
 
         # the names to use for this job in the TaskCluster index
         'job-name': Any(

taskcluster/taskgraph/try_option_syntax.py

@@ -31,6 +31,7 @@ BUILD_KINDS = set([
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
+    'source-check',
 ])