Bug 1387199 - Do spidermonkey windows in taskgraph. r=dustin

Set these as tier 3 right now, because there are frequent intermittent test failures. MozReview-Commit-ID: Gs6AXLKaVrF --HG-- extra : rebase_source : b86c6ffd527dc8af963967c2722a3cee66a6ed3f
2017-08-16 15:27:30 -04:00 · 2017-08-16 15:27:30 -04:00 · d11db1f68c
--- a/taskcluster/ci/spidermonkey/kind.yml
+++ b/taskcluster/ci/spidermonkey/kind.yml
@ -8,6 +8,7 @@ kind-dependencies:
   - toolchain

 transforms:
+   - taskgraph.transforms.spidermonkey:transforms
   - taskgraph.transforms.build_attrs:transforms
   - taskgraph.transforms.toolchain:transforms
   - taskgraph.transforms.job:transforms
@ -21,11 +22,6 @@ job-defaults:
    index:
        product: firefox
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
-    worker:
-        max-run-time: 36000
-        docker-image: {in-tree: desktop-build}
-        env:
-            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
    run:
        using: spidermonkey
    when:
@ -59,8 +55,12 @@ job-defaults:
            - toolkit/mozapps/installer/package-name.mk
            - toolkit/mozapps/installer/upload-files.mk
    toolchains:
-        - linux64-clang
-        - linux64-gcc
+        by-worker-type:
+            .*-b-win2012:
+                - win64-clang-cl
+            default:
+                - linux64-clang
+                - linux64-gcc

 jobs:
    sm-package/opt:
@ -69,6 +69,11 @@ jobs:
            job-name: sm-package-opt
        treeherder:
            symbol: SM-tc(pkg)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
        run:
            using: spidermonkey-package
            spidermonkey-variant: plain
@ -80,6 +85,11 @@ jobs:
        treeherder:
            platform: linux64/debug
            symbol: SM-tc(mozjs-crate)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
        run:
            using: spidermonkey-mozjs-crate
            spidermonkey-variant: plain
@ -92,6 +102,27 @@ jobs:
        treeherder:
            platform: linux64/debug
            symbol: SM-tc(p)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+        run:
+            spidermonkey-variant: plaindebug
+
+    sm-plain-windows/debug:
+        description: "Spidermonkey Plain"
+        index:
+            job-name: sm-win-plaindebug-debug
+        treeherder:
+            platform: windows2012-32/debug
+            symbol: SM-tc(p)
+            tier: 3
+        worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+        worker:
+            max-run-time: 36000
+            env:
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
        run:
            spidermonkey-variant: plaindebug

@ -101,6 +132,27 @@ jobs:
            job-name: sm-plain-opt
        treeherder:
            symbol: SM-tc(p)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+        run:
+            spidermonkey-variant: plain
+
+    sm-plain-windows/opt:
+        description: "Spidermonkey Plain"
+        index:
+            job-name: sm-win-plain-opt
+        treeherder:
+            platform: windows2012-32/opt
+            symbol: SM-tc(p)
+            tier: 3
+        worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+        worker:
+            max-run-time: 36000
+            env:
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
        run:
            spidermonkey-variant: plain

@ -112,6 +164,8 @@ jobs:
            platform: linux32/debug
            symbol: SM-tc(arm)
        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
            env:
                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux32/releng.manifest"
        run:
@ -124,6 +178,11 @@ jobs:
        treeherder:
            platform: linux64/debug
            symbol: SM-tc(arm64)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
        run:
            spidermonkey-variant: arm64-sim

@ -134,6 +193,8 @@ jobs:
        treeherder:
            symbol: SM-tc(asan)
        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
            env:
                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
        run:
@ -146,6 +207,27 @@ jobs:
        treeherder:
            platform: linux64/debug
            symbol: SM-tc(cgc)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+        run:
+            spidermonkey-variant: compacting
+
+    sm-compacting-windows/debug:
+        description: "Spidermonkey Compacting"
+        index:
+            job-name: sm-win-compacting-debug
+        treeherder:
+            platform: windows2012-32/debug
+            symbol: SM-tc(cgc)
+            tier: 3
+        worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+        worker:
+            max-run-time: 36000
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
        run:
            spidermonkey-variant: compacting

@ -156,6 +238,8 @@ jobs:
        treeherder:
            symbol: SM-tc(msan)
        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
            env:
                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
        run:
@ -169,6 +253,8 @@ jobs:
            symbol: SM-tc(tsan)
            tier: 3
        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
            env:
                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/tsan.manifest"
        run:
@ -181,6 +267,11 @@ jobs:
        treeherder:
            platform: linux64/debug
            symbol: SM-tc(r)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
        run:
            spidermonkey-variant: rootanalysis

@ -191,6 +282,11 @@ jobs:
        treeherder:
            platform: linux64/debug
            symbol: SM-tc(nu)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
        run:
            spidermonkey-variant: nonunified

@ -201,5 +297,10 @@ jobs:
        treeherder:
            platform: linux64/opt
            symbol: SM-tc(f)
+        worker:
+            max-run-time: 36000
+            docker-image: {in-tree: desktop-build}
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
        run:
            spidermonkey-variant: fuzzing
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@ -77,13 +77,16 @@ JOB_NAME_WHITELIST = set([
    'sm-arm64-sim-debug',
    'sm-asan-opt',
    'sm-compacting-debug',
+    'sm-win-compacting-debug',
    'sm-fuzzing',
    'sm-mozjs-sys-debug',
    'sm-msan-opt',
    'sm-nonunified-debug',
    'sm-package-opt',
    'sm-plain-opt',
+    'sm-win-plain-opt',
    'sm-plaindebug-debug',
+    'sm-win-plaindebug-debug',
    'sm-rootanalysis-debug',
    'sm-tsan-opt',
    'win32-add-on-devel',
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@ -13,6 +13,8 @@ from voluptuous import Required, Any
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
    docker_worker_add_public_artifacts,
+    generic_worker_add_public_artifacts,
+    docker_worker_add_gecko_vcs_env_vars,
    support_vcs_checkout,
 )

@ -80,3 +82,55 @@ def docker_worker_spidermonkey(config, job, taskdesc):
        '-c',
        'cd /home/worker && workspace/build/src/taskcluster/scripts/builder/%s' % script
    ]
+
+
+@run_job_using("generic-worker", "spidermonkey", schema=sm_run_schema)
+def generic_worker_spidermonkey(config, job, taskdesc):
+    assert job['worker']['os'] == 'windows', 'only supports windows right now'
+
+    run = job['run']
+
+    worker = taskdesc['worker']
+
+    generic_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker.setdefault('env', {})
+    env.update({
+        'MOZHARNESS_DISABLE': 'true',
+        'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
+        'MOZ_BUILD_DATE': config.params['moz_build_date'],
+        'MOZ_SCM_LEVEL': config.params['level'],
+        'SCCACHE_DISABLE': "1",
+        'WORK': ".",  # Override the defaults in build scripts
+        'SRCDIR': "./src",  # with values suiteable for windows generic worker
+        'UPLOAD_DIR': "./public/build"
+    })
+
+    script = "build-sm.sh"
+    if run['using'] == 'spidermonkey-package':
+        script = "build-sm-package.sh"
+        # Don't allow untested configurations yet
+        raise Exception("spidermonkey-package is not a supported configuration")
+    elif run['using'] == 'spidermonkey-mozjs-crate':
+        script = "build-sm-mozjs-crate.sh"
+        # Don't allow untested configurations yet
+        raise Exception("spidermonkey-mozhs-crate is not a supported configuration")
+
+    hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
+    hg_command.append('robustcheckout')
+    hg_command.extend(['--sharebase', 'y:\\hg-shared'])
+    hg_command.append('--purge')
+    hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
+    hg_command.extend(['--revision', env['GECKO_HEAD_REV']])
+    hg_command.append(env['GECKO_HEAD_REPOSITORY'])
+    hg_command.append('.\\src')
+
+    command = ['c:\\mozilla-build\\msys\\bin\\bash.exe '  # string concat
+               '"./src/taskcluster/scripts/builder/%s"' % script]
+
+    worker['command'] = []
+    worker['command'].extend([
+        ' '.join(hg_command),
+        ' '.join(command)
+    ])
--- a/taskcluster/taskgraph/transforms/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/spidermonkey.py
@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+import copy
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def handle_keyed_by(config, jobs):
+    """Resolve fields that can be keyed by platform, etc."""
+    fields = [
+        "toolchains"
+    ]
+    for job in jobs:
+        job = copy.deepcopy(job)  # don't overwrite dict values here
+        for field in fields:
+            resolve_keyed_by(item=job, field=field, item_name=job['name'])
+
+        yield job