From d11db1f68cafa18bcbf811640bcd6421f3247c1b Mon Sep 17 00:00:00 2001 From: Justin Wood Date: Wed, 16 Aug 2017 15:27:30 -0400 Subject: [PATCH] Bug 1387199 - Do spidermonkey windows in taskgraph. r=dustin Set these as tier 3 right now, because there are frequent intermittent test failures. MozReview-Commit-ID: Gs6AXLKaVrF --HG-- extra : rebase_source : b86c6ffd527dc8af963967c2722a3cee66a6ed3f --- taskcluster/ci/spidermonkey/kind.yml | 115 ++++++++++++++++-- .../transforms/gecko_v2_whitelist.py | 3 + .../taskgraph/transforms/job/spidermonkey.py | 54 ++++++++ .../taskgraph/transforms/spidermonkey.py | 24 ++++ 4 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 taskcluster/taskgraph/transforms/spidermonkey.py diff --git a/taskcluster/ci/spidermonkey/kind.yml b/taskcluster/ci/spidermonkey/kind.yml index 88ea79476397..2dbdca1142f9 100644 --- a/taskcluster/ci/spidermonkey/kind.yml +++ b/taskcluster/ci/spidermonkey/kind.yml @@ -8,6 +8,7 @@ kind-dependencies: - toolchain transforms: + - taskgraph.transforms.spidermonkey:transforms - taskgraph.transforms.build_attrs:transforms - taskgraph.transforms.toolchain:transforms - taskgraph.transforms.job:transforms @@ -21,11 +22,6 @@ job-defaults: index: product: firefox worker-type: aws-provisioner-v1/gecko-{level}-b-linux - worker: - max-run-time: 36000 - docker-image: {in-tree: desktop-build} - env: - TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: using: spidermonkey when: @@ -59,8 +55,12 @@ job-defaults: - toolkit/mozapps/installer/package-name.mk - toolkit/mozapps/installer/upload-files.mk toolchains: - - linux64-clang - - linux64-gcc + by-worker-type: + .*-b-win2012: + - win64-clang-cl + default: + - linux64-clang + - linux64-gcc jobs: sm-package/opt: @@ -69,6 +69,11 @@ jobs: job-name: sm-package-opt treeherder: symbol: SM-tc(pkg) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: using: spidermonkey-package spidermonkey-variant: plain @@ -80,6 +85,11 @@ jobs: treeherder: platform: linux64/debug symbol: SM-tc(mozjs-crate) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: using: spidermonkey-mozjs-crate spidermonkey-variant: plain @@ -92,6 +102,27 @@ jobs: treeherder: platform: linux64/debug symbol: SM-tc(p) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" + run: + spidermonkey-variant: plaindebug + + sm-plain-windows/debug: + description: "Spidermonkey Plain" + index: + job-name: sm-win-plaindebug-debug + treeherder: + platform: windows2012-32/debug + symbol: SM-tc(p) + tier: 3 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + max-run-time: 36000 + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest" run: spidermonkey-variant: plaindebug @@ -101,6 +132,27 @@ jobs: job-name: sm-plain-opt treeherder: symbol: SM-tc(p) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" + run: + spidermonkey-variant: plain + + sm-plain-windows/opt: + description: "Spidermonkey Plain" + index: + job-name: sm-win-plain-opt + treeherder: + platform: windows2012-32/opt + symbol: SM-tc(p) + tier: 3 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + max-run-time: 36000 + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest" run: spidermonkey-variant: plain @@ -112,6 +164,8 @@ jobs: platform: linux32/debug symbol: SM-tc(arm) worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} env: TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux32/releng.manifest" run: @@ -124,6 +178,11 @@ jobs: treeherder: platform: linux64/debug symbol: SM-tc(arm64) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: spidermonkey-variant: arm64-sim @@ -134,6 +193,8 @@ jobs: treeherder: symbol: SM-tc(asan) worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} env: TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: @@ -146,6 +207,27 @@ jobs: treeherder: platform: linux64/debug symbol: SM-tc(cgc) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" + run: + spidermonkey-variant: compacting + + sm-compacting-windows/debug: + description: "Spidermonkey Compacting" + index: + job-name: sm-win-compacting-debug + treeherder: + platform: windows2012-32/debug + symbol: SM-tc(cgc) + tier: 3 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + max-run-time: 36000 + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest" run: spidermonkey-variant: compacting @@ -156,6 +238,8 @@ jobs: treeherder: symbol: SM-tc(msan) worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} env: TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: @@ -169,6 +253,8 @@ jobs: symbol: SM-tc(tsan) tier: 3 worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} env: TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/tsan.manifest" run: @@ -181,6 +267,11 @@ jobs: treeherder: platform: linux64/debug symbol: SM-tc(r) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: spidermonkey-variant: rootanalysis @@ -191,6 +282,11 @@ jobs: treeherder: platform: linux64/debug symbol: SM-tc(nu) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: spidermonkey-variant: nonunified @@ -201,5 +297,10 @@ jobs: treeherder: platform: linux64/opt symbol: SM-tc(f) + worker: + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" run: spidermonkey-variant: fuzzing diff --git a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py index e87192601ec2..3b121f23539e 100644 --- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py +++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py @@ -77,13 +77,16 @@ JOB_NAME_WHITELIST = set([ 'sm-arm64-sim-debug', 'sm-asan-opt', 'sm-compacting-debug', + 'sm-win-compacting-debug', 'sm-fuzzing', 'sm-mozjs-sys-debug', 'sm-msan-opt', 'sm-nonunified-debug', 'sm-package-opt', 'sm-plain-opt', + 'sm-win-plain-opt', 'sm-plaindebug-debug', + 'sm-win-plaindebug-debug', 'sm-rootanalysis-debug', 'sm-tsan-opt', 'win32-add-on-devel', diff --git a/taskcluster/taskgraph/transforms/job/spidermonkey.py b/taskcluster/taskgraph/transforms/job/spidermonkey.py index d38cc5ddc5bb..35bb735b8fc7 100644 --- a/taskcluster/taskgraph/transforms/job/spidermonkey.py +++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py @@ -13,6 +13,8 @@ from voluptuous import Required, Any from taskgraph.transforms.job import run_job_using from taskgraph.transforms.job.common import ( docker_worker_add_public_artifacts, + generic_worker_add_public_artifacts, + docker_worker_add_gecko_vcs_env_vars, support_vcs_checkout, ) @@ -80,3 +82,55 @@ def docker_worker_spidermonkey(config, job, taskdesc): '-c', 'cd /home/worker && workspace/build/src/taskcluster/scripts/builder/%s' % script ] + + +@run_job_using("generic-worker", "spidermonkey", schema=sm_run_schema) +def generic_worker_spidermonkey(config, job, taskdesc): + assert job['worker']['os'] == 'windows', 'only supports windows right now' + + run = job['run'] + + worker = taskdesc['worker'] + + generic_worker_add_public_artifacts(config, job, taskdesc) + docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) + + env = worker.setdefault('env', {}) + env.update({ + 'MOZHARNESS_DISABLE': 'true', + 'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'], + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + 'SCCACHE_DISABLE': "1", + 'WORK': ".", # Override the defaults in build scripts + 'SRCDIR': "./src", # with values suiteable for windows generic worker + 'UPLOAD_DIR': "./public/build" + }) + + script = "build-sm.sh" + if run['using'] == 'spidermonkey-package': + script = "build-sm-package.sh" + # Don't allow untested configurations yet + raise Exception("spidermonkey-package is not a supported configuration") + elif run['using'] == 'spidermonkey-mozjs-crate': + script = "build-sm-mozjs-crate.sh" + # Don't allow untested configurations yet + raise Exception("spidermonkey-mozhs-crate is not a supported configuration") + + hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"'] + hg_command.append('robustcheckout') + hg_command.extend(['--sharebase', 'y:\\hg-shared']) + hg_command.append('--purge') + hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified']) + hg_command.extend(['--revision', env['GECKO_HEAD_REV']]) + hg_command.append(env['GECKO_HEAD_REPOSITORY']) + hg_command.append('.\\src') + + command = ['c:\\mozilla-build\\msys\\bin\\bash.exe ' # string concat + '"./src/taskcluster/scripts/builder/%s"' % script] + + worker['command'] = [] + worker['command'].extend([ + ' '.join(hg_command), + ' '.join(command) + ]) diff --git a/taskcluster/taskgraph/transforms/spidermonkey.py b/taskcluster/taskgraph/transforms/spidermonkey.py new file mode 100644 index 000000000000..e35b3f25e652 --- /dev/null +++ b/taskcluster/taskgraph/transforms/spidermonkey.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by +import copy + +transforms = TransformSequence() + + +@transforms.add +def handle_keyed_by(config, jobs): + """Resolve fields that can be keyed by platform, etc.""" + fields = [ + "toolchains" + ] + for job in jobs: + job = copy.deepcopy(job) # don't overwrite dict values here + for field in fields: + resolve_keyed_by(item=job, field=field, item_name=job['name']) + + yield job