зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1286075: add a build kind, modify tests to use it; r=jlund
MozReview-Commit-ID: DkpkkSRxVB9 --HG-- extra : rebase_source : e9d8764b64dff35e374bc669b1769cb6c2ce0eeb
This commit is contained in:
Dustin J. Mitchell
Родитель
9d6518161a
Коммит
ea5260dea0
|
|
@ -1,7 +1,7 @@
|
|||
implementation: taskgraph.task.test:TestTask
|
||||
|
||||
kind-dependencies:
|
||||
- legacy
|
||||
- build
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.tests.test_description:validate
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
android-partner-sample1/opt:
|
||||
description: "Android armv7 API 15+ partner sample 1"
|
||||
index:
|
||||
product: mobile
|
||||
job-name: android-api-15-partner-sample1-opt
|
||||
treeherder:
|
||||
platform: android-4-0-armv7-api15-partner1/opt
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
run-on-projects:
|
||||
- try
|
||||
worker-type: aws-provisioner-v1/android-api-15
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
env:
|
||||
EXTRA_CHECKOUT_REPOSITORIES: "PARTNER\n"
|
||||
PARTNER_BASE_REPOSITORY: "https://github.com/mozilla/fennec-distribution-sample"
|
||||
PARTNER_DEST_DIR: "/home/worker/workspace/build/partner"
|
||||
PARTNER_HEAD_REPOSITORY: "https://github.com/mozilla/fennec-distribution-sample"
|
||||
PARTNER_HEAD_REV: "master"
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build multi-l10n update]
|
||||
config:
|
||||
- builds/releng_base_android_64_builds.py
|
||||
- disable_signing.py
|
||||
- platform_supports_post_upload_to_latest.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
custom-build-variant-cfg: api-15-partner-sample1
|
||||
tooltool-downloads: internal
|
||||
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
android-api-15/debug:
|
||||
description: "Android 4.0 API15+ Debug"
|
||||
index:
|
||||
product: mobile
|
||||
job-name:
|
||||
buildbot: android-api-15-debug
|
||||
gecko-v1: android-api-15.debug
|
||||
gecko-v2: android-api-15-debug
|
||||
treeherder:
|
||||
platform: android-4-0-armv7-api15/debug
|
||||
symbol: tc(B)
|
||||
worker-type: aws-provisioner-v1/android-api-15
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build multi-l10n update]
|
||||
config:
|
||||
- builds/releng_base_android_64_builds.py
|
||||
- disable_signing.py
|
||||
- platform_supports_post_upload_to_latest.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
custom-build-variant-cfg: api-15-debug
|
||||
tooltool-downloads: internal
|
||||
|
||||
android-x86/opt:
|
||||
description: "Android 4.2 x86 Opt"
|
||||
index:
|
||||
product: mobile
|
||||
job-name: android-x86-opt
|
||||
treeherder:
|
||||
platform: android-4-2-x86/opt
|
||||
symbol: tc(B)
|
||||
tier: 1
|
||||
worker-type: aws-provisioner-v1/android-api-15
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build multi-l10n update]
|
||||
config:
|
||||
- builds/releng_base_android_64_builds.py
|
||||
- disable_signing.py
|
||||
- platform_supports_post_upload_to_latest.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
custom-build-variant-cfg: x86
|
||||
tooltool-downloads: internal
|
||||
|
||||
android-api-15/opt:
|
||||
description: "Android 4.0 API15+ Opt"
|
||||
index:
|
||||
product: mobile
|
||||
job-name: android-api-15-opt
|
||||
treeherder:
|
||||
platform: android-4-0-armv7-api15/opt
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/android-api-15
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build multi-l10n update]
|
||||
config:
|
||||
- builds/releng_base_android_64_builds.py
|
||||
- disable_signing.py
|
||||
- platform_supports_post_upload_to_latest.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
custom-build-variant-cfg: api-15
|
||||
tooltool-downloads: internal
|
||||
|
||||
android-api-15-gradle/opt:
|
||||
description: "Android 4.0 API15+ (Gradle) Opt"
|
||||
index:
|
||||
product: mobile
|
||||
job-name: android-api-15-gradle-opt
|
||||
treeherder:
|
||||
platform: android-4-0-armv7-api15/opt
|
||||
symbol: tc(Bg)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/android-api-15
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 7200
|
||||
env:
|
||||
# Bug 1292762 - Set GRADLE_USER_HOME to avoid sdk-manager-plugin intermittent
|
||||
GRADLE_USER_HOME: /home/worker/workspace/build/src/dotgradle
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build multi-l10n update]
|
||||
config:
|
||||
- builds/releng_base_android_64_builds.py
|
||||
- disable_signing.py
|
||||
- platform_supports_post_upload_to_latest.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
custom-build-variant-cfg: api-15-gradle
|
||||
tooltool-downloads: internal
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
implementation: taskgraph.task.transform:TransformTask
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.build:transforms
|
||||
- taskgraph.transforms.build_attrs:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
jobs-from:
|
||||
- android-partner.yml
|
||||
- android.yml
|
||||
- linux.yml
|
||||
- macosx.yml
|
||||
- mulet.yml
|
||||
- windows.yml
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
linux64/opt:
|
||||
description: "Linux64 Opt"
|
||||
index:
|
||||
product: firefox
|
||||
job-name: linux64-opt
|
||||
treeherder:
|
||||
platform: linux64/opt
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build check-test generate-build-stats update]
|
||||
config:
|
||||
- builds/releng_base_linux_64_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
tooltool-downloads: public
|
||||
need-xvfb: true
|
||||
|
||||
linux64/pgo:
|
||||
description: "Linux64 Opt PGO"
|
||||
index:
|
||||
product: firefox
|
||||
job-name: linux64-pgo-opt
|
||||
treeherder:
|
||||
platform: linux64/pgo
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
coalesce-name: linux64-pgo
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build check-test generate-build-stats update]
|
||||
options: [enable-pgo]
|
||||
config:
|
||||
- builds/releng_base_linux_64_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
tooltool-downloads: public
|
||||
need-xvfb: true
|
||||
|
||||
linux64/debug:
|
||||
description: "Linux64 Debug"
|
||||
index:
|
||||
product: firefox
|
||||
job-name:
|
||||
buildbot: linux64-debug
|
||||
gecko-v1: linux64.debug
|
||||
gecko-v2: linux64-debug
|
||||
treeherder:
|
||||
platform: linux64/debug
|
||||
symbol: tc(B)
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build check-test generate-build-stats update]
|
||||
config:
|
||||
- builds/releng_base_linux_64_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
custom-build-variant-cfg: debug
|
||||
tooltool-downloads: public
|
||||
need-xvfb: true
|
||||
|
||||
linux/opt:
|
||||
description: "Linux32 Opt"
|
||||
index:
|
||||
product: firefox
|
||||
job-name: linux-opt
|
||||
treeherder:
|
||||
platform: linux32/opt
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
coalesce-name: opt_linux32
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build check-test generate-build-stats update]
|
||||
config:
|
||||
- builds/releng_base_linux_32_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
tooltool-downloads: public
|
||||
need-xvfb: true
|
||||
|
||||
linux/debug:
|
||||
description: "Linux32 Debug"
|
||||
index:
|
||||
product: firefox
|
||||
job-name: linux-debug
|
||||
treeherder:
|
||||
platform: linux32/debug
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
coalesce-name: dbg_linux32
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build check-test generate-build-stats update]
|
||||
config:
|
||||
- builds/releng_base_linux_32_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
custom-build-variant-cfg: debug
|
||||
tooltool-downloads: public
|
||||
need-xvfb: true
|
||||
|
||||
linux64-asan/opt:
|
||||
description: "Linux64 Opt ASAN"
|
||||
index:
|
||||
product: firefox
|
||||
job-name: linux64-asan-opt
|
||||
treeherder:
|
||||
platform: linux64/asan
|
||||
symbol: tc(Bo)
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build check-test generate-build-stats update]
|
||||
config:
|
||||
- builds/releng_base_linux_64_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
custom-build-variant-cfg: asan-tc
|
||||
tooltool-downloads: public
|
||||
need-xvfb: true
|
||||
|
||||
linux64-asan/debug:
|
||||
description: "Linux64 Debug ASAN"
|
||||
index:
|
||||
product: firefox
|
||||
job-name: linux64-asan-debug
|
||||
treeherder:
|
||||
platform: linux64/asan
|
||||
symbol: tc(Bd)
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-linux
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build check-test generate-build-stats update]
|
||||
config:
|
||||
- builds/releng_base_linux_64_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
custom-build-variant-cfg: asan-tc-and-debug
|
||||
tooltool-downloads: public
|
||||
need-xvfb: true
|
||||
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
macosx64/debug:
|
||||
description: "MacOS X x64 Cross-compile"
|
||||
index:
|
||||
product: firefox
|
||||
job-name: macosx64-debug
|
||||
treeherder:
|
||||
platform: osx-10-7/debug
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/dbg-macosx64
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build generate-build-stats update]
|
||||
config:
|
||||
- builds/releng_base_mac_64_cross_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
custom-build-variant-cfg: cross-debug
|
||||
tooltool-downloads: internal
|
||||
|
||||
macosx64/opt:
|
||||
description: "MacOS X x64 Cross-compile"
|
||||
index:
|
||||
product: firefox
|
||||
job-name: macosx64-opt
|
||||
treeherder:
|
||||
platform: osx-10-7/opt
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/opt-macosx64
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 36000
|
||||
run:
|
||||
using: mozharness
|
||||
actions: [get-secrets build generate-build-stats update]
|
||||
config:
|
||||
- builds/releng_base_mac_64_cross_builds.py
|
||||
- balrog/production.py
|
||||
script: "mozharness/scripts/fx_desktop_build.py"
|
||||
secrets: true
|
||||
tooltool-downloads: internal
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
linux64-mulet/debug:
|
||||
description: "Linux64 Mulet Debug"
|
||||
index:
|
||||
product: b2g
|
||||
job-name:
|
||||
buildbot: linux64-mulet
|
||||
gecko-v1: mulet.dbg
|
||||
gecko-v2: mulet-dbg
|
||||
treeherder:
|
||||
platform: mulet-linux64/debug # See bug 1286086
|
||||
symbol: B
|
||||
tier: 3
|
||||
worker-type: aws-provisioner-v1/mulet-debug
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 3600
|
||||
run:
|
||||
using: mach-via-build-mulet-linux.sh
|
||||
mozconfig: b2g/dev/config/mozconfigs/linux64/mulet_dbg
|
||||
tooltool-manifest: b2g/dev/config/tooltool-manifests/linux64/releng.manifest
|
||||
|
||||
linux64-mulet/opt:
|
||||
description: "Linux64 Mulet Opt"
|
||||
index:
|
||||
product: b2g
|
||||
job-name:
|
||||
buildbot: linux64-mulet
|
||||
gecko-v1: mulet.opt
|
||||
gecko-v2: mulet-opt
|
||||
treeherder:
|
||||
platform: mulet-linux64/opt # ?!?%
|
||||
symbol: B
|
||||
tier: 3
|
||||
worker-type: aws-provisioner-v1/mulet-opt
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
max-run-time: 3600
|
||||
run:
|
||||
using: mach-via-build-mulet-linux.sh
|
||||
mozconfig: b2g/dev/config/mozconfigs/linux64/mulet
|
||||
tooltool-manifest: b2g/dev/config/tooltool-manifests/linux64/releng.manifest
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
win32/debug:
|
||||
description: "Win32 Debug"
|
||||
index:
|
||||
product: firefox
|
||||
job-name:
|
||||
gecko-v2: win32-debug
|
||||
treeherder:
|
||||
platform: windowsxp/debug
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
|
||||
worker:
|
||||
implementation: generic-worker
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: mozharness
|
||||
script: mozharness/scripts/fx_desktop_build.py
|
||||
config:
|
||||
- builds/taskcluster_firefox_win32_debug.py
|
||||
|
||||
win32/opt:
|
||||
description: "Win32 Opt"
|
||||
index:
|
||||
product: firefox
|
||||
job-name:
|
||||
gecko-v2: win32-opt
|
||||
treeherder:
|
||||
platform: windowsxp/opt
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
|
||||
worker:
|
||||
implementation: generic-worker
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: mozharness
|
||||
script: mozharness/scripts/fx_desktop_build.py
|
||||
config:
|
||||
- builds/taskcluster_firefox_win32_opt.py
|
||||
|
||||
win64/debug:
|
||||
description: "Win64 Debug"
|
||||
index:
|
||||
product: firefox
|
||||
job-name:
|
||||
gecko-v2: win64-debug
|
||||
treeherder:
|
||||
platform: windows8-64/debug
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
|
||||
worker:
|
||||
implementation: generic-worker
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: mozharness
|
||||
script: mozharness/scripts/fx_desktop_build.py
|
||||
config:
|
||||
- builds/taskcluster_firefox_win64_debug.py
|
||||
|
||||
win64/opt:
|
||||
description: "Win64 Opt"
|
||||
index:
|
||||
product: firefox
|
||||
job-name:
|
||||
gecko-v2: win64-opt
|
||||
treeherder:
|
||||
platform: windows8-64/opt
|
||||
symbol: tc(B)
|
||||
tier: 2
|
||||
worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
|
||||
worker:
|
||||
implementation: generic-worker
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: mozharness
|
||||
script: mozharness/scripts/fx_desktop_build.py
|
||||
config:
|
||||
- builds/taskcluster_firefox_win64_opt.py
|
||||
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
implementation: taskgraph.task.test:TestTask
|
||||
|
||||
kind-dependencies:
|
||||
- legacy
|
||||
- build
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.tests.test_description:validate
|
||||
|
|
|
|||
|
|
@ -3,10 +3,13 @@ Task Kinds
|
|||
|
||||
This section lists and documents the available task kinds.
|
||||
|
||||
Builds
|
||||
build
|
||||
------
|
||||
|
||||
Builds are currently implemented by the ``legacy`` kind.
|
||||
Builds are tasks that produce an installer or other output that can be run by
|
||||
users or automated tests. This is more restrictive than most definitions of
|
||||
"build" in a Mozilla context: it does not include tasks that run build-like
|
||||
actions for static analysis or to produce instrumented artifacts.
|
||||
|
||||
Tests
|
||||
-----
|
||||
|
|
|
|||
|
|
@ -62,9 +62,6 @@ class TestTask(transform.TransformTask):
|
|||
for task in loaded_tasks:
|
||||
if task.kind != dep_kind:
|
||||
continue
|
||||
# remove this check when builds are no longer legacy
|
||||
if task.attributes['legacy_kind'] != 'build':
|
||||
continue
|
||||
|
||||
build_platform = task.attributes.get('build_platform')
|
||||
build_type = task.attributes.get('build_type')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def set_defaults(config, jobs):
|
||||
"""Set defaults, including those that differ per worker implementation"""
|
||||
for job in jobs:
|
||||
job['treeherder'].setdefault('kind', 'build')
|
||||
job['treeherder'].setdefault('tier', 1)
|
||||
if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
|
||||
job['worker'].setdefault('docker-image', {'in-tree': 'desktop-build'})
|
||||
yield job
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def set_build_attributes(config, jobs):
|
||||
"""
|
||||
Set the build_platform and build_type attributes based on the job name.
|
||||
Although not all jobs using this transform are actual "builds", the try
|
||||
option syntax treats them as such, and this arranges the attributes
|
||||
appropriately for that purpose.
|
||||
"""
|
||||
for job in jobs:
|
||||
build_platform, build_type = job['name'].split('/')
|
||||
|
||||
# pgo builds are represented as a different platform, type opt
|
||||
if build_type == 'pgo':
|
||||
build_platform = build_platform + '-pgo'
|
||||
build_type = 'opt'
|
||||
|
||||
attributes = job.setdefault('attributes', {})
|
||||
attributes.update({
|
||||
'build_platform': build_platform,
|
||||
'build_type': build_type,
|
||||
})
|
||||
|
||||
yield job
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Common support for various job types. These functions are all named after the
|
||||
worker implementation they operate on, and take the same three parameters, for
|
||||
consistency.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
|
||||
|
||||
|
||||
def docker_worker_add_workspace_cache(config, job, taskdesc):
|
||||
"""Add the workspace cache based on the build platform/type and level,
|
||||
except on try where workspace caches are not used."""
|
||||
if config.params['project'] == 'try':
|
||||
return
|
||||
|
||||
taskdesc['worker'].setdefault('caches', []).append({
|
||||
'type': 'persistent',
|
||||
'name': 'level-{}-{}-build-{}-{}-workspace'.format(
|
||||
config.params['level'], config.params['project'],
|
||||
taskdesc['attributes']['build_platform'],
|
||||
taskdesc['attributes']['build_type'],
|
||||
),
|
||||
'mount-point': "/home/worker/workspace",
|
||||
})
|
||||
|
||||
|
||||
def docker_worker_add_tc_vcs_cache(config, job, taskdesc):
|
||||
taskdesc['worker'].setdefault('caches', []).append({
|
||||
'type': 'persistent',
|
||||
'name': 'level-{}-{}-tc-vcs'.format(
|
||||
config.params['level'], config.params['project']),
|
||||
'mount-point': "/home/worker/.tc-vcs",
|
||||
})
|
||||
|
||||
|
||||
def docker_worker_add_public_artifacts(config, job, taskdesc):
|
||||
taskdesc['worker'].setdefault('artifacts', []).append({
|
||||
'name': 'public/build',
|
||||
'path': '/home/worker/artifacts/',
|
||||
'type': 'directory',
|
||||
})
|
||||
|
||||
|
||||
def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc):
|
||||
"""Add the GECKO_BASE_* and GECKO_HEAD_* env vars to the worker."""
|
||||
env = taskdesc['worker'].setdefault('env', {})
|
||||
env.update({
|
||||
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
|
||||
'GECKO_HEAD_REF': config.params['head_rev'],
|
||||
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
|
||||
'GECKO_HEAD_REV': config.params['head_rev'],
|
||||
})
|
||||
|
||||
|
||||
def docker_worker_setup_secrets(config, job, taskdesc):
|
||||
"""Set up access to secrets via taskcluster-proxy. The value of
|
||||
run['secrets'] should be a boolean or a list of secret names that
|
||||
can be accessed."""
|
||||
if not job['run'].get('secrets'):
|
||||
return
|
||||
|
||||
taskdesc['worker']['taskcluster-proxy'] = True
|
||||
secrets = job['run']['secrets']
|
||||
if secrets is True:
|
||||
secrets = ['*']
|
||||
for sec in secrets:
|
||||
taskdesc['scopes'].append(SECRET_SCOPE.format(
|
||||
job['treeherder']['kind'], config.params['level'], sec))
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
|
||||
Support for running jobs via mozharness. Ideally, most stuff gets run this
|
||||
way, and certainly anything using mozharness should use this approach.
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import time
|
||||
from voluptuous import Schema, Required, Optional, Any
|
||||
|
||||
from taskgraph.transforms.job import run_job_using
|
||||
from taskgraph.transforms.job.common import (
|
||||
docker_worker_add_workspace_cache,
|
||||
docker_worker_add_tc_vcs_cache,
|
||||
docker_worker_add_gecko_vcs_env_vars,
|
||||
docker_worker_setup_secrets,
|
||||
docker_worker_add_public_artifacts
|
||||
)
|
||||
|
||||
COALESCE_KEY = 'builds.{project}.{name}'
|
||||
|
||||
mozharness_run_schema = Schema({
|
||||
Required('using'): 'mozharness',
|
||||
|
||||
# the mozharness script used to run this task, relative to the testing/
|
||||
# directory and using forward slashes even on Windows
|
||||
Required('script'): basestring,
|
||||
|
||||
# the config files required for the task, relative to
|
||||
# testing/mozharness/configs and using forward slashes even on Windows
|
||||
Required('config'): [basestring],
|
||||
|
||||
# any additional actions to pass to the mozharness command; not supported
|
||||
# on Windows
|
||||
Optional('actions'): [basestring],
|
||||
|
||||
# any additional options (without leading --) to be passed to mozharness;
|
||||
# not supported on Windows
|
||||
Optional('options'): [basestring],
|
||||
|
||||
# --custom-build-variant-cfg value (not supported on Windows)
|
||||
Optional('custom-build-variant-cfg'): basestring,
|
||||
|
||||
# If not false, tooltool downloads will be enabled via relengAPIProxy
|
||||
# for either just public files, or all files. Not supported on Windows
|
||||
Required('tooltool-downloads', default=False): Any(
|
||||
False,
|
||||
'public',
|
||||
'internal',
|
||||
),
|
||||
|
||||
# The set of secret names to which the task has access; these are prefixed
|
||||
# with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
|
||||
# this will enable any worker features required and set the task's scopes
|
||||
# appropriately. `true` here means ['*'], all secrets. Not supported on
|
||||
# Windows
|
||||
Required('secrets', default=False): Any(bool, [basestring]),
|
||||
|
||||
# If true, taskcluster proxy will be enabled; note that it may also be enabled
|
||||
# automatically e.g., for secrets support. Not supported on Windows.
|
||||
Required('taskcluster-proxy', default=False): bool,
|
||||
|
||||
# If true, the build scripts will start Xvfb. Not supported on Windows.
|
||||
Required('need-xvfb', default=False): bool,
|
||||
|
||||
# If false, indicate that builds should skip producing artifacts. Not
|
||||
# supported on Windows.
|
||||
Required('keep-artifacts', default=True): bool,
|
||||
})
|
||||
|
||||
|
||||
@run_job_using("docker-worker", "mozharness", schema=mozharness_run_schema)
|
||||
def mozharness_on_docker_worker_setup(config, job, taskdesc):
|
||||
run = job['run']
|
||||
|
||||
worker = taskdesc['worker']
|
||||
worker['implementation'] = job['worker']['implementation']
|
||||
|
||||
# running via mozharness assumes desktop-build (which contains build.sh)
|
||||
taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
|
||||
|
||||
worker['relengapi-proxy'] = False # but maybe enabled for tooltool below
|
||||
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
|
||||
|
||||
docker_worker_add_public_artifacts(config, job, taskdesc)
|
||||
docker_worker_add_tc_vcs_cache(config, job, taskdesc)
|
||||
docker_worker_add_workspace_cache(config, job, taskdesc)
|
||||
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
|
||||
|
||||
env = worker.setdefault('env', {})
|
||||
env.update({
|
||||
'MOZHARNESS_CONFIG': ' '.join(run['config']),
|
||||
'MOZHARNESS_SCRIPT': run['script'],
|
||||
'MH_BRANCH': config.params['project'],
|
||||
'MH_BUILD_POOL': 'taskcluster',
|
||||
'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
|
||||
'MOZ_SCM_LEVEL': config.params['level'],
|
||||
})
|
||||
|
||||
if 'actions' in run:
|
||||
env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
|
||||
|
||||
if 'options' in run:
|
||||
env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
|
||||
|
||||
if 'custom-build-variant-cfg' in run:
|
||||
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
|
||||
|
||||
# if we're not keeping artifacts, set some env variables to empty values
|
||||
# that will cause the build process to skip copying the results to the
|
||||
# artifacts directory. This will have no effect for operations that are
|
||||
# not builds.
|
||||
if not run['keep-artifacts']:
|
||||
env['DIST_TARGET_UPLOADS'] = ''
|
||||
env['DIST_UPLOADS'] = ''
|
||||
|
||||
# Xvfb
|
||||
if run['need-xvfb']:
|
||||
env['NEED_XVFB'] = 'true'
|
||||
|
||||
# tooltool downloads
|
||||
if run['tooltool-downloads']:
|
||||
worker['relengapi-proxy'] = True
|
||||
worker['caches'].append({
|
||||
'type': 'persistent',
|
||||
'name': 'tooltool-cache',
|
||||
'mount-point': '/home/worker/tooltool-cache',
|
||||
})
|
||||
taskdesc['scopes'].extend([
|
||||
'docker-worker:relengapi-proxy:tooltool.download.public',
|
||||
])
|
||||
if run['tooltool-downloads'] == 'internal':
|
||||
taskdesc['scopes'].append(
|
||||
'docker-worker:relengapi-proxy:tooltool.download.internal')
|
||||
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
|
||||
env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
|
||||
env['TOOLTOOL_REV'] = 'master'
|
||||
|
||||
docker_worker_setup_secrets(config, job, taskdesc)
|
||||
|
||||
worker['command'] = ["/bin/bash", "bin/build.sh"]
|
||||
|
||||
|
||||
# We use the generic worker to run tasks on Windows
|
||||
@run_job_using("generic-worker", "mozharness", schema=mozharness_run_schema)
|
||||
def mozharness_on_windows(config, job, taskdesc):
|
||||
run = job['run']
|
||||
|
||||
# fail if invalid run options are included
|
||||
invalid = []
|
||||
for prop in ['actions', 'options', 'custom-build-variant-cfg',
|
||||
'tooltool-downloads', 'secrets', 'taskcluster-proxy',
|
||||
'need-xvfb']:
|
||||
if prop in run and run[prop]:
|
||||
invalid.append(prop)
|
||||
if not run.get('keep-artifacts', True):
|
||||
invalid.append('keep-artifacts')
|
||||
if invalid:
|
||||
raise Exception("Jobs run using mozharness on Windows do not support properties " +
|
||||
', '.join(invalid))
|
||||
|
||||
worker = taskdesc['worker']
|
||||
|
||||
worker['artifacts'] = [{
|
||||
'path': r'public\build',
|
||||
'type': 'directory',
|
||||
}]
|
||||
|
||||
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
|
||||
|
||||
env = worker['env']
|
||||
env.update({
|
||||
'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
|
||||
'MOZ_SCM_LEVEL': config.params['level'],
|
||||
'TOOLTOOL_REPO': 'https://github.com/mozilla/build-tooltool',
|
||||
'TOOLTOOL_REV': 'master',
|
||||
})
|
||||
|
||||
mh_command = [r'c:\mozilla-build\python\python.exe']
|
||||
mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')]))
|
||||
for cfg in run['config']:
|
||||
mh_command.append('--config ' + cfg.replace('/', '\\'))
|
||||
mh_command.append('--branch ' + config.params['project'])
|
||||
mh_command.append(r'--skip-buildbot-actions --work-dir %cd:Z:=z:%\build')
|
||||
worker['command'] = [
|
||||
r'mkdir .\build\src',
|
||||
r'hg share c:\builds\hg-shared\mozilla-central .\build\src',
|
||||
r'hg pull -u -R .\build\src --rev %GECKO_HEAD_REV% %GECKO_HEAD_REPOSITORY%',
|
||||
' '.join(mh_command),
|
||||
]
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Support for running mulet tasks via build-mulet-linux.sh
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import time
|
||||
from voluptuous import Schema, Required
|
||||
|
||||
from taskgraph.transforms.job import run_job_using
|
||||
from taskgraph.transforms.job.common import (
|
||||
docker_worker_add_workspace_cache,
|
||||
docker_worker_add_tc_vcs_cache,
|
||||
docker_worker_add_gecko_vcs_env_vars,
|
||||
docker_worker_add_public_artifacts
|
||||
)
|
||||
|
||||
COALESCE_KEY = 'builds.{project}.{name}'
|
||||
|
||||
build_mulet_linux_schema = Schema({
|
||||
Required('using'): 'mach-via-build-mulet-linux.sh',
|
||||
|
||||
# The pathname of the mozconfig to use
|
||||
Required('mozconfig'): basestring,
|
||||
|
||||
# The tooltool manifest to use
|
||||
Required('tooltool-manifest'): basestring,
|
||||
})
|
||||
|
||||
|
||||
@run_job_using("docker-worker", "mach-via-build-mulet-linux.sh", schema=build_mulet_linux_schema)
|
||||
def docker_worker_make_via_build_mulet_linux_sh(config, job, taskdesc):
|
||||
run = job['run']
|
||||
worker = taskdesc.get('worker')
|
||||
|
||||
# assumes the builder image (which contains the gecko checkout command)
|
||||
taskdesc['worker']['docker-image'] = {"in-tree": "builder"}
|
||||
|
||||
worker['taskcluster-proxy'] = False
|
||||
|
||||
docker_worker_add_public_artifacts(config, job, taskdesc)
|
||||
docker_worker_add_tc_vcs_cache(config, job, taskdesc)
|
||||
docker_worker_add_workspace_cache(config, job, taskdesc)
|
||||
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
|
||||
|
||||
env = worker.setdefault('env', {})
|
||||
env.update({
|
||||
'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
|
||||
'MOZ_SCM_LEVEL': config.params['level'],
|
||||
})
|
||||
|
||||
env['MOZCONFIG'] = run['mozconfig']
|
||||
env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
|
||||
|
||||
# tooltool downloads
|
||||
worker['relengapi-proxy'] = True
|
||||
worker['caches'].append({
|
||||
'type': 'persistent',
|
||||
'name': 'tooltool-cache',
|
||||
# N.B. different from build.sh
|
||||
# TODO(taskdiff): grepping suggests this isn't used..
|
||||
'mount-point': '/home/worker/tools/tooltool-cache',
|
||||
})
|
||||
taskdesc['scopes'].extend([
|
||||
'docker-worker:relengapi-proxy:tooltool.download.public',
|
||||
])
|
||||
env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
|
||||
env['TOOLTOOL_REV'] = 'master'
|
||||
|
||||
worker['command'] = [
|
||||
"/bin/bash",
|
||||
"-c",
|
||||
"checkout-gecko workspace"
|
||||
" && cd ./workspace/gecko/taskcluster/scripts/builder"
|
||||
" && buildbot_step 'Build' ./build-mulet-linux.sh $HOME/workspace",
|
||||
]
|
||||
|
||||
mulet_simulator_schema = Schema({
|
||||
Required('using'): 'mulet-simulator',
|
||||
|
||||
# The shell command to run with `bash -exc`. This will have parameters
|
||||
# substituted for {..} and will be enclosed in a {task-reference: ..} block
|
||||
# so it can refer to the parent task as <build>
|
||||
Required('shell-command'): basestring,
|
||||
})
|
||||
|
||||
|
||||
@run_job_using("docker-worker", "mulet-simulator", schema=mulet_simulator_schema)
|
||||
def docker_worker_mulet_simulator(config, job, taskdesc):
|
||||
run = job['run']
|
||||
worker = taskdesc.get('worker')
|
||||
|
||||
# assumes the builder image (which contains the gecko checkout command)
|
||||
taskdesc['worker']['docker-image'] = {"in-tree": "builder"}
|
||||
|
||||
worker['taskcluster-proxy'] = False
|
||||
|
||||
docker_worker_add_public_artifacts(config, job, taskdesc)
|
||||
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
|
||||
|
||||
taskdesc.setdefault('routes', []).extend([
|
||||
'index.gecko.v1.{project}.latest.simulator.opt'.format(**config.params),
|
||||
])
|
||||
|
||||
# TODO(taskdiff): has the scope for this cache, but not the cache
|
||||
taskdesc.setdefault('scopes', []).extend([
|
||||
'docker-worker:cache:level-{level}-{project}-tc-vcs'.format(**config.params),
|
||||
])
|
||||
|
||||
shell_command = run['shell-command'].format(**config.params)
|
||||
|
||||
worker['command'] = [
|
||||
"/bin/bash",
|
||||
"-exc",
|
||||
{'task-reference': shell_command},
|
||||
]
|
||||
|
|
@ -23,6 +23,7 @@ BUILD_TYPE_ALIASES = {
|
|||
|
||||
# consider anything in this whitelist of kinds to be governed by -b/-p
|
||||
BUILD_KINDS = set([
|
||||
'build',
|
||||
])
|
||||
|
||||
# anything in this list is governed by -j
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче