gecko-dev/taskcluster/ci/test/mochitest.yml

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

530 строки
16 KiB
YAML
Исходник Обычный вид История

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
job-defaults:
Bug 1523303 - [taskgraph] Define suite "categories" rather than flavours task configs, r=gbrown Currently we have the concept of a "suite" and a "flavour" in our task configuration. Typically, the "suite" refers to the high-level test harness like "mochitest" or "reftest", whereas the flavour is more specific, e.g "browser-chrome-instrumentation" or "crashtest". However the line between suite and flavour is not applied with any semblance of consistency which results in inconsistent naming throughout the tree. This patch gets rid of the concept of "flavours" entirely (at least when it comes to task configuration). A suite is a type of test run, for example: - mochitest-plain - mochitest-devtools-chrome - mochitest-browser-chrome-instrumentation - jsreftest - reftest - firefox-ui-functional-remote etc There is no confusion here between suites and flavours because flavours don't exist. However, there are a couple of places where we *do* need to know what "test harness" is used to run a suite. These cases are: 1. For SCHEDULES moz.build rules 2. For the desktop_unittest.py mozharness script which takes arguments like --mochitest-suite=browser (this is not a compelling use of this information and should be refactored to work more like the android_emulator_unittest.py script) So to get this information, this patch introduces a new concept of a "category" which is the overall "test harness" that runs the suite. For many suites, the "category" is identical to the suite name. Unlike flavours, "categories" have no bearing on how we call or refer to the suite. Differential Revision: https://phabricator.services.mozilla.com/D27554 --HG-- extra : moz-landing-system : lando
2019-04-22 23:44:01 +03:00
suite:
category: mochitest
target:
by-test-platform:
android-em-7.*: geckoview-androidTest.apk
android-hw.*: geckoview-androidTest.apk
default: null
variants:
by-test-platform:
linux.*64/debug: ['fission', 'socketprocess_networking']
default: ['fission']
run-on-projects: built-projects
fission-run-on-projects:
by-test-platform:
linux.*64-qr/debug: ['trunk']
linux.*64-shippable-qr/opt: ['mozilla-central']
linux.*64-shippable/.*: ['mozilla-central']
linux.*64/debug: ['mozilla-central']
windows10-64-shippable(-qr)?/opt: ['mozilla-central']
default: []
fission-tier:
by-test-platform:
linux.*64.*-qr/debug: 1
default: 2
mozharness:
script:
by-test-platform:
android-em.*: android_emulator_unittest.py
android-hw.*: android_hardware_unittest.py
default: desktop_unittest.py
config:
by-test-platform:
android-em-7.*:
- android/android_common.py
- android/androidx86_7_0.py
android-hw.*:
- android/android_common.py
- android/android_hw.py
linux.*:
- unittests/linux_unittest.py
- remove_executables.py
macosx.*:
- unittests/mac_unittest.py
windows.*:
- unittests/win_unittest.py
mochitest-plain:
description: "Mochitest plain run"
treeherder-symbol: M()
schedules-component: mochitest-plain
loopback-video: true
tier: default
virtualization:
by-test-platform:
windows10-64.*: hardware
default: virtual
variants:
by-test-platform:
linux1804-64/opt: ['fission-xorigin']
linux1804-64/debug: ['fission-xorigin']
default: ['fission']
chunks:
by-test-platform:
android-em-7.*: 4
linux.*/debug: 16
linux.*64-asan/opt: 10
linux.*64-tsan/opt: 20
linux.*64-.*cov/opt: 10
windows10-64-ccov/.*: 10
macosx.*64-ccov/.*: 10
default: 5
instance-size:
by-test-platform:
linux.*64-tsan/opt: xlarge # runs out of memory on default/m3.large
default: default
e10s: true
max-run-time: 5400
allow-software-gl-layers: false
mozharness:
mochitest-flavor: plain
extra-options:
by-test-platform:
android-em.*:
- --test-suite=mochitest-plain
default: []
chunked:
by-test-platform:
android-em.*: false
default: true
mochitest-a11y:
description: "Mochitest a11y run"
treeherder-symbol: M(a11y)
schedules-component: mochitest-a11y
loopback-video: true
tier: default
e10s: false
run-on-projects: built-projects
mozharness:
mochitest-flavor: a11y
mochitest-browser-chrome:
description: "Mochitest browser-chrome run"
Bug 1523303 - [taskgraph] Define suite "categories" rather than flavours task configs, r=gbrown Currently we have the concept of a "suite" and a "flavour" in our task configuration. Typically, the "suite" refers to the high-level test harness like "mochitest" or "reftest", whereas the flavour is more specific, e.g "browser-chrome-instrumentation" or "crashtest". However the line between suite and flavour is not applied with any semblance of consistency which results in inconsistent naming throughout the tree. This patch gets rid of the concept of "flavours" entirely (at least when it comes to task configuration). A suite is a type of test run, for example: - mochitest-plain - mochitest-devtools-chrome - mochitest-browser-chrome-instrumentation - jsreftest - reftest - firefox-ui-functional-remote etc There is no confusion here between suites and flavours because flavours don't exist. However, there are a couple of places where we *do* need to know what "test harness" is used to run a suite. These cases are: 1. For SCHEDULES moz.build rules 2. For the desktop_unittest.py mozharness script which takes arguments like --mochitest-suite=browser (this is not a compelling use of this information and should be refactored to work more like the android_emulator_unittest.py script) So to get this information, this patch introduces a new concept of a "category" which is the overall "test harness" that runs the suite. For many suites, the "category" is identical to the suite name. Unlike flavours, "categories" have no bearing on how we call or refer to the suite. Differential Revision: https://phabricator.services.mozilla.com/D27554 --HG-- extra : moz-landing-system : lando
2019-04-22 23:44:01 +03:00
suite:
name: mochitest-browser-chrome
treeherder-symbol: M(bc)
schedules-component: mochitest-browser-chrome
loopback-video: true
fission-run-on-projects:
by-test-platform:
linux.*64(-qr)?/debug: ['trunk']
linux.*64-shippable(-qr)?/opt: ['mozilla-central']
windows10-64-shippable(-qr)?/opt: ['mozilla-central']
default: []
tier: default
fission-tier:
by-test-platform:
linux.*64/debug: 1
default: 2
chunks:
by-test-platform:
linux.*/debug: 16
linux.*64-asan/opt: 16
linux.*64-tsan/opt: 32
macosx.*64/debug: 16
windows10-64-ccov/.*: 14
windows10.*-asan/opt: 9
default: 7
max-run-time:
by-test-platform:
linux.*64-ccov/.*: 9000
windows7-32/debug: 5400
windows10-64/debug: 5400
macosx.*64/debug: 5400
windows10-64-ccov/.*: 10800
macosx.*64-ccov/.*: 10800
linux.*/debug: 5400
windows10-64-qr/debug: 5400
default: 3600
mozharness:
mochitest-flavor: browser
chunked: true
# Bug 1281241: migrating to m3.large instances
instance-size: default
allow-software-gl-layers: false
browser-screenshots:
description: "Browser Screenshots"
Bug 1523303 - [taskgraph] Define suite "categories" rather than flavours task configs, r=gbrown Currently we have the concept of a "suite" and a "flavour" in our task configuration. Typically, the "suite" refers to the high-level test harness like "mochitest" or "reftest", whereas the flavour is more specific, e.g "browser-chrome-instrumentation" or "crashtest". However the line between suite and flavour is not applied with any semblance of consistency which results in inconsistent naming throughout the tree. This patch gets rid of the concept of "flavours" entirely (at least when it comes to task configuration). A suite is a type of test run, for example: - mochitest-plain - mochitest-devtools-chrome - mochitest-browser-chrome-instrumentation - jsreftest - reftest - firefox-ui-functional-remote etc There is no confusion here between suites and flavours because flavours don't exist. However, there are a couple of places where we *do* need to know what "test harness" is used to run a suite. These cases are: 1. For SCHEDULES moz.build rules 2. For the desktop_unittest.py mozharness script which takes arguments like --mochitest-suite=browser (this is not a compelling use of this information and should be refactored to work more like the android_emulator_unittest.py script) So to get this information, this patch introduces a new concept of a "category" which is the overall "test harness" that runs the suite. For many suites, the "category" is identical to the suite name. Unlike flavours, "categories" have no bearing on how we call or refer to the suite. Differential Revision: https://phabricator.services.mozilla.com/D27554 --HG-- extra : moz-landing-system : lando
2019-04-22 23:44:01 +03:00
suite:
name: mochitest-browser-chrome-screenshots
treeherder-symbol: M(ss)
loopback-video: true
run-on-projects:
by-test-platform:
windows7-32(?:-pgo|-shippable)(?:-qr)?/opt: ['mozilla-central']
windows10-64(?:-pgo|-shippable)(?:-qr)?/opt: ['mozilla-central']
(?:windows10-64|windows7-32|linux1804-64|macosx1014-64)(?:-qr)?/opt: ['integration']
linux1804-64-(?:pgo|shippable)(?:-qr)?/opt: ['mozilla-central']
macosx.*64-shippable/opt: ['mozilla-central']
default: []
fission-run-on-projects: []
fission-tier: 2
max-run-time: 3600
mozharness:
mochitest-flavor: browser
allow-software-gl-layers: false
mochitest-chrome:
description: "Mochitest chrome run"
treeherder-symbol: M(c)
schedules-component: mochitest-chrome
loopback-video: true
tier: default
chunks:
by-test-platform:
.*-ccov/.*: 3
.*-qr/.*: 3
.*-asan/opt: 3
.*-tsan/opt: 6
(linux.*64|windows.*-..|macosx10..)/debug: 3
default: 2
max-run-time: 3600
e10s: false
mozharness:
mochitest-flavor: chrome
chunked: true
mochitest-devtools-chrome:
description: "Mochitest devtools-chrome run"
Bug 1523303 - [taskgraph] Define suite "categories" rather than flavours task configs, r=gbrown Currently we have the concept of a "suite" and a "flavour" in our task configuration. Typically, the "suite" refers to the high-level test harness like "mochitest" or "reftest", whereas the flavour is more specific, e.g "browser-chrome-instrumentation" or "crashtest". However the line between suite and flavour is not applied with any semblance of consistency which results in inconsistent naming throughout the tree. This patch gets rid of the concept of "flavours" entirely (at least when it comes to task configuration). A suite is a type of test run, for example: - mochitest-plain - mochitest-devtools-chrome - mochitest-browser-chrome-instrumentation - jsreftest - reftest - firefox-ui-functional-remote etc There is no confusion here between suites and flavours because flavours don't exist. However, there are a couple of places where we *do* need to know what "test harness" is used to run a suite. These cases are: 1. For SCHEDULES moz.build rules 2. For the desktop_unittest.py mozharness script which takes arguments like --mochitest-suite=browser (this is not a compelling use of this information and should be refactored to work more like the android_emulator_unittest.py script) So to get this information, this patch introduces a new concept of a "category" which is the overall "test harness" that runs the suite. For many suites, the "category" is identical to the suite name. Unlike flavours, "categories" have no bearing on how we call or refer to the suite. Differential Revision: https://phabricator.services.mozilla.com/D27554 --HG-- extra : moz-landing-system : lando
2019-04-22 23:44:01 +03:00
suite:
name: mochitest-devtools-chrome
schedules-component: mochitest-browser-chrome
treeherder-symbol: M(dt)
loopback-video: true
tier: default
max-run-time:
by-test-platform:
windows10-64-ccov/.*: 10800
macosx.*64-ccov/.*: 9000
linux.*64-ccov/.*: 7200
default: 5400
chunks:
by-test-platform:
.*-ccov/.*: 16
linux.*64/debug: 12
macosx.*64/debug: 8
.*-asan/opt: 8
.*-tsan/opt: 16
default: 5
mozharness:
mochitest-flavor: chrome
chunked: true
instance-size:
by-test-platform:
linux.*64-[at]san/opt: xlarge # runs out of memory on default/m3.large
default: default
# Bug 1296086: high number of intermittents observed with software GL and large instances
allow-software-gl-layers: false
mochitest-plain-gpu:
description: "Mochitest plain GPU run"
Bug 1523303 - [taskgraph] Define suite "categories" rather than flavours task configs, r=gbrown Currently we have the concept of a "suite" and a "flavour" in our task configuration. Typically, the "suite" refers to the high-level test harness like "mochitest" or "reftest", whereas the flavour is more specific, e.g "browser-chrome-instrumentation" or "crashtest". However the line between suite and flavour is not applied with any semblance of consistency which results in inconsistent naming throughout the tree. This patch gets rid of the concept of "flavours" entirely (at least when it comes to task configuration). A suite is a type of test run, for example: - mochitest-plain - mochitest-devtools-chrome - mochitest-browser-chrome-instrumentation - jsreftest - reftest - firefox-ui-functional-remote etc There is no confusion here between suites and flavours because flavours don't exist. However, there are a couple of places where we *do* need to know what "test harness" is used to run a suite. These cases are: 1. For SCHEDULES moz.build rules 2. For the desktop_unittest.py mozharness script which takes arguments like --mochitest-suite=browser (this is not a compelling use of this information and should be refactored to work more like the android_emulator_unittest.py script) So to get this information, this patch introduces a new concept of a "category" which is the overall "test harness" that runs the suite. For many suites, the "category" is identical to the suite name. Unlike flavours, "categories" have no bearing on how we call or refer to the suite. Differential Revision: https://phabricator.services.mozilla.com/D27554 --HG-- extra : moz-landing-system : lando
2019-04-22 23:44:01 +03:00
suite:
name: mochitest-plain-gpu
treeherder-symbol: M(gpu)
schedules-component: mochitest-plain
loopback-video: true
fission-run-on-projects: []
fission-tier: 2
tier: default
virtualization:
by-test-platform:
windows10-64.*: hardware
default: virtual-with-gpu
e10s: true
mozharness:
mochitest-flavor: plain
extra-options:
by-test-platform:
android.*:
# note that Android runs fewer suites than other platforms
- --test-suite=mochitest-plain-gpu
default:
- --mochitest-suite=mochitest-plain-gpu
mochitest-chrome-gpu:
description: "Mochitest chrome GPU run"
suite:
name: mochitest-chrome-gpu
treeherder-symbol: M(gpu-c)
loopback-video: true
fission-run-on-projects: []
fission-tier: 2
tier: default
virtualization:
by-test-platform:
windows10-64.*: hardware
default: virtual-with-gpu
e10s: true
run-on-projects:
by-test-platform:
android.*/.*: []
default: built-projects
mozharness:
mochitest-flavor: chrome
extra-options:
by-test-platform:
android.*:
# note that Android runs fewer suites than other platforms
- --test-suite=mochitest-chrome-gpu
default:
- --mochitest-suite=mochitest-chrome-gpu
mochitest-media:
description: "Mochitest media run"
treeherder-symbol: M(mda)
schedules-component: mochitest-plain
max-run-time:
by-test-platform:
windows10-64-ccov/.*: 7200
macosx.*64-ccov/.*: 7200
default: 5400
run-on-projects:
by-test-platform:
android-hw-.*(?<!-shippable)/opt: ['autoland']
android-hw-.*-api-16/(?:debug|pgo)?: ['trunk', 'mozilla-beta', 'mozilla-release']
android-hw-.*-api-16-shippable/opt: ['trunk', 'mozilla-beta', 'mozilla-release']
windows10-aarch64/.*: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
default: built-projects
variants:
by-test-platform:
android.*: ['socketprocess']
macosx.*64-ccov/.*: []
default: ['fission', 'socketprocess', 'webgl-ipc']
loopback-video: true
instance-size: large
chunks:
by-test-platform:
android-em-7.*: 1
windows10-64.*: 1
macosx.*64.*/.*: 2
windows10-aarch64/.*: 1
windows7-32-shippable/.*: 2
linux1804-64(-shippable|-devedition|-.*qr)/opt: 2
default: 3
mozharness:
mochitest-flavor: plain
chunked:
by-test-platform:
android.*: false
macosx.*64.*: false
windows10-64.*: false
default: true
tier:
by-test-platform:
android-em.*: 1
windows10-aarch64.*: 2
android-hw.*: 1
default: default
mochitest-plain-headless:
description: "Mochitest plain headless run"
Bug 1523303 - [taskgraph] Define suite "categories" rather than flavours task configs, r=gbrown Currently we have the concept of a "suite" and a "flavour" in our task configuration. Typically, the "suite" refers to the high-level test harness like "mochitest" or "reftest", whereas the flavour is more specific, e.g "browser-chrome-instrumentation" or "crashtest". However the line between suite and flavour is not applied with any semblance of consistency which results in inconsistent naming throughout the tree. This patch gets rid of the concept of "flavours" entirely (at least when it comes to task configuration). A suite is a type of test run, for example: - mochitest-plain - mochitest-devtools-chrome - mochitest-browser-chrome-instrumentation - jsreftest - reftest - firefox-ui-functional-remote etc There is no confusion here between suites and flavours because flavours don't exist. However, there are a couple of places where we *do* need to know what "test harness" is used to run a suite. These cases are: 1. For SCHEDULES moz.build rules 2. For the desktop_unittest.py mozharness script which takes arguments like --mochitest-suite=browser (this is not a compelling use of this information and should be refactored to work more like the android_emulator_unittest.py script) So to get this information, this patch introduces a new concept of a "category" which is the overall "test harness" that runs the suite. For many suites, the "category" is identical to the suite name. Unlike flavours, "categories" have no bearing on how we call or refer to the suite. Differential Revision: https://phabricator.services.mozilla.com/D27554 --HG-- extra : moz-landing-system : lando
2019-04-22 23:44:01 +03:00
suite:
name: mochitest-plain
treeherder-symbol: M(h)
loopback-video: true
chunks:
by-test-platform:
linux.*/debug: 16
default: 5
e10s: true
max-run-time: 5400
allow-software-gl-layers: false
tier: 2
run-on-projects: ['mozilla-central']
mozharness:
mochitest-flavor: plain
chunked: true
extra-options:
- --headless
mochitest-valgrind:
description: "Mochitest plain Valgrind run"
Bug 1523303 - [taskgraph] Define suite "categories" rather than flavours task configs, r=gbrown Currently we have the concept of a "suite" and a "flavour" in our task configuration. Typically, the "suite" refers to the high-level test harness like "mochitest" or "reftest", whereas the flavour is more specific, e.g "browser-chrome-instrumentation" or "crashtest". However the line between suite and flavour is not applied with any semblance of consistency which results in inconsistent naming throughout the tree. This patch gets rid of the concept of "flavours" entirely (at least when it comes to task configuration). A suite is a type of test run, for example: - mochitest-plain - mochitest-devtools-chrome - mochitest-browser-chrome-instrumentation - jsreftest - reftest - firefox-ui-functional-remote etc There is no confusion here between suites and flavours because flavours don't exist. However, there are a couple of places where we *do* need to know what "test harness" is used to run a suite. These cases are: 1. For SCHEDULES moz.build rules 2. For the desktop_unittest.py mozharness script which takes arguments like --mochitest-suite=browser (this is not a compelling use of this information and should be refactored to work more like the android_emulator_unittest.py script) So to get this information, this patch introduces a new concept of a "category" which is the overall "test harness" that runs the suite. For many suites, the "category" is identical to the suite name. Unlike flavours, "categories" have no bearing on how we call or refer to the suite. Differential Revision: https://phabricator.services.mozilla.com/D27554 --HG-- extra : moz-landing-system : lando
2019-04-22 23:44:01 +03:00
suite:
name: mochitest-valgrind-plain
treeherder-symbol: M-V()
run-on-projects: []
tier: 3
loopback-video: true
chunks: 40
max-run-time: 14400
# We could re-enable e10s later.
# There's no intrinsic reason not to use it.
e10s: false
variants: []
allow-software-gl-layers: false
mozharness:
mochitest-flavor: plain
chunked: true
mochitest-webgl1-core:
description: "Mochitest webgl1-core run"
treeherder-symbol: M(gl1c)
schedules-component: mochitest-plain
virtualization: virtual-with-gpu
variants:
by-test-platform:
android.*: []
default: ['webgl-ipc']
e10s: true
loopback-video: true
tier: default
max-run-time:
by-test-platform:
macosx.*64-ccov/.*: 7200
default: 1800
run-on-projects:
by-test-platform:
android-hw-.*(?<!-shippable)/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
android-hw.*aarch.*-shippable/opt: ['mozilla-central']
default: built-projects
# Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
allow-software-gl-layers: false
mozharness:
mochitest-flavor: plain
mochitest-webgl1-ext:
description: "Mochitest webgl1-ext run"
treeherder-symbol: M(gl1e)
schedules-component: mochitest-plain
virtualization: virtual-with-gpu
variants:
by-test-platform:
android.*: []
macosx.*64-ccov/.*: []
default: ['webgl-ipc']
chunks:
by-test-platform:
android.*: 2
default: 1
e10s: true
loopback-video: true
tier: default
max-run-time: 2700
run-on-projects:
by-test-platform:
android-hw-.*(?<!-shippable)/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
android-hw.*aarch.*-shippable/opt: ['mozilla-central']
default: built-projects
# Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
allow-software-gl-layers: false
mozharness:
mochitest-flavor: plain
chunked: true
mochitest-webgl2-core:
description: "Mochitest webgl2-core run"
treeherder-symbol: M(gl2c)
schedules-component: mochitest-plain
virtualization: virtual-with-gpu
variants:
by-test-platform:
android.*: []
default: ['webgl-ipc']
chunks:
by-test-platform:
android.*: 2
default: 1
e10s: true
loopback-video: true
tier: default
max-run-time: 1800
run-on-projects:
by-test-platform:
android-hw-.*(?<!-shippable)/opt: ['mozilla-central', 'mozilla-beta', 'mozilla-release']
android-hw.*aarch.*-shippable/opt: ['mozilla-central']
default: built-projects
# Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
allow-software-gl-layers: false
mozharness:
mochitest-flavor: plain
chunked: true
mochitest-webgl2-ext:
description: "Mochitest webgl2-ext run"
treeherder-symbol: M(gl2e)
schedules-component: mochitest-plain
virtualization: virtual-with-gpu
variants:
by-test-platform:
android.*: []
macosx.*64-ccov/.*: []
default: ['webgl-ipc']
chunks: 4
e10s: true
loopback-video: true
tier: default
max-run-time: 2700
# Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
allow-software-gl-layers: false
mozharness:
mochitest-flavor: plain
chunked: true
mochitest-webgl2-deqp:
description: "Mochitest webgl2-deqp run"
treeherder-symbol: M(gl2d)
schedules-component: mochitest-plain
run-on-projects: [] # Don't run this for now.
virtualization: virtual-with-gpu
variants:
by-test-platform:
android.*: []
default: ['webgl-ipc']
chunks: 4
e10s: true
loopback-video: true
tier: default
max-run-time: 1800
# Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
allow-software-gl-layers: false
mozharness:
mochitest-flavor: plain
chunked: true
mochitest-webgpu:
description: "Mochitest webgpu run"
treeherder-symbol: M(webgpu)
schedules-component: mochitest-plain
virtualization: virtual-with-gpu
e10s: true
loopback-video: true
run-on-projects:
by-test-platform:
.*mingw.*: ["release"]
.*shippable.*: ["mozilla-central"]
default: ["trunk"]
tier: default
max-run-time:
by-test-platform:
macosx.*64-ccov/.*: 7200
default: 1800
# Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
allow-software-gl-layers: false
mozharness:
mochitest-flavor: plain
mochitest-remote:
description: "Mochitest for the remote agent (/remote folder)"
suite:
name: mochitest-remote
treeherder-symbol: M(remote)
loopback-video: true
run-on-projects:
by-test-platform:
.*shippable.*: ["mozilla-central"]
(linux1804-64|windows7-32|windows10-64|macosx1014-64)(?:-qr)?/opt: ['integration']
default: ["trunk"]
fission-run-on-projects: []
fission-tier: 2
max-run-time: 5400
mozharness:
mochitest-flavor: browser
extra-options:
- --setpref=remote.log.level=Trace