Bug 1539932 - [ci] Handle serviceworker/socketprocess test variants more generically, r=jmaher

We are starting to spin off more and more "variants" of test suites. These are usually just duplicates of our pre-existing tasks, except with an additional pref set. Currently there are two variants (serviceworker-e10s and socketprocess-e10s), but a third will be added soon (fission). This change ensures we handle these types of requests in a consistent and well defined manner. It also splits tasks in a loop, so we don't accidentally risk combinatorial explosion. Variants should typically be reserved for very large changes that will impact the entire codebase (think e10s). Differential Revision: https://phabricator.services.mozilla.com/D28061 --HG-- extra : moz-landing-system : lando
2019-04-18 14:40:08 +00:00 · 2019-04-18 14:40:08 +00:00 · e105223614
--- a/taskcluster/ci/test/mochitest.yml
+++ b/taskcluster/ci/test/mochitest.yml
@ -3,11 +3,10 @@ job-defaults:
        by-test-platform:
            android-em-7.*: geckoview-androidTest.apk
            default: null
-    serviceworker-e10s:
+    variants:
        by-test-platform:
-            linux64/debug: both
-            default: false
-    socketprocess-e10s: false
+            linux64/debug: ['serviceworker']
+            default: []
    run-on-projects:
        by-test-platform:
            android-em-4.3-arm7-api-16/opt: ['try']
@ -319,10 +318,11 @@ mochitest-media:
            android-em-4.3-arm7-api-16/opt: ['try']
            windows10-aarch64/opt: ['try', 'mozilla-central']
            default: built-projects
-    socketprocess-e10s:
+    variants:
        by-test-platform:
-            android.*: false
-            default: both
+            android.*: []
+            linux64/debug: ['serviceworker', 'socketprocess']
+            default: ['socketprocess']
    loopback-video: true
    virtualization:
        by-test-platform:
@ -386,7 +386,7 @@ mochitest-valgrind:
    # We could re-enable e10s later.
    # There's no intrinsic reason not to use it.
    e10s: false
-    serviceworker-e10s: false
+    variants: []
    allow-software-gl-layers: false
    mozharness:
        mochitest-flavor: plain
--- a/taskcluster/ci/test/reftest.yml
+++ b/taskcluster/ci/test/reftest.yml
@ -3,10 +3,10 @@ job-defaults:
        by-test-platform:
            android-em-7.*: geckoview-androidTest.apk
            default: null
-    serviceworker-e10s:
+    variants:
        by-test-platform:
-            linux64/debug: both
-            default: false
+            linux64/debug: ['serviceworker']
+            default: []
    run-on-projects:
        by-test-platform:
            android-em-4.3-arm7-api-16/opt: ['try']
--- a/taskcluster/ci/test/web-platform.yml
+++ b/taskcluster/ci/test/web-platform.yml
@ -1,10 +1,10 @@
 job-defaults:
    instance-size: xlarge
    max-run-time: 5400
-    serviceworker-e10s:
+    variants:
        by-test-platform:
-            linux64/debug: both
-            default: false
+            linux64/debug: ['serviceworker']
+            default: []
    virtualization:
        by-test-platform:
            windows10-64(?:-pgo|-shippable)?-qr/.*: virtual-with-gpu
--- a/taskcluster/ci/test/xpcshell.yml
+++ b/taskcluster/ci/test/xpcshell.yml
@ -1,8 +1,8 @@
 job-defaults:
-    serviceworker-e10s:
+    variants:
        by-test-platform:
-            linux64/debug: both
-            default: false
+            linux64/debug: ['serviceworker']
+            default: []
    mozharness:
        script:
            by-test-platform:
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@ -108,6 +108,13 @@ unittest_try_name
 This is the name used to refer to a unit test via try syntax.  It
 may not match either of ``unittest_suite`` or ``unittest_flavor``.

+unittest_variant
+================
+
+The configuration variant the test suite is running with. If set, this usually
+means the tests are running with a special pref enabled. These are defined in
+``taskgraph.transforms.tests.TEST_VARIANTS``.
+
 talos_try_name
 ==============

@ -130,18 +137,6 @@ test_chunk
 This is the chunk number of a chunked test suite (talos or unittest).  Note
 that this is a string!

-serviceworker_e10s
-==================
-
-For test suites which distinguish whether or not they run with the serviceworker
-e10s redesign enabled.
-
-socketprocess_e10s
-==================
-
-For test suites which distinguish whether or not they run with the socket
-process enabled.
-
 e10s
 ====

--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@ -22,6 +22,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import match_run_on_projects
 from taskgraph.util.schema import resolve_keyed_by, OptimizationSchema
+from taskgraph.util.templates import merge
 from taskgraph.util.treeherder import split_symbol, join_symbol, add_suffix
 from taskgraph.util.platforms import platform_family
 from taskgraph.util.schema import (
@ -147,6 +148,39 @@ MACOSX_WORKER_TYPES = {
    'macosx64': 'releng-hardware/gecko-t-osx-1010',
 }

+
+def runs_on_central(test):
+    return match_run_on_projects('mozilla-central', test['run-on-projects'])
+
+
+TEST_VARIANTS = {
+    'serviceworker': {
+        'description': "{description} with serviceworker-e10s redesign enabled",
+        'filterfn': runs_on_central,
+        'suffix': 'sw',
+        'config': {
+            'run-on-projects': ['mozilla-central'],
+            'tier': 2,
+            'mozharness': {
+                'extra-options': ['--setpref="dom.serviceWorkers.parent_intercept=true"'],
+            },
+        }
+    },
+    'socketprocess': {
+        'description': "{description} with socket process enabled",
+        'suffix': 'spi',
+        'config': {
+            'mozharness': {
+                'extra-options': [
+                    '--setpref="media.peerconnection.mtransport_process=true"',
+                    '--setpref="network.process.enabled=true"',
+                ],
+            }
+        }
+    }
+}
+
+
 logger = logging.getLogger(__name__)

 transforms = TransformSequence()
@ -225,13 +259,11 @@ test_description_schema = Schema({
    # the branch (see below)
    Optional('expires-after'): basestring,

-    # Whether to run this task with the serviceworker e10s redesign enabled
-    # (desktop-test only).  If 'both', run one task with and one task without.
-    # Tasks with this enabled have have "-sw" appended to the test name and
-    # treeherder group.
-    Optional('serviceworker-e10s'): optionally_keyed_by(
+    # The different configurations that should be run against this task, defined
+    # in the TEST_VARIANTS object.
+    Optional('variants'): optionally_keyed_by(
        'test-platform', 'project',
-        Any(bool, 'both')),
+        Any(TEST_VARIANTS.keys())),

    # Whether to run this task with e10s.  If false, run
    # without e10s; if true, run with e10s; if 'both', run one task with and
@ -241,14 +273,6 @@ test_description_schema = Schema({
        'test-platform', 'project',
        Any(bool, 'both')),

-    # Whether to run this task with the socket process enabled (desktop-test
-    # only).  If 'both', run one task with and one task without.  Tasks with
-    # this enabled have have "-spi" appended to the test name and treeherder
-    # group.
-    Optional('socketprocess-e10s'): optionally_keyed_by(
-        'test-platform', 'project',
-        Any(bool, 'both')),
-
    # Whether the task should run with WebRender enabled or not.
    Optional('webrender'): bool,

@ -508,9 +532,8 @@ def set_defaults(config, tests):
        test.setdefault('loopback-video', False)
        test.setdefault('docker-image', {'in-tree': 'desktop1604-test'})
        test.setdefault('checkout', False)
-        test.setdefault('serviceworker-e10s', False)
-        test.setdefault('socketprocess-e10s', False)
        test.setdefault('require-signed-extensions', False)
+        test.setdefault('variants', [])

        test['mozharness'].setdefault('extra-options', [])
        test['mozharness'].setdefault('requires-signed-builds', False)
@ -768,9 +791,8 @@ def handle_keyed_by(config, tests):
        'docker-image',
        'max-run-time',
        'chunks',
-        'serviceworker-e10s',
+        'variants',
        'e10s',
-        'socketprocess-e10s',
        'suite',
        'run-on-projects',
        'os-groups',
@ -938,39 +960,34 @@ def handle_run_on_projects(config, tests):


@transforms.add
-def split_serviceworker_e10s(config, tests):
+def split_variants(config, tests):
    for test in tests:
-        if test['attributes'].get('socketprocess_e10s'):
-            yield test
-            continue
+        variants = test.pop('variants')

-        sw = test.pop('serviceworker-e10s')
+        yield copy.deepcopy(test)

-        test['serviceworker-e10s'] = False
-        test['attributes']['serviceworker_e10s'] = False
+        for name in variants:
+            testv = copy.deepcopy(test)
+            variant = TEST_VARIANTS[name]

-        if sw == 'both':
-            yield copy.deepcopy(test)
-            sw = True
-        if sw:
-            if not match_run_on_projects('mozilla-central', test['run-on-projects']):
+            if 'filterfn' in variant and not variant['filterfn'](testv):
                continue

-            test['description'] += " with serviceworker-e10s redesign enabled"
-            test['run-on-projects'] = ['mozilla-central']
-            test['test-name'] += '-sw'
-            test['try-name'] += '-sw'
-            test['attributes']['serviceworker_e10s'] = True
-            group, symbol = split_symbol(test['treeherder-symbol'])
+            testv['attributes']['unittest_variant'] = name
+            testv['description'] = variant['description'].format(**testv)
+
+            suffix = '-' + variant['suffix']
+            testv['test-name'] += suffix
+            testv['try-name'] += suffix
+
+            group, symbol = split_symbol(testv['treeherder-symbol'])
            if group != '?':
-                group += '-sw'
+                group += suffix
            else:
-                symbol += '-sw'
-            test['treeherder-symbol'] = join_symbol(group, symbol)
-            test['mozharness']['extra-options'].append(
-                '--setpref="dom.serviceWorkers.parent_intercept=true"')
-            test['tier'] = 2
-        yield test
+                symbol += suffix
+            testv['treeherder-symbol'] = join_symbol(group, symbol)
+
+            yield merge(testv, variant['config'])


@transforms.add
@ -998,39 +1015,6 @@ def split_e10s(config, tests):
            yield test


-@transforms.add
-def split_socketprocess_e10s(config, tests):
-    for test in tests:
-        if test['attributes'].get('serviceworker_e10s'):
-            yield test
-            continue
-
-        sw = test.pop('socketprocess-e10s')
-
-        test['socketprocess-e10s'] = False
-        test['attributes']['socketprocess_e10s'] = False
-
-        if sw == 'both':
-            yield copy.deepcopy(test)
-            sw = True
-        if sw:
-            test['description'] += " with socket process enabled"
-            test['test-name'] += '-spi'
-            test['try-name'] += '-spi'
-            test['attributes']['socketprocess_e10s'] = True
-            group, symbol = split_symbol(test['treeherder-symbol'])
-            if group != '?':
-                group += '-spi'
-            else:
-                symbol += '-spi'
-            test['treeherder-symbol'] = join_symbol(group, symbol)
-            test['mozharness']['extra-options'].append(
-                '--setpref="media.peerconnection.mtransport_process=true"')
-            test['mozharness']['extra-options'].append(
-                '--setpref="network.process.enabled=true"')
-        yield test
-
-
@transforms.add
 def split_chunks(config, tests):
    """Based on the 'chunks' key, split tests up into chunks by duplicating