Bug 1587826 - [ci] Use a transform to split raptor apps r=tomprince,perftest-reviewers,rwood

Differential Revision: https://phabricator.services.mozilla.com/D48979

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/browsertime-fenix.yml

@@ -1,46 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
----
-job-defaults:
-    max-run-time: 1800
-    suite: raptor
-    workdir: /builds/worker
-    run-on-projects: []
-    tier: 3
-    target:
-        by-test-platform:
-            android-hw.*-aarch64.*/.*:
-                index: project.mobile.fenix.v2.performance-test.latest.aarch64
-                name: target.apk
-            default:
-                index: project.mobile.fenix.v2.performance-test.latest.arm
-                name: target.apk
-    e10s: true
-    virtualization: hardware
-    mozharness:
-        script: raptor_script.py
-        config:
-            - raptor/android_hw_config.py
-
-browsertime-tp6m-1-fenix:
-    description: "Raptor (browsertime) tp6m-1 on Fenix"
-    treeherder-symbol: Btime-fenix(tp6m-1)
-    mozharness:
-        extra-options:
-            - --browsertime
-            - --app=fenix
-            - --binary-path=org.mozilla.fenix.performancetest
-            - --activity=org.mozilla.fenix.IntentReceiverActivity
-            - --test=raptor-tp6m-1
-
-browsertime-tp6m-1-fenix-cold:
-    description: "Raptor (browsertime) tp6m-1 cold page-load on Fenix"
-    treeherder-symbol: Btime-fenix(tp6m-c-1)
-    mozharness:
-        extra-options:
-            - --browsertime
-            - --app=fenix
-            - --binary-path=org.mozilla.fenix.performancetest
-            - --activity=org.mozilla.fenix.IntentReceiverActivity
-            - --test=raptor-tp6m-cold-1

taskcluster/ci/test/browsertime-gve.yml

@@ -1,40 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
----
-job-defaults:
-    max-run-time: 1800
-    suite: raptor
-    workdir: /builds/worker
-    run-on-projects: ['integration', 'try']
-    tier: 3
-    target: geckoview_example.apk
-    e10s: true
-    virtualization: hardware
-    mozharness:
-        script: raptor_script.py
-        config:
-            - raptor/android_hw_config.py
-
-browsertime-tp6m-1-geckoview:
-    description: "Raptor (browsertime) tp6m-1 on GeckoView Example"
-    treeherder-symbol: Btime(tp6m-1)
-    mozharness:
-        extra-options:
-            - --browsertime
-            - --app=geckoview
-            - --binary=org.mozilla.geckoview_example
-            - --activity=org.mozilla.geckoview_example.GeckoViewActivity
-            - --test=raptor-tp6m-1
-
-browsertime-tp6m-1-geckoview-cold:
-    description: "Raptor (browsertime) tp6m-1 cold page-load on GeckoView Example"
-    max-run-time: 2700
-    treeherder-symbol: Btime(tp6m-c-1)
-    mozharness:
-        extra-options:
-            - --browsertime
-            - --app=geckoview
-            - --binary=org.mozilla.geckoview_example
-            - --activity=org.mozilla.geckoview_example.GeckoViewActivity
-            - --test=raptor-tp6m-cold-1

taskcluster/ci/test/browsertime-mobile.yml

@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    max-run-time: 1800
+    suite: raptor
+    workdir: /builds/worker
+    run-on-projects:
+        by-app:
+            fenix: []
+            geckoview: ["integration", "try"]
+            refbrow: []
+    tier: 3
+    target:
+        by-app:
+            fenix:
+                by-test-platform:
+                    android-hw.*-aarch64.*/.*:
+                        index: project.mobile.fenix.v2.performance-test.latest.aarch64
+                        name: target.apk
+                    default:
+                        index: project.mobile.fenix.v2.performance-test.latest.arm
+                        name: target.apk
+            geckoview: geckoview_example.apk
+            refbrow:
+                by-test-platform:
+                    android-hw.*-aarch64.*/.*:
+                        index: project.mobile.reference-browser.v2.raptor.latest.aarch64
+                        name: target.apk
+                    default:
+                        index: project.mobile.reference-browser.v2.raptor.latest.arm
+                        name: target.apk
+    e10s: true
+    virtualization: hardware
+    activity:
+        by-app:
+            fenix: org.mozilla.fenix.IntentReceiverActivity
+            geckoview: org.mozilla.geckoview_example.GeckoViewActivity
+            refbrow: org.mozilla.reference.browser.BrowserTestActivity
+    binary-path:
+        by-app:
+            fenix: org.mozilla.fenix.performancetest
+            geckoview: org.mozilla.geckoview_example
+            refbrow: org.mozilla.reference.browser.raptor
+    mozharness:
+        script: raptor_script.py
+        config:
+            - raptor/android_hw_config.py
+        extra-options:
+            - --browsertime
+
+browsertime-tp6m-1:
+    description: "Raptor (browsertime) tp6m-1"
+    apps: ["fenix", "geckoview", "refbrow"]
+    treeherder-symbol: Btime(tp6m-1)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-1
+
+browsertime-tp6m-1-cold:
+    description: "Raptor (browsertime) tp6m-1 cold page-load"
+    apps: ["fenix", "geckoview", "refbrow"]
+    treeherder-symbol: Btime(tp6m-c-1)
+    max-run-time:
+        by-app:
+            fenix: 1800
+            geckoview: 2700
+            refbrow: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-1

taskcluster/ci/test/browsertime-refbrow.yml

@@ -1,46 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
----
-job-defaults:
-    max-run-time: 1800
-    suite: raptor
-    workdir: /builds/worker
-    run-on-projects: []
-    tier: 3
-    target:
-        by-test-platform:
-            android-hw.*-aarch64.*/.*:
-                index: project.mobile.reference-browser.v2.raptor.latest.aarch64
-                name: target.apk
-            default:
-                index: project.mobile.reference-browser.v2.raptor.latest.arm
-                name: target.apk
-    e10s: true
-    virtualization: hardware
-    mozharness:
-        script: raptor_script.py
-        config:
-            - raptor/android_hw_config.py
-
-browsertime-tp6m-1-refbrow:
-    description: "Raptor (browsertime) tp6m-1 on the reference browser"
-    treeherder-symbol: Btime-refbrow(tp6m-1)
-    mozharness:
-        extra-options:
-            - --browsertime
-            - --app=refbrow
-            - --binary-path=org.mozilla.reference.browser.raptor
-            - --activity=org.mozilla.reference.browser.BrowserTestActivity
-            - --test=raptor-tp6m-1
-
-browsertime-tp6m-1-refbrow-cold:
-    description: "Raptor (browsertime) tp6m-1 cold page-load on the reference browser"
-    treeherder-symbol: Btime-refbrow(tp6m-c-1)
-    mozharness:
-        extra-options:
-            - --browsertime
-            - --app=refbrow
-            - --binary-path=org.mozilla.reference.browser.raptor
-            - --activity=org.mozilla.reference.browser.BrowserTestActivity
-            - --test=raptor-tp6m-cold-1

taskcluster/ci/test/kind.yml

@@ -27,9 +27,7 @@ transforms:
 
 jobs-from:
     - awsy.yml
-    - browsertime-fenix.yml
-    - browsertime-gve.yml
-    - browsertime-refbrow.yml
+    - browsertime-mobile.yml
     - compiled.yml
     - firefox-ui.yml
     - marionette.yml

taskcluster/ci/test/raptor.yml

@@ -1013,11 +1013,11 @@ raptor-jetstream2-firefox-profiling:
         fetch:
             - jetstream2
 
-browsertime-tp6-1-firefox:
-    description: "Raptor (browsertime) tp6-1 on Firefox"
+browsertime-tp6-1:
+    description: "Raptor (browsertime) tp6-1"
     run-on-projects: ['integration', 'try']
+    apps: ["firefox"]
     variants: ["fission"]
-    try-name: browsertime-tp6-1-firefox
     treeherder-symbol: Btime(tp6-1)
     max-run-time: 3000
     tier: 3
@@ -1026,11 +1026,11 @@ browsertime-tp6-1-firefox:
             - --browsertime
             - --test=raptor-tp6-1
 
-browsertime-tp6-1-firefox-cold:
-    description: "Raptor (browsertime) tp6-1 cold page-load on Firefox"
+browsertime-tp6-1-cold:
+    description: "Raptor (browsertime) tp6-1 cold page-load"
     run-on-projects: ['integration', 'try']
+    apps: ["firefox"]
     variants: ["fission"]
-    try-name: browsertime-tp6-1-firefox-cold
     treeherder-symbol: Btime(tp6-c-1)
     max-run-time: 3000
     tier: 3

taskcluster/ci/test/test-platforms.yml

@@ -42,7 +42,7 @@ linux64/opt:
         - mochitest-headless
         - linux-talos-flex
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - raptor-fetch-firefox
         - marionette-media-tests
 linux64-shippable/opt:
@@ -62,7 +62,7 @@ linux64-shippable/opt:
         - raptor-chromium
         - raptor-fetch-chromium
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - raptor-fetch-firefox
         - raptor-profiling
         - raptor-fetch-profiling
@@ -96,7 +96,7 @@ linux64-qr/opt:
         - awsy
         - web-platform-tests
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - raptor-fetch-firefox
 linux64-shippable-qr/opt:
     build-platform: linux64-shippable/opt
@@ -106,7 +106,7 @@ linux64-shippable-qr/opt:
         - awsy
         - web-platform-tests
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - raptor-fetch-firefox
 linux64-qr/debug:
     build-platform: linux64/debug
@@ -148,7 +148,7 @@ windows7-32/opt:
         - windows-tests
         - web-platform-tests
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
 
 windows7-32-shippable/opt:
     build-platform: win32-shippable/opt
@@ -163,7 +163,7 @@ windows7-32-shippable/opt:
         - raptor-chrome
         - raptor-chromium
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
 
 windows7-32-devedition/opt:
     build-platform: win32-devedition-nightly/opt
@@ -204,7 +204,7 @@ windows10-64/opt:
         - windows-tests
         - web-platform-tests
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - marionette-media-tests
 
 windows10-aarch64/opt:
@@ -213,7 +213,7 @@ windows10-aarch64/opt:
         - awsy
         - marionette-gpu-tests
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - windows-aarch64-tests
         - windows-talos
 
@@ -221,7 +221,7 @@ windows10-64-ref-hw-2017/opt:
     build-platform: win64-shippable/opt
     test-sets:
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - talos-ref-hw
 
 windows10-64-shippable/opt:
@@ -237,7 +237,7 @@ windows10-64-shippable/opt:
         - raptor-chrome
         - raptor-chromium
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - raptor-profiling
 
 windows10-64-devedition/opt:
@@ -267,7 +267,7 @@ windows10-64-qr/opt:
         - windows-qr-tests
         - windows-talos
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - web-platform-tests
 
 windows10-64-shippable-qr/opt:
@@ -277,7 +277,7 @@ windows10-64-shippable-qr/opt:
         - windows-qr-tests
         - windows-talos
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - web-platform-tests
 
 windows10-64-mingwclang/debug:
@@ -313,7 +313,7 @@ macosx1014-64-shippable/opt:
         - raptor-chrome
         - raptor-chromium
         - raptor-firefox
-        - browsertime-firefox
+        - browsertime
         - raptor-firefox-power
         - raptor-profiling
         - marionette-media-tests

taskcluster/ci/test/test-sets.yml

@@ -889,22 +889,14 @@ android-hw-arm7-raptor-nightly:
     - raptor-tp6m-9-fennec68
     - raptor-tp6m-10-fennec68
 
-browsertime-firefox:
-    - browsertime-tp6-1-firefox
-    - browsertime-tp6-1-firefox-cold
+browsertime:
+    - browsertime-tp6-1
+    - browsertime-tp6-1-cold
 
 android-hw-aarch64-browsertime:
-    - browsertime-tp6m-1-fenix
-    - browsertime-tp6m-1-geckoview
-    - browsertime-tp6m-1-refbrow
-    - browsertime-tp6m-1-fenix-cold
-    - browsertime-tp6m-1-geckoview-cold
-    - browsertime-tp6m-1-refbrow-cold
+    - browsertime-tp6m-1
+    - browsertime-tp6m-1-cold
 
 android-hw-arm7-browsertime:
-    - browsertime-tp6m-1-fenix
-    - browsertime-tp6m-1-geckoview
-    - browsertime-tp6m-1-refbrow
-    - browsertime-tp6m-1-fenix-cold
-    - browsertime-tp6m-1-geckoview-cold
-    - browsertime-tp6m-1-refbrow-cold
+    - browsertime-tp6m-1
+    - browsertime-tp6m-1-cold

taskcluster/taskgraph/transforms/raptor.py

@@ -4,22 +4,53 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+from copy import deepcopy
+
 from voluptuous import (
+    Optional,
     Required,
     Extra,
 )
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.tests import test_description_schema
-from taskgraph.util.schema import Schema
+from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
+from taskgraph.util.treeherder import split_symbol, join_symbol
 
 transforms = TransformSequence()
 
 
 raptor_description_schema = Schema({
+    # Raptor specific configs.
+    Optional('apps'): optionally_keyed_by(
+        'test-platform',
+        [basestring]
+    ),
+    Optional('activity'): optionally_keyed_by(
+        'app',
+        basestring
+    ),
+    Optional('binary-path'): optionally_keyed_by(
+        'app',
+        basestring
+    ),
+    # Configs defined in the 'test_description_schema'.
+    Optional('max-run-time'): optionally_keyed_by(
+        'app',
+        test_description_schema['max-run-time']
+    ),
+    Optional('run-on-projects'): optionally_keyed_by(
+        'app',
+        test_description_schema['run-on-projects']
+    ),
+    Optional('target'): optionally_keyed_by(
+        'app',
+        test_description_schema['target']
+    ),
     Required('test-name'): test_description_schema['test-name'],
     Required('test-platform'): test_description_schema['test-platform'],
     Required('require-signed-extensions'): test_description_schema['require-signed-extensions'],
+    Required('treeherder-symbol'): test_description_schema['treeherder-symbol'],
     # Any unrecognized keys will be validated against the test_description_schema.
     Extra: object,
 })
@@ -27,11 +58,73 @@ raptor_description_schema = Schema({
 transforms.add_validate(raptor_description_schema)
 
 
+@transforms.add
+def split_apps(config, tests):
+    app_symbols = {
+        'chrome': 'Chr',
+        'chromium': 'Cr',
+        'fenix': 'fenix',
+        'refbrow': 'refbrow',
+    }
+
+    for test in tests:
+        apps = test.pop('apps', None)
+        if not apps:
+            yield test
+            continue
+
+        for app in apps:
+            atest = deepcopy(test)
+            suffix = "-{}".format(app)
+            atest['app'] = app
+            atest['description'] += " on {}".format(app.capitalize())
+
+            name = atest['test-name']
+            if name.endswith('-cold'):
+                name = atest['test-name'][:-len('-cold')] + suffix + '-cold'
+            else:
+                name += suffix
+
+            atest['test-name'] = name
+            atest['try-name'] = name
+
+            if app in app_symbols:
+                group, symbol = split_symbol(atest['treeherder-symbol'])
+                group += "-{}".format(app_symbols[app])
+                atest['treeherder-symbol'] = join_symbol(group, symbol)
+
+            yield atest
+
+
+@transforms.add
+def handle_keyed_by_app(config, tests):
+    fields = [
+        'activity',
+        'binary-path',
+        'max-run-time',
+        'run-on-projects',
+        'target',
+    ]
+    for test in tests:
+        for field in fields:
+            resolve_keyed_by(test, field, item_name=test['test-name'])
+        yield test
+
+
 @transforms.add
 def add_extra_options(config, tests):
     for test in tests:
         extra_options = test.setdefault('mozharness', {}).setdefault('extra-options', [])
 
+        if 'app' in test:
+            extra_options.append('--app={}'.format(test.pop('app')))
+
+        if 'activity' in test:
+            extra_options.append('--activity={}'.format(test.pop('activity')))
+
+        if 'binary-path' in test:
+            extra_options.append('--binary-path={}'.format(test.pop('binary-path')))
+
         if test['require-signed-extensions']:
             extra_options.append('--is-release-build')
 

taskcluster/taskgraph/transforms/tests.py

@@ -1091,7 +1091,7 @@ def handle_run_on_projects(config, tests):
 @transforms.add
 def split_variants(config, tests):
     for test in tests:
-        variants = test.pop('variants') or []
+        variants = test.pop('variants', [])
 
         yield copy.deepcopy(test)