Bug 1591466 - split raptor tests into tier-1 (high value) and tier-2 (lower value). r=sparky

split raptor tests into tier-1 (high value) and tier-2 (lower value)

Differential Revision: https://phabricator.services.mozilla.com/D78247

taskcluster/ci/test/raptor.yml

@@ -74,7 +74,7 @@ job-defaults:
                 by-test-platform:
                     windows10-64-ccov/.*: 3
                     linux64-ccov/.*: 3
-                    default: 1
+                    default: 2
             default: 2
     fission-tier: 2
     virtualization:
@@ -131,6 +131,7 @@ job-defaults:
                             - linux64-chromium
                 default: []
 
+
 raptor-tp6:
     description: "Raptor (webextension) tp6"
     apps: ["firefox", "chrome", "chromium"]
@@ -166,8 +167,8 @@ raptor-tp6:
         - [facebook-redesign, fb-r]
         - fandom
         - [docs, gdoc]
-        - [google-mail, gmail]
         - google
+        - [google-mail, gmail]
         - [sheets, gsheets]
         - [slides, gslides]
         - imdb
@@ -181,14 +182,27 @@ raptor-tp6:
         - paypal
         - pinterest
         - reddit
+        - tumblr
         - twitch
         - twitter
         - wikipedia
         - yahoo-mail
+        - yahoo-news
         - yandex
         - youtube
-        - tumblr
-        - yahoo-news
+    tier:
+        by-app:
+            firefox:
+                by-subtest:
+                    amazon: 1
+                    .*gmail.*: 1
+                    .*gslides.*: 1
+                    imgur: 1
+                    tumblr: 1
+                    twitch: 1
+                    twitter: 1
+                    default: 2
+            default: 2
 
 raptor-tp6-binast-firefox:
     description: "Raptor Binast on Firefox"
@@ -213,7 +227,6 @@ raptor-speedometer-firefox-profiling:
     try-name: raptor-speedometer-firefox-profiling
     treeherder-symbol: Rap-Prof(sp)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-speedometer
@@ -233,7 +246,6 @@ raptor-stylebench-firefox-profiling:
     try-name: raptor-stylebench-firefox-profiling
     treeherder-symbol: Rap-Prof(sb)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-stylebench
@@ -252,7 +264,6 @@ raptor-motionmark-htmlsuite-firefox-profiling:
     description: "Raptor MotionMark HtmlSuite on Firefox with Gecko Profiling"
     try-name: raptor-motionmark-htmlsuite-firefox-profiling
     treeherder-symbol: Rap-Prof(mm-h)
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-motionmark-htmlsuite
@@ -263,6 +274,7 @@ raptor-motionmark-animometer-firefox:
     variants: ["fission"]
     try-name: raptor-motionmark-animometer-firefox
     treeherder-symbol: Rap(mm-a)
+    tier: 1
     mozharness:
         extra-options:
             - --test=raptor-motionmark-animometer
@@ -272,7 +284,6 @@ raptor-motionmark-animometer-firefox-profiling:
     try-name: raptor-motionmark-animometer-firefox-profiling
     treeherder-symbol: Rap-Prof(mm-a)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-motionmark-animometer
@@ -283,6 +294,7 @@ raptor-webaudio-firefox:
     variants: ["fission"]
     try-name: raptor-webaudio-firefox
     treeherder-symbol: Rap(wa)
+    tier: 1
     mozharness:
         extra-options:
             - --test=raptor-webaudio
@@ -292,7 +304,6 @@ raptor-webaudio-firefox-profiling:
     try-name: raptor-webaudio-firefox
     treeherder-symbol: Rap-Prof(wa)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-webaudio
@@ -312,7 +323,6 @@ raptor-sunspider-firefox-profiling:
     try-name: raptor-sunspider-firefox-profiling
     treeherder-symbol: Rap-Prof(ss)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-sunspider
@@ -339,7 +349,6 @@ raptor-unity-webgl-firefox-profiling:
     description: "Raptor Unity WebGL on Firefox with Gecko Profiling"
     try-name: raptor-unity-webgl-firefox-profiling
     treeherder-symbol: Rap-Prof(ugl)
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-unity-webgl
@@ -358,7 +367,6 @@ raptor-wasm-misc-cranelift-firefox:
             (linux|windows|macos)(?!.*shippable).*: []
             default: ["mozilla-central", "try"]
     max-run-time: 1800
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-misc-cranelift
@@ -383,7 +391,6 @@ raptor-wasm-misc-firefox-profiling:
     try-name: raptor-wasm-misc-firefox-profiling
     treeherder-symbol: Rap-Prof(wm)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-misc
@@ -409,7 +416,6 @@ raptor-wasm-misc-baseline-firefox-profiling:
     try-name: raptor-wasm-misc-baseline-firefox-profiling
     treeherder-symbol: Rap-Prof(wm-b)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-misc-baseline
@@ -435,7 +441,6 @@ raptor-wasm-misc-ion-firefox-profiling:
     try-name: raptor-wasm-misc-ion-firefox-profiling
     treeherder-symbol: Rap-Prof(wm-i)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-misc-ion
@@ -466,7 +471,6 @@ raptor-assorted-dom-firefox-profiling:
     try-name: raptor-assorted-dom-firefox-profiling
     treeherder-symbol: Rap-Prof(dom)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-assorted-dom
@@ -494,7 +498,6 @@ raptor-wasm-godot-firefox-profiling:
     try-name: raptor-wasm-godot-firefox-profiling
     treeherder-symbol: Rap-Prof(godot)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot
@@ -519,7 +522,6 @@ raptor-wasm-godot-baseline-firefox-profiling:
     try-name: raptor-wasm-godot-baseline-firefox-profiling
     treeherder-symbol: Rap-Prof(godot-b)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-baseline
@@ -539,7 +541,6 @@ raptor-wasm-godot-cranelift-firefox:
             .*-qr/.*: 2100
             .*-ref-hw-2017/.*: 2100
             default: 1500
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-cranelift
@@ -549,7 +550,6 @@ raptor-wasm-godot-cranelift-firefox-profiling:
     try-name: raptor-wasm-godot-cranelift-firefox-profiling
     treeherder-symbol: Rap-Prof(godot-c)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-cranelift
@@ -574,7 +574,6 @@ raptor-wasm-godot-ion-firefox-profiling:
     try-name: raptor-wasm-godot-ion-firefox-profiling
     treeherder-symbol: Rap-Prof(godot-i)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-ion
@@ -591,7 +590,6 @@ raptor-youtube-playback-firefox:
             (linux|windows|macos)(?!.*shippable).*: []
             default: ["mozilla-central", "try"]
     max-run-time: 3000
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback
@@ -601,7 +599,6 @@ raptor-youtube-playback-firefox-profiling:
     try-name: raptor-youtube-playback-firefox-profiling
     treeherder-symbol: Rap-Prof(ytp)
     max-run-time: 4000
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback
@@ -654,7 +651,6 @@ raptor-youtube-playback-av1-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 1800
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-av1-sfr
@@ -673,7 +669,6 @@ raptor-youtube-playback-h264-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 1800
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-h264-sfr
@@ -711,7 +706,6 @@ raptor-youtube-playback-vp9-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 1800
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-vp9-sfr
@@ -730,7 +724,6 @@ raptor-youtube-playback-widevine-hfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 2700
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-widevine-hfr
@@ -749,7 +742,6 @@ raptor-youtube-playback-widevine-h264-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 2700
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-widevine-h264-sfr
@@ -768,7 +760,6 @@ raptor-youtube-playback-widevine-vp9-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 2700
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-widevine-vp9-sfr

taskcluster/taskgraph/transforms/raptor.py

@@ -26,6 +26,7 @@ raptor_description_schema = Schema({
     # Raptor specific configs.
     Optional('apps'): optionally_keyed_by(
         'test-platform',
+        'subtest',
         [text_type]
     ),
     Optional('raptor-test'): text_type,
@@ -69,6 +70,7 @@ raptor_description_schema = Schema({
     Optional('tier'): optionally_keyed_by(
         'app',
         'raptor-test',
+        'subtest',
         test_description_schema['tier']
     ),
     Optional('run-visual-metrics'): optionally_keyed_by(
@@ -177,7 +179,11 @@ def split_raptor_subtests(config, tests):
             chunked = deepcopy(test)
             chunked['chunk-number'] = chunk_number
             chunked['subtest'] = subtest
-
+            chunked['subtest-symbol'] = subtest
+            if isinstance(chunked['subtest'], list):
+                chunked['subtest'] = subtest[0]
+                chunked['subtest-symbol'] = subtest[1]
+            chunked = resolve_keyed_by(chunked, 'tier', chunked['subtest'])
             yield chunked
 
 
@@ -238,17 +244,12 @@ def split_page_load_by_url(config, tests):
         # definition for `raptor-subtests`
         chunk_number = test.pop('chunk-number', None)
         subtest = test.pop('subtest', None)
+        subtest_symbol = test.pop('subtest-symbol', None)
 
         if not chunk_number or not subtest:
             yield test
             continue
 
-        if isinstance(subtest, list):
-            subtest, subtest_symbol = subtest
-        else:
-            subtest_symbol = subtest
-            subtest = subtest
-
         if len(subtest_symbol) > 10:
             raise Exception(
                 "Treeherder symbol %s is lager than 10 char! Please use a different symbol."
@@ -353,3 +354,12 @@ def add_extra_options(config, tests):
                             extra_options.append(add_extra_params_option)
 
         yield test
+
+
+@transforms.add
+def apply_tier_optimization(config, tests):
+    for test in tests:
+        test['optimization'] = {'push-interval-10': None}
+        if test['tier'] > 1:
+            test['optimization'] = {'push-interval-25': None}
+        yield test