Bug 1670284 - Run android perf tests with webrender-enabled. r=perftest-reviewers,Bebe

This patch makes webrender-enabled testing the main method for testing and also adds a subset of tests that will without it enabled until the switch is fully complete.

Differential Revision: https://phabricator.services.mozilla.com/D95744

taskcluster/ci/config.yml

@@ -29,9 +29,13 @@ treeherder:
         'Btime-ChR': 'Browsertime performance tests on Google Chrome Release'
         'Btime-Cr': 'Browsertime performance tests on Google Chromium'
         'Btime-P': 'Browsertime power tests on Firefox/Geckoview'
+        'Btime-P-wr': 'Browsertime power tests on Firefox/Geckoview with webrender enabled'
         'Btime-P-fenix': 'Browsertime power tests on Fenix'
+        'Btime-P-fenix-wr': 'Browsertime power tests on Fenix with webrender enabled'
         'Btime-P-1proc': 'Browsertime power tests on Fennec'
+        'Btime-P-1proc-wr': 'Browsertime power tests on Fennec with webrender enabled'
         'Btime-P-refbrow': 'Browsertime power tests on reference browser'
+        'Btime-P-refbrow-wr': 'Browsertime power tests on reference browser with webrender enabled'
         'Btime-fenix': 'Browsertime performance tests on Fenix'
         'Btime-fenix-wr': 'Browsertime performance tests on Fenix with webrender enabled'
         'Btime-fennec': 'Browsertime performance tests on Fennec'
@@ -68,6 +72,7 @@ treeherder:
         'R-fis': 'Reftests with fission enabled'
         'R-swr': 'Reftests with software webrender enabled'
         'Rap': 'Raptor performance tests on Firefox'
+        'Rap-wr': 'Raptor performance tests on Firefox'
         'Rap-live': 'Raptor performance tests on Firefox with live sites'
         'Rap-1proc': 'Raptor performance tests on Firefox without e10s'
         'Rap-fis': 'Raptor performance tests on Firefox with fission enabled'
@@ -81,6 +86,7 @@ treeherder:
         'Rap-fennec': 'Raptor performance tests on Fennec'
         'Rap-fennec-1proc': 'Raptor performance tests on Fennec without e10s'
         'Rap-refbrow': 'Raptor performance tests on the reference browser'
+        'Rap-refbrow-wr': 'Raptor performance tests on the reference browser'
         'remote': 'Remote debugging protocol tests'
         'T': 'Talos performance tests'
         'T-1proc': 'Talos performance tests without e10s'

taskcluster/ci/test/browsertime-mobile.yml

@@ -128,7 +128,7 @@ browsertime-tp6m:
                     refbrow: 2
                     fennec: 2
                     default: 3
-    run-on-projects:
+    webrender-run-on-projects:
         by-raptor-test:
             amazon:
                 by-app:
@@ -181,6 +181,54 @@ browsertime-tp6m:
                             geckoview:
                                 by-test-platform: *android-warm-settings
                             default: []
+    run-on-projects:
+        by-app:
+            geckoview:
+                by-subtest:
+                    allrecipes:
+                        by-pageload:
+                            cold:
+                                by-test-platform: *android-cold-settings
+                            warm:
+                                by-test-platform: *android-warm-settings
+                    amazon-search:
+                        by-pageload:
+                            cold:
+                                by-test-platform: *android-cold-settings
+                            warm:
+                                by-test-platform: *android-warm-settings
+                    espn:
+                        by-pageload:
+                            cold:
+                                by-test-platform: *android-cold-settings
+                            warm:
+                                by-test-platform: *android-warm-settings
+                    facebook:
+                        by-pageload:
+                            cold:
+                                by-test-platform: *android-cold-settings
+                            warm:
+                                by-test-platform: *android-warm-settings
+                    google:
+                        by-pageload:
+                            cold:
+                                by-test-platform: *android-cold-settings
+                            warm:
+                                by-test-platform: *android-warm-settings
+                    microsoft-support:
+                        by-pageload:
+                            cold:
+                                by-test-platform: *android-cold-settings
+                            warm:
+                                by-test-platform: *android-warm-settings
+                    youtube-watch:
+                        by-pageload:
+                            cold:
+                                by-test-platform: *android-cold-settings
+                            warm:
+                                by-test-platform: *android-warm-settings
+                    default: []
+            default: []
     pageload: cold
     run-visual-metrics: true
     treeherder-symbol: Btime(tp6m)
@@ -206,7 +254,7 @@ browsertime-youtube-playback-mobile:
                 - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
                 - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
     apps: ["fenix", "geckoview", "refbrow"]
-    run-on-projects:
+    webrender-run-on-projects:
         by-app:
             geckoview:
                 by-test-platform:
@@ -288,7 +336,7 @@ browsertime-tp6m-live:
                 - youtube
                 - [youtube-watch, youtube-w]
     apps: ["fenix", "chrome-m", "geckoview"]
-    run-on-projects:
+    webrender-run-on-projects:
         by-app:
             geckoview:
                 by-test-platform:
@@ -326,7 +374,7 @@ browsertime-speedometer-mobile:
         by-app:
             fenix: 1
             default: 3
-    run-on-projects:
+    webrender-run-on-projects:
         by-app:
             geckoview:
                 by-test-platform:
@@ -344,7 +392,7 @@ browsertime-unity-webgl-mobile:
     description: "Browsertime Unity WebGL on Android"
     apps: ["geckoview", "refbrow", "fenix", "chrome-m"]
     treeherder-symbol: Btime(ugl)
-    run-on-projects:
+    webrender-run-on-projects:
         by-app:
             geckoview:
                 by-test-platform:
@@ -368,7 +416,7 @@ browsertime-power:
         - idle
         - idle-bg
         - [speedometer, sp]
-    variants: []
+    variants: ["webrender"]
     run-on-projects: []
     treeherder-symbol: Btime-P(power)
     mozharness:

taskcluster/ci/test/raptor-gve.yml

@@ -5,6 +5,8 @@
 job-defaults:
     max-run-time: 1800
     suite: raptor
+    webrender-run-on-projects: []
+    variants: ["webrender"]
     run-on-projects:
         by-test-platform:
             android-hw-p2-.*-api-16/(?!opt).*: []

taskcluster/ci/test/raptor-refbrow.yml

@@ -4,6 +4,8 @@
 job-defaults:
     max-run-time: 1800
     suite: raptor
+    webrender-run-on-projects: []
+    variants: ["webrender"]
     run-on-projects: []
     tier: 2
     test-manifest-loader: null  # don't load tests in the taskgraph

taskcluster/taskgraph/target_tasks.py

@@ -741,6 +741,8 @@ def target_tasks_fennec_v68(full_task_graph, parameters, graph_config):
             return False
         if not accept_raptor_android_build(test_platform):
             return False
+        if "-wr" not in try_name:
+            return False
 
         if "-fennec" in try_name:
             if "-power" in try_name:
@@ -783,6 +785,8 @@ def target_tasks_live_site_perf_testing(full_task_graph, parameters, graph_confi
 
         if not accept_raptor_android_build(platform):
             return False
+        if "-wr" not in try_name:
+            return False
         if "fenix" not in try_name:
             return False
         if "browsertime" not in try_name:
@@ -790,7 +794,7 @@ def target_tasks_live_site_perf_testing(full_task_graph, parameters, graph_confi
         if "live" not in try_name:
             return False
         for test in LIVE_SITES:
-            if try_name.endswith(test) or try_name.endswith(test + "-e10s"):
+            if try_name.endswith(test + "-wr") or try_name.endswith(test + "-wr-e10s"):
                 # These tests run 3 times a week, ignore them
                 return False
 
@@ -820,7 +824,11 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config)
 
         def _run_live_site():
             for test in LIVE_SITES:
-                if try_name.endswith(test) or try_name.endswith(test + "-e10s"):
+                if try_name.endswith(test + "-wr") or try_name.endswith(
+                    test + "-wr-e10s"
+                ):
+                    return True
+                elif try_name.endswith(test) or try_name.endswith(test + "-e10s"):
                     return True
             return False
 
@@ -860,6 +868,9 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config)
             # Ignore all fennec tests here, we run those weekly
             if "fennec" in try_name:
                 return False
+            # Only run webrender tests
+            if "chrome-m" not in try_name and "-wr" not in try_name:
+                return False
             # Select live site tests
             if "-live" in try_name and ("fenix" in try_name or "chrome-m" in try_name):
                 return _run_live_site()

taskcluster/taskgraph/transforms/raptor.py

@@ -48,7 +48,13 @@ raptor_description_schema = Schema(
             test_description_schema["run-on-projects"],
         ),
         Optional("webrender-run-on-projects"): optionally_keyed_by(
-            "app", test_description_schema["webrender-run-on-projects"]
+            "app",
+            "pageload",
+            "test-name",
+            "raptor-test",
+            "subtest",
+            "test-platform",
+            test_description_schema["webrender-run-on-projects"],
         ),
         Optional("variants"): optionally_keyed_by(
             "app", "subtest", test_description_schema["variants"]