Bug 1820893 - Run browsertime performance tests on nightly-as-release builds. r=perftest-reviewers,taskgraph-reviewers,jmaher,afinder

This patch adds three new shippable platforms running the nightly-as-release build. At the same time, browsertime essential tests are enabled for all three platforms on autoland.

Depends on D172029

Differential Revision: https://phabricator.services.mozilla.com/D172030

taskcluster/ci/test/browsertime-desktop.yml

@@ -83,7 +83,7 @@ browsertime-tp6:
                         firefox:
                             by-test-platform:
                                 windows.*-32.*: []
-                                .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+                                .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
                                 default: []
                         default: []
         treeherder-symbol: Btime(tp6)
@@ -149,7 +149,7 @@ browsertime-responsiveness:
             firefox:
                 by-test-platform:
                     windows.*-32.*: []
-                    .*64.*shippable-qr/.*: [trunk]
+                    .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
                     default: []
             default: []
     tier:
@@ -195,7 +195,7 @@ browsertime-tp6-essential:
                     by-test-platform:
                         windows.*-32.*: []
                         macosx1100.*shippable.*: [mozilla-central, mozilla-beta]
-                        .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+                        .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
                         default: []
                 default: []
         tier:
@@ -223,6 +223,15 @@ browsertime-tp6-bytecode:
             - twitter
             - wikipedia
             - yahoo-mail
+    run-on-projects:
+        by-app:
+            firefox:
+                by-test-platform:
+                    windows.*-32.*: []
+                    macosx1100.*shippable.*: [mozilla-central, mozilla-beta]
+                    .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+                    default: []
+            default: []
     mozharness:
         extra-options:
             by-test-platform:
@@ -317,7 +326,8 @@ browsertime-benchmark:
                     firefox:
                         by-test-platform:
                             windows.*-32.*: []
-                            .*64.*shippable-qr/.*: [trunk, mozilla-release]
+                            .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-release]
+                            .*nightlyasrelease.*: [autoland]
                             default: []
                     default: []
             unity-webgl:
@@ -325,7 +335,16 @@ browsertime-benchmark:
                     firefox:
                         by-test-platform:
                             windows.*-32.*: []
-                            .*64.*shippable-qr/.*: [trunk, mozilla-release]
+                            .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-release]
+                            default: []
+                    default: []
+            speedometer:
+                by-app:
+                    firefox:
+                        by-test-platform:
+                            windows.*-32.*: []
+                            .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+                            .*nightlyasrelease.*: [autoland]
                             default: []
                     default: []
             default:
@@ -333,7 +352,7 @@ browsertime-benchmark:
                     firefox:
                         by-test-platform:
                             windows.*-32.*: []
-                            .*64.*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
+                            .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk, mozilla-beta, mozilla-release]
                             default: []
                     default: []
     max-run-time:
@@ -372,7 +391,7 @@ browsertime-benchmark-wasm:
             firefox:
                 by-test-platform:
                     windows.*-32.*: []
-                    .*64.*shippable-qr/.*: [mozilla-central, mozilla-release]
+                    .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central, mozilla-release]
                     default: []
             default: []
     max-run-time:
@@ -399,7 +418,7 @@ browsertime-youtube-playback:
     run-on-projects:
         by-test-platform:
             windows.*-32.*: []
-            .*64.*shippable-qr/.*: [mozilla-central, mozilla-release]
+            .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central, mozilla-release]
             default: []
     tier: 2
     limit-platforms:
@@ -594,7 +613,7 @@ browsertime-custom:
     run-on-projects:
         by-test-platform:
             windows.*-32.*: []
-            .*64.*shippable-qr/.*: [mozilla-central]
+            .*64(?!.*nightlyasrelease).*shippable-qr/.*: [mozilla-central]
             default: []
     tier: 1
     treeherder-symbol: Btime(ps)
@@ -613,7 +632,7 @@ browsertime-first-install:
     run-on-projects:
         by-test-platform:
             windows.*-32.*: []
-            .*64.*shippable-qr/.*: [trunk]
+            .*64(?!.*nightlyasrelease).*shippable-qr/.*: [trunk]
             default: []
     tier: 1
     treeherder-symbol: Btime(welcome)
@@ -629,7 +648,8 @@ browsertime-grandprix:
             firefox:
                 by-test-platform:
                     windows.*-32.*: []
-                    .*64.*shippable-qr/.*: [autoland, mozilla-central]
+                    .*64(?!.*nightlyasrelease).*shippable-qr/.*: [autoland, mozilla-central]
+                    .*nightlyasrelease.*: [autoland]
                     default: []
             default: []
     tier: 3

taskcluster/ci/test/test-platforms.yml

@@ -63,6 +63,11 @@ linux1804-64-shippable-qr/opt:
         - browsertime
         - browsertime-profiling
 
+linux1804-64-nightlyasrelease-qr/opt:
+    build-platform: linux64-nightlyasrelease/opt
+    test-sets:
+        - browsertime
+
 linux1804-64-qr/debug:
     build-platform: linux64/debug
     test-sets:
@@ -190,6 +195,11 @@ windows10-64-shippable-qr/opt:
         - windows-talos
         - browsertime
 
+windows10-64-nightlyasrelease-qr/opt:
+    build-platform: win64-nightlyasrelease/opt
+    test-sets:
+        - browsertime
+
 # win10-64 2004
 windows10-64-2004-qr/debug:
     build-platform: win64/debug
@@ -322,6 +332,11 @@ macosx1015-64-shippable-qr/opt:
         - web-platform-tests
         - web-platform-tests-backlog
 
+macosx1015-64-nightlyasrelease-qr/opt:
+    build-platform: macosx64-nightlyasrelease/opt
+    test-sets:
+        - browsertime
+
 macosx1015-64-devedition-qr/opt:
     build-platform: macosx64-devedition/opt
     test-sets: