Bug 1567177 - Create cold tests for cnn mobile website. r=perftest-reviewers,sparky

The new cold cnn mobile tests can be found in 'raptor-tp6m-cold-27.ini'.

Differential Revision: https://phabricator.services.mozilla.com/D38671

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/raptor.yml

@@ -1536,6 +1536,28 @@ raptor-tp6m-26-fenix-cold:
             - --binary-path=org.mozilla.fenix.performancetest
             - --activity=org.mozilla.fenix.IntentReceiverActivity
 
+raptor-tp6m-27-fenix-cold:
+    description: "Raptor tp6m-27 cold page-load on Fenix"
+    try-name: raptor-tp6m-27-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-27)
+    run-on-projects: []
+    e10s: true
+    target:
+        by-test-platform:
+            android-hw.*-aarch64.*/.*:
+                index: project.mobile.fenix.v2.performance-test.latest.aarch64
+                name: target.apk
+            default:
+                index: project.mobile.fenix.v2.performance-test.latest.arm
+                name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-27
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix.performancetest
+            - --activity=org.mozilla.fenix.IntentReceiverActivity
+
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     variants: ["fission"]
@@ -2888,6 +2910,28 @@ raptor-tp6m-26-geckoview-cold:
             - --binary=org.mozilla.geckoview_example
             - --activity=org.mozilla.geckoview_example.GeckoViewActivity
 
+raptor-tp6m-27-geckoview-cold:
+    description: "Raptor tp6m-27 cold page-load on GeckoView Example"
+    max-run-time: 2700
+    try-name: raptor-tp6m-27-geckoview-cold
+    treeherder-symbol: Rap(tp6m-c-27)
+    run-on-projects:
+        by-test-platform:
+            android-hw-.*/opt: ['mozilla-central', 'comm-central']
+            android-hw-p2-.*-api-16/pgo: ['trunk']
+            android-hw-p2-.*aarch64.*/pgo: ['trunk', 'mozilla-beta']
+            android-hw-g5.*/pgo: ['trunk', 'mozilla-beta']
+            default: ['trunk', 'mozilla-beta']
+    e10s: true
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-27
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+            - --activity=org.mozilla.geckoview_example.GeckoViewActivity
+
 raptor-tp6m-1-fennec-cold:
     description: "Raptor tp6m-1 cold page-load on Fennec"
     max-run-time: 2700
@@ -3252,6 +3296,20 @@ raptor-tp6m-26-fennec-cold:
             - --app=fennec
             - --binary=org.mozilla.fennec_aurora
 
+raptor-tp6m-27-fennec-cold:
+    description: "Raptor tp6m-27 cold page-load on Fennec"
+    max-run-time: 2700
+    try-name: raptor-tp6m-27-fennec-cold
+    treeherder-symbol: Rap(tp6m-c-27)
+    target: target.apk
+    run-on-projects: []
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-27
+            - --app=fennec
+            - --binary=org.mozilla.fennec_aurora
+
 raptor-tp6m-1-fennec64:
     description: "Raptor tp6m-1 on Fennec64"
     max-run-time: 2700
@@ -4330,6 +4388,28 @@ raptor-tp6m-26-refbrow-cold:
             - --binary-path=org.mozilla.reference.browser.raptor
             - --activity=org.mozilla.reference.browser.BrowserTestActivity
 
+raptor-tp6m-27-refbrow-cold:
+    description: "Raptor tp6m-27 cold page-load on the reference browser"
+    try-name: raptor-tp6m-27-refbrow-cold
+    treeherder-symbol: Rap-refbrow(tp6m-c-27)
+    run-on-projects: []
+    e10s: true
+    target:
+        by-test-platform:
+            android-hw.*-aarch64.*/.*:
+                index: project.mobile.reference-browser.v2.raptor.latest.aarch64
+                name: target.apk
+            default:
+                index: project.mobile.reference-browser.v2.raptor.latest.arm
+                name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-27
+            - --app=refbrow
+            - --binary-path=org.mozilla.reference.browser.raptor
+            - --activity=org.mozilla.reference.browser.BrowserTestActivity
+
 raptor-ares6-firefox:
     description: "Raptor Ares6 on Firefox"
     variants: ["fission"]

taskcluster/ci/test/test-sets.yml

@@ -500,6 +500,7 @@ android-hw-arm7-raptor:
     - raptor-tp6m-24-fenix-cold
     - raptor-tp6m-25-fenix-cold
     - raptor-tp6m-26-fenix-cold
+    - raptor-tp6m-27-fenix-cold
     - raptor-tp6m-1-geckoview-cold
     - raptor-tp6m-2-geckoview-cold
     - raptor-tp6m-3-geckoview-cold
@@ -526,6 +527,7 @@ android-hw-arm7-raptor:
     - raptor-tp6m-24-geckoview-cold
     - raptor-tp6m-25-geckoview-cold
     - raptor-tp6m-26-geckoview-cold
+    - raptor-tp6m-27-geckoview-cold
     - raptor-tp6m-1-fennec64
     - raptor-tp6m-2-fennec64
     - raptor-tp6m-3-fennec64
@@ -562,6 +564,7 @@ android-hw-arm7-raptor:
     - raptor-tp6m-24-fennec-cold
     - raptor-tp6m-25-fennec-cold
     - raptor-tp6m-26-fennec-cold
+    - raptor-tp6m-27-fennec-cold
     - raptor-tp6m-1-fennec64-cold
     - raptor-tp6m-2-fennec64-cold
     - raptor-tp6m-3-fennec64-cold
@@ -602,6 +605,7 @@ android-hw-arm7-raptor:
     - raptor-tp6m-24-refbrow-cold
     - raptor-tp6m-25-refbrow-cold
     - raptor-tp6m-26-refbrow-cold
+    - raptor-tp6m-27-refbrow-cold
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
@@ -664,6 +668,7 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-24-fenix-cold
     - raptor-tp6m-25-fenix-cold
     - raptor-tp6m-26-fenix-cold
+    - raptor-tp6m-27-fenix-cold
     - raptor-tp6m-1-geckoview-cold
     - raptor-tp6m-2-geckoview-cold
     - raptor-tp6m-3-geckoview-cold
@@ -690,6 +695,7 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-24-geckoview-cold
     - raptor-tp6m-25-geckoview-cold
     - raptor-tp6m-26-geckoview-cold
+    - raptor-tp6m-27-geckoview-cold
     - raptor-tp6m-1-fennec-cold
     - raptor-tp6m-2-fennec-cold
     - raptor-tp6m-3-fennec-cold
@@ -716,6 +722,7 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-24-fennec-cold
     - raptor-tp6m-25-fennec-cold
     - raptor-tp6m-26-fennec-cold
+    - raptor-tp6m-27-fennec-cold
     - raptor-tp6m-1-fennec64-cold
     - raptor-tp6m-2-fennec64-cold
     - raptor-tp6m-3-fennec64-cold
@@ -756,6 +763,7 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-24-refbrow-cold
     - raptor-tp6m-25-refbrow-cold
     - raptor-tp6m-26-refbrow-cold
+    - raptor-tp6m-27-refbrow-cold
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power

testing/raptor/raptor/raptor.ini

@@ -68,6 +68,7 @@
 [include:tests/tp6/mobile/raptor-tp6m-cold-24.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-25.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-26.ini]
+[include:tests/tp6/mobile/raptor-tp6m-cold-27.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-1-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-2-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-3-fennec64.ini]

testing/raptor/raptor/tests/tp6/mobile/raptor-tp6m-cold-27.ini

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-27
+
+[DEFAULT]
+type =  pageload
+test_url = https://edition.cnn.com
+playback = mitmproxy-android
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitm4-motog5-gve-cnn.manifest
+playback_recordings = android-cnn.mp
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-cnn-fenix-cold]
+apps = fenix
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-cnn-geckoview-cold]
+apps = geckoview
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-cnn-fennec-cold]
+apps = fennec
+measure = fnbpaint, dcf, loadtime
+alert_on = loadtime
+
+[raptor-tp6m-cnn-refbrow-cold]
+apps = refbrow
+measure = fnbpaint, fcp, dcf, loadtime