Bug 1503990 add additional pages to tp6 r=rwood

Differential Revision: https://phabricator.services.mozilla.com/D11058 --HG-- extra : moz-landing-system : lando
2018-11-20 01:21:38 +00:00 · 2018-11-20 01:21:38 +00:00 · 8ef2b84cb6
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@ -85,6 +85,110 @@ raptor-tp6-2-chrome:
            - --test=raptor-tp6-2
            - --app=chrome

+raptor-tp6-3-firefox:
+    description: "Raptor tp6-3 on Firefox"
+    try-name: raptor-tp6-3-firefox
+    treeherder-symbol: Rap(tp6-3)
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 2
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-3
+
+raptor-tp6-3-chrome:
+    description: "Raptor tp6-3 on Chrome"
+    try-name: raptor-tp6-3-chrome
+    treeherder-symbol: Rap-C(tp6-3)
+    run-on-projects: ['try', 'mozilla-central']
+    tier:
+        by-test-platform:
+            linux64.*: 3
+            default: 2
+    max-run-time: 2400
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-3
+            - --app=chrome
+
+raptor-tp6-4-firefox:
+    description: "Raptor tp6-4 on Firefox"
+    try-name: raptor-tp6-4-firefox
+    treeherder-symbol: Rap(tp6-4)
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 2
+    max-run-time: 1200
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-4
+
+raptor-tp6-4-chrome:
+    description: "Raptor tp6-4 on Chrome"
+    try-name: raptor-tp6-4-chrome
+    treeherder-symbol: Rap-C(tp6-4)
+    run-on-projects: ['try', 'mozilla-central']
+    tier:
+        by-test-platform:
+            linux64.*: 3
+            default: 2
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-4
+            - --app=chrome
+
+raptor-tp6-5-firefox:
+    description: "Raptor tp6-5 on Firefox"
+    try-name: raptor-tp6-5-firefox
+    treeherder-symbol: Rap(tp6-5)
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 2
+    max-run-time: 1200
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-5
+
+raptor-tp6-5-chrome:
+    description: "Raptor tp6-5 on Chrome"
+    try-name: raptor-tp6-5-chrome
+    treeherder-symbol: Rap-C(tp6-5)
+    run-on-projects: ['try', 'mozilla-central']
+    tier:
+        by-test-platform:
+            linux64.*: 3
+            default: 2
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-5
+            - --app=chrome
+
+raptor-tp6-6-firefox:
+    description: "Raptor tp6-6 on Firefox"
+    try-name: raptor-tp6-6-firefox
+    treeherder-symbol: Rap(tp6-6)
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 2
+    max-run-time: 1200
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-6
+
+raptor-tp6-6-chrome:
+    description: "Raptor tp6-6 on Chrome"
+    try-name: raptor-tp6-6-chrome
+    treeherder-symbol: Rap-C(tp6-6)
+    run-on-projects: ['try', 'mozilla-central']
+    tier:
+        by-test-platform:
+            linux64.*: 3
+            default: 2
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-6
+            - --app=chrome
+
 raptor-speedometer-firefox:
    description: "Raptor Speedometer on Firefox"
    try-name: raptor-speedometer-firefox
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@ -79,6 +79,10 @@ talos:
 raptor-firefox:
    - raptor-tp6-1-firefox
    - raptor-tp6-2-firefox
+    - raptor-tp6-3-firefox
+    - raptor-tp6-4-firefox
+    - raptor-tp6-5-firefox
+    - raptor-tp6-6-firefox
    - raptor-speedometer-firefox
    - raptor-stylebench-firefox
    - raptor-motionmark-htmlsuite-firefox
@ -90,6 +94,10 @@ raptor-firefox:
 raptor-chrome:
    - raptor-tp6-1-chrome
    - raptor-tp6-2-chrome
+    - raptor-tp6-3-chrome
+    - raptor-tp6-4-chrome
+    - raptor-tp6-5-chrome
+    - raptor-tp6-6-chrome
    - raptor-speedometer-chrome
    - raptor-stylebench-chrome
    - raptor-motionmark-htmlsuite-chrome
--- a/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-3.manifest
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-3.manifest
@ -0,0 +1,9 @@
+[
+    {
+        "filename": "raptor-tp6-3.zip",
+        "size": 24512088,
+        "digest": "5b8080cf842a50fe2055127daa084fa9f133307965e4d8af246978150b8b3b11f1fb06cdf65ba69e13c875b77db14e7494cdb940376b264c3aefb5b53e22b892",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]
--- a/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-4.manifest
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-4.manifest
@ -0,0 +1,9 @@
+[
+    {
+        "filename": "raptor-tp6-4.zip",
+        "size": 1643188,
+        "digest": "0e9f2a23323f93f7a9839ab49d84555cce03bfab2196aa8670e604d6df390b22f0594527cecf7b2efd0449dd87507ec934bdcc74f1ebee68c22161f4104b6513",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]
--- a/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-5.manifest
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-5.manifest
@ -0,0 +1,9 @@
+[
+    {
+        "filename": "raptor-tp6-5.zip",
+        "size": 27670268,
+        "digest": "4a52cb6770062231f9f283ab42ac53634d91677146b4d91931f508de22bff262512775d0c93d80730287000907e438d467136768883691c3e20c1e6a8f475a03",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]
--- a/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-6.manifest
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-6.manifest
@ -0,0 +1,9 @@
+[
+    {
+        "filename": "raptor-tp6-6.zip",
+        "size": 2240008,
+        "digest": "21110ee297074413a72343bcb7d22ae88efce104de6e7eea5916e2dae004b29746ec6dd6155686cffed816e88d38ad1bbc04fe1253623ed8c0d4256a146d8b77",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@ -1,6 +1,10 @@
 # raptor pageload tests
 [include:tests/raptor-tp6-1.ini]
 [include:tests/raptor-tp6-2.ini]
+[include:tests/raptor-tp6-3.ini]
+[include:tests/raptor-tp6-4.ini]
+[include:tests/raptor-tp6-5.ini]
+[include:tests/raptor-tp6-6.ini]

 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
--- a/testing/raptor/raptor/tests/raptor-tp6-3.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-3.ini
@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-3
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-3.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
+gecko_profile_interval = 1
+gecko_profile_entries = 2000000
+
+[raptor-tp6-imdb-firefox]
+apps = firefox
+test_url = https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2
+playback_recordings = imdb.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-imgur-firefox]
+apps = firefox
+test_url = https://imgur.com/gallery/m5tYJL6
+playback_recordings = imgur.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-wikia-firefox]
+apps = firefox
+test_url = http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase
+playback_recordings = wikia.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-imdb-chrome]
+apps = chrome
+test_url = https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2
+playback_recordings = imdb.mp
+measure = fcp
+
+[raptor-tp6-imgur-chrome]
+apps = chrome
+test_url = https://imgur.com/gallery/m5tYJL6
+playback_recordings = imgur.mp
+measure = fcp
+
+[raptor-tp6-wikia-chrome]
+apps = chrome
+test_url = http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase
+playback_recordings = wikia.mp
+measure = fcp
--- a/testing/raptor/raptor/tests/raptor-tp6-4.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-4.ini
@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-4
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-4.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
+gecko_profile_interval = 1
+gecko_profile_entries = 2000000
+
+[raptor-tp6-bing-firefox]
+apps = firefox
+test_url = https://www.bing.com/search?q=barack+obama
+playback_recordings = bing.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-yandex-firefox]
+apps = firefox
+test_url = https://yandex.ru/search/?text=barack%20obama&lr=10115
+playback_recordings = yandex.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-bing-chrome]
+apps = chrome
+test_url = https://www.bing.com/search?q=barack+obama
+playback_recordings = bing.mp
+measure = fcp
+
+[raptor-tp6-yandex-chrome]
+apps = chrome
+test_url = https://yandex.ru/search/?text=barack%20obama&lr=10115
+playback_recordings = yandex.mp
+measure = fcp
--- a/testing/raptor/raptor/tests/raptor-tp6-5.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-5.ini
@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-5
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-5.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
+gecko_profile_interval = 1
+gecko_profile_entries = 2000000
+
+[raptor-tp6-apple-firefox]
+apps = firefox
+test_url = https://www.apple.com/macbook-pro/
+playback_recordings = apple.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-microsoft-firefox]
+apps = firefox
+test_url = https://www.microsoft.com/en-us/windows/get-windows-10
+page_timeout = 60000
+playback_recordings = microsoft.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-apple-chrome]
+apps = chrome
+test_url = https://www.apple.com/macbook-pro/
+playback_recordings = apple.mp
+measure = fcp
+
+[raptor-tp6-microsoft-chrome]
+apps = chrome
+test_url = https://www.microsoft.com/en-us/windows/get-windows-10
+playback_recordings = microsoft.mp
+measure = fcp
--- a/testing/raptor/raptor/tests/raptor-tp6-6.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6-6.ini
@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-6
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-6.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
+gecko_profile_interval = 1
+gecko_profile_entries = 2000000
+
+[raptor-tp6-reddit-firefox]
+apps = firefox
+test_url = https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/
+playback_recordings = reddit.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-reddit-chrome]
+apps = chrome
+test_url = https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/
+playback_recordings = reddit.mp
+measure = fcp
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@ -13,10 +13,22 @@
  },
  "content_scripts": [
    {
-      "matches": ["*://*.amazon.com/*",
+      "matches": [
+                  "*://*.apple.com/*",
+                  "*://*.amazon.com/*",
+                  "*://*.bing.com/*",
                  "*://*.facebook.com/*",
                  "*://*.google.com/*",
-                  "*://*.youtube.com/*"],
+                  "*://*.imdb.com/*",
+                  "*://*.imgur.com/*",
+                  "*://*.microsoft.com/*",
+                  "*://*.reddit.com/*",
+                  "*://*.vice.com/*",
+                  "*://*.wikia.com/*",
+                  "*://*.wikipedia.org/*",
+                  "*://*.youtube.com/*",
+                  "*://*.yandex.ru/*"
+                  ],
      "js": ["measure.js"]
    },
    {