Bug 1522972 - Raptor cold-load tests on Firefox and Chromium desktop; #perftest r=davehunt

Differential Revision: https://phabricator.services.mozilla.com/D29032

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/raptor-chromium.yml

@@ -272,3 +272,39 @@ raptor-wasm-godot-chromium:
         extra-options:
             - --test=raptor-wasm-godot
             - --app=chromium
+
+raptor-tp6-1-chromium-cold:
+    description: "Raptor tp6-1 cold page-load on Chromium"
+    try-name: raptor-tp6-1-chromium-cold
+    treeherder-symbol: Rap-Cr(tp6-c-1)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-1
+            - --app=chromium
+
+raptor-tp6-2-chromium-cold:
+    description: "Raptor tp6-2 cold page-load on Chromium"
+    try-name: raptor-tp6-2-chromium-cold
+    treeherder-symbol: Rap-Cr(tp6-c-2)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-2
+            - --app=chromium
+
+raptor-tp6-3-chromium-cold:
+    description: "Raptor tp6-3 cold page-load on Chromium"
+    try-name: raptor-tp6-3-chromium-cold
+    treeherder-symbol: Rap-Cr(tp6-c-3)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-3
+            - --app=chromium
+
+raptor-tp6-4-chromium-cold:
+    description: "Raptor tp6-4 cold page-load on Chromium"
+    try-name: raptor-tp6-4-chromium-cold
+    treeherder-symbol: Rap-Cr(tp6-c-4)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-4
+            - --app=chromium

taskcluster/ci/test/raptor.yml

@@ -1471,3 +1471,39 @@ raptor-wasm-godot-ion-firefox-profiling:
         extra-options:
             - --test=raptor-wasm-godot-ion
             - --gecko-profile
+
+raptor-tp6-1-firefox-cold:
+    description: "Raptor tp6-1 cold page-load on Firefox"
+    try-name: raptor-tp6-1-firefox-cold
+    treeherder-symbol: Rap(tp6-c-1)
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-1
+
+raptor-tp6-2-firefox-cold:
+    description: "Raptor tp6-2 cold page-load on Firefox"
+    try-name: raptor-tp6-2-firefox-cold
+    treeherder-symbol: Rap(tp6-c-2)
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-2
+
+raptor-tp6-3-firefox-cold:
+    description: "Raptor tp6-3 cold page-load on Firefox"
+    try-name: raptor-tp6-3-firefox-cold
+    treeherder-symbol: Rap(tp6-c-3)
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-3
+
+raptor-tp6-4-firefox-cold:
+    description: "Raptor tp6-4 cold page-load on Firefox"
+    try-name: raptor-tp6-4-firefox-cold
+    treeherder-symbol: Rap(tp6-c-4)
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-4

taskcluster/ci/test/test-sets.yml

@@ -96,6 +96,10 @@ raptor-firefox:
     - raptor-webaudio-firefox
     - raptor-sunspider-firefox
     - raptor-wasm-godot-firefox
+    - raptor-tp6-1-firefox-cold
+    - raptor-tp6-2-firefox-cold
+    - raptor-tp6-3-firefox-cold
+    - raptor-tp6-4-firefox-cold
 
 raptor-profiling:
     - raptor-tp6-1-firefox-profiling
@@ -134,6 +138,10 @@ raptor-chromium:
     - raptor-webaudio-chromium
     - raptor-sunspider-chromium
     - raptor-wasm-godot-chromium
+    - raptor-tp6-1-chromium-cold
+    - raptor-tp6-2-chromium-cold
+    - raptor-tp6-3-chromium-cold
+    - raptor-tp6-4-chromium-cold
 
 # Fetch tasks are only supported on Linux for now,
 # so these need to be separate sets.

testing/raptor/raptor/raptor.ini

@@ -1,4 +1,4 @@
-# raptor pageload tests desktop
+# raptor warm pageload tests desktop
 [include:tests/raptor-tp6-1.ini]
 [include:tests/raptor-tp6-2.ini]
 [include:tests/raptor-tp6-3.ini]
@@ -10,6 +10,12 @@
 [include:tests/raptor-tp6-9.ini]
 [include:tests/raptor-tp6-10.ini]
 
+# raptor cold pageload tests desktop
+[include:tests/raptor-tp6-cold-1.ini]
+[include:tests/raptor-tp6-cold-2.ini]
+[include:tests/raptor-tp6-cold-3.ini]
+[include:tests/raptor-tp6-cold-4.ini]
+
 # raptor pageload binast tests desktop
 [include:tests/raptor-tp6-binast-1.ini]
 

testing/raptor/raptor/raptor.py

@@ -436,6 +436,69 @@ class RaptorDesktop(Raptor):
         self.control_server.browser_proc = proc
 
     def run_test(self, test, timeout=None):
+        # tests will be run warm (i.e. NO browser restart between page-cycles)
+        # unless otheriwse specified in the test INI by using 'cold = true'
+        if test.get('cold', False) is True:
+            self.run_test_cold(test, timeout)
+        else:
+            self.run_test_warm(test, timeout)
+
+    def run_test_cold(self, test, timeout=None):
+        '''
+        Run the Raptor test but restart the entire browser app between page-cycles.
+
+        Note: For page-load tests, playback will only be started once - at the beginning of all
+        browser cycles, and then stopped after all cycles are finished. That includes the import
+        of the mozproxy ssl cert and turning on the browser proxy.
+
+        Since we're running in cold-mode, before this point (in manifest.py) the
+        'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
+        and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
+        browser restart.
+
+        The 'browser-cycle' value is the current overall browser start iteration. The control
+        server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
+        each results set received; and will pass that on as part of the results so that the
+        results processing will know results for multiple browser cycles are being received.
+
+        The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
+        '''
+        self.log.info("test %s is running in cold mode; browser WILL be restarted between "
+                      "page cycles" % test['name'])
+
+        for test['browser_cycle'] in range(1, test['expected_browser_cycles'] + 1):
+
+            self.log.info("begin browser cycle %d of %d for test %s"
+                          % (test['browser_cycle'], test['expected_browser_cycles'], test['name']))
+
+            self.run_test_setup(test)
+
+            if test['browser_cycle'] == 1:
+
+                if test.get('playback') is not None:
+                    self.start_playback(test)
+
+                if self.config['host'] not in ('localhost', '127.0.0.1'):
+                    self.delete_proxy_settings_from_profile()
+
+            else:
+                # initial browser profile was already created before run_test was called;
+                # now additional browser cycles we want to create a new one each time
+                self.create_browser_profile()
+
+                self.run_test_setup(test)
+
+            # now start the browser/app under test
+            self.launch_desktop_browser(test)
+
+            # set our control server flag to indicate we are running the browser/app
+            self.control_server._finished = False
+
+            self.wait_for_test_finish(test, timeout)
+
+        self.run_test_teardown()
+
+    def run_test_warm(self, test, timeout=None):
         self.run_test_setup(test)
 
         try:

testing/raptor/raptor/tests/raptor-tp6-cold-1.ini

@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cold-1
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
+browser_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6-amazon-firefox-cold]
+apps = firefox
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = amazon.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-amazon-chromium-cold]
+apps = chromium
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = amazon.mp
+measure = fcp, loadtime
+
+[raptor-tp6-amazon-chrome-cold]
+apps = chrome
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = amazon.mp
+measure = fcp, loadtime

testing/raptor/raptor/tests/raptor-tp6-cold-2.ini

@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cold-2
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
+browser_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6-facebook-firefox-cold]
+apps = firefox
+test_url = https://www.facebook.com
+playback_recordings = facebook.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-facebook-chromium-cold]
+apps = chromium
+test_url = https://www.facebook.com
+playback_recordings = facebook.mp
+measure = fcp, loadtime
+
+[raptor-tp6-facebook-chrome-cold]
+apps = chrome
+test_url = https://www.facebook.com
+playback_recordings = facebook.mp
+measure = fcp, loadtime

testing/raptor/raptor/tests/raptor-tp6-cold-3.ini

@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cold-3
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
+browser_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6-google-firefox-cold]
+apps = firefox
+# note: use the full url as the first part (without '&cad=h') redirects
+# to the url + '&cad=h'; that redirection causes measure.js content
+# to be loaded into that page also; resulting in 2 fnbpaint values etc.
+test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
+playback_recordings = google-search.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-google-chromium-cold]
+apps = chromium
+test_url = https://www.google.com/#hl=en&q=barack+obama
+playback_recordings = google-search.mp
+measure = fcp, loadtime
+
+[raptor-tp6-google-chrome-cold]
+apps = chrome
+test_url = https://www.google.com/#hl=en&q=barack+obama
+playback_recordings = google-search.mp
+measure = fcp, loadtime

testing/raptor/raptor/tests/raptor-tp6-cold-4.ini

@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cold-4
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
+browser_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6-youtube-firefox-cold]
+apps = firefox
+test_url = https://www.youtube.com
+playback_recordings = youtube.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-youtube-chromium-cold]
+apps = chromium
+test_url = https://www.youtube.com
+playback_recordings = youtube.mp
+measure = fcp, loadtime
+
+[raptor-tp6-youtube-chrome-cold]
+apps = chrome
+test_url = https://www.youtube.com
+playback_recordings = youtube.mp
+measure = fcp, loadtime