Bug 1575089 - Add a new Talos test that measures warm startup time with a number of real-world WebExtensions installed. r=rwood

Differential Revision: https://phabricator.services.mozilla.com/D43309

--HG--
extra : moz-landing-system : lando

.hgignore

@@ -156,6 +156,8 @@ _OPT\.OBJ/
 ^testing/talos/talos/tests/tp5n.tar.gz
 ^testing/talos/talos/tests/tp5n
 ^testing/talos/talos/tests/devtools/damp.manifest.develop
+^testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint.manifest.develop
+^testing/talos/talos/webextensions/
 ^talos-venv
 ^py3venv
 ^testing/talos/talos/mitmproxy/mitmdump

taskcluster/ci/test/talos.yml

@@ -405,3 +405,20 @@ talos-xperf:
             by-test-platform:
                 windows.*:
                     - talos/windows_vm_config.py
+
+talos-realworld-webextensions:
+    description: "Talos tests with realworld WebExtensions"
+    try-name: "realworld-webextensions"
+    treeherder-symbol: T(realwebext)
+    run-on-projects:
+        by-test-platform:
+            linux64-ccov/.*: ['try']
+            windows10-64-ccov/.*: ['try']
+            (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
+            windows10-aarch64/opt: ['mozilla-central', 'try']
+            macosx.*64(?:-shippable)?(?:-qr)?/opt: ['mozilla-central', 'try']
+            default: ['mozilla-beta', 'trunk', 'try']
+    max-run-time: 1500
+    mozharness:
+        extra-options:
+            - --suite=realworld-webextensions

taskcluster/ci/test/test-sets.yml

@@ -73,6 +73,7 @@ talos:
     - talos-perf-reftest
     - talos-perf-reftest-singletons
     - talos-tabswitch
+    - talos-realworld-webextensions
     # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions
     # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions
 
@@ -321,6 +322,7 @@ windows-talos:
     - talos-tp5o
     - talos-xperf
     - talos-tabswitch
+    - talos-realworld-webextensions
     # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions
 
 marionette-gpu-tests:
@@ -379,6 +381,7 @@ macosx64-talos:
     - talos-tp5o
     - talos-perf-reftest
     - talos-perf-reftest-singletons
+    - talos-realworld-webextensions
     # - talos-tabswitch # Bug 1453007 times out
     # - talos-h1 # too long to unpack profile- Bug 1442893
 

testing/mozharness/mozharness/mozilla/testing/talos.py

@@ -207,6 +207,7 @@ class Talos(TestingMixin, MercurialScript, TooltoolMixin,
         self.gecko_profile_interval = self.config.get('gecko_profile_interval')
         self.pagesets_name = None
         self.benchmark_zip = None
+        self.webextensions_zip = None
 
     # We accept some configuration options from the try commit message in the format
     # mozharness: <options>
@@ -265,6 +266,18 @@ class Talos(TestingMixin, MercurialScript, TooltoolMixin,
             self.benchmark_zip_manifest = 'jetstream-benchmark.manifest'
             return self.benchmark_zip
 
+    def query_webextensions_zip(self):
+        """Certain suites require external WebExtension sets to be downloaded and
+        extracted.
+        """
+        if self.webextensions_zip:
+            return self.webextensions_zip
+        if self.query_talos_json_config() and self.suite is not None:
+            self.webextensions_zip = \
+                self.talos_json_config['suites'][self.suite].get('webextensions_zip')
+            self.webextensions_zip_manifest = 'webextensions.manifest'
+            return self.webextensions_zip
+
     def get_suite_from_test(self):
         """ Retrieve the talos suite name from a given talos test name."""
         # running locally, single test name provided instead of suite; go through tests and
@@ -367,31 +380,40 @@ class Talos(TestingMixin, MercurialScript, TooltoolMixin,
             self.suite = self.config['suite']
 
         tooltool_artifacts = []
+        src_talos_pageset_dest = os.path.join(self.talos_path, 'talos', 'tests')
+        webextension_dest = os.path.join(self.talos_path, 'talos', 'webextensions')
+
         if self.query_pagesets_name():
             tooltool_artifacts.append({'name': self.pagesets_name,
-                                       'manifest': self.pagesets_name_manifest})
+                                       'manifest': self.pagesets_name_manifest,
+                                       'dest': src_talos_pageset_dest})
 
         if self.query_benchmark_zip():
             tooltool_artifacts.append({'name': self.benchmark_zip,
-                                       'manifest': self.benchmark_zip_manifest})
+                                       'manifest': self.benchmark_zip_manifest,
+                                       'dest': src_talos_pageset_dest})
+
+        if self.query_webextensions_zip():
+            tooltool_artifacts.append({'name': self.webextensions_zip,
+                                       'manifest': self.webextensions_zip_manifest,
+                                       'dest': webextension_dest})
 
         # now that have the suite name, check if artifact is required, if so download it
         # the --no-download option will override this
         for artifact in tooltool_artifacts:
             if '--no-download' not in self.config.get('talos_extra_options', []):
                 self.info("Downloading %s with tooltool..." % artifact)
-                self.src_talos_webdir = os.path.join(self.talos_path, 'talos')
-                src_talos_pageset = os.path.join(self.src_talos_webdir, 'tests')
-                if not os.path.exists(os.path.join(src_talos_pageset, artifact['name'])):
+
+                if not os.path.exists(os.path.join(artifact['dest'], artifact['name'])):
                     manifest_file = os.path.join(self.talos_path, artifact['manifest'])
                     self.tooltool_fetch(
                         manifest_file,
-                        output_dir=src_talos_pageset,
+                        output_dir=artifact['dest'],
                         cache=self.config.get('tooltool_cache')
                     )
-                    archive = os.path.join(src_talos_pageset, artifact['name'])
+                    archive = os.path.join(artifact['dest'], artifact['name'])
                     unzip = self.query_exe('unzip')
-                    unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
+                    unzip_cmd = [unzip, '-q', '-o', archive, '-d', artifact['dest']]
                     self.run_command(unzip_cmd, halt_on_failure=True)
                 else:
                     self.info("%s already available" % artifact)

testing/talos/talos.json

@@ -67,6 +67,10 @@
         },
         "h1": {
             "tests": ["ts_paint_heavy"]
+        },
+        "realworld-webextensions": {
+            "tests": ["startup_about_home_paint_realworld_webextensions"],
+            "webextensions_zip": "webextensions.zip"
         }
     }
 }

testing/talos/talos/ffsetup.py

@@ -151,13 +151,21 @@ class FFSetup(object):
         profile.addons.install(extensions)
 
         # installing webextensions
+        webextensions_to_install = []
+        webextensions_folder = self.test_config.get('webextensions_folder', None)
+        if isinstance(webextensions_folder, basestring):
+            folder = utils.interpolate(webextensions_folder)
+            for file in os.listdir(folder):
+                if file.endswith(".xpi"):
+                    webextensions_to_install.append(os.path.join(folder, file))
+
         webextensions = self.test_config.get('webextensions', None)
         if isinstance(webextensions, basestring):
-            webextensions = [webextensions]
+            webextensions_to_install.append(webextensions)
 
-        if webextensions is not None:
+        if webextensions_to_install is not None:
             LOG.info("Installing Webextensions:")
-            for webext in webextensions:
+            for webext in webextensions_to_install:
                 filename = utils.interpolate(webext)
                 if mozinfo.os == 'win':
                     filename = filename.replace('/', '\\')

testing/talos/talos/getinfooffline/api.js

@@ -0,0 +1,17 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80 filetype=javascript: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* globals ExtensionAPI */
+
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+this.getinfooffline = class extends ExtensionAPI {
+  onStartup() {
+    Services.io.offline = true;
+  }
+};

testing/talos/talos/getinfooffline/background.js

@@ -0,0 +1,6 @@
+// We've run once, and hopefully bypassed any network connections that add-ons
+// might have tried to make after first install. Now let's uninstall ourselves
+// so that subsequent starts run in online mode.
+
+/* globals browser */
+browser.management.uninstallSelf({});

testing/talos/talos/getinfooffline/manifest.json

@@ -0,0 +1,27 @@
+{
+  "manifest_version": 2,
+  "name": "Make the getinfo step happen offline for Talos",
+  "description": "Puts the browser in offline mode during the getinfo step and then uninstalls itself so that the browser runs in online mode for subsequent starts. This is useful when running startup tests that instal add-ons that like to open up network requests after being installed for the first time, which normally would cause the test runner to crash.",
+  "version": "0.1",
+
+  "applications": {
+    "gecko": {
+      "id": "getinfooffline@mozilla.org"
+    }
+  },
+
+  "background": {
+    "scripts": ["background.js"]
+  },
+
+  "experiment_apis": {
+    "getinfooffline": {
+      "schema": "schema.json",
+      "parent": {
+        "scopes": ["addon_parent"],
+        "script": "api.js",
+        "events": ["startup"]
+      }
+    }
+  }
+}

testing/talos/talos/getinfooffline/schema.json

@@ -0,0 +1 @@
+[]

testing/talos/talos/test.py

@@ -119,6 +119,7 @@ class TsBase(Test):
         'setup',
         'cleanup',
         'webextensions',
+        'webextensions_folder',
         'reinstall',     # A list of files from the profile directory that
                          # should be copied to the temporary profile prior to
                          # running each cycle, to avoid one cycle overwriting
@@ -177,6 +178,18 @@ class startup_about_home_paint(ts_paint):
     tpmanifest = '${talos}/startup_test/startup_about_home_paint/startup_about_home_paint.manifest'
 
 
+@register_test()
+class startup_about_home_paint_realworld_webextensions(ts_paint):
+    url = None
+    cycles = 20
+    extensions = [
+        '${talos}/startup_test/startup_about_home_paint/addon',
+        '${talos}/getinfooffline'
+    ]
+    tpmanifest = '${talos}/startup_test/startup_about_home_paint/startup_about_home_paint.manifest'
+    webextensions_folder = '${talos}/webextensions'
+
+
 @register_test()
 class sessionrestore(TsBase):
     """
@@ -253,7 +266,7 @@ class PageloaderTest(Test):
             'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
             'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup',
             'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile',
-            'subtest_alerts', 'perfherder_framework', 'pdfpaint']
+            'subtest_alerts', 'perfherder_framework', 'pdfpaint', 'webextensions_folder']
 
 
 class QuantumPageloadTest(PageloaderTest):

testing/talos/webextensions.manifest

@@ -0,0 +1,9 @@
+[
+  {
+    "size": 2608427,
+    "visibility": "public",
+    "digest": "f3198257ca82043bb9fe9a7085cff7af6f0d11685a0416ebb73e8edec547b970dd044666120e44898e2de36372add88ef93cd08af79dcd03e32fa1e9ef2053c8",
+    "algorithm": "sha512",
+    "filename": "webextensions.zip"
+  }
+]