From bf3a67ed9b8ebecf299e9afea525c5bffab14892 Mon Sep 17 00:00:00 2001 From: Mike Conley Date: Sat, 31 Aug 2019 06:58:00 +0000 Subject: [PATCH] Bug 1575089 - Add a new Talos test that measures warm startup time with a number of real-world WebExtensions installed. r=rwood Differential Revision: https://phabricator.services.mozilla.com/D43309 --HG-- extra : moz-landing-system : lando --- .hgignore | 2 + taskcluster/ci/test/talos.yml | 17 +++++++++ taskcluster/ci/test/test-sets.yml | 3 ++ .../mozharness/mozilla/testing/talos.py | 38 +++++++++++++++---- testing/talos/talos.json | 4 ++ testing/talos/talos/ffsetup.py | 14 +++++-- testing/talos/talos/getinfooffline/api.js | 17 +++++++++ .../talos/talos/getinfooffline/background.js | 6 +++ .../talos/talos/getinfooffline/manifest.json | 27 +++++++++++++ .../talos/talos/getinfooffline/schema.json | 1 + testing/talos/talos/test.py | 15 +++++++- testing/talos/webextensions.manifest | 9 +++++ 12 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 testing/talos/talos/getinfooffline/api.js create mode 100644 testing/talos/talos/getinfooffline/background.js create mode 100644 testing/talos/talos/getinfooffline/manifest.json create mode 100644 testing/talos/talos/getinfooffline/schema.json create mode 100644 testing/talos/webextensions.manifest diff --git a/.hgignore b/.hgignore index 30a799d14248..9cd9e5ebe227 100644 --- a/.hgignore +++ b/.hgignore @@ -156,6 +156,8 @@ _OPT\.OBJ/ ^testing/talos/talos/tests/tp5n.tar.gz ^testing/talos/talos/tests/tp5n ^testing/talos/talos/tests/devtools/damp.manifest.develop +^testing/talos/talos/startup_test/startup_about_home_paint/startup_about_home_paint.manifest.develop +^testing/talos/talos/webextensions/ ^talos-venv ^py3venv ^testing/talos/talos/mitmproxy/mitmdump diff --git a/taskcluster/ci/test/talos.yml b/taskcluster/ci/test/talos.yml index aa9d6fd3b278..2a5a71f537da 100644 --- a/taskcluster/ci/test/talos.yml +++ b/taskcluster/ci/test/talos.yml @@ -405,3 +405,20 @@ talos-xperf: by-test-platform: windows.*: - talos/windows_vm_config.py + +talos-realworld-webextensions: + description: "Talos tests with realworld WebExtensions" + try-name: "realworld-webextensions" + treeherder-symbol: T(realwebext) + run-on-projects: + by-test-platform: + linux64-ccov/.*: ['try'] + windows10-64-ccov/.*: ['try'] + (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try'] + windows10-aarch64/opt: ['mozilla-central', 'try'] + macosx.*64(?:-shippable)?(?:-qr)?/opt: ['mozilla-central', 'try'] + default: ['mozilla-beta', 'trunk', 'try'] + max-run-time: 1500 + mozharness: + extra-options: + - --suite=realworld-webextensions diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index d149f5b994c2..bb86257028ac 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -73,6 +73,7 @@ talos: - talos-perf-reftest - talos-perf-reftest-singletons - talos-tabswitch + - talos-realworld-webextensions # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions @@ -321,6 +322,7 @@ windows-talos: - talos-tp5o - talos-xperf - talos-tabswitch + - talos-realworld-webextensions # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions marionette-gpu-tests: @@ -379,6 +381,7 @@ macosx64-talos: - talos-tp5o - talos-perf-reftest - talos-perf-reftest-singletons + - talos-realworld-webextensions # - talos-tabswitch # Bug 1453007 times out # - talos-h1 # too long to unpack profile- Bug 1442893 diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py index 5bc3e6c02235..14e586da6333 100755 --- a/testing/mozharness/mozharness/mozilla/testing/talos.py +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -207,6 +207,7 @@ class Talos(TestingMixin, MercurialScript, TooltoolMixin, self.gecko_profile_interval = self.config.get('gecko_profile_interval') self.pagesets_name = None self.benchmark_zip = None + self.webextensions_zip = None # We accept some configuration options from the try commit message in the format # mozharness: @@ -265,6 +266,18 @@ class Talos(TestingMixin, MercurialScript, TooltoolMixin, self.benchmark_zip_manifest = 'jetstream-benchmark.manifest' return self.benchmark_zip + def query_webextensions_zip(self): + """Certain suites require external WebExtension sets to be downloaded and + extracted. + """ + if self.webextensions_zip: + return self.webextensions_zip + if self.query_talos_json_config() and self.suite is not None: + self.webextensions_zip = \ + self.talos_json_config['suites'][self.suite].get('webextensions_zip') + self.webextensions_zip_manifest = 'webextensions.manifest' + return self.webextensions_zip + def get_suite_from_test(self): """ Retrieve the talos suite name from a given talos test name.""" # running locally, single test name provided instead of suite; go through tests and @@ -367,31 +380,40 @@ class Talos(TestingMixin, MercurialScript, TooltoolMixin, self.suite = self.config['suite'] tooltool_artifacts = [] + src_talos_pageset_dest = os.path.join(self.talos_path, 'talos', 'tests') + webextension_dest = os.path.join(self.talos_path, 'talos', 'webextensions') + if self.query_pagesets_name(): tooltool_artifacts.append({'name': self.pagesets_name, - 'manifest': self.pagesets_name_manifest}) + 'manifest': self.pagesets_name_manifest, + 'dest': src_talos_pageset_dest}) if self.query_benchmark_zip(): tooltool_artifacts.append({'name': self.benchmark_zip, - 'manifest': self.benchmark_zip_manifest}) + 'manifest': self.benchmark_zip_manifest, + 'dest': src_talos_pageset_dest}) + + if self.query_webextensions_zip(): + tooltool_artifacts.append({'name': self.webextensions_zip, + 'manifest': self.webextensions_zip_manifest, + 'dest': webextension_dest}) # now that have the suite name, check if artifact is required, if so download it # the --no-download option will override this for artifact in tooltool_artifacts: if '--no-download' not in self.config.get('talos_extra_options', []): self.info("Downloading %s with tooltool..." % artifact) - self.src_talos_webdir = os.path.join(self.talos_path, 'talos') - src_talos_pageset = os.path.join(self.src_talos_webdir, 'tests') - if not os.path.exists(os.path.join(src_talos_pageset, artifact['name'])): + + if not os.path.exists(os.path.join(artifact['dest'], artifact['name'])): manifest_file = os.path.join(self.talos_path, artifact['manifest']) self.tooltool_fetch( manifest_file, - output_dir=src_talos_pageset, + output_dir=artifact['dest'], cache=self.config.get('tooltool_cache') ) - archive = os.path.join(src_talos_pageset, artifact['name']) + archive = os.path.join(artifact['dest'], artifact['name']) unzip = self.query_exe('unzip') - unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset] + unzip_cmd = [unzip, '-q', '-o', archive, '-d', artifact['dest']] self.run_command(unzip_cmd, halt_on_failure=True) else: self.info("%s already available" % artifact) diff --git a/testing/talos/talos.json b/testing/talos/talos.json index e9c964b9334d..af12f88c3139 100644 --- a/testing/talos/talos.json +++ b/testing/talos/talos.json @@ -67,6 +67,10 @@ }, "h1": { "tests": ["ts_paint_heavy"] + }, + "realworld-webextensions": { + "tests": ["startup_about_home_paint_realworld_webextensions"], + "webextensions_zip": "webextensions.zip" } } } diff --git a/testing/talos/talos/ffsetup.py b/testing/talos/talos/ffsetup.py index 03e4f67507e0..7adaf7d16b79 100644 --- a/testing/talos/talos/ffsetup.py +++ b/testing/talos/talos/ffsetup.py @@ -151,13 +151,21 @@ class FFSetup(object): profile.addons.install(extensions) # installing webextensions + webextensions_to_install = [] + webextensions_folder = self.test_config.get('webextensions_folder', None) + if isinstance(webextensions_folder, basestring): + folder = utils.interpolate(webextensions_folder) + for file in os.listdir(folder): + if file.endswith(".xpi"): + webextensions_to_install.append(os.path.join(folder, file)) + webextensions = self.test_config.get('webextensions', None) if isinstance(webextensions, basestring): - webextensions = [webextensions] + webextensions_to_install.append(webextensions) - if webextensions is not None: + if webextensions_to_install is not None: LOG.info("Installing Webextensions:") - for webext in webextensions: + for webext in webextensions_to_install: filename = utils.interpolate(webext) if mozinfo.os == 'win': filename = filename.replace('/', '\\') diff --git a/testing/talos/talos/getinfooffline/api.js b/testing/talos/talos/getinfooffline/api.js new file mode 100644 index 000000000000..6f8ecf882bc0 --- /dev/null +++ b/testing/talos/talos/getinfooffline/api.js @@ -0,0 +1,17 @@ +/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80 filetype=javascript: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +/* globals ExtensionAPI */ + +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +this.getinfooffline = class extends ExtensionAPI { + onStartup() { + Services.io.offline = true; + } +}; diff --git a/testing/talos/talos/getinfooffline/background.js b/testing/talos/talos/getinfooffline/background.js new file mode 100644 index 000000000000..4db0e8d23d7f --- /dev/null +++ b/testing/talos/talos/getinfooffline/background.js @@ -0,0 +1,6 @@ +// We've run once, and hopefully bypassed any network connections that add-ons +// might have tried to make after first install. Now let's uninstall ourselves +// so that subsequent starts run in online mode. + +/* globals browser */ +browser.management.uninstallSelf({}); diff --git a/testing/talos/talos/getinfooffline/manifest.json b/testing/talos/talos/getinfooffline/manifest.json new file mode 100644 index 000000000000..3fd6fbc172d3 --- /dev/null +++ b/testing/talos/talos/getinfooffline/manifest.json @@ -0,0 +1,27 @@ +{ + "manifest_version": 2, + "name": "Make the getinfo step happen offline for Talos", + "description": "Puts the browser in offline mode during the getinfo step and then uninstalls itself so that the browser runs in online mode for subsequent starts. This is useful when running startup tests that instal add-ons that like to open up network requests after being installed for the first time, which normally would cause the test runner to crash.", + "version": "0.1", + + "applications": { + "gecko": { + "id": "getinfooffline@mozilla.org" + } + }, + + "background": { + "scripts": ["background.js"] + }, + + "experiment_apis": { + "getinfooffline": { + "schema": "schema.json", + "parent": { + "scopes": ["addon_parent"], + "script": "api.js", + "events": ["startup"] + } + } + } +} diff --git a/testing/talos/talos/getinfooffline/schema.json b/testing/talos/talos/getinfooffline/schema.json new file mode 100644 index 000000000000..fe51488c7066 --- /dev/null +++ b/testing/talos/talos/getinfooffline/schema.json @@ -0,0 +1 @@ +[] diff --git a/testing/talos/talos/test.py b/testing/talos/talos/test.py index ce0850010037..c98c5137f36a 100644 --- a/testing/talos/talos/test.py +++ b/testing/talos/talos/test.py @@ -119,6 +119,7 @@ class TsBase(Test): 'setup', 'cleanup', 'webextensions', + 'webextensions_folder', 'reinstall', # A list of files from the profile directory that # should be copied to the temporary profile prior to # running each cycle, to avoid one cycle overwriting @@ -177,6 +178,18 @@ class startup_about_home_paint(ts_paint): tpmanifest = '${talos}/startup_test/startup_about_home_paint/startup_about_home_paint.manifest' +@register_test() +class startup_about_home_paint_realworld_webextensions(ts_paint): + url = None + cycles = 20 + extensions = [ + '${talos}/startup_test/startup_about_home_paint/addon', + '${talos}/getinfooffline' + ] + tpmanifest = '${talos}/startup_test/startup_about_home_paint/startup_about_home_paint.manifest' + webextensions_folder = '${talos}/webextensions' + + @register_test() class sessionrestore(TsBase): """ @@ -253,7 +266,7 @@ class PageloaderTest(Test): 'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk', 'format_pagename', 'filters', 'preferences', 'extensions', 'setup', 'cleanup', 'lower_is_better', 'alert_threshold', 'unit', 'webextensions', 'profile', - 'subtest_alerts', 'perfherder_framework', 'pdfpaint'] + 'subtest_alerts', 'perfherder_framework', 'pdfpaint', 'webextensions_folder'] class QuantumPageloadTest(PageloaderTest): diff --git a/testing/talos/webextensions.manifest b/testing/talos/webextensions.manifest new file mode 100644 index 000000000000..293cd6709af4 --- /dev/null +++ b/testing/talos/webextensions.manifest @@ -0,0 +1,9 @@ +[ + { + "size": 2608427, + "visibility": "public", + "digest": "f3198257ca82043bb9fe9a7085cff7af6f0d11685a0416ebb73e8edec547b970dd044666120e44898e2de36372add88ef93cd08af79dcd03e32fa1e9ef2053c8", + "algorithm": "sha512", + "filename": "webextensions.zip" + } +] \ No newline at end of file