From 2719b0d9cf4a9652209dd4dcf8b512d47c3cc5b6 Mon Sep 17 00:00:00 2001 From: Marian-Vasile Laza Date: Wed, 30 Mar 2022 11:18:07 -0700 Subject: [PATCH] Backed out 2 changesets (bug 1749928) for causing webgl failures. CLOSED TREE Backed out changeset d654d3fcfa8e (bug 1749928) Backed out changeset 49e91c2434f7 (bug 1749928) --- taskcluster/ci/test/talos.yml | 98 +---- taskcluster/ci/test/test-sets.yml | 12 +- .../mozharness/mozilla/testing/talos.py | 17 +- testing/perfdocs/generated/talos.rst | 172 +------- testing/talos/perfdocs/config.yml | 4 +- testing/talos/talos.json | 18 +- testing/talos/talos/ffsetup.py | 2 +- testing/talos/talos/output.py | 11 - .../devtools/addon/content/damp-tests.js | 387 +++++++++--------- .../tests/devtools/addon/content/damp.js | 17 +- 10 files changed, 225 insertions(+), 513 deletions(-) diff --git a/taskcluster/ci/test/talos.yml b/taskcluster/ci/test/talos.yml index c4cdec0923ea..4a5fc3122755 100644 --- a/taskcluster/ci/test/talos.yml +++ b/taskcluster/ci/test/talos.yml @@ -98,10 +98,10 @@ talos-chrome-profiling: - --suite=chromez - --gecko-profile -talos-damp-inspector: - description: Talos devtools inspector (damp-inspector) - try-name: damp-inspector - treeherder-symbol: T(damp-inspector) +talos-damp: + description: Talos devtools (damp) + try-name: damp + treeherder-symbol: T(damp) max-run-time: 5400 variants: - fission @@ -121,62 +121,12 @@ talos-damp-inspector: default: [mozilla-beta, trunk] mozharness: extra-options: - - --suite=damp-inspector + - --suite=damp -talos-damp-webconsole: - description: Talos devtools webconsole (damp-webconsole) - try-name: damp-webconsole - treeherder-symbol: T(damp-webconsole) - max-run-time: 5400 - variants: - - fission - - webrender-sw+fission - run-without-variant: false - run-on-projects: - by-variant: - webrender-sw+fission: - by-test-platform: - (linux1804|windows10|macosx1015)-64-shippable-qr.*: [mozilla-beta, trunk] - default: [] - default: - by-test-platform: - windows.*-32-shippable.*/.*: [] - (linux|windows|macos)(?!.*shippable).*: [] - (linux|windows10-64|macos)(?!.*-qr).*: [] - default: [mozilla-beta, trunk] - mozharness: - extra-options: - - --suite=damp-webconsole - -talos-damp-other: - description: Talos devtools other (damp-other) - try-name: damp-other - treeherder-symbol: T(damp-other) - max-run-time: 5400 - variants: - - fission - - webrender-sw+fission - run-without-variant: false - run-on-projects: - by-variant: - webrender-sw+fission: - by-test-platform: - (linux1804|windows10|macosx1015)-64-shippable-qr.*: [mozilla-beta, trunk] - default: [] - default: - by-test-platform: - windows.*-32-shippable.*/.*: [] - (linux|windows|macos)(?!.*shippable).*: [] - (linux|windows10-64|macos)(?!.*-qr).*: [] - default: [mozilla-beta, trunk] - mozharness: - extra-options: - - --suite=damp-other - -talos-damp-inspector-profiling: - description: Talos devtools inspector (damp-inspector) with gecko profiling - try-name: damp-inspector-prof - treeherder-symbol: T-Prof(damp-inspector) +talos-damp-profiling: + description: Talos devtools (damp) with gecko profiling + try-name: damp-prof + treeherder-symbol: T-Prof(damp) tier: 2 max-run-time: 5400 variants: [fission] @@ -184,35 +134,7 @@ talos-damp-inspector-profiling: run-on-projects: [] mozharness: extra-options: - - --suite=damp-inspector - - --gecko-profile - -talos-damp-webconsole-profiling: - description: Talos devtools webconsole (damp-webconsole) with gecko profiling - try-name: damp-webconsole-prof - treeherder-symbol: T-Prof(damp-webconsole) - tier: 2 - max-run-time: 5400 - variants: [fission] - run-without-variant: false - run-on-projects: [] - mozharness: - extra-options: - - --suite=damp-webconsole - - --gecko-profile - -talos-damp-other-profiling: - description: Talos devtools other (damp-other) with gecko profiling - try-name: damp-other-prof - treeherder-symbol: T-Prof(damp-other) - tier: 2 - max-run-time: 5400 - variants: [fission] - run-without-variant: false - run-on-projects: [] - mozharness: - extra-options: - - --suite=damp-other + - --suite=damp - --gecko-profile talos-dromaeojs: diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index b7e40047fd6a..4c2814384fa3 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -35,9 +35,7 @@ mochitest-webgpu: talos: - talos-bcv - talos-chrome - - talos-damp-inspector - - talos-damp-webconsole - - talos-damp-other + - talos-damp - talos-dromaeojs - talos-g1 - talos-g3 @@ -278,9 +276,7 @@ windows-aarch64-tests: windows-talos: - talos-bcv - talos-chrome - - talos-damp-inspector - - talos-damp-webconsole - - talos-damp-other + - talos-damp - talos-dromaeojs - talos-g1 - talos-g4 @@ -351,9 +347,7 @@ macosx1015-64-qr-tests: macosx64-talos: - talos-bcv - talos-chrome - - talos-damp-inspector - - talos-damp-webconsole - - talos-damp-other + - talos-damp - talos-dromaeojs - talos-g1 - talos-g4 diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py index 70c5adf4c1ad..4c9b0bae202f 100755 --- a/testing/mozharness/mozharness/mozilla/testing/talos.py +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -474,12 +474,6 @@ class Talos( self.fatal("Talos json config not found, cannot verify suite") return suite_name - def query_suite_extra_prefs(self): - if self.query_talos_json_config() and self.suite is not None: - return self.talos_json_config["suites"][self.suite].get("extra_prefs") - - return [] - def validate_suite(self): """Ensure suite name is a valid talos suite.""" if self.query_talos_json_config() and self.suite is not None: @@ -529,15 +523,10 @@ class Talos( options += self.config["talos_extra_options"] if self.config.get("code_coverage", False): options.extend(["--code-coverage"]) - - # Add extra_prefs defined by individual test suites in talos.json - extra_prefs = self.query_suite_extra_prefs() - # Add extra_prefs from the configuration if self.config["extra_prefs"]: - extra_prefs.extend(self.config["extra_prefs"]) - - options.extend(["--setpref={}".format(p) for p in extra_prefs]) - + options.extend( + ["--setpref={}".format(p) for p in self.config["extra_prefs"]] + ) # disabling fission can come from the --disable-fission cmd line argument; or in CI # it comes from a taskcluster transform which adds a --setpref for fission.autostart if (not self.config["fission"]) or "fission.autostart=false" in self.config[ diff --git a/testing/perfdocs/generated/talos.rst b/testing/perfdocs/generated/talos.rst index baf0c69fd63a..ef86e8dd7378 100644 --- a/testing/perfdocs/generated/talos.rst +++ b/testing/perfdocs/generated/talos.rst @@ -1450,13 +1450,13 @@ Talos test lists * contact: :ochameau and devtools team * source: `damp `__ * type: `Page load`_ - * measuring: Developer Tools toolbox performance. Split in test suites covering different DevTools areas (inspector, webconsole, other). + * measuring: Developer Tools toolbox startup, shutdown, and reload performance * reporting: intervals in ms (lower is better) * see below for details * data: there are 36 reported subtests from DAMP which we load 25 times, resulting in 36 sets of 25 data points. * summarization: * subtest: `ignore first`_ data point, then take the `median`_ of the remaining 24 data points; `source: test.py `__ - * suite: No value for the suite, only individual subtests are relevant. + * suite: `geometric mean`_ of the 36 subtest results. * description: To run this locally, you'll need to pull down the `tp5 page set <#page-sets>`__ and run it in a local web server. See the `tp5 @@ -1528,32 +1528,12 @@ Talos test lists - autoland - mozilla-release - mozilla-beta - * - **talos-damp-inspector-fis-e10s** + * - **talos-damp-fis-e10s** - ❌ - ❌ - ❌ - ❌ - * - **talos-damp-inspector-swr-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-other-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-other-swr-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-webconsole-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-webconsole-swr-fis-e10s** + * - **talos-damp-swr-fis-e10s** - ❌ - ❌ - ❌ @@ -1569,32 +1549,12 @@ Talos test lists - autoland - mozilla-release - mozilla-beta - * - **talos-damp-inspector-fis-e10s** + * - **talos-damp-fis-e10s** - ✅ - ✅ - ❌ - ✅ - * - **talos-damp-inspector-swr-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-other-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-other-swr-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-webconsole-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-webconsole-swr-fis-e10s** + * - **talos-damp-swr-fis-e10s** - ✅ - ✅ - ❌ @@ -1610,32 +1570,12 @@ Talos test lists - autoland - mozilla-release - mozilla-beta - * - **talos-damp-inspector-fis-e10s** + * - **talos-damp-fis-e10s** - ✅ - ✅ - ❌ - ✅ - * - **talos-damp-inspector-swr-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-other-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-other-swr-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-webconsole-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-webconsole-swr-fis-e10s** + * - **talos-damp-swr-fis-e10s** - ✅ - ✅ - ❌ @@ -1651,32 +1591,12 @@ Talos test lists - autoland - mozilla-release - mozilla-beta - * - **talos-damp-inspector-fis-e10s** + * - **talos-damp-fis-e10s** - ❌ - ❌ - ❌ - ❌ - * - **talos-damp-inspector-swr-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-other-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-other-swr-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-webconsole-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-webconsole-swr-fis-e10s** + * - **talos-damp-swr-fis-e10s** - ❌ - ❌ - ❌ @@ -1692,32 +1612,12 @@ Talos test lists - autoland - mozilla-release - mozilla-beta - * - **talos-damp-inspector-fis-e10s** + * - **talos-damp-fis-e10s** - ❌ - ❌ - ❌ - ❌ - * - **talos-damp-inspector-swr-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-other-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-other-swr-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-webconsole-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-webconsole-swr-fis-e10s** + * - **talos-damp-swr-fis-e10s** - ❌ - ❌ - ❌ @@ -1733,32 +1633,12 @@ Talos test lists - autoland - mozilla-release - mozilla-beta - * - **talos-damp-inspector-fis-e10s** + * - **talos-damp-fis-e10s** - ❌ - ❌ - ❌ - ❌ - * - **talos-damp-inspector-swr-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-other-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-other-swr-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-webconsole-fis-e10s** - - ❌ - - ❌ - - ❌ - - ❌ - * - **talos-damp-webconsole-swr-fis-e10s** + * - **talos-damp-swr-fis-e10s** - ❌ - ❌ - ❌ @@ -1774,32 +1654,12 @@ Talos test lists - autoland - mozilla-release - mozilla-beta - * - **talos-damp-inspector-fis-e10s** + * - **talos-damp-fis-e10s** - ✅ - ✅ - ❌ - ✅ - * - **talos-damp-inspector-swr-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-other-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-other-swr-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-webconsole-fis-e10s** - - ✅ - - ✅ - - ❌ - - ✅ - * - **talos-damp-webconsole-swr-fis-e10s** + * - **talos-damp-swr-fis-e10s** - ✅ - ✅ - ❌ diff --git a/testing/talos/perfdocs/config.yml b/testing/talos/perfdocs/config.yml index cf44d99805d1..3e6e764d8a9a 100644 --- a/testing/talos/perfdocs/config.yml +++ b/testing/talos/perfdocs/config.yml @@ -142,12 +142,12 @@ suites: - contact: :ochameau and devtools team - source: `damp `__ - type: `Page load`_ - - measuring: Developer Tools toolbox performance. Split in test suites covering different DevTools areas (inspector, webconsole, other). + - measuring: Developer Tools toolbox startup, shutdown, and reload performance - reporting: intervals in ms (lower is better) - see below for details - data: there are 36 reported subtests from DAMP which we load 25 times, resulting in 36 sets of 25 data points. - summarization: * subtest: `ignore first`_ data point, then take the `median`_ of the remaining 24 data points; `source: test.py `__ - * suite: No value for the suite, only individual subtests are relevant. + * suite: `geometric mean`_ of the 36 subtest results. - description: To run this locally, you'll need to pull down the `tp5 page set <#page-sets>`__ and run it in a local web server. See the `tp5 diff --git a/testing/talos/talos.json b/testing/talos/talos.json index d02d47db168f..af816c334af0 100644 --- a/testing/talos/talos.json +++ b/testing/talos/talos.json @@ -24,23 +24,7 @@ }, "damp": { "tests": ["damp"], - "pagesets_name": "tp5n.zip", - "extra_prefs": ["talos.damp.suite=all"] - }, - "damp-inspector": { - "tests": ["damp"], - "pagesets_name": "tp5n.zip", - "extra_prefs": ["talos.damp.suite=inspector"] - }, - "damp-webconsole": { - "tests": ["damp"], - "pagesets_name": "tp5n.zip", - "extra_prefs": ["talos.damp.suite=webconsole"] - }, - "damp-other": { - "tests": ["damp"], - "pagesets_name": "tp5n.zip", - "extra_prefs": ["talos.damp.suite=other"] + "pagesets_name": "tp5n.zip" }, "tabswitch": { "tests": ["tabswitch"], diff --git a/testing/talos/talos/ffsetup.py b/testing/talos/talos/ffsetup.py index 6b3adc836fd3..47de13a10de7 100644 --- a/testing/talos/talos/ffsetup.py +++ b/testing/talos/talos/ffsetup.py @@ -333,7 +333,7 @@ class FFSetup(object): self._init_env() self._init_profile() try: - if not self.debug_mode and not self.test_config["name"].startswith("damp"): + if not self.debug_mode and self.test_config["name"] != "damp": self._run_profile() except BaseException: self.clean() diff --git a/testing/talos/talos/output.py b/testing/talos/talos/output.py index ca75c6655d02..f5d0e3e1db5a 100644 --- a/testing/talos/talos/output.py +++ b/testing/talos/talos/output.py @@ -322,15 +322,6 @@ class Output(object): score = 60 * 1000 / filter.geometric_mean(results) / correctionFactor return score - @classmethod - def damp_score(cls, val_list): - """ - damp_score: damp is only interested in the value of subtests and will - aggregate data from several suites. - Use a hardcoded value for the suite to avoid inconsistencies. - """ - return 100 - def construct_results(self, vals, testname): if "responsiveness" in testname: return filter.responsiveness_Metric([val for (val, page) in vals]) @@ -346,8 +337,6 @@ class Output(object): return self.speedometer_score(vals) elif testname.startswith("stylebench"): return self.stylebench_score(vals) - elif testname.startswith("damp"): - return self.damp_score(vals) elif len(vals) > 1: return filter.geometric_mean([i for i, j in vals]) else: diff --git a/testing/talos/talos/tests/devtools/addon/content/damp-tests.js b/testing/talos/talos/tests/devtools/addon/content/damp-tests.js index 596501ca5eff..bd5fe4245ce6 100644 --- a/testing/talos/talos/tests/devtools/addon/content/damp-tests.js +++ b/testing/talos/talos/tests/devtools/addon/content/damp-tests.js @@ -6,13 +6,6 @@ const Services = require("Services"); const isWindows = Services.appinfo.OS === "WINNT"; - -const TEST_SUITES = { - INSPECTOR: "inspector", - WEBCONSOLE: "webconsole", - OTHER: "other", -}; - /** * This is the registry for all DAMP tests. Tests will be run in the order specified by * the DAMP_TESTS array. @@ -26,197 +19,189 @@ const TEST_SUITES = { * - {Boolean} cold: set to true to run the test only during the first run of the browser */ -module.exports = { - [TEST_SUITES.INSPECTOR]: [ - // The first cold-open test is *colder* than the other cold-open tests, it will also - // assess the impact of loading shared DevTools modules for the first time. - // This test will assert the impact of base loader/Loader.jsm modules loading, - // typically gDevtools/gDevToolsBrowser/Framework modules, while the others will mostly - // track panel-specific modules (Browser loader, but not only). - { - name: "inspector.cold-open", - path: "inspector/cold-open.js", - description: - "Measure first open toolbox on inspector panel (incl. shared modules)", - cold: true, - }, - { - name: "accessibility.cold-open", - path: "accessibility/cold-open.js", - description: "Measure first open toolbox on accessibility panel", - cold: true, - }, - // Run all tests against "simple" document - { - name: "simple.inspector", - path: "inspector/simple.js", - description: - "Measure open/close toolbox on inspector panel against simple document", - }, - { - name: "simple.styleeditor", - path: "styleeditor/simple.js", - description: - "Measure open/close toolbox on style editor panel against simple document", - }, - { - name: "simple.accessibility", - path: "accessibility/simple.js", - description: - "Measure open/close toolbox on accessibility panel against simple document", - // Bug 1660854 - disable on Windows due to frequent failures - disabled: isWindows, - }, - // Run all tests against "complicated" document - { - name: "complicated.inspector", - path: "inspector/complicated.js", - description: - "Measure open/close toolbox on inspector panel against complicated document", - }, - { - name: "complicated.styleeditor", - path: "styleeditor/complicated.js", - description: - "Measure open/close toolbox on style editor panel against complicated document", - }, - { - name: "custom.inspector", - path: "inspector/custom.js", - }, - // Run individual tests covering a very precise tool feature. - { - name: "inspector.mutations", - path: "inspector/mutations.js", - description: - "Measure the time to perform childList mutations when inspector is enabled", - }, - { - name: "inspector.layout", - path: "inspector/layout.js", - description: - "Measure the time to open/close toolbox on inspector with layout tab against big document with grid containers", - }, - ], - [TEST_SUITES.WEBCONSOLE]: [ - { - name: "webconsole.cold-open", - path: "webconsole/cold-open.js", - description: "Measure first open toolbox on webconsole panel", - cold: true, - }, - { - name: "simple.webconsole", - path: "webconsole/simple.js", - description: - "Measure open/close toolbox on webconsole panel against simple document", - }, - { - name: "complicated.webconsole", - path: "webconsole/complicated.js", - description: - "Measure open/close toolbox on webconsole panel against complicated document", - }, - { - name: "custom.webconsole", - path: "webconsole/custom.js", - }, - { - name: "console.bulklog", - path: "webconsole/bulklog.js", - description: - "Measure time for a bunch of sync console.log statements to appear", - }, - { - name: "console.autocomplete", - path: "webconsole/autocomplete.js", - description: "Measure time for autocomplete popup to appear", - }, - { - name: "console.streamlog", - path: "webconsole/streamlog.js", - description: - "Measure rAF on page during a stream of console.log statements", - }, - { - name: "console.objectexpand", - path: "webconsole/objectexpand.js", - description: - "Measure time to expand a large object and close the console", - }, - { - name: "console.openwithcache", - path: "webconsole/openwithcache.js", - description: - "Measure time to render last logged messages in console for a page with 100 logged messages", - }, - { - name: "console.typing", - path: "webconsole/typing.js", - description: - "Measure time it takes to type something in the console input", - }, - ], - [TEST_SUITES.OTHER]: [ - { - name: "debugger.cold-open", - path: "debugger/cold-open.js", - description: "Measure first open toolbox on debugger panel", - cold: true, - }, - { - name: "netmonitor.cold-open", - path: "netmonitor/cold-open.js", - description: "Measure first open toolbox on netmonitor panel", - cold: true, - }, - { - name: "simple.debugger", - path: "debugger/simple.js", - description: - "Measure open/close toolbox on debugger panel against simple document", - }, - { - name: "simple.netmonitor", - path: "netmonitor/simple.js", - description: - "Measure open/close toolbox on network monitor panel against simple document", - }, - { - name: "complicated.debugger", - path: "debugger/complicated.js", - description: - "Measure open/close toolbox on debugger panel against complicated document", - }, - // Bug 1693975 - disable test due to frequent failures - // { - // name: "complicated.netmonitor", - // path: "netmonitor/complicated.js", - // description: - // "Measure open/close toolbox on network monitor panel against complicated document", - // }, - // Run all tests against a document specific to each tool - { - name: "custom.debugger", - path: "debugger/custom.js", - }, - { - name: "panelsInBackground.reload", - path: "toolbox/panels-in-background.js", - description: "Measure page reload time when all panels are in background", - }, - { - name: "toolbox.screenshot", - path: "toolbox/screenshot.js", - description: "Measure the time to take a fullpage screenshot", - }, - { - name: "server.protocoljs", - path: "server/protocol.js", - description: "Measure RDP/protocol.js performance", - }, - // ⚠ Adding new individual tests slows down DAMP execution ⚠ - // ⚠ Consider contributing to custom.${tool} rather than adding isolated tests ⚠ - // ⚠ See https://firefox-source-docs.mozilla.org/devtools/tests/writing-perf-tests.html ⚠ - ], -}; +module.exports = [ + // The first cold-open test is *colder* than the other cold-open tests, it will also + // assess the impact of loading shared DevTools modules for the first time. + // This test will assert the impact of base loader/Loader.jsm modules loading, + // typically gDevtools/gDevToolsBrowser/Framework modules, while the others will mostly + // track panel-specific modules (Browser loader, but not only). + { + name: "inspector.cold-open", + path: "inspector/cold-open.js", + description: + "Measure first open toolbox on inspector panel (incl. shared modules)", + cold: true, + }, + { + name: "debugger.cold-open", + path: "debugger/cold-open.js", + description: "Measure first open toolbox on debugger panel", + cold: true, + }, + { + name: "webconsole.cold-open", + path: "webconsole/cold-open.js", + description: "Measure first open toolbox on webconsole panel", + cold: true, + }, + { + name: "netmonitor.cold-open", + path: "netmonitor/cold-open.js", + description: "Measure first open toolbox on netmonitor panel", + cold: true, + }, + { + name: "accessibility.cold-open", + path: "accessibility/cold-open.js", + description: "Measure first open toolbox on accessibility panel", + cold: true, + }, + // Run all tests against "simple" document + { + name: "simple.webconsole", + path: "webconsole/simple.js", + description: + "Measure open/close toolbox on webconsole panel against simple document", + }, + { + name: "simple.inspector", + path: "inspector/simple.js", + description: + "Measure open/close toolbox on inspector panel against simple document", + }, + { + name: "simple.debugger", + path: "debugger/simple.js", + description: + "Measure open/close toolbox on debugger panel against simple document", + }, + { + name: "simple.styleeditor", + path: "styleeditor/simple.js", + description: + "Measure open/close toolbox on style editor panel against simple document", + }, + { + name: "simple.netmonitor", + path: "netmonitor/simple.js", + description: + "Measure open/close toolbox on network monitor panel against simple document", + }, + { + name: "simple.accessibility", + path: "accessibility/simple.js", + description: + "Measure open/close toolbox on accessibility panel against simple document", + // Bug 1660854 - disable on Windows due to frequent failures + disabled: isWindows, + }, + // Run all tests against "complicated" document + { + name: "complicated.webconsole", + path: "webconsole/complicated.js", + description: + "Measure open/close toolbox on webconsole panel against complicated document", + }, + { + name: "complicated.inspector", + path: "inspector/complicated.js", + description: + "Measure open/close toolbox on inspector panel against complicated document", + }, + { + name: "complicated.debugger", + path: "debugger/complicated.js", + description: + "Measure open/close toolbox on debugger panel against complicated document", + }, + { + name: "complicated.styleeditor", + path: "styleeditor/complicated.js", + description: + "Measure open/close toolbox on style editor panel against complicated document", + }, + // Bug 1693975 - disable test due to frequent failures + // { + // name: "complicated.netmonitor", + // path: "netmonitor/complicated.js", + // description: + // "Measure open/close toolbox on network monitor panel against complicated document", + // }, + // Run all tests against a document specific to each tool + { + name: "custom.webconsole", + path: "webconsole/custom.js", + }, + { + name: "custom.inspector", + path: "inspector/custom.js", + }, + { + name: "custom.debugger", + path: "debugger/custom.js", + }, + // Run individual tests covering a very precise tool feature. + { + name: "console.bulklog", + path: "webconsole/bulklog.js", + description: + "Measure time for a bunch of sync console.log statements to appear", + }, + { + name: "console.autocomplete", + path: "webconsole/autocomplete.js", + description: "Measure time for autocomplete popup to appear", + }, + { + name: "console.streamlog", + path: "webconsole/streamlog.js", + description: + "Measure rAF on page during a stream of console.log statements", + }, + { + name: "console.objectexpand", + path: "webconsole/objectexpand.js", + description: "Measure time to expand a large object and close the console", + }, + { + name: "console.openwithcache", + path: "webconsole/openwithcache.js", + description: + "Measure time to render last logged messages in console for a page with 100 logged messages", + }, + { + name: "console.typing", + path: "webconsole/typing.js", + description: "Measure time it takes to type something in the console input", + }, + { + name: "inspector.mutations", + path: "inspector/mutations.js", + description: + "Measure the time to perform childList mutations when inspector is enabled", + }, + { + name: "inspector.layout", + path: "inspector/layout.js", + description: + "Measure the time to open/close toolbox on inspector with layout tab against big document with grid containers", + }, + { + name: "panelsInBackground.reload", + path: "toolbox/panels-in-background.js", + description: "Measure page reload time when all panels are in background", + }, + { + name: "toolbox.screenshot", + path: "toolbox/screenshot.js", + description: "Measure the time to take a fullpage screenshot", + }, + { + name: "server.protocoljs", + path: "server/protocol.js", + description: "Measure RDP/protocol.js performance", + }, + // ⚠ Adding new individual tests slows down DAMP execution ⚠ + // ⚠ Consider contributing to custom.${tool} rather than adding isolated tests ⚠ + // ⚠ See https://firefox-source-docs.mozilla.org/devtools/tests/writing-perf-tests.html ⚠ +]; diff --git a/testing/talos/talos/tests/devtools/addon/content/damp.js b/testing/talos/talos/tests/devtools/addon/content/damp.js index 37831c71eed8..bd442ad389a0 100644 --- a/testing/talos/talos/tests/devtools/addon/content/damp.js +++ b/testing/talos/talos/tests/devtools/addon/content/damp.js @@ -442,20 +442,9 @@ Damp.prototype = { // Filter tests via `./mach --subtests filter` command line argument let filter = Services.prefs.getCharPref("talos.subtests", ""); - const suite = Services.prefs.getCharPref("talos.damp.suite", ""); - let testSuite; - if (suite === "all") { - testSuite = Object.values(DAMP_TESTS).flat(); - } else { - testSuite = DAMP_TESTS[suite]; - if (!testSuite) { - this.error(`Unable to find any test suite matching '${suite}'`); - } - } - - let tests = testSuite - .filter(test => !test.disabled) - .filter(test => test.name.includes(filter)); + let tests = DAMP_TESTS.filter(test => !test.disabled).filter(test => + test.name.includes(filter) + ); if (tests.length === 0) { this.error(`Unable to find any test matching '${filter}'`);