diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index ed9667fa1bdc..0b10c94f9c1b 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -36,6 +36,7 @@ treeherder: 'Btime-live-fenix': 'Browsertime performance tests on Firefox' 'Btime-live-ChR': 'Browsertime performance tests on Firefox' 'Btime-live-Cr': 'Browsertime performance tests on Google Chromium' + 'Btime-live-Saf': 'Browsertime performance tests on Safari' 'Btime-ChR': 'Browsertime performance tests on Google Chrome Release' 'Btime-nofis-ChR': 'Browsertime performance tests on Google Chrome Release without fission enabled' 'Btime-Cr': 'Browsertime performance tests on Google Chromium' @@ -44,6 +45,7 @@ treeherder: 'Btime-P-refbrow': 'Browsertime power tests on reference browser' 'Btime-fenix': 'Browsertime performance tests on Fenix' 'Btime-refbrow': 'Browsertime performance tests on the reference browser' + 'Btime-Saf': 'Browsertime performance tests on Safari' 'Fetch': 'Fetch and store content' 'Fxfn': 'Firefox functional tests' 'Fxfn-nofis': 'Firefox functional tests without fission enabled' diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml index 6ff612cf8022..95578dc8aad9 100644 --- a/taskcluster/ci/test/browsertime-desktop.yml +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -18,6 +18,8 @@ job-defaults: - .*shippable.* chromium: - .*shippable.* + safari: + - .*macos.*shippable.* default: [] test-manifest-loader: null # don't load tests in the taskgraph tier: 3 @@ -44,6 +46,8 @@ browsertime-tp6: - .*shippable.* chromium: - .*shippable.* + safari: + - .*macos.*shippable.* default: [] max-run-time: by-subtest: @@ -244,7 +248,7 @@ browsertime-tp6-bytecode: browsertime-benchmark: description: Raptor (browsertime) Benchmark tests raptor: - apps: [firefox, chrome, chromium] + apps: [firefox, chrome, chromium, safari] subtests: by-app: firefox: @@ -260,6 +264,18 @@ browsertime-benchmark: - [twitch-animation, tw-a] - [unity-webgl, ugl] - [webaudio, wa] + safari: + - ares6 + - [assorted-dom, dom] + - [jetstream2, js2] + # - [matrix-react-bench, mrb] (Bug 1802922) + - [motionmark-animometer, mm-a] + - [motionmark-htmlsuite, mm-h] + - [speedometer, sp] + - [stylebench, sb] + - [sunspider, ss] + - [unity-webgl, ugl] + # - [webaudio, wa] (Bug 1802922) default: - ares6 - [assorted-dom, dom] @@ -335,7 +351,7 @@ browsertime-benchmark: browsertime-benchmark-wasm: description: Raptor (browsertime) WASM benchmarks raptor: - apps: [firefox, chrome, chromium] + apps: [firefox, chrome, chromium] # Safari depends on Bug 1802922 subtests: by-app: firefox: @@ -415,36 +431,65 @@ browsertime-tp6-live: - --live-sites - --extra-profiler-run raptor: - apps: [firefox, chrome, chromium] + apps: [firefox, chrome, chromium, safari] test: tp6 run-visual-metrics: true subtests: - - amazon - - [bing-search, bing] - - ebay - - [facebook, fb] - - fandom - - [google-docs, gdocs] - - [google-mail, gmail] - - [google-search, gsearch] - - [google-slides, gslides] - - imdb - - imgur - # - instagram (see bug 1736999) - - linkedin - - microsoft - - netflix - # - office (site loads blank page if not signed in) - - outlook - - paypal - - pinterest - - reddit - - tumblr - - twitch - - twitter - - wikipedia - - yahoo-mail - - youtube + by-app: + safari: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + # - fandom (Bug 1802922) + - [google-docs, gdocs] + # - [google-mail, gmail] (Bug 1802922) + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + # - imgur (Bug 1802922) + # - instagram (see bug 1736999) + - linkedin + - microsoft + # - netflix (Bug 1802922) + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + # - yahoo-mail (Bug 1802922) + - youtube + default: + - amazon + - [bing-search, bing] + - ebay + - [facebook, fb] + - fandom + - [google-docs, gdocs] + - [google-mail, gmail] + - [google-search, gsearch] + - [google-slides, gslides] + - imdb + - imgur + # - instagram (see bug 1736999) + - linkedin + - microsoft + - netflix + # - office (site loads blank page if not signed in) + - outlook + - paypal + - pinterest + - reddit + - tumblr + - twitch + - twitter + - wikipedia + - yahoo-mail + - youtube run-on-projects: [] tier: 3 treeherder-symbol: Btime-live(tp6) diff --git a/taskcluster/gecko_taskgraph/transforms/test/raptor.py b/taskcluster/gecko_taskgraph/transforms/test/raptor.py index d7e9847ff199..3cd7dcd2e1a3 100644 --- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py +++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py @@ -5,16 +5,11 @@ from copy import deepcopy -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema -from taskgraph.util.treeherder import join_symbol, split_symbol -from voluptuous import ( - Optional, - Required, - Extra, -) - from gecko_taskgraph.transforms.test import test_description_schema +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by +from taskgraph.util.treeherder import join_symbol, split_symbol +from voluptuous import Extra, Optional, Required transforms = TransformSequence() task_transforms = TransformSequence() @@ -81,6 +76,7 @@ def split_apps(config, tests): "chromium": "Cr", "fenix": "fenix", "refbrow": "refbrow", + "safari": "Saf", } for test in tests: diff --git a/testing/mozharness/external_tools/performance-artifact-schema.json b/testing/mozharness/external_tools/performance-artifact-schema.json index aaf4312d0725..a5df29b26c48 100644 --- a/testing/mozharness/external_tools/performance-artifact-schema.json +++ b/testing/mozharness/external_tools/performance-artifact-schema.json @@ -12,7 +12,8 @@ "fennec", "geckoview", "refbrow", - "fenix" + "fenix", + "safari" ], "maxLength": 10, "type": "string" @@ -23,7 +24,9 @@ "type": "string" } }, - "required": ["name"], + "required": [ + "name" + ], "type": "object" }, "framework_schema": { @@ -206,7 +209,7 @@ "description": "Structure for submitting performance data as part of a job", "id": "https://treeherder.mozilla.org/schemas/v1/performance-artifact.json#", "properties": { - "application":{ + "application": { "$ref": "#/definitions/application_schema" }, "framework": { diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index b025d9f66e2a..7c15b733b295 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -12,33 +12,30 @@ import glob import multiprocessing import os import re -import sys import subprocess +import sys import tempfile - from shutil import copyfile, rmtree -from six import string_types - import mozharness - from mozharness.base.errors import PythonErrorList -from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL, INFO +from mozharness.base.log import CRITICAL, DEBUG, ERROR, INFO, OutputParser +from mozharness.base.python import Python3Virtualenv +from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.automation import ( EXIT_STATUS_DICT, - TBPL_SUCCESS, TBPL_RETRY, + TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE, ) -from mozharness.base.python import Python3Virtualenv from mozharness.mozilla.testing.android import AndroidMixin -from mozharness.mozilla.testing.errors import HarnessErrorList, TinderBoxPrintRe -from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options -from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.testing.codecoverage import ( CodeCoverageMixin, code_coverage_config_options, ) +from mozharness.mozilla.testing.errors import HarnessErrorList, TinderBoxPrintRe +from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options +from six import string_types scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))) external_tools_path = os.path.join(scripts_path, "external_tools") @@ -197,6 +194,7 @@ class Raptor( "geckoview", "refbrow", "fenix", + "safari", ], "dest": "app", "help": "Name of the application we are testing (default: firefox).", @@ -944,6 +942,9 @@ class Raptor( "Set binary to %s instead of %s" % (kw_options["binary"], binary_path) ) + elif self.app == "safari" and not self.run_local: + binary_path = "/Applications/Safari.app/Contents/MacOS/Safari" + kw_options["binary"] = binary_path else: # Running on Chromium if not self.run_local: # When running locally we already set the Chromium binary above, in init. diff --git a/testing/perfdocs/generated/raptor.rst b/testing/perfdocs/generated/raptor.rst index 6e13193bc65d..e4d8c9de19d4 100644 --- a/testing/perfdocs/generated/raptor.rst +++ b/testing/perfdocs/generated/raptor.rst @@ -47,7 +47,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: :jandem and SpiderMonkey Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **gecko profile entries**: 14000000 * **gecko profile interval**: 1 @@ -143,6 +143,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-ares6** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -236,7 +241,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: PerfTest Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **fetch task**: assorted-dom * **gecko profile entries**: 2000000 @@ -332,6 +337,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-assorted-dom** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -425,7 +435,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: :jandem and SpiderMonkey Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **fetch task**: jetstream2 * **gecko profile entries**: 14000000 @@ -522,6 +532,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-jetstream2** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -805,7 +820,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: :jgilbert and Graphics(gfx) Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **gecko profile entries**: 8000000 * **gecko profile interval**: 1 @@ -899,6 +914,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-motionmark-animometer** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -992,7 +1012,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: :jgilbert and Graphics(gfx) Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **gecko profile entries**: 8000000 * **gecko profile interval**: 1 @@ -1086,6 +1106,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-motionmark-htmlsuite** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -1350,7 +1375,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: SpiderMonkey Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **gecko profile entries**: 14000000 * **gecko profile interval**: 1 @@ -1669,6 +1694,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-speedometer** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -2077,6 +2107,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-speedometer** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -2170,7 +2205,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: :emelio and Layout Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **gecko profile entries**: 8000000 * **gecko profile interval**: 1 @@ -2266,6 +2301,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-stylebench** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -2359,7 +2399,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: :jandem and SpiderMonkey Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **gecko profile entries**: 8000000 * **gecko profile interval**: 1 @@ -2453,6 +2493,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ✅ + * - **browsertime-benchmark-safari-sunspider** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -2933,6 +2978,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ❌ + * - **browsertime-benchmark-safari-unity-webgl** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -3022,7 +3072,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr **Owner**: :jgilbert and Graphics(gfx) Team * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **expected**: pass * **fetch task**: unity-webgl * **gecko profile entries**: 8000000 @@ -3258,6 +3308,11 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr - ✅ - ✅ - ❌ + * - **browsertime-benchmark-safari-unity-webgl** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -6345,7 +6400,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -6836,6 +6891,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-amazon** + - ❌ + - ❌ + - ❌ + - ❌ * - **browsertime-tp6-profiling-firefox-amazon** - ✅ - ❌ @@ -6995,7 +7055,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -7154,6 +7214,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-bing-search** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -7308,7 +7373,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -7502,7 +7567,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -8097,7 +8162,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -8237,6 +8302,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-ebay** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -8371,7 +8441,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -8886,7 +8956,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -9221,7 +9291,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -9682,6 +9752,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-facebook** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -9816,7 +9891,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -10129,7 +10204,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -10270,6 +10345,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-google-docs** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -10404,7 +10484,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -10677,7 +10757,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -10816,6 +10896,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-google-search** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -10950,7 +11035,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -11110,6 +11195,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-google-slides** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -11264,7 +11354,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -11725,6 +11815,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-imdb** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -11859,7 +11954,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -12134,7 +12229,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -12689,7 +12784,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -12829,6 +12924,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-linkedin** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -12963,7 +13063,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -13103,6 +13203,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-microsoft** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -13237,7 +13342,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -13511,7 +13616,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -13846,7 +13951,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -14040,7 +14145,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -14179,6 +14284,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-outlook** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -14313,7 +14423,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -14452,6 +14562,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-paypal** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -14586,7 +14701,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -14726,6 +14841,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-pinterest** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -14860,7 +14980,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -15321,6 +15441,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-reddit** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -15455,7 +15580,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -15595,6 +15720,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-tumblr** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -15729,7 +15859,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -15870,6 +16000,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-twitch** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -16004,7 +16139,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -16163,6 +16298,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-twitter** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -16317,7 +16457,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -16511,7 +16651,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -16992,6 +17132,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-wikipedia** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -17146,7 +17291,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -17459,7 +17604,7 @@ Tests for page-load performance. The links direct to the actual websites that ar * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 25 * **expected**: pass * **gecko profile entries**: 14000000 @@ -17925,6 +18070,11 @@ Tests for page-load performance. The links direct to the actual websites that ar - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-youtube** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -18065,7 +18215,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests * **accept zero vismet**: true * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 10 * **expected**: pass * **gecko profile entries**: 14000000 @@ -18261,7 +18411,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests * **accept zero vismet**: true * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 10 * **expected**: pass * **gecko profile entries**: 14000000 @@ -18457,7 +18607,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests * **accept zero vismet**: true * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 10 * **expected**: pass * **gecko profile entries**: 14000000 @@ -18653,7 +18803,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests * **accept zero vismet**: true * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 10 * **expected**: pass * **gecko profile entries**: 14000000 @@ -18849,7 +18999,7 @@ Browsertime tests that interact with the webpage. Includes responsiveness tests * **accept zero vismet**: true * **alert on**: fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium + * **apps**: firefox, chrome, chromium, safari * **browser cycles**: 10 * **expected**: pass * **gecko profile entries**: 14000000 @@ -23406,6 +23556,11 @@ Page-load performance test suite on Android. The links direct to the actual webs - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-amazon** + - ❌ + - ❌ + - ❌ + - ❌ * - **browsertime-tp6-profiling-firefox-amazon** - ✅ - ❌ @@ -27880,6 +28035,11 @@ Page-load performance test suite on Android. The links direct to the actual webs - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-facebook** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -29850,6 +30010,11 @@ Page-load performance test suite on Android. The links direct to the actual webs - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-imdb** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -31339,6 +31504,11 @@ Page-load performance test suite on Android. The links direct to the actual webs - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-reddit** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -32908,6 +33078,11 @@ Page-load performance test suite on Android. The links direct to the actual webs - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-wikipedia** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** @@ -33525,6 +33700,11 @@ Page-load performance test suite on Android. The links direct to the actual webs - ❌ - ❌ - ❌ + * - **browsertime-tp6-live-safari-youtube** + - ❌ + - ❌ + - ❌ + - ❌ .. list-table:: **test-windows10-32-qr/opt** diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py index baa04da3386c..ea8b88834667 100644 --- a/testing/raptor/raptor/browsertime/base.py +++ b/testing/raptor/raptor/browsertime/base.py @@ -84,7 +84,8 @@ class Browsertime(Perftest): def build_browser_profile(self): super(Browsertime, self).build_browser_profile() - self.remove_mozprofile_delimiters_from_profile() + if self.profile is not None: + self.remove_mozprofile_delimiters_from_profile() def remove_mozprofile_delimiters_from_profile(self): # Perftest.build_browser_profile uses mozprofile to create the profile and merge in prefs; @@ -370,9 +371,10 @@ class Browsertime(Perftest): self.results_handler.result_dir_for_test_profiling(test), ] ) - priority1_options.extend( - ["--firefox.profileTemplate", str(self.profile.profile)] - ) + if self.profile is not None: + priority1_options.extend( + ["--firefox.profileTemplate", str(self.profile.profile)] + ) # This argument can have duplicates of the value "--firefox.env" so we do not need # to check if it conflicts diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index a4f286895bb1..2a28a8523dde 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -3,14 +3,19 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function -import six import argparse import os import platform +import six from mozlog.commandline import add_logging_group -(FIREFOX, CHROME, CHROMIUM) = DESKTOP_APPS = ["firefox", "chrome", "chromium"] +(FIREFOX, CHROME, CHROMIUM, SAFARI) = DESKTOP_APPS = [ + "firefox", + "chrome", + "chromium", + "safari", +] (GECKOVIEW, REFBROW, FENIX, CHROME_ANDROID) = FIREFOX_ANDROID_APPS = [ "geckoview", "refbrow", @@ -23,6 +28,7 @@ APPS = { FIREFOX: {"long_name": "Firefox Desktop"}, CHROME: {"long_name": "Google Chrome Desktop"}, CHROMIUM: {"long_name": "Google Chromium Desktop"}, + SAFARI: {"long_name": "Safari Desktop"}, GECKOVIEW: { "long_name": "Firefox GeckoView on Android", "default_activity": "org.mozilla.geckoview_example.GeckoViewActivity", diff --git a/testing/raptor/raptor/perftest.py b/testing/raptor/raptor/perftest.py index bc21680e1b9a..66d2ade1e7db 100644 --- a/testing/raptor/raptor/perftest.py +++ b/testing/raptor/raptor/perftest.py @@ -22,7 +22,6 @@ import mozprocess import mozproxy.utils as mpu import mozversion import six - from mozprofile import create_profile from mozproxy import get_playback @@ -39,10 +38,10 @@ for path in paths: sys.path.insert(0, path) from cmdline import FIREFOX_ANDROID_APPS -from condprof.client import get_profile, ProfileNotFoundError +from condprof.client import ProfileNotFoundError, get_profile from condprof.util import get_current_platform -from logger.logger import RaptorLogger from gecko_profile import GeckoProfile +from logger.logger import RaptorLogger from results import RaptorResultsHandler LOG = RaptorLogger(component="raptor-perftest") @@ -250,11 +249,11 @@ class Perftest(object): def build_conditioned_profile(self): # Late import so python-test doesn't import it import asyncio - from condprof.runner import Runner # The following import patchs an issue with invalid # content-type, see bug 1655869 from condprof import patch # noqa + from condprof.runner import Runner if not getattr(self, "browsertime"): raise Exception( @@ -385,7 +384,10 @@ class Perftest(object): return self.conditioned_profile_copy def build_browser_profile(self): - if ( + if self.config["app"] in ["safari"]: + self.profile = None + return + elif ( self.config["app"] in ["chrome", "chromium", "chrome-m"] or self.config.get("conditioned_profile") is None ): @@ -448,7 +450,7 @@ class Perftest(object): if test.get("playback") is not None and self.playback is None: self.start_playback(test) - if test.get("preferences") is not None: + if test.get("preferences") is not None and self.config["app"] not in "safari": self.set_browser_test_prefs(test["preferences"]) @abstractmethod @@ -808,10 +810,20 @@ class PerftestDesktop(Perftest): # Fall-back method to get browser version on desktop try: - if ( - "linux" in self.config["platform"] - or "mac" in self.config["platform"] - ): + if "mac" in self.config["platform"]: + import plistlib + + for plist_file in ("version.plist", "Info.plist"): + try: + binary_path = pathlib.Path(self.config["binary"]) + plist_path = binary_path.parent.parent.joinpath(plist_file) + with plist_path.open("rb") as plist: + plist = plistlib.load(plist) + except FileNotFoundError: + pass + browser_name = self.config["app"] + browser_version = plist.get("CFBundleShortVersionString") + elif "linux" in self.config["platform"]: command = [self.config["binary"], "--version"] proc = mozprocess.ProcessHandler(command) proc.run(timeout=10, outputTimeout=10) diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py index 94edbd940374..118e1f845210 100644 --- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -24,19 +24,18 @@ try: except ImportError: build = None -from mozlog import commandline -from mozprofile.cli import parse_preferences, parse_key_value - -from browsertime import BrowsertimeDesktop, BrowsertimeAndroid -from cmdline import parse_args, CHROMIUM_DISTROS +from browsertime import BrowsertimeAndroid, BrowsertimeDesktop +from cmdline import CHROMIUM_DISTROS, DESKTOP_APPS, parse_args from logger.logger import RaptorLogger from manifest import get_raptor_test_list +from mozlog import commandline +from mozprofile.cli import parse_key_value, parse_preferences from signal_handler import SignalHandler from utils import view_gecko_profile_from_raptor from webextension import ( - WebExtensionFirefox, - WebExtensionDesktopChrome, WebExtensionAndroid, + WebExtensionDesktopChrome, + WebExtensionFirefox, ) LOG = RaptorLogger(component="raptor-main") @@ -95,7 +94,7 @@ def main(args=sys.argv[1:]): if key.startswith("browsertime_"): inner_kwargs[key] = outer_kwargs.get(key) - if args.app == "firefox" or args.app in CHROMIUM_DISTROS: + if args.app in DESKTOP_APPS: klass = BrowsertimeDesktop else: klass = BrowsertimeAndroid diff --git a/testing/raptor/raptor/results.py b/testing/raptor/raptor/results.py index f9660b15da96..3eaae7e96282 100644 --- a/testing/raptor/raptor/results.py +++ b/testing/raptor/raptor/results.py @@ -15,7 +15,7 @@ from io import open import six from logger.logger import RaptorLogger -from output import RaptorOutput, BrowsertimeOutput +from output import BrowsertimeOutput, RaptorOutput LOG = RaptorLogger(component="perftest-results-handler") KNOWN_TEST_MODIFIERS = [ @@ -628,14 +628,15 @@ class BrowsertimeResultsHandler(PerftestResultsHandler): for bt, raptor in conversion: if measure is not None and bt not in measure: continue - # chrome we just measure fcp and loadtime; skip fnbpaint and dcf + # chrome and safari we just measure fcp and loadtime; skip fnbpaint and dcf if ( self.app - and ( - "chrome" in self.app.lower() - or "chromium" in self.app.lower() + and self.app.lower() in ["chrome", "chromium", "safari"] + and bt + in ( + "fnbpaint", + "dcf", ) - and bt in ("fnbpaint", "dcf") ): continue diff --git a/testing/raptor/raptor/tests/benchmarks/ares6.ini b/testing/raptor/raptor/tests/benchmarks/ares6.ini index c42bd39865ae..a2cc22d79189 100644 --- a/testing/raptor/raptor/tests/benchmarks/ares6.ini +++ b/testing/raptor/raptor/tests/benchmarks/ares6.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari gecko_profile_entries = 14000000 gecko_profile_interval = 1 lower_is_better = true diff --git a/testing/raptor/raptor/tests/benchmarks/assorted-dom.ini b/testing/raptor/raptor/tests/benchmarks/assorted-dom.ini index 025f36076b3d..2394fec7c92a 100644 --- a/testing/raptor/raptor/tests/benchmarks/assorted-dom.ini +++ b/testing/raptor/raptor/tests/benchmarks/assorted-dom.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari fetch_task = assorted-dom gecko_profile_entries = 2000000 gecko_profile_interval = 1 diff --git a/testing/raptor/raptor/tests/benchmarks/jetstream2.ini b/testing/raptor/raptor/tests/benchmarks/jetstream2.ini index 6b5cbe2b2389..24b9d7928b2a 100644 --- a/testing/raptor/raptor/tests/benchmarks/jetstream2.ini +++ b/testing/raptor/raptor/tests/benchmarks/jetstream2.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari fetch_task = jetstream2 gecko_profile_entries = 14000000 gecko_profile_interval = 1 diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer.ini b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer.ini index cb1d6647a9f1..3fb89358333a 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-animometer.ini +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-animometer.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari gecko_profile_entries = 8000000 gecko_profile_interval = 1 lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite.ini b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite.ini index e116394ba10f..83a8110d586d 100644 --- a/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite.ini +++ b/testing/raptor/raptor/tests/benchmarks/motionmark-htmlsuite.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari gecko_profile_entries = 8000000 gecko_profile_interval = 1 lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini index 5ccfb8761d18..e65a908d6009 100644 --- a/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini +++ b/testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari gecko_profile_entries = 14000000 gecko_profile_interval = 1 lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/stylebench.ini b/testing/raptor/raptor/tests/benchmarks/stylebench.ini index e70ad96bb3a8..3b24d35c2eb4 100644 --- a/testing/raptor/raptor/tests/benchmarks/stylebench.ini +++ b/testing/raptor/raptor/tests/benchmarks/stylebench.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari gecko_profile_entries = 8000000 gecko_profile_interval = 1 lower_is_better = false diff --git a/testing/raptor/raptor/tests/benchmarks/sunspider.ini b/testing/raptor/raptor/tests/benchmarks/sunspider.ini index 5d03acb0222e..15ef1eef063c 100644 --- a/testing/raptor/raptor/tests/benchmarks/sunspider.ini +++ b/testing/raptor/raptor/tests/benchmarks/sunspider.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari gecko_profile_entries = 8000000 gecko_profile_interval = 1 lower_is_better = true diff --git a/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.ini b/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.ini index c28878581808..1dcaf9640e6a 100644 --- a/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.ini +++ b/testing/raptor/raptor/tests/benchmarks/unity-webgl-desktop.ini @@ -6,7 +6,7 @@ [DEFAULT] alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari fetch_task = unity-webgl gecko_profile_entries = 8000000 gecko_profile_interval = 1 diff --git a/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini b/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini index 99fb961f226c..9df1ac9366e4 100644 --- a/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini +++ b/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini @@ -8,7 +8,7 @@ accept_zero_vismet = true alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari browser_cycles = 25 gecko_profile_entries = 14000000 gecko_profile_interval = 1 diff --git a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini index 415049f736c1..c010d228ea6b 100644 --- a/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini +++ b/testing/raptor/raptor/tests/tp6/desktop/browsertime-tp6.ini @@ -7,7 +7,7 @@ [DEFAULT] alert_on = fcp, loadtime, ContentfulSpeedIndex, PerceptualSpeedIndex, SpeedIndex, FirstVisualChange, LastVisualChange alert_threshold = 2.0 -apps = firefox, chrome, chromium +apps = firefox, chrome, chromium, safari browser_cycles = 25 gecko_profile_entries = 14000000 gecko_profile_interval = 1