diff --git a/testing/perfdocs/generated/raptor.rst b/testing/perfdocs/generated/raptor.rst index cb0c215205f0..1c8694ebccb7 100644 --- a/testing/perfdocs/generated/raptor.rst +++ b/testing/perfdocs/generated/raptor.rst @@ -29,2142 +29,7 @@ The following documents all testing we have for Raptor. Benchmarks ---------- -Standard benchmarks are third-party tests (i.e. Speedometer) that we have integrated into Raptor to run per-commit in our production CI. (FF: Firefox, CH: Chrome, CU: Chromium) - -.. dropdown:: ares6 (FF, CH, CU) - :container: + anchor-id-ares6-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **page cycles**: 4 - * **page timeout**: 270000 - * **subtest lower is better**: true - * **subtest unit**: ms - * **test url**: `:/ARES-6/index.html?raptor>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-ares6-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-ares6-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-ares6-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-ares6-e10s: None - * browsertime-benchmark-chromium-ares6-e10s: None - * browsertime-benchmark-firefox-ares6-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-firefox-ares6-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-ares6-e10s: trunk - * browsertime-benchmark-firefox-ares6-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-ares6-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-ares6-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-ares6-e10s: None - * browsertime-benchmark-chromium-ares6-e10s: None - * browsertime-benchmark-firefox-ares6-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-ares6-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-ares6-e10s: None - * browsertime-benchmark-firefox-ares6-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-ares6-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-ares6-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-ares6-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-ares6-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-ares6-e10s: None - * browsertime-benchmark-chromium-ares6-e10s: None - * browsertime-benchmark-firefox-ares6-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-firefox-ares6-e10s: None - - -.. dropdown:: assorted-dom (FF, CH, CU) - :container: + anchor-id-assorted-dom-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **fetch task**: assorted-dom - * **gecko profile entries**: 2000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 60000 - * **screen capture**: true - * **test url**: `:/assorted-dom/assorted/driver.html?raptor>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-assorted-dom-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-assorted-dom-e10s: None - * browsertime-benchmark-chromium-assorted-dom-e10s: None - * browsertime-benchmark-firefox-assorted-dom-e10s: autoland - * test-linux1804-64/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: trunk - * browsertime-benchmark-firefox-assorted-dom-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-assorted-dom-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-assorted-dom-e10s: None - * browsertime-benchmark-chromium-assorted-dom-e10s: None - * browsertime-benchmark-firefox-assorted-dom-e10s: autoland - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: None - * browsertime-benchmark-firefox-assorted-dom-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-assorted-dom-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-assorted-dom-e10s: None - * browsertime-benchmark-chromium-assorted-dom-e10s: None - * browsertime-benchmark-firefox-assorted-dom-e10s: autoland - * test-windows10-64/opt - * browsertime-benchmark-firefox-assorted-dom-e10s: None - - -.. dropdown:: jetstream2 (FF, CH, CU) - :container: + anchor-id-jetstream2-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **fetch task**: jetstream2 - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 4 - * **page timeout**: 2000000 - * **subtest lower is better**: false - * **subtest unit**: score - * **test url**: `:/JetStream2/index.html?raptor>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-jetstream2-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-jetstream2-e10s: mozilla-central - * browsertime-benchmark-firefox-jetstream2-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-jetstream2-e10s: None - * browsertime-benchmark-chromium-jetstream2-e10s: None - * browsertime-benchmark-firefox-jetstream2-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-firefox-jetstream2-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-jetstream2-e10s: mozilla-central - * browsertime-benchmark-firefox-jetstream2-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-jetstream2-e10s: mozilla-central - * browsertime-benchmark-firefox-jetstream2-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-jetstream2-e10s: None - * browsertime-benchmark-chromium-jetstream2-e10s: None - * browsertime-benchmark-firefox-jetstream2-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-jetstream2-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-jetstream2-e10s: None - * browsertime-benchmark-firefox-jetstream2-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-jetstream2-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-jetstream2-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-jetstream2-e10s: mozilla-central - * browsertime-benchmark-firefox-jetstream2-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-jetstream2-e10s: None - * browsertime-benchmark-chromium-jetstream2-e10s: None - * browsertime-benchmark-firefox-jetstream2-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-firefox-jetstream2-e10s: None - - -.. dropdown:: motionmark-animometer (FF, CH, CU) - :container: + anchor-id-motionmark-animometer-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 1 - * **page timeout**: 600000 - * **test url**: `:/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=Animometer&raptor=true&oskey={platform}>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-motionmark-animometer-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-motionmark-animometer-e10s: None - * browsertime-benchmark-chromium-motionmark-animometer-e10s: None - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: trunk - * browsertime-benchmark-firefox-motionmark-animometer-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-motionmark-animometer-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-motionmark-animometer-e10s: None - * browsertime-benchmark-chromium-motionmark-animometer-e10s: None - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * browsertime-benchmark-firefox-motionmark-animometer-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-motionmark-animometer-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-motionmark-animometer-e10s: None - * browsertime-benchmark-chromium-motionmark-animometer-e10s: None - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-firefox-motionmark-animometer-e10s: None - - -.. dropdown:: motionmark-htmlsuite (FF, CH, CU) - :container: + anchor-id-motionmark-htmlsuite-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 5 - * **page timeout**: 600000 - * **test url**: `:/MotionMark/developer.html?test-interval=15&display=minimal&tiles=big&controller=fixed&frame-rate=30&kalman-process-error=1&kalman-measurement-error=4&time-measurement=performance&suite-name=HTMLsuite&raptor=true&oskey={platform}>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-motionmark-htmlsuite-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-motionmark-htmlsuite-e10s: None - * browsertime-benchmark-chromium-motionmark-htmlsuite-e10s: None - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: autoland - * test-linux1804-64/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: trunk - * browsertime-benchmark-firefox-motionmark-htmlsuite-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-motionmark-htmlsuite-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-motionmark-htmlsuite-e10s: None - * browsertime-benchmark-chromium-motionmark-htmlsuite-e10s: None - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: autoland - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: None - * browsertime-benchmark-firefox-motionmark-htmlsuite-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-motionmark-htmlsuite-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-motionmark-htmlsuite-e10s: None - * browsertime-benchmark-chromium-motionmark-htmlsuite-e10s: None - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: autoland - * test-windows10-64/opt - * browsertime-benchmark-firefox-motionmark-htmlsuite-e10s: None - - -.. dropdown:: raptor-speedometer-geckoview (GV) - :container: + anchor-id-raptor-speedometer-geckoview-b - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 1 - * **page timeout**: 420000 - * **subtest lower is better**: true - * **subtest unit**: ms - * **test url**: `:/Speedometer/index.html?raptor>`__ - * **type**: benchmark - * **unit**: score - - -.. dropdown:: raptor-youtube-playback-av1-sfr-chrome (CH) - :container: + anchor-id-raptor-youtube-playback-av1-sfr-chrome-b - - * **alert threshold**: 2.0 - * **apps**: chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-av1-sfr-fenix (FE) - :container: + anchor-id-raptor-youtube-playback-av1-sfr-fenix-b - - * **alert threshold**: 2.0 - * **apps**: fenix - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-av1-sfr-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-av1-sfr-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-av1-sfr-geckoview (GV) - :container: + anchor-id-raptor-youtube-playback-av1-sfr-geckoview-b - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-h264-1080p30-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-h264-1080p30-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **lower is better**: true - * **page cycles**: 20 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-h264-1080p60-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-h264-1080p60-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **lower is better**: true - * **page cycles**: 20 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-h264-full-1080p30-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-h264-full-1080p30-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **lower is better**: true - * **page cycles**: 20 - * **page timeout**: 2700000 - * **preferences**: {"full-screen-api.allow-trusted-requests-only": false, -"full-screen-api.warning.timeout": 0} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-h264-full-1080p60-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-h264-full-1080p60-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **lower is better**: true - * **page cycles**: 20 - * **page timeout**: 2700000 - * **preferences**: {"full-screen-api.allow-trusted-requests-only": false, -"full-screen-api.warning.timeout": 0} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-h264-sfr-chrome (CH) - :container: + anchor-id-raptor-youtube-playback-h264-sfr-chrome-b - - * **alert threshold**: 2.0 - * **apps**: chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-h264-sfr-fenix (FE) - :container: + anchor-id-raptor-youtube-playback-h264-sfr-fenix-b - - * **alert threshold**: 2.0 - * **apps**: fenix - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-h264-sfr-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-h264-sfr-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-h264-sfr-geckoview (GV) - :container: + anchor-id-raptor-youtube-playback-h264-sfr-geckoview-b - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-hfr-chrome (CH) - :container: + anchor-id-raptor-youtube-playback-hfr-chrome-b - - * **alert on**: H2641080p60fps@1X_dropped_frames, -H264720p60fps@1X_dropped_frames - * **alert threshold**: 2.0 - * **apps**: chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-hfr-fenix (FE) - :container: + anchor-id-raptor-youtube-playback-hfr-fenix-b - - * **alert on**: H2641080p60fps@1X_dropped_frames, -H264720p60fps@1X_dropped_frames - * **alert threshold**: 2.0 - * **apps**: fenix - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-hfr-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-hfr-firefox-b - - * **alert on**: H2641080p60fps@1X_dropped_frames, -H264720p60fps@1X_dropped_frames - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-hfr-geckoview (GV) - :container: + anchor-id-raptor-youtube-playback-hfr-geckoview-b - - * **alert on**: H2641080p60fps@1X_dropped_frames, -H264720p60fps@1X_dropped_frames - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-v9-1080p30-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-v9-1080p30-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **lower is better**: true - * **page cycles**: 20 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-v9-1080p60-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-v9-1080p60-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **lower is better**: true - * **page cycles**: 20 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-v9-full-1080p30-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-v9-full-1080p30-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **lower is better**: true - * **page cycles**: 20 - * **page timeout**: 2700000 - * **preferences**: {"full-screen-api.allow-trusted-requests-only": false, -"full-screen-api.warning.timeout": 0} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-v9-full-1080p60-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-v9-full-1080p60-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **lower is better**: true - * **page cycles**: 20 - * **page timeout**: 2700000 - * **preferences**: {"full-screen-api.allow-trusted-requests-only": false, -"full-screen-api.warning.timeout": 0} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-vp9-sfr-chrome (CH) - :container: + anchor-id-raptor-youtube-playback-vp9-sfr-chrome-b - - * **alert threshold**: 2.0 - * **apps**: chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-vp9-sfr-fenix (FE) - :container: + anchor-id-raptor-youtube-playback-vp9-sfr-fenix-b - - * **alert threshold**: 2.0 - * **apps**: fenix - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-vp9-sfr-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-vp9-sfr-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-vp9-sfr-geckoview (GV) - :container: + anchor-id-raptor-youtube-playback-vp9-sfr-geckoview-b - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-h264-sfr-chrome (CH) - :container: + anchor-id-raptor-youtube-playback-widevine-h264-sfr-chrome-b - - * **alert threshold**: 2.0 - * **apps**: chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-h264-sfr-fenix (FE) - :container: + anchor-id-raptor-youtube-playback-widevine-h264-sfr-fenix-b - - * **alert threshold**: 2.0 - * **apps**: fenix - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-h264-sfr-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-widevine-h264-sfr-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-h264-sfr-geckoview (GV) - :container: + anchor-id-raptor-youtube-playback-widevine-h264-sfr-geckoview-b - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-hfr-chrome (CH) - :container: + anchor-id-raptor-youtube-playback-widevine-hfr-chrome-b - - * **alert threshold**: 2.0 - * **apps**: chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-hfr-fenix (FE) - :container: + anchor-id-raptor-youtube-playback-widevine-hfr-fenix-b - - * **alert threshold**: 2.0 - * **apps**: fenix - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-hfr-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-widevine-hfr-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-hfr-geckoview (GV) - :container: + anchor-id-raptor-youtube-playback-widevine-hfr-geckoview-b - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 1800000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-vp9-sfr-chrome (CH) - :container: + anchor-id-raptor-youtube-playback-widevine-vp9-sfr-chrome-b - - * **alert threshold**: 2.0 - * **apps**: chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-vp9-sfr-fenix (FE) - :container: + anchor-id-raptor-youtube-playback-widevine-vp9-sfr-fenix-b - - * **alert threshold**: 2.0 - * **apps**: fenix - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-vp9-sfr-firefox (FF) - :container: + anchor-id-raptor-youtube-playback-widevine-vp9-sfr-firefox-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: raptor-youtube-playback-widevine-vp9-sfr-geckoview (GV) - :container: + anchor-id-raptor-youtube-playback-widevine-vp9-sfr-geckoview-b - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: speedometer (FF, CH, CU, FE, GV, RB, CH-M) - :container: + anchor-id-speedometer-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 5 - * **page timeout**: 180000 - * **subtest lower is better**: true - * **subtest unit**: ms - * **test url**: `:/Speedometer/index.html?raptor>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-speedometer-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-speedometer-e10s: None - * browsertime-benchmark-chromium-speedometer-e10s: None - * browsertime-benchmark-firefox-speedometer-e10s: autoland - * test-linux1804-64/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: trunk - * browsertime-benchmark-firefox-speedometer-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-speedometer-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-speedometer-e10s: None - * browsertime-benchmark-chromium-speedometer-e10s: None - * browsertime-benchmark-firefox-speedometer-e10s: autoland - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * browsertime-benchmark-firefox-speedometer-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-speedometer-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-speedometer-e10s: None - * browsertime-benchmark-chromium-speedometer-e10s: None - * browsertime-benchmark-firefox-speedometer-e10s: autoland - * test-windows10-64/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - - * **alert threshold**: 2.0 - * **apps**: fenix, geckoview, refbrow, chrome-m - * **expected**: pass - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 1 - * **page timeout**: 420000 - * **subtest lower is better**: true - * **subtest unit**: ms - * **test url**: `:/Speedometer/index.html?raptor>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-speedometer-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-speedometer-e10s: None - * browsertime-benchmark-chromium-speedometer-e10s: None - * browsertime-benchmark-firefox-speedometer-e10s: autoland - * test-linux1804-64/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: trunk - * browsertime-benchmark-firefox-speedometer-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-speedometer-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-speedometer-e10s: None - * browsertime-benchmark-chromium-speedometer-e10s: None - * browsertime-benchmark-firefox-speedometer-e10s: autoland - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * browsertime-benchmark-firefox-speedometer-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-speedometer-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-speedometer-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-speedometer-e10s: None - * browsertime-benchmark-chromium-speedometer-e10s: None - * browsertime-benchmark-firefox-speedometer-e10s: autoland - * test-windows10-64/opt - * browsertime-benchmark-firefox-speedometer-e10s: None - - -.. dropdown:: stylebench (FF, CH, CU) - :container: + anchor-id-stylebench-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 5 - * **page timeout**: 120000 - * **subtest lower is better**: true - * **subtest unit**: ms - * **test url**: `:/StyleBench/index.html?raptor>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-stylebench-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-stylebench-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-stylebench-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-stylebench-e10s: None - * browsertime-benchmark-chromium-stylebench-e10s: None - * browsertime-benchmark-firefox-stylebench-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-firefox-stylebench-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-stylebench-e10s: trunk - * browsertime-benchmark-firefox-stylebench-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-stylebench-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-stylebench-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-stylebench-e10s: None - * browsertime-benchmark-chromium-stylebench-e10s: None - * browsertime-benchmark-firefox-stylebench-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-stylebench-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-stylebench-e10s: None - * browsertime-benchmark-firefox-stylebench-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-stylebench-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-stylebench-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-stylebench-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-stylebench-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-stylebench-e10s: None - * browsertime-benchmark-chromium-stylebench-e10s: None - * browsertime-benchmark-firefox-stylebench-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-firefox-stylebench-e10s: None - - -.. dropdown:: sunspider (FF, CH, CU) - :container: + anchor-id-sunspider-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **page cycles**: 5 - * **page timeout**: 55000 - * **test url**: `:/SunSpider/sunspider-1.0.1/sunspider-1.0.1/driver.html?raptor>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-sunspider-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-sunspider-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-sunspider-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-sunspider-e10s: None - * browsertime-benchmark-chromium-sunspider-e10s: None - * browsertime-benchmark-firefox-sunspider-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-firefox-sunspider-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-sunspider-e10s: trunk - * browsertime-benchmark-firefox-sunspider-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-sunspider-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-sunspider-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-sunspider-e10s: None - * browsertime-benchmark-chromium-sunspider-e10s: None - * browsertime-benchmark-firefox-sunspider-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-sunspider-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-sunspider-e10s: None - * browsertime-benchmark-firefox-sunspider-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-sunspider-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-sunspider-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-sunspider-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-sunspider-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-sunspider-e10s: None - * browsertime-benchmark-chromium-sunspider-e10s: None - * browsertime-benchmark-firefox-sunspider-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-firefox-sunspider-e10s: None - - -.. dropdown:: unity-webgl (FF, CH, CU, FE, RB, FE, CH-M) - :container: + anchor-id-unity-webgl-b - - * **alert threshold**: 2.0 - * **apps**: geckoview, refbrow, fenix, chrome-m - * **expected**: pass - * **fetch task**: unity-webgl - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 1 - * **page timeout**: 420000 - * **test url**: `:/unity-webgl/index.html?raptor>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: autoland - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-unity-webgl-e10s: None - * browsertime-benchmark-chromium-unity-webgl-e10s: None - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: autoland - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: autoland - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-unity-webgl-e10s: None - * browsertime-benchmark-chromium-unity-webgl-e10s: None - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: autoland - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-unity-webgl-e10s: None - * browsertime-benchmark-chromium-unity-webgl-e10s: None - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **fetch task**: unity-webgl - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: false - * **page cycles**: 5 - * **page timeout**: 420000 - * **test url**: `:/unity-webgl/index.html?raptor>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: autoland - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-unity-webgl-e10s: None - * browsertime-benchmark-chromium-unity-webgl-e10s: None - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: autoland - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: autoland - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-unity-webgl-e10s: None - * browsertime-benchmark-chromium-unity-webgl-e10s: None - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: autoland - * browsertime-benchmark-firefox-unity-webgl-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-unity-webgl-e10s: None - * browsertime-benchmark-chromium-unity-webgl-e10s: None - * browsertime-benchmark-firefox-unity-webgl-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-firefox-unity-webgl-e10s: None - - -.. dropdown:: wasm-godot (FF, CH, CU) - :container: + anchor-id-wasm-godot-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **newtab per cycle**: true - * **page cycles**: 5 - * **page timeout**: 120000 - * **test url**: `/wasm-godot/index.html>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-wasm-chrome-wasm-godot-e10s: None - * browsertime-benchmark-wasm-chromium-wasm-godot-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-fis-e10s: None - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-wasm-chrome-wasm-godot-e10s: None - * browsertime-benchmark-wasm-chromium-wasm-godot-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-wasm-chrome-wasm-godot-e10s: None - * browsertime-benchmark-wasm-chromium-wasm-godot-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-e10s: None - - -.. dropdown:: wasm-godot-baseline (FF) - :container: + anchor-id-wasm-godot-baseline-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **newtab per cycle**: true - * **page cycles**: 5 - * **page timeout**: 120000 - * **preferences**: {"javascript.options.wasm_baselinejit": true, -"javascript.options.wasm_optimizingjit": false} - * **test url**: `/wasm-godot/index.html>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-fis-e10s: None - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-baseline-e10s: None - - -.. dropdown:: wasm-godot-optimizing (FF) - :container: + anchor-id-wasm-godot-optimizing-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 8000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **newtab per cycle**: true - * **page cycles**: 5 - * **page timeout**: 120000 - * **preferences**: {"javascript.options.wasm_baselinejit": false, -"javascript.options.wasm_optimizingjit": true} - * **test url**: `/wasm-godot/index.html>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-fis-e10s: None - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-wasm-firefox-wasm-godot-optimizing-e10s: None - - -.. dropdown:: wasm-misc (FF, CH, CU) - :container: + anchor-id-wasm-misc-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **fetch task**: wasm-misc - * **gecko profile entries**: 4000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **page cycles**: 5 - * **page timeout**: 1200000 - * **test url**: `:/wasm-misc/index.html?raptor>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-wasm-chrome-wasm-misc-e10s: None - * browsertime-benchmark-wasm-chromium-wasm-misc-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-fis-e10s: None - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-wasm-chrome-wasm-misc-e10s: None - * browsertime-benchmark-wasm-chromium-wasm-misc-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-wasm-chrome-wasm-misc-e10s: None - * browsertime-benchmark-wasm-chromium-wasm-misc-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-e10s: None - - -.. dropdown:: wasm-misc-baseline (FF) - :container: + anchor-id-wasm-misc-baseline-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **fetch task**: wasm-misc - * **gecko profile entries**: 4000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **page cycles**: 5 - * **page timeout**: 1200000 - * **preferences**: {"javascript.options.wasm_baselinejit": true, -"javascript.options.wasm_optimizingjit": false} - * **test url**: `:/wasm-misc/index.html?raptor>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-fis-e10s: None - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-baseline-e10s: None - - -.. dropdown:: wasm-misc-optimizing (FF) - :container: + anchor-id-wasm-misc-optimizing-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **fetch task**: wasm-misc - * **gecko profile entries**: 4000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **page cycles**: 5 - * **page timeout**: 1200000 - * **preferences**: {"javascript.options.wasm_baselinejit": false, -"javascript.options.wasm_optimizingjit": true} - * **test url**: `:/wasm-misc/index.html?raptor>`__ - * **type**: benchmark - * **unit**: ms - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-fis-e10s: None - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: mozilla-beta, trunk - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-wasm-firefox-wasm-misc-optimizing-e10s: None - - -.. dropdown:: webaudio (FF, CH, CU) - :container: + anchor-id-webaudio-b - - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **expected**: pass - * **gecko profile entries**: 4000000 - * **gecko profile interval**: 1 - * **lower is better**: true - * **page cycles**: 5 - * **page timeout**: 360000 - * **test url**: `:/webaudio/index.html?raptor>`__ - * **type**: benchmark - * **unit**: score - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-benchmark-firefox-webaudio-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-benchmark-firefox-webaudio-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-webaudio-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-benchmark-chrome-webaudio-e10s: None - * browsertime-benchmark-chromium-webaudio-e10s: None - * browsertime-benchmark-firefox-webaudio-e10s: None - * test-linux1804-64/opt - * browsertime-benchmark-firefox-webaudio-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-benchmark-firefox-webaudio-e10s: trunk - * browsertime-benchmark-firefox-webaudio-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-benchmark-firefox-webaudio-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-webaudio-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-benchmark-chrome-webaudio-e10s: None - * browsertime-benchmark-chromium-webaudio-e10s: None - * browsertime-benchmark-firefox-webaudio-e10s: None - * test-windows10-32-qr/opt - * browsertime-benchmark-firefox-webaudio-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-benchmark-firefox-webaudio-e10s: None - * browsertime-benchmark-firefox-webaudio-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-benchmark-firefox-webaudio-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-benchmark-firefox-webaudio-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-benchmark-firefox-webaudio-e10s: mozilla-beta, trunk - * browsertime-benchmark-firefox-webaudio-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-benchmark-chrome-webaudio-e10s: None - * browsertime-benchmark-chromium-webaudio-e10s: None - * browsertime-benchmark-firefox-webaudio-e10s: None - * test-windows10-64/opt - * browsertime-benchmark-firefox-webaudio-e10s: None - - -.. dropdown:: youtube-playback (FF, GV, FE, RB, CH) - :container: + anchor-id-youtube-playback-b - - * **alert on**: H264.1080p30@1X_dropped_frames, -H264.1080p60@1X_dropped_frames, -H264.1440p30@1X_dropped_frames, -H264.144p15@1X_dropped_frames, -H264.2160p30@1X_dropped_frames, -H264.240p30@1X_dropped_frames, -H264.360p30@1X_dropped_frames, -H264.480p30@1X_dropped_frames, -H264.720p30@1X_dropped_frames, -H264.720p60@1X_dropped_frames, -VP9.1080p30@1X_dropped_frames, -VP9.1080p60@1X_dropped_frames, -VP9.1440p30@1X_dropped_frames, -VP9.1440p60@1X_dropped_frames, -VP9.144p30@1X_dropped_frames, -VP9.2160p30@1X_dropped_frames, -VP9.2160p60@1X_dropped_frames, -VP9.240p30@1X_dropped_frames, -VP9.360p30@1X_dropped_frames, -VP9.480p30@1X_dropped_frames, -VP9.720p30@1X_dropped_frames, -VP9.720p60@1X_dropped_frames - * **alert threshold**: 2.0 - * **apps**: firefox, geckoview, fenix,refbrow, chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-av1-sfr (FF , GV, FE, RB, CH) - :container: + anchor-id-youtube-playback-av1-sfr-b - - * **alert threshold**: 2.0 - * **apps**: firefox, geckoview, fenix, refbrow, chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-h264-1080p30 (FF) - :container: + anchor-id-youtube-playback-h264-1080p30-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-h264-1080p60 (FF) - :container: + anchor-id-youtube-playback-h264-1080p60-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-h264-full-1080p30 (FF) - :container: + anchor-id-youtube-playback-h264-full-1080p30-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"full-screen-api.allow-trusted-requests-only": false, -"full-screen-api.warning.timeout": 0} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-h264-full-1080p60 (FF) - :container: + anchor-id-youtube-playback-h264-full-1080p60-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"full-screen-api.allow-trusted-requests-only": false, -"full-screen-api.warning.timeout": 0} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-h264-sfr (FF , GV, FE, RB, CH) - :container: + anchor-id-youtube-playback-h264-sfr-b - - * **alert threshold**: 2.0 - * **apps**: firefox, geckoview, fenix, refbrow, chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: youtube-playback-hfr (FF , GV, FE, RB, CH) - :container: + anchor-id-youtube-playback-hfr-b - - * **alert on**: H2641080p60fps@1X_dropped_frames, -H264720p60fps@1X_dropped_frames - * **alert threshold**: 2.0 - * **apps**: firefox, geckoview, fenix, refbrow, chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: youtube-playback-v9-1080p30 (FF) - :container: + anchor-id-youtube-playback-v9-1080p30-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-v9-1080p60 (FF) - :container: + anchor-id-youtube-playback-v9-1080p60-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-v9-full-1080p30 (FF) - :container: + anchor-id-youtube-playback-v9-full-1080p30-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"full-screen-api.allow-trusted-requests-only": false, -"full-screen-api.warning.timeout": 0} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-v9-full-1080p60 (FF) - :container: + anchor-id-youtube-playback-v9-full-1080p60-b - - * **alert threshold**: 2.0 - * **apps**: firefox - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"full-screen-api.allow-trusted-requests-only": false, -"full-screen-api.warning.timeout": 0} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-vp9-sfr (FF , GV, FE, RB, CH) - :container: + anchor-id-youtube-playback-vp9-sfr-b - - * **alert threshold**: 2.0 - * **apps**: firefox, geckoview, fenix, refbrow, chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: youtube-playback-widevine-h264-sfr (FF , GV, FE, RB, CH) - :container: + anchor-id-youtube-playback-widevine-h264-sfr-b - - * **alert threshold**: 2.0 - * **apps**: firefox, geckoview, fenix, refbrow, chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-widevine-hfr (FF , GV, FE, RB, CH) - :container: + anchor-id-youtube-playback-widevine-hfr-b - - * **alert threshold**: 2.0 - * **apps**: firefox, geckoview, fenix, refbrow, chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - - -.. dropdown:: youtube-playback-widevine-vp9-sfr (FF , GV, FE, RB, CH) - :container: + anchor-id-youtube-playback-widevine-vp9-sfr-b - - * **alert threshold**: 2.0 - * **apps**: firefox, geckoview, fenix, refbrow, chrome - * **expected**: pass - * **gecko profile entries**: 50000000 - * **gecko profile interval**: 1000 - * **gecko profile threads**: MediaPlayback - * **lower is better**: true - * **page cycles**: 1 - * **page timeout**: 2700000 - * **preferences**: {"media.eme.enabled": true, -"media.gmp-manager.updateEnabled": true, -"media.eme.require-app-approval": false} - * **subtest lower is better**: true - * **subtest unit**: score - * **test url**: ``__ - * **type**: benchmark - * **unit**: score - * **use live sites**: true - +Standard benchmarks are third-party tests (i.e. Speedometer) that we have integrated into Raptor to run per-commit in our production CI. Custom @@ -5366,431 +3231,7 @@ Tests for page-load performance. The links direct to the actual websites that ar Live ---- -A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms. (WX: WebExtension, BT: Browsertime, FF: Firefox, CH: Chrome, CU: Chromium, GV: Geckoview, RB: Refbrow, FE: Fenix, CH-M: Chrome mobile) - -.. dropdown:: booking-sf (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-booking-sf-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: cnn (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-cnn-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-tp6-essential-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-tp6-essential-firefox-cnn-e10s: mozilla-beta, trunk - * browsertime-tp6-essential-firefox-cnn-fis-e10s: mozilla-central - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-fis-e10s: None - * test-linux1804-64-shippable/opt - * browsertime-tp6-essential-chrome-cnn-e10s: None - * browsertime-tp6-essential-chromium-cnn-e10s: None - * browsertime-tp6-essential-firefox-cnn-e10s: mozilla-beta, trunk - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * test-linux1804-64/opt - * browsertime-tp6-essential-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-tp6-essential-firefox-cnn-e10s: trunk - * browsertime-tp6-essential-firefox-cnn-fis-e10s: mozilla-central - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-fis-e10s: None - * test-macosx1015-64-shippable-qr/opt - * browsertime-tp6-essential-firefox-cnn-e10s: mozilla-beta, trunk - * browsertime-tp6-essential-firefox-cnn-fis-e10s: mozilla-central - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-fis-e10s: None - * test-macosx1015-64-shippable/opt - * browsertime-tp6-essential-chrome-cnn-e10s: None - * browsertime-tp6-essential-chromium-cnn-e10s: None - * browsertime-tp6-essential-firefox-cnn-e10s: mozilla-beta, trunk - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * test-windows10-32-qr/opt - * browsertime-tp6-essential-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-tp6-essential-firefox-cnn-e10s: None - * browsertime-tp6-essential-firefox-cnn-fis-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-tp6-essential-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-tp6-essential-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-tp6-essential-firefox-cnn-e10s: mozilla-beta, trunk - * browsertime-tp6-essential-firefox-cnn-fis-e10s: mozilla-central - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-fis-e10s: None - * test-windows10-64-shippable/opt - * browsertime-tp6-essential-chrome-cnn-e10s: None - * browsertime-tp6-essential-chromium-cnn-e10s: None - * browsertime-tp6-essential-firefox-cnn-e10s: mozilla-beta, trunk - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - * test-windows10-64/opt - * browsertime-tp6-essential-firefox-cnn-e10s: None - * browsertime-tp6-live-sheriffed-firefox-cnn-e10s: None - - -.. dropdown:: cnn-ampstories (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-cnn-ampstories-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: discord (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-discord-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: expedia (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-expedia-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-tp6-firefox-expedia-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-tp6-firefox-expedia-e10s: mozilla-beta, trunk - * browsertime-tp6-firefox-expedia-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-tp6-chrome-expedia-e10s: None - * browsertime-tp6-chromium-expedia-e10s: None - * browsertime-tp6-firefox-expedia-e10s: None - * test-linux1804-64/opt - * browsertime-tp6-firefox-expedia-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-tp6-firefox-expedia-e10s: trunk - * browsertime-tp6-firefox-expedia-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-tp6-firefox-expedia-e10s: mozilla-beta, trunk - * browsertime-tp6-firefox-expedia-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-tp6-chrome-expedia-e10s: None - * browsertime-tp6-chromium-expedia-e10s: None - * browsertime-tp6-firefox-expedia-e10s: None - * test-windows10-32-qr/opt - * browsertime-tp6-firefox-expedia-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-tp6-firefox-expedia-e10s: None - * browsertime-tp6-firefox-expedia-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-tp6-firefox-expedia-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-tp6-firefox-expedia-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-tp6-firefox-expedia-e10s: mozilla-beta, trunk - * browsertime-tp6-firefox-expedia-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-tp6-chrome-expedia-e10s: None - * browsertime-tp6-chromium-expedia-e10s: None - * browsertime-tp6-firefox-expedia-e10s: None - * test-windows10-64/opt - * browsertime-tp6-firefox-expedia-e10s: None - - -.. dropdown:: fashionbeans (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-fashionbeans-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: google-accounts (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-google-accounts-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: imdb-firefox (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-imdb-firefox-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: medium-article (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-medium-article-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: nytimes (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-nytimes-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - * test-linux1804-64-qr/opt - * browsertime-tp6-firefox-nytimes-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-tp6-firefox-nytimes-e10s: mozilla-beta, trunk - * browsertime-tp6-firefox-nytimes-fis-e10s: mozilla-central - * test-linux1804-64-shippable/opt - * browsertime-tp6-chrome-nytimes-e10s: None - * browsertime-tp6-chromium-nytimes-e10s: None - * browsertime-tp6-firefox-nytimes-e10s: None - * test-linux1804-64/opt - * browsertime-tp6-firefox-nytimes-e10s: None - * test-macosx1014-64-shippable-qr/opt - * browsertime-tp6-firefox-nytimes-e10s: trunk - * browsertime-tp6-firefox-nytimes-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-tp6-firefox-nytimes-e10s: mozilla-beta, trunk - * browsertime-tp6-firefox-nytimes-fis-e10s: mozilla-central - * test-macosx1015-64-shippable/opt - * browsertime-tp6-chrome-nytimes-e10s: None - * browsertime-tp6-chromium-nytimes-e10s: None - * browsertime-tp6-firefox-nytimes-e10s: None - * test-windows10-32-qr/opt - * browsertime-tp6-firefox-nytimes-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-tp6-firefox-nytimes-e10s: None - * browsertime-tp6-firefox-nytimes-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-tp6-firefox-nytimes-e10s: None - * test-windows10-64-ref-hw-2017/opt - * browsertime-tp6-firefox-nytimes-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-tp6-firefox-nytimes-e10s: mozilla-beta, trunk - * browsertime-tp6-firefox-nytimes-fis-e10s: mozilla-central - * test-windows10-64-shippable/opt - * browsertime-tp6-chrome-nytimes-e10s: None - * browsertime-tp6-chromium-nytimes-e10s: None - * browsertime-tp6-firefox-nytimes-e10s: None - * test-windows10-64/opt - * browsertime-tp6-firefox-nytimes-e10s: None - - -.. dropdown:: people-article (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-people-article-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: reddit-thread (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-reddit-thread-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: rumble-fox (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-rumble-fox-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: stackoverflow-question (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-stackoverflow-question-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: urbandictionary-define (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-urbandictionary-define-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - - -.. dropdown:: wikia-marvel (GV, FE, RB, CH-M, FF, CH, CU) - :container: + anchor-id-wikia-marvel-l - - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium - * **browser cycles**: 15 - * **expected**: pass - * **lower is better**: true - * **page cycles**: 25 - * **page timeout**: 60000 - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: true - * **Test Task**: - +A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms. Mobile @@ -8852,141 +6293,7 @@ Page-load performance test suite on Android. The links direct to the actual webs Scenario -------- -Tests that perform a specific action (a scenario), i.e. idle application, idle application in background, etc. (FE: Fenix, GV: Geckoview, RB: Refbrow) - -.. dropdown:: idle (FE, GV, RB) - :container: + anchor-id-idle-s - - * **alert threshold**: 2.0 - * **apps**: fenix, geckoview, refbrow - * **expected**: pass - * **lower is better**: true - * **measure**: fakeMeasure - * **page cycles**: 1 - * **page timeout**: 1320000 - * **scenario time**: 1200000 - * **test url**: ``__ - * **type**: scenario - * **unit**: scenarioComplete - * **Test Task**: - - -.. dropdown:: idle-bg (FE, GV, RB) - :container: + anchor-id-idle-bg-s - - * **alert threshold**: 2.0 - * **apps**: fenix, geckoview, refbrow - * **background app**: true - * **expected**: pass - * **lower is better**: true - * **measure**: fakeMeasure - * **page cycles**: 1 - * **page timeout**: 1320000 - * **scenario time**: 600000 - * **test url**: ``__ - * **type**: scenario - * **unit**: scenarioComplete - * **Test Task**: - - -.. dropdown:: raptor-scn-power-idle-bg-fenix (FE) - :container: + anchor-id-raptor-scn-power-idle-bg-fenix-s - - * **alert threshold**: 2.0 - * **apps**: fenix - * **background test**: true - * **expected**: pass - * **lower is better**: true - * **measure**: fakeMeasure - * **page cycles**: 1 - * **page timeout**: 660000 - * **scenario time**: 600000 - * **test url**: ``__ - * **type**: scenario - * **unit**: scenarioComplete - - -.. dropdown:: raptor-scn-power-idle-bg-geckoview (GV) - :container: + anchor-id-raptor-scn-power-idle-bg-geckoview-s - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **background test**: true - * **expected**: pass - * **lower is better**: true - * **measure**: fakeMeasure - * **page cycles**: 1 - * **page timeout**: 660000 - * **scenario time**: 600000 - * **test url**: ``__ - * **type**: scenario - * **unit**: scenarioComplete - - -.. dropdown:: raptor-scn-power-idle-bg-refbrow (RB) - :container: + anchor-id-raptor-scn-power-idle-bg-refbrow-s - - * **alert threshold**: 2.0 - * **apps**: refbrow - * **background test**: true - * **expected**: pass - * **lower is better**: true - * **measure**: fakeMeasure - * **page cycles**: 1 - * **page timeout**: 660000 - * **scenario time**: 600000 - * **test url**: ``__ - * **type**: scenario - * **unit**: scenarioComplete - - -.. dropdown:: raptor-scn-power-idle-fenix (FE) - :container: + anchor-id-raptor-scn-power-idle-fenix-s - - * **alert threshold**: 2.0 - * **apps**: fenix - * **expected**: pass - * **lower is better**: true - * **measure**: fakeMeasure - * **page cycles**: 1 - * **page timeout**: 1320000 - * **scenario time**: 1200000 - * **test url**: ``__ - * **type**: scenario - * **unit**: scenarioComplete - - -.. dropdown:: raptor-scn-power-idle-geckoview (GV) - :container: + anchor-id-raptor-scn-power-idle-geckoview-s - - * **alert threshold**: 2.0 - * **apps**: geckoview - * **expected**: pass - * **lower is better**: true - * **measure**: fakeMeasure - * **page cycles**: 1 - * **page timeout**: 1320000 - * **scenario time**: 1200000 - * **test url**: ``__ - * **type**: scenario - * **unit**: scenarioComplete - - -.. dropdown:: raptor-scn-power-idle-refbrow (RB) - :container: + anchor-id-raptor-scn-power-idle-refbrow-s - - * **alert threshold**: 2.0 - * **apps**: refbrow - * **expected**: pass - * **lower is better**: true - * **measure**: fakeMeasure - * **page cycles**: 1 - * **page timeout**: 1320000 - * **scenario time**: 1200000 - * **test url**: ``__ - * **type**: scenario - * **unit**: scenarioComplete - +Tests that perform a specific action (a scenario), i.e. idle application, idle application in background, etc. Unittests diff --git a/testing/raptor/raptor/perfdocs/config.yml b/testing/raptor/raptor/perfdocs/config.yml index 161c9b6396d8..274668a00c5c 100644 --- a/testing/raptor/raptor/perfdocs/config.yml +++ b/testing/raptor/raptor/perfdocs/config.yml @@ -72,107 +72,11 @@ suites: wikipedia: "BT, GV, FE, RB, CH-M" youtube-watch: "BT, GV, FE, RB, CH-M" live: - description: "A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms. (WX: WebExtension, BT: Browsertime, FF: Firefox, CH: Chrome, CU: Chromium, GV: Geckoview, RB: Refbrow, FE: Fenix, CH-M: Chrome mobile)" - tests: - booking-sf: "GV, FE, RB, CH-M, FF, CH, CU" - cnn: "GV, FE, RB, CH-M, FF, CH, CU" - cnn-ampstories: "GV, FE, RB, CH-M, FF, CH, CU" - discord: "GV, FE, RB, CH-M, FF, CH, CU" - expedia: "GV, FE, RB, CH-M, FF, CH, CU" - fashionbeans: "GV, FE, RB, CH-M, FF, CH, CU" - google-accounts: "GV, FE, RB, CH-M, FF, CH, CU" - imdb-firefox: "GV, FE, RB, CH-M, FF, CH, CU" - medium-article: "GV, FE, RB, CH-M, FF, CH, CU" - nytimes: "GV, FE, RB, CH-M, FF, CH, CU" - people-article: "GV, FE, RB, CH-M, FF, CH, CU" - reddit-thread: "GV, FE, RB, CH-M, FF, CH, CU" - rumble-fox: "GV, FE, RB, CH-M, FF, CH, CU" - stackoverflow-question: "GV, FE, RB, CH-M, FF, CH, CU" - urbandictionary-define: "GV, FE, RB, CH-M, FF, CH, CU" - wikia-marvel: "GV, FE, RB, CH-M, FF, CH, CU" + description: "A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms." benchmarks: - description: "Standard benchmarks are third-party tests (i.e. Speedometer) that we have integrated into Raptor to run per-commit in our production CI. (FF: Firefox, CH: Chrome, CU: Chromium)" - tests: - ares6: "FF, CH, CU" - assorted-dom: "FF, CH, CU" - jetstream2: "FF, CH, CU" - motionmark-animometer: "FF, CH, CU" - motionmark-htmlsuite: "FF, CH, CU" - raptor-speedometer-geckoview: "GV" - raptor-youtube-playback-av1-sfr-firefox: "FF" - raptor-youtube-playback-av1-sfr-geckoview: "GV" - raptor-youtube-playback-av1-sfr-fenix: "FE" - raptor-youtube-playback-av1-sfr-chrome: "CH" - raptor-youtube-playback-h264-sfr-firefox: "FF" - raptor-youtube-playback-h264-sfr-geckoview: "GV" - raptor-youtube-playback-h264-sfr-fenix: "FE" - raptor-youtube-playback-h264-sfr-chrome: "CH" - raptor-youtube-playback-h264-1080p30-firefox: "FF" - raptor-youtube-playback-h264-1080p60-firefox: "FF" - raptor-youtube-playback-h264-full-1080p30-firefox: "FF" - raptor-youtube-playback-h264-full-1080p60-firefox: "FF" - raptor-youtube-playback-hfr-firefox: "FF" - raptor-youtube-playback-hfr-geckoview: "GV" - raptor-youtube-playback-hfr-fenix: "FE" - raptor-youtube-playback-hfr-chrome: "CH" - raptor-youtube-playback-v9-1080p30-firefox: "FF" - raptor-youtube-playback-v9-1080p60-firefox: "FF" - raptor-youtube-playback-v9-full-1080p30-firefox: "FF" - raptor-youtube-playback-v9-full-1080p60-firefox: "FF" - raptor-youtube-playback-vp9-sfr-firefox: "FF" - raptor-youtube-playback-vp9-sfr-geckoview: "GV" - raptor-youtube-playback-vp9-sfr-fenix: "FE" - raptor-youtube-playback-vp9-sfr-chrome: "CH" - raptor-youtube-playback-widevine-h264-sfr-firefox: "FF" - raptor-youtube-playback-widevine-h264-sfr-geckoview: "GV" - raptor-youtube-playback-widevine-h264-sfr-fenix: "FE" - raptor-youtube-playback-widevine-h264-sfr-chrome: "CH" - raptor-youtube-playback-widevine-hfr-firefox: "FF" - raptor-youtube-playback-widevine-hfr-geckoview: "GV" - raptor-youtube-playback-widevine-hfr-fenix: "FE" - raptor-youtube-playback-widevine-hfr-chrome: "CH" - raptor-youtube-playback-widevine-vp9-sfr-firefox: "FF" - raptor-youtube-playback-widevine-vp9-sfr-geckoview: "GV" - raptor-youtube-playback-widevine-vp9-sfr-fenix: "FE" - raptor-youtube-playback-widevine-vp9-sfr-chrome: "CH" - speedometer: "FF, CH, CU, FE, GV, RB, CH-M" - stylebench: "FF, CH, CU" - sunspider: "FF, CH, CU" - unity-webgl: "FF, CH, CU, FE, RB, FE, CH-M" - wasm-godot-baseline: "FF" - wasm-godot-optimizing: "FF" - wasm-godot: "FF, CH, CU" - wasm-misc-baseline: "FF" - wasm-misc-optimizing: "FF" - wasm-misc: "FF, CH, CU" - webaudio: "FF, CH, CU" - youtube-playback: "FF, GV, FE, RB, CH" - youtube-playback-av1-sfr: "FF , GV, FE, RB, CH" - youtube-playback-h264-1080p30: "FF" - youtube-playback-h264-1080p60: "FF" - youtube-playback-h264-full-1080p30: "FF" - youtube-playback-h264-full-1080p60: "FF" - youtube-playback-h264-sfr: "FF , GV, FE, RB, CH" - youtube-playback-hfr: "FF , GV, FE, RB, CH" - youtube-playback-v9-1080p30: "FF" - youtube-playback-v9-1080p60: "FF" - youtube-playback-v9-full-1080p30: "FF" - youtube-playback-v9-full-1080p60: "FF" - youtube-playback-vp9-sfr: "FF , GV, FE, RB, CH" - youtube-playback-widevine-h264-sfr: "FF , GV, FE, RB, CH" - youtube-playback-widevine-hfr: "FF , GV, FE, RB, CH" - youtube-playback-widevine-vp9-sfr: "FF , GV, FE, RB, CH" + description: "Standard benchmarks are third-party tests (i.e. Speedometer) that we have integrated into Raptor to run per-commit in our production CI. " scenario: - description: "Tests that perform a specific action (a scenario), i.e. idle application, idle application in background, etc. (FE: Fenix, GV: Geckoview, RB: Refbrow)" - tests: - idle: "FE, GV, RB" - idle-bg: "FE, GV, RB" - raptor-scn-power-idle-bg-fenix: "FE" - raptor-scn-power-idle-bg-geckoview: "GV" - raptor-scn-power-idle-bg-refbrow: "RB" - raptor-scn-power-idle-fenix: "FE" - raptor-scn-power-idle-geckoview: "GV" - raptor-scn-power-idle-refbrow: "RB" + description: "Tests that perform a specific action (a scenario), i.e. idle application, idle application in background, etc." custom: description: "Browsertime tests that use a custom pageload test script. These use the pageload type, but may have other intentions." tests: @@ -180,7 +84,3 @@ suites: welcome: "Measures pageload metrics for the first-install about:welcome page" unittests: description: "These tests aren't used in standard testing, they are only used in the Raptor unit tests (they are similar to raptor-tp6 tests though)." - raptor-tp6-unittest-amazon-firefox: "FF" - raptor-tp6-unittest-facebook-firefox: "FF" - raptor-tp6-unittest-google-firefox: "FF" - raptor-tp6-unittest-youtube-firefox: "FF"