зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1896524 - Add desktop/android browsertime tests to measure video playback latency. r=perftest-reviewers,sparky
The video under test is a simple recording of solid colors, where the first frame is red, second frame is blue, subsequent frames alternate between green and purple, and the final frame is yellow. This allows us to identify what frame is being displayed and when in a browser agnostic way. This is achieved by updating the visualmetrics python script in browsertime to identify frames which match a given color range, called key color frames. The script will produce a list of which frames appear when that we can use to estimate the playback latency. Note that the annotated videos produced by the jobs will also be annotated to make it clear which frames it believes are key color frames. Until the tests have proven stable, they have been set at tier 3. The status of the merge can be seen at: https://github.com/sitespeedio/browsertime/pull/2119 Differential Revision: https://phabricator.services.mozilla.com/D210242
This commit is contained in:
Andrew Osmond
Родитель
66bee6b30a
Коммит
e28f7e8cc6
|
|
@ -49,7 +49,7 @@ job-defaults:
|
|||
- linux64-custom-car
|
||||
- linux64-minidump-stackwalk
|
||||
- linux64-fix-stacks
|
||||
windows10-64.*:
|
||||
windows.*-64.*:
|
||||
- win64-custom-car
|
||||
- win64-minidump-stackwalk
|
||||
- win64-fix-stacks
|
||||
|
|
@ -73,12 +73,15 @@ job-defaults:
|
|||
windows.*-32.*.:
|
||||
- win32-minidump-stackwalk
|
||||
- win32-fix-stacks
|
||||
windows10-64.*:
|
||||
windows.*-64.*:
|
||||
- win64-minidump-stackwalk
|
||||
- win64-fix-stacks
|
||||
macos.*:
|
||||
macosx1015.*:
|
||||
- macosx64-minidump-stackwalk
|
||||
- macosx64-fix-stacks
|
||||
macosx1400.*:
|
||||
- macosx64-aarch64-minidump-stackwalk
|
||||
- macosx64-aarch64-fix-stacks
|
||||
default:
|
||||
- linux64-minidump-stackwalk
|
||||
- linux64-fix-stacks
|
||||
|
|
@ -884,6 +887,36 @@ browsertime-pageload-benchmark:
|
|||
tier: 2
|
||||
treeherder-symbol: Btime(tp6-bench)
|
||||
|
||||
browsertime-video-playback-latency:
|
||||
<<: *tp6-defaults
|
||||
raptor:
|
||||
apps: [firefox, chrome, safari]
|
||||
test: video-playback-latency
|
||||
run-visual-metrics: true
|
||||
subtests:
|
||||
by-app:
|
||||
safari:
|
||||
- vpl-h264
|
||||
default:
|
||||
- vpl-av1
|
||||
- vpl-h264
|
||||
- vpl-vp9
|
||||
description: Raptor (browsertime) Video playback latency test
|
||||
mozharness:
|
||||
extra-options:
|
||||
- --chimera
|
||||
- --browsertime-no-ffwindowrecorder
|
||||
- --extra-profiler-run
|
||||
run-on-projects:
|
||||
by-app:
|
||||
firefox:
|
||||
by-test-platform:
|
||||
(macosx1015|linux|windows10-64).*shippable-qr.*: [autoland, mozilla-central]
|
||||
default: []
|
||||
default: []
|
||||
tier: 3
|
||||
treeherder-symbol: Btime(vpl)
|
||||
|
||||
browsertime-speculative:
|
||||
raptor:
|
||||
apps: [firefox]
|
||||
|
|
|
|||
|
|
@ -368,6 +368,33 @@ browsertime-tp6m-profiling:
|
|||
android-hw-(a51).*aarch64-shippable-qr/opt: [mozilla-central]
|
||||
default: []
|
||||
|
||||
browsertime-video-playback-latency-mobile:
|
||||
<<: *tp6m-defaults
|
||||
description: Browsertime Video Playback Latency on Android
|
||||
treeherder-symbol: Btime(vpl)
|
||||
run-without-variant: false
|
||||
raptor:
|
||||
apps: [chrome-m, fenix, geckoview]
|
||||
run-visual-metrics: true
|
||||
test: video-playback-latency
|
||||
subtests:
|
||||
- vpl-av1
|
||||
- vpl-h264
|
||||
- vpl-vp9
|
||||
mozharness:
|
||||
extra-options:
|
||||
- --chimera
|
||||
- --browsertime-no-ffwindowrecorder
|
||||
- --extra-profiler-run
|
||||
run-on-projects:
|
||||
by-app:
|
||||
chrome-m: []
|
||||
default:
|
||||
by-test-platform:
|
||||
android-hw-a51.*shippable.*: [autoland, mozilla-central]
|
||||
default: []
|
||||
tier: 3
|
||||
|
||||
browsertime-benchmark-speedometer-mobile:
|
||||
description: Browsertime Speedometer on Android
|
||||
run-without-variant: false
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ browsertime:
|
|||
- browsertime-throttled
|
||||
- browsertime-pageload-benchmark
|
||||
- browsertime-indexeddb
|
||||
- browsertime-video-playback-latency
|
||||
|
||||
browsertime-clang:
|
||||
- browsertime-tp6
|
||||
|
|
@ -350,6 +351,7 @@ windows-talos:
|
|||
# - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions
|
||||
|
||||
windows-ref-hw-tests:
|
||||
- browsertime-video-playback-latency
|
||||
- mochitest-media
|
||||
|
||||
talos-xperf:
|
||||
|
|
@ -493,6 +495,7 @@ android-hw-browsertime:
|
|||
- browsertime-benchmark-unity-webgl-mobile
|
||||
- browsertime-benchmark-motionmark
|
||||
- browsertime-benchmark-motionmark-1-3
|
||||
- browsertime-video-playback-latency-mobile
|
||||
|
||||
android-hw-browsertime-profiling:
|
||||
- browsertime-tp6m-profiling
|
||||
|
|
|
|||
|
|
@ -100,6 +100,26 @@ Similar to SpeedIndex, except that it uses the contentfulness of a frame to dete
|
|||
* **Mobile**: `allrecipes <raptor.html#allrecipes-m>`__, `amazon <raptor.html#amazon-m>`__, `amazon-search <raptor.html#amazon-search-m>`__, `bild-de <raptor.html#bild-de-m>`__, `bing <raptor.html#bing-m>`__, `bing-search-restaurants <raptor.html#bing-search-restaurants-m>`__, `booking <raptor.html#booking-m>`__, `cnn <raptor.html#cnn-m>`__, `cnn-ampstories <raptor.html#cnn-ampstories-m>`__, `dailymail <raptor.html#dailymail-m>`__, `ebay-kleinanzeigen <raptor.html#ebay-kleinanzeigen-m>`__, `ebay-kleinanzeigen-search <raptor.html#ebay-kleinanzeigen-search-m>`__, `espn <raptor.html#espn-m>`__, `facebook <raptor.html#facebook-m>`__, `facebook-cristiano <raptor.html#facebook-cristiano-m>`__, `google <raptor.html#google-m>`__, `google-maps <raptor.html#google-maps-m>`__, `google-search-restaurants <raptor.html#google-search-restaurants-m>`__, `imdb <raptor.html#imdb-m>`__, `instagram <raptor.html#instagram-m>`__, `microsoft-support <raptor.html#microsoft-support-m>`__, `reddit <raptor.html#reddit-m>`__, `sina <raptor.html#sina-m>`__, `stackoverflow <raptor.html#stackoverflow-m>`__, `wikipedia <raptor.html#wikipedia-m>`__, `youtube <raptor.html#youtube-m>`__, `youtube-watch <raptor.html#youtube-watch-m>`__
|
||||
|
||||
|
||||
Estimated Frame Latency (Any)
|
||||
=============================
|
||||
Similar to estimatedFirstFrameLatency, except that it uses all identified frames during video playback, normalized to be an estimate of when the first frame was displayed by using the expected time offset from the video itself.
|
||||
|
||||
|
||||
* **Aliases**: estimatedAnyFrameLatency
|
||||
* **Tests using it**:
|
||||
* **Custom**: `vpl-av1 <raptor.html#vpl-av1-c>`__, `vpl-h264 <raptor.html#vpl-h264-c>`__, `vpl-vp9 <raptor.html#vpl-vp9-c>`__
|
||||
|
||||
|
||||
Estimated Frame Latency (First)
|
||||
===============================
|
||||
A metric used to denote the latency on displaying the first frame of a video. Calculated by using videos of the pageload from which key frames are identified by matching a given solid RGB color with fuzz.
|
||||
|
||||
|
||||
* **Aliases**: estimatedFirstFrameLatency
|
||||
* **Tests using it**:
|
||||
* **Custom**: `vpl-av1 <raptor.html#vpl-av1-c>`__, `vpl-h264 <raptor.html#vpl-h264-c>`__, `vpl-vp9 <raptor.html#vpl-vp9-c>`__
|
||||
|
||||
|
||||
First Paint
|
||||
===========
|
||||
Denotes the first time the browser performs a paint that has content in it (in ms).
|
||||
|
|
|
|||
|
|
@ -16497,6 +16497,950 @@ Browsertime tests that use a custom pageload test script. These use the pageload
|
|||
|
||||
|
||||
|
||||
.. dropdown:: vpl-av1
|
||||
:class-container: anchor-id-vpl-av1-c
|
||||
|
||||
**Owner**: :aosmond and the Media Team
|
||||
|
||||
**Description**: Measures AV1 1080p video playback latency.
|
||||
|
||||
* **alert on**: `estimatedFirstFrameLatency <raptor-metrics.html#estimated-frame-latency-(first)>`__, `estimatedAnyFrameLatency <raptor-metrics.html#estimated-frame-latency-(any)>`__
|
||||
* **alert threshold**: 2.0
|
||||
* **apps**: chrome, chrome-m, geckoview, fenix, firefox, safari
|
||||
* **benchmark webserver**: true
|
||||
* **browser cycles**: 20
|
||||
* **expected**: pass
|
||||
* **gecko profile features**: stackwalk,js,cpu,java,processcpu,screenshots,memory
|
||||
* **gecko profile interval**: 1
|
||||
* **gecko profile threads**: BackgroundThreadPool,Compositor,decoder,GeckoMain,gmp,graph,grph,InotifyEventThread,IPDL Background,media,ModuleProcessThread,PacerThread,RemVidChild,RenderBackend,Renderer,SwComposite,TextureUpdate,ImageBridgeChld
|
||||
* **link searchfox**: `<https://searchfox.org/mozilla-central/source/testing/raptor/raptor/tests/custom/browsertime-video-playback-latency.toml#36>`__
|
||||
* **lower is better**: true
|
||||
* **measure**: firstFrame, secondFrame, lastFrame, estimatedFirstFrameLatency, estimatedAnyFrameLatency
|
||||
* **newtab per cycle**: true
|
||||
* **page cycles**: 20
|
||||
* **page timeout**: 20000
|
||||
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
|
||||
* **repository**: https://github.com/mozilla/perf-automation
|
||||
* **repository path**: benchmarks/video-playback-latency
|
||||
* **repository revision**: e3d0295c15cfdd6f32e6d4270de0ed5d6c1632cd
|
||||
* **support class**: video_playback_latency.py
|
||||
* **test url**: `<http://\<host\>:\<port\>/av1_1080p.html>`__
|
||||
* **type**: pageload
|
||||
* **unit**: ms
|
||||
* **Test Task**:
|
||||
|
||||
.. list-table:: **test-android-hw-a51-11-0-aarch64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-mobile-fenix-vpl-av1-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-geckoview-vpl-av1-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-android-hw-a51-11-0-aarch64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-mobile-chrome-m-vpl-av1-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-fenix-vpl-av1-nofis**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-geckoview-vpl-av1-nofis**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-32-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-32-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows11-64-2009-hw-ref-shippable/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows11-64-2009-hw-ref/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-av1**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
|
||||
.. dropdown:: vpl-h264
|
||||
:class-container: anchor-id-vpl-h264-c
|
||||
|
||||
**Owner**: :aosmond and the Media Team
|
||||
|
||||
**Description**: Measures H264 1080p video playback latency.
|
||||
|
||||
* **alert on**: `estimatedFirstFrameLatency <raptor-metrics.html#estimated-frame-latency-(first)>`__, `estimatedAnyFrameLatency <raptor-metrics.html#estimated-frame-latency-(any)>`__
|
||||
* **alert threshold**: 2.0
|
||||
* **apps**: chrome, chrome-m, geckoview, fenix, firefox, safari
|
||||
* **benchmark webserver**: true
|
||||
* **browser cycles**: 20
|
||||
* **expected**: pass
|
||||
* **gecko profile features**: stackwalk,js,cpu,java,processcpu,screenshots,memory
|
||||
* **gecko profile interval**: 1
|
||||
* **gecko profile threads**: BackgroundThreadPool,Compositor,decoder,GeckoMain,gmp,graph,grph,InotifyEventThread,IPDL Background,media,ModuleProcessThread,PacerThread,RemVidChild,RenderBackend,Renderer,SwComposite,TextureUpdate,ImageBridgeChld
|
||||
* **link searchfox**: `<https://searchfox.org/mozilla-central/source/testing/raptor/raptor/tests/custom/browsertime-video-playback-latency.toml#30>`__
|
||||
* **lower is better**: true
|
||||
* **measure**: firstFrame, secondFrame, lastFrame, estimatedFirstFrameLatency, estimatedAnyFrameLatency
|
||||
* **newtab per cycle**: true
|
||||
* **page cycles**: 20
|
||||
* **page timeout**: 20000
|
||||
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
|
||||
* **repository**: https://github.com/mozilla/perf-automation
|
||||
* **repository path**: benchmarks/video-playback-latency
|
||||
* **repository revision**: e3d0295c15cfdd6f32e6d4270de0ed5d6c1632cd
|
||||
* **support class**: video_playback_latency.py
|
||||
* **test url**: `<http://\<host\>:\<port\>/h264_1080p.html>`__
|
||||
* **type**: pageload
|
||||
* **unit**: ms
|
||||
* **Test Task**:
|
||||
|
||||
.. list-table:: **test-android-hw-a51-11-0-aarch64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-mobile-fenix-vpl-h264-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-geckoview-vpl-h264-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-android-hw-a51-11-0-aarch64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-mobile-chrome-m-vpl-h264-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-fenix-vpl-h264-nofis**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-geckoview-vpl-h264-nofis**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-safari-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-32-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-32-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows11-64-2009-hw-ref-shippable/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows11-64-2009-hw-ref/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-h264**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
|
||||
.. dropdown:: vpl-vp9
|
||||
:class-container: anchor-id-vpl-vp9-c
|
||||
|
||||
**Owner**: :aosmond and the Media Team
|
||||
|
||||
**Description**: Measures VP9 1080p video playback latency.
|
||||
|
||||
* **alert on**: `estimatedFirstFrameLatency <raptor-metrics.html#estimated-frame-latency-(first)>`__, `estimatedAnyFrameLatency <raptor-metrics.html#estimated-frame-latency-(any)>`__
|
||||
* **alert threshold**: 2.0
|
||||
* **apps**: chrome, chrome-m, geckoview, fenix, firefox, safari
|
||||
* **benchmark webserver**: true
|
||||
* **browser cycles**: 20
|
||||
* **expected**: pass
|
||||
* **gecko profile features**: stackwalk,js,cpu,java,processcpu,screenshots,memory
|
||||
* **gecko profile interval**: 1
|
||||
* **gecko profile threads**: BackgroundThreadPool,Compositor,decoder,GeckoMain,gmp,graph,grph,InotifyEventThread,IPDL Background,media,ModuleProcessThread,PacerThread,RemVidChild,RenderBackend,Renderer,SwComposite,TextureUpdate,ImageBridgeChld
|
||||
* **link searchfox**: `<https://searchfox.org/mozilla-central/source/testing/raptor/raptor/tests/custom/browsertime-video-playback-latency.toml#33>`__
|
||||
* **lower is better**: true
|
||||
* **measure**: firstFrame, secondFrame, lastFrame, estimatedFirstFrameLatency, estimatedAnyFrameLatency
|
||||
* **newtab per cycle**: true
|
||||
* **page cycles**: 20
|
||||
* **page timeout**: 20000
|
||||
* **preferences**: media.autoplay.default=0 media.autoplay.ask-permission=false media.autoplay.blocking_policy=0 media.allowed-to-play.enabled=true media.block-autoplay-until-in-foreground=false
|
||||
* **repository**: https://github.com/mozilla/perf-automation
|
||||
* **repository path**: benchmarks/video-playback-latency
|
||||
* **repository revision**: e3d0295c15cfdd6f32e6d4270de0ed5d6c1632cd
|
||||
* **support class**: video_playback_latency.py
|
||||
* **test url**: `<http://\<host\>:\<port\>/vp9_1080p.html>`__
|
||||
* **type**: pageload
|
||||
* **unit**: ms
|
||||
* **Test Task**:
|
||||
|
||||
.. list-table:: **test-android-hw-a51-11-0-aarch64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-mobile-fenix-vpl-vp9-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-geckoview-vpl-vp9-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-android-hw-a51-11-0-aarch64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-mobile-chrome-m-vpl-vp9-nofis**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-fenix-vpl-vp9-nofis**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-mobile-geckoview-vpl-vp9-nofis**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-32-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-32-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-nightlyasrelease-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows11-64-2009-hw-ref-shippable/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-chrome-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows11-64-2009-hw-ref/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-video-playback-latency-firefox-vpl-vp9**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
|
||||
.. dropdown:: welcome
|
||||
:class-container: anchor-id-welcome-c
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,151 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import filters
|
||||
from base_python_support import BasePythonSupport
|
||||
|
||||
|
||||
class VideoPlaybackLatency(BasePythonSupport):
|
||||
def __init__(self, **kwargs):
|
||||
self._is_android = False
|
||||
self._is_chrome = False
|
||||
|
||||
def setup_test(self, test, args):
|
||||
from cmdline import CHROME_ANDROID_APPS, CHROMIUM_DISTROS, DESKTOP_APPS
|
||||
|
||||
self._is_android = args.app not in DESKTOP_APPS
|
||||
self._is_chrome = (
|
||||
args.app in CHROMIUM_DISTROS or args.app in CHROME_ANDROID_APPS
|
||||
)
|
||||
|
||||
def modify_command(self, cmd, test):
|
||||
# Because of the aspect ratio of Android during recording,
|
||||
# most pixels are white, so we need to use a lower fraction.
|
||||
fraction = "0.25" if self._is_android else "0.7"
|
||||
|
||||
# Firefox/Safari configuration allows video playback but
|
||||
# Chrome needs an explicit switch to enable it.
|
||||
if self._is_chrome:
|
||||
cmd += [
|
||||
"--chrome.enableVideoAutoplay",
|
||||
"true",
|
||||
]
|
||||
|
||||
cmd += [
|
||||
"--visualMetricsKeyColor",
|
||||
"poster",
|
||||
"0",
|
||||
"128",
|
||||
"220",
|
||||
"255",
|
||||
"220",
|
||||
"255",
|
||||
fraction,
|
||||
"--visualMetricsKeyColor",
|
||||
"firstFrame",
|
||||
"220",
|
||||
"255",
|
||||
"0",
|
||||
"60",
|
||||
"0",
|
||||
"60",
|
||||
fraction,
|
||||
"--visualMetricsKeyColor",
|
||||
"secondFrame",
|
||||
"0",
|
||||
"60",
|
||||
"0",
|
||||
"60",
|
||||
"220",
|
||||
"255",
|
||||
fraction,
|
||||
"--visualMetricsKeyColor",
|
||||
"lastFrame",
|
||||
"220",
|
||||
"255",
|
||||
"220",
|
||||
"255",
|
||||
"0",
|
||||
"128",
|
||||
fraction,
|
||||
]
|
||||
|
||||
def handle_result(self, bt_result, raw_result, last_result=False, **kwargs):
|
||||
measurements = {
|
||||
"poster": [],
|
||||
"posterEnd": [],
|
||||
"firstFrame": [],
|
||||
"secondFrame": [],
|
||||
"lastFrame": [],
|
||||
"estimatedFirstFrameLatency": [],
|
||||
"estimatedAnyFrameLatency": [],
|
||||
}
|
||||
|
||||
fps = 30.0
|
||||
total_duration_ms = 1000.0
|
||||
frame_duration_ms = total_duration_ms / fps
|
||||
|
||||
offsets = {
|
||||
"firstFrame": 0.0,
|
||||
"posterEnd": 0.0,
|
||||
"secondFrame": frame_duration_ms * 3.0,
|
||||
"lastFrame": total_duration_ms - frame_duration_ms,
|
||||
}
|
||||
|
||||
# Gather the key frame start times of each page/cycle
|
||||
for cycle in raw_result["visualMetrics"]:
|
||||
measurement = {}
|
||||
for key, frames in cycle["KeyColorFrames"].items():
|
||||
if key not in measurements or not len(frames):
|
||||
continue
|
||||
measurement[key] = frames[0]["startTimestamp"]
|
||||
if key == "poster":
|
||||
measurement["posterEnd"] = frames[0]["endTimestamp"]
|
||||
|
||||
for key in ["firstFrame", "posterEnd", "secondFrame", "lastFrame"]:
|
||||
if key not in measurement:
|
||||
continue
|
||||
normalized_value = measurement[key] - offsets[key]
|
||||
if normalized_value <= 0:
|
||||
continue
|
||||
measurements["estimatedFirstFrameLatency"].append(normalized_value)
|
||||
break
|
||||
|
||||
for key, value in measurement.items():
|
||||
measurements[key].append(value)
|
||||
if key not in offsets:
|
||||
continue
|
||||
normalized_value = value - offsets[key]
|
||||
if normalized_value <= 0:
|
||||
continue
|
||||
measurements["estimatedAnyFrameLatency"].append(normalized_value)
|
||||
|
||||
for measurement, values in measurements.items():
|
||||
bt_result["measurements"].setdefault(measurement, []).extend(values)
|
||||
|
||||
def _build_subtest(self, measurement_name, replicates, test):
|
||||
unit = test.get("unit", "ms")
|
||||
if test.get("subtest_unit"):
|
||||
unit = test.get("subtest_unit")
|
||||
|
||||
return {
|
||||
"name": measurement_name,
|
||||
"lowerIsBetter": test.get("lower_is_better", True),
|
||||
"alertThreshold": float(test.get("alert_threshold", 2.0)),
|
||||
"unit": unit,
|
||||
"replicates": replicates,
|
||||
"value": round(filters.geometric_mean(replicates), 3),
|
||||
}
|
||||
|
||||
def summarize_test(self, test, suite, **kwargs):
|
||||
suite["type"] = "pageload"
|
||||
if suite["subtests"] == {}:
|
||||
suite["subtests"] = []
|
||||
for measurement_name, replicates in test["measurements"].items():
|
||||
if not replicates:
|
||||
continue
|
||||
suite["subtests"].append(
|
||||
self._build_subtest(measurement_name, replicates, test)
|
||||
)
|
||||
suite["subtests"].sort(key=lambda subtest: subtest["name"])
|
||||
|
|
@ -151,7 +151,7 @@ class Browsertime(Perftest):
|
|||
|
||||
super(Browsertime, self).run_test_setup(test)
|
||||
|
||||
if test.get("type") == "benchmark":
|
||||
if test.get("type") == "benchmark" or test.get("benchmark_webserver", False):
|
||||
# benchmark-type tests require the benchmark test to be served out
|
||||
self.benchmark = Benchmark(self.config, test, debug_mode=self.debug_mode)
|
||||
test["test_url"] = test["test_url"].replace("<host>", self.benchmark.host)
|
||||
|
|
@ -936,7 +936,7 @@ class Browsertime(Perftest):
|
|||
# Change the timeout for scenarios since they
|
||||
# don't output much for a long period of time
|
||||
output_timeout = timeout
|
||||
elif self.benchmark:
|
||||
elif test.get("type", "") == "benchmark":
|
||||
output_timeout = BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT
|
||||
|
||||
if self.debug_mode:
|
||||
|
|
@ -1015,7 +1015,7 @@ class Browsertime(Perftest):
|
|||
|
||||
proc_timeout = self._compute_process_timeout(test, timeout, cmd)
|
||||
output_timeout = BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT
|
||||
if self.benchmark:
|
||||
if test.get("type", "") == "benchmark":
|
||||
output_timeout = BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT
|
||||
elif test.get("output_timeout", None) is not None:
|
||||
output_timeout = int(test.get("output_timeout"))
|
||||
|
|
|
|||
|
|
@ -147,7 +147,9 @@ def validate_test_toml(test_details):
|
|||
"`repository_revision` is required when a `repository` is defined."
|
||||
)
|
||||
valid_settings = False
|
||||
elif test_details.get("type") not in ("benchmark"):
|
||||
elif test_details.get("type") not in ("benchmark") and not test_details.get(
|
||||
"benchmark_webserver", False
|
||||
):
|
||||
LOG.error("`repository` is only available for benchmark test types.")
|
||||
valid_settings = False
|
||||
|
||||
|
|
|
|||
|
|
@ -64,6 +64,20 @@ metrics:
|
|||
Similar to SpeedIndex, except that it uses the structural similarity index measure (ssim)
|
||||
to determine visual completeness. This technique compares the luminance, contrast, and
|
||||
structure of the frames (a given frame vs. a final frame) to determine the completeness.
|
||||
"Estimated Frame Latency (First)":
|
||||
aliases:
|
||||
- estimatedFirstFrameLatency
|
||||
description: >
|
||||
A metric used to denote the latency on displaying the first frame of a video. Calculated
|
||||
by using videos of the pageload from which key frames are identified by matching a given
|
||||
solid RGB color with fuzz.
|
||||
"Estimated Frame Latency (Any)":
|
||||
aliases:
|
||||
- estimatedAnyFrameLatency
|
||||
description: >
|
||||
Similar to estimatedFirstFrameLatency, except that it uses all identified frames during
|
||||
video playback, normalized to be an estimate of when the first frame was displayed by
|
||||
using the expected time offset from the video itself.
|
||||
|
||||
suites:
|
||||
desktop:
|
||||
|
|
@ -231,6 +245,9 @@ suites:
|
|||
A pageload benchmark test that quickly runs through a selection of the
|
||||
pages available to produce a single overall metric for fcp, lcp, and
|
||||
loadtime. The metrics from all pages get summarized with the geomean, and mean.
|
||||
vpl-av1: "Measures AV1 1080p video playback latency."
|
||||
vpl-h264: "Measures H264 1080p video playback latency."
|
||||
vpl-vp9: "Measures VP9 1080p video playback latency."
|
||||
interactive:
|
||||
description: "Browsertime tests that interact with the webpage. Includes responsiveness tests as they make use of this support for navigation. These form of tests allow the specification of browsertime commands through the test manifest."
|
||||
tests:
|
||||
|
|
|
|||
|
|
@ -85,6 +85,9 @@
|
|||
["include:tests/custom/browsertime-welcome.toml"]
|
||||
# First-install pageload test
|
||||
|
||||
["include:tests/custom/browsertime-video-playback-latency.toml"]
|
||||
# Video playback latency tests
|
||||
|
||||
["include:tests/interactive/browsertime-responsiveness.toml"]
|
||||
# Interactive raptor-browsertime tests
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
[DEFAULT]
|
||||
alert_threshold = 2.0
|
||||
alert_on = "estimatedFirstFrameLatency, estimatedAnyFrameLatency"
|
||||
gecko_profile_interval = 1
|
||||
gecko_profile_features = "stackwalk,js,cpu,java,processcpu,screenshots,memory"
|
||||
gecko_profile_threads = "BackgroundThreadPool,Compositor,decoder,GeckoMain,gmp,graph,grph,InotifyEventThread,IPDL Background,media,ModuleProcessThread,PacerThread,RemVidChild,RenderBackend,Renderer,SwComposite,TextureUpdate,ImageBridgeChld"
|
||||
lower_is_better = true
|
||||
newtab_per_cycle = true
|
||||
owner = ":aosmond and the Media Team"
|
||||
page_cycles = 20
|
||||
browser_cycles = 20
|
||||
page_timeout = 20000
|
||||
apps = "chrome, chrome-m, geckoview, fenix, firefox, safari"
|
||||
type = "pageload"
|
||||
benchmark_webserver = true
|
||||
support_class = "video_playback_latency.py"
|
||||
measure = "firstFrame, secondFrame, lastFrame, estimatedFirstFrameLatency, estimatedAnyFrameLatency"
|
||||
unit = "ms"
|
||||
preferences = [
|
||||
"media.autoplay.default=0",
|
||||
"media.autoplay.ask-permission=false",
|
||||
"media.autoplay.blocking_policy=0",
|
||||
"media.allowed-to-play.enabled=true",
|
||||
"media.block-autoplay-until-in-foreground=false",
|
||||
]
|
||||
repository = "https://github.com/mozilla/perf-automation"
|
||||
repository_revision = "e3d0295c15cfdd6f32e6d4270de0ed5d6c1632cd"
|
||||
repository_path = "benchmarks/video-playback-latency"
|
||||
|
||||
["vpl-h264"]
|
||||
test_url = "http://<host>:<port>/h264_1080p.html"
|
||||
|
||||
["vpl-vp9"]
|
||||
test_url = "http://<host>:<port>/vp9_1080p.html"
|
||||
|
||||
["vpl-av1"]
|
||||
test_url = "http://<host>:<port>/av1_1080p.html"
|
||||
Загрузка…
Ссылка в новой задаче