mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1621344 - Add Android Browsertime youtube-playback tests r=perftest-reviewers,sparky,AlexandruIonescu 

Differential Revision: https://phabricator.services.mozilla.com/D67437
This commit is contained in:
Florin Strugariu 2020-07-02 09:33:41 +00:00
Родитель bcb2ab8676
Коммит 6999f35c94
8 изменённых файлов: 271 добавлений и 111 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

18
taskcluster/ci/test/browsertime-mobile.yml
Просмотреть файл

@ -193,6 +193,24 @@ browsertime-tp6m:
run-visual-metrics: true
treeherder-symbol: Btime(tp6m)
browsertime-youtube-playback-mobile:
description: "Browsertime YouTube Playback on Android"
treeherder-symbol: Btime(ytp)
raptor-subtests:
- [youtube-playback, ytp]
- [youtube-playback-av1-sfr, ytp-av1-sfr]
- [youtube-playback-h264-sfr, ytp-h264-sfr]
- [youtube-playback-h264-std, ytp-h264-std]
- [youtube-playback-vp9-sfr, ytp-vp9-sfr]
- [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
- [youtube-playback-widevine-hfr, ytp-widevine-hfr]
- [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
apps: ["fenix", "geckoview", "refbrow"]
run-on-projects: []
mozharness:
extra-options:
- --browsertime
browsertime-tp6m-live:
description: "Raptor (browsertime) tp6 live site page-load tests on android"
raptor-test: tp6m

2
taskcluster/ci/test/test-sets.yml
Просмотреть файл

@ -626,12 +626,14 @@ android-hw-aarch64-browsertime:
- browsertime-tp6m-live
- browsertime-tp6m-profiling
- browsertime-speedometer-mobile
- browsertime-youtube-playback-mobile
android-hw-arm7-browsertime:
- browsertime-tp6m
- browsertime-tp6m-live
- browsertime-tp6m-profiling
- browsertime-speedometer-mobile
- browsertime-youtube-playback-mobile
linux-tsan-tests:
- mochitest-plain

15
taskcluster/taskgraph/transforms/raptor.py
Просмотреть файл

@ -255,13 +255,17 @@ def split_page_load_by_url(config, tests):
yield test
continue
if len(subtest_symbol) > 10:
if len(subtest_symbol) > 10 and \
'ytp' not in subtest_symbol:
raise Exception(
"Treeherder symbol %s is lager than 10 char! Please use a different symbol."
% subtest_symbol)
if test['test-name'].startswith('browsertime-tp6'):
if test['test-name'].startswith('browsertime-'):
test['raptor-test'] = subtest
# Remove youtube-playback in the test name to avoid duplication
test['test-name'] = test['test-name'].replace("youtube-playback-", "")
else:
# Use full test name if running on webextension
test['raptor-test'] = 'raptor-tp6-' + subtest + "-{}".format(test['app'])
@ -332,10 +336,11 @@ def add_extra_options(config, tests):
"android-hw-g5": [
{
"branches": [], # For all branches
"testnames": ["youtube-playback"],
"testnames": ["youtube-playback", "raptor-youtube-playback"],
"urlparams": [
# It excludes all VP9 tests
"exclude=1-34"
"exclude=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,"
"24,25,26,27,28,29,30,31,32,33,34"
]
},
]
@ -349,7 +354,7 @@ def add_extra_options(config, tests):
for testurlparams_by_project in testurlparams_by_project_definitions:
# The test should contain at least one defined testname
if any(
testname in test['test-name']
testname == test['test-name']
for testname in testurlparams_by_project['testnames']
):
branches = testurlparams_by_project['branches']

2
testing/raptor/raptor/browsertime/base.py
Просмотреть файл

@ -23,7 +23,7 @@ LOG = RaptorLogger(component="raptor-browsertime")
DEFAULT_CHROMEVERSION = "77"
BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT = 120 # 2 minutes
BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT = 900 # 15 minutes
BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT = None # Disable output timeout for benchmark tests
class Browsertime(Perftest):

6
testing/raptor/raptor/manifest.py
Просмотреть файл

@ -482,7 +482,11 @@ def get_raptor_test_list(args, oskey):
# of supporting both webext and browsertime, just provide a dummy 'measure' setting
# here to prevent having to check in multiple places; it has no effect on what
# browsertime actually measures; remove this when eventually we remove webext support
if args.browsertime and next_test.get('measure') is None:
if (
args.browsertime
and next_test.get("measure") is None
and next_test.get("type") == "pageload"
):
next_test['measure'] = "fnbpaint, fcp, dcf, loadtime"
# convert 'measure =' test INI line to list

199
testing/raptor/raptor/output.py
Просмотреть файл

@ -575,6 +575,96 @@ class PerftestOutput(object):
return _subtests.values(), sorted(vals, reverse=True)
def parseYoutubePlaybackPerformanceOutput(self, test):
"""Parse the metrics for the Youtube playback performance test.
For each video measured values for dropped and decoded frames will be
available from the benchmark site.
{u'PlaybackPerf.VP9.2160p60@2X': {u'droppedFrames': 1, u'decodedFrames': 796}
With each page cycle / iteration of the test multiple values can be present.
Raptor will calculate the percentage of dropped frames to decoded frames.
All those three values will then be emitted as separate sub tests.
"""
_subtests = {}
test_name = [measurement
for measurement in test["measurements"].keys()
if "youtube-playback" in measurement
]
if len(test_name) > 0:
data = test["measurements"].get(test_name[0])
else:
raise Exception("No measurements found for youtube test!")
def create_subtest_entry(
name,
value,
unit=test["subtest_unit"],
lower_is_better=test["subtest_lower_is_better"],
):
# build a list of subtests and append all related replicates
if name not in _subtests.keys():
# subtest not added yet, first pagecycle, so add new one
_subtests[name] = {
"name": name,
"unit": unit,
"lowerIsBetter": lower_is_better,
"replicates": [],
}
_subtests[name]["replicates"].append(value)
if self.subtest_alert_on is not None:
if name in self.subtest_alert_on:
LOG.info(
"turning on subtest alerting for measurement type: %s" % name
)
_subtests[name]["shouldAlert"] = True
for pagecycle in data:
for _sub, _value in pagecycle[0].iteritems():
try:
percent_dropped = (
float(_value["droppedFrames"]) / _value["decodedFrames"] * 100.0
)
except ZeroDivisionError:
# if no frames have been decoded the playback failed completely
percent_dropped = 100.0
# Remove the not needed "PlaybackPerf." prefix from each test
_sub = _sub.split("PlaybackPerf", 1)[-1]
if _sub.startswith("."):
_sub = _sub[1:]
# build a list of subtests and append all related replicates
create_subtest_entry(
"{}_decoded_frames".format(_sub),
_value["decodedFrames"],
lower_is_better=False,
)
create_subtest_entry(
"{}_dropped_frames".format(_sub), _value["droppedFrames"]
)
create_subtest_entry(
"{}_%_dropped_frames".format(_sub), percent_dropped
)
vals = []
subtests = []
names = _subtests.keys()
names.sort(reverse=True)
for name in names:
_subtests[name]["value"] = round(
filters.median(_subtests[name]["replicates"]), 2
)
subtests.append(_subtests[name])
# only include dropped_frames values, without the %_dropped_frames values
if name.endswith("X_dropped_frames"):
vals.append([_subtests[name]["value"], name])
return subtests, vals
class RaptorOutput(PerftestOutput):
"""class for raptor output"""
@ -683,27 +773,9 @@ class RaptorOutput(PerftestOutput):
elif test["type"] == "benchmark":
# to reduce the conditions in the if below, i will use this list
# to check if we are running an youtube playback perf test
youtube_playback_tests = [
"youtube-playbackperf-test",
"youtube-playbackperf-sfr-vp9-test",
"youtube-playbackperf-sfr-h264-test",
"youtube-playbackperf-sfr-av1-test",
"youtube-playbackperf-hfr-test",
"youtube-playbackperf-widevine-sfr-vp9-test",
"youtube-playbackperf-widevine-sfr-h264-test",
"youtube-playbackperf-widevine-hfr-test",
]
youtube_playback_test_name = [
i for i in youtube_playback_tests if i in test["measurements"]
]
if youtube_playback_test_name:
subtests, vals = self.parseYoutubePlaybackPerformanceOutput(
test, test_name=youtube_playback_test_name[0]
)
if any(["youtube-playback" in measurement
for measurement in test["measurements"].keys()]):
subtests, vals = self.parseYoutubePlaybackPerformanceOutput(test)
elif "assorted-dom" in test["measurements"]:
subtests, vals = self.parseAssortedDomOutput(test)
elif "ares6" in test["measurements"]:
@ -1232,89 +1304,6 @@ class RaptorOutput(PerftestOutput):
return subtests, vals
def parseYoutubePlaybackPerformanceOutput(self, test, test_name):
"""Parse the metrics for the Youtube playback performance test.
For each video measured values for dropped and decoded frames will be
available from the benchmark site.
{u'PlaybackPerf.VP9.2160p60@2X': {u'droppedFrames': 1, u'decodedFrames': 796}
With each page cycle / iteration of the test multiple values can be present.
Raptor will calculate the percentage of dropped frames to decoded frames.
All those three values will then be emitted as separate sub tests.
"""
_subtests = {}
data = test["measurements"].get(test_name)
def create_subtest_entry(
name,
value,
unit=test["subtest_unit"],
lower_is_better=test["subtest_lower_is_better"],
):
# build a list of subtests and append all related replicates
if name not in _subtests.keys():
# subtest not added yet, first pagecycle, so add new one
_subtests[name] = {
"name": name,
"unit": unit,
"lowerIsBetter": lower_is_better,
"replicates": [],
}
_subtests[name]["replicates"].append(value)
if self.subtest_alert_on is not None:
if name in self.subtest_alert_on:
LOG.info(
"turning on subtest alerting for measurement type: %s" % name
)
_subtests[name]["shouldAlert"] = True
for pagecycle in data:
for _sub, _value in pagecycle[0].items():
try:
percent_dropped = (
float(_value["droppedFrames"]) / _value["decodedFrames"] * 100.0
)
except ZeroDivisionError:
# if no frames have been decoded the playback failed completely
percent_dropped = 100.0
# Remove the not needed "PlaybackPerf." prefix from each test
_sub = _sub.split("PlaybackPerf", 1)[-1]
if _sub.startswith("."):
_sub = _sub[1:]
# build a list of subtests and append all related replicates
create_subtest_entry(
"{}_decoded_frames".format(_sub),
_value["decodedFrames"],
lower_is_better=False,
)
create_subtest_entry(
"{}_dropped_frames".format(_sub), _value["droppedFrames"]
)
create_subtest_entry(
"{}_%_dropped_frames".format(_sub), percent_dropped
)
vals = []
subtests = []
names = _subtests.keys()
names.sort(reverse=True)
for name in names:
_subtests[name]["value"] = round(
filters.median(_subtests[name]["replicates"]), 2
)
subtests.append(_subtests[name])
# only include dropped_frames values, without the %_dropped_frames values
if name.endswith("X_dropped_frames"):
vals.append([_subtests[name]["value"], name])
return subtests, vals
def summarize_screenshots(self, screenshots):
if len(screenshots) == 0:
return
@ -1489,6 +1478,8 @@ class BrowsertimeOutput(PerftestOutput):
subtests, vals = self.parseSpeedometerOutput(test)
if "ares6" in test["name"]:
subtests, vals = self.parseAresSixOutput(test)
if any("youtube-playback" in key for key in test["measurements"].keys()):
subtests, vals = self.parseYoutubePlaybackPerformanceOutput(test)
suite["subtests"] = subtests
# summarize results for both benchmark type tests

3
testing/raptor/raptor/raptor.ini
Просмотреть файл

@ -114,3 +114,6 @@
# raptor-browsertime live page-load tests
[include:tests/tp6/live/browsertime-live.ini]
# raptor-browsertime benchmark tests
[include:tests/benchmarks/youtube-playback.ini]

137
testing/raptor/raptor/tests/benchmarks/youtube-playback.ini Normal file
Просмотреть файл

@ -0,0 +1,137 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Youtube playback performance benchmark Browsertime tests
#
# Original location of source and media files:
# https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
#
# Bug 1547717 - Cannot override autoplay preference due to GeckoRuntime Settings
# Bug 1554966 - With GeckoView there is no way yet to get around the problem that autoplay is blocked
# As such using muted playback everywhere the numbers across platforms should be closer
# Bug 1618566 - change test_url to production server, once all is done
# test_url = https://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
[DEFAULT]
type = benchmark
use_live_sites = true
gecko_profile_interval = 1000
gecko_profile_entries = 50000000
gecko_profile_threads = MediaPlayback
page_cycles = 1
# account for a page cycle duration of at maximum 45 minutes
page_timeout = 2700000
alert_threshold = 2.0
lower_is_better = true
unit = score
subtest_lower_is_better = true
subtest_unit = score
[youtube-playback]
apps = firefox, geckoview, fenix, fennec,refbrow, chrome
alert_on = H264.1080p30@1X_dropped_frames,
H264.1080p60@1X_dropped_frames,
H264.1440p30@1X_dropped_frames,
H264.144p15@1X_dropped_frames,
H264.2160p30@1X_dropped_frames,
H264.240p30@1X_dropped_frames,
H264.360p30@1X_dropped_frames,
H264.480p30@1X_dropped_frames,
H264.720p30@1X_dropped_frames,
H264.720p60@1X_dropped_frames,
VP9.1080p30@1X_dropped_frames,
VP9.1080p60@1X_dropped_frames,
VP9.1440p30@1X_dropped_frames,
VP9.1440p60@1X_dropped_frames,
VP9.144p30@1X_dropped_frames,
VP9.2160p30@1X_dropped_frames,
VP9.2160p60@1X_dropped_frames,
VP9.240p30@1X_dropped_frames,
VP9.360p30@1X_dropped_frames,
VP9.480p30@1X_dropped_frames,
VP9.720p30@1X_dropped_frames,
VP9.720p60@1X_dropped_frames
test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&exclude=1,2&muted=true
[youtube-playback-av1-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-sfr-av1-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-h264-1080p30]
apps = firefox
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run
[youtube-playback-h264-1080p60]
apps = firefox
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run
[youtube-playback-h264-full-1080p30]
apps = firefox
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-sfr-h264-test&tests=18&raptor=true&muted=true&command=run&fullscreen=true
preferences = {"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0}
[youtube-playback-h264-full-1080p60]
apps = firefox
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=46&muted=true&command=run&fullscreen=true
preferences = {"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0}
[youtube-playback-h264-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-h264-std]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome
alert_on = H2641080p60fps@1X_dropped_frames,
H264720p60fps@1X_dropped_frames
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-v9-1080p30]
apps = firefox
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run
[youtube-playback-v9-1080p60]
apps = firefox
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run
[youtube-playback-v9-full-1080p30]
apps = firefox
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&tests=18&muted=true&command=run&fullscreen=true
preferences = {"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0}
[youtube-playback-v9-full-1080p60]
apps = firefox
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-hfr-test&raptor=true&tests=14&muted=true&command=run&fullscreen=true
preferences = {"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0}
[youtube-playback-vp9-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-widevine-h264-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-widevine-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run
preferences = {"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true,
"media.eme.require-app-approval": false}
[youtube-playback-widevine-hfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-widevine-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
preferences = {"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true,
"media.eme.require-app-approval": false}
[youtube-playback-widevine-vp9-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome
test_url = https://yttest.dev.mozaws.net/testing_2020/perf-youtube-playback/2019/main.html?test_type=playbackperf-widevine-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run
preferences = {"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true,
"media.eme.require-app-approval": false}