mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1766009 - Create WPT chrome job r=perftest-reviewers,sparky,davehunt 

Differential Revision: https://phabricator.services.mozilla.com/D144444
This commit is contained in:
andrej 2022-05-04 17:10:39 +00:00
Родитель 8ebf835d75
Коммит 23829f29b7
11 изменённых файлов: 357 добавлений и 58 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

76
python/mozperftest/mozperftest/test/webpagetest.py
Просмотреть файл

@ -8,25 +8,6 @@ import mozperftest.utils as utils
import pathlib
from mozperftest.runner import HERE
WPT_METRICS = [
"firstContentfulPaint",
"timeToContentfulPaint",
"visualComplete90",
"firstPaint",
"visualComplete99",
"visualComplete",
"SpeedIndex",
"bytesIn",
"bytesOut",
"TTFB",
"fullyLoadedCPUms",
"fullyLoadedCPUpct",
"domElements",
"domContentLoadedEventStart",
"domContentLoadedEventEnd",
"loadEventStart",
"loadEventEnd",
]
ACCEPTED_BROWSERS = ["Chrome", "Firefox"]
ACCEPTED_CONNECTIONS = [
@ -196,6 +177,7 @@ class WebPageTest(Layer):
options = metadata.script["options"]
test_list = options["test_list"]
self.statistic_types = options["test_parameters"]["statistics"]
self.wpt_browser_metrics = options["browser_metrics"]
self.pre_run_error_checks(options["test_parameters"], test_list)
self.create_and_run_wpt_threaded_tests(test_list, metadata)
return metadata
@ -339,22 +321,23 @@ class WebPageTest(Layer):
def add_wpt_run_to_metadata(self, wbt_run, metadata, website):
requested_values = self.extract_desired_values_from_wpt_run(wbt_run)
metadata.add_result(
{
"name": ("WebPageTest:" + re.match(r"(^.\w+)", website)[0]),
"framework": {"name": "mozperftest"},
"transformer": "mozperftest.test.webpagetest:WebPageTestData",
"shouldAlert": True,
"results": [
{
"values": [int(metric_value)],
"name": metric_name,
"shouldAlert": True,
}
for metric_name, metric_value in requested_values.items()
],
}
)
if requested_values is not None:
metadata.add_result(
{
"name": ("WebPageTest:" + re.match(r"(^.\w+)", website)[0]),
"framework": {"name": "mozperftest"},
"transformer": "mozperftest.test.webpagetest:WebPageTestData",
"shouldAlert": True,
"results": [
{
"values": [metric_value],
"name": metric_name,
"shouldAlert": True,
}
for metric_name, metric_value in requested_values.items()
],
}
)
def extract_desired_values_from_wpt_run(self, wpt_run):
view_types = ["firstView"]
@ -363,10 +346,23 @@ class WebPageTest(Layer):
desired_values = {}
for statistic in self.statistic_types:
for view in view_types:
for value in WPT_METRICS:
for value in self.wpt_browser_metrics:
if isinstance(wpt_run["data"][statistic][view], list):
self.error(f"Fail {wpt_run['data']['url']}")
return
if value not in wpt_run["data"][statistic][view].keys():
raise WPTDataProcessingError(f"{value} not found wpt results ")
desired_values[f"{value}.{view}.{statistic}"] = wpt_run["data"][
statistic
][view][value]
raise WPTDataProcessingError(f"{value} not found wpt results")
desired_values[f"{value}.{view}.{statistic}"] = int(
wpt_run["data"][statistic][view][value]
)
try:
desired_values["browserVersion"] = float(
re.match(
r"\d+.\d+",
wpt_run["data"]["runs"]["1"]["firstView"]["browserVersion"],
)[0]
)
desired_values["webPagetestVersion"] = float(wpt_run["webPagetestVersion"])
except Exception:
self.error("Issue found with processing browser/WPT version")
return desired_values

30
testing/performance/perftest_WPT_init_file.js → python/mozperftest/mozperftest/tests/data/samples/test_perftest_WPT_init_file.js
Просмотреть файл

@ -16,11 +16,12 @@ module.exports = {
owner: "Performance Testing Team",
name: "webpagetest",
description:
"Run webpagetest performance pageload test against Alexa top 50 websites",
longDescription: `This mozperftest gets webpagetest to run pageload tests against the 50 most popular websites and provide data.
The full list of data returned from webpagetest: firstContentfulPaint, timeToContentfulPaint, visualComplete90,
firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut,TTFB,fullyLoadedCPUms, fullyLoadedCPUpct,
domElements, domContentLoadedEventStart, domContentLoadedEventEnd, loadEventStart, loadEventEnd`,
"Run webpagetest performance pageload tests on Firefox against Alexa top 50 websites",
longDescription: `This mozperftest gets webpagetest to run pageload tests on Firefox against the 50 most popular
websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint,
timeToContentfulPaint, visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn, bytesOut,
TTFB, fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd,
loadEventStart, loadEventEnd`,
options: {
test_parameters: {
location: "ec2-us-east-1",
@ -90,5 +91,24 @@ module.exports = {
"amazon.co.jp",
"okezone.com",
],
browser_metrics: [
"firstContentfulPaint",
"timeToContentfulPaint",
"visualComplete90",
"firstPaint",
"visualComplete99",
"visualComplete",
"SpeedIndex",
"bytesIn",
"bytesOut",
"TTFB",
"fullyLoadedCPUms",
"fullyLoadedCPUpct",
"domElements",
"domContentLoadedEventStart",
"domContentLoadedEventEnd",
"loadEventStart",
"loadEventEnd",
],
},
};

2
python/mozperftest/mozperftest/tests/support.py
Просмотреть файл

@ -20,7 +20,7 @@ EXAMPLE_TESTS_DIR = os.path.join(HERE, "data", "samples")
EXAMPLE_TEST = os.path.join(EXAMPLE_TESTS_DIR, "perftest_example.js")
EXAMPLE_XPCSHELL_TEST = Path(EXAMPLE_TESTS_DIR, "test_xpcshell.js")
EXAMPLE_XPCSHELL_TEST2 = Path(EXAMPLE_TESTS_DIR, "test_xpcshell_flavor2.js")
EXAMPLE_WPT_TEST = Path(ROOT, "testing", "performance", "perftest_WPT_init_file.js")
EXAMPLE_WPT_TEST = Path(EXAMPLE_TESTS_DIR, "test_perftest_WPT_init_file.js")
BT_DATA = Path(HERE, "data", "browsertime-results", "browsertime.json")
BT_DATA_VIDEO = Path(HERE, "data", "browsertime-results-video", "browsertime.json")
DMG = Path(HERE, "data", "firefox.dmg")

25
python/mozperftest/mozperftest/tests/test_webpagetest.py
Просмотреть файл

@ -13,13 +13,32 @@ from mozperftest.test.webpagetest import (
WPTBrowserSelectionError,
WPTInvalidURLError,
WPTLocationSelectionError,
WPT_METRICS,
WPTInvalidConnectionSelection,
ACCEPTED_STATISTICS,
WPTInvalidStatisticsError,
WPTDataProcessingError,
)
WPT_METRICS = [
"firstContentfulPaint",
"timeToContentfulPaint",
"visualComplete90",
"firstPaint",
"visualComplete99",
"visualComplete",
"SpeedIndex",
"bytesIn",
"bytesOut",
"TTFB",
"fullyLoadedCPUms",
"fullyLoadedCPUpct",
"domElements",
"domContentLoadedEventStart",
"domContentLoadedEventEnd",
"loadEventStart",
"loadEventEnd",
]
class WPTTests:
def __init__(self, log):
@ -48,7 +67,9 @@ def init_placeholder_wpt_data(fvonly=False, invalid_results=False):
"average": views,
"standardDeviation": views,
"median": views,
}
"runs": {"1": {"firstView": {"browserVersion": 101.1}}},
},
"webPagetestVersion": 21.0,
}
exclude_metrics = 0 if not invalid_results else 2
for metric in WPT_METRICS[exclude_metrics:]:

3
python/mozperftest/perfdocs/config.yml
Просмотреть файл

@ -34,7 +34,8 @@ suites:
pageload: ""
JSConf (warm): ""
perfstats: ""
webpagetest: ""
webpagetest-firefox: ""
webpagetest-chrome: ""
browser/base/content/test:
description: "Performance tests from the 'browser/base/content/test' folder."
tests:

1
taskcluster/ci/config.yml
Просмотреть файл

@ -203,6 +203,7 @@ treeherder:
'doc': 'Documentation'
'GhS': 'GitHub Synchronization'
'perftest': 'Performance tests'
'perftest-chrome': 'Performance tests using Chrone'
'perftest-http3': 'Performance tests with HTTP/3'
'l10n': 'Localization checks'
'fxrec': 'Desktop startup recorder (fxrecord)'

28
taskcluster/ci/perftest/linux.yml
Просмотреть файл

@ -37,8 +37,8 @@ try-xpcshell:
--xpcshell-nodejs ${MOZ_FETCHES_DIR}/node/bin/node
--xpcshell-xre-path ${MOZ_FETCHES_DIR}/firefox
webpagetest:
description: Run webpagetest performance pageload test against Alexa top 50 websites
webpagetest-firefox:
description: Run webpagetest performance pageload tests on Firefox against Alexa top 50 websites
treeherder:
symbol: perftest(linux-webpagetest)
attributes:
@ -50,11 +50,33 @@ webpagetest:
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
cd $MOZ_FETCHES_DIR &&
python3.8 python/mozperftest/mozperftest/runner.py
testing/performance/perftest_WPT_init_file.js
testing/performance/perftest_WPT_firefox_init_file.js
--flavor=webpagetest
--webpagetest
--perfherder
--perfherder-metrics median
--perfherder-app=firefox
--output $MOZ_FETCHES_DIR/../artifacts
webpagetest-chrome:
description: Run webpagetest performance pageload tests on Chrome against Alexa top 50 websites
treeherder:
symbol: perftest-chrome(linux-webpagetest)
attributes:
cron: true
scopes:
- secrets:get:project/perftest/gecko/level-{level}/perftest-login
run:
command: >-
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
cd $MOZ_FETCHES_DIR &&
python3.8 python/mozperftest/mozperftest/runner.py
testing/performance/perftest_WPT_chrome_init_file.js
--flavor=webpagetest
--webpagetest
--perfherder
--perfherder-metrics median
--perfherder-app=chrome
--output $MOZ_FETCHES_DIR/../artifacts
try-browsertime:

20
testing/perfdocs/generated/mozperftest.rst
Просмотреть файл

@ -211,15 +211,25 @@ perftest_perfstats.js
This test launches browsertime with the perfStats option (will collect low-overhead timings, see Bug 1553254). The test currently runs a short user journey. A selection of popular sites are visited, first as cold pageloads, and then as warm.
perftest_WPT_init_file.js
=========================
perftest_WPT_chrome_init_file.js
================================
:owner: Performance Testing Team
:name: webpagetest
:name: webpagetest-chrome
**Run webpagetest performance pageload test against Alexa top 50 websites**
**Run webpagetest performance pageload tests on Chrome against Alexa top 50 websites**
This mozperftest gets webpagetest to run pageload tests against the 50 most popular websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint, timeToContentfulPaint, visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut,TTFB,fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd, loadEventStart, loadEventEnd
This mozperftest gets webpagetest to run pageload tests on Chrome against the 50 most popular websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint, visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut, TTFB, fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd, loadEventStart, loadEventEnd
perftest_WPT_firefox_init_file.js
=================================
:owner: Performance Testing Team
:name: webpagetest-firefox
**Run webpagetest performance pageload tests on Firefox against Alexa top 50 websites**
This mozperftest gets webpagetest to run pageload tests on Firefox against the 50 most popular websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint, timeToContentfulPaint, visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn, bytesOut, TTFB, fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd, loadEventStart, loadEventEnd
If you have any questions, please see this `wiki page <https://wiki.mozilla.org/TestEngineering/Performance#Where_to_find_us>`_.

3
testing/performance/perftest.ini
Просмотреть файл

@ -8,4 +8,5 @@
[perftest_perfstats.js]
[perftest_politico_link.js]
[perftest_youtube_link.js]
[perftest_WPT_init_file.js]
[perftest_WPT_firefox_init_file.js]
[perftest_WPT_chrome_init_file.js]

113
testing/performance/perftest_WPT_chrome_init_file.js Normal file
Просмотреть файл

@ -0,0 +1,113 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
/* eslint-env node */
"use strict";
function setUp() {}
function tearDown() {}
function test() {}
module.exports = {
setUp,
tearDown,
test,
owner: "Performance Testing Team",
name: "webpagetest-chrome",
description:
"Run webpagetest performance pageload tests on Chrome against Alexa top 50 websites",
longDescription: `This mozperftest gets webpagetest to run pageload tests on Chrome against the 50 most popular
websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint,
visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn,bytesOut,
TTFB, fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd,
loadEventStart, loadEventEnd`,
options: {
test_parameters: {
location: "ec2-us-east-1",
browser: "Chrome",
connection: "Cable",
timeout_limit: 21600,
wait_between_requests: 5,
statistics: ["average", "median", "standardDeviation"],
label: "",
runs: 3,
fvonly: 0,
private: 1,
web10: 0,
script: "",
block: "",
video: 1,
tcpdump: 0,
noimages: 0,
keepua: 1,
uastring: "",
htmlbody: 0,
custom: "",
ignoreSSL: 0,
appendua: "",
injectScript: "",
disableAVIF: 0,
disableWEBP: 0,
disableJXL: 0,
},
test_list: [
"google.com",
"youtube.com",
"facebook.com",
"qq.com",
"baidu.com",
"sohu.com",
"360.cn",
"jd.com",
"amazon.com",
"yahoo.com",
"zoom.us",
"weibo.com",
"sina.com.cn",
"live.com",
"reddit.com",
"netflix.com",
"microsoft.com",
"instagram.com",
"panda.tv",
"google.com.hk",
"csdn.net",
"bing.com",
"vk.com",
"yahoo.co.jp",
"twitter.com",
"naver.com",
"canva.com",
"ebay.com",
"force.com",
"amazon.in",
"adobe.com",
"aliexpress.com",
"linkedin.com",
"tianya.cn",
"yy.com",
"huanqiu.com",
"amazon.co.jp",
"okezone.com",
],
browser_metrics: [
"firstContentfulPaint",
"visualComplete90",
"firstPaint",
"visualComplete99",
"visualComplete",
"SpeedIndex",
"bytesIn",
"bytesOut",
"TTFB",
"fullyLoadedCPUms",
"fullyLoadedCPUpct",
"domElements",
"domContentLoadedEventStart",
"domContentLoadedEventEnd",
"loadEventStart",
"loadEventEnd",
],
},
};

114
testing/performance/perftest_WPT_firefox_init_file.js Normal file
Просмотреть файл

@ -0,0 +1,114 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
/* eslint-env node */
"use strict";
function setUp() {}
function tearDown() {}
function test() {}
module.exports = {
setUp,
tearDown,
test,
owner: "Performance Testing Team",
name: "webpagetest-firefox",
description:
"Run webpagetest performance pageload tests on Firefox against Alexa top 50 websites",
longDescription: `This mozperftest gets webpagetest to run pageload tests on Firefox against the 50 most popular
websites and provide data. The full list of data returned from webpagetest: firstContentfulPaint,
timeToContentfulPaint, visualComplete90, firstPaint, visualComplete99, visualComplete, SpeedIndex, bytesIn, bytesOut,
TTFB, fullyLoadedCPUms, fullyLoadedCPUpct, domElements, domContentLoadedEventStart, domContentLoadedEventEnd,
loadEventStart, loadEventEnd`,
options: {
test_parameters: {
location: "ec2-us-east-1",
browser: "Firefox",
connection: "Cable",
timeout_limit: 21600,
wait_between_requests: 5,
statistics: ["average", "median", "standardDeviation"],
label: "",
runs: 3,
fvonly: 0,
private: 1,
web10: 0,
script: "",
block: "",
video: 1,
tcpdump: 0,
noimages: 0,
keepua: 1,
uastring: "",
htmlbody: 0,
custom: "",
ignoreSSL: 0,
appendua: "",
injectScript: "",
disableAVIF: 0,
disableWEBP: 0,
disableJXL: 0,
},
test_list: [
"google.com",
"youtube.com",
"facebook.com",
"qq.com",
"baidu.com",
"sohu.com",
"360.cn",
"jd.com",
"amazon.com",
"yahoo.com",
"zoom.us",
"weibo.com",
"sina.com.cn",
"live.com",
"reddit.com",
"netflix.com",
"microsoft.com",
"instagram.com",
"panda.tv",
"google.com.hk",
"csdn.net",
"bing.com",
"vk.com",
"yahoo.co.jp",
"twitter.com",
"naver.com",
"canva.com",
"ebay.com",
"force.com",
"amazon.in",
"adobe.com",
"aliexpress.com",
"linkedin.com",
"tianya.cn",
"yy.com",
"huanqiu.com",
"amazon.co.jp",
"okezone.com",
],
browser_metrics: [
"firstContentfulPaint",
"timeToContentfulPaint",
"visualComplete90",
"firstPaint",
"visualComplete99",
"visualComplete",
"SpeedIndex",
"bytesIn",
"bytesOut",
"TTFB",
"fullyLoadedCPUms",
"fullyLoadedCPUpct",
"domElements",
"domContentLoadedEventStart",
"domContentLoadedEventEnd",
"loadEventStart",
"loadEventEnd",
],
},
};