зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1752350 - Add matrix-react-bench to raptor-browsertime. r=jandem,perftest-reviewers,AlexandruIonescu
This patch adds the matrix-react-bench to raptor-browsertime. It will run alongside the other benchmarks on trunk and mozilla-beta. It will also be possible to run these on chrome and chromium. Differential Revision: https://phabricator.services.mozilla.com/D137344
This commit is contained in:
Родитель
9e561c3134
Коммит
5fee015336
|
@ -50,3 +50,11 @@ jetstream2:
|
|||
url: https://github.com/mozilla/perf-automation/releases/download/jetstream2-v1.2/jetstream2-54c3c3d9d3f6.zip
|
||||
sha256: 978f5920f27099d4a6b854c5e4a50d2a706fad8b5082a6fdcab95bb71fb6dc12
|
||||
size: 25481279
|
||||
|
||||
matrix-react-bench:
|
||||
description: Matrix-React Benchmark
|
||||
fetch:
|
||||
type: static-url
|
||||
url: https://github.com/mozilla/perf-automation/releases/download/matrix-react-bench-v1/matrix-react-bench-7eef4a5eb4bd.zip
|
||||
sha256: 785b8053466b4de089f9fda69f367e508b8bd50baf8a01cda0ee9f18be662dca
|
||||
size: 57514580
|
||||
|
|
|
@ -202,6 +202,7 @@ browsertime-benchmark:
|
|||
- ares6
|
||||
- [assorted-dom, dom]
|
||||
- [jetstream2, js2]
|
||||
- [matrix-react-bench, mrb]
|
||||
- [motionmark-animometer, mm-a]
|
||||
- [motionmark-htmlsuite, mm-h]
|
||||
- [speedometer, sp]
|
||||
|
@ -271,6 +272,7 @@ browsertime-benchmark:
|
|||
.*-qr/.*: 2100
|
||||
default: 1500
|
||||
jetstream2: 8000
|
||||
matrix-react-bench: 1500
|
||||
motionmark-htmlsuite: 1500
|
||||
unity-webgl: 1500
|
||||
default: 900
|
||||
|
@ -278,6 +280,7 @@ browsertime-benchmark:
|
|||
fetch:
|
||||
- assorted-dom
|
||||
- jetstream2
|
||||
- matrix-react-bench
|
||||
- unity-webgl
|
||||
|
||||
browsertime-benchmark-wasm:
|
||||
|
|
|
@ -728,6 +728,236 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
|
|||
|
||||
|
||||
|
||||
.. dropdown:: matrix-react-bench
|
||||
:container: + anchor-id-matrix-react-bench-b
|
||||
|
||||
**Owner**: :jandem and SpiderMonkey Team
|
||||
|
||||
* **alert threshold**: 2.0
|
||||
* **apps**: firefox, chrome, chromium
|
||||
* **expected**: pass
|
||||
* **fetch task**: matrix-react-bench
|
||||
* **gecko profile entries**: 14000000
|
||||
* **gecko profile interval**: 1
|
||||
* **lower is better**: true
|
||||
* **page cycles**: 30
|
||||
* **page timeout**: 2000000
|
||||
* **subtest lower is better**: true
|
||||
* **subtest unit**: ms
|
||||
* **test url**: `<http://\<host\>:\<port\>/matrix-react-bench/matrix_demo.html>`__
|
||||
* **type**: benchmark
|
||||
* **unit**: ms
|
||||
* **Test Task**:
|
||||
|
||||
.. list-table:: **test-linux1804-64-clang-trunk-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-linux1804-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-benchmark-chrome-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-chromium-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ✅
|
||||
|
||||
|
||||
.. list-table:: **test-macosx1015-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-benchmark-chrome-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-chromium-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ✅
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-32-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-32-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-benchmark-chrome-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-chromium-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
|
||||
|
||||
.. list-table:: **test-windows10-64-shippable-qr/opt**
|
||||
:widths: 30 15 15 15 15
|
||||
:header-rows: 1
|
||||
|
||||
* - **Test Name**
|
||||
- mozilla-central
|
||||
- autoland
|
||||
- mozilla-release
|
||||
- mozilla-beta
|
||||
* - **browsertime-benchmark-chrome-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-chromium-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
- ❌
|
||||
* - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
|
||||
- ✅
|
||||
- ✅
|
||||
- ❌
|
||||
- ✅
|
||||
|
||||
|
||||
|
||||
.. dropdown:: motionmark-animometer
|
||||
:container: + anchor-id-motionmark-animometer-b
|
||||
|
||||
|
|
|
@ -28,7 +28,7 @@ from mozbuild.base import MachCommandConditions as Conditions
|
|||
HERE = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
BENCHMARK_REPOSITORY = "https://github.com/mozilla/perf-automation"
|
||||
BENCHMARK_REVISION = "54c3c3d9d3f651f0d8ebb809d25232f72b5b06f2"
|
||||
BENCHMARK_REVISION = "7eef4a5eb4bd2d7558967e947b57cda33e287133"
|
||||
|
||||
ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m"]
|
||||
|
||||
|
|
|
@ -1061,6 +1061,40 @@ class PerftestOutput(object):
|
|||
|
||||
return subtests, vals
|
||||
|
||||
def parseMatrixReactBenchOutput(self, test):
|
||||
# https://github.com/jandem/matrix-react-bench
|
||||
|
||||
_subtests = {}
|
||||
data = test["measurements"]["matrix-react-bench"]
|
||||
for page_cycle in data:
|
||||
# Each cycle is formatted like `[[iterations, val], [iterations, val2], ...]`
|
||||
for iteration, val in page_cycle:
|
||||
sub = f"{iteration}-iterations"
|
||||
_subtests.setdefault(
|
||||
sub,
|
||||
{
|
||||
"unit": test["subtest_unit"],
|
||||
"alertThreshold": float(test["alert_threshold"]),
|
||||
"lowerIsBetter": test["subtest_lower_is_better"],
|
||||
"name": sub,
|
||||
"replicates": [],
|
||||
},
|
||||
)
|
||||
|
||||
# The values produced are far too large for perfherder
|
||||
_subtests[sub]["replicates"].append(val)
|
||||
|
||||
vals = []
|
||||
subtests = []
|
||||
names = list(_subtests)
|
||||
names.sort(reverse=True)
|
||||
for name in names:
|
||||
_subtests[name]["value"] = filters.mean(_subtests[name]["replicates"])
|
||||
subtests.append(_subtests[name])
|
||||
vals.append([_subtests[name]["value"], name])
|
||||
|
||||
return subtests, vals
|
||||
|
||||
|
||||
class RaptorOutput(PerftestOutput):
|
||||
"""class for raptor output"""
|
||||
|
@ -1582,6 +1616,8 @@ class BrowsertimeOutput(PerftestOutput):
|
|||
subtests, vals = self.parseAssortedDomOutput(test)
|
||||
if "jetstream2" in test["measurements"]:
|
||||
subtests, vals = self.parseJetstreamTwoOutput(test)
|
||||
if "matrix-react-bench" in test["name"]:
|
||||
subtests, vals = self.parseMatrixReactBenchOutput(test)
|
||||
|
||||
if subtests is None:
|
||||
raise Exception("No benchmark metrics found in browsertime results")
|
||||
|
|
|
@ -97,6 +97,7 @@ suites:
|
|||
ares6: "FF, CH, CU"
|
||||
assorted-dom: "FF, CH, CU"
|
||||
jetstream2: "FF, CH, CU"
|
||||
matrix-react-bench: "FF, CH, CU"
|
||||
motionmark-animometer: "FF, CH, CU"
|
||||
motionmark-htmlsuite: "FF, CH, CU"
|
||||
raptor-speedometer-geckoview: "GV"
|
||||
|
|
|
@ -40,6 +40,7 @@
|
|||
[include:tests/benchmarks/sunspider.ini]
|
||||
[include:tests/benchmarks/assorted-dom.ini]
|
||||
[include:tests/benchmarks/jetstream2.ini]
|
||||
[include:tests/benchmarks/matrix-react-bench.ini]
|
||||
[include:tests/benchmarks/unity-webgl-desktop.ini]
|
||||
|
||||
# Fission process switch time test
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# Matrix-react-bench benchmark for firefox and chromium distributions
|
||||
|
||||
[DEFAULT]
|
||||
alert_threshold = 2.0
|
||||
apps = firefox, chrome, chromium
|
||||
fetch_task = matrix-react-bench
|
||||
gecko_profile_entries = 14000000
|
||||
gecko_profile_interval = 1
|
||||
lower_is_better = true
|
||||
owner = :jandem and SpiderMonkey Team
|
||||
page_cycles = 30
|
||||
page_timeout = 2000000
|
||||
subtest_lower_is_better = true
|
||||
subtest_unit = ms
|
||||
test_url = http://<host>:<port>/matrix-react-bench/matrix_demo.html
|
||||
type = benchmark
|
||||
unit = ms
|
||||
|
||||
[matrix-react-bench]
|
Загрузка…
Ссылка в новой задаче