Bug 1752350 - Add matrix-react-bench to raptor-browsertime. r=jandem,perftest-reviewers,AlexandruIonescu

This patch adds the matrix-react-bench to raptor-browsertime. It will run alongside the other benchmarks on trunk and mozilla-beta. It will also be possible to run these on chrome and chromium.

Differential Revision: https://phabricator.services.mozilla.com/D137344

taskcluster/ci/fetch/benchmarks.yml

@@ -50,3 +50,11 @@ jetstream2:
         url: https://github.com/mozilla/perf-automation/releases/download/jetstream2-v1.2/jetstream2-54c3c3d9d3f6.zip
         sha256: 978f5920f27099d4a6b854c5e4a50d2a706fad8b5082a6fdcab95bb71fb6dc12
         size: 25481279
+
+matrix-react-bench:
+    description: Matrix-React Benchmark
+    fetch:
+        type: static-url
+        url: https://github.com/mozilla/perf-automation/releases/download/matrix-react-bench-v1/matrix-react-bench-7eef4a5eb4bd.zip
+        sha256: 785b8053466b4de089f9fda69f367e508b8bd50baf8a01cda0ee9f18be662dca
+        size: 57514580

taskcluster/ci/test/browsertime-desktop.yml

@@ -202,6 +202,7 @@ browsertime-benchmark:
             - ares6
             - [assorted-dom, dom]
             - [jetstream2, js2]
+            - [matrix-react-bench, mrb]
             - [motionmark-animometer, mm-a]
             - [motionmark-htmlsuite, mm-h]
             - [speedometer, sp]
@@ -271,6 +272,7 @@ browsertime-benchmark:
                     .*-qr/.*: 2100
                     default: 1500
             jetstream2: 8000
+            matrix-react-bench: 1500
             motionmark-htmlsuite: 1500
             unity-webgl: 1500
             default: 900
@@ -278,6 +280,7 @@ browsertime-benchmark:
         fetch:
             - assorted-dom
             - jetstream2
+            - matrix-react-bench
             - unity-webgl
 
 browsertime-benchmark-wasm:

testing/perfdocs/generated/raptor.rst

@@ -728,6 +728,236 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
 
 
 
+.. dropdown:: matrix-react-bench
+   :container: + anchor-id-matrix-react-bench-b
+
+   **Owner**: :jandem and SpiderMonkey Team
+
+   * **alert threshold**: 2.0
+   * **apps**: firefox, chrome, chromium
+   * **expected**: pass
+   * **fetch task**: matrix-react-bench
+   * **gecko profile entries**: 14000000
+   * **gecko profile interval**: 1
+   * **lower is better**: true
+   * **page cycles**: 30
+   * **page timeout**: 2000000
+   * **subtest lower is better**: true
+   * **subtest unit**: ms
+   * **test url**: `<http://\<host\>:\<port\>/matrix-react-bench/matrix_demo.html>`__
+   * **type**: benchmark
+   * **unit**: ms
+   * **Test Task**:
+
+   .. list-table:: **test-linux1804-64-clang-trunk-qr/opt**
+      :widths: 30 15 15 15 15
+      :header-rows: 1
+
+      * - **Test Name**
+        - mozilla-central
+        - autoland
+        - mozilla-release
+        - mozilla-beta
+      * - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+
+
+   .. list-table:: **test-linux1804-64-qr/opt**
+      :widths: 30 15 15 15 15
+      :header-rows: 1
+
+      * - **Test Name**
+        - mozilla-central
+        - autoland
+        - mozilla-release
+        - mozilla-beta
+      * - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+
+
+   .. list-table:: **test-linux1804-64-shippable-qr/opt**
+      :widths: 30 15 15 15 15
+      :header-rows: 1
+
+      * - **Test Name**
+        - mozilla-central
+        - autoland
+        - mozilla-release
+        - mozilla-beta
+      * - **browsertime-benchmark-chrome-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-chromium-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
+        - ✅
+        - ✅
+        - ❌
+        - ✅
+
+
+   .. list-table:: **test-macosx1015-64-shippable-qr/opt**
+      :widths: 30 15 15 15 15
+      :header-rows: 1
+
+      * - **Test Name**
+        - mozilla-central
+        - autoland
+        - mozilla-release
+        - mozilla-beta
+      * - **browsertime-benchmark-chrome-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-chromium-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
+        - ✅
+        - ✅
+        - ❌
+        - ✅
+
+
+   .. list-table:: **test-windows10-32-qr/opt**
+      :widths: 30 15 15 15 15
+      :header-rows: 1
+
+      * - **Test Name**
+        - mozilla-central
+        - autoland
+        - mozilla-release
+        - mozilla-beta
+      * - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+
+
+   .. list-table:: **test-windows10-32-shippable-qr/opt**
+      :widths: 30 15 15 15 15
+      :header-rows: 1
+
+      * - **Test Name**
+        - mozilla-central
+        - autoland
+        - mozilla-release
+        - mozilla-beta
+      * - **browsertime-benchmark-chrome-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-chromium-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+
+
+   .. list-table:: **test-windows10-64-qr/opt**
+      :widths: 30 15 15 15 15
+      :header-rows: 1
+
+      * - **Test Name**
+        - mozilla-central
+        - autoland
+        - mozilla-release
+        - mozilla-beta
+      * - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+
+
+   .. list-table:: **test-windows10-64-shippable-qr/opt**
+      :widths: 30 15 15 15 15
+      :header-rows: 1
+
+      * - **Test Name**
+        - mozilla-central
+        - autoland
+        - mozilla-release
+        - mozilla-beta
+      * - **browsertime-benchmark-chrome-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-chromium-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-e10s**
+        - ❌
+        - ❌
+        - ❌
+        - ❌
+      * - **browsertime-benchmark-firefox-matrix-react-bench-fis-e10s**
+        - ✅
+        - ✅
+        - ❌
+        - ✅
+
+
+
 .. dropdown:: motionmark-animometer
    :container: + anchor-id-motionmark-animometer-b
 

testing/raptor/mach_commands.py

@@ -28,7 +28,7 @@ from mozbuild.base import MachCommandConditions as Conditions
 HERE = os.path.dirname(os.path.realpath(__file__))
 
 BENCHMARK_REPOSITORY = "https://github.com/mozilla/perf-automation"
-BENCHMARK_REVISION = "54c3c3d9d3f651f0d8ebb809d25232f72b5b06f2"
+BENCHMARK_REVISION = "7eef4a5eb4bd2d7558967e947b57cda33e287133"
 
 ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m"]
 

testing/raptor/raptor/output.py

@@ -1061,6 +1061,40 @@ class PerftestOutput(object):
 
         return subtests, vals
 
+    def parseMatrixReactBenchOutput(self, test):
+        # https://github.com/jandem/matrix-react-bench
+
+        _subtests = {}
+        data = test["measurements"]["matrix-react-bench"]
+        for page_cycle in data:
+            # Each cycle is formatted like `[[iterations, val], [iterations, val2], ...]`
+            for iteration, val in page_cycle:
+                sub = f"{iteration}-iterations"
+                _subtests.setdefault(
+                    sub,
+                    {
+                        "unit": test["subtest_unit"],
+                        "alertThreshold": float(test["alert_threshold"]),
+                        "lowerIsBetter": test["subtest_lower_is_better"],
+                        "name": sub,
+                        "replicates": [],
+                    },
+                )
+
+                # The values produced are far too large for perfherder
+                _subtests[sub]["replicates"].append(val)
+
+        vals = []
+        subtests = []
+        names = list(_subtests)
+        names.sort(reverse=True)
+        for name in names:
+            _subtests[name]["value"] = filters.mean(_subtests[name]["replicates"])
+            subtests.append(_subtests[name])
+            vals.append([_subtests[name]["value"], name])
+
+        return subtests, vals
+
 
 class RaptorOutput(PerftestOutput):
     """class for raptor output"""
@@ -1582,6 +1616,8 @@ class BrowsertimeOutput(PerftestOutput):
                     subtests, vals = self.parseAssortedDomOutput(test)
                 if "jetstream2" in test["measurements"]:
                     subtests, vals = self.parseJetstreamTwoOutput(test)
+                if "matrix-react-bench" in test["name"]:
+                    subtests, vals = self.parseMatrixReactBenchOutput(test)
 
                 if subtests is None:
                     raise Exception("No benchmark metrics found in browsertime results")

testing/raptor/raptor/perfdocs/config.yml

@@ -97,6 +97,7 @@ suites:
             ares6: "FF, CH, CU"
             assorted-dom: "FF, CH, CU"
             jetstream2: "FF, CH, CU"
+            matrix-react-bench: "FF, CH, CU"
             motionmark-animometer: "FF, CH, CU"
             motionmark-htmlsuite: "FF, CH, CU"
             raptor-speedometer-geckoview: "GV"

testing/raptor/raptor/raptor.ini

@@ -40,6 +40,7 @@
 [include:tests/benchmarks/sunspider.ini]
 [include:tests/benchmarks/assorted-dom.ini]
 [include:tests/benchmarks/jetstream2.ini]
+[include:tests/benchmarks/matrix-react-bench.ini]
 [include:tests/benchmarks/unity-webgl-desktop.ini]
 
 # Fission process switch time test

testing/raptor/raptor/tests/benchmarks/matrix-react-bench.ini

@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Matrix-react-bench benchmark for firefox and chromium distributions
+
+[DEFAULT]
+alert_threshold = 2.0
+apps = firefox, chrome, chromium
+fetch_task = matrix-react-bench
+gecko_profile_entries = 14000000
+gecko_profile_interval = 1
+lower_is_better = true
+owner = :jandem and SpiderMonkey Team
+page_cycles = 30
+page_timeout = 2000000
+subtest_lower_is_better = true
+subtest_unit = ms
+test_url = http://<host>:<port>/matrix-react-bench/matrix_demo.html
+type = benchmark
+unit = ms
+
+[matrix-react-bench]