Bug 1672142 - Add perf test for the controlled website r=sparky,dragana

Differential Revision: https://phabricator.services.mozilla.com/D94089

netwerk/test/perf/hooks_throttling.py

@@ -0,0 +1,201 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Drives the throttling feature when the test calls our
+controlled server.
+"""
+import time
+import http.client
+import os
+import json
+from urllib.parse import urlparse
+import sys
+
+from mozperftest.test.browsertime import add_option
+from mozperftest.utils import get_tc_secret
+
+
+ENDPOINTS = {
+    "linux": "h3.dev.mozaws.net",
+    "darwin": "h3.mac.dev.mozaws.net",
+    "win32": "h3.win.dev.mozaws.net",
+}
+CTRL_SERVER = ENDPOINTS[sys.platform]
+TASK_CLUSTER = "TASK_ID" in os.environ.keys()
+_SECRET = {
+    "throttler_host": f"https://{CTRL_SERVER}/_throttler",
+    "throttler_key": os.environ.get("WEBNETEM_KEY", ""),
+}
+if TASK_CLUSTER:
+    _SECRET.update(get_tc_secret())
+
+if _SECRET["throttler_key"] == "":
+    raise Exception("WEBNETEM_KEY not set")
+
+_TIMEOUT = 30
+WAIT_TIME = 60 * 10
+IDLE_TIME = 10
+BREATHE_TIME = 20
+
+
+class Throttler:
+    def __init__(self, env, host, key):
+        self.env = env
+        self.host = host
+        self.key = key
+        self.verbose = env.get_arg("verbose", False)
+        self.logger = self.verbose and self.env.info or self.env.debug
+
+    def log(self, msg):
+        self.logger("[throttler] " + msg)
+
+    def _request(self, action, data=None):
+        kw = {}
+        headers = {b"X-WEBNETEM-KEY": self.key}
+        verb = data is None and "GET" or "POST"
+        if data is not None:
+            data = json.dumps(data)
+            headers[b"Content-type"] = b"application/json"
+
+        parsed = urlparse(self.host)
+        server = parsed.netloc
+        path = parsed.path
+        if action != "status":
+            path += "/" + action
+
+        self.log(f"Calling {verb} {path}")
+        conn = http.client.HTTPSConnection(server, timeout=_TIMEOUT)
+        conn.request(verb, path, body=data, headers=headers, **kw)
+        resp = conn.getresponse()
+        res = resp.read()
+        if resp.status >= 400:
+            raise Exception(res)
+        res = json.loads(res)
+        return res
+
+    def start(self, data=None):
+        self.log("Starting")
+        now = time.time()
+        acquired = False
+
+        while time.time() - now < WAIT_TIME:
+            status = self._request("status")
+            if status.get("test_running"):
+                # a test is running
+                self.log("A test is already controlling the server")
+                self.log(f"Waiting {IDLE_TIME} seconds")
+            else:
+                try:
+                    self._request("start_test")
+                    acquired = True
+                    break
+                except Exception:
+                    # we got beat in the race
+                    self.log("Someone else beat us")
+            time.sleep(IDLE_TIME)
+
+        if not acquired:
+            raise Exception("Could not acquire the test server")
+
+        if data is not None:
+            self._request("shape", data)
+
+    def stop(self):
+        self.log("Stopping")
+        try:
+            self._request("reset")
+        finally:
+            self._request("stop_test")
+
+
+def get_throttler(env):
+    host = _SECRET["throttler_host"]
+    key = _SECRET["throttler_key"].encode()
+    return Throttler(env, host, key)
+
+
+_PROTOCOL = "h2", "h3"
+_PAGE = "gallery", "news", "shopping", "photoblog"
+
+# set the network condition here.
+# each item has a name and some netem options:
+#
+# loss_ratio: specify percentage of packets that will be lost
+# loss_corr: specify a correlation factor for the random packet loss
+# dup_ratio: specify percentage of packets that will be duplicated
+# delay: specify an overall delay for each packet
+# jitter: specify amount of jitter in milliseconds
+# delay_jitter_corr: specify a correlation factor for the random jitter
+# reorder_ratio: specify percentage of packets that will be reordered
+# reorder_corr: specify a correlation factor for the random reordering
+#
+_THROTTLING = (
+    {"name": "full"},  # no throttling.
+    {"name": "one", "delay": "20"},
+    {"name": "two", "delay": "50"},
+    {"name": "three", "delay": "100"},
+    {"name": "four", "delay": "200"},
+    {"name": "five", "delay": "300"},
+)
+
+
+def get_test():
+    """Iterate on test conditions.
+
+    For each cycle, we return a combination of: protocol, page, throttling
+    settings. Each combination has a name, and that name will be used along with
+    the protocol as a prefix for each metrics.
+    """
+    for proto in _PROTOCOL:
+        for page in _PAGE:
+            url = f"https://{CTRL_SERVER}/{page}.html"
+            for throttler_settings in _THROTTLING:
+                yield proto, page, url, throttler_settings
+
+
+combo = get_test()
+
+
+def before_cycle(metadata, env, cycle, script):
+    global combo
+    if "throttlable" not in script["tags"]:
+        return
+    throttler = get_throttler(env)
+    try:
+        proto, page, url, throttler_settings = next(combo)
+    except StopIteration:
+        combo = get_test()
+        proto, page, url, throttler_settings = next(combo)
+
+    # setting the url for the browsertime script
+    add_option(env, "browsertime.url", url, overwrite=True)
+
+    # enabling http if needed
+    if proto == "h3":
+        add_option(env, "firefox.preference", "network.http.http3.enabled:true")
+
+    # prefix used to differenciate metrics
+    name = throttler_settings["name"]
+    script["name"] = f"{name}_{proto}_{page}"
+
+    # throttling the controlled server if needed
+    if throttler_settings != {"name": "full"}:
+        env.info("Calling the controlled server")
+        throttler.start(throttler_settings)
+    else:
+        env.info("No throttling for this call")
+        throttler.start()
+
+
+def after_cycle(metadata, env, cycle, script):
+    if "throttlable" not in script["tags"]:
+        return
+    throttler = get_throttler(env)
+    try:
+        throttler.stop()
+    except Exception:
+        pass
+
+    # give a chance for a competitive job to take over
+    time.sleep(BREATHE_TIME)

netwerk/test/perf/perftest.ini

@@ -1,4 +1,5 @@
 [perftest_http3_cloudflareblog.js]
+[perftest_http3_controlled.js]
 [perftest_http3_facebook_scroll.js]
 [perftest_http3_google_image.js]
 [perftest_http3_google_search.js]

netwerk/test/perf/perftest_http3_controlled.js

@@ -0,0 +1,32 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+/* eslint-env node */
+
+/*
+Ensure the `--firefox.preference=network.http.http3.enabled:true` is
+set for this test.
+*/
+
+async function test(context, commands) {
+  let url = context.options.browsertime.url;
+
+  // Make firefox learn of HTTP/3 server
+  // XXX: Need to build an HTTP/3-specific conditioned profile
+  // to handle these pre-navigations.
+  await commands.navigate(url);
+
+  // Measure initial pageload
+  await commands.measure.start("pageload");
+  await commands.navigate(url);
+  await commands.measure.stop();
+  commands.measure.result[0].browserScripts.pageinfo.url = url;
+}
+
+module.exports = {
+  test,
+  owner: "Network Team",
+  name: "controlled",
+  description: "User-journey live site test for controlled server",
+  tags: ["throttlable"],
+};

python/mozperftest/mozperftest/metadata.py

@@ -16,10 +16,10 @@ class Metadata(MachLogger):
         self._env = env
         self.script = script
 
-    def run_hook(self, name, **kw):
+    def run_hook(self, name, *args, **kw):
         # this bypasses layer restrictions on args,
         # which is fine since it's a user script
-        return self._env.hooks.run(name, **kw)
+        return self._env.hooks.run(name, *args, **kw)
 
     def set_output(self, output):
         self._output = output

python/mozperftest/mozperftest/metrics/common.py

@@ -248,22 +248,13 @@ class MetricsStorage(object):
 
         # Simplify the filtered metric names
         if simplify_names:
-            previous = []
-            for data_type, data_info in filtered.items():
-                for res in data_info:
-                    if any([met in res["subtest"] for met in simplify_exclude]):
-                        continue
 
-                    new = res["subtest"].split(".")[-1]
-                    if new in previous:
-                        self.logger.warning(
-                            f"Another metric which ends with `{new}` was already found. "
-                            f"{res['subtest']} will not be simplified."
-                        )
-                        continue
+            def _simplify(name):
+                if any([met in name for met in simplify_exclude]):
+                    return None
+                return name.split(".")[-1]
 
-                    res["subtest"] = new
-                    previous.append(new)
+            self._alter_name(filtered, res, filter=_simplify)
 
         # Split the filtered results
         if split_by is not None:
@@ -290,6 +281,22 @@ class MetricsStorage(object):
 
         return filtered
 
+    def _alter_name(self, filtered, res, filter):
+        previous = []
+        for data_type, data_info in filtered.items():
+            for res in data_info:
+                new = filter(res["subtest"])
+                if new is None:
+                    continue
+                if new in previous:
+                    self.logger.warning(
+                        f"Another metric which ends with `{new}` was already found. "
+                        f"{res['subtest']} will not be simplified."
+                    )
+                    continue
+                res["subtest"] = new
+                previous.append(new)
+
 
 _metrics = {}
 

python/mozperftest/mozperftest/metrics/perfherder.py

@@ -6,6 +6,7 @@ import jsonschema
 import os
 import pathlib
 import statistics
+import sys
 
 from mozperftest.utils import strtobool
 from mozperftest.layers import Layer
@@ -170,7 +171,14 @@ class Perfherder(Layer):
 
         # XXX "suites" key error occurs when using self.info so a print
         # is being done for now.
-        print("PERFHERDER_DATA: " + json.dumps(all_perfherder_data))
+
+        # print() will produce a BlockingIOError on large outputs, so we use
+        # sys.stdout
+        sys.stdout.write("PERFHERDER_DATA: ")
+        json.dump(all_perfherder_data, sys.stdout)
+        sys.stdout.write("\n")
+        sys.stdout.flush()
+
         metadata.set_output(write_json(all_perfherder_data, output, file))
         return metadata
 

python/mozperftest/mozperftest/system/macos.py

@@ -38,7 +38,7 @@ class MacosDevice(Layer):
             universal_newlines=True,
         )
 
-        stdout, stderr = p.communicate(timeout=15)
+        stdout, stderr = p.communicate(timeout=45)
         if p.returncode != 0:
             raise subprocess.CalledProcessError(
                 stdout=stdout, stderr=stderr, returncode=p.returncode
@@ -49,6 +49,9 @@ class MacosDevice(Layer):
     def extract_app(self, dmg, target):
         mount = Path(tempfile.mkdtemp())
 
+        if not Path(dmg).exists():
+            raise FileNotFoundError(dmg)
+
         # mounting the DMG with hdiutil
         cmd = f"hdiutil attach -nobrowse -mountpoint {str(mount)} {dmg}"
         try:

python/mozperftest/mozperftest/test/browsertime/__init__.py

@@ -5,9 +5,12 @@
 from mozperftest.test.browsertime.runner import BrowsertimeRunner  # noqa
 
 
-def add_option(env, name, value):
-    options = env.get_arg("browsertime-extra-options", "")
-    options += ",%s=%s" % (name, value)
+def add_option(env, name, value, overwrite=False):
+    if not overwrite:
+        options = env.get_arg("browsertime-extra-options", "")
+        options += f",{name}={value}"
+    else:
+        options = f"{name}={value}"
     env.set_arg("browsertime-extra-options", options)
 
 

python/mozperftest/mozperftest/test/browsertime/runner.py

@@ -335,11 +335,15 @@ class BrowsertimeRunner(NodeRunner):
             result_dir = result_dir.resolve()
 
             # Run the test cycle
-            metadata.run_hook("before_cycle", cycle=cycle)
+            metadata.run_hook(
+                "before_cycle", metadata, self.env, cycle, self._test_script
+            )
             try:
                 metadata = self._one_cycle(metadata, result_dir)
             finally:
-                metadata.run_hook("after_cycle", cycle=cycle)
+                metadata.run_hook(
+                    "after_cycle", metadata, self.env, cycle, self._test_script
+                )
         return metadata
 
     def _one_cycle(self, metadata, result_dir):

python/mozperftest/mozperftest/utils.py

@@ -14,6 +14,7 @@ import shutil
 import importlib
 import subprocess
 import shlex
+import functools
 
 from redo import retry
 from requests.packages.urllib3.util.retry import Retry
@@ -412,6 +413,7 @@ _URL = (
 _DEFAULT_SERVER = "https://firefox-ci-tc.services.mozilla.com"
 
 
+@functools.lru_cache()
 def get_tc_secret():
     """Returns the Taskcluster secret.
 

taskcluster/ci/perftest/linux.yml

@@ -182,3 +182,27 @@ livesites:
             --perfherder-simplify-names
             --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
             --output $MOZ_FETCHES_DIR/../artifacts
+
+controlled:
+    description: Controlled performance testing
+    treeherder:
+        symbol: perftest(controlled)
+    attributes:
+        batch: false
+        cron: true
+    run:
+        command: >-
+            mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+            cd $MOZ_FETCHES_DIR &&
+            python3.8 python/mozperftest/mozperftest/runner.py
+            netwerk/test/perf/perftest_http3_controlled.js
+            --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin
+            --browsertime-iterations 1
+            --browsertime-cycles 96
+            --hooks netwerk/test/perf/hooks_throttling.py
+            --flavor desktop-browser
+            --perfherder
+            --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms
+            --perfherder-simplify-names
+            --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+            --output $MOZ_FETCHES_DIR/../artifacts

taskcluster/ci/perftest/macosx.yml

@@ -18,6 +18,8 @@ job-defaults:
     platform: macosx64-shippable/opt
     require-build:
         macosx64-shippable/opt: build-macosx64-shippable/opt
+    scopes:
+        - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles
 
 try-xpcshell:
     description: Run ./mach perftest on macOs
@@ -189,3 +191,29 @@ livesites:
             --perfherder-simplify-names
             --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
             --output $MOZ_FETCHES_DIR/../artifacts
+
+controlled:
+    description: Controlled performance testing
+    treeherder:
+        symbol: perftest(controlled)
+    attributes:
+        batch: false
+        cron: true
+    run:
+        command: >-
+            mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+            cd $MOZ_FETCHES_DIR &&
+            python3 -m venv . &&
+            python3 python/mozperftest/mozperftest/runner.py
+            netwerk/test/perf/perftest_http3_controlled.js
+            --browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg
+            --browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node
+            --browsertime-iterations 1
+            --browsertime-cycles 96
+            --hooks netwerk/test/perf/hooks_throttling.py
+            --flavor desktop-browser
+            --perfherder
+            --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms
+            --perfherder-simplify-names
+            --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+            --output $MOZ_FETCHES_DIR/../artifacts

taskcluster/ci/perftest/windows.yml

@@ -14,6 +14,8 @@ job-defaults:
     platform: win64-shippable/opt
     require-build:
         win64-shippable/opt: build-win64-shippable/opt
+    scopes:
+        - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles
 
 try-browsertime:
     description: Run ./mach perftest on windows
@@ -139,3 +141,28 @@ livesites:
             --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe
             --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe
             --output $MOZ_FETCHES_DIR/../artifacts
+
+controlled:
+    description: Controlled performance testing
+    treeherder:
+        symbol: perftest(controlled)
+    attributes:
+        batch: false
+        cron: true
+    run:
+        command: >-
+            mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+            cd $MOZ_FETCHES_DIR &&
+            python3.exe python/mozperftest/mozperftest/runner.py
+            netwerk/test/perf/perftest_http3_controlled.js
+            --browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe
+            --browsertime-iterations 1
+            --browsertime-cycles 96
+            --hooks netwerk/test/perf/hooks_throttling.py
+            --flavor desktop-browser
+            --perfherder
+            --perfherder-metrics name:navigationTiming,unit:ms name:pageTimings,unit:ms name:resources,unit:ms name:firstPaint,unit:ms name:timeToContentfulPaint,unit:ms
+            --perfherder-simplify-names
+            --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe
+            --browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe
+            --output $MOZ_FETCHES_DIR/../artifacts