Bug 1644706 - Make a multi-commit test for app-link in CI. r=mozperftest-reviewers,tarek

This patch adds a new method to test multiple Fenix builds within one test run.

Depends on D78926

Differential Revision: https://phabricator.services.mozilla.com/D79107

python/mozperftest/mozperftest/system/android.py

@@ -24,6 +24,9 @@ _PERMALINKS = {
     "fenix_fennec_nightly_arm64_v8a": _ROOT_URL
     + _FENIX_FENNEC_BUILDS
     + "arm64-v8a/geckoNightly/target.apk",
+    # The two following aliases are used for Fenix multi-commit testing in CI
+    "fenix_nightlysim_multicommit_arm64_v8a": None,
+    "fenix_nightlysim_multicommit_armeabi_v7a": None,
     "gve_nightly_aarch64": _ROOT_URL
     + _GV_BUILDS
     + "aarch64-opt/artifacts/public/build/geckoview_example.apk",
@@ -158,8 +161,10 @@ class AndroidDevice(Layer):
         if self.clear_logcat:
             self.device.clear_logcat()
 
-        # install APKs
+        # Install APKs
         for apk in self.get_arg("android-install-apk"):
+            self.info("Uninstalling old version")
+            self.device.uninstall_app(self.get_arg("android-app-name"))
             self.info("Installing %s" % apk)
             if apk in _PERMALINKS:
                 apk = _PERMALINKS[apk]
@@ -169,7 +174,7 @@ class AndroidDevice(Layer):
                     self.info("Downloading %s" % apk)
                     download_file(apk, target)
                     self.info("Installing downloaded APK")
-                    self.device.install_app(str(target), replace=True)
+                    self.device.install_app(str(target))
             else:
                 self.device.install_app(apk, replace=True)
             self.info("Done.")

python/mozperftest/mozperftest/test/browsertime/runner.py

@@ -142,7 +142,7 @@ class BrowsertimeRunner(NodeRunner):
 
         # check if the browsertime package has been deployed correctly
         # for this we just check for the browsertime directory presence
-        if self.browsertime_js.exists():
+        if self.browsertime_js.exists() and not self.get_arg("clobber"):
             return
 
         if install_url is None:
@@ -154,8 +154,8 @@ class BrowsertimeRunner(NodeRunner):
         for file in ("package.json", "package-lock.json"):
             src = BROWSERTIME_SRC_ROOT / file
             target = self.state_path / file
-            if not target.exists():
-                shutil.copyfile(str(src), str(target))
+            # Overwrite the existing files
+            shutil.copyfile(str(src), str(target))
 
         package_json_path = self.state_path / "package.json"
 
@@ -330,6 +330,12 @@ class BrowsertimeRunner(NodeRunner):
                     continue
                 option = option.split("=")
                 if len(option) != 2:
+                    self.warning(
+                        f"Skipping browsertime option {option} as it "
+                        "is missing a name/value pairing. We expect options "
+                        "to be formatted as: --browsertime-extra-options "
+                        "'browserRestartTries=1,timeouts.browserStart=10'"
+                    )
                     continue
                 name, value = option
                 args += ["--" + name, value]

python/mozperftest/mozperftest/tests/test_android.py

@@ -13,6 +13,9 @@ class FakeDevice:
     def __init__(self, **args):
         self.apps = []
 
+    def uninstall_app(self, apk_name):
+        return True
+
     def install_app(self, apk, replace=True):
         if apk not in self.apps:
             self.apps.append(apk)
@@ -62,7 +65,7 @@ def test_android_apk_alias(device):
         "flavor": "mobile-browser",
         "android-install-apk": ["fenix_fennec_nightly_armeabi_v7a"],
         "android": True,
-        "android-app-name": "org.mozilla.fenned_aurora",
+        "android-app-name": "org.mozilla.fennec_aurora",
         "android-capture-adb": "stdout",
     }
 
@@ -71,7 +74,8 @@ def test_android_apk_alias(device):
     with system as android, silence(system):
         android(metadata)
     # XXX really ?
-    assert device.mock_calls[1][1][0].endswith("target.apk")
+    assert device.mock_calls[1][1][0] == "org.mozilla.fennec_aurora"
+    assert device.mock_calls[2][1][0].endswith("target.apk")
 
 
 @mock.patch("mozperftest.utils.requests.get", new=requests_content())

python/mozperftest/mozperftest/tests/test_androidlog.py

@@ -19,6 +19,9 @@ class FakeDevice:
     def __init__(self, **args):
         self.apps = []
 
+    def uninstall_app(self, apk_name):
+        return True
+
     def install_app(self, apk, replace=True):
         if apk not in self.apps:
             self.apps.append(apk)

python/mozperftest/mozperftest/tests/test_environment.py

@@ -132,12 +132,15 @@ def test_metrics_last():
     system = create_mock()
     browser = create_mock()
 
-    # check that the metrics layer is entered after
-    # other have finished
+    # Check that the metrics layer is entered after
+    # other have finished and that the other layers
+    # were only called once
     class M:
         def __enter__(self):
-            system.teardown.assert_called()
-            browser.teardown.assert_called()
+            system.setup.assert_called_once()
+            browser.setup.assert_called_once()
+            system.teardown.assert_called_once()
+            browser.teardown.assert_called_once()
             return self
 
         def __exit__(self, *args, **kw):

python/mozperftest/mozperftest/utils.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import logging
 import contextlib
+from datetime import date, timedelta
 import sys
 import os
 import random
@@ -15,6 +16,7 @@ import tempfile
 
 
 RETRY_SLEEP = 10
+MULTI_TASK_ROOT = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/tasks/"
 
 
 @contextlib.contextmanager
@@ -189,3 +191,15 @@ def temporary_env(**env):
                 del os.environ[key]
             else:
                 os.environ[key] = value
+
+
+def get_multi_tasks_url(route, previous=True):
+    """Builds a URL to obtain all the tasks of a given build route for a single day.
+
+    If previous is true, then we get builds from the previous day,
+    otherwise, we look at the current day.
+    """
+    curr = date.today()
+    if previous:
+        curr = curr - timedelta(1)
+    return f"""{MULTI_TASK_ROOT}{route}.{curr.strftime("%Y.%m.%d")}.revision"""

taskcluster/ci/perftest/kind.yml

@@ -78,7 +78,7 @@ jobs:
                 --android-clear-logcat
                 --android-capture-logcat logcat
                 --perfherder-metrics processLaunchToNavStart
-                --android-install-apk fenix_fennec_nightly_armeabi_v7a
+                --android-install-apk fenix_nightlysim_multicommit_armeabi_v7a
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
                 --perfherder-app fenix
@@ -104,7 +104,7 @@ jobs:
                 --android
                 --android-app-name org.mozilla.fennec_aurora
                 --perfherder-metrics processLaunchToNavStart
-                --android-install-apk fenix_fennec_nightly_arm64_v8a
+                --android-install-apk fenix_nightlysim_multicommit_arm64_v8a
                 --android-activity org.mozilla.fenix.IntentReceiverActivity
                 --android-clear-logcat
                 --android-capture-logcat logcat

testing/performance/hooks_applink.py

@@ -1,17 +1,69 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import json
+import tempfile
+import pathlib
+
 from mozperftest.test.browsertime import add_options
+from mozperftest.system.android import _ROOT_URL
+from mozperftest.utils import download_file, get_multi_tasks_url
 
-url = "'https://www.example.com'"
+URL = "'https://www.example.com'"
 
-common_options = [("processStartTime", "true"),
+COMMON_OPTIONS = [("processStartTime", "true"),
                   ("firefox.disableBrowsertimeExtension", "true"),
                   ("firefox.android.intentArgument", "'-a'"),
                   ("firefox.android.intentArgument", "'android.intent.action.VIEW'"),
                   ("firefox.android.intentArgument", "'-d'"),
-                  ("firefox.android.intentArgument", url)]
+                  ("firefox.android.intentArgument", URL)]
+
+NIGHTLY_SIM_ROUTE = "project.mobile.fenix.v2.nightly-simulation"
+G5_SUFFIX = "artifacts/public/build/armeabi-v7a/geckoNightly/target.apk"
+P2_SUFFIX = "artifacts/public/build/arm64-v8a/geckoNightly/target.apk"
+
+build_generator = None
+
+
+def before_iterations(kw):
+    global build_generator
+
+    if "fenix_nightlysim_multicommit" not in kw.get("android_install_apk")[0]:
+        return
+
+    # Get the builds to test
+    build_url = get_multi_tasks_url(NIGHTLY_SIM_ROUTE)
+    tmpfile = pathlib.Path(tempfile.mkdtemp(), "alltasks.json")
+    download_file(build_url, tmpfile)
+
+    # Set the number of test-iterations to the number of builds
+    with tmpfile.open() as f:
+        tasks = json.load(f)["tasks"]
+    kw["test_iterations"] = len(tasks)
+
+    # Finally, make an iterator for the builds (used in `before_runs`)
+    route_suffix = G5_SUFFIX
+    if "arm64_v8a" in kw.get("android_install_apk"):
+        route_suffix = P2_SUFFIX
+
+    def _build_iterator(route_suffix):
+        for task in tasks:
+            route = task["namespace"]
+            revision = route.split(".")[-1]
+            print("Testing revision %s" % revision)
+            download_url = "%s%s/%s" % (_ROOT_URL, route, route_suffix)
+            yield revision, [download_url]
+
+    build_generator = _build_iterator(route_suffix)
+
+    return kw
 
 
 def before_runs(env, **kw):
-    add_options(env, common_options)
+    global build_generator
+
+    add_options(env, COMMON_OPTIONS)
+    if build_generator:
+        revision, build = next(build_generator)
+        env.set_arg("android-install-apk", build)
+        env.set_arg("perfherder-prefix", revision)