Backed out 3 changesets (bug 1644706, bug 1644437, bug 1644368) for python related bustage CLOSED TREE

Backed out changeset 2a6645398c5a (bug 1644706)
Backed out changeset e695f5748e04 (bug 1644437)
Backed out changeset d615591507b9 (bug 1644368)

python/mozperftest/docs/writing.rst

@@ -95,8 +95,6 @@ Hooks
 
 A Python module can be used to run functions during a run lifecycle. Available hooks are:
 
-- **before_iterations(args)** runs before everything is started. Gets the args, which
-  can be changed.
 - **before_runs(env)** runs before the test is launched. Can be used to
   change the running environment.
 - **after_runs(env)** runs after the test is done.

python/mozperftest/mozperftest/argparser.py

@@ -38,11 +38,6 @@ class Options:
             "help": "Test to run. Can be a single test file or URL or a directory"
             " of tests (to run recursively). If omitted, the entire suite is run.",
         },
-        "--test-iterations": {
-            "type": int,
-            "default": 1,
-            "help": "Number of times the whole test is executed",
-        },
         "--output": {
             "type": str,
             "default": "artifacts",

python/mozperftest/mozperftest/environment.py

@@ -3,20 +3,23 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import copy
 import contextlib
+import importlib
+from pathlib import Path
+import tempfile
+import shutil
 
 from mozperftest.test import pick_test
 from mozperftest.system import pick_system
 from mozperftest.metrics import pick_metrics
 from mozperftest.layers import Layers, StopRunError
-from mozperftest.utils import MachLogger
-from mozperftest.hooks import Hooks
+from mozperftest.utils import download_file, MachLogger
 
 
 SYSTEM, TEST, METRICS = 0, 1, 2
 
 
 class MachEnvironment(MachLogger):
-    def __init__(self, mach_cmd, flavor="desktop-browser", hooks=None, **kwargs):
+    def __init__(self, mach_cmd, flavor="desktop-browser", **kwargs):
         MachLogger.__init__(self, mach_cmd)
         self._mach_cmd = mach_cmd
         self._mach_args = dict(
@@ -27,10 +30,8 @@ class MachEnvironment(MachLogger):
             raise NotImplementedError(flavor)
         for layer in (pick_system, pick_test, pick_metrics):
             self.add_layer(layer(self, flavor, mach_cmd))
-        if hooks is None:
-            # we just load an empty Hooks instance
-            hooks = Hooks(mach_cmd)
-        self.hooks = hooks
+        self.tmp_dir = tempfile.mkdtemp()
+        self._load_hooks()
 
     @contextlib.contextmanager
     def frozen(self):
@@ -104,3 +105,47 @@ class MachEnvironment(MachLogger):
 
     def __exit__(self, type, value, traceback):
         return
+
+    def cleanup(self):
+        if self.tmp_dir is None:
+            return
+        shutil.rmtree(self.tmp_dir)
+        self.tmp_dir = None
+
+    def _load_hooks(self):
+        self._hooks = None
+        hooks = self.get_arg("hooks")
+        if hooks is None:
+            return
+
+        if hooks.startswith("http"):
+            target = Path(self.tmp_dir, hooks.split("/")[-1])
+            hooks = download_file(hooks, target)
+        else:
+            hooks = Path(hooks)
+
+        if hooks.exists():
+            spec = importlib.util.spec_from_file_location("hooks", str(hooks))
+            hooks = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(hooks)
+            self._hooks = hooks
+        else:
+            raise IOError(str(hooks))
+
+    def has_hook(self, name):
+        if self._hooks is None:
+            return False
+        return hasattr(self._hooks, name)
+
+    def get_hook(self, name):
+        if self._hooks is None:
+            return False
+        return getattr(self._hooks, name)
+
+    def run_hook(self, name, *args, **kw):
+        if self._hooks is None:
+            return
+        if not hasattr(self._hooks, name):
+            return
+        self.debug("Running hook %s" % name)
+        return getattr(self._hooks, name)(self, *args, **kw)

python/mozperftest/mozperftest/hooks.py

@@ -1,58 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import importlib
-import tempfile
-from pathlib import Path
-import shutil
-
-from mozperftest.utils import download_file, MachLogger
-
-
-class Hooks(MachLogger):
-    def __init__(self, mach_cmd, hook_module=None):
-        MachLogger.__init__(self, mach_cmd)
-        self.tmp_dir = tempfile.mkdtemp()
-
-        if hook_module is None:
-            self._hooks = None
-            return
-
-        if not isinstance(hook_module, Path):
-            if hook_module.startswith("http"):
-                target = Path(self.tmp_dir, hook_module.split("/")[-1])
-                hook_module = download_file(hook_module, target)
-            else:
-                hook_module = Path(hook_module)
-
-        if hook_module.exists():
-            spec = importlib.util.spec_from_file_location("hooks", str(hook_module))
-            hook_module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(hook_module)
-            self._hooks = hook_module
-        else:
-            raise IOError(str(hook_module))
-
-    def cleanup(self):
-        if self.tmp_dir is None:
-            return
-        shutil.rmtree(self.tmp_dir)
-        self.tmp_dir = None
-
-    def exists(self, name):
-        if self._hooks is None:
-            return False
-        return hasattr(self._hooks, name)
-
-    def get(self, name):
-        if self._hooks is None:
-            return False
-        return getattr(self._hooks, name)
-
-    def run(self, name, *args, **kw):
-        if self._hooks is None:
-            return
-        if not hasattr(self._hooks, name):
-            return
-        self.debug("Running hook %s" % name)
-        return getattr(self._hooks, name)(*args, **kw)

python/mozperftest/mozperftest/layers.py

@@ -62,7 +62,7 @@ class Layer(MachLogger):
         self.teardown()
 
     def __call__(self, metadata):
-        has_exc_handler = self.env.hooks.exists("on_exception")
+        has_exc_handler = self.env.has_hook("on_exception")
         self.debug("Running %s:run" % self.name)
         try:
             metadata = self.run(metadata)
@@ -71,7 +71,7 @@ class Layer(MachLogger):
                 self.error("User handled error")
                 for line in traceback.format_exc().splitlines():
                     self.error(line)
-                resume_run = self.env.hooks.run("on_exception", self.env, self, e)
+                resume_run = self.env.run_hook("on_exception", self, e)
                 if resume_run:
                     return metadata
                 raise StopRunError()

python/mozperftest/mozperftest/metadata.py

@@ -17,8 +17,8 @@ class Metadata(MachLogger):
 
     def run_hook(self, name, **kw):
         # this bypasses layer restrictions on args,
-        # which is fine since it's a user script
-        return self._env.hooks.run(name, **kw)
+        # which is fine since it's user script
+        return self._env.run_hook(name, **kw)
 
     def set_output(self, output):
         self._output = output

python/mozperftest/mozperftest/runner.py

@@ -94,53 +94,44 @@ def run_tests(mach_cmd, **kwargs):
         try_options = json.loads(os.environ["PERFTEST_OPTIONS"])
         kwargs.update(try_options)
 
-    from mozperftest.utils import build_test_list
+    from mozperftest.utils import build_test_list, install_package
     from mozperftest import MachEnvironment, Metadata
-    from mozperftest.hooks import Hooks
 
-    hooks = Hooks(mach_cmd, kwargs.pop("hooks", None))
+    flavor = kwargs["flavor"]
+    kwargs["tests"], tmp_dir = build_test_list(
+        kwargs["tests"], randomized=flavor != "doc"
+    )
     verbose = kwargs.get("verbose", False)
     log_level = logging.DEBUG if verbose else logging.INFO
-
-    # If we run through mach, we just  want to set the level
-    # of the existing termminal handler.
-    # Otherwise, we're adding it.
-    if mach_cmd.log_manager.terminal_handler is not None:
-        mach_cmd.log_manager.terminal_handler.level = log_level
-    else:
-        mach_cmd.log_manager.add_terminal_logging(level=log_level)
+    mach_cmd.log_manager.add_terminal_logging(level=log_level)
 
     try:
-        hooks.run("before_iterations", kwargs)
-
-        for iteration in range(kwargs.get("test_iterations", 1)):
-            flavor = kwargs["flavor"]
-            kwargs["tests"], tmp_dir = build_test_list(
-                kwargs["tests"], randomized=flavor != "doc"
+        if flavor == "doc":
+            location = os.path.join(
+                mach_cmd.topsrcdir, "third_party", "python", "esprima"
             )
+            install_package(mach_cmd.virtualenv_manager, location)
+
+            from mozperftest.scriptinfo import ScriptInfo
+
+            for test in kwargs["tests"]:
+                print(ScriptInfo(test))
+            return
+
+        env = MachEnvironment(mach_cmd, **kwargs)
+        try:
+            metadata = Metadata(mach_cmd, env, flavor)
+            env.run_hook("before_runs")
             try:
-                # XXX this doc is specific to browsertime scripts
-                # maybe we want to move it
-                if flavor == "doc":
-                    from mozperftest.test.browsertime.script import ScriptInfo
-
-                    for test in kwargs["tests"]:
-                        print(ScriptInfo(test))
-                    return
-
-                env = MachEnvironment(mach_cmd, hooks=hooks, **kwargs)
-                metadata = Metadata(mach_cmd, env, flavor)
-                hooks.run("before_runs", env)
-                try:
-                    with env.frozen() as e:
-                        e.run(metadata)
-                finally:
-                    hooks.run("after_runs", env)
+                with env.frozen() as e:
+                    e.run(metadata)
             finally:
-                if tmp_dir is not None:
-                    shutil.rmtree(tmp_dir)
+                env.run_hook("after_runs")
+        finally:
+            env.cleanup()
     finally:
-        hooks.cleanup()
+        if tmp_dir is not None:
+            shutil.rmtree(tmp_dir)
 
 
 def main(argv=sys.argv[1:]):

python/mozperftest/mozperftest/scriptinfo.py

@@ -3,27 +3,32 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from collections import defaultdict
 import re
+import os
 import textwrap
-from pathlib import Path
 
+# This import will fail if esprima is not installed.
+# The package is vendored in python/third_party and also available at PyPI
+# The mach perftest command will make sure it's installed when --flavor doc
+# is being used.
 import esprima
 
 
-# list of metadata, each item is the name and if the field is mandatory
-METADATA = [
-    ("setUp", False),
-    ("tearDown", False),
-    ("test", True),
-    ("owner", True),
-    ("author", False),
-    ("name", True),
-    ("description", True),
-    ("longDescription", False),
-    ("usage", False),
-    ("supportedBrowsers", False),
-    ("supportedPlatforms", False),
-    ("filename", True),
-]
+METADATA = set(
+    [
+        "setUp",
+        "tearDown",
+        "test",
+        "owner",
+        "author",
+        "name",
+        "description",
+        "longDescription",
+        "usage",
+        "supportedBrowsers",
+        "supportedPlatforms",
+        "filename",
+    ]
+)
 
 
 _INFO = """\
@@ -42,19 +47,18 @@ Description:
 """
 
 
-class MissingFieldError(Exception):
-    pass
+class MetadataDict(defaultdict):
+    def __missing__(self, key):
+        return "N/A"
 
 
-class ScriptInfo(defaultdict):
-    """Loads and parses a Browsertime test script."""
-
-    def __init__(self, path):
+class ScriptInfo(MetadataDict):
+    def __init__(self, script):
         super(ScriptInfo, self).__init__()
-        self.script = Path(path)
-        self["filename"] = str(self.script)
-
-        with self.script.open() as f:
+        filename = os.path.basename(script)
+        self["filename"] = script, filename
+        self.script = script
+        with open(script) as f:
             self.parsed = esprima.parseScript(f.read())
 
         # looking for the exports statement
@@ -62,7 +66,6 @@ class ScriptInfo(defaultdict):
             if (
                 stmt.type != "ExpressionStatement"
                 or stmt.expression.left is None
-                or stmt.expression.left.property is None
                 or stmt.expression.left.property.name != "exports"
                 or stmt.expression.right is None
                 or stmt.expression.right.properties is None
@@ -83,33 +86,21 @@ class ScriptInfo(defaultdict):
                     value = [e.value for e in prop.value.elements]
                 else:
                     raise ValueError(prop.value.type)
+                #  line wrapping
+                if isinstance(value, str):
+                    repr = "\n".join(textwrap.wrap(value, break_on_hyphens=False))
+                elif isinstance(value, list):
+                    repr = ", ".join(value)
 
-                self[prop.key.name] = value
+                self[prop.key.name] = value, repr
 
-        # If the fields found, don't match our known ones, then an error is raised
-        for field, required in METADATA:
-            if not required:
-                continue
-            if field not in self:
-                raise MissingFieldError(field)
+        # If the fields found, don't match our known ones,
+        # then an error is raised
+        assert set(list(self.keys())) - METADATA == set()
 
     def __str__(self):
-        """Used to generate docs."""
-        d = {}
-        for field, value in self.items():
-            if field == "filename":
-                d[field] = self.script.name
-                continue
-
-            # line wrapping
-            if isinstance(value, str):
-                value = "\n".join(textwrap.wrap(value, break_on_hyphens=False))
-            elif isinstance(value, list):
-                value = ", ".join(value)
-            d[field] = value
-
-        d["filename_underline"] = "-" * len(d["filename"])
+        reprs = dict((k, v[1]) for k, v in self.items())
+        d = MetadataDict()
+        d.update(reprs)
+        d.update({"filename_underline": "-" * len(self["filename"])})
         return _INFO % d
-
-    def __missing__(self, key):
-        return "N/A"

python/mozperftest/mozperftest/system/android.py

@@ -24,9 +24,6 @@ _PERMALINKS = {
     "fenix_fennec_nightly_arm64_v8a": _ROOT_URL
     + _FENIX_FENNEC_BUILDS
     + "arm64-v8a/geckoNightly/target.apk",
-    # The two following aliases are used for Fenix multi-commit testing in CI
-    "fenix_nightlysim_multicommit_arm64_v8a": None,
-    "fenix_nightlysim_multicommit_armeabi_v7a": None,
     "gve_nightly_aarch64": _ROOT_URL
     + _GV_BUILDS
     + "aarch64-opt/artifacts/public/build/geckoview_example.apk",
@@ -161,10 +158,8 @@ class AndroidDevice(Layer):
         if self.clear_logcat:
             self.device.clear_logcat()
 
-        # Install APKs
+        # install APKs
         for apk in self.get_arg("android-install-apk"):
-            self.info("Uninstalling old version")
-            self.device.uninstall_app(self.get_arg("android-app-name"))
             self.info("Installing %s" % apk)
             if apk in _PERMALINKS:
                 apk = _PERMALINKS[apk]
@@ -174,7 +169,7 @@ class AndroidDevice(Layer):
                     self.info("Downloading %s" % apk)
                     download_file(apk, target)
                     self.info("Installing downloaded APK")
-                    self.device.install_app(str(target))
+                    self.device.install_app(str(target), replace=True)
             else:
                 self.device.install_app(apk, replace=True)
             self.info("Done.")

python/mozperftest/mozperftest/test/androidlog.py

@@ -46,7 +46,7 @@ class AndroidLog(Layer):
         options = {
             "first-timestamp": self.get_arg("first-timestamp", first_ts),
             "second-timestamp": self.get_arg("second-timestamp", second_ts),
-            "processor": self.env.hooks.get("logcat_processor"),
+            "processor": self.env.get_hook("logcat_processor"),
             "transform-subtest-name": self.get_arg("subtest-name"),
         }
 

python/mozperftest/mozperftest/test/browsertime/runner.py

@@ -10,13 +10,13 @@ import re
 import shutil
 from pathlib import Path
 
+from mozperftest.scriptinfo import ScriptInfo
 from mozperftest.utils import install_package
 from mozperftest.test.noderunner import NodeRunner
 from mozperftest.test.browsertime.setup import (
     system_prerequisites,
     append_system_env,
 )
-from mozperftest.test.browsertime.script import ScriptInfo
 
 
 BROWSERTIME_SRC_ROOT = Path(__file__).parent
@@ -89,7 +89,7 @@ class BrowsertimeRunner(NodeRunner):
         self._mach_context = mach_cmd._mach_context
         self.virtualenv_manager = mach_cmd.virtualenv_manager
         self._created_dirs = []
-        self._test_script = None
+        self._test_info = {}
         self._setup_helper = None
         self.get_binary_path = mach_cmd.get_binary_path
 
@@ -128,13 +128,8 @@ class BrowsertimeRunner(NodeRunner):
         super(BrowsertimeRunner, self).setup()
         install_url = self.get_arg("install-url")
 
-        tests = self.get_arg("tests", [])
-        if len(tests) != 1:
-            # we don't support auto-discovery (no test passed) or multiple
-            # tests here yet.
-            raise NotImplementedError()
-
-        self._test_script = ScriptInfo(tests[0])
+        if self.get_arg("tests"):
+            self._test_info = ScriptInfo(self.get_arg("tests")[0])
 
         # installing Python deps on the fly
         for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION):
@@ -142,7 +137,7 @@ class BrowsertimeRunner(NodeRunner):
 
         # check if the browsertime package has been deployed correctly
         # for this we just check for the browsertime directory presence
-        if self.browsertime_js.exists() and not self.get_arg("clobber"):
+        if self.browsertime_js.exists():
             return
 
         if install_url is None:
@@ -154,8 +149,8 @@ class BrowsertimeRunner(NodeRunner):
         for file in ("package.json", "package-lock.json"):
             src = BROWSERTIME_SRC_ROOT / file
             target = self.state_path / file
-            # Overwrite the existing files
-            shutil.copyfile(str(src), str(target))
+            if not target.exists():
+                shutil.copyfile(str(src), str(target))
 
         package_json_path = self.state_path / "package.json"
 
@@ -308,6 +303,7 @@ class BrowsertimeRunner(NodeRunner):
 
     def _one_cycle(self, metadata, result_dir):
         profile = self.get_arg("profile-directory")
+        test_script = self.get_arg("tests")[0]
 
         args = [
             "--resultDir",
@@ -316,7 +312,7 @@ class BrowsertimeRunner(NodeRunner):
             profile,
             "--iterations",
             str(self.get_arg("iterations")),
-            self._test_script["filename"],
+            test_script,
         ]
 
         if self.get_arg("verbose"):
@@ -330,12 +326,6 @@ class BrowsertimeRunner(NodeRunner):
                     continue
                 option = option.split("=")
                 if len(option) != 2:
-                    self.warning(
-                        f"Skipping browsertime option {option} as it "
-                        "is missing a name/value pairing. We expect options "
-                        "to be formatted as: --browsertime-extra-options "
-                        "'browserRestartTries=1,timeouts.browserStart=10'"
-                    )
                     continue
                 name, value = option
                 args += ["--" + name, value]
@@ -351,7 +341,10 @@ class BrowsertimeRunner(NodeRunner):
             raise NodeException(exit_code)
 
         metadata.add_result(
-            {"results": str(result_dir), "name": self._test_script["name"]}
+            {
+                "results": str(result_dir),
+                "name": self._test_info.get("name", ("browsertime",))[0],
+            }
         )
 
         return metadata

python/mozperftest/mozperftest/tests/data/hooks_iteration.py

@@ -1,2 +0,0 @@
-def before_iterations(kwargs):
-    kwargs["test_iterations"] = 5

python/mozperftest/mozperftest/tests/data/samples/perftest_example.js

@@ -6,8 +6,6 @@
 
 var someVar;
 
-someVar = 2;
-
 async function setUp(context) {
   context.log.info("setUp example!");
 }

python/mozperftest/mozperftest/tests/support.py

@@ -7,7 +7,6 @@ from pathlib import Path
 
 from mozperftest.metadata import Metadata
 from mozperftest.environment import MachEnvironment
-from mozperftest.hooks import Hooks
 
 
 HERE = Path(__file__).parent
@@ -58,8 +57,7 @@ def get_running_env(**kwargs):
         "browsertime-install-url": None,
     }
     mach_args.update(kwargs)
-    hooks = Hooks(mach_cmd, mach_args.pop("hooks", None))
-    env = MachEnvironment(mach_cmd, hooks=hooks, **mach_args)
+    env = MachEnvironment(mach_cmd, **mach_args)
     metadata = Metadata(mach_cmd, env, "desktop-browser")
     return mach_cmd, metadata, env
 

python/mozperftest/mozperftest/tests/test_android.py

@@ -13,9 +13,6 @@ class FakeDevice:
     def __init__(self, **args):
         self.apps = []
 
-    def uninstall_app(self, apk_name):
-        return True
-
     def install_app(self, apk, replace=True):
         if apk not in self.apps:
             self.apps.append(apk)
@@ -65,7 +62,7 @@ def test_android_apk_alias(device):
         "flavor": "mobile-browser",
         "android-install-apk": ["fenix_fennec_nightly_armeabi_v7a"],
         "android": True,
-        "android-app-name": "org.mozilla.fennec_aurora",
+        "android-app-name": "org.mozilla.fenned_aurora",
         "android-capture-adb": "stdout",
     }
 
@@ -74,8 +71,7 @@ def test_android_apk_alias(device):
     with system as android, silence(system):
         android(metadata)
     # XXX really ?
-    assert device.mock_calls[1][1][0] == "org.mozilla.fennec_aurora"
-    assert device.mock_calls[2][1][0].endswith("target.apk")
+    assert device.mock_calls[1][1][0].endswith("target.apk")
 
 
 @mock.patch("mozperftest.utils.requests.get", new=requests_content())

python/mozperftest/mozperftest/tests/test_androidlog.py

@@ -19,9 +19,6 @@ class FakeDevice:
     def __init__(self, **args):
         self.apps = []
 
-    def uninstall_app(self, apk_name):
-        return True
-
     def install_app(self, apk, replace=True):
         if apk not in self.apps:
             self.apps.append(apk)

python/mozperftest/mozperftest/tests/test_environment.py

@@ -10,24 +10,21 @@ import pytest
 from mozperftest.environment import MachEnvironment
 from mozperftest.tests.support import get_running_env, requests_content
 from mozperftest.layers import Layer
-from mozperftest.hooks import Hooks
 
 
 HERE = Path(__file__).parent.resolve()
 
 
-def _get_env(hooks_path):
-    return MachEnvironment(mock.MagicMock(), hooks=Hooks(mock.MagicMock(), hooks_path))
-
-
 def test_run_hooks():
-    env = _get_env(Path(HERE, "data", "hook.py"))
-    assert env.hooks.run("doit", env) == "OK"
+    hooks = str(Path(HERE, "data", "hook.py"))
+    env = MachEnvironment(mock.MagicMock(), hooks=hooks)
+    assert env.run_hook("doit") == "OK"
 
 
 def test_bad_hooks():
+    hooks = "Idontexists"
     with pytest.raises(IOError):
-        _get_env("Idontexists")
+        MachEnvironment(mock.MagicMock(), hooks=hooks)
 
 
 doit = [b"def doit(*args, **kw):\n", b"    return 'OK'\n"]
@@ -35,8 +32,9 @@ doit = [b"def doit(*args, **kw):\n", b"    return 'OK'\n"]
 
 @mock.patch("mozperftest.utils.requests.get", requests_content(doit))
 def test_run_hooks_url():
-    env = _get_env("http://somewhere/hooks.py")
-    assert env.hooks.run("doit", env) == "OK"
+    hooks = "http://somewhere/hooks.py"
+    env = MachEnvironment(mock.MagicMock(), hooks=hooks)
+    assert env.run_hook("doit") == "OK"
 
 
 def test_layers():
@@ -132,15 +130,12 @@ def test_metrics_last():
     system = create_mock()
     browser = create_mock()
 
-    # Check that the metrics layer is entered after
-    # other have finished and that the other layers
-    # were only called once
+    # check that the metrics layer is entered after
+    # other have finished
     class M:
         def __enter__(self):
-            system.setup.assert_called_once()
-            browser.setup.assert_called_once()
-            system.teardown.assert_called_once()
-            browser.teardown.assert_called_once()
+            system.teardown.assert_called()
+            browser.teardown.assert_called()
             return self
 
         def __exit__(self, *args, **kw):

python/mozperftest/mozperftest/tests/test_mach_commands.py

@@ -8,7 +8,6 @@ from unittest import mock
 import tempfile
 import shutil
 from contextlib import contextmanager
-from pathlib import Path
 
 from mach.registrar import Registrar
 
@@ -23,16 +22,8 @@ from mozperftest.tests.support import EXAMPLE_TESTS_DIR  # noqa
 from mozperftest.utils import temporary_env, silence  # noqa
 
 
-ITERATION_HOOKS = Path(__file__).parent / "data" / "hooks_iteration.py"
-
-
 class _TestMachEnvironment(MachEnvironment):
-    def __init__(self, mach_cmd, flavor="desktop-browser", hooks=None, **kwargs):
-        MachEnvironment.__init__(self, mach_cmd, flavor, hooks, **kwargs)
-        self.runs = 0
-
     def run(self, metadata):
-        self.runs += 1
         return metadata
 
     def __enter__(self):
@@ -69,21 +60,6 @@ def test_command(mocked_func):
         # XXX add assertions
 
 
-@mock.patch("mozperftest.MachEnvironment")
-@mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv")
-def test_command_iterations(venv, env):
-    kwargs = {
-        "tests": [EXAMPLE_TESTS_DIR],
-        "hooks": ITERATION_HOOKS,
-        "flavor": "desktop-browser",
-    }
-    with _get_command() as test, silence(test):
-        test.run_perftest(**kwargs)
-    # the hook changes the iteration value to 5.
-    # each iteration generates 5 calls, so we want to see 25
-    assert len(env.mock_calls) == 25
-
-
 @mock.patch("mozperftest.MachEnvironment", new=_TestMachEnvironment)
 @mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv")
 @mock.patch("tryselect.push.push_to_try")

python/mozperftest/mozperftest/tests/test_scriptinfo.py

@@ -5,7 +5,7 @@
 import mozunit
 import pytest
 
-from mozperftest.test.browsertime.script import ScriptInfo, MissingFieldError
+from mozperftest.scriptinfo import ScriptInfo, METADATA
 from mozperftest.tests.support import EXAMPLE_TEST, HERE
 
 
@@ -16,10 +16,13 @@ def test_scriptinfo():
     display = str(info)
     assert "The description of the example test." in display
 
+    # Ensure that all known fields are in the script info
+    assert set(list(info.keys())) - METADATA == set()
+
 
 def test_scriptinfo_failure():
     bad_example = HERE / "data" / "failing-samples" / "perftest_doc_failure_example.js"
-    with pytest.raises(MissingFieldError):
+    with pytest.raises(AssertionError):
         ScriptInfo(bad_example)
 
 

python/mozperftest/mozperftest/utils.py

@@ -3,7 +3,6 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import logging
 import contextlib
-from datetime import date, timedelta
 import sys
 import os
 import random
@@ -16,7 +15,6 @@ import tempfile
 
 
 RETRY_SLEEP = 10
-MULTI_TASK_ROOT = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/tasks/"
 
 
 @contextlib.contextmanager
@@ -191,15 +189,3 @@ def temporary_env(**env):
                 del os.environ[key]
             else:
                 os.environ[key] = value
-
-
-def get_multi_tasks_url(route, previous=True):
-    """Builds a URL to obtain all the tasks of a given build route for a single day.
-
-    If previous is true, then we get builds from the previous day,
-    otherwise, we look at the current day.
-    """
-    curr = date.today()
-    if previous:
-        curr = curr - timedelta(1)
-    return f"""{MULTI_TASK_ROOT}{route}.{curr.strftime("%Y.%m.%d")}.revision"""

taskcluster/ci/perftest/kind.yml

@@ -78,7 +78,7 @@ jobs:
                 --android-clear-logcat
                 --android-capture-logcat logcat
                 --perfherder-metrics processLaunchToNavStart
-                --android-install-apk fenix_nightlysim_multicommit_armeabi_v7a
+                --android-install-apk fenix_fennec_nightly_armeabi_v7a
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
                 --perfherder-app fenix
@@ -104,7 +104,7 @@ jobs:
                 --android
                 --android-app-name org.mozilla.fennec_aurora
                 --perfherder-metrics processLaunchToNavStart
-                --android-install-apk fenix_nightlysim_multicommit_arm64_v8a
+                --android-install-apk fenix_fennec_nightly_arm64_v8a
                 --android-activity org.mozilla.fenix.IntentReceiverActivity
                 --android-clear-logcat
                 --android-capture-logcat logcat

testing/performance/hooks_applink.py

@@ -1,69 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import json
-import tempfile
-import pathlib
-
 from mozperftest.test.browsertime import add_options
-from mozperftest.system.android import _ROOT_URL
-from mozperftest.utils import download_file, get_multi_tasks_url
 
-URL = "'https://www.example.com'"
+url = "'https://www.example.com'"
 
-COMMON_OPTIONS = [("processStartTime", "true"),
+common_options = [("processStartTime", "true"),
                   ("firefox.disableBrowsertimeExtension", "true"),
                   ("firefox.android.intentArgument", "'-a'"),
                   ("firefox.android.intentArgument", "'android.intent.action.VIEW'"),
                   ("firefox.android.intentArgument", "'-d'"),
-                  ("firefox.android.intentArgument", URL)]
-
-NIGHTLY_SIM_ROUTE = "project.mobile.fenix.v2.nightly-simulation"
-G5_SUFFIX = "artifacts/public/build/armeabi-v7a/geckoNightly/target.apk"
-P2_SUFFIX = "artifacts/public/build/arm64-v8a/geckoNightly/target.apk"
-
-build_generator = None
-
-
-def before_iterations(kw):
-    global build_generator
-
-    if "fenix_nightlysim_multicommit" not in kw.get("android_install_apk")[0]:
-        return
-
-    # Get the builds to test
-    build_url = get_multi_tasks_url(NIGHTLY_SIM_ROUTE)
-    tmpfile = pathlib.Path(tempfile.mkdtemp(), "alltasks.json")
-    download_file(build_url, tmpfile)
-
-    # Set the number of test-iterations to the number of builds
-    with tmpfile.open() as f:
-        tasks = json.load(f)["tasks"]
-    kw["test_iterations"] = len(tasks)
-
-    # Finally, make an iterator for the builds (used in `before_runs`)
-    route_suffix = G5_SUFFIX
-    if "arm64_v8a" in kw.get("android_install_apk"):
-        route_suffix = P2_SUFFIX
-
-    def _build_iterator(route_suffix):
-        for task in tasks:
-            route = task["namespace"]
-            revision = route.split(".")[-1]
-            print("Testing revision %s" % revision)
-            download_url = "%s%s/%s" % (_ROOT_URL, route, route_suffix)
-            yield revision, [download_url]
-
-    build_generator = _build_iterator(route_suffix)
-
-    return kw
+                  ("firefox.android.intentArgument", url)]
 
 
 def before_runs(env, **kw):
-    global build_generator
-
-    add_options(env, COMMON_OPTIONS)
-    if build_generator:
-        revision, build = next(build_generator)
-        env.set_arg("android-install-apk", build)
-        env.set_arg("perfherder-prefix", revision)
+    add_options(env, common_options)