Bug 1659072 - Add commit timestamps to Fenix batch-mode tests. r=tarek

Differential Revision: https://phabricator.services.mozilla.com/D89642

python/mozperftest/mozperftest/mach_commands.py

@@ -133,9 +133,6 @@ class Perftest(MachCommandBase):
             push_to_try("perftest", "perftest", try_task_config=task_config)
             return
 
-        # run locally
-        MachCommandBase.activate_virtualenv(self)
-
         from mozperftest.runner import run_tests
 
         run_tests(self, kwargs, original_parser.get_user_args(kwargs))

python/mozperftest/mozperftest/metrics/perfherder.py

@@ -51,6 +51,14 @@ class Perfherder(Layer):
                 "default": False,
                 "help": "If set, browsertime statistics will be reported.",
             },
+            "timestamp": {
+                "type": float,
+                "default": None,
+                "help": (
+                    "Timestamp to use for the perfherder data. Can be the "
+                    "current date or a past date if needed."
+                ),
+            },
         }
     )
 
@@ -146,6 +154,10 @@ class Perfherder(Layer):
             # If a prefix was given, store it in the perfherder data as well
             all_perfherder_data["prefix"] = prefix
 
+        timestamp = self.get_arg("timestamp")
+        if timestamp is not None:
+            all_perfherder_data["pushTimestamp"] = timestamp
+
         # Validate the final perfherder data blob
         with pathlib.Path(metadata._mach_cmd.topsrcdir, PERFHERDER_SCHEMA).open() as f:
             schema = json.load(f)

python/mozperftest/mozperftest/runner.py

@@ -124,7 +124,13 @@ def run_tests(mach_cmd, kwargs, client_args):
         mach_cmd.log_manager.enable_unstructured()
 
     try:
+        # Only pass the virtualenv to the before_iterations hook
+        # so that users can install test-specific packages if needed.
+        mach_cmd.activate_virtualenv()
+        kwargs["virtualenv"] = mach_cmd.virtualenv_manager
         hooks.run("before_iterations", kwargs)
+        del kwargs["virtualenv"]
+
         tests, tmp_dir = build_test_list(kwargs["tests"])
 
         for test in tests:

python/mozperftest/mozperftest/test/noderunner.py

@@ -25,7 +25,6 @@ class NodeRunner(Layer):
 
     def setup(self):
         """Install the Node.js package."""
-        self.mach_cmd.activate_virtualenv()
         self.verify_node_install()
 
     def node(self, args):

python/mozperftest/mozperftest/test/xpcshell.py

@@ -71,7 +71,7 @@ class XPCShell(Layer):
         self.topsrcdir = mach_cmd.topsrcdir
 
     def setup(self):
-        self.mach_cmd.activate_virtualenv()
+        pass
 
     def run(self, metadata):
         test = Path(metadata.script["filename"])

python/mozperftest/mozperftest/tests/test_perfherder.py

@@ -38,6 +38,7 @@ def test_perfherder():
         "perfherder-stats": True,
         "perfherder-prefix": "",
         "perfherder-metrics": [metric_fields("firstPaint")],
+        "perfherder-timestamp": 1.0,
     }
 
     metrics, metadata, env = setup_env(options)
@@ -53,6 +54,7 @@ def test_perfherder():
     # Check some metadata
     assert output["application"]["name"] == "firefox"
     assert output["framework"]["name"] == "browsertime"
+    assert output["pushTimestamp"] == 1.0
 
     # Check some numbers in our data
     assert len(output["suites"]) == 1

python/mozperftest/perfdocs/writing.rst

@@ -139,7 +139,8 @@ Hooks
 A Python module can be used to run functions during a run lifecycle. Available hooks are:
 
 - **before_iterations(args)** runs before everything is started. Gets the args, which
-  can be changed.
+  can be changed. The **args** argument also contains a **virtualenv** variable that
+  can be used for installing Python packages (e.g. through `install_package <https://searchfox.org/mozilla-central/source/python/mozperftest/mozperftest/utils.py#115-144>`_).
 - **before_runs(env)** runs before the test is launched. Can be used to
   change the running environment.
 - **after_runs(env)** runs after the test is done.

testing/perfdocs/generated/writing.rst

@@ -139,7 +139,8 @@ Hooks
 A Python module can be used to run functions during a run lifecycle. Available hooks are:
 
 - **before_iterations(args)** runs before everything is started. Gets the args, which
-  can be changed.
+  can be changed. The **args** argument also contains a **virtualenv** variable that
+  can be used for installing Python packages (e.g. through `install_package <https://searchfox.org/mozilla-central/source/python/mozperftest/mozperftest/utils.py#115-144>`_).
 - **before_runs(env)** runs before the test is launched. Can be used to
   change the running environment.
 - **after_runs(env)** runs after the test is done.

testing/performance/hooks_android_view.py

@@ -3,11 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import json
 import tempfile
+import time
 import pathlib
 
 from mozperftest.test.browsertime import add_options
 from mozperftest.system.android import _ROOT_URL
-from mozperftest.utils import download_file, get_multi_tasks_url, get_revision_namespace_url
+from mozperftest.utils import (
+    download_file,
+    get_multi_tasks_url,
+    get_revision_namespace_url,
+    install_package,
+)
 
 URL = "'https://www.example.com'"
 
@@ -26,11 +32,30 @@ build_generator = None
 
 def before_iterations(kw):
     global build_generator
+
     install_list = kw.get("android_install_apk")
     if (len(install_list) == 0 or
         all(["fenix_nightlysim_multicommit" not in apk for apk in install_list])):
         return
 
+    # Install gitpython
+    install_package(kw["virtualenv"], "gitpython==3.1.0")
+    import git
+
+    class _GitProgress(git.RemoteProgress):
+        def update(self, op_code, cur_count, max_count=None, message=''):
+            if message:
+                print(message)
+
+    # Setup the local fenix github repo
+    print("Cloning fenix repo...")
+    fenix_repo = git.Repo.clone_from(
+        'https://github.com/mozilla-mobile/fenix',
+        tempfile.mkdtemp(),
+        branch='master',
+        progress=_GitProgress()
+    )
+
     # Get the builds to test
     architecture = "arm64-v8a" if "arm64_v8a" in kw.get("android_install_apk") else "armeabi-v7a"
     json_ = _fetch_json(get_revision_namespace_url, NIGHTLY_SIM_ROUTE, day=kw["test_date"])
@@ -39,25 +64,37 @@ def before_iterations(kw):
 
     tasks = []
     for revision in revisions:
+        try:
+            commitdate = fenix_repo.commit(revision).committed_date
+        except ValueError:
+            print("Commit %s is not from the Fenix master branch" % revision)
+            continue
+
         json_ = _fetch_json(get_multi_tasks_url, NIGHTLY_SIM_ROUTE, revision, day=kw["test_date"])
         for task in json_["tasks"]:
             route = task["namespace"]
             task_architecture = route.split(".")[-1]
             if task_architecture == architecture:
                 tasks.append({
+                    "timestamp": commitdate,
                     "revision": revision,
                     "route": route,
                     "route_suffix": ROUTE_SUFFIX.format(architecture=task_architecture),
                 })
 
+    # Set the number of test-iterations to the number of builds
     kw["test_iterations"] = len(tasks)
 
     def _build_iterator():
         for task in tasks:
             revision = task["revision"]
-            print(f"Testing revision {revision}")
+            timestamp = task["timestamp"]
+
+            humandate = time.ctime(int(timestamp))
+            print(f"Testing revision {revision} from {humandate}")
+
             download_url = f'{_ROOT_URL}{task["route"]}/{task["route_suffix"]}'
-            yield revision, [download_url]
+            yield revision, timestamp, [download_url]
 
     build_generator = _build_iterator()
 
@@ -69,7 +106,6 @@ def _fetch_json(get_url_function, *args, **kwargs):
     tmpfile = pathlib.Path(tempfile.mkdtemp(), "temp.json")
     download_file(build_url, tmpfile)
 
-    # Set the number of test-iterations to the number of builds
     with tmpfile.open() as f:
         return json.load(f)
 
@@ -79,6 +115,7 @@ def before_runs(env, **kw):
 
     add_options(env, COMMON_OPTIONS)
     if build_generator:
-        revision, build = next(build_generator)
+        revision, timestamp, build = next(build_generator)
         env.set_arg("android-install-apk", build)
         env.set_arg("perfherder-prefix", revision)
+        env.set_arg("perfherder-timestamp", timestamp)