Bug 1639212 - allow URLs for hooks and scripts r=sparky

This patch allows using URLS for --hooks and scripts

Differential Revision: https://phabricator.services.mozilla.com/D75967

python/mozperftest/mozperftest/argparser.py

@@ -35,8 +35,8 @@ class Options:
             "nargs": "*",
             "metavar": "TEST",
             "default": [],
-            "help": "Test to run. Can be a single test file or a directory of tests "
-            "(to run recursively). If omitted, the entire suite is run.",
+            "help": "Test to run. Can be a single test file or URL or a directory"
+            " of tests (to run recursively). If omitted, the entire suite is run.",
         },
         "--output": {
             "type": str,
@@ -44,7 +44,11 @@ class Options:
             "help": "Path to where data will be stored, defaults to a top-level "
             "`artifacts` folder.",
         },
-        "--hooks": {"type": str, "default": "", "help": "Python hooks"},
+        "--hooks": {
+            "type": str,
+            "default": "",
+            "help": "Script containing hooks. Can be a path or a URL.",
+        },
         "--verbose": {"action": "store_true", "default": False, "help": "Verbose mode"},
         "--push-to-try": {
             "action": "store_true",

python/mozperftest/mozperftest/environment.py

@@ -4,12 +4,15 @@
 import copy
 import contextlib
 import importlib
-import os
+from pathlib import Path
+import tempfile
+import shutil
 
 from mozperftest.browser import pick_browser
 from mozperftest.system import pick_system
 from mozperftest.metrics import pick_metrics
 from mozperftest.layers import Layers
+from mozperftest.utils import download_file
 
 
 SYSTEM, BROWSER, METRICS = 0, 1, 2
@@ -27,6 +30,7 @@ class MachEnvironment:
             raise NotImplementedError(flavor)
         for layer in (pick_system, pick_browser, pick_metrics):
             self.add_layer(layer(self, flavor, mach_cmd))
+        self.tmp_dir = tempfile.mkdtemp()
         self._load_hooks()
 
     @contextlib.contextmanager
@@ -94,14 +98,31 @@ class MachEnvironment:
         for layer in self.layers:
             layer.__exit__(type, value, traceback)
 
+    def cleanup(self):
+        if self.tmp_dir is None:
+            return
+        shutil.rmtree(self.tmp_dir)
+        self.tmp_dir = None
+
     def _load_hooks(self):
         self._hooks = None
         hooks = self.get_arg("hooks")
-        if hooks is not None and os.path.exists(hooks):
-            spec = importlib.util.spec_from_file_location("hooks", hooks)
+        if hooks is None:
+            return
+
+        if hooks.startswith("http"):
+            target = Path(self.tmp_dir, hooks.split("/")[-1])
+            hooks = download_file(hooks, target)
+        else:
+            hooks = Path(hooks)
+
+        if hooks.exists():
+            spec = importlib.util.spec_from_file_location("hooks", str(hooks))
             hooks = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(hooks)
             self._hooks = hooks
+        else:
+            raise IOError(str(hooks))
 
     def run_hook(self, name, **kw):
         if self._hooks is None:

python/mozperftest/mozperftest/runner.py

@@ -25,6 +25,7 @@ The --push-to-try flow is:
 """
 import sys
 import os
+import shutil
 
 
 HERE = os.path.dirname(__file__)
@@ -113,26 +114,37 @@ def run_tests(mach_cmd, **kwargs):
     from mozperftest import MachEnvironment, Metadata
 
     flavor = kwargs["flavor"]
-    kwargs["tests"] = build_test_list(kwargs["tests"], randomized=flavor != "doc")
+    kwargs["tests"], tmp_dir = build_test_list(
+        kwargs["tests"], randomized=flavor != "doc"
+    )
 
-    if flavor == "doc":
-        location = os.path.join(mach_cmd.topsrcdir, "third_party", "python", "esprima")
-        install_package(mach_cmd.virtualenv_manager, location)
-
-        from mozperftest.scriptinfo import ScriptInfo
-
-        for test in kwargs["tests"]:
-            print(ScriptInfo(test))
-        return
-
-    env = MachEnvironment(mach_cmd, **kwargs)
-    metadata = Metadata(mach_cmd, env, flavor)
-    env.run_hook("before_runs")
     try:
-        with env.frozen() as e:
-            e.run(metadata)
+        if flavor == "doc":
+            location = os.path.join(
+                mach_cmd.topsrcdir, "third_party", "python", "esprima"
+            )
+            install_package(mach_cmd.virtualenv_manager, location)
+
+            from mozperftest.scriptinfo import ScriptInfo
+
+            for test in kwargs["tests"]:
+                print(ScriptInfo(test))
+            return
+
+        env = MachEnvironment(mach_cmd, **kwargs)
+        try:
+            metadata = Metadata(mach_cmd, env, flavor)
+            env.run_hook("before_runs")
+            try:
+                with env.frozen() as e:
+                    e.run(metadata)
+            finally:
+                env.run_hook("after_runs")
+        finally:
+            env.cleanup()
     finally:
-        env.run_hook("after_runs")
+        if tmp_dir is not None:
+            shutil.rmtree(tmp_dir)
 
 
 def main(argv=sys.argv[1:]):

python/mozperftest/mozperftest/tests/test_environment.py

@@ -5,9 +5,10 @@
 from pathlib import Path
 import mozunit
 import mock
+import pytest
 
 from mozperftest.environment import MachEnvironment
-from mozperftest.tests.support import get_running_env
+from mozperftest.tests.support import get_running_env, requests_content
 
 
 HERE = Path(__file__).parent.resolve()
@@ -19,6 +20,22 @@ def test_run_hooks():
     assert env.run_hook("doit") == "OK"
 
 
+def test_bad_hooks():
+    hooks = "Idontexists"
+    with pytest.raises(IOError):
+        MachEnvironment(mock.MagicMock(), hooks=hooks)
+
+
+doit = [b"def doit(*args, **kw):\n", b"    return 'OK'\n"]
+
+
+@mock.patch("mozperftest.utils.requests.get", requests_content(doit))
+def test_run_hooks_url():
+    hooks = "http://somewhere/hooks.py"
+    env = MachEnvironment(mock.MagicMock(), hooks=hooks)
+    assert env.run_hook("doit") == "OK"
+
+
 def test_layers():
     env = MachEnvironment(mock.MagicMock())
     assert env.get_layer("browsertime").name == "browsertime"

python/mozperftest/mozperftest/tests/test_utils.py

@@ -7,9 +7,16 @@ import mozunit
 import mock
 import pytest
 from pathlib import Path
+import shutil
 
-from mozperftest.utils import host_platform, silence, download_file, install_package
-from mozperftest.tests.support import temp_file, requests_content
+from mozperftest.utils import (
+    host_platform,
+    silence,
+    download_file,
+    install_package,
+    build_test_list,
+)
+from mozperftest.tests.support import temp_file, requests_content, EXAMPLE_TESTS_DIR
 
 
 def test_silence():
@@ -70,5 +77,15 @@ def test_install_package():
     vem._run_pip.assert_called()
 
 
+@mock.patch("mozperftest.utils.requests.get", requests_content())
+def test_build_test_list():
+    tests = [EXAMPLE_TESTS_DIR, "https://some/location/perftest_one.js"]
+    try:
+        files, tmp_dir = build_test_list(tests)
+        assert len(files) == 2
+    finally:
+        shutil.rmtree(tmp_dir)
+
+
 if __name__ == "__main__":
     mozunit.main()

python/mozperftest/mozperftest/utils.py

@@ -10,6 +10,8 @@ from io import StringIO
 from redo import retry
 import requests
 from collections import defaultdict
+from pathlib import Path
+import tempfile
 
 
 RETRY_SLEEP = 10
@@ -108,25 +110,40 @@ def install_package(virtualenv_manager, package):
 
 
 def build_test_list(tests, randomized=False):
+    """Collects tests given a list of directories, files and URLs.
+
+    Returns a tuple containing the list of tests found and a temp dir for tests
+    that were downloaded from an URL.
+    """
+    temp_dir = None
+
     if isinstance(tests, str):
         tests = [tests]
     res = []
     for test in tests:
-        if os.path.isfile(test):
+        if test.startswith("http"):
+            if temp_dir is None:
+                temp_dir = tempfile.mkdtemp()
+            target = Path(temp_dir, test.split("/")[-1])
+            download_file(test, target)
+            res.append(str(target))
+            continue
+
+        test = Path(test)
+
+        if test.is_file():
             res.append(test)
-        elif os.path.isdir(test):
-            for root, dirs, files in os.walk(test):
-                for file in files:
-                    if not file.startswith("perftest"):
-                        continue
-                    res.append(os.path.join(root, file))
+        elif test.is_dir():
+            for file in test.rglob("perftest_*.js"):
+                res.append(str(file))
     if not randomized:
         res.sort()
     else:
         # random shuffling is used to make sure
         # we don't always run tests in the same order
         random.shuffle(res)
-    return res
+
+    return res, temp_dir
 
 
 def download_file(url, target, retry_sleep=RETRY_SLEEP, attempts=3):