diff --git a/python/mozperftest/mozperftest/argparser.py b/python/mozperftest/mozperftest/argparser.py index 188a169b75a4..1b73dce6f04d 100644 --- a/python/mozperftest/mozperftest/argparser.py +++ b/python/mozperftest/mozperftest/argparser.py @@ -28,7 +28,7 @@ class Options: "--flavor": { "choices": FLAVORS, "metavar": "{{{}}}".format(", ".join(FLAVORS)), - "default": None, + "default": "script", "help": "Only run tests of this flavor.", }, "tests": { diff --git a/python/mozperftest/mozperftest/browser/browsertime/runner.py b/python/mozperftest/mozperftest/browser/browsertime/runner.py index a6d19c7a8d60..dbe59b14bae1 100644 --- a/python/mozperftest/mozperftest/browser/browsertime/runner.py +++ b/python/mozperftest/mozperftest/browser/browsertime/runner.py @@ -12,6 +12,7 @@ import shutil from mozbuild.util import mkdir import mozpack.path as mozpath +from mozperftest.utils import install_package from mozperftest.browser.noderunner import NodeRunner from mozperftest.browser.browsertime.setup import ( system_prerequisites, @@ -106,8 +107,7 @@ class BrowsertimeRunner(NodeRunner): # installing Python deps on the fly for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION): - if self._need_install(dep): - self.virtualenv_manager._run_pip(["install", dep]) + install_package(self.virtualenv_manager, dep) # check if the browsertime package has been deployed correctly # for this we just check for the browsertime directory presence diff --git a/python/mozperftest/mozperftest/mach_commands.py b/python/mozperftest/mozperftest/mach_commands.py index 46d8cea9844b..e0b76b4cbf1b 100644 --- a/python/mozperftest/mozperftest/mach_commands.py +++ b/python/mozperftest/mozperftest/mach_commands.py @@ -1,8 +1,6 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import os -import random from functools import partial from mach.decorators import CommandProvider, Command from mozbuild.base import MachCommandBase, MachCommandConditions as conditions @@ -16,25 +14,6 @@ def get_perftest_parser(): @CommandProvider class Perftest(MachCommandBase): - def _build_test_list(self, tests, randomized=False): - res = [] - for test in tests: - if os.path.isfile(test): - tests.append(test) - elif os.path.isdir(test): - for root, dirs, files in os.walk(test): - for file in files: - if not file.startswith("perftest"): - continue - res.append(os.path.join(root, file)) - if not randomized: - res.sort() - else: - # random shuffling is used to make sure - # we don't always run tests in the same order - random.shuffle(res) - return res - @Command( "perftest", category="testing", @@ -42,36 +21,9 @@ class Perftest(MachCommandBase): description="Run any flavor of perftest", parser=get_perftest_parser, ) - def run_perftest( - self, flavor="script", test_objects=None, resolve_tests=True, **kwargs - ): - + def run_perftest(self, **kwargs): MachCommandBase._activate_virtualenv(self) - kwargs["tests"] = self._build_test_list( - kwargs["tests"], randomized=flavor != "doc" - ) - if flavor == "doc": - from mozperftest.utils import install_package + from mozperftest.runner import run_tests - location = os.path.join(self.topsrcdir, "third_party", "python", "esprima") - install_package(self.virtualenv_manager, location) - - from mozperftest.scriptinfo import ScriptInfo - - for test in kwargs["tests"]: - print(ScriptInfo(test)) - return - - from mozperftest import MachEnvironment, Metadata - - kwargs["test_objects"] = test_objects - kwargs["resolve_tests"] = resolve_tests - env = MachEnvironment(self, flavor, **kwargs) - metadata = Metadata(self, env, flavor) - env.run_hook("before_runs") - try: - with env.frozen() as e: - e.run(metadata) - finally: - env.run_hook("after_runs") + run_tests(mach_cmd=self, **kwargs) diff --git a/python/mozperftest/mozperftest/runner.py b/python/mozperftest/mozperftest/runner.py new file mode 100644 index 000000000000..82cbc6a6fd49 --- /dev/null +++ b/python/mozperftest/mozperftest/runner.py @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Pure Python runner so we can execute perftest in the CI without +depending on the mach toolchain, that is not fully available in +all worker environments. +""" +import sys +import os + + +HERE = os.path.dirname(__file__) +SRC_ROOT = os.path.join(HERE, "..", "..", "..") +SEARCH_PATHS = [ + "python/mach", + "python/mozboot", + "python/mozbuild", + "python/mozperftest", + "python/mozterm", + "python/mozversioncontrol", + "testing/mozbase/mozdevice", + "testing/mozbase/mozfile", + "testing/mozbase/mozinfo", + "testing/mozbase/mozlog", + "testing/mozbase/mozprocess", + "testing/mozbase/mozprofile", + "testing/mozbase/mozproxy", + "third_party/python/dlmanager", + "third_party/python/esprima", + "third_party/python/pyyaml/lib3", + "third_party/python/redo", + "third_party/python/requests", + "third_party/python/six", +] + + +# XXX need to make that for all systems flavors +if "SHELL" not in os.environ: + os.environ["SHELL"] = "/bin/bash" + + +def main(): + for path in SEARCH_PATHS: + path = os.path.abspath(path) + if not os.path.exists(path): + raise IOError("Can't find %s" % path) + sys.path.insert(0, os.path.join(SRC_ROOT, path)) + + from mozbuild.base import MachCommandBase, MozbuildObject + from mozperftest import PerftestArgumentParser + from mozboot.util import get_state_dir + + config = MozbuildObject.from_environment() + config.topdir = config.topsrcdir + config.cwd = os.getcwd() + config.state_dir = get_state_dir() + mach_cmd = MachCommandBase(config) + parser = PerftestArgumentParser(description="vanilla perftest") + args = parser.parse_args() + run_tests(mach_cmd, **dict(args._get_kwargs())) + + +def run_tests(mach_cmd, **kwargs): + from mozperftest.utils import build_test_list, install_package + from mozperftest import MachEnvironment, Metadata + + flavor = kwargs["flavor"] + kwargs["tests"] = build_test_list(kwargs["tests"], randomized=flavor != "doc") + + if flavor == "doc": + install_package(mach_cmd.virtualenv_manager, "esprima") + + from mozperftest.scriptinfo import ScriptInfo + + for test in kwargs["tests"]: + print(ScriptInfo(test)) + return + + env = MachEnvironment(mach_cmd, **kwargs) + metadata = Metadata(mach_cmd, env, flavor) + env.run_hook("before_runs") + try: + with env.frozen() as e: + e.run(metadata) + finally: + env.run_hook("after_runs") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/mozperftest/mozperftest/tests/support.py b/python/mozperftest/mozperftest/tests/support.py index 01b2f7ae1779..87db73299ac3 100644 --- a/python/mozperftest/mozperftest/tests/support.py +++ b/python/mozperftest/mozperftest/tests/support.py @@ -47,12 +47,13 @@ def get_running_env(**kwargs): mach_cmd._mach_context.state_dir = tempfile.mkdtemp() mach_args = { + "flavor": "script", "test_objects": None, "resolve_tests": True, "browsertime-clobber": False, "browsertime-install-url": None, } mach_args.update(kwargs) - env = MachEnvironment(mach_cmd, "script", **mach_args) + env = MachEnvironment(mach_cmd, **mach_args) metadata = Metadata(mach_cmd, env, "script") return mach_cmd, metadata, env diff --git a/python/mozperftest/mozperftest/tests/test_browsertime.py b/python/mozperftest/mozperftest/tests/test_browsertime.py index 7d0752209cc9..6c95d849b590 100644 --- a/python/mozperftest/mozperftest/tests/test_browsertime.py +++ b/python/mozperftest/mozperftest/tests/test_browsertime.py @@ -17,6 +17,7 @@ def fetch(self, url): return os.path.join(HERE, "fetched_artifact.zip") +@mock.patch("mozperftest.browser.browsertime.runner.install_package") @mock.patch( "mozperftest.browser.noderunner.NodeRunner.verify_node_install", new=lambda x: True ) @@ -25,7 +26,7 @@ def fetch(self, url): "mozperftest.browser.browsertime.runner.BrowsertimeRunner._setup_node_packages", new=lambda x, y: None, ) -def test_browser(): +def test_browser(*mocked): mach_cmd, metadata, env = get_running_env() browser = env.layers[BROWSER] env.set_arg("tests", [EXAMPLE_TEST]) diff --git a/python/mozperftest/mozperftest/tests/test_mach_commands.py b/python/mozperftest/mozperftest/tests/test_mach_commands.py index 05e68fc1d91b..d27061028286 100644 --- a/python/mozperftest/mozperftest/tests/test_mach_commands.py +++ b/python/mozperftest/mozperftest/tests/test_mach_commands.py @@ -54,7 +54,7 @@ def _get_perftest(): @mock.patch("mozperftest.mach_commands.MachCommandBase._activate_virtualenv") def test_command(mocked_func): with _get_perftest() as test: - test.run_perftest(tests=[EXAMPLE_TESTS_DIR]) + test.run_perftest(tests=[EXAMPLE_TESTS_DIR], flavor="script") # XXX add assertions diff --git a/python/mozperftest/mozperftest/utils.py b/python/mozperftest/mozperftest/utils.py index 8991803a2ac9..aae3eeab4a21 100644 --- a/python/mozperftest/mozperftest/utils.py +++ b/python/mozperftest/mozperftest/utils.py @@ -5,6 +5,7 @@ import logging import contextlib import sys import os +import random from six import StringIO @@ -73,3 +74,25 @@ def install_package(virtualenv_manager, package): # already installed in this venv, we can skip return virtualenv_manager._run_pip(["install", package]) + + +def build_test_list(tests, randomized=False): + if isinstance(tests, str): + tests = [tests] + res = [] + for test in tests: + if os.path.isfile(test): + res.append(test) + elif os.path.isdir(test): + for root, dirs, files in os.walk(test): + for file in files: + if not file.startswith("perftest"): + continue + res.append(os.path.join(root, file)) + if not randomized: + res.sort() + else: + # random shuffling is used to make sure + # we don't always run tests in the same order + random.shuffle(res) + return res