diff --git a/build/ipython_virtualenv_packages.txt b/build/ipython_virtualenv_packages.txt new file mode 100644 index 000000000000..a569a360b3d9 --- /dev/null +++ b/build/ipython_virtualenv_packages.txt @@ -0,0 +1 @@ +pypi:ipython==7.16.1 diff --git a/build/psutil_virtualenv_packages.txt b/build/psutil_virtualenv_packages.txt new file mode 100644 index 000000000000..1fdad6983d8f --- /dev/null +++ b/build/psutil_virtualenv_packages.txt @@ -0,0 +1 @@ +pypi:psutil==5.8.0 diff --git a/python/mach_commands.py b/python/mach_commands.py index af4852881ca7..5bc0a7fbb065 100644 --- a/python/mach_commands.py +++ b/python/mach_commands.py @@ -7,8 +7,6 @@ from __future__ import absolute_import, print_function, unicode_literals import argparse import logging import os -import subprocess -import sys import tempfile from multiprocessing import cpu_count @@ -18,24 +16,14 @@ from concurrent.futures import ThreadPoolExecutor, as_completed, thread import mozinfo from mozfile import which +from mach.decorators import CommandArgument, Command from manifestparser import TestManifest from manifestparser import filters as mpf - -from mach.decorators import CommandArgument, Command -from mach.requirements import MachEnvRequirements -from mach.util import UserError - here = os.path.abspath(os.path.dirname(__file__)) @Command("python", category="devenv", description="Run Python.") -@CommandArgument( - "--no-virtualenv", action="store_true", help="Do not set up a virtualenv" -) -@CommandArgument( - "--no-activate", action="store_true", help="Do not activate the virtualenv" -) @CommandArgument( "--exec-file", default=None, help="Execute this Python file using `exec`" ) @@ -46,18 +34,17 @@ here = os.path.abspath(os.path.dirname(__file__)) help="Use ipython instead of the default Python REPL.", ) @CommandArgument( - "--requirements", + "--virtualenv", default=None, - help="Install this requirements file before running Python", + help="Prepare and use the virtualenv with the provided name. If not specified, " + "then the Mach context is used instead.", ) @CommandArgument("args", nargs=argparse.REMAINDER) def python( command_context, - no_virtualenv, - no_activate, exec_file, ipython, - requirements, + virtualenv, args, ): # Avoid logging the command @@ -66,59 +53,34 @@ def python( # Note: subprocess requires native strings in os.environ on Windows. append_env = {"PYTHONDONTWRITEBYTECODE": str("1")} - if requirements and no_virtualenv: - raise UserError("Cannot pass both --requirements and --no-virtualenv.") - - if no_virtualenv: - python_path = sys.executable - requirements = MachEnvRequirements.from_requirements_definition( - command_context.topsrcdir, - False, - True, - os.path.join( - command_context.topsrcdir, "build", "mach_virtualenv_packages.txt" - ), - ) - - append_env["PYTHONPATH"] = os.pathsep.join( - requirements.pths_as_absolute(command_context.topsrcdir) - ) - else: - if not no_activate: - command_context.virtualenv_manager.activate() - else: - command_context.virtualenv_manager.ensure() - python_path = command_context.virtualenv_manager.python_path - if requirements: - command_context.virtualenv_manager.install_pip_requirements( - requirements, require_hashes=False - ) + if virtualenv: + command_context._virtualenv_name = virtualenv if exec_file: + command_context.activate_virtualenv() exec(open(exec_file).read()) return 0 if ipython: - bindir = os.path.dirname(python_path) - python_path = which("ipython", path=bindir) - if not python_path: - if not no_virtualenv: - # Use `pip` directly rather than `install_pip_package()` to bypass - # `req.check_if_exists()` which may detect a system installed ipython. - subprocess.check_call( - [ - command_context.virtualenv_manager.python_path, - "-m", - "pip", - "install", - "ipython", - ] - ) - python_path = which("ipython", path=bindir) - + if virtualenv: + command_context.virtualenv_manager.ensure() + python_path = which( + "ipython", path=command_context.virtualenv_manager.bin_path + ) if not python_path: - print("error: could not detect or install ipython") - return 1 + raise Exception( + "--ipython was specified, but the provided " + '--virtualenv doesn\'t have "ipython" installed.' + ) + else: + command_context._virtualenv_name = "ipython" + command_context.virtualenv_manager.ensure() + python_path = which( + "ipython", path=command_context.virtualenv_manager.bin_path + ) + else: + command_context.virtualenv_manager.ensure() + python_path = command_context.virtualenv_manager.python_path return command_context.run_process( [python_path] + args, diff --git a/taskcluster/docker/diffoscope/get_and_diffoscope b/taskcluster/docker/diffoscope/get_and_diffoscope index 1283e7a2558e..9c51af250885 100644 --- a/taskcluster/docker/diffoscope/get_and_diffoscope +++ b/taskcluster/docker/diffoscope/get_and_diffoscope @@ -72,7 +72,7 @@ for option; do for dir in a b; do # Need to run mach python from inside the gecko source. # See bug #1533642. - (cd $GECKO_PATH && ./mach python --no-virtualenv toolkit/mozapps/installer/unpack.py --omnijar $OMNIJAR $CURDIR/$dir) + (cd $GECKO_PATH && ./mach python toolkit/mozapps/installer/unpack.py --omnijar $OMNIJAR $CURDIR/$dir) done ;; --fail) diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py index a6e4d0d716db..aa8a5d632971 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py @@ -300,7 +300,6 @@ def mozharness_on_generic_worker(config, job, taskdesc): mh_command += [ f"{gecko_path}/mach", "python", - "--no-activate", "{}/testing/{}".format(gecko_path, run.pop("script")), ] diff --git a/taskcluster/scripts/builder/build-l10n.sh b/taskcluster/scripts/builder/build-l10n.sh index 4c4bee2f66bd..caff42bd54e2 100755 --- a/taskcluster/scripts/builder/build-l10n.sh +++ b/taskcluster/scripts/builder/build-l10n.sh @@ -97,7 +97,7 @@ fi cd /builds/worker $GECKO_PATH/mach python \ - --requirements $GECKO_PATH/build/psutil_requirements.txt \ + --virtualenv psutil \ -- \ $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ ${config_path_cmds} \ diff --git a/taskcluster/scripts/builder/build-linux.sh b/taskcluster/scripts/builder/build-linux.sh index db501f1cace3..65c78779a56e 100755 --- a/taskcluster/scripts/builder/build-linux.sh +++ b/taskcluster/scripts/builder/build-linux.sh @@ -112,7 +112,7 @@ fi cd /builds/worker $GECKO_PATH/mach python \ - --requirements $GECKO_PATH/build/psutil_requirements.txt \ + --virtualenv psutil \ -- \ $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ ${config_path_cmds} \