Bug 1731836: Simplify `./mach python` arguments r=ahal

`--no-virtualenv` was needed for one use case: the
`get_and_diffoscope` task, despite not needing `psutil`, would
run into failures during virtualenv-creation because we used to
unconditionally build the `psutil` package, and `get_and_diffoscope`
didn't have the environment needed for such a build.

Since we no longer build and install `psutil` into every virtualenv,
it's no longer needed for its one usage, which means that it can be
removed.

`--requirements` is replaced by `--virtualenv`, which removes an
ad-hoc pip package installation and embraces the centralized dep
system.

`--no-activate` is now implied by default: a virtualenv is only
created and activated if `--virtualenv` is provided.`

`ipython==7.16.1` was the chosen version because it is the last
one compatible with Python 3.6.

Differential Revision: https://phabricator.services.mozilla.com/D131529

build/ipython_virtualenv_packages.txt

@@ -0,0 +1 @@
+pypi:ipython==7.16.1

build/psutil_virtualenv_packages.txt

@@ -0,0 +1 @@
+pypi:psutil==5.8.0

python/mach_commands.py

@@ -7,8 +7,6 @@ from __future__ import absolute_import, print_function, unicode_literals
 import argparse
 import logging
 import os
-import subprocess
-import sys
 import tempfile
 from multiprocessing import cpu_count
 
@@ -18,24 +16,14 @@ from concurrent.futures import ThreadPoolExecutor, as_completed, thread
 
 import mozinfo
 from mozfile import which
+from mach.decorators import CommandArgument, Command
 from manifestparser import TestManifest
 from manifestparser import filters as mpf
 
-
-from mach.decorators import CommandArgument, Command
-from mach.requirements import MachEnvRequirements
-from mach.util import UserError
-
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 @Command("python", category="devenv", description="Run Python.")
-@CommandArgument(
-    "--no-virtualenv", action="store_true", help="Do not set up a virtualenv"
-)
-@CommandArgument(
-    "--no-activate", action="store_true", help="Do not activate the virtualenv"
-)
 @CommandArgument(
     "--exec-file", default=None, help="Execute this Python file using `exec`"
 )
@@ -46,18 +34,17 @@ here = os.path.abspath(os.path.dirname(__file__))
     help="Use ipython instead of the default Python REPL.",
 )
 @CommandArgument(
-    "--requirements",
+    "--virtualenv",
     default=None,
-    help="Install this requirements file before running Python",
+    help="Prepare and use the virtualenv with the provided name. If not specified, "
+    "then the Mach context is used instead.",
 )
 @CommandArgument("args", nargs=argparse.REMAINDER)
 def python(
     command_context,
-    no_virtualenv,
-    no_activate,
     exec_file,
     ipython,
-    requirements,
+    virtualenv,
     args,
 ):
     # Avoid logging the command
@@ -66,59 +53,34 @@ def python(
     # Note: subprocess requires native strings in os.environ on Windows.
     append_env = {"PYTHONDONTWRITEBYTECODE": str("1")}
 
-    if requirements and no_virtualenv:
-        raise UserError("Cannot pass both --requirements and --no-virtualenv.")
-
-    if no_virtualenv:
-        python_path = sys.executable
-        requirements = MachEnvRequirements.from_requirements_definition(
-            command_context.topsrcdir,
-            False,
-            True,
-            os.path.join(
-                command_context.topsrcdir, "build", "mach_virtualenv_packages.txt"
-            ),
-        )
-
-        append_env["PYTHONPATH"] = os.pathsep.join(
-            requirements.pths_as_absolute(command_context.topsrcdir)
-        )
-    else:
-        if not no_activate:
-            command_context.virtualenv_manager.activate()
-        else:
-            command_context.virtualenv_manager.ensure()
-        python_path = command_context.virtualenv_manager.python_path
-        if requirements:
-            command_context.virtualenv_manager.install_pip_requirements(
-                requirements, require_hashes=False
-            )
+    if virtualenv:
+        command_context._virtualenv_name = virtualenv
 
     if exec_file:
+        command_context.activate_virtualenv()
         exec(open(exec_file).read())
         return 0
 
     if ipython:
-        bindir = os.path.dirname(python_path)
-        python_path = which("ipython", path=bindir)
-        if not python_path:
-            if not no_virtualenv:
-                # Use `pip` directly rather than `install_pip_package()` to bypass
-                # `req.check_if_exists()` which may detect a system installed ipython.
-                subprocess.check_call(
-                    [
-                        command_context.virtualenv_manager.python_path,
-                        "-m",
-                        "pip",
-                        "install",
-                        "ipython",
-                    ]
-                )
-                python_path = which("ipython", path=bindir)
-
+        if virtualenv:
+            command_context.virtualenv_manager.ensure()
+            python_path = which(
+                "ipython", path=command_context.virtualenv_manager.bin_path
+            )
             if not python_path:
-                print("error: could not detect or install ipython")
-                return 1
+                raise Exception(
+                    "--ipython was specified, but the provided "
+                    '--virtualenv doesn\'t have "ipython" installed.'
+                )
+        else:
+            command_context._virtualenv_name = "ipython"
+            command_context.virtualenv_manager.ensure()
+            python_path = which(
+                "ipython", path=command_context.virtualenv_manager.bin_path
+            )
+    else:
+        command_context.virtualenv_manager.ensure()
+        python_path = command_context.virtualenv_manager.python_path
 
     return command_context.run_process(
         [python_path] + args,

taskcluster/docker/diffoscope/get_and_diffoscope

@@ -72,7 +72,7 @@ for option; do
 		for dir in a b; do
 			# Need to run mach python from inside the gecko source.
 			# See bug #1533642.
-			(cd $GECKO_PATH && ./mach python --no-virtualenv toolkit/mozapps/installer/unpack.py --omnijar $OMNIJAR $CURDIR/$dir)
+			(cd $GECKO_PATH && ./mach python toolkit/mozapps/installer/unpack.py --omnijar $OMNIJAR $CURDIR/$dir)
 		done
 		;;
 	--fail)

taskcluster/gecko_taskgraph/transforms/job/mozharness.py

@@ -300,7 +300,6 @@ def mozharness_on_generic_worker(config, job, taskdesc):
     mh_command += [
         f"{gecko_path}/mach",
         "python",
-        "--no-activate",
         "{}/testing/{}".format(gecko_path, run.pop("script")),
     ]
 

taskcluster/scripts/builder/build-l10n.sh

@@ -97,7 +97,7 @@ fi
 cd /builds/worker
 
 $GECKO_PATH/mach python \
-  --requirements $GECKO_PATH/build/psutil_requirements.txt \
+  --virtualenv psutil \
   -- \
   $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \
   ${config_path_cmds} \

taskcluster/scripts/builder/build-linux.sh

@@ -112,7 +112,7 @@ fi
 cd /builds/worker
 
 $GECKO_PATH/mach python \
-  --requirements $GECKO_PATH/build/psutil_requirements.txt \
+  --virtualenv psutil \
   -- \
   $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \
   ${config_path_cmds} \