Bug 1766497: Mach: use `venv` instead of `virtualenv` r=ahal

Brew's Python 3.10 causes `virtualenv==20.7.2` to produce a wonky folder
structure (`$venv/opt/homebrew/lib/python3.10/site-packages`?).

This is likely fixed with newer `virtualenv`, but the simpler workaround
here is to use `venv` instead now that Python 3 is always used.

Adds `python3-venv` to docker image so that tests and debian-based tasks
can leverage it.

Differential Revision: https://phabricator.services.mozilla.com/D144872

python/mach/mach/site.py

@@ -35,6 +35,18 @@ METADATA_FILENAME = "moz_virtualenv_metadata.json"
 # python packages via the system environment.
 PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS = ("mach", "build", "common")
 
+_is_windows = sys.platform == "cygwin" or (sys.platform == "win32" and os.sep == "\\")
+
+
+class VenvModuleNotFoundException(Exception):
+    def __init__(self):
+        msg = (
+            'Mach was unable to find the "venv" module, which is needed '
+            "to create virtual environments in Python. You may need to "
+            "install it manually using the package manager for your system."
+        )
+        super(Exception, self).__init__(msg)
+
 
 class VirtualenvOutOfDateException(Exception):
     pass
@@ -220,6 +232,7 @@ class MozSiteMetadata:
 
         yield
         MozSiteMetadata.current = self
+
         sys.executable = executable
 
         if pkg_resources:
@@ -321,14 +334,11 @@ class MachSiteManager:
         if self._site_packages_source == SitePackagesSource.NONE:
             return SiteUpToDateResult(True)
         elif self._site_packages_source == SitePackagesSource.SYSTEM:
-            _assert_pip_check(
-                self._topsrcdir, self._sys_path(), "mach", self._requirements
-            )
+            _assert_pip_check(self._sys_path(), "mach", self._requirements)
             return SiteUpToDateResult(True)
         elif self._site_packages_source == SitePackagesSource.VENV:
             environment = self._virtualenv()
             return _is_venv_up_to_date(
-                self._topsrcdir,
                 environment,
                 self._pthfile_lines(environment),
                 self._requirements,
@@ -384,7 +394,6 @@ class MachSiteManager:
 
         environment = self._virtualenv()
         _create_venv_with_pthfile(
-            self._topsrcdir,
             environment,
             self._pthfile_lines(environment),
             True,
@@ -563,7 +572,6 @@ class CommandSiteManager:
                 )
 
             _create_venv_with_pthfile(
-                self._topsrcdir,
                 self._virtualenv,
                 self._pthfile_lines(),
                 self._populate_virtualenv,
@@ -732,14 +740,12 @@ class CommandSiteManager:
         pthfile_lines = self._pthfile_lines()
         if self._mach_site_packages_source == SitePackagesSource.SYSTEM:
             _assert_pip_check(
-                self._topsrcdir,
                 pthfile_lines,
                 self._site_name,
                 self._requirements if not self._populate_virtualenv else None,
             )
 
         return _is_venv_up_to_date(
-            self._topsrcdir,
             self._virtualenv,
             pthfile_lines,
             self._requirements,
@@ -751,11 +757,7 @@ class PythonVirtualenv:
     """Calculates paths of interest for general python virtual environments"""
 
     def __init__(self, prefix):
-        is_windows = sys.platform == "cygwin" or (
-            sys.platform == "win32" and os.sep == "\\"
-        )
-
-        if is_windows:
+        if _is_windows:
             self.bin_path = os.path.join(prefix, "Scripts")
             self.python_path = os.path.join(self.bin_path, "python.exe")
         else:
@@ -1023,12 +1025,6 @@ def resolve_requirements(topsrcdir, site_name):
         )
 
 
-def _virtualenv_py_path(topsrcdir):
-    return os.path.join(
-        topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py"
-    )
-
-
 def _resolve_installed_packages(python_executable):
     pip_json = subprocess.check_output(
         [
@@ -1047,7 +1043,34 @@ def _resolve_installed_packages(python_executable):
     return {package["name"]: package["version"] for package in installed_packages}
 
 
-def _assert_pip_check(topsrcdir, pthfile_lines, virtualenv_name, requirements):
+def _ensure_python_exe(python_exe_root: Path):
+    """On some machines in CI venv does not behave consistently. Sometimes
+    only a "python3" executable is created, but we expect "python". Since
+    they are functionally identical, we can just copy "python3" to "python"
+    (and vice-versa) to solve the problem.
+    """
+    python3_exe_path = python_exe_root / "python3"
+    python_exe_path = python_exe_root / "python"
+
+    if _is_windows:
+        python3_exe_path = python3_exe_path.with_suffix(".exe")
+        python_exe_path = python_exe_path.with_suffix(".exe")
+
+    if python3_exe_path.exists() and not python_exe_path.exists():
+        shutil.copy(str(python3_exe_path), str(python_exe_path))
+
+    if python_exe_path.exists() and not python3_exe_path.exists():
+        shutil.copy(str(python_exe_path), str(python3_exe_path))
+
+    if not python_exe_path.exists() and not python3_exe_path.exists():
+        raise Exception(
+            f'Neither a "{python_exe_path.name}" or "{python3_exe_path.name}" '
+            f"were found. This means something unexpected happened during the "
+            f"virtual environment creation and we cannot proceed."
+        )
+
+
+def _assert_pip_check(pthfile_lines, virtualenv_name, requirements):
     """Check if the provided pthfile lines have a package incompatibility
 
     If there's an incompatibility, raise an exception and allow it to bubble up since
@@ -1077,17 +1100,30 @@ def _assert_pip_check(topsrcdir, pthfile_lines, virtualenv_name, requirements):
         # we create a new virtualenv that has our pinned pip version, so that
         # we get consistent results (there's been lots of pip resolver behaviour
         # changes recently).
-
-        subprocess.check_call(
-            [
-                sys.executable,
-                _virtualenv_py_path(topsrcdir),
-                "--no-download",
-                check_env_path,
-            ],
-            stdout=subprocess.DEVNULL,
+        process = subprocess.run(
+            [sys.executable, "-m", "venv", "--without-pip", check_env_path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            encoding="UTF-8",
         )
+
+        if process.returncode != 0:
+            if "No module named venv" in process.stderr:
+                raise VenvModuleNotFoundException()
+            else:
+                raise subprocess.CalledProcessError(
+                    process.returncode,
+                    process.args,
+                    output=process.stdout,
+                    stderr=process.stderr,
+                )
+
+        if process.stdout:
+            print(process.stdout)
+
         check_env = PythonVirtualenv(check_env_path)
+        _ensure_python_exe(Path(check_env.python_path).parent)
+
         with open(
             os.path.join(
                 os.path.join(check_env.resolve_sysconfig_packages_path("platlib")),
@@ -1161,7 +1197,6 @@ def _deprioritize_venv_packages(virtualenv, populate_virtualenv):
 
 
 def _create_venv_with_pthfile(
-    topsrcdir,
     target_venv,
     pthfile_lines,
     populate_with_pip,
@@ -1175,17 +1210,29 @@ def _create_venv_with_pthfile(
     os.makedirs(virtualenv_root)
     metadata.write(is_finalized=False)
 
-    subprocess.check_call(
-        [
-            metadata.original_python.python_path,
-            _virtualenv_py_path(topsrcdir),
-            # pip, setuptools and wheel are vendored and inserted into the virtualenv
-            # scope automatically, so "virtualenv" doesn't need to seed it.
-            "--no-seed",
-            virtualenv_root,
-        ]
+    process = subprocess.run(
+        [sys.executable, "-m", "venv", "--without-pip", virtualenv_root],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding="UTF-8",
     )
 
+    if process.returncode != 0:
+        if "No module named venv" in process.stderr:
+            raise VenvModuleNotFoundException()
+        else:
+            raise subprocess.CalledProcessError(
+                process.returncode,
+                process.args,
+                output=process.stdout,
+                stderr=process.stderr,
+            )
+
+    if process.stdout:
+        print(process.stdout)
+
+    _ensure_python_exe(Path(target_venv.python_path).parent)
+
     platlib_site_packages_dir = target_venv.resolve_sysconfig_packages_path("platlib")
     pthfile_contents = "\n".join(pthfile_lines)
     with open(os.path.join(platlib_site_packages_dir, PTH_FILENAME), "w") as f:
@@ -1200,7 +1247,6 @@ def _create_venv_with_pthfile(
 
 
 def _is_venv_up_to_date(
-    topsrcdir,
     target_venv,
     expected_pthfile_lines,
     requirements,
@@ -1209,23 +1255,12 @@ def _is_venv_up_to_date(
     if not os.path.exists(target_venv.prefix):
         return SiteUpToDateResult(False, f'"{target_venv.prefix}" does not exist')
 
-    # Modifications to any of the following files mean the virtualenv should be
-    # rebuilt:
-    # * The `virtualenv` package
-    # * Any of our requirements manifest files
-    virtualenv_package = os.path.join(
-        topsrcdir,
-        "third_party",
-        "python",
-        "virtualenv",
-        "virtualenv",
-        "version.py",
-    )
-    deps = [virtualenv_package] + requirements.requirements_paths
+    # Modifications to any of the requirements manifest files mean the virtualenv should
+    # be rebuilt:
     metadata_mtime = os.path.getmtime(
         os.path.join(target_venv.prefix, METADATA_FILENAME)
     )
-    for dep_file in deps:
+    for dep_file in requirements.requirements_paths:
         if os.path.getmtime(dep_file) > metadata_mtime:
             return SiteUpToDateResult(
                 False, f'"{dep_file}" has changed since the virtualenv was created'

python/mach/mach/test/test_site_activation.py

@@ -71,14 +71,8 @@ def test_new_package_appears_in_pkg_resources():
         subprocess.check_call(
             [
                 sys.executable,
-                os.path.join(
-                    buildconfig.topsrcdir,
-                    "third_party",
-                    "python",
-                    "virtualenv",
-                    "virtualenv.py",
-                ),
-                "--no-download",
+                "-m",
+                "venv",
                 venv_dir,
             ]
         )
@@ -327,6 +321,7 @@ def _activation_context():
         topsrcdir / "python" / "mach",
         topsrcdir / "third_party" / "python" / "packaging",
         topsrcdir / "third_party" / "python" / "pyparsing",
+        topsrcdir / "third_party" / "python" / "pip",
     ]
 
     with tempfile.TemporaryDirectory() as work_dir:

python/mach/mach/test/test_site_compatibility.py

@@ -1,7 +1,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
+import os
 import shutil
 import subprocess
 import sys
@@ -10,6 +10,7 @@ from pathlib import Path
 import mozunit
 from buildconfig import topsrcdir
 from mach.requirements import MachEnvRequirements
+from mach.site import PythonVirtualenv
 
 
 def _resolve_command_site_names():
@@ -117,20 +118,28 @@ def test_sites_compatible(tmpdir: str):
     mach_requirements = _requirement_definition_to_pip_format("mach", cache, True)
 
     # Create virtualenv to try to install all dependencies into.
+    virtualenv = PythonVirtualenv(str(work_dir / "env"))
     subprocess.check_call(
         [
             sys.executable,
-            str(
-                Path(topsrcdir)
-                / "third_party"
-                / "python"
-                / "virtualenv"
-                / "virtualenv.py"
-            ),
-            "--no-download",
-            str(work_dir / "env"),
+            "-m",
+            "venv",
+            "--without-pip",
+            virtualenv.prefix,
         ]
     )
+    platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib")
+    third_party = Path(topsrcdir) / "third_party" / "python"
+    with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile:
+        pthfile.write(
+            "\n".join(
+                [
+                    str(third_party / "pip"),
+                    str(third_party / "wheel"),
+                    str(third_party / "setuptools"),
+                ]
+            )
+        )
 
     for name in command_site_names:
         print(f'Checking compatibility of "{name}" site')
@@ -146,7 +155,9 @@ def test_sites_compatible(tmpdir: str):
         # command)
         subprocess.check_call(
             [
-                str(work_dir / "env" / "bin" / "pip"),
+                virtualenv.python_path,
+                "-m",
+                "pip",
                 "install",
                 "-r",
                 str(work_dir / "requirements.txt"),

python/mozbuild/mozbuild/mozconfig.py

@@ -144,11 +144,11 @@ class MozconfigLoader(object):
             shell = shell + ".exe"
 
         command = [
-            shell,
+            mozpath.normsep(shell),
             mozpath.normsep(self._loader_script),
             mozpath.normsep(self.topsrcdir),
-            path,
-            sys.executable,
+            mozpath.normsep(path),
+            mozpath.normsep(sys.executable),
             mozpath.join(mozpath.dirname(self._loader_script), "action", "dump_env.py"),
         ]
 

taskcluster/docker/debian-base/Dockerfile

@@ -42,6 +42,7 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG
       python3-minimal \
       python3-zstandard \
       python3-psutil \
+      python3-venv \
       vim-tiny \
       xz-utils \
       zstd