diff --git a/python/mach/mach/site.py b/python/mach/mach/site.py index efc25b5575ad..0b4ba71778ea 100644 --- a/python/mach/mach/site.py +++ b/python/mach/mach/site.py @@ -35,6 +35,18 @@ METADATA_FILENAME = "moz_virtualenv_metadata.json" # python packages via the system environment. PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS = ("mach", "build", "common") +_is_windows = sys.platform == "cygwin" or (sys.platform == "win32" and os.sep == "\\") + + +class VenvModuleNotFoundException(Exception): + def __init__(self): + msg = ( + 'Mach was unable to find the "venv" module, which is needed ' + "to create virtual environments in Python. You may need to " + "install it manually using the package manager for your system." + ) + super(Exception, self).__init__(msg) + class VirtualenvOutOfDateException(Exception): pass @@ -220,6 +232,7 @@ class MozSiteMetadata: yield MozSiteMetadata.current = self + sys.executable = executable if pkg_resources: @@ -321,14 +334,11 @@ class MachSiteManager: if self._site_packages_source == SitePackagesSource.NONE: return SiteUpToDateResult(True) elif self._site_packages_source == SitePackagesSource.SYSTEM: - _assert_pip_check( - self._topsrcdir, self._sys_path(), "mach", self._requirements - ) + _assert_pip_check(self._sys_path(), "mach", self._requirements) return SiteUpToDateResult(True) elif self._site_packages_source == SitePackagesSource.VENV: environment = self._virtualenv() return _is_venv_up_to_date( - self._topsrcdir, environment, self._pthfile_lines(environment), self._requirements, @@ -384,7 +394,6 @@ class MachSiteManager: environment = self._virtualenv() _create_venv_with_pthfile( - self._topsrcdir, environment, self._pthfile_lines(environment), True, @@ -563,7 +572,6 @@ class CommandSiteManager: ) _create_venv_with_pthfile( - self._topsrcdir, self._virtualenv, self._pthfile_lines(), self._populate_virtualenv, @@ -732,14 +740,12 @@ class CommandSiteManager: pthfile_lines = self._pthfile_lines() if self._mach_site_packages_source == SitePackagesSource.SYSTEM: _assert_pip_check( - self._topsrcdir, pthfile_lines, self._site_name, self._requirements if not self._populate_virtualenv else None, ) return _is_venv_up_to_date( - self._topsrcdir, self._virtualenv, pthfile_lines, self._requirements, @@ -751,11 +757,7 @@ class PythonVirtualenv: """Calculates paths of interest for general python virtual environments""" def __init__(self, prefix): - is_windows = sys.platform == "cygwin" or ( - sys.platform == "win32" and os.sep == "\\" - ) - - if is_windows: + if _is_windows: self.bin_path = os.path.join(prefix, "Scripts") self.python_path = os.path.join(self.bin_path, "python.exe") else: @@ -1023,12 +1025,6 @@ def resolve_requirements(topsrcdir, site_name): ) -def _virtualenv_py_path(topsrcdir): - return os.path.join( - topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py" - ) - - def _resolve_installed_packages(python_executable): pip_json = subprocess.check_output( [ @@ -1047,7 +1043,34 @@ def _resolve_installed_packages(python_executable): return {package["name"]: package["version"] for package in installed_packages} -def _assert_pip_check(topsrcdir, pthfile_lines, virtualenv_name, requirements): +def _ensure_python_exe(python_exe_root: Path): + """On some machines in CI venv does not behave consistently. Sometimes + only a "python3" executable is created, but we expect "python". Since + they are functionally identical, we can just copy "python3" to "python" + (and vice-versa) to solve the problem. + """ + python3_exe_path = python_exe_root / "python3" + python_exe_path = python_exe_root / "python" + + if _is_windows: + python3_exe_path = python3_exe_path.with_suffix(".exe") + python_exe_path = python_exe_path.with_suffix(".exe") + + if python3_exe_path.exists() and not python_exe_path.exists(): + shutil.copy(str(python3_exe_path), str(python_exe_path)) + + if python_exe_path.exists() and not python3_exe_path.exists(): + shutil.copy(str(python_exe_path), str(python3_exe_path)) + + if not python_exe_path.exists() and not python3_exe_path.exists(): + raise Exception( + f'Neither a "{python_exe_path.name}" or "{python3_exe_path.name}" ' + f"were found. This means something unexpected happened during the " + f"virtual environment creation and we cannot proceed." + ) + + +def _assert_pip_check(pthfile_lines, virtualenv_name, requirements): """Check if the provided pthfile lines have a package incompatibility If there's an incompatibility, raise an exception and allow it to bubble up since @@ -1077,17 +1100,30 @@ def _assert_pip_check(topsrcdir, pthfile_lines, virtualenv_name, requirements): # we create a new virtualenv that has our pinned pip version, so that # we get consistent results (there's been lots of pip resolver behaviour # changes recently). - - subprocess.check_call( - [ - sys.executable, - _virtualenv_py_path(topsrcdir), - "--no-download", - check_env_path, - ], - stdout=subprocess.DEVNULL, + process = subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", check_env_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8", ) + + if process.returncode != 0: + if "No module named venv" in process.stderr: + raise VenvModuleNotFoundException() + else: + raise subprocess.CalledProcessError( + process.returncode, + process.args, + output=process.stdout, + stderr=process.stderr, + ) + + if process.stdout: + print(process.stdout) + check_env = PythonVirtualenv(check_env_path) + _ensure_python_exe(Path(check_env.python_path).parent) + with open( os.path.join( os.path.join(check_env.resolve_sysconfig_packages_path("platlib")), @@ -1161,7 +1197,6 @@ def _deprioritize_venv_packages(virtualenv, populate_virtualenv): def _create_venv_with_pthfile( - topsrcdir, target_venv, pthfile_lines, populate_with_pip, @@ -1175,17 +1210,29 @@ def _create_venv_with_pthfile( os.makedirs(virtualenv_root) metadata.write(is_finalized=False) - subprocess.check_call( - [ - metadata.original_python.python_path, - _virtualenv_py_path(topsrcdir), - # pip, setuptools and wheel are vendored and inserted into the virtualenv - # scope automatically, so "virtualenv" doesn't need to seed it. - "--no-seed", - virtualenv_root, - ] + process = subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", virtualenv_root], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="UTF-8", ) + if process.returncode != 0: + if "No module named venv" in process.stderr: + raise VenvModuleNotFoundException() + else: + raise subprocess.CalledProcessError( + process.returncode, + process.args, + output=process.stdout, + stderr=process.stderr, + ) + + if process.stdout: + print(process.stdout) + + _ensure_python_exe(Path(target_venv.python_path).parent) + platlib_site_packages_dir = target_venv.resolve_sysconfig_packages_path("platlib") pthfile_contents = "\n".join(pthfile_lines) with open(os.path.join(platlib_site_packages_dir, PTH_FILENAME), "w") as f: @@ -1200,7 +1247,6 @@ def _create_venv_with_pthfile( def _is_venv_up_to_date( - topsrcdir, target_venv, expected_pthfile_lines, requirements, @@ -1209,23 +1255,12 @@ def _is_venv_up_to_date( if not os.path.exists(target_venv.prefix): return SiteUpToDateResult(False, f'"{target_venv.prefix}" does not exist') - # Modifications to any of the following files mean the virtualenv should be - # rebuilt: - # * The `virtualenv` package - # * Any of our requirements manifest files - virtualenv_package = os.path.join( - topsrcdir, - "third_party", - "python", - "virtualenv", - "virtualenv", - "version.py", - ) - deps = [virtualenv_package] + requirements.requirements_paths + # Modifications to any of the requirements manifest files mean the virtualenv should + # be rebuilt: metadata_mtime = os.path.getmtime( os.path.join(target_venv.prefix, METADATA_FILENAME) ) - for dep_file in deps: + for dep_file in requirements.requirements_paths: if os.path.getmtime(dep_file) > metadata_mtime: return SiteUpToDateResult( False, f'"{dep_file}" has changed since the virtualenv was created' diff --git a/python/mach/mach/test/test_site_activation.py b/python/mach/mach/test/test_site_activation.py index c1a96ffdc1d2..6b74d610c2da 100644 --- a/python/mach/mach/test/test_site_activation.py +++ b/python/mach/mach/test/test_site_activation.py @@ -71,14 +71,8 @@ def test_new_package_appears_in_pkg_resources(): subprocess.check_call( [ sys.executable, - os.path.join( - buildconfig.topsrcdir, - "third_party", - "python", - "virtualenv", - "virtualenv.py", - ), - "--no-download", + "-m", + "venv", venv_dir, ] ) @@ -327,6 +321,7 @@ def _activation_context(): topsrcdir / "python" / "mach", topsrcdir / "third_party" / "python" / "packaging", topsrcdir / "third_party" / "python" / "pyparsing", + topsrcdir / "third_party" / "python" / "pip", ] with tempfile.TemporaryDirectory() as work_dir: diff --git a/python/mach/mach/test/test_site_compatibility.py b/python/mach/mach/test/test_site_compatibility.py index f130f5c3176e..142e0c41b46d 100644 --- a/python/mach/mach/test/test_site_compatibility.py +++ b/python/mach/mach/test/test_site_compatibility.py @@ -1,7 +1,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - +import os import shutil import subprocess import sys @@ -10,6 +10,7 @@ from pathlib import Path import mozunit from buildconfig import topsrcdir from mach.requirements import MachEnvRequirements +from mach.site import PythonVirtualenv def _resolve_command_site_names(): @@ -117,20 +118,28 @@ def test_sites_compatible(tmpdir: str): mach_requirements = _requirement_definition_to_pip_format("mach", cache, True) # Create virtualenv to try to install all dependencies into. + virtualenv = PythonVirtualenv(str(work_dir / "env")) subprocess.check_call( [ sys.executable, - str( - Path(topsrcdir) - / "third_party" - / "python" - / "virtualenv" - / "virtualenv.py" - ), - "--no-download", - str(work_dir / "env"), + "-m", + "venv", + "--without-pip", + virtualenv.prefix, ] ) + platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib") + third_party = Path(topsrcdir) / "third_party" / "python" + with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile: + pthfile.write( + "\n".join( + [ + str(third_party / "pip"), + str(third_party / "wheel"), + str(third_party / "setuptools"), + ] + ) + ) for name in command_site_names: print(f'Checking compatibility of "{name}" site') @@ -146,7 +155,9 @@ def test_sites_compatible(tmpdir: str): # command) subprocess.check_call( [ - str(work_dir / "env" / "bin" / "pip"), + virtualenv.python_path, + "-m", + "pip", "install", "-r", str(work_dir / "requirements.txt"), diff --git a/python/mozbuild/mozbuild/mozconfig.py b/python/mozbuild/mozbuild/mozconfig.py index 803f0b0fa1cf..22e407e9a56c 100644 --- a/python/mozbuild/mozbuild/mozconfig.py +++ b/python/mozbuild/mozbuild/mozconfig.py @@ -144,11 +144,11 @@ class MozconfigLoader(object): shell = shell + ".exe" command = [ - shell, + mozpath.normsep(shell), mozpath.normsep(self._loader_script), mozpath.normsep(self.topsrcdir), - path, - sys.executable, + mozpath.normsep(path), + mozpath.normsep(sys.executable), mozpath.join(mozpath.dirname(self._loader_script), "action", "dump_env.py"), ] diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile index f46e07ca677f..ad48749a38ea 100644 --- a/taskcluster/docker/debian-base/Dockerfile +++ b/taskcluster/docker/debian-base/Dockerfile @@ -42,6 +42,7 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG python3-minimal \ python3-zstandard \ python3-psutil \ + python3-venv \ vim-tiny \ xz-utils \ zstd