mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1717645: Resolve nested virtualenv requirements up-front r=ahal 

This simplifies consumer logic, since they get the parsed list of pypi
and pth requirements, as well as the list of input files that were
parsed.

One benefit of this simplification is that we no longer
recursively create VirtualenvManagers.

Note that mach_bootstrap cannot (yet) take advantage
of `ParseMachEnvRequirements` because of a dependency cycle:
* `mach_bootstrap` must set up the `sys.path` to import
  `ParseMachEnvRequirements`.
* `mach_bootstrap` would want `ParseMachEnvRequirements` to
  determine which paths to add to the `sys.path`.

Differential Revision: https://phabricator.services.mozilla.com/D119685
This commit is contained in:
Mitchell Hentges 2021-07-20 21:42:01 +00:00
Родитель 6edd219c43
Коммит 6e4b9c1e3b
2 изменённых файлов: 138 добавлений и 87 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

96
python/mozbuild/mozbuild/requirements.py Normal file
Просмотреть файл

@ -0,0 +1,96 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
class PthSpecifier:
def __init__(self, path):
self.path = path
class PypiSpecifier:
def __init__(self, package_name, version, full_specifier):
self.package_name = package_name
self.version = version
self.full_specifier = full_specifier
class MachEnvRequirements:
"""Requirements associated with a "virtualenv_packages.txt" definition
Represents the dependencies of a virtualenv. The source files consist
of colon-delimited fields. The first field
specifies the action. The remaining fields are arguments to that
action. The following actions are supported:
pth -- Adds the path given as argument to "mach.pth" under
the virtualenv site packages directory.
pypi -- Fetch the package, plus dependencies, from PyPI.
packages.txt -- Denotes that the specified path is a child manifest. It
will be read and processed as if its contents were concatenated
into the manifest being read.
thunderbird -- This denotes the action as to only occur for Thunderbird
checkouts. The initial "thunderbird" field is stripped, then the
remaining line is processed like normal. e.g.
"thunderbird:pth:python/foo"
"""
def __init__(self):
self.requirements_paths = []
self.pth_requirements = []
self.pypi_requirements = []
@classmethod
def from_requirements_definition(
cls, topsrcdir, is_thunderbird, requirements_definition
):
requirements = cls()
_parse_mach_env_requirements(
requirements, requirements_definition, topsrcdir, is_thunderbird
)
return requirements
def _parse_mach_env_requirements(
requirements_output, root_requirements_path, topsrcdir, is_thunderbird
):
def _parse_requirements_line(line):
action, params = line.rstrip().split(":", maxsplit=1)
if action == "pth":
requirements_output.pth_requirements.append(PthSpecifier(params))
elif action == "pypi":
if len(params.split("==")) != 2:
raise Exception(
"Expected pypi package version to be pinned in the "
'format "package==version", found "{}"'.format(params)
)
package_name, version = params.split("==")
requirements_output.pypi_requirements.append(
PypiSpecifier(package_name, version, params)
)
elif action == "packages.txt":
nested_definition_path = os.path.join(topsrcdir, params)
assert os.path.isfile(nested_definition_path)
_parse_requirements_definition_file(nested_definition_path)
elif action == "thunderbird":
if is_thunderbird:
_parse_requirements_line(params)
else:
raise Exception("Unknown requirements definition action: %s" % action)
def _parse_requirements_definition_file(requirements_path):
"""Parse requirements file into list of requirements"""
requirements_output.requirements_paths.append(requirements_path)
with open(requirements_path, "r") as requirements_file:
lines = [line for line in requirements_file]
for line in lines:
_parse_requirements_line(line)
_parse_requirements_definition_file(root_requirements_path)

129
python/mozbuild/mozbuild/virtualenv.py
Просмотреть файл

@ -155,14 +155,15 @@ class VirtualenvManager(VirtualenvHelper):
built with then this method will return False.
"""
deps = [self.manifest_path, __file__]
# check if virtualenv exists
if not os.path.exists(self.virtualenv_root) or not os.path.exists(
self.activate_path
):
return False
env_requirements = self._requirements()
deps = [__file__] + env_requirements.requirements_paths
# Modifications to our package dependency list or to this file mean the
# virtualenv should be rebuilt.
activate_mtime = os.path.getmtime(self.activate_path)
@ -179,9 +180,7 @@ class VirtualenvManager(VirtualenvHelper):
if (python != self.python_path) and (hexversion != orig_version):
return False
packages = self.packages()
pypi_packages = [package for action, package in packages if action == "pypi"]
if pypi_packages:
if env_requirements.pypi_requirements:
pip_json = self._run_pip(
["list", "--format", "json"], capture_output=True
).stdout
@ -189,23 +188,13 @@ class VirtualenvManager(VirtualenvHelper):
installed_packages = {
package["name"]: package["version"] for package in installed_packages
}
for pypi_package in pypi_packages:
name, version = pypi_package.split("==")
if installed_packages.get(name, None) != version:
for requirement in env_requirements.pypi_requirements:
if (
installed_packages.get(requirement.package_name, None)
!= requirement.version
):
return False
# recursively check sub packages.txt files
submanifests = [
package for action, package in packages if action == "packages.txt"
]
for submanifest in submanifests:
submanifest = os.path.join(self.topsrcdir, submanifest)
submanager = VirtualenvManager(
self.topsrcdir, self.virtualenv_root, self.log_handle, submanifest
)
if not submanager.up_to_date(python):
return False
return True
def ensure(self, python=sys.executable):
@ -270,31 +259,28 @@ class VirtualenvManager(VirtualenvHelper):
return self.virtualenv_root
def packages(self):
with open(self.manifest_path, "r") as fh:
return [line.rstrip().split(":", maxsplit=1) for line in fh]
def _requirements(self):
try:
# When `virtualenv.py` is invoked from an existing Mach process,
# import MachEnvRequirements in the expected way.
from mozbuild.requirements import MachEnvRequirements
except ImportError:
# When `virtualenv.py` is invoked standalone, import
# MachEnvRequirements from the adjacent "standalone"
# requirements module.
from requirements import MachEnvRequirements
thunderbird_dir = os.path.join(self.topsrcdir, "comm")
is_thunderbird = os.path.exists(thunderbird_dir) and bool(
os.listdir(thunderbird_dir)
)
return MachEnvRequirements.from_requirements_definition(
self.topsrcdir, is_thunderbird, self.manifest_path
)
def populate(self):
"""Populate the virtualenv.
The manifest file consists of colon-delimited fields. The first field
specifies the action. The remaining fields are arguments to that
action. The following actions are supported:
pth -- Adds the path given as argument to "mach.pth" under
the virtualenv site packages directory.
pypi -- Fetch the package, plus dependencies, from PyPI.
thunderbird -- This denotes the action as to only occur for Thunderbird
checkouts. The initial "thunderbird" field is stripped, then the
remaining line is processed like normal. e.g.
"thunderbird:pth:python/foo"
packages.txt -- Denotes that the specified path is a child manifest. It
will be read and processed as if its contents were concatenated
into the manifest being read.
Note that the Python interpreter running this function should be the
one from the virtualenv. If it is the system Python or if the
environment is not configured properly, packages could be installed
@ -302,52 +288,9 @@ class VirtualenvManager(VirtualenvHelper):
"""
import distutils.sysconfig
thunderbird_dir = os.path.join(self.topsrcdir, "comm")
is_thunderbird = os.path.exists(thunderbird_dir) and bool(
os.listdir(thunderbird_dir)
)
python_lib = distutils.sysconfig.get_python_lib()
def handle_package(action, package):
if action == "packages.txt":
src = os.path.join(self.topsrcdir, package)
assert os.path.isfile(src), "'%s' does not exist" % src
submanager = VirtualenvManager(
self.topsrcdir,
self.virtualenv_root,
self.log_handle,
src,
populate_local_paths=self.populate_local_paths,
)
submanager.populate()
elif action == "pth":
if not self.populate_local_paths:
return
path = os.path.join(self.topsrcdir, package)
with open(os.path.join(python_lib, "mach.pth"), "a") as f:
# This path is relative to the .pth file. Using a
# relative path allows the srcdir/objdir combination
# to be moved around (as long as the paths relative to
# each other remain the same).
f.write("%s\n" % os.path.relpath(path, python_lib))
elif action == "thunderbird":
if is_thunderbird:
handle_package(*package.split(":", maxsplit=1))
elif action == "pypi":
if len(package.split("==")) != 2:
raise Exception(
"Expected pypi package version to be pinned in the "
'format "package==version", found "{}"'.format(package)
)
self.install_pip_package(package)
else:
raise Exception("Unknown action: %s" % action)
# We always target the OS X deployment target that Python itself was
# built with, regardless of what's in the current environment. If we
# don't do # this, we may run into a Python bug. See
# don't do this, we may run into a Python bug. See
# http://bugs.python.org/issue9516 and bug 659881.
#
# Note that this assumes that nothing compiled in the virtualenv is
@ -382,8 +325,20 @@ class VirtualenvManager(VirtualenvHelper):
old_env_variables[k] = os.environ[k]
del os.environ[k]
for current_action, current_package in self.packages():
handle_package(current_action, current_package)
env_requirements = self._requirements()
if self.populate_local_paths:
python_lib = distutils.sysconfig.get_python_lib()
with open(os.path.join(python_lib, "mach.pth"), "a") as f:
for pth_requirement in env_requirements.pth_requirements:
path = os.path.join(self.topsrcdir, pth_requirement.path)
# This path is relative to the .pth file. Using a
# relative path allows the srcdir/objdir combination
# to be moved around (as long as the paths relative to
# each other remain the same).
f.write("{}\n".format(os.path.relpath(path, python_lib)))
for pypi_requirement in env_requirements.pypi_requirements:
self.install_pip_package(pypi_requirement.full_specifier)
finally:
os.environ.pop("MACOSX_DEPLOYMENT_TARGET", None)