Bug 1724274: Move virtualenv dependency manifests to python/virtualenvs r=ahal

This is needed for a few reasons:
* All mach commands can use virtualenvs, not just `build`-related
  commands, so the files don't make sense to be in `build/`.
* When locking is added, more files associated with virtualenvs will be
  added, and this will change will ease the related directory structure
  setup.
* This removes the need for a redundant "_virtualenv_packages" keyword
  as part of the manifest filenames.

Differential Revision: https://phabricator.services.mozilla.com/D140382
This commit is contained in:
Mitchell Hentges 2022-03-24 14:04:34 +00:00
Родитель 1195180f57
Коммит 23cbe80fd0
20 изменённых файлов: 36 добавлений и 39 удалений

Просмотреть файл

@ -7,9 +7,6 @@ path:build/moz.configure/init.configure
path:build/moz.configure/util.configure
# Used for bootstrapping the mach driver.
path:build/mach_initialize.py
path:build/build_virtualenv_packages.txt
path:build/common_virtualenv_packages.txt
path:build/mach_virtualenv_packages.txt
path:build/psutil_requirements.txt
path:build/zstandard_requirements.txt
path:mach

Просмотреть файл

@ -2,7 +2,7 @@
[include]
# List of dependencies for the command
path:build/docs_virtualenv_packages.txt
path:python/sites/docs.txt
# Code for generating docs.
glob:docs/**

Просмотреть файл

@ -1058,9 +1058,9 @@ def config_status_deps(build_env, build_project):
os.path.join(topsrcdir, "config", "milestone.txt"),
os.path.join(topsrcdir, "browser", "config", "version.txt"),
os.path.join(topsrcdir, "browser", "config", "version_display.txt"),
os.path.join(topsrcdir, "build", "build_virtualenv_packages.txt"),
os.path.join(topsrcdir, "build", "common_virtualenv_packages.txt"),
os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
os.path.join(topsrcdir, "python", "sites", "build.txt"),
os.path.join(topsrcdir, "python", "sites", "common.txt"),
os.path.join(topsrcdir, "python", "sites", "mach.txt"),
os.path.join(topsrcdir, "python", "mach", "mach", "site.py"),
os.path.join(topsrcdir, "aclocal.m4"),
os.path.join(topsrcdir, "old-configure.in"),

Просмотреть файл

@ -5,11 +5,11 @@ Using third-party Python packages
Mach and its associated commands have a variety of 3rd-party Python dependencies. Many of these
are vendored in ``third_party/python``, while others are installed at runtime via ``pip``.
The dependencies of Mach itself can be found at ``build/mach_virtualenv_packages.txt``. Mach commands
may have additional dependencies which are specified at ``build/<site>_virtualenv_packages.txt``.
The dependencies of Mach itself can be found at ``python/sites/mach.txt``. Mach commands
may have additional dependencies which are specified at ``python/sites/<site>.txt``.
For example, the following Mach command would have its 3rd-party dependencies declared at
``build/foo_virtualenv_packages.txt``.
``python/sites/foo.txt``.
.. code:: python
@ -45,7 +45,7 @@ There's two ways of using 3rd-party Python dependencies:
~~~~~~~~~~~~~~~~~~~~~~~~~~~
To add a ``pip install``-d package dependency, add it to your site's
``build/<site>_virtualenv_packages.txt`` manifest file:
``python/sites/<site>.txt`` manifest file:
.. code::
@ -72,7 +72,7 @@ into the ``third_party/python`` directory.
Next, add that package and any new transitive dependencies (you'll see them added in
``third_party/python/requirements.txt``) to the associated site's dependency manifest in
``build/<site>_virtualenv_packages.txt``:
``python/sites/<site>.txt``:
.. code::
@ -226,4 +226,4 @@ other, more mature commands may still only be compatible with a much older versi
If a Mach command's dependency conflicts with a vendored package, and that vendored package
isn't needed by Mach itself, then that vendored dependency should be moved from
``mach_virtualenv_packages.txt`` to its associated environment.
``python/sites/mach.txt`` to its associated environment.

Просмотреть файл

@ -29,7 +29,8 @@ class PypiOptionalSpecifier(PypiSpecifier):
class MachEnvRequirements:
"""Requirements associated with a "virtualenv_packages.txt" definition
"""Requirements associated with a "site dependency manifest", as
defined in "python/sites/".
Represents the dependencies of a site. The source files consist
of colon-delimited fields. The first field

Просмотреть файл

@ -275,7 +275,7 @@ class MachSiteManager:
virtualenv_root: The path to the the associated Mach virtualenv,
if any
requirements: The requirements associated with the Mach site, parsed from
the file at build/mach_virtualenv_packages.txt
the file at python/sites/mach.txt
original_python: The external Python site that was used to invoke Mach.
If Mach invocations are nested, then "original_python" refers to
Python site that was used to start Mach first.
@ -490,7 +490,7 @@ class CommandSiteManager:
site_packages_source: Where this site will import its pip-installed
dependencies from
requirements: The requirements associated with this site, parsed from
the file at build/<site_name>_virtualenv_packages.txt
the file at python/sites/<site_name>.txt
"""
self._topsrcdir = topsrcdir
self._mach_virtualenv_root = mach_virtualenv_root
@ -992,14 +992,12 @@ class ExternalPythonSite:
@functools.lru_cache(maxsize=None)
def resolve_requirements(topsrcdir, virtualenv_name):
manifest_path = os.path.join(
topsrcdir, "build", f"{virtualenv_name}_virtualenv_packages.txt"
)
def resolve_requirements(topsrcdir, site_name):
manifest_path = os.path.join(topsrcdir, "python", "sites", f"{site_name}.txt")
if not os.path.exists(manifest_path):
raise Exception(
f'The current command is using the "{virtualenv_name}" '
"virtualenv. However, that virtualenv is missing its associated "
f'The current command is using the "{site_name}" '
"site. However, that site is missing its associated "
f'requirements definition file at "{manifest_path}".'
)
@ -1011,14 +1009,14 @@ def resolve_requirements(topsrcdir, virtualenv_name):
return MachEnvRequirements.from_requirements_definition(
topsrcdir,
is_thunderbird,
virtualenv_name not in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS,
site_name not in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS,
manifest_path,
)
except UnexpectedFlexibleRequirementException as e:
raise Exception(
f'The "{virtualenv_name}" virtualenv does not have all pypi packages pinned '
f'The "{site_name}" site does not have all pypi packages pinned '
f'in the format "package==version" (found "{e.raw_requirement}").\n'
f"Only the {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS} virtualenvs are "
f"Only the {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS} sites are "
"allowed to have unpinned packages."
)

Просмотреть файл

@ -13,23 +13,21 @@ from mach.requirements import MachEnvRequirements
def _resolve_command_site_names():
virtualenv_names = []
for child in (Path(topsrcdir) / "build").iterdir():
if not child.name.endswith("_virtualenv_packages.txt"):
site_names = []
for child in (Path(topsrcdir) / "python" / "sites").iterdir():
if not child.is_file():
continue
if child.name == "mach_virtualenv_packages.txt":
if child.name == "mach.txt":
continue
virtualenv_names.append(child.name[: -len("_virtualenv_packages.txt")])
return virtualenv_names
site_names.append(child.stem)
return site_names
def _requirement_definition_to_pip_format(virtualenv_name, cache, is_mach_or_build_env):
def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env):
"""Convert from parsed requirements object to pip-consumable format"""
requirements_path = (
Path(topsrcdir) / "build" / f"{virtualenv_name}_virtualenv_packages.txt"
)
requirements_path = Path(topsrcdir) / "python" / "sites" / f"{site_name}.txt"
requirements = MachEnvRequirements.from_requirements_definition(
topsrcdir, False, not is_mach_or_build_env, requirements_path
)

Просмотреть файл

@ -44,6 +44,9 @@ with Files("mach_commands.py"):
with Files("mozperftest/**"):
BUG_COMPONENT = ("Testing", "mozperftest")
with Files("sites/**"):
BUG_COMPONENT = ("Firefox Build System", "Mach Core")
SPHINX_PYTHON_PACKAGE_DIRS += [
"mach",

Просмотреть файл

@ -391,15 +391,15 @@ ARCHIVE_FILES = {
{"source": buildconfig.topsrcdir, "pattern": "build/mach_initialize.py"},
{
"source": buildconfig.topsrcdir,
"pattern": "build/build_virtualenv_packages.txt",
"pattern": "python/sites/build.txt",
},
{
"source": buildconfig.topsrcdir,
"pattern": "build/common_virtualenv_packages.txt",
"pattern": "python/sites/common.txt",
},
{
"source": buildconfig.topsrcdir,
"pattern": "build/mach_virtualenv_packages.txt",
"pattern": "python/sites/mach.txt",
},
{"source": buildconfig.topsrcdir, "pattern": "mach/**"},
{

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

Просмотреть файл

@ -12,7 +12,7 @@ To update the dependency:
2. Run `mach vendor rust`.
This fetches all dependencies and adds them to `third_pary/rust`.
3. Update the version of `gleanVersion` in `build.gradle` to the same version.
4. Update the version of `glean-sdk` in `build/mach_virtualenv_packages.txt`.
4. Update the version of `glean-sdk` in `python/sites/mach.txt`.
## Version mismatches of Rust dependencies