Bug 1724274: Move virtualenv dependency manifests to python/virtualenvs r=ahal

This is needed for a few reasons:
* All mach commands can use virtualenvs, not just `build`-related
  commands, so the files don't make sense to be in `build/`.
* When locking is added, more files associated with virtualenvs will be
  added, and this will change will ease the related directory structure
  setup.
* This removes the need for a redundant "_virtualenv_packages" keyword
  as part of the manifest filenames.

Differential Revision: https://phabricator.services.mozilla.com/D140382

build/sparse-profiles/mach

@@ -7,9 +7,6 @@ path:build/moz.configure/init.configure
 path:build/moz.configure/util.configure
 # Used for bootstrapping the mach driver.
 path:build/mach_initialize.py
-path:build/build_virtualenv_packages.txt
-path:build/common_virtualenv_packages.txt
-path:build/mach_virtualenv_packages.txt
 path:build/psutil_requirements.txt
 path:build/zstandard_requirements.txt
 path:mach

build/sparse-profiles/sphinx-docs

@@ -2,7 +2,7 @@
 
 [include]
 # List of dependencies for the command
-path:build/docs_virtualenv_packages.txt
+path:python/sites/docs.txt
 
 # Code for generating docs.
 glob:docs/**

moz.configure

@@ -1058,9 +1058,9 @@ def config_status_deps(build_env, build_project):
             os.path.join(topsrcdir, "config", "milestone.txt"),
             os.path.join(topsrcdir, "browser", "config", "version.txt"),
             os.path.join(topsrcdir, "browser", "config", "version_display.txt"),
-            os.path.join(topsrcdir, "build", "build_virtualenv_packages.txt"),
-            os.path.join(topsrcdir, "build", "common_virtualenv_packages.txt"),
-            os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
+            os.path.join(topsrcdir, "python", "sites", "build.txt"),
+            os.path.join(topsrcdir, "python", "sites", "common.txt"),
+            os.path.join(topsrcdir, "python", "sites", "mach.txt"),
             os.path.join(topsrcdir, "python", "mach", "mach", "site.py"),
             os.path.join(topsrcdir, "aclocal.m4"),
             os.path.join(topsrcdir, "old-configure.in"),

python/docs/index.rst

@@ -5,11 +5,11 @@ Using third-party Python packages
 Mach and its associated commands have a variety of 3rd-party Python dependencies. Many of these
 are vendored in ``third_party/python``, while others are installed at runtime via ``pip``.
 
-The dependencies of Mach itself can be found at ``build/mach_virtualenv_packages.txt``. Mach commands
-may have additional dependencies which are specified at ``build/<site>_virtualenv_packages.txt``.
+The dependencies of Mach itself can be found at ``python/sites/mach.txt``. Mach commands
+may have additional dependencies which are specified at ``python/sites/<site>.txt``.
 
 For example, the following Mach command would have its 3rd-party dependencies declared at
-``build/foo_virtualenv_packages.txt``.
+``python/sites/foo.txt``.
 
 .. code:: python
 
@@ -45,7 +45,7 @@ There's two ways of using 3rd-party Python dependencies:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To add a ``pip install``-d package dependency, add it to your site's
-``build/<site>_virtualenv_packages.txt`` manifest file:
+``python/sites/<site>.txt`` manifest file:
 
 .. code::
 
@@ -72,7 +72,7 @@ into the ``third_party/python`` directory.
 
 Next, add that package and any new transitive dependencies (you'll see them added in
 ``third_party/python/requirements.txt``) to the associated site's dependency manifest in
-``build/<site>_virtualenv_packages.txt``:
+``python/sites/<site>.txt``:
 
 .. code::
 
@@ -226,4 +226,4 @@ other, more mature commands may still only be compatible with a much older versi
 
     If a Mach command's dependency conflicts with a vendored package, and that vendored package
     isn't needed by Mach itself, then that vendored dependency should be moved from
-    ``mach_virtualenv_packages.txt`` to its associated environment.
+    ``python/sites/mach.txt`` to its associated environment.

python/mach/mach/requirements.py

@@ -29,7 +29,8 @@ class PypiOptionalSpecifier(PypiSpecifier):
 
 
 class MachEnvRequirements:
-    """Requirements associated with a "virtualenv_packages.txt" definition
+    """Requirements associated with a "site dependency manifest", as
+    defined in "python/sites/".
 
     Represents the dependencies of a site. The source files consist
     of colon-delimited fields. The first field

python/mach/mach/site.py

@@ -275,7 +275,7 @@ class MachSiteManager:
             virtualenv_root: The path to the the associated Mach virtualenv,
                 if any
             requirements: The requirements associated with the Mach site, parsed from
-                the file at build/mach_virtualenv_packages.txt
+                the file at python/sites/mach.txt
             original_python: The external Python site that was used to invoke Mach.
                 If Mach invocations are nested, then "original_python" refers to
                 Python site that was used to start Mach first.
@@ -490,7 +490,7 @@ class CommandSiteManager:
             site_packages_source: Where this site will import its pip-installed
                 dependencies from
             requirements: The requirements associated with this site, parsed from
-                the file at build/<site_name>_virtualenv_packages.txt
+                the file at python/sites/<site_name>.txt
         """
         self._topsrcdir = topsrcdir
         self._mach_virtualenv_root = mach_virtualenv_root
@@ -992,14 +992,12 @@ class ExternalPythonSite:
 
 
 @functools.lru_cache(maxsize=None)
-def resolve_requirements(topsrcdir, virtualenv_name):
-    manifest_path = os.path.join(
-        topsrcdir, "build", f"{virtualenv_name}_virtualenv_packages.txt"
-    )
+def resolve_requirements(topsrcdir, site_name):
+    manifest_path = os.path.join(topsrcdir, "python", "sites", f"{site_name}.txt")
     if not os.path.exists(manifest_path):
         raise Exception(
-            f'The current command is using the "{virtualenv_name}" '
-            "virtualenv. However, that virtualenv is missing its associated "
+            f'The current command is using the "{site_name}" '
+            "site. However, that site is missing its associated "
             f'requirements definition file at "{manifest_path}".'
         )
 
@@ -1011,14 +1009,14 @@ def resolve_requirements(topsrcdir, virtualenv_name):
         return MachEnvRequirements.from_requirements_definition(
             topsrcdir,
             is_thunderbird,
-            virtualenv_name not in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS,
+            site_name not in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS,
             manifest_path,
         )
     except UnexpectedFlexibleRequirementException as e:
         raise Exception(
-            f'The "{virtualenv_name}" virtualenv does not have all pypi packages pinned '
+            f'The "{site_name}" site does not have all pypi packages pinned '
             f'in the format "package==version" (found "{e.raw_requirement}").\n'
-            f"Only the {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS} virtualenvs are "
+            f"Only the {PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS} sites are "
             "allowed to have unpinned packages."
         )
 

python/mach/mach/test/test_site_compatibility.py

@@ -13,23 +13,21 @@ from mach.requirements import MachEnvRequirements
 
 
 def _resolve_command_site_names():
-    virtualenv_names = []
-    for child in (Path(topsrcdir) / "build").iterdir():
-        if not child.name.endswith("_virtualenv_packages.txt"):
+    site_names = []
+    for child in (Path(topsrcdir) / "python" / "sites").iterdir():
+        if not child.is_file():
             continue
 
-        if child.name == "mach_virtualenv_packages.txt":
+        if child.name == "mach.txt":
             continue
 
-        virtualenv_names.append(child.name[: -len("_virtualenv_packages.txt")])
-    return virtualenv_names
+        site_names.append(child.stem)
+    return site_names
 
 
-def _requirement_definition_to_pip_format(virtualenv_name, cache, is_mach_or_build_env):
+def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env):
     """Convert from parsed requirements object to pip-consumable format"""
-    requirements_path = (
-        Path(topsrcdir) / "build" / f"{virtualenv_name}_virtualenv_packages.txt"
-    )
+    requirements_path = Path(topsrcdir) / "python" / "sites" / f"{site_name}.txt"
     requirements = MachEnvRequirements.from_requirements_definition(
         topsrcdir, False, not is_mach_or_build_env, requirements_path
     )

python/moz.build

@@ -44,6 +44,9 @@ with Files("mach_commands.py"):
 with Files("mozperftest/**"):
     BUG_COMPONENT = ("Testing", "mozperftest")
 
+with Files("sites/**"):
+    BUG_COMPONENT = ("Firefox Build System", "Mach Core")
+
 
 SPHINX_PYTHON_PACKAGE_DIRS += [
     "mach",

python/mozbuild/mozbuild/action/test_archive.py

@@ -391,15 +391,15 @@ ARCHIVE_FILES = {
         {"source": buildconfig.topsrcdir, "pattern": "build/mach_initialize.py"},
         {
             "source": buildconfig.topsrcdir,
-            "pattern": "build/build_virtualenv_packages.txt",
+            "pattern": "python/sites/build.txt",
         },
         {
             "source": buildconfig.topsrcdir,
-            "pattern": "build/common_virtualenv_packages.txt",
+            "pattern": "python/sites/common.txt",
         },
         {
             "source": buildconfig.topsrcdir,
-            "pattern": "build/mach_virtualenv_packages.txt",
+            "pattern": "python/sites/mach.txt",
         },
         {"source": buildconfig.topsrcdir, "pattern": "mach/**"},
         {

toolkit/components/glean/docs/dev/updating_sdk.md

@@ -12,7 +12,7 @@ To update the dependency:
 2. Run `mach vendor rust`.
    This fetches all dependencies and adds them to `third_pary/rust`.
 3. Update the version of `gleanVersion` in `build.gradle` to the same version.
-4. Update the version of `glean-sdk` in `build/mach_virtualenv_packages.txt`.
+4. Update the version of `glean-sdk` in `python/sites/mach.txt`.
 
 ## Version mismatches of Rust dependencies