Bug 1717645: Resolve nested virtualenv requirements up-front r=ahal

This simplifies consumer logic, since they get the parsed list of pypi
and pth requirements, as well as the list of input files that were
parsed.

One benefit of this simplification is that we no longer
recursively create VirtualenvManagers.

Note that mach_bootstrap cannot (yet) take advantage
of `ParseMachEnvRequirements` because of a dependency cycle:
* `mach_bootstrap` must set up the `sys.path` to import
  `ParseMachEnvRequirements`.
* `mach_bootstrap` would want `ParseMachEnvRequirements` to
  determine which paths to add to the `sys.path`.

Differential Revision: https://phabricator.services.mozilla.com/D119685

python/mozbuild/mozbuild/requirements.py

@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+
+class PthSpecifier:
+    def __init__(self, path):
+        self.path = path
+
+
+class PypiSpecifier:
+    def __init__(self, package_name, version, full_specifier):
+        self.package_name = package_name
+        self.version = version
+        self.full_specifier = full_specifier
+
+
+class MachEnvRequirements:
+    """Requirements associated with a "virtualenv_packages.txt" definition
+
+    Represents the dependencies of a virtualenv. The source files consist
+    of colon-delimited fields. The first field
+    specifies the action. The remaining fields are arguments to that
+    action. The following actions are supported:
+
+    pth -- Adds the path given as argument to "mach.pth" under
+        the virtualenv site packages directory.
+
+    pypi -- Fetch the package, plus dependencies, from PyPI.
+
+    packages.txt -- Denotes that the specified path is a child manifest. It
+        will be read and processed as if its contents were concatenated
+        into the manifest being read.
+
+    thunderbird -- This denotes the action as to only occur for Thunderbird
+        checkouts. The initial "thunderbird" field is stripped, then the
+        remaining line is processed like normal. e.g.
+        "thunderbird:pth:python/foo"
+    """
+
+    def __init__(self):
+        self.requirements_paths = []
+        self.pth_requirements = []
+        self.pypi_requirements = []
+
+    @classmethod
+    def from_requirements_definition(
+        cls, topsrcdir, is_thunderbird, requirements_definition
+    ):
+        requirements = cls()
+        _parse_mach_env_requirements(
+            requirements, requirements_definition, topsrcdir, is_thunderbird
+        )
+        return requirements
+
+
+def _parse_mach_env_requirements(
+    requirements_output, root_requirements_path, topsrcdir, is_thunderbird
+):
+    def _parse_requirements_line(line):
+        action, params = line.rstrip().split(":", maxsplit=1)
+        if action == "pth":
+            requirements_output.pth_requirements.append(PthSpecifier(params))
+        elif action == "pypi":
+            if len(params.split("==")) != 2:
+                raise Exception(
+                    "Expected pypi package version to be pinned in the "
+                    'format "package==version", found "{}"'.format(params)
+                )
+            package_name, version = params.split("==")
+            requirements_output.pypi_requirements.append(
+                PypiSpecifier(package_name, version, params)
+            )
+        elif action == "packages.txt":
+            nested_definition_path = os.path.join(topsrcdir, params)
+            assert os.path.isfile(nested_definition_path)
+            _parse_requirements_definition_file(nested_definition_path)
+        elif action == "thunderbird":
+            if is_thunderbird:
+                _parse_requirements_line(params)
+        else:
+            raise Exception("Unknown requirements definition action: %s" % action)
+
+    def _parse_requirements_definition_file(requirements_path):
+        """Parse requirements file into list of requirements"""
+        requirements_output.requirements_paths.append(requirements_path)
+
+        with open(requirements_path, "r") as requirements_file:
+            lines = [line for line in requirements_file]
+
+        for line in lines:
+            _parse_requirements_line(line)
+
+    _parse_requirements_definition_file(root_requirements_path)

python/mozbuild/mozbuild/virtualenv.py

@@ -155,14 +155,15 @@ class VirtualenvManager(VirtualenvHelper):
                 built with then this method will return False.
         """
 
-        deps = [self.manifest_path, __file__]
-
         # check if virtualenv exists
         if not os.path.exists(self.virtualenv_root) or not os.path.exists(
             self.activate_path
         ):
             return False
 
+        env_requirements = self._requirements()
+        deps = [__file__] + env_requirements.requirements_paths
+
         # Modifications to our package dependency list or to this file mean the
         # virtualenv should be rebuilt.
         activate_mtime = os.path.getmtime(self.activate_path)
@@ -179,9 +180,7 @@ class VirtualenvManager(VirtualenvHelper):
         if (python != self.python_path) and (hexversion != orig_version):
             return False
 
-        packages = self.packages()
+        if env_requirements.pypi_requirements:
-        pypi_packages = [package for action, package in packages if action == "pypi"]
-        if pypi_packages:
             pip_json = self._run_pip(
                 ["list", "--format", "json"], capture_output=True
             ).stdout
@@ -189,23 +188,13 @@ class VirtualenvManager(VirtualenvHelper):
             installed_packages = {
                 package["name"]: package["version"] for package in installed_packages
             }
-            for pypi_package in pypi_packages:
+            for requirement in env_requirements.pypi_requirements:
-                name, version = pypi_package.split("==")
+                if (
-                if installed_packages.get(name, None) != version:
+                    installed_packages.get(requirement.package_name, None)
+                    != requirement.version
+                ):
                     return False
 
-        # recursively check sub packages.txt files
-        submanifests = [
-            package for action, package in packages if action == "packages.txt"
-        ]
-        for submanifest in submanifests:
-            submanifest = os.path.join(self.topsrcdir, submanifest)
-            submanager = VirtualenvManager(
-                self.topsrcdir, self.virtualenv_root, self.log_handle, submanifest
-            )
-            if not submanager.up_to_date(python):
-                return False
-
         return True
 
     def ensure(self, python=sys.executable):
@@ -270,31 +259,28 @@ class VirtualenvManager(VirtualenvHelper):
 
         return self.virtualenv_root
 
-    def packages(self):
+    def _requirements(self):
-        with open(self.manifest_path, "r") as fh:
+        try:
-            return [line.rstrip().split(":", maxsplit=1) for line in fh]
+            # When `virtualenv.py` is invoked from an existing Mach process,
+            # import MachEnvRequirements in the expected way.
+            from mozbuild.requirements import MachEnvRequirements
+        except ImportError:
+            # When `virtualenv.py` is invoked standalone, import
+            # MachEnvRequirements from the adjacent "standalone"
+            # requirements module.
+            from requirements import MachEnvRequirements
+
+        thunderbird_dir = os.path.join(self.topsrcdir, "comm")
+        is_thunderbird = os.path.exists(thunderbird_dir) and bool(
+            os.listdir(thunderbird_dir)
+        )
+        return MachEnvRequirements.from_requirements_definition(
+            self.topsrcdir, is_thunderbird, self.manifest_path
+        )
 
     def populate(self):
         """Populate the virtualenv.
 
-        The manifest file consists of colon-delimited fields. The first field
-        specifies the action. The remaining fields are arguments to that
-        action. The following actions are supported:
-
-        pth -- Adds the path given as argument to "mach.pth" under
-            the virtualenv site packages directory.
-
-        pypi -- Fetch the package, plus dependencies, from PyPI.
-
-        thunderbird -- This denotes the action as to only occur for Thunderbird
-            checkouts. The initial "thunderbird" field is stripped, then the
-            remaining line is processed like normal. e.g.
-            "thunderbird:pth:python/foo"
-
-        packages.txt -- Denotes that the specified path is a child manifest. It
-            will be read and processed as if its contents were concatenated
-            into the manifest being read.
-
         Note that the Python interpreter running this function should be the
         one from the virtualenv. If it is the system Python or if the
         environment is not configured properly, packages could be installed
@@ -302,52 +288,9 @@ class VirtualenvManager(VirtualenvHelper):
         """
         import distutils.sysconfig
 
-        thunderbird_dir = os.path.join(self.topsrcdir, "comm")
-        is_thunderbird = os.path.exists(thunderbird_dir) and bool(
-            os.listdir(thunderbird_dir)
-        )
-        python_lib = distutils.sysconfig.get_python_lib()
-
-        def handle_package(action, package):
-            if action == "packages.txt":
-                src = os.path.join(self.topsrcdir, package)
-                assert os.path.isfile(src), "'%s' does not exist" % src
-                submanager = VirtualenvManager(
-                    self.topsrcdir,
-                    self.virtualenv_root,
-                    self.log_handle,
-                    src,
-                    populate_local_paths=self.populate_local_paths,
-                )
-                submanager.populate()
-            elif action == "pth":
-                if not self.populate_local_paths:
-                    return
-
-                path = os.path.join(self.topsrcdir, package)
-
-                with open(os.path.join(python_lib, "mach.pth"), "a") as f:
-                    # This path is relative to the .pth file.  Using a
-                    # relative path allows the srcdir/objdir combination
-                    # to be moved around (as long as the paths relative to
-                    # each other remain the same).
-                    f.write("%s\n" % os.path.relpath(path, python_lib))
-            elif action == "thunderbird":
-                if is_thunderbird:
-                    handle_package(*package.split(":", maxsplit=1))
-            elif action == "pypi":
-                if len(package.split("==")) != 2:
-                    raise Exception(
-                        "Expected pypi package version to be pinned in the "
-                        'format "package==version", found "{}"'.format(package)
-                    )
-                self.install_pip_package(package)
-            else:
-                raise Exception("Unknown action: %s" % action)
-
         # We always target the OS X deployment target that Python itself was
         # built with, regardless of what's in the current environment. If we
-        # don't do # this, we may run into a Python bug. See
+        # don't do this, we may run into a Python bug. See
         # http://bugs.python.org/issue9516 and bug 659881.
         #
         # Note that this assumes that nothing compiled in the virtualenv is
@@ -382,8 +325,20 @@ class VirtualenvManager(VirtualenvHelper):
                 old_env_variables[k] = os.environ[k]
                 del os.environ[k]
 
-            for current_action, current_package in self.packages():
+            env_requirements = self._requirements()
-                handle_package(current_action, current_package)
+            if self.populate_local_paths:
+                python_lib = distutils.sysconfig.get_python_lib()
+                with open(os.path.join(python_lib, "mach.pth"), "a") as f:
+                    for pth_requirement in env_requirements.pth_requirements:
+                        path = os.path.join(self.topsrcdir, pth_requirement.path)
+                        # This path is relative to the .pth file.  Using a
+                        # relative path allows the srcdir/objdir combination
+                        # to be moved around (as long as the paths relative to
+                        # each other remain the same).
+                        f.write("{}\n".format(os.path.relpath(path, python_lib)))
+
+            for pypi_requirement in env_requirements.pypi_requirements:
+                self.install_pip_package(pypi_requirement.full_specifier)
 
         finally:
             os.environ.pop("MACOSX_DEPLOYMENT_TARGET", None)