Bug 1713610: Require PyPI-vendored packages be added to requirements.in r=ahal

`./mach vendor python <package>` was already adding its new package
to `requirements.txt`, so we were getting full resolver support, which
is good. However, it caused our `requirements.in` file to start
getting out-of-date, and therefore made it harder to identify the
top-level dependencies.

Arguably, we could have `./mach vendor python <package>` automatically
update `requirements.in`, too, but then we need to solve the edge cases,
such as "What if the package is already in `requirements.in`? What if
that existing item has a different version?"

The hardest part of updating `requirements.in` was finding it, so I've
also modified the `./mach vendor python` help text to make it more
identifiable.

Differential Revision: https://phabricator.services.mozilla.com/D116386

python/docs/index.rst

@@ -30,17 +30,10 @@ Where possible, the following policy applies to **ALL** vendored packages:
 Adding a Python package
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-To vendor a Python package, run ``mach vendor python [PACKAGE]``, where
-``[PACKAGE]`` is one or more package names along with a version number in the
-format ``pytest==3.5.1``. The package will be installed, transient dependencies
-will be determined, and a ``requirements.txt`` file will be generated with the
-full list of dependencies. The requirements file is then used with ``pip`` to
-download and extract the source distributions of all packages into the
-``third_party/python`` directory.
-
-Alternatively, you can also modify the direct dependencies in
-``third_party/python/requirements.in`` and then run ``mach vendor python`` for your
-changes to take effect.
+To vendor a Python package, add it to ``third_party/python/requirements.in``
+and then run ``mach vendor python``. This will update the tree of pinned
+dependencies in ``third_party/python/requirements.txt`` and download them all
+into the ``third_party/python`` directory.
 
 What if the package isn't on PyPI?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

python/mozbuild/mozbuild/vendor/mach_commands.py

@@ -7,12 +7,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 import sys
 import logging
 
-from mach.decorators import (
-    CommandArgument,
-    CommandProvider,
-    Command,
-    SubCommand,
-)
+from mach.decorators import CommandArgument, CommandProvider, Command, SubCommand
 
 from mozbuild.base import MachCommandBase
 from mozbuild.vendor.moz_yaml import load_moz_yaml, MozYamlVerifyError
@@ -151,7 +146,9 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore-
         "vendor",
         "python",
         description="Vendor Python packages from pypi.org into third_party/python. "
-        "Some extra files like docs and tests will automatically be excluded.",
+        "Some extra files like docs and tests will automatically be excluded."
+        "Installs the packages listed in third_party/python/requirements.in and "
+        "their dependencies.",
     )
     @CommandArgument(
         "--keep-extra-files",
@@ -159,15 +156,6 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore-
         default=False,
         help="Keep all files, including tests and documentation.",
     )
-    @CommandArgument(
-        "packages",
-        default=None,
-        nargs="*",
-        help="Packages to vendor. If omitted, packages and their dependencies "
-        "defined in Pipfile.lock will be vendored. If Pipfile has been modified, "
-        "then Pipfile.lock will be regenerated. Note that transient dependencies "
-        "may be updated when running this command.",
-    )
     def vendor_python(self, command_context, **kwargs):
         from mozbuild.vendor.vendor_python import VendorPython
 

python/mozbuild/mozbuild/vendor/vendor_python.py

@@ -16,14 +16,12 @@ from mozpack.files import FileFinder
 
 
 class VendorPython(MozbuildObject):
-    def vendor(self, packages=None, keep_extra_files=False):
+    def vendor(self, keep_extra_files=False):
         self.populate_logger()
         self.log_manager.enable_unstructured()
 
         vendor_dir = mozpath.join(self.topsrcdir, os.path.join("third_party", "python"))
 
-        packages = packages or []
-
         self.activate_virtualenv()
         pip_compile = os.path.join(self.virtualenv_manager.bin_path, "pip-compile")
         if not os.path.exists(pip_compile):
@@ -38,7 +36,7 @@ class VendorPython(MozbuildObject):
             tmpspec = "requirements-mach-vendor-python.in"
             tmpspec_absolute = os.path.join(spec_dir, tmpspec)
             shutil.copyfile(spec, tmpspec_absolute)
-            self._update_packages(tmpspec_absolute, packages)
+            self._update_packages(tmpspec_absolute)
 
             # resolve the dependencies and update requirements.txt
             subprocess.check_output(
@@ -75,13 +73,7 @@ class VendorPython(MozbuildObject):
             shutil.copyfile(tmpspec_absolute, spec)
             self.repository.add_remove_files(vendor_dir)
 
-    def _update_packages(self, spec, packages):
-        for package in packages:
-            if not all(package.partition("==")):
-                raise Exception(
-                    "Package {} must be in the format name==version".format(package)
-                )
-
+    def _update_packages(self, spec):
         requirements = {}
         with open(spec, "r") as f:
             comments = []
@@ -94,10 +86,6 @@ class VendorPython(MozbuildObject):
                 requirements[name] = version, comments
                 comments = []
 
-        for package in packages:
-            name, version = package.split("==")
-            requirements[name] = version, []
-
         with open(spec, "w") as f:
             for name, (version, comments) in sorted(requirements.items()):
                 if comments:
@@ -109,12 +97,7 @@ class VendorPython(MozbuildObject):
 
         ignore = ()
         if not keep_extra_files:
-            ignore = (
-                "*/doc",
-                "*/docs",
-                "*/test",
-                "*/tests",
-            )
+            ignore = ("*/doc", "*/docs", "*/test", "*/tests")
         finder = FileFinder(src)
         for path, _ in finder.find("*"):
             base, ext = os.path.splitext(path)