Bug 1490253 - Replace pipenv with pip-tools for vendoring packages and dependencies; r=ahal

Depends on D7869

Differential Revision: https://phabricator.services.mozilla.com/D7870

--HG--
extra : moz-landing-system : lando

Pipfile

@@ -1,20 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[dev-packages]
-
-[packages]
-attrs = "==18.1.0"
-blessings = "==1.7"
-jsmin = "==2.1.0"
-json-e = "==2.7.0"
-pip-tools = "==3.0.0"
-pipenv = "==2018.5.18"
-pytest = "==3.6.2"
-python-hglib = "==2.4"
-requests = "==2.9.1"
-six = "==1.10.0"
-virtualenv = "==15.2.0"
-voluptuous = "==0.11.5"

Pipfile.lock

@@ -1,172 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "e756c316803705f9230eb8dd3b53fd9a9aa0a146c7387e3caffb668e0f7ea223"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "atomicwrites": {
-            "hashes": [
-                "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585",
-                "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6"
-            ],
-            "version": "==1.1.5"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265",
-                "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b"
-            ],
-            "index": "pypi",
-            "version": "==18.1.0"
-        },
-        "blessings": {
-            "hashes": [
-                "sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d",
-                "sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3",
-                "sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e"
-            ],
-            "index": "pypi",
-            "version": "==1.7"
-        },
-        "certifi": {
-            "hashes": [
-                "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7",
-                "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0"
-            ],
-            "version": "==2018.4.16"
-        },
-        "click": {
-            "hashes": [
-                "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
-                "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
-            ],
-            "version": "==7.0"
-        },
-        "funcsigs": {
-            "hashes": [
-                "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca",
-                "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50"
-            ],
-            "markers": "python_version < '3.0'",
-            "version": "==1.0.2"
-        },
-        "jsmin": {
-            "hashes": [
-                "sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56"
-            ],
-            "index": "pypi",
-            "version": "==2.1.0"
-        },
-        "json-e": {
-            "hashes": [
-                "sha256:d8c1ec3f5bbc7728c3a504ebe58829f283c64eca230871e4eefe974b4cdaae4a"
-            ],
-            "index": "pypi",
-            "version": "==2.7.0"
-        },
-        "more-itertools": {
-            "hashes": [
-                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
-                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
-                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
-            ],
-            "version": "==4.3.0"
-        },
-        "pip-tools": {
-            "hashes": [
-                "sha256:4a94997602848f77ff02f660c0fcdfeaf316924ebb236c865f9742ce212aa6f9",
-                "sha256:e45e5198ce3799068642ebb0e7c9be5520bcff944c0186f79c1199a2759c970a"
-            ],
-            "index": "pypi",
-            "version": "==3.0.0"
-        },
-        "pipenv": {
-            "hashes": [
-                "sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece",
-                "sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741"
-            ],
-            "index": "pypi",
-            "version": "==2018.5.18"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff",
-                "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c",
-                "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5"
-            ],
-            "version": "==0.6.0"
-        },
-        "py": {
-            "hashes": [
-                "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7",
-                "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e"
-            ],
-            "version": "==1.5.4"
-        },
-        "pytest": {
-            "hashes": [
-                "sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c",
-                "sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd"
-            ],
-            "index": "pypi",
-            "version": "==3.6.2"
-        },
-        "python-hglib": {
-            "hashes": [
-                "sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462"
-            ],
-            "index": "pypi",
-            "version": "==2.4"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8",
-                "sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f"
-            ],
-            "index": "pypi",
-            "version": "==2.9.1"
-        },
-        "six": {
-            "hashes": [
-                "sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1",
-                "sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a"
-            ],
-            "index": "pypi",
-            "version": "==1.10.0"
-        },
-        "virtualenv": {
-            "hashes": [
-                "sha256:1d7e241b431e7afce47e77f8843a276f652699d1fa4f93b9d8ce0076fd7b0b54",
-                "sha256:e8e05d4714a1c51a2f5921e62f547fcb0f713ebbe959e0a7f585cc8bef71d11f"
-            ],
-            "index": "pypi",
-            "version": "==15.2.0"
-        },
-        "virtualenv-clone": {
-            "hashes": [
-                "sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7",
-                "sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8"
-            ],
-            "version": "==0.3.0"
-        },
-        "voluptuous": {
-            "hashes": [
-                "sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1",
-                "sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef"
-            ],
-            "index": "pypi",
-            "version": "==0.11.5"
-        }
-    },
-    "develop": {}
-}

build/virtualenv_packages.txt

@@ -9,6 +9,7 @@ mozilla.pth:python/l10n
 mozilla.pth:third_party/python/atomicwrites
 mozilla.pth:third_party/python/attrs/src
 mozilla.pth:third_party/python/blessings
+mozilla.pth:third_party/python/Click
 mozilla.pth:third_party/python/compare-locales
 mozilla.pth:third_party/python/configobj
 mozilla.pth:third_party/python/cram

moz.build

@@ -37,9 +37,6 @@ with Files('*moz*'):
 with Files('GNUmakefile'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
-with Files('Pipfile*'):
-    BUG_COMPONENT = ('Firefox Build System', 'General')
-
 with Files('*gradle*'):
     BUG_COMPONENT = ('Firefox for Android', 'Build Config & IDE Support')
     SCHEDULES.exclusive = ['android']

python/mozbuild/mozbuild/vendor_python.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
+import shutil
 import subprocess
 
 import mozfile
@@ -24,36 +25,59 @@ class VendorPython(MozbuildObject):
             self.topsrcdir, os.path.join('third_party', 'python'))
 
         packages = packages or []
-        pipenv = self.ensure_pipenv()
 
-        for package in packages:
-            if not all(package.partition('==')):
-                raise Exception('Package {} must be in the format name==version'.format(package))
+        self._activate_virtualenv()
+        pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile')
+        if not os.path.exists(pip_compile):
+            path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools'))
+            self.virtualenv_manager.install_pip_package(path, vendored=True)
+        spec = os.path.join(vendor_dir, 'requirements.in')
+        requirements = os.path.join(vendor_dir, 'requirements.txt')
 
-        for package in packages:
-            subprocess.check_call(
-                [pipenv, 'install', package],
-                cwd=self.topsrcdir)
+        with NamedTemporaryFile('w') as tmpspec:
+            shutil.copyfile(spec, tmpspec.name)
+            self._update_packages(tmpspec.name, packages)
 
-        with NamedTemporaryFile('w') as requirements:
-            # determine the dependency graph and generate requirements.txt
-            subprocess.check_call(
-                [pipenv, 'lock', '--requirements'],
-                cwd=self.topsrcdir,
-                stdout=requirements)
+            # resolve the dependencies and update requirements.txt
+            subprocess.check_output([
+                pip_compile,
+                tmpspec.name,
+                '--no-header',
+                '--no-index',
+                '--output-file', requirements,
+                '--generate-hashes'])
 
             with TemporaryDirectory() as tmp:
                 # use requirements.txt to download archived source distributions of all packages
                 self.virtualenv_manager._run_pip([
                     'download',
-                    '-r', requirements.name,
+                    '-r', requirements,
                     '--no-deps',
                     '--dest', tmp,
                     '--no-binary', ':all:',
                     '--disable-pip-version-check'])
                 self._extract(tmp, vendor_dir)
 
-        self.repository.add_remove_files(vendor_dir)
+            shutil.copyfile(tmpspec.name, spec)
+            self.repository.add_remove_files(vendor_dir)
+
+    def _update_packages(self, spec, packages):
+        for package in packages:
+            if not all(package.partition('==')):
+                raise Exception('Package {} must be in the format name==version'.format(package))
+
+        requirements = {}
+        with open(spec, 'r') as f:
+            for line in f.readlines():
+                name, version = line.rstrip().split('==')
+                requirements[name] = version
+        for package in packages:
+            name, version = package.split('==')
+            requirements[name] = version
+
+        with open(spec, 'w') as f:
+            for name, version in sorted(requirements.items()):
+                f.write('{}=={}\n'.format(name, version))
 
     def _extract(self, src, dest):
         """extract source distribution into vendor directory"""

third_party/python/moz.build

@@ -74,6 +74,9 @@ with Files('redo/**'):
 with Files('requests*/**'):
     BUG_COMPONENT = ('Firefox Build System', 'General')
 
+with Files('requirements.*'):
+    BUG_COMPONENT = ('Firefox Build System', 'General')
+
 with Files('rsa/**'):
     BUG_COMPONENT = ('Core', 'Security: PSM')
 

third_party/python/requirements.in

@@ -0,0 +1,12 @@
+attrs==18.1.0
+blessings==1.7
+jsmin==2.1.0
+json-e==2.7.0
+pip-tools==3.0.0
+pipenv==2018.5.18
+pytest==3.6.2
+python-hglib==2.4
+requests==2.9.1
+six==1.10.0
+virtualenv==15.2.0
+voluptuous==0.11.5

third_party/python/requirements.txt

@@ -0,0 +1,68 @@
+atomicwrites==1.1.5 \
+    --hash=sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585 \
+    --hash=sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6 \
+    # via pytest
+attrs==18.1.0 \
+    --hash=sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265 \
+    --hash=sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b
+blessings==1.7 \
+    --hash=sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d \
+    --hash=sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3 \
+    --hash=sha256:caad5211e7ba5afe04367cdd4cfc68fa886e2e08f6f35e76b7387d2109ccea6e
+certifi==2018.4.16 \
+    --hash=sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7 \
+    --hash=sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 \
+    # via pipenv
+click==7.0 \
+    --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
+    --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
+    # via pip-tools
+funcsigs==1.0.2 \
+    --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
+    --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \
+    # via pytest
+jsmin==2.1.0 \
+    --hash=sha256:5d07bf0251a4128e5e8e8eef603849b6b5741c337bff087731a248f9cc774f56
+json-e==2.7.0 \
+    --hash=sha256:d8c1ec3f5bbc7728c3a504ebe58829f283c64eca230871e4eefe974b4cdaae4a
+more-itertools==4.3.0 \
+    --hash=sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092 \
+    --hash=sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e \
+    --hash=sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d \
+    # via pytest
+pip-tools==3.0.0 \
+    --hash=sha256:4a94997602848f77ff02f660c0fcdfeaf316924ebb236c865f9742ce212aa6f9 \
+    --hash=sha256:e45e5198ce3799068642ebb0e7c9be5520bcff944c0186f79c1199a2759c970a
+pipenv==2018.5.18 \
+    --hash=sha256:04b9a8b02a3ff12a5502b335850cfdb192adcfd1d6bbdb7a7c47cae9ab9ddece \
+    --hash=sha256:e96d5bfa6822a17b2200d455aa5f9002c14361c50df1b1e51921479d7c09e741
+pluggy==0.6.0 \
+    --hash=sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff \
+    --hash=sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c \
+    --hash=sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5 \
+    # via pytest
+py==1.5.4 \
+    --hash=sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7 \
+    --hash=sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e \
+    # via pytest
+pytest==3.6.2 \
+    --hash=sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c \
+    --hash=sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd
+python-hglib==2.4 \
+    --hash=sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462
+requests==2.9.1 \
+    --hash=sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8 \
+    --hash=sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f
+six==1.10.0 \
+    --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \
+    --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a
+virtualenv-clone==0.3.0 \
+    --hash=sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7 \
+    --hash=sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8 \
+    # via pipenv
+virtualenv==15.2.0 \
+    --hash=sha256:1d7e241b431e7afce47e77f8843a276f652699d1fa4f93b9d8ce0076fd7b0b54 \
+    --hash=sha256:e8e05d4714a1c51a2f5921e62f547fcb0f713ebbe959e0a7f585cc8bef71d11f
+voluptuous==0.11.5 \
+    --hash=sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1 \
+    --hash=sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef