split local dependencies from pure test dependencies and only test deps in travis (#8639)

2018-06-25 11:53:45 +01:00 · 2018-06-25 11:53:45 +01:00 · fd916c252e
--- a/11
+++ b/11
@ -97,14 +97,19 @@ populate_data:
 	python manage.py cron category_totals
 	python manage.py update_permissions_from_mc

-install_python_dependencies:
+install_python_common_dependencies:
 	# Can't use --progress-bar=off for system packages as long as our docker image
 	# doesn't have pip 10 by default.
 	pip install --no-deps --exists-action=w -r requirements/system.txt
+	pip install --progress-bar=off --no-deps --exists-action=w -r requirements/prod_without_hash.txt
+
+install_python_test_dependencies: install_python_common_dependencies
+	pip install --progress-bar=off --no-deps --exists-action=w -r requirements/tests.txt
+
+install_python_dev_dependencies: install_python_common_dependencies
 	pip install --progress-bar=off --no-deps --exists-action=w -r requirements/flake8.txt
 	pip install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
 	pip install --progress-bar=off --no-deps --exists-action=w -r requirements/docs.txt
-	pip install --progress-bar=off --no-deps --exists-action=w -r requirements/prod_without_hash.txt
 	pip install -e .

 install_node_dependencies: install_node_js copy_node_js
@ -117,7 +122,7 @@ copy_node_js:
 	for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done
 	for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done

-update_deps: install_python_dependencies install_node_dependencies
+update_deps: install_python_dev_dependencies install_node_dependencies
    # Make sure we pull our latest mozilla-product-data.
    # There were scenarios where this wouldn't happen automatically during our CircleCI ui-tests so we enforce it here as a workaround.
 	python manage.py update_product_details
--- a/requirements/dev.txt
+++ b/requirements/dev.txt
@ -1,53 +1,7 @@
-r flake8.txt
-r prod.txt
+-r tests.txt

-# apipkg is required by execnet
-apipkg==1.4 \
-    --hash=sha256:65d2aa68b28e7d31233bb2ba8eb31cda40e4671f8ac2d6b241e358c9652a74b9 \
-    --hash=sha256:2e38399dbe842891fe85392601aab8f40a8f4cc5a9053c326de35a1cc0297ac6
-# execnet is required by pytest-cache, pytest-xdist
-execnet==1.5.0 \
-    --hash=sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83 \
-    --hash=sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a
-psutil==5.4.6 \
-    --hash=sha256:319e12f6bae4d4d988fbff3bed792953fa3b44c791f085b0a1a230f755671ef7 \
-    --hash=sha256:7789885a72aa3075d28d028236eb3f2b84d908f81d38ad41769a6ddc2fd81b7c \
-    --hash=sha256:0ff2b16e9045d01edb1dd10d7fbcc184012e37f6cd38029e959f2be9c6223f50 \
-    --hash=sha256:dc85fad15ef98103ecc047a0d81b55bbf5fe1b03313b96e883acc2e2fa87ed5c \
-    --hash=sha256:7f4616bcb44a6afda930cfc40215e5e9fa7c6896e683b287c771c937712fbe2f \
-    --hash=sha256:529ae235896efb99a6f77653a7138273ab701ec9f0343a1f5030945108dee3c4 \
-    --hash=sha256:254adb6a27c888f141d2a6032ae231d8ed4fc5f7583b4c825e5f7d7c78d26d2e \
-    --hash=sha256:a9b85b335b40a528a8e2a6b549592138de8429c6296e7361892958956e6a73cf \
-    --hash=sha256:686e5a35fe4c0acc25f3466c32e716f2d498aaae7b7edc03e2305b682226bcf6
-# py is required by pytest, pytest-xdist
-py==1.5.3 \
-    --hash=sha256:983f77f3331356039fdd792e9220b7b8ee1aa6bd2b25f567a963ff1de5a64f6a \
-    --hash=sha256:29c9fab495d7528e80ba1e343b958684f4ace687327e6f789a94bf3d1915f881
 pyinotify==0.9.6 \
    --hash=sha256:9c998a5d7606ca835065cdabc013ae6c66eb9ea76a00a1e3bc6e0cfe2b4f71f4
-# pytest is required by amo-validator, pytest-base-url, pytest-cache, pytest-cov, pytest-django, pytest-html, pytest-instafail, pytest-selenium, pytest-variables, pytest-xdist
-pytest==3.6.2 \
-    --hash=sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd \
-    --hash=sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c
-pytest-cache==1.0 \
-    --hash=sha256:be7468edd4d3d83f1e844959fd6e3fd28e77a481440a7118d430130ea31b07a9
-pytest-django==3.1.2 \
-    --hash=sha256:038ccc5a9daa1b1b0eb739ab7dce54e495811eca5ea3af4815a2a3ac45152309 \
-    --hash=sha256:00995c2999b884a38ae9cd30a8c00ed32b3d38c1041250ea84caf18085589662 # pyup: <3.2.0
-pytest-xdist==1.22.2 \
-    --hash=sha256:be2662264b035920ba740ed6efb1c816a83c8a22253df7766d129f6a7bfdbd35 \
-    --hash=sha256:e8f5744acc270b3e7d915bdb4d5f471670f049b6fbd163d4cbd52203b075d30f
-pytest-forked==0.2 \
-    --hash=sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08
-freezegun==0.3.10 \
-    --hash=sha256:94c59d69bb99c9ec3ca5a3adb41930d3ea09d2a9756c23a02d89fa75646e78dd \
-    --hash=sha256:703caac155dcaad61f78de4cb0666dca778d854dfb90b3699930adee0559a622
-cookies==2.2.1 \
-    --hash=sha256:15bee753002dff684987b8df8c235288eb8d45f8191ae056254812dfd42c81d3
-responses==0.9.0 \
-    --hash=sha256:f23a29dca18b815d9d64a516b4a0abb1fbdccff6141d988ad8100facb81cf7b3 \
-    --hash=sha256:c6082710f4abfb60793899ca5f21e7ceb25aabf321560cc0726f8b59006811c9
-
 # ipython / ipdb for easier debugging, supervisor to run services
 # Remove ipython<6 version restriction when we move to python 3, see
 # https://github.com/mozilla/addons-server/issues/5380
@ -57,31 +11,8 @@ ipython==5.7.0 \
    --hash=sha256:707d1bbfc81e41e39ead1012af931bec6f80357b87e520af352e539cf5961dc0 \
    --hash=sha256:fc0464e68f9c65cd8c453474b4175432cc29ecb6c83775baedf6dbfcee9275ab \
    --hash=sha256:8db43a7fb7619037c98626613ff08d03dda9d5d12c84814a4504c78c0da8323c # pyup: <6.0
-# meld3 is required by supervisor
-meld3==1.0.2 \
-    --hash=sha256:b28a9bfac342aadb4557aa144bea9f8e6208bfb0596190570d10a892d35ff7dc
-supervisor==3.3.4 \
-    --hash=sha256:212201a3fd1d35c150ef0c35bf0676fd1a6c195fb60bf0f2147fe7dbd317e672
 watchdog==0.8.3 \
    --hash=sha256:7e65882adb7746039b6f3876ee174952f8eaaa34491ba34333ddf1fe35de4162
-PyYAML==3.12 \
-    --hash=sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f \
-    --hash=sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736 \
-    --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \
-    --hash=sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca \
-    --hash=sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3 \
-    --hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7
-argh==0.26.2 \
-    --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 \
-    --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65
-more-itertools==4.2.0 \
-    --hash=sha256:a18d870ef2ffca2b8463c0070ad17b5978056f403fb64e3f15fe62a52db21cc0 \
-    --hash=sha256:6703844a52d3588f951883005efcf555e49566a48afd4db4e965d69b883980d3 \
-    --hash=sha256:2b6b9893337bfd9166bee6a62c2b0c9fe7735dcf85948b387ec8cba30e85d8e8
-pathtools==0.1.2 \
-    --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0
-pytest-responses==0.3.0 \
-    --hash=sha256:4556395e4d1d69620027e618c302f0dbe16eef05b1037dc5955c921b6d3bc0ee

 # Dependencies for IPython 5.5
 traitlets==4.3.2 \
@ -114,15 +45,7 @@ wcwidth==0.1.7 \
    --hash=sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e
 simplegeneric==0.8.1 \
    --hash=sha256:dc972e06094b9af5b855b3df4a646395e43d1c9d0d39ed345b7393560d0b9173
-attrs==18.1.0 \
-    --hash=sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265 \
-    --hash=sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b
-pluggy==0.6.0 \
-    --hash=sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c
 isort==4.3.4 \
    --hash=sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497 \
    --hash=sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af \
    --hash=sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8
-atomicwrites==1.1.5 \
-    --hash=sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585 \
-    --hash=sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6
--- a/requirements/tests.txt
+++ b/requirements/tests.txt
@ -0,0 +1,80 @@
+-r flake8.txt
+-r prod.txt
+
+# apipkg is required by execnet
+apipkg==1.4 \
+    --hash=sha256:65d2aa68b28e7d31233bb2ba8eb31cda40e4671f8ac2d6b241e358c9652a74b9 \
+    --hash=sha256:2e38399dbe842891fe85392601aab8f40a8f4cc5a9053c326de35a1cc0297ac6
+# execnet is required by pytest-cache, pytest-xdist
+execnet==1.5.0 \
+    --hash=sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83 \
+    --hash=sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a
+# meld3 is required by supervisor
+meld3==1.0.2 \
+    --hash=sha256:b28a9bfac342aadb4557aa144bea9f8e6208bfb0596190570d10a892d35ff7dc
+psutil==5.4.6 \
+    --hash=sha256:319e12f6bae4d4d988fbff3bed792953fa3b44c791f085b0a1a230f755671ef7 \
+    --hash=sha256:7789885a72aa3075d28d028236eb3f2b84d908f81d38ad41769a6ddc2fd81b7c \
+    --hash=sha256:0ff2b16e9045d01edb1dd10d7fbcc184012e37f6cd38029e959f2be9c6223f50 \
+    --hash=sha256:dc85fad15ef98103ecc047a0d81b55bbf5fe1b03313b96e883acc2e2fa87ed5c \
+    --hash=sha256:7f4616bcb44a6afda930cfc40215e5e9fa7c6896e683b287c771c937712fbe2f \
+    --hash=sha256:529ae235896efb99a6f77653a7138273ab701ec9f0343a1f5030945108dee3c4 \
+    --hash=sha256:254adb6a27c888f141d2a6032ae231d8ed4fc5f7583b4c825e5f7d7c78d26d2e \
+    --hash=sha256:a9b85b335b40a528a8e2a6b549592138de8429c6296e7361892958956e6a73cf \
+    --hash=sha256:686e5a35fe4c0acc25f3466c32e716f2d498aaae7b7edc03e2305b682226bcf6
+# py is required by pytest, pytest-xdist
+py==1.5.3 \
+    --hash=sha256:983f77f3331356039fdd792e9220b7b8ee1aa6bd2b25f567a963ff1de5a64f6a \
+    --hash=sha256:29c9fab495d7528e80ba1e343b958684f4ace687327e6f789a94bf3d1915f881
+# pytest is required by amo-validator, pytest-base-url, pytest-cache, pytest-cov, pytest-django, pytest-html, pytest-instafail, pytest-selenium, pytest-variables, pytest-xdist
+pytest==3.6.2 \
+    --hash=sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd \
+    --hash=sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c
+pytest-cache==1.0 \
+    --hash=sha256:be7468edd4d3d83f1e844959fd6e3fd28e77a481440a7118d430130ea31b07a9
+pytest-django==3.1.2 \
+    --hash=sha256:038ccc5a9daa1b1b0eb739ab7dce54e495811eca5ea3af4815a2a3ac45152309 \
+    --hash=sha256:00995c2999b884a38ae9cd30a8c00ed32b3d38c1041250ea84caf18085589662 # pyup: <3.2.0
+pytest-xdist==1.22.2 \
+    --hash=sha256:be2662264b035920ba740ed6efb1c816a83c8a22253df7766d129f6a7bfdbd35 \
+    --hash=sha256:e8f5744acc270b3e7d915bdb4d5f471670f049b6fbd163d4cbd52203b075d30f
+pytest-forked==0.2 \
+    --hash=sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08
+supervisor==3.3.4 \
+    --hash=sha256:212201a3fd1d35c150ef0c35bf0676fd1a6c195fb60bf0f2147fe7dbd317e672
+
+freezegun==0.3.10 \
+    --hash=sha256:94c59d69bb99c9ec3ca5a3adb41930d3ea09d2a9756c23a02d89fa75646e78dd \
+    --hash=sha256:703caac155dcaad61f78de4cb0666dca778d854dfb90b3699930adee0559a622
+cookies==2.2.1 \
+    --hash=sha256:15bee753002dff684987b8df8c235288eb8d45f8191ae056254812dfd42c81d3
+responses==0.9.0 \
+    --hash=sha256:f23a29dca18b815d9d64a516b4a0abb1fbdccff6141d988ad8100facb81cf7b3 \
+    --hash=sha256:c6082710f4abfb60793899ca5f21e7ceb25aabf321560cc0726f8b59006811c9
+
+PyYAML==3.12 \
+    --hash=sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f \
+    --hash=sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736 \
+    --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \
+    --hash=sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca \
+    --hash=sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3 \
+    --hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7
+argh==0.26.2 \
+    --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 \
+    --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65
+more-itertools==4.2.0 \
+    --hash=sha256:a18d870ef2ffca2b8463c0070ad17b5978056f403fb64e3f15fe62a52db21cc0 \
+    --hash=sha256:6703844a52d3588f951883005efcf555e49566a48afd4db4e965d69b883980d3 \
+    --hash=sha256:2b6b9893337bfd9166bee6a62c2b0c9fe7735dcf85948b387ec8cba30e85d8e8
+pathtools==0.1.2 \
+    --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0
+pytest-responses==0.3.0 \
+    --hash=sha256:4556395e4d1d69620027e618c302f0dbe16eef05b1037dc5955c921b6d3bc0ee
+attrs==18.1.0 \
+    --hash=sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265 \
+    --hash=sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b
+pluggy==0.6.0 \
+    --hash=sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c
+atomicwrites==1.1.5 \
+    --hash=sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585 \
+    --hash=sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6
--- a/scripts/travis-extract-l10n.sh
+++ b/scripts/travis-extract-l10n.sh
@ -33,7 +33,7 @@ function init_environment {
    git checkout master
    git checkout -b "$BRANCH_NAME"

-    make -f Makefile-docker install_python_dependencies
+    make -f Makefile-docker install_python_test_dependencies
    make -f Makefile-docker install_node_js
 }

--- a/tox.ini
+++ b/tox.ini
@ -17,39 +17,39 @@ whitelist_externals =

 [testenv:es]
 commands =
-    make -f Makefile-docker update_deps
+    make -f Makefile-docker install_python_test_dependencies
    pytest -m "es_tests and not needs_locales_compilation" --ignore=tests/ui/ -v {posargs}

 [testenv:addons]
 commands =
-    make -f Makefile-docker update_deps
+    make -f Makefile-docker install_python_test_dependencies
    pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/addons/ {posargs}

 [testenv:devhub]
 commands =
-    make -f Makefile-docker update_deps
+    make -f Makefile-docker install_python_test_dependencies install_node_dependencies
    pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/devhub/ {posargs}

 [testenv:reviewers]
 commands =
-    make -f Makefile-docker update_deps
+    make -f Makefile-docker install_python_test_dependencies
    pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/reviewers/ {posargs}

 [testenv:amo-locales-and-signing]
 commands =
-    make -f Makefile-docker update_deps
+    make -f Makefile-docker install_python_test_dependencies install_node_dependencies
    pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/amo/ src/olympia/lib/crypto/ src/olympia/signing {posargs}
    bash {toxinidir}/locale/compile-mo.sh {toxinidir}/locale/
    pytest -n 2 -m 'needs_locales_compilation' -v src/olympia/ {posargs}

 [testenv:users-and-accounts]
 commands =
-    make -f Makefile-docker update_deps
+    make -f Makefile-docker install_python_test_dependencies
    pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/users/ src/olympia/accounts/ {posargs}

 [testenv:main]
 commands =
-    make -f Makefile-docker update_deps
+    make -f Makefile-docker install_python_test_dependencies install_node_dependencies
    pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/ --ignore src/olympia/addons/ --ignore src/olympia/devhub/ --ignore src/olympia/reviewers/ --ignore src/olympia/amo/ --ignore src/olympia/users/ --ignore src/olympia/accounts/ --ignore src/olympia/lib/crypto --ignore src/olympia/signing {posargs}

 [testenv:ui-tests]