Backed out 7 changesets (bug 1717645, bug 1717051, bug 1720215) for busting the gecko decision task. CLOSED TREE DONTBUILD

Backed out changeset dbdd100c60d0 (bug 1717051)
Backed out changeset 4f32438176a0 (bug 1717051)
Backed out changeset 6597db537fa2 (bug 1717051)
Backed out changeset cc686ee3fb6e (bug 1720215)
Backed out changeset 82dc0771b98b (bug 1717645)
Backed out changeset 16b0fb30aff2 (bug 1717645)
Backed out changeset 370e02f73923 (bug 1717645)

build/common_virtualenv_packages.txt

@@ -20,6 +20,8 @@ pth:testing/condprofile
 pth:testing/firefox-ui/harness
 pth:testing/marionette/client
 pth:testing/marionette/harness
+pth:testing/marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
+pth:testing/marionette/puppeteer/firefox
 pth:testing/mozbase/manifestparser
 pth:testing/mozbase/mozcrash
 pth:testing/mozbase/mozdebug
@@ -44,6 +46,7 @@ pth:testing/mozbase/mozversion
 pth:testing/raptor
 pth:testing/talos
 pth:testing/web-platform
+pth:testing/web-platform/tests/tools/six
 pth:testing/web-platform/tests/tools/third_party/certifi
 pth:testing/web-platform/tests/tools/third_party/h2
 pth:testing/web-platform/tests/tools/third_party/hpack

build/glean_requirements.in

@@ -0,0 +1 @@
+glean_sdk==36.0.0

build/glean_requirements.txt

@@ -0,0 +1,174 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=glean_requirements.txt glean_requirements.in
+#
+appdirs==1.4.4 \
+    --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41 \
+    --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128
+    # via glean-parser
+attrs==20.3.0 \
+    --hash=sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 \
+    --hash=sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700
+    # via jsonschema
+cffi==1.14.4 \
+    --hash=sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e \
+    --hash=sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d \
+    --hash=sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a \
+    --hash=sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec \
+    --hash=sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362 \
+    --hash=sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668 \
+    --hash=sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c \
+    --hash=sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b \
+    --hash=sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06 \
+    --hash=sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698 \
+    --hash=sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2 \
+    --hash=sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c \
+    --hash=sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7 \
+    --hash=sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009 \
+    --hash=sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03 \
+    --hash=sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b \
+    --hash=sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909 \
+    --hash=sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53 \
+    --hash=sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35 \
+    --hash=sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26 \
+    --hash=sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b \
+    --hash=sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01 \
+    --hash=sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb \
+    --hash=sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293 \
+    --hash=sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd \
+    --hash=sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d \
+    --hash=sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3 \
+    --hash=sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d \
+    --hash=sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e \
+    --hash=sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca \
+    --hash=sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d \
+    --hash=sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775 \
+    --hash=sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375 \
+    --hash=sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b \
+    --hash=sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b \
+    --hash=sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f
+    # via glean-sdk
+click==7.1.2 \
+    --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \
+    --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc
+    # via glean-parser
+diskcache==5.1.0 \
+    --hash=sha256:bc7928df986dbc8a8d6e34c33b0da89d668cfa65e7fcc91298a6959a35076993 \
+    --hash=sha256:d8d608363f1b0ecbc216b5b34b6c8e269a5d4cce338269f3b1c1f8f11816f71d
+    # via glean-parser
+glean-parser==2.5.0 \
+    --hash=sha256:50d53cb23c7b2ecdfed497187c63ea03bff8f78a0bbfd32dbaffb0affa0b6d2d \
+    --hash=sha256:b2c1b670b73e7df21de335de34e9ece457ef3996110567d7ad6f09fb546e6e5e
+    # via glean-sdk
+glean_sdk==36.0.0 \
+    --hash=sha256:325606a32442fb6956def031568904e3ad663db49c31992082645e4f57827e49 \
+    --hash=sha256:45aba5df378584aedf2890a1c918846f3f29f52ff63e9cb24d1bc5135edd8f4c \
+    --hash=sha256:48401bb9bc2bb4b44b7897305007fa67e0bde044ac73309050d60d78a4ee2cca \
+    --hash=sha256:64b91f1ba273db500a2ced9a549ce5a95ccbeaf22fc9568d8d4fda6fe38a7e6c \
+    --hash=sha256:81ddbf2d471009b6a07f33ebca37aa4694a2dfa7c8362f4d6aa101b0293c6f12 \
+    --hash=sha256:8baf391b64445c8bb40f2a36e511a87653c60e481c0ac2a89521d9f4e9b66425
+    # via -r glean_requirements.in
+importlib-metadata==3.7.0 \
+    --hash=sha256:24499ffde1b80be08284100393955842be4a59c7c16bbf2738aad0e464a8e0aa \
+    --hash=sha256:c6af5dbf1126cd959c4a8d8efd61d4d3c83bddb0459a17e554284a077574b614
+    # via jsonschema
+iso8601==0.1.14 \
+    --hash=sha256:8aafd56fa0290496c5edbb13c311f78fa3a241f0853540da09d9363eae3ebd79 \
+    --hash=sha256:e7e1122f064d626e17d47cd5106bed2c620cb38fe464999e0ddae2b6d2de6004
+    # via
+    #   glean-parser
+    #   glean-sdk
+jinja2==2.11.2 \
+    --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \
+    --hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035
+    # via glean-parser
+jsonschema==3.2.0 \
+    --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \
+    --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a
+    # via glean-parser
+markupsafe==1.1.1 \
+    --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
+    --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
+    --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \
+    --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \
+    --hash=sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42 \
+    --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \
+    --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \
+    --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \
+    --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \
+    --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \
+    --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \
+    --hash=sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b \
+    --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \
+    --hash=sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15 \
+    --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \
+    --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \
+    --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \
+    --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \
+    --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \
+    --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \
+    --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \
+    --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \
+    --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \
+    --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \
+    --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \
+    --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \
+    --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \
+    --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \
+    --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \
+    --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \
+    --hash=sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2 \
+    --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 \
+    --hash=sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be
+    # via jinja2
+pathspec==0.8.1 \
+    --hash=sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd \
+    --hash=sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d
+    # via yamllint
+pycparser==2.20 \
+    --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
+    --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705
+    # via cffi
+pyrsistent==0.17.3 \
+    --hash=sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e
+    # via jsonschema
+pyyaml==5.3.1 \
+    --hash=sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97 \
+    --hash=sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76 \
+    --hash=sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2 \
+    --hash=sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e \
+    --hash=sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648 \
+    --hash=sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf \
+    --hash=sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f \
+    --hash=sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2 \
+    --hash=sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee \
+    --hash=sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a \
+    --hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
+    --hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
+    --hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a
+    # via
+    #   glean-parser
+    #   yamllint
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+    # via jsonschema
+typing-extensions==3.7.4.3 \
+    --hash=sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 \
+    --hash=sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c \
+    --hash=sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f
+    # via importlib-metadata
+yamllint==1.26.0 \
+    --hash=sha256:8a5f8e442f49309eaf3e9d7232ce76f2fc8026f5c0c0b164b83f33fed1399637 \
+    --hash=sha256:b0e4c89985c7f5f8451c2eb8c67d804d10ac13a4abe031cbf49bdf3465d01087
+    # via glean-parser
+zipp==3.4.0 \
+    --hash=sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108 \
+    --hash=sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb
+    # via importlib-metadata
+
+# WARNING: The following packages were not pinned, but pip requires them to be
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
+# setuptools

build/mach_bootstrap.py

@@ -144,11 +144,7 @@ CATEGORIES = {
 
 def search_path(mozilla_dir, packages_txt):
     with open(os.path.join(mozilla_dir, packages_txt)) as f:
-        packages = [
-            line.strip().split(":", maxsplit=1)
-            for line in f
-            if not line.lstrip().startswith("#")
-        ]
+        packages = [line.rstrip().split(":", maxsplit=1) for line in f]
 
     def handle_package(action, package):
         if action == "packages.txt":

build/mach_virtualenv_packages.txt

@@ -1,7 +1 @@
 packages.txt:build/common_virtualenv_packages.txt
-# glean_sdk may not be installable if a wheel isn't available
-# and it has to be built from source.
-pypi-optional:glean_sdk==36.0.0:telemetry will not be collected
-# Mach gracefully handles the case where `psutil` is unavailable.
-pypi-optional:psutil==5.8.0:telemetry will be missing some data
-pypi:zstandard==0.15.2

build/psutil_requirements.in

@@ -0,0 +1,2 @@
+psutil==5.8.0
+

build/psutil_requirements.txt

@@ -0,0 +1,36 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=build/psutil_requirements.txt build/psutil_requirements.in
+#
+psutil==5.8.0 \
+    --hash=sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64 \
+    --hash=sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131 \
+    --hash=sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c \
+    --hash=sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6 \
+    --hash=sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023 \
+    --hash=sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df \
+    --hash=sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394 \
+    --hash=sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4 \
+    --hash=sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b \
+    --hash=sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2 \
+    --hash=sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d \
+    --hash=sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65 \
+    --hash=sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d \
+    --hash=sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef \
+    --hash=sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7 \
+    --hash=sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60 \
+    --hash=sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6 \
+    --hash=sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8 \
+    --hash=sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b \
+    --hash=sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d \
+    --hash=sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac \
+    --hash=sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935 \
+    --hash=sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d \
+    --hash=sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28 \
+    --hash=sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876 \
+    --hash=sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0 \
+    --hash=sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3 \
+    --hash=sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563
+    # via -r build/psutil_requirements.in

build/zstandard_requirements.in

@@ -0,0 +1,2 @@
+zstandard==0.15.2
+

python/mozbuild/mozbuild/mach_commands.py

@@ -2370,6 +2370,40 @@ class CreateMachEnvironment(MachCommandBase):
             print("virtualenv at %s is already up to date." % virtualenv_path)
         else:
             manager.build(sys.executable)
+
+        try:
+            # `mach` can handle it perfectly fine if `psutil` is missing, so
+            # there's no reason to freak out in this case.
+            manager.install_pip_requirements(
+                os.path.join(
+                    command_context.topsrcdir, "build", "psutil_requirements.txt"
+                )
+            )
+        except subprocess.CalledProcessError:
+            print(
+                "Could not install psutil, so telemetry will be missing some "
+                "data. Continuing."
+            )
+
+        manager.install_pip_requirements(
+            os.path.join(
+                command_context.topsrcdir, "build", "zstandard_requirements.txt"
+            )
+        )
+
+        # This can fail on some platforms. See
+        # https://bugzilla.mozilla.org/show_bug.cgi?id=1660120
+        try:
+            manager.install_pip_requirements(
+                os.path.join(
+                    command_context.topsrcdir, "build", "glean_requirements.txt"
+                )
+            )
+        except subprocess.CalledProcessError:
+            print(
+                "Could not install glean_sdk, so telemetry will not be "
+                "collected. Continuing."
+            )
         print("Mach environment created.")
 
 

python/mozbuild/mozbuild/requirements.py

@@ -1,125 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-
-
-class PthSpecifier:
-    def __init__(self, path):
-        self.path = path
-
-
-class PypiSpecifier:
-    def __init__(self, package_name, version, full_specifier):
-        self.package_name = package_name
-        self.version = version
-        self.full_specifier = full_specifier
-
-
-class PypiOptionalSpecifier:
-    def __init__(self, repercussion, package_name, version, full_specifier):
-        self.repercussion = repercussion
-        self.package_name = package_name
-        self.version = version
-        self.full_specifier = full_specifier
-
-
-class MachEnvRequirements:
-    """Requirements associated with a "virtualenv_packages.txt" definition
-
-    Represents the dependencies of a virtualenv. The source files consist
-    of colon-delimited fields. The first field
-    specifies the action. The remaining fields are arguments to that
-    action. The following actions are supported:
-
-    pth -- Adds the path given as argument to "mach.pth" under
-        the virtualenv site packages directory.
-
-    pypi -- Fetch the package, plus dependencies, from PyPI.
-
-    packages.txt -- Denotes that the specified path is a child manifest. It
-        will be read and processed as if its contents were concatenated
-        into the manifest being read.
-
-    thunderbird -- This denotes the action as to only occur for Thunderbird
-        checkouts. The initial "thunderbird" field is stripped, then the
-        remaining line is processed like normal. e.g.
-        "thunderbird:pth:python/foo"
-    """
-
-    def __init__(self):
-        self.requirements_paths = []
-        self.pth_requirements = []
-        self.pypi_requirements = []
-        self.pypi_optional_requirements = []
-
-    @classmethod
-    def from_requirements_definition(
-        cls, topsrcdir, is_thunderbird, requirements_definition
-    ):
-        requirements = cls()
-        _parse_mach_env_requirements(
-            requirements, requirements_definition, topsrcdir, is_thunderbird
-        )
-        return requirements
-
-
-def _parse_mach_env_requirements(
-    requirements_output, root_requirements_path, topsrcdir, is_thunderbird
-):
-    def _parse_requirements_line(line):
-        line = line.strip()
-        if line.startswith("#"):
-            return
-
-        action, params = line.rstrip().split(":", maxsplit=1)
-        if action == "pth":
-            requirements_output.pth_requirements.append(PthSpecifier(params))
-        elif action == "pypi":
-            package_name, version = _parse_package_specifier(params)
-            requirements_output.pypi_requirements.append(
-                PypiSpecifier(package_name, version, params)
-            )
-        elif action == "pypi-optional":
-            if len(params.split(":", maxsplit=1)) != 2:
-                raise Exception(
-                    "Expected pypi-optional package to have a repercussion"
-                    'description in the format "package:fallback explanation", '
-                    'found "{}"'.format(params)
-                )
-            package, repercussion = params.split(":")
-            package_name, version = _parse_package_specifier(package)
-            requirements_output.pypi_optional_requirements.append(
-                PypiOptionalSpecifier(repercussion, package_name, version, package)
-            )
-        elif action == "packages.txt":
-            nested_definition_path = os.path.join(topsrcdir, params)
-            assert os.path.isfile(nested_definition_path)
-            _parse_requirements_definition_file(nested_definition_path)
-        elif action == "thunderbird":
-            if is_thunderbird:
-                _parse_requirements_line(params)
-        else:
-            raise Exception("Unknown requirements definition action: %s" % action)
-
-    def _parse_requirements_definition_file(requirements_path):
-        """Parse requirements file into list of requirements"""
-        requirements_output.requirements_paths.append(requirements_path)
-
-        with open(requirements_path, "r") as requirements_file:
-            lines = [line for line in requirements_file]
-
-        for line in lines:
-            _parse_requirements_line(line)
-
-    _parse_requirements_definition_file(root_requirements_path)
-
-
-def _parse_package_specifier(specifier):
-    if len(specifier.split("==")) != 2:
-        raise Exception(
-            "Expected pypi package version to be pinned in the "
-            'format "package==version", found "{}"'.format(specifier)
-        )
-    return specifier.split("==")

python/mozbuild/mozbuild/virtualenv.py

@@ -14,11 +14,9 @@ import platform
 import shutil
 import subprocess
 import sys
-from tempfile import TemporaryDirectory
 
 IS_NATIVE_WIN = sys.platform == "win32" and os.sep == "\\"
 IS_CYGWIN = sys.platform == "cygwin"
-PTH_FILENAME = "mach.pth"
 
 
 UPGRADE_WINDOWS = """
@@ -157,15 +155,14 @@ class VirtualenvManager(VirtualenvHelper):
                 built with then this method will return False.
         """
 
+        deps = [self.manifest_path, __file__]
+
         # check if virtualenv exists
         if not os.path.exists(self.virtualenv_root) or not os.path.exists(
             self.activate_path
         ):
             return False
 
-        env_requirements = self._requirements()
-        deps = [__file__] + env_requirements.requirements_paths
-
         # Modifications to our package dependency list or to this file mean the
         # virtualenv should be rebuilt.
         activate_mtime = os.path.getmtime(self.activate_path)
@@ -182,43 +179,33 @@ class VirtualenvManager(VirtualenvHelper):
         if (python != self.python_path) and (hexversion != orig_version):
             return False
 
-        if env_requirements.pth_requirements and self.populate_local_paths:
-            try:
-                with open(
-                    os.path.join(self._site_packages_dir(), PTH_FILENAME)
-                ) as file:
-                    pth_lines = file.read().strip().split("\n")
-            except FileNotFoundError:
-                return False
-
-            current_paths = [
-                os.path.abspath(os.path.join(self._site_packages_dir(), path))
-                for path in pth_lines
-            ]
-
-            required_paths = [
-                os.path.abspath(os.path.join(self.topsrcdir, pth.path))
-                for pth in env_requirements.pth_requirements
-            ]
-
-            if current_paths != required_paths:
-                return False
-
-        if env_requirements.pypi_requirements:
+        packages = self.packages()
+        pypi_packages = [package for action, package in packages if action == "pypi"]
+        if pypi_packages:
             pip_json = self._run_pip(
-                ["list", "--format", "json"], stdout=subprocess.PIPE
+                ["list", "--format", "json"], capture_output=True
             ).stdout
             installed_packages = json.loads(pip_json)
             installed_packages = {
                 package["name"]: package["version"] for package in installed_packages
             }
-            for requirement in env_requirements.pypi_requirements:
-                if (
-                    installed_packages.get(requirement.package_name, None)
-                    != requirement.version
-                ):
+            for pypi_package in pypi_packages:
+                name, version = pypi_package.split("==")
+                if installed_packages.get(name, None) != version:
                     return False
 
+        # recursively check sub packages.txt files
+        submanifests = [
+            package for action, package in packages if action == "packages.txt"
+        ]
+        for submanifest in submanifests:
+            submanifest = os.path.join(self.topsrcdir, submanifest)
+            submanager = VirtualenvManager(
+                self.topsrcdir, self.virtualenv_root, self.log_handle, submanifest
+            )
+            if not submanager.up_to_date(python):
+                return False
+
         return True
 
     def ensure(self, python=sys.executable):
@@ -283,28 +270,31 @@ class VirtualenvManager(VirtualenvHelper):
 
         return self.virtualenv_root
 
-    def _requirements(self):
-        try:
-            # When `virtualenv.py` is invoked from an existing Mach process,
-            # import MachEnvRequirements in the expected way.
-            from mozbuild.requirements import MachEnvRequirements
-        except ImportError:
-            # When `virtualenv.py` is invoked standalone, import
-            # MachEnvRequirements from the adjacent "standalone"
-            # requirements module.
-            from requirements import MachEnvRequirements
-
-        thunderbird_dir = os.path.join(self.topsrcdir, "comm")
-        is_thunderbird = os.path.exists(thunderbird_dir) and bool(
-            os.listdir(thunderbird_dir)
-        )
-        return MachEnvRequirements.from_requirements_definition(
-            self.topsrcdir, is_thunderbird, self.manifest_path
-        )
+    def packages(self):
+        with open(self.manifest_path, "r") as fh:
+            return [line.rstrip().split(":", maxsplit=1) for line in fh]
 
     def populate(self):
         """Populate the virtualenv.
 
+        The manifest file consists of colon-delimited fields. The first field
+        specifies the action. The remaining fields are arguments to that
+        action. The following actions are supported:
+
+        pth -- Adds the path given as argument to "mach.pth" under
+            the virtualenv site packages directory.
+
+        pypi -- Fetch the package, plus dependencies, from PyPI.
+
+        thunderbird -- This denotes the action as to only occur for Thunderbird
+            checkouts. The initial "thunderbird" field is stripped, then the
+            remaining line is processed like normal. e.g.
+            "thunderbird:pth:python/foo"
+
+        packages.txt -- Denotes that the specified path is a child manifest. It
+            will be read and processed as if its contents were concatenated
+            into the manifest being read.
+
         Note that the Python interpreter running this function should be the
         one from the virtualenv. If it is the system Python or if the
         environment is not configured properly, packages could be installed
@@ -312,13 +302,78 @@ class VirtualenvManager(VirtualenvHelper):
         """
         import distutils.sysconfig
 
-        # We ignore environment variables that may have been altered by
+        thunderbird_dir = os.path.join(self.topsrcdir, "comm")
+        is_thunderbird = os.path.exists(thunderbird_dir) and bool(
+            os.listdir(thunderbird_dir)
+        )
+        python_lib = distutils.sysconfig.get_python_lib()
+
+        def handle_package(action, package):
+            if action == "packages.txt":
+                src = os.path.join(self.topsrcdir, package)
+                assert os.path.isfile(src), "'%s' does not exist" % src
+                submanager = VirtualenvManager(
+                    self.topsrcdir,
+                    self.virtualenv_root,
+                    self.log_handle,
+                    src,
+                    populate_local_paths=self.populate_local_paths,
+                )
+                submanager.populate()
+            elif action == "pth":
+                if not self.populate_local_paths:
+                    return
+
+                path = os.path.join(self.topsrcdir, package)
+
+                with open(os.path.join(python_lib, "mach.pth"), "a") as f:
+                    # This path is relative to the .pth file.  Using a
+                    # relative path allows the srcdir/objdir combination
+                    # to be moved around (as long as the paths relative to
+                    # each other remain the same).
+                    f.write("%s\n" % os.path.relpath(path, python_lib))
+            elif action == "thunderbird":
+                if is_thunderbird:
+                    handle_package(*package.split(":", maxsplit=1))
+            elif action == "pypi":
+                if len(package.split("==")) != 2:
+                    raise Exception(
+                        "Expected pypi package version to be pinned in the "
+                        'format "package==version", found "{}"'.format(package)
+                    )
+                self.install_pip_package(package)
+            else:
+                raise Exception("Unknown action: %s" % action)
+
+        # We always target the OS X deployment target that Python itself was
+        # built with, regardless of what's in the current environment. If we
+        # don't do # this, we may run into a Python bug. See
+        # http://bugs.python.org/issue9516 and bug 659881.
+        #
+        # Note that this assumes that nothing compiled in the virtualenv is
+        # shipped as part of a distribution. If we do ship anything, the
+        # deployment target here may be different from what's targeted by the
+        # shipping binaries and # virtualenv-produced binaries may fail to
+        # work.
+        #
+        # We also ignore environment variables that may have been altered by
         # configure or a mozconfig activated in the current shell. We trust
         # Python is smart enough to find a proper compiler and to use the
         # proper compiler flags. If it isn't your Python is likely broken.
         IGNORE_ENV_VARIABLES = ("CC", "CXX", "CFLAGS", "CXXFLAGS", "LDFLAGS")
 
         try:
+            old_target = os.environ.get("MACOSX_DEPLOYMENT_TARGET", None)
+            sysconfig_target = distutils.sysconfig.get_config_var(
+                "MACOSX_DEPLOYMENT_TARGET"
+            )
+
+            if sysconfig_target is not None:
+                # MACOSX_DEPLOYMENT_TARGET is usually a string (e.g.: "10.14.6"), but
+                # in some cases it is an int (e.g.: 11). Since environment variables
+                # must all be str, explicitly convert it.
+                os.environ["MACOSX_DEPLOYMENT_TARGET"] = str(sysconfig_target)
+
             old_env_variables = {}
             for k in IGNORE_ENV_VARIABLES:
                 if k not in os.environ:
@@ -327,31 +382,15 @@ class VirtualenvManager(VirtualenvHelper):
                 old_env_variables[k] = os.environ[k]
                 del os.environ[k]
 
-            env_requirements = self._requirements()
-            if self.populate_local_paths:
-                python_lib = distutils.sysconfig.get_python_lib()
-                with open(os.path.join(python_lib, PTH_FILENAME), "a") as f:
-                    for pth_requirement in env_requirements.pth_requirements:
-                        path = os.path.join(self.topsrcdir, pth_requirement.path)
-                        # This path is relative to the .pth file.  Using a
-                        # relative path allows the srcdir/objdir combination
-                        # to be moved around (as long as the paths relative to
-                        # each other remain the same).
-                        f.write("{}\n".format(os.path.relpath(path, python_lib)))
-
-            for pypi_requirement in env_requirements.pypi_requirements:
-                self.install_pip_package(pypi_requirement.full_specifier)
-
-            for requirement in env_requirements.pypi_optional_requirements:
-                try:
-                    self.install_pip_package(requirement.full_specifier)
-                except subprocess.CalledProcessError:
-                    print(
-                        f"Could not install {requirement.package_name}, so "
-                        f"{requirement.repercussion}. Continuing."
-                    )
+            for current_action, current_package in self.packages():
+                handle_package(current_action, current_package)
 
         finally:
+            os.environ.pop("MACOSX_DEPLOYMENT_TARGET", None)
+
+            if old_target is not None:
+                os.environ["MACOSX_DEPLOYMENT_TARGET"] = old_target
+
             os.environ.update(old_env_variables)
 
     def call_setup(self, directory, arguments):
@@ -447,6 +486,9 @@ class VirtualenvManager(VirtualenvHelper):
         If vendored is True, no package index will be used and no dependencies
         will be installed.
         """
+        import mozfile
+        from mozfile import TemporaryDirectory
+
         if sys.executable.startswith(self.bin_path):
             # If we're already running in this interpreter, we can optimize in
             # the case that the package requirement is already satisfied.
@@ -489,7 +531,7 @@ class VirtualenvManager(VirtualenvHelper):
                     tmp, "{}-1.0-py3-none-any.whl".format(os.path.basename(package))
                 )
                 shutil.make_archive(wheel_file, "zip", package)
-                shutil.move("{}.zip".format(wheel_file), wheel_file)
+                mozfile.move("{}.zip".format(wheel_file), wheel_file)
                 package = wheel_file
 
             args.append(package)
@@ -543,7 +585,22 @@ class VirtualenvManager(VirtualenvHelper):
 
         https://github.com/pypa/pip/blob/5ee933aab81273da3691c97f2a6e7016ecbe0ef9/src/pip/_internal/self_outdated_check.py#L100-L101 # noqa F401
         """
-        site_packages = self._site_packages_dir()
+
+        # Defer "distutils" import until this function is called so that
+        # "mach bootstrap" doesn't fail due to Linux distro python-distutils
+        # package not being installed.
+        # By the time this function is called, "distutils" must be installed
+        # because it's needed by the "virtualenv" package.
+        from distutils import dist
+
+        distribution = dist.Distribution({"script_args": "--no-user-cfg"})
+        installer = distribution.get_command_obj("install")
+        installer.prefix = os.path.normpath(self.virtualenv_root)
+        installer.finalize_options()
+
+        # Path to virtualenv's "site-packages" directory
+        site_packages = installer.install_purelib
+
         pip_dist_info = next(
             (
                 file
@@ -576,22 +633,6 @@ class VirtualenvManager(VirtualenvHelper):
             [pip] + args, cwd=self.topsrcdir, env=env, universal_newlines=True, **kwargs
         )
 
-    def _site_packages_dir(self):
-        # Defer "distutils" import until this function is called so that
-        # "mach bootstrap" doesn't fail due to Linux distro python-distutils
-        # package not being installed.
-        # By the time this function is called, "distutils" must be installed
-        # because it's needed by the "virtualenv" package.
-        from distutils import dist
-
-        distribution = dist.Distribution({"script_args": "--no-user-cfg"})
-        installer = distribution.get_command_obj("install")
-        installer.prefix = os.path.normpath(self.virtualenv_root)
-        installer.finalize_options()
-
-        # Path to virtualenv's "site-packages" directory
-        return installer.install_purelib
-
 
 def get_archflags():
     # distutils will use the architecture of the running Python instance when building packages.