diff --git a/build/debian-packages/python-zstandard-wheezy.diff b/build/debian-packages/python-zstandard-wheezy.diff index 0d3eb521dc70..ea4f34377b60 100644 --- a/build/debian-packages/python-zstandard-wheezy.diff +++ b/build/debian-packages/python-zstandard-wheezy.diff @@ -3,11 +3,11 @@ index 84028db..d6c86c4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ -+python-zstandard (0.9.1-1.deb7moz1) unstable; urgency=low ++python-zstandard (0.11.1-1.deb7moz1) wheezy; urgency=low + + * Remove build dependencies so package builds on Wheezy. + -+ -- Gregory Szorc Mon, 04 Jun 2018 22:35:00 -0700 ++ -- Mike Hommey Thu, 25 Jul 2019 14:43:17 +0900 + python-zstandard (0.9.1-1) unstable; urgency=low diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 7e2cdb3a1915..58cb67f0846c 100755 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -15,7 +15,7 @@ treeherder: 'py3': 'Python 3 unit tests' 'A': 'Android Gradle tests' 'Bpgo': 'Profile-guided optimization builds' - 'Fetch-URL': 'Fetch and store content' + 'Fetch': 'Fetch and store content' 'Fxfn-l': 'Firefox functional tests (local)' 'Fxfn-l-1proc': 'Firefox functional tests (local) without e10s' 'Fxfn-r': 'Firefox functional tests (remote)' diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index 07b13b6119b9..b437dbf05f52 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -195,7 +195,9 @@ jobs: parent: debian9-base fetch: symbol: I(fetch) - parent: debian9-raw + parent: debian10-raw + packages: + - deb10-python-zstandard static-analysis-build: symbol: I(static-analysis-build) parent: android-build diff --git a/taskcluster/ci/fetch/toolchains.yml b/taskcluster/ci/fetch/toolchains.yml index 9091383608fb..285659b2e5e0 100644 --- a/taskcluster/ci/fetch/toolchains.yml +++ b/taskcluster/ci/fetch/toolchains.yml @@ -240,3 +240,31 @@ wine-3.0.3: gpg-signature: sig-url: "{url}.sign" key-path: build/unix/build-gcc/DA23579A74D4AD9AF9D3F945CEFAC8EAAF17519D.key + +cbindgen-0.9.0: + description: cbindgen source code + fetch: + type: git + repo: https://github.com/eqrion/cbindgen + revision: e19526e00b3fe6921b881682147a1fe5d6b64124 + +cctools-port: + description: cctools-port source code + fetch: + type: git + repo: https://github.com/tpoechtrager/cctools-port + revision: 3f979bbcd7ee29d79fb93f829edf3d1d16441147 + +libtapi: + description: libtapi source code + fetch: + type: git + repo: https://github.com/tpoechtrager/apple-libtapi + revision: 3efb201881e7a76a21e0554906cf306432539cef + +libdmg-hfsplus: + description: libdmg-hfsplug source code + fetch: + type: git + repo: https://github.com/mozilla/libdmg-hfsplus/ + revision: 2ee327795680101d36f9700bd0fb618362237718 diff --git a/taskcluster/ci/packages/kind.yml b/taskcluster/ci/packages/kind.yml index 94a424300a10..808b642267bc 100644 --- a/taskcluster/ci/packages/kind.yml +++ b/taskcluster/ci/packages/kind.yml @@ -284,8 +284,9 @@ jobs: run: using: debian-package tarball: - url: https://github.com/indygreg/python-zstandard/releases/download/0.9.1/python-zstandard-0.9.1.tar.gz - sha256: 59c7d6f1f85cebb5124abb50d8ec281c5311e0812e18785e28b197cf1515dd3b + url: https://github.com/indygreg/python-zstandard/archive/0.11.1.tar.gz + sha256: 2df90698e22f0da49cdd360438479f363e58eb8c3ac6e8bac84d831cf8e7baf4 + name: python-zstandard patch: python-zstandard-wheezy.diff packages: - deb7-dh-python @@ -301,8 +302,10 @@ jobs: using: debian-package dist: stretch tarball: - url: https://github.com/indygreg/python-zstandard/releases/download/0.9.1/python-zstandard-0.9.1.tar.gz - sha256: 59c7d6f1f85cebb5124abb50d8ec281c5311e0812e18785e28b197cf1515dd3b + url: https://github.com/indygreg/python-zstandard/archive/0.11.1.tar.gz + sha256: 2df90698e22f0da49cdd360438479f363e58eb8c3ac6e8bac84d831cf8e7baf4 + name: python-zstandard + pre-build-command: debchange -v 0.11.1-1.deb9moz --distribution stretch "Mozilla backport for stretch" deb10-python-zstandard: description: "python-zstandard for Debian buster" @@ -312,8 +315,10 @@ jobs: using: debian-package dist: buster tarball: - url: https://github.com/indygreg/python-zstandard/releases/download/0.9.1/python-zstandard-0.9.1.tar.gz - sha256: 59c7d6f1f85cebb5124abb50d8ec281c5311e0812e18785e28b197cf1515dd3b + url: https://github.com/indygreg/python-zstandard/archive/0.11.1.tar.gz + sha256: 2df90698e22f0da49cdd360438479f363e58eb8c3ac6e8bac84d831cf8e7baf4 + name: python-zstandard + pre-build-command: debchange -v 0.11.1-1.deb10moz --distribution buster "Mozilla backport for buster" deb7-pcre3: description: "pcre3 8.31 for Debian Wheezy" diff --git a/taskcluster/docker/fetch/Dockerfile b/taskcluster/docker/fetch/Dockerfile index 678789f9b8f5..fa3b244f1188 100644 --- a/taskcluster/docker/fetch/Dockerfile +++ b/taskcluster/docker/fetch/Dockerfile @@ -11,11 +11,16 @@ RUN mkdir /builds && \ # Declare default working folder WORKDIR /builds/worker -RUN apt-get update && \ +# %ARG TASKCLUSTER_ROOT_URL +# %ARG DOCKER_IMAGE_PACKAGES +RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \ + apt-get update && \ apt-get install \ gnupg \ bzip2 \ + git \ python3-requests \ + python3-zstandard \ unzip # %include taskcluster/scripts/run-task diff --git a/taskcluster/docker/image_builder/requirements/py2.txt b/taskcluster/docker/image_builder/requirements/py2.txt index cbd986a3a808..9b290c99ca80 100644 --- a/taskcluster/docker/image_builder/requirements/py2.txt +++ b/taskcluster/docker/image_builder/requirements/py2.txt @@ -37,31 +37,31 @@ cffi==1.12.1 \ pycparser==2.19 \ --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \ # via cffi -zstandard==0.10.2 \ - --hash=sha256:08114ac056944e7f70c0faf99d0afbce08b078eacf8ee6698985654c7e725234 \ - --hash=sha256:087276799ddf3200b4724e3d6f57b11ba975d9243b4af9e95721397d795a2497 \ - --hash=sha256:0c21feac9f7c850a457b1c707c3cc4f3b8f475a3c9120f8cec82ebc3b215b80a \ - --hash=sha256:0fe6403a01e996a7247239691101148dc4071ccf7fe12b680d7b6c91a04aefbb \ - --hash=sha256:1383412acd5356ff543c434723f2e7794c77e1ed4efc1062464cc2112c09af50 \ - --hash=sha256:2acd18eeac4fcecef8c1b95d4ffaa606222aa1ba0d4372e829dc516b0504e6ef \ - --hash=sha256:302bd7b3bc7281015cd6f975207755c534551d0a32c79147518f2de0459dbef4 \ - --hash=sha256:390acfced0106fb12247e12c2aa399836e6686f5ba9daec332957ff830f215cd \ - --hash=sha256:43ec51075547d498ec6e7952e459c3817e610d6e4ca68f4fa43a16ccea01d496 \ - --hash=sha256:53f89a65d52d6fb56b2c5dd0445f30ca25852f344ba20de325ce6767dd842fca \ - --hash=sha256:5f4f650b83b8085862de9e555d87f6053ca577b4070f4c6610a870116c4dd1f4 \ - --hash=sha256:72ef2361d90a717457376351acb5b1b0c189a09dbd95adcb51907a96b79a6add \ - --hash=sha256:7ef5c7ede8e8cda2a37c0ecab456f4cfae2c42049f51b24edb5303dbfe318ea6 \ - --hash=sha256:86c9dee0fe6d4ea5bf394767929fdf5f924d161d9a6d23adcd58a690c5e160b0 \ - --hash=sha256:8b587c9a17f4b050274d9b7f9284d5fae0a8d6a8021f88f779345593326bc33d \ - --hash=sha256:91025801859a60b7761dea6a8b645f25be6d3639ef828423f094d90b3f60850e \ - --hash=sha256:9d2940e2801cc768d2cb71e71dca3b025ca3737e9d1d0fad0c95b2e7db0c947a \ - --hash=sha256:aa520b90eede823632013a319e91652d8226a6309a104cffdc7e00d5a2b5e66b \ - --hash=sha256:b10fba39049595827f228e77e7b5070cb39c46466bf8fef51da73220a20cc717 \ - --hash=sha256:c794b5c21485fb3232f5693995ba1a497267b1aecb70b218107cf131f8dc1d3d \ - --hash=sha256:d05516bc197c5b7b2aa2f834ea7c5ee9fd9aa3034f4193cc05d899b18251aa9c \ - --hash=sha256:d085c2c676f03357e5d6b11dbbf4e8c1b0d20b1066ac87e6cccc45d4b6c19675 \ - --hash=sha256:dd40e26aaee67b9078618b0fce3d5f209e328852f2c72c6772cf6352f57d2ed1 \ - --hash=sha256:e7b84c10ed30c1c997d81ef271945372fba9e18ac58d77a17d43fd9c42392ed4 \ - --hash=sha256:e982d8af9618d45b25456f1f80e6d628295772d74d755f9a46b90711b7a56067 \ - --hash=sha256:ef24c8ec97f93b2bdf1080553cdf38ea9ab195846b679cdcfe683c945ed2f1ee \ - --hash=sha256:f46c5021c3663f82c2ff994295a8574638d56a831ca2a26d736d47fbcf4f9187 +zstandard==0.11.1 \ + --hash=sha256:19f5ad81590acd20dbdfb930b87a035189778662fdc67ab8cbcc106269ed1be8 \ + --hash=sha256:1a1db0c9774181e806a418c32d511aa085c7e2c28c257a58f6c107f5decb3109 \ + --hash=sha256:22d7aa898f36f78108cc1ef0c8da8225f0add518441d815ad4fdd1d577378209 \ + --hash=sha256:357873afdd7cd0e653d169c36ce837ce2b3e5926dd4a5c0f0476c813f6765373 \ + --hash=sha256:3c31da5d78a7b07e722e8a3e0b1295bc9b316b7e90a1666659c451a42750ffe4 \ + --hash=sha256:3f76562ec63fabc6f4b5be0cd986f911c97105c35c31b4d655b90c4d2fe07f40 \ + --hash=sha256:42fa4462e0563fe17e73dfeb95eef9b00429b86282f8f6ca0e2765b1855a8324 \ + --hash=sha256:51aad01a5709ca6f45768c69ffd4c887528e5ad9e09302426b735560752c4e82 \ + --hash=sha256:6cd81819a02e57e38e27c53c5c0a7015e059b0e148a18bf27b46b4f808840879 \ + --hash=sha256:717fd2494f222164396e03d08ef57174d2a889920b81ca49f276caf9381e6405 \ + --hash=sha256:71c8711458212c973a9b719275db8111f22803e0caf675affde50703b96e9be1 \ + --hash=sha256:76a331b5a6258fce3906551557db9be83bdd89a62f66f509a55a4a307239c782 \ + --hash=sha256:7c92dfcdf7e0c540f9718b40b4c54516a968ef6b81567b75df81866a1af2189d \ + --hash=sha256:7f3db21223a8bb4ffcf6c36b9c20d38278967723b47fce249dcb6ec6d4082b83 \ + --hash=sha256:7fa9deba4c904e76870e08324adff94ec3a4bc56a50bbe1a9f859a4aed11c0d2 \ + --hash=sha256:88912cbcf68cc40037c113460a166ebfbbb24864ceebb89ad221ea346f22e995 \ + --hash=sha256:94aa5bb817f1c747b21214f6ef83a022bcb63bf81e4dae2954768165c13a510b \ + --hash=sha256:951e382a2ea47179ecb3e314e8c70f2e5189e3652ccbbcb71c6443dd71bc20fc \ + --hash=sha256:978a500ae1184f602dc902977ec208c7cf02c10caae9c159b10976a7cb29f879 \ + --hash=sha256:991c4a40171d87854b219cdf2ba56c1c34b3b3a8ebe5d1ab63bd357ff71271b2 \ + --hash=sha256:9ca84187182743d2e6bbf9d3f79d3834db205cddc98add27ad20f2189d080a60 \ + --hash=sha256:ae50bc839cf1ff549f55a3e55922563f246fb692f77497175a8d8d4cddc294da \ + --hash=sha256:b7abae5b17e82d5f78aaa641077b4619c6ad204e30c6f3445d422acff5f35d3e \ + --hash=sha256:b8fce0c961654f77c81a6ae1f2cd40633b41ef16a12ae02f0382ed6692f9bb90 \ + --hash=sha256:d8f047d3647a5cd1b77b4580f35208c938da00c101a092571c85bcefaa2d725d \ + --hash=sha256:f1785b31bf428e964a9670dd4f721023f2741ef7fd67c663bf01e3d4d3f9ec2a \ + --hash=sha256:fcf70e1e9d38035a15482e954ba064f3b701cf84cfe571576d15af93ac2a2fb1 diff --git a/taskcluster/docker/image_builder/requirements/py3.txt b/taskcluster/docker/image_builder/requirements/py3.txt index 1da0def30d57..8389ac886f72 100644 --- a/taskcluster/docker/image_builder/requirements/py3.txt +++ b/taskcluster/docker/image_builder/requirements/py3.txt @@ -74,31 +74,31 @@ websocket-client==0.54.0 \ --hash=sha256:8c8bf2d4f800c3ed952df206b18c28f7070d9e3dcbd6ca6291127574f57ee786 \ --hash=sha256:e51562c91ddb8148e791f0155fdb01325d99bb52c4cdbb291aee7a3563fd0849 \ # via docker -zstandard==0.10.2 \ - --hash=sha256:08114ac056944e7f70c0faf99d0afbce08b078eacf8ee6698985654c7e725234 \ - --hash=sha256:087276799ddf3200b4724e3d6f57b11ba975d9243b4af9e95721397d795a2497 \ - --hash=sha256:0c21feac9f7c850a457b1c707c3cc4f3b8f475a3c9120f8cec82ebc3b215b80a \ - --hash=sha256:0fe6403a01e996a7247239691101148dc4071ccf7fe12b680d7b6c91a04aefbb \ - --hash=sha256:1383412acd5356ff543c434723f2e7794c77e1ed4efc1062464cc2112c09af50 \ - --hash=sha256:2acd18eeac4fcecef8c1b95d4ffaa606222aa1ba0d4372e829dc516b0504e6ef \ - --hash=sha256:302bd7b3bc7281015cd6f975207755c534551d0a32c79147518f2de0459dbef4 \ - --hash=sha256:390acfced0106fb12247e12c2aa399836e6686f5ba9daec332957ff830f215cd \ - --hash=sha256:43ec51075547d498ec6e7952e459c3817e610d6e4ca68f4fa43a16ccea01d496 \ - --hash=sha256:53f89a65d52d6fb56b2c5dd0445f30ca25852f344ba20de325ce6767dd842fca \ - --hash=sha256:5f4f650b83b8085862de9e555d87f6053ca577b4070f4c6610a870116c4dd1f4 \ - --hash=sha256:72ef2361d90a717457376351acb5b1b0c189a09dbd95adcb51907a96b79a6add \ - --hash=sha256:7ef5c7ede8e8cda2a37c0ecab456f4cfae2c42049f51b24edb5303dbfe318ea6 \ - --hash=sha256:86c9dee0fe6d4ea5bf394767929fdf5f924d161d9a6d23adcd58a690c5e160b0 \ - --hash=sha256:8b587c9a17f4b050274d9b7f9284d5fae0a8d6a8021f88f779345593326bc33d \ - --hash=sha256:91025801859a60b7761dea6a8b645f25be6d3639ef828423f094d90b3f60850e \ - --hash=sha256:9d2940e2801cc768d2cb71e71dca3b025ca3737e9d1d0fad0c95b2e7db0c947a \ - --hash=sha256:aa520b90eede823632013a319e91652d8226a6309a104cffdc7e00d5a2b5e66b \ - --hash=sha256:b10fba39049595827f228e77e7b5070cb39c46466bf8fef51da73220a20cc717 \ - --hash=sha256:c794b5c21485fb3232f5693995ba1a497267b1aecb70b218107cf131f8dc1d3d \ - --hash=sha256:d05516bc197c5b7b2aa2f834ea7c5ee9fd9aa3034f4193cc05d899b18251aa9c \ - --hash=sha256:d085c2c676f03357e5d6b11dbbf4e8c1b0d20b1066ac87e6cccc45d4b6c19675 \ - --hash=sha256:dd40e26aaee67b9078618b0fce3d5f209e328852f2c72c6772cf6352f57d2ed1 \ - --hash=sha256:e7b84c10ed30c1c997d81ef271945372fba9e18ac58d77a17d43fd9c42392ed4 \ - --hash=sha256:e982d8af9618d45b25456f1f80e6d628295772d74d755f9a46b90711b7a56067 \ - --hash=sha256:ef24c8ec97f93b2bdf1080553cdf38ea9ab195846b679cdcfe683c945ed2f1ee \ - --hash=sha256:f46c5021c3663f82c2ff994295a8574638d56a831ca2a26d736d47fbcf4f9187 +zstandard==0.11.1 \ + --hash=sha256:19f5ad81590acd20dbdfb930b87a035189778662fdc67ab8cbcc106269ed1be8 \ + --hash=sha256:1a1db0c9774181e806a418c32d511aa085c7e2c28c257a58f6c107f5decb3109 \ + --hash=sha256:22d7aa898f36f78108cc1ef0c8da8225f0add518441d815ad4fdd1d577378209 \ + --hash=sha256:357873afdd7cd0e653d169c36ce837ce2b3e5926dd4a5c0f0476c813f6765373 \ + --hash=sha256:3c31da5d78a7b07e722e8a3e0b1295bc9b316b7e90a1666659c451a42750ffe4 \ + --hash=sha256:3f76562ec63fabc6f4b5be0cd986f911c97105c35c31b4d655b90c4d2fe07f40 \ + --hash=sha256:42fa4462e0563fe17e73dfeb95eef9b00429b86282f8f6ca0e2765b1855a8324 \ + --hash=sha256:51aad01a5709ca6f45768c69ffd4c887528e5ad9e09302426b735560752c4e82 \ + --hash=sha256:6cd81819a02e57e38e27c53c5c0a7015e059b0e148a18bf27b46b4f808840879 \ + --hash=sha256:717fd2494f222164396e03d08ef57174d2a889920b81ca49f276caf9381e6405 \ + --hash=sha256:71c8711458212c973a9b719275db8111f22803e0caf675affde50703b96e9be1 \ + --hash=sha256:76a331b5a6258fce3906551557db9be83bdd89a62f66f509a55a4a307239c782 \ + --hash=sha256:7c92dfcdf7e0c540f9718b40b4c54516a968ef6b81567b75df81866a1af2189d \ + --hash=sha256:7f3db21223a8bb4ffcf6c36b9c20d38278967723b47fce249dcb6ec6d4082b83 \ + --hash=sha256:7fa9deba4c904e76870e08324adff94ec3a4bc56a50bbe1a9f859a4aed11c0d2 \ + --hash=sha256:88912cbcf68cc40037c113460a166ebfbbb24864ceebb89ad221ea346f22e995 \ + --hash=sha256:94aa5bb817f1c747b21214f6ef83a022bcb63bf81e4dae2954768165c13a510b \ + --hash=sha256:951e382a2ea47179ecb3e314e8c70f2e5189e3652ccbbcb71c6443dd71bc20fc \ + --hash=sha256:978a500ae1184f602dc902977ec208c7cf02c10caae9c159b10976a7cb29f879 \ + --hash=sha256:991c4a40171d87854b219cdf2ba56c1c34b3b3a8ebe5d1ab63bd357ff71271b2 \ + --hash=sha256:9ca84187182743d2e6bbf9d3f79d3834db205cddc98add27ad20f2189d080a60 \ + --hash=sha256:ae50bc839cf1ff549f55a3e55922563f246fb692f77497175a8d8d4cddc294da \ + --hash=sha256:b7abae5b17e82d5f78aaa641077b4619c6ad204e30c6f3445d422acff5f35d3e \ + --hash=sha256:b8fce0c961654f77c81a6ae1f2cd40633b41ef16a12ae02f0382ed6692f9bb90 \ + --hash=sha256:d8f047d3647a5cd1b77b4580f35208c938da00c101a092571c85bcefaa2d725d \ + --hash=sha256:f1785b31bf428e964a9670dd4f721023f2741ef7fd67c663bf01e3d4d3f9ec2a \ + --hash=sha256:fcf70e1e9d38035a15482e954ba064f3b701cf84cfe571576d15af93ac2a2fb1 diff --git a/taskcluster/scripts/misc/fetch-content b/taskcluster/scripts/misc/fetch-content index 00248ca58ed7..e73ad1d050b0 100755 --- a/taskcluster/scripts/misc/fetch-content +++ b/taskcluster/scripts/misc/fetch-content @@ -342,6 +342,59 @@ def fetch_urls(downloads): f.result() +def git_checkout_archive(dest_path: pathlib.Path, repo: str, commit: str, + prefix=None): + """Produce an archive of the files comprising a Git checkout.""" + dest_path.parent.mkdir(parents=True, exist_ok=True) + + if dest_path.suffixes[-2:] != ['.tar', '.zst']: + raise Exception('Only producing .tar.zst archives is supported.') + + if not zstandard: + raise ValueError('zstandard Python package not available') + + with tempfile.TemporaryDirectory() as td: + temp_dir = pathlib.Path(td) + + git_dir = temp_dir / 'git_dir' + if not prefix: + prefix = repo.rstrip('/').rsplit('/', 1)[-1] + + # This could be faster with a shallow clone. However, Git requires a ref + # to initiate a clone. Since the commit-ish may not refer to a ref, we + # simply perform a full clone followed by a checkout. + print('cloning %s to %s' % (repo, git_dir)) + subprocess.run(['git', 'clone', '--bare', repo, str(git_dir)], + check=True) + + print('creating archive %s of commit %s' % (dest_path, commit)) + proc = subprocess.Popen([ + 'git', '--git-dir={}'.format(git_dir), 'archive', + '--format=tar', '--prefix=%s/' % prefix, commit + ], stdout=subprocess.PIPE) + + with open(dest_path, 'wb') as out: + ctx = zstandard.ZstdCompressor() + ctx.copy_stream(proc.stdout, out) + + proc.wait() + + +def command_git_checkout_archive(args): + dest = pathlib.Path(args.dest) + + try: + git_checkout_archive(dest, args.repo, args.commit, + prefix=args.path_prefix) + except Exception: + try: + dest.unlink() + except FileNotFoundError: + pass + + raise + + def command_static_url(args): gpg_sig_url = args.gpg_sig_url gpg_env_key = args.gpg_key_env @@ -409,6 +462,19 @@ def main(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(title='sub commands') + git_checkout = subparsers.add_parser( + 'git-checkout-archive', + help='Obtain an archive of files from a Git repository checkout') + git_checkout.set_defaults(func=command_git_checkout_archive) + git_checkout.add_argument('--path-prefix', + help='Prefix for paths in produced archive') + git_checkout.add_argument('repo', + help='URL to Git repository to be cloned') + git_checkout.add_argument('commit', + help='Git commit to check out') + git_checkout.add_argument('dest', + help='Destination path of archive') + url = subparsers.add_parser('static-url', help='Download a static URL') url.set_defaults(func=command_static_url) url.add_argument('--sha256', required=True, diff --git a/taskcluster/taskgraph/transforms/fetch.py b/taskcluster/taskgraph/transforms/fetch.py index f4490621b60f..521c06a93bbe 100644 --- a/taskcluster/taskgraph/transforms/fetch.py +++ b/taskcluster/taskgraph/transforms/fetch.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, unicode_literals from mozbuild.shellutil import quote as shell_quote import os +import re from voluptuous import ( Any, @@ -89,6 +90,13 @@ FETCH_SCHEMA = Schema({ # The name to give to the generated artifact. Required('artifact-name'): basestring + }, + { + 'type': 'git', + Required('repo'): basestring, + Required('revision'): basestring, + Optional('artifact-name'): basestring, + Optional('path-prefix'): basestring, } ), }) @@ -110,6 +118,8 @@ def process_fetch_job(config, jobs): yield create_fetch_url_task(config, job) elif typ == 'chromium-fetch': yield create_chromium_fetch_task(config, job) + elif typ == 'git': + yield create_git_fetch_task(config, job) else: # validate() should have caught this. assert False @@ -185,7 +195,7 @@ def create_fetch_url_task(config, job): ]) task = make_base_task(config, name, job['description'], args) - task['treeherder']['symbol'] = join_symbol('Fetch-URL', name) + task['treeherder']['symbol'] = join_symbol('Fetch', name) task['worker']['artifacts'] = [{ 'type': 'directory', 'name': 'public', @@ -212,6 +222,54 @@ def create_fetch_url_task(config, job): return task +def create_git_fetch_task(config, job): + name = job['name'] + fetch = job['fetch'] + path_prefix = fetch.get('path-prefix') + if not path_prefix: + path_prefix = fetch['repo'].rstrip('/').rsplit('/', 1)[-1] + artifact_name = fetch.get('artifact-name') + if not artifact_name: + artifact_name = '{}.tar.zst'.format(path_prefix) + + if not re.match(r'[0-9a-fA-F]{40}', fetch['revision']): + raise Exception( + 'Revision is not a sha1 in fetch task "{}"'.format(name)) + + args = [ + '/builds/worker/bin/fetch-content', + 'git-checkout-archive', + '--path-prefix', + path_prefix, + fetch['repo'], + fetch['revision'], + '/builds/worker/artifacts/%s' % artifact_name, + ] + + task = make_base_task(config, name, job['description'], args) + task['treeherder']['symbol'] = join_symbol('Fetch', name) + task['worker']['artifacts'] = [{ + 'type': 'directory', + 'name': 'public', + 'path': '/builds/worker/artifacts', + }] + task['attributes']['fetch-artifact'] = 'public/%s' % artifact_name + + if not taskgraph.fast: + cache_name = task['label'].replace('{}-'.format(config.kind), '', 1) + + # This adds the level to the index path automatically. + add_optimization( + config, + task, + cache_type=CACHE_TYPE, + cache_name=cache_name, + digest_data=[fetch['revision'], path_prefix, artifact_name], + ) + + return task + + def create_chromium_fetch_task(config, job): name = job['name'] fetch = job['fetch'] @@ -240,7 +298,7 @@ def create_chromium_fetch_task(config, job): } task = make_base_task(config, name, job['description'], cmd) - task['treeherder']['symbol'] = join_symbol('Fetch-URL', name) + task['treeherder']['symbol'] = join_symbol('Fetch', name) task['worker']['artifacts'] = [{ 'type': 'directory', 'name': 'public', diff --git a/taskcluster/taskgraph/transforms/job/debian_package.py b/taskcluster/taskgraph/transforms/job/debian_package.py index cc35db966691..bdc523ebb2a8 100644 --- a/taskcluster/taskgraph/transforms/job/debian_package.py +++ b/taskcluster/taskgraph/transforms/job/debian_package.py @@ -43,6 +43,11 @@ run_schema = Schema({ # control (.dsc), or a tarball. Required(Any('dsc', 'tarball')): source_definition, + # Package name. Normally derived from the source control or tarball file + # name. Use in case the name doesn't match DSC_PACKAGE_RE or + # SOURCE_PACKAGE_RE. + Optional('name'): basestring, + # Patch to apply to the extracted source. Optional('patch'): basestring, @@ -110,7 +115,9 @@ def docker_worker_debian_package(config, job, taskdesc): src_url = src['url'] src_file = os.path.basename(src_url) src_sha256 = src['sha256'] - package = package_re.match(src_file).group(0) + package = run.get('name') + if not package: + package = package_re.match(src_file).group(0) unpack = unpack.format(src_file=src_file, package=package) resolver = run.get('resolver', 'apt-get')