Merge mozilla-central to autoland on a CLOSED TREE

2019-07-30 13:37:01 +03:00 · 2019-07-30 13:37:01 +03:00 · 63aab19d58
--- a/build/debian-packages/python-zstandard-wheezy.diff
+++ b/build/debian-packages/python-zstandard-wheezy.diff
@ -3,11 +3,11 @@ index 84028db..d6c86c4 100644
 --- a/debian/changelog
 +++ b/debian/changelog
@@ -1,3 +1,9 @@
-+python-zstandard (0.9.1-1.deb7moz1) unstable; urgency=low
+python-zstandard (0.11.1-1.deb7moz1) wheezy; urgency=low
 +
 +  * Remove build dependencies so package builds on Wheezy.
 +
-+ -- Gregory Szorc <gps@mozilla.com>  Mon, 04 Jun 2018 22:35:00 -0700
+ -- Mike Hommey <glandium@mozilla.com>  Thu, 25 Jul 2019 14:43:17 +0900
 +
 python-zstandard (0.9.1-1) unstable; urgency=low

--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@ -15,7 +15,7 @@ treeherder:
        'py3': 'Python 3 unit tests'
        'A': 'Android Gradle tests'
        'Bpgo': 'Profile-guided optimization builds'
-        'Fetch-URL': 'Fetch and store content'
+        'Fetch': 'Fetch and store content'
        'Fxfn-l': 'Firefox functional tests (local)'
        'Fxfn-l-1proc': 'Firefox functional tests (local) without e10s'
        'Fxfn-r': 'Firefox functional tests (remote)'
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@ -195,7 +195,9 @@ jobs:
        parent: debian9-base
    fetch:
        symbol: I(fetch)
-        parent: debian9-raw
+        parent: debian10-raw
+        packages:
+            - deb10-python-zstandard
    static-analysis-build:
        symbol: I(static-analysis-build)
        parent: android-build
--- a/taskcluster/ci/fetch/toolchains.yml
+++ b/taskcluster/ci/fetch/toolchains.yml
@ -240,3 +240,31 @@ wine-3.0.3:
        gpg-signature:
            sig-url: "{url}.sign"
            key-path: build/unix/build-gcc/DA23579A74D4AD9AF9D3F945CEFAC8EAAF17519D.key
+
+cbindgen-0.9.0:
+    description: cbindgen source code
+    fetch:
+        type: git
+        repo: https://github.com/eqrion/cbindgen
+        revision: e19526e00b3fe6921b881682147a1fe5d6b64124
+
+cctools-port:
+    description: cctools-port source code
+    fetch:
+        type: git
+        repo: https://github.com/tpoechtrager/cctools-port
+        revision: 3f979bbcd7ee29d79fb93f829edf3d1d16441147
+
+libtapi:
+    description: libtapi source code
+    fetch:
+        type: git
+        repo: https://github.com/tpoechtrager/apple-libtapi
+        revision: 3efb201881e7a76a21e0554906cf306432539cef
+
+libdmg-hfsplus:
+    description: libdmg-hfsplug source code
+    fetch:
+        type: git
+        repo: https://github.com/mozilla/libdmg-hfsplus/
+        revision: 2ee327795680101d36f9700bd0fb618362237718
--- a/taskcluster/ci/packages/kind.yml
+++ b/taskcluster/ci/packages/kind.yml
@ -284,8 +284,9 @@ jobs:
        run:
            using: debian-package
            tarball:
-                url: https://github.com/indygreg/python-zstandard/releases/download/0.9.1/python-zstandard-0.9.1.tar.gz
-                sha256: 59c7d6f1f85cebb5124abb50d8ec281c5311e0812e18785e28b197cf1515dd3b
+                url: https://github.com/indygreg/python-zstandard/archive/0.11.1.tar.gz
+                sha256: 2df90698e22f0da49cdd360438479f363e58eb8c3ac6e8bac84d831cf8e7baf4
+            name: python-zstandard
            patch: python-zstandard-wheezy.diff
            packages:
                - deb7-dh-python
@ -301,8 +302,10 @@ jobs:
            using: debian-package
            dist: stretch
            tarball:
-                url: https://github.com/indygreg/python-zstandard/releases/download/0.9.1/python-zstandard-0.9.1.tar.gz
-                sha256: 59c7d6f1f85cebb5124abb50d8ec281c5311e0812e18785e28b197cf1515dd3b
+                url: https://github.com/indygreg/python-zstandard/archive/0.11.1.tar.gz
+                sha256: 2df90698e22f0da49cdd360438479f363e58eb8c3ac6e8bac84d831cf8e7baf4
+            name: python-zstandard
+            pre-build-command: debchange -v 0.11.1-1.deb9moz --distribution stretch "Mozilla backport for stretch"

    deb10-python-zstandard:
        description: "python-zstandard for Debian buster"
@ -312,8 +315,10 @@ jobs:
            using: debian-package
            dist: buster
            tarball:
-                url: https://github.com/indygreg/python-zstandard/releases/download/0.9.1/python-zstandard-0.9.1.tar.gz
-                sha256: 59c7d6f1f85cebb5124abb50d8ec281c5311e0812e18785e28b197cf1515dd3b
+                url: https://github.com/indygreg/python-zstandard/archive/0.11.1.tar.gz
+                sha256: 2df90698e22f0da49cdd360438479f363e58eb8c3ac6e8bac84d831cf8e7baf4
+            name: python-zstandard
+            pre-build-command: debchange -v 0.11.1-1.deb10moz --distribution buster "Mozilla backport for buster"

    deb7-pcre3:
        description: "pcre3 8.31 for Debian Wheezy"
--- a/taskcluster/docker/fetch/Dockerfile
+++ b/taskcluster/docker/fetch/Dockerfile
@ -11,11 +11,16 @@ RUN mkdir /builds && \
 # Declare default working folder
 WORKDIR /builds/worker

-RUN apt-get update && \
+# %ARG TASKCLUSTER_ROOT_URL
+# %ARG DOCKER_IMAGE_PACKAGES
+RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \
+    apt-get update && \
    apt-get install \
      gnupg \
      bzip2 \
+      git \
      python3-requests \
+      python3-zstandard \
      unzip

 # %include taskcluster/scripts/run-task
--- a/taskcluster/docker/image_builder/requirements/py2.txt
+++ b/taskcluster/docker/image_builder/requirements/py2.txt
@ -37,31 +37,31 @@ cffi==1.12.1 \
 pycparser==2.19 \
    --hash=sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3 \
    # via cffi
-zstandard==0.10.2 \
-    --hash=sha256:08114ac056944e7f70c0faf99d0afbce08b078eacf8ee6698985654c7e725234 \
-    --hash=sha256:087276799ddf3200b4724e3d6f57b11ba975d9243b4af9e95721397d795a2497 \
-    --hash=sha256:0c21feac9f7c850a457b1c707c3cc4f3b8f475a3c9120f8cec82ebc3b215b80a \
-    --hash=sha256:0fe6403a01e996a7247239691101148dc4071ccf7fe12b680d7b6c91a04aefbb \
-    --hash=sha256:1383412acd5356ff543c434723f2e7794c77e1ed4efc1062464cc2112c09af50 \
-    --hash=sha256:2acd18eeac4fcecef8c1b95d4ffaa606222aa1ba0d4372e829dc516b0504e6ef \
-    --hash=sha256:302bd7b3bc7281015cd6f975207755c534551d0a32c79147518f2de0459dbef4 \
-    --hash=sha256:390acfced0106fb12247e12c2aa399836e6686f5ba9daec332957ff830f215cd \
-    --hash=sha256:43ec51075547d498ec6e7952e459c3817e610d6e4ca68f4fa43a16ccea01d496 \
-    --hash=sha256:53f89a65d52d6fb56b2c5dd0445f30ca25852f344ba20de325ce6767dd842fca \
-    --hash=sha256:5f4f650b83b8085862de9e555d87f6053ca577b4070f4c6610a870116c4dd1f4 \
-    --hash=sha256:72ef2361d90a717457376351acb5b1b0c189a09dbd95adcb51907a96b79a6add \
-    --hash=sha256:7ef5c7ede8e8cda2a37c0ecab456f4cfae2c42049f51b24edb5303dbfe318ea6 \
-    --hash=sha256:86c9dee0fe6d4ea5bf394767929fdf5f924d161d9a6d23adcd58a690c5e160b0 \
-    --hash=sha256:8b587c9a17f4b050274d9b7f9284d5fae0a8d6a8021f88f779345593326bc33d \
-    --hash=sha256:91025801859a60b7761dea6a8b645f25be6d3639ef828423f094d90b3f60850e \
-    --hash=sha256:9d2940e2801cc768d2cb71e71dca3b025ca3737e9d1d0fad0c95b2e7db0c947a \
-    --hash=sha256:aa520b90eede823632013a319e91652d8226a6309a104cffdc7e00d5a2b5e66b \
-    --hash=sha256:b10fba39049595827f228e77e7b5070cb39c46466bf8fef51da73220a20cc717 \
-    --hash=sha256:c794b5c21485fb3232f5693995ba1a497267b1aecb70b218107cf131f8dc1d3d \
-    --hash=sha256:d05516bc197c5b7b2aa2f834ea7c5ee9fd9aa3034f4193cc05d899b18251aa9c \
-    --hash=sha256:d085c2c676f03357e5d6b11dbbf4e8c1b0d20b1066ac87e6cccc45d4b6c19675 \
-    --hash=sha256:dd40e26aaee67b9078618b0fce3d5f209e328852f2c72c6772cf6352f57d2ed1 \
-    --hash=sha256:e7b84c10ed30c1c997d81ef271945372fba9e18ac58d77a17d43fd9c42392ed4 \
-    --hash=sha256:e982d8af9618d45b25456f1f80e6d628295772d74d755f9a46b90711b7a56067 \
-    --hash=sha256:ef24c8ec97f93b2bdf1080553cdf38ea9ab195846b679cdcfe683c945ed2f1ee \
-    --hash=sha256:f46c5021c3663f82c2ff994295a8574638d56a831ca2a26d736d47fbcf4f9187
+zstandard==0.11.1 \
+    --hash=sha256:19f5ad81590acd20dbdfb930b87a035189778662fdc67ab8cbcc106269ed1be8 \
+    --hash=sha256:1a1db0c9774181e806a418c32d511aa085c7e2c28c257a58f6c107f5decb3109 \
+    --hash=sha256:22d7aa898f36f78108cc1ef0c8da8225f0add518441d815ad4fdd1d577378209 \
+    --hash=sha256:357873afdd7cd0e653d169c36ce837ce2b3e5926dd4a5c0f0476c813f6765373 \
+    --hash=sha256:3c31da5d78a7b07e722e8a3e0b1295bc9b316b7e90a1666659c451a42750ffe4 \
+    --hash=sha256:3f76562ec63fabc6f4b5be0cd986f911c97105c35c31b4d655b90c4d2fe07f40 \
+    --hash=sha256:42fa4462e0563fe17e73dfeb95eef9b00429b86282f8f6ca0e2765b1855a8324 \
+    --hash=sha256:51aad01a5709ca6f45768c69ffd4c887528e5ad9e09302426b735560752c4e82 \
+    --hash=sha256:6cd81819a02e57e38e27c53c5c0a7015e059b0e148a18bf27b46b4f808840879 \
+    --hash=sha256:717fd2494f222164396e03d08ef57174d2a889920b81ca49f276caf9381e6405 \
+    --hash=sha256:71c8711458212c973a9b719275db8111f22803e0caf675affde50703b96e9be1 \
+    --hash=sha256:76a331b5a6258fce3906551557db9be83bdd89a62f66f509a55a4a307239c782 \
+    --hash=sha256:7c92dfcdf7e0c540f9718b40b4c54516a968ef6b81567b75df81866a1af2189d \
+    --hash=sha256:7f3db21223a8bb4ffcf6c36b9c20d38278967723b47fce249dcb6ec6d4082b83 \
+    --hash=sha256:7fa9deba4c904e76870e08324adff94ec3a4bc56a50bbe1a9f859a4aed11c0d2 \
+    --hash=sha256:88912cbcf68cc40037c113460a166ebfbbb24864ceebb89ad221ea346f22e995 \
+    --hash=sha256:94aa5bb817f1c747b21214f6ef83a022bcb63bf81e4dae2954768165c13a510b \
+    --hash=sha256:951e382a2ea47179ecb3e314e8c70f2e5189e3652ccbbcb71c6443dd71bc20fc \
+    --hash=sha256:978a500ae1184f602dc902977ec208c7cf02c10caae9c159b10976a7cb29f879 \
+    --hash=sha256:991c4a40171d87854b219cdf2ba56c1c34b3b3a8ebe5d1ab63bd357ff71271b2 \
+    --hash=sha256:9ca84187182743d2e6bbf9d3f79d3834db205cddc98add27ad20f2189d080a60 \
+    --hash=sha256:ae50bc839cf1ff549f55a3e55922563f246fb692f77497175a8d8d4cddc294da \
+    --hash=sha256:b7abae5b17e82d5f78aaa641077b4619c6ad204e30c6f3445d422acff5f35d3e \
+    --hash=sha256:b8fce0c961654f77c81a6ae1f2cd40633b41ef16a12ae02f0382ed6692f9bb90 \
+    --hash=sha256:d8f047d3647a5cd1b77b4580f35208c938da00c101a092571c85bcefaa2d725d \
+    --hash=sha256:f1785b31bf428e964a9670dd4f721023f2741ef7fd67c663bf01e3d4d3f9ec2a \
+    --hash=sha256:fcf70e1e9d38035a15482e954ba064f3b701cf84cfe571576d15af93ac2a2fb1
--- a/taskcluster/docker/image_builder/requirements/py3.txt
+++ b/taskcluster/docker/image_builder/requirements/py3.txt
@ -74,31 +74,31 @@ websocket-client==0.54.0 \
    --hash=sha256:8c8bf2d4f800c3ed952df206b18c28f7070d9e3dcbd6ca6291127574f57ee786 \
    --hash=sha256:e51562c91ddb8148e791f0155fdb01325d99bb52c4cdbb291aee7a3563fd0849 \
    # via docker
-zstandard==0.10.2 \
-    --hash=sha256:08114ac056944e7f70c0faf99d0afbce08b078eacf8ee6698985654c7e725234 \
-    --hash=sha256:087276799ddf3200b4724e3d6f57b11ba975d9243b4af9e95721397d795a2497 \
-    --hash=sha256:0c21feac9f7c850a457b1c707c3cc4f3b8f475a3c9120f8cec82ebc3b215b80a \
-    --hash=sha256:0fe6403a01e996a7247239691101148dc4071ccf7fe12b680d7b6c91a04aefbb \
-    --hash=sha256:1383412acd5356ff543c434723f2e7794c77e1ed4efc1062464cc2112c09af50 \
-    --hash=sha256:2acd18eeac4fcecef8c1b95d4ffaa606222aa1ba0d4372e829dc516b0504e6ef \
-    --hash=sha256:302bd7b3bc7281015cd6f975207755c534551d0a32c79147518f2de0459dbef4 \
-    --hash=sha256:390acfced0106fb12247e12c2aa399836e6686f5ba9daec332957ff830f215cd \
-    --hash=sha256:43ec51075547d498ec6e7952e459c3817e610d6e4ca68f4fa43a16ccea01d496 \
-    --hash=sha256:53f89a65d52d6fb56b2c5dd0445f30ca25852f344ba20de325ce6767dd842fca \
-    --hash=sha256:5f4f650b83b8085862de9e555d87f6053ca577b4070f4c6610a870116c4dd1f4 \
-    --hash=sha256:72ef2361d90a717457376351acb5b1b0c189a09dbd95adcb51907a96b79a6add \
-    --hash=sha256:7ef5c7ede8e8cda2a37c0ecab456f4cfae2c42049f51b24edb5303dbfe318ea6 \
-    --hash=sha256:86c9dee0fe6d4ea5bf394767929fdf5f924d161d9a6d23adcd58a690c5e160b0 \
-    --hash=sha256:8b587c9a17f4b050274d9b7f9284d5fae0a8d6a8021f88f779345593326bc33d \
-    --hash=sha256:91025801859a60b7761dea6a8b645f25be6d3639ef828423f094d90b3f60850e \
-    --hash=sha256:9d2940e2801cc768d2cb71e71dca3b025ca3737e9d1d0fad0c95b2e7db0c947a \
-    --hash=sha256:aa520b90eede823632013a319e91652d8226a6309a104cffdc7e00d5a2b5e66b \
-    --hash=sha256:b10fba39049595827f228e77e7b5070cb39c46466bf8fef51da73220a20cc717 \
-    --hash=sha256:c794b5c21485fb3232f5693995ba1a497267b1aecb70b218107cf131f8dc1d3d \
-    --hash=sha256:d05516bc197c5b7b2aa2f834ea7c5ee9fd9aa3034f4193cc05d899b18251aa9c \
-    --hash=sha256:d085c2c676f03357e5d6b11dbbf4e8c1b0d20b1066ac87e6cccc45d4b6c19675 \
-    --hash=sha256:dd40e26aaee67b9078618b0fce3d5f209e328852f2c72c6772cf6352f57d2ed1 \
-    --hash=sha256:e7b84c10ed30c1c997d81ef271945372fba9e18ac58d77a17d43fd9c42392ed4 \
-    --hash=sha256:e982d8af9618d45b25456f1f80e6d628295772d74d755f9a46b90711b7a56067 \
-    --hash=sha256:ef24c8ec97f93b2bdf1080553cdf38ea9ab195846b679cdcfe683c945ed2f1ee \
-    --hash=sha256:f46c5021c3663f82c2ff994295a8574638d56a831ca2a26d736d47fbcf4f9187
+zstandard==0.11.1 \
+    --hash=sha256:19f5ad81590acd20dbdfb930b87a035189778662fdc67ab8cbcc106269ed1be8 \
+    --hash=sha256:1a1db0c9774181e806a418c32d511aa085c7e2c28c257a58f6c107f5decb3109 \
+    --hash=sha256:22d7aa898f36f78108cc1ef0c8da8225f0add518441d815ad4fdd1d577378209 \
+    --hash=sha256:357873afdd7cd0e653d169c36ce837ce2b3e5926dd4a5c0f0476c813f6765373 \
+    --hash=sha256:3c31da5d78a7b07e722e8a3e0b1295bc9b316b7e90a1666659c451a42750ffe4 \
+    --hash=sha256:3f76562ec63fabc6f4b5be0cd986f911c97105c35c31b4d655b90c4d2fe07f40 \
+    --hash=sha256:42fa4462e0563fe17e73dfeb95eef9b00429b86282f8f6ca0e2765b1855a8324 \
+    --hash=sha256:51aad01a5709ca6f45768c69ffd4c887528e5ad9e09302426b735560752c4e82 \
+    --hash=sha256:6cd81819a02e57e38e27c53c5c0a7015e059b0e148a18bf27b46b4f808840879 \
+    --hash=sha256:717fd2494f222164396e03d08ef57174d2a889920b81ca49f276caf9381e6405 \
+    --hash=sha256:71c8711458212c973a9b719275db8111f22803e0caf675affde50703b96e9be1 \
+    --hash=sha256:76a331b5a6258fce3906551557db9be83bdd89a62f66f509a55a4a307239c782 \
+    --hash=sha256:7c92dfcdf7e0c540f9718b40b4c54516a968ef6b81567b75df81866a1af2189d \
+    --hash=sha256:7f3db21223a8bb4ffcf6c36b9c20d38278967723b47fce249dcb6ec6d4082b83 \
+    --hash=sha256:7fa9deba4c904e76870e08324adff94ec3a4bc56a50bbe1a9f859a4aed11c0d2 \
+    --hash=sha256:88912cbcf68cc40037c113460a166ebfbbb24864ceebb89ad221ea346f22e995 \
+    --hash=sha256:94aa5bb817f1c747b21214f6ef83a022bcb63bf81e4dae2954768165c13a510b \
+    --hash=sha256:951e382a2ea47179ecb3e314e8c70f2e5189e3652ccbbcb71c6443dd71bc20fc \
+    --hash=sha256:978a500ae1184f602dc902977ec208c7cf02c10caae9c159b10976a7cb29f879 \
+    --hash=sha256:991c4a40171d87854b219cdf2ba56c1c34b3b3a8ebe5d1ab63bd357ff71271b2 \
+    --hash=sha256:9ca84187182743d2e6bbf9d3f79d3834db205cddc98add27ad20f2189d080a60 \
+    --hash=sha256:ae50bc839cf1ff549f55a3e55922563f246fb692f77497175a8d8d4cddc294da \
+    --hash=sha256:b7abae5b17e82d5f78aaa641077b4619c6ad204e30c6f3445d422acff5f35d3e \
+    --hash=sha256:b8fce0c961654f77c81a6ae1f2cd40633b41ef16a12ae02f0382ed6692f9bb90 \
+    --hash=sha256:d8f047d3647a5cd1b77b4580f35208c938da00c101a092571c85bcefaa2d725d \
+    --hash=sha256:f1785b31bf428e964a9670dd4f721023f2741ef7fd67c663bf01e3d4d3f9ec2a \
+    --hash=sha256:fcf70e1e9d38035a15482e954ba064f3b701cf84cfe571576d15af93ac2a2fb1
--- a/taskcluster/scripts/misc/fetch-content
+++ b/taskcluster/scripts/misc/fetch-content
@ -342,6 +342,59 @@ def fetch_urls(downloads):
            f.result()


+def git_checkout_archive(dest_path: pathlib.Path, repo: str, commit: str,
+                         prefix=None):
+    """Produce an archive of the files comprising a Git checkout."""
+    dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if dest_path.suffixes[-2:] != ['.tar', '.zst']:
+        raise Exception('Only producing .tar.zst archives is supported.')
+
+    if not zstandard:
+        raise ValueError('zstandard Python package not available')
+
+    with tempfile.TemporaryDirectory() as td:
+        temp_dir = pathlib.Path(td)
+
+        git_dir = temp_dir / 'git_dir'
+        if not prefix:
+            prefix = repo.rstrip('/').rsplit('/', 1)[-1]
+
+        # This could be faster with a shallow clone. However, Git requires a ref
+        # to initiate a clone. Since the commit-ish may not refer to a ref, we
+        # simply perform a full clone followed by a checkout.
+        print('cloning %s to %s' % (repo, git_dir))
+        subprocess.run(['git', 'clone', '--bare', repo, str(git_dir)],
+                       check=True)
+
+        print('creating archive %s of commit %s' % (dest_path, commit))
+        proc = subprocess.Popen([
+            'git', '--git-dir={}'.format(git_dir), 'archive',
+            '--format=tar', '--prefix=%s/' % prefix, commit
+        ], stdout=subprocess.PIPE)
+
+        with open(dest_path, 'wb') as out:
+            ctx = zstandard.ZstdCompressor()
+            ctx.copy_stream(proc.stdout, out)
+
+        proc.wait()
+
+
+def command_git_checkout_archive(args):
+    dest = pathlib.Path(args.dest)
+
+    try:
+        git_checkout_archive(dest, args.repo, args.commit,
+                             prefix=args.path_prefix)
+    except Exception:
+        try:
+            dest.unlink()
+        except FileNotFoundError:
+            pass
+
+        raise
+
+
 def command_static_url(args):
    gpg_sig_url = args.gpg_sig_url
    gpg_env_key = args.gpg_key_env
@ -409,6 +462,19 @@ def main():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(title='sub commands')

+    git_checkout = subparsers.add_parser(
+        'git-checkout-archive',
+        help='Obtain an archive of files from a Git repository checkout')
+    git_checkout.set_defaults(func=command_git_checkout_archive)
+    git_checkout.add_argument('--path-prefix',
+                              help='Prefix for paths in produced archive')
+    git_checkout.add_argument('repo',
+                              help='URL to Git repository to be cloned')
+    git_checkout.add_argument('commit',
+                              help='Git commit to check out')
+    git_checkout.add_argument('dest',
+                              help='Destination path of archive')
+
    url = subparsers.add_parser('static-url', help='Download a static URL')
    url.set_defaults(func=command_static_url)
    url.add_argument('--sha256', required=True,
--- a/taskcluster/taskgraph/transforms/fetch.py
+++ b/taskcluster/taskgraph/transforms/fetch.py
@ -10,6 +10,7 @@ from __future__ import absolute_import, unicode_literals
 from mozbuild.shellutil import quote as shell_quote

 import os
+import re

 from voluptuous import (
    Any,
@ -89,6 +90,13 @@ FETCH_SCHEMA = Schema({

            # The name to give to the generated artifact.
            Required('artifact-name'): basestring
+        },
+        {
+            'type': 'git',
+            Required('repo'): basestring,
+            Required('revision'): basestring,
+            Optional('artifact-name'): basestring,
+            Optional('path-prefix'): basestring,
        }
    ),
 })
@ -110,6 +118,8 @@ def process_fetch_job(config, jobs):
            yield create_fetch_url_task(config, job)
        elif typ == 'chromium-fetch':
            yield create_chromium_fetch_task(config, job)
+        elif typ == 'git':
+            yield create_git_fetch_task(config, job)
        else:
            # validate() should have caught this.
            assert False
@ -185,7 +195,7 @@ def create_fetch_url_task(config, job):
    ])

    task = make_base_task(config, name, job['description'], args)
-    task['treeherder']['symbol'] = join_symbol('Fetch-URL', name)
+    task['treeherder']['symbol'] = join_symbol('Fetch', name)
    task['worker']['artifacts'] = [{
        'type': 'directory',
        'name': 'public',
@ -212,6 +222,54 @@ def create_fetch_url_task(config, job):
    return task


+def create_git_fetch_task(config, job):
+    name = job['name']
+    fetch = job['fetch']
+    path_prefix = fetch.get('path-prefix')
+    if not path_prefix:
+        path_prefix = fetch['repo'].rstrip('/').rsplit('/', 1)[-1]
+    artifact_name = fetch.get('artifact-name')
+    if not artifact_name:
+        artifact_name = '{}.tar.zst'.format(path_prefix)
+
+    if not re.match(r'[0-9a-fA-F]{40}', fetch['revision']):
+        raise Exception(
+            'Revision is not a sha1 in fetch task "{}"'.format(name))
+
+    args = [
+        '/builds/worker/bin/fetch-content',
+        'git-checkout-archive',
+        '--path-prefix',
+        path_prefix,
+        fetch['repo'],
+        fetch['revision'],
+        '/builds/worker/artifacts/%s' % artifact_name,
+    ]
+
+    task = make_base_task(config, name, job['description'], args)
+    task['treeherder']['symbol'] = join_symbol('Fetch', name)
+    task['worker']['artifacts'] = [{
+        'type': 'directory',
+        'name': 'public',
+        'path': '/builds/worker/artifacts',
+    }]
+    task['attributes']['fetch-artifact'] = 'public/%s' % artifact_name
+
+    if not taskgraph.fast:
+        cache_name = task['label'].replace('{}-'.format(config.kind), '', 1)
+
+        # This adds the level to the index path automatically.
+        add_optimization(
+            config,
+            task,
+            cache_type=CACHE_TYPE,
+            cache_name=cache_name,
+            digest_data=[fetch['revision'], path_prefix, artifact_name],
+        )
+
+    return task
+
+
 def create_chromium_fetch_task(config, job):
    name = job['name']
    fetch = job['fetch']
@ -240,7 +298,7 @@ def create_chromium_fetch_task(config, job):
    }

    task = make_base_task(config, name, job['description'], cmd)
-    task['treeherder']['symbol'] = join_symbol('Fetch-URL', name)
+    task['treeherder']['symbol'] = join_symbol('Fetch', name)
    task['worker']['artifacts'] = [{
        'type': 'directory',
        'name': 'public',
--- a/taskcluster/taskgraph/transforms/job/debian_package.py
+++ b/taskcluster/taskgraph/transforms/job/debian_package.py
@ -43,6 +43,11 @@ run_schema = Schema({
    # control (.dsc), or a tarball.
    Required(Any('dsc', 'tarball')): source_definition,

+    # Package name. Normally derived from the source control or tarball file
+    # name. Use in case the name doesn't match DSC_PACKAGE_RE or
+    # SOURCE_PACKAGE_RE.
+    Optional('name'): basestring,
+
    # Patch to apply to the extracted source.
    Optional('patch'): basestring,

@ -110,7 +115,9 @@ def docker_worker_debian_package(config, job, taskdesc):
    src_url = src['url']
    src_file = os.path.basename(src_url)
    src_sha256 = src['sha256']
-    package = package_re.match(src_file).group(0)
+    package = run.get('name')
+    if not package:
+        package = package_re.match(src_file).group(0)
    unpack = unpack.format(src_file=src_file, package=package)

    resolver = run.get('resolver', 'apt-get')