Compress Docker images with zstandard

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
 known_first_party = taskboot
-known_third_party = boto3,botocore,docker,dockerfile_parse,github,pytest,requests,setuptools,taskcluster,taskcluster_urls,twine,yaml
+known_third_party = boto3,botocore,docker,dockerfile_parse,github,pytest,requests,setuptools,taskcluster,taskcluster_urls,twine,yaml,zstandard
 force_single_line = True
 default_section=FIRSTPARTY
 line_length=159

.taskcluster.yml

@@ -79,9 +79,9 @@ tasks:
             pip install --no-cache-dir --quiet . &&
             taskboot --target=/src build --image=$IMAGE --tag=$VERSION --write /image.tar Dockerfile"
         artifacts:
-          public/taskboot/image.tar:
+          public/taskboot/image.tar.zst:
             expires: {$fromNow: '2 weeks'}
-            path: /image.tar
+            path: /image.tar.zst
             type: file
       scopes:
         - docker-worker:capability:privileged
@@ -102,7 +102,7 @@ tasks:
         features:
           dind: true
         maxRunTime: 3600
-        image: python:3.7-alpine
+        image: python:3.7
         env:
           IMAGE: mozilla/taskboot
           REGISTRY: registry.hub.docker.com
@@ -110,14 +110,13 @@ tasks:
         command:
           - sh
           - -lxce
-          - "apk add --no-cache git --quiet &&
-            git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot &&
+          - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b taskboot &&
             pip install --no-cache-dir --quiet . &&
             taskboot --target=/src build --build-tool=dind --image=$IMAGE --tag=$VERSION --write /image.tar tests/dockerfile.empty"
         artifacts:
-          public/taskboot/test-dind.tar:
+          public/taskboot/test-dind.tar.zst:
             expires: {$fromNow: '2 weeks'}
-            path: /image.tar
+            path: /image.tar.zst
             type: file
       metadata:
         name: TaskBoot docker build using Docker in Docker
@@ -138,7 +137,7 @@ tasks:
         maxRunTime: 3600
         image:
           type: task-image
-          path: public/taskboot/test-dind.tar
+          path: public/taskboot/test-dind.tar.zst
           taskId: {$eval: as_slugid("docker_build_dind")}
       metadata:
         name: TaskBoot docker run DinD image

requirements.txt

@@ -8,3 +8,4 @@ taskcluster==38.0.4
 taskcluster-urls==13.0.1
 twine==2.0.0
 yarl<1.4.0
+zstandard==0.15.1

taskboot/build.py

@@ -18,6 +18,7 @@ from taskboot.docker import Docker
 from taskboot.docker import Img
 from taskboot.docker import patch_dockerfile
 from taskboot.utils import retry
+from taskboot.utils import zstd_compress
 
 logger = logging.getLogger(__name__)
 
@@ -95,6 +96,7 @@ def build_image(target, args):
     # Write the produced image
     if output:
         build_tool.save(tags, output)
+        zstd_compress(output)
 
     # Push the produced image
     if args.push:
@@ -167,7 +169,11 @@ def build_compose(target, args):
 
         # Write the produced image
         if output:
-            build_tool.save(tags, os.path.join(output, "{}.tar".format(name)))
+            output_path = os.path.join(output, f"{name}.tar")
+
+            build_tool.save(tags, output_path)
+
+            zstd_compress(output_path)
 
     logger.info("Compose file fully processed.")
 

taskboot/push.py

@@ -4,6 +4,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import logging
+import os
 
 import requests
 import taskcluster
@@ -15,6 +16,7 @@ from taskboot.docker import docker_id_archive
 from taskboot.utils import download_artifact
 from taskboot.utils import load_artifacts
 from taskboot.utils import load_named_artifacts
+from taskboot.utils import zstd_decompress
 
 logger = logging.getLogger(__name__)
 
@@ -62,6 +64,9 @@ def push_artifact(queue, push_tool, task_id, artifact_name, custom_tag=None):
     and push it on remote repo
     """
     path = download_artifact(queue, task_id, artifact_name)
+    path, ext = os.path.splitext(path)
+    assert ext == "zst"
+    zstd_decompress(path)
     push_tool.push_archive(path, custom_tag)
 
 
@@ -91,6 +96,10 @@ def heroku_release(target, args):
         # Push the Docker image
         custom_tag_name = f"{HEROKU_REGISTRY}/{args.heroku_app}/{heroku_dyno_name}"
 
+        artifact_path, ext = os.path.splitext(artifact_path)
+        assert ext == "zst"
+        zstd_decompress(artifact_path)
+
         skopeo.push_archive(artifact_path, custom_tag_name)
 
         # Get the Docker image id

taskboot/utils.py

@@ -12,6 +12,7 @@ from fnmatch import fnmatch
 
 import requests
 import taskcluster
+import zstandard
 
 logger = logging.getLogger(__name__)
 
@@ -172,3 +173,21 @@ def load_named_artifacts(config, source_task_id, arguments, output_directory=Non
         )
 
         yield (name, artifact_name, artifact_path)
+
+
+def zstd_compress(path: str) -> None:
+    cctx = zstandard.ZstdCompressor(threads=-1)
+    with open(path, "rb") as input_f:
+        with open(f"{path}.zst", "wb") as output_f:
+            cctx.copy_stream(input_f, output_f)
+
+    os.remove(path)
+
+
+def zstd_decompress(path: str) -> None:
+    dctx = zstandard.ZstdDecompressor()
+    with open(f"{path}.zst", "rb") as input_f:
+        with open(path, "wb") as output_f:
+            dctx.copy_stream(input_f, output_f)
+
+    os.remove(f"{path}.zst")