Bug 1657199: Generate the same docker-image digests locally as in automation; r=Callek

In bug 1626058, I changed how the docker image digest was generated:

- I used the same directory structure to generate the digest as was used for generating the context
- I moved context generation to the decision task, and used the hash of that as part of the digest.

Unfortunately, it turns out the file name in the gzip header of the context
.tar.gz differed between when we are creating a context to write out, and when
were just generating the hash.

This adjust the name used in the gzip header to be consistent.

Differential Revision: https://phabricator.services.mozilla.com/D84753

taskcluster/taskgraph/docker.py

@@ -65,7 +65,7 @@ def build_context(name, outputFile, args=None):
     if not os.path.isdir(image_dir):
         raise Exception('image directory does not exist: %s' % image_dir)
 
-    docker.create_context_tar(GECKO, image_dir, outputFile, args)
+    docker.create_context_tar(GECKO, image_dir, outputFile, image_name=name, args=args)
 
 
 def build_image(name, tag, args=None):
@@ -83,7 +83,7 @@ def build_image(name, tag, args=None):
     tag = tag or docker.docker_image(name, by_tag=True)
 
     buf = BytesIO()
-    docker.stream_context_tar(GECKO, image_dir, buf, '', args)
+    docker.stream_context_tar(GECKO, image_dir, buf, name, args)
     docker.post_to_docker(buf.getvalue(), '/build', nocache=1, t=tag)
 
     print('Successfully built %s and tagged with %s' % (name, tag))

taskcluster/taskgraph/test/test_util_docker.py

@@ -35,10 +35,10 @@ class TestDocker(unittest.TestCase):
                 f.write("data\n")
             os.chmod(p, MODE_STANDARD)
             self.assertEqual(
-                docker.generate_context_hash(tmpdir,
-                                             os.path.join(tmpdir, 'docker/my-image'),
-                                             'my-image'),
-                'e1649b3427bd7a0387f4508d25057c2e89228748517aad6c70e3df54f47bd13a'
+                docker.generate_context_hash(
+                    tmpdir, os.path.join(tmpdir, "docker/my-image"), "my-image", {},
+                ),
+                '680532a33c845e3b4f8ea8a7bd697da579b647f28c29f7a0a71e51e6cca33983'
             )
         finally:
             shutil.rmtree(tmpdir)
@@ -89,8 +89,8 @@ class TestDocker(unittest.TestCase):
             os.chmod(os.path.join(d, 'extra'), MODE_STANDARD)
 
             tp = os.path.join(tmp, 'tar')
-            h = docker.create_context_tar(tmp, d, tp, 'my_image')
-            self.assertEqual(h, '6c1cc23357625f64f775a08eace7bbc3877dd08d2f3546e0f2e308bac8491865')
+            h = docker.create_context_tar(tmp, d, tp, 'my_image', {})
+            self.assertEqual(h, 'eae3ad00936085eb3e5958912f79fb06ee8e14a91f7157c5f38625f7ddacb9c7')
 
             # File prefix should be "my_image"
             with tarfile.open(tp, 'r:gz') as tf:
@@ -117,8 +117,8 @@ class TestDocker(unittest.TestCase):
             os.chmod(os.path.join(extra, 'file0'), MODE_STANDARD)
 
             tp = os.path.join(tmp, 'tar')
-            h = docker.create_context_tar(tmp, d, tp, 'test_image')
-            self.assertEqual(h, 'e7f14044b8ec1ba42e251d4b293af212ad08b30ec8ab6613abbdbe73c3c2b61f')
+            h = docker.create_context_tar(tmp, d, tp, 'test_image', {})
+            self.assertEqual(h, '49dc3827530cd344d7bcc52e1fdd4aefc632568cf442cffd3dd9633a58f271bf')
 
             with tarfile.open(tp, 'r:gz') as tf:
                 self.assertEqual(tf.getnames(), [
@@ -139,7 +139,7 @@ class TestDocker(unittest.TestCase):
                 fh.write(b'# %include /etc/shadow\n')
 
             with self.assertRaisesRegexp(Exception, 'cannot be absolute'):
-                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test')
+                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test', {})
         finally:
             shutil.rmtree(tmp)
 
@@ -153,7 +153,7 @@ class TestDocker(unittest.TestCase):
                 fh.write(b'# %include foo/../../../etc/shadow\n')
 
             with self.assertRaisesRegexp(Exception, 'path outside topsrcdir'):
-                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test')
+                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test', {})
         finally:
             shutil.rmtree(tmp)
 
@@ -167,7 +167,7 @@ class TestDocker(unittest.TestCase):
                 fh.write(b'# %include does/not/exist\n')
 
             with self.assertRaisesRegexp(Exception, 'path does not exist'):
-                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test')
+                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test', {})
         finally:
             shutil.rmtree(tmp)
 
@@ -195,9 +195,9 @@ class TestDocker(unittest.TestCase):
             os.chmod(os.path.join(tmp, 'file0'), MODE_STANDARD)
 
             tp = os.path.join(tmp, 'tar')
-            h = docker.create_context_tar(tmp, d, tp, 'my_image')
+            h = docker.create_context_tar(tmp, d, tp, 'my_image', {})
 
-            self.assertEqual(h, 'd2a3363b15d0eb547a6c81a72ddf3980e2f6e6360c29b4fb6818102896f43180')
+            self.assertEqual(h, 'a392f23cd6606ae43116390a4d0113354cff1e688a41d46f48b0fb25e90baa13')
 
             with tarfile.open(tp, 'r:gz') as tf:
                 self.assertEqual(tf.getnames(), [

taskcluster/taskgraph/transforms/docker_image.py

@@ -107,10 +107,12 @@ def fill_template(config, tasks):
                 context_hash = create_context_tar(
                     GECKO, context_path,
                     context_file,
+                    image_name,
                     args)
             else:
                 context_hash = generate_context_hash(
                     GECKO, context_path,
+                    image_name,
                     args)
         else:
             if config.write_artifacts:

taskcluster/taskgraph/util/docker.py

@@ -168,10 +168,10 @@ class VoidWriter(object):
         pass
 
 
-def generate_context_hash(topsrcdir, image_path, args=None):
+def generate_context_hash(topsrcdir, image_path, image_name, args):
     """Generates a sha256 hash for context directory used to build an image."""
 
-    return stream_context_tar(topsrcdir, image_path, VoidWriter(), args=args)
+    return stream_context_tar(topsrcdir, image_path, VoidWriter(), image_name, args=args)
 
 
 class HashingWriter(object):
@@ -189,7 +189,7 @@ class HashingWriter(object):
         return six.ensure_text(self._hash.hexdigest())
 
 
-def create_context_tar(topsrcdir, context_dir, out_path, args=None):
+def create_context_tar(topsrcdir, context_dir, out_path, image_name, args):
     """Create a context tarball.
 
     A directory ``context_dir`` containing a Dockerfile will be assembled into
@@ -213,11 +213,11 @@ def create_context_tar(topsrcdir, context_dir, out_path, args=None):
     """
     with open(out_path, 'wb') as fh:
         return stream_context_tar(
-            topsrcdir, context_dir, fh, image_name=os.path.basename(out_path), args=args,
+            topsrcdir, context_dir, fh, image_name=image_name, args=args,
         )
 
 
-def stream_context_tar(topsrcdir, context_dir, out_file, image_name=None, args=None):
+def stream_context_tar(topsrcdir, context_dir, out_file, image_name, args):
     """Like create_context_tar, but streams the tar file to the `out_file` file
     object."""
     archive_files = {}
@@ -276,7 +276,7 @@ def stream_context_tar(topsrcdir, context_dir, out_file, image_name=None, args=N
     archive_files['Dockerfile'] = GeneratedFile(b''.join(six.ensure_binary(s) for s in content))
 
     writer = HashingWriter(out_file)
-    create_tar_gz_from_files(writer, archive_files, image_name)
+    create_tar_gz_from_files(writer, archive_files, "{}.tar".format(image_name))
     return writer.hexdigest()