Bug 1391476 - Capture Docker volumes in docker-worker config; r=dustin

Docker volumes are host-mounted filesystems. We typically mount
caches at their location. But not always. The reason we define
VOLUME in Dockerfiles is we're guaranteed to get a fast host
filesystem instead of AUFS when a cache isn't mounted.

In this commit, we teach the docker-worker payload builder about
the existence of Docker volumes. Docker volumes can be declared
inline in the YAML. More conveniently, we automatically parse out
VOLUME lines from corresponding in-tree Dockerfile.

We'll do useful things with this data in subsequent commits.

MozReview-Commit-ID: BNxp8EDEYw

--HG--
extra : rebase_source : 1aa6dbb000386cd2789d526708ce369640d532c9

taskcluster/taskgraph/transforms/task.py

@@ -25,6 +25,7 @@ from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import get_release_config
 from voluptuous import Any, Required, Optional, Extra
 from taskgraph import GECKO
+from ..util import docker as dockerutil
 
 from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
 
@@ -193,6 +194,18 @@ task_description_schema = Schema({
         Required('loopback-audio', default=False): bool,
         Required('docker-in-docker', default=False): bool,  # (aka 'dind')
 
+        # Paths to Docker volumes.
+        #
+        # For in-tree Docker images, volumes can be parsed from Dockerfile.
+        # This only works for the Dockerfile itself: if a volume is defined in
+        # a base image, it will need to be declared here. Out-of-tree Docker
+        # images will also require explicit volume annotation.
+        #
+        # Caches are often mounted to the same path as Docker volumes. In this
+        # case, they take precedence over a Docker volume. But a volume still
+        # needs to be declared for the path.
+        Optional('volumes', default=[]): [basestring],
+
         # caches to set up for the task
         Optional('caches'): [{
             # only one type is supported by any of the workers right now
@@ -595,6 +608,7 @@ def build_docker_worker_payload(config, task, task_def):
     image = worker['docker-image']
     if isinstance(image, dict):
         if 'in-tree' in image:
+            name = image['in-tree']
             docker_image_task = 'build-docker-image-' + image['in-tree']
             task.setdefault('dependencies', {})['docker-image'] = docker_image_task
 
@@ -603,6 +617,18 @@ def build_docker_worker_payload(config, task, task_def):
                 "taskId": {"task-reference": "<docker-image>"},
                 "type": "task-image",
             }
+
+            # Find VOLUME in Dockerfile.
+            volumes = dockerutil.parse_volumes(name)
+            for v in sorted(volumes):
+                if v in worker['volumes']:
+                    raise Exception('volume %s already defined; '
+                                    'if it is defined in a Dockerfile, '
+                                    'it does not need to be specified in the '
+                                    'worker definition' % v)
+
+                worker['volumes'].append(v)
+
         elif 'indexed' in image:
             image = {
                 "path": "public/image.tar.zst",

taskcluster/taskgraph/util/docker.py

@@ -11,6 +11,7 @@ import subprocess
 import tarfile
 import tempfile
 
+from mozbuild.util import memoize
 from mozpack.archive import (
     create_tar_gz_from_files,
 )
@@ -163,3 +164,24 @@ def build_from_context(docker_bin, context_path, prefix, tag=None):
             raise Exception('error building image')
     finally:
         shutil.rmtree(d)
+
+
+@memoize
+def parse_volumes(image):
+    """Parse VOLUME entries from a Dockerfile for an image."""
+    volumes = set()
+
+    with open(os.path.join(IMAGE_DIR, image, 'Dockerfile'), 'rb') as fh:
+        for line in fh:
+            line = line.strip()
+            if not line.startswith(b'VOLUME '):
+                continue
+
+            v = line.split(None, 1)[1]
+            if v.startswith(b'['):
+                raise ValueError('cannot parse array syntax for VOLUME; '
+                                 'convert to multiple entries')
+
+            volumes |= set(v.split())
+
+    return volumes