Bug 1431523 - Use docker images for debian package tasks. r=dustin

We however leave moving the packages building to a script for another
day.

Differential Revision: https://phabricator.services.mozilla.com/D19624


--HG--
rename : taskcluster/docker/debian-base/cloud-mirror-workaround.sh => taskcluster/docker/debian-raw/cloud-mirror-workaround.sh
rename : taskcluster/docker/debian-base/setup_packages.sh => taskcluster/docker/debian-raw/setup_packages.sh

taskcluster/ci/docker-image/kind.yml

@@ -22,13 +22,34 @@ jobs:
     symbol: I(ib)
   desktop1604-test:
     symbol: I(dt16t)
+  # Neither the debian7-*raw nor the debian7-*packages images can have
+  # packages dependencies.
+  debian7-raw:
+    symbol: I(deb7-raw)
+    definition: debian-raw
+    args:
+      BASE_IMAGE: debian:wheezy-20171210
+      DIST: wheezy
+      SNAPSHOT: '20171210T214726Z'
+  debian7-packages:
+    symbol: I(deb7-pkg)
+    definition: debian-packages
+    parent: debian7-raw
+  debian7-i386-raw:
+    symbol: I(deb7-32-raw)
+    definition: debian-raw
+    args:
+      BASE_IMAGE: i386/debian:wheezy-20171210
+      DIST: wheezy
+      SNAPSHOT: '20171210T214726Z'
+  debian7-i386-packages:
+    symbol: I(deb7-32-pkg)
+    definition: debian-packages
+    parent: debian7-i386-raw
   debian7-base:
     symbol: I(deb7-base)
     definition: debian-base
-    args:
-      DIST: wheezy
-      BASE_TAG: '20171210'
-      SNAPSHOT: '20171210T214726Z'
+    parent: debian7-raw
     packages:
       - deb7-gdb
       - deb7-git
@@ -111,13 +132,23 @@ jobs:
     parent: debian7-amd64-build
   lint:
     symbol: I(lnt)
+  # Neither the debian9-raw nor the debian9-packages images can have
+  # packages dependencies.
+  debian9-raw:
+    symbol: I(deb9-raw)
+    definition: debian-raw
+    args:
+      BASE_IMAGE: debian:stretch-20170620
+      DIST: stretch
+      SNAPSHOT: '20170830T000511Z'
+  debian9-packages:
+    symbol: I(deb9-pkg)
+    definition: debian-packages
+    parent: debian9-raw
   debian9-base:
     symbol: I(deb9-base)
     definition: debian-base
-    args:
-      DIST: stretch
-      BASE_TAG: '20170620'
-      SNAPSHOT: '20170830T000511Z'
+    parent: debian9-raw
     packages:
       - deb9-mercurial
       - deb9-python-zstandard

taskcluster/docker/debian-base/Dockerfile

@@ -1,6 +1,5 @@
-# %ARG DIST
-# %ARG BASE_TAG
-FROM debian:$DIST-$BASE_TAG
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
 MAINTAINER Mike Hommey <mhommey@mozilla.com>
 
 ### Add worker user and setup its workspace.
@@ -23,36 +22,14 @@ ENV HOME=/builds/worker \
     SHELL=/bin/bash \
     USER=worker \
     LOGNAME=worker \
-    HOSTNAME=taskcluster-worker \
-    DEBIAN_FRONTEND=noninteractive
+    HOSTNAME=taskcluster-worker
 
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
 
-# %ARG SNAPSHOT
-# Set apt sources list to a snapshot.
-RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \
-      echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \
-    done > /etc/apt/sources.list && \
-    ( echo 'quiet "true";'; \
-      echo 'APT::Get::Assume-Yes "true";'; \
-      echo 'APT::Install-Recommends "false";'; \
-      echo 'Acquire::Check-Valid-Until "false";'; \
-      echo 'Acquire::Retries "5";'; \
-    ) > /etc/apt/apt.conf.d/99taskcluster
-
-RUN apt-get update && \
-    apt-get install \
-      apt-transport-https \
-      ca-certificates
-
-COPY setup_packages.sh /usr/local/sbin/
-COPY cloud-mirror-workaround.sh /usr/local/sbin/
-
 # %ARG TASKCLUSTER_ROOT_URL
 # %ARG DOCKER_IMAGE_PACKAGES
 RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \
-    echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";' > /etc/apt/apt.conf.d/99cloud-mirror-workaround && \
     apt-get update && \
     apt-get install \
       git \

taskcluster/docker/debian-packages/Dockerfile

@@ -0,0 +1,10 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+RUN apt-get install --install-recommends \
+      apt-utils \
+      aptitude \
+      build-essential \
+      devscripts \
+      fakeroot

taskcluster/docker/debian-raw/Dockerfile

@@ -0,0 +1,30 @@
+# %ARG BASE_IMAGE
+FROM $BASE_IMAGE
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]
+
+COPY setup_packages.sh /usr/local/sbin/
+COPY cloud-mirror-workaround.sh /usr/local/sbin/
+
+# %ARG DIST
+# %ARG SNAPSHOT
+# Set apt sources list to a snapshot.
+RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \
+      echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \
+    done > /etc/apt/sources.list && \
+    ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+      echo 'Acquire::Retries "5";'; \
+      echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";'; \
+    ) > /etc/apt/apt.conf.d/99taskcluster
+
+RUN apt-get update && \
+    apt-get install \
+      apt-transport-https \
+      ca-certificates

taskcluster/taskgraph/transforms/job/debian_package.py

@@ -9,7 +9,6 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import re
-import taskcluster_urls
 
 from taskgraph.util.schema import Schema
 from voluptuous import Any, Optional, Required
@@ -73,17 +72,21 @@ def docker_worker_debian_package(config, job, taskdesc):
 
     name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
 
-    docker_repo = 'debian'
     arch = run.get('arch', 'amd64')
-    if arch != 'amd64':
-        docker_repo = '{}/{}'.format(arch, docker_repo)
 
     worker = taskdesc['worker']
     worker['artifacts'] = []
-    worker['docker-image'] = '{repo}:{dist}-{date}'.format(
-        repo=docker_repo,
-        dist=run['dist'],
-        date=run['snapshot'][:8])
+    version = {
+        'wheezy': 7,
+        'jessie': 8,
+        'stretch': 9,
+        'buster': 10,
+    }[run['dist']]
+    image = 'debian%d' % version
+    if arch != 'amd64':
+        image += '-' + arch
+    image += '-packages'
+    worker['docker-image'] = {'in-tree': image}
     # Retry on apt-get errors.
     worker['retry-exit-status'] = [100]
 
@@ -110,13 +113,6 @@ def docker_worker_debian_package(config, job, taskdesc):
     package = package_re.match(src_file).group(0)
     unpack = unpack.format(src_file=src_file, package=package)
 
-    base_deps = [
-        'apt-utils',
-        'build-essential',
-        'devscripts',
-        'fakeroot',
-    ]
-
     resolver = run.get('resolver', 'apt-get')
     if resolver == 'apt-get':
         resolver = 'apt-get -yyq --no-install-recommends'
@@ -124,15 +120,13 @@ def docker_worker_debian_package(config, job, taskdesc):
         resolver = ('aptitude -y --without-recommends -o '
                     'Aptitude::ProblemResolver::Hints::KeepBuildDeps='
                     '"reject {}-build-deps :UNINST"').format(package)
-        base_deps.append('aptitude')
     else:
         raise RuntimeError('Unreachable')
 
     adjust = ''
     if 'patch' in run:
-        # We can't depend on docker images, so we don't have robustcheckout or
-        # or run-task to get a checkout. So for this one file we'd need
-        # from a checkout, download it.
+        # We don't use robustcheckout or run-task to get a checkout. So for
+        # this one file we'd need from a checkout, download it.
         env['PATCH_URL'] = '{head_repo}/raw-file/{head_rev}/build/debian-packages/{patch}'.format(
             head_repo=config.params['head_repository'],
             head_rev=config.params['head_rev'],
@@ -154,33 +148,15 @@ def docker_worker_debian_package(config, job, taskdesc):
             dist=run['dist'],
         )
 
-    queue_url = taskcluster_urls.api(get_root_url(), 'queue', 'v1', '')
-
-    # We can't depend on docker images (since docker images depend on packages),
-    # so we inline the whole script here.
     worker['command'] = [
         'sh',
         '-x',
         '-c',
-        # Fill /etc/apt/sources.list with the relevant snapshot repository.
-        'echo "deb http://snapshot.debian.org/archive/debian'
-        '/{snapshot}/ {dist} main" > /etc/apt/sources.list && '
-        'echo "deb http://snapshot.debian.org/archive/debian'
-        '/{snapshot}/ {dist}-updates main" >> /etc/apt/sources.list && '
-        'echo "deb http://snapshot.debian.org/archive/debian'
-        '/{snapshot}/ {dist}-backports main" >> /etc/apt/sources.list && '
-        'echo "deb http://snapshot.debian.org/archive/debian-security'
-        '/{snapshot}/ {dist}/updates main" >> /etc/apt/sources.list && '
-        'apt-get update -o Acquire::Check-Valid-Until=false -q && '
         # Add sources for packages coming from other package tasks.
-        'apt-get install -yyq apt-transport-https ca-certificates && '
-        'for task in $PACKAGES; do '
-        '  echo "deb [trusted=yes] {queue_url}task/$task/artifacts/public/build/ debian/" '
-        '>> /etc/apt/sources.list; '
-        'done && '
-        # Install the base utilities required to build debian packages.
-        'apt-get update -o Acquire::Check-Valid-Until=false -q && '
-        'apt-get install -yyq {base_deps} && '
+        '/usr/local/sbin/setup_packages.sh {queue_url} $PACKAGES && '
+        'apt-get update && '
+        # Upgrade packages that might have new versions in package tasks.
+        'apt-get dist-upgrade && '
         'cd /tmp && '
         # Get, validate and extract the package source.
         'dget -d -u {src_url} && '
@@ -201,7 +177,7 @@ def docker_worker_debian_package(config, job, taskdesc):
         'apt-ftparchive sources debian | gzip -c9 > debian/Sources.gz && '
         'apt-ftparchive packages debian | gzip -c9 > debian/Packages.gz'
         .format(
-            queue_url=queue_url,
+            queue_url=get_root_url(),
             package=package,
             snapshot=run['snapshot'],
             dist=run['dist'],
@@ -211,7 +187,6 @@ def docker_worker_debian_package(config, job, taskdesc):
             unpack=unpack,
             adjust=adjust,
             artifacts='/tmp/artifacts',
-            base_deps=' '.join(base_deps),
             resolver=resolver,
         )
     ]
@@ -233,9 +208,6 @@ def docker_worker_debian_package(config, job, taskdesc):
         digest_data.append(
             hash_path(os.path.join(GECKO, 'build', 'debian-packages', run['patch'])))
 
-    if docker_repo != 'debian':
-        digest_data.append(docker_repo)
-
     if not taskgraph.fast:
         taskdesc['cache'] = {
             'type': 'packages.v1',