Backout changeset 8f7958c7d3e5 (bug 1431523) to give time to docker images, etc. to build without blocking other landings.

--HG--
rename : taskcluster/docker/debian-raw/cloud-mirror-workaround.sh => taskcluster/docker/debian-base/cloud-mirror-workaround.sh
rename : taskcluster/docker/debian-raw/setup_packages.sh => taskcluster/docker/debian-base/setup_packages.sh

taskcluster/ci/docker-image/kind.yml

@@ -22,34 +22,13 @@ jobs:
     symbol: I(ib)
   desktop1604-test:
     symbol: I(dt16t)
-  # Neither the debian7-*raw nor the debian7-*packages images can have
-  # packages dependencies.
-  debian7-raw:
-    symbol: I(deb7-raw)
-    definition: debian-raw
-    args:
-      BASE_IMAGE: debian:wheezy-20171210
-      DIST: wheezy
-      SNAPSHOT: '20171210T214726Z'
-  debian7-packages:
-    symbol: I(deb7-pkg)
-    definition: debian-packages
-    parent: debian7-raw
-  debian7-i386-raw:
-    symbol: I(deb7-32-raw)
-    definition: debian-raw
-    args:
-      BASE_IMAGE: i386/debian:wheezy-20171210
-      DIST: wheezy
-      SNAPSHOT: '20171210T214726Z'
-  debian7-i386-packages:
-    symbol: I(deb7-32-pkg)
-    definition: debian-packages
-    parent: debian7-i386-raw
   debian7-base:
     symbol: I(deb7-base)
     definition: debian-base
-    parent: debian7-raw
+    args:
+      DIST: wheezy
+      BASE_TAG: '20171210'
+      SNAPSHOT: '20171210T214726Z'
     packages:
       - deb7-gdb
       - deb7-git
@@ -132,23 +111,13 @@ jobs:
     parent: debian7-amd64-build
   lint:
     symbol: I(lnt)
-  # Neither the debian9-raw nor the debian9-packages images can have
-  # packages dependencies.
-  debian9-raw:
-    symbol: I(deb9-raw)
-    definition: debian-raw
-    args:
-      BASE_IMAGE: debian:stretch-20170620
-      DIST: stretch
-      SNAPSHOT: '20170830T000511Z'
-  debian9-packages:
-    symbol: I(deb9-pkg)
-    definition: debian-packages
-    parent: debian9-raw
   debian9-base:
     symbol: I(deb9-base)
     definition: debian-base
-    parent: debian9-raw
+    args:
+      DIST: stretch
+      BASE_TAG: '20170620'
+      SNAPSHOT: '20170830T000511Z'
     packages:
       - deb9-mercurial
       - deb9-python-zstandard

taskcluster/docker/debian-base/Dockerfile

@@ -1,5 +1,6 @@
-# %ARG DOCKER_IMAGE_PARENT
-FROM $DOCKER_IMAGE_PARENT
+# %ARG DIST
+# %ARG BASE_TAG
+FROM debian:$DIST-$BASE_TAG
 MAINTAINER Mike Hommey <mhommey@mozilla.com>
 
 ### Add worker user and setup its workspace.
@@ -22,14 +23,36 @@ ENV HOME=/builds/worker \
     SHELL=/bin/bash \
     USER=worker \
     LOGNAME=worker \
-    HOSTNAME=taskcluster-worker
+    HOSTNAME=taskcluster-worker \
+    DEBIAN_FRONTEND=noninteractive
 
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
 
+# %ARG SNAPSHOT
+# Set apt sources list to a snapshot.
+RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \
+      echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \
+    done > /etc/apt/sources.list && \
+    ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+      echo 'Acquire::Retries "5";'; \
+    ) > /etc/apt/apt.conf.d/99taskcluster
+
+RUN apt-get update && \
+    apt-get install \
+      apt-transport-https \
+      ca-certificates
+
+COPY setup_packages.sh /usr/local/sbin/
+COPY cloud-mirror-workaround.sh /usr/local/sbin/
+
 # %ARG TASKCLUSTER_ROOT_URL
 # %ARG DOCKER_IMAGE_PACKAGES
 RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \
+    echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";' > /etc/apt/apt.conf.d/99cloud-mirror-workaround && \
     apt-get update && \
     apt-get install \
       git \

taskcluster/docker/debian-packages/Dockerfile

@@ -1,10 +0,0 @@
-# %ARG DOCKER_IMAGE_PARENT
-FROM $DOCKER_IMAGE_PARENT
-MAINTAINER Mike Hommey <mhommey@mozilla.com>
-
-RUN apt-get install --install-recommends \
-      apt-utils \
-      aptitude \
-      build-essential \
-      devscripts \
-      fakeroot

taskcluster/docker/debian-raw/Dockerfile

@@ -1,30 +0,0 @@
-# %ARG BASE_IMAGE
-FROM $BASE_IMAGE
-MAINTAINER Mike Hommey <mhommey@mozilla.com>
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
-
-COPY setup_packages.sh /usr/local/sbin/
-COPY cloud-mirror-workaround.sh /usr/local/sbin/
-
-# %ARG DIST
-# %ARG SNAPSHOT
-# Set apt sources list to a snapshot.
-RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \
-      echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \
-    done > /etc/apt/sources.list && \
-    ( echo 'quiet "true";'; \
-      echo 'APT::Get::Assume-Yes "true";'; \
-      echo 'APT::Install-Recommends "false";'; \
-      echo 'Acquire::Check-Valid-Until "false";'; \
-      echo 'Acquire::Retries "5";'; \
-      echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";'; \
-    ) > /etc/apt/apt.conf.d/99taskcluster
-
-RUN apt-get update && \
-    apt-get install \
-      apt-transport-https \
-      ca-certificates

taskcluster/taskgraph/transforms/job/debian_package.py

@@ -9,6 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import re
+import taskcluster_urls
 
 from taskgraph.util.schema import Schema
 from voluptuous import Any, Optional, Required
@@ -72,21 +73,17 @@ def docker_worker_debian_package(config, job, taskdesc):
 
     name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
 
+    docker_repo = 'debian'
     arch = run.get('arch', 'amd64')
+    if arch != 'amd64':
+        docker_repo = '{}/{}'.format(arch, docker_repo)
 
     worker = taskdesc['worker']
     worker['artifacts'] = []
-    version = {
-        'wheezy': 7,
-        'jessie': 8,
-        'stretch': 9,
-        'buster': 10,
-    }[run['dist']]
-    image = 'debian%d' % version
-    if arch != 'amd64':
-        image += '-' + arch
-    image += '-packages'
-    worker['docker-image'] = {'in-tree': image}
+    worker['docker-image'] = '{repo}:{dist}-{date}'.format(
+        repo=docker_repo,
+        dist=run['dist'],
+        date=run['snapshot'][:8])
     # Retry on apt-get errors.
     worker['retry-exit-status'] = [100]
 
@@ -113,6 +110,13 @@ def docker_worker_debian_package(config, job, taskdesc):
     package = package_re.match(src_file).group(0)
     unpack = unpack.format(src_file=src_file, package=package)
 
+    base_deps = [
+        'apt-utils',
+        'build-essential',
+        'devscripts',
+        'fakeroot',
+    ]
+
     resolver = run.get('resolver', 'apt-get')
     if resolver == 'apt-get':
         resolver = 'apt-get -yyq --no-install-recommends'
@@ -120,13 +124,15 @@ def docker_worker_debian_package(config, job, taskdesc):
         resolver = ('aptitude -y --without-recommends -o '
                     'Aptitude::ProblemResolver::Hints::KeepBuildDeps='
                     '"reject {}-build-deps :UNINST"').format(package)
+        base_deps.append('aptitude')
     else:
         raise RuntimeError('Unreachable')
 
     adjust = ''
     if 'patch' in run:
-        # We don't use robustcheckout or run-task to get a checkout. So for
-        # this one file we'd need from a checkout, download it.
+        # We can't depend on docker images, so we don't have robustcheckout or
+        # or run-task to get a checkout. So for this one file we'd need
+        # from a checkout, download it.
         env['PATCH_URL'] = '{head_repo}/raw-file/{head_rev}/build/debian-packages/{patch}'.format(
             head_repo=config.params['head_repository'],
             head_rev=config.params['head_rev'],
@@ -148,15 +154,33 @@ def docker_worker_debian_package(config, job, taskdesc):
             dist=run['dist'],
         )
 
+    queue_url = taskcluster_urls.api(get_root_url(), 'queue', 'v1', '')
+
+    # We can't depend on docker images (since docker images depend on packages),
+    # so we inline the whole script here.
     worker['command'] = [
         'sh',
         '-x',
         '-c',
+        # Fill /etc/apt/sources.list with the relevant snapshot repository.
+        'echo "deb http://snapshot.debian.org/archive/debian'
+        '/{snapshot}/ {dist} main" > /etc/apt/sources.list && '
+        'echo "deb http://snapshot.debian.org/archive/debian'
+        '/{snapshot}/ {dist}-updates main" >> /etc/apt/sources.list && '
+        'echo "deb http://snapshot.debian.org/archive/debian'
+        '/{snapshot}/ {dist}-backports main" >> /etc/apt/sources.list && '
+        'echo "deb http://snapshot.debian.org/archive/debian-security'
+        '/{snapshot}/ {dist}/updates main" >> /etc/apt/sources.list && '
+        'apt-get update -o Acquire::Check-Valid-Until=false -q && '
         # Add sources for packages coming from other package tasks.
-        '/usr/local/sbin/setup_packages.sh {queue_url} $PACKAGES && '
-        'apt-get update && '
-        # Upgrade packages that might have new versions in package tasks.
-        'apt-get dist-upgrade && '
+        'apt-get install -yyq apt-transport-https ca-certificates && '
+        'for task in $PACKAGES; do '
+        '  echo "deb [trusted=yes] {queue_url}task/$task/artifacts/public/build/ debian/" '
+        '>> /etc/apt/sources.list; '
+        'done && '
+        # Install the base utilities required to build debian packages.
+        'apt-get update -o Acquire::Check-Valid-Until=false -q && '
+        'apt-get install -yyq {base_deps} && '
         'cd /tmp && '
         # Get, validate and extract the package source.
         'dget -d -u {src_url} && '
@@ -177,7 +201,7 @@ def docker_worker_debian_package(config, job, taskdesc):
         'apt-ftparchive sources debian | gzip -c9 > debian/Sources.gz && '
         'apt-ftparchive packages debian | gzip -c9 > debian/Packages.gz'
         .format(
-            queue_url=get_root_url(),
+            queue_url=queue_url,
             package=package,
             snapshot=run['snapshot'],
             dist=run['dist'],
@@ -187,6 +211,7 @@ def docker_worker_debian_package(config, job, taskdesc):
             unpack=unpack,
             adjust=adjust,
             artifacts='/tmp/artifacts',
+            base_deps=' '.join(base_deps),
             resolver=resolver,
         )
     ]
@@ -208,6 +233,9 @@ def docker_worker_debian_package(config, job, taskdesc):
         digest_data.append(
             hash_path(os.path.join(GECKO, 'build', 'debian-packages', run['patch'])))
 
+    if docker_repo != 'debian':
+        digest_data.append(docker_repo)
+
     if not taskgraph.fast:
         taskdesc['cache'] = {
             'type': 'packages.v1',