diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index b68665ab7fe9..7526906392a1 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -22,34 +22,13 @@ jobs: symbol: I(ib) desktop1604-test: symbol: I(dt16t) - # Neither the debian7-*raw nor the debian7-*packages images can have - # packages dependencies. - debian7-raw: - symbol: I(deb7-raw) - definition: debian-raw - args: - BASE_IMAGE: debian:wheezy-20171210 - DIST: wheezy - SNAPSHOT: '20171210T214726Z' - debian7-packages: - symbol: I(deb7-pkg) - definition: debian-packages - parent: debian7-raw - debian7-i386-raw: - symbol: I(deb7-32-raw) - definition: debian-raw - args: - BASE_IMAGE: i386/debian:wheezy-20171210 - DIST: wheezy - SNAPSHOT: '20171210T214726Z' - debian7-i386-packages: - symbol: I(deb7-32-pkg) - definition: debian-packages - parent: debian7-i386-raw debian7-base: symbol: I(deb7-base) definition: debian-base - parent: debian7-raw + args: + DIST: wheezy + BASE_TAG: '20171210' + SNAPSHOT: '20171210T214726Z' packages: - deb7-gdb - deb7-git @@ -132,23 +111,13 @@ jobs: parent: debian7-amd64-build lint: symbol: I(lnt) - # Neither the debian9-raw nor the debian9-packages images can have - # packages dependencies. - debian9-raw: - symbol: I(deb9-raw) - definition: debian-raw - args: - BASE_IMAGE: debian:stretch-20170620 - DIST: stretch - SNAPSHOT: '20170830T000511Z' - debian9-packages: - symbol: I(deb9-pkg) - definition: debian-packages - parent: debian9-raw debian9-base: symbol: I(deb9-base) definition: debian-base - parent: debian9-raw + args: + DIST: stretch + BASE_TAG: '20170620' + SNAPSHOT: '20170830T000511Z' packages: - deb9-mercurial - deb9-python-zstandard diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile index 4336a7cc7e50..db2018674eb3 100644 --- a/taskcluster/docker/debian-base/Dockerfile +++ b/taskcluster/docker/debian-base/Dockerfile @@ -1,5 +1,6 @@ -# %ARG DOCKER_IMAGE_PARENT -FROM $DOCKER_IMAGE_PARENT +# %ARG DIST +# %ARG BASE_TAG +FROM debian:$DIST-$BASE_TAG MAINTAINER Mike Hommey ### Add worker user and setup its workspace. @@ -22,14 +23,36 @@ ENV HOME=/builds/worker \ SHELL=/bin/bash \ USER=worker \ LOGNAME=worker \ - HOSTNAME=taskcluster-worker + HOSTNAME=taskcluster-worker \ + DEBIAN_FRONTEND=noninteractive # Set a default command useful for debugging CMD ["/bin/bash", "--login"] +# %ARG SNAPSHOT +# Set apt sources list to a snapshot. +RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \ + done > /etc/apt/sources.list && \ + ( echo 'quiet "true";'; \ + echo 'APT::Get::Assume-Yes "true";'; \ + echo 'APT::Install-Recommends "false";'; \ + echo 'Acquire::Check-Valid-Until "false";'; \ + echo 'Acquire::Retries "5";'; \ + ) > /etc/apt/apt.conf.d/99taskcluster + +RUN apt-get update && \ + apt-get install \ + apt-transport-https \ + ca-certificates + +COPY setup_packages.sh /usr/local/sbin/ +COPY cloud-mirror-workaround.sh /usr/local/sbin/ + # %ARG TASKCLUSTER_ROOT_URL # %ARG DOCKER_IMAGE_PACKAGES RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \ + echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";' > /etc/apt/apt.conf.d/99cloud-mirror-workaround && \ apt-get update && \ apt-get install \ git \ diff --git a/taskcluster/docker/debian-raw/cloud-mirror-workaround.sh b/taskcluster/docker/debian-base/cloud-mirror-workaround.sh similarity index 100% rename from taskcluster/docker/debian-raw/cloud-mirror-workaround.sh rename to taskcluster/docker/debian-base/cloud-mirror-workaround.sh diff --git a/taskcluster/docker/debian-raw/setup_packages.sh b/taskcluster/docker/debian-base/setup_packages.sh similarity index 100% rename from taskcluster/docker/debian-raw/setup_packages.sh rename to taskcluster/docker/debian-base/setup_packages.sh diff --git a/taskcluster/docker/debian-packages/Dockerfile b/taskcluster/docker/debian-packages/Dockerfile deleted file mode 100644 index 15e8b33855b2..000000000000 --- a/taskcluster/docker/debian-packages/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -# %ARG DOCKER_IMAGE_PARENT -FROM $DOCKER_IMAGE_PARENT -MAINTAINER Mike Hommey - -RUN apt-get install --install-recommends \ - apt-utils \ - aptitude \ - build-essential \ - devscripts \ - fakeroot diff --git a/taskcluster/docker/debian-raw/Dockerfile b/taskcluster/docker/debian-raw/Dockerfile deleted file mode 100644 index 7dcc7b7769d4..000000000000 --- a/taskcluster/docker/debian-raw/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# %ARG BASE_IMAGE -FROM $BASE_IMAGE -MAINTAINER Mike Hommey - -ENV DEBIAN_FRONTEND=noninteractive - -# Set a default command useful for debugging -CMD ["/bin/bash", "--login"] - -COPY setup_packages.sh /usr/local/sbin/ -COPY cloud-mirror-workaround.sh /usr/local/sbin/ - -# %ARG DIST -# %ARG SNAPSHOT -# Set apt sources list to a snapshot. -RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \ - echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \ - done > /etc/apt/sources.list && \ - ( echo 'quiet "true";'; \ - echo 'APT::Get::Assume-Yes "true";'; \ - echo 'APT::Install-Recommends "false";'; \ - echo 'Acquire::Check-Valid-Until "false";'; \ - echo 'Acquire::Retries "5";'; \ - echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";'; \ - ) > /etc/apt/apt.conf.d/99taskcluster - -RUN apt-get update && \ - apt-get install \ - apt-transport-https \ - ca-certificates diff --git a/taskcluster/taskgraph/transforms/job/debian_package.py b/taskcluster/taskgraph/transforms/job/debian_package.py index fec8f7901479..812450b91dc5 100644 --- a/taskcluster/taskgraph/transforms/job/debian_package.py +++ b/taskcluster/taskgraph/transforms/job/debian_package.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, print_function, unicode_literals import os import re +import taskcluster_urls from taskgraph.util.schema import Schema from voluptuous import Any, Optional, Required @@ -72,21 +73,17 @@ def docker_worker_debian_package(config, job, taskdesc): name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1) + docker_repo = 'debian' arch = run.get('arch', 'amd64') + if arch != 'amd64': + docker_repo = '{}/{}'.format(arch, docker_repo) worker = taskdesc['worker'] worker['artifacts'] = [] - version = { - 'wheezy': 7, - 'jessie': 8, - 'stretch': 9, - 'buster': 10, - }[run['dist']] - image = 'debian%d' % version - if arch != 'amd64': - image += '-' + arch - image += '-packages' - worker['docker-image'] = {'in-tree': image} + worker['docker-image'] = '{repo}:{dist}-{date}'.format( + repo=docker_repo, + dist=run['dist'], + date=run['snapshot'][:8]) # Retry on apt-get errors. worker['retry-exit-status'] = [100] @@ -113,6 +110,13 @@ def docker_worker_debian_package(config, job, taskdesc): package = package_re.match(src_file).group(0) unpack = unpack.format(src_file=src_file, package=package) + base_deps = [ + 'apt-utils', + 'build-essential', + 'devscripts', + 'fakeroot', + ] + resolver = run.get('resolver', 'apt-get') if resolver == 'apt-get': resolver = 'apt-get -yyq --no-install-recommends' @@ -120,13 +124,15 @@ def docker_worker_debian_package(config, job, taskdesc): resolver = ('aptitude -y --without-recommends -o ' 'Aptitude::ProblemResolver::Hints::KeepBuildDeps=' '"reject {}-build-deps :UNINST"').format(package) + base_deps.append('aptitude') else: raise RuntimeError('Unreachable') adjust = '' if 'patch' in run: - # We don't use robustcheckout or run-task to get a checkout. So for - # this one file we'd need from a checkout, download it. + # We can't depend on docker images, so we don't have robustcheckout or + # or run-task to get a checkout. So for this one file we'd need + # from a checkout, download it. env['PATCH_URL'] = '{head_repo}/raw-file/{head_rev}/build/debian-packages/{patch}'.format( head_repo=config.params['head_repository'], head_rev=config.params['head_rev'], @@ -148,15 +154,33 @@ def docker_worker_debian_package(config, job, taskdesc): dist=run['dist'], ) + queue_url = taskcluster_urls.api(get_root_url(), 'queue', 'v1', '') + + # We can't depend on docker images (since docker images depend on packages), + # so we inline the whole script here. worker['command'] = [ 'sh', '-x', '-c', + # Fill /etc/apt/sources.list with the relevant snapshot repository. + 'echo "deb http://snapshot.debian.org/archive/debian' + '/{snapshot}/ {dist} main" > /etc/apt/sources.list && ' + 'echo "deb http://snapshot.debian.org/archive/debian' + '/{snapshot}/ {dist}-updates main" >> /etc/apt/sources.list && ' + 'echo "deb http://snapshot.debian.org/archive/debian' + '/{snapshot}/ {dist}-backports main" >> /etc/apt/sources.list && ' + 'echo "deb http://snapshot.debian.org/archive/debian-security' + '/{snapshot}/ {dist}/updates main" >> /etc/apt/sources.list && ' + 'apt-get update -o Acquire::Check-Valid-Until=false -q && ' # Add sources for packages coming from other package tasks. - '/usr/local/sbin/setup_packages.sh {queue_url} $PACKAGES && ' - 'apt-get update && ' - # Upgrade packages that might have new versions in package tasks. - 'apt-get dist-upgrade && ' + 'apt-get install -yyq apt-transport-https ca-certificates && ' + 'for task in $PACKAGES; do ' + ' echo "deb [trusted=yes] {queue_url}task/$task/artifacts/public/build/ debian/" ' + '>> /etc/apt/sources.list; ' + 'done && ' + # Install the base utilities required to build debian packages. + 'apt-get update -o Acquire::Check-Valid-Until=false -q && ' + 'apt-get install -yyq {base_deps} && ' 'cd /tmp && ' # Get, validate and extract the package source. 'dget -d -u {src_url} && ' @@ -177,7 +201,7 @@ def docker_worker_debian_package(config, job, taskdesc): 'apt-ftparchive sources debian | gzip -c9 > debian/Sources.gz && ' 'apt-ftparchive packages debian | gzip -c9 > debian/Packages.gz' .format( - queue_url=get_root_url(), + queue_url=queue_url, package=package, snapshot=run['snapshot'], dist=run['dist'], @@ -187,6 +211,7 @@ def docker_worker_debian_package(config, job, taskdesc): unpack=unpack, adjust=adjust, artifacts='/tmp/artifacts', + base_deps=' '.join(base_deps), resolver=resolver, ) ] @@ -208,6 +233,9 @@ def docker_worker_debian_package(config, job, taskdesc): digest_data.append( hash_path(os.path.join(GECKO, 'build', 'debian-packages', run['patch']))) + if docker_repo != 'debian': + digest_data.append(docker_repo) + if not taskgraph.fast: taskdesc['cache'] = { 'type': 'packages.v1',