From 9d82e727e0efc4f4278f7007b2da5ea490f93cc5 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 26 Jul 2018 08:54:45 -0400 Subject: [PATCH] bug 1477021: create a docker image that can update Pipfile.lock, and attach diffs to phabricator. r=sfraser --HG-- extra : rebase_source : 15c8c6ea7f2124863f8e9198a6962cbb37a28ab2 --- taskcluster/ci/docker-image/kind.yml | 2 + taskcluster/docker/pipfile-updates/Dockerfile | 33 +++++ taskcluster/docker/pipfile-updates/README.md | 14 ++ taskcluster/docker/pipfile-updates/runme.sh | 57 ++++++++ .../scripts/update_pipfiles.sh | 132 ++++++++++++++++++ taskcluster/docker/pipfile-updates/setup.sh | 35 +++++ 6 files changed, 273 insertions(+) create mode 100644 taskcluster/docker/pipfile-updates/Dockerfile create mode 100644 taskcluster/docker/pipfile-updates/README.md create mode 100755 taskcluster/docker/pipfile-updates/runme.sh create mode 100755 taskcluster/docker/pipfile-updates/scripts/update_pipfiles.sh create mode 100755 taskcluster/docker/pipfile-updates/setup.sh diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index e4d545e7a081..ba0919ed6c1c 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -105,5 +105,7 @@ jobs: definition: partner-repack periodic-updates: symbol: I(file) + pipfile-updates: + symbol: I(pip) firefox-snap: symbol: I(snap) diff --git a/taskcluster/docker/pipfile-updates/Dockerfile b/taskcluster/docker/pipfile-updates/Dockerfile new file mode 100644 index 000000000000..cb4ac2a3813e --- /dev/null +++ b/taskcluster/docker/pipfile-updates/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:bionic +MAINTAINER Ben Hearsum + +# Required software +ENV DEBIAN_FRONTEND noninteractive + +# %include python/mozbuild/mozbuild/action/tooltool.py +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py + +# %include testing/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py + +# %include taskcluster/docker/recipes/hgrc +COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc + +# %include taskcluster/docker/recipes/install-mercurial.sh +ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh + +ADD setup.sh /setup/setup.sh + +RUN cd /setup && ./setup.sh + +COPY runme.sh / +COPY scripts/* /home/worker/scripts/ + +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +CMD ["/runme.sh"] diff --git a/taskcluster/docker/pipfile-updates/README.md b/taskcluster/docker/pipfile-updates/README.md new file mode 100644 index 000000000000..b0690ecf6138 --- /dev/null +++ b/taskcluster/docker/pipfile-updates/README.md @@ -0,0 +1,14 @@ + +==Pipfile Updates== + +This docker image contains the necessary dependencies and scripts to update in-tree Pipfile.lock's, +produce a diff, and submit it to Phabricator. + + +==Quick Start== + +```sh +docker build -t pipfile-update --no-cache --rm . + +docker run -e PYTHON3="1" -e BRANCH="mozilla-central" -e PIPFILE_DIRECTORY="taskcluster/docker/funsize-update-generator" pipfile-update +``` diff --git a/taskcluster/docker/pipfile-updates/runme.sh b/taskcluster/docker/pipfile-updates/runme.sh new file mode 100755 index 000000000000..2181152ce4a4 --- /dev/null +++ b/taskcluster/docker/pipfile-updates/runme.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -xe + +# Things to be set by task definition. +# -b branch +# -p pipfile_directory +# -3 use python3 + + +test "${BRANCH}" +test "${PIPFILE_DIRECTORY}" + +PIP_ARG="-2" +if [ -n "${PYTHON3}" ]; then + PIP_ARG="-3" +fi + +export ARTIFACTS_DIR="/home/worker/artifacts" +mkdir -p "$ARTIFACTS_DIR" + +# Get Arcanist API token + +if [ -n "${TASK_ID}" ] +then + curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json \ + "https://queue.taskcluster.net/v1/task/$TASK_ID" + ARC_SECRET=$(jq -r '.scopes[] | select(contains ("arc-phabricator-token"))' /home/worker/task.json | awk -F: '{print $3}') +fi +if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster +then + set +x # Don't echo these + secrets_url="http://taskcluster/secrets/v1/secret/${ARC_SECRET}" + SECRET=$(curl "${secrets_url}") + TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') +elif [ -n "${ARC_TOKEN}" ] # Allow for local testing. +then + TOKEN="${ARC_TOKEN}" +fi + +if [ -n "${TOKEN}" ] +then + cat >"${HOME}/.arcrc" <&2 + usage + exit 12 +fi + +# Must supply a code branch to work with. +if [ "${BRANCH}" == "" ]; then + echo "Error: You must specify a branch with -b branchname." >&2 + usage + exit 13 +fi + +if [ "${REPODIR}" == "" ]; then + REPODIR="$(basename "${BRANCH}")" +fi + +if [ "${BRANCH}" == "mozilla-central" ]; then + HGREPO="https://${HGHOST}/${BRANCH}" +elif [[ "${BRANCH}" == mozilla-* ]]; then + HGREPO="https://${HGHOST}/releases/${BRANCH}" +else + HGREPO="https://${HGHOST}/projects/${BRANCH}" +fi + +clone_repo + +${PIP} install pipenv + +update_pipfile "${PIPFILE_DIRECTORY}" +echo "INFO: diffing old/new Pipfile.lock into ${DIFF_ARTIFACT}" +hg -R "${REPODIR}" diff "${BASEDIR}/${BRANCH}/${PIPFILE_DIRECTORY}/Pipfile.lock" | tee "${DIFF_ARTIFACT}" + +COMMIT_MESSAGE="No Bug, ${PIPFILE_DIRECTORY} pipfile-update." + +if ${HG} -R "${REPODIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}" +then + ${HG} -R "${REPODIR}" out + push_repo +fi + +echo "All done" diff --git a/taskcluster/docker/pipfile-updates/setup.sh b/taskcluster/docker/pipfile-updates/setup.sh new file mode 100755 index 000000000000..f3fb5192f0ba --- /dev/null +++ b/taskcluster/docker/pipfile-updates/setup.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -ve + +tooltool_fetch() { + cat >manifest.tt + python2.7 /setup/tooltool.py fetch + rm manifest.tt +} + +useradd -d /home/worker -s /bin/bash -m worker + +apt-get update -q +apt-get install -y --no-install-recommends \ + arcanist \ + curl \ + gcc \ + jq \ + libdpkg-perl \ + liblzma-dev \ + python \ + python-dev \ + python-pip \ + python3 \ + python3-dev \ + python3-pip + +apt-get clean + +. install-mercurial.sh + +rm -rf /setup