зеркало из https://github.com/mozilla/gecko-dev.git
bug 1477021: create a docker image that can update Pipfile.lock, and attach diffs to phabricator. r=sfraser
--HG-- extra : rebase_source : 15c8c6ea7f2124863f8e9198a6962cbb37a28ab2
This commit is contained in:
Ben Hearsum
Родитель
505d868007
Коммит
9d82e727e0
|
|
@ -105,5 +105,7 @@ jobs:
|
|||
definition: partner-repack
|
||||
periodic-updates:
|
||||
symbol: I(file)
|
||||
pipfile-updates:
|
||||
symbol: I(pip)
|
||||
firefox-snap:
|
||||
symbol: I(snap)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
FROM ubuntu:bionic
|
||||
MAINTAINER Ben Hearsum <bhearsum@mozilla.com>
|
||||
|
||||
# Required software
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
# %include python/mozbuild/mozbuild/action/tooltool.py
|
||||
ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
|
||||
|
||||
# %include testing/mozharness/external_tools/robustcheckout.py
|
||||
ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
|
||||
|
||||
# %include taskcluster/docker/recipes/hgrc
|
||||
COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
|
||||
|
||||
# %include taskcluster/docker/recipes/install-mercurial.sh
|
||||
ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
|
||||
|
||||
ADD setup.sh /setup/setup.sh
|
||||
|
||||
RUN cd /setup && ./setup.sh
|
||||
|
||||
COPY runme.sh /
|
||||
COPY scripts/* /home/worker/scripts/
|
||||
|
||||
ENV HOME /home/worker
|
||||
ENV SHELL /bin/bash
|
||||
ENV USER worker
|
||||
ENV LOGNAME worker
|
||||
ENV LC_ALL C.UTF-8
|
||||
ENV LANG C.UTF-8
|
||||
|
||||
CMD ["/runme.sh"]
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
|
||||
==Pipfile Updates==
|
||||
|
||||
This docker image contains the necessary dependencies and scripts to update in-tree Pipfile.lock's,
|
||||
produce a diff, and submit it to Phabricator.
|
||||
|
||||
|
||||
==Quick Start==
|
||||
|
||||
```sh
|
||||
docker build -t pipfile-update --no-cache --rm .
|
||||
|
||||
docker run -e PYTHON3="1" -e BRANCH="mozilla-central" -e PIPFILE_DIRECTORY="taskcluster/docker/funsize-update-generator" pipfile-update
|
||||
```
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
# Things to be set by task definition.
|
||||
# -b branch
|
||||
# -p pipfile_directory
|
||||
# -3 use python3
|
||||
|
||||
|
||||
test "${BRANCH}"
|
||||
test "${PIPFILE_DIRECTORY}"
|
||||
|
||||
PIP_ARG="-2"
|
||||
if [ -n "${PYTHON3}" ]; then
|
||||
PIP_ARG="-3"
|
||||
fi
|
||||
|
||||
export ARTIFACTS_DIR="/home/worker/artifacts"
|
||||
mkdir -p "$ARTIFACTS_DIR"
|
||||
|
||||
# Get Arcanist API token
|
||||
|
||||
if [ -n "${TASK_ID}" ]
|
||||
then
|
||||
curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json \
|
||||
"https://queue.taskcluster.net/v1/task/$TASK_ID"
|
||||
ARC_SECRET=$(jq -r '.scopes[] | select(contains ("arc-phabricator-token"))' /home/worker/task.json | awk -F: '{print $3}')
|
||||
fi
|
||||
if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster
|
||||
then
|
||||
set +x # Don't echo these
|
||||
secrets_url="http://taskcluster/secrets/v1/secret/${ARC_SECRET}"
|
||||
SECRET=$(curl "${secrets_url}")
|
||||
TOKEN=$(echo "${SECRET}" | jq -r '.secret.token')
|
||||
elif [ -n "${ARC_TOKEN}" ] # Allow for local testing.
|
||||
then
|
||||
TOKEN="${ARC_TOKEN}"
|
||||
fi
|
||||
|
||||
if [ -n "${TOKEN}" ]
|
||||
then
|
||||
cat >"${HOME}/.arcrc" <<END
|
||||
{
|
||||
"hosts": {
|
||||
"https://phabricator.services.mozilla.com/api/": {
|
||||
"token": "${TOKEN}"
|
||||
}
|
||||
}
|
||||
}
|
||||
END
|
||||
set -x
|
||||
chmod 600 "${HOME}/.arcrc"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
/home/worker/scripts/update_pipfiles.sh -b "${BRANCH}" -p "${PIPFILE_DIRECTORY}" ${PIP_ARG}
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
function usage {
|
||||
cat <<EOF
|
||||
|
||||
Usage: $(basename "$0") -h # Displays this usage/help text
|
||||
Usage: $(basename "$0") -x # lists exit codes
|
||||
Usage: $(basename "$0") -b branch -p pipfile_directory [-2] [-3]
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
BRANCH=""
|
||||
PIP=""
|
||||
COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>'
|
||||
REPODIR=''
|
||||
HGHOST="hg.mozilla.org"
|
||||
BASEDIR="${HOME}"
|
||||
PIPFILE_DIRECTORY=""
|
||||
DIFF_ARTIFACT="${ARTIFACTS_DIR}/Pipfile.lock.diff"
|
||||
|
||||
HG="$(command -v hg)"
|
||||
|
||||
# Clones an hg repo
|
||||
function clone_repo {
|
||||
cd "${BASEDIR}"
|
||||
if [ ! -d "${REPODIR}" ]; then
|
||||
CLONE_CMD="${HG} clone ${HGREPO} ${REPODIR}"
|
||||
${CLONE_CMD}
|
||||
fi
|
||||
|
||||
${HG} -R ${REPODIR} pull
|
||||
${HG} -R ${REPODIR} update -C default
|
||||
}
|
||||
|
||||
# Push all pending commits to Phabricator
|
||||
function push_repo {
|
||||
cd "${REPODIR}"
|
||||
if [ ! -r "${HOME}/.arcrc" ]
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
if ! ARC=$(command -v arc)
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
if [ -z "${REVIEWERS}" ]
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
# Clean up older review requests
|
||||
# Turn Needs Review D624: No bug, Automated HSTS ...
|
||||
# into D624
|
||||
for diff in $($ARC list | grep "Needs Review" | grep "${PIPFILE_DIRECTORY} pipfile-update" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH) }')
|
||||
do
|
||||
echo "Removing old request $diff"
|
||||
# There is no 'arc abandon', see bug 1452082
|
||||
echo '{"transactions": [{"type":"abandon"}], "objectIdentifier": "'"${diff}"'"}' | arc call-conduit differential.revision.edit
|
||||
done
|
||||
|
||||
$ARC diff --verbatim --reviewers "${REVIEWERS}"
|
||||
}
|
||||
|
||||
function update_pipfile {
|
||||
pushd "${REPODIR}/${1}"
|
||||
pipenv update
|
||||
popd
|
||||
}
|
||||
|
||||
# Main
|
||||
|
||||
# Parse our command-line options.
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-h) usage; exit 0 ;;
|
||||
-b) BRANCH="$2"; shift ;;
|
||||
-r) REPODIR="$2"; shift ;;
|
||||
-2) PIP="pip" ;;
|
||||
-3) PIP="pip3" ;;
|
||||
-p) PIPFILE_DIRECTORY="$2"; shift ;;
|
||||
-*) usage
|
||||
exit 11 ;;
|
||||
*) break ;; # terminate while loop
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Must supply a code branch to work with.
|
||||
if [ "${PIP}" == "" ]; then
|
||||
echo "Error: You must specify a python version with -2 or -3" >&2
|
||||
usage
|
||||
exit 12
|
||||
fi
|
||||
|
||||
# Must supply a code branch to work with.
|
||||
if [ "${BRANCH}" == "" ]; then
|
||||
echo "Error: You must specify a branch with -b branchname." >&2
|
||||
usage
|
||||
exit 13
|
||||
fi
|
||||
|
||||
if [ "${REPODIR}" == "" ]; then
|
||||
REPODIR="$(basename "${BRANCH}")"
|
||||
fi
|
||||
|
||||
if [ "${BRANCH}" == "mozilla-central" ]; then
|
||||
HGREPO="https://${HGHOST}/${BRANCH}"
|
||||
elif [[ "${BRANCH}" == mozilla-* ]]; then
|
||||
HGREPO="https://${HGHOST}/releases/${BRANCH}"
|
||||
else
|
||||
HGREPO="https://${HGHOST}/projects/${BRANCH}"
|
||||
fi
|
||||
|
||||
clone_repo
|
||||
|
||||
${PIP} install pipenv
|
||||
|
||||
update_pipfile "${PIPFILE_DIRECTORY}"
|
||||
echo "INFO: diffing old/new Pipfile.lock into ${DIFF_ARTIFACT}"
|
||||
hg -R "${REPODIR}" diff "${BASEDIR}/${BRANCH}/${PIPFILE_DIRECTORY}/Pipfile.lock" | tee "${DIFF_ARTIFACT}"
|
||||
|
||||
COMMIT_MESSAGE="No Bug, ${PIPFILE_DIRECTORY} pipfile-update."
|
||||
|
||||
if ${HG} -R "${REPODIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}"
|
||||
then
|
||||
${HG} -R "${REPODIR}" out
|
||||
push_repo
|
||||
fi
|
||||
|
||||
echo "All done"
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
#!/bin/bash
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
set -ve
|
||||
|
||||
tooltool_fetch() {
|
||||
cat >manifest.tt
|
||||
python2.7 /setup/tooltool.py fetch
|
||||
rm manifest.tt
|
||||
}
|
||||
|
||||
useradd -d /home/worker -s /bin/bash -m worker
|
||||
|
||||
apt-get update -q
|
||||
apt-get install -y --no-install-recommends \
|
||||
arcanist \
|
||||
curl \
|
||||
gcc \
|
||||
jq \
|
||||
libdpkg-perl \
|
||||
liblzma-dev \
|
||||
python \
|
||||
python-dev \
|
||||
python-pip \
|
||||
python3 \
|
||||
python3-dev \
|
||||
python3-pip
|
||||
|
||||
apt-get clean
|
||||
|
||||
. install-mercurial.sh
|
||||
|
||||
rm -rf /setup
|
||||
Загрузка…
Ссылка в новой задаче