From 8c4ff0fb122a0211645f6bb49d25feb90353a3ca Mon Sep 17 00:00:00 2001 From: Noemi Erli Date: Fri, 24 Jan 2020 17:48:10 +0200 Subject: [PATCH] Backed out changeset cf3d74d0cf82 per Callek's request DONTBUILD CLOSED TREE --- taskcluster/docker/debian-raw/Dockerfile | 2 ++ .../docker/debian-raw/cloud-mirror-workaround.sh | 10 ++++++++++ taskcluster/docker/debian-raw/setup_packages.sh | 6 +++++- taskcluster/docker/funsize-update-generator/README | 2 +- taskcluster/docker/funsize-update-generator/runme.sh | 6 +++++- .../docker/funsize-update-generator/scripts/funsize.py | 4 +++- taskcluster/docker/periodic-updates/README.md | 4 ++-- taskcluster/docker/periodic-updates/runme.sh | 6 +++++- .../periodic-updates/scripts/periodic_file_updates.sh | 9 +++++++-- taskcluster/docker/pipfile-updates/runme.sh | 6 +++++- taskcluster/docs/actions.rst | 2 +- taskcluster/docs/cron.rst | 6 +++--- taskcluster/docs/signing.rst | 6 +++--- taskcluster/scripts/misc/build-libdmg-hfsplus.sh | 6 +++++- taskcluster/scripts/misc/fetch-content | 3 +++ taskcluster/taskgraph/__init__.py | 2 +- taskcluster/taskgraph/transforms/task.py | 2 +- taskcluster/taskgraph/util/verify.py | 2 +- 18 files changed, 63 insertions(+), 21 deletions(-) create mode 100755 taskcluster/docker/debian-raw/cloud-mirror-workaround.sh diff --git a/taskcluster/docker/debian-raw/Dockerfile b/taskcluster/docker/debian-raw/Dockerfile index 27f8b6831da3..da5a7fffa737 100644 --- a/taskcluster/docker/debian-raw/Dockerfile +++ b/taskcluster/docker/debian-raw/Dockerfile @@ -8,6 +8,7 @@ ENV DEBIAN_FRONTEND=noninteractive CMD ["/bin/bash", "--login"] COPY setup_packages.sh /usr/local/sbin/ +COPY cloud-mirror-workaround.sh /usr/local/sbin/ # %ARG DIST # %ARG SNAPSHOT @@ -20,6 +21,7 @@ RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-sec echo 'APT::Install-Recommends "false";'; \ echo 'Acquire::Check-Valid-Until "false";'; \ echo 'Acquire::Retries "5";'; \ + echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";'; \ ) > /etc/apt/apt.conf.d/99taskcluster RUN apt-get update && \ diff --git a/taskcluster/docker/debian-raw/cloud-mirror-workaround.sh b/taskcluster/docker/debian-raw/cloud-mirror-workaround.sh new file mode 100755 index 000000000000..53819a4d7f9b --- /dev/null +++ b/taskcluster/docker/debian-raw/cloud-mirror-workaround.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# The apt in Debian stretch mishandles the HTTP redirects from queue.taskcluster.net to +# cloud-mirror.taskcluster.net, and unescapes the url. This apt method wrapper +# strips redirections and sends directly to the S3 bucket. This has the downside of +# always hitting us-west-2 independently of the zone this runs on, but this is only +# used when creating docker images, so shouldn't generate huge cross-AWS-zone S3 +# transfers. + +/usr/lib/apt/methods/https | sed -u 's,^New-URI: https://cloud-mirror.taskcluster.net/v1/redirect/[^/]*/[^/]*/https://,New-URI: https://,' diff --git a/taskcluster/docker/debian-raw/setup_packages.sh b/taskcluster/docker/debian-raw/setup_packages.sh index 2f61283528bb..f75ce742405c 100755 --- a/taskcluster/docker/debian-raw/setup_packages.sh +++ b/taskcluster/docker/debian-raw/setup_packages.sh @@ -4,7 +4,11 @@ TASKCLUSTER_ROOT_URL=$1 shift # duplicate the functionality of taskcluster-lib-urls, but in bash.. -queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +if [ "$TASKCLUSTER_ROOT_URL" = "https://taskcluster.net" ]; then + queue_base='https://queue.taskcluster.net/v1' +else + queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +fi for task in "$@"; do diff --git a/taskcluster/docker/funsize-update-generator/README b/taskcluster/docker/funsize-update-generator/README index 921fafa58776..96417e97f86b 100644 --- a/taskcluster/docker/funsize-update-generator/README +++ b/taskcluster/docker/funsize-update-generator/README @@ -3,6 +3,6 @@ To run this locally for testing/development purposes: 1. Find a funsize generating task ID 2. make pull DOCKERIO_USERNAME=mozillareleases -3. docker run -t -e TASKCLUSTER_ROOT_URL="https://firefox-ci-tc.services.mozilla.com/" -e SHA1_SIGNING_CERT='nightly_sha1' -e SHA384_SIGNING_CERT='nightly_sha384' -e TASK_ID="${TASK_ID}" -e EXTRA_PARAMS="--arch=x86_64" mozillareleases/funsize-update-generator /runme.sh +3. docker run -t -e TASKCLUSTER_ROOT_URL="https://taskcluster.net" -e SHA1_SIGNING_CERT='nightly_sha1' -e SHA384_SIGNING_CERT='nightly_sha384' -e TASK_ID="${TASK_ID}" -e EXTRA_PARAMS="--arch=x86_64" mozillareleases/funsize-update-generator /runme.sh The TASK_ID should be a recent "partials" Task. diff --git a/taskcluster/docker/funsize-update-generator/runme.sh b/taskcluster/docker/funsize-update-generator/runme.sh index f755594aa89e..ccde49365e3e 100644 --- a/taskcluster/docker/funsize-update-generator/runme.sh +++ b/taskcluster/docker/funsize-update-generator/runme.sh @@ -10,7 +10,11 @@ ARTIFACTS_DIR="/home/worker/artifacts" mkdir -p "$ARTIFACTS_DIR" # duplicate the functionality of taskcluster-lib-urls, but in bash.. -queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +if [ "$TASKCLUSTER_ROOT_URL" = "https://taskcluster.net" ]; then + queue_base='https://queue.taskcluster.net/v1' +else + queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +fi curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID" diff --git a/taskcluster/docker/funsize-update-generator/scripts/funsize.py b/taskcluster/docker/funsize-update-generator/scripts/funsize.py index 4f47d29dc5a6..18e208501797 100755 --- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py +++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py @@ -28,7 +28,9 @@ log = logging.getLogger(__name__) ROOT_URL = os.environ['TASKCLUSTER_ROOT_URL'] -QUEUE_PREFIX = ROOT_URL + '/api/queue/' +QUEUE_PREFIX = ("https://queue.taskcluster.net/" + if ROOT_URL == 'https://taskcluster.net' + else ROOT_URL + '/api/queue/') ALLOWED_URL_PREFIXES = ( "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/", "http://download.cdn.mozilla.net/pub/firefox/nightly/", diff --git a/taskcluster/docker/periodic-updates/README.md b/taskcluster/docker/periodic-updates/README.md index 6e1a45aad53c..ad021d979a78 100644 --- a/taskcluster/docker/periodic-updates/README.md +++ b/taskcluster/docker/periodic-updates/README.md @@ -46,7 +46,7 @@ up to date. ==Example Taskcluster Task== -https://firefox-ci-tc.services.mozilla.com/tasks/create/ +https://tools.taskcluster.net/tasks/create ```yaml provisionerId: aws-provisioner-v1 @@ -84,7 +84,7 @@ metadata: name: Periodic updates testing description: Produce diffs for HSTS and HPKP in-tree files. owner: sfraser@mozilla.com - source: 'https://firefox-ci-tc.services.mozilla.com/tasks/create' + source: 'https://tools.taskcluster.net/task-creator/' tags: {} extra: treeherder: diff --git a/taskcluster/docker/periodic-updates/runme.sh b/taskcluster/docker/periodic-updates/runme.sh index f1890fa41149..232940331923 100755 --- a/taskcluster/docker/periodic-updates/runme.sh +++ b/taskcluster/docker/periodic-updates/runme.sh @@ -57,7 +57,11 @@ export ARTIFACTS_DIR="/home/worker/artifacts" mkdir -p "$ARTIFACTS_DIR" # duplicate the functionality of taskcluster-lib-urls, but in bash.. -queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +if [ "$TASKCLUSTER_ROOT_URL" = "https://taskcluster.net" ]; then + queue_base='https://queue.taskcluster.net/v1' +else + queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +fi # Get Arcanist API token diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh index 62d3d9f3d62b..dc22c951246f 100755 --- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh +++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh @@ -95,8 +95,13 @@ REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT: SUFFIX_LIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${SUFFIX_LIST_DIFF_ARTIFACT:-"effective_tld_names.diff"}" # duplicate the functionality of taskcluster-lib-urls, but in bash.. -queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" -index_base="$TASKCLUSTER_ROOT_URL/api/index/v1" +if [ "$TASKCLUSTER_ROOT_URL" = "https://taskcluster.net" ]; then + queue_base='https://queue.taskcluster.net/v1' + index_base='https://index.taskcluster.net/v1' +else + queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" + index_base="$TASKCLUSTER_ROOT_URL/api/index/v1" +fi # Get the current in-tree version for a code branch. function get_version { diff --git a/taskcluster/docker/pipfile-updates/runme.sh b/taskcluster/docker/pipfile-updates/runme.sh index e91d0157ca2c..daed99d91398 100755 --- a/taskcluster/docker/pipfile-updates/runme.sh +++ b/taskcluster/docker/pipfile-updates/runme.sh @@ -20,7 +20,11 @@ export ARTIFACTS_DIR="/home/worker/artifacts" mkdir -p "$ARTIFACTS_DIR" # duplicate the functionality of taskcluster-lib-urls, but in bash.. -queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +if [ "$TASKCLUSTER_ROOT_URL" = "https://taskcluster.net" ]; then + queue_base='https://queue.taskcluster.net/v1' +else + queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +fi # Get Arcanist API token diff --git a/taskcluster/docs/actions.rst b/taskcluster/docs/actions.rst index 0d416192a4f2..a88eefc042d1 100644 --- a/taskcluster/docs/actions.rst +++ b/taskcluster/docs/actions.rst @@ -266,6 +266,6 @@ More Information For further details on actions in general, see `the actions.json spec`_. The hooks used for in-tree actions are set up by `ci-admin`_ based on configuration in `ci-configuration`_. -.. _the actions.json spec: https://firefox-ci-tc.services.mozilla.com/docs/manual/tasks/actions/spec +.. _the actions.json spec: https://docs.taskcluster.net/manual/tasks/actions/spec .. _ci-admin: http://hg.mozilla.org/ci/ci-admin/ .. _ci-configuration: http://hg.mozilla.org/ci/ci-configuration/ diff --git a/taskcluster/docs/cron.rst b/taskcluster/docs/cron.rst index 6f384aa6962a..603d6c8f74e8 100644 --- a/taskcluster/docs/cron.rst +++ b/taskcluster/docs/cron.rst @@ -17,9 +17,9 @@ meaning of this file. How It Works ------------ -The `TaskCluster Hooks Service `_ -has a hook configured for each repository supporting periodic task graphs. The -hook runs every 15 minutes, and the resulting task is referred to as a "cron task". +The `TaskCluster Hooks Service `_ has a +hook configured for each repository supporting periodic task graphs. The hook +runs every 15 minutes, and the resulting task is referred to as a "cron task". That cron task runs `./mach taskgraph cron` in a checkout of the Gecko source tree. diff --git a/taskcluster/docs/signing.rst b/taskcluster/docs/signing.rst index 8afc29a6b3a7..e077feb4dbf1 100644 --- a/taskcluster/docs/signing.rst +++ b/taskcluster/docs/signing.rst @@ -169,13 +169,13 @@ any other team. .. _addonscript: https://github.com/mozilla-releng/addonscript/ .. _code signing: https://en.wikipedia.org/wiki/Code_signing .. _chain of trust: https://scriptworker.readthedocs.io/en/latest/chain_of_trust.html -.. _depsigning: https://firefox-ci-tc.services.mozilla.com/provisioners/scriptworker-k8s/worker-types/gecko-t-signing +.. _depsigning: https://tools.taskcluster.net/provisioners/scriptworker-prov-v1/worker-types/depsigning .. _should_sign_windows: https://github.com/mozilla-releng/signingscript/blob/65cbb99ea53896fda9f4844e050a9695c762d24f/signingscript/sign.py#L369 .. _Encrypted Media Extensions: https://hacks.mozilla.org/2014/05/reconciling-mozillas-mission-and-w3c-eme/ .. _signing password files: https://github.com/mozilla/build-puppet/tree/feff5e12ab70f2c060b29940464e77208c7f0ef2/modules/signing_scriptworker/templates .. _signingscript: https://github.com/mozilla-releng/signingscript/ -.. _signing-linux-dev: https://firefox-ci-tc.services.mozilla.com/provisioners/scriptworker-k8s/worker-types/gecko-t-signing-dev -.. _signing-linux-v1: https://firefox-ci-tc.services.mozilla.com/provisioners/scriptworker-k8s/worker-types/gecko-3-signing +.. _signing-linux-dev: https://tools.taskcluster.net/provisioners/scriptworker-prov-v1/worker-types/signing-linux-dev +.. _signing-linux-v1: https://tools.taskcluster.net/provisioners/scriptworker-prov-v1/worker-types/signing-linux-v1 .. _signtool: https://github.com/mozilla-releng/signtool .. _Scriptworker: https://github.com/mozilla-releng/scriptworker/ .. _widevine site: https://www.widevine.com/wv_drm.html diff --git a/taskcluster/scripts/misc/build-libdmg-hfsplus.sh b/taskcluster/scripts/misc/build-libdmg-hfsplus.sh index c76b722d8d8a..ef1273b64805 100755 --- a/taskcluster/scripts/misc/build-libdmg-hfsplus.sh +++ b/taskcluster/scripts/misc/build-libdmg-hfsplus.sh @@ -19,7 +19,11 @@ strip dmg/dmg hfs/hfsplus cp dmg/dmg hfs/hfsplus $STAGE # duplicate the functionality of taskcluster-lib-urls, but in bash.. -queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +if [ "$TASKCLUSTER_ROOT_URL" = "https://taskcluster.net" ]; then + queue_base='https://queue.taskcluster.net/v1' +else + queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" +fi cat >$STAGE/README<