From b3bebb6ecaa8f2346dabb41c4673ea6d173bb5f6 Mon Sep 17 00:00:00 2001 From: Kartikaya Gupta Date: Tue, 16 Jul 2019 14:09:47 +0000 Subject: [PATCH] Bug 1523321 - Run the wrupdater on m-c pushes that touch gfx/wr. r=tomprince,jrmuizel Differential Revision: https://phabricator.services.mozilla.com/D33548 --HG-- extra : moz-landing-system : lando --- build/sparse-profiles/docker-image | 3 ++ gfx/wr/ci-scripts/wrupdater/docker-setup.sh | 31 +++++++++++++++++++ gfx/wr/ci-scripts/wrupdater/requirements.txt | 3 ++ taskcluster/ci/docker-image/kind.yml | 3 ++ taskcluster/ci/webrender/kind.yml | 21 +++++++++++++ .../docker/webrender-updater/Dockerfile | 17 ++++++++++ taskcluster/taskgraph/transforms/webrender.py | 27 ++++++++++++++++ 7 files changed, 105 insertions(+) create mode 100755 gfx/wr/ci-scripts/wrupdater/docker-setup.sh create mode 100644 gfx/wr/ci-scripts/wrupdater/requirements.txt create mode 100644 taskcluster/docker/webrender-updater/Dockerfile create mode 100644 taskcluster/taskgraph/transforms/webrender.py diff --git a/build/sparse-profiles/docker-image b/build/sparse-profiles/docker-image index b9a74e83349d..20bb5e690bb0 100644 --- a/build/sparse-profiles/docker-image +++ b/build/sparse-profiles/docker-image @@ -19,3 +19,6 @@ path:tools/lint/tox/tox_requirements.txt # Required for the webrender docker image path:gfx/wr/ci-scripts/docker-image/setup.sh +# Required for the webrender-updater docker image +path:gfx/wr/ci-scripts/wrupdater/docker-setup.sh +path:gfx/wr/ci-scripts/wrupdater/requirements.txt diff --git a/gfx/wr/ci-scripts/wrupdater/docker-setup.sh b/gfx/wr/ci-scripts/wrupdater/docker-setup.sh new file mode 100755 index 000000000000..8550dea3bc60 --- /dev/null +++ b/gfx/wr/ci-scripts/wrupdater/docker-setup.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +test "$(whoami)" == 'root' + +# Install stuff we need +apt-get -y update +apt-get install -y \ + cmake \ + curl \ + gcc \ + git \ + g++ \ + libffi-dev \ + libgit2-dev/stretch-backports \ + libssl-dev \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools + +# Python packages +pip3 install -r /tmp/requirements.txt diff --git a/gfx/wr/ci-scripts/wrupdater/requirements.txt b/gfx/wr/ci-scripts/wrupdater/requirements.txt new file mode 100644 index 000000000000..fcd6117ab1c9 --- /dev/null +++ b/gfx/wr/ci-scripts/wrupdater/requirements.txt @@ -0,0 +1,3 @@ +requests==2.21.0 +pygit2==0.27.0 # this requires libgit2 v0.27.* +python-hglib==2.6.1 diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index 013315f06501..942828baf7e5 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -202,3 +202,6 @@ jobs: webrender: symbol: I(webrender) parent: debian9-base + webrender-updater: + symbol: I(wrupdater) + parent: debian9-base diff --git a/taskcluster/ci/webrender/kind.yml b/taskcluster/ci/webrender/kind.yml index 1a4ab7b2a505..82e568805152 100644 --- a/taskcluster/ci/webrender/kind.yml +++ b/taskcluster/ci/webrender/kind.yml @@ -8,6 +8,7 @@ kind-dependencies: - toolchain transforms: + - taskgraph.transforms.webrender:transforms - taskgraph.transforms.use_toolchains:transforms - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms @@ -462,3 +463,23 @@ jobs: - 'gfx/wr/**' - 'testing/mozharness/scripts/android_wrench.py' - 'testing/mozharness/configs/android/wrench.py' + + wrupdater-sync: + run-on-projects: + - mozilla-central + description: Sync webrender to github mirror + worker-type: b-linux + worker: + max-run-time: 3600 + docker-image: {in-tree: webrender-updater} + wrupdater-secrets: true + run: + using: run-task + command: '$GECKO_PATH/gfx/wr/ci-scripts/wrupdater/sync-to-github.sh' + treeherder: + platform: linux64-qr/opt + symbol: WR(sync) + tier: 2 + when: + files-changed: + - 'gfx/wr/**' diff --git a/taskcluster/docker/webrender-updater/Dockerfile b/taskcluster/docker/webrender-updater/Dockerfile new file mode 100644 index 000000000000..4f908bc12354 --- /dev/null +++ b/taskcluster/docker/webrender-updater/Dockerfile @@ -0,0 +1,17 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Kartikaya Gupta + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache +VOLUME /builds/worker/.wrupdater + +# %include gfx/wr/ci-scripts/wrupdater/docker-setup.sh +ADD topsrcdir/gfx/wr/ci-scripts/wrupdater/docker-setup.sh /tmp/wrup-setup.sh +# %include gfx/wr/ci-scripts/wrupdater/requirements.txt +ADD topsrcdir/gfx/wr/ci-scripts/wrupdater/requirements.txt /tmp/requirements.txt +RUN /bin/bash /tmp/wrup-setup.sh && rm /tmp/wrup-setup.sh + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/taskgraph/transforms/webrender.py b/taskcluster/taskgraph/transforms/webrender.py new file mode 100644 index 000000000000..7fe469e7b7e8 --- /dev/null +++ b/taskcluster/taskgraph/transforms/webrender.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + + +@transforms.add +def webrender(config, tasks): + """Do transforms specific to webrender tasks. + """ + for task in tasks: + # Replace the worker attribute `wrupdater-secrets: true` with scopes + # to access the github token, but only on m-c. Doing this on any other + # tree will result in decision task failure because m-c is the only one + # allowed to have that scope. + worker = task['worker'] + if worker.get('wrupdater-secrets', False): + del worker['wrupdater-secrets'] + if config.params['project'] == 'mozilla-central': + task.setdefault('scopes', []) + task['scopes'].append('secrets:get:project/webrender-ci/wrupdater-github-token') + yield task