diff --git a/.taskcluster.yml b/.taskcluster.yml new file mode 100644 index 00000000..2cfe69e0 --- /dev/null +++ b/.taskcluster.yml @@ -0,0 +1,52 @@ +version: 0 + +metadata: + name: Normandy + description: Normandy CI Tasks + owner: "{{ event.head.user.email }}" + source: "{{ event.head.repo.url }}" + +tasks: + - metadata: + name: normandy:decision + description: Normandy decision task + owner: "{{ event.head.user.email }}" + source: "{{ event.head.repo.url }}" + provisionerId: "{{ taskcluster.docker.provisionerId }}" + workerType: "{{ taskcluster.docker.workerType }}" + scopes: + - queue:create-task:aws-provisioner-v1/github-worker + payload: + env: + maxRunTime: 300 # 5 minutes + image: ubuntu:zesty + features: + taskclusterProxy: true + command: + - "/bin/bash" + - "-c" + - >- + apt-get update && + apt-get install -y git python3.6 python3.6-venv && + cd ~ && + + git clone $GITHUB_HEAD_REPO_URL normandy && + pushd normandy && + git checkout $GITHUB_HEAD_SHA && + popd && + + pushd normandy/ci/taskcluster && + python3.6 -m venv venv && + source venv/bin/activate && + pip install -r requirements/default.txt -c requirements/constraints.txt && + popd && + + python3.6 normandy/ci/taskcluster/bin/decision.py + extra: + github: + env: true + events: + - pull_request.opened + - pull_request.synchronize + - push + - release diff --git a/ci/bin/pipstrap.py b/ci/circleci/bin/pipstrap.py similarity index 100% rename from ci/bin/pipstrap.py rename to ci/circleci/bin/pipstrap.py diff --git a/ci/bin/runner.sh b/ci/circleci/bin/runner.sh similarity index 93% rename from ci/bin/runner.sh rename to ci/circleci/bin/runner.sh index ea360f28..c64af4c4 100755 --- a/ci/bin/runner.sh +++ b/ci/circleci/bin/runner.sh @@ -6,7 +6,7 @@ if [[ $# -eq 0 ]]; then exit 1 fi -BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)" +BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../../.. && pwd)" PROJECTS=( recipe-client-addon recipe-server diff --git a/ci/requirements/constraints.txt b/ci/circleci/requirements/constraints.txt similarity index 100% rename from ci/requirements/constraints.txt rename to ci/circleci/requirements/constraints.txt diff --git a/ci/requirements/default.txt b/ci/circleci/requirements/default.txt similarity index 100% rename from ci/requirements/default.txt rename to ci/circleci/requirements/default.txt diff --git a/ci/requirements/pip.txt b/ci/circleci/requirements/pip.txt similarity index 100% rename from ci/requirements/pip.txt rename to ci/circleci/requirements/pip.txt diff --git a/ci/taskcluster/bin/decision.py b/ci/taskcluster/bin/decision.py new file mode 100755 index 00000000..5c05b677 --- /dev/null +++ b/ci/taskcluster/bin/decision.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +import os + +import requests +from taskcluster import fromNow +from taskcluster.utils import stableSlugId, dumpJson + +tasks = [ + { + 'name': 'recipe-client-addon:test', + 'description': 'Test recipe-client-addon with gecko-dev', + 'command': 'normandy/recipe-client-addon/bin/tc/test.sh', + 'env': { + 'GECKO_DEV_URL': 'https://github.com/mozilla/gecko-dev', + }, + }, + { + 'name': 'recipe-client-addon:make-xpi', + 'description': 'Build XPI for recipe-client-addon', + 'command': 'normandy/recipe-client-addon/bin/tc/make-xpi.sh', + 'dependencies': ['recipe-client-addon:test'], + }, +] + + +def main(): + _idMaker = stableSlugId() + + def idMaker(name): + return _idMaker(name).decode() + + decisionTaskId = os.environ['TASK_ID'] + + with requests.Session() as session: + for task in tasks: + dependencies = [idMaker(name) for name in task.get('dependencies', [])] + dependencies.append(decisionTaskId) + + env = task.get('env', {}) + for key, val in os.environ.items(): + if key.startswith('GITHUB_'): + env.setdefault(key, val) + + task_id = idMaker(task['name']) + res = session.put(f'http://taskcluster/queue/v1/task/{task_id}', data=dumpJson({ + 'metadata': { + 'name': task['name'], + 'description': task['description'], + 'owner': "mcooper@mozilla.com", + 'source': "https://github.com/mozilla/normandy", + }, + 'provisionerId': 'aws-provisioner-v1', + 'workerType': 'github-worker', + 'schedulerId': 'taskcluster-github', + 'taskGroupId': decisionTaskId, + 'created': fromNow('0 seconds'), + 'deadline': fromNow('4 hours'), + 'expires': fromNow('365 days'), + 'payload': { + 'image': 'ubuntu:zesty', + 'command': [ + '/bin/bash', + '-c', + ' && '.join([ + 'apt-get update', + 'apt-get install -y git', + 'mkdir /artifacts', + 'cd ~', + 'git clone $GITHUB_HEAD_REPO_URL normandy', + 'pushd normandy', + 'git checkout $GITHUB_HEAD_SHA', + 'popd', + task['command'], + ]) + ], + 'maxRunTime': 14400, # 4 hours + 'env': env, + 'artifacts': { + 'public': { + 'type': 'directory', + 'path': '/artifacts', + 'expires': fromNow('364 days'), # must expire before task + }, + }, + }, + 'dependencies': dependencies, + })) + print(res.text) + res.raise_for_status() + + +if __name__ == '__main__': + main() diff --git a/ci/taskcluster/requirements/constraints.txt b/ci/taskcluster/requirements/constraints.txt new file mode 100644 index 00000000..7b286cb0 --- /dev/null +++ b/ci/taskcluster/requirements/constraints.txt @@ -0,0 +1,21 @@ +mohawk==0.3.4 \ + --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3 +slugid==1.0.7 \ + --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779 +six==1.10.0 \ + --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \ + --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a +aiohttp==1.3.1 \ + --hash=sha256:8e71993f16870ebb1e8af21cb65dbac0cb6418ce9ab5a8cfaefce1c986a306bf \ + --hash=sha256:13f44a971da01093a5abcc6436596558e0c5bfac2532407a097be8607510ee1a \ + --hash=sha256:e147b0cea568773443683becce9de4071506431118609b5d477fe61508417af1 +async_timeout==1.1.0 \ + --hash=sha256:b88bd1fe001b800ec23c7bf27a81b32819e2a56668e9fba5646a7f3618143081 \ + --hash=sha256:d848623229614d2a239375f003c65a0ecf3bcfed3a2fdba6ce83b31a8038c2a0 +yarl==0.9.2 \ + --hash=sha256:4c63264bdf022e4ab509dd8946147dec026a4110c90e0a2dda9499385c312ece +multidict==2.1.4 \ + --hash=sha256:a77aa8c9f68846c3b5db43ff8ed2a7a884dbe845d01f55113a3fba78518c4cd7 +chardet==2.3.0 \ + --hash=sha256:aaf514bde38020b4f1e42c6a6e141f2827a8a58ccfc3b22b6ff5a1a4b50be56e \ + --hash=sha256:e53e38b3a4afe6d1132de62b7400a4ac363452dc5dfcf8d88e8e0cce663c68aa diff --git a/ci/taskcluster/requirements/default.txt b/ci/taskcluster/requirements/default.txt new file mode 100644 index 00000000..94550b2e --- /dev/null +++ b/ci/taskcluster/requirements/default.txt @@ -0,0 +1,5 @@ +taskcluster==1.0.2 \ + --hash=sha256:11cfd462a333e0a084f94c9ce55e349036dbcf04656767f9773da7d148aa5115 +requests==2.13.0 \ + --hash=sha256:1a720e8862a41aa22e339373b526f508ef0c8988baf48b84d3fc891a8e237efb \ + --hash=sha256:5722cd09762faa01276230270ff16af7acf7c5c45d623868d9ba116f15791ce8 diff --git a/circle.yml b/circle.yml index 734382cb..cb0d5182 100644 --- a/circle.yml +++ b/circle.yml @@ -15,32 +15,30 @@ dependencies: - docker info # Use pipstrap to bootstrap ourselves to a trusted pip 8 for hash # checking. - - ./ci/bin/pipstrap.py + - ./ci/circleci/bin/pipstrap.py # Install latest pip (for constraint support), then the rest of the # CI requirements. - - pip install -r ./ci/requirements/pip.txt - - pip install -r ./ci/requirements/default.txt -c ./ci/requirements/constraints.txt + - pip install -r ./ci/circleci/requirements/pip.txt + - pip install -r ./ci/circleci/requirements/default.txt -c ./ci/circleci/requirements/constraints.txt override: - - ./ci/bin/runner.sh dependencies + - ./ci/circleci/bin/runner.sh dependencies compile: override: - - ./ci/bin/runner.sh compile + - ./ci/circleci/bin/runner.sh compile test: override: - - ./ci/bin/runner.sh test - post: - - ./ci/bin/runner.sh test-post + - ./ci/circleci/bin/runner.sh test deployment: latest: branch: master commands: - - ./ci/bin/runner.sh deploy latest + - ./ci/circleci/bin/runner.sh deploy latest tags: # push all tags tag: /.*/ commands: - - ./ci/bin/runner.sh deploy "$CIRCLE_TAG" + - ./ci/circleci/bin/runner.sh deploy "$CIRCLE_TAG" diff --git a/recipe-client-addon/bin/ci/dependencies b/recipe-client-addon/bin/ci/dependencies deleted file mode 100755 index 233d6170..00000000 --- a/recipe-client-addon/bin/ci/dependencies +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -eu - -npm install -./node_modules/.bin/get-firefox -b nightly -t nightly -e diff --git a/recipe-client-addon/bin/ci/test b/recipe-client-addon/bin/ci/test deleted file mode 100755 index d5f84464..00000000 --- a/recipe-client-addon/bin/ci/test +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -eu - -./node_modules/.bin/eslint lib -./node_modules/.bin/jpm test --tbpl --binary ./nightly/firefox | tee jpm.tbpl -# exit with exit code of jpm, not tee -exit ${PIPESTATUS[0]} diff --git a/recipe-client-addon/bin/ci/test-post b/recipe-client-addon/bin/ci/test-post deleted file mode 100755 index 35249cbf..00000000 --- a/recipe-client-addon/bin/ci/test-post +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -eu - -./node_modules/.bin/woodchipper --timeout 600000 < jpm.tbpl > $CIRCLE_TEST_REPORTS/jpm.xml -./node_modules/.bin/jpm xpi && mv shield-recipe-client.xpi $CIRCLE_ARTIFACTS/shield-recipe-client.xpi diff --git a/recipe-client-addon/bin/make-xpi.sh b/recipe-client-addon/bin/make-xpi.sh new file mode 100755 index 00000000..62090758 --- /dev/null +++ b/recipe-client-addon/bin/make-xpi.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -eu + +BASE_DIR="$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")" +TMP_DIR=$(mktemp -d) +DEST="${TMP_DIR}/shield-recipe-client" + +mkdir -p $DEST + +# deletes the temp directory +function cleanup { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +while read -r LINE || [[ -n "${LINE}" ]]; do + mkdir -p "$(dirname "${DEST}/${LINE}")" + cp -r "${BASE_DIR}/${LINE}" "$(dirname "${DEST}/${LINE}")" +done < "${BASE_DIR}/build-includes.txt" + +rm "${DEST}/install.rdf.in" +cp "${BASE_DIR}/install.rdf" "${DEST}" + +pushd $DEST +zip -r shield-recipe-client.xpi * +mv shield-recipe-client.xpi $BASE_DIR +popd diff --git a/recipe-client-addon/bin/tc/make-xpi.sh b/recipe-client-addon/bin/tc/make-xpi.sh new file mode 100755 index 00000000..ff73b0c9 --- /dev/null +++ b/recipe-client-addon/bin/tc/make-xpi.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -eu + +apt-get install -y npm zip + +pushd normandy/recipe-client-addon +npm install +./bin/make-xpi.sh +cp shield-recipe-client.xpi /artifacts/ +popd diff --git a/recipe-client-addon/bin/tc/test.sh b/recipe-client-addon/bin/tc/test.sh new file mode 100755 index 00000000..fab0a8d9 --- /dev/null +++ b/recipe-client-addon/bin/tc/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eu + +# mach wants this +export SHELL=$(which bash) + +apt-get install -y npm curl python2.7 xvfb + +# Creates gecko-dev-master +echo 'Downloading gecko-dev...' +curl -sL https://github.com/mozilla/gecko-dev/archive/master.tar.gz | tar xz + +pushd normandy/recipe-client-addon +npm install +./bin/update-mozilla-central.sh ../gecko-dev/ +popd + +pushd gecko-dev-master +python2.7 ./python/mozboot/bin/bootstrap.py --no-interactive --application-choice=browser +source /root/.cargo/env +./mach build +xvfb-run ./mach test browser/extensions/shield-recipe-client/ +popd diff --git a/recipe-client-addon/chrome.manifest b/recipe-client-addon/chrome.manifest deleted file mode 100644 index 6c72f685..00000000 --- a/recipe-client-addon/chrome.manifest +++ /dev/null @@ -1,5 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -resource shield-recipe-client . diff --git a/recipe-client-addon/package.json b/recipe-client-addon/package.json index ba9baba3..88810d48 100644 --- a/recipe-client-addon/package.json +++ b/recipe-client-addon/package.json @@ -1,38 +1,21 @@ { "title": "Shield Recipe Client", - "name": "shield-recipe-client", - "id": "shield-recipe-client@mozilla.org", "version": "0.1.3", "description": "Client to download and run recipes for SHIELD, Heartbeat, etc.", "main": "index.js", "author": "Mike Cooper ", "repository": "https://github.com/mozilla/normandy-addon", "scripts": { - "test": "jpm test", - "build": "jpm xpi" - }, - "engines": { - "firefox": ">=51.0a1" }, "license": "MPL-2.0", - "keywords": [ - "jetpack" - ], "devDependencies": { "babel-eslint": "6.1.2", "eslint": "^3.8.1", "eslint-config-normandy": "1.0.0", "eslint-plugin-babel": "3.3.0", - "eslint-plugin-mozilla": "0.2.5", - "get-firefox": "^1.5.0", - "jpm": "1.2.2", - "woodchipper": "0.9.1" + "eslint-plugin-mozilla": "0.2.5" }, "dependencies": { - "jexl": "1.1.4", - "sha.js": "2.4.5" - }, - "permissions": { - "multiprocess": true + "jexl": "1.1.4" } }