recipe-client-addon: test and build xpi in TC

2017-02-09 09:30:43 -08:00 · 2017-02-09 09:30:43 -08:00 · 9aebecda4a
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@ -0,0 +1,52 @@
+version: 0
+
+metadata:
+  name: Normandy
+  description: Normandy CI Tasks
+  owner: "{{ event.head.user.email }}"
+  source: "{{ event.head.repo.url }}"
+
+tasks:
+  - metadata:
+      name: normandy:decision
+      description: Normandy decision task
+      owner: "{{ event.head.user.email }}"
+      source: "{{ event.head.repo.url }}"
+    provisionerId: "{{ taskcluster.docker.provisionerId }}"
+    workerType: "{{ taskcluster.docker.workerType }}"
+    scopes:
+      - queue:create-task:aws-provisioner-v1/github-worker
+    payload:
+      env:
+      maxRunTime: 300 # 5 minutes
+      image: ubuntu:zesty
+      features:
+        taskclusterProxy: true
+      command:
+        - "/bin/bash"
+        - "-c"
+        - >-
+          apt-get update &&
+          apt-get install -y git python3.6 python3.6-venv &&
+          cd ~ &&
+
+          git clone $GITHUB_HEAD_REPO_URL normandy &&
+          pushd normandy &&
+          git checkout $GITHUB_HEAD_SHA &&
+          popd &&
+
+          pushd normandy/ci/taskcluster &&
+          python3.6 -m venv venv &&
+          source venv/bin/activate &&
+          pip install -r requirements/default.txt -c requirements/constraints.txt &&
+          popd &&
+
+          python3.6 normandy/ci/taskcluster/bin/decision.py
+    extra:
+      github:
+        env: true
+        events:
+          - pull_request.opened
+          - pull_request.synchronize
+          - push
+          - release
--- a/ci/circleci/bin/pipstrap.py
+++ b/ci/circleci/bin/pipstrap.py
--- a/ci/circleci/bin/runner.sh
+++ b/ci/circleci/bin/runner.sh
@ -6,7 +6,7 @@ if [[ $# -eq 0 ]]; then
    exit 1
 fi

-BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)"
+BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../../.. && pwd)"
 PROJECTS=(
    recipe-client-addon
    recipe-server
--- a/ci/circleci/requirements/constraints.txt
+++ b/ci/circleci/requirements/constraints.txt
--- a/ci/circleci/requirements/default.txt
+++ b/ci/circleci/requirements/default.txt
--- a/ci/circleci/requirements/pip.txt
+++ b/ci/circleci/requirements/pip.txt
--- a/ci/taskcluster/bin/decision.py
+++ b/ci/taskcluster/bin/decision.py
@ -0,0 +1,93 @@
+#!/usr/bin/env python
+import os
+
+import requests
+from taskcluster import fromNow
+from taskcluster.utils import stableSlugId, dumpJson
+
+tasks = [
+    {
+        'name': 'recipe-client-addon:test',
+        'description': 'Test recipe-client-addon with gecko-dev',
+        'command': 'normandy/recipe-client-addon/bin/tc/test.sh',
+        'env': {
+            'GECKO_DEV_URL': 'https://github.com/mozilla/gecko-dev',
+        },
+    },
+    {
+        'name': 'recipe-client-addon:make-xpi',
+        'description': 'Build XPI for recipe-client-addon',
+        'command': 'normandy/recipe-client-addon/bin/tc/make-xpi.sh',
+        'dependencies': ['recipe-client-addon:test'],
+    },
+]
+
+
+def main():
+    _idMaker = stableSlugId()
+
+    def idMaker(name):
+        return _idMaker(name).decode()
+
+    decisionTaskId = os.environ['TASK_ID']
+
+    with requests.Session() as session:
+        for task in tasks:
+            dependencies = [idMaker(name) for name in task.get('dependencies', [])]
+            dependencies.append(decisionTaskId)
+
+            env = task.get('env', {})
+            for key, val in os.environ.items():
+                if key.startswith('GITHUB_'):
+                    env.setdefault(key, val)
+
+            task_id = idMaker(task['name'])
+            res = session.put(f'http://taskcluster/queue/v1/task/{task_id}', data=dumpJson({
+                'metadata': {
+                    'name': task['name'],
+                    'description': task['description'],
+                    'owner': "mcooper@mozilla.com",
+                    'source': "https://github.com/mozilla/normandy",
+                },
+                'provisionerId': 'aws-provisioner-v1',
+                'workerType': 'github-worker',
+                'schedulerId': 'taskcluster-github',
+                'taskGroupId': decisionTaskId,
+                'created': fromNow('0 seconds'),
+                'deadline': fromNow('4 hours'),
+                'expires': fromNow('365 days'),
+                'payload': {
+                    'image': 'ubuntu:zesty',
+                    'command': [
+                        '/bin/bash',
+                        '-c',
+                        ' && '.join([
+                            'apt-get update',
+                            'apt-get install -y git',
+                            'mkdir /artifacts',
+                            'cd ~',
+                            'git clone $GITHUB_HEAD_REPO_URL normandy',
+                            'pushd normandy',
+                            'git checkout $GITHUB_HEAD_SHA',
+                            'popd',
+                            task['command'],
+                        ])
+                    ],
+                    'maxRunTime': 14400,  # 4 hours
+                    'env': env,
+                    'artifacts': {
+                        'public': {
+                            'type': 'directory',
+                            'path': '/artifacts',
+                            'expires': fromNow('364 days'),  # must expire before task
+                        },
+                    },
+                },
+                'dependencies': dependencies,
+            }))
+            print(res.text)
+            res.raise_for_status()
+
+
+if __name__ == '__main__':
+    main()
--- a/ci/taskcluster/requirements/constraints.txt
+++ b/ci/taskcluster/requirements/constraints.txt
@ -0,0 +1,21 @@
+mohawk==0.3.4 \
+    --hash=sha256:e98b331d9fa9ece7b8be26094cbe2d57613ae882133cc755167268a984bc0ab3
+slugid==1.0.7 \
+    --hash=sha256:6dab3c7eef0bb423fb54cb7752e0f466ddd0ee495b78b763be60e8a27f69e779
+six==1.10.0 \
+    --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \
+    --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a
+aiohttp==1.3.1 \
+    --hash=sha256:8e71993f16870ebb1e8af21cb65dbac0cb6418ce9ab5a8cfaefce1c986a306bf \
+    --hash=sha256:13f44a971da01093a5abcc6436596558e0c5bfac2532407a097be8607510ee1a \
+    --hash=sha256:e147b0cea568773443683becce9de4071506431118609b5d477fe61508417af1
+async_timeout==1.1.0 \
+    --hash=sha256:b88bd1fe001b800ec23c7bf27a81b32819e2a56668e9fba5646a7f3618143081 \
+    --hash=sha256:d848623229614d2a239375f003c65a0ecf3bcfed3a2fdba6ce83b31a8038c2a0
+yarl==0.9.2 \
+    --hash=sha256:4c63264bdf022e4ab509dd8946147dec026a4110c90e0a2dda9499385c312ece
+multidict==2.1.4 \
+    --hash=sha256:a77aa8c9f68846c3b5db43ff8ed2a7a884dbe845d01f55113a3fba78518c4cd7
+chardet==2.3.0 \
+    --hash=sha256:aaf514bde38020b4f1e42c6a6e141f2827a8a58ccfc3b22b6ff5a1a4b50be56e \
+    --hash=sha256:e53e38b3a4afe6d1132de62b7400a4ac363452dc5dfcf8d88e8e0cce663c68aa
--- a/ci/taskcluster/requirements/default.txt
+++ b/ci/taskcluster/requirements/default.txt
@ -0,0 +1,5 @@
+taskcluster==1.0.2 \
+    --hash=sha256:11cfd462a333e0a084f94c9ce55e349036dbcf04656767f9773da7d148aa5115
+requests==2.13.0 \
+    --hash=sha256:1a720e8862a41aa22e339373b526f508ef0c8988baf48b84d3fc891a8e237efb \
+    --hash=sha256:5722cd09762faa01276230270ff16af7acf7c5c45d623868d9ba116f15791ce8
--- a/circle.yml
+++ b/circle.yml
@ -15,32 +15,30 @@ dependencies:
    - docker info
    # Use pipstrap to bootstrap ourselves to a trusted pip 8 for hash
    # checking.
-    - ./ci/bin/pipstrap.py
+    - ./ci/circleci/bin/pipstrap.py
    # Install latest pip (for constraint support), then the rest of the
    # CI requirements.
-    - pip install -r ./ci/requirements/pip.txt
-    - pip install -r ./ci/requirements/default.txt -c ./ci/requirements/constraints.txt
+    - pip install -r ./ci/circleci/requirements/pip.txt
+    - pip install -r ./ci/circleci/requirements/default.txt -c ./ci/circleci/requirements/constraints.txt
  override:
-    - ./ci/bin/runner.sh dependencies
+    - ./ci/circleci/bin/runner.sh dependencies

 compile:
  override:
-    - ./ci/bin/runner.sh compile
+    - ./ci/circleci/bin/runner.sh compile

 test:
  override:
-    - ./ci/bin/runner.sh test
-  post:
-    - ./ci/bin/runner.sh test-post
+    - ./ci/circleci/bin/runner.sh test

 deployment:
  latest:
    branch: master
    commands:
-      - ./ci/bin/runner.sh deploy latest
+      - ./ci/circleci/bin/runner.sh deploy latest

  tags:
    # push all tags
    tag: /.*/
    commands:
-      - ./ci/bin/runner.sh deploy "$CIRCLE_TAG"
+      - ./ci/circleci/bin/runner.sh deploy "$CIRCLE_TAG"
--- a/recipe-client-addon/bin/ci/dependencies
+++ b/recipe-client-addon/bin/ci/dependencies
@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -eu
-
-npm install
-./node_modules/.bin/get-firefox -b nightly -t nightly -e
--- a/recipe-client-addon/bin/ci/test
+++ b/recipe-client-addon/bin/ci/test
@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-set -eu
-
-./node_modules/.bin/eslint lib
-./node_modules/.bin/jpm test --tbpl --binary ./nightly/firefox | tee jpm.tbpl
-# exit with exit code of jpm, not tee
-exit ${PIPESTATUS[0]}
--- a/recipe-client-addon/bin/ci/test-post
+++ b/recipe-client-addon/bin/ci/test-post
@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -eu
-
-./node_modules/.bin/woodchipper --timeout 600000 < jpm.tbpl > $CIRCLE_TEST_REPORTS/jpm.xml
-./node_modules/.bin/jpm xpi && mv shield-recipe-client.xpi $CIRCLE_ARTIFACTS/shield-recipe-client.xpi
--- a/recipe-client-addon/bin/make-xpi.sh
+++ b/recipe-client-addon/bin/make-xpi.sh
@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -eu
+
+BASE_DIR="$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")"
+TMP_DIR=$(mktemp -d)
+DEST="${TMP_DIR}/shield-recipe-client"
+
+mkdir -p $DEST
+
+# deletes the temp directory
+function cleanup {
+  rm -rf "$TMP_DIR"
+}
+trap cleanup EXIT
+
+while read -r LINE || [[ -n "${LINE}" ]]; do
+  mkdir -p "$(dirname "${DEST}/${LINE}")"
+  cp -r "${BASE_DIR}/${LINE}" "$(dirname "${DEST}/${LINE}")"
+done < "${BASE_DIR}/build-includes.txt"
+
+rm "${DEST}/install.rdf.in"
+cp "${BASE_DIR}/install.rdf" "${DEST}"
+
+pushd $DEST
+zip -r shield-recipe-client.xpi *
+mv shield-recipe-client.xpi $BASE_DIR
+popd
--- a/recipe-client-addon/bin/tc/make-xpi.sh
+++ b/recipe-client-addon/bin/tc/make-xpi.sh
@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+apt-get install -y npm zip
+
+pushd normandy/recipe-client-addon
+npm install
+./bin/make-xpi.sh
+cp shield-recipe-client.xpi /artifacts/
+popd
--- a/recipe-client-addon/bin/tc/test.sh
+++ b/recipe-client-addon/bin/tc/test.sh
@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -eu
+
+# mach wants this
+export SHELL=$(which bash)
+
+apt-get install -y npm curl python2.7 xvfb
+
+# Creates gecko-dev-master
+echo 'Downloading gecko-dev...'
+curl -sL https://github.com/mozilla/gecko-dev/archive/master.tar.gz | tar xz
+
+pushd normandy/recipe-client-addon
+npm install
+./bin/update-mozilla-central.sh ../gecko-dev/
+popd
+
+pushd gecko-dev-master
+python2.7 ./python/mozboot/bin/bootstrap.py --no-interactive --application-choice=browser
+source /root/.cargo/env
+./mach build
+xvfb-run ./mach test browser/extensions/shield-recipe-client/
+popd
--- a/recipe-client-addon/chrome.manifest
+++ b/recipe-client-addon/chrome.manifest
@ -1,5 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-resource shield-recipe-client .
--- a/recipe-client-addon/package.json
+++ b/recipe-client-addon/package.json
@ -1,38 +1,21 @@
 {
  "title": "Shield Recipe Client",
-  "name": "shield-recipe-client",
-  "id": "shield-recipe-client@mozilla.org",
  "version": "0.1.3",
  "description": "Client to download and run recipes for SHIELD, Heartbeat, etc.",
  "main": "index.js",
  "author": "Mike Cooper <mcooper@mozilla.com>",
  "repository": "https://github.com/mozilla/normandy-addon",
  "scripts": {
-    "test": "jpm test",
-    "build": "jpm xpi"
-  },
-  "engines": {
-    "firefox": ">=51.0a1"
  },
  "license": "MPL-2.0",
-  "keywords": [
-    "jetpack"
-  ],
  "devDependencies": {
    "babel-eslint": "6.1.2",
    "eslint": "^3.8.1",
    "eslint-config-normandy": "1.0.0",
    "eslint-plugin-babel": "3.3.0",
-    "eslint-plugin-mozilla": "0.2.5",
-    "get-firefox": "^1.5.0",
-    "jpm": "1.2.2",
-    "woodchipper": "0.9.1"
+    "eslint-plugin-mozilla": "0.2.5"
  },
  "dependencies": {
-    "jexl": "1.1.4",
-    "sha.js": "2.4.5"
-  },
-  "permissions": {
-    "multiprocess": true
+    "jexl": "1.1.4"
  }
 }