Deploy demos with Jenkins

* Disable circle demo deploy
* Add IRC notification to Jenkinsfile deployments
* Fix the deployment docs for new pipeline

.editorconfig

@@ -31,6 +31,9 @@ indent_size = 4
 [*.html]
 indent_size = 2
 
+[Jenkinsfile]
+indent_size = 4
+
 # Matches the exact files package.json and .travis.yml
 [{package.json, .travis.yml}]
 indent_size = 2

Jenkinsfile

@@ -1,16 +1,76 @@
 def branch = env.BRANCH_NAME
+env.DEIS_PROFILE = 'usw'
+env.PRIVATE_REGISTRY = 'localhost:5001'
+
+/** Send a notice to #www on irc.mozilla.org with the build result
+ *
+ * @param stage step of build/deploy
+ * @param result outcome of build (will be uppercased)
+*/
+def ircNotification(stage, result) {
+    def nick = "bedrock-deployer-${env.BUILD_NUMBER}"
+    def channel = '#www'
+    result = result.toUpperCase()
+    def message = "${stage}: ${result}: Branch ${env.BRANCH_NAME} build #${env.BUILD_NUMBER}: ${env.BUILD_URL}"
+    sh """
+        (
+        echo NICK ${nick}
+        echo USER ${nick} 8 * : ${nick}
+        sleep 5
+        echo "JOIN ${channel}"
+        echo "NOTICE ${channel} :${message}"
+        echo QUIT
+        ) | openssl s_client -connect irc.mozilla.org:6697
+    """
+}
 
 if ( branch == 'master') {
-    echo "Building master"
+    ircNotification('Dev Deploy', 'starting')
     build 'bedrock_base_image'
 }
 else if ( branch == 'prod') {
-    echo "Building prod"
+    ircNotification('Prod Deploy', 'starting')
     build 'bedrock_base_image'
 }
-else if ( branch ==~ /^demo__[a-z_-]+$/ ) {
-    echo "Building a demo: ${branch} (just for testing. does not work yet.)"
-    echo "TODO: make this work"
+else if ( branch ==~ /^demo__[\w-]+$/ ) {
+    node {
+        ircNotification('Demo Deploy', 'starting')
+
+        stage ('git') {
+            checkout scm
+            sh 'git submodule sync'
+            sh 'git submodule update --init --recursive'
+            env.GIT_COMMIT = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
+        }
+
+        try {
+            stage ('build') {
+                sh 'make clean'
+                sh 'make sync-all'
+                sh 'echo "ENV GIT_SHA ${GIT_COMMIT}" >> docker/dockerfiles/bedrock_dev_final'
+                sh 'echo "RUN echo ${GIT_COMMIT} > static/revision.txt" >> docker/dockerfiles/bedrock_dev_final'
+                sh 'make build-final'
+            }
+        } catch(err) {
+            ircNotification('Demo Build', 'failure')
+            throw err
+        }
+
+        try {
+            stage ('deploy') {
+                withCredentials([[$class: 'StringBinding',
+                                  credentialsId: 'SENTRY_DEMO_DSN',
+                                  variable: 'SENTRY_DEMO_DSN']]) {
+                    sh './docker/jenkins/demo_deploy.sh'
+                }
+            }
+        } catch(err) {
+            ircNotification('Demo Deploy', 'failure')
+            throw err
+        }
+
+        ircNotification('Demo Deploy', 'success')
+    }
 }
 else {
     echo "Doing nothing for ${branch}"

Makefile

@@ -1,6 +1,7 @@
 BASE_IMG_NAME = bedrock_base
 DEV_IMG_NAME = bedrock_dev
-FINAL_IMG_NAME = bedrock_dev_final
+GIT_SHA = $(shell git rev-parse HEAD)
+FINAL_IMG_NAME = bedrock_dev_final:${GIT_SHA}
 
 default:
 	@echo "You need to specify a subcommand."
@@ -30,7 +31,7 @@ build:
 	touch .docker-build
 
 build-final: .docker-build
-	docker build -f docker/dockerfiles/bedrock_dev_final -t bedrock_dev_final .
+	docker build -f docker/dockerfiles/bedrock_dev_final -t ${FINAL_IMG_NAME} .
 	touch .docker-build-final
 
 run: .docker-build
@@ -60,7 +61,8 @@ clean:
 	-rm -rf docs/_build/
 
 	# state files
-	-rm .docker-build
+	-rm -f .docker-build
+	-rm -f .docker-build-final
 
 test: .docker-build
 	docker run --user `id -u` --env-file docker/test.env -v "$$PWD:/app" ${DEV_IMG_NAME} docker/run-tests.sh

bin/circleci-demo-deploy.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-set -e
-
-docker tag bedrock_dev_final "$DOCKER_IMAGE_TAG"
-
-echo "Logging into quay.io"
-docker login -e "$QUAY_EMAIL" -u "$QUAY_USERNAME" -p "$QUAY_PASSWORD" quay.io
-echo "Pushing ${DOCKER_IMAGE_TAG} to quay.io"
-docker push ${DOCKER_IMAGE_TAG}
-
-# Install deis client
-echo "Installing Deis client"
-curl -sSL http://deis.io/deis-cli/install.sh | sh
-
-DEIS_APP_NAME="bedrock-demo-${CIRCLE_BRANCH#demo__}"
-# convert underscores to dashes. Deis does _not_ like underscores.
-DEIS_APP_NAME=$( echo "$DEIS_APP_NAME" | tr "_" "-" )
-echo "Logging into the Deis Controller at $DEIS_CONTROLLER"
-./deis login "$DEIS_CONTROLLER" --username "$DEIS_USERNAME" --password "$DEIS_PASSWORD"
-echo "Creating the demo app $DEIS_APP_NAME"
-if ./deis apps:create "$DEIS_APP_NAME" --no-remote; then
-  echo "Giving github user $CIRCLE_USERNAME perms for the app"
-  ./deis perms:create "$CIRCLE_USERNAME" -a "$DEIS_APP_NAME" || true
-  echo "Configuring the new demo app"
-  ./deis config:push -a "$DEIS_APP_NAME" -p docker/demo.env
-  if [[ -n "$SENTRY_DEMO_DSN" ]]; then
-    ./deis config:set -a "$DEIS_APP_NAME" "SENTRY_DSN=$SENTRY_DEMO_DSN"
-  fi
-fi
-echo "Pulling $DOCKER_IMAGE_TAG into Deis app $DEIS_APP_NAME"
-./deis pull "$DOCKER_IMAGE_TAG" -a "$DEIS_APP_NAME"

bin/circleci-docker-build.sh

@@ -1,49 +1,6 @@
 #!/bin/bash
 set -ex
 
-DOCKER_CACHE_PATH=~/docker
-DOCKER_CACHE_FILE="${DOCKER_CACHE_PATH}/image.tgz"
-
-mkdir -p $DOCKER_CACHE_PATH
 make clean
-
-if [[ -f $DOCKER_CACHE_FILE ]]; then
-  gunzip -c "$DOCKER_CACHE_FILE" | docker load;
-fi
-
-if [[ -f $DOCKER_CACHE_PATH/bedrock.db ]]; then
-  mv $DOCKER_CACHE_PATH/bedrock.db ./
-fi
-
-LOCALES_TAR_FILE="$DOCKER_CACHE_PATH/locales.tgz"
-if [[ -f "$LOCALES_TAR_FILE" ]]; then
-  tar xzf "$LOCALES_TAR_FILE"
-  rm -f "$LOCALES_TAR_FILE"
-fi
-
-MFSA_TAR_FILE="$DOCKER_CACHE_PATH/mfsa_repo.tgz"
-if [[ -f "$MFSA_TAR_FILE" ]]; then
-  tar xzf "$MFSA_TAR_FILE"
-  rm -f "$MFSA_TAR_FILE"
-fi
-
-PD_TAR_FILE="$DOCKER_CACHE_PATH/pd_files.tgz"
-if [[ -f "$PD_TAR_FILE" ]]; then
-  tar xzf "$PD_TAR_FILE"
-  rm -f "$PD_TAR_FILE"
-fi
-
-if [[ "$CIRCLE_BRANCH" == demo__* ]]; then
-  make sync-all
-  cp bedrock.db $DOCKER_CACHE_PATH/
-  tar czf "$LOCALES_TAR_FILE" locale
-  tar czf "$MFSA_TAR_FILE" mofo_security_advisories
-  if [[ -d product_details_json ]]; then
-    tar czf "$PD_TAR_FILE" product_details_json
-  fi
-fi
-
 echo "ENV GIT_SHA ${CIRCLE_SHA1}" >> docker/dockerfiles/bedrock_dev_final
 make build-final
-docker save $(docker history -q bedrock_dev_final | grep -v '<missing>') | gzip > $DOCKER_CACHE_FILE
-

circle.yml

@@ -36,10 +36,3 @@ test:
     - gulp js:lint
     - gulp js:test
     - make test-image
-
-deployment:
-  demo:
-    branch: /demo__.+/
-    owner: mozilla
-    commands:
-      - bin/circleci-demo-deploy.sh

docker/jenkins/demo_deploy.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -ex
+
+DEIS_APP_NAME="bedrock-demo-${BRANCH_NAME#demo__}"
+# convert underscores to dashes. Deis does _not_ like underscores.
+DEIS_APP_NAME=$( echo "$DEIS_APP_NAME" | tr "_" "-" )
+# used for pulling from deis
+DOCKER_IMAGE_TAG="${DEIS_APP_NAME}:${GIT_COMMIT}"
+# used for pushing to registry
+PRIVATE_IMAGE_TAG="${PRIVATE_REGISTRY}/${DOCKER_IMAGE_TAG}"
+
+docker tag "bedrock_dev_final:${GIT_COMMIT}" "$PRIVATE_IMAGE_TAG"
+docker push "$PRIVATE_IMAGE_TAG"
+
+echo "Creating the demo app $DEIS_APP_NAME"
+if deis apps:create "$DEIS_APP_NAME" --no-remote; then
+  echo "Configuring the new demo app"
+  deis config:push -a "$DEIS_APP_NAME" -p docker/demo.env
+  # Sentry DSN is potentially sensitive. Turn off command echo.
+  set +x
+  if [[ -n "$SENTRY_DEMO_DSN" ]]; then
+    deis config:set -a "$DEIS_APP_NAME" "SENTRY_DSN=$SENTRY_DEMO_DSN"
+  fi
+  set -x
+fi
+echo "Pulling $DOCKER_IMAGE_TAG into Deis app $DEIS_APP_NAME"
+deis pull "$DOCKER_IMAGE_TAG" -a "$DEIS_APP_NAME"

docs/pipeline.rst

@@ -64,8 +64,7 @@ pushes, this is all handled by the pipeline, and is subject to change according
 #. Check that staging deployment is green:
     #. View `deployment pipeline <https://ci.us-west.moz.works/view/Bedrock%20Pipeline/?fullscreen=true>`_
     #. If any staging tests fail above, check retries, e.g. `bedrock_test_stage_eu_west`_
-#. Add a tag for the deployment by running ``bin/tag-release.sh``
-#. Push tags to trigger prod push (``git push --tags``)
+#. Tag and push the deployment by running ``bin/tag-release.sh --push``
 
 Pipeline integration
 --------------------