Build and deploy via Jenkinsfile

* Add jenkins.yml for config and other groovy files for functions.
* Make the ircNotification function more flexible
* Add locking for deploy/test steps
* Allow a branch config to require a commit be tagged for deployment
* Deploy all apps in a region before moving to next region
* Update deployment pipeline docs
* Add Jenkinsfile milestones
* Optimize steps for BlueOcean

.editorconfig

@@ -31,11 +31,13 @@ indent_size = 4
 [*.html]
 indent_size = 2
 
+[*.yml]
+indent_size = 2
+
 [Jenkinsfile]
 indent_size = 4
 
-# Matches the exact files package.json and .travis.yml
+[package.json]
-[{package.json, .travis.yml}]
 indent_size = 2
 
 [Makefile]

Jenkinsfile

@@ -1,62 +1,219 @@
-env.DEIS_PROFILE = 'usw'
+#!groovy
-env.PRIVATE_REGISTRY = 'localhost:5001'
 
-/** Send a notice to #www on irc.mozilla.org with the build result
+@Library('github.com/mozmar/jenkins-pipeline@20170126.1')
- *
- * @param stage step of build/deploy
- * @param result outcome of build (will be uppercased)
-*/
-def ircNotification(stage, status) {
-    sh "bin/irc-notify.sh --stage '${stage}' --status '${status}'"
-}
 
-node {
+def config
-    def branch = env.BRANCH_NAME
+def utils
 
-    stage ('git') {
+stage ('Checkout') {
+    node {
         checkout scm
         sh 'git submodule sync'
         sh 'git submodule update --init --recursive'
-        env.GIT_COMMIT = sh([returnStdout: true, script: 'git rev-parse HEAD']).trim()
+        // defined in the Library loaded above
+        setGitEnvironmentVariables()
+        // load the config
+        config = readYaml file: 'jenkins.yml'
+        // load the utility functions used below
+        utils = load 'docker/jenkins/utils.groovy'
+        // save the files for later
+        stash name: 'scripts', includes: 'bin/,docker/'
+        stash name: 'tests', includes: 'tests/,requirements/'
+        stash 'workspace'
+    }
+}
+
+if ( config.branches.containsKey(env.BRANCH_NAME) ) {
+    def branchConfig = config.branches[env.BRANCH_NAME]
+    milestone()
+    stage ('Build Images') {
+        node {
+            unstash 'workspace'
+            // make sure we should continue
+            if ( branchConfig.require_tag ) {
+                try {
+                    sh 'docker/jenkins/check_if_tag.sh'
+                } catch(err) {
+                    utils.ircNotification(config, [stage: 'Git Tag Check', status: 'failure'])
+                    throw err
+                }
+            }
+            utils.ircNotification(config, [stage: 'Test & Deploy', status: 'starting'])
+            try {
+                utils.buildDockerImage(dockerfile: 'bedrock_base', update: true)
+                utils.buildDockerImage(dockerfile: 'bedrock_code', fromDockerfile: 'bedrock_base')
+            } catch(err) {
+                utils.ircNotification(config, [stage: 'Docker Build', status: 'failure'])
+                throw err
+            }
+        }
     }
 
-    if ( branch == 'master') {
+    milestone()
-        ircNotification('Dev Deploy', 'starting')
+    stage ('Test Images') {
-        build 'bedrock_base_image'
+        node {
+            unstash 'scripts'
+            try {
+                withEnv(['DOCKER_REPOSITORY=mozorg/bedrock_code']) {
+                    sh 'docker/jenkins/run_tests.sh'
+                }
+            } catch(err) {
+                utils.ircNotification(config, [stage: 'Unit Test', status: 'failure'])
+                throw err
+            }
+            utils.ircNotification(config, [stage: 'Docker Builds & Tests', status: 'complete'])
+        }
     }
-    else if ( branch == 'prod') {
+
-        ircNotification('Stage & Prod Deploys', 'starting')
+    milestone()
-        build 'bedrock_base_image'
+    stage ('Push Public Images') {
+        parallel([
+            dockerhub: {
+                node {
+                    unstash 'scripts'
+                    try {
+                        utils.buildDockerImage(dockerfile: 'bedrock_l10n', fromDockerfile: 'bedrock_code', script: 'include_l10n.sh')
+                    } catch(err) {
+                        utils.ircNotification(config, [stage: 'L10n Build', status: 'failure'])
+                        throw err
+                    }
+                    try {
+                        utils.pushDockerhub('mozorg/bedrock_base')
+                        utils.pushDockerhub('mozorg/bedrock_code')
+                        utils.pushDockerhub('mozorg/bedrock_l10n', 'mozorg/bedrock')
+                    } catch(err) {
+                        utils.ircNotification(config, [stage: 'Dockerhub Push Failed', status: 'warning'])
+                    }
+                }
+            },
+            integration_tests: {
+                node {
+                    unstash 'scripts'
+                    unstash 'tests'
+                    // prep for next stage
+                    sh 'docker/jenkins/build_integration_test_image.sh'
+                }
+            },
+        ])
     }
-    else if ( branch ==~ /^demo__[\w-]+$/ ) {
+
-        ircNotification('Demo Deploy', 'starting')
+    /**
-        try {
+     * Do region first because deployment and testing should work like this:
-            stage ('build') {
+     * region1:
+     *   push image -> deploy app1 -> test app1 -> deploy app2 -> test app2
+     * region2:
+     *   push image -> deploy app1 -> test app1 -> deploy app2 -> test app2
+     *
+     * A failure at any step of the above should fail the entire job
+     */
+    milestone()
+    for (regionId in branchConfig.regions) {
+        def region = config.regions[regionId]
+        def stageName = "Private Push: ${region.name}"
+        stage (stageName) {
+            node {
+                unstash 'scripts'
+                try {
+                    utils.pushPrivateReg(region.registry_port)
+                } catch(err) {
+                    utils.ircNotification(config, [stage: stageName, status: 'failure'])
+                    throw err
+                }
+            }
+        }
+        for (appname in branchConfig.apps) {
+            def appURL = "https://${appname}.${region.name}.moz.works"
+            stageName = "Deploy ${appname}-${region.name}"
+            // ensure no deploy/test cycle happens in parallel for an app/region
+            lock (stageName) {
+                milestone()
+                stage (stageName) {
+                    node {
+                        unstash 'scripts'
+                        withEnv(["DEIS_PROFILE=${region.deis_profile}",
+                                 "DOCKER_REPOSITORY=${appname}",
+                                 "DEIS_APPLICATION=${appname}"]) {
+                            try {
+                                retry(3) {
+                                    sh 'docker/jenkins/push2deis.sh'
+                                }
+                            } catch(err) {
+                                utils.ircNotification(config, [stage: stageName, status: 'failure'])
+                                throw err
+                            }
+                        }
+                    }
+                }
+                // queue up test closures
+                def allTests = [:]
+                for (filename in branchConfig.integration_tests) {
+                    allTests[filename] = utils.integrationTestJob(filename, appname, region.name)
+                }
+                stage ("Test ${appname}-${region.name}") {
+                    try {
+                        // wait for server to be ready
+                        sleep(time: 10, unit: 'SECONDS')
+                        parallel allTests
+                    } catch(err) {
+                        node {
+                            unstash 'scripts'
+                            utils.ircNotification(config, [stage: "Integration Tests ${region.name}", status: 'failure'])
+                        }
+                        throw err
+                    }
+                }
+                node {
+                    unstash 'scripts'
+                    // huge success \o/
+                    utils.ircNotification(config, [message: appURL, status: 'shipped'])
+                }
+            }
+        }
+    }
+}
+/**
+ * Deploy demo branches
+ */
+else if ( env.BRANCH_NAME ==~ /^demo__[\w-]+$/ ) {
+    node {
+        utils.ircNotification(config, [stage: 'Demo Deploy', status: 'starting'])
+        appname = utils.demoAppName(env.BRANCH_NAME)
+        stage ('build') {
+            milestone()
+            try {
                 sh 'make clean'
                 sh 'make sync-all'
                 sh 'echo "ENV GIT_SHA ${GIT_COMMIT}" >> docker/dockerfiles/bedrock_dev_final'
                 sh 'echo "RUN echo ${GIT_COMMIT} > static/revision.txt" >> docker/dockerfiles/bedrock_dev_final'
                 sh 'make build-final'
+            } catch(err) {
+                utils.ircNotification(config, [stage: 'Demo Build', status: 'failure'])
+                throw err
             }
-        } catch(err) {
-            ircNotification('Demo Build', 'failure')
-            throw err
         }
 
-        try {
+        stage ('deploy') {
-            stage ('deploy') {
+            lock (appname) {
-                withCredentials([[$class: 'StringBinding',
+                milestone()
-                                  credentialsId: 'SENTRY_DEMO_DSN',
+                try {
-                                  variable: 'SENTRY_DEMO_DSN']]) {
+                    withCredentials([[$class: 'StringBinding',
-                    sh './docker/jenkins/demo_deploy.sh'
+                                      credentialsId: 'SENTRY_DEMO_DSN',
+                                      variable: 'SENTRY_DEMO_DSN']]) {
+                        withEnv(['DEIS_PROFILE=usw',
+                                 "DEIS_APP_NAME=${appname}",
+                                 "PRIVATE_REGISTRY=localhost:${config.regions.usw.registry_port}"]) {
+                            sh './docker/jenkins/demo_deploy.sh'
+                        }
+                    }
+                    utils.ircNotification(config, [app_url: "https://${appname}.us-west.moz.works/"])
+                } catch(err) {
+                    utils.ircNotification(config, [stage: 'Demo Deploy', status: 'failure'])
+                    throw err
                 }
             }
-        } catch(err) {
-            ircNotification('Demo Deploy', 'failure')
-            throw err
         }
     }
-    else {
+}
-        echo "Doing nothing for ${branch}"
+else {
-    }
+    echo "Doing nothing for ${env.BRANCH_NAME}"
 }

bin/irc-notify.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 set -eo pipefail
 
-# Required environment variables if using --stage and --status:
+# Required environment variables if using --stage:
 # BRANCH_NAME, BUILD_NUMBER, BUILD_URL
 
 # defaults and constants
-BUILD_NUMBER="${BUILD_NUMBER:-0}"
+NICK="hms-flintstone"
-NICK="bedrock-deployer-$BUILD_NUMBER"
 CHANNEL="#www"
 SERVER="irc.mozilla.org:6697"
 # colors and styles: values from the following links
 # http://www.mirc.com/colors.html
 # http://stackoverflow.com/a/13382032
 RED=$'\x034'
+YELLOW=$'\x038'
 GREEN=$'\x039'
 BLUE=$'\x0311'
 BOLD=$'\x02'
@@ -30,10 +30,6 @@ while [[ $# -gt 1 ]]; do
             STATUS="$2"
             shift # past argument
             ;;
-        --demo_url)
-            DEMO_URL="$2"
-            shift # past argument
-            ;;
         -m|--message)
             MESSAGE="$2"
             shift # past argument
@@ -54,31 +50,43 @@ while [[ $# -gt 1 ]]; do
     shift # past argument or value
 done
 
-if [[ -z "$MESSAGE" ]]; then
+if [[ -n "$STATUS" ]]; then
-    if [[ -n "$STATUS" ]]; then
+    STATUS=$(echo "$STATUS" | tr '[:lower:]' '[:upper:]')
-        STATUS=$(echo "$STATUS" | tr '[:lower:]' '[:upper:]')
+    case "$STATUS" in
-        case "$STATUS" in
+      'SUCCESS')
-          'SUCCESS')
+        STATUS_COLOR="🎉 ${BOLD}${GREEN}"
-            STATUS_COLOR="${BOLD}${GREEN}"
+        ;;
-            ;;
+      'SHIPPED')
-          'FAILURE')
+        STATUS_COLOR="🚢 ${BOLD}${GREEN}"
-            STATUS_COLOR="${BOLD}${RED}"
+        ;;
-            ;;
+      'WARNING')
-          *)
+        STATUS_COLOR="⚠️ ${BOLD}${YELLOW}"
-            STATUS_COLOR="$BLUE"
+        ;;
-            ;;
+      'FAILURE')
-        esac
+        STATUS_COLOR="🚨 ${BOLD}${RED}"
-        MESSAGE="${STAGE}: ${STATUS_COLOR}${STATUS}${NORMAL}:"
+        ;;
-        MESSAGE="$MESSAGE Branch ${BOLD}${BRANCH_NAME}${NORMAL} build #${BUILD_NUMBER}: ${BUILD_URL}"
+      *)
-    elif [[ -n "$DEMO_URL" ]]; then
+        STATUS_COLOR="✨ $BLUE"
-        MESSAGE="${BOLD}${GREEN}SUCCESS${NORMAL}: Demo deployed: ${DEMO_URL}"
+        ;;
-    else
+    esac
-        echo "Missing required arguments"
+    STATUS="${STATUS_COLOR}${STATUS}${NORMAL}: "
-        echo
+fi
-        echo "Usage: irc-notify.sh [--stage STAGE --status STATUS] [--demo_url DEMO_URL]"
+
-        echo "Optional args: --irc_nick, --irc_server, --irc_channel"
+if [[ -n "$STAGE" ]]; then
-        exit 1
+    MESSAGE="${STATUS}${STAGE}:"
-    fi
+    MESSAGE="$MESSAGE Branch ${BOLD}${BRANCH_NAME}${NORMAL} build #${BUILD_NUMBER}: ${BUILD_URL}"
+elif [[ -n "$MESSAGE" ]]; then
+    MESSAGE="${STATUS}${MESSAGE}"
+else
+    echo "Missing required arguments"
+    echo
+    echo "Usage: irc-notify.sh [--stage STAGE]|[-m MESSAGE]"
+    echo "Optional args: --status, --irc_nick, --irc_server, --irc_channel"
+    exit 1
+fi
+
+if [[ -n "$BUILD_NUMBER" ]]; then
+    NICK="${NICK}-${BUILD_NUMBER}"
 fi
 
 (

bin/run-integration-tests.sh

@@ -20,6 +20,9 @@ CMD="${CMD} -r a"
 CMD="${CMD} --verbose"
 CMD="${CMD} -n ${PYTEST_PROCESSES}"
 CMD="${CMD} --base-url ${BASE_URL}"
+# rerun a flaky test once
+# DO NOT INCREASE THIS
+CMD="${CMD} --reruns 1"
 # Disable HTML report due to https://github.com/pytest-dev/pytest/issues/1351
 # CMD="${CMD} --html ${RESULTS_PATH}/index.html"
 CMD="${CMD} --junitxml ${RESULTS_PATH}/junit.xml"

docker/jenkins/build_image.sh

@@ -2,7 +2,7 @@
 #
 set -ex
 DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT}
-TMP_DOCKER_TAG=${JOB_NAME}${BUILD_NUMBER}
+TMP_DOCKER_TAG=${BUILD_TAG}
 
 # If docker image exists and no force rebuild do nothing
 FORCE_REBUILD=`echo "$FORCE_REBUILD" | tr '[:upper:]' '[:lower:]'`

docker/jenkins/build_integration_test_image.sh

@@ -0,0 +1,2 @@
+#!/bin/bash -xe
+docker build -t bedrock_integration_tests:${GIT_COMMIT} -f docker/dockerfiles/bedrock_integration_tests --pull=true .

docker/jenkins/check_if_tag.sh

@@ -1,10 +1,15 @@
 #!/bin/bash
 
-# Used to trigger downstream Jenkins jobs
-TRIGGER_FILE=".commit_is_tag"
-rm -rf $TRIGGER_FILE
-
 TAG=$(git describe --tags --exact-match $GIT_COMMIT 2> /dev/null)
-if [[ -n "$TAG" ]] && [[ "$TAG" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}(\.[0-9])?$ ]]; then
+if [[ -n "$TAG" ]]; then
-    touch $TRIGGER_FILE
+    if [[ "$TAG" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}(\.[0-9])?$ ]]; then
-fi;
+        echo "Build tagged as $TAG"
+        exit 0
+    else
+        echo "Build tagged but in the wrong format: $TAG"
+        exit 1
+    fi
+else
+    echo "Build not tagged"
+    exit 1
+fi

docker/jenkins/demo_deploy.sh

@@ -1,9 +1,6 @@
 #!/bin/bash
 set -ex
 
-DEIS_APP_NAME="bedrock-demo-${BRANCH_NAME#demo__}"
-# convert underscores to dashes. Deis does _not_ like underscores.
-DEIS_APP_NAME=$( echo "$DEIS_APP_NAME" | tr "_" "-" )
 # used for pulling from deis
 DOCKER_IMAGE_TAG="${DEIS_APP_NAME}:${GIT_COMMIT}"
 # used for pushing to registry
@@ -25,5 +22,3 @@ if deis apps:create "$DEIS_APP_NAME" --no-remote; then
 fi
 echo "Pulling $DOCKER_IMAGE_TAG into Deis app $DEIS_APP_NAME"
 deis pull "$DOCKER_IMAGE_TAG" -a "$DEIS_APP_NAME"
-
-bin/irc-notify.sh --demo_url "https://${DEIS_APP_NAME}.us-west.moz.works/"

docker/jenkins/include_l10n.sh

@@ -6,26 +6,18 @@
 #
 set -xe
 
-# Used to trigger downstream Jenkins jobs
-PARAM_FILE=.parameters
-TRIGGER_FILE=.docker-updated
-FORCE_TIME_TRIGGER_UPDATE=.timetriggerupdate
-rm -rf $TRIGGER_FILE $PARAM_FILE
-
 if [[ $BUILD_CAUSE == "REMOTECAUSE" ]]
 then
     LATEST_TAG=$(git describe --abbrev=0 --tags)
     # parent (~0) of latest tag is the commit that was tagged
     GIT_COMMIT=$(git rev-parse ${LATEST_TAG}~0)
 fi
-echo "GIT_COMMIT=$GIT_COMMIT" >> $PARAM_FILE
 
 DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT}
 
-touch $TRIGGER_FILE
+if [[ ! -d locale/.git ]];
-
-if [[ ! -e locale ]];
 then
+    rm -rf locale
     git clone --depth 1 https://github.com/mozilla-l10n/bedrock-l10n locale
 fi;
 

docker/jenkins/properties/integration_tests/chrome.properties

@@ -0,0 +1,4 @@
+export DRIVER=SauceLabs
+export BROWSER_NAME=chrome
+export PLATFORM="Windows 10"
+export MARK_EXPRESSION="not headless"

docker/jenkins/properties/integration_tests/firefox.properties

@@ -0,0 +1,5 @@
+export DRIVER=SauceLabs
+export BROWSER_NAME=firefox
+export BROWSER_VERSION=45.0
+export PLATFORM="Windows 10"
+export MARK_EXPRESSION="not headless"

docker/jenkins/properties/integration_tests/headless.properties

@@ -0,0 +1 @@
+export MARK_EXPRESSION=headless

docker/jenkins/properties/integration_tests/ie.properties

@@ -0,0 +1,4 @@
+export DRIVER=SauceLabs
+export BROWSER_NAME="internet explorer"
+export PLATFORM="Windows 10"
+export MARK_EXPRESSION="not headless"

docker/jenkins/properties/integration_tests/ie6.properties

@@ -0,0 +1,5 @@
+export DRIVER=SauceLabs
+export BROWSER_NAME="internet explorer"
+export BROWSER_VERSION="6.0"
+export PLATFORM="Windows XP"
+export MARK_EXPRESSION="sanity and not headless"

docker/jenkins/properties/integration_tests/ie7.properties

@@ -0,0 +1,5 @@
+export DRIVER="SauceLabs"
+export BROWSER_NAME="internet explorer"
+export BROWSER_VERSION="7.0"
+export PLATFORM="Windows XP"
+export MARK_EXPRESSION="sanity and not headless"

docker/jenkins/properties/integration_tests/per_push/firefox.properties

@@ -1,5 +0,0 @@
-DRIVER=SauceLabs
-BROWSER_NAME=firefox
-BROWSER_VERSION=45.0
-PLATFORM=Windows 10
-MARK_EXPRESSION=not headless

docker/jenkins/properties/integration_tests/per_push/headless.properties

@@ -1 +0,0 @@
-MARK_EXPRESSION=headless

docker/jenkins/properties/integration_tests/per_tag/chrome.properties

@@ -1,4 +0,0 @@
-DRIVER=SauceLabs
-BROWSER_NAME=chrome
-PLATFORM=Windows 10
-MARK_EXPRESSION=not headless

docker/jenkins/properties/integration_tests/per_tag/ie.properties

@@ -1,4 +0,0 @@
-DRIVER=SauceLabs
-BROWSER_NAME=internet explorer
-PLATFORM=Windows 10
-MARK_EXPRESSION=not headless

docker/jenkins/properties/integration_tests/per_tag/ie6.properties

@@ -1,5 +0,0 @@
-DRIVER=SauceLabs
-BROWSER_NAME=internet explorer
-BROWSER_VERSION=6.0
-PLATFORM=Windows XP
-MARK_EXPRESSION=sanity and not headless

docker/jenkins/properties/integration_tests/per_tag/ie7.properties

@@ -1,5 +0,0 @@
-DRIVER=SauceLabs
-BROWSER_NAME=internet explorer
-BROWSER_VERSION=7.0
-PLATFORM=Windows XP
-MARK_EXPRESSION=sanity and not headless

docker/jenkins/push2deis.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-# Needs DEIS_CONTROLLER URL, DEIS_USERNAME, DEIS_PASSWORD,
+# Needs DEIS_PROFILE, DOCKER_REPOSITORY, DEIS_APPLICATION, NEWRELIC_API_KEY and
-# DOCKER_REPOSITORY, DEIS_APPLICATION, NEWRELIC_API_KEY and
 # NEWRELIC_APP_NAME environment variables.
 #
 # To set them go to Job -> Configure -> Build Environment -> Inject
@@ -9,10 +8,11 @@
 
 set -ex
 
-deis login $DEIS_CONTROLLER  --username $DEIS_USERNAME --password $DEIS_PASSWORD
 deis pull $DOCKER_REPOSITORY:$GIT_COMMIT -a $DEIS_APPLICATION
-curl -H "x-api-key:$NEWRELIC_API_KEY" \
+if [[ -n "$NEWRELIC_API_KEY" ]]; then
-     -d "deployment[app_name]=$NEWRELIC_APP_NAME" \
+    curl -H "x-api-key:$NEWRELIC_API_KEY" \
-     -d "deployment[revision]=$GIT_COMMIT" \
+         -d "deployment[app_name]=$NEWRELIC_APP_NAME" \
-     -d "deployment[user]=EE Jenkins" \
+         -d "deployment[revision]=$GIT_COMMIT" \
-     https://api.newrelic.com/deployments.xml
+         -d "deployment[user]=EE Jenkins" \
+         https://api.newrelic.com/deployments.xml
+fi

docker/jenkins/run_integration_tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -xe
-GIT_COMMIT=${GIT_COMMIT:-$(git rev-parse HEAD)}
+
-cp docker/dockerfiles/bedrock_integration_tests Dockerfile
+# $1 should be the properties file for this run
-docker build -t bedrock_integration_tests:${GIT_COMMIT} --pull=true .
+source "$1"
 
 if [ -z "${BASE_URL}" ]; then
   # start bedrock
@@ -48,7 +48,12 @@ if [ "${DRIVER}" = "Remote" ]; then
   done
 fi
 
-docker run -v `pwd`/results:/app/results \
+# make sure results dir exists or docker will create it
+# and it will be owned by root
+RESULTS_DIR="$PWD/results"
+rm -rf "$RESULTS_DIR"
+mkdir -p "$RESULTS_DIR"
+docker run -v "${RESULTS_DIR}:/app/results" -u $(stat -c "%u:%g" "$RESULTS_DIR") \
   ${DOCKER_LINKS[@]} \
   -e BASE_URL=${BASE_URL} \
   -e DRIVER=${DRIVER} \

docker/jenkins/utils.groovy

@@ -0,0 +1,93 @@
+/**
+ * Define utility functions.
+ */
+
+def demoAppName(branchname) {
+    def appname = branchname[5..-1].replaceAll('_', '-')
+    return "bedrock-demo-${appname}".toString()
+}
+
+/**
+ * Send a notice to #www on irc.mozilla.org with the build result
+ *
+ * @param stage step of build/deploy
+ * @param result outcome of build (will be uppercased)
+*/
+def ircNotification(config, Map args) {
+    def command = "bin/irc-notify.sh --irc_nick '${config.irc.nick}' --irc_channel '${config.irc.channel}'"
+    for (arg in args) {
+        command += " --${arg.key} '${arg.value}'"
+    }
+    sh command
+}
+
+def buildDockerImage(Map kwargs) {
+    def update = kwargs.update ? 'true' : 'false'
+    def repo = kwargs.dockerRepo ?: 'mozorg'
+    def script = kwargs.script ?: 'build_image.sh'
+    def environs = ["UPDATE_DOCKER_IMAGES=${update}",
+                    "DOCKERFILE=${kwargs.dockerfile}",
+                    "DOCKER_REPOSITORY=${repo}/${kwargs.dockerfile}"]
+    if (kwargs.fromDockerfile) {
+        environs << "FROM_DOCKER_REPOSITORY=${repo}/${kwargs.fromDockerfile}"
+    }
+    withEnv(environs) {
+        sh "docker/jenkins/${script}"
+    }
+}
+
+def pushDockerhub(from_repo, to_repo='') {
+    to_repo = to_repo ?: from_repo
+    withCredentials([[$class: 'StringBinding',
+                      credentialsId: 'DOCKER_PASSWORD',
+                      variable: 'DOCKER_PASSWORD']]) {
+        withEnv(['DOCKER_USERNAME=mozjenkins',
+                 "FROM_DOCKER_REPOSITORY=${from_repo}",
+                 "DOCKER_REPOSITORY=${to_repo}"]) {
+            retry(2) {
+                sh 'docker/jenkins/push2dockerhub.sh'
+            }
+        }
+    }
+}
+
+def pushPrivateReg(port) {
+    retry(3) {
+        // TODO Fix DEIS_APPS before merge
+        // DEIS_APPS=bedrock-dev,bedrock-stage,bedrock-prod
+        withEnv(['FROM_DOCKER_REPOSITORY=mozorg/bedrock_l10n',
+                 "PRIVATE_REGISTRIES=localhost:${port}",
+                 'DEIS_APPS=bedrock-jenkinsfile-test']) {
+            sh 'docker/jenkins/push2privateregistries.sh'
+        }
+    }
+}
+
+def integrationTestJob(propFileName, appName, region) {
+    def testsBaseDir = 'docker/jenkins/properties/integration_tests'
+    def testsFileExt = '.properties'
+    return {
+        node {
+            unstash 'scripts'
+            unstash 'tests'
+            def fullFilename = "${testsBaseDir}/${propFileName}${testsFileExt}"
+            def testScript = "docker/jenkins/run_integration_tests.sh ${fullFilename}".toString()
+            withCredentials([[$class: 'UsernamePasswordMultiBinding',
+                              credentialsId: 'SAUCELABS_CREDENTIALS',
+                              usernameVariable: 'SAUCELABS_USERNAME',
+                              passwordVariable: 'SAUCELABS_API_KEY']]) {
+                withEnv(["BASE_URL=https://${appName}.${region}.moz.works",
+                         "SELENIUM_VERSION=2.52.0"]) {
+                    try {
+                        sh testScript
+                    }
+                    finally {
+                        junit 'results/*.xml'
+                    }
+                }
+            }
+        }
+    }
+}
+
+return this;

docs/pipeline.rst

@@ -34,15 +34,15 @@ Pull request
 Once a pull request is submitted, `CircleCI`_ will run both the Python and  JavaScript
 unit tests, as well as the smoke suite of redirect headless HTTP(s) response checks.
 
-Push to master branch (not tagged)
+Push to master branch
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~
 
-Whenever a change is pushed to the master branch but not tagged, the smoke suite of
+Whenever a change is pushed to the master branch, the smoke suite of
 headless (see :ref:`testing-redirects`) and UI tests (see :ref:`smoke-functional-tests`)
 are run against Firefox on Linux. If successful, the change is pushed to the dev environment,
 and the full suite of headless and UI tests are then run against
 Firefox on Windows 10 using `Sauce Labs`_. This is handled by the pipeline, and is subject
-to change according to the `per-push properties file`_ in the repository.
+to change according to the settings in the `jenkins.yml file`_ in the repository.
 
 .. _tagged-commit:
 
@@ -55,15 +55,15 @@ Chrome and Internet Explorer on Windows 10, and the sanity suite is run against
 versions of Internet Explorer (currently IE6 & IE7). If successful, the change is
 pushed to staging, tested, and then to production and the same tests are then run against
 production. As with untagged pushes, this is all handled by the pipeline, and is subject
-to change according to the `per-tag properties file`_ in the repository.
+to change according to the settings in the `jenkins.yml file`_ in the repository.
 
 **Push to prod cheat sheet**
 
 #. Check out the ``master`` branch
 #. Make sure the ``master`` branch is up to date with ``mozilla/bedrock master``
 #. Check that dev deployment is green:
-    #. View `deployment pipeline <https://ci.us-west.moz.works/view/Bedrock%20Pipeline/?fullscreen=true>`_
+    #. View `deployment pipeline <https://ci.us-west.moz.works/blue/organizations/jenkins/bedrock_multibranch_pipeline/branches/>`_
-    #. If any staging tests fail above, check retries, e.g. `bedrock_test_dev_eu_west`_
+       and look at ``master`` branch
 #. Tag and push the deployment by running ``bin/tag-release.sh --push``
 
 .. note::
@@ -82,28 +82,23 @@ to change according to the `per-tag properties file`_ in the repository.
 Pipeline integration
 --------------------
 
-The following diagram shows an overview of when the tests are run:
+Our `Jenkinsfile`_ will run the integration tests based on information in our `jenkins.yml file`_.
-
+This file specifies various test names per branch that will cause it to load different
-.. image:: pipeline.png
+parameters, allowing it to be called in many different ways to cover the testing
-    :width: 100%
+needs. The job finds the parameters file and executes `this script <https://github.com/mozilla/bedrock/blob/master/docker/jenkins/run_integration_tests.sh>`_,
-
-A dedicated **Test Runner** job exists to run the integration tests. This job takes various
-parameters, allowing it to be called from multiple upstream jobs to cover the testing
-needs. This job takes the parameters passed to it and executes `this script <https://github.com/mozilla/bedrock/blob/master/docker/jenkins/run_integration_tests.sh>`_,
 which then runs `this Docker image <https://github.com/mozilla/bedrock/blob/master/docker/dockerfiles/bedrock_integration_tests>`_,
 and ultimately runs `another script <https://github.com/mozilla/bedrock/blob/master/bin/run-integration-tests.sh>`_.
 The two scripts can also be executed locally to replicate the way Jenkins operates.
 
-During the **Build** stage, the Test Runner job is called without a ``BASE_URL``. This means
+During the **Test Images** stage, the Test Runner job is called without a ``BASE_URL``. This means
 that a local instance of the application will be started, and the URL of this instance
 will be used for testing. The ``DRIVER`` parameter is set to ``Remote``, which causes a
 local instance of Selenium Grid to be started in Docker and used for the browser-based
 functional UI tests.
 
-During the various **Test** stages, the **Test Runner** job is called once for every `per-push properties file`_.
+The test scripts above will be run once for each properties file specified in the `jenkins.yml file`_
-If the push was also tagged, the job is also called once for every `per-tag properties file`_.
+for the branch being built and tested. Pushes to `master` will run different tests than pushes to `prod`
-The content of these files are used as the parameters for the Test Runner job, allowing
+for example.
-the configuration to be reviewed and under version control.
 
 Configuration
 ~~~~~~~~~~~~~
@@ -129,7 +124,8 @@ Adding test runs
 ~~~~~~~~~~~~~~~~
 
 Test runs can be added by creating new `properties files`_ with the parameters of the new
-test run. For example, if you wanted to run tests in Firefox on both Windows 10 and
+test run. These are simply bash syntax files that set environment variables.
+ For example, if you wanted to run tests in Firefox on both Windows 10 and
 OS X, you could create the following files
 
 win10-firefox.properties
@@ -137,20 +133,20 @@ win10-firefox.properties
 
 .. code-block:: none
 
-    DRIVER=SauceLabs
+    export DRIVER=SauceLabs
-    BROWSER_NAME=firefox
+    export BROWSER_NAME=firefox
-    PLATFORM=Windows 10
+    export PLATFORM="Windows 10"
-    MARK_EXPRESSION=not headless
+    export MARK_EXPRESSION="not headless""
 
 osx-firefox.properties
 ......................
 
 .. code-block:: none
 
-    DRIVER=SauceLabs
+    export DRIVER=SauceLabs
-    BROWSER_NAME=firefox
+    export BROWSER_NAME=firefox
-    PLATFORM=OS X 10.11
+    export PLATFORM="OS X 10.11"
-    MARK_EXPRESSION=not headless
+    export MARK_EXPRESSION="not headless"
 
 You can use `Sauce Labs platform configurator`_ to help with the parameter values.
 
@@ -159,53 +155,9 @@ job and pass in the ``BASE_URL`` and other parameters of your choosing. This is
 testing against deployed demo environments. For a good baseline, use the values from ``win10-firefox.properties``_
 above.
 
-Investigating failures
-~~~~~~~~~~~~~~~~~~~~~~
-
-Due to the configuration of the pipeline, it can be a little confusing when
-investigating failed builds. The actual `bedrock_integration_tests_runner`_ job will fail,
-and cause the upstream job that triggered it to fail. The best approach to investigating
-failures is to first look at the upstream job, which will either be
-`bedrock_integration_tests`_ (during the **Build** stage) or one of the ``bedrock_test_`` jobs in
-the various **Test** stages:
-
-* `bedrock_test_dev_eu_west`_
-* `bedrock_test_dev_us_west`_
-* `bedrock_test_stage_eu_west`_
-* `bedrock_test_stage_us_west`_
-* `bedrock_test_prod_eu_west`_
-* `bedrock_test_prod_us_west`_
-
-In the console log for the failed upstream job you will see a line similar to:
-
-    ``bedrock_integration_tests_runner #n completed. Result was FAILURE``
-
-The ``#n`` will be a valid build number, and this text will be a link directly to the
-failed test runner build. Clicking this will take you to that build, where you can use
-the **Console Output** or the **Test Results** links to find out what caused the build
-to fail.
-
-If you’re looking at a failed `bedrock_integration_tests_runner`_ build, you can determine
-the upstream job by looking for the following text in the build status page, or in the
-console output:
-
-    ``Started by upstream project bedrock_test_ build number n``
-
-If the job was run in `Sauce Labs`_, you can also find a link to the failed job in the
-console output, which will contain a **video of the failure**.
-
 Known issues in Jenkins
 -----------------------
 
-Shared test runner job shown in pipeline view
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-In order to avoid duplicating the test runner job for every necessary environment and
-platform configuration, we use a shared job that accepts parameters. Unfortunately this
-means that the pipeline view tries to show the job with the various upstream
-relationships. It would be better to simply hide this job, however this functionality
-does not exist. An `enhancement request`_ has been raised.
-
 Jenkins stalls after global configuration changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -217,17 +169,10 @@ A `bug for the IRC plugin`_ has been raised.
 .. _Deployment Pipeline: https://ci.us-west.moz.works/view/Bedrock%20Pipeline/
 .. _CircleCI: https://circleci.com/
 .. _Sauce Labs: https://saucelabs.com/
-.. _per-push properties file: https://github.com/mozilla/bedrock/tree/master/docker/jenkins/properties/integration_tests/per_push
+.. _Jenkinsfile: https://github.com/mozilla/bedrock/tree/master/Jenkinsfile
-.. _per-tag properties file: https://github.com/mozilla/bedrock/tree/master/docker/jenkins/properties/integration_tests/per_tag
+.. _jenkins.yml file: https://github.com/mozilla/bedrock/tree/master/jenkins.yml
 .. _properties files: https://github.com/mozilla/bedrock/tree/master/docker/jenkins/properties/integration_tests
 .. _bedrock_integration_tests_runner: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_integration_tests_runner/
-.. _bedrock_integrations_tests: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_integration_tests/
-.. _bedrock_test_dev_eu_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_dev_eu_west/
-.. _bedrock_test_dev_us_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_dev_us_west/
-.. _bedrock_test_stage_eu_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_stage_eu_west/
-.. _bedrock_test_stage_us_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_stage_us_west/
-.. _bedrock_test_prod_eu_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_prod_eu_west/
-.. _bedrock_test_prod_us_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_prod_us_west/
 .. _configured in Jenkins: https://ci.us-west.moz.works/configure
 .. _become unresponsive: https://issues.jenkins-ci.org/browse/JENKINS-28175
 .. _test dependencies: https://github.com/mozilla/bedrock/blob/master/requirements/test.txt
@@ -235,4 +180,3 @@ A `bug for the IRC plugin`_ has been raised.
 .. _Sauce Labs platform configurator: https://wiki.saucelabs.com/display/DOCS/Platform+Configurator/
 .. _enhancement request: https://issues.jenkins-ci.org/browse/JENKINS-26210
 .. _bug for the IRC plugin: https://issues.jenkins-ci.org/browse/JENKINS-28175
-.. _bedrock_integration_tests_runner: https://ci.us-west.moz.works/job/bedrock_integration_tests_runner/build?delay=0sec

jenkins.yml

@@ -0,0 +1,51 @@
+branches:
+  master:
+    regions:
+      - usw
+      - euw
+    apps:
+      - bedrock-dev
+    integration_tests:
+      - firefox
+      - headless
+  prod:
+    require_tag: true
+    regions:
+      - usw
+      - euw
+    apps:
+      - bedrock-stage
+      - bedrock-prod
+    integration_tests:
+      - chrome
+      - ie
+      - ie6
+      - ie7
+  # for testing
+  deploy-via-jenkinsfile:
+    regions:
+      - usw
+      - euw
+    apps:
+      - bedrock-jenkinsfile-test
+    integration_tests:
+      - firefox
+      - headless
+      - chrome
+      - ie
+      - ie6
+      - ie7
+
+regions:
+  usw:
+    deis_profile: usw
+    name: us-west
+    registry_port: 5001
+  euw:
+    deis_profile: euw
+    name: eu-west
+    registry_port: 5000
+
+irc:
+  nick: hms-flintstone
+  channel: "#www"