Add more options to jenkins.yml for deployments

* Add ability to toggle smoke tests
* Add ability to toggle deployment
* Add ability to toggle integration tests
* Add ability to toggle pushing docker images to public registry

Jenkinsfile

@@ -53,34 +53,38 @@ if ( config.branches.containsKey(env.BRANCH_NAME) ) {
         }
     }
 
-    milestone()
-    stage ('Test Images') {
-        parallel([
-            smoke_tests: utils.integrationTestJob('smoke'),
-            unit_tests: {
-                node {
-                    unstash 'scripts'
-                    try {
-                        sh 'docker/bin/run_tests.sh'
-                    } catch(err) {
-                        utils.ircNotification(config, [stage: 'Unit Test', status: 'failure'])
-                        throw err
+    if ( branchConfig.smoke_tests ) {
+        milestone()
+        stage ('Test Images') {
+            parallel([
+                smoke_tests: utils.integrationTestJob('smoke'),
+                unit_tests: {
+                    node {
+                        unstash 'scripts'
+                        try {
+                            sh 'docker/bin/run_tests.sh'
+                        } catch(err) {
+                            utils.ircNotification(config, [stage: 'Unit Test', status: 'failure'])
+                            throw err
+                        }
                     }
-                }
-            },
-        ])
+                },
+            ])
+        }
     }
 
-    milestone()
-    stage ('Push Public Images') {
-        node {
-            unstash 'scripts'
-            try {
-                utils.pushDockerhub('mozorg/bedrock_base')
-                utils.pushDockerhub('mozorg/bedrock_code')
-                utils.pushDockerhub('mozorg/bedrock_l10n', 'mozorg/bedrock')
-            } catch(err) {
-                utils.ircNotification(config, [stage: 'Dockerhub Push Failed', status: 'warning'])
+    if ( branchConfig.push_public_registry ) {
+        milestone()
+        stage ('Push Public Images') {
+            node {
+                unstash 'scripts'
+                try {
+                    utils.pushDockerhub('mozorg/bedrock_base')
+                    utils.pushDockerhub('mozorg/bedrock_code')
+                    utils.pushDockerhub('mozorg/bedrock_l10n', 'mozorg/bedrock')
+                } catch(err) {
+                    utils.ircNotification(config, [stage: 'Dockerhub Push Failed', status: 'warning'])
+                }
             }
         }
     }
@@ -94,68 +98,74 @@ if ( config.branches.containsKey(env.BRANCH_NAME) ) {
      *
      * A failure at any step of the above should fail the entire job
      */
-    milestone()
-    for (regionId in branchConfig.regions) {
-        def region = config.regions[regionId]
-        def stageName = "Private Push: ${region.name}"
-        stage (stageName) {
-            node {
-                unstash 'scripts'
-                try {
-                    utils.pushPrivateReg(region.registry_port, branchConfig.apps)
-                } catch(err) {
-                    utils.ircNotification(config, [stage: stageName, status: 'failure'])
-                    throw err
+    if ( branchConfig.apps ) {
+        milestone()
+        // default to usw only
+        def regions = branchConfig.regions ?: ['usw']
+        for (regionId in regions) {
+            def region = config.regions[regionId]
+            def stageName = "Private Push: ${region.name}"
+            stage (stageName) {
+                node {
+                    unstash 'scripts'
+                    try {
+                        utils.pushPrivateReg(region.registry_port, branchConfig.apps)
+                    } catch(err) {
+                        utils.ircNotification(config, [stage: stageName, status: 'failure'])
+                        throw err
+                    }
                 }
             }
-        }
-        for (appname in branchConfig.apps) {
-            def appSuffix = branchConfig.app_name_suffix ?: ''
-            def appURL = "https://${appname}${appSuffix}.${region.name}.moz.works"
-            stageName = "Deploy ${appname}-${region.name}"
-            // ensure no deploy/test cycle happens in parallel for an app/region
-            lock (stageName) {
-                milestone()
-                stage (stageName) {
-                    node {
-                        unstash 'scripts'
-                        withEnv(["DEIS_PROFILE=${region.deis_profile}",
-                                 "DOCKER_REPOSITORY=${appname}",
-                                 "DEIS_APPLICATION=${appname}"]) {
-                            try {
-                                retry(3) {
-                                    sh 'docker/bin/push2deis.sh'
+            for (appname in branchConfig.apps) {
+                def appSuffix = branchConfig.app_name_suffix ?: ''
+                def appURL = "https://${appname}${appSuffix}.${region.name}.moz.works"
+                stageName = "Deploy ${appname}-${region.name}"
+                // ensure no deploy/test cycle happens in parallel for an app/region
+                lock (stageName) {
+                    milestone()
+                    stage (stageName) {
+                        node {
+                            unstash 'scripts'
+                            withEnv(["DEIS_PROFILE=${region.deis_profile}",
+                                     "DOCKER_REPOSITORY=${appname}",
+                                     "DEIS_APPLICATION=${appname}"]) {
+                                try {
+                                    retry(3) {
+                                        sh 'docker/bin/push2deis.sh'
+                                    }
+                                } catch(err) {
+                                    utils.ircNotification(config, [stage: stageName, status: 'failure'])
+                                    throw err
                                 }
+                            }
+                        }
+                    }
+                    if ( branchConfig.apps ) {
+                        // queue up test closures
+                        def allTests = [:]
+                        for (filename in branchConfig.integration_tests) {
+                            allTests[filename] = utils.integrationTestJob(filename, appURL)
+                        }
+                        stage ("Test ${appname}-${region.name}") {
+                            try {
+                                // wait for server to be ready
+                                sleep(time: 10, unit: 'SECONDS')
+                                parallel allTests
                             } catch(err) {
-                                utils.ircNotification(config, [stage: stageName, status: 'failure'])
+                                node {
+                                    unstash 'scripts'
+                                    utils.ircNotification(config, [stage: "Integration Tests ${appname}-${region.name}", status: 'failure'])
+                                }
                                 throw err
                             }
                         }
                     }
-                }
-                // queue up test closures
-                def allTests = [:]
-                for (filename in branchConfig.integration_tests) {
-                    allTests[filename] = utils.integrationTestJob(filename, appURL)
-                }
-                stage ("Test ${appname}-${region.name}") {
-                    try {
-                        // wait for server to be ready
-                        sleep(time: 10, unit: 'SECONDS')
-                        parallel allTests
-                    } catch(err) {
-                        node {
-                            unstash 'scripts'
-                            utils.ircNotification(config, [stage: "Integration Tests ${appname}-${region.name}", status: 'failure'])
-                        }
-                        throw err
+                    node {
+                        unstash 'scripts'
+                        // huge success \o/
+                        utils.ircNotification(config, [message: appURL, status: 'shipped'])
                     }
                 }
-                node {
-                    unstash 'scripts'
-                    // huge success \o/
-                    utils.ircNotification(config, [message: appURL, status: 'shipped'])
-                }
             }
         }
     }

docker/bin/push2dockerhub.sh

@@ -9,8 +9,10 @@ set -ex
 
 docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD -e $DOCKER_USERNAME@example.com
 
-# Tag using git hash
-docker tag $FROM_DOCKER_REPOSITORY:${GIT_COMMIT} $DOCKER_REPOSITORY:${GIT_COMMIT}
+if [[ "$FROM_DOCKER_REPOSITORY" != "$DOCKER_REPOSITORY" ]]; then
+    # Tag using git hash
+    docker tag $FROM_DOCKER_REPOSITORY:${GIT_COMMIT} $DOCKER_REPOSITORY:${GIT_COMMIT}
+fi
 
 # Push to docker hub
 docker push $DOCKER_REPOSITORY:${GIT_COMMIT}

docs/pipeline.rst

@@ -104,10 +104,34 @@ Configuration
 ~~~~~~~~~~~~~
 
 Many of the options are configured via environment variables passed from the initial
-script, to the Docker image and onto the final script. This means that global defaults
-can be `configured in Jenkins`_. Note that admin access is required to make changes to the
-global configuration, and there is a known issue that may cause Jenkins to `become
-unresponsive`_ after a configuration change.
+script, to the Docker image and onto the final script. Many of these options can be
+set in the `jenkins.yml file`_. In the ``branches`` section of ``jenkins.yml`` you may specify
+any branch name and set how it should be built by jenkins. Take the following example:
+
+.. code-block:: yaml
+
+    branches:
+      change-all-the-things:
+        smoke_tests: true
+        apps:
+          - bedrock-probably-broken
+
+This configuration would cause commits pushed to a branch named ``change-all-the-things`` to have docker
+images built for them, have the smoke and unit tests run, and deploy to a deis app named ``bedrock-probably-broken``
+in our us-west deis cluster. It will not (yet) create that deis app for you like a demo deployment would,
+but you can set it to deploy to any existing app you want. Note that said app must have the ``jenkins`` user added via the
+``deis perms:create jenkins -a <your app name>`` command.
+
+The available branch configuration options are as follows:
+
+* ``smoke_tests``: boolean. Set to ``true`` to cause the unit and smoke test suites run against the docker images.
+* ``push_public_registry``: boolean. Set to ``true`` to cause the built images to be pushed to the public docker hub.
+* ``require_tag``: boolean. Set to ``true`` to require that the commit being built have a git tag in the format YYYY-MM-DD.X.
+* ``app_name_suffix``: string. Set to a value to have the IRC notification alter the URL of the deployed app. (only useful for stage and prod)
+* ``regions``: list. A list of strings indicating the deployment regions for the set of apps. The valid values are in the ``regions`` area of
+  the jenkins.yml file. If omitted a deployment to only ``usw`` is assumed.
+* ``apps``: list. A list of strings indicating the deis app name(s) to which to deploy. If omitted no deployments will occur.
+* ``integration_tests``: list. A list of strings indicating the types of integration tests to run. If omitted no tests will run.
 
 Updating Selenium
 ~~~~~~~~~~~~~~~~~

jenkins.yml

@@ -1,8 +1,12 @@
 environment:
   SELENIUM_VERSION: 2.52.0
 
+# see the following for documentation of these options
+# http://bedrock.readthedocs.io/en/latest/pipeline.html#configuration
 branches:
   master:
+    push_public_registry: true
+    smoke_tests: true
     regions:
       - usw
       - euw
@@ -12,6 +16,8 @@ branches:
       - firefox
       - headless
   prod:
+    push_public_registry: true
+    smoke_tests: true
     require_tag: true
     app_name_suffix: "-deis"
     regions:
@@ -28,6 +34,7 @@ branches:
       - ie6
       - ie7
   run-integration-tests:
+    smoke_tests: true
     regions:
       - usw
     apps:
@@ -39,6 +46,8 @@ branches:
       - ie
       - ie6
       - ie7
+  run-smoke-tests:
+    smoke_tests: true
 
 regions:
   usw: