Move docker scripts, envfiles, and groovy files

Jenkinsfile

@@ -15,7 +15,7 @@ stage ('Checkout') {
         // load the config
         config = readYaml file: 'jenkins.yml'
         // load the utility functions used below
-        utils = load 'docker/jenkins/utils.groovy'
+        utils = load 'jenkins/utils.groovy'
         // defined in the Library loaded above
         setGitEnvironmentVariables()
         setConfigEnvironmentVariables(config)
@@ -35,7 +35,7 @@ if ( config.branches.containsKey(env.BRANCH_NAME) ) {
             // make sure we should continue
             if ( branchConfig.require_tag ) {
                 try {
-                    sh 'docker/jenkins/check_if_tag.sh'
+                    sh 'docker/bin/check_if_tag.sh'
                 } catch(err) {
                     utils.ircNotification(config, [stage: 'Git Tag Check', status: 'failure'])
                     throw err
@@ -44,7 +44,7 @@ if ( config.branches.containsKey(env.BRANCH_NAME) ) {
             utils.ircNotification(config, [stage: 'Test & Deploy', status: 'starting'])
             lock ("bedrock-docker-${env.GIT_COMMIT}") {
                 try {
-                    sh 'docker/jenkins/build_images.sh --prod --test'
+                    sh 'docker/bin/build_images.sh --prod --test'
                 } catch(err) {
                     utils.ircNotification(config, [stage: 'Docker Build', status: 'failure'])
                     throw err
@@ -56,12 +56,12 @@ if ( config.branches.containsKey(env.BRANCH_NAME) ) {
     milestone()
     stage ('Test Images') {
         parallel([
-            integration_tests: utils.integrationTestJob('smoke'),
+            smoke_tests: utils.integrationTestJob('smoke'),
             unit_tests: {
                 node {
                     unstash 'scripts'
                     try {
-                        sh 'docker/jenkins/run_tests.sh'
+                        sh 'docker/bin/run_tests.sh'
                     } catch(err) {
                         utils.ircNotification(config, [stage: 'Unit Test', status: 'failure'])
                         throw err
@@ -124,7 +124,7 @@ if ( config.branches.containsKey(env.BRANCH_NAME) ) {
                                  "DEIS_APPLICATION=${appname}"]) {
                             try {
                                 retry(3) {
-                                    sh 'docker/jenkins/push2deis.sh'
+                                    sh 'docker/bin/push2deis.sh'
                                 }
                             } catch(err) {
                                 utils.ircNotification(config, [stage: stageName, status: 'failure'])
@@ -170,7 +170,7 @@ else if ( env.BRANCH_NAME ==~ /^demo__[\w-]+$/ ) {
             lock ("bedrock-docker-${env.GIT_COMMIT}") {
                 milestone()
                 try {
-                    sh 'docker/jenkins/build_images.sh --demo'
+                    sh 'docker/bin/build_images.sh --demo'
                 } catch(err) {
                     utils.ircNotification(config, [stage: 'Demo Build', status: 'failure'])
                     throw err
@@ -189,7 +189,7 @@ else if ( env.BRANCH_NAME ==~ /^demo__[\w-]+$/ ) {
                         withEnv(['DEIS_PROFILE=usw',
                                  "DEIS_APP_NAME=${appname}",
                                  "PRIVATE_REGISTRY=localhost:${config.regions.usw.registry_port}"]) {
-                            sh './docker/jenkins/demo_deploy.sh'
+                            sh './docker/bin/demo_deploy.sh'
                         }
                     }
                 } catch(err) {

Makefile

@@ -35,16 +35,16 @@ build-final: .docker-build
 	touch .docker-build-final
 
 run: .docker-build
-	docker run --env-file docker/dev.env -p 8000:8000 -v "$$PWD:/app" ${DEV_IMG_NAME}
+	docker run --env-file docker/envfiles/dev.env -p 8000:8000 -v "$$PWD:/app" ${DEV_IMG_NAME}
 
 django-shell: .docker-build
-	docker run --user `id -u` -it --env-file docker/dev.env -v "$$PWD:/app" ${DEV_IMG_NAME} python manage.py shell
+	docker run --user `id -u` -it --env-file docker/envfiles/dev.env -v "$$PWD:/app" ${DEV_IMG_NAME} python manage.py shell
 
 shell: .docker-build
-	docker run --user `id -u` -it --env-file docker/dev.env -v "$$PWD:/app" ${DEV_IMG_NAME} bash
+	docker run --user `id -u` -it --env-file docker/envfiles/dev.env -v "$$PWD:/app" ${DEV_IMG_NAME} bash
 
 sync-all: .docker-build
-	docker run --user `id -u` --env-file docker/demo.env -v "$$PWD:/app" ${DEV_IMG_NAME} bin/sync-all.sh
+	docker run --user `id -u` --env-file docker/envfiles/demo.env -v "$$PWD:/app" ${DEV_IMG_NAME} bin/sync-all.sh
 
 clean:
 	# python related things
@@ -69,12 +69,12 @@ clean:
 	-rm -f .docker-build-final
 
 test: .docker-build
-	docker run --user `id -u` --env-file docker/test.env -v "$$PWD:/app" ${DEV_IMG_NAME} bin/run-tests.sh
+	docker run --user `id -u` --env-file docker/envfiles/test.env -v "$$PWD:/app" ${DEV_IMG_NAME} bin/run-tests.sh
 
 test-image: .docker-build-final
-	docker run --env-file docker/test.env ${FINAL_IMG_NAME} bin/run-tests.sh
+	docker run --env-file docker/envfiles/test.env ${FINAL_IMG_NAME} bin/run-tests.sh
 
 docs:
-	docker run --user `id -u` --env-file docker/dev.env -v "$$PWD:/app" ${DEV_IMG_NAME} bash -c "make -C docs/ clean && make -C docs/ html"
+	docker run --user `id -u` --env-file docker/envfiles/dev.env -v "$$PWD:/app" ${DEV_IMG_NAME} bash -c "make -C docs/ clean && make -C docs/ html"
 
 .PHONY: default clean build build-final docs run test sync-all test-image shell django-shell

circle.yml

@@ -14,7 +14,7 @@ dependencies:
     - node_modules
     - "~/docker"
   override:
-    - docker/jenkins/build_images.sh --test
+    - docker/bin/build_images.sh --test
     - npm install -g gulp-cli
     - npm install
 
@@ -25,4 +25,4 @@ test:
     - gulp css:lint
     - gulp js:lint
     - gulp js:test
-    - docker/jenkins/run_tests.sh
+    - docker/bin/run_tests.sh

docker/bin/build_images.sh

@@ -48,11 +48,11 @@ function dockerRun() {
     env_file="$1"
     image_tag="mozorg/bedrock_${2}:${GIT_COMMIT}"
     cmd="$3"
-    docker run --user $(id -u) -v "$PWD:/app" --env-file "docker/${env_file}.env" "$image_tag" bash -c "$cmd"
+    docker run --user $(id -u) -v "$PWD:/app" --env-file "docker/envfiles/${env_file}.env" "$image_tag" bash -c "$cmd"
 }
 
 if ! imageExists "base"; then
-    docker/jenkins/docker_build.sh --pull "base"
+    docker/bin/docker_build.sh --pull "base"
 fi
 
 # build the static files using the builder image
@@ -60,24 +60,24 @@ fi
 if ! imageExists "code"; then
     # build a staticfiles builder image
     if ! imageExists "build"; then
-        docker/jenkins/docker_build.sh "build"
+        docker/bin/docker_build.sh "build"
     fi
-    dockerRun prod build docker/jenkins/build_staticfiles.sh
+    dockerRun prod build docker/bin/build_staticfiles.sh
     echo "${GIT_COMMIT}" > static/revision.txt
-    docker/jenkins/docker_build.sh "code"
+    docker/bin/docker_build.sh "code"
 fi
 
 # build a tester image for non-demo deploys
 if $TEST_MODE && ! imageExists "test"; then
-    docker/jenkins/docker_build.sh "test"
+    docker/bin/docker_build.sh "test"
 fi
 
 # include the data that the deployments need
 if $DEMO_MODE && ! imageExists "demo"; then
     dockerRun demo code bin/sync-all.sh
-    docker/jenkins/docker_build.sh "demo"
+    docker/bin/docker_build.sh "demo"
 fi
 if $PROD_MODE && ! imageExists "l10n"; then
     dockerRun prod code "python manage.py l10n_update"
-    docker/jenkins/docker_build.sh -c "locale" "l10n"
+    docker/bin/docker_build.sh -c "locale" "l10n"
 fi

docker/bin/demo_deploy.sh

@@ -12,7 +12,7 @@ docker push "$PRIVATE_IMAGE_TAG"
 echo "Creating the demo app $DEIS_APP_NAME"
 if deis apps:create "$DEIS_APP_NAME" --no-remote; then
   echo "Configuring the new demo app"
-  deis config:push -a "$DEIS_APP_NAME" -p docker/demo.env
+  deis config:push -a "$DEIS_APP_NAME" -p docker/envfiles/demo.env
   # Sentry DSN is potentially sensitive. Turn off command echo.
   set +x
   if [[ -n "$SENTRY_DEMO_DSN" ]]; then

docker/bin/run_tests.sh

@@ -9,4 +9,4 @@ if [[ -z "$GIT_COMMIT" ]]; then
 fi
 
 TEST_IMAGE_TAG="mozorg/bedrock_test:${GIT_COMMIT}"
-docker run --env-file docker/test.env "$TEST_IMAGE_TAG"
+docker run --env-file docker/envfiles/test.env "$TEST_IMAGE_TAG"

docker/bin/trigger.py

@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-import requests
-
-URL = os.getenv('URL', ('https://ci.us-west.moz.works/job/bedrock_base_image/'
-                        'buildWithParameters?delay=0&token={secret}&'
-                        'TAG={tag}&BUILDONLY={buildonly}'))
-
-if len(sys.argv) < 2:
-    print('Usage: {} <tag> [buildonly=true|false]'.format(sys.argv[0]))
-    sys.exit(0)
-
-SECRET = os.getenv('WEBHOOK_SECRET')
-if not SECRET:
-    print('Set WEBHOOK_SECRET environment variable')
-    sys.exit(1)
-TAG = sys.argv[1]
-
-try:
-    BUILDONLY = sys.argv[2]
-except IndexError:
-    BUILDONLY = 'False'
-
-if BUILDONLY.lower() not in ['false', 'true']:
-    print('Valid options for BuildOnly are "true" and "false"')
-    sys.exit(1)
-
-requests.post(URL.format(secret=SECRET, tag=TAG, buildonly=BUILDONLY))

docs/install.rst

@@ -88,9 +88,9 @@ To test a single app, specify the app by name in the command above. e.g.::
     If your local tests run fine, but when you submit a pull-request the tests fail in
     `CircleCI <https://circleci.com/gh/mozilla/bedrock>`_, it could be due to the
     difference in settings between what you have in ``.env``
-    and what CircleCI uses: ``docker/demo.env``. You can run tests as close to Circle
+    and what CircleCI uses: ``docker/envfiles/demo.env``. You can run tests as close to Circle
     as possible by moving your ``.env`` file to another name (e.g. ``.env-backup``), then
-    copying ``docker/demo.env`` to ``.env``, and running tests again.
+    copying ``docker/envfiles/demo.env`` to ``.env``, and running tests again.
 
 Make it run
 -----------

docs/pipeline.rst

@@ -85,7 +85,7 @@ Pipeline integration
 Our `Jenkinsfile`_ will run the integration tests based on information in our `jenkins.yml file`_.
 This file specifies various test names per branch that will cause it to use different
 parameters, allowing it to be called in many different ways to cover the testing
-needs. The job executes `this script <https://github.com/mozilla/bedrock/blob/master/docker/jenkins/run_integration_tests.sh>`_,
+needs. The job executes `this script <https://github.com/mozilla/bedrock/blob/master/docker/bin/run_integration_tests.sh>`_,
 which then runs `this Docker image <https://github.com/mozilla/bedrock/blob/master/docker/dockerfiles/bedrock_test>`_,
 and ultimately runs `another script <https://github.com/mozilla/bedrock/blob/master/bin/run-integration-tests.sh>`_.
 The two scripts can also be executed locally to replicate the way Jenkins operates.
@@ -124,7 +124,7 @@ Adding test runs
 ~~~~~~~~~~~~~~~~
 
 Test runs can be added by creating a new properties section in the
-`integration tests script <https://github.com/mozilla/bedrock/blob/master/docker/jenkins/run_integration_tests.sh>`_
+`integration tests script <https://github.com/mozilla/bedrock/blob/master/docker/bin/run_integration_tests.sh>`_
 with the parameters of the new test run. This is simply a bash script and you can duplicate a clause of the case staement.
 For example, if you wanted to run tests in Firefox on both Windows 10 and
 OS X, you could create the following clauses:

jenkins/utils.groovy

@@ -37,7 +37,7 @@ def pushDockerhub(from_repo, to_repo='') {
                  "FROM_DOCKER_REPOSITORY=${from_repo}",
                  "DOCKER_REPOSITORY=${to_repo}"]) {
             retry(2) {
-                sh 'docker/jenkins/push2dockerhub.sh'
+                sh 'docker/bin/push2dockerhub.sh'
             }
         }
     }
@@ -48,7 +48,7 @@ def pushPrivateReg(port, apps) {
         withEnv(['FROM_DOCKER_REPOSITORY=mozorg/bedrock_l10n',
                  "PRIVATE_REGISTRIES=localhost:${port}",
                  "DEIS_APPS=${apps.join(',')}"]) {
-            sh 'docker/jenkins/push2privateregistries.sh'
+            sh 'docker/bin/push2privateregistries.sh'
         }
     }
 }
@@ -58,7 +58,7 @@ def integrationTestJob(propFileName, appURL='') {
         node {
             unstash 'scripts'
             unstash 'tests'
-            def testScript = "docker/jenkins/run_integration_tests.sh ${propFileName}".toString()
+            def testScript = "docker/bin/run_integration_tests.sh ${propFileName}".toString()
             withCredentials([[$class: 'UsernamePasswordMultiBinding',
                               credentialsId: 'SAUCELABS_CREDENTIALS',
                               usernameVariable: 'SAUCELABS_USERNAME',
@@ -70,7 +70,7 @@ def integrationTestJob(propFileName, appURL='') {
                     finally {
                         junit 'results/*.xml'
                         if ( propFileName == 'local' ) {
-                            sh 'docker/jenkins/cleanup_after_functional_tests.sh'
+                            sh 'docker/bin/cleanup_after_functional_tests.sh'
                         }
                     }
                 }