Docker delivery pipeline with jenkins and deis.

* Don't include git directory.

* Build bedrock_base

* Build bedrock_code

* Update run_tests

* Build bedrock_l10n

* Update push to docker hub.

* Update push2deis

* bedrock_l10n: When time triggered use the currently deployed commit.

* Script to check if commit is tag.

* Skip docker image squashing if not needed.

* Allow pulling base docker images.

* Build only if commit in master branch or commit is tagged.

.dockerignore

@@ -1,2 +1,3 @@
 locale/.svn
 .env
+.git

docker/bin/deploy.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+URL=${URL:-"https://ci.us-west.moz.works/job/bedrock_base_image/buildWithParameters"}
+TAG=$1
+
+if [ -z "$1" ]
+  then
+      echo "Usage: $0 <tag>"
+      exit 0;
+fi
+
+curl -X POST "$URL?delay=0&token=$WEBHOOK_SECRET&TAG=$TAG"

docker/docker-compose.yml

@@ -1,7 +1,7 @@
 db:
   image: postgres:9.4
 web:
-  build: ../
+  image: ${FROM_DOCKER_REPOSITORY}:${GIT_COMMIT}
   ports:
     - "8000:8000"
   links:

docker/dockerfiles/bedrock_base

@@ -20,19 +20,8 @@ COPY ./bin/peep.py /app/bin/peep.py
 RUN ./bin/peep.py install --no-cache-dir -r requirements/dev.txt
 RUN ./bin/peep.py install --no-cache-dir -r requirements/prod.txt
 RUN ./bin/peep.py install --no-cache-dir -r requirements/docker.txt
-COPY . /app
-
-RUN git rev-parse HEAD > static/revision.txt
-RUN ./manage.py collectstatic --noinput
-RUN ./manage.py update_product_details
 
 # Cleanup
 RUN apt-get purge -y python-dev build-essential libxml2-dev libxslt1-dev libmemcached-dev git
 RUN apt-get autoremove -y
-RUN rm -rf /var/lib/{apt,dpkg,cache,log} /usr/share/doc /usr/share/man /tmp/* /var/cache/* /app/.git
-RUN find /app -name *.pyc -delete
-RUN ./docker/softlinkstatic.py
-
-# Change User
-RUN chown webdev.webdev -R .
-USER webdev
+RUN rm -rf /var/lib/{apt,dpkg,cache,log} /usr/share/doc /usr/share/man /tmp/* /var/cache/*

docker/dockerfiles/bedrock_code

@@ -0,0 +1,14 @@
+FROM ${FROM_DOCKER_REPOSITORY}:${GIT_COMMIT}
+
+COPY . /app
+
+RUN echo "${DOCKER_IMAGE_TAG}" > static/revision.txt
+RUN ./manage.py collectstatic --noinput
+RUN ./manage.py update_product_details
+
+# Cleanup
+RUN ./docker/softlinkstatic.py
+
+# Change User
+RUN chown webdev.webdev -R .
+USER webdev

docker/dockerfiles/bedrock_l10n

@@ -0,0 +1,2 @@
+FROM ${FROM_DOCKER_REPOSITORY}:${GIT_COMMIT}
+COPY . /app/locale

docker/jenkins/build_image.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+set -ex
+DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT}
+
+# If docker image exists and no force rebuild do nothing
+FORCE_REBUILD=`echo "$FORCE_REBUILD" | tr '[:upper:]' '[:lower:]'`
+if [[ $FORCE_REBUILD != "true" ]];
+then
+    if docker history -q $DOCKER_IMAGE_TAG 2> /dev/null;
+    then
+        echo "Docker image already exists, do nothing"
+        exit 0;
+    fi
+fi
+
+# Workaround to ignore mtime until we upgrade to Docker 1.8
+# See https://github.com/docker/docker/pull/12031
+find . -newerat 20140101 -exec touch -t 201401010000 {} \;
+
+cat docker/dockerfiles/${DOCKERFILE} | envsubst > Dockerfile
+docker build -t $DOCKER_IMAGE_TAG --pull=${UPDATE_DOCKER_IMAGES:-false} . | tee docker-build.log
+
+if [[ $FORCE_REBUILD != "true" ]];
+then
+    if tail -n 3 docker-build.log | grep "Using cache";
+    then
+        echo "Docker image already squashed, skip squashing";
+        TAG=`tail -n 1 docker-build.log | awk '{ print $(NF) }'`
+        docker tag -f $TAG $DOCKER_IMAGE_TAG
+        exit 0;
+    fi
+fi
+
+docker save $DOCKER_IMAGE_TAG  | sudo docker-squash -t $DOCKER_IMAGE_TAG | docker load

docker/jenkins/check_if_allowed_sha.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# If commit is not a tag
+if ! git describe --tags --exact-match $GIT_COMMIT > /dev/null 2>&1;
+then
+    # And if commit is not in master branch then exit.
+    if ! git branch --contains $GIT_COMMIT 2> /dev/null | grep "* master" > /dev/null;
+    then
+        exit 1;
+    fi;
+fi;
+
+# Commit is a tag or in master branch.
+exit 0;

docker/jenkins/check_if_tag.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Used to trigger downstream Jenkins jobs
+TRIGGER_FILE=.commit_is_tag
+rm -rf $TRIGGER_FILE
+
+if git describe --tags --exact-match $GIT_COMMIT 2> /dev/null;
+then
+    touch $TRIGGER_FILE
+fi;

docker/jenkins/include_l10n.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Needs DOCKER_REPOSITORY and FROM_DOCKER_REPOSITORY
+#
+# To set them go to Job -> Configure -> Build Environment -> Inject
+# passwords and Inject env variables
+#
+set -xe
+
+# Used to trigger downstream Jenkins jobs
+TRIGGER_FILE=.docker-updated
+rm -rf $TRIGGER_FILE
+
+if [[ $BUILD_CAUSE == "TIMERTRIGGER" ]]
+then
+    SVN_STATUS=`svn status -uq locale | wc -l`
+    if [[ $SVN_STATUS == "0" ]]
+    then
+        # No updates, just exit
+        echo "No locale updates"
+        exit 0;
+    else
+        # Set GIT_COMMIT to the current deployed to prod commit
+        COMMIT_URL=${COMMIT_URL:-https://www.mozilla.org/static/revision.txt}
+        GIT_COMMIT=`curl  $COMMIT_URL 2> /dev/null`
+    fi
+fi
+
+DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT}
+
+touch $TRIGGER_FILE
+
+set +e
+svn cleanup locale
+set -e
+svn co http://svn.mozilla.org/projects/mozilla.com/trunk/locales/ locale
+
+
+cat docker/dockerfiles/bedrock_l10n | envsubst > ./locale/Dockerfile
+echo ".svn" > ./locale/.dockerignore
+
+docker build -f locale/Dockerfile -t $DOCKER_IMAGE_TAG locale

docker/jenkins/push2deis.sh

@@ -8,12 +8,5 @@
 
 set -ex
 
-# Create a temporary virtualenv to install deis client
-TDIR=`mktemp -d`
-virtualenv $TDIR
-. $TDIR/bin/activate
-pip install deis==1.8.0
-
-
 deis login $DEIS_CONTROLLER  --username $DEIS_USERNAME --password $DEIS_PASSWORD
-deis pull $DOCKER_REPOSITORY:`git rev-parse HEAD` -a $DEIS_APPLICATION
+deis pull $DOCKER_REPOSITORY:$GIT_COMMIT -a $DEIS_APPLICATION

docker/jenkins/push2dockerhub.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
-# Needs DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REPOSITORY
-# environment variables.
+# Needs DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_REPOSITORY,
+# FROM_DOCKER_REPOSITORY environment variables.
 #
 # To set them go to Job -> Configure -> Build Environment -> Inject
 # passwords and Inject env variables
@@ -13,9 +13,5 @@ docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD -e $DOCKER_USERNAME@example
 COMMIT="${ghprbActualCommit:=$GIT_COMMIT}"
 
 # Tag using git hash
-docker tag -f `echo jenkins${JOB_NAME}${BUILD_NUMBER}| sed s/_//g`_web $DOCKER_REPOSITORY:$COMMIT
+docker tag -f $FROM_DOCKER_REPOSITORY:$COMMIT $DOCKER_REPOSITORY:$COMMIT
 docker push $DOCKER_REPOSITORY:$COMMIT
-
-# Tag as latest
-docker tag -f `echo jenkins${JOB_NAME}${BUILD_NUMBER}| sed s/_//g`_web $DOCKER_REPOSITORY:latest
-docker push $DOCKER_REPOSITORY:latest

docker/jenkins/run_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Runs unit_tests
+#
+set -ex
+
+# Create a temporary virtualenv to install docker-compose
+TDIR=`mktemp -d`
+virtualenv $TDIR
+. $TDIR/bin/activate
+pip install docker-compose==1.2.0
+
+cat docker/docker-compose.yml | envsubst > ./docker-compose.yml
+
+DOCKER_COMPOSE="docker-compose --project-name jenkins${JOB_NAME}${BUILD_NUMBER}"
+# Start the database and give it some time to boot up
+# TODO Uncomment when unit tests can run without locales.
+## $DOCKER_COMPOSE up -d db
+## sleep 10s;
+## $DOCKER_COMPOSE run -T web ./manage.py test
+
+# Cleanup
+$DOCKER_COMPOSE stop
+rm -rf $TDIR

docker/jenkins/runtests.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-#
-# Runs unit_tests
-#
-set -ex
-
-# Create a temporary virtualenv to install docker-compose
-TDIR=`mktemp -d`
-virtualenv $TDIR
-. $TDIR/bin/activate
-pip install docker-compose==1.2.0
-
-# Pull locales, we probably need to move this elsewhere.
-git submodule update --init --recursive
-
-set +e
-svn cleanup locale
-set -e
-svn co http://svn.mozilla.org/projects/mozilla.com/trunk/locales/ locale
-
-DOCKER_COMPOSE="docker-compose --project-name jenkins${JOB_NAME}${BUILD_NUMBER} -f docker/docker-compose.yml"
-
-$DOCKER_COMPOSE build
-
-# Start the database and give it some time to boot up
-$DOCKER_COMPOSE up -d db
-
-docker save `echo jenkins${JOB_NAME}${BUILD_NUMBER}| sed s/_//g`_web | sudo docker-squash -t `echo jenkins${JOB_NAME}${BUILD_NUMBER}| sed s/_//g`_web | docker load
-
-
-$DOCKER_COMPOSE run -T web ./manage.py test
-
-# Cleanup
-$DOCKER_COMPOSE stop
-rm -rf $TDIR