diff --git a/.dockerignore b/.dockerignore index 3417ffe524..a715c9d790 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,2 @@ -locale/.git .git .env diff --git a/.editorconfig b/.editorconfig index 5d69374c36..97cd47a363 100644 --- a/.editorconfig +++ b/.editorconfig @@ -28,6 +28,9 @@ indent_size = 4 [*.scss] indent_size = 4 +[*.sh] +indent_size = 4 + [*.html] indent_size = 2 @@ -37,6 +40,9 @@ indent_size = 2 [Jenkinsfile] indent_size = 4 +[*.groovy] +indent_size = 4 + [package.json] indent_size = 2 diff --git a/.gitignore b/.gitignore index a301131206..2982c18bd5 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ tmp/* *.less.css db.sql /locale +/results Thumbs.db Desktop.ini venv @@ -41,3 +42,4 @@ tests/functional/results.html tests/functional/assets/style.css .eslintcache .docker-build* +Dockerfile-* diff --git a/Jenkinsfile b/Jenkinsfile index a6dbbe2daf..7196d16e9e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!groovy -@Library('github.com/mozmar/jenkins-pipeline@20170126.1') +@Library('github.com/mozmar/jenkins-pipeline@20170214.1') def config def utils @@ -10,12 +10,15 @@ stage ('Checkout') { checkout scm sh 'git submodule sync' sh 'git submodule update --init --recursive' - // defined in the Library loaded above - setGitEnvironmentVariables() + // clean up + sh 'make clean' // load the config config = readYaml file: 'jenkins.yml' // load the utility functions used below utils = load 'docker/jenkins/utils.groovy' + // defined in the Library loaded above + setGitEnvironmentVariables() + setConfigEnvironmentVariables(config) // save the files for later stash name: 'scripts', includes: 'bin/,docker/' stash name: 'tests', includes: 'tests/,requirements/' @@ -39,62 +42,47 @@ if ( config.branches.containsKey(env.BRANCH_NAME) ) { } } utils.ircNotification(config, [stage: 'Test & Deploy', status: 'starting']) - try { - utils.buildDockerImage(dockerfile: 'bedrock_base', update: true) - utils.buildDockerImage(dockerfile: 'bedrock_code', fromDockerfile: 'bedrock_base') - } catch(err) { - utils.ircNotification(config, [stage: 'Docker Build', status: 'failure']) - throw err + lock ("bedrock-docker-${env.GIT_COMMIT}") { + try { + sh 'docker/jenkins/build_images.sh --prod --test' + } catch(err) { + utils.ircNotification(config, [stage: 'Docker Build', status: 'failure']) + throw err + } } } } milestone() stage ('Test Images') { - node { - unstash 'scripts' - try { - withEnv(['DOCKER_REPOSITORY=mozorg/bedrock_code']) { - sh 'docker/jenkins/run_tests.sh' + parallel([ + integration_tests: utils.integrationTestJob('local'), + unit_tests: { + node { + unstash 'scripts' + try { + sh 'docker/jenkins/run_tests.sh' + } catch(err) { + utils.ircNotification(config, [stage: 'Unit Test', status: 'failure']) + throw err + } } - } catch(err) { - utils.ircNotification(config, [stage: 'Unit Test', status: 'failure']) - throw err - } - utils.ircNotification(config, [stage: 'Docker Builds & Tests', status: 'complete']) - } + }, + ]) } milestone() stage ('Push Public Images') { - parallel([ - dockerhub: { - node { - unstash 'scripts' - try { - utils.buildDockerImage(dockerfile: 'bedrock_l10n', fromDockerfile: 'bedrock_code', script: 'include_l10n.sh') - } catch(err) { - utils.ircNotification(config, [stage: 'L10n Build', status: 'failure']) - throw err - } - try { - utils.pushDockerhub('mozorg/bedrock_base') - utils.pushDockerhub('mozorg/bedrock_code') - utils.pushDockerhub('mozorg/bedrock_l10n', 'mozorg/bedrock') - } catch(err) { - utils.ircNotification(config, [stage: 'Dockerhub Push Failed', status: 'warning']) - } - } - }, - integration_tests: { - node { - unstash 'scripts' - unstash 'tests' - // prep for next stage - sh 'docker/jenkins/build_integration_test_image.sh' - } - }, - ]) + node { + unstash 'scripts' + try { + utils.pushDockerhub('mozorg/bedrock_base') + utils.pushDockerhub('mozorg/bedrock_code') + utils.pushDockerhub('mozorg/bedrock_l10n', 'mozorg/bedrock') + } catch(err) { + utils.ircNotification(config, [stage: 'Dockerhub Push Failed', status: 'warning']) + } + } } /** @@ -179,16 +167,14 @@ else if ( env.BRANCH_NAME ==~ /^demo__[\w-]+$/ ) { node { utils.ircNotification(config, [stage: 'Demo Deploy', status: 'starting']) stage ('build') { - milestone() - try { - sh 'make clean' - sh 'make sync-all' - sh 'echo "ENV GIT_SHA ${GIT_COMMIT}" >> docker/dockerfiles/bedrock_dev_final' - sh 'echo "RUN echo ${GIT_COMMIT} > static/revision.txt" >> docker/dockerfiles/bedrock_dev_final' - sh 'make build-final' - } catch(err) { - utils.ircNotification(config, [stage: 'Demo Build', status: 'failure']) - throw err + lock ("bedrock-docker-${env.GIT_COMMIT}") { + milestone() + try { + sh 'docker/jenkins/build_images.sh --demo' + } catch(err) { + utils.ircNotification(config, [stage: 'Demo Build', status: 'failure']) + throw err + } } } diff --git a/Makefile b/Makefile index 8ebe67ace6..03596e19aa 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,10 @@ clean: # test related things -rm -f .coverage + -rm -rf results + + # static files + git clean -fdx static # docs files -rm -rf docs/_build/ diff --git a/circle.yml b/circle.yml index 1464ab568a..c1aed5f70b 100644 --- a/circle.yml +++ b/circle.yml @@ -2,18 +2,7 @@ machine: services: - docker node: - version: 4.2.6 - environment: - # Deis - DEIS_CONTROLLER: https://deis.us-west.moz.works - DEIS_USERNAME: circleci - # Docker - QUAY_EMAIL: mozmar@example.com - QUAY_USERNAME: mozmar+circleci - DOCKER_REPOSITORY: quay.io/mozmar/bedrock - DOCKER_IMAGE_TAG: "$DOCKER_REPOSITORY:$CIRCLE_BRANCH" - BASE_DOCKER_REPOSITORY: quay.io/mozmar/bedrock_base - BASE_DOCKER_IMAGE_TAG: "$BASE_DOCKER_REPOSITORY:$CIRCLE_BRANCH" + version: 6.1.0 checkout: post: @@ -25,9 +14,9 @@ dependencies: - node_modules - "~/docker" override: + - docker/jenkins/build_images.sh --test - npm install -g gulp-cli - npm install - - bin/circleci-docker-build.sh test: pre: @@ -36,4 +25,4 @@ test: - gulp css:lint - gulp js:lint - gulp js:test - - make test-image + - docker/jenkins/run_tests.sh diff --git a/docker/dockerfiles/bedrock_base b/docker/dockerfiles/bedrock_base index fa36e0ce4f..ae3426199b 100644 --- a/docker/dockerfiles/bedrock_base +++ b/docker/dockerfiles/bedrock_base @@ -1,25 +1,24 @@ -FROM quay.io/mozmar/base +FROM quay.io/mozmar/base:latest +# Set Python-related environment variables to reduce annoying-ness +ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 ENV PIP_DISABLE_PIP_VERSION_CHECK=1 ENV LANG=C.UTF-8 +RUN update-alternatives --install /bin/sh sh /bin/bash 10 +RUN adduser --uid 1000 --disabled-password --gecos '' --no-create-home webdev WORKDIR /app EXPOSE 8000 CMD ["./docker/run.sh"] -RUN adduser --uid 1000 --disabled-password --gecos '' --no-create-home webdev - RUN apt-get update && \ apt-get install -y --no-install-recommends \ - gettext build-essential python python-dev python-pip python-setuptools \ - libxml2-dev libxslt1.1 libxslt1-dev \ - nodejs-legacy npm git libpq-dev + gettext build-essential python-{dev,pip,setuptools} \ + libxml2-dev libxslt1.1 libxslt1-dev git libpq-dev COPY ./requirements /app/requirements -# Install app -RUN pip install --no-cache-dir -r requirements/dev.txt +# Install Python deps RUN pip install --no-cache-dir -r requirements/prod.txt RUN pip install --no-cache-dir -r requirements/docker.txt -RUN pip install --no-cache-dir -r requirements/test.txt diff --git a/docker/dockerfiles/bedrock_build b/docker/dockerfiles/bedrock_build new file mode 100644 index 0000000000..d2afab9e9f --- /dev/null +++ b/docker/dockerfiles/bedrock_build @@ -0,0 +1,14 @@ +FROM mozorg/bedrock_base:${GIT_COMMIT} + +ENV PATH=/node_modules/.bin:$PATH +ENV PIPELINE_LESS_BINARY=lessc +ENV PIPELINE_SASS_BINARY=node-sass +ENV PIPELINE_YUGLIFY_BINARY=yuglify + +RUN apt-get install -y --no-install-recommends nodejs-legacy npm + +COPY ./node_modules / +COPY ./package.json / +COPY ./lockdown.json / +# --unsafe-perm required for lockdown to function +RUN cd / && npm install --production --unsafe-perm diff --git a/docker/dockerfiles/bedrock_code b/docker/dockerfiles/bedrock_code index adb11d1722..279a20cc52 100644 --- a/docker/dockerfiles/bedrock_code +++ b/docker/dockerfiles/bedrock_code @@ -1,20 +1,25 @@ -FROM ${FROM_DOCKER_REPOSITORY}:${GIT_COMMIT} +FROM mozorg/bedrock_base:${GIT_COMMIT} -COPY . /app +# changes infrequently +COPY ./bin ./bin +COPY ./etc ./etc +COPY ./lib ./lib +COPY ./root_files ./root_files +COPY ./scripts ./scripts +COPY ./wsgi ./wsgi +COPY ./manage.py ./ +COPY ./LICENSE ./ +COPY ./newrelic.ini ./ +COPY ./contribute.json ./ -RUN echo "${GIT_COMMIT}" > static/revision.txt - -# --unsafe-perm required for lockdown to function -RUN npm install --production --unsafe-perm -RUN ./manage.py collectstatic -l --noinput - -# Cleanup -RUN rm -rf node_modules -RUN ./docker/bin/softlinkstatic.py -RUN apt-get purge -y nodejs npm python-dev build-essential libxml2-dev libxslt1-dev -RUN apt-get autoremove -y -RUN rm -rf /var/lib/{apt,dpkg,cache,log} /usr/share/doc /usr/share/man /tmp/* /var/cache/* +# changes more frequently +COPY ./docker ./docker +COPY ./vendor-local ./vendor-local +COPY ./bedrock ./bedrock +COPY ./media ./media +COPY ./static ./static +ENV GIT_SHA ${GIT_COMMIT} # Change User RUN chown webdev.webdev -R . USER webdev diff --git a/docker/dockerfiles/bedrock_demo b/docker/dockerfiles/bedrock_demo new file mode 100644 index 0000000000..e5c353c0fe --- /dev/null +++ b/docker/dockerfiles/bedrock_demo @@ -0,0 +1,9 @@ +FROM mozorg/bedrock_code:${GIT_COMMIT} + +COPY ./locale ./locale +COPY ./bedrock.db ./ + +# Change User +USER root +RUN chown webdev.webdev -R . +USER webdev diff --git a/docker/dockerfiles/bedrock_integration_tests b/docker/dockerfiles/bedrock_integration_tests deleted file mode 100644 index bfccac3e21..0000000000 --- a/docker/dockerfiles/bedrock_integration_tests +++ /dev/null @@ -1,26 +0,0 @@ -FROM quay.io/mozmar/base - -WORKDIR /app -# Run the tests -CMD ["/app/bin/run-integration-tests.sh"] - - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential python python-dev python-pip python-setuptools \ - libxml2-dev libxslt1.1 libxslt1-dev zlib1g-dev - -# Defaults -ENV PYTEST_PROCESSES 5 -ENV PRIVACY "public restricted" -ENV TESTS_PATH /app/tests -ENV RESULTS_PATH /app/results -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PIP_DISABLE_PIP_VERSION_CHECK=1 - -COPY ./requirements /app/requirements - -# Install requirements -RUN pip install --no-cache-dir -r requirements/test.txt - -COPY . /app diff --git a/docker/dockerfiles/bedrock_l10n b/docker/dockerfiles/bedrock_l10n index 6e313939d7..d3ef2dba4e 100644 --- a/docker/dockerfiles/bedrock_l10n +++ b/docker/dockerfiles/bedrock_l10n @@ -1,5 +1,8 @@ -FROM ${FROM_DOCKER_REPOSITORY}:${GIT_COMMIT} -COPY . /app/locale +FROM mozorg/bedrock_code:${GIT_COMMIT} + +COPY . ./locale/ + +# Change User USER root -RUN chown -R webdev.webdev /app/locale +RUN chown webdev.webdev -R locale USER webdev diff --git a/docker/dockerfiles/bedrock_linkchecker b/docker/dockerfiles/bedrock_linkchecker index 203a2f2b48..2c8fe74306 100644 --- a/docker/dockerfiles/bedrock_linkchecker +++ b/docker/dockerfiles/bedrock_linkchecker @@ -1,4 +1,4 @@ -FROM quay.io/mozmar/base +FROM quay.io/mozmar/base:latest RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential python python-dev python-pip python-setuptools diff --git a/docker/dockerfiles/bedrock_test b/docker/dockerfiles/bedrock_test new file mode 100644 index 0000000000..6d3c82ecde --- /dev/null +++ b/docker/dockerfiles/bedrock_test @@ -0,0 +1,11 @@ +FROM mozorg/bedrock_code:${GIT_COMMIT} + +CMD ["./docker/run-tests.sh"] +USER root + +RUN pip install --no-cache-dir -r requirements/test.txt +COPY ./setup.cfg ./ +COPY ./tests ./tests + +RUN chown webdev.webdev -R . +USER webdev diff --git a/docker/jenkins/build_image.sh b/docker/jenkins/build_image.sh deleted file mode 100755 index d0e7588e8a..0000000000 --- a/docker/jenkins/build_image.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -set -ex -DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT} -TMP_DOCKER_TAG=${BUILD_TAG} - -# If docker image exists and no force rebuild do nothing -FORCE_REBUILD=`echo "$FORCE_REBUILD" | tr '[:upper:]' '[:lower:]'` -if [[ $FORCE_REBUILD != "true" ]]; -then - if docker history -q $DOCKER_IMAGE_TAG > /dev/null; - then - echo "Docker image already exists, do nothing" - exit 0; - fi -fi - -cat docker/dockerfiles/${DOCKERFILE} | envsubst > Dockerfile - -if [[ $FORCE_REBUILD == "true" ]]; -then - NO_CACHE="true" -fi; - -docker build -t ${TMP_DOCKER_TAG} --pull=${UPDATE_DOCKER_IMAGES:-true} --no-cache=${NO_CACHE:-false} . | tee docker-build.log - -TAG=`tail -n 1 docker-build.log | awk '{ print $(NF) }'` - -if [[ $FORCE_REBUILD != "true" ]]; -then - if [[ $(tail -n 3 docker-build.log | grep "Using cache") && $(docker images | grep "${TAG}-squashed") ]]; - then - echo "Docker image already squashed, skip squashing"; - docker tag ${TAG}-squashed $DOCKER_IMAGE_TAG - exit 0; - fi -fi - -docker-squash -t ${TAG}-squashed ${TMP_DOCKER_TAG} -docker tag ${TAG}-squashed ${DOCKER_IMAGE_TAG} diff --git a/docker/jenkins/build_images.sh b/docker/jenkins/build_images.sh new file mode 100755 index 0000000000..d355ce637e --- /dev/null +++ b/docker/jenkins/build_images.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +set -exo pipefail + +if [[ -z "$GIT_COMMIT" ]]; then + GIT_COMMIT=$(git rev-parse HEAD) +fi + +BUILD_IMAGE_TAG="mozorg/bedrock_build:${GIT_COMMIT}" +CODE_IMAGE_TAG="mozorg/bedrock_code:${GIT_COMMIT}" +DOCKER_REBUILD=false +# demo mode will build the demo image containing a db file full of data +DEMO_MODE=false +# prod mode will build the l10n image containing the locale dir +PROD_MODE=false +# test mode will build the unit testing image containing the testing requirements +TEST_MODE=false + +# parse cli args +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -r|--rebuild) + DOCKER_REBUILD=true + ;; + -d|--demo) + DEMO_MODE=true + ;; + -p|--prod) + PROD_MODE=true + ;; + -t|--test) + TEST_MODE=true + ;; + esac + shift # past argument or value +done + +function imageExists() { + if $DOCKER_REBUILD; then + return 1 + fi + docker history -q "mozorg/bedrock_${1}:${GIT_COMMIT}" > /dev/null 2>&1 + return $? +} + +function dockerRun() { + env_file="$1" + image_tag="mozorg/bedrock_${2}:${GIT_COMMIT}" + cmd="$3" + docker run --user $(id -u) -v "$PWD:/app" --env-file "docker/${env_file}.env" "$image_tag" bash -c "$cmd" +} + +if ! imageExists "base"; then + docker/jenkins/docker_build.sh --pull "base" +fi + +# build the static files using the builder image +# and include those and the app in a code image +if ! imageExists "code"; then + # build a staticfiles builder image + if ! imageExists "build"; then + docker/jenkins/docker_build.sh "build" + fi + dockerRun prod build docker/jenkins/build_staticfiles.sh + echo "${GIT_COMMIT}" > static/revision.txt + docker/jenkins/docker_build.sh "code" +fi + +# build a tester image for non-demo deploys +if $TEST_MODE && ! imageExists "test"; then + docker/jenkins/docker_build.sh "test" +fi + +# include the data that the deployments need +if $DEMO_MODE && ! imageExists "demo"; then + dockerRun demo code bin/sync_all + docker/jenkins/docker_build.sh "demo" +fi +if $PROD_MODE && ! imageExists "l10n"; then + dockerRun prod code "python manage.py l10n_update" + docker/jenkins/docker_build.sh -c "locale" "l10n" +fi diff --git a/docker/jenkins/build_integration_test_image.sh b/docker/jenkins/build_integration_test_image.sh deleted file mode 100755 index b4ae041a8f..0000000000 --- a/docker/jenkins/build_integration_test_image.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -xe -docker build -t bedrock_integration_tests:${GIT_COMMIT} -f docker/dockerfiles/bedrock_integration_tests --pull=true . diff --git a/docker/jenkins/build_staticfiles.sh b/docker/jenkins/build_staticfiles.sh new file mode 100755 index 0000000000..6422c31997 --- /dev/null +++ b/docker/jenkins/build_staticfiles.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -euxo pipefail + +python manage.py collectstatic --link --noinput -v 0 +python docker/bin/softlinkstatic.py diff --git a/docker/jenkins/cleanup_after_functional_tests.sh b/docker/jenkins/cleanup_after_functional_tests.sh index 9ce4784ae1..869da09722 100755 --- a/docker/jenkins/cleanup_after_functional_tests.sh +++ b/docker/jenkins/cleanup_after_functional_tests.sh @@ -1,15 +1,18 @@ -#!/bin/bash +#!/bin/bash -x -if [ "${DRIVER}" = "Remote" ]; then - docker stop bedrock-code-${BUILD_NUMBER} - docker rm bedrock-code-${BUILD_NUMBER} +BUILD_NUMBER="${BUILD_NUMBER:-0}" - for NODE_NUMBER in `seq ${NUMBER_OF_NODES:-5}`; - do - docker stop bedrock-selenium-node-${NODE_NUMBER}-${BUILD_NUMBER} - docker rm bedrock-selenium-node-${NODE_NUMBER}-${BUILD_NUMBER} - done; +docker stop bedrock-code-${BUILD_NUMBER} +docker rm bedrock-code-${BUILD_NUMBER} - docker stop bedrock-selenium-hub-${BUILD_NUMBER} - docker rm bedrock-selenium-hub-${BUILD_NUMBER} -fi +for NODE_NUMBER in `seq ${NUMBER_OF_NODES:-5}`; +do + docker stop bedrock-selenium-node-${NODE_NUMBER}-${BUILD_NUMBER} + docker rm bedrock-selenium-node-${NODE_NUMBER}-${BUILD_NUMBER} +done; + +docker stop bedrock-selenium-hub-${BUILD_NUMBER} +docker rm bedrock-selenium-hub-${BUILD_NUMBER} + +# always report success +exit 0 diff --git a/docker/jenkins/demo_deploy.sh b/docker/jenkins/demo_deploy.sh index c9a68c2eea..fdee80c3eb 100755 --- a/docker/jenkins/demo_deploy.sh +++ b/docker/jenkins/demo_deploy.sh @@ -6,7 +6,7 @@ DOCKER_IMAGE_TAG="${DEIS_APP_NAME}:${GIT_COMMIT}" # used for pushing to registry PRIVATE_IMAGE_TAG="${PRIVATE_REGISTRY}/${DOCKER_IMAGE_TAG}" -docker tag "bedrock_dev_final:${GIT_COMMIT}" "$PRIVATE_IMAGE_TAG" +docker tag "mozorg/bedrock_demo:${GIT_COMMIT}" "$PRIVATE_IMAGE_TAG" docker push "$PRIVATE_IMAGE_TAG" echo "Creating the demo app $DEIS_APP_NAME" diff --git a/docker/jenkins/docker_build.sh b/docker/jenkins/docker_build.sh new file mode 100755 index 0000000000..e6f8ddabbf --- /dev/null +++ b/docker/jenkins/docker_build.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -exo pipefail + +if [[ -z "$GIT_COMMIT" ]]; then + export GIT_COMMIT=$(git rev-parse HEAD) +fi + +DOCKER_REPO="${DOCKER_REPO:-mozorg}" +DOCKER_NO_CACHE=false +DOCKER_PULL=false +DOCKER_CTX='.' +DOCKERFILE='base' + +# parse cli args +while [[ $# -gt 1 ]]; do + key="$1" + case $key in + -c|--context) + DOCKER_CTX="$2" + shift + ;; + -n|--no-cache) + DOCKER_NO_CACHE=true + ;; + -p|--pull) + DOCKER_PULL=true + ;; + esac + shift # past argument or value +done + +DOCKERFILE="$1" +FINAL_DOCKERFILE="${DOCKER_CTX}/Dockerfile-$DOCKERFILE" +DOCKER_IMAGE_TAG="${DOCKER_REPO}/bedrock_${DOCKERFILE}:${GIT_COMMIT}" + +# generate the dockerfile +rm -f "$FINAL_DOCKERFILE" +cat "docker/dockerfiles/bedrock_$DOCKERFILE" | envsubst '$GIT_COMMIT' > "$FINAL_DOCKERFILE" + +# build the docker image +docker build -t "$DOCKER_IMAGE_TAG" --pull="$DOCKER_PULL" --no-cache="$DOCKER_NO_CACHE" -f "$FINAL_DOCKERFILE" "$DOCKER_CTX" diff --git a/docker/jenkins/include_l10n.sh b/docker/jenkins/include_l10n.sh deleted file mode 100755 index 2b8d9e7f67..0000000000 --- a/docker/jenkins/include_l10n.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Needs DOCKER_REPOSITORY and FROM_DOCKER_REPOSITORY -# -# To set them go to Job -> Configure -> Build Environment -> Inject -# passwords and Inject env variables -# -set -xe - -if [[ $BUILD_CAUSE == "REMOTECAUSE" ]] -then - LATEST_TAG=$(git describe --abbrev=0 --tags) - # parent (~0) of latest tag is the commit that was tagged - GIT_COMMIT=$(git rev-parse ${LATEST_TAG}~0) -fi - -DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT} - -if [[ ! -d locale/.git ]]; -then - rm -rf locale - git clone --depth 1 https://github.com/mozilla-l10n/bedrock-l10n locale -fi; - -pushd locale -git fetch origin -git checkout -f origin/master -popd - -cat docker/dockerfiles/bedrock_l10n | envsubst > ./locale/Dockerfile - -docker build -f locale/Dockerfile -t $DOCKER_IMAGE_TAG locale diff --git a/docker/jenkins/periodic_db_update.sh b/docker/jenkins/periodic_db_update.sh old mode 100644 new mode 100755 diff --git a/docker/jenkins/properties/integration_tests/local.properties b/docker/jenkins/properties/integration_tests/local.properties new file mode 100644 index 0000000000..d44fa86842 --- /dev/null +++ b/docker/jenkins/properties/integration_tests/local.properties @@ -0,0 +1,2 @@ +export DRIVER=Remote +export MARK_EXPRESSION=smoke diff --git a/docker/jenkins/run_integration_tests.sh b/docker/jenkins/run_integration_tests.sh index ff37d4e8d7..77becf155b 100755 --- a/docker/jenkins/run_integration_tests.sh +++ b/docker/jenkins/run_integration_tests.sh @@ -1,7 +1,13 @@ #!/bin/bash -xe # $1 should be the properties file for this run -source "$1" +source "docker/jenkins/properties/integration_tests/${1}.properties" + +if [[ -z "$GIT_COMMIT" ]]; then + GIT_COMMIT=$(git rev-parse HEAD) +fi + +BUILD_NUMBER="${BUILD_NUMBER:-0}" if [ -z "${BASE_URL}" ]; then # start bedrock @@ -51,9 +57,10 @@ fi # make sure results dir exists or docker will create it # and it will be owned by root RESULTS_DIR="$PWD/results" +DOCKER_RESULTS_DIR="/app/results" rm -rf "$RESULTS_DIR" mkdir -p "$RESULTS_DIR" -docker run -v "${RESULTS_DIR}:/app/results" -u $(stat -c "%u:%g" "$RESULTS_DIR") \ +docker run -v "${RESULTS_DIR}:${DOCKER_RESULTS_DIR}" -u $(stat -c "%u:%g" "$RESULTS_DIR") \ ${DOCKER_LINKS[@]} \ -e BASE_URL=${BASE_URL} \ -e DRIVER=${DRIVER} \ @@ -69,4 +76,7 @@ docker run -v "${RESULTS_DIR}:/app/results" -u $(stat -c "%u:%g" "$RESULTS_DIR") -e SCREEN_RESOLUTION=${SCREEN_RESOLUTION} \ -e MARK_EXPRESSION="${MARK_EXPRESSION}" \ -e TESTS_PATH="${TESTS_PATH}" \ - bedrock_integration_tests:${GIT_COMMIT} + -e RESULTS_PATH="${DOCKER_RESULTS_DIR}" \ + -e PYTEST_PROCESSES=5 \ + -e PRIVACY="public restricted" \ + mozorg/bedrock_test:${GIT_COMMIT} bin/run-integration-tests.sh diff --git a/docker/jenkins/run_tests.sh b/docker/jenkins/run_tests.sh index 3b4e9b3384..99ca704772 100755 --- a/docker/jenkins/run_tests.sh +++ b/docker/jenkins/run_tests.sh @@ -2,14 +2,11 @@ # # Runs unit_tests # -set -ex +set -exo pipefail -ENV_FILE=`mktemp` -cat << EOF > $ENV_FILE -DEBUG=True -ALLOWED_HOSTS=localhost,127.0.0.1, -SECRET_KEY=39114b6a-2858-4caf-8878-482a24ee9542 -ADMINS=["thedude@example.com"] -EOF +if [[ -z "$GIT_COMMIT" ]]; then + GIT_COMMIT=$(git rev-parse HEAD) +fi -docker run --env-file $ENV_FILE ${DOCKER_REPOSITORY}:${GIT_COMMIT:-$(git rev-parse HEAD)} py.test lib bedrock +TEST_IMAGE_TAG="mozorg/bedrock_test:${GIT_COMMIT}" +docker run --env-file docker/test.env "$TEST_IMAGE_TAG" diff --git a/docker/jenkins/utils.groovy b/docker/jenkins/utils.groovy index 4a9db146b8..fd0eceb07a 100644 --- a/docker/jenkins/utils.groovy +++ b/docker/jenkins/utils.groovy @@ -28,21 +28,6 @@ def ircNotification(config, Map args) { sh command } -def buildDockerImage(Map kwargs) { - def update = kwargs.update ? 'true' : 'false' - def repo = kwargs.dockerRepo ?: 'mozorg' - def script = kwargs.script ?: 'build_image.sh' - def environs = ["UPDATE_DOCKER_IMAGES=${update}", - "DOCKERFILE=${kwargs.dockerfile}", - "DOCKER_REPOSITORY=${repo}/${kwargs.dockerfile}"] - if (kwargs.fromDockerfile) { - environs << "FROM_DOCKER_REPOSITORY=${repo}/${kwargs.fromDockerfile}" - } - withEnv(environs) { - sh "docker/jenkins/${script}" - } -} - def pushDockerhub(from_repo, to_repo='') { to_repo = to_repo ?: from_repo withCredentials([[$class: 'StringBinding', @@ -68,26 +53,25 @@ def pushPrivateReg(port, apps) { } } -def integrationTestJob(propFileName, appURL) { - def testsBaseDir = 'docker/jenkins/properties/integration_tests' - def testsFileExt = '.properties' +def integrationTestJob(propFileName, appURL='') { return { node { unstash 'scripts' unstash 'tests' - def fullFilename = "${testsBaseDir}/${propFileName}${testsFileExt}" - def testScript = "docker/jenkins/run_integration_tests.sh ${fullFilename}".toString() + def testScript = "docker/jenkins/run_integration_tests.sh ${propFileName}".toString() withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'SAUCELABS_CREDENTIALS', usernameVariable: 'SAUCELABS_USERNAME', passwordVariable: 'SAUCELABS_API_KEY']]) { - withEnv(["BASE_URL=${appURL}", - "SELENIUM_VERSION=2.52.0"]) { + withEnv(["BASE_URL=${appURL}"]) { try { sh testScript } finally { junit 'results/*.xml' + if ( propFileName == 'local' ) { + sh 'docker/jenkins/cleanup_after_functional_tests.sh' + } } } } diff --git a/docker/prod.env b/docker/prod.env new file mode 100644 index 0000000000..06897e976d --- /dev/null +++ b/docker/prod.env @@ -0,0 +1,5 @@ +DATABASE_URL=sqlite:///bedrock.db +DEBUG=False +DEV=False +ALLOWED_HOSTS=* +SECRET_KEY=59114b6a-2858-4caf-8878-482a24ee9542 diff --git a/docker/test.env b/docker/test.env index 7886884ae4..1b114d292f 100644 --- a/docker/test.env +++ b/docker/test.env @@ -1,4 +1,5 @@ -DATABASE_URL=sqlite:///bedrock.db DEBUG=False -ALLOWED_HOSTS=localhost,127.0.0.1, -SECRET_KEY=59114b6a-2858-4caf-8878-482a24ee9542 +DEV=False +ALLOWED_HOSTS=* +SECRET_KEY=39114b6a-2858-4caf-8878-482a24ee9542 +ADMINS=["thedude@example.com"] diff --git a/jenkins.yml b/jenkins.yml index b61efb54c3..6f41ec8b12 100644 --- a/jenkins.yml +++ b/jenkins.yml @@ -1,3 +1,6 @@ +environment: + SELENIUM_VERSION: 2.52.0 + branches: master: regions: @@ -18,6 +21,8 @@ branches: - bedrock-stage - bedrock-prod integration_tests: + - headless + - firefox - chrome - ie - ie6