зеркало из https://github.com/mozilla/bedrock.git
Change how we do docker
* More simple and streamlined base image build file * Use a builder image to decrease the size and security footprint of running containers * Make demos more prod-like * Add l10n for prod builds via docker commit * Specifically include only those files required for the running site * Tests in circleci now use the same docker build and test scripts * Add lock for docker image building per git hash * Add environment variable support to jenkins.yml * Add headless and firefox integration tests to prod deploys * Run integration smoke tests before deployment
This commit is contained in:
Paul McLanahan
Родитель
fe1e4c5450
Коммит
a2cc3e9b2d
|
|
@ -1,3 +1,2 @@
|
|||
locale/.git
|
||||
.git
|
||||
.env
|
||||
|
|
|
|||
|
|
@ -28,6 +28,9 @@ indent_size = 4
|
|||
[*.scss]
|
||||
indent_size = 4
|
||||
|
||||
[*.sh]
|
||||
indent_size = 4
|
||||
|
||||
[*.html]
|
||||
indent_size = 2
|
||||
|
||||
|
|
@ -37,6 +40,9 @@ indent_size = 2
|
|||
[Jenkinsfile]
|
||||
indent_size = 4
|
||||
|
||||
[*.groovy]
|
||||
indent_size = 4
|
||||
|
||||
[package.json]
|
||||
indent_size = 2
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ tmp/*
|
|||
*.less.css
|
||||
db.sql
|
||||
/locale
|
||||
/results
|
||||
Thumbs.db
|
||||
Desktop.ini
|
||||
venv
|
||||
|
|
@ -41,3 +42,4 @@ tests/functional/results.html
|
|||
tests/functional/assets/style.css
|
||||
.eslintcache
|
||||
.docker-build*
|
||||
Dockerfile-*
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!groovy
|
||||
|
||||
@Library('github.com/mozmar/jenkins-pipeline@20170126.1')
|
||||
@Library('github.com/mozmar/jenkins-pipeline@20170214.1')
|
||||
|
||||
def config
|
||||
def utils
|
||||
|
|
@ -10,12 +10,15 @@ stage ('Checkout') {
|
|||
checkout scm
|
||||
sh 'git submodule sync'
|
||||
sh 'git submodule update --init --recursive'
|
||||
// defined in the Library loaded above
|
||||
setGitEnvironmentVariables()
|
||||
// clean up
|
||||
sh 'make clean'
|
||||
// load the config
|
||||
config = readYaml file: 'jenkins.yml'
|
||||
// load the utility functions used below
|
||||
utils = load 'docker/jenkins/utils.groovy'
|
||||
// defined in the Library loaded above
|
||||
setGitEnvironmentVariables()
|
||||
setConfigEnvironmentVariables(config)
|
||||
// save the files for later
|
||||
stash name: 'scripts', includes: 'bin/,docker/'
|
||||
stash name: 'tests', includes: 'tests/,requirements/'
|
||||
|
|
@ -39,62 +42,47 @@ if ( config.branches.containsKey(env.BRANCH_NAME) ) {
|
|||
}
|
||||
}
|
||||
utils.ircNotification(config, [stage: 'Test & Deploy', status: 'starting'])
|
||||
try {
|
||||
utils.buildDockerImage(dockerfile: 'bedrock_base', update: true)
|
||||
utils.buildDockerImage(dockerfile: 'bedrock_code', fromDockerfile: 'bedrock_base')
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Docker Build', status: 'failure'])
|
||||
throw err
|
||||
lock ("bedrock-docker-${env.GIT_COMMIT}") {
|
||||
try {
|
||||
sh 'docker/jenkins/build_images.sh --prod --test'
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Docker Build', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
milestone()
|
||||
stage ('Test Images') {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
try {
|
||||
withEnv(['DOCKER_REPOSITORY=mozorg/bedrock_code']) {
|
||||
sh 'docker/jenkins/run_tests.sh'
|
||||
parallel([
|
||||
integration_tests: utils.integrationTestJob('local'),
|
||||
unit_tests: {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
try {
|
||||
sh 'docker/jenkins/run_tests.sh'
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Unit Test', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
}
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Unit Test', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
utils.ircNotification(config, [stage: 'Docker Builds & Tests', status: 'complete'])
|
||||
}
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
milestone()
|
||||
stage ('Push Public Images') {
|
||||
parallel([
|
||||
dockerhub: {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
try {
|
||||
utils.buildDockerImage(dockerfile: 'bedrock_l10n', fromDockerfile: 'bedrock_code', script: 'include_l10n.sh')
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'L10n Build', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
try {
|
||||
utils.pushDockerhub('mozorg/bedrock_base')
|
||||
utils.pushDockerhub('mozorg/bedrock_code')
|
||||
utils.pushDockerhub('mozorg/bedrock_l10n', 'mozorg/bedrock')
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Dockerhub Push Failed', status: 'warning'])
|
||||
}
|
||||
}
|
||||
},
|
||||
integration_tests: {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
unstash 'tests'
|
||||
// prep for next stage
|
||||
sh 'docker/jenkins/build_integration_test_image.sh'
|
||||
}
|
||||
},
|
||||
])
|
||||
node {
|
||||
unstash 'scripts'
|
||||
try {
|
||||
utils.pushDockerhub('mozorg/bedrock_base')
|
||||
utils.pushDockerhub('mozorg/bedrock_code')
|
||||
utils.pushDockerhub('mozorg/bedrock_l10n', 'mozorg/bedrock')
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Dockerhub Push Failed', status: 'warning'])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -179,16 +167,14 @@ else if ( env.BRANCH_NAME ==~ /^demo__[\w-]+$/ ) {
|
|||
node {
|
||||
utils.ircNotification(config, [stage: 'Demo Deploy', status: 'starting'])
|
||||
stage ('build') {
|
||||
milestone()
|
||||
try {
|
||||
sh 'make clean'
|
||||
sh 'make sync-all'
|
||||
sh 'echo "ENV GIT_SHA ${GIT_COMMIT}" >> docker/dockerfiles/bedrock_dev_final'
|
||||
sh 'echo "RUN echo ${GIT_COMMIT} > static/revision.txt" >> docker/dockerfiles/bedrock_dev_final'
|
||||
sh 'make build-final'
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Demo Build', status: 'failure'])
|
||||
throw err
|
||||
lock ("bedrock-docker-${env.GIT_COMMIT}") {
|
||||
milestone()
|
||||
try {
|
||||
sh 'docker/jenkins/build_images.sh --demo'
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Demo Build', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
4
Makefile
4
Makefile
|
|
@ -56,6 +56,10 @@ clean:
|
|||
|
||||
# test related things
|
||||
-rm -f .coverage
|
||||
-rm -rf results
|
||||
|
||||
# static files
|
||||
git clean -fdx static
|
||||
|
||||
# docs files
|
||||
-rm -rf docs/_build/
|
||||
|
|
|
|||
17
circle.yml
17
circle.yml
|
|
@ -2,18 +2,7 @@ machine:
|
|||
services:
|
||||
- docker
|
||||
node:
|
||||
version: 4.2.6
|
||||
environment:
|
||||
# Deis
|
||||
DEIS_CONTROLLER: https://deis.us-west.moz.works
|
||||
DEIS_USERNAME: circleci
|
||||
# Docker
|
||||
QUAY_EMAIL: mozmar@example.com
|
||||
QUAY_USERNAME: mozmar+circleci
|
||||
DOCKER_REPOSITORY: quay.io/mozmar/bedrock
|
||||
DOCKER_IMAGE_TAG: "$DOCKER_REPOSITORY:$CIRCLE_BRANCH"
|
||||
BASE_DOCKER_REPOSITORY: quay.io/mozmar/bedrock_base
|
||||
BASE_DOCKER_IMAGE_TAG: "$BASE_DOCKER_REPOSITORY:$CIRCLE_BRANCH"
|
||||
version: 6.1.0
|
||||
|
||||
checkout:
|
||||
post:
|
||||
|
|
@ -25,9 +14,9 @@ dependencies:
|
|||
- node_modules
|
||||
- "~/docker"
|
||||
override:
|
||||
- docker/jenkins/build_images.sh --test
|
||||
- npm install -g gulp-cli
|
||||
- npm install
|
||||
- bin/circleci-docker-build.sh
|
||||
|
||||
test:
|
||||
pre:
|
||||
|
|
@ -36,4 +25,4 @@ test:
|
|||
- gulp css:lint
|
||||
- gulp js:lint
|
||||
- gulp js:test
|
||||
- make test-image
|
||||
- docker/jenkins/run_tests.sh
|
||||
|
|
|
|||
|
|
@ -1,25 +1,24 @@
|
|||
FROM quay.io/mozmar/base
|
||||
FROM quay.io/mozmar/base:latest
|
||||
|
||||
# Set Python-related environment variables to reduce annoying-ness
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
ENV LANG=C.UTF-8
|
||||
RUN update-alternatives --install /bin/sh sh /bin/bash 10
|
||||
RUN adduser --uid 1000 --disabled-password --gecos '' --no-create-home webdev
|
||||
|
||||
WORKDIR /app
|
||||
EXPOSE 8000
|
||||
CMD ["./docker/run.sh"]
|
||||
|
||||
RUN adduser --uid 1000 --disabled-password --gecos '' --no-create-home webdev
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
gettext build-essential python python-dev python-pip python-setuptools \
|
||||
libxml2-dev libxslt1.1 libxslt1-dev \
|
||||
nodejs-legacy npm git libpq-dev
|
||||
gettext build-essential python-{dev,pip,setuptools} \
|
||||
libxml2-dev libxslt1.1 libxslt1-dev git libpq-dev
|
||||
|
||||
COPY ./requirements /app/requirements
|
||||
|
||||
# Install app
|
||||
RUN pip install --no-cache-dir -r requirements/dev.txt
|
||||
# Install Python deps
|
||||
RUN pip install --no-cache-dir -r requirements/prod.txt
|
||||
RUN pip install --no-cache-dir -r requirements/docker.txt
|
||||
RUN pip install --no-cache-dir -r requirements/test.txt
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
FROM mozorg/bedrock_base:${GIT_COMMIT}
|
||||
|
||||
ENV PATH=/node_modules/.bin:$PATH
|
||||
ENV PIPELINE_LESS_BINARY=lessc
|
||||
ENV PIPELINE_SASS_BINARY=node-sass
|
||||
ENV PIPELINE_YUGLIFY_BINARY=yuglify
|
||||
|
||||
RUN apt-get install -y --no-install-recommends nodejs-legacy npm
|
||||
|
||||
COPY ./node_modules /
|
||||
COPY ./package.json /
|
||||
COPY ./lockdown.json /
|
||||
# --unsafe-perm required for lockdown to function
|
||||
RUN cd / && npm install --production --unsafe-perm
|
||||
|
|
@ -1,20 +1,25 @@
|
|||
FROM ${FROM_DOCKER_REPOSITORY}:${GIT_COMMIT}
|
||||
FROM mozorg/bedrock_base:${GIT_COMMIT}
|
||||
|
||||
COPY . /app
|
||||
# changes infrequently
|
||||
COPY ./bin ./bin
|
||||
COPY ./etc ./etc
|
||||
COPY ./lib ./lib
|
||||
COPY ./root_files ./root_files
|
||||
COPY ./scripts ./scripts
|
||||
COPY ./wsgi ./wsgi
|
||||
COPY ./manage.py ./
|
||||
COPY ./LICENSE ./
|
||||
COPY ./newrelic.ini ./
|
||||
COPY ./contribute.json ./
|
||||
|
||||
RUN echo "${GIT_COMMIT}" > static/revision.txt
|
||||
|
||||
# --unsafe-perm required for lockdown to function
|
||||
RUN npm install --production --unsafe-perm
|
||||
RUN ./manage.py collectstatic -l --noinput
|
||||
|
||||
# Cleanup
|
||||
RUN rm -rf node_modules
|
||||
RUN ./docker/bin/softlinkstatic.py
|
||||
RUN apt-get purge -y nodejs npm python-dev build-essential libxml2-dev libxslt1-dev
|
||||
RUN apt-get autoremove -y
|
||||
RUN rm -rf /var/lib/{apt,dpkg,cache,log} /usr/share/doc /usr/share/man /tmp/* /var/cache/*
|
||||
# changes more frequently
|
||||
COPY ./docker ./docker
|
||||
COPY ./vendor-local ./vendor-local
|
||||
COPY ./bedrock ./bedrock
|
||||
COPY ./media ./media
|
||||
COPY ./static ./static
|
||||
|
||||
ENV GIT_SHA ${GIT_COMMIT}
|
||||
# Change User
|
||||
RUN chown webdev.webdev -R .
|
||||
USER webdev
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
FROM mozorg/bedrock_code:${GIT_COMMIT}
|
||||
|
||||
COPY ./locale ./locale
|
||||
COPY ./bedrock.db ./
|
||||
|
||||
# Change User
|
||||
USER root
|
||||
RUN chown webdev.webdev -R .
|
||||
USER webdev
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
FROM quay.io/mozmar/base
|
||||
|
||||
WORKDIR /app
|
||||
# Run the tests
|
||||
CMD ["/app/bin/run-integration-tests.sh"]
|
||||
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential python python-dev python-pip python-setuptools \
|
||||
libxml2-dev libxslt1.1 libxslt1-dev zlib1g-dev
|
||||
|
||||
# Defaults
|
||||
ENV PYTEST_PROCESSES 5
|
||||
ENV PRIVACY "public restricted"
|
||||
ENV TESTS_PATH /app/tests
|
||||
ENV RESULTS_PATH /app/results
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
COPY ./requirements /app/requirements
|
||||
|
||||
# Install requirements
|
||||
RUN pip install --no-cache-dir -r requirements/test.txt
|
||||
|
||||
COPY . /app
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
FROM ${FROM_DOCKER_REPOSITORY}:${GIT_COMMIT}
|
||||
COPY . /app/locale
|
||||
FROM mozorg/bedrock_code:${GIT_COMMIT}
|
||||
|
||||
COPY . ./locale/
|
||||
|
||||
# Change User
|
||||
USER root
|
||||
RUN chown -R webdev.webdev /app/locale
|
||||
RUN chown webdev.webdev -R locale
|
||||
USER webdev
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM quay.io/mozmar/base
|
||||
FROM quay.io/mozmar/base:latest
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential python python-dev python-pip python-setuptools
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
FROM mozorg/bedrock_code:${GIT_COMMIT}
|
||||
|
||||
CMD ["./docker/run-tests.sh"]
|
||||
USER root
|
||||
|
||||
RUN pip install --no-cache-dir -r requirements/test.txt
|
||||
COPY ./setup.cfg ./
|
||||
COPY ./tests ./tests
|
||||
|
||||
RUN chown webdev.webdev -R .
|
||||
USER webdev
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
set -ex
|
||||
DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT}
|
||||
TMP_DOCKER_TAG=${BUILD_TAG}
|
||||
|
||||
# If docker image exists and no force rebuild do nothing
|
||||
FORCE_REBUILD=`echo "$FORCE_REBUILD" | tr '[:upper:]' '[:lower:]'`
|
||||
if [[ $FORCE_REBUILD != "true" ]];
|
||||
then
|
||||
if docker history -q $DOCKER_IMAGE_TAG > /dev/null;
|
||||
then
|
||||
echo "Docker image already exists, do nothing"
|
||||
exit 0;
|
||||
fi
|
||||
fi
|
||||
|
||||
cat docker/dockerfiles/${DOCKERFILE} | envsubst > Dockerfile
|
||||
|
||||
if [[ $FORCE_REBUILD == "true" ]];
|
||||
then
|
||||
NO_CACHE="true"
|
||||
fi;
|
||||
|
||||
docker build -t ${TMP_DOCKER_TAG} --pull=${UPDATE_DOCKER_IMAGES:-true} --no-cache=${NO_CACHE:-false} . | tee docker-build.log
|
||||
|
||||
TAG=`tail -n 1 docker-build.log | awk '{ print $(NF) }'`
|
||||
|
||||
if [[ $FORCE_REBUILD != "true" ]];
|
||||
then
|
||||
if [[ $(tail -n 3 docker-build.log | grep "Using cache") && $(docker images | grep "${TAG}-squashed") ]];
|
||||
then
|
||||
echo "Docker image already squashed, skip squashing";
|
||||
docker tag ${TAG}-squashed $DOCKER_IMAGE_TAG
|
||||
exit 0;
|
||||
fi
|
||||
fi
|
||||
|
||||
docker-squash -t ${TAG}-squashed ${TMP_DOCKER_TAG}
|
||||
docker tag ${TAG}-squashed ${DOCKER_IMAGE_TAG}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -exo pipefail
|
||||
|
||||
if [[ -z "$GIT_COMMIT" ]]; then
|
||||
GIT_COMMIT=$(git rev-parse HEAD)
|
||||
fi
|
||||
|
||||
BUILD_IMAGE_TAG="mozorg/bedrock_build:${GIT_COMMIT}"
|
||||
CODE_IMAGE_TAG="mozorg/bedrock_code:${GIT_COMMIT}"
|
||||
DOCKER_REBUILD=false
|
||||
# demo mode will build the demo image containing a db file full of data
|
||||
DEMO_MODE=false
|
||||
# prod mode will build the l10n image containing the locale dir
|
||||
PROD_MODE=false
|
||||
# test mode will build the unit testing image containing the testing requirements
|
||||
TEST_MODE=false
|
||||
|
||||
# parse cli args
|
||||
while [[ $# -gt 0 ]]; do
|
||||
key="$1"
|
||||
case $key in
|
||||
-r|--rebuild)
|
||||
DOCKER_REBUILD=true
|
||||
;;
|
||||
-d|--demo)
|
||||
DEMO_MODE=true
|
||||
;;
|
||||
-p|--prod)
|
||||
PROD_MODE=true
|
||||
;;
|
||||
-t|--test)
|
||||
TEST_MODE=true
|
||||
;;
|
||||
esac
|
||||
shift # past argument or value
|
||||
done
|
||||
|
||||
function imageExists() {
|
||||
if $DOCKER_REBUILD; then
|
||||
return 1
|
||||
fi
|
||||
docker history -q "mozorg/bedrock_${1}:${GIT_COMMIT}" > /dev/null 2>&1
|
||||
return $?
|
||||
}
|
||||
|
||||
function dockerRun() {
|
||||
env_file="$1"
|
||||
image_tag="mozorg/bedrock_${2}:${GIT_COMMIT}"
|
||||
cmd="$3"
|
||||
docker run --user $(id -u) -v "$PWD:/app" --env-file "docker/${env_file}.env" "$image_tag" bash -c "$cmd"
|
||||
}
|
||||
|
||||
if ! imageExists "base"; then
|
||||
docker/jenkins/docker_build.sh --pull "base"
|
||||
fi
|
||||
|
||||
# build the static files using the builder image
|
||||
# and include those and the app in a code image
|
||||
if ! imageExists "code"; then
|
||||
# build a staticfiles builder image
|
||||
if ! imageExists "build"; then
|
||||
docker/jenkins/docker_build.sh "build"
|
||||
fi
|
||||
dockerRun prod build docker/jenkins/build_staticfiles.sh
|
||||
echo "${GIT_COMMIT}" > static/revision.txt
|
||||
docker/jenkins/docker_build.sh "code"
|
||||
fi
|
||||
|
||||
# build a tester image for non-demo deploys
|
||||
if $TEST_MODE && ! imageExists "test"; then
|
||||
docker/jenkins/docker_build.sh "test"
|
||||
fi
|
||||
|
||||
# include the data that the deployments need
|
||||
if $DEMO_MODE && ! imageExists "demo"; then
|
||||
dockerRun demo code bin/sync_all
|
||||
docker/jenkins/docker_build.sh "demo"
|
||||
fi
|
||||
if $PROD_MODE && ! imageExists "l10n"; then
|
||||
dockerRun prod code "python manage.py l10n_update"
|
||||
docker/jenkins/docker_build.sh -c "locale" "l10n"
|
||||
fi
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
#!/bin/bash -xe
|
||||
docker build -t bedrock_integration_tests:${GIT_COMMIT} -f docker/dockerfiles/bedrock_integration_tests --pull=true .
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
python manage.py collectstatic --link --noinput -v 0
|
||||
python docker/bin/softlinkstatic.py
|
||||
|
|
@ -1,15 +1,18 @@
|
|||
#!/bin/bash
|
||||
#!/bin/bash -x
|
||||
|
||||
if [ "${DRIVER}" = "Remote" ]; then
|
||||
docker stop bedrock-code-${BUILD_NUMBER}
|
||||
docker rm bedrock-code-${BUILD_NUMBER}
|
||||
BUILD_NUMBER="${BUILD_NUMBER:-0}"
|
||||
|
||||
for NODE_NUMBER in `seq ${NUMBER_OF_NODES:-5}`;
|
||||
do
|
||||
docker stop bedrock-selenium-node-${NODE_NUMBER}-${BUILD_NUMBER}
|
||||
docker rm bedrock-selenium-node-${NODE_NUMBER}-${BUILD_NUMBER}
|
||||
done;
|
||||
docker stop bedrock-code-${BUILD_NUMBER}
|
||||
docker rm bedrock-code-${BUILD_NUMBER}
|
||||
|
||||
docker stop bedrock-selenium-hub-${BUILD_NUMBER}
|
||||
docker rm bedrock-selenium-hub-${BUILD_NUMBER}
|
||||
fi
|
||||
for NODE_NUMBER in `seq ${NUMBER_OF_NODES:-5}`;
|
||||
do
|
||||
docker stop bedrock-selenium-node-${NODE_NUMBER}-${BUILD_NUMBER}
|
||||
docker rm bedrock-selenium-node-${NODE_NUMBER}-${BUILD_NUMBER}
|
||||
done;
|
||||
|
||||
docker stop bedrock-selenium-hub-${BUILD_NUMBER}
|
||||
docker rm bedrock-selenium-hub-${BUILD_NUMBER}
|
||||
|
||||
# always report success
|
||||
exit 0
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ DOCKER_IMAGE_TAG="${DEIS_APP_NAME}:${GIT_COMMIT}"
|
|||
# used for pushing to registry
|
||||
PRIVATE_IMAGE_TAG="${PRIVATE_REGISTRY}/${DOCKER_IMAGE_TAG}"
|
||||
|
||||
docker tag "bedrock_dev_final:${GIT_COMMIT}" "$PRIVATE_IMAGE_TAG"
|
||||
docker tag "mozorg/bedrock_demo:${GIT_COMMIT}" "$PRIVATE_IMAGE_TAG"
|
||||
docker push "$PRIVATE_IMAGE_TAG"
|
||||
|
||||
echo "Creating the demo app $DEIS_APP_NAME"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -exo pipefail
|
||||
|
||||
if [[ -z "$GIT_COMMIT" ]]; then
|
||||
export GIT_COMMIT=$(git rev-parse HEAD)
|
||||
fi
|
||||
|
||||
DOCKER_REPO="${DOCKER_REPO:-mozorg}"
|
||||
DOCKER_NO_CACHE=false
|
||||
DOCKER_PULL=false
|
||||
DOCKER_CTX='.'
|
||||
DOCKERFILE='base'
|
||||
|
||||
# parse cli args
|
||||
while [[ $# -gt 1 ]]; do
|
||||
key="$1"
|
||||
case $key in
|
||||
-c|--context)
|
||||
DOCKER_CTX="$2"
|
||||
shift
|
||||
;;
|
||||
-n|--no-cache)
|
||||
DOCKER_NO_CACHE=true
|
||||
;;
|
||||
-p|--pull)
|
||||
DOCKER_PULL=true
|
||||
;;
|
||||
esac
|
||||
shift # past argument or value
|
||||
done
|
||||
|
||||
DOCKERFILE="$1"
|
||||
FINAL_DOCKERFILE="${DOCKER_CTX}/Dockerfile-$DOCKERFILE"
|
||||
DOCKER_IMAGE_TAG="${DOCKER_REPO}/bedrock_${DOCKERFILE}:${GIT_COMMIT}"
|
||||
|
||||
# generate the dockerfile
|
||||
rm -f "$FINAL_DOCKERFILE"
|
||||
cat "docker/dockerfiles/bedrock_$DOCKERFILE" | envsubst '$GIT_COMMIT' > "$FINAL_DOCKERFILE"
|
||||
|
||||
# build the docker image
|
||||
docker build -t "$DOCKER_IMAGE_TAG" --pull="$DOCKER_PULL" --no-cache="$DOCKER_NO_CACHE" -f "$FINAL_DOCKERFILE" "$DOCKER_CTX"
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Needs DOCKER_REPOSITORY and FROM_DOCKER_REPOSITORY
|
||||
#
|
||||
# To set them go to Job -> Configure -> Build Environment -> Inject
|
||||
# passwords and Inject env variables
|
||||
#
|
||||
set -xe
|
||||
|
||||
if [[ $BUILD_CAUSE == "REMOTECAUSE" ]]
|
||||
then
|
||||
LATEST_TAG=$(git describe --abbrev=0 --tags)
|
||||
# parent (~0) of latest tag is the commit that was tagged
|
||||
GIT_COMMIT=$(git rev-parse ${LATEST_TAG}~0)
|
||||
fi
|
||||
|
||||
DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT}
|
||||
|
||||
if [[ ! -d locale/.git ]];
|
||||
then
|
||||
rm -rf locale
|
||||
git clone --depth 1 https://github.com/mozilla-l10n/bedrock-l10n locale
|
||||
fi;
|
||||
|
||||
pushd locale
|
||||
git fetch origin
|
||||
git checkout -f origin/master
|
||||
popd
|
||||
|
||||
cat docker/dockerfiles/bedrock_l10n | envsubst > ./locale/Dockerfile
|
||||
|
||||
docker build -f locale/Dockerfile -t $DOCKER_IMAGE_TAG locale
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
export DRIVER=Remote
|
||||
export MARK_EXPRESSION=smoke
|
||||
|
|
@ -1,7 +1,13 @@
|
|||
#!/bin/bash -xe
|
||||
|
||||
# $1 should be the properties file for this run
|
||||
source "$1"
|
||||
source "docker/jenkins/properties/integration_tests/${1}.properties"
|
||||
|
||||
if [[ -z "$GIT_COMMIT" ]]; then
|
||||
GIT_COMMIT=$(git rev-parse HEAD)
|
||||
fi
|
||||
|
||||
BUILD_NUMBER="${BUILD_NUMBER:-0}"
|
||||
|
||||
if [ -z "${BASE_URL}" ]; then
|
||||
# start bedrock
|
||||
|
|
@ -51,9 +57,10 @@ fi
|
|||
# make sure results dir exists or docker will create it
|
||||
# and it will be owned by root
|
||||
RESULTS_DIR="$PWD/results"
|
||||
DOCKER_RESULTS_DIR="/app/results"
|
||||
rm -rf "$RESULTS_DIR"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
docker run -v "${RESULTS_DIR}:/app/results" -u $(stat -c "%u:%g" "$RESULTS_DIR") \
|
||||
docker run -v "${RESULTS_DIR}:${DOCKER_RESULTS_DIR}" -u $(stat -c "%u:%g" "$RESULTS_DIR") \
|
||||
${DOCKER_LINKS[@]} \
|
||||
-e BASE_URL=${BASE_URL} \
|
||||
-e DRIVER=${DRIVER} \
|
||||
|
|
@ -69,4 +76,7 @@ docker run -v "${RESULTS_DIR}:/app/results" -u $(stat -c "%u:%g" "$RESULTS_DIR")
|
|||
-e SCREEN_RESOLUTION=${SCREEN_RESOLUTION} \
|
||||
-e MARK_EXPRESSION="${MARK_EXPRESSION}" \
|
||||
-e TESTS_PATH="${TESTS_PATH}" \
|
||||
bedrock_integration_tests:${GIT_COMMIT}
|
||||
-e RESULTS_PATH="${DOCKER_RESULTS_DIR}" \
|
||||
-e PYTEST_PROCESSES=5 \
|
||||
-e PRIVACY="public restricted" \
|
||||
mozorg/bedrock_test:${GIT_COMMIT} bin/run-integration-tests.sh
|
||||
|
|
|
|||
|
|
@ -2,14 +2,11 @@
|
|||
#
|
||||
# Runs unit_tests
|
||||
#
|
||||
set -ex
|
||||
set -exo pipefail
|
||||
|
||||
ENV_FILE=`mktemp`
|
||||
cat << EOF > $ENV_FILE
|
||||
DEBUG=True
|
||||
ALLOWED_HOSTS=localhost,127.0.0.1,
|
||||
SECRET_KEY=39114b6a-2858-4caf-8878-482a24ee9542
|
||||
ADMINS=["thedude@example.com"]
|
||||
EOF
|
||||
if [[ -z "$GIT_COMMIT" ]]; then
|
||||
GIT_COMMIT=$(git rev-parse HEAD)
|
||||
fi
|
||||
|
||||
docker run --env-file $ENV_FILE ${DOCKER_REPOSITORY}:${GIT_COMMIT:-$(git rev-parse HEAD)} py.test lib bedrock
|
||||
TEST_IMAGE_TAG="mozorg/bedrock_test:${GIT_COMMIT}"
|
||||
docker run --env-file docker/test.env "$TEST_IMAGE_TAG"
|
||||
|
|
|
|||
|
|
@ -28,21 +28,6 @@ def ircNotification(config, Map args) {
|
|||
sh command
|
||||
}
|
||||
|
||||
def buildDockerImage(Map kwargs) {
|
||||
def update = kwargs.update ? 'true' : 'false'
|
||||
def repo = kwargs.dockerRepo ?: 'mozorg'
|
||||
def script = kwargs.script ?: 'build_image.sh'
|
||||
def environs = ["UPDATE_DOCKER_IMAGES=${update}",
|
||||
"DOCKERFILE=${kwargs.dockerfile}",
|
||||
"DOCKER_REPOSITORY=${repo}/${kwargs.dockerfile}"]
|
||||
if (kwargs.fromDockerfile) {
|
||||
environs << "FROM_DOCKER_REPOSITORY=${repo}/${kwargs.fromDockerfile}"
|
||||
}
|
||||
withEnv(environs) {
|
||||
sh "docker/jenkins/${script}"
|
||||
}
|
||||
}
|
||||
|
||||
def pushDockerhub(from_repo, to_repo='') {
|
||||
to_repo = to_repo ?: from_repo
|
||||
withCredentials([[$class: 'StringBinding',
|
||||
|
|
@ -68,26 +53,25 @@ def pushPrivateReg(port, apps) {
|
|||
}
|
||||
}
|
||||
|
||||
def integrationTestJob(propFileName, appURL) {
|
||||
def testsBaseDir = 'docker/jenkins/properties/integration_tests'
|
||||
def testsFileExt = '.properties'
|
||||
def integrationTestJob(propFileName, appURL='') {
|
||||
return {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
unstash 'tests'
|
||||
def fullFilename = "${testsBaseDir}/${propFileName}${testsFileExt}"
|
||||
def testScript = "docker/jenkins/run_integration_tests.sh ${fullFilename}".toString()
|
||||
def testScript = "docker/jenkins/run_integration_tests.sh ${propFileName}".toString()
|
||||
withCredentials([[$class: 'UsernamePasswordMultiBinding',
|
||||
credentialsId: 'SAUCELABS_CREDENTIALS',
|
||||
usernameVariable: 'SAUCELABS_USERNAME',
|
||||
passwordVariable: 'SAUCELABS_API_KEY']]) {
|
||||
withEnv(["BASE_URL=${appURL}",
|
||||
"SELENIUM_VERSION=2.52.0"]) {
|
||||
withEnv(["BASE_URL=${appURL}"]) {
|
||||
try {
|
||||
sh testScript
|
||||
}
|
||||
finally {
|
||||
junit 'results/*.xml'
|
||||
if ( propFileName == 'local' ) {
|
||||
sh 'docker/jenkins/cleanup_after_functional_tests.sh'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
DATABASE_URL=sqlite:///bedrock.db
|
||||
DEBUG=False
|
||||
DEV=False
|
||||
ALLOWED_HOSTS=*
|
||||
SECRET_KEY=59114b6a-2858-4caf-8878-482a24ee9542
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
DATABASE_URL=sqlite:///bedrock.db
|
||||
DEBUG=False
|
||||
ALLOWED_HOSTS=localhost,127.0.0.1,
|
||||
SECRET_KEY=59114b6a-2858-4caf-8878-482a24ee9542
|
||||
DEV=False
|
||||
ALLOWED_HOSTS=*
|
||||
SECRET_KEY=39114b6a-2858-4caf-8878-482a24ee9542
|
||||
ADMINS=["thedude@example.com"]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
environment:
|
||||
SELENIUM_VERSION: 2.52.0
|
||||
|
||||
branches:
|
||||
master:
|
||||
regions:
|
||||
|
|
@ -18,6 +21,8 @@ branches:
|
|||
- bedrock-stage
|
||||
- bedrock-prod
|
||||
integration_tests:
|
||||
- headless
|
||||
- firefox
|
||||
- chrome
|
||||
- ie
|
||||
- ie6
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче