зеркало из https://github.com/mozilla/bedrock.git
Build and deploy via Jenkinsfile
* Add jenkins.yml for config and other groovy files for functions. * Make the ircNotification function more flexible * Add locking for deploy/test steps * Allow a branch config to require a commit be tagged for deployment * Deploy all apps in a region before moving to next region * Update deployment pipeline docs * Add Jenkinsfile milestones * Optimize steps for BlueOcean
This commit is contained in:
Paul McLanahan
Родитель
3f492296d6
Коммит
160d6e55d8
|
|
@ -31,11 +31,13 @@ indent_size = 4
|
|||
[*.html]
|
||||
indent_size = 2
|
||||
|
||||
[*.yml]
|
||||
indent_size = 2
|
||||
|
||||
[Jenkinsfile]
|
||||
indent_size = 4
|
||||
|
||||
# Matches the exact files package.json and .travis.yml
|
||||
[{package.json, .travis.yml}]
|
||||
[package.json]
|
||||
indent_size = 2
|
||||
|
||||
[Makefile]
|
||||
|
|
|
|||
|
|
@ -1,62 +1,219 @@
|
|||
env.DEIS_PROFILE = 'usw'
|
||||
env.PRIVATE_REGISTRY = 'localhost:5001'
|
||||
#!groovy
|
||||
|
||||
/** Send a notice to #www on irc.mozilla.org with the build result
|
||||
*
|
||||
* @param stage step of build/deploy
|
||||
* @param result outcome of build (will be uppercased)
|
||||
*/
|
||||
def ircNotification(stage, status) {
|
||||
sh "bin/irc-notify.sh --stage '${stage}' --status '${status}'"
|
||||
}
|
||||
@Library('github.com/mozmar/jenkins-pipeline@20170126.1')
|
||||
|
||||
node {
|
||||
def branch = env.BRANCH_NAME
|
||||
def config
|
||||
def utils
|
||||
|
||||
stage ('git') {
|
||||
stage ('Checkout') {
|
||||
node {
|
||||
checkout scm
|
||||
sh 'git submodule sync'
|
||||
sh 'git submodule update --init --recursive'
|
||||
env.GIT_COMMIT = sh([returnStdout: true, script: 'git rev-parse HEAD']).trim()
|
||||
// defined in the Library loaded above
|
||||
setGitEnvironmentVariables()
|
||||
// load the config
|
||||
config = readYaml file: 'jenkins.yml'
|
||||
// load the utility functions used below
|
||||
utils = load 'docker/jenkins/utils.groovy'
|
||||
// save the files for later
|
||||
stash name: 'scripts', includes: 'bin/,docker/'
|
||||
stash name: 'tests', includes: 'tests/,requirements/'
|
||||
stash 'workspace'
|
||||
}
|
||||
}
|
||||
|
||||
if ( config.branches.containsKey(env.BRANCH_NAME) ) {
|
||||
def branchConfig = config.branches[env.BRANCH_NAME]
|
||||
milestone()
|
||||
stage ('Build Images') {
|
||||
node {
|
||||
unstash 'workspace'
|
||||
// make sure we should continue
|
||||
if ( branchConfig.require_tag ) {
|
||||
try {
|
||||
sh 'docker/jenkins/check_if_tag.sh'
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Git Tag Check', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
}
|
||||
utils.ircNotification(config, [stage: 'Test & Deploy', status: 'starting'])
|
||||
try {
|
||||
utils.buildDockerImage(dockerfile: 'bedrock_base', update: true)
|
||||
utils.buildDockerImage(dockerfile: 'bedrock_code', fromDockerfile: 'bedrock_base')
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Docker Build', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ( branch == 'master') {
|
||||
ircNotification('Dev Deploy', 'starting')
|
||||
build 'bedrock_base_image'
|
||||
milestone()
|
||||
stage ('Test Images') {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
try {
|
||||
withEnv(['DOCKER_REPOSITORY=mozorg/bedrock_code']) {
|
||||
sh 'docker/jenkins/run_tests.sh'
|
||||
}
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Unit Test', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
utils.ircNotification(config, [stage: 'Docker Builds & Tests', status: 'complete'])
|
||||
}
|
||||
}
|
||||
else if ( branch == 'prod') {
|
||||
ircNotification('Stage & Prod Deploys', 'starting')
|
||||
build 'bedrock_base_image'
|
||||
|
||||
milestone()
|
||||
stage ('Push Public Images') {
|
||||
parallel([
|
||||
dockerhub: {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
try {
|
||||
utils.buildDockerImage(dockerfile: 'bedrock_l10n', fromDockerfile: 'bedrock_code', script: 'include_l10n.sh')
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'L10n Build', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
try {
|
||||
utils.pushDockerhub('mozorg/bedrock_base')
|
||||
utils.pushDockerhub('mozorg/bedrock_code')
|
||||
utils.pushDockerhub('mozorg/bedrock_l10n', 'mozorg/bedrock')
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Dockerhub Push Failed', status: 'warning'])
|
||||
}
|
||||
}
|
||||
},
|
||||
integration_tests: {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
unstash 'tests'
|
||||
// prep for next stage
|
||||
sh 'docker/jenkins/build_integration_test_image.sh'
|
||||
}
|
||||
},
|
||||
])
|
||||
}
|
||||
else if ( branch ==~ /^demo__[\w-]+$/ ) {
|
||||
ircNotification('Demo Deploy', 'starting')
|
||||
try {
|
||||
stage ('build') {
|
||||
|
||||
/**
|
||||
* Do region first because deployment and testing should work like this:
|
||||
* region1:
|
||||
* push image -> deploy app1 -> test app1 -> deploy app2 -> test app2
|
||||
* region2:
|
||||
* push image -> deploy app1 -> test app1 -> deploy app2 -> test app2
|
||||
*
|
||||
* A failure at any step of the above should fail the entire job
|
||||
*/
|
||||
milestone()
|
||||
for (regionId in branchConfig.regions) {
|
||||
def region = config.regions[regionId]
|
||||
def stageName = "Private Push: ${region.name}"
|
||||
stage (stageName) {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
try {
|
||||
utils.pushPrivateReg(region.registry_port)
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: stageName, status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
for (appname in branchConfig.apps) {
|
||||
def appURL = "https://${appname}.${region.name}.moz.works"
|
||||
stageName = "Deploy ${appname}-${region.name}"
|
||||
// ensure no deploy/test cycle happens in parallel for an app/region
|
||||
lock (stageName) {
|
||||
milestone()
|
||||
stage (stageName) {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
withEnv(["DEIS_PROFILE=${region.deis_profile}",
|
||||
"DOCKER_REPOSITORY=${appname}",
|
||||
"DEIS_APPLICATION=${appname}"]) {
|
||||
try {
|
||||
retry(3) {
|
||||
sh 'docker/jenkins/push2deis.sh'
|
||||
}
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: stageName, status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// queue up test closures
|
||||
def allTests = [:]
|
||||
for (filename in branchConfig.integration_tests) {
|
||||
allTests[filename] = utils.integrationTestJob(filename, appname, region.name)
|
||||
}
|
||||
stage ("Test ${appname}-${region.name}") {
|
||||
try {
|
||||
// wait for server to be ready
|
||||
sleep(time: 10, unit: 'SECONDS')
|
||||
parallel allTests
|
||||
} catch(err) {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
utils.ircNotification(config, [stage: "Integration Tests ${region.name}", status: 'failure'])
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
node {
|
||||
unstash 'scripts'
|
||||
// huge success \o/
|
||||
utils.ircNotification(config, [message: appURL, status: 'shipped'])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Deploy demo branches
|
||||
*/
|
||||
else if ( env.BRANCH_NAME ==~ /^demo__[\w-]+$/ ) {
|
||||
node {
|
||||
utils.ircNotification(config, [stage: 'Demo Deploy', status: 'starting'])
|
||||
appname = utils.demoAppName(env.BRANCH_NAME)
|
||||
stage ('build') {
|
||||
milestone()
|
||||
try {
|
||||
sh 'make clean'
|
||||
sh 'make sync-all'
|
||||
sh 'echo "ENV GIT_SHA ${GIT_COMMIT}" >> docker/dockerfiles/bedrock_dev_final'
|
||||
sh 'echo "RUN echo ${GIT_COMMIT} > static/revision.txt" >> docker/dockerfiles/bedrock_dev_final'
|
||||
sh 'make build-final'
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Demo Build', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
} catch(err) {
|
||||
ircNotification('Demo Build', 'failure')
|
||||
throw err
|
||||
}
|
||||
|
||||
try {
|
||||
stage ('deploy') {
|
||||
withCredentials([[$class: 'StringBinding',
|
||||
credentialsId: 'SENTRY_DEMO_DSN',
|
||||
variable: 'SENTRY_DEMO_DSN']]) {
|
||||
sh './docker/jenkins/demo_deploy.sh'
|
||||
stage ('deploy') {
|
||||
lock (appname) {
|
||||
milestone()
|
||||
try {
|
||||
withCredentials([[$class: 'StringBinding',
|
||||
credentialsId: 'SENTRY_DEMO_DSN',
|
||||
variable: 'SENTRY_DEMO_DSN']]) {
|
||||
withEnv(['DEIS_PROFILE=usw',
|
||||
"DEIS_APP_NAME=${appname}",
|
||||
"PRIVATE_REGISTRY=localhost:${config.regions.usw.registry_port}"]) {
|
||||
sh './docker/jenkins/demo_deploy.sh'
|
||||
}
|
||||
}
|
||||
utils.ircNotification(config, [app_url: "https://${appname}.us-west.moz.works/"])
|
||||
} catch(err) {
|
||||
utils.ircNotification(config, [stage: 'Demo Deploy', status: 'failure'])
|
||||
throw err
|
||||
}
|
||||
}
|
||||
} catch(err) {
|
||||
ircNotification('Demo Deploy', 'failure')
|
||||
throw err
|
||||
}
|
||||
}
|
||||
else {
|
||||
echo "Doing nothing for ${branch}"
|
||||
}
|
||||
}
|
||||
else {
|
||||
echo "Doing nothing for ${env.BRANCH_NAME}"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
# Required environment variables if using --stage and --status:
|
||||
# Required environment variables if using --stage:
|
||||
# BRANCH_NAME, BUILD_NUMBER, BUILD_URL
|
||||
|
||||
# defaults and constants
|
||||
BUILD_NUMBER="${BUILD_NUMBER:-0}"
|
||||
NICK="bedrock-deployer-$BUILD_NUMBER"
|
||||
NICK="hms-flintstone"
|
||||
CHANNEL="#www"
|
||||
SERVER="irc.mozilla.org:6697"
|
||||
# colors and styles: values from the following links
|
||||
# http://www.mirc.com/colors.html
|
||||
# http://stackoverflow.com/a/13382032
|
||||
RED=$'\x034'
|
||||
YELLOW=$'\x038'
|
||||
GREEN=$'\x039'
|
||||
BLUE=$'\x0311'
|
||||
BOLD=$'\x02'
|
||||
|
|
@ -30,10 +30,6 @@ while [[ $# -gt 1 ]]; do
|
|||
STATUS="$2"
|
||||
shift # past argument
|
||||
;;
|
||||
--demo_url)
|
||||
DEMO_URL="$2"
|
||||
shift # past argument
|
||||
;;
|
||||
-m|--message)
|
||||
MESSAGE="$2"
|
||||
shift # past argument
|
||||
|
|
@ -54,31 +50,43 @@ while [[ $# -gt 1 ]]; do
|
|||
shift # past argument or value
|
||||
done
|
||||
|
||||
if [[ -z "$MESSAGE" ]]; then
|
||||
if [[ -n "$STATUS" ]]; then
|
||||
STATUS=$(echo "$STATUS" | tr '[:lower:]' '[:upper:]')
|
||||
case "$STATUS" in
|
||||
'SUCCESS')
|
||||
STATUS_COLOR="${BOLD}${GREEN}"
|
||||
;;
|
||||
'FAILURE')
|
||||
STATUS_COLOR="${BOLD}${RED}"
|
||||
;;
|
||||
*)
|
||||
STATUS_COLOR="$BLUE"
|
||||
;;
|
||||
esac
|
||||
MESSAGE="${STAGE}: ${STATUS_COLOR}${STATUS}${NORMAL}:"
|
||||
MESSAGE="$MESSAGE Branch ${BOLD}${BRANCH_NAME}${NORMAL} build #${BUILD_NUMBER}: ${BUILD_URL}"
|
||||
elif [[ -n "$DEMO_URL" ]]; then
|
||||
MESSAGE="${BOLD}${GREEN}SUCCESS${NORMAL}: Demo deployed: ${DEMO_URL}"
|
||||
else
|
||||
echo "Missing required arguments"
|
||||
echo
|
||||
echo "Usage: irc-notify.sh [--stage STAGE --status STATUS] [--demo_url DEMO_URL]"
|
||||
echo "Optional args: --irc_nick, --irc_server, --irc_channel"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$STATUS" ]]; then
|
||||
STATUS=$(echo "$STATUS" | tr '[:lower:]' '[:upper:]')
|
||||
case "$STATUS" in
|
||||
'SUCCESS')
|
||||
STATUS_COLOR="🎉 ${BOLD}${GREEN}"
|
||||
;;
|
||||
'SHIPPED')
|
||||
STATUS_COLOR="🚢 ${BOLD}${GREEN}"
|
||||
;;
|
||||
'WARNING')
|
||||
STATUS_COLOR="⚠️ ${BOLD}${YELLOW}"
|
||||
;;
|
||||
'FAILURE')
|
||||
STATUS_COLOR="🚨 ${BOLD}${RED}"
|
||||
;;
|
||||
*)
|
||||
STATUS_COLOR="✨ $BLUE"
|
||||
;;
|
||||
esac
|
||||
STATUS="${STATUS_COLOR}${STATUS}${NORMAL}: "
|
||||
fi
|
||||
|
||||
if [[ -n "$STAGE" ]]; then
|
||||
MESSAGE="${STATUS}${STAGE}:"
|
||||
MESSAGE="$MESSAGE Branch ${BOLD}${BRANCH_NAME}${NORMAL} build #${BUILD_NUMBER}: ${BUILD_URL}"
|
||||
elif [[ -n "$MESSAGE" ]]; then
|
||||
MESSAGE="${STATUS}${MESSAGE}"
|
||||
else
|
||||
echo "Missing required arguments"
|
||||
echo
|
||||
echo "Usage: irc-notify.sh [--stage STAGE]|[-m MESSAGE]"
|
||||
echo "Optional args: --status, --irc_nick, --irc_server, --irc_channel"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -n "$BUILD_NUMBER" ]]; then
|
||||
NICK="${NICK}-${BUILD_NUMBER}"
|
||||
fi
|
||||
|
||||
(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ CMD="${CMD} -r a"
|
|||
CMD="${CMD} --verbose"
|
||||
CMD="${CMD} -n ${PYTEST_PROCESSES}"
|
||||
CMD="${CMD} --base-url ${BASE_URL}"
|
||||
# rerun a flaky test once
|
||||
# DO NOT INCREASE THIS
|
||||
CMD="${CMD} --reruns 1"
|
||||
# Disable HTML report due to https://github.com/pytest-dev/pytest/issues/1351
|
||||
# CMD="${CMD} --html ${RESULTS_PATH}/index.html"
|
||||
CMD="${CMD} --junitxml ${RESULTS_PATH}/junit.xml"
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
#
|
||||
set -ex
|
||||
DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT}
|
||||
TMP_DOCKER_TAG=${JOB_NAME}${BUILD_NUMBER}
|
||||
TMP_DOCKER_TAG=${BUILD_TAG}
|
||||
|
||||
# If docker image exists and no force rebuild do nothing
|
||||
FORCE_REBUILD=`echo "$FORCE_REBUILD" | tr '[:upper:]' '[:lower:]'`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
#!/bin/bash -xe
|
||||
docker build -t bedrock_integration_tests:${GIT_COMMIT} -f docker/dockerfiles/bedrock_integration_tests --pull=true .
|
||||
|
|
@ -1,10 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Used to trigger downstream Jenkins jobs
|
||||
TRIGGER_FILE=".commit_is_tag"
|
||||
rm -rf $TRIGGER_FILE
|
||||
|
||||
TAG=$(git describe --tags --exact-match $GIT_COMMIT 2> /dev/null)
|
||||
if [[ -n "$TAG" ]] && [[ "$TAG" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}(\.[0-9])?$ ]]; then
|
||||
touch $TRIGGER_FILE
|
||||
fi;
|
||||
if [[ -n "$TAG" ]]; then
|
||||
if [[ "$TAG" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}(\.[0-9])?$ ]]; then
|
||||
echo "Build tagged as $TAG"
|
||||
exit 0
|
||||
else
|
||||
echo "Build tagged but in the wrong format: $TAG"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Build not tagged"
|
||||
exit 1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
#!/bin/bash
|
||||
set -ex
|
||||
|
||||
DEIS_APP_NAME="bedrock-demo-${BRANCH_NAME#demo__}"
|
||||
# convert underscores to dashes. Deis does _not_ like underscores.
|
||||
DEIS_APP_NAME=$( echo "$DEIS_APP_NAME" | tr "_" "-" )
|
||||
# used for pulling from deis
|
||||
DOCKER_IMAGE_TAG="${DEIS_APP_NAME}:${GIT_COMMIT}"
|
||||
# used for pushing to registry
|
||||
|
|
@ -25,5 +22,3 @@ if deis apps:create "$DEIS_APP_NAME" --no-remote; then
|
|||
fi
|
||||
echo "Pulling $DOCKER_IMAGE_TAG into Deis app $DEIS_APP_NAME"
|
||||
deis pull "$DOCKER_IMAGE_TAG" -a "$DEIS_APP_NAME"
|
||||
|
||||
bin/irc-notify.sh --demo_url "https://${DEIS_APP_NAME}.us-west.moz.works/"
|
||||
|
|
|
|||
|
|
@ -6,26 +6,18 @@
|
|||
#
|
||||
set -xe
|
||||
|
||||
# Used to trigger downstream Jenkins jobs
|
||||
PARAM_FILE=.parameters
|
||||
TRIGGER_FILE=.docker-updated
|
||||
FORCE_TIME_TRIGGER_UPDATE=.timetriggerupdate
|
||||
rm -rf $TRIGGER_FILE $PARAM_FILE
|
||||
|
||||
if [[ $BUILD_CAUSE == "REMOTECAUSE" ]]
|
||||
then
|
||||
LATEST_TAG=$(git describe --abbrev=0 --tags)
|
||||
# parent (~0) of latest tag is the commit that was tagged
|
||||
GIT_COMMIT=$(git rev-parse ${LATEST_TAG}~0)
|
||||
fi
|
||||
echo "GIT_COMMIT=$GIT_COMMIT" >> $PARAM_FILE
|
||||
|
||||
DOCKER_IMAGE_TAG=${DOCKER_REPOSITORY}:${GIT_COMMIT}
|
||||
|
||||
touch $TRIGGER_FILE
|
||||
|
||||
if [[ ! -e locale ]];
|
||||
if [[ ! -d locale/.git ]];
|
||||
then
|
||||
rm -rf locale
|
||||
git clone --depth 1 https://github.com/mozilla-l10n/bedrock-l10n locale
|
||||
fi;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
export DRIVER=SauceLabs
|
||||
export BROWSER_NAME=chrome
|
||||
export PLATFORM="Windows 10"
|
||||
export MARK_EXPRESSION="not headless"
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
export DRIVER=SauceLabs
|
||||
export BROWSER_NAME=firefox
|
||||
export BROWSER_VERSION=45.0
|
||||
export PLATFORM="Windows 10"
|
||||
export MARK_EXPRESSION="not headless"
|
||||
|
|
@ -0,0 +1 @@
|
|||
export MARK_EXPRESSION=headless
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
export DRIVER=SauceLabs
|
||||
export BROWSER_NAME="internet explorer"
|
||||
export PLATFORM="Windows 10"
|
||||
export MARK_EXPRESSION="not headless"
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
export DRIVER=SauceLabs
|
||||
export BROWSER_NAME="internet explorer"
|
||||
export BROWSER_VERSION="6.0"
|
||||
export PLATFORM="Windows XP"
|
||||
export MARK_EXPRESSION="sanity and not headless"
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
export DRIVER="SauceLabs"
|
||||
export BROWSER_NAME="internet explorer"
|
||||
export BROWSER_VERSION="7.0"
|
||||
export PLATFORM="Windows XP"
|
||||
export MARK_EXPRESSION="sanity and not headless"
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
DRIVER=SauceLabs
|
||||
BROWSER_NAME=firefox
|
||||
BROWSER_VERSION=45.0
|
||||
PLATFORM=Windows 10
|
||||
MARK_EXPRESSION=not headless
|
||||
|
|
@ -1 +0,0 @@
|
|||
MARK_EXPRESSION=headless
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
DRIVER=SauceLabs
|
||||
BROWSER_NAME=chrome
|
||||
PLATFORM=Windows 10
|
||||
MARK_EXPRESSION=not headless
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
DRIVER=SauceLabs
|
||||
BROWSER_NAME=internet explorer
|
||||
PLATFORM=Windows 10
|
||||
MARK_EXPRESSION=not headless
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
DRIVER=SauceLabs
|
||||
BROWSER_NAME=internet explorer
|
||||
BROWSER_VERSION=6.0
|
||||
PLATFORM=Windows XP
|
||||
MARK_EXPRESSION=sanity and not headless
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
DRIVER=SauceLabs
|
||||
BROWSER_NAME=internet explorer
|
||||
BROWSER_VERSION=7.0
|
||||
PLATFORM=Windows XP
|
||||
MARK_EXPRESSION=sanity and not headless
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
#!/bin/bash
|
||||
# Needs DEIS_CONTROLLER URL, DEIS_USERNAME, DEIS_PASSWORD,
|
||||
# DOCKER_REPOSITORY, DEIS_APPLICATION, NEWRELIC_API_KEY and
|
||||
# Needs DEIS_PROFILE, DOCKER_REPOSITORY, DEIS_APPLICATION, NEWRELIC_API_KEY and
|
||||
# NEWRELIC_APP_NAME environment variables.
|
||||
#
|
||||
# To set them go to Job -> Configure -> Build Environment -> Inject
|
||||
|
|
@ -9,10 +8,11 @@
|
|||
|
||||
set -ex
|
||||
|
||||
deis login $DEIS_CONTROLLER --username $DEIS_USERNAME --password $DEIS_PASSWORD
|
||||
deis pull $DOCKER_REPOSITORY:$GIT_COMMIT -a $DEIS_APPLICATION
|
||||
curl -H "x-api-key:$NEWRELIC_API_KEY" \
|
||||
-d "deployment[app_name]=$NEWRELIC_APP_NAME" \
|
||||
-d "deployment[revision]=$GIT_COMMIT" \
|
||||
-d "deployment[user]=EE Jenkins" \
|
||||
https://api.newrelic.com/deployments.xml
|
||||
if [[ -n "$NEWRELIC_API_KEY" ]]; then
|
||||
curl -H "x-api-key:$NEWRELIC_API_KEY" \
|
||||
-d "deployment[app_name]=$NEWRELIC_APP_NAME" \
|
||||
-d "deployment[revision]=$GIT_COMMIT" \
|
||||
-d "deployment[user]=EE Jenkins" \
|
||||
https://api.newrelic.com/deployments.xml
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash -xe
|
||||
GIT_COMMIT=${GIT_COMMIT:-$(git rev-parse HEAD)}
|
||||
cp docker/dockerfiles/bedrock_integration_tests Dockerfile
|
||||
docker build -t bedrock_integration_tests:${GIT_COMMIT} --pull=true .
|
||||
|
||||
# $1 should be the properties file for this run
|
||||
source "$1"
|
||||
|
||||
if [ -z "${BASE_URL}" ]; then
|
||||
# start bedrock
|
||||
|
|
@ -48,7 +48,12 @@ if [ "${DRIVER}" = "Remote" ]; then
|
|||
done
|
||||
fi
|
||||
|
||||
docker run -v `pwd`/results:/app/results \
|
||||
# make sure results dir exists or docker will create it
|
||||
# and it will be owned by root
|
||||
RESULTS_DIR="$PWD/results"
|
||||
rm -rf "$RESULTS_DIR"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
docker run -v "${RESULTS_DIR}:/app/results" -u $(stat -c "%u:%g" "$RESULTS_DIR") \
|
||||
${DOCKER_LINKS[@]} \
|
||||
-e BASE_URL=${BASE_URL} \
|
||||
-e DRIVER=${DRIVER} \
|
||||
|
|
|
|||
|
|
@ -0,0 +1,93 @@
|
|||
/**
|
||||
* Define utility functions.
|
||||
*/
|
||||
|
||||
def demoAppName(branchname) {
|
||||
def appname = branchname[5..-1].replaceAll('_', '-')
|
||||
return "bedrock-demo-${appname}".toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a notice to #www on irc.mozilla.org with the build result
|
||||
*
|
||||
* @param stage step of build/deploy
|
||||
* @param result outcome of build (will be uppercased)
|
||||
*/
|
||||
def ircNotification(config, Map args) {
|
||||
def command = "bin/irc-notify.sh --irc_nick '${config.irc.nick}' --irc_channel '${config.irc.channel}'"
|
||||
for (arg in args) {
|
||||
command += " --${arg.key} '${arg.value}'"
|
||||
}
|
||||
sh command
|
||||
}
|
||||
|
||||
def buildDockerImage(Map kwargs) {
|
||||
def update = kwargs.update ? 'true' : 'false'
|
||||
def repo = kwargs.dockerRepo ?: 'mozorg'
|
||||
def script = kwargs.script ?: 'build_image.sh'
|
||||
def environs = ["UPDATE_DOCKER_IMAGES=${update}",
|
||||
"DOCKERFILE=${kwargs.dockerfile}",
|
||||
"DOCKER_REPOSITORY=${repo}/${kwargs.dockerfile}"]
|
||||
if (kwargs.fromDockerfile) {
|
||||
environs << "FROM_DOCKER_REPOSITORY=${repo}/${kwargs.fromDockerfile}"
|
||||
}
|
||||
withEnv(environs) {
|
||||
sh "docker/jenkins/${script}"
|
||||
}
|
||||
}
|
||||
|
||||
def pushDockerhub(from_repo, to_repo='') {
|
||||
to_repo = to_repo ?: from_repo
|
||||
withCredentials([[$class: 'StringBinding',
|
||||
credentialsId: 'DOCKER_PASSWORD',
|
||||
variable: 'DOCKER_PASSWORD']]) {
|
||||
withEnv(['DOCKER_USERNAME=mozjenkins',
|
||||
"FROM_DOCKER_REPOSITORY=${from_repo}",
|
||||
"DOCKER_REPOSITORY=${to_repo}"]) {
|
||||
retry(2) {
|
||||
sh 'docker/jenkins/push2dockerhub.sh'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def pushPrivateReg(port) {
|
||||
retry(3) {
|
||||
// TODO Fix DEIS_APPS before merge
|
||||
// DEIS_APPS=bedrock-dev,bedrock-stage,bedrock-prod
|
||||
withEnv(['FROM_DOCKER_REPOSITORY=mozorg/bedrock_l10n',
|
||||
"PRIVATE_REGISTRIES=localhost:${port}",
|
||||
'DEIS_APPS=bedrock-jenkinsfile-test']) {
|
||||
sh 'docker/jenkins/push2privateregistries.sh'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def integrationTestJob(propFileName, appName, region) {
|
||||
def testsBaseDir = 'docker/jenkins/properties/integration_tests'
|
||||
def testsFileExt = '.properties'
|
||||
return {
|
||||
node {
|
||||
unstash 'scripts'
|
||||
unstash 'tests'
|
||||
def fullFilename = "${testsBaseDir}/${propFileName}${testsFileExt}"
|
||||
def testScript = "docker/jenkins/run_integration_tests.sh ${fullFilename}".toString()
|
||||
withCredentials([[$class: 'UsernamePasswordMultiBinding',
|
||||
credentialsId: 'SAUCELABS_CREDENTIALS',
|
||||
usernameVariable: 'SAUCELABS_USERNAME',
|
||||
passwordVariable: 'SAUCELABS_API_KEY']]) {
|
||||
withEnv(["BASE_URL=https://${appName}.${region}.moz.works",
|
||||
"SELENIUM_VERSION=2.52.0"]) {
|
||||
try {
|
||||
sh testScript
|
||||
}
|
||||
finally {
|
||||
junit 'results/*.xml'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return this;
|
||||
Двоичные данные
docs/pipeline.png
Двоичные данные
docs/pipeline.png
Двоичный файл не отображается.
|
До Ширина: | Высота: | Размер: 14 KiB |
|
|
@ -34,15 +34,15 @@ Pull request
|
|||
Once a pull request is submitted, `CircleCI`_ will run both the Python and JavaScript
|
||||
unit tests, as well as the smoke suite of redirect headless HTTP(s) response checks.
|
||||
|
||||
Push to master branch (not tagged)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
Push to master branch
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Whenever a change is pushed to the master branch but not tagged, the smoke suite of
|
||||
Whenever a change is pushed to the master branch, the smoke suite of
|
||||
headless (see :ref:`testing-redirects`) and UI tests (see :ref:`smoke-functional-tests`)
|
||||
are run against Firefox on Linux. If successful, the change is pushed to the dev environment,
|
||||
and the full suite of headless and UI tests are then run against
|
||||
Firefox on Windows 10 using `Sauce Labs`_. This is handled by the pipeline, and is subject
|
||||
to change according to the `per-push properties file`_ in the repository.
|
||||
to change according to the settings in the `jenkins.yml file`_ in the repository.
|
||||
|
||||
.. _tagged-commit:
|
||||
|
||||
|
|
@ -55,15 +55,15 @@ Chrome and Internet Explorer on Windows 10, and the sanity suite is run against
|
|||
versions of Internet Explorer (currently IE6 & IE7). If successful, the change is
|
||||
pushed to staging, tested, and then to production and the same tests are then run against
|
||||
production. As with untagged pushes, this is all handled by the pipeline, and is subject
|
||||
to change according to the `per-tag properties file`_ in the repository.
|
||||
to change according to the settings in the `jenkins.yml file`_ in the repository.
|
||||
|
||||
**Push to prod cheat sheet**
|
||||
|
||||
#. Check out the ``master`` branch
|
||||
#. Make sure the ``master`` branch is up to date with ``mozilla/bedrock master``
|
||||
#. Check that dev deployment is green:
|
||||
#. View `deployment pipeline <https://ci.us-west.moz.works/view/Bedrock%20Pipeline/?fullscreen=true>`_
|
||||
#. If any staging tests fail above, check retries, e.g. `bedrock_test_dev_eu_west`_
|
||||
#. View `deployment pipeline <https://ci.us-west.moz.works/blue/organizations/jenkins/bedrock_multibranch_pipeline/branches/>`_
|
||||
and look at ``master`` branch
|
||||
#. Tag and push the deployment by running ``bin/tag-release.sh --push``
|
||||
|
||||
.. note::
|
||||
|
|
@ -82,28 +82,23 @@ to change according to the `per-tag properties file`_ in the repository.
|
|||
Pipeline integration
|
||||
--------------------
|
||||
|
||||
The following diagram shows an overview of when the tests are run:
|
||||
|
||||
.. image:: pipeline.png
|
||||
:width: 100%
|
||||
|
||||
A dedicated **Test Runner** job exists to run the integration tests. This job takes various
|
||||
parameters, allowing it to be called from multiple upstream jobs to cover the testing
|
||||
needs. This job takes the parameters passed to it and executes `this script <https://github.com/mozilla/bedrock/blob/master/docker/jenkins/run_integration_tests.sh>`_,
|
||||
Our `Jenkinsfile`_ will run the integration tests based on information in our `jenkins.yml file`_.
|
||||
This file specifies various test names per branch that will cause it to load different
|
||||
parameters, allowing it to be called in many different ways to cover the testing
|
||||
needs. The job finds the parameters file and executes `this script <https://github.com/mozilla/bedrock/blob/master/docker/jenkins/run_integration_tests.sh>`_,
|
||||
which then runs `this Docker image <https://github.com/mozilla/bedrock/blob/master/docker/dockerfiles/bedrock_integration_tests>`_,
|
||||
and ultimately runs `another script <https://github.com/mozilla/bedrock/blob/master/bin/run-integration-tests.sh>`_.
|
||||
The two scripts can also be executed locally to replicate the way Jenkins operates.
|
||||
|
||||
During the **Build** stage, the Test Runner job is called without a ``BASE_URL``. This means
|
||||
During the **Test Images** stage, the Test Runner job is called without a ``BASE_URL``. This means
|
||||
that a local instance of the application will be started, and the URL of this instance
|
||||
will be used for testing. The ``DRIVER`` parameter is set to ``Remote``, which causes a
|
||||
local instance of Selenium Grid to be started in Docker and used for the browser-based
|
||||
functional UI tests.
|
||||
|
||||
During the various **Test** stages, the **Test Runner** job is called once for every `per-push properties file`_.
|
||||
If the push was also tagged, the job is also called once for every `per-tag properties file`_.
|
||||
The content of these files are used as the parameters for the Test Runner job, allowing
|
||||
the configuration to be reviewed and under version control.
|
||||
The test scripts above will be run once for each properties file specified in the `jenkins.yml file`_
|
||||
for the branch being built and tested. Pushes to `master` will run different tests than pushes to `prod`
|
||||
for example.
|
||||
|
||||
Configuration
|
||||
~~~~~~~~~~~~~
|
||||
|
|
@ -129,7 +124,8 @@ Adding test runs
|
|||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Test runs can be added by creating new `properties files`_ with the parameters of the new
|
||||
test run. For example, if you wanted to run tests in Firefox on both Windows 10 and
|
||||
test run. These are simply bash syntax files that set environment variables.
|
||||
For example, if you wanted to run tests in Firefox on both Windows 10 and
|
||||
OS X, you could create the following files
|
||||
|
||||
win10-firefox.properties
|
||||
|
|
@ -137,20 +133,20 @@ win10-firefox.properties
|
|||
|
||||
.. code-block:: none
|
||||
|
||||
DRIVER=SauceLabs
|
||||
BROWSER_NAME=firefox
|
||||
PLATFORM=Windows 10
|
||||
MARK_EXPRESSION=not headless
|
||||
export DRIVER=SauceLabs
|
||||
export BROWSER_NAME=firefox
|
||||
export PLATFORM="Windows 10"
|
||||
export MARK_EXPRESSION="not headless""
|
||||
|
||||
osx-firefox.properties
|
||||
......................
|
||||
|
||||
.. code-block:: none
|
||||
|
||||
DRIVER=SauceLabs
|
||||
BROWSER_NAME=firefox
|
||||
PLATFORM=OS X 10.11
|
||||
MARK_EXPRESSION=not headless
|
||||
export DRIVER=SauceLabs
|
||||
export BROWSER_NAME=firefox
|
||||
export PLATFORM="OS X 10.11"
|
||||
export MARK_EXPRESSION="not headless"
|
||||
|
||||
You can use `Sauce Labs platform configurator`_ to help with the parameter values.
|
||||
|
||||
|
|
@ -159,53 +155,9 @@ job and pass in the ``BASE_URL`` and other parameters of your choosing. This is
|
|||
testing against deployed demo environments. For a good baseline, use the values from ``win10-firefox.properties``_
|
||||
above.
|
||||
|
||||
Investigating failures
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Due to the configuration of the pipeline, it can be a little confusing when
|
||||
investigating failed builds. The actual `bedrock_integration_tests_runner`_ job will fail,
|
||||
and cause the upstream job that triggered it to fail. The best approach to investigating
|
||||
failures is to first look at the upstream job, which will either be
|
||||
`bedrock_integration_tests`_ (during the **Build** stage) or one of the ``bedrock_test_`` jobs in
|
||||
the various **Test** stages:
|
||||
|
||||
* `bedrock_test_dev_eu_west`_
|
||||
* `bedrock_test_dev_us_west`_
|
||||
* `bedrock_test_stage_eu_west`_
|
||||
* `bedrock_test_stage_us_west`_
|
||||
* `bedrock_test_prod_eu_west`_
|
||||
* `bedrock_test_prod_us_west`_
|
||||
|
||||
In the console log for the failed upstream job you will see a line similar to:
|
||||
|
||||
``bedrock_integration_tests_runner #n completed. Result was FAILURE``
|
||||
|
||||
The ``#n`` will be a valid build number, and this text will be a link directly to the
|
||||
failed test runner build. Clicking this will take you to that build, where you can use
|
||||
the **Console Output** or the **Test Results** links to find out what caused the build
|
||||
to fail.
|
||||
|
||||
If you’re looking at a failed `bedrock_integration_tests_runner`_ build, you can determine
|
||||
the upstream job by looking for the following text in the build status page, or in the
|
||||
console output:
|
||||
|
||||
``Started by upstream project bedrock_test_ build number n``
|
||||
|
||||
If the job was run in `Sauce Labs`_, you can also find a link to the failed job in the
|
||||
console output, which will contain a **video of the failure**.
|
||||
|
||||
Known issues in Jenkins
|
||||
-----------------------
|
||||
|
||||
Shared test runner job shown in pipeline view
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
In order to avoid duplicating the test runner job for every necessary environment and
|
||||
platform configuration, we use a shared job that accepts parameters. Unfortunately this
|
||||
means that the pipeline view tries to show the job with the various upstream
|
||||
relationships. It would be better to simply hide this job, however this functionality
|
||||
does not exist. An `enhancement request`_ has been raised.
|
||||
|
||||
Jenkins stalls after global configuration changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -217,17 +169,10 @@ A `bug for the IRC plugin`_ has been raised.
|
|||
.. _Deployment Pipeline: https://ci.us-west.moz.works/view/Bedrock%20Pipeline/
|
||||
.. _CircleCI: https://circleci.com/
|
||||
.. _Sauce Labs: https://saucelabs.com/
|
||||
.. _per-push properties file: https://github.com/mozilla/bedrock/tree/master/docker/jenkins/properties/integration_tests/per_push
|
||||
.. _per-tag properties file: https://github.com/mozilla/bedrock/tree/master/docker/jenkins/properties/integration_tests/per_tag
|
||||
.. _Jenkinsfile: https://github.com/mozilla/bedrock/tree/master/Jenkinsfile
|
||||
.. _jenkins.yml file: https://github.com/mozilla/bedrock/tree/master/jenkins.yml
|
||||
.. _properties files: https://github.com/mozilla/bedrock/tree/master/docker/jenkins/properties/integration_tests
|
||||
.. _bedrock_integration_tests_runner: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_integration_tests_runner/
|
||||
.. _bedrock_integrations_tests: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_integration_tests/
|
||||
.. _bedrock_test_dev_eu_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_dev_eu_west/
|
||||
.. _bedrock_test_dev_us_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_dev_us_west/
|
||||
.. _bedrock_test_stage_eu_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_stage_eu_west/
|
||||
.. _bedrock_test_stage_us_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_stage_us_west/
|
||||
.. _bedrock_test_prod_eu_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_prod_eu_west/
|
||||
.. _bedrock_test_prod_us_west: https://ci.us-west.moz.works/view/Bedrock/job/bedrock_test_prod_us_west/
|
||||
.. _configured in Jenkins: https://ci.us-west.moz.works/configure
|
||||
.. _become unresponsive: https://issues.jenkins-ci.org/browse/JENKINS-28175
|
||||
.. _test dependencies: https://github.com/mozilla/bedrock/blob/master/requirements/test.txt
|
||||
|
|
@ -235,4 +180,3 @@ A `bug for the IRC plugin`_ has been raised.
|
|||
.. _Sauce Labs platform configurator: https://wiki.saucelabs.com/display/DOCS/Platform+Configurator/
|
||||
.. _enhancement request: https://issues.jenkins-ci.org/browse/JENKINS-26210
|
||||
.. _bug for the IRC plugin: https://issues.jenkins-ci.org/browse/JENKINS-28175
|
||||
.. _bedrock_integration_tests_runner: https://ci.us-west.moz.works/job/bedrock_integration_tests_runner/build?delay=0sec
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
branches:
|
||||
master:
|
||||
regions:
|
||||
- usw
|
||||
- euw
|
||||
apps:
|
||||
- bedrock-dev
|
||||
integration_tests:
|
||||
- firefox
|
||||
- headless
|
||||
prod:
|
||||
require_tag: true
|
||||
regions:
|
||||
- usw
|
||||
- euw
|
||||
apps:
|
||||
- bedrock-stage
|
||||
- bedrock-prod
|
||||
integration_tests:
|
||||
- chrome
|
||||
- ie
|
||||
- ie6
|
||||
- ie7
|
||||
# for testing
|
||||
deploy-via-jenkinsfile:
|
||||
regions:
|
||||
- usw
|
||||
- euw
|
||||
apps:
|
||||
- bedrock-jenkinsfile-test
|
||||
integration_tests:
|
||||
- firefox
|
||||
- headless
|
||||
- chrome
|
||||
- ie
|
||||
- ie6
|
||||
- ie7
|
||||
|
||||
regions:
|
||||
usw:
|
||||
deis_profile: usw
|
||||
name: us-west
|
||||
registry_port: 5001
|
||||
euw:
|
||||
deis_profile: euw
|
||||
name: eu-west
|
||||
registry_port: 5000
|
||||
|
||||
irc:
|
||||
nick: hms-flintstone
|
||||
channel: "#www"
|
||||
Загрузка…
Ссылка в новой задаче