From 7cb811dc8db73b9d2153a672ad59999bfd907530 Mon Sep 17 00:00:00 2001
From: Menghua Xiao <menxiao@microsoft.com>
Date: Fri, 1 Dec 2017 11:23:11 +0800
Subject: [PATCH] Script extension to bootstrap the Jenkins and pipeline for
 blue-green ACS deployment

---
 .gitignore                                    |   3 +
 jenkins/blue-green/acs-k8s-blue-green-job.xml | 282 ++++++++++++++++++
 jenkins/blue-green/add-blue-green-job.sh      | 225 ++++++++++++++
 jenkins/blue-green/sp-credentials.xml         |  12 +
 jenkins/blue-green/ssh-credentials.xml        |   9 +
 jenkins/install_jenkins.sh                    |  20 ++
 .../301-jenkins-k8s-blue-green.sh             | 202 +++++++++++++
 7 files changed, 753 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 jenkins/blue-green/acs-k8s-blue-green-job.xml
 create mode 100644 jenkins/blue-green/add-blue-green-job.sh
 create mode 100644 jenkins/blue-green/sp-credentials.xml
 create mode 100644 jenkins/blue-green/ssh-credentials.xml
 create mode 100644 quickstart_template/301-jenkins-k8s-blue-green.sh

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..6ef72f8
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.idea/
+*.iml
+
diff --git a/jenkins/blue-green/acs-k8s-blue-green-job.xml b/jenkins/blue-green/acs-k8s-blue-green-job.xml
new file mode 100644
index 0000000..d9d3de8
--- /dev/null
+++ b/jenkins/blue-green/acs-k8s-blue-green-job.xml
@@ -0,0 +1,282 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<flow-definition plugin="workflow-job@2.15">
+  <actions />
+  <description>{insert-job-description}</description>
+  <displayName>{insert-job-display-name}</displayName>
+  <keepDependencies>false</keepDependencies>
+  <properties>
+    <org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+      <triggers/>
+    </org.jenkinsci.plugins.workflow.job.properties.PipelineTriggersJobProperty>
+  </properties>
+  <definition class="org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition" plugin="workflow-cps@2.41">
+    <script><![CDATA[
+node {
+    def resourceGroup = '{insert-acs-resource-group}'
+    def acs = '{insert-acs-name}'
+    def currentEnvironment = 'blue'
+    def newEnvironment = { ->
+        currentEnvironment == 'blue' ? 'green' : 'blue'
+    }
+    def expectedTomcatVersion = { ->
+        newEnvironment() == 'blue' ? '7' : '8'
+    }
+
+    stage('Prepare Sources') {
+        // here we generate all the Kubernetes configurations in the pipeline just for demostration,
+        // in the real world projects, we will fetch the configurations from SCM most of the time.
+        writeFile file: 'k8s/service-blue.yml', text: '''
+kind: Service
+apiVersion: v1
+metadata:
+  name: tomcat-service
+  labels:
+    app: tomcat
+    role: blue
+    env: prod
+spec:
+  type: LoadBalancer
+  selector:
+    app: tomcat
+    role: blue
+  ports:
+    - port: 80
+      targetPort: 8080'''
+      
+        writeFile file: 'k8s/test-endpoint-blue.yml', text: '''
+kind: Service
+apiVersion: v1
+metadata:
+  name: tomcat-test-blue
+  labels:
+    app: tomcat
+    role: test-blue
+spec:
+  type: LoadBalancer
+  selector:
+    app: tomcat
+    role: blue
+  ports:
+    - port: 80
+      targetPort: 8080'''
+        
+        writeFile file: 'k8s/service-green.yml', text: '''
+kind: Service
+apiVersion: v1
+metadata:
+  name: tomcat-service
+  labels:
+    app: tomcat
+    role: green
+    env: prod
+spec:
+  type: LoadBalancer
+  selector:
+    app: tomcat
+    role: green
+  ports:
+    - port: 80
+      targetPort: 8080'''
+
+        writeFile file: 'k8s/test-endpoint-green.yml', text: '''
+kind: Service
+apiVersion: v1
+metadata:
+  name: tomcat-test-green
+  labels:
+    app: tomcat
+    role: test-green
+spec:
+  type: LoadBalancer
+  selector:
+    app: tomcat
+    role: green
+  ports:
+    - port: 80
+      targetPort: 8080'''
+        
+        writeFile file: 'k8s/deployment-blue.yml', text: '''
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: tomcat-deployment-blue
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: tomcat
+        role: blue
+    spec:
+      containers:
+      - name: tomcat-container
+        image: tomcat:7.0-jre7
+        ports:
+        - containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 8080
+      strategy:
+        type: RollingUpdate
+        rollingUpdate:
+          maxUnavailable: 1
+          maxSurge: 50%'''
+        
+        writeFile file: 'k8s/deployment-green.yml', text: '''
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: tomcat-deployment-green
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: tomcat
+        role: green
+    spec:
+      containers:
+      - name: tomcat-container
+        image: tomcat:8.0-jre8
+        ports:
+        - containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 8080
+      strategy:
+        type: RollingUpdate
+        rollingUpdate:
+          maxUnavailable: 1
+          maxSurge: 50%'''
+    }
+
+    stage('Ensure Test Endpoints') {
+        // Generally, this should be prepared out of the deployment pipeline.
+        // And it's less likely to be changed during the deployment.
+        // We add it here to keep it simple so that we don't have to configure another job for demonstration.
+        acsDeploy azureCredentialsId: 'sp', 
+                  configFilePaths: 'k8s/test-endpoint-*.yml',
+                  containerService: "$acs | Kubernetes",
+                  resourceGroupName: resourceGroup,
+                  sshCredentialsId: 'k8s-ssh'
+    }
+
+    stage('Check Env') {
+        withCredentials([azureServicePrincipal('sp')]) {
+            sshagent(['k8s-ssh']) {
+                sh """
+az login --service-principal -u "\$AZURE_CLIENT_ID" -p "\$AZURE_CLIENT_SECRET" -t "\$AZURE_TENANT_ID"
+az account set --subscription "\$AZURE_SUBSCRIPTION_ID"
+acs_host=\$(az acs show --resource-group "${resourceGroup}" --name "${acs}" --query "[linuxProfile.adminUsername, masterProfile.fqdn] | join('@', @)" --output tsv)
+#az acs kubernetes get-credentials --resource-group "${resourceGroup}" --name "${acs}" --file ./kubeconfig
+scp -o StrictHostKeyChecking=no "\$acs_host:.kube/config" kubeconfig
+if kubectl --kubeconfig kubeconfig get services --selector='app=tomcat,role=green' --no-headers --output json | jq -r '.items[].metadata.name' | grep -e '^tomcat-service\$'; then
+    echo "Current environment: green"
+    echo green >current-environment
+else
+    echo "Current environment: blue"
+    echo blue >current-environment
+fi
+"""
+            }
+        }
+    }
+
+    stage('Deploy') {
+        // here we always deploy the same Kubernetes configuration to the same deployment, but this is only for a deployment demonstration.
+        // In the real world projects, we would update the image with a new version and upgrade the corresponding deployment.
+        currentEnvironment = readFile('current-environment').trim()
+
+        echo "***************************  CURRENT: $currentEnvironment     NEW: ${newEnvironment()}  *****************************"
+
+        currentBuild.displayName = newEnvironment().toUpperCase() + ' Deployment'
+        
+        acsDeploy azureCredentialsId: 'sp',
+                  configFilePaths: "k8s/deployment-${newEnvironment()}.yml",
+                  containerService: "$acs | Kubernetes",
+                  resourceGroupName: resourceGroup,
+                  sshCredentialsId: 'k8s-ssh'
+    }
+
+    stage('Verify Staged') {
+        sh """
+count=0
+while true; do
+    count=\$(expr \$count + 1)
+    test_ip=\$(kubectl --kubeconfig=kubeconfig get services --selector='app=tomcat,role=test-${newEnvironment()}' --output json | jq -r '.items[0].status.loadBalancer.ingress[0].ip')
+    if [ "\$test_ip" != null ]; then
+        echo 'Test Server IP: \$test_ip'
+        break
+    fi
+    if [ "\$count" -gt 30 ]; then
+        echo 'Timeout while waiting for the test server IP'
+        exit -1
+    fi
+    echo "Test server IP not ready, sleep 10 seconds..."
+    sleep 10
+done
+count=0
+while true; do
+    count=\$(expr \$count + 1)
+    if curl "http://\$test_ip" | grep -o "Apache Tomcat/${expectedTomcatVersion()}"; then
+        break;
+    fi
+    if [ "\$count" -gt 30 ]; then
+        echo 'Timeout while waiting for the staging environment to be ready'
+        exit -1
+    fi
+    echo "Staging environment not ready, wait 10 seconds..."
+    sleep 10
+done
+"""
+    }
+
+    stage('Switch') {
+        acsDeploy azureCredentialsId: 'sp',
+                  configFilePaths: "k8s/service-${newEnvironment()}.yml",
+                  containerService: "$acs | Kubernetes",
+                  resourceGroupName: resourceGroup,
+                  sshCredentialsId: 'k8s-ssh'
+    }
+
+    stage('Verify Prod') {
+        sh """
+count=0
+while true; do
+    count=\$(expr \$count + 1)
+    prod_ip=\$(kubectl --kubeconfig=kubeconfig get services --selector='app=tomcat,env=prod' --output json | jq -r '.items[0].status.loadBalancer.ingress[0].ip')
+    if [ "\$prod_ip" != null ]; then
+        echo 'Production Server IP: \$prod_ip'
+        break
+    fi
+    if [ "\$count" -gt 30 ]; then
+        echo 'Timeout while waiting for the production server IP'
+        exit -1
+    fi
+    echo "Production server IP not ready, sleep 10 seconds..."
+    sleep 10
+done
+count=0
+while true; do
+    count=\$(expr \$count + 1)
+    if curl "http://\$prod_ip" | grep -o "Apache Tomcat/${expectedTomcatVersion()}"; then
+        break;
+    fi
+    if [ "\$count" -gt 30 ]; then
+        echo 'Timeout while waiting for the prod environment to be ready'
+        exit -1
+    fi
+    echo "Prod environment not ready, wait 10 seconds..."
+    sleep 10
+done
+"""
+    }
+}
+]]></script>
+    <sandbox>true</sandbox>
+  </definition>
+  <triggers/>
+  <disabled>false</disabled>
+</flow-definition>
diff --git a/jenkins/blue-green/add-blue-green-job.sh b/jenkins/blue-green/add-blue-green-job.sh
new file mode 100644
index 0000000..a26e2b2
--- /dev/null
+++ b/jenkins/blue-green/add-blue-green-job.sh
@@ -0,0 +1,225 @@
+#!/bin/bash
+
+function print_usage() {
+  cat <<EOF
+Command
+  $0
+Arguments
+  --jenkins_url|-j                [Required]: Jenkins URL
+  --jenkins_username|-ju          [Required]: Jenkins user name
+  --jenkins_password|-jp                    : Jenkins password. If not specified and the user name is "admin", the initialAdminPassword will be used
+  --acs_resource_group|-ag        [Required]: Resource group for the target ACS with Kubernetes orchestrator
+  --acs_name|-an                  [Required]: Name of the ACS cluster
+  --ssh_credentials_id|-sci                 : Desired Jenkins SSH credentials ID
+  --ssh_credentials_desc|-scd               : Desired Jenkins SSH credentials description
+  --ssh_credentials_username|-scu [Required]: Username for the SSH credentials
+  --ssh_credentials_key_file|-scp [Required]: Private key file for the SSH credentials
+  --sp_credentials_id|-spi                  : Desired Jenkins Azure service principal ID
+  --sp_credentials_desc|-spd                : Desired Jenkins Azure service princiapl description
+  --sp_subscription_id|-sps       [Required]: Subscription ID for the Azure service principal
+  --sp_client_id|-spc             [Required]: Client ID for the Azure service principal
+  --sp_client_password|-spp       [Required]: Client secrets for the Azure service principal
+  --sp_tenant_id|-spt             [Required]: Tenant ID for the Azure service principal
+  --sp_environment|-spe                     : Azure environment for the Azure service principal
+  --job_short_name|-jsn                     : Desired Jenkins job short name
+  --job_display_name|-jdn                   : Desired Jenkins job display name
+  --job_description|-jd                     : Desired Jenkins job description
+  --artifacts_location|-al                  : Url used to reference other scripts/artifacts.
+  --sas_token|-st                           : A sas token needed if the artifacts location is private.
+EOF
+}
+
+function throw_if_empty() {
+  local name="$1"
+  local value="$2"
+  if [ -z "$value" ]; then
+    echo "Parameter '$name' cannot be empty." 1>&2
+    print_usage
+    exit -1
+  fi
+}
+
+function run_util_script() {
+  local script_path="$1"
+  shift
+  curl --silent "${artifacts_location}${script_path}${artifacts_location_sas_token}" | sudo bash -s -- "$@"
+  local return_value=$?
+  if [ $return_value -ne 0 ]; then
+    >&2 echo "Failed while executing script '$script_path'."
+    exit $return_value
+  fi
+}
+
+#set defaults
+ssh_credentials_id="k8s-ssh"
+ssh_credentials_desc="SSH credentials to login to ACS Kubernetes master"
+sp_credentials_id="sp"
+sp_credentials_desc="Service Principal to manage Azure resources"
+sp_environment="Azure"
+job_short_name="acs-k8s-blue-green-deployment"
+job_display_name="ACS Kubernetes Blue-green Deployment"
+job_description="A pipeline that demonstrates the blue-green deployment to ACS Kubernetes with the azure-acs Jenkins plugin."
+artifacts_location="https://raw.githubusercontent.com/Azure/azure-devops-utils/master/"
+
+while [[ $# > 0 ]]
+do
+  key="$1"
+  shift
+  case $key in
+    --jenkins_url|-j)
+      jenkins_url="$1"
+      shift
+      ;;
+    --jenkins_username|-ju)
+      jenkins_username="$1"
+      shift
+      ;;
+    --jenkins_password|-jp)
+      jenkins_password="$1"
+      shift
+      ;;
+    --acs_resource_group|-ag)
+      acs_resource_group="$1"
+      shift
+      ;;
+    --acs_name|-an)
+      acs_name="$1"
+      shift
+      ;;
+    --ssh_credentials_id|-sci)
+      ssh_credentials_id="$1"
+      shift
+      ;;
+    --ssh_credentials_desc|-scd)
+      ssh_credentials_desc="$1"
+      shift
+      ;;
+    --ssh_credentials_username|-scu)
+      ssh_credentials_username="$1"
+      shift
+      ;;
+    --ssh_credentials_key_file|-scp)
+      ssh_credentials_key_file="$1"
+      shift
+      ;;
+    --sp_credentials_id|-spi)
+      sp_credentials_id="$1"
+      shift
+      ;;
+    --sp_credentials_desc|-spd)
+      sp_credentials_desc="$1"
+      shift
+      ;;
+    --sp_subscription_id|-sps)
+      sp_subscription_id="$1"
+      shift
+      ;;
+    --sp_client_id|-spc)
+      sp_client_id="$1"
+      shift
+      ;;
+    --sp_client_password|-spp)
+      sp_client_password="$1"
+      shift
+      ;;
+    --sp_tenant_id|-spt)
+      sp_tenant_id="$1"
+      shift
+      ;;
+    --sp_environment|-spe)
+      sp_environment="$1"
+      shift
+      ;;
+    --job_short_name|-jsn)
+      job_short_name="$1"
+      shift
+      ;;
+    --job_display_name|-jdn)
+      job_display_name="$1"
+      shift
+      ;;
+    --job_description|-jd)
+      job_description="$1"
+      shift
+      ;;
+    --artifacts_location|-al)
+      artifacts_location="$1"
+      shift
+      ;;
+    --sas_token|-st)
+      artifacts_location_sas_token="$1"
+      shift
+      ;;
+    --help|-help|-h)
+      print_usage
+      exit 13
+      ;;
+    *)
+      echo "ERROR: Unknown argument '$key' to script '$0'" 1>&2
+      exit -1
+  esac
+done
+
+throw_if_empty --jenkins_url "$jenkins_url"
+throw_if_empty --jenkins_username "$jenkins_username"
+if [ "$jenkins_username" != "admin" ]; then
+  throw_if_empty --jenkins_password "$jenkins_password"
+fi
+throw_if_empty --acs_resource_group "$acs_resource_group"
+throw_if_empty --acs_name "$acs_name"
+throw_if_empty --ssh_credentials_id "$ssh_credentials_id"
+throw_if_empty --ssh_credentials_username "$ssh_credentials_username"
+throw_if_empty --ssh_credentials_key_file "$ssh_credentials_key_file"
+if [ ! -f "$ssh_credentials_key_file" ]; then
+    echo "ERROR: Cannot find SSH key file $ssh_credentials_key_file" >&2
+    exit -1
+fi
+ssh_credentials_private_key=$(cat "$ssh_credentials_key_file")
+throw_if_empty "SSH private key" "$ssh_credentials_private_key"
+throw_if_empty --sp_credentials_id "$sp_credentials_id"
+throw_if_empty --sp_subscription_id "$sp_subscription_id"
+throw_if_empty --sp_client_id "$sp_client_id"
+throw_if_empty --sp_client_password "$sp_client_password"
+throw_if_empty --sp_tenant_id "$sp_tenant_id"
+throw_if_empty --sp_environment "$sp_environment"
+
+#download dependencies
+job_xml=$(curl -s ${artifacts_location}/jenkins/blue-green/acs-k8s-blue-green-job.xml${artifacts_location_sas_token})
+ssh_credentials_xml=$(curl -s ${artifacts_location}/jenkins/blue-green/ssh-credentials.xml${artifacts_location_sas_token})
+sp_credentials_xml=$(curl -s ${artifacts_location}/jenkins/blue-green/sp-credentials.xml${artifacts_location_sas_token})
+
+#prepare job.xml
+job_xml=${job_xml//'{insert-job-display-name}'/${job_display_name}}
+job_xml=${job_xml//'{insert-job-description}'/${job_description}}
+job_xml=${job_xml//'{insert-acs-resource-group}'/${acs_resource_group}}
+job_xml=${job_xml//'{insert-acs-name}'/${acs_name}}
+
+#prepare ssh-credentials.xml
+ssh_credentials_xml=${ssh_credentials_xml//'{insert-ssh-credentials-id}'/${ssh_credentials_id}}
+ssh_credentials_xml=${ssh_credentials_xml//'{insert-ssh-credentials-desc}'/${ssh_credentials_desc}}
+ssh_credentials_xml=${ssh_credentials_xml//'{insert-ssh-username}'/${ssh_credentials_username}}
+ssh_credentials_xml=${ssh_credentials_xml//'{insert-ssh-private-key}'/${ssh_credentials_private_key}}
+
+#prepare sp-credentials.xml
+sp_credentials_xml=${sp_credentials_xml//'{insert-sp-credentials-id}'/${sp_credentials_id}}
+sp_credentials_xml=${sp_credentials_xml//'{insert-sp-credentials-desc}'/${sp_credentials_desc}}
+sp_credentials_xml=${sp_credentials_xml//'{insert-sp-subscription-id}'/${sp_subscription_id}}
+sp_credentials_xml=${sp_credentials_xml//'{insert-sp-client-id}'/${sp_client_id}}
+sp_credentials_xml=${sp_credentials_xml//'{insert-sp-client-password}'/${sp_client_password}}
+sp_credentials_xml=${sp_credentials_xml//'{insert-sp-tenant-id}'/${sp_tenant_id}}
+sp_credentials_xml=${sp_credentials_xml//'{insert-sp-environment}'/${sp_environment}}
+
+#add SSH credentials
+echo "${ssh_credentials_xml}" >ssh-credentials.xml
+run_util_script "jenkins/run-cli-command.sh" -j "$jenkins_url" -ju "$jenkins_username" -jp "$jenkins_password" -c 'create-credentials-by-xml SystemCredentialsProvider::SystemContextResolver::jenkins (global)' -cif "ssh-credentials.xml"
+
+#add Azure service principal credentials
+echo "${sp_credentials_xml}" >sp-credentials.xml
+run_util_script "jenkins/run-cli-command.sh" -j "$jenkins_url" -ju "$jenkins_username" -jp "$jenkins_password" -c 'create-credentials-by-xml SystemCredentialsProvider::SystemContextResolver::jenkins (global)' -cif "sp-credentials.xml"
+
+#add job
+echo "${job_xml}" >job.xml
+run_util_script "jenkins/run-cli-command.sh" -j "$jenkins_url" -ju "$jenkins_username" -jp "$jenkins_password" -c "create-job ${job_short_name}" -cif "job.xml"
+
+# clean up
+rm -f ssh-credentials.xml sp-credentials.xml job.xml
diff --git a/jenkins/blue-green/sp-credentials.xml b/jenkins/blue-green/sp-credentials.xml
new file mode 100644
index 0000000..eaa17f3
--- /dev/null
+++ b/jenkins/blue-green/sp-credentials.xml
@@ -0,0 +1,12 @@
+<com.microsoft.azure.util.AzureCredentials plugin="azure-credentials@1.3.1">
+    <scope>GLOBAL</scope>
+    <id>{insert-sp-credentials-id}</id>
+    <description>{insert-sp-credentials-desc}</description>
+    <data>
+        <subscriptionId>{insert-sp-subscription-id}</subscriptionId>
+        <clientId>{insert-sp-client-id}</clientId>
+        <clientSecret>{insert-sp-client-password}</clientSecret>
+        <tenant>{insert-sp-tenant-id}</tenant>
+        <azureEnvironmentName>{insert-sp-environment}</azureEnvironmentName>
+    </data>
+</com.microsoft.azure.util.AzureCredentials>
\ No newline at end of file
diff --git a/jenkins/blue-green/ssh-credentials.xml b/jenkins/blue-green/ssh-credentials.xml
new file mode 100644
index 0000000..a2c50cf
--- /dev/null
+++ b/jenkins/blue-green/ssh-credentials.xml
@@ -0,0 +1,9 @@
+<com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey plugin="ssh-credentials@1.13">
+    <scope>GLOBAL</scope>
+    <id>{insert-ssh-credentials-id}</id>
+    <description>{insert-ssh-credentials-desc}</description>
+    <username>{insert-ssh-username}</username>
+    <privateKeySource class="com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey$DirectEntryPrivateKeySource">
+        <privateKey>{insert-ssh-private-key}</privateKey>
+    </privateKeySource>
+</com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey>
\ No newline at end of file
diff --git a/jenkins/install_jenkins.sh b/jenkins/install_jenkins.sh
index 4c57785..1771be5 100644
--- a/jenkins/install_jenkins.sh
+++ b/jenkins/install_jenkins.sh
@@ -44,6 +44,20 @@ function run_util_script() {
   fi
 }
 
+function retry_until_successful {
+  counter=0
+  "${@}"
+  while [ $? -ne 0 ]; do
+    if [[ "$counter" -gt 20 ]]; then
+        exit 1
+    else
+        let counter++
+    fi
+    sleep 5
+    "${@}"
+  done;
+}
+
 #defaults
 artifacts_location="https://raw.githubusercontent.com/Azure/azure-devops-utils/master/"
 jenkins_version_location="https://raw.githubusercontent.com/Azure/azure-devops-utils/master/jenkins/jenkins-verified-ver"
@@ -263,6 +277,9 @@ else
   sudo apt-get install jenkins --yes # sometime the first apt-get install jenkins command fails, so we try it twice
 fi
 
+retry_until_successful sudo test -f /var/lib/jenkins/secrets/initialAdminPassword
+retry_until_successful run_util_script "jenkins/run-cli-command.sh" -c "version"
+
 #We need to install workflow-aggregator so all the options in the auth matrix are valid
 plugins=(azure-vm-agents windows-azure-storage matrix-auth workflow-aggregator azure-app-service tfs azure-acs azure-container-agents)
 for plugin in "${plugins[@]}"; do
@@ -320,6 +337,9 @@ sp_cred=$(cat <<EOF
 </com.microsoft.azure.util.AzureCredentials>
 EOF
 )
+
+retry_until_successful run_util_script "jenkins/run-cli-command.sh" -c "version"
+
 if [ "${service_principal_type}" == 'msi' ]; then
   echo "${msi_cred}" > msi_cred.xml
   run_util_script "jenkins/run-cli-command.sh" -c "create-credentials-by-xml system::system::jenkins _" -cif msi_cred.xml
diff --git a/quickstart_template/301-jenkins-k8s-blue-green.sh b/quickstart_template/301-jenkins-k8s-blue-green.sh
new file mode 100644
index 0000000..e1fdd43
--- /dev/null
+++ b/quickstart_template/301-jenkins-k8s-blue-green.sh
@@ -0,0 +1,202 @@
+#!/bin/bash
+
+function print_usage() {
+  cat <<EOF
+https://github.com/Azure/azure-quickstart-templates/tree/master/301-jenkins-k8s-blue-green
+Command
+  $0
+Arguments
+  --app_id|-ai                       [Required] : Service principal app id  used to dynamically manage resource in your subscription
+  --app_key|-ak                      [Required] : Service principal app key used to dynamically manage resource in your subscription
+  --subscription_id|-si              [Required] : Subscription Id
+  --tenant_id|-ti                    [Required] : Tenant Id
+  --resource_group|-rg               [Required] : Resource group containing your Kubernetes cluster
+  --acs_name|-an                     [Required] : Name of the ACS cluster with Kubernetes orchestrator
+  --jenkins_fqdn|-jf                 [Required] : Jenkins FQDN
+  --artifacts_location|-al                      : Url used to reference other scripts/artifacts.
+  --sas_token|-st                               : A sas token needed if the artifacts location is private.
+EOF
+}
+
+function throw_if_empty() {
+  local name="$1"
+  local value="$2"
+  if [ -z "$value" ]; then
+    echo "Parameter '$name' cannot be empty." 1>&2
+    print_usage
+    exit -1
+  fi
+}
+
+function run_util_script() {
+  local script_path="$1"
+  shift
+  curl --silent "${artifacts_location}${script_path}${artifacts_location_sas_token}" | sudo bash -s -- "$@"
+  local return_value=$?
+  if [ $return_value -ne 0 ]; then
+    >&2 echo "Failed while executing script '$script_path'."
+    exit $return_value
+  fi
+}
+
+function install_kubectl() {
+  if !(command -v kubectl >/dev/null); then
+    kubectl_file="/usr/local/bin/kubectl"
+    sudo curl -L -s -o $kubectl_file https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
+    sudo chmod +x $kubectl_file
+  fi
+}
+
+function install_az() {
+  if !(command -v az >/dev/null); then
+    sudo apt-get update && sudo apt-get install -y libssl-dev libffi-dev python-dev
+    echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/azure-cli/ wheezy main" | sudo tee /etc/apt/sources.list.d/azure-cli.list
+    sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893
+    sudo apt-get install -y apt-transport-https
+    sudo apt-get -y update && sudo apt-get install -y azure-cli
+  fi
+}
+
+function allow_acs_nsg_access()
+{
+  local source_ip=$1
+  local resource_group=$2
+
+  local nsgs=($(az network nsg list --resource-group "$resource_group" --query '[].name' --output tsv | grep -e "^k8s-master-"))
+  local port_range=22
+  if [ "$source_ip" = Internet ]; then
+    # web job deletes the rule if the port is set to 22 for wildcard internet access
+    port_range="21-23"
+  fi
+  for nsg in "${nsgs[@]}"; do
+    local name="allow_$source_ip"
+    # used a fixed priority here
+    local max_priority="$(az network nsg rule list -g "$resource_group" --nsg-name "$nsg" --query '[].priority' --output tsv | sort -n | tail -n1)"
+    local priority="$(expr "$max_priority" + 50)"
+    log_info "Add allow $source_ip rules to NSG $nsg in resource group $resource_group, with priority $priority"
+    az network nsg rule create --priority "$priority" --destination-port-ranges "$port_range" --resource-group "$resource_group" \
+        --nsg-name "$nsg" --name "$name" --source-address-prefixes "$source_ip"
+    #az network nsg rule create --priority "$priority" --destination-port-ranges 22 --resource-group "$resource_group" \
+    #    --nsg-name "$nsg" --name "$name" --source-address-prefixes "$source_ip"
+  done
+}
+
+artifacts_location="https://raw.githubusercontent.com/Azure/azure-devops-utils/master/"
+
+while [[ $# > 0 ]]
+do
+  key="$1"
+  shift
+  case "$key" in
+    --app_id|-ai)
+      app_id="$1"
+      shift
+      ;;
+    --app_key|-ak)
+      app_key="$1"
+      shift
+      ;;
+    --subscription_id|-si)
+      subscription_id="$1"
+      shift
+      ;;
+    --tenant_id|-ti)
+      tenant_id="$1"
+      shift
+      ;;
+    --resource_group|-rg)
+      resource_group="$1"
+      shift
+      ;;
+    --acs_name|-an)
+      acs_name="$1"
+      shift
+      ;;
+    --jenkins_fqdn|-jf)
+      jenkins_fqdn="$1"
+      shift
+      ;;
+    --artifacts_location|-al)
+      artifacts_location="$1"
+      shift
+      ;;
+    --sas_token|-st)
+      artifacts_location_sas_token="$1"
+      shift
+      ;;
+    --help|-help|-h)
+      print_usage
+      exit 13
+      ;;
+    *)
+      echo "ERROR: Unknown argument '$key' to script '$0'" 1>&2
+      exit -1
+  esac
+done
+
+throw_if_empty --app_id "$app_id"
+throw_if_empty --app_key "$app_key"
+throw_if_empty --subscription_id "$subscription_id"
+throw_if_empty --tenant_id "$tenant_id"
+throw_if_empty --resource_group "$resource_group"
+throw_if_empty --acs_name "$acs_name"
+throw_if_empty --jenkins_fqdn "$jenkins_fqdn"
+
+install_kubectl
+
+install_az
+
+sudo apt-get install --yes jq
+
+az login --service-principal -u "$app_id" -p "$app_key" --tenant "$tenant_id"
+az account set --subscription "$subscription_id"
+master_fqdn="$(az acs show --resource-group "$resource_group" --name "$acs_name" --query masterProfile.fqdn --output tsv)"
+master_username="$(az acs show --resource-group "$resource_group" --name "$acs_name" --query linuxProfile.adminUsername --output tsv)"
+
+temp_user_name="$(uuidgen | sed 's/-//g')"
+temp_key_path="$(mktemp -d)/temp_key"
+ssh-keygen -t rsa -N "" -f "$temp_key_path"
+temp_pub_key="${temp_key_path}.pub"
+
+# Allow Jenkins master to access the ACS K8s master via SSH
+jenkins_ip=($(dig +short "$jenkins_fqdn"))
+for ip in "${jenkins_ip[@]}"; do
+  [[ -z "$ip" ]] && continue
+  allow_acs_nsg_access "$ip" "$resource_group"
+done
+
+master_vm_ids=$(az vm list -g "$resource_group" --query "[].id" -o tsv | grep "k8s-master-")
+>&2 echo "Master VM ids: $master_vm_ids"
+
+# Add the generated SSH public key to the authroized keys for the Kubernetes master admin user in two steps:
+#   1. add a temporary user using Azure CLI with the generated username and public key
+#   2. login with the temporary user, and append its .ssh/authorized_keys which is the generated public key to the master user's authorized_keys list.
+# this will be used in Jenkins to authenticate with the Kubernetes master node via SSH
+az vm user update -u "$temp_user_name" --ssh-key-value "$temp_pub_key" --ids "$master_vm_ids"
+ssh -o StrictHostKeyChecking=no -i "$temp_key_path" "$temp_user_name@$master_fqdn" "[ -d '/home/$master_username' ] && (cat .ssh/authorized_keys | sudo tee -a /home/$master_username/.ssh/authorized_keys)"
+
+# Remove temporary credentials on every kubernetes master vm
+az vm user delete -u "$temp_user_name" --ids "$master_vm_ids"
+az logout
+
+#install jenkins
+run_util_script "jenkins/install_jenkins.sh" -jf "${jenkins_fqdn}" -al "${artifacts_location}" -st "${artifacts_location_sas_token}"
+
+run_util_script "jenkins/run-cli-command.sh" -c "install-plugin ssh-agent -deploy"
+
+run_util_script "jenkins/blue-green/add-blue-green-job.sh" \
+    -j "http://localhost:8080/" \
+    -ju "admin" \
+    --acs_resource_group "$resource_group" \
+    --acs_name "$acs_name" \
+    --ssh_credentials_username "$master_username" \
+    --ssh_credentials_key_file "$temp_key_path" \
+    --sp_subscription_id "$subscription_id" \
+    --sp_client_id "$app_id" \
+    --sp_client_password "$app_key" \
+    --sp_tenant_id "$tenant_id" \
+    --artifacts_location "$artifacts_location" \
+    --sas_token "$artifacts_location_sas_token"
+
+rm -f "$temp_key_path"
+rm -f "$temp_pub_key"