diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6ef72f8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea/ +*.iml + diff --git a/jenkins/blue-green/acs-k8s-blue-green-job.xml b/jenkins/blue-green/acs-k8s-blue-green-job.xml new file mode 100644 index 0000000..d9d3de8 --- /dev/null +++ b/jenkins/blue-green/acs-k8s-blue-green-job.xml @@ -0,0 +1,282 @@ + + + + {insert-job-description} + {insert-job-display-name} + false + + + + + + + + true + + + false + diff --git a/jenkins/blue-green/add-blue-green-job.sh b/jenkins/blue-green/add-blue-green-job.sh new file mode 100644 index 0000000..a26e2b2 --- /dev/null +++ b/jenkins/blue-green/add-blue-green-job.sh @@ -0,0 +1,225 @@ +#!/bin/bash + +function print_usage() { + cat <&2 + print_usage + exit -1 + fi +} + +function run_util_script() { + local script_path="$1" + shift + curl --silent "${artifacts_location}${script_path}${artifacts_location_sas_token}" | sudo bash -s -- "$@" + local return_value=$? + if [ $return_value -ne 0 ]; then + >&2 echo "Failed while executing script '$script_path'." + exit $return_value + fi +} + +#set defaults +ssh_credentials_id="k8s-ssh" +ssh_credentials_desc="SSH credentials to login to ACS Kubernetes master" +sp_credentials_id="sp" +sp_credentials_desc="Service Principal to manage Azure resources" +sp_environment="Azure" +job_short_name="acs-k8s-blue-green-deployment" +job_display_name="ACS Kubernetes Blue-green Deployment" +job_description="A pipeline that demonstrates the blue-green deployment to ACS Kubernetes with the azure-acs Jenkins plugin." +artifacts_location="https://raw.githubusercontent.com/Azure/azure-devops-utils/master/" + +while [[ $# > 0 ]] +do + key="$1" + shift + case $key in + --jenkins_url|-j) + jenkins_url="$1" + shift + ;; + --jenkins_username|-ju) + jenkins_username="$1" + shift + ;; + --jenkins_password|-jp) + jenkins_password="$1" + shift + ;; + --acs_resource_group|-ag) + acs_resource_group="$1" + shift + ;; + --acs_name|-an) + acs_name="$1" + shift + ;; + --ssh_credentials_id|-sci) + ssh_credentials_id="$1" + shift + ;; + --ssh_credentials_desc|-scd) + ssh_credentials_desc="$1" + shift + ;; + --ssh_credentials_username|-scu) + ssh_credentials_username="$1" + shift + ;; + --ssh_credentials_key_file|-scp) + ssh_credentials_key_file="$1" + shift + ;; + --sp_credentials_id|-spi) + sp_credentials_id="$1" + shift + ;; + --sp_credentials_desc|-spd) + sp_credentials_desc="$1" + shift + ;; + --sp_subscription_id|-sps) + sp_subscription_id="$1" + shift + ;; + --sp_client_id|-spc) + sp_client_id="$1" + shift + ;; + --sp_client_password|-spp) + sp_client_password="$1" + shift + ;; + --sp_tenant_id|-spt) + sp_tenant_id="$1" + shift + ;; + --sp_environment|-spe) + sp_environment="$1" + shift + ;; + --job_short_name|-jsn) + job_short_name="$1" + shift + ;; + --job_display_name|-jdn) + job_display_name="$1" + shift + ;; + --job_description|-jd) + job_description="$1" + shift + ;; + --artifacts_location|-al) + artifacts_location="$1" + shift + ;; + --sas_token|-st) + artifacts_location_sas_token="$1" + shift + ;; + --help|-help|-h) + print_usage + exit 13 + ;; + *) + echo "ERROR: Unknown argument '$key' to script '$0'" 1>&2 + exit -1 + esac +done + +throw_if_empty --jenkins_url "$jenkins_url" +throw_if_empty --jenkins_username "$jenkins_username" +if [ "$jenkins_username" != "admin" ]; then + throw_if_empty --jenkins_password "$jenkins_password" +fi +throw_if_empty --acs_resource_group "$acs_resource_group" +throw_if_empty --acs_name "$acs_name" +throw_if_empty --ssh_credentials_id "$ssh_credentials_id" +throw_if_empty --ssh_credentials_username "$ssh_credentials_username" +throw_if_empty --ssh_credentials_key_file "$ssh_credentials_key_file" +if [ ! -f "$ssh_credentials_key_file" ]; then + echo "ERROR: Cannot find SSH key file $ssh_credentials_key_file" >&2 + exit -1 +fi +ssh_credentials_private_key=$(cat "$ssh_credentials_key_file") +throw_if_empty "SSH private key" "$ssh_credentials_private_key" +throw_if_empty --sp_credentials_id "$sp_credentials_id" +throw_if_empty --sp_subscription_id "$sp_subscription_id" +throw_if_empty --sp_client_id "$sp_client_id" +throw_if_empty --sp_client_password "$sp_client_password" +throw_if_empty --sp_tenant_id "$sp_tenant_id" +throw_if_empty --sp_environment "$sp_environment" + +#download dependencies +job_xml=$(curl -s ${artifacts_location}/jenkins/blue-green/acs-k8s-blue-green-job.xml${artifacts_location_sas_token}) +ssh_credentials_xml=$(curl -s ${artifacts_location}/jenkins/blue-green/ssh-credentials.xml${artifacts_location_sas_token}) +sp_credentials_xml=$(curl -s ${artifacts_location}/jenkins/blue-green/sp-credentials.xml${artifacts_location_sas_token}) + +#prepare job.xml +job_xml=${job_xml//'{insert-job-display-name}'/${job_display_name}} +job_xml=${job_xml//'{insert-job-description}'/${job_description}} +job_xml=${job_xml//'{insert-acs-resource-group}'/${acs_resource_group}} +job_xml=${job_xml//'{insert-acs-name}'/${acs_name}} + +#prepare ssh-credentials.xml +ssh_credentials_xml=${ssh_credentials_xml//'{insert-ssh-credentials-id}'/${ssh_credentials_id}} +ssh_credentials_xml=${ssh_credentials_xml//'{insert-ssh-credentials-desc}'/${ssh_credentials_desc}} +ssh_credentials_xml=${ssh_credentials_xml//'{insert-ssh-username}'/${ssh_credentials_username}} +ssh_credentials_xml=${ssh_credentials_xml//'{insert-ssh-private-key}'/${ssh_credentials_private_key}} + +#prepare sp-credentials.xml +sp_credentials_xml=${sp_credentials_xml//'{insert-sp-credentials-id}'/${sp_credentials_id}} +sp_credentials_xml=${sp_credentials_xml//'{insert-sp-credentials-desc}'/${sp_credentials_desc}} +sp_credentials_xml=${sp_credentials_xml//'{insert-sp-subscription-id}'/${sp_subscription_id}} +sp_credentials_xml=${sp_credentials_xml//'{insert-sp-client-id}'/${sp_client_id}} +sp_credentials_xml=${sp_credentials_xml//'{insert-sp-client-password}'/${sp_client_password}} +sp_credentials_xml=${sp_credentials_xml//'{insert-sp-tenant-id}'/${sp_tenant_id}} +sp_credentials_xml=${sp_credentials_xml//'{insert-sp-environment}'/${sp_environment}} + +#add SSH credentials +echo "${ssh_credentials_xml}" >ssh-credentials.xml +run_util_script "jenkins/run-cli-command.sh" -j "$jenkins_url" -ju "$jenkins_username" -jp "$jenkins_password" -c 'create-credentials-by-xml SystemCredentialsProvider::SystemContextResolver::jenkins (global)' -cif "ssh-credentials.xml" + +#add Azure service principal credentials +echo "${sp_credentials_xml}" >sp-credentials.xml +run_util_script "jenkins/run-cli-command.sh" -j "$jenkins_url" -ju "$jenkins_username" -jp "$jenkins_password" -c 'create-credentials-by-xml SystemCredentialsProvider::SystemContextResolver::jenkins (global)' -cif "sp-credentials.xml" + +#add job +echo "${job_xml}" >job.xml +run_util_script "jenkins/run-cli-command.sh" -j "$jenkins_url" -ju "$jenkins_username" -jp "$jenkins_password" -c "create-job ${job_short_name}" -cif "job.xml" + +# clean up +rm -f ssh-credentials.xml sp-credentials.xml job.xml diff --git a/jenkins/blue-green/sp-credentials.xml b/jenkins/blue-green/sp-credentials.xml new file mode 100644 index 0000000..eaa17f3 --- /dev/null +++ b/jenkins/blue-green/sp-credentials.xml @@ -0,0 +1,12 @@ + + GLOBAL + {insert-sp-credentials-id} + {insert-sp-credentials-desc} + + {insert-sp-subscription-id} + {insert-sp-client-id} + {insert-sp-client-password} + {insert-sp-tenant-id} + {insert-sp-environment} + + \ No newline at end of file diff --git a/jenkins/blue-green/ssh-credentials.xml b/jenkins/blue-green/ssh-credentials.xml new file mode 100644 index 0000000..a2c50cf --- /dev/null +++ b/jenkins/blue-green/ssh-credentials.xml @@ -0,0 +1,9 @@ + + GLOBAL + {insert-ssh-credentials-id} + {insert-ssh-credentials-desc} + {insert-ssh-username} + + {insert-ssh-private-key} + + \ No newline at end of file diff --git a/jenkins/install_jenkins.sh b/jenkins/install_jenkins.sh index 4c57785..1771be5 100644 --- a/jenkins/install_jenkins.sh +++ b/jenkins/install_jenkins.sh @@ -44,6 +44,20 @@ function run_util_script() { fi } +function retry_until_successful { + counter=0 + "${@}" + while [ $? -ne 0 ]; do + if [[ "$counter" -gt 20 ]]; then + exit 1 + else + let counter++ + fi + sleep 5 + "${@}" + done; +} + #defaults artifacts_location="https://raw.githubusercontent.com/Azure/azure-devops-utils/master/" jenkins_version_location="https://raw.githubusercontent.com/Azure/azure-devops-utils/master/jenkins/jenkins-verified-ver" @@ -263,6 +277,9 @@ else sudo apt-get install jenkins --yes # sometime the first apt-get install jenkins command fails, so we try it twice fi +retry_until_successful sudo test -f /var/lib/jenkins/secrets/initialAdminPassword +retry_until_successful run_util_script "jenkins/run-cli-command.sh" -c "version" + #We need to install workflow-aggregator so all the options in the auth matrix are valid plugins=(azure-vm-agents windows-azure-storage matrix-auth workflow-aggregator azure-app-service tfs azure-acs azure-container-agents) for plugin in "${plugins[@]}"; do @@ -320,6 +337,9 @@ sp_cred=$(cat < EOF ) + +retry_until_successful run_util_script "jenkins/run-cli-command.sh" -c "version" + if [ "${service_principal_type}" == 'msi' ]; then echo "${msi_cred}" > msi_cred.xml run_util_script "jenkins/run-cli-command.sh" -c "create-credentials-by-xml system::system::jenkins _" -cif msi_cred.xml diff --git a/quickstart_template/301-jenkins-k8s-blue-green.sh b/quickstart_template/301-jenkins-k8s-blue-green.sh new file mode 100644 index 0000000..e1fdd43 --- /dev/null +++ b/quickstart_template/301-jenkins-k8s-blue-green.sh @@ -0,0 +1,202 @@ +#!/bin/bash + +function print_usage() { + cat <&2 + print_usage + exit -1 + fi +} + +function run_util_script() { + local script_path="$1" + shift + curl --silent "${artifacts_location}${script_path}${artifacts_location_sas_token}" | sudo bash -s -- "$@" + local return_value=$? + if [ $return_value -ne 0 ]; then + >&2 echo "Failed while executing script '$script_path'." + exit $return_value + fi +} + +function install_kubectl() { + if !(command -v kubectl >/dev/null); then + kubectl_file="/usr/local/bin/kubectl" + sudo curl -L -s -o $kubectl_file https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl + sudo chmod +x $kubectl_file + fi +} + +function install_az() { + if !(command -v az >/dev/null); then + sudo apt-get update && sudo apt-get install -y libssl-dev libffi-dev python-dev + echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/azure-cli/ wheezy main" | sudo tee /etc/apt/sources.list.d/azure-cli.list + sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 + sudo apt-get install -y apt-transport-https + sudo apt-get -y update && sudo apt-get install -y azure-cli + fi +} + +function allow_acs_nsg_access() +{ + local source_ip=$1 + local resource_group=$2 + + local nsgs=($(az network nsg list --resource-group "$resource_group" --query '[].name' --output tsv | grep -e "^k8s-master-")) + local port_range=22 + if [ "$source_ip" = Internet ]; then + # web job deletes the rule if the port is set to 22 for wildcard internet access + port_range="21-23" + fi + for nsg in "${nsgs[@]}"; do + local name="allow_$source_ip" + # used a fixed priority here + local max_priority="$(az network nsg rule list -g "$resource_group" --nsg-name "$nsg" --query '[].priority' --output tsv | sort -n | tail -n1)" + local priority="$(expr "$max_priority" + 50)" + log_info "Add allow $source_ip rules to NSG $nsg in resource group $resource_group, with priority $priority" + az network nsg rule create --priority "$priority" --destination-port-ranges "$port_range" --resource-group "$resource_group" \ + --nsg-name "$nsg" --name "$name" --source-address-prefixes "$source_ip" + #az network nsg rule create --priority "$priority" --destination-port-ranges 22 --resource-group "$resource_group" \ + # --nsg-name "$nsg" --name "$name" --source-address-prefixes "$source_ip" + done +} + +artifacts_location="https://raw.githubusercontent.com/Azure/azure-devops-utils/master/" + +while [[ $# > 0 ]] +do + key="$1" + shift + case "$key" in + --app_id|-ai) + app_id="$1" + shift + ;; + --app_key|-ak) + app_key="$1" + shift + ;; + --subscription_id|-si) + subscription_id="$1" + shift + ;; + --tenant_id|-ti) + tenant_id="$1" + shift + ;; + --resource_group|-rg) + resource_group="$1" + shift + ;; + --acs_name|-an) + acs_name="$1" + shift + ;; + --jenkins_fqdn|-jf) + jenkins_fqdn="$1" + shift + ;; + --artifacts_location|-al) + artifacts_location="$1" + shift + ;; + --sas_token|-st) + artifacts_location_sas_token="$1" + shift + ;; + --help|-help|-h) + print_usage + exit 13 + ;; + *) + echo "ERROR: Unknown argument '$key' to script '$0'" 1>&2 + exit -1 + esac +done + +throw_if_empty --app_id "$app_id" +throw_if_empty --app_key "$app_key" +throw_if_empty --subscription_id "$subscription_id" +throw_if_empty --tenant_id "$tenant_id" +throw_if_empty --resource_group "$resource_group" +throw_if_empty --acs_name "$acs_name" +throw_if_empty --jenkins_fqdn "$jenkins_fqdn" + +install_kubectl + +install_az + +sudo apt-get install --yes jq + +az login --service-principal -u "$app_id" -p "$app_key" --tenant "$tenant_id" +az account set --subscription "$subscription_id" +master_fqdn="$(az acs show --resource-group "$resource_group" --name "$acs_name" --query masterProfile.fqdn --output tsv)" +master_username="$(az acs show --resource-group "$resource_group" --name "$acs_name" --query linuxProfile.adminUsername --output tsv)" + +temp_user_name="$(uuidgen | sed 's/-//g')" +temp_key_path="$(mktemp -d)/temp_key" +ssh-keygen -t rsa -N "" -f "$temp_key_path" +temp_pub_key="${temp_key_path}.pub" + +# Allow Jenkins master to access the ACS K8s master via SSH +jenkins_ip=($(dig +short "$jenkins_fqdn")) +for ip in "${jenkins_ip[@]}"; do + [[ -z "$ip" ]] && continue + allow_acs_nsg_access "$ip" "$resource_group" +done + +master_vm_ids=$(az vm list -g "$resource_group" --query "[].id" -o tsv | grep "k8s-master-") +>&2 echo "Master VM ids: $master_vm_ids" + +# Add the generated SSH public key to the authroized keys for the Kubernetes master admin user in two steps: +# 1. add a temporary user using Azure CLI with the generated username and public key +# 2. login with the temporary user, and append its .ssh/authorized_keys which is the generated public key to the master user's authorized_keys list. +# this will be used in Jenkins to authenticate with the Kubernetes master node via SSH +az vm user update -u "$temp_user_name" --ssh-key-value "$temp_pub_key" --ids "$master_vm_ids" +ssh -o StrictHostKeyChecking=no -i "$temp_key_path" "$temp_user_name@$master_fqdn" "[ -d '/home/$master_username' ] && (cat .ssh/authorized_keys | sudo tee -a /home/$master_username/.ssh/authorized_keys)" + +# Remove temporary credentials on every kubernetes master vm +az vm user delete -u "$temp_user_name" --ids "$master_vm_ids" +az logout + +#install jenkins +run_util_script "jenkins/install_jenkins.sh" -jf "${jenkins_fqdn}" -al "${artifacts_location}" -st "${artifacts_location_sas_token}" + +run_util_script "jenkins/run-cli-command.sh" -c "install-plugin ssh-agent -deploy" + +run_util_script "jenkins/blue-green/add-blue-green-job.sh" \ + -j "http://localhost:8080/" \ + -ju "admin" \ + --acs_resource_group "$resource_group" \ + --acs_name "$acs_name" \ + --ssh_credentials_username "$master_username" \ + --ssh_credentials_key_file "$temp_key_path" \ + --sp_subscription_id "$subscription_id" \ + --sp_client_id "$app_id" \ + --sp_client_password "$app_key" \ + --sp_tenant_id "$tenant_id" \ + --artifacts_location "$artifacts_location" \ + --sas_token "$artifacts_location_sas_token" + +rm -f "$temp_key_path" +rm -f "$temp_pub_key"