Add Packer (#3650)

* add packer

* rename packer builder

* az-login as a dependency

* remove extra )

* use $(MAKE)

* remove installFlexVolDrivers

.gitignore

@@ -33,4 +33,9 @@ pkg/openshift/translations/
 # test outputs
 cmd/_test_output
 
+# packer
+packer/settings.json
+packer/sp.json
+
 .idea
+.vs

.vsts-ci.yaml

@@ -1,8 +1,51 @@
 trigger: none
 
+# steps: 
+# - create an VHD in Packer to normal storage account
+# - copy from Packer storage account to classic storage account using AzCopy
+# - generate SAS link from azure CLI
+# - POST a new SKU to azure marketplace
+
 phases:
 - phase: build_vhd
   queue: Hosted Linux Preview
   steps:
-  - script: make info
+  - script: |
+      docker run --rm \
+      -v ${PWD}:/go/src/github.com/Azure/acs-engine \
+      -w /go/src/github.com/Azure/acs-engine \
+      -e CLIENT_ID=${CLIENT_ID} \
+      -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+      -e TENANT_ID=${TENANT_ID} \
+      -e AZURE_VM_SIZE=${AZURE_VM_SIZE} \
+      -e AZURE_RESOURCE_GROUP_NAME=${AZURE_RESOURCE_GROUP_NAME} \
+      -e AZURE_LOCATION=${AZURE_LOCATION} \
+      ${DEIS_GO_DEV_IMAGE} make run-packer
     displayName: Building VHD
+  - script: |
+      OS_DISK_SAS="$(cat packer-output | grep "OSDiskUriReadOnlySas:" | cut -d " " -f 2)" && \
+      VHD_NAME="$(echo $OS_DISK_SAS | cut -d "/" -f 8 | cut -d "?" -f 1)" && \
+      docker run --rm \
+      -v ${PWD}:/go/src/github.com/Azure/acs-engine \
+      -w /go/src/github.com/Azure/acs-engine \
+      -e CLIENT_ID=${CLIENT_ID} \
+      -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+      -e TENANT_ID=${TENANT_ID} \
+      -e CLASSIC_BLOB=${CLASSIC_BLOB} \
+      -e CLASSIC_SAS_TOKEN="$(SAS_TOKEN)" \
+      -e OS_DISK_SAS=${OS_DISK_SAS} \
+      -e VHD_NAME=${VHD_NAME} \
+      ${DEIS_GO_DEV_IMAGE} make az-copy
+    displayName: Copying resource to Classic Storage Account
+  - script: |
+      docker run --rm \
+      -v ${PWD}:/go/src/github.com/Azure/acs-engine \
+      -w /go/src/github.com/Azure/acs-engine \
+      -e CLIENT_ID=${CLIENT_ID} \
+      -e CLIENT_SECRET="$(CLIENT_SECRET)" \
+      -e TENANT_ID=${TENANT_ID} \
+      -e CLASSIC_SA_CONNECTION_STRING="$(CLASSIC_SA_CONNECTION_STRING)" \
+      -e START_DATE=${START_DATE} \
+      -e EXPIRY_DATE=${EXPIRY_DATE} \
+      ${DEIS_GO_DEV_IMAGE} make generate-sas
+    displayName: Getting Shared Access Signature URI

Makefile

@@ -153,3 +153,4 @@ devenv:
 
 include versioning.mk
 include test.mk
+include packer.mk

packer.mk

@@ -0,0 +1,17 @@
+build-packer:
+	@packer build -var-file=packer/settings.json packer/vhd-image-builder.json
+
+init-packer:
+	@./packer/init-variables
+
+az-login:
+	az login --service-principal -u ${CLIENT_ID} -p ${CLIENT_SECRET} --tenant ${TENANT_ID}
+
+run-packer: az-login
+	@packer version && $(MAKE) init-packer && ($(MAKE) build-packer | tee packer-output)
+
+az-copy: az-login
+	azcopy --source "${OS_DISK_SAS}" --destination "${CLASSIC_BLOB}/${VHD_NAME}" --dest-sas "${CLASSIC_SAS_TOKEN}"
+
+generate-sas: az-login
+	az storage container generate-sas --name vhds --permissions lr --connection-string "${CLASSIC_SA_CONNECTION_STRING}" --start ${START_DATE} --expiry ${EXPIRY_DATE} | tee vhd-sas

packer/cleanup-vhd.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -eux
+
+## Cleanup packer SSH key and machine ID generated for this boot
+rm -f /root/.ssh/authorized_keys
+rm -f /home/packer/.ssh/authorized_keys
+rm -f /etc/machine-id
+touch /etc/machine-id

packer/init-variables.sh

@@ -0,0 +1,65 @@
+#!/bin/bash -e
+
+CDIR=$(dirname "${BASH_SOURCE}")
+
+SETTINGS_JSON="${SETTINGS_JSON:-./packer/settings.json}"
+SP_JSON="${SP_JSON:-./packer/sp.json}"
+SUBSCRIPTION_ID="${SUBSCRIPTION_ID:-`az account show -o json --query="id" | tr -d '"'`}"
+STORAGE_ACCOUNT_NAME="aksimages$(date +%s)"
+
+echo "Subscription ID: ${SUBSCRIPTION_ID}"
+echo "Service Principal Path: ${SP_JSON}"
+
+if [ -a "${SP_JSON}" ]; then
+	echo "Existing credentials file found."
+	exit 0
+elif [ -z "${CLIENT_ID}" ]; then
+	echo "Service principal not found! Generating one @ ${SP_JSON}"
+	az ad sp create-for-rbac -n aks-images-packer$(date +%s) -o json > ${SP_JSON}
+	CLIENT_ID=`cat ${SP_JSON} | jq -r .appId`
+	CLIENT_SECRET=`cat ${SP_JSON} | jq -r .password`
+	TENANT_ID=`cat ${SP_JSON} | jq -r .tenant`
+fi
+
+avail=$(az storage account check-name -n ${STORAGE_ACCOUNT_NAME} -o json | jq -r .nameAvailable)
+if $avail ; then
+	echo "creating new storage account ${STORAGE_ACCOUNT_NAME}"
+	az storage account create -n $STORAGE_ACCOUNT_NAME -g $AZURE_RESOURCE_GROUP_NAME --sku "Standard_RAGRS"
+	echo "creating new container system"
+	key=$(az storage account keys list -n $STORAGE_ACCOUNT_NAME -g $AZURE_RESOURCE_GROUP_NAME | jq -r '.[0].value')
+	az storage container create --name system --public-access container --account-key=$key --account-name=$STORAGE_ACCOUNT_NAME
+else
+	echo "storage account ${STORAGE_ACCOUNT_NAME} already exists."
+fi
+
+if [ -z "${CLIENT_ID}" ]; then
+	echo "CLIENT_ID was not set! Something happened when generating the service principal or when trying to read the sp file!"
+	exit 1
+fi
+
+if [ -z "${CLIENT_SECRET}" ]; then
+	echo "CLIENT_SECRET was not set! Something happened when generating the service principal or when trying to read the sp file!"
+	exit 1
+fi
+
+if [ -z "${TENANT_ID}" ]; then
+	echo "TENANT_ID was not set! Something happened when generating the service principal or when trying to read the sp file!"
+	exit 1
+fi
+
+echo "storage name: ${STORAGE_ACCOUNT_NAME}"
+
+cat <<EOF > packer/settings.json
+{
+  "subscription_id":  "${SUBSCRIPTION_ID}",
+  "client_id": "${CLIENT_ID}",
+  "client_secret": "${CLIENT_SECRET}",
+  "tenant_id":      "${TENANT_ID}",
+  "resource_group_name": "${AZURE_RESOURCE_GROUP_NAME}",
+  "location": "${AZURE_LOCATION}",
+  "storage_account_name": "${STORAGE_ACCOUNT_NAME}",
+  "vm_size": "${AZURE_VM_SIZE}"
+}
+EOF
+
+cat packer/settings.json

packer/install-dependencies.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+source /home/packer/provision_installs.sh
+source /home/packer/provision_source.sh
+
+# TODO: deal with etcd versions
+ETCD_VERSION="3.2.23"
+ETCD_DOWNLOAD_URL="https://acs-mirror.azureedge.net/github-coreos"
+installEtcd
+
+installDeps
+
+DOCKER_REPO="https://apt.dockerproject.org/repo"
+DOCKER_ENGINE_VERSION="1.13.*"
+installDocker
+
+installClearContainersRuntime
+
+VNET_CNI_PLUGINS_URL="https://acs-mirror.azureedge.net/cni/azure-vnet-cni-linux-amd64-latest.tgz"
+CNI_PLUGINS_URL="https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-latest.tgz"
+
+installAzureCNI
+
+CONTAINERD_DOWNLOAD_URL_BASE="https://storage.googleapis.com/cri-containerd-release/"
+installContainerd
+
+# TODO: install multiple versions
+HYPERKUBE_VERSION="v1.10.5"
+HYPERKUBE_URL="k8s.gcr.io/hyperkube-amd64:${HYPERKUBE_VERSION}"
+extractHyperkube
+
+echo "Install complete successfully" > /var/log/azure/golden-image-install.complete

packer/settings-example.json

@@ -0,0 +1,11 @@
+{
+  "client_id": "",
+  "client_secret": "",
+  "tenant_id": "",
+  "subscription_id": "",
+  "resource_group_name": "aksimages",
+  "managed_image_resource_group_name": "aksimages",
+  "storage_account_name": "",
+  "location": "West US 2",
+  "vm_size": "Standard_D2_v2"
+}

packer/vhd-image-builder.json

@@ -0,0 +1,70 @@
+{
+    "variables": {
+        "client_id": "{{env `AZURE_CLIENT_ID`}}",
+        "client_secret": "{{env `AZURE_CLIENT_SECRET`}}",
+        "tenant_id": "{{env `AZURE_TENANT_ID`}}",
+        "subscription_id": "{{env `AZURE_SUBSCRIPTION_ID`}}",
+        "location": "{{env `AZURE_LOCATION`}}",
+        "vm_size": "{{env `AZURE_VM_SIZE`}}"
+    },
+    "builders": [
+        {
+            "type": "azure-arm",
+            "client_id": "{{user `client_id`}}",
+            "client_secret": "{{user `client_secret`}}",
+            "tenant_id": "{{user `tenant_id`}}",
+            "subscription_id": "{{user `subscription_id`}}",
+            "resource_group_name": "{{user `resource_group_name`}}",
+            "capture_container_name": "acsengine-vhds",
+            "capture_name_prefix": "acsengine-{{timestamp}}",
+            "storage_account": "{{user `storage_account_name`}}",
+            "os_type": "Linux",
+            "image_publisher": "Canonical",
+            "image_offer": "UbuntuServer",
+            "image_sku": "16.04-LTS",
+            "image_version": "latest",
+            "azure_tags": {
+                "dept": "Azure Container Service"
+            },
+            "location": "{{user `location`}}",
+            "vm_size": "{{user `vm_size`}}"
+        }
+    ],
+    "provisioners": [
+        {
+            "type": "shell",
+            "inline": [
+                "sudo mkdir -p /opt/azure/containers",
+                "sudo chown -R $USER /opt/azure/containers"
+            ]
+        },
+        {
+            "type": "file",
+            "source": "packer/cleanup-vhd.sh",
+            "destination": "/home/packer/cleanup-vhd.sh"
+        },
+        {
+            "type": "file",
+            "source": "parts/k8s/kubernetesinstalls.sh",
+            "destination": "/home/packer/provision_installs.sh"
+        },
+        {
+            "type": "file",
+            "source": "parts/k8s/kubernetesprovisionsource.sh",
+            "destination": "/home/packer/provision_source.sh"
+        },
+        {
+            "type": "file",
+            "source": "packer/install-dependencies.sh",
+            "destination": "/home/packer/install-dependencies.sh"
+        },
+        {
+            "type": "shell",
+            "inline": [
+                "sudo /bin/bash -eux /home/packer/install-dependencies.sh",
+                "sudo /bin/bash -eux /home/packer/cleanup-vhd.sh",
+                "rm /home/packer/*.sh"
+            ]
+        }
+    ]
+}