From d6e7cf5364e7f7e01b84f3b1ad11422fc2cbac82 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 14 Aug 2018 11:07:49 -0700 Subject: [PATCH] refactor CSE to separate installs (#3642) * refactor CSE * simplify master vars cse variables * fix cloud provider params * fix quotes * remove hostedmaster duplicate * use key vault name instead of extra var * missing , * move servicePrincipalObjectId to inside EnableEncryptionWithExternalKms case * replace vars with params * remove servicePrincipalObjectId var * only install etcd for master nodes * etcdDownloadURL to parameter * param -> var fix * clusterKeyVaultName is a variable * indent * more indents * restore retries for cloud init * restore wait for cloud init * remove installFlexVolDrivers --- docs/kubernetes/troubleshooting.md | 2 +- parts/k8s/kubernetesagentcustomdata.yml | 14 + parts/k8s/kubernetesconfigs.sh | 346 +++++++++++++ parts/k8s/kubernetescustomscript.sh | 587 ++--------------------- parts/k8s/kubernetesinstalls.sh | 142 ++++++ parts/k8s/kubernetesmastercustomdata.yml | 25 +- parts/k8s/kubernetesmastervars.t | 18 +- parts/k8s/kubernetesprovisionsource.sh | 41 ++ pkg/acsengine/const.go | 2 + pkg/acsengine/template_generator.go | 6 + 10 files changed, 617 insertions(+), 566 deletions(-) create mode 100644 parts/k8s/kubernetesconfigs.sh create mode 100644 parts/k8s/kubernetesinstalls.sh diff --git a/docs/kubernetes/troubleshooting.md b/docs/kubernetes/troubleshooting.md index fa5577d99..31a7042cb 100644 --- a/docs/kubernetes/troubleshooting.md +++ b/docs/kubernetes/troubleshooting.md @@ -46,7 +46,7 @@ How to determine the above? execute command: command terminated with exit status=20\n[stdout]\n\n[stderr]\n"." ``` -Look for the exit code. In the above example, the exit code is `20`. The list of exit codes and their meaning can be found [here](../../parts/k8s/kubernetescustomscript.sh). +Look for the exit code. In the above example, the exit code is `20`. The list of exit codes and their meaning can be found [here](../../parts/k8s/kubernetesprovisionsource.sh). If after following the above you are still unable to troubleshoot your deployment error, please open a Github issue with title "CSE error: exit code " and include the following in the description: diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index c992a619f..917175e7a 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -14,6 +14,20 @@ write_files: owner: "root" content: !!binary | {{WrapAsVariable "sshdConfig"}} + +- path: "/opt/azure/containers/provision_installs.sh" + permissions: "0744" + encoding: gzip + owner: "root" + content: !!binary | + {{WrapAsVariable "provisionInstalls"}} + +- path: "/opt/azure/containers/provision_configs.sh" + permissions: "0744" + encoding: gzip + owner: "root" + content: !!binary | + {{WrapAsVariable "provisionConfigs"}} {{if .KubernetesConfig.RequiresDocker}} {{if not .IsCoreOS}} diff --git a/parts/k8s/kubernetesconfigs.sh b/parts/k8s/kubernetesconfigs.sh new file mode 100644 index 000000000..49a1f705e --- /dev/null +++ b/parts/k8s/kubernetesconfigs.sh @@ -0,0 +1,346 @@ +#!/bin/bash + +function systemctlEnableAndStart() { + systemctl_restart 100 5 30 $1 + RESTART_STATUS=$? + systemctl status $1 --no-pager -l > /var/log/azure/$1-status.log + if [ $RESTART_STATUS -ne 0 ]; then + echo "$1 could not be started" + exit $ERR_SYSTEMCTL_START_FAIL + fi + retrycmd_if_failure 10 5 3 systemctl enable $1 + if [ $? -ne 0 ]; then + echo "$1 could not be enabled by systemctl" + exit $ERR_SYSTEMCTL_ENABLE_FAIL + fi +} + +function configureEtcd() { + useradd -U "etcd" + usermod -p "$(head -c 32 /dev/urandom | base64)" "etcd" + passwd -u "etcd" + id "etcd" + + APISERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/apiserver.key" + touch "${APISERVER_PRIVATE_KEY_PATH}" + chmod 0600 "${APISERVER_PRIVATE_KEY_PATH}" + chown root:root "${APISERVER_PRIVATE_KEY_PATH}" + + CA_PRIVATE_KEY_PATH="/etc/kubernetes/certs/ca.key" + touch "${CA_PRIVATE_KEY_PATH}" + chmod 0600 "${CA_PRIVATE_KEY_PATH}" + chown root:root "${CA_PRIVATE_KEY_PATH}" + + ETCD_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdserver.key" + touch "${ETCD_SERVER_PRIVATE_KEY_PATH}" + chmod 0600 "${ETCD_SERVER_PRIVATE_KEY_PATH}" + chown etcd:etcd "${ETCD_SERVER_PRIVATE_KEY_PATH}" + + ETCD_CLIENT_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdclient.key" + touch "${ETCD_CLIENT_PRIVATE_KEY_PATH}" + chmod 0600 "${ETCD_CLIENT_PRIVATE_KEY_PATH}" + chown root:root "${ETCD_CLIENT_PRIVATE_KEY_PATH}" + + ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.key" + touch "${ETCD_PEER_PRIVATE_KEY_PATH}" + chmod 0600 "${ETCD_PEER_PRIVATE_KEY_PATH}" + chown etcd:etcd "${ETCD_PEER_PRIVATE_KEY_PATH}" + + ETCD_SERVER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdserver.crt" + touch "${ETCD_SERVER_CERTIFICATE_PATH}" + chmod 0644 "${ETCD_SERVER_CERTIFICATE_PATH}" + chown root:root "${ETCD_SERVER_CERTIFICATE_PATH}" + + ETCD_CLIENT_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdclient.crt" + touch "${ETCD_CLIENT_CERTIFICATE_PATH}" + chmod 0644 "${ETCD_CLIENT_CERTIFICATE_PATH}" + chown root:root "${ETCD_CLIENT_CERTIFICATE_PATH}" + + ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.crt" + touch "${ETCD_PEER_CERTIFICATE_PATH}" + chmod 0644 "${ETCD_PEER_CERTIFICATE_PATH}" + chown root:root "${ETCD_PEER_CERTIFICATE_PATH}" + + set +x + echo "${APISERVER_PRIVATE_KEY}" | base64 --decode > "${APISERVER_PRIVATE_KEY_PATH}" + echo "${CA_PRIVATE_KEY}" | base64 --decode > "${CA_PRIVATE_KEY_PATH}" + echo "${ETCD_SERVER_PRIVATE_KEY}" | base64 --decode > "${ETCD_SERVER_PRIVATE_KEY_PATH}" + echo "${ETCD_CLIENT_PRIVATE_KEY}" | base64 --decode > "${ETCD_CLIENT_PRIVATE_KEY_PATH}" + echo "${ETCD_PEER_KEY}" | base64 --decode > "${ETCD_PEER_PRIVATE_KEY_PATH}" + echo "${ETCD_SERVER_CERTIFICATE}" | base64 --decode > "${ETCD_SERVER_CERTIFICATE_PATH}" + echo "${ETCD_CLIENT_CERTIFICATE}" | base64 --decode > "${ETCD_CLIENT_CERTIFICATE_PATH}" + echo "${ETCD_PEER_CERT}" | base64 --decode > "${ETCD_PEER_CERTIFICATE_PATH}" + set -x + + /opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1 + RET=$? + if [ $RET -ne 0 ]; then + exit $RET + fi + + /opt/azure/containers/mountetcd.sh || exit $ERR_ETCD_VOL_MOUNT_FAIL + systemctlEnableAndStart etcd + for i in $(seq 1 600); do + MEMBER="$(sudo etcdctl member list | grep -E ${MASTER_VM_NAME} | cut -d':' -f 1)" + if [ "$MEMBER" != "" ]; then + break + else + sleep 1 + fi + done + retrycmd_if_failure 10 1 5 sudo etcdctl member update $MEMBER ${ETCD_PEER_URL} || exit $ERR_ETCD_CONFIG_FAIL +} + +function ensureRPC() { + systemctlEnableAndStart rpcbind + systemctlEnableAndStart rpc-statd +} + +function runAptDaily() { + /usr/lib/apt/apt.systemd.daily +} + +function generateAggregatedAPICerts() { + wait_for_file 1 1 /etc/kubernetes/generate-proxy-certs.sh && /etc/kubernetes/generate-proxy-certs.sh +} + +function configureK8s() { + KUBELET_PRIVATE_KEY_PATH="/etc/kubernetes/certs/client.key" + touch "${KUBELET_PRIVATE_KEY_PATH}" + chmod 0600 "${KUBELET_PRIVATE_KEY_PATH}" + chown root:root "${KUBELET_PRIVATE_KEY_PATH}" + + APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt" + touch "${APISERVER_PUBLIC_KEY_PATH}" + chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}" + chown root:root "${APISERVER_PUBLIC_KEY_PATH}" + + AZURE_JSON_PATH="/etc/kubernetes/azure.json" + touch "${AZURE_JSON_PATH}" + chmod 0600 "${AZURE_JSON_PATH}" + chown root:root "${AZURE_JSON_PATH}" + + set +x + echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}" + echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}" + cat << EOF > "${AZURE_JSON_PATH}" +{ + "cloud":"${TARGET_ENVIRONMENT}", + "tenantId": "${TENANT_ID}", + "subscriptionId": "${SUBSCRIPTION_ID}", + "aadClientId": "${SERVICE_PRINCIPAL_CLIENT_ID}", + "aadClientSecret": "${SERVICE_PRINCIPAL_CLIENT_SECRET}", + "resourceGroup": "${RESOURCE_GROUP}", + "location": "${LOCATION}", + "vmType": "${VM_TYPE}", + "subnetName": "${SUBNET}", + "securityGroupName": "${NETWORK_SECURITY_GROUP}", + "vnetName": "${VIRTUAL_NETWORK}", + "vnetResourceGroup": "${VIRTUAL_NETWORK_RESOURCE_GROUP}", + "routeTableName": "${ROUTE_TABLE}", + "primaryAvailabilitySetName": "${PRIMARY_AVAILABILITY_SET}", + "primaryScaleSetName": "${PRIMARY_SCALE_SET}", + "cloudProviderBackoff": ${CLOUDPROVIDER_BACKOFF}, + "cloudProviderBackoffRetries": ${CLOUDPROVIDER_BACKOFF_RETRIES}, + "cloudProviderBackoffExponent": ${CLOUDPROVIDER_BACKOFF_EXPONENT}, + "cloudProviderBackoffDuration": ${CLOUDPROVIDER_BACKOFF_DURATION}, + "cloudProviderBackoffJitter": ${CLOUDPROVIDER_BACKOFF_JITTER}, + "cloudProviderRatelimit": ${CLOUDPROVIDER_RATELIMIT}, + "cloudProviderRateLimitQPS": ${CLOUDPROVIDER_RATELIMIT_QPS}, + "cloudProviderRateLimitBucket": ${CLOUDPROVIDER_RATELIMIT_BUCKET}, + "useManagedIdentityExtension": ${USE_MANAGED_IDENTITY_EXTENSION}, + "useInstanceMetadata": ${USE_INSTANCE_METADATA}, + "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", + "providerKeyName": "k8s", + "providerKeyVersion": "" +} +EOF + set -x + generateAggregatedAPICerts +} + +function configureCNI() { + # Turn on br_netfilter (needed for the iptables rules to work on bridges) + # and permanently enable it + retrycmd_if_failure 30 6 10 modprobe br_netfilter || exit $ERR_MODPROBE_FAIL + # /etc/modules-load.d is the location used by systemd to load modules + echo -n "br_netfilter" > /etc/modules-load.d/br_netfilter.conf + if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then + mv $CNI_BIN_DIR/10-azure.conflist $CNI_CONFIG_DIR/ + chmod 600 $CNI_CONFIG_DIR/10-azure.conflist + /sbin/ebtables -t nat --list + fi +} + +function setKubeletOpts () { + sed -i "s#^KUBELET_OPTS=.*#KUBELET_OPTS=${1}#" /etc/default/kubelet +} + +function ensureCCProxy() { + cat $CC_SERVICE_IN_TMP | sed 's#@libexecdir@#/usr/libexec#' > /etc/systemd/system/cc-proxy.service + cat $CC_SOCKET_IN_TMP sed 's#@localstatedir@#/var#' > /etc/systemd/system/cc-proxy.socket + # Enable and start Clear Containers proxy service + echo "Enabling and starting Clear Containers proxy service..." + systemctlEnableAndStart cc-proxy +} + +function setupContainerd() { + echo "Configuring cri-containerd..." + mkdir -p "/etc/containerd" + CRI_CONTAINERD_CONFIG="/etc/containerd/config.toml" + echo "subreaper = false" > "$CRI_CONTAINERD_CONFIG" + echo "oom_score = 0" >> "$CRI_CONTAINERD_CONFIG" + echo "[plugins.cri]" >> "$CRI_CONTAINERD_CONFIG" + echo "sandbox_image = \"$POD_INFRA_CONTAINER_SPEC\"" >> "$CRI_CONTAINERD_CONFIG" + echo "[plugins.cri.containerd.untrusted_workload_runtime]" >> "$CRI_CONTAINERD_CONFIG" + echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG" + if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then + echo "runtime_engine = '/usr/bin/cc-runtime'" >> "$CRI_CONTAINERD_CONFIG" + elif [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then + echo "runtime_engine = '/usr/bin/kata-runtime'" >> "$CRI_CONTAINERD_CONFIG" + else + echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG" + fi + echo "[plugins.cri.containerd.default_runtime]" >> "$CRI_CONTAINERD_CONFIG" + echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG" + echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG" + setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock" +} + +function ensureContainerd() { + if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then + setupContainerd + # Enable and start cri-containerd service + # Make sure this is done after networking plugins are installed + echo "Enabling and starting cri-containerd service..." + systemctlEnableAndStart containerd + fi +} + +function ensureDocker() { + echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf + usermod -aG docker ${ADMINUSER} + wait_for_file 600 1 $DOCKER || exit $ERR_FILE_WATCH_TIMEOUT + systemctlEnableAndStart docker + retrycmd_if_failure 6 1 10 docker pull busybox # pre-pull busybox, but don't exit if fail + systemctlEnableAndStart docker-health-probe +} +function ensureKMS() { + systemctlEnableAndStart kms +} + +function ensureKubelet() { + systemctlEnableAndStart kubelet +} + +function ensureJournal(){ + echo "Storage=persistent" >> /etc/systemd/journald.conf + echo "SystemMaxUse=1G" >> /etc/systemd/journald.conf + echo "RuntimeMaxUse=1G" >> /etc/systemd/journald.conf + echo "ForwardToSyslog=no" >> /etc/systemd/journald.conf + systemctlEnableAndStart systemd-journald +} + +function ensurePodSecurityPolicy() { + POD_SECURITY_POLICY_FILE="/etc/kubernetes/manifests/pod-security-policy.yaml" + if [ -f $POD_SECURITY_POLICY_FILE ]; then + $KUBECTL create -f $POD_SECURITY_POLICY_FILE + fi +} + +function ensureK8sControlPlane() { + if $REBOOTREQUIRED; then + return + fi + wait_for_file 600 1 $KUBECTL || exit $ERR_FILE_WATCH_TIMEOUT + retrycmd_if_failure 900 1 20 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT + ensurePodSecurityPolicy +} + +function ensureEtcd() { + retrycmd_if_failure 120 5 10 curl --cacert /etc/kubernetes/certs/ca.crt --cert /etc/kubernetes/certs/etcdclient.crt --key /etc/kubernetes/certs/etcdclient.key ${ETCD_CLIENT_URL}/v2/machines || exit $ERR_ETCD_RUNNING_TIMEOUT +} + +function writeKubeConfig() { + KUBECONFIGDIR=/home/$ADMINUSER/.kube + KUBECONFIGFILE=$KUBECONFIGDIR/config + mkdir -p $KUBECONFIGDIR + touch $KUBECONFIGFILE + chown $ADMINUSER:$ADMINUSER $KUBECONFIGDIR + chown $ADMINUSER:$ADMINUSER $KUBECONFIGFILE + chmod 700 $KUBECONFIGDIR + chmod 600 $KUBECONFIGFILE + + # disable logging after secret output + set +x + echo " +--- +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: \"$CA_CERTIFICATE\" + server: $KUBECONFIG_SERVER + name: \"$MASTER_FQDN\" +contexts: +- context: + cluster: \"$MASTER_FQDN\" + user: \"$MASTER_FQDN-admin\" + name: \"$MASTER_FQDN\" +current-context: \"$MASTER_FQDN\" +kind: Config +users: +- name: \"$MASTER_FQDN-admin\" + user: + client-certificate-data: \"$KUBECONFIG_CERTIFICATE\" + client-key-data: \"$KUBECONFIG_KEY\" +" > $KUBECONFIGFILE + # renable logging after secrets + set -x +} + +function configClusterAutoscalerAddon() { + if [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == true ]]; then + CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT="- mountPath: /var/lib/waagent/\n\ name: waagent\n\ readOnly: true" + CLUSTER_AUTOSCALER_MSI_VOLUME="- hostPath:\n\ path: /var/lib/waagent/\n\ name: waagent" + CLUSTER_AUTOSCALER_MSI_HOST_NETWORK="hostNetwork: true" + + sed -i "s||${CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||${CLUSTER_AUTOSCALER_MSI_VOLUME}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo "${CLUSTER_AUTOSCALER_MSI_HOST_NETWORK}")|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + elif [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == false ]]; then + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + fi + + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" +} + +configACIConnectorAddon() { + ACI_CONNECTOR_CREDENTIALS=$(printf "{\"clientId\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_ID)\", \"clientSecret\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET)\", \"tenantId\": \"$(echo $TENANT_ID)\", \"subscriptionId\": \"$(echo $SUBSCRIPTION_ID)\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",\"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\"}" | base64 -w 0) + + openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 -keyout /etc/kubernetes/certs/aci-connector-key.pem -out /etc/kubernetes/certs/aci-connector-cert.pem -subj "/C=US/ST=CA/L=virtualkubelet/O=virtualkubelet/OU=virtualkubelet/CN=virtualkubelet" + ACI_CONNECTOR_KEY=$(base64 /etc/kubernetes/certs/aci-connector-key.pem -w0) + ACI_CONNECTOR_CERT=$(base64 /etc/kubernetes/certs/aci-connector-cert.pem -w0) + + sed -i "s||$ACI_CONNECTOR_CREDENTIALS|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" + sed -i "s||$(echo $RESOURCE_GROUP)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" + sed -i "s||$(echo $ACI_CONNECTOR_CERT)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" + sed -i "s||$(echo $ACI_CONNECTOR_KEY)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" +} + +configAddons() { + if [[ "${CLUSTER_AUTOSCALER_ADDON}" = True ]]; then + configClusterAutoscalerAddon + fi + + if [[ "${ACI_CONNECTOR_ADDON}" = True ]]; then + configACIConnectorAddon + fi +} \ No newline at end of file diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 228842d89..07cbd4c7e 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -2,47 +2,9 @@ set -x echo `date`,`hostname`, startscript>>/opt/m source /opt/azure/containers/provision_source.sh -# TODO standardize/generalize CSE exit codes -ERR_SYSTEMCTL_ENABLE_FAIL=3 # Service could not be enabled by systemctl -ERR_SYSTEMCTL_START_FAIL=4 # Service could not be started by systemctl -ERR_CLOUD_INIT_TIMEOUT=5 # Timeout waiting for cloud-init runcmd to complete -ERR_FILE_WATCH_TIMEOUT=6 # Timeout waiting for a file -ERR_HOLD_WALINUXAGENT=7 # Unable to place walinuxagent apt package on hold during install -ERR_RELEASE_HOLD_WALINUXAGENT=8 # Unable to release hold on walinuxagent apt package after install -ERR_APT_INSTALL_TIMEOUT=9 # Timeout installing required apt packages -ERR_ETCD_DATA_DIR_NOT_FOUND=10 # Etcd data dir not found -ERR_ETCD_RUNNING_TIMEOUT=11 # Timeout waiting for etcd to be accessible -ERR_ETCD_DOWNLOAD_TIMEOUT=12 # Timeout waiting for etcd to download -ERR_ETCD_VOL_MOUNT_FAIL=13 # Unable to mount etcd disk volume -ERR_ETCD_START_TIMEOUT=14 # Unable to start etcd runtime -ERR_ETCD_CONFIG_FAIL=15 # Unable to configure etcd cluster -ERR_DOCKER_INSTALL_TIMEOUT=20 # Timeout waiting for docker install -ERR_DOCKER_DOWNLOAD_TIMEOUT=21 # Timout waiting for docker download(s) -ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT=22 # Timeout waiting to download docker repo key -ERR_DOCKER_APT_KEY_TIMEOUT=23 # Timeout waiting for docker apt-key -ERR_K8S_RUNNING_TIMEOUT=30 # Timeout waiting for k8s cluster to be healthy -ERR_K8S_DOWNLOAD_TIMEOUT=31 # Timeout waiting for Kubernetes download(s) -ERR_KUBECTL_NOT_FOUND=32 # kubectl client binary not found on local disk -ERR_CNI_DOWNLOAD_TIMEOUT=41 # Timeout waiting for CNI download(s) -ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT=42 # Timeout waiting for https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -ERR_MS_PROD_DEB_PKG_ADD_FAIL=43 # Failed to add repo pkg file -#ERR_FLEXVOLUME_DOWNLOAD_TIMEOUT=44 # Failed to add repo pkg file -- DEPRECATED -ERR_MODPROBE_FAIL=49 # Unable to load a kernel module using modprobe -ERR_OUTBOUND_CONN_FAIL=50 # Unable to establish outbound connection -ERR_KATA_KEY_DOWNLOAD_TIMEOUT=60 # Timeout waiting to download kata repo key -ERR_KATA_APT_KEY_TIMEOUT=61 # Timeout waiting for kata apt-key -ERR_KATA_INSTALL_TIMEOUT=62 # Timeout waiting for kata install -ERR_CUSTOM_SEARCH_DOMAINS_FAIL=80 # Unable to configure custom search domains -ERR_APT_DAILY_TIMEOUT=98 # Timeout waiting for apt daily updates -ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete +source /opt/azure/containers/provision_installs.sh +source /opt/azure/containers/provision_configs.sh -OS=$(cat /etc/*-release | grep ^ID= | tr -d 'ID="' | awk '{print toupper($0)}') -UBUNTU_OS_NAME="UBUNTU" -RHEL_OS_NAME="RHEL" -COREOS_OS_NAME="COREOS" -KUBECTL=/usr/local/bin/kubectl -DOCKER=/usr/bin/docker -CNI_BIN_DIR=/opt/cni/bin CUSTOM_SEARCH_DOMAIN_SCRIPT=/opt/azure/containers/setup-custom-search-domains.sh set +x @@ -61,6 +23,13 @@ else REBOOTREQUIRED=false fi +if [ -f /var/log/azure/golden-image-install.complete ]; then + echo "detected golden image pre-install" + FULL_INSTALL_REQUIRED=false +else + FULL_INSTALL_REQUIRED=true +fi + function testOutboundConnection() { retrycmd_if_failure 20 1 3 nc -v www.google.com 443 || retrycmd_if_failure 20 1 3 nc -v www.1688.com 443 || exit $ERR_OUTBOUND_CONN_FAIL } @@ -69,538 +38,72 @@ function waitForCloudInit() { wait_for_file 900 1 /var/log/azure/cloud-init.complete || exit $ERR_CLOUD_INIT_TIMEOUT } -function systemctlEnableAndStart() { - systemctl_restart 100 5 30 $1 - RESTART_STATUS=$? - systemctl status $1 --no-pager -l > /var/log/azure/$1-status.log - if [ $RESTART_STATUS -ne 0 ]; then - echo "$1 could not be started" - exit $ERR_SYSTEMCTL_START_FAIL - fi - retrycmd_if_failure 10 5 3 systemctl enable $1 - if [ $? -ne 0 ]; then - echo "$1 could not be enabled by systemctl" - exit $ERR_SYSTEMCTL_ENABLE_FAIL - fi -} - -function installEtcd() { - useradd -U "etcd" - usermod -p "$(head -c 32 /dev/urandom | base64)" "etcd" - passwd -u "etcd" - id "etcd" - - APISERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/apiserver.key" - touch "${APISERVER_PRIVATE_KEY_PATH}" - chmod 0600 "${APISERVER_PRIVATE_KEY_PATH}" - chown root:root "${APISERVER_PRIVATE_KEY_PATH}" - - CA_PRIVATE_KEY_PATH="/etc/kubernetes/certs/ca.key" - touch "${CA_PRIVATE_KEY_PATH}" - chmod 0600 "${CA_PRIVATE_KEY_PATH}" - chown root:root "${CA_PRIVATE_KEY_PATH}" - - ETCD_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdserver.key" - touch "${ETCD_SERVER_PRIVATE_KEY_PATH}" - chmod 0600 "${ETCD_SERVER_PRIVATE_KEY_PATH}" - chown etcd:etcd "${ETCD_SERVER_PRIVATE_KEY_PATH}" - - ETCD_CLIENT_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdclient.key" - touch "${ETCD_CLIENT_PRIVATE_KEY_PATH}" - chmod 0600 "${ETCD_CLIENT_PRIVATE_KEY_PATH}" - chown root:root "${ETCD_CLIENT_PRIVATE_KEY_PATH}" - - ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.key" - touch "${ETCD_PEER_PRIVATE_KEY_PATH}" - chmod 0600 "${ETCD_PEER_PRIVATE_KEY_PATH}" - chown etcd:etcd "${ETCD_PEER_PRIVATE_KEY_PATH}" - - ETCD_SERVER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdserver.crt" - touch "${ETCD_SERVER_CERTIFICATE_PATH}" - chmod 0644 "${ETCD_SERVER_CERTIFICATE_PATH}" - chown root:root "${ETCD_SERVER_CERTIFICATE_PATH}" - - ETCD_CLIENT_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdclient.crt" - touch "${ETCD_CLIENT_CERTIFICATE_PATH}" - chmod 0644 "${ETCD_CLIENT_CERTIFICATE_PATH}" - chown root:root "${ETCD_CLIENT_CERTIFICATE_PATH}" - - ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.crt" - touch "${ETCD_PEER_CERTIFICATE_PATH}" - chmod 0644 "${ETCD_PEER_CERTIFICATE_PATH}" - chown root:root "${ETCD_PEER_CERTIFICATE_PATH}" - - set +x - echo "${APISERVER_PRIVATE_KEY}" | base64 --decode > "${APISERVER_PRIVATE_KEY_PATH}" - echo "${CA_PRIVATE_KEY}" | base64 --decode > "${CA_PRIVATE_KEY_PATH}" - echo "${ETCD_SERVER_PRIVATE_KEY}" | base64 --decode > "${ETCD_SERVER_PRIVATE_KEY_PATH}" - echo "${ETCD_CLIENT_PRIVATE_KEY}" | base64 --decode > "${ETCD_CLIENT_PRIVATE_KEY_PATH}" - echo "${ETCD_PEER_KEY}" | base64 --decode > "${ETCD_PEER_PRIVATE_KEY_PATH}" - echo "${ETCD_SERVER_CERTIFICATE}" | base64 --decode > "${ETCD_SERVER_CERTIFICATE_PATH}" - echo "${ETCD_CLIENT_CERTIFICATE}" | base64 --decode > "${ETCD_CLIENT_CERTIFICATE_PATH}" - echo "${ETCD_PEER_CERT}" | base64 --decode > "${ETCD_PEER_CERTIFICATE_PATH}" - set -x - - /opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1 - RET=$? - if [ $RET -ne 0 ]; then - exit $RET - fi - - /opt/azure/containers/mountetcd.sh || exit $ERR_ETCD_VOL_MOUNT_FAIL - systemctlEnableAndStart etcd - for i in $(seq 1 600); do - MEMBER="$(sudo etcdctl member list | grep -E ${MASTER_VM_NAME} | cut -d':' -f 1)" - if [ "$MEMBER" != "" ]; then - break - else - sleep 1 - fi - done - retrycmd_if_failure 10 1 5 sudo etcdctl member update $MEMBER ${ETCD_PEER_URL} || exit $ERR_ETCD_CONFIG_FAIL -} - -function installDeps() { - retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT - retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL - echo `date`,`hostname`, apt-get_update_begin>>/opt/m - apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - echo `date`,`hostname`, apt-get_update_end>>/opt/m - # make sure walinuxagent doesn't get updated in the middle of running this script - retrycmd_if_failure 20 5 30 apt-mark hold walinuxagent || exit $ERR_HOLD_WALINUXAGENT - # See https://github.com/kubernetes/kubernetes/blob/master/build/debian-hyperkube-base/Dockerfile#L25-L44 - apt_get_install 20 30 300 apt-transport-https ca-certificates iptables iproute2 ebtables socat util-linux mount ethtool init-system-helpers nfs-common ceph-common conntrack glusterfs-client ipset jq cgroup-lite git pigz xz-utils blobfuse fuse cifs-utils || exit $ERR_APT_INSTALL_TIMEOUT - systemctlEnableAndStart rpcbind - systemctlEnableAndStart rpc-statd -} - -function installDocker() { - retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg || exit $ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT - retrycmd_if_failure 10 5 10 apt-key add /tmp/aptdocker.gpg || exit $ERR_DOCKER_APT_KEY_TIMEOUT - echo "deb ${DOCKER_REPO} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list - printf "Package: docker-engine\nPin: version ${DOCKER_ENGINE_VERSION}\nPin-Priority: 550\n" > /etc/apt/preferences.d/docker.pref - apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - apt_get_install 20 30 120 docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT - echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf - usermod -aG docker ${ADMINUSER} - touch /var/log/azure/docker-install.complete -} - -function runAptDaily() { - /usr/lib/apt/apt.systemd.daily -} - -function generateAggregatedAPICerts() { - wait_for_file 1 1 /etc/kubernetes/generate-proxy-certs.sh && /etc/kubernetes/generate-proxy-certs.sh -} - -function configureK8s() { - KUBELET_PRIVATE_KEY_PATH="/etc/kubernetes/certs/client.key" - touch "${KUBELET_PRIVATE_KEY_PATH}" - chmod 0600 "${KUBELET_PRIVATE_KEY_PATH}" - chown root:root "${KUBELET_PRIVATE_KEY_PATH}" - - APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt" - touch "${APISERVER_PUBLIC_KEY_PATH}" - chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}" - chown root:root "${APISERVER_PUBLIC_KEY_PATH}" - - AZURE_JSON_PATH="/etc/kubernetes/azure.json" - touch "${AZURE_JSON_PATH}" - chmod 0600 "${AZURE_JSON_PATH}" - chown root:root "${AZURE_JSON_PATH}" - - set +x - echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}" - echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}" - cat << EOF > "${AZURE_JSON_PATH}" -{ - "cloud":"${TARGET_ENVIRONMENT}", - "tenantId": "${TENANT_ID}", - "subscriptionId": "${SUBSCRIPTION_ID}", - "aadClientId": "${SERVICE_PRINCIPAL_CLIENT_ID}", - "aadClientSecret": "${SERVICE_PRINCIPAL_CLIENT_SECRET}", - "resourceGroup": "${RESOURCE_GROUP}", - "location": "${LOCATION}", - "vmType": "${VM_TYPE}", - "subnetName": "${SUBNET}", - "securityGroupName": "${NETWORK_SECURITY_GROUP}", - "vnetName": "${VIRTUAL_NETWORK}", - "vnetResourceGroup": "${VIRTUAL_NETWORK_RESOURCE_GROUP}", - "routeTableName": "${ROUTE_TABLE}", - "primaryAvailabilitySetName": "${PRIMARY_AVAILABILITY_SET}", - "primaryScaleSetName": "${PRIMARY_SCALE_SET}", - "cloudProviderBackoff": ${CLOUDPROVIDER_BACKOFF}, - "cloudProviderBackoffRetries": ${CLOUDPROVIDER_BACKOFF_RETRIES}, - "cloudProviderBackoffExponent": ${CLOUDPROVIDER_BACKOFF_EXPONENT}, - "cloudProviderBackoffDuration": ${CLOUDPROVIDER_BACKOFF_DURATION}, - "cloudProviderBackoffJitter": ${CLOUDPROVIDER_BACKOFF_JITTER}, - "cloudProviderRatelimit": ${CLOUDPROVIDER_RATELIMIT}, - "cloudProviderRateLimitQPS": ${CLOUDPROVIDER_RATELIMIT_QPS}, - "cloudProviderRateLimitBucket": ${CLOUDPROVIDER_RATELIMIT_BUCKET}, - "useManagedIdentityExtension": ${USE_MANAGED_IDENTITY_EXTENSION}, - "useInstanceMetadata": ${USE_INSTANCE_METADATA}, - "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", - "providerKeyName": "k8s", - "providerKeyVersion": "" -} -EOF - set -x - generateAggregatedAPICerts -} - -function setKubeletOpts () { - sed -i "s#^KUBELET_OPTS=.*#KUBELET_OPTS=${1}#" /etc/default/kubelet -} - -function installCNI() { - mkdir -p $CNI_BIN_DIR - CONTAINERNETWORKING_CNI_TGZ_TMP=/tmp/containernetworking_cni.tgz - retrycmd_get_tarball 60 5 $CONTAINERNETWORKING_CNI_TGZ_TMP ${CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT - tar -xzf $CONTAINERNETWORKING_CNI_TGZ_TMP -C $CNI_BIN_DIR - chown -R root:root $CNI_BIN_DIR - chmod -R 755 $CNI_BIN_DIR - # Turn on br_netfilter (needed for the iptables rules to work on bridges) - # and permanently enable it - retrycmd_if_failure 30 6 10 modprobe br_netfilter || exit $ERR_MODPROBE_FAIL - # /etc/modules-load.d is the location used by systemd to load modules - echo -n "br_netfilter" > /etc/modules-load.d/br_netfilter.conf -} - -function configAzureCNI() { - CNI_CONFIG_DIR=/etc/cni/net.d - mkdir -p $CNI_CONFIG_DIR - chown -R root:root $CNI_CONFIG_DIR - chmod 755 $CNI_CONFIG_DIR - mkdir -p $CNI_BIN_DIR - AZURE_CNI_TGZ_TMP=/tmp/azure_cni.tgz - retrycmd_get_tarball 60 5 $AZURE_CNI_TGZ_TMP ${VNET_CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT - tar -xzf $AZURE_CNI_TGZ_TMP -C $CNI_BIN_DIR - installCNI - mv $CNI_BIN_DIR/10-azure.conflist $CNI_CONFIG_DIR/ - chmod 600 $CNI_CONFIG_DIR/10-azure.conflist - /sbin/ebtables -t nat --list -} - -function configNetworkPlugin() { - if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then - configAzureCNI - elif [[ "${NETWORK_PLUGIN}" = "kubenet" ]]; then - installCNI - elif [[ "${NETWORK_PLUGIN}" = "flannel" ]]; then - installCNI - fi -} - -function installKataContainersRuntime() { - # Add Kata Containers repository key - echo "Adding Kata Containers repository key..." - KATA_RELEASE_KEY_TMP=/tmp/kata-containers-release.key - KATA_URL=http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/Release.key - retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $KATA_URL > $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_KEY_DOWNLOAD_TIMEOUT - retrycmd_if_failure 10 5 10 apt-key add $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_APT_KEY_TIMEOUT - - # Add Kata Container repository - echo "Adding Kata Containers repository..." - echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/kata-containers.list - - # Install Kata Containers runtime - echo "Installing Kata Containers runtime..." - apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - apt_get_install 20 30 120 kata-runtime || exit $ERR_KATA_INSTALL_TIMEOUT -} - -function installClearContainersRuntime() { - # Add Clear Containers repository key - echo "Adding Clear Containers repository key..." - CC_RELEASE_KEY_TMP=/tmp/clear-containers-release.key - CC_URL=https://download.opensuse.org/repositories/home:clearcontainers:clear-containers-3/xUbuntu_16.04/Release.key - retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $CC_URL > $CC_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT - retrycmd_if_failure 10 5 10 apt-key add $CC_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT - - # Add Clear Container repository - echo "Adding Clear Containers repository..." - echo 'deb http://download.opensuse.org/repositories/home:/clearcontainers:/clear-containers-3/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/cc-runtime.list - - # Install Clear Containers runtime - echo "Installing Clear Containers runtime..." - apt_get_update - apt_get_install 20 30 120 cc-runtime - - # Install the systemd service and socket files. - local repo_uri="https://raw.githubusercontent.com/clearcontainers/proxy/3.0.23" - CC_SERVICE_IN_TMP=/tmp/cc-proxy.service.in - CC_SOCKET_IN_TMP=/tmp/cc-proxy.socket.in - retrycmd_if_failure_no_stats 20 1 5 curl -fsSL "${repo_uri}/cc-proxy.service.in" > $CC_SERVICE_IN_TMP - retrycmd_if_failure_no_stats 20 1 5 curl -fsSL "${repo_uri}/cc-proxy.socket.in" > $CC_SOCKET_IN_TMP - cat $CC_SERVICE_IN_TMP | sed 's#@libexecdir@#/usr/libexec#' > /etc/systemd/system/cc-proxy.service - cat $CC_SOCKET_IN_TMP sed 's#@localstatedir@#/var#' > /etc/systemd/system/cc-proxy.socket - - # Enable and start Clear Containers proxy service - echo "Enabling and starting Clear Containers proxy service..." - systemctlEnableAndStart cc-proxy -} - -function setupContainerd() { - echo "Configuring cri-containerd..." - - mkdir -p "/etc/containerd" - CRI_CONTAINERD_CONFIG="/etc/containerd/config.toml" - echo "subreaper = false" > "$CRI_CONTAINERD_CONFIG" - echo "oom_score = 0" >> "$CRI_CONTAINERD_CONFIG" - - echo "[plugins.cri]" >> "$CRI_CONTAINERD_CONFIG" - echo "sandbox_image = \"$POD_INFRA_CONTAINER_SPEC\"" >> "$CRI_CONTAINERD_CONFIG" - - echo "[plugins.cri.containerd.untrusted_workload_runtime]" >> "$CRI_CONTAINERD_CONFIG" - echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG" - if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then - echo "runtime_engine = '/usr/bin/cc-runtime'" >> "$CRI_CONTAINERD_CONFIG" - elif [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then - echo "runtime_engine = '/usr/bin/kata-runtime'" >> "$CRI_CONTAINERD_CONFIG" - else - echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG" - fi - echo "[plugins.cri.containerd.default_runtime]" >> "$CRI_CONTAINERD_CONFIG" - echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG" - echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG" - - setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock" -} - -function installContainerd() { - CRI_CONTAINERD_VERSION="1.1.0" - CONTAINERD_DOWNLOAD_URL="${CONTAINERD_DOWNLOAD_URL_BASE}cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz" - - CONTAINERD_TGZ_TMP=/tmp/containerd.tar.gz - retrycmd_get_tarball 60 5 "$CONTAINERD_TGZ_TMP" "$CONTAINERD_DOWNLOAD_URL" - tar -xzf "$CONTAINERD_TGZ_TMP" -C / - rm -f "$CONTAINERD_TGZ_TMP" - sed -i '/\[Service\]/a ExecStartPost=\/sbin\/iptables -P FORWARD ACCEPT' /etc/systemd/system/containerd.service - - echo "Successfully installed cri-containerd..." - if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then - setupContainerd - fi -} - -function ensureContainerd() { - if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then - # Enable and start cri-containerd service - # Make sure this is done after networking plugins are installed - echo "Enabling and starting cri-containerd service..." - systemctlEnableAndStart containerd - fi -} - -function ensureDocker() { - wait_for_file 600 1 $DOCKER || exit $ERR_FILE_WATCH_TIMEOUT - systemctlEnableAndStart docker - retrycmd_if_failure 6 1 10 docker pull busybox # pre-pull busybox, but don't exit if fail - systemctlEnableAndStart docker-health-probe -} -function ensureKMS() { - systemctlEnableAndStart kms -} - -function ensureKubelet() { - systemctlEnableAndStart kubelet -} - -function extractHyperkube(){ - TMP_DIR=$(mktemp -d) - retrycmd_if_failure 100 1 30 curl -sSL -o /usr/local/bin/img "https://acs-mirror.azureedge.net/img/img-linux-amd64-v0.4.6" - chmod +x /usr/local/bin/img - retrycmd_if_failure 75 1 60 img pull $HYPERKUBE_URL || exit $ERR_K8S_DOWNLOAD_TIMEOUT - path=$(find /tmp/img -name "hyperkube") - - if [[ $OS == $COREOS_OS_NAME ]]; then - cp "$path" "/opt/kubelet" - cp "$path" "/opt/kubectl" - chmod a+x /opt/kubelet /opt/kubectl - else - cp "$path" "/usr/local/bin/kubelet" - cp "$path" "/usr/local/bin/kubectl" - chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl - fi - rm -rf /tmp/hyperkube.tar "/tmp/img" -} - -function ensureJournal(){ - echo "Storage=persistent" >> /etc/systemd/journald.conf - echo "SystemMaxUse=1G" >> /etc/systemd/journald.conf - echo "RuntimeMaxUse=1G" >> /etc/systemd/journald.conf - echo "ForwardToSyslog=no" >> /etc/systemd/journald.conf - systemctlEnableAndStart systemd-journald -} - -function ensurePodSecurityPolicy() { - POD_SECURITY_POLICY_FILE="/etc/kubernetes/manifests/pod-security-policy.yaml" - if [ -f $POD_SECURITY_POLICY_FILE ]; then - $KUBECTL create -f $POD_SECURITY_POLICY_FILE - fi -} - -function ensureK8sControlPlane() { - if $REBOOTREQUIRED; then - return - fi - wait_for_file 600 1 $KUBECTL || exit $ERR_FILE_WATCH_TIMEOUT - retrycmd_if_failure 600 1 20 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT - ensurePodSecurityPolicy -} - -function ensureEtcd() { - retrycmd_if_failure 120 5 10 curl --cacert /etc/kubernetes/certs/ca.crt --cert /etc/kubernetes/certs/etcdclient.crt --key /etc/kubernetes/certs/etcdclient.key ${ETCD_CLIENT_URL}/v2/machines || exit $ERR_ETCD_RUNNING_TIMEOUT -} - -function writeKubeConfig() { - KUBECONFIGDIR=/home/$ADMINUSER/.kube - KUBECONFIGFILE=$KUBECONFIGDIR/config - mkdir -p $KUBECONFIGDIR - touch $KUBECONFIGFILE - chown $ADMINUSER:$ADMINUSER $KUBECONFIGDIR - chown $ADMINUSER:$ADMINUSER $KUBECONFIGFILE - chmod 700 $KUBECONFIGDIR - chmod 600 $KUBECONFIGFILE - - # disable logging after secret output - set +x - echo " ---- -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: \"$CA_CERTIFICATE\" - server: $KUBECONFIG_SERVER - name: \"$MASTER_FQDN\" -contexts: -- context: - cluster: \"$MASTER_FQDN\" - user: \"$MASTER_FQDN-admin\" - name: \"$MASTER_FQDN\" -current-context: \"$MASTER_FQDN\" -kind: Config -users: -- name: \"$MASTER_FQDN-admin\" - user: - client-certificate-data: \"$KUBECONFIG_CERTIFICATE\" - client-key-data: \"$KUBECONFIG_KEY\" -" > $KUBECONFIGFILE - # renable logging after secrets - set -x -} - -function configClusterAutoscalerAddon() { - if [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == true ]]; then - CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT="- mountPath: /var/lib/waagent/\n\ name: waagent\n\ readOnly: true" - CLUSTER_AUTOSCALER_MSI_VOLUME="- hostPath:\n\ path: /var/lib/waagent/\n\ name: waagent" - CLUSTER_AUTOSCALER_MSI_HOST_NETWORK="hostNetwork: true" - - sed -i "s||${CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||${CLUSTER_AUTOSCALER_MSI_VOLUME}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo "${CLUSTER_AUTOSCALER_MSI_HOST_NETWORK}")|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - elif [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == false ]]; then - sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - fi - - sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" -} - -configACIConnectorAddon() { - ACI_CONNECTOR_CREDENTIALS=$(printf "{\"clientId\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_ID)\", \"clientSecret\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET)\", \"tenantId\": \"$(echo $TENANT_ID)\", \"subscriptionId\": \"$(echo $SUBSCRIPTION_ID)\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",\"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\"}" | base64 -w 0) - - openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 -keyout /etc/kubernetes/certs/aci-connector-key.pem -out /etc/kubernetes/certs/aci-connector-cert.pem -subj "/C=US/ST=CA/L=virtualkubelet/O=virtualkubelet/OU=virtualkubelet/CN=virtualkubelet" - ACI_CONNECTOR_KEY=$(base64 /etc/kubernetes/certs/aci-connector-key.pem -w0) - ACI_CONNECTOR_CERT=$(base64 /etc/kubernetes/certs/aci-connector-cert.pem -w0) - - sed -i "s||$ACI_CONNECTOR_CREDENTIALS|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||$(echo $RESOURCE_GROUP)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||$(echo $ACI_CONNECTOR_CERT)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||$(echo $ACI_CONNECTOR_KEY)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" -} - -configAddons() { - if [[ "${CLUSTER_AUTOSCALER_ADDON}" = True ]]; then - configClusterAutoscalerAddon - fi - - if [[ "${ACI_CONNECTOR_ADDON}" = True ]]; then - configACIConnectorAddon +function holdWALinuxAgent() { + if [[ $OS == $UBUNTU_OS_NAME ]]; then + # make sure walinuxagent doesn't get updated in the middle of running this script + retrycmd_if_failure 20 5 30 apt-mark hold walinuxagent || exit $ERR_HOLD_WALINUXAGENT fi } testOutboundConnection waitForCloudInit +holdWALinuxAgent + +if $FULL_INSTALL_REQUIRED; then + if [[ ! -z "${MASTER_NODE}" ]]; then + installEtcd + fi + installDeps + installContainerRuntime + installNetworkPlugin + installContainerd + extractHyperkube +else + echo "Golden image; skipping dependencies installation" +fi + +ensureRPC if [[ ! -z "${MASTER_NODE}" ]]; then - echo "executing master node provision operations" - installEtcd -else - echo "skipping master node provision operations, this is an agent node" + configureEtcd fi if [ -f $CUSTOM_SEARCH_DOMAIN_SCRIPT ]; then $CUSTOM_SEARCH_DOMAIN_SCRIPT > /opt/azure/containers/setup-custom-search-domain.log 2>&1 || exit $ERR_CUSTOM_SEARCH_DOMAINS_FAIL fi -installDeps - if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then - installDocker ensureDocker +elif [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then + # Ensure we can nest virtualization + if grep -q vmx /proc/cpuinfo; then + ensureCCProxy + fi +elif [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then + # Ensure we can nest virtualization + if grep -q vmx /proc/cpuinfo; then + installKataContainersRuntime + fi fi + configureK8s - -configNetworkPlugin +configureCNI if [[ ! -z "${MASTER_NODE}" ]]; then - echo `date`,`hostname`, configAddonsStart>>/opt/m configAddons - echo `date`,`hostname`, configAddonsDone>>/opt/m fi -# containerd needs to be installed before extractHyperkube -# so runc is present. -echo `date`,`hostname`, installContainerdStart>>/opt/m -installContainerd -echo `date`,`hostname`, extractHyperkubeStart>>/opt/m -extractHyperkube -echo `date`,`hostname`, extractHyperkubeDone>>/opt/m - -if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then - # Ensure we can nest virtualization - if grep -q vmx /proc/cpuinfo; then - installClearContainersRuntime - fi -fi - -if [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then - # Ensure we can nest virtualization - if grep -q vmx /proc/cpuinfo; then - installKataContainersRuntime - fi -fi - -echo `date`,`hostname`, ensureContainerdStart>>/opt/m ensureContainerd -if [[ ! -z "${MASTER_NODE}" && ! -z "${EnableEncryptionWithExternalKms}" ]]; then +if [[ ! -z "${MASTER_NODE}" && "${KMS_PROVIDER_VAULT_NAME}" != "" ]]; then ensureKMS fi ensureKubelet ensureJournal + if [[ ! -z "${MASTER_NODE}" ]]; then writeKubeConfig ensureEtcd @@ -615,7 +118,7 @@ if [[ $OS == $UBUNTU_OS_NAME ]]; then retrycmd_if_failure 20 5 30 apt-mark unhold walinuxagent || exit $ERR_RELEASE_HOLD_WALINUXAGENTs fi -echo "Install complete successfully" +echo "Custom script finished successfully" mkdir -p /opt/azure/containers && touch /opt/azure/containers/provision.complete ps auxfww > /opt/azure/provision-ps.log & diff --git a/parts/k8s/kubernetesinstalls.sh b/parts/k8s/kubernetesinstalls.sh new file mode 100644 index 000000000..ea78d32d5 --- /dev/null +++ b/parts/k8s/kubernetesinstalls.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +CC_SERVICE_IN_TMP=/opt/azure/containers/cc-proxy.service.in +CC_SOCKET_IN_TMP=/opt/azure/containers/cc-proxy.socket.in +CNI_CONFIG_DIR="/etc/cni/net.d" +CNI_BIN_DIR="/opt/cni/bin" +AZURE_CNI_TGZ_TMP="/tmp/azure_cni.tgz" +CONTAINERNETWORKING_CNI_TGZ_TMP="/tmp/containernetworking_cni.tgz" + +function installEtcd() { + retrycmd_get_tarball 60 10 /tmp/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz ${ETCD_DOWNLOAD_URL}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz || exit $ERR_ETCD_DOWNLOAD_TIMEOUT + tar -xzvf /tmp/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -C /usr/bin/ --strip-components=1 || exit $ERR_ETCD_DOWNLOAD_TIMEOUT +} + +function installDeps() { + retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT + retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + # See https://github.com/kubernetes/kubernetes/blob/master/build/debian-hyperkube-base/Dockerfile#L25-L44 + apt_get_install 20 30 300 apt-transport-https ca-certificates iptables iproute2 ebtables socat util-linux mount ethtool init-system-helpers nfs-common ceph-common conntrack glusterfs-client ipset jq cgroup-lite git pigz xz-utils blobfuse fuse cifs-utils || exit $ERR_APT_INSTALL_TIMEOUT +} + +function installContainerRuntime() { + if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then + installDocker + elif [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then + # Ensure we can nest virtualization + if grep -q vmx /proc/cpuinfo; then + installClearContainersRuntime + fi + fi +} + +function installDocker() { + retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg || exit $ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 apt-key add /tmp/aptdocker.gpg || exit $ERR_DOCKER_APT_KEY_TIMEOUT + echo "deb ${DOCKER_REPO} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list + printf "Package: docker-engine\nPin: version ${DOCKER_ENGINE_VERSION}\nPin-Priority: 550\n" > /etc/apt/preferences.d/docker.pref + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + apt_get_install 20 30 120 docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT + touch /var/log/azure/docker-install.complete +} + +function installKataContainersRuntime() { + # Add Kata Containers repository key + echo "Adding Kata Containers repository key..." + KATA_RELEASE_KEY_TMP=/tmp/kata-containers-release.key + KATA_URL=http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/Release.key + retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $KATA_URL > $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 apt-key add $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_APT_KEY_TIMEOUT + + # Add Kata Container repository + echo "Adding Kata Containers repository..." + echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/kata-containers.list + + # Install Kata Containers runtime + echo "Installing Kata Containers runtime..." + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + apt_get_install 20 30 120 kata-runtime || exit $ERR_KATA_INSTALL_TIMEOUT +} + +function installClearContainersRuntime() { + # Add Clear Containers repository key + echo "Adding Clear Containers repository key..." + CC_RELEASE_KEY_TMP=/tmp/clear-containers-release.key + CC_URL=https://download.opensuse.org/repositories/home:clearcontainers:clear-containers-3/xUbuntu_16.04/Release.key + retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $CC_URL > $CC_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT + retrycmd_if_failure 10 5 10 apt-key add $CC_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT + + # Add Clear Container repository + echo "Adding Clear Containers repository..." + echo 'deb http://download.opensuse.org/repositories/home:/clearcontainers:/clear-containers-3/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/cc-runtime.list + + # Install Clear Containers runtime + echo "Installing Clear Containers runtime..." + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + apt_get_install 20 30 120 cc-runtime + + # Install the systemd service and socket files. + local repo_uri="https://raw.githubusercontent.com/clearcontainers/proxy/3.0.23" + retrycmd_if_failure_no_stats 20 1 5 curl -fsSL "${repo_uri}/cc-proxy.service.in" > $CC_SERVICE_IN_TMP + retrycmd_if_failure_no_stats 20 1 5 curl -fsSL "${repo_uri}/cc-proxy.socket.in" > $CC_SOCKET_IN_TMP +} + +function installNetworkPlugin() { + if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then + installAzureCNI + elif [[ "${NETWORK_PLUGIN}" = "kubenet" ]]; then + installCNI + elif [[ "${NETWORK_PLUGIN}" = "flannel" ]]; then + installCNI + fi +} + +function installCNI() { + mkdir -p $CNI_BIN_DIR + retrycmd_get_tarball 60 5 $CONTAINERNETWORKING_CNI_TGZ_TMP ${CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT + tar -xzf $CONTAINERNETWORKING_CNI_TGZ_TMP -C $CNI_BIN_DIR + chown -R root:root $CNI_BIN_DIR + chmod -R 755 $CNI_BIN_DIR +} + +function installAzureCNI() { + mkdir -p $CNI_CONFIG_DIR + chown -R root:root $CNI_CONFIG_DIR + chmod 755 $CNI_CONFIG_DIR + mkdir -p $CNI_BIN_DIR + retrycmd_get_tarball 60 5 $AZURE_CNI_TGZ_TMP ${VNET_CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT + tar -xzf $AZURE_CNI_TGZ_TMP -C $CNI_BIN_DIR + installCNI +} + +function installContainerd() { + CRI_CONTAINERD_VERSION="1.1.0" + CONTAINERD_DOWNLOAD_URL="${CONTAINERD_DOWNLOAD_URL_BASE}cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz" + CONTAINERD_TGZ_TMP=/tmp/containerd.tar.gz + retrycmd_get_tarball 60 5 "$CONTAINERD_TGZ_TMP" "$CONTAINERD_DOWNLOAD_URL" || exit $ERR_CONTAINERD_DOWNLOAD_TIMEOUT + tar -xzf "$CONTAINERD_TGZ_TMP" -C / + rm -f "$CONTAINERD_TGZ_TMP" + sed -i '/\[Service\]/a ExecStartPost=\/sbin\/iptables -P FORWARD ACCEPT' /etc/systemd/system/containerd.service + echo "Successfully installed cri-containerd..." +} + +function extractHyperkube() { + TMP_DIR=$(mktemp -d) + retrycmd_if_failure 100 1 30 curl -sSL -o /usr/local/bin/img "https://acs-mirror.azureedge.net/img/img-linux-amd64-v0.4.6" + chmod +x /usr/local/bin/img + retrycmd_if_failure 75 1 60 img pull $HYPERKUBE_URL || exit $ERR_K8S_DOWNLOAD_TIMEOUT + img unpack -o "/home/rootfs" $HYPERKUBE_URL + path=$(find /home/rootfs -name "hyperkube") + + if [[ $OS == $COREOS_OS_NAME ]]; then + cp "$path" "/opt/kubelet" + cp "$path" "/opt/kubectl" + chmod a+x /opt/kubelet /opt/kubectl + else + cp "$path" "/usr/local/bin/kubelet" + cp "$path" "/usr/local/bin/kubectl" + chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl + fi + rm -rf /tmp/hyperkube.tar "/tmp/img" +} \ No newline at end of file diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index c17d9ede0..1a7a25b79 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -21,6 +21,20 @@ write_files: content: !!binary | {{WrapAsVariable "sshdConfig"}} +- path: "/opt/azure/containers/provision_installs.sh" + permissions: "0744" + encoding: gzip + owner: "root" + content: !!binary | + {{WrapAsVariable "provisionInstalls"}} + +- path: "/opt/azure/containers/provision_configs.sh" + permissions: "0744" + encoding: gzip + owner: "root" + content: !!binary | + {{WrapAsVariable "provisionConfigs"}} + {{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}} {{if not .MasterProfile.IsCoreOS}} - path: "/etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf" @@ -394,17 +408,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER content: | #!/bin/bash set -x - source /opt/azure/containers/provision_source.sh - # TODO standardize/generalize CSE exit codes - ERR_ETCD_DOWNLOAD_TIMEOUT=12 - ERR_SYSTEMCTL_ENABLE_FAIL=3 - ETCD_VER=v{{WrapAsParameter "etcdVersion"}} - DOWNLOAD_URL={{WrapAsParameter "etcdDownloadURLBase"}} - retrycmd_get_tarball 60 10 /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz ${DOWNLOAD_URL}/etcd-${ETCD_VER}-linux-amd64.tar.gz - if [ $? -ne 0 ]; then - exit $ERR_ETCD_DOWNLOAD_TIMEOUT - fi - tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /usr/bin/ --strip-components=1 sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:2379" /etc/environment sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index cbf860a1d..f83ff884f 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -126,23 +126,15 @@ {{end}} "provisionScript": "{{GetKubernetesB64Provision}}", "provisionSource": "{{GetKubernetesB64ProvisionSource}}", + "provisionInstalls": "{{GetKubernetesB64Installs}}", + "provisionConfigs": "{{GetKubernetesB64Configs}}", "mountetcdScript": "{{GetKubernetesB64Mountetcd}}", "customSearchDomainsScript": "{{GetKubernetesB64CustomSearchDomainsScript}}", "sshdConfig": "{{GetB64sshdConfig}}", {{if not IsOpenShift}} + "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", {{if not IsHostedMaster}} - "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')))]", - {{if EnableEncryptionWithExternalKms}} - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'), ' EnableEncryptionWithExternalKms=true')]", - {{else}} - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", - {{end}} - {{else}} - {{if EnableEncryptionWithExternalKms}} - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'), ' EnableEncryptionWithExternalKms=true')]", - {{else}} - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'))]", - {{end}} + "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{end}} {{end}} "generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}", @@ -316,4 +308,6 @@ "apiVersionStorage": "2015-06-15", {{end}} "clusterKeyVaultName": "[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]" +{{else}} + ,"clusterKeyVaultName": "" {{end}} diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index 2148530f2..181f545e5 100644 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -1,5 +1,46 @@ #!/bin/sh +ERR_SYSTEMCTL_ENABLE_FAIL=3 # Service could not be enabled by systemctl +ERR_SYSTEMCTL_START_FAIL=4 # Service could not be started by systemctl +ERR_CLOUD_INIT_TIMEOUT=5 # Timeout waiting for cloud-init runcmd to complete +ERR_FILE_WATCH_TIMEOUT=6 # Timeout waiting for a file +ERR_HOLD_WALINUXAGENT=7 # Unable to place walinuxagent apt package on hold during install +ERR_RELEASE_HOLD_WALINUXAGENT=8 # Unable to release hold on walinuxagent apt package after install +ERR_APT_INSTALL_TIMEOUT=9 # Timeout installing required apt packages +ERR_ETCD_DATA_DIR_NOT_FOUND=10 # Etcd data dir not found +ERR_ETCD_RUNNING_TIMEOUT=11 # Timeout waiting for etcd to be accessible +ERR_ETCD_DOWNLOAD_TIMEOUT=12 # Timeout waiting for etcd to download +ERR_ETCD_VOL_MOUNT_FAIL=13 # Unable to mount etcd disk volume +ERR_ETCD_START_TIMEOUT=14 # Unable to start etcd runtime +ERR_ETCD_CONFIG_FAIL=15 # Unable to configure etcd cluster +ERR_DOCKER_INSTALL_TIMEOUT=20 # Timeout waiting for docker install +ERR_DOCKER_DOWNLOAD_TIMEOUT=21 # Timout waiting for docker download(s) +ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT=22 # Timeout waiting to download docker repo key +ERR_DOCKER_APT_KEY_TIMEOUT=23 # Timeout waiting for docker apt-key +ERR_K8S_RUNNING_TIMEOUT=30 # Timeout waiting for k8s cluster to be healthy +ERR_K8S_DOWNLOAD_TIMEOUT=31 # Timeout waiting for Kubernetes download(s) +ERR_KUBECTL_NOT_FOUND=32 # kubectl client binary not found on local disk +ERR_CNI_DOWNLOAD_TIMEOUT=41 # Timeout waiting for CNI download(s) +ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT=42 # Timeout waiting for https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb +ERR_MS_PROD_DEB_PKG_ADD_FAIL=43 # Failed to add repo pkg file +#ERR_FLEXVOLUME_DOWNLOAD_TIMEOUT=44 # Failed to add repo pkg file -- DEPRECATED +ERR_MODPROBE_FAIL=49 # Unable to load a kernel module using modprobe +ERR_OUTBOUND_CONN_FAIL=50 # Unable to establish outbound connection +ERR_KATA_KEY_DOWNLOAD_TIMEOUT=60 # Timeout waiting to download kata repo key +ERR_KATA_APT_KEY_TIMEOUT=61 # Timeout waiting for kata apt-key +ERR_KATA_INSTALL_TIMEOUT=62 # Timeout waiting for kata install +ERR_CONTAINERD_DOWNLOAD_TIMEOUT=70 # Timeout waiting for containerd download(s) +ERR_CUSTOM_SEARCH_DOMAINS_FAIL=80 # Unable to configure custom search domains +ERR_APT_DAILY_TIMEOUT=98 # Timeout waiting for apt daily updates +ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete + +OS=$(cat /etc/*-release | grep ^ID= | tr -d 'ID="' | awk '{print toupper($0)}') +UBUNTU_OS_NAME="UBUNTU" +RHEL_OS_NAME="RHEL" +COREOS_OS_NAME="COREOS" +KUBECTL=/usr/local/bin/kubectl +DOCKER=/usr/bin/docker + retrycmd_if_failure() { retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift for i in $(seq 1 $retries); do diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 26a89c9d4..2510d86a7 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -205,6 +205,8 @@ const ( kubernetesMasterCustomDataYaml = "k8s/kubernetesmastercustomdata.yml" kubernetesCustomScript = "k8s/kubernetescustomscript.sh" kubernetesProvisionSourceScript = "k8s/kubernetesprovisionsource.sh" + kubernetesInstalls = "k8s/kubernetesinstalls.sh" + kubernetesConfigurations = "k8s/kubernetesconfigs.sh" kubernetesMountetcd = "k8s/kubernetes_mountetcd.sh" kubernetesCustomSearchDomainsScript = "k8s/setup-custom-search-domains.sh" kubernetesMasterGenerateProxyCertsScript = "k8s/kubernetesmastergenerateproxycertscript.sh" diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index bd0c05a20..e5df92340 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -528,6 +528,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat "GetKubernetesB64ProvisionSource": func() string { return getBase64CustomScript(kubernetesProvisionSourceScript) }, + "GetKubernetesB64Installs": func() string { + return getBase64CustomScript(kubernetesInstalls) + }, + "GetKubernetesB64Configs": func() string { + return getBase64CustomScript(kubernetesConfigurations) + }, "GetKubernetesB64Mountetcd": func() string { return getBase64CustomScript(kubernetesMountetcd) },