зеркало из https://github.com/Azure/acs-engine.git
refactor CSE to separate installs (#3642)
* refactor CSE * simplify master vars cse variables * fix cloud provider params * fix quotes * remove hostedmaster duplicate * use key vault name instead of extra var * missing , * move servicePrincipalObjectId to inside EnableEncryptionWithExternalKms case * replace vars with params * remove servicePrincipalObjectId var * only install etcd for master nodes * etcdDownloadURL to parameter * param -> var fix * clusterKeyVaultName is a variable * indent * more indents * restore retries for cloud init * restore wait for cloud init * remove installFlexVolDrivers
This commit is contained in:
Cecile Robert-Michon
GitHub
Родитель
36aa21cb39
Коммит
d6e7cf5364
|
|
@ -46,7 +46,7 @@ How to determine the above?
|
|||
execute command: command terminated with exit status=20\n[stdout]\n\n[stderr]\n"."
|
||||
```
|
||||
|
||||
Look for the exit code. In the above example, the exit code is `20`. The list of exit codes and their meaning can be found [here](../../parts/k8s/kubernetescustomscript.sh).
|
||||
Look for the exit code. In the above example, the exit code is `20`. The list of exit codes and their meaning can be found [here](../../parts/k8s/kubernetesprovisionsource.sh).
|
||||
|
||||
If after following the above you are still unable to troubleshoot your deployment error, please open a Github issue with title "CSE error: exit code <INSERT_YOUR_EXIT_CODE>" and include the following in the description:
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,20 @@ write_files:
|
|||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "sshdConfig"}}
|
||||
|
||||
- path: "/opt/azure/containers/provision_installs.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "provisionInstalls"}}
|
||||
|
||||
- path: "/opt/azure/containers/provision_configs.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "provisionConfigs"}}
|
||||
|
||||
{{if .KubernetesConfig.RequiresDocker}}
|
||||
{{if not .IsCoreOS}}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,346 @@
|
|||
#!/bin/bash
|
||||
|
||||
function systemctlEnableAndStart() {
|
||||
systemctl_restart 100 5 30 $1
|
||||
RESTART_STATUS=$?
|
||||
systemctl status $1 --no-pager -l > /var/log/azure/$1-status.log
|
||||
if [ $RESTART_STATUS -ne 0 ]; then
|
||||
echo "$1 could not be started"
|
||||
exit $ERR_SYSTEMCTL_START_FAIL
|
||||
fi
|
||||
retrycmd_if_failure 10 5 3 systemctl enable $1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "$1 could not be enabled by systemctl"
|
||||
exit $ERR_SYSTEMCTL_ENABLE_FAIL
|
||||
fi
|
||||
}
|
||||
|
||||
function configureEtcd() {
|
||||
useradd -U "etcd"
|
||||
usermod -p "$(head -c 32 /dev/urandom | base64)" "etcd"
|
||||
passwd -u "etcd"
|
||||
id "etcd"
|
||||
|
||||
APISERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/apiserver.key"
|
||||
touch "${APISERVER_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${APISERVER_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${APISERVER_PRIVATE_KEY_PATH}"
|
||||
|
||||
CA_PRIVATE_KEY_PATH="/etc/kubernetes/certs/ca.key"
|
||||
touch "${CA_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${CA_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${CA_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdserver.key"
|
||||
touch "${ETCD_SERVER_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${ETCD_SERVER_PRIVATE_KEY_PATH}"
|
||||
chown etcd:etcd "${ETCD_SERVER_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_CLIENT_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdclient.key"
|
||||
touch "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.key"
|
||||
touch "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
chown etcd:etcd "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_SERVER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdserver.crt"
|
||||
touch "${ETCD_SERVER_CERTIFICATE_PATH}"
|
||||
chmod 0644 "${ETCD_SERVER_CERTIFICATE_PATH}"
|
||||
chown root:root "${ETCD_SERVER_CERTIFICATE_PATH}"
|
||||
|
||||
ETCD_CLIENT_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdclient.crt"
|
||||
touch "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
chmod 0644 "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
chown root:root "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
|
||||
ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.crt"
|
||||
touch "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
chmod 0644 "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
chown root:root "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
|
||||
set +x
|
||||
echo "${APISERVER_PRIVATE_KEY}" | base64 --decode > "${APISERVER_PRIVATE_KEY_PATH}"
|
||||
echo "${CA_PRIVATE_KEY}" | base64 --decode > "${CA_PRIVATE_KEY_PATH}"
|
||||
echo "${ETCD_SERVER_PRIVATE_KEY}" | base64 --decode > "${ETCD_SERVER_PRIVATE_KEY_PATH}"
|
||||
echo "${ETCD_CLIENT_PRIVATE_KEY}" | base64 --decode > "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
echo "${ETCD_PEER_KEY}" | base64 --decode > "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
echo "${ETCD_SERVER_CERTIFICATE}" | base64 --decode > "${ETCD_SERVER_CERTIFICATE_PATH}"
|
||||
echo "${ETCD_CLIENT_CERTIFICATE}" | base64 --decode > "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
echo "${ETCD_PEER_CERT}" | base64 --decode > "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
set -x
|
||||
|
||||
/opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1
|
||||
RET=$?
|
||||
if [ $RET -ne 0 ]; then
|
||||
exit $RET
|
||||
fi
|
||||
|
||||
/opt/azure/containers/mountetcd.sh || exit $ERR_ETCD_VOL_MOUNT_FAIL
|
||||
systemctlEnableAndStart etcd
|
||||
for i in $(seq 1 600); do
|
||||
MEMBER="$(sudo etcdctl member list | grep -E ${MASTER_VM_NAME} | cut -d':' -f 1)"
|
||||
if [ "$MEMBER" != "" ]; then
|
||||
break
|
||||
else
|
||||
sleep 1
|
||||
fi
|
||||
done
|
||||
retrycmd_if_failure 10 1 5 sudo etcdctl member update $MEMBER ${ETCD_PEER_URL} || exit $ERR_ETCD_CONFIG_FAIL
|
||||
}
|
||||
|
||||
function ensureRPC() {
|
||||
systemctlEnableAndStart rpcbind
|
||||
systemctlEnableAndStart rpc-statd
|
||||
}
|
||||
|
||||
function runAptDaily() {
|
||||
/usr/lib/apt/apt.systemd.daily
|
||||
}
|
||||
|
||||
function generateAggregatedAPICerts() {
|
||||
wait_for_file 1 1 /etc/kubernetes/generate-proxy-certs.sh && /etc/kubernetes/generate-proxy-certs.sh
|
||||
}
|
||||
|
||||
function configureK8s() {
|
||||
KUBELET_PRIVATE_KEY_PATH="/etc/kubernetes/certs/client.key"
|
||||
touch "${KUBELET_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${KUBELET_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${KUBELET_PRIVATE_KEY_PATH}"
|
||||
|
||||
APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt"
|
||||
touch "${APISERVER_PUBLIC_KEY_PATH}"
|
||||
chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}"
|
||||
chown root:root "${APISERVER_PUBLIC_KEY_PATH}"
|
||||
|
||||
AZURE_JSON_PATH="/etc/kubernetes/azure.json"
|
||||
touch "${AZURE_JSON_PATH}"
|
||||
chmod 0600 "${AZURE_JSON_PATH}"
|
||||
chown root:root "${AZURE_JSON_PATH}"
|
||||
|
||||
set +x
|
||||
echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}"
|
||||
echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
|
||||
cat << EOF > "${AZURE_JSON_PATH}"
|
||||
{
|
||||
"cloud":"${TARGET_ENVIRONMENT}",
|
||||
"tenantId": "${TENANT_ID}",
|
||||
"subscriptionId": "${SUBSCRIPTION_ID}",
|
||||
"aadClientId": "${SERVICE_PRINCIPAL_CLIENT_ID}",
|
||||
"aadClientSecret": "${SERVICE_PRINCIPAL_CLIENT_SECRET}",
|
||||
"resourceGroup": "${RESOURCE_GROUP}",
|
||||
"location": "${LOCATION}",
|
||||
"vmType": "${VM_TYPE}",
|
||||
"subnetName": "${SUBNET}",
|
||||
"securityGroupName": "${NETWORK_SECURITY_GROUP}",
|
||||
"vnetName": "${VIRTUAL_NETWORK}",
|
||||
"vnetResourceGroup": "${VIRTUAL_NETWORK_RESOURCE_GROUP}",
|
||||
"routeTableName": "${ROUTE_TABLE}",
|
||||
"primaryAvailabilitySetName": "${PRIMARY_AVAILABILITY_SET}",
|
||||
"primaryScaleSetName": "${PRIMARY_SCALE_SET}",
|
||||
"cloudProviderBackoff": ${CLOUDPROVIDER_BACKOFF},
|
||||
"cloudProviderBackoffRetries": ${CLOUDPROVIDER_BACKOFF_RETRIES},
|
||||
"cloudProviderBackoffExponent": ${CLOUDPROVIDER_BACKOFF_EXPONENT},
|
||||
"cloudProviderBackoffDuration": ${CLOUDPROVIDER_BACKOFF_DURATION},
|
||||
"cloudProviderBackoffJitter": ${CLOUDPROVIDER_BACKOFF_JITTER},
|
||||
"cloudProviderRatelimit": ${CLOUDPROVIDER_RATELIMIT},
|
||||
"cloudProviderRateLimitQPS": ${CLOUDPROVIDER_RATELIMIT_QPS},
|
||||
"cloudProviderRateLimitBucket": ${CLOUDPROVIDER_RATELIMIT_BUCKET},
|
||||
"useManagedIdentityExtension": ${USE_MANAGED_IDENTITY_EXTENSION},
|
||||
"useInstanceMetadata": ${USE_INSTANCE_METADATA},
|
||||
"providerVaultName": "${KMS_PROVIDER_VAULT_NAME}",
|
||||
"providerKeyName": "k8s",
|
||||
"providerKeyVersion": ""
|
||||
}
|
||||
EOF
|
||||
set -x
|
||||
generateAggregatedAPICerts
|
||||
}
|
||||
|
||||
function configureCNI() {
|
||||
# Turn on br_netfilter (needed for the iptables rules to work on bridges)
|
||||
# and permanently enable it
|
||||
retrycmd_if_failure 30 6 10 modprobe br_netfilter || exit $ERR_MODPROBE_FAIL
|
||||
# /etc/modules-load.d is the location used by systemd to load modules
|
||||
echo -n "br_netfilter" > /etc/modules-load.d/br_netfilter.conf
|
||||
if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then
|
||||
mv $CNI_BIN_DIR/10-azure.conflist $CNI_CONFIG_DIR/
|
||||
chmod 600 $CNI_CONFIG_DIR/10-azure.conflist
|
||||
/sbin/ebtables -t nat --list
|
||||
fi
|
||||
}
|
||||
|
||||
function setKubeletOpts () {
|
||||
sed -i "s#^KUBELET_OPTS=.*#KUBELET_OPTS=${1}#" /etc/default/kubelet
|
||||
}
|
||||
|
||||
function ensureCCProxy() {
|
||||
cat $CC_SERVICE_IN_TMP | sed 's#@libexecdir@#/usr/libexec#' > /etc/systemd/system/cc-proxy.service
|
||||
cat $CC_SOCKET_IN_TMP sed 's#@localstatedir@#/var#' > /etc/systemd/system/cc-proxy.socket
|
||||
# Enable and start Clear Containers proxy service
|
||||
echo "Enabling and starting Clear Containers proxy service..."
|
||||
systemctlEnableAndStart cc-proxy
|
||||
}
|
||||
|
||||
function setupContainerd() {
|
||||
echo "Configuring cri-containerd..."
|
||||
mkdir -p "/etc/containerd"
|
||||
CRI_CONTAINERD_CONFIG="/etc/containerd/config.toml"
|
||||
echo "subreaper = false" > "$CRI_CONTAINERD_CONFIG"
|
||||
echo "oom_score = 0" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "[plugins.cri]" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "sandbox_image = \"$POD_INFRA_CONTAINER_SPEC\"" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "[plugins.cri.containerd.untrusted_workload_runtime]" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
|
||||
echo "runtime_engine = '/usr/bin/cc-runtime'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
elif [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then
|
||||
echo "runtime_engine = '/usr/bin/kata-runtime'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
else
|
||||
echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
fi
|
||||
echo "[plugins.cri.containerd.default_runtime]" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
|
||||
}
|
||||
|
||||
function ensureContainerd() {
|
||||
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
|
||||
setupContainerd
|
||||
# Enable and start cri-containerd service
|
||||
# Make sure this is done after networking plugins are installed
|
||||
echo "Enabling and starting cri-containerd service..."
|
||||
systemctlEnableAndStart containerd
|
||||
fi
|
||||
}
|
||||
|
||||
function ensureDocker() {
|
||||
echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf
|
||||
usermod -aG docker ${ADMINUSER}
|
||||
wait_for_file 600 1 $DOCKER || exit $ERR_FILE_WATCH_TIMEOUT
|
||||
systemctlEnableAndStart docker
|
||||
retrycmd_if_failure 6 1 10 docker pull busybox # pre-pull busybox, but don't exit if fail
|
||||
systemctlEnableAndStart docker-health-probe
|
||||
}
|
||||
function ensureKMS() {
|
||||
systemctlEnableAndStart kms
|
||||
}
|
||||
|
||||
function ensureKubelet() {
|
||||
systemctlEnableAndStart kubelet
|
||||
}
|
||||
|
||||
function ensureJournal(){
|
||||
echo "Storage=persistent" >> /etc/systemd/journald.conf
|
||||
echo "SystemMaxUse=1G" >> /etc/systemd/journald.conf
|
||||
echo "RuntimeMaxUse=1G" >> /etc/systemd/journald.conf
|
||||
echo "ForwardToSyslog=no" >> /etc/systemd/journald.conf
|
||||
systemctlEnableAndStart systemd-journald
|
||||
}
|
||||
|
||||
function ensurePodSecurityPolicy() {
|
||||
POD_SECURITY_POLICY_FILE="/etc/kubernetes/manifests/pod-security-policy.yaml"
|
||||
if [ -f $POD_SECURITY_POLICY_FILE ]; then
|
||||
$KUBECTL create -f $POD_SECURITY_POLICY_FILE
|
||||
fi
|
||||
}
|
||||
|
||||
function ensureK8sControlPlane() {
|
||||
if $REBOOTREQUIRED; then
|
||||
return
|
||||
fi
|
||||
wait_for_file 600 1 $KUBECTL || exit $ERR_FILE_WATCH_TIMEOUT
|
||||
retrycmd_if_failure 900 1 20 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT
|
||||
ensurePodSecurityPolicy
|
||||
}
|
||||
|
||||
function ensureEtcd() {
|
||||
retrycmd_if_failure 120 5 10 curl --cacert /etc/kubernetes/certs/ca.crt --cert /etc/kubernetes/certs/etcdclient.crt --key /etc/kubernetes/certs/etcdclient.key ${ETCD_CLIENT_URL}/v2/machines || exit $ERR_ETCD_RUNNING_TIMEOUT
|
||||
}
|
||||
|
||||
function writeKubeConfig() {
|
||||
KUBECONFIGDIR=/home/$ADMINUSER/.kube
|
||||
KUBECONFIGFILE=$KUBECONFIGDIR/config
|
||||
mkdir -p $KUBECONFIGDIR
|
||||
touch $KUBECONFIGFILE
|
||||
chown $ADMINUSER:$ADMINUSER $KUBECONFIGDIR
|
||||
chown $ADMINUSER:$ADMINUSER $KUBECONFIGFILE
|
||||
chmod 700 $KUBECONFIGDIR
|
||||
chmod 600 $KUBECONFIGFILE
|
||||
|
||||
# disable logging after secret output
|
||||
set +x
|
||||
echo "
|
||||
---
|
||||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
certificate-authority-data: \"$CA_CERTIFICATE\"
|
||||
server: $KUBECONFIG_SERVER
|
||||
name: \"$MASTER_FQDN\"
|
||||
contexts:
|
||||
- context:
|
||||
cluster: \"$MASTER_FQDN\"
|
||||
user: \"$MASTER_FQDN-admin\"
|
||||
name: \"$MASTER_FQDN\"
|
||||
current-context: \"$MASTER_FQDN\"
|
||||
kind: Config
|
||||
users:
|
||||
- name: \"$MASTER_FQDN-admin\"
|
||||
user:
|
||||
client-certificate-data: \"$KUBECONFIG_CERTIFICATE\"
|
||||
client-key-data: \"$KUBECONFIG_KEY\"
|
||||
" > $KUBECONFIGFILE
|
||||
# renable logging after secrets
|
||||
set -x
|
||||
}
|
||||
|
||||
function configClusterAutoscalerAddon() {
|
||||
if [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == true ]]; then
|
||||
CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT="- mountPath: /var/lib/waagent/\n\ name: waagent\n\ readOnly: true"
|
||||
CLUSTER_AUTOSCALER_MSI_VOLUME="- hostPath:\n\ path: /var/lib/waagent/\n\ name: waagent"
|
||||
CLUSTER_AUTOSCALER_MSI_HOST_NETWORK="hostNetwork: true"
|
||||
|
||||
sed -i "s|<kubernetesClusterAutoscalerVolumeMounts>|${CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerVolumes>|${CLUSTER_AUTOSCALER_MSI_VOLUME}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerHostNetwork>|$(echo "${CLUSTER_AUTOSCALER_MSI_HOST_NETWORK}")|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
elif [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == false ]]; then
|
||||
sed -i "s|<kubernetesClusterAutoscalerVolumeMounts>|""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerVolumes>|""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerHostNetwork>|""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
fi
|
||||
|
||||
sed -i "s|<kubernetesClusterAutoscalerClientId>|$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerClientSecret>|$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerSubscriptionId>|$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerTenantId>|$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerResourceGroup>|$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerVmType>|$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerVMSSName>|$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
}
|
||||
|
||||
configACIConnectorAddon() {
|
||||
ACI_CONNECTOR_CREDENTIALS=$(printf "{\"clientId\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_ID)\", \"clientSecret\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET)\", \"tenantId\": \"$(echo $TENANT_ID)\", \"subscriptionId\": \"$(echo $SUBSCRIPTION_ID)\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",\"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\"}" | base64 -w 0)
|
||||
|
||||
openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 -keyout /etc/kubernetes/certs/aci-connector-key.pem -out /etc/kubernetes/certs/aci-connector-cert.pem -subj "/C=US/ST=CA/L=virtualkubelet/O=virtualkubelet/OU=virtualkubelet/CN=virtualkubelet"
|
||||
ACI_CONNECTOR_KEY=$(base64 /etc/kubernetes/certs/aci-connector-key.pem -w0)
|
||||
ACI_CONNECTOR_CERT=$(base64 /etc/kubernetes/certs/aci-connector-cert.pem -w0)
|
||||
|
||||
sed -i "s|<kubernetesACIConnectorCredentials>|$ACI_CONNECTOR_CREDENTIALS|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorResourceGroup>|$(echo $RESOURCE_GROUP)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorCert>|$(echo $ACI_CONNECTOR_CERT)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorKey>|$(echo $ACI_CONNECTOR_KEY)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
}
|
||||
|
||||
configAddons() {
|
||||
if [[ "${CLUSTER_AUTOSCALER_ADDON}" = True ]]; then
|
||||
configClusterAutoscalerAddon
|
||||
fi
|
||||
|
||||
if [[ "${ACI_CONNECTOR_ADDON}" = True ]]; then
|
||||
configACIConnectorAddon
|
||||
fi
|
||||
}
|
||||
|
|
@ -2,47 +2,9 @@
|
|||
set -x
|
||||
echo `date`,`hostname`, startscript>>/opt/m
|
||||
source /opt/azure/containers/provision_source.sh
|
||||
# TODO standardize/generalize CSE exit codes
|
||||
ERR_SYSTEMCTL_ENABLE_FAIL=3 # Service could not be enabled by systemctl
|
||||
ERR_SYSTEMCTL_START_FAIL=4 # Service could not be started by systemctl
|
||||
ERR_CLOUD_INIT_TIMEOUT=5 # Timeout waiting for cloud-init runcmd to complete
|
||||
ERR_FILE_WATCH_TIMEOUT=6 # Timeout waiting for a file
|
||||
ERR_HOLD_WALINUXAGENT=7 # Unable to place walinuxagent apt package on hold during install
|
||||
ERR_RELEASE_HOLD_WALINUXAGENT=8 # Unable to release hold on walinuxagent apt package after install
|
||||
ERR_APT_INSTALL_TIMEOUT=9 # Timeout installing required apt packages
|
||||
ERR_ETCD_DATA_DIR_NOT_FOUND=10 # Etcd data dir not found
|
||||
ERR_ETCD_RUNNING_TIMEOUT=11 # Timeout waiting for etcd to be accessible
|
||||
ERR_ETCD_DOWNLOAD_TIMEOUT=12 # Timeout waiting for etcd to download
|
||||
ERR_ETCD_VOL_MOUNT_FAIL=13 # Unable to mount etcd disk volume
|
||||
ERR_ETCD_START_TIMEOUT=14 # Unable to start etcd runtime
|
||||
ERR_ETCD_CONFIG_FAIL=15 # Unable to configure etcd cluster
|
||||
ERR_DOCKER_INSTALL_TIMEOUT=20 # Timeout waiting for docker install
|
||||
ERR_DOCKER_DOWNLOAD_TIMEOUT=21 # Timout waiting for docker download(s)
|
||||
ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT=22 # Timeout waiting to download docker repo key
|
||||
ERR_DOCKER_APT_KEY_TIMEOUT=23 # Timeout waiting for docker apt-key
|
||||
ERR_K8S_RUNNING_TIMEOUT=30 # Timeout waiting for k8s cluster to be healthy
|
||||
ERR_K8S_DOWNLOAD_TIMEOUT=31 # Timeout waiting for Kubernetes download(s)
|
||||
ERR_KUBECTL_NOT_FOUND=32 # kubectl client binary not found on local disk
|
||||
ERR_CNI_DOWNLOAD_TIMEOUT=41 # Timeout waiting for CNI download(s)
|
||||
ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT=42 # Timeout waiting for https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb
|
||||
ERR_MS_PROD_DEB_PKG_ADD_FAIL=43 # Failed to add repo pkg file
|
||||
#ERR_FLEXVOLUME_DOWNLOAD_TIMEOUT=44 # Failed to add repo pkg file -- DEPRECATED
|
||||
ERR_MODPROBE_FAIL=49 # Unable to load a kernel module using modprobe
|
||||
ERR_OUTBOUND_CONN_FAIL=50 # Unable to establish outbound connection
|
||||
ERR_KATA_KEY_DOWNLOAD_TIMEOUT=60 # Timeout waiting to download kata repo key
|
||||
ERR_KATA_APT_KEY_TIMEOUT=61 # Timeout waiting for kata apt-key
|
||||
ERR_KATA_INSTALL_TIMEOUT=62 # Timeout waiting for kata install
|
||||
ERR_CUSTOM_SEARCH_DOMAINS_FAIL=80 # Unable to configure custom search domains
|
||||
ERR_APT_DAILY_TIMEOUT=98 # Timeout waiting for apt daily updates
|
||||
ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete
|
||||
source /opt/azure/containers/provision_installs.sh
|
||||
source /opt/azure/containers/provision_configs.sh
|
||||
|
||||
OS=$(cat /etc/*-release | grep ^ID= | tr -d 'ID="' | awk '{print toupper($0)}')
|
||||
UBUNTU_OS_NAME="UBUNTU"
|
||||
RHEL_OS_NAME="RHEL"
|
||||
COREOS_OS_NAME="COREOS"
|
||||
KUBECTL=/usr/local/bin/kubectl
|
||||
DOCKER=/usr/bin/docker
|
||||
CNI_BIN_DIR=/opt/cni/bin
|
||||
CUSTOM_SEARCH_DOMAIN_SCRIPT=/opt/azure/containers/setup-custom-search-domains.sh
|
||||
|
||||
set +x
|
||||
|
|
@ -61,6 +23,13 @@ else
|
|||
REBOOTREQUIRED=false
|
||||
fi
|
||||
|
||||
if [ -f /var/log/azure/golden-image-install.complete ]; then
|
||||
echo "detected golden image pre-install"
|
||||
FULL_INSTALL_REQUIRED=false
|
||||
else
|
||||
FULL_INSTALL_REQUIRED=true
|
||||
fi
|
||||
|
||||
function testOutboundConnection() {
|
||||
retrycmd_if_failure 20 1 3 nc -v www.google.com 443 || retrycmd_if_failure 20 1 3 nc -v www.1688.com 443 || exit $ERR_OUTBOUND_CONN_FAIL
|
||||
}
|
||||
|
|
@ -69,538 +38,72 @@ function waitForCloudInit() {
|
|||
wait_for_file 900 1 /var/log/azure/cloud-init.complete || exit $ERR_CLOUD_INIT_TIMEOUT
|
||||
}
|
||||
|
||||
function systemctlEnableAndStart() {
|
||||
systemctl_restart 100 5 30 $1
|
||||
RESTART_STATUS=$?
|
||||
systemctl status $1 --no-pager -l > /var/log/azure/$1-status.log
|
||||
if [ $RESTART_STATUS -ne 0 ]; then
|
||||
echo "$1 could not be started"
|
||||
exit $ERR_SYSTEMCTL_START_FAIL
|
||||
fi
|
||||
retrycmd_if_failure 10 5 3 systemctl enable $1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "$1 could not be enabled by systemctl"
|
||||
exit $ERR_SYSTEMCTL_ENABLE_FAIL
|
||||
fi
|
||||
}
|
||||
|
||||
function installEtcd() {
|
||||
useradd -U "etcd"
|
||||
usermod -p "$(head -c 32 /dev/urandom | base64)" "etcd"
|
||||
passwd -u "etcd"
|
||||
id "etcd"
|
||||
|
||||
APISERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/apiserver.key"
|
||||
touch "${APISERVER_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${APISERVER_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${APISERVER_PRIVATE_KEY_PATH}"
|
||||
|
||||
CA_PRIVATE_KEY_PATH="/etc/kubernetes/certs/ca.key"
|
||||
touch "${CA_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${CA_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${CA_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdserver.key"
|
||||
touch "${ETCD_SERVER_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${ETCD_SERVER_PRIVATE_KEY_PATH}"
|
||||
chown etcd:etcd "${ETCD_SERVER_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_CLIENT_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdclient.key"
|
||||
touch "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.key"
|
||||
touch "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
chown etcd:etcd "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_SERVER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdserver.crt"
|
||||
touch "${ETCD_SERVER_CERTIFICATE_PATH}"
|
||||
chmod 0644 "${ETCD_SERVER_CERTIFICATE_PATH}"
|
||||
chown root:root "${ETCD_SERVER_CERTIFICATE_PATH}"
|
||||
|
||||
ETCD_CLIENT_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdclient.crt"
|
||||
touch "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
chmod 0644 "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
chown root:root "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
|
||||
ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.crt"
|
||||
touch "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
chmod 0644 "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
chown root:root "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
|
||||
set +x
|
||||
echo "${APISERVER_PRIVATE_KEY}" | base64 --decode > "${APISERVER_PRIVATE_KEY_PATH}"
|
||||
echo "${CA_PRIVATE_KEY}" | base64 --decode > "${CA_PRIVATE_KEY_PATH}"
|
||||
echo "${ETCD_SERVER_PRIVATE_KEY}" | base64 --decode > "${ETCD_SERVER_PRIVATE_KEY_PATH}"
|
||||
echo "${ETCD_CLIENT_PRIVATE_KEY}" | base64 --decode > "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
echo "${ETCD_PEER_KEY}" | base64 --decode > "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
echo "${ETCD_SERVER_CERTIFICATE}" | base64 --decode > "${ETCD_SERVER_CERTIFICATE_PATH}"
|
||||
echo "${ETCD_CLIENT_CERTIFICATE}" | base64 --decode > "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
echo "${ETCD_PEER_CERT}" | base64 --decode > "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
set -x
|
||||
|
||||
/opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1
|
||||
RET=$?
|
||||
if [ $RET -ne 0 ]; then
|
||||
exit $RET
|
||||
fi
|
||||
|
||||
/opt/azure/containers/mountetcd.sh || exit $ERR_ETCD_VOL_MOUNT_FAIL
|
||||
systemctlEnableAndStart etcd
|
||||
for i in $(seq 1 600); do
|
||||
MEMBER="$(sudo etcdctl member list | grep -E ${MASTER_VM_NAME} | cut -d':' -f 1)"
|
||||
if [ "$MEMBER" != "" ]; then
|
||||
break
|
||||
else
|
||||
sleep 1
|
||||
fi
|
||||
done
|
||||
retrycmd_if_failure 10 1 5 sudo etcdctl member update $MEMBER ${ETCD_PEER_URL} || exit $ERR_ETCD_CONFIG_FAIL
|
||||
}
|
||||
|
||||
function installDeps() {
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT
|
||||
retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL
|
||||
echo `date`,`hostname`, apt-get_update_begin>>/opt/m
|
||||
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
|
||||
echo `date`,`hostname`, apt-get_update_end>>/opt/m
|
||||
# make sure walinuxagent doesn't get updated in the middle of running this script
|
||||
retrycmd_if_failure 20 5 30 apt-mark hold walinuxagent || exit $ERR_HOLD_WALINUXAGENT
|
||||
# See https://github.com/kubernetes/kubernetes/blob/master/build/debian-hyperkube-base/Dockerfile#L25-L44
|
||||
apt_get_install 20 30 300 apt-transport-https ca-certificates iptables iproute2 ebtables socat util-linux mount ethtool init-system-helpers nfs-common ceph-common conntrack glusterfs-client ipset jq cgroup-lite git pigz xz-utils blobfuse fuse cifs-utils || exit $ERR_APT_INSTALL_TIMEOUT
|
||||
systemctlEnableAndStart rpcbind
|
||||
systemctlEnableAndStart rpc-statd
|
||||
}
|
||||
|
||||
function installDocker() {
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg || exit $ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT
|
||||
retrycmd_if_failure 10 5 10 apt-key add /tmp/aptdocker.gpg || exit $ERR_DOCKER_APT_KEY_TIMEOUT
|
||||
echo "deb ${DOCKER_REPO} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
|
||||
printf "Package: docker-engine\nPin: version ${DOCKER_ENGINE_VERSION}\nPin-Priority: 550\n" > /etc/apt/preferences.d/docker.pref
|
||||
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
|
||||
apt_get_install 20 30 120 docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT
|
||||
echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf
|
||||
usermod -aG docker ${ADMINUSER}
|
||||
touch /var/log/azure/docker-install.complete
|
||||
}
|
||||
|
||||
function runAptDaily() {
|
||||
/usr/lib/apt/apt.systemd.daily
|
||||
}
|
||||
|
||||
function generateAggregatedAPICerts() {
|
||||
wait_for_file 1 1 /etc/kubernetes/generate-proxy-certs.sh && /etc/kubernetes/generate-proxy-certs.sh
|
||||
}
|
||||
|
||||
function configureK8s() {
|
||||
KUBELET_PRIVATE_KEY_PATH="/etc/kubernetes/certs/client.key"
|
||||
touch "${KUBELET_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${KUBELET_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${KUBELET_PRIVATE_KEY_PATH}"
|
||||
|
||||
APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt"
|
||||
touch "${APISERVER_PUBLIC_KEY_PATH}"
|
||||
chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}"
|
||||
chown root:root "${APISERVER_PUBLIC_KEY_PATH}"
|
||||
|
||||
AZURE_JSON_PATH="/etc/kubernetes/azure.json"
|
||||
touch "${AZURE_JSON_PATH}"
|
||||
chmod 0600 "${AZURE_JSON_PATH}"
|
||||
chown root:root "${AZURE_JSON_PATH}"
|
||||
|
||||
set +x
|
||||
echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}"
|
||||
echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
|
||||
cat << EOF > "${AZURE_JSON_PATH}"
|
||||
{
|
||||
"cloud":"${TARGET_ENVIRONMENT}",
|
||||
"tenantId": "${TENANT_ID}",
|
||||
"subscriptionId": "${SUBSCRIPTION_ID}",
|
||||
"aadClientId": "${SERVICE_PRINCIPAL_CLIENT_ID}",
|
||||
"aadClientSecret": "${SERVICE_PRINCIPAL_CLIENT_SECRET}",
|
||||
"resourceGroup": "${RESOURCE_GROUP}",
|
||||
"location": "${LOCATION}",
|
||||
"vmType": "${VM_TYPE}",
|
||||
"subnetName": "${SUBNET}",
|
||||
"securityGroupName": "${NETWORK_SECURITY_GROUP}",
|
||||
"vnetName": "${VIRTUAL_NETWORK}",
|
||||
"vnetResourceGroup": "${VIRTUAL_NETWORK_RESOURCE_GROUP}",
|
||||
"routeTableName": "${ROUTE_TABLE}",
|
||||
"primaryAvailabilitySetName": "${PRIMARY_AVAILABILITY_SET}",
|
||||
"primaryScaleSetName": "${PRIMARY_SCALE_SET}",
|
||||
"cloudProviderBackoff": ${CLOUDPROVIDER_BACKOFF},
|
||||
"cloudProviderBackoffRetries": ${CLOUDPROVIDER_BACKOFF_RETRIES},
|
||||
"cloudProviderBackoffExponent": ${CLOUDPROVIDER_BACKOFF_EXPONENT},
|
||||
"cloudProviderBackoffDuration": ${CLOUDPROVIDER_BACKOFF_DURATION},
|
||||
"cloudProviderBackoffJitter": ${CLOUDPROVIDER_BACKOFF_JITTER},
|
||||
"cloudProviderRatelimit": ${CLOUDPROVIDER_RATELIMIT},
|
||||
"cloudProviderRateLimitQPS": ${CLOUDPROVIDER_RATELIMIT_QPS},
|
||||
"cloudProviderRateLimitBucket": ${CLOUDPROVIDER_RATELIMIT_BUCKET},
|
||||
"useManagedIdentityExtension": ${USE_MANAGED_IDENTITY_EXTENSION},
|
||||
"useInstanceMetadata": ${USE_INSTANCE_METADATA},
|
||||
"providerVaultName": "${KMS_PROVIDER_VAULT_NAME}",
|
||||
"providerKeyName": "k8s",
|
||||
"providerKeyVersion": ""
|
||||
}
|
||||
EOF
|
||||
set -x
|
||||
generateAggregatedAPICerts
|
||||
}
|
||||
|
||||
function setKubeletOpts () {
|
||||
sed -i "s#^KUBELET_OPTS=.*#KUBELET_OPTS=${1}#" /etc/default/kubelet
|
||||
}
|
||||
|
||||
function installCNI() {
|
||||
mkdir -p $CNI_BIN_DIR
|
||||
CONTAINERNETWORKING_CNI_TGZ_TMP=/tmp/containernetworking_cni.tgz
|
||||
retrycmd_get_tarball 60 5 $CONTAINERNETWORKING_CNI_TGZ_TMP ${CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT
|
||||
tar -xzf $CONTAINERNETWORKING_CNI_TGZ_TMP -C $CNI_BIN_DIR
|
||||
chown -R root:root $CNI_BIN_DIR
|
||||
chmod -R 755 $CNI_BIN_DIR
|
||||
# Turn on br_netfilter (needed for the iptables rules to work on bridges)
|
||||
# and permanently enable it
|
||||
retrycmd_if_failure 30 6 10 modprobe br_netfilter || exit $ERR_MODPROBE_FAIL
|
||||
# /etc/modules-load.d is the location used by systemd to load modules
|
||||
echo -n "br_netfilter" > /etc/modules-load.d/br_netfilter.conf
|
||||
}
|
||||
|
||||
function configAzureCNI() {
|
||||
CNI_CONFIG_DIR=/etc/cni/net.d
|
||||
mkdir -p $CNI_CONFIG_DIR
|
||||
chown -R root:root $CNI_CONFIG_DIR
|
||||
chmod 755 $CNI_CONFIG_DIR
|
||||
mkdir -p $CNI_BIN_DIR
|
||||
AZURE_CNI_TGZ_TMP=/tmp/azure_cni.tgz
|
||||
retrycmd_get_tarball 60 5 $AZURE_CNI_TGZ_TMP ${VNET_CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT
|
||||
tar -xzf $AZURE_CNI_TGZ_TMP -C $CNI_BIN_DIR
|
||||
installCNI
|
||||
mv $CNI_BIN_DIR/10-azure.conflist $CNI_CONFIG_DIR/
|
||||
chmod 600 $CNI_CONFIG_DIR/10-azure.conflist
|
||||
/sbin/ebtables -t nat --list
|
||||
}
|
||||
|
||||
function configNetworkPlugin() {
|
||||
if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then
|
||||
configAzureCNI
|
||||
elif [[ "${NETWORK_PLUGIN}" = "kubenet" ]]; then
|
||||
installCNI
|
||||
elif [[ "${NETWORK_PLUGIN}" = "flannel" ]]; then
|
||||
installCNI
|
||||
fi
|
||||
}
|
||||
|
||||
function installKataContainersRuntime() {
|
||||
# Add Kata Containers repository key
|
||||
echo "Adding Kata Containers repository key..."
|
||||
KATA_RELEASE_KEY_TMP=/tmp/kata-containers-release.key
|
||||
KATA_URL=http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/Release.key
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $KATA_URL > $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_KEY_DOWNLOAD_TIMEOUT
|
||||
retrycmd_if_failure 10 5 10 apt-key add $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_APT_KEY_TIMEOUT
|
||||
|
||||
# Add Kata Container repository
|
||||
echo "Adding Kata Containers repository..."
|
||||
echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/kata-containers.list
|
||||
|
||||
# Install Kata Containers runtime
|
||||
echo "Installing Kata Containers runtime..."
|
||||
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
|
||||
apt_get_install 20 30 120 kata-runtime || exit $ERR_KATA_INSTALL_TIMEOUT
|
||||
}
|
||||
|
||||
function installClearContainersRuntime() {
|
||||
# Add Clear Containers repository key
|
||||
echo "Adding Clear Containers repository key..."
|
||||
CC_RELEASE_KEY_TMP=/tmp/clear-containers-release.key
|
||||
CC_URL=https://download.opensuse.org/repositories/home:clearcontainers:clear-containers-3/xUbuntu_16.04/Release.key
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $CC_URL > $CC_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT
|
||||
retrycmd_if_failure 10 5 10 apt-key add $CC_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT
|
||||
|
||||
# Add Clear Container repository
|
||||
echo "Adding Clear Containers repository..."
|
||||
echo 'deb http://download.opensuse.org/repositories/home:/clearcontainers:/clear-containers-3/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/cc-runtime.list
|
||||
|
||||
# Install Clear Containers runtime
|
||||
echo "Installing Clear Containers runtime..."
|
||||
apt_get_update
|
||||
apt_get_install 20 30 120 cc-runtime
|
||||
|
||||
# Install the systemd service and socket files.
|
||||
local repo_uri="https://raw.githubusercontent.com/clearcontainers/proxy/3.0.23"
|
||||
CC_SERVICE_IN_TMP=/tmp/cc-proxy.service.in
|
||||
CC_SOCKET_IN_TMP=/tmp/cc-proxy.socket.in
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL "${repo_uri}/cc-proxy.service.in" > $CC_SERVICE_IN_TMP
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL "${repo_uri}/cc-proxy.socket.in" > $CC_SOCKET_IN_TMP
|
||||
cat $CC_SERVICE_IN_TMP | sed 's#@libexecdir@#/usr/libexec#' > /etc/systemd/system/cc-proxy.service
|
||||
cat $CC_SOCKET_IN_TMP sed 's#@localstatedir@#/var#' > /etc/systemd/system/cc-proxy.socket
|
||||
|
||||
# Enable and start Clear Containers proxy service
|
||||
echo "Enabling and starting Clear Containers proxy service..."
|
||||
systemctlEnableAndStart cc-proxy
|
||||
}
|
||||
|
||||
function setupContainerd() {
|
||||
echo "Configuring cri-containerd..."
|
||||
|
||||
mkdir -p "/etc/containerd"
|
||||
CRI_CONTAINERD_CONFIG="/etc/containerd/config.toml"
|
||||
echo "subreaper = false" > "$CRI_CONTAINERD_CONFIG"
|
||||
echo "oom_score = 0" >> "$CRI_CONTAINERD_CONFIG"
|
||||
|
||||
echo "[plugins.cri]" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "sandbox_image = \"$POD_INFRA_CONTAINER_SPEC\"" >> "$CRI_CONTAINERD_CONFIG"
|
||||
|
||||
echo "[plugins.cri.containerd.untrusted_workload_runtime]" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
|
||||
echo "runtime_engine = '/usr/bin/cc-runtime'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
elif [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then
|
||||
echo "runtime_engine = '/usr/bin/kata-runtime'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
else
|
||||
echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
fi
|
||||
echo "[plugins.cri.containerd.default_runtime]" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG"
|
||||
|
||||
setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
|
||||
}
|
||||
|
||||
function installContainerd() {
|
||||
CRI_CONTAINERD_VERSION="1.1.0"
|
||||
CONTAINERD_DOWNLOAD_URL="${CONTAINERD_DOWNLOAD_URL_BASE}cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz"
|
||||
|
||||
CONTAINERD_TGZ_TMP=/tmp/containerd.tar.gz
|
||||
retrycmd_get_tarball 60 5 "$CONTAINERD_TGZ_TMP" "$CONTAINERD_DOWNLOAD_URL"
|
||||
tar -xzf "$CONTAINERD_TGZ_TMP" -C /
|
||||
rm -f "$CONTAINERD_TGZ_TMP"
|
||||
sed -i '/\[Service\]/a ExecStartPost=\/sbin\/iptables -P FORWARD ACCEPT' /etc/systemd/system/containerd.service
|
||||
|
||||
echo "Successfully installed cri-containerd..."
|
||||
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
|
||||
setupContainerd
|
||||
fi
|
||||
}
|
||||
|
||||
function ensureContainerd() {
|
||||
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
|
||||
# Enable and start cri-containerd service
|
||||
# Make sure this is done after networking plugins are installed
|
||||
echo "Enabling and starting cri-containerd service..."
|
||||
systemctlEnableAndStart containerd
|
||||
fi
|
||||
}
|
||||
|
||||
function ensureDocker() {
|
||||
wait_for_file 600 1 $DOCKER || exit $ERR_FILE_WATCH_TIMEOUT
|
||||
systemctlEnableAndStart docker
|
||||
retrycmd_if_failure 6 1 10 docker pull busybox # pre-pull busybox, but don't exit if fail
|
||||
systemctlEnableAndStart docker-health-probe
|
||||
}
|
||||
function ensureKMS() {
|
||||
systemctlEnableAndStart kms
|
||||
}
|
||||
|
||||
function ensureKubelet() {
|
||||
systemctlEnableAndStart kubelet
|
||||
}
|
||||
|
||||
function extractHyperkube(){
|
||||
TMP_DIR=$(mktemp -d)
|
||||
retrycmd_if_failure 100 1 30 curl -sSL -o /usr/local/bin/img "https://acs-mirror.azureedge.net/img/img-linux-amd64-v0.4.6"
|
||||
chmod +x /usr/local/bin/img
|
||||
retrycmd_if_failure 75 1 60 img pull $HYPERKUBE_URL || exit $ERR_K8S_DOWNLOAD_TIMEOUT
|
||||
path=$(find /tmp/img -name "hyperkube")
|
||||
|
||||
if [[ $OS == $COREOS_OS_NAME ]]; then
|
||||
cp "$path" "/opt/kubelet"
|
||||
cp "$path" "/opt/kubectl"
|
||||
chmod a+x /opt/kubelet /opt/kubectl
|
||||
else
|
||||
cp "$path" "/usr/local/bin/kubelet"
|
||||
cp "$path" "/usr/local/bin/kubectl"
|
||||
chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
|
||||
fi
|
||||
rm -rf /tmp/hyperkube.tar "/tmp/img"
|
||||
}
|
||||
|
||||
function ensureJournal(){
|
||||
echo "Storage=persistent" >> /etc/systemd/journald.conf
|
||||
echo "SystemMaxUse=1G" >> /etc/systemd/journald.conf
|
||||
echo "RuntimeMaxUse=1G" >> /etc/systemd/journald.conf
|
||||
echo "ForwardToSyslog=no" >> /etc/systemd/journald.conf
|
||||
systemctlEnableAndStart systemd-journald
|
||||
}
|
||||
|
||||
function ensurePodSecurityPolicy() {
|
||||
POD_SECURITY_POLICY_FILE="/etc/kubernetes/manifests/pod-security-policy.yaml"
|
||||
if [ -f $POD_SECURITY_POLICY_FILE ]; then
|
||||
$KUBECTL create -f $POD_SECURITY_POLICY_FILE
|
||||
fi
|
||||
}
|
||||
|
||||
function ensureK8sControlPlane() {
|
||||
if $REBOOTREQUIRED; then
|
||||
return
|
||||
fi
|
||||
wait_for_file 600 1 $KUBECTL || exit $ERR_FILE_WATCH_TIMEOUT
|
||||
retrycmd_if_failure 600 1 20 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT
|
||||
ensurePodSecurityPolicy
|
||||
}
|
||||
|
||||
function ensureEtcd() {
|
||||
retrycmd_if_failure 120 5 10 curl --cacert /etc/kubernetes/certs/ca.crt --cert /etc/kubernetes/certs/etcdclient.crt --key /etc/kubernetes/certs/etcdclient.key ${ETCD_CLIENT_URL}/v2/machines || exit $ERR_ETCD_RUNNING_TIMEOUT
|
||||
}
|
||||
|
||||
function writeKubeConfig() {
|
||||
KUBECONFIGDIR=/home/$ADMINUSER/.kube
|
||||
KUBECONFIGFILE=$KUBECONFIGDIR/config
|
||||
mkdir -p $KUBECONFIGDIR
|
||||
touch $KUBECONFIGFILE
|
||||
chown $ADMINUSER:$ADMINUSER $KUBECONFIGDIR
|
||||
chown $ADMINUSER:$ADMINUSER $KUBECONFIGFILE
|
||||
chmod 700 $KUBECONFIGDIR
|
||||
chmod 600 $KUBECONFIGFILE
|
||||
|
||||
# disable logging after secret output
|
||||
set +x
|
||||
echo "
|
||||
---
|
||||
apiVersion: v1
|
||||
clusters:
|
||||
- cluster:
|
||||
certificate-authority-data: \"$CA_CERTIFICATE\"
|
||||
server: $KUBECONFIG_SERVER
|
||||
name: \"$MASTER_FQDN\"
|
||||
contexts:
|
||||
- context:
|
||||
cluster: \"$MASTER_FQDN\"
|
||||
user: \"$MASTER_FQDN-admin\"
|
||||
name: \"$MASTER_FQDN\"
|
||||
current-context: \"$MASTER_FQDN\"
|
||||
kind: Config
|
||||
users:
|
||||
- name: \"$MASTER_FQDN-admin\"
|
||||
user:
|
||||
client-certificate-data: \"$KUBECONFIG_CERTIFICATE\"
|
||||
client-key-data: \"$KUBECONFIG_KEY\"
|
||||
" > $KUBECONFIGFILE
|
||||
# renable logging after secrets
|
||||
set -x
|
||||
}
|
||||
|
||||
function configClusterAutoscalerAddon() {
|
||||
if [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == true ]]; then
|
||||
CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT="- mountPath: /var/lib/waagent/\n\ name: waagent\n\ readOnly: true"
|
||||
CLUSTER_AUTOSCALER_MSI_VOLUME="- hostPath:\n\ path: /var/lib/waagent/\n\ name: waagent"
|
||||
CLUSTER_AUTOSCALER_MSI_HOST_NETWORK="hostNetwork: true"
|
||||
|
||||
sed -i "s|<kubernetesClusterAutoscalerVolumeMounts>|${CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerVolumes>|${CLUSTER_AUTOSCALER_MSI_VOLUME}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerHostNetwork>|$(echo "${CLUSTER_AUTOSCALER_MSI_HOST_NETWORK}")|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
elif [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == false ]]; then
|
||||
sed -i "s|<kubernetesClusterAutoscalerVolumeMounts>|""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerVolumes>|""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerHostNetwork>|""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
fi
|
||||
|
||||
sed -i "s|<kubernetesClusterAutoscalerClientId>|$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerClientSecret>|$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerSubscriptionId>|$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerTenantId>|$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerResourceGroup>|$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerVmType>|$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerVMSSName>|$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
}
|
||||
|
||||
configACIConnectorAddon() {
|
||||
ACI_CONNECTOR_CREDENTIALS=$(printf "{\"clientId\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_ID)\", \"clientSecret\": \"$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET)\", \"tenantId\": \"$(echo $TENANT_ID)\", \"subscriptionId\": \"$(echo $SUBSCRIPTION_ID)\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",\"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\"}" | base64 -w 0)
|
||||
|
||||
openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 -keyout /etc/kubernetes/certs/aci-connector-key.pem -out /etc/kubernetes/certs/aci-connector-cert.pem -subj "/C=US/ST=CA/L=virtualkubelet/O=virtualkubelet/OU=virtualkubelet/CN=virtualkubelet"
|
||||
ACI_CONNECTOR_KEY=$(base64 /etc/kubernetes/certs/aci-connector-key.pem -w0)
|
||||
ACI_CONNECTOR_CERT=$(base64 /etc/kubernetes/certs/aci-connector-cert.pem -w0)
|
||||
|
||||
sed -i "s|<kubernetesACIConnectorCredentials>|$ACI_CONNECTOR_CREDENTIALS|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorResourceGroup>|$(echo $RESOURCE_GROUP)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorCert>|$(echo $ACI_CONNECTOR_CERT)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorKey>|$(echo $ACI_CONNECTOR_KEY)|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
}
|
||||
|
||||
configAddons() {
|
||||
if [[ "${CLUSTER_AUTOSCALER_ADDON}" = True ]]; then
|
||||
configClusterAutoscalerAddon
|
||||
fi
|
||||
|
||||
if [[ "${ACI_CONNECTOR_ADDON}" = True ]]; then
|
||||
configACIConnectorAddon
|
||||
function holdWALinuxAgent() {
|
||||
if [[ $OS == $UBUNTU_OS_NAME ]]; then
|
||||
# make sure walinuxagent doesn't get updated in the middle of running this script
|
||||
retrycmd_if_failure 20 5 30 apt-mark hold walinuxagent || exit $ERR_HOLD_WALINUXAGENT
|
||||
fi
|
||||
}
|
||||
|
||||
testOutboundConnection
|
||||
waitForCloudInit
|
||||
holdWALinuxAgent
|
||||
|
||||
if $FULL_INSTALL_REQUIRED; then
|
||||
if [[ ! -z "${MASTER_NODE}" ]]; then
|
||||
installEtcd
|
||||
fi
|
||||
installDeps
|
||||
installContainerRuntime
|
||||
installNetworkPlugin
|
||||
installContainerd
|
||||
extractHyperkube
|
||||
else
|
||||
echo "Golden image; skipping dependencies installation"
|
||||
fi
|
||||
|
||||
ensureRPC
|
||||
|
||||
if [[ ! -z "${MASTER_NODE}" ]]; then
|
||||
echo "executing master node provision operations"
|
||||
installEtcd
|
||||
else
|
||||
echo "skipping master node provision operations, this is an agent node"
|
||||
configureEtcd
|
||||
fi
|
||||
|
||||
if [ -f $CUSTOM_SEARCH_DOMAIN_SCRIPT ]; then
|
||||
$CUSTOM_SEARCH_DOMAIN_SCRIPT > /opt/azure/containers/setup-custom-search-domain.log 2>&1 || exit $ERR_CUSTOM_SEARCH_DOMAINS_FAIL
|
||||
fi
|
||||
|
||||
installDeps
|
||||
|
||||
if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then
|
||||
installDocker
|
||||
ensureDocker
|
||||
elif [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
|
||||
# Ensure we can nest virtualization
|
||||
if grep -q vmx /proc/cpuinfo; then
|
||||
ensureCCProxy
|
||||
fi
|
||||
elif [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then
|
||||
# Ensure we can nest virtualization
|
||||
if grep -q vmx /proc/cpuinfo; then
|
||||
installKataContainersRuntime
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
configureK8s
|
||||
|
||||
configNetworkPlugin
|
||||
configureCNI
|
||||
|
||||
if [[ ! -z "${MASTER_NODE}" ]]; then
|
||||
echo `date`,`hostname`, configAddonsStart>>/opt/m
|
||||
configAddons
|
||||
echo `date`,`hostname`, configAddonsDone>>/opt/m
|
||||
fi
|
||||
|
||||
# containerd needs to be installed before extractHyperkube
|
||||
# so runc is present.
|
||||
echo `date`,`hostname`, installContainerdStart>>/opt/m
|
||||
installContainerd
|
||||
echo `date`,`hostname`, extractHyperkubeStart>>/opt/m
|
||||
extractHyperkube
|
||||
echo `date`,`hostname`, extractHyperkubeDone>>/opt/m
|
||||
|
||||
if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
|
||||
# Ensure we can nest virtualization
|
||||
if grep -q vmx /proc/cpuinfo; then
|
||||
installClearContainersRuntime
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then
|
||||
# Ensure we can nest virtualization
|
||||
if grep -q vmx /proc/cpuinfo; then
|
||||
installKataContainersRuntime
|
||||
fi
|
||||
fi
|
||||
|
||||
echo `date`,`hostname`, ensureContainerdStart>>/opt/m
|
||||
ensureContainerd
|
||||
|
||||
if [[ ! -z "${MASTER_NODE}" && ! -z "${EnableEncryptionWithExternalKms}" ]]; then
|
||||
if [[ ! -z "${MASTER_NODE}" && "${KMS_PROVIDER_VAULT_NAME}" != "" ]]; then
|
||||
ensureKMS
|
||||
fi
|
||||
|
||||
ensureKubelet
|
||||
ensureJournal
|
||||
|
||||
|
||||
if [[ ! -z "${MASTER_NODE}" ]]; then
|
||||
writeKubeConfig
|
||||
ensureEtcd
|
||||
|
|
@ -615,7 +118,7 @@ if [[ $OS == $UBUNTU_OS_NAME ]]; then
|
|||
retrycmd_if_failure 20 5 30 apt-mark unhold walinuxagent || exit $ERR_RELEASE_HOLD_WALINUXAGENTs
|
||||
fi
|
||||
|
||||
echo "Install complete successfully"
|
||||
echo "Custom script finished successfully"
|
||||
|
||||
mkdir -p /opt/azure/containers && touch /opt/azure/containers/provision.complete
|
||||
ps auxfww > /opt/azure/provision-ps.log &
|
||||
|
|
|
|||
|
|
@ -0,0 +1,142 @@
|
|||
#!/bin/bash
|
||||
|
||||
CC_SERVICE_IN_TMP=/opt/azure/containers/cc-proxy.service.in
|
||||
CC_SOCKET_IN_TMP=/opt/azure/containers/cc-proxy.socket.in
|
||||
CNI_CONFIG_DIR="/etc/cni/net.d"
|
||||
CNI_BIN_DIR="/opt/cni/bin"
|
||||
AZURE_CNI_TGZ_TMP="/tmp/azure_cni.tgz"
|
||||
CONTAINERNETWORKING_CNI_TGZ_TMP="/tmp/containernetworking_cni.tgz"
|
||||
|
||||
function installEtcd() {
|
||||
retrycmd_get_tarball 60 10 /tmp/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz ${ETCD_DOWNLOAD_URL}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz || exit $ERR_ETCD_DOWNLOAD_TIMEOUT
|
||||
tar -xzvf /tmp/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -C /usr/bin/ --strip-components=1 || exit $ERR_ETCD_DOWNLOAD_TIMEOUT
|
||||
}
|
||||
|
||||
function installDeps() {
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT
|
||||
retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL
|
||||
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
|
||||
# See https://github.com/kubernetes/kubernetes/blob/master/build/debian-hyperkube-base/Dockerfile#L25-L44
|
||||
apt_get_install 20 30 300 apt-transport-https ca-certificates iptables iproute2 ebtables socat util-linux mount ethtool init-system-helpers nfs-common ceph-common conntrack glusterfs-client ipset jq cgroup-lite git pigz xz-utils blobfuse fuse cifs-utils || exit $ERR_APT_INSTALL_TIMEOUT
|
||||
}
|
||||
|
||||
function installContainerRuntime() {
|
||||
if [[ "$CONTAINER_RUNTIME" == "docker" ]]; then
|
||||
installDocker
|
||||
elif [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
|
||||
# Ensure we can nest virtualization
|
||||
if grep -q vmx /proc/cpuinfo; then
|
||||
installClearContainersRuntime
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function installDocker() {
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg || exit $ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT
|
||||
retrycmd_if_failure 10 5 10 apt-key add /tmp/aptdocker.gpg || exit $ERR_DOCKER_APT_KEY_TIMEOUT
|
||||
echo "deb ${DOCKER_REPO} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
|
||||
printf "Package: docker-engine\nPin: version ${DOCKER_ENGINE_VERSION}\nPin-Priority: 550\n" > /etc/apt/preferences.d/docker.pref
|
||||
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
|
||||
apt_get_install 20 30 120 docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT
|
||||
touch /var/log/azure/docker-install.complete
|
||||
}
|
||||
|
||||
function installKataContainersRuntime() {
|
||||
# Add Kata Containers repository key
|
||||
echo "Adding Kata Containers repository key..."
|
||||
KATA_RELEASE_KEY_TMP=/tmp/kata-containers-release.key
|
||||
KATA_URL=http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/Release.key
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $KATA_URL > $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_KEY_DOWNLOAD_TIMEOUT
|
||||
retrycmd_if_failure 10 5 10 apt-key add $KATA_RELEASE_KEY_TMP || exit $ERR_KATA_APT_KEY_TIMEOUT
|
||||
|
||||
# Add Kata Container repository
|
||||
echo "Adding Kata Containers repository..."
|
||||
echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/kata-containers.list
|
||||
|
||||
# Install Kata Containers runtime
|
||||
echo "Installing Kata Containers runtime..."
|
||||
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
|
||||
apt_get_install 20 30 120 kata-runtime || exit $ERR_KATA_INSTALL_TIMEOUT
|
||||
}
|
||||
|
||||
function installClearContainersRuntime() {
|
||||
# Add Clear Containers repository key
|
||||
echo "Adding Clear Containers repository key..."
|
||||
CC_RELEASE_KEY_TMP=/tmp/clear-containers-release.key
|
||||
CC_URL=https://download.opensuse.org/repositories/home:clearcontainers:clear-containers-3/xUbuntu_16.04/Release.key
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $CC_URL > $CC_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT
|
||||
retrycmd_if_failure 10 5 10 apt-key add $CC_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT
|
||||
|
||||
# Add Clear Container repository
|
||||
echo "Adding Clear Containers repository..."
|
||||
echo 'deb http://download.opensuse.org/repositories/home:/clearcontainers:/clear-containers-3/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/cc-runtime.list
|
||||
|
||||
# Install Clear Containers runtime
|
||||
echo "Installing Clear Containers runtime..."
|
||||
apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT
|
||||
apt_get_install 20 30 120 cc-runtime
|
||||
|
||||
# Install the systemd service and socket files.
|
||||
local repo_uri="https://raw.githubusercontent.com/clearcontainers/proxy/3.0.23"
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL "${repo_uri}/cc-proxy.service.in" > $CC_SERVICE_IN_TMP
|
||||
retrycmd_if_failure_no_stats 20 1 5 curl -fsSL "${repo_uri}/cc-proxy.socket.in" > $CC_SOCKET_IN_TMP
|
||||
}
|
||||
|
||||
function installNetworkPlugin() {
|
||||
if [[ "${NETWORK_PLUGIN}" = "azure" ]]; then
|
||||
installAzureCNI
|
||||
elif [[ "${NETWORK_PLUGIN}" = "kubenet" ]]; then
|
||||
installCNI
|
||||
elif [[ "${NETWORK_PLUGIN}" = "flannel" ]]; then
|
||||
installCNI
|
||||
fi
|
||||
}
|
||||
|
||||
function installCNI() {
|
||||
mkdir -p $CNI_BIN_DIR
|
||||
retrycmd_get_tarball 60 5 $CONTAINERNETWORKING_CNI_TGZ_TMP ${CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT
|
||||
tar -xzf $CONTAINERNETWORKING_CNI_TGZ_TMP -C $CNI_BIN_DIR
|
||||
chown -R root:root $CNI_BIN_DIR
|
||||
chmod -R 755 $CNI_BIN_DIR
|
||||
}
|
||||
|
||||
function installAzureCNI() {
|
||||
mkdir -p $CNI_CONFIG_DIR
|
||||
chown -R root:root $CNI_CONFIG_DIR
|
||||
chmod 755 $CNI_CONFIG_DIR
|
||||
mkdir -p $CNI_BIN_DIR
|
||||
retrycmd_get_tarball 60 5 $AZURE_CNI_TGZ_TMP ${VNET_CNI_PLUGINS_URL} || exit $ERR_CNI_DOWNLOAD_TIMEOUT
|
||||
tar -xzf $AZURE_CNI_TGZ_TMP -C $CNI_BIN_DIR
|
||||
installCNI
|
||||
}
|
||||
|
||||
function installContainerd() {
|
||||
CRI_CONTAINERD_VERSION="1.1.0"
|
||||
CONTAINERD_DOWNLOAD_URL="${CONTAINERD_DOWNLOAD_URL_BASE}cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz"
|
||||
CONTAINERD_TGZ_TMP=/tmp/containerd.tar.gz
|
||||
retrycmd_get_tarball 60 5 "$CONTAINERD_TGZ_TMP" "$CONTAINERD_DOWNLOAD_URL" || exit $ERR_CONTAINERD_DOWNLOAD_TIMEOUT
|
||||
tar -xzf "$CONTAINERD_TGZ_TMP" -C /
|
||||
rm -f "$CONTAINERD_TGZ_TMP"
|
||||
sed -i '/\[Service\]/a ExecStartPost=\/sbin\/iptables -P FORWARD ACCEPT' /etc/systemd/system/containerd.service
|
||||
echo "Successfully installed cri-containerd..."
|
||||
}
|
||||
|
||||
function extractHyperkube() {
|
||||
TMP_DIR=$(mktemp -d)
|
||||
retrycmd_if_failure 100 1 30 curl -sSL -o /usr/local/bin/img "https://acs-mirror.azureedge.net/img/img-linux-amd64-v0.4.6"
|
||||
chmod +x /usr/local/bin/img
|
||||
retrycmd_if_failure 75 1 60 img pull $HYPERKUBE_URL || exit $ERR_K8S_DOWNLOAD_TIMEOUT
|
||||
img unpack -o "/home/rootfs" $HYPERKUBE_URL
|
||||
path=$(find /home/rootfs -name "hyperkube")
|
||||
|
||||
if [[ $OS == $COREOS_OS_NAME ]]; then
|
||||
cp "$path" "/opt/kubelet"
|
||||
cp "$path" "/opt/kubectl"
|
||||
chmod a+x /opt/kubelet /opt/kubectl
|
||||
else
|
||||
cp "$path" "/usr/local/bin/kubelet"
|
||||
cp "$path" "/usr/local/bin/kubectl"
|
||||
chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
|
||||
fi
|
||||
rm -rf /tmp/hyperkube.tar "/tmp/img"
|
||||
}
|
||||
|
|
@ -21,6 +21,20 @@ write_files:
|
|||
content: !!binary |
|
||||
{{WrapAsVariable "sshdConfig"}}
|
||||
|
||||
- path: "/opt/azure/containers/provision_installs.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "provisionInstalls"}}
|
||||
|
||||
- path: "/opt/azure/containers/provision_configs.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "provisionConfigs"}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}}
|
||||
{{if not .MasterProfile.IsCoreOS}}
|
||||
- path: "/etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf"
|
||||
|
|
@ -394,17 +408,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
|
|||
content: |
|
||||
#!/bin/bash
|
||||
set -x
|
||||
source /opt/azure/containers/provision_source.sh
|
||||
# TODO standardize/generalize CSE exit codes
|
||||
ERR_ETCD_DOWNLOAD_TIMEOUT=12
|
||||
ERR_SYSTEMCTL_ENABLE_FAIL=3
|
||||
ETCD_VER=v{{WrapAsParameter "etcdVersion"}}
|
||||
DOWNLOAD_URL={{WrapAsParameter "etcdDownloadURLBase"}}
|
||||
retrycmd_get_tarball 60 10 /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz ${DOWNLOAD_URL}/etcd-${ETCD_VER}-linux-amd64.tar.gz
|
||||
if [ $? -ne 0 ]; then
|
||||
exit $ERR_ETCD_DOWNLOAD_TIMEOUT
|
||||
fi
|
||||
tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /usr/bin/ --strip-components=1
|
||||
sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:2379" /etc/environment
|
||||
sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment
|
||||
sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment
|
||||
|
|
|
|||
|
|
@ -126,23 +126,15 @@
|
|||
{{end}}
|
||||
"provisionScript": "{{GetKubernetesB64Provision}}",
|
||||
"provisionSource": "{{GetKubernetesB64ProvisionSource}}",
|
||||
"provisionInstalls": "{{GetKubernetesB64Installs}}",
|
||||
"provisionConfigs": "{{GetKubernetesB64Configs}}",
|
||||
"mountetcdScript": "{{GetKubernetesB64Mountetcd}}",
|
||||
"customSearchDomainsScript": "{{GetKubernetesB64CustomSearchDomainsScript}}",
|
||||
"sshdConfig": "{{GetB64sshdConfig}}",
|
||||
{{if not IsOpenShift}}
|
||||
"provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]",
|
||||
{{if not IsHostedMaster}}
|
||||
"provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')))]",
|
||||
{{if EnableEncryptionWithExternalKms}}
|
||||
"provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'), ' EnableEncryptionWithExternalKms=true')]",
|
||||
{{else}}
|
||||
"provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]",
|
||||
{{end}}
|
||||
{{else}}
|
||||
{{if EnableEncryptionWithExternalKms}}
|
||||
"provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'), ' EnableEncryptionWithExternalKms=true')]",
|
||||
{{else}}
|
||||
"provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'))]",
|
||||
{{end}}
|
||||
"provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]",
|
||||
{{end}}
|
||||
{{end}}
|
||||
"generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}",
|
||||
|
|
@ -316,4 +308,6 @@
|
|||
"apiVersionStorage": "2015-06-15",
|
||||
{{end}}
|
||||
"clusterKeyVaultName": "[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]"
|
||||
{{else}}
|
||||
,"clusterKeyVaultName": ""
|
||||
{{end}}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,46 @@
|
|||
#!/bin/sh
|
||||
|
||||
ERR_SYSTEMCTL_ENABLE_FAIL=3 # Service could not be enabled by systemctl
|
||||
ERR_SYSTEMCTL_START_FAIL=4 # Service could not be started by systemctl
|
||||
ERR_CLOUD_INIT_TIMEOUT=5 # Timeout waiting for cloud-init runcmd to complete
|
||||
ERR_FILE_WATCH_TIMEOUT=6 # Timeout waiting for a file
|
||||
ERR_HOLD_WALINUXAGENT=7 # Unable to place walinuxagent apt package on hold during install
|
||||
ERR_RELEASE_HOLD_WALINUXAGENT=8 # Unable to release hold on walinuxagent apt package after install
|
||||
ERR_APT_INSTALL_TIMEOUT=9 # Timeout installing required apt packages
|
||||
ERR_ETCD_DATA_DIR_NOT_FOUND=10 # Etcd data dir not found
|
||||
ERR_ETCD_RUNNING_TIMEOUT=11 # Timeout waiting for etcd to be accessible
|
||||
ERR_ETCD_DOWNLOAD_TIMEOUT=12 # Timeout waiting for etcd to download
|
||||
ERR_ETCD_VOL_MOUNT_FAIL=13 # Unable to mount etcd disk volume
|
||||
ERR_ETCD_START_TIMEOUT=14 # Unable to start etcd runtime
|
||||
ERR_ETCD_CONFIG_FAIL=15 # Unable to configure etcd cluster
|
||||
ERR_DOCKER_INSTALL_TIMEOUT=20 # Timeout waiting for docker install
|
||||
ERR_DOCKER_DOWNLOAD_TIMEOUT=21 # Timout waiting for docker download(s)
|
||||
ERR_DOCKER_KEY_DOWNLOAD_TIMEOUT=22 # Timeout waiting to download docker repo key
|
||||
ERR_DOCKER_APT_KEY_TIMEOUT=23 # Timeout waiting for docker apt-key
|
||||
ERR_K8S_RUNNING_TIMEOUT=30 # Timeout waiting for k8s cluster to be healthy
|
||||
ERR_K8S_DOWNLOAD_TIMEOUT=31 # Timeout waiting for Kubernetes download(s)
|
||||
ERR_KUBECTL_NOT_FOUND=32 # kubectl client binary not found on local disk
|
||||
ERR_CNI_DOWNLOAD_TIMEOUT=41 # Timeout waiting for CNI download(s)
|
||||
ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT=42 # Timeout waiting for https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb
|
||||
ERR_MS_PROD_DEB_PKG_ADD_FAIL=43 # Failed to add repo pkg file
|
||||
#ERR_FLEXVOLUME_DOWNLOAD_TIMEOUT=44 # Failed to add repo pkg file -- DEPRECATED
|
||||
ERR_MODPROBE_FAIL=49 # Unable to load a kernel module using modprobe
|
||||
ERR_OUTBOUND_CONN_FAIL=50 # Unable to establish outbound connection
|
||||
ERR_KATA_KEY_DOWNLOAD_TIMEOUT=60 # Timeout waiting to download kata repo key
|
||||
ERR_KATA_APT_KEY_TIMEOUT=61 # Timeout waiting for kata apt-key
|
||||
ERR_KATA_INSTALL_TIMEOUT=62 # Timeout waiting for kata install
|
||||
ERR_CONTAINERD_DOWNLOAD_TIMEOUT=70 # Timeout waiting for containerd download(s)
|
||||
ERR_CUSTOM_SEARCH_DOMAINS_FAIL=80 # Unable to configure custom search domains
|
||||
ERR_APT_DAILY_TIMEOUT=98 # Timeout waiting for apt daily updates
|
||||
ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete
|
||||
|
||||
OS=$(cat /etc/*-release | grep ^ID= | tr -d 'ID="' | awk '{print toupper($0)}')
|
||||
UBUNTU_OS_NAME="UBUNTU"
|
||||
RHEL_OS_NAME="RHEL"
|
||||
COREOS_OS_NAME="COREOS"
|
||||
KUBECTL=/usr/local/bin/kubectl
|
||||
DOCKER=/usr/bin/docker
|
||||
|
||||
retrycmd_if_failure() {
|
||||
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
|
||||
for i in $(seq 1 $retries); do
|
||||
|
|
|
|||
|
|
@ -205,6 +205,8 @@ const (
|
|||
kubernetesMasterCustomDataYaml = "k8s/kubernetesmastercustomdata.yml"
|
||||
kubernetesCustomScript = "k8s/kubernetescustomscript.sh"
|
||||
kubernetesProvisionSourceScript = "k8s/kubernetesprovisionsource.sh"
|
||||
kubernetesInstalls = "k8s/kubernetesinstalls.sh"
|
||||
kubernetesConfigurations = "k8s/kubernetesconfigs.sh"
|
||||
kubernetesMountetcd = "k8s/kubernetes_mountetcd.sh"
|
||||
kubernetesCustomSearchDomainsScript = "k8s/setup-custom-search-domains.sh"
|
||||
kubernetesMasterGenerateProxyCertsScript = "k8s/kubernetesmastergenerateproxycertscript.sh"
|
||||
|
|
|
|||
|
|
@ -528,6 +528,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
|
|||
"GetKubernetesB64ProvisionSource": func() string {
|
||||
return getBase64CustomScript(kubernetesProvisionSourceScript)
|
||||
},
|
||||
"GetKubernetesB64Installs": func() string {
|
||||
return getBase64CustomScript(kubernetesInstalls)
|
||||
},
|
||||
"GetKubernetesB64Configs": func() string {
|
||||
return getBase64CustomScript(kubernetesConfigurations)
|
||||
},
|
||||
"GetKubernetesB64Mountetcd": func() string {
|
||||
return getBase64CustomScript(kubernetesMountetcd)
|
||||
},
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче