aks-engine/parts/k8s/cloud-init/nodecustomdata.yml

442 строки
11 KiB
YAML

#cloud-config
write_files:
{{- if .RequiresCloudproviderConfig}}
- path: /opt/azure/needs_azure.json
permissions: "0644"
owner: root
content: |
#EOF
{{end}}
- path: {{GetCSEHelpersScriptFilepath}}
permissions: "0744"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "provisionSource"}}
- path: /opt/azure/containers/provision.sh
permissions: "0744"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "provisionScript"}}
- path: {{GetCSEInstallScriptFilepath}}
permissions: "0744"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "provisionInstalls"}}
- path: {{GetCSEConfigScriptFilepath}}
permissions: "0744"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "provisionConfigs"}}
{{- if not .IsVHDDistro}}
- path: /opt/azure/containers/provision_cis.sh
permissions: "0744"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "provisionCIS"}}
{{end}}
{{- if .IsAuditDEnabled}}
- path: /etc/audit/rules.d/CIS.rules
permissions: "0744"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "auditdRules"}}
{{end}}
{{- if .IsUbuntu1804}}
{{- if not .IsVHDDistro}}
- path: /var/run/reboot-required
permissions: "0644"
owner: root
content: |
{{end}}
{{end}}
{{- if IsCustomCloudProfile}}
- path: {{GetCustomCloudConfigCSEScriptFilepath}}
permissions: "0744"
encoding: gzip
owner: root
content: !!binary |
{{WrapAsVariable "provisionConfigsCustomCloud"}}
{{end}}
{{- if HasKubeReservedCgroup}}
- path: /etc/systemd/system/{{- GetKubeReservedCgroup -}}.slice
permissions: "0644"
owner: root
content: |
[Unit]
Description=Limited resources slice for Kubernetes services
Documentation=man:systemd.special(7)
DefaultDependencies=no
Before=slices.target
Requires=-.slice
After=-.slice
#EOF
- path: /etc/systemd/system/kubelet.service.d/kubereserved-slice.conf
permissions: "0644"
owner: root
content: |
[Service]
Slice={{- GetKubeReservedCgroup -}}.slice
#EOF
- path: /etc/systemd/system/{{GetContainerRuntime}}.service.d/kubereserved-slice.conf
permissions: "0644"
owner: root
content: |
[Service]
Slice={{- GetKubeReservedCgroup -}}.slice
#EOF
{{- end}}
{{- if HasKubeletHealthZPort}}
- path: /etc/systemd/system/kubelet-monitor.service
permissions: "0644"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "kubeletMonitorSystemdService"}}
{{- end}}
{{- if .IsFlatcar}}
- path: /opt/bin/health-monitor.sh
{{else}}
- path: /usr/local/bin/health-monitor.sh
{{- end}}
permissions: "0544"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "healthMonitorScript"}}
- path: /etc/systemd/system/kubelet.service
permissions: "0644"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "kubeletSystemdService"}}
{{- /* for historical reasons, we overload the name "docker" here; in fact this monitor service supports both docker and containerd */}}
- path: /etc/systemd/system/docker-monitor.service
permissions: "0644"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "dockerMonitorSystemdService"}}
{{- if not .IsVHDDistro}}
- path: /etc/apt/preferences
permissions: "0644"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "aptPreferences"}}
{{end}}
- path: /etc/apt/apt.conf.d/99periodic
permissions: "0644"
owner: root
content: |
APT::Periodic::Update-Package-Lists "0";
APT::Periodic::Download-Upgradeable-Packages "0";
APT::Periodic::AutocleanInterval "0";
APT::Periodic::Unattended-Upgrade "0";
{{- if and IsIPv6Enabled .IsUbuntu1604}}
- path: {{GetDHCPv6ServiceCSEScriptFilepath}}
permissions: "0644"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "dhcpv6SystemdService"}}
- path: {{GetDHCPv6ConfigCSEScriptFilepath}}
permissions: "0544"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "dhcpv6ConfigurationScript"}}
{{end}}
{{- if .KubernetesConfig.RequiresDocker}}
{{- if not .IsFlatcar}}
{{- if not .IsVHDDistro}}
- path: /etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf
permissions: "0644"
encoding: gzip
owner: "root"
content: !!binary |
{{CloudInitData "dockerClearMountPropagationFlags"}}
{{- end}}
{{- end}}
- path: /etc/systemd/system/docker.service.d/exec_start.conf
permissions: "0644"
owner: root
content: |
[Service]
Restart=always
ExecStart=
{{- if .IsFlatcar}}
ExecStart=/usr/bin/env PATH=${TORCX_BINDIR}:${PATH} ${TORCX_BINDIR}/dockerd --host=fd:// --containerd=/var/run/docker/libcontainerd/docker-containerd.sock --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} $DOCKER_SELINUX $DOCKER_OPTS $DOCKER_CGROUPS $DOCKER_OPT_BIP $DOCKER_OPT_MTU $DOCKER_OPT_IPMASQ
{{else}}
ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}}
{{- end}}
ExecStartPost=/sbin/iptables -P FORWARD ACCEPT
#EOF
- path: /etc/docker/daemon.json
permissions: "0644"
owner: root
content: |
{{IndentString (GetDockerConfig (IsNSeriesSKU .VMSize)) 4}}
{{end}}
{{- if HasCiliumNetworkPlugin}}
- path: /etc/systemd/system/sys-fs-bpf.mount
permissions: "0644"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "systemdBPFMount"}}
{{end}}
- path: /etc/sysctl.d/11-aks-engine.conf
permissions: "0644"
owner: root
content: |
{{GetSysctlDConfigKeyVals .SysctlDConfig}}
#EOF
{{- if NeedsContainerd}}
- path: /etc/systemd/system/containerd.service.d/exec_start.conf
permissions: "0644"
owner: root
content: |
[Service]
ExecStartPre=/sbin/iptables -P FORWARD ACCEPT
#EOF
- path: /etc/containerd/config.toml
permissions: "0644"
owner: root
content: |
{{- if IsNSeriesSKU .VMSize}}
{{IndentString GetNvidiaContainerdConfig 4}}
{{else}}
{{IndentString GetContainerdConfig 4}}
{{- end}}
#EOF
{{if IsKubenet }}
- path: /etc/containerd/kubenet_template.conf
permissions: "0644"
owner: root
content: |
{
"cniVersion": "0.3.1",
"name": "kubenet",
"plugins": [{
"type": "bridge",
"bridge": "cbr0",
"mtu": 1500,
"addIf": "eth0",
"isGateway": true,
"ipMasq": false,
"hairpinMode": false,
"ipam": {
"type": "host-local",
"subnet": "{{`{{.PodCIDR}}`}}",
"routes": [{ "dst": "0.0.0.0/0" }]
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true},
"snat": false
}]
}
{{end}}
{{end}}
{{- if IsNSeriesSKU .VMSize}}
- path: /etc/systemd/system/nvidia-modprobe.service
permissions: "0644"
owner: root
content: |
[Unit]
Description=Installs and loads Nvidia GPU kernel module
[Service]
Type=oneshot
RemainAfterExit=true
ExecStartPre=/bin/sh -c "dkms autoinstall --verbose"
ExecStart=/bin/sh -c "nvidia-modprobe -u -c0"
ExecStartPost=/bin/sh -c "sleep 10 && systemctl restart kubelet"
[Install]
WantedBy=multi-user.target
{{end}}
- path: /etc/kubernetes/certs/ca.crt
permissions: "0644"
encoding: base64
owner: root
content: |
{{WrapAsParameter "caCertificate"}}
- path: /etc/kubernetes/certs/client.crt
permissions: "0644"
encoding: base64
owner: root
content: |
{{WrapAsParameter "clientCertificate"}}
{{- if HasCustomSearchDomain}}
- path: {{GetCustomSearchDomainsCSEScriptFilepath}}
permissions: "0744"
encoding: gzip
owner: root
content: !!binary |
{{CloudInitData "customSearchDomainsScript"}}
{{end}}
- path: /var/lib/kubelet/kubeconfig
permissions: "0644"
owner: root
content: |
apiVersion: v1
kind: Config
clusters:
- name: localcluster
cluster:
certificate-authority: /etc/kubernetes/certs/ca.crt
server: https://{{WrapAsVariable "kubernetesAPIServerIP"}}:443
users:
- name: client
user:
client-certificate: /etc/kubernetes/certs/client.crt
client-key: /etc/kubernetes/certs/client.key
contexts:
- context:
cluster: localcluster
user: client
name: localclustercontext
current-context: localclustercontext
#EOF
- path: /etc/default/kubelet
permissions: "0644"
owner: root
content: |
KUBELET_CONFIG={{GetKubeletConfigKeyVals .KubernetesConfig }}
KUBELET_NODE_LABELS={{GetAgentKubernetesLabels . "',variables('labelResourceGroup'),'"}}
{{- if IsCustomCloudProfile }}
AZURE_ENVIRONMENT_FILEPATH=/etc/kubernetes/azurestackcloud.json
{{end}}
#EOF
- path: /opt/azure/containers/kubelet.sh
permissions: "0755"
owner: root
content: |
#!/bin/bash
MOUNT_DIR=/var/lib/kubelet
mkdir -p $MOUNT_DIR /var/lib/cni
if ! [[ $(findmnt -rno SOURCE,TARGET ${MOUNT_DIR}) ]]; then
mount --bind $MOUNT_DIR $MOUNT_DIR
fi
mount --make-shared $MOUNT_DIR
{{- if IsAzureCNI}}
ifconfig eth0 mtu {{GetEth0MTU}} up
{{- end}}
{{- if and (IsVirtualMachineScaleSets .) IsAADPodIdentityAddonEnabled UseManagedIdentity}}
{{- /* Disable TCP access to IMDS endpoint, aad-pod-identity nmi component will provide a complementary iptables rule to re-route this traffic */}}
iptables -A OUTPUT -s 127.0.0.1/32 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DROP
{{end}}
{{- if not IsIPMasqAgentEnabled}}
{{if IsAzureCNI}}
iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
{{end}}
{{end}}
{{- /* Ensure that container traffic can't connect to internal Azure IP endpoint */}}
iptables -I FORWARD -d 168.63.129.16 -p tcp --dport 80 -j DROP
#EOF
{{- if IsCustomCloudProfile}}
- path: "/etc/kubernetes/azurestackcloud.json"
permissions: "0600"
owner: "root"
content: |
{{WrapAsVariable "environmentJSON"}}
{{end}}
{{- if .IsFlatcar}}
- path: "/etc/kubernetes/manifests/.keep"
{{- if .KubernetesConfig.RequiresDocker}}
groups:
- docker: [{{WrapAsParameter "linuxAdminUsername"}}]
{{end}}
coreos:
units:
- name: kubelet.service
enable: true
drop-ins:
- name: "10-flatcar.conf"
content: |
[Unit]
Requires=rpc-statd.service
ConditionPathExists=
ConditionPathExists=/opt/bin/kubelet
[Service]
ExecStart=
ExecStart=/opt/bin/kubelet \
--enable-server \
--node-labels="${KUBELET_NODE_LABELS}" \
--v=2 \
--volume-plugin-dir=/etc/kubernetes/volumeplugins \
$KUBELET_CONFIG $KUBELET_OPTS \
$KUBELET_REGISTER_NODE $KUBELET_REGISTER_WITH_TAINTS
- name: kubelet-monitor.service
enable: true
drop-ins:
- name: "10-flatcar.conf"
content: |
After=kubelet.service
[Service]
ExecStart=
ExecStart=/opt/bin/health-monitor.sh kubelet
- name: docker-monitor.service
enable: true
drop-ins:
- name: "10-flatcar.conf"
content: |
After={{GetContainerRuntime}}.service
[Service]
ExecStart=
ExecStart=/opt/bin/health-monitor.sh container-runtime
- name: rpcbind.service
enable: true
{{else}}
runcmd:
- set -x
- source {{GetCSEHelpersScriptFilepath}}
- aptmarkWALinuxAgent hold{{GetKubernetesAgentPreprovisionYaml .}}
{{- end}}