Azure
/
acs-engine
зеркало из https://github.com/Azure/acs-engine.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add VMSS support for k8s master (#3769)

This commit is contained in:
Rita Zhang 2018-09-14 17:21:55 -07:00 коммит произвёл Jack Francis
Родитель 40800b91cf
Коммит c83e1028ba
33 изменённых файлов: 2412 добавлений и 134 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
cmd/deploy.go
Просмотреть файл

@ -314,6 +314,9 @@ func autofillApimodel(dc *deployCmd) error {
useManagedIdentity := k8sConfig != nil &&
k8sConfig.UseManagedIdentity
if dc.containerService.Properties.MasterProfile.IsVirtualMachineScaleSets() {
k8sConfig.UserAssignedID = acsengine.DefaultUserAssignedID
}
userAssignedID := k8sConfig != nil &&
k8sConfig.UseManagedIdentity &&
k8sConfig.UserAssignedID != ""

10
docs/clusterdefinition.md
Просмотреть файл

@ -67,7 +67,7 @@ To learn more about supported orchestrators and versions, run the orchestrators
| schedulerConfig | no | Configure various runtime configuration for scheduler. See `schedulerConfig` [below](#feat-scheduler-config) |
| serviceCidr | no | IP range for Service IPs, Default is "10.0.0.0/16". This range is never routed outside of a node so does not need to lie within clusterSubnet or the VNET |
| useInstanceMetadata | no | Use the Azure cloudprovider instance metadata service for appropriate resource discovery operations. Default is `true` |
| useManagedIdentity | no | Includes and uses MSI identities for all interactions with the Azure Resource Manager (ARM) API. Instead of using a static service principal written to /etc/kubernetes/azure.json, Kubernetes will use a dynamic, time-limited token fetched from the MSI extension running on master and agent nodes. This support is currently alpha and requires Kubernetes v1.9.1 or newer. (boolean - default == false) |
| useManagedIdentity | no | Includes and uses MSI identities for all interactions with the Azure Resource Manager (ARM) API. Instead of using a static service principal written to /etc/kubernetes/azure.json, Kubernetes will use a dynamic, time-limited token fetched from the MSI extension running on master and agent nodes. This support is currently alpha and requires Kubernetes v1.9.1 or newer. (boolean - default == false). When MasterProfile is using `VirtualMachineScaleSets`, this feature requires Kubernetes v1.12 or newer as we default to using user assigned identity. |
#### addons
@ -487,16 +487,18 @@ We consider `kubeletConfig`, `controllerManagerConfig`, `apiServerConfig`, and `
| count | yes | Masters have count value of 1, 3, or 5 masters |
| dnsPrefix | yes | The dns prefix for the master FQDN. The master FQDN is used for SSH or commandline access. This must be a unique name. ([bring your own VNET examples](../examples/vnet)) |
| subjectAltNames | no | An array of fully qualified domain names using which a user can reach API server. These domains are added as Subject Alternative Names to the generated API server certificate. **NOTE**: These domains **will not** be automatically provisioned. |
| firstConsecutiveStaticIP | only required when vnetSubnetId specified | The IP address of the first master. IP Addresses will be assigned consecutively to additional master nodes |
| firstConsecutiveStaticIP | only required when vnetSubnetId specified and when MasterProfile is not `VirtualMachineScaleSets` | The IP address of the first master. IP Addresses will be assigned consecutively to additional master nodes. When MasterProfile is using `VirtualMachineScaleSets`, this value will be determined by an offset from the first IP in the `vnetCidr`. For example, if `vnetCidr` is `10.239.0.0/16`, then `firstConsecutiveStaticIP` will be `10.239.0.4` |
| vmsize | yes | Describes a valid [Azure VM Sizes](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes/). These are restricted to machines with at least 2 cores and 100GB of ephemeral disk space |
| osDiskSizeGB | no | Describes the OS Disk Size in GB |
| vnetSubnetId | no | Specifies the Id of an alternate VNET subnet. The subnet id must specify a valid VNET ID owned by the same subscription. ([bring your own VNET examples](../examples/vnet)) |
| vnetSubnetId | only required when using custom VNET | Specifies the Id of an alternate VNET subnet. The subnet id must specify a valid VNET ID owned by the same subscription. ([bring your own VNET examples](../examples/vnet)). When MasterProfile is set to `VirtualMachineScaleSets`, this value should be the subnetId of the master subnet. When MasterProfile is set to `AvailabilitySet`, this value should be the subnetId shared by both master and agent nodes. |
| extensions | no | This is an array of extensions. This indicates that the extension be run on a single master. The name in the extensions array must exactly match the extension name in the extensionProfiles |
| vnetCidr | no | Specifies the VNET cidr when using a custom VNET ([bring your own VNET examples](../examples/vnet)) |
| vnetCidr | no | Specifies the VNET cidr when using a custom VNET ([bring your own VNET examples](../examples/vnet)). This VNET cidr should include both the master and the agent subnets. |
| imageReference.name | no | The name of the Linux OS image. Needs to be used in conjunction with resourceGroup, below |
| imageReference.resourceGroup | no | Resource group that contains the Linux OS image. Needs to be used in conjunction with name, above |
| distro | no | Select Master(s) Operating System (Linux only). Currently supported values are: `ubuntu`, `aks` and `coreos` (CoreOS support is currently experimental). Defaults to `aks` if undefined. `aks` is a custom image based on `ubuntu` that comes with pre-installed software necessary for Kubernetes deployments. Currently supported OS and orchestrator configurations -- `ubuntu` and `aks`: DCOS, Docker Swarm, Kubernetes; `RHEL`: OpenShift; `coreos`: Kubernetes. [Example of CoreOS Master with CoreOS Agents](../examples/coreos/kubernetes-coreos.json) |
| customFiles | no | The custom files to be provisioned to the master nodes. Defined as an array of json objects with each defined as `"source":"absolute-local-path", "dest":"absolute-path-on-masternodes"`.[See examples](../examples/customfiles) |
| availabilityProfile | no | Supported values are `AvailabilitySet` (default) and `VirtualMachineScaleSets` (requires Kubernetes clusters version 1.10+ and agent pool availabilityProfile must also be `VirtualMachineScaleSets`). |
| agentVnetSubnetId | only required when using custom VNET and when MasterProfile is using `VirtualMachineScaleSets` | Specifies the Id of an alternate VNET subnet for all the agent pool nodes. The subnet id must specify a valid VNET ID owned by the same subscription. ([bring your own VNET examples](../examples/vnet)). When MasterProfile is using `VirtualMachineScaleSets`, this value should be the subnetId of the subnet for all agent pool nodes. |
### agentPoolProfiles

108
examples/kubernetes-vmss-master/customvnet.json Normal file
Просмотреть файл

@ -0,0 +1,108 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"kubernetesConfig": {
"clusterSubnet": "10.239.0.0/16",
"addons": [
{
"name": "tiller",
"enabled": true,
"config": {
"max-history": "10"
},
"containers": [
{
"name": "tiller",
"cpuRequests": "1",
"memoryRequests": "1Gi",
"cpuLimits": "1",
"memoryLimits": "1Gi"
}
]
},
{
"name": "kubernetes-dashboard",
"enabled": true,
"containers": [
{
"name": "kubernetes-dashboard",
"cpuRequests": "50m",
"memoryRequests": "512Mi",
"cpuLimits": "50m",
"memoryLimits": "512Mi"
}
]
},
{
"name": "rescheduler",
"enabled": true,
"containers": [
{
"name": "rescheduler",
"cpuRequests": "20m",
"memoryRequests": "200Mi",
"cpuLimits": "20m",
"memoryLimits": "200Mi"
}
]
}
]
}
},
"masterProfile": {
"count": 3,
"dnsPrefix": "",
"vmSize": "Standard_DS2_v2",
"OSDiskSizeGB": 200,
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
"agentVnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
"vnetCidr": "10.239.0.0/16",
"availabilityProfile": "VirtualMachineScaleSets"
},
"agentPoolProfiles": [
{
"name": "agentmd",
"count": 3,
"vmSize": "Standard_DS2_v2",
"OSDiskSizeGB": 200,
"storageProfile": "ManagedDisks",
"diskSizesGB": [
128,
128,
128,
128
],
"availabilityProfile": "VirtualMachineScaleSets",
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
},
{
"name": "agentsa",
"count": 3,
"vmSize": "Standard_DS2_v2",
"OSDiskSizeGB": 200,
"storageProfile": "ManagedDisks",
"diskSizesGB": [
128
],
"availabilityProfile": "VirtualMachineScaleSets",
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
}
],
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}

34
examples/kubernetes-vmss-master/kubernetes.json Normal file
Просмотреть файл

@ -0,0 +1,34 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.10"
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v2",
"availabilityProfile": "VirtualMachineScaleSets"
},
"agentPoolProfiles": [
{
"name": "agentpool1",
"count": 3,
"vmSize": "Standard_D2_v2",
"availabilityProfile": "VirtualMachineScaleSets",
"storageProfile": "ManagedDisks"
}
],
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
}
}
}

48
examples/kubernetes-vmss-master/windows.json Normal file
Просмотреть файл

@ -0,0 +1,48 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"orchestratorRelease": "1.10"
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v2",
"availabilityProfile": "VirtualMachineScaleSets"
},
"agentPoolProfiles": [
{
"name": "linuxpool1",
"count": 1,
"vmSize": "Standard_D2_v2",
"availabilityProfile": "VirtualMachineScaleSets"
},
{
"name": "agentwin",
"count": 2,
"vmSize": "Standard_D2_v2",
"availabilityProfile": "VirtualMachineScaleSets",
"osType": "Windows"
}
],
"windowsProfile": {
"adminUsername": "azureuser",
"adminPassword": "replacepassword1234$"
},
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}

108
examples/vnet/kubernetes-master-vmss.json Normal file
Просмотреть файл

@ -0,0 +1,108 @@
{
"apiVersion": "vlabs",
"properties": {
"orchestratorProfile": {
"orchestratorType": "Kubernetes",
"kubernetesConfig": {
"clusterSubnet": "10.239.0.0/16",
"addons": [
{
"name": "tiller",
"enabled": true,
"config": {
"max-history": "10"
},
"containers": [
{
"name": "tiller",
"cpuRequests": "1",
"memoryRequests": "1Gi",
"cpuLimits": "1",
"memoryLimits": "1Gi"
}
]
},
{
"name": "kubernetes-dashboard",
"enabled": true,
"containers": [
{
"name": "kubernetes-dashboard",
"cpuRequests": "50m",
"memoryRequests": "512Mi",
"cpuLimits": "50m",
"memoryLimits": "512Mi"
}
]
},
{
"name": "rescheduler",
"enabled": true,
"containers": [
{
"name": "rescheduler",
"cpuRequests": "20m",
"memoryRequests": "200Mi",
"cpuLimits": "20m",
"memoryLimits": "200Mi"
}
]
}
]
}
},
"masterProfile": {
"count": 3,
"dnsPrefix": "",
"vmSize": "Standard_DS2_v2",
"OSDiskSizeGB": 200,
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
"agentVnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
"vnetCidr": "10.239.0.0/16",
"availabilityProfile": "VirtualMachineScaleSets"
},
"agentPoolProfiles": [
{
"name": "agentmd",
"count": 3,
"vmSize": "Standard_DS2_v2",
"OSDiskSizeGB": 200,
"storageProfile": "ManagedDisks",
"diskSizesGB": [
128,
128,
128,
128
],
"availabilityProfile": "VirtualMachineScaleSets",
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
},
{
"name": "agentsa",
"count": 3,
"vmSize": "Standard_DS2_v2",
"OSDiskSizeGB": 200,
"storageProfile": "ManagedDisks",
"diskSizesGB": [
128
],
"availabilityProfile": "VirtualMachineScaleSets",
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
}
],
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
},
"servicePrincipalProfile": {
"clientId": "",
"secret": ""
}
}
}

8
parts/k8s/kubernetesbase.t
Просмотреть файл

@ -49,7 +49,11 @@
"{{.Name}}AccountName": "[concat(variables('storageAccountBaseName'), 'agnt{{$index}}')]",
{{end}}
{{end}}
{{if IsMasterVirtualMachineScaleSets}}
{{template "k8s/kubernetesmastervarsvmss.t" .}}
{{else}}
{{template "k8s/kubernetesmastervars.t" .}}
{{end}}
},
"resources": [
{{if UserAssignedIDEnabled}}
@ -80,7 +84,11 @@
{{end}}
{{end}}
{{if not IsHostedMaster}}
{{if IsMasterVirtualMachineScaleSets}}
,{{template "k8s/kubernetesmasterresourcesvmss.t" .}}
{{else}}
,{{template "k8s/kubernetesmasterresources.t" .}}
{{end}}
{{else}}
{{if not IsCustomVNET}}
,{

11
parts/k8s/kubernetesconfigs.sh
Просмотреть файл

@ -1,4 +1,9 @@
#!/bin/bash
NODE_INDEX=$(hostname | tail -c 2)
NODE_NAME=$(hostname)
PRIVATE_IP=$(hostname -I | cut -d' ' -f1)
ETCD_PEER_URL="https://${PRIVATE_IP}:2380"
ETCD_CLIENT_URL="https://${PRIVATE_IP}:2379"
function systemctlEnableAndStart() {
systemctl_restart 100 5 30 $1
@ -41,7 +46,7 @@ function configureEtcd() {
chmod 0600 "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
chown root:root "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.key"
ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${NODE_INDEX}.key"
touch "${ETCD_PEER_PRIVATE_KEY_PATH}"
chmod 0600 "${ETCD_PEER_PRIVATE_KEY_PATH}"
chown etcd:etcd "${ETCD_PEER_PRIVATE_KEY_PATH}"
@ -56,7 +61,7 @@ function configureEtcd() {
chmod 0644 "${ETCD_CLIENT_CERTIFICATE_PATH}"
chown root:root "${ETCD_CLIENT_CERTIFICATE_PATH}"
ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.crt"
ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${NODE_INDEX}.crt"
touch "${ETCD_PEER_CERTIFICATE_PATH}"
chmod 0644 "${ETCD_PEER_CERTIFICATE_PATH}"
chown root:root "${ETCD_PEER_CERTIFICATE_PATH}"
@ -85,7 +90,7 @@ function configureEtcd() {
$MOUNT_ETCD_FILE || exit $ERR_ETCD_VOL_MOUNT_FAIL
systemctlEnableAndStart etcd || exit $ERR_ETCD_START_TIMEOUT
for i in $(seq 1 600); do
MEMBER="$(sudo etcdctl member list | grep -E ${MASTER_VM_NAME} | cut -d':' -f 1)"
MEMBER="$(sudo etcdctl member list | grep -E ${NODE_NAME} | cut -d':' -f 1)"
if [ "$MEMBER" != "" ]; then
break
else

4
parts/k8s/kubernetescustomscript.sh
Просмотреть файл

@ -8,8 +8,8 @@ source /opt/azure/containers/provision_configs.sh
CUSTOM_SEARCH_DOMAIN_SCRIPT=/opt/azure/containers/setup-custom-search-domains.sh
set +x
ETCD_PEER_CERT=$(echo ${ETCD_PEER_CERTIFICATES} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${MASTER_INDEX}+1)))
ETCD_PEER_KEY=$(echo ${ETCD_PEER_PRIVATE_KEYS} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${MASTER_INDEX}+1)))
ETCD_PEER_CERT=$(echo ${ETCD_PEER_CERTIFICATES} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${NODE_INDEX}+1)))
ETCD_PEER_KEY=$(echo ${ETCD_PEER_PRIVATE_KEYS} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${NODE_INDEX}+1)))
set -x
if [[ $OS == $COREOS_OS_NAME ]]; then

540
parts/k8s/kubernetesmastercustomdatavmss.yml Normal file
Просмотреть файл

@ -0,0 +1,540 @@
#cloud-config
{{if not .MasterProfile.IsCoreOS}}
packages:
- jq
- traceroute
{{end}}
write_files:
- path: "/opt/azure/containers/provision_source.sh"
permissions: "0744"
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "provisionSource"}}
- path: "/etc/ssh/sshd_config"
permissions: "0644"
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "sshdConfig"}}
- path: "/opt/azure/containers/provision_installs.sh"
permissions: "0744"
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "provisionInstalls"}}
- path: "/opt/azure/containers/provision_configs.sh"
permissions: "0744"
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "provisionConfigs"}}
{{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}}
{{if not .MasterProfile.IsCoreOS}}
- path: "/etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf"
permissions: "0644"
owner: "root"
content: |
[Service]
MountFlags=shared
{{end}}
- path: "/etc/systemd/system/docker.service.d/exec_start.conf"
permissions: "0644"
owner: "root"
content: |
[Service]
ExecStart=
{{if .MasterProfile.IsCoreOS}}
ExecStart=/usr/bin/env PATH=${TORCX_BINDIR}:${PATH} ${TORCX_BINDIR}/dockerd --host=fd:// --containerd=/var/run/docker/libcontainerd/docker-containerd.sock --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} $DOCKER_SELINUX $DOCKER_OPTS $DOCKER_CGROUPS $DOCKER_OPT_BIP $DOCKER_OPT_MTU $DOCKER_OPT_IPMASQ
{{else}}
ExecStart=/usr/bin/docker daemon -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}}
{{end}}
- path: "/etc/docker/daemon.json"
permissions: "0644"
owner: "root"
content: |
{
"live-restore": true,
"log-driver": "json-file",
"log-opts": {
"max-size": "50m",
"max-file": "5"
}
}
- path: "/usr/local/bin/health-monitor.sh"
permissions: "0544"
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "healthMonitorScript"}}
- path: "/etc/systemd/system/docker-monitor.service"
permissions: "0644"
owner: "root"
content: |
[Unit]
Description=a script that checks docker health and restarts if needed
After=docker.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
ExecStart=/usr/local/bin/health-monitor.sh container-runtime
[Install]
WantedBy=multi-user.target
- path: "/etc/systemd/system/kubelet-monitor.service"
permissions: "0644"
owner: "root"
content: |
[Unit]
Description=a script that checks kubelet health and restarts if needed
After=kubelet.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
ExecStart=/usr/local/bin/health-monitor.sh kubelet
[Install]
WantedBy=multi-user.target
{{end}}
- path: "/etc/kubernetes/certs/ca.crt"
permissions: "0644"
encoding: "base64"
owner: "root"
content: |
{{WrapAsParameter "caCertificate"}}
- path: "/etc/kubernetes/certs/client.crt"
permissions: "0644"
encoding: "base64"
owner: "root"
content: |
{{WrapAsParameter "clientCertificate"}}
{{if EnableAggregatedAPIs}}
- path: "/etc/kubernetes/generate-proxy-certs.sh"
permissions: "0744"
encoding: "gzip"
owner: "root"
content: !!binary |
{{WrapAsVariable "generateProxyCertsScript"}}
{{end}}
{{if HasCustomSearchDomain}}
- path: "/opt/azure/containers/setup-custom-search-domains.sh"
permissions: "0744"
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "customSearchDomainsScript"}}
{{end}}
- path: "/var/lib/kubelet/kubeconfig"
permissions: "0644"
owner: "root"
content: |
apiVersion: v1
kind: Config
clusters:
- name: localcluster
cluster:
certificate-authority: /etc/kubernetes/certs/ca.crt
server: <SERVERIP>
users:
- name: client
user:
client-certificate: /etc/kubernetes/certs/client.crt
client-key: /etc/kubernetes/certs/client.key
contexts:
- context:
cluster: localcluster
user: client
name: localclustercontext
current-context: localclustercontext
{{if EnableDataEncryptionAtRest}}
- path: "/etc/kubernetes/encryption-config.yaml"
permissions: "0600"
owner: "root"
content: |
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- aescbc:
keys:
- name: key1
secret: <etcdEncryptionSecret>
- identity: {}
{{end}}
{{if EnableEncryptionWithExternalKms}}
- path: "/etc/kubernetes/encryption-config.yaml"
permissions: "0444"
owner: "root"
content: |
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- kms:
name: azurekmsprovider
endpoint: unix:///opt/azurekms.socket
cachesize: 0
- identity: {}
{{end}}
MASTER_MANIFESTS_CONFIG_PLACEHOLDER
MASTER_ADDONS_CONFIG_PLACEHOLDER
MASTER_CUSTOM_FILES_PLACEHOLDER
- path: "/etc/default/kubelet"
permissions: "0644"
owner: "root"
content: |
{{if IsKubernetesVersionLt "1.8.0"}}
KUBELET_OPTS=--require-kubeconfig
{{else}}
KUBELET_OPTS=
{{end}}
KUBELET_CONFIG={{GetKubeletConfigKeyVals .MasterProfile.KubernetesConfig}}
KUBELET_IMAGE={{WrapAsParameter "kubernetesHyperkubeSpec"}}
KUBELET_NODE_LABELS={{GetMasterKubernetesLabels "',variables('labelResourceGroup'),'"}}
{{if IsKubernetesVersionGe "1.6.0"}}
{{if HasLinuxAgents}}
KUBELET_REGISTER_NODE=--register-node=true
KUBELET_REGISTER_WITH_TAINTS=--register-with-taints=node-role.kubernetes.io/master=true:NoSchedule
{{end}}
{{else}}
KUBELET_REGISTER_SCHEDULABLE={{WrapAsVariable "registerSchedulable"}}
{{end}}
MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
- path: "/opt/azure/containers/kubelet.sh"
permissions: "0755"
owner: "root"
content: |
#!/bin/bash
set -e
PRIVATE_IP=$(hostname -I | cut -d" " -f1)
ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
{{if gt .MasterProfile.Count 1}}
# Azure does not support two LoadBalancers(LB) sharing the same nic and backend port.
# As a workaround, the Internal LB(ILB) listens for apiserver traffic on port 4443 and the External LB(ELB) on port 443
# This IPTable rule then redirects ILB traffic to port 443 in the prerouting chain
iptables -t nat -A PREROUTING -p tcp --dport 4443 -j REDIRECT --to-port 443
{{end}}
{{if IsAzureCNI}}
# SNAT outbound traffic from pods to destinations outside of VNET.
iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
sed -i "s|<azureCNINetworkMonitorImage>|{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml"
{{end}}
sed -i "s|<kubernetesAddonManagerSpec>|{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml"
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml"
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsParameter "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsParameter "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsParameter "kubernetesExecHealthzSpec"}}|g; s|<kubernetesDNSSidecarSpec>|{{WrapAsParameter "kubernetesDNSSidecarSpec"}}|g; s|<kubernetesKubeletClusterDomain>|{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s|<kubeDNSServiceIP>|{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}}
sed -i "s|<kubernetesDashboardSpec>|{{WrapAsParameter "kubernetesDashboardSpec"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
sed -i "s|<kubernetesDashboardCPURequests>|{{WrapAsParameter "kubernetesDashboardCPURequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
sed -i "s|<kubernetesDashboardMemoryRequests>|{{WrapAsParameter "kubernetesDashboardMemoryRequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
sed -i "s|<kubernetesDashboardCPULimit>|{{WrapAsParameter "kubernetesDashboardCPULimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
sed -i "s|<kubernetesDashboardMemoryLimit>|{{WrapAsParameter "kubernetesDashboardMemoryLimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsTillerEnabled}}
sed -i "s|<kubernetesTillerSpec>|{{WrapAsParameter "kubernetesTillerSpec"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
sed -i "s|<kubernetesTillerCPURequests>|{{WrapAsParameter "kubernetesTillerCPURequests"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
sed -i "s|<kubernetesTillerMemoryRequests>|{{WrapAsParameter "kubernetesTillerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
sed -i "s|<kubernetesTillerCPULimit>|{{WrapAsParameter "kubernetesTillerCPULimit"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
sed -i "s|<kubernetesTillerMemoryLimit>|{{WrapAsParameter "kubernetesTillerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
sed -i "s|<kubernetesTillerMaxHistory>|{{WrapAsParameter "kubernetesTillerMaxHistory"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
{{end}}
{{if AdminGroupID }}
sed -i "s|<aadAdminGroupId>|{{WrapAsParameter "aadAdminGroupId"}}|g" "/etc/kubernetes/addons/aad-default-admin-group-rbac.yaml"
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled}}
sed -i "s|<kubernetesACIConnectorSpec>|{{WrapAsParameter "kubernetesACIConnectorSpec"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
sed -i "s|<kubernetesACIConnectorNodeName>|{{WrapAsParameter "kubernetesACIConnectorNodeName"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
sed -i "s|<kubernetesACIConnectorOS>|{{WrapAsParameter "kubernetesACIConnectorOS"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
sed -i "s|<kubernetesACIConnectorTaint>|{{WrapAsParameter "kubernetesACIConnectorTaint"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
sed -i "s|<kubernetesACIConnectorRegion>|{{WrapAsParameter "kubernetesACIConnectorRegion"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
sed -i "s|<kubernetesACIConnectorCPURequests>|{{WrapAsParameter "kubernetesACIConnectorCPURequests"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
sed -i "s|<kubernetesACIConnectorMemoryRequests>|{{WrapAsParameter "kubernetesACIConnectorMemoryRequests"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
sed -i "s|<kubernetesACIConnectorCPULimit>|{{WrapAsParameter "kubernetesACIConnectorCPULimit"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
sed -i "s|<kubernetesACIConnectorMemoryLimit>|{{WrapAsParameter "kubernetesACIConnectorMemoryLimit"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}}
sed -i "s|<kubernetesClusterAutoscalerAzureCloud>|{{WrapAsParameter "kubernetesClusterAutoscalerAzureCloud"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
sed -i "s|<kubernetesClusterAutoscalerSpec>|{{WrapAsParameter "kubernetesClusterAutoscalerSpec"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
sed -i "s|<kubernetesClusterAutoscalerCPULimit>|{{WrapAsParameter "kubernetesClusterAutoscalerCPULimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
sed -i "s|<kubernetesClusterAutoscalerMemoryLimit>|{{WrapAsParameter "kubernetesClusterAutoscalerMemoryLimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
sed -i "s|<kubernetesClusterAutoscalerCPURequests>|{{WrapAsParameter "kubernetesClusterAutoscalerCPURequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
sed -i "s|<kubernetesClusterAutoscalerMemoryRequests>|{{WrapAsParameter "kubernetesClusterAutoscalerMemoryRequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
sed -i "s|<kubernetesClusterAutoscalerMinNodes>|{{WrapAsParameter "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
sed -i "s|<kubernetesClusterAutoscalerMaxNodes>|{{WrapAsParameter "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
sed -i "s|<kubernetesClusterAutoscalerUseManagedIdentity>|{{WrapAsParameter "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
{{end}}
{{if eq .OrchestratorProfile.KubernetesConfig.LoadBalancerSku "Standard"}}
sed -i "s|<kuberneteselbsvcname>|{{WrapAsParameter "kuberneteselbsvcname"}}|g" "/etc/kubernetes/addons/elb-svc.yaml"
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsBlobfuseFlexVolumeEnabled}}
sed -i "s|<kubernetesBlobfuseFlexVolumeInstallerCPURequests>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml"
sed -i "s|<kubernetesBlobfuseFlexVolumeInstallerMemoryRequests>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml"
sed -i "s|<kubernetesBlobfuseFlexVolumeInstallerCPULimit>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml"
sed -i "s|<kubernetesBlobfuseFlexVolumeInstallerMemoryLimit>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml"
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsSMBFlexVolumeEnabled}}
sed -i "s|<kubernetesSMBFlexVolumeInstallerCPURequests>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml"
sed -i "s|<kubernetesSMBFlexVolumeInstallerMemoryRequests>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml"
sed -i "s|<kubernetesSMBFlexVolumeInstallerCPULimit>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml"
sed -i "s|<kubernetesSMBFlexVolumeInstallerMemoryLimit>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml"
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled}}
sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerCPURequests>|{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerMemoryRequests>|{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerCPULimit>|{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerMemoryLimit>|{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}}
sed -i "s|<kubernetesReschedulerSpec>|{{WrapAsParameter "kubernetesReschedulerSpec"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
sed -i "s|<kubernetesReschedulerCPURequests>|{{WrapAsParameter "kubernetesReschedulerCPURequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
sed -i "s|<kubernetesReschedulerMemoryRequests>|{{WrapAsParameter "kubernetesReschedulerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
sed -i "s|<kubernetesReschedulerCPULimit>|{{WrapAsParameter "kubernetesReschedulerCPULimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
sed -i "s|<kubernetesReschedulerMemoryLimit>|{{WrapAsParameter "kubernetesReschedulerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
{{end}}
{{if .OrchestratorProfile.IsMetricsServerEnabled}}
sed -i "s|<kubernetesMetricsServerSpec>|{{WrapAsParameter "kubernetesMetricsServerSpec"}}|g" "/etc/kubernetes/addons/kube-metrics-server-deployment.yaml"
{{end}}
{{if IsNVIDIADevicePluginEnabled}}
sed -i "s|<kubernetesNVIDIADevicePluginSpec>|{{WrapAsParameter "kubernetesNVIDIADevicePluginSpec"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
sed -i "s|<kubernetesNVIDIADevicePluginCPURequests>|{{WrapAsParameter "kubernetesNVIDIADevicePluginCPURequests"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
sed -i "s|<kubernetesNVIDIADevicePluginMemoryRequests>|{{WrapAsParameter "kubernetesNVIDIADevicePluginMemoryRequests"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
sed -i "s|<kubernetesNVIDIADevicePluginCPULimit>|{{WrapAsParameter "kubernetesNVIDIADevicePluginCPULimit"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
sed -i "s|<kubernetesNVIDIADevicePluginMemoryLimit>|{{WrapAsParameter "kubernetesNVIDIADevicePluginMemoryLimit"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
{{end}}
{{if EnableDataEncryptionAtRest }}
sed -i "s|<etcdEncryptionSecret>|\"{{WrapAsParameter "etcdEncryptionKey"}}\"|g" "/etc/kubernetes/encryption-config.yaml"
{{end}}
{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}}
# If Calico Policy enabled then update Cluster Cidr
sed -i "s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml"
{{end}}
{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPlugin "flannel"}}
# If Flannel is enabled then update Cluster Cidr
sed -i "s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/flannel-daemonset.yaml"
{{end}}
{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "cilium"}}
# If Cilium Policy enabled then update the etcd certs and address
sed -i "s|<ETCD_URL>|"https://$PRIVATE_IP:$ETCD_CLIENT_PORT"|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
sed -i "s|<ETCD_CA>|$(base64 -w 0 /etc/kubernetes/certs/ca.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
sed -i "s|<ETCD_CLIENT_KEY>|$(base64 -w 0 /etc/kubernetes/certs/etcdclient.key)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
sed -i "s|<ETCD_CLIENT_CERT>|$(base64 -w 0 /etc/kubernetes/certs/etcdclient.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
{{end}}
{{if UseCloudControllerManager }}
sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsParameter "kubernetesCcmImageSpec"}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml"
sed -i "s|<kubernetesCloudControllerManagerConfig>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml"
{{end}}
sed -i "s|<kubernetesControllerManagerConfig>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.ControllerManagerConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml"
sed -i "s|<kubernetesAPIServerConfig>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.APIServerConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
sed -i "s|<kubernetesSchedulerConfig>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.SchedulerConfig}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
sed -i "s|<kubernetesAPIServerIP>|{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
{{if not EnablePodSecurityPolicy}}
sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service"
{{end}}
{{if EnableEncryptionWithExternalKms}}
sed -i "s|# Required|Requires=kms.service|g" "/etc/systemd/system/kubelet.service"
{{end}}
{{if HasCustomSearchDomain}}
sed -i "s|<searchDomainName>|{{WrapAsParameter "searchDomainName"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh"
sed -i "s|<searchDomainRealmUser>|{{WrapAsParameter "searchDomainRealmUser"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh"
sed -i "s|<searchDomainRealmPassword>|{{WrapAsParameter "searchDomainRealmPassword"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh"
{{end}}
{{if .OrchestratorProfile.KubernetesConfig.IsContainerMonitoringEnabled}}
sed -i "s|<omsAgentVersion>|{{WrapAsParameter "omsAgentVersion"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
sed -i "s|<dockerProviderVersion>|{{WrapAsParameter "omsAgentDockerProviderVersion"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
sed -i "s|<kubernetesContainerMonitoringSpec>|{{WrapAsParameter "omsAgentImage"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
sed -i "s|<workspaceGuid>|{{WrapAsParameter "omsAgentWorkspaceGuid"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
sed -i "s|<workspaceKey>|{{WrapAsParameter "omsAgentWorkspaceKey"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
sed -i "s|<kubernetesOMSAgentCPURequests>|{{WrapAsParameter "kubernetesOMSAgentCPURequests"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
sed -i "s|<kubernetesOMSAgentMemoryRequests>|{{WrapAsParameter "kubernetesOMSAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
sed -i "s|<kubernetesOMSAgentCPULimit>|{{WrapAsParameter "kubernetesOMSAgentCPULimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
sed -i "s|<kubernetesOMSAgentMemoryLimit>|{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
{{end}}
- path: "/opt/azure/containers/provision.sh"
permissions: "0744"
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "provisionScript"}}
- path: "/opt/azure/containers/mountetcd.sh"
permissions: "0744"
encoding: gzip
owner: "root"
content: !!binary |
{{WrapAsVariable "mountetcdScript"}}
- path: "/etc/systemd/system/etcd.service"
permissions: "0644"
owner: "root"
content: |
[Unit]
Description=etcd - highly-available key value store
Documentation=https://github.com/coreos/etcd
Documentation=man:etcd
After=network.target
Wants=network-online.target
[Service]
Environment=DAEMON_ARGS=
Environment=ETCD_NAME=%H
Environment=ETCD_DATA_DIR=
EnvironmentFile=-/etc/default/%p
Type=notify
User=etcd
PermissionsStartOnly=true
ExecStart=/usr/bin/etcd $DAEMON_ARGS
Restart=always
[Install]
WantedBy=multi-user.target
- path: "/opt/azure/containers/setup-etcd.sh"
permissions: "0744"
owner: "root"
content: |
#!/bin/bash
set -x
MASTER_VM_NAME=$(hostname)
MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//")
MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}}
MASTER_INDEX=$(hostname | tail -c 2)
PRIVATE_IP=$(hostname -I | cut -d" " -f1)
PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3)
MASTER_COUNT={{WrapAsVariable "masterCount"}}
IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}}
echo $IPADDRESS_COUNT
ETCD_SERVER_PORT={{WrapAsVariable "masterEtcdServerPort"}}
ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
MASTER_URLS=""
index=0
while [ $index -lt $MASTER_COUNT ]
do
echo $index
offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4`
echo $offset
MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT,"
index=`expr $index + 1`
done
MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//")
echo $MASTER_URLS
sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:$ETCD_CLIENT_PORT" /etc/environment
sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment
sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment
sudo sed -i "1iETCDCTL_CERT_FILE={{WrapAsVariable "etcdClientCertFilepath"}}" /etc/environment
sudo sed -i "s|<SERVERIP>|https://$PRIVATE_IP:443|g" "/var/lib/kubelet/kubeconfig"
/bin/echo DAEMON_ARGS=--name $MASTER_VM_NAME --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.crt --peer-key-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.key --initial-advertise-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --listen-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT" --listen-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT,https://127.0.0.1:$ETCD_CLIENT_PORT" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster $MASTER_URLS --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd
{{if .MasterProfile.IsCoreOS}}
- path: "/opt/azure/containers/provision-setup.sh"
permissions: "0755"
owner: "root"
content: |
#!/bin/bash
# the first arg is the number of retries, the second arg is the wait duration between two retries and the rest of the args are the cmd to run
source /opt/azure/containers/provision_source.sh
set -x
MASTER_VM_NAME=$(hostname)
MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//")
MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}}
MASTER_INDEX=$(hostname | tail -c 2)
PRIVATE_IP=$(hostname -I | cut -d" " -f1)
PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3)
MASTER_COUNT={{WrapAsVariable "masterCount"}}
IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}}
echo $IPADDRESS_COUNT
ETCD_SERVER_PORT={{WrapAsVariable "masterEtcdServerPort"}}
ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
MASTER_URLS=""
index=0
while [ $index -lt $MASTER_COUNT ]
do
echo $index
offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4`
echo $offset
MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT,"
index=`expr $index + 1`
done
MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//")
echo $MASTER_URLS
/bin/echo DAEMON_ARGS=--name $MASTER_VM_NAME --initial-advertise-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --listen-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --advertise-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT" --listen-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT,https://127.0.0.1:$ETCD_CLIENT_PORT" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster $MASTER_URLS --data-dir "/var/lib/etcddisk"" --initial-cluster-state "new" | tee -a /etc/default/etcd
/opt/azure/containers/mountetcd.sh
sudo /bin/chown -R etcd:etcd /var/lib/etcddisk
systemctl stop etcd-member
sudo /bin/sed -i s/Restart=on-failure/Restart=always/g /lib/systemd/system/etcd-member.service
systemctl daemon-reload
systemctl restart etcd-member
retrycmd_if_failure 5 5 10 curl --retry 5 --retry-delay 10 --retry-max-time 10 --max-time 60 http://127.0.0.1:2379/v2/machines
mkdir -p /etc/kubernetes/manifests
{{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}}
usermod -aG docker {{WrapAsParameter "linuxAdminUsername"}}
{{end}}
{{if EnableAggregatedAPIs}}
sudo bash /etc/kubernetes/generate-proxy-certs.sh
{{end}}
touch /opt/azure/containers/runcmd.complete
coreos:
units:
- name: start-provision-setup.service
command: "start"
content: |
# Note: Initiated as a service since there is no runcmd within CoreOS on cloud-config/Ignition
[Unit]
Description=Start provision setup service
[Service]
ExecStart=/opt/azure/containers/provision-setup.sh
{{else}}
runcmd:
- set -x
- timeout 10 apt-mark hold walinuxagent{{GetKubernetesMasterPreprovisionYaml}}
- timeout 10 apt-mark unhold walinuxagent
{{end}}

2
parts/k8s/kubernetesmasterresources.t
Просмотреть файл

@ -924,7 +924,7 @@
{{if IsOpenShift}}
"script": "{{ Base64 OpenShiftGetMasterSh }}"
{{else}}
"commandToExecute": "[concat('for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' MASTER_INDEX=',copyIndex(variables('masterOffset')),' /usr/bin/nohup /bin/bash -c \"stat /opt/azure/containers/provision.complete > /dev/null 2>&1 || /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]"
"commandToExecute": "[concat('for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"stat /opt/azure/containers/provision.complete > /dev/null 2>&1 || /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]"
{{end}}
}
}

468
parts/k8s/kubernetesmasterresourcesvmss.t Normal file
Просмотреть файл

@ -0,0 +1,468 @@
{{if and UseManagedIdentity (not UserAssignedIDEnabled)}}
{
"apiVersion": "2014-10-01-preview",
"name": "[guid(concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmidentity'))]",
"type": "Microsoft.Authorization/roleAssignments",
"properties": {
"roleDefinitionId": "[variables('contributorRoleDefinitionId')]",
"principalId": "[reference(concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmss'), '2017-03-30', 'Full').identity.principalId]"
}
},
{{end}}
{{if EnableEncryptionWithExternalKms}}
{
"type": "Microsoft.Storage/storageAccounts",
"name": "[variables('clusterKeyVaultName')]",
"apiVersion": "[variables('apiVersionStorage')]",
"location": "[variables('location')]",
"properties": {
"accountType": "Standard_LRS"
}
},
{
"type": "Microsoft.KeyVault/vaults",
"name": "[variables('clusterKeyVaultName')]",
"apiVersion": "[variables('apiVersionKeyVault')]",
"location": "[variables('location')]",
{{ if UseManagedIdentity}}
"dependsOn":
[
"[concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmss')]",
"[concat('Microsoft.Authorization/roleAssignments/', guid(concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmidentity')))]",
],
{{end}}
"properties": {
"enabledForDeployment": "false",
"enabledForDiskEncryption": "false",
"enabledForTemplateDeployment": "false",
"tenantId": "[variables('tenantID')]",
{{if not UseManagedIdentity}}
"accessPolicies":
[
{
"tenantId": "[variables('tenantID')]",
"objectId": "[parameters('servicePrincipalObjectId')]",
"permissions": {
"keys": ["create", "encrypt", "decrypt", "get", "list"]
}
}
],
{{else}}
"accessPolicies":
[
{
"objectId": "[reference(concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmss'), '2017-03-30', 'Full').identity.principalId]",
"permissions": {
"keys": [
"create",
"encrypt",
"decrypt",
"get",
"list"
]
},
"tenantId": "[variables('tenantID')]"
},
],
{{end}}
"sku": {
"name": "[parameters('clusterKeyVaultSku')]",
"family": "A"
}
}
},
{{end}}
{
"apiVersion": "[variables('apiVersionDefault')]",
"location": "[variables('location')]",
"name": "[variables('nsgName')]",
"properties": {
"securityRules": [
{{if .HasWindows}}
{
"name": "allow_rdp",
"properties": {
"access": "Allow",
"description": "Allow RDP traffic to master",
"destinationAddressPrefix": "*",
"destinationPortRange": "3389-3389",
"direction": "Inbound",
"priority": 102,
"protocol": "Tcp",
"sourceAddressPrefix": "*",
"sourcePortRange": "*"
}
},
{{end}}
{
"name": "allow_ssh",
"properties": {
"access": "Allow",
"description": "Allow SSH traffic to master",
"destinationAddressPrefix": "*",
"destinationPortRange": "22-22",
"direction": "Inbound",
"priority": 101,
"protocol": "Tcp",
"sourceAddressPrefix": "*",
"sourcePortRange": "*"
}
},
{
"name": "allow_kube_tls",
"properties": {
"access": "Allow",
"description": "Allow kube-apiserver (tls) traffic to master",
"destinationAddressPrefix": "*",
"destinationPortRange":"443-443",
"direction": "Inbound",
"priority": 100,
"protocol": "Tcp",
"sourceAddressPrefix": "*",
"sourcePortRange": "*"
}
}
]
},
"type": "Microsoft.Network/networkSecurityGroups"
},
{{if RequireRouteTable}}
{
"apiVersion": "[variables('apiVersionDefault')]",
"location": "[variables('location')]",
"name": "[variables('routeTableName')]",
"type": "Microsoft.Network/routeTables"
},
{{end}}
{{if not .MasterProfile.IsCustomVNET}}
{
"apiVersion": "[variables('apiVersionDefault')]",
"dependsOn": [
{{if RequireRouteTable}}
"[concat('Microsoft.Network/routeTables/', variables('routeTableName'))]",
{{end}}
"[concat('Microsoft.Network/networkSecurityGroups/', variables('nsgName'))]"
],
"location": "[variables('location')]",
"name": "[variables('virtualNetworkName')]",
"properties": {
"addressSpace": {
"addressPrefixes": [
"[parameters('vnetCidr')]"
]
},
"subnets": [
{
"name": "subnetmaster",
"properties": {
"addressPrefix": "[parameters('masterSubnet')]"
,"networkSecurityGroup": {
"id": "[variables('nsgID')]"
}
{{if RequireRouteTable}}
,"routeTable": {
"id": "[variables('routeTableID')]"
}
{{end}}
}
},
{
"name":"subnetagent",
"properties":{
"addressPrefix": "[parameters('agentSubnet')]",
"networkSecurityGroup": {
"id": "[variables('nsgID')]"
}
{{if RequireRouteTable}}
,"routeTable": {
"id": "[variables('routeTableID')]"
}
{{end}}
}
}
]
},
"type": "Microsoft.Network/virtualNetworks"
},
{{end}}
{
"apiVersion": "[variables('apiVersionDefault')]",
"location": "[variables('location')]",
"name": "[variables('masterPublicIPAddressName')]",
"properties": {
"dnsSettings": {
"domainNameLabel": "[variables('masterFqdnPrefix')]"
},
"publicIPAllocationMethod": "Dynamic"
},
"type": "Microsoft.Network/publicIPAddresses"
},
{
"type": "Microsoft.Network/loadBalancers",
"name": "[variables('masterLbName')]",
"location": "[variables('location')]",
"apiVersion": "[variables('apiVersionDefault')]",
"dependsOn": [
"[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]"
],
"properties": {
"frontendIPConfigurations": [
{
"name": "[variables('masterLbIPConfigName')]",
"properties": {
"publicIPAddress": {
"id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('masterPublicIPAddressName'))]"
}
}
}
],
"backendAddressPools": [
{
"name": "[variables('masterLbBackendPoolName')]"
}
],
"probes": [
{
"name": "tcpHTTPSProbe",
"properties": {
"protocol": "tcp",
"port": 443,
"intervalInSeconds": 5,
"numberOfProbes": 2
}
}
],
"inboundNatPools": [
{
"name": "[concat('SSH-', variables('masterVMNamePrefix'), 'natpools')]",
"properties": {
"frontendIPConfiguration": {
"id": "[variables('masterLbIPConfigID')]"
},
"protocol": "tcp",
"backendPort": "22",
"frontendPortRangeStart": "50001",
"frontendPortRangeEnd": "50119",
"enableFloatingIP": false
}
}
],
"loadBalancingRules": [
{
"name": "LBRuleHTTPS",
"properties": {
"frontendIPConfiguration": {
"id": "[variables('masterLbIPConfigID')]"
},
"backendAddressPool": {
"id": "[concat(variables('masterLbID'), '/backendAddressPools/', variables('masterLbBackendPoolName'))]"
},
"protocol": "tcp",
"frontendPort": 443,
"backendPort": 443,
"enableFloatingIP": false,
"idleTimeoutInMinutes": 5,
"loadDistribution": "Default",
"probe": {
"id": "[concat(variables('masterLbID'),'/probes/tcpHTTPSProbe')]"
}
}
}
]
}
},
{
"apiVersion": "[variables('apiVersionVirtualMachineScaleSets')]",
"dependsOn": [
{{if .MasterProfile.IsCustomVNET}}
"[variables('nsgID')]"
{{else}}
"[variables('vnetID')]",
"[variables('masterLbID')]"
{{end}}
],
"tags":
{
"creationSource": "[concat(parameters('generatorCode'), '-', variables('masterVMNamePrefix'), 'vmss')]",
"resourceNameSuffix": "[parameters('nameSuffix')]",
"orchestrator": "[variables('orchestratorNameVersionTag')]",
"acsengineVersion" : "[parameters('acsengineVersion')]",
"poolName": "master"
},
"location": "[variables('location')]",
"name": "[concat(variables('masterVMNamePrefix'), 'vmss')]",
{{if UseManagedIdentity}}
{{if UserAssignedIDEnabled}}
"identity": {
"type": "userAssigned",
"identityIds": [
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('userAssignedID'))]"
]
},
{{else}}
"identity": {
"type": "systemAssigned"
},
{{end}}
{{end}}
"sku": {
"tier": "Standard",
"capacity": "[variables('masterCount')]",
"name": "[parameters('masterVMSize')]"
},
"properties": {
"overprovision": false,
"upgradePolicy": {
"mode": "Manual"
},
"virtualMachineProfile": {
"networkProfile": {
"networkInterfaceConfigurations": [
{
"name": "[concat(variables('masterVMNamePrefix'), 'netintconfig')]",
"properties": {
"primary": true,
{{if .MasterProfile.IsCustomVNET}}
"networkSecurityGroup": {
"id": "[variables('nsgID')]"
},
{{end}}
"ipConfigurations": [
{{range $seq := loop 1 .MasterProfile.IPAddressCount}}
{
"name": "ipconfig{{$seq}}",
"properties": {
{{if eq $seq 1}}
"loadBalancerBackendAddressPools": [
{
"id": "[concat(variables('masterLbID'), '/backendAddressPools/', variables('masterLbBackendPoolName'))]"
}
],
"loadBalancerInboundNatPools": [
{
"id": "[concat(variables('masterLbID'),'/inboundNatPools/SSH-', variables('masterVMNamePrefix'), 'natpools')]"
}
],
"primary": true,
{{else}}
"primary": false,
{{end}}
"subnet": {
"id": "[variables('vnetSubnetIDMaster')]"
}
}
}
{{if lt $seq $.MasterProfile.IPAddressCount}},{{end}}
{{end}}
]
{{if HasCustomNodesDNS}}
,"dnsSettings": {
"dnsServers": [
"[parameters('dnsServer')]"
]
}
{{end}}
{{if not IsAzureCNI}}
,"enableIPForwarding": true
{{end}}
}
}
]
},
"osProfile": {
"adminUsername": "[parameters('linuxAdminUsername')]",
"computerNamePrefix": "[concat(variables('masterVMNamePrefix'), 'vmss')]",
{{GetKubernetesMasterCustomDataVMSS .}}
"linuxConfiguration": {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{
"keyData": "[parameters('sshRSAPublicKey')]",
"path": "[variables('sshKeyPath')]"
}
]
}
}
{{if .LinuxProfile.HasSecrets}}
,
"secrets": "[variables('linuxProfileSecrets')]"
{{end}}
},
"storageProfile": {
{{if not UseMasterCustomImage }}
"dataDisks": [
{
"createOption": "Empty",
"diskSizeGB": "[parameters('etcdDiskSizeGB')]",
"lun": 0
}
],
{{end}}
"imageReference": {
{{if UseMasterCustomImage}}
"id": "[resourceId(parameters('osImageResourceGroup'), 'Microsoft.Compute/images', parameters('osImageName'))]"
{{else}}
"offer": "[parameters('osImageOffer')]",
"publisher": "[parameters('osImagePublisher')]",
"sku": "[parameters('osImageSku')]",
"version": "[parameters('osImageVersion')]"
{{end}}
},
"osDisk": {
"caching": "ReadWrite"
,"createOption": "FromImage"
{{if ne .MasterProfile.OSDiskSizeGB 0}}
,"diskSizeGB": {{.MasterProfile.OSDiskSizeGB}}
{{end}}
}
},
"extensionProfile": {
"extensions": [
{{if UseManagedIdentity}}
{
"name": "[concat(variables('masterVMNamePrefix'), 'vmss-ManagedIdentityExtension')]",
"properties": {
"publisher": "Microsoft.ManagedIdentity",
"type": "ManagedIdentityExtensionForLinux",
"typeHandlerVersion": "1.0",
"autoUpgradeMinorVersion": true,
"settings": {
"port": 50343
},
"protectedSettings": {}
}
},
{{end}}
{
"name": "[concat(variables('masterVMNamePrefix'), 'vmssCSE')]",
"properties": {
"publisher": "Microsoft.Azure.Extensions",
"type": "CustomScript",
"typeHandlerVersion": "2.0",
"autoUpgradeMinorVersion": true,
"settings": {},
"protectedSettings": {
"commandToExecute": "[concat('for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"stat /opt/azure/containers/provision.complete > /dev/null 2>&1 || /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]"
}
}
}
{{if UseAksExtension}}
,{
"name": "[concat(variables('masterVMNamePrefix'), 'vmss-computeAksLinuxBilling')]",
"location": "[variables('location')]",
"properties": {
"publisher": "Microsoft.AKS",
"type": "Compute.AKS-Engine.Linux.Billing",
"typeHandlerVersion": "1.0",
"autoUpgradeMinorVersion": true,
"settings": {}
}
}
{{end}}
]
}
}
},
"type": "Microsoft.Compute/virtualMachineScaleSets"
}

286
parts/k8s/kubernetesmastervarsvmss.t Normal file
Просмотреть файл

@ -0,0 +1,286 @@
"maxVMsPerPool": 100,
{{ if not IsHostedMaster }}
{{if eq .MasterProfile.Count 1}}
"etcdPeerPrivateKeys": [
"[parameters('etcdPeerPrivateKey0')]"
],
"etcdPeerCertificates": [
"[parameters('etcdPeerCertificate0')]"
],
{{end}}
{{if eq .MasterProfile.Count 3}}
"etcdPeerPrivateKeys": [
"[parameters('etcdPeerPrivateKey0')]",
"[parameters('etcdPeerPrivateKey1')]",
"[parameters('etcdPeerPrivateKey2')]"
],
"etcdPeerCertificates": [
"[parameters('etcdPeerCertificate0')]",
"[parameters('etcdPeerCertificate1')]",
"[parameters('etcdPeerCertificate2')]"
],
{{end}}
{{if eq .MasterProfile.Count 5}}
"etcdPeerPrivateKeys": [
"[parameters('etcdPeerPrivateKey0')]",
"[parameters('etcdPeerPrivateKey1')]",
"[parameters('etcdPeerPrivateKey2')]",
"[parameters('etcdPeerPrivateKey3')]",
"[parameters('etcdPeerPrivateKey4')]"
],
"etcdPeerCertificates": [
"[parameters('etcdPeerCertificate0')]",
"[parameters('etcdPeerCertificate1')]",
"[parameters('etcdPeerCertificate2')]",
"[parameters('etcdPeerCertificate3')]",
"[parameters('etcdPeerCertificate4')]"
],
{{end}}
"etcdPeerCertFilepath":[
"/etc/kubernetes/certs/etcdpeer0.crt",
"/etc/kubernetes/certs/etcdpeer1.crt",
"/etc/kubernetes/certs/etcdpeer2.crt",
"/etc/kubernetes/certs/etcdpeer3.crt",
"/etc/kubernetes/certs/etcdpeer4.crt"
],
"etcdPeerKeyFilepath":[
"/etc/kubernetes/certs/etcdpeer0.key",
"/etc/kubernetes/certs/etcdpeer1.key",
"/etc/kubernetes/certs/etcdpeer2.key",
"/etc/kubernetes/certs/etcdpeer3.key",
"/etc/kubernetes/certs/etcdpeer4.key"
],
"etcdCaFilepath": "/etc/kubernetes/certs/ca.crt",
"etcdClientCertFilepath": "/etc/kubernetes/certs/etcdclient.crt",
"etcdClientKeyFilepath": "/etc/kubernetes/certs/etcdclient.key",
"etcdServerCertFilepath": "/etc/kubernetes/certs/etcdserver.crt",
"etcdServerKeyFilepath": "/etc/kubernetes/certs/etcdserver.key",
{{end}}
"useManagedIdentityExtension": "{{ UseManagedIdentity }}",
"userAssignedID": "{{UserAssignedID}}",
"userAssignedClientID": "{{UserAssignedClientID}}",
"useInstanceMetadata": "{{ UseInstanceMetadata }}",
"loadBalancerSku": "{{ LoadBalancerSku }}",
"excludeMasterFromStandardLB": "{{ ExcludeMasterFromStandardLB }}",
{{ if UseManagedIdentity }}
"servicePrincipalClientId": "msi",
"servicePrincipalClientSecret": "msi",
{{ else }}
"servicePrincipalClientId": "[parameters('servicePrincipalClientId')]",
"servicePrincipalClientSecret": "[parameters('servicePrincipalClientSecret')]",
{{ end }}
"masterFqdnPrefix": "[tolower(parameters('masterEndpointDNSNamePrefix'))]",
{{if not IsHostedMaster}}
"masterCount": {{.MasterProfile.Count}},
"masterOffset": "",
"masterIpAddressCount": {{.MasterProfile.IPAddressCount}},
{{end}}
"apiVersionDefault": "2016-03-30",
"apiVersionLinkDefault": "2015-01-01",
"locations": [
"[resourceGroup().location]",
"[parameters('location')]"
],
"location": "[variables('locations')[mod(add(2,length(parameters('location'))),add(1,length(parameters('location'))))]]",
"resourceGroup": "[resourceGroup().name]",
"truncatedResourceGroup": "[take(replace(replace(resourceGroup().name, '(', '-'), ')', '-'), 63)]",
"labelResourceGroup": "[if(or(or(endsWith(variables('truncatedResourceGroup'), '-'), endsWith(variables('truncatedResourceGroup'), '_')), endsWith(variables('truncatedResourceGroup'), '.')), concat(take(variables('truncatedResourceGroup'), 62), 'z'), variables('truncatedResourceGroup'))]",
{{if not IsHostedMaster}}
"routeTableName": "[concat(variables('masterVMNamePrefix'),'routetable')]",
{{else}}
"routeTableName": "[concat(variables('agentNamePrefix'), 'routetable')]",
{{end}}
"routeTableID": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]",
"sshNatPorts": [22,2201,2202,2203,2204],
"sshKeyPath": "[concat('/home/',parameters('linuxAdminUsername'),'/.ssh/authorized_keys')]",
{{if .HasStorageAccountDisks}}
"apiVersionStorage": "2015-06-15",
"maxVMsPerStorageAccount": 20,
"maxStorageAccountsPerAgent": "[div(variables('maxVMsPerPool'),variables('maxVMsPerStorageAccount'))]",
"dataStorageAccountPrefixSeed": 97,
"storageAccountPrefixes": [ "0", "6", "c", "i", "o", "u", "1", "7", "d", "j", "p", "v", "2", "8", "e", "k", "q", "w", "3", "9", "f", "l", "r", "x", "4", "a", "g", "m", "s", "y", "5", "b", "h", "n", "t", "z" ],
"storageAccountPrefixesCount": "[length(variables('storageAccountPrefixes'))]",
"vmsPerStorageAccount": 20,
"storageAccountBaseName": "[uniqueString(concat(variables('masterFqdnPrefix'),variables('location')))]",
{{GetSizeMap}},
{{else}}
"storageAccountPrefixes": [],
"storageAccountBaseName": "",
{{end}}
{{if UserAssignedIDEnabled}}
"apiVersionUserMSI": "2018-06-01",
{{end}}
{{if .HasManagedDisks}}
"apiVersionStorageManagedDisks": "2016-04-30-preview",
{{end}}
{{if .HasVirtualMachineScaleSets}}
"apiVersionVirtualMachineScaleSets": "2017-12-01",
{{end}}
{{if not IsHostedMaster}}
{{if .MasterProfile.IsStorageAccount}}
"masterStorageAccountName": "[concat(variables('storageAccountBaseName'), 'mstr0')]",
{{end}}
{{end}}
"provisionScript": "{{GetKubernetesB64Provision}}",
"provisionSource": "{{GetKubernetesB64ProvisionSource}}",
"healthMonitorScript": "{{GetKubernetesB64HealthMonitorScript}}",
"provisionInstalls": "{{GetKubernetesB64Installs}}",
"provisionConfigs": "{{GetKubernetesB64Configs}}",
"mountetcdScript": "{{GetKubernetesB64Mountetcd}}",
"customSearchDomainsScript": "{{GetKubernetesB64CustomSearchDomainsScript}}",
"sshdConfig": "{{GetB64sshdConfig}}",
"provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]",
{{if not IsHostedMaster}}
"provisionScriptParametersMaster": "[concat('MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]",
{{end}}
"generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}",
"orchestratorNameVersionTag": "{{.OrchestratorProfile.OrchestratorType}}:{{.OrchestratorProfile.OrchestratorVersion}}",
{{if IsAzureCNI}}
"allocateNodeCidrs": false,
{{else}}
"allocateNodeCidrs": true,
{{end}}
{{if not IsHostedMaster}}
{{if .MasterProfile.IsCustomVNET}}
"vnetSubnetID": "[parameters('agentVnetSubnetID')]",
"vnetSubnetIDMaster": "[parameters('masterVnetSubnetID')]",
"subnetNameResourceSegmentIndex": 10,
"subnetName": "[split(parameters('masterVnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]",
"vnetNameResourceSegmentIndex": 8,
"virtualNetworkName": "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]",
"vnetResourceGroupNameResourceSegmentIndex": 4,
"virtualNetworkResourceGroupName": "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]",
{{else}}
"subnetName": "subnetmaster",
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]",
"vnetSubnetID": "[concat(variables('vnetID'),'/subnets/subnetagent')]",
"vnetSubnetIDMaster": "[concat(variables('vnetID'),'/subnets/subnetmaster')]",
"virtualNetworkName": "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]",
"virtualNetworkResourceGroupName": "''",
{{end}}
{{else}}
{{if IsCustomVNET}}
"vnetSubnetID": "[parameters('{{ (index .AgentPoolProfiles 0).Name }}VnetSubnetID')]",
"subnetNameResourceSegmentIndex": 10,
"subnetName": "[split(variables('vnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]",
"vnetNameResourceSegmentIndex": 8,
"virtualNetworkName": "[split(variables('vnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]",
"vnetResourceGroupNameResourceSegmentIndex": 4,
"virtualNetworkResourceGroupName": "[split(variables('vnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]",
{{else}}
"subnetName": "[concat(parameters('orchestratorName'), '-subnet')]",
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]",
"vnetSubnetID": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]",
"virtualNetworkName": "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]",
"virtualNetworkResourceGroupName": "",
{{end}}
{{end}}
{{if not IsHostedMaster }}
"nsgName": "[concat(variables('masterVMNamePrefix'), 'nsg')]",
{{else}}
"nsgName": "[concat(variables('agentNamePrefix'), 'nsg')]",
{{end}}
"nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]",
{{if not AnyAgentUsesVirtualMachineScaleSets}}
"primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',parameters('nameSuffix'))]",
"primaryScaleSetName": "",
"vmType": "standard",
{{else}}
"primaryScaleSetName": "[concat(parameters('orchestratorName'), '-{{ (index .AgentPoolProfiles 0).Name }}-',parameters('nameSuffix'), '-vmss')]",
"primaryAvailabilitySetName": "",
"vmType": "vmss",
{{end}}
{{if not IsHostedMaster }}
{{if IsPrivateCluster}}
"kubeconfigServer": "[concat('https://', variables('kubernetesAPIServerIP'), ':443')]",
{{if ProvisionJumpbox}}
"jumpboxOSDiskName": "[concat(parameters('jumpboxVMName'), '-osdisk')]",
"jumpboxPublicIpAddressName": "[concat(parameters('jumpboxVMName'), '-ip')]",
"jumpboxNetworkInterfaceName": "[concat(parameters('jumpboxVMName'), '-nic')]",
"jumpboxNetworkSecurityGroupName": "[concat(parameters('jumpboxVMName'), '-nsg')]",
"kubeconfig": "{{GetKubeConfig}}",
{{if not JumpboxIsManagedDisks}}
"jumpboxStorageAccountName": "[concat(variables('storageAccountBaseName'), 'jb')]",
{{end}}
{{if not .HasStorageAccountDisks}}
{{GetSizeMap}},
{{end}}
{{end}}
{{else}}
"masterPublicIPAddressName": "[concat(parameters('orchestratorName'), '-master-ip-', variables('masterFqdnPrefix'), '-', parameters('nameSuffix'))]",
"masterLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterLbName'))]",
"masterLbIPConfigID": "[concat(variables('masterLbID'),'/frontendIPConfigurations/', variables('masterLbIPConfigName'))]",
"masterLbIPConfigName": "[concat(parameters('orchestratorName'), '-master-lbFrontEnd-', parameters('nameSuffix'))]",
"masterLbName": "[concat(parameters('orchestratorName'), '-master-lb-', parameters('nameSuffix'))]",
"kubeconfigServer": "[concat('https://', variables('masterFqdnPrefix'), '.', variables('location'), '.', parameters('fqdnEndpointSuffix'))]",
{{end}}
{{if gt .MasterProfile.Count 1}}
"masterInternalLbName": "[concat(parameters('orchestratorName'), '-master-internal-lb-', parameters('nameSuffix'))]",
"masterInternalLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterInternalLbName'))]",
"masterInternalLbIPConfigName": "[concat(parameters('orchestratorName'), '-master-internal-lbFrontEnd-', parameters('nameSuffix'))]",
"masterInternalLbIPConfigID": "[concat(variables('masterInternalLbID'),'/frontendIPConfigurations/', variables('masterInternalLbIPConfigName'))]",
"masterInternalLbIPOffset": {{GetDefaultInternalLbStaticIPOffset}},
"kubernetesAPIServerIP": "[parameters('firstConsecutiveStaticIP')]",
{{else}}
"kubernetesAPIServerIP": "[parameters('firstConsecutiveStaticIP')]",
{{end}}
"masterLbBackendPoolName": "[concat(parameters('orchestratorName'), '-master-pool-', parameters('nameSuffix'))]",
"masterFirstAddrComment": "these MasterFirstAddrComment are used to place multiple masters consecutively in the address space",
"masterFirstAddrOctets": "[split(parameters('firstConsecutiveStaticIP'),'.')]",
"masterFirstAddrOctet4": "[variables('masterFirstAddrOctets')[3]]",
"masterFirstAddrPrefix": "[concat(variables('masterFirstAddrOctets')[0],'.',variables('masterFirstAddrOctets')[1],'.',variables('masterFirstAddrOctets')[2],'.')]",
"masterVMNamePrefix": "[concat(parameters('orchestratorName'), '-master-', parameters('nameSuffix'), '-')]",
"masterVMNames": [
"[concat(variables('masterVMNamePrefix'), 'vmss000000')]",
"[concat(variables('masterVMNamePrefix'), 'vmss000001')]",
"[concat(variables('masterVMNamePrefix'), 'vmss000002')]",
"[concat(variables('masterVMNamePrefix'), 'vmss000003')]",
"[concat(variables('masterVMNamePrefix'), 'vmss000004')]"
],
"masterEtcdServerPort": {{GetMasterEtcdServerPort}},
"masterEtcdClientPort": {{GetMasterEtcdClientPort}},
{{else}}
"kubernetesAPIServerIP": "[parameters('kubernetesEndpoint')]",
"agentNamePrefix": "[concat(parameters('orchestratorName'), '-agentpool-', parameters('nameSuffix'), '-')]",
{{end}}
"subscriptionId": "[subscription().subscriptionId]",
"contributorRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
"readerRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]",
"scope": "[resourceGroup().id]",
"tenantId": "[subscription().tenantId]",
"singleQuote": "'"
{{if .LinuxProfile.HasSecrets}}
, "linuxProfileSecrets" :
[
{{range $vIndex, $vault := .LinuxProfile.Secrets}}
{{if $vIndex}} , {{end}}
{
"sourceVault":{
"id":"[parameters('linuxKeyVaultID{{$vIndex}}')]"
},
"vaultCertificates":[
{{range $cIndex, $cert := $vault.VaultCertificates}}
{{if $cIndex}} , {{end}}
{
"certificateUrl" :"[parameters('linuxKeyVaultID{{$vIndex}}CertificateURL{{$cIndex}}')]"
}
{{end}}
]
}
{{end}}
]
{{end}}
{{if .HasWindows}}
, "windowsCustomScriptSuffix": " $inputFile = '%SYSTEMDRIVE%\\AzureData\\CustomData.bin' ; $outputFile = '%SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.ps1' ; Copy-Item $inputFile $outputFile ; Invoke-Expression('{0} {1}' -f $outputFile, $arguments) ; "
{{end}}
{{if EnableEncryptionWithExternalKms}}
,"apiVersionKeyVault": "2016-10-01",
{{if not .HasStorageAccountDisks}}
"apiVersionStorage": "2015-06-15",
{{end}}
"clusterKeyVaultName": "[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]"
{{else}}
,"clusterKeyVaultName": ""
{{end}}

15
parts/masterparams.t
Просмотреть файл

@ -32,6 +32,14 @@
},
"type": "string"
},
{{if .MasterProfile.IsVirtualMachineScaleSets}}
"agentVnetSubnetID": {
"metadata": {
"description": "Sets the vnet subnet of the agent."
},
"type": "string"
},
{{end}}
{{else}}
"masterSubnet": {
"defaultValue": "{{.MasterProfile.Subnet}}",
@ -40,6 +48,13 @@
},
"type": "string"
},
"agentSubnet": {
"defaultValue": "{{.MasterProfile.AgentSubnet}}",
"metadata": {
"description": "Sets the subnet of the agent node(s)."
},
"type": "string"
},
{{end}}
{{end}}
{{if IsHostedMaster}}

17
pkg/acsengine/const.go
Просмотреть файл

@ -20,21 +20,17 @@ const (
// DefaultDCOSBootstrapStaticIP specifies the static IP address on bootstrap for a DCOS cluster
DefaultDCOSBootstrapStaticIP = "192.168.255.240"
// DefaultKubernetesMasterSubnet specifies the default subnet for masters and agents.
// Except when master VMSS is used, this specifies the default subnet for masters.
DefaultKubernetesMasterSubnet = "10.240.0.0/16"
// DefaultKubernetesClusterSubnet specifies the default subnet for pods.
DefaultKubernetesClusterSubnet = "10.244.0.0/16"
// DefaultDockerBridgeSubnet specifies the default subnet for the docker bridge network for masters and agents.
DefaultDockerBridgeSubnet = "172.17.0.1/16"
// DefaultFirstConsecutiveKubernetesStaticIP specifies the static IP address on Kubernetes master 0
DefaultFirstConsecutiveKubernetesStaticIP = "10.240.255.5"
// DefaultAgentSubnetTemplate specifies a default agent subnet
DefaultAgentSubnetTemplate = "10.%d.0.0/16"
// DefaultKubernetesSubnet specifies the default subnet used for all masters, agents and pods
// when VNET integration is enabled.
DefaultKubernetesSubnet = "10.240.0.0/12"
// DefaultKubernetesFirstConsecutiveStaticIPOffset specifies the IP address offset of master 0
// when VNET integration is enabled.
DefaultKubernetesFirstConsecutiveStaticIPOffset = 5
// DefaultKubernetesMaxPods is the maximum number of pods to run on a node.
DefaultKubernetesMaxPods = 110
// DefaultKubernetesMaxPodsVNETIntegrated is the maximum number of pods to run on a node when VNET integration is enabled.
@ -196,6 +192,10 @@ const (
DefaultJumpboxUsername = "azureuser"
// DefaultKubeletPodMaxPIDs specifies the default max pid authorized by pods
DefaultKubeletPodMaxPIDs = 100
// DefaultKubernetesAgentSubnetVMSS specifies the default subnet for agents when master is VMSS
DefaultKubernetesAgentSubnetVMSS = "10.248.0.0/13"
// DefaultUserAssignedID specifies the default name for the user assigned identity
DefaultUserAssignedID = "acsenginetestid"
)
const (
@ -229,6 +229,7 @@ const (
const (
kubernetesMasterCustomDataYaml = "k8s/kubernetesmastercustomdata.yml"
kubernetesMasterCustomDataVMSSYaml = "k8s/kubernetesmastercustomdatavmss.yml"
kubernetesCustomScript = "k8s/kubernetescustomscript.sh"
kubernetesProvisionSourceScript = "k8s/kubernetesprovisionsource.sh"
kubernetesHealthMonitorScript = "k8s/health-monitor.sh"
@ -294,8 +295,10 @@ const (
kubernetesAgentResourcesVMAS = "k8s/kubernetesagentresourcesvmas.t"
kubernetesAgentResourcesVMSS = "k8s/kubernetesagentresourcesvmss.t"
kubernetesAgentVars = "k8s/kubernetesagentvars.t"
kubernetesMasterResources = "k8s/kubernetesmasterresources.t"
kubernetesMasterVars = "k8s/kubernetesmastervars.t"
kubernetesMasterResourcesVMAS = "k8s/kubernetesmasterresources.t"
kubernetesMasterResourcesVMSS = "k8s/kubernetesmasterresourcesvmss.t"
kubernetesMasterVarsVMAS = "k8s/kubernetesmastervars.t"
kubernetesMasterVarsVMSS = "k8s/kubernetesmastervarsvmss.t"
kubernetesParams = "k8s/kubernetesparams.t"
kubernetesWinAgentVars = "k8s/kuberneteswinagentresourcesvmas.t"
kubernetesWinAgentVarsVMSS = "k8s/kuberneteswinagentresourcesvmss.t"

65
pkg/acsengine/defaults.go
Просмотреть файл

@ -316,6 +316,7 @@ func setOrchestratorDefaults(cs *api.ContainerService, isUpdate bool) {
if o.KubernetesConfig.ClusterSubnet == "" {
if o.IsAzureCNI() {
// When Azure CNI is enabled, all masters, agents and pods share the same large subnet.
// Except when master is VMSS, then masters and agents have separate subnets within the same large subnet.
o.KubernetesConfig.ClusterSubnet = DefaultKubernetesSubnet
} else {
o.KubernetesConfig.ClusterSubnet = DefaultKubernetesClusterSubnet
@ -496,6 +497,10 @@ func setMasterProfileDefaults(a *api.Properties, isUpgrade bool) {
a.MasterProfile.Distro = api.AKS
}
}
// set default to VMAS for now
if len(a.MasterProfile.AvailabilityProfile) == 0 {
a.MasterProfile.AvailabilityProfile = api.AvailabilitySet
}
if !a.MasterProfile.IsCustomVNET() {
if a.OrchestratorProfile.OrchestratorType == api.Kubernetes {
@ -504,13 +509,24 @@ func setMasterProfileDefaults(a *api.Properties, isUpgrade bool) {
a.MasterProfile.Subnet = a.OrchestratorProfile.KubernetesConfig.ClusterSubnet
// FirstConsecutiveStaticIP is not reset if it is upgrade and some value already exists
if !isUpgrade || len(a.MasterProfile.FirstConsecutiveStaticIP) == 0 {
a.MasterProfile.FirstConsecutiveStaticIP = getFirstConsecutiveStaticIPAddress(a.MasterProfile.Subnet)
if a.MasterProfile.IsVirtualMachineScaleSets() {
a.MasterProfile.FirstConsecutiveStaticIP = api.DefaultFirstConsecutiveKubernetesStaticIPVMSS
a.MasterProfile.Subnet = DefaultKubernetesMasterSubnet
a.MasterProfile.AgentSubnet = DefaultKubernetesAgentSubnetVMSS
} else {
a.MasterProfile.FirstConsecutiveStaticIP = a.MasterProfile.GetFirstConsecutiveStaticIPAddress(a.MasterProfile.Subnet)
}
}
} else {
a.MasterProfile.Subnet = DefaultKubernetesMasterSubnet
// FirstConsecutiveStaticIP is not reset if it is upgrade and some value already exists
if !isUpgrade || len(a.MasterProfile.FirstConsecutiveStaticIP) == 0 {
a.MasterProfile.FirstConsecutiveStaticIP = DefaultFirstConsecutiveKubernetesStaticIP
if a.MasterProfile.IsVirtualMachineScaleSets() {
a.MasterProfile.FirstConsecutiveStaticIP = api.DefaultFirstConsecutiveKubernetesStaticIPVMSS
a.MasterProfile.AgentSubnet = DefaultKubernetesAgentSubnetVMSS
} else {
a.MasterProfile.FirstConsecutiveStaticIP = api.DefaultFirstConsecutiveKubernetesStaticIP
}
}
}
} else if a.OrchestratorProfile.OrchestratorType == api.OpenShift {
@ -544,6 +560,11 @@ func setMasterProfileDefaults(a *api.Properties, isUpgrade bool) {
}
}
if a.MasterProfile.IsCustomVNET() && a.MasterProfile.IsVirtualMachineScaleSets() {
if a.OrchestratorProfile.OrchestratorType == api.Kubernetes {
a.MasterProfile.FirstConsecutiveStaticIP = a.MasterProfile.GetFirstConsecutiveStaticIPAddress(a.MasterProfile.VnetCidr)
}
}
// Set the default number of IP addresses allocated for masters.
if a.MasterProfile.IPAddressCount == 0 {
// Allocate one IP address for the node.
@ -592,7 +613,9 @@ func setAgentProfileDefaults(a *api.Properties, isUpgrade, isScale bool) {
for _, profile := range a.AgentPoolProfiles {
if a.OrchestratorProfile.OrchestratorType == api.Kubernetes ||
a.OrchestratorProfile.OrchestratorType == api.OpenShift {
if !a.MasterProfile.IsVirtualMachineScaleSets() {
profile.Subnet = a.MasterProfile.Subnet
}
} else {
profile.Subnet = fmt.Sprintf(DefaultAgentSubnetTemplate, subnetCounter)
}
@ -697,16 +720,23 @@ func setDefaultCerts(a *api.Properties) (bool, error) {
}
ips := []net.IP{firstMasterIP}
// Add the Internal Loadbalancer IP which is always at at a known offset from the firstMasterIP
ips = append(ips, net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(DefaultInternalLbStaticIPOffset)})
// Include the Internal load balancer as well
if a.MasterProfile.IsVirtualMachineScaleSets() {
// Include the Internal load balancer as well
for i := 1; i < a.MasterProfile.Count; i++ {
offset := i * a.MasterProfile.IPAddressCount
ip := net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(offset)}
ips = append(ips, ip)
}
} else {
for i := 1; i < a.MasterProfile.Count; i++ {
ip := net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(i)}
ips = append(ips, ip)
}
}
if a.CertificateProfile == nil {
a.CertificateProfile = &api.CertificateProfile{}
}
@ -803,31 +833,6 @@ func certsAlreadyPresent(c *api.CertificateProfile, m int) map[string]bool {
return g
}
// getFirstConsecutiveStaticIPAddress returns the first static IP address of the given subnet.
func getFirstConsecutiveStaticIPAddress(subnetStr string) string {
_, subnet, err := net.ParseCIDR(subnetStr)
if err != nil {
return DefaultFirstConsecutiveKubernetesStaticIP
}
// Find the first and last octet of the host bits.
ones, bits := subnet.Mask.Size()
firstOctet := ones / 8
lastOctet := bits/8 - 1
// Set the remaining host bits in the first octet.
subnet.IP[firstOctet] |= (1 << byte((8 - (ones % 8)))) - 1
// Fill the intermediate octets with 1s and last octet with offset. This is done so to match
// the existing behavior of allocating static IP addresses from the last /24 of the subnet.
for i := firstOctet + 1; i < lastOctet; i++ {
subnet.IP[i] = 255
}
subnet.IP[lastOctet] = DefaultKubernetesFirstConsecutiveStaticIPOffset
return subnet.IP.String()
}
// combine user-provided --feature-gates vals with defaults
// a minimum k8s version may be declared as required for defaults assignment
func addDefaultFeatureGates(m map[string]string, version string, minVersion string, defaults string) {

126
pkg/acsengine/defaults_test.go
Просмотреть файл

@ -497,6 +497,132 @@ func TestStorageProfile(t *testing.T) {
}
// TestMasterProfileDefaults covers tests for setMasterProfileDefaults
func TestMasterProfileDefaults(t *testing.T) {
// this validates default masterProfile configuration
mockCS := getMockBaseContainerService("1.10.3")
properties := mockCS.Properties
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure"
properties.MasterProfile.AvailabilityProfile = ""
properties.MasterProfile.Count = 3
mockCS.Properties = properties
setPropertiesDefaults(&mockCS, false, false)
if properties.MasterProfile.IsVirtualMachineScaleSets() {
t.Fatalf("Master VMAS, AzureCNI: MasterProfile AvailabilityProfile did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.AvailabilityProfile, api.AvailabilitySet)
}
if properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet != DefaultKubernetesSubnet {
t.Fatalf("Master VMAS, AzureCNI: MasterProfile ClusterSubnet did not have the expected default configuration, got %s, expected %s",
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet, DefaultKubernetesSubnet)
}
if properties.MasterProfile.Subnet != properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet {
t.Fatalf("Master VMAS, AzureCNI: MasterProfile Subnet did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.Subnet, properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet)
}
if properties.AgentPoolProfiles[0].Subnet != properties.MasterProfile.Subnet {
t.Fatalf("Master VMAS, AzureCNI: AgentPoolProfiles Subnet did not have the expected default configuration, got %s, expected %s",
properties.AgentPoolProfiles[0].Subnet, properties.MasterProfile.Subnet)
}
if properties.MasterProfile.FirstConsecutiveStaticIP != "10.255.255.5" {
t.Fatalf("Master VMAS, AzureCNI: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.FirstConsecutiveStaticIP, "10.255.255.5")
}
// this validates default vmss masterProfile configuration
mockCS = getMockBaseContainerService("1.10.3")
properties = mockCS.Properties
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure"
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
properties.MasterProfile.AvailabilityProfile = api.VirtualMachineScaleSets
setPropertiesDefaults(&mockCS, false, true)
if !properties.MasterProfile.IsVirtualMachineScaleSets() {
t.Fatalf("Master VMSS, AzureCNI: MasterProfile AvailabilityProfile did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.AvailabilityProfile, api.VirtualMachineScaleSets)
}
if properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet != DefaultKubernetesSubnet {
t.Fatalf("Master VMSS, AzureCNI: MasterProfile ClusterSubnet did not have the expected default configuration, got %s, expected %s",
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet, DefaultKubernetesSubnet)
}
if properties.MasterProfile.FirstConsecutiveStaticIP != api.DefaultFirstConsecutiveKubernetesStaticIPVMSS {
t.Fatalf("Master VMSS, AzureCNI: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.FirstConsecutiveStaticIP, api.DefaultFirstConsecutiveKubernetesStaticIPVMSS)
}
if properties.MasterProfile.Subnet != DefaultKubernetesMasterSubnet {
t.Fatalf("Master VMSS, AzureCNI: MasterProfile Subnet did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.Subnet, DefaultKubernetesMasterSubnet)
}
if properties.MasterProfile.AgentSubnet != DefaultKubernetesAgentSubnetVMSS {
t.Fatalf("Master VMSS, AzureCNI: MasterProfile AgentSubnet did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.AgentSubnet, DefaultKubernetesAgentSubnetVMSS)
}
// this validates default masterProfile configuration and kubenet
mockCS = getMockBaseContainerService("1.10.3")
properties = mockCS.Properties
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "kubenet"
properties.MasterProfile.AvailabilityProfile = api.VirtualMachineScaleSets
setPropertiesDefaults(&mockCS, false, true)
if properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet != DefaultKubernetesClusterSubnet {
t.Fatalf("Master VMSS, kubenet: MasterProfile ClusterSubnet did not have the expected default configuration, got %s, expected %s",
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet, DefaultKubernetesClusterSubnet)
}
if properties.MasterProfile.Subnet != DefaultKubernetesMasterSubnet {
t.Fatalf("Master VMSS, kubenet: MasterProfile Subnet did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.Subnet, DefaultKubernetesMasterSubnet)
}
if properties.MasterProfile.FirstConsecutiveStaticIP != api.DefaultFirstConsecutiveKubernetesStaticIPVMSS {
t.Fatalf("Master VMSS, kubenet: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.FirstConsecutiveStaticIP, api.DefaultFirstConsecutiveKubernetesStaticIPVMSS)
}
if properties.MasterProfile.AgentSubnet != DefaultKubernetesAgentSubnetVMSS {
t.Fatalf("Master VMSS, kubenet: MasterProfile AgentSubnet did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.AgentSubnet, DefaultKubernetesAgentSubnetVMSS)
}
properties.MasterProfile.AvailabilityProfile = api.AvailabilitySet
setPropertiesDefaults(&mockCS, false, true)
if properties.MasterProfile.FirstConsecutiveStaticIP != api.DefaultFirstConsecutiveKubernetesStaticIP {
t.Fatalf("Master VMAS, kubenet: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.FirstConsecutiveStaticIP, api.DefaultFirstConsecutiveKubernetesStaticIP)
}
// this validates default vmas masterProfile configuration, AzureCNI, and custom vnet
mockCS = getMockBaseContainerService("1.10.3")
properties = mockCS.Properties
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
properties.MasterProfile.VnetSubnetID = "/subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/Microsoft.Network/virtualNetworks/ExampleCustomVNET/subnets/ExampleMasterSubnet"
properties.MasterProfile.VnetCidr = "10.239.0.0/16"
properties.MasterProfile.FirstConsecutiveStaticIP = "10.239.255.239"
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure"
properties.MasterProfile.AvailabilityProfile = api.AvailabilitySet
setPropertiesDefaults(&mockCS, false, true)
if properties.MasterProfile.FirstConsecutiveStaticIP != "10.239.255.239" {
t.Fatalf("Master VMAS, AzureCNI, customvnet: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.FirstConsecutiveStaticIP, "10.239.255.239")
}
// this validates default vmss masterProfile configuration, AzureCNI, and custom vnet
mockCS = getMockBaseContainerService("1.10.3")
properties = mockCS.Properties
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
properties.MasterProfile.VnetSubnetID = "/subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/Microsoft.Network/virtualNetworks/ExampleCustomVNET/subnets/ExampleMasterSubnet"
properties.MasterProfile.VnetCidr = "10.239.0.0/16"
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure"
properties.MasterProfile.AvailabilityProfile = api.VirtualMachineScaleSets
setPropertiesDefaults(&mockCS, false, true)
if properties.MasterProfile.FirstConsecutiveStaticIP != "10.239.0.4" {
t.Fatalf("Master VMSS, AzureCNI, customvnet: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
properties.MasterProfile.FirstConsecutiveStaticIP, "10.239.0.4")
}
}
func TestAgentPoolProfile(t *testing.T) {
mockCS := getMockBaseContainerService("1.10")
properties := mockCS.Properties

2
pkg/acsengine/engine.go
Просмотреть файл

@ -28,7 +28,7 @@ import (
var commonTemplateFiles = []string{agentOutputs, agentParams, masterOutputs, iaasOutputs, masterParams, windowsParams}
var dcosTemplateFiles = []string{dcosBaseFile, dcosAgentResourcesVMAS, dcosAgentResourcesVMSS, dcosAgentVars, dcosMasterResources, dcosMasterVars, dcosParams, dcosWindowsAgentResourcesVMAS, dcosWindowsAgentResourcesVMSS}
var dcos2TemplateFiles = []string{dcos2BaseFile, dcosAgentResourcesVMAS, dcosAgentResourcesVMSS, dcosAgentVars, dcos2MasterResources, dcos2BootstrapResources, dcos2MasterVars, dcosParams, dcosWindowsAgentResourcesVMAS, dcosWindowsAgentResourcesVMSS, dcos2BootstrapVars, dcos2BootstrapParams}
var kubernetesTemplateFiles = []string{kubernetesBaseFile, kubernetesAgentResourcesVMAS, kubernetesAgentResourcesVMSS, kubernetesAgentVars, kubernetesMasterResources, kubernetesMasterVars, kubernetesParams, kubernetesWinAgentVars, kubernetesWinAgentVarsVMSS}
var kubernetesTemplateFiles = []string{kubernetesBaseFile, kubernetesAgentResourcesVMAS, kubernetesAgentResourcesVMSS, kubernetesAgentVars, kubernetesMasterResourcesVMAS, kubernetesMasterResourcesVMSS, kubernetesMasterVarsVMAS, kubernetesMasterVarsVMSS, kubernetesParams, kubernetesWinAgentVars, kubernetesWinAgentVarsVMSS}
var swarmTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentResourcesVMAS, swarmAgentVars, swarmAgentResourcesVMSS, swarmBaseFile, swarmMasterResources, swarmMasterVars, swarmWinAgentResourcesVMAS, swarmWinAgentResourcesVMSS}
var swarmModeTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentResourcesVMAS, swarmAgentVars, swarmAgentResourcesVMSS, swarmBaseFile, swarmMasterResources, swarmMasterVars, swarmWinAgentResourcesVMAS, swarmWinAgentResourcesVMSS}
var openshiftTemplateFiles = append(

4
pkg/acsengine/params.go
Просмотреть файл

@ -57,11 +57,15 @@ func getParameters(cs *api.ContainerService, generatorCode string, acsengineVers
if properties.MasterProfile != nil {
if properties.MasterProfile.IsCustomVNET() {
addValue(parametersMap, "masterVnetSubnetID", properties.MasterProfile.VnetSubnetID)
if properties.MasterProfile.IsVirtualMachineScaleSets() {
addValue(parametersMap, "agentVnetSubnetID", properties.MasterProfile.AgentVnetSubnetID)
}
if properties.OrchestratorProfile.IsKubernetes() || properties.OrchestratorProfile.IsOpenShift() {
addValue(parametersMap, "vnetCidr", properties.MasterProfile.VnetCidr)
}
} else {
addValue(parametersMap, "masterSubnet", properties.MasterProfile.Subnet)
addValue(parametersMap, "agentSubnet", properties.MasterProfile.AgentSubnet)
}
addValue(parametersMap, "firstConsecutiveStaticIP", properties.MasterProfile.FirstConsecutiveStaticIP)
addValue(parametersMap, "masterVMSize", properties.MasterProfile.VMSize)

92
pkg/acsengine/template_generator.go
Просмотреть файл

@ -156,9 +156,55 @@ func (t *TemplateGenerator) prepareTemplateFiles(properties *api.Properties) ([]
return files, baseFile, nil
}
func (t *TemplateGenerator) getMasterCustomData(cs *api.ContainerService, textFilename string, profile *api.Properties) string {
str, e := t.getSingleLineForTemplate(textFilename, cs, profile)
if e != nil {
panic(e)
}
// add manifests
str = substituteConfigString(str,
kubernetesManifestSettingsInit(profile),
"k8s/manifests",
"/etc/kubernetes/manifests",
"MASTER_MANIFESTS_CONFIG_PLACEHOLDER",
profile.OrchestratorProfile.OrchestratorVersion)
// add artifacts
str = substituteConfigString(str,
kubernetesArtifactSettingsInitMaster(profile),
"k8s/artifacts",
"/etc/systemd/system",
"MASTER_ARTIFACTS_CONFIG_PLACEHOLDER",
profile.OrchestratorProfile.OrchestratorVersion)
// add addons
str = substituteAddonConfigString(str,
kubernetesAddonSettingsInit(profile),
"k8s/addons",
"/etc/kubernetes/addons",
"MASTER_ADDONS_CONFIG_PLACEHOLDER",
profile.OrchestratorProfile.OrchestratorVersion)
// add custom files
customFilesReader, err := customfilesIntoReaders(masterCustomFiles(profile))
if err != nil {
log.Fatalf("Could not read custom files: %s", err.Error())
}
str = substituteConfigStringCustomFiles(str,
customFilesReader,
"MASTER_CUSTOM_FILES_PLACEHOLDER")
// return the custom data
return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str)
}
// getTemplateFuncMap returns all functions used in template generation
func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) template.FuncMap {
return template.FuncMap{
"IsMasterVirtualMachineScaleSets": func() bool {
return cs.Properties.MasterProfile != nil && cs.Properties.MasterProfile.IsVirtualMachineScaleSets()
},
"IsHostedMaster": func() bool {
return cs.Properties.HostedMasterProfile != nil
},
@ -498,46 +544,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
return DefaultInternalLbStaticIPOffset
},
"GetKubernetesMasterCustomData": func(profile *api.Properties) string {
str, e := t.getSingleLineForTemplate(kubernetesMasterCustomDataYaml, cs, profile)
if e != nil {
panic(e)
}
// add manifests
str = substituteConfigString(str,
kubernetesManifestSettingsInit(profile),
"k8s/manifests",
"/etc/kubernetes/manifests",
"MASTER_MANIFESTS_CONFIG_PLACEHOLDER",
profile.OrchestratorProfile.OrchestratorVersion)
// add artifacts
str = substituteConfigString(str,
kubernetesArtifactSettingsInitMaster(profile),
"k8s/artifacts",
"/etc/systemd/system",
"MASTER_ARTIFACTS_CONFIG_PLACEHOLDER",
profile.OrchestratorProfile.OrchestratorVersion)
// add addons
str = substituteAddonConfigString(str,
kubernetesAddonSettingsInit(profile),
"k8s/addons",
"/etc/kubernetes/addons",
"MASTER_ADDONS_CONFIG_PLACEHOLDER",
profile.OrchestratorProfile.OrchestratorVersion)
// add custom files
customFilesReader, err := customfilesIntoReaders(masterCustomFiles(profile))
if err != nil {
log.Fatalf("Could not read custom files: %s", err.Error())
}
str = substituteConfigStringCustomFiles(str,
customFilesReader,
"MASTER_CUSTOM_FILES_PLACEHOLDER")
// return the custom data
return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str)
str := t.getMasterCustomData(cs, kubernetesMasterCustomDataYaml, profile)
return str
},
"GetKubernetesMasterCustomDataVMSS": func(profile *api.Properties) string {
str := t.getMasterCustomData(cs, kubernetesMasterCustomDataVMSSYaml, profile)
return str
},
"GetKubernetesAgentCustomData": func(profile *api.AgentPoolProfile) string {
str, e := t.getSingleLineForTemplate(kubernetesAgentCustomDataYaml, cs, profile)

10
pkg/api/const.go
Просмотреть файл

@ -62,6 +62,16 @@ const (
const (
// AvailabilitySet means that the vms are in an availability set
AvailabilitySet = "AvailabilitySet"
// DefaultFirstConsecutiveKubernetesStaticIP specifies the static IP address on Kubernetes master 0
DefaultFirstConsecutiveKubernetesStaticIP = "10.240.255.5"
// DefaultFirstConsecutiveKubernetesStaticIPVMSS specifies the static IP address on Kubernetes master 0 of VMSS
DefaultFirstConsecutiveKubernetesStaticIPVMSS = "10.240.0.4"
// DefaultKubernetesFirstConsecutiveStaticIPOffset specifies the IP address offset of master 0
// when VNET integration is enabled.
DefaultKubernetesFirstConsecutiveStaticIPOffset = 5
// DefaultKubernetesFirstConsecutiveStaticIPOffsetVMSS specifies the IP address offset of master 0 in VMSS
// when VNET integration is enabled.
DefaultKubernetesFirstConsecutiveStaticIPOffsetVMSS = 4
// VirtualMachineScaleSets means that the vms are in a virtual machine scaleset
VirtualMachineScaleSets = "VirtualMachineScaleSets"
// ScaleSetPriorityRegular is the default ScaleSet Priority

4
pkg/api/converterfromapi.go
Просмотреть файл

@ -894,6 +894,7 @@ func convertMasterProfileToVLabs(api *MasterProfile, vlabsProfile *vlabs.MasterP
vlabsProfile.VMSize = api.VMSize
vlabsProfile.OSDiskSizeGB = api.OSDiskSizeGB
vlabsProfile.VnetSubnetID = api.VnetSubnetID
vlabsProfile.AgentVnetSubnetID = api.AgentVnetSubnetID
vlabsProfile.FirstConsecutiveStaticIP = api.FirstConsecutiveStaticIP
vlabsProfile.VnetCidr = api.VnetCidr
vlabsProfile.SetSubnet(api.Subnet)
@ -920,7 +921,8 @@ func convertMasterProfileToVLabs(api *MasterProfile, vlabsProfile *vlabs.MasterP
vlabsProfile.ImageRef.Name = api.ImageRef.Name
vlabsProfile.ImageRef.ResourceGroup = api.ImageRef.ResourceGroup
}
vlabsProfile.AvailabilityProfile = api.AvailabilityProfile
vlabsProfile.AgentSubnet = api.AgentSubnet
convertCustomFilesToVlabs(api, vlabsProfile)
}

3
pkg/api/convertertoapi.go
Просмотреть файл

@ -893,6 +893,7 @@ func convertVLabsMasterProfile(vlabs *vlabs.MasterProfile, api *MasterProfile) {
api.VMSize = vlabs.VMSize
api.OSDiskSizeGB = vlabs.OSDiskSizeGB
api.VnetSubnetID = vlabs.VnetSubnetID
api.AgentVnetSubnetID = vlabs.AgentVnetSubnetID
api.FirstConsecutiveStaticIP = vlabs.FirstConsecutiveStaticIP
api.VnetCidr = vlabs.VnetCidr
api.Subnet = vlabs.GetSubnet()
@ -930,6 +931,8 @@ func convertVLabsMasterProfile(vlabs *vlabs.MasterProfile, api *MasterProfile) {
api.ImageRef.ResourceGroup = vlabs.ImageRef.ResourceGroup
}
api.AvailabilityProfile = vlabs.AvailabilityProfile
api.AgentSubnet = vlabs.AgentSubnet
convertCustomFilesToAPI(vlabs, api)
}

38
pkg/api/types.go
Просмотреть файл

@ -1,6 +1,7 @@
package api
import (
"net"
neturl "net/url"
"strconv"
"strings"
@ -395,6 +396,7 @@ type MasterProfile struct {
OSDiskSizeGB int `json:"osDiskSizeGB,omitempty"`
VnetSubnetID string `json:"vnetSubnetID,omitempty"`
VnetCidr string `json:"vnetCidr,omitempty"`
AgentVnetSubnetID string `json:"agentVnetSubnetID,omitempty"`
FirstConsecutiveStaticIP string `json:"firstConsecutiveStaticIP,omitempty"`
Subnet string `json:"subnet"`
IPAddressCount int `json:"ipAddressCount,omitempty"`
@ -407,6 +409,8 @@ type MasterProfile struct {
KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"`
ImageRef *ImageReference `json:"imageReference,omitempty"`
CustomFiles *[]CustomFile `json:"customFiles,omitempty"`
AvailabilityProfile string `json:"availabilityProfile"`
AgentSubnet string `json:"agentSubnet,omitempty"`
// Master LB public endpoint/FQDN with port
// The format will be FQDN:2376
@ -736,6 +740,40 @@ func (m *MasterProfile) IsCoreOS() bool {
return m.Distro == CoreOS
}
// IsVirtualMachineScaleSets returns true if the master availability profile is VMSS
func (m *MasterProfile) IsVirtualMachineScaleSets() bool {
return m.AvailabilityProfile == VirtualMachineScaleSets
}
// GetFirstConsecutiveStaticIPAddress returns the first static IP address of the given subnet.
func (m *MasterProfile) GetFirstConsecutiveStaticIPAddress(subnetStr string) string {
_, subnet, err := net.ParseCIDR(subnetStr)
if err != nil {
return DefaultFirstConsecutiveKubernetesStaticIP
}
// Find the first and last octet of the host bits.
ones, bits := subnet.Mask.Size()
firstOctet := ones / 8
lastOctet := bits/8 - 1
if m.IsVirtualMachineScaleSets() {
subnet.IP[lastOctet] = DefaultKubernetesFirstConsecutiveStaticIPOffsetVMSS
} else {
// Set the remaining host bits in the first octet.
subnet.IP[firstOctet] |= (1 << byte((8 - (ones % 8)))) - 1
// Fill the intermediate octets with 1s and last octet with offset. This is done so to match
// the existing behavior of allocating static IP addresses from the last /24 of the subnet.
for i := firstOctet + 1; i < lastOctet; i++ {
subnet.IP[i] = 255
}
subnet.IP[lastOctet] = DefaultKubernetesFirstConsecutiveStaticIPOffset
}
return subnet.IP.String()
}
// IsCustomVNET returns true if the customer brought their own VNET
func (a *AgentPoolProfile) IsCustomVNET() bool {
return len(a.VnetSubnetID) > 0

34
pkg/api/types_test.go
Просмотреть файл

@ -392,7 +392,41 @@ func TestTotalNodes(t *testing.T) {
}
}
}
func TestMasterAvailabilityProfile(t *testing.T) {
cases := []struct {
p Properties
expectedISVMSS bool
}{
{
p: Properties{
MasterProfile: &MasterProfile{},
},
expectedISVMSS: false,
},
{
p: Properties{
MasterProfile: &MasterProfile{
AvailabilityProfile: AvailabilitySet,
},
},
expectedISVMSS: false,
},
{
p: Properties{
MasterProfile: &MasterProfile{
AvailabilityProfile: VirtualMachineScaleSets,
},
},
expectedISVMSS: true,
},
}
for _, c := range cases {
if c.p.MasterProfile.IsVirtualMachineScaleSets() != c.expectedISVMSS {
t.Fatalf("expected MasterProfile.IsVirtualMachineScaleSets() to return %t but instead returned %t", c.expectedISVMSS, c.p.MasterProfile.IsVirtualMachineScaleSets())
}
}
}
func TestAvailabilityProfile(t *testing.T) {
cases := []struct {
p Properties

22
pkg/api/vlabs/types.go
Просмотреть файл

@ -365,6 +365,7 @@ type MasterProfile struct {
OSDiskSizeGB int `json:"osDiskSizeGB,omitempty" validate:"min=0,max=1023"`
VnetSubnetID string `json:"vnetSubnetID,omitempty"`
VnetCidr string `json:"vnetCidr,omitempty"`
AgentVnetSubnetID string `json:"agentVnetSubnetID,omitempty"`
FirstConsecutiveStaticIP string `json:"firstConsecutiveStaticIP,omitempty"`
IPAddressCount int `json:"ipAddressCount,omitempty" validate:"min=0,max=256"`
StorageProfile string `json:"storageProfile,omitempty" validate:"eq=StorageAccount|eq=ManagedDisks|len=0"`
@ -376,6 +377,8 @@ type MasterProfile struct {
KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"`
ImageRef *ImageReference `json:"imageReference,omitempty"`
CustomFiles *[]CustomFile `json:"customFiles,omitempty"`
AvailabilityProfile string `json:"availabilityProfile"`
AgentSubnet string `json:"agentSubnet,omitempty"`
// subnet is internal
subnet string
@ -547,6 +550,25 @@ func (m *MasterProfile) IsCoreOS() bool {
return m.Distro == CoreOS
}
// IsVirtualMachineScaleSets returns true if the master availability profile is VMSS
func (m *MasterProfile) IsVirtualMachineScaleSets() bool {
return m.AvailabilityProfile == VirtualMachineScaleSets
}
// IsAllVirtualMachineScaleSets returns true if the cluster contains only Virtual Machine Scale Sets
func (p *Properties) IsAllVirtualMachineScaleSets() bool {
isAll := p.MasterProfile.IsVirtualMachineScaleSets()
if isAll {
for _, agentPoolProfile := range p.AgentPoolProfiles {
if agentPoolProfile.AvailabilityProfile != VirtualMachineScaleSets {
isAll = false
break
}
}
}
return isAll
}
// IsCustomVNET returns true if the customer brought their own VNET
func (a *AgentPoolProfile) IsCustomVNET() bool {
return len(a.VnetSubnetID) > 0

58
pkg/api/vlabs/types_test.go
Просмотреть файл

@ -81,6 +81,64 @@ func TestOrchestratorProfile(t *testing.T) {
}
}
func TestMasterProfile(t *testing.T) {
// With osType not specified
MasterProfileText := `{"count" : 0, "storageProfile" : "StorageAccount", "vnetSubnetID" : "1234", "agentVnetSubnetID" : "5678"}`
mp := &MasterProfile{}
if e := json.Unmarshal([]byte(MasterProfileText), mp); e != nil {
t.Fatalf("unexpectedly detected unmarshal failure for MasterProfile, %+v", e)
}
if mp.Count != 0 {
t.Fatalf("unexpectedly detected MasterProfile.Count != 1 after unmarshal")
}
if !mp.IsCustomVNET() {
t.Fatalf("unexpectedly detected nil MasterProfile.VNetSubNetID after unmarshal")
}
if !mp.IsStorageAccount() {
t.Fatalf("unexpectedly detected MasterProfile.StorageProfile != ManagedDisks after unmarshal")
}
// With vmas
MasterProfileText = `{ "count": 1, "vmSize": "Standard_D2_v2", "storageProfile" : "ManagedDisks", "diskSizesGB" : [750, 250, 600, 1000] }`
mp = &MasterProfile{}
if e := json.Unmarshal([]byte(MasterProfileText), mp); e != nil {
t.Fatalf("unexpectedly detected unmarshal failure for MasterProfile, %+v", e)
}
if mp.Count != 1 {
t.Fatalf("unexpectedly detected MasterProfile.Count != 1 after unmarshal")
}
if !mp.IsManagedDisks() {
t.Fatalf("unexpectedly detected MasterProfile.StorageProfile != ManagedDisks after unmarshal")
}
if mp.IsVirtualMachineScaleSets() {
t.Fatalf("unexpectedly detected MasterProfile.AvailabilitySets == VirtualMachineScaleSets after unmarshal")
}
// With vmss
MasterProfileText = `{ "count": 3, "vmSize": "Standard_D2_v2", "availabilityProfile": "VirtualMachineScaleSets", "storageProfile" : "ManagedDisks", "diskSizesGB" : [750, 250, 600, 1000] }`
mp = &MasterProfile{}
if e := json.Unmarshal([]byte(MasterProfileText), mp); e != nil {
t.Fatalf("unexpectedly detected unmarshal failure for MasterProfile, %+v", e)
}
if mp.Count != 3 {
t.Fatalf("unexpectedly detected MasterProfile.Count != 3 after unmarshal")
}
if !mp.IsManagedDisks() {
t.Fatalf("unexpectedly detected MasterProfile.StorageProfile != ManagedDisks after unmarshal")
}
if !mp.IsVirtualMachineScaleSets() {
t.Fatalf("unexpectedly detected MasterProfile.AvailabilitySets != VirtualMachineScaleSets after unmarshal")
}
}
func TestAgentPoolProfile(t *testing.T) {
// With osType not specified
AgentPoolProfileText := `{"count" : 0, "storageProfile" : "StorageAccount", "vnetSubnetID" : "1234"}`

67
pkg/api/vlabs/validate.go
Просмотреть файл

@ -129,6 +129,10 @@ func (a *Properties) Validate(isUpdate bool) error {
return e
}
if e := a.validateManagedIdentity(); e != nil {
return e
}
if e := a.validateAADProfile(); e != nil {
return e
}
@ -354,6 +358,17 @@ func (a *Properties) validateMasterProfile() error {
return err
}
}
if m.IsVirtualMachineScaleSets() && a.OrchestratorProfile.OrchestratorType == Kubernetes {
e := validateVMSS(a.OrchestratorProfile, false, m.StorageProfile)
if e != nil {
return e
}
if !a.IsAllVirtualMachineScaleSets() {
return errors.New("VirtualMachineScaleSets for master profile must be used together with virtualMachineScaleSets for agent profiles. Set \"availabilityProfile\" to \"VirtualMachineScaleSets\" for agent profiles")
}
}
return common.ValidateDNSPrefix(m.DNSPrefix)
}
@ -406,9 +421,12 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error {
return e
}
if e := agentPoolProfile.validateVMSS(a.OrchestratorProfile, isUpdate); agentPoolProfile.AvailabilityProfile == VirtualMachineScaleSets && e != nil {
if agentPoolProfile.AvailabilityProfile == VirtualMachineScaleSets {
e := validateVMSS(a.OrchestratorProfile, isUpdate, agentPoolProfile.StorageProfile)
if e != nil {
return e
}
}
if a.OrchestratorProfile.OrchestratorType == Kubernetes {
if a.AgentPoolProfiles[i].AvailabilityProfile != a.AgentPoolProfiles[0].AvailabilityProfile {
@ -550,6 +568,10 @@ func (a *Properties) validateVNET() error {
}
}
if isCustomVNET {
if a.MasterProfile.IsVirtualMachineScaleSets() && a.MasterProfile.AgentVnetSubnetID == "" {
return errors.New("when master profile is using VirtualMachineScaleSets and is custom vnet, set \"vnetsubnetid\" and \"agentVnetSubnetID\" for master profile")
}
subscription, resourcegroup, vnetname, _, e := common.GetVNETSubnetIDComponents(a.MasterProfile.VnetSubnetID)
if e != nil {
return e
@ -568,7 +590,7 @@ func (a *Properties) validateVNET() error {
}
masterFirstIP := net.ParseIP(a.MasterProfile.FirstConsecutiveStaticIP)
if masterFirstIP == nil {
if masterFirstIP == nil && !a.MasterProfile.IsVirtualMachineScaleSets() {
return errors.Errorf("MasterProfile.FirstConsecutiveStaticIP (with VNET Subnet specification) '%s' is an invalid IP address", a.MasterProfile.FirstConsecutiveStaticIP)
}
@ -619,6 +641,43 @@ func (a *Properties) validateServicePrincipalProfile() error {
return nil
}
func (a *Properties) validateManagedIdentity() error {
if a.OrchestratorProfile.OrchestratorType == Kubernetes {
useManagedIdentity := a.OrchestratorProfile.KubernetesConfig != nil &&
a.OrchestratorProfile.KubernetesConfig.UseManagedIdentity
if useManagedIdentity {
version := common.RationalizeReleaseAndVersion(
a.OrchestratorProfile.OrchestratorType,
a.OrchestratorProfile.OrchestratorRelease,
a.OrchestratorProfile.OrchestratorVersion,
false,
false)
if version == "" {
return errors.Errorf("the following user supplied OrchestratorProfile configuration is not supported: OrchestratorType: %s, OrchestratorRelease: %s, OrchestratorVersion: %s. Please check supported Release or Version for this build of acs-engine", a.OrchestratorProfile.OrchestratorType, a.OrchestratorProfile.OrchestratorRelease, a.OrchestratorProfile.OrchestratorVersion)
}
sv, err := semver.Make(version)
if err != nil {
return errors.Errorf("could not validate version %s", version)
}
minVersion, err := semver.Make("1.12.0-beta.0")
if err != nil {
return errors.New("could not validate version")
}
if a.MasterProfile.IsVirtualMachineScaleSets() {
if sv.LT(minVersion) {
return errors.New("managed identity and VMSS masters can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"")
}
} else if a.OrchestratorProfile.KubernetesConfig.UserAssignedID != "" && sv.LT(minVersion) {
return errors.New("user assigned identity can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"")
}
}
}
return nil
}
func (a *Properties) validateAADProfile() error {
if profile := a.AADProfile; profile != nil {
if a.OrchestratorProfile.OrchestratorType != Kubernetes {
@ -744,7 +803,7 @@ func (a *AgentPoolProfile) validateCustomNodeLabels(orchestratorType string) err
return nil
}
func (a *AgentPoolProfile) validateVMSS(o *OrchestratorProfile, isUpdate bool) error {
func validateVMSS(o *OrchestratorProfile, isUpdate bool, storageProfile string) error {
if o.OrchestratorType == Kubernetes {
version := common.RationalizeReleaseAndVersion(
o.OrchestratorType,
@ -777,7 +836,7 @@ func (a *AgentPoolProfile) validateVMSS(o *OrchestratorProfile, isUpdate bool) e
return errors.Errorf("VirtualMachineScaleSets with instance metadata is supported for Kubernetes version %s or greater. Please set \"useInstanceMetadata\": false in \"kubernetesConfig\" or set \"orchestratorVersion\" to %s or above", minVersion.String(), minVersion.String())
}
}
if (a.AvailabilityProfile == VirtualMachineScaleSets || len(a.AvailabilityProfile) == 0) && a.StorageProfile == StorageAccount {
if storageProfile == StorageAccount {
return errors.Errorf("VirtualMachineScaleSets does not support %s disks. Please specify \"storageProfile\": \"%s\" (recommended) or \"availabilityProfile\": \"%s\"", StorageAccount, ManagedDisks, AvailabilitySet)
}
}

169
pkg/api/vlabs/validate_test.go
Просмотреть файл

@ -1377,12 +1377,103 @@ func TestValidateImageNameAndGroup(t *testing.T) {
})
}
}
func TestProperties_ValidateManagedIdentity(t *testing.T) {
tests := []struct {
name string
orchestratorRelease string
useManagedIdentity bool
userAssignedID string
masterProfile MasterProfile
expectedErr string
agentPoolProfiles []*AgentPoolProfile
}{
{
name: "use managed identity with master vmss",
orchestratorRelease: "1.11",
useManagedIdentity: true,
masterProfile: MasterProfile{
DNSPrefix: "dummy",
Count: 3,
AvailabilityProfile: VirtualMachineScaleSets,
},
agentPoolProfiles: []*AgentPoolProfile{
{
Name: "agentpool",
VMSize: "Standard_DS2_v2",
Count: 1,
AvailabilityProfile: VirtualMachineScaleSets,
},
},
expectedErr: "managed identity and VMSS masters can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"",
},
{
name: "use managed identity with master vmas",
orchestratorRelease: "1.11",
useManagedIdentity: true,
masterProfile: MasterProfile{
DNSPrefix: "dummy",
Count: 3,
},
agentPoolProfiles: []*AgentPoolProfile{
{
Name: "agentpool",
VMSize: "Standard_DS2_v2",
Count: 1,
},
},
},
{
name: "use user assigned identity with master vmas",
orchestratorRelease: "1.11",
useManagedIdentity: true,
userAssignedID: "acsenginetestid",
masterProfile: MasterProfile{
DNSPrefix: "dummy",
Count: 3,
},
agentPoolProfiles: []*AgentPoolProfile{
{
Name: "agentpool",
VMSize: "Standard_DS2_v2",
Count: 1,
},
},
expectedErr: "user assigned identity can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
properties := getK8sDefaultProperties(true)
properties.MasterProfile = &test.masterProfile
properties.MasterProfile.VMSize = "Standard_DS2_v2"
properties.OrchestratorProfile = &OrchestratorProfile{
OrchestratorRelease: test.orchestratorRelease,
OrchestratorType: Kubernetes,
KubernetesConfig: &KubernetesConfig{
UseManagedIdentity: test.useManagedIdentity,
UserAssignedID: test.userAssignedID,
},
}
properties.AgentPoolProfiles = test.agentPoolProfiles
err := properties.Validate(false)
if test.expectedErr == "" && err != nil ||
test.expectedErr != "" && (err == nil || test.expectedErr != err.Error()) {
t.Errorf("test %s: unexpected error %q\n", test.name, err)
}
})
}
}
func TestMasterProfileValidate(t *testing.T) {
tests := []struct {
name string
orchestratorType string
orchestratorVersion string
orchestratorRelease string
useInstanceMetadata bool
masterProfile MasterProfile
agentPoolProfiles []*AgentPoolProfile
expectedErr string
}{
{
@ -1412,6 +1503,7 @@ func TestMasterProfileValidate(t *testing.T) {
masterProfile: MasterProfile{
DNSPrefix: "dummy",
Count: 1,
StorageProfile: ManagedDisks,
},
},
{
@ -1429,6 +1521,7 @@ func TestMasterProfileValidate(t *testing.T) {
masterProfile: MasterProfile{
VnetSubnetID: "testvnetstring",
Count: 1,
StorageProfile: ManagedDisks,
},
expectedErr: "when specifying a vnetsubnetid the firstconsecutivestaticip is required",
},
@ -1440,6 +1533,7 @@ func TestMasterProfileValidate(t *testing.T) {
VnetSubnetID: "testvnetstring",
FirstConsecutiveStaticIP: "10.0.0.1",
Count: 1,
StorageProfile: ManagedDisks,
},
},
{
@ -1455,17 +1549,65 @@ func TestMasterProfileValidate(t *testing.T) {
},
expectedErr: "imageName needs to be specified when imageResourceGroup is provided",
},
{
name: "Master Profile with VMSS and Kubernetes v1.9.6",
orchestratorType: Kubernetes,
orchestratorRelease: "1.9",
masterProfile: MasterProfile{
DNSPrefix: "dummy",
Count: 3,
AvailabilityProfile: VirtualMachineScaleSets,
},
expectedErr: "VirtualMachineScaleSets are only available in Kubernetes version 1.10.0 or greater. Please set \"orchestratorVersion\" to 1.10.0 or above",
},
{
name: "Master Profile with VMSS and storage account",
orchestratorType: Kubernetes,
orchestratorRelease: "1.10",
masterProfile: MasterProfile{
DNSPrefix: "dummy",
Count: 3,
AvailabilityProfile: VirtualMachineScaleSets,
StorageProfile: StorageAccount,
},
expectedErr: "VirtualMachineScaleSets does not support StorageAccount disks. Please specify \"storageProfile\": \"ManagedDisks\" (recommended) or \"availabilityProfile\": \"AvailabilitySet\"",
},
{
name: "Master Profile with VMSS and agent profiles with VMAS",
orchestratorType: Kubernetes,
orchestratorRelease: "1.10",
masterProfile: MasterProfile{
DNSPrefix: "dummy",
Count: 3,
AvailabilityProfile: VirtualMachineScaleSets,
},
agentPoolProfiles: []*AgentPoolProfile{
{
Name: "agentpool",
VMSize: "Standard_DS2_v2",
Count: 1,
AvailabilityProfile: AvailabilitySet,
},
},
expectedErr: "VirtualMachineScaleSets for master profile must be used together with virtualMachineScaleSets for agent profiles. Set \"availabilityProfile\" to \"VirtualMachineScaleSets\" for agent profiles",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
properties := &Properties{}
properties := getK8sDefaultProperties(true)
properties.MasterProfile = &test.masterProfile
properties.MasterProfile.StorageProfile = ManagedDisks
properties.MasterProfile.StorageProfile = test.masterProfile.StorageProfile
properties.OrchestratorProfile = &OrchestratorProfile{
OrchestratorType: test.orchestratorType,
OrchestratorVersion: test.orchestratorVersion,
OrchestratorRelease: test.orchestratorRelease,
KubernetesConfig: &KubernetesConfig{
UseInstanceMetadata: helpers.PointerToBool(test.useInstanceMetadata),
},
}
properties.AgentPoolProfiles = test.agentPoolProfiles
err := properties.validateMasterProfile()
if test.expectedErr == "" && err != nil ||
test.expectedErr != "" && (err == nil || test.expectedErr != err.Error()) {
@ -1670,7 +1812,7 @@ func TestProperties_ValidateVNET(t *testing.T) {
expectedMsg: "Multiple VNETS specified. The master profile and each agent pool must reference the same VNET (but it is ok to reference different subnets on that VNET)",
},
{
name: "Invalid MasterProfile FirstConsecutiveStaticIP",
name: "Invalid MasterProfile FirstConsecutiveStaticIP when master is VMAS",
masterProfile: &MasterProfile{
VnetSubnetID: validVNetSubnetID,
Count: 1,
@ -1689,6 +1831,26 @@ func TestProperties_ValidateVNET(t *testing.T) {
},
expectedMsg: "MasterProfile.FirstConsecutiveStaticIP (with VNET Subnet specification) '10.0.0.invalid' is an invalid IP address",
},
{
name: "Empty MasterProfile FirstConsecutiveStaticIP and empty agentVnetSubnetID when master is VMSS",
masterProfile: &MasterProfile{
VnetSubnetID: validVNetSubnetID,
Count: 1,
DNSPrefix: "foo",
VMSize: "Standard_DS2_v2",
AvailabilityProfile: VirtualMachineScaleSets,
},
agentPoolProfiles: []*AgentPoolProfile{
{
Name: "agentpool",
VMSize: "Standard_D2_v2",
Count: 1,
AvailabilityProfile: VirtualMachineScaleSets,
VnetSubnetID: validVNetSubnetID,
},
},
expectedMsg: "when master profile is using VirtualMachineScaleSets and is custom vnet, set \"vnetsubnetid\" and \"agentVnetSubnetID\" for master profile",
},
{
name: "Invalid vnetcidr",
masterProfile: &MasterProfile{
@ -1716,6 +1878,7 @@ func TestProperties_ValidateVNET(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
p := getK8sDefaultProperties(true)
p.OrchestratorProfile.OrchestratorRelease = "1.10"
p.MasterProfile = test.masterProfile
p.AgentPoolProfiles = test.agentPoolProfiles
err := p.Validate(true)

19
test/e2e/azure/cli.go
Просмотреть файл

@ -243,6 +243,25 @@ func (a *Account) CreateVnet(vnet, addressPrefixes, subnetName, subnetPrefix str
return nil
}
// CreateSubnet will create a subnet in a vnet in a resource group
func (a *Account) CreateSubnet(vnet, subnetName, subnetPrefix string) error {
var cmd *exec.Cmd
if a.TimeoutCommands {
cmd = exec.Command("timeout", "60", "az", "network", "vnet", "subnet", "create", "-g",
a.ResourceGroup.Name, "--vnet-name", vnet, "--name", subnetName, "--address-prefix", subnetPrefix)
} else {
cmd = exec.Command("az", "network", "vnet", "subnet", "create", "-g",
a.ResourceGroup.Name, "--vnet-name", vnet, "--name", subnetName, "--address-prefix", subnetPrefix)
}
util.PrintCommand(cmd)
out, err := cmd.CombinedOutput()
if err != nil {
log.Printf("Error while trying to create subnet in vnet with the following command:\n az network vnet subnet create -g %s --vnet-name %s --name %s --address-prefix %s \n Output:%s\n", a.ResourceGroup.Name, vnet, subnetName, subnetPrefix, out)
return err
}
return nil
}
// RouteTable holds information from running az network route-table list
type RouteTable struct {
ID string `json:"id"`

14
test/e2e/engine/template.go
Просмотреть файл

@ -76,7 +76,7 @@ func ParseConfig(cwd, clusterDefinition, name string) (*Config, error) {
// Build takes a template path and will inject values based on provided environment variables
// it will then serialize the structs back into json and save it to outputPath
func Build(cfg *config.Config, subnetID string) (*Engine, error) {
func Build(cfg *config.Config, masterSubnetID string, agentSubnetID string, isVMSS bool) (*Engine, error) {
config, err := ParseConfig(cfg.CurrentWorkingDir, cfg.ClusterDefinition, cfg.Name)
if err != nil {
log.Printf("Error while trying to build Engine Configuration:%s\n", err)
@ -168,9 +168,17 @@ func Build(cfg *config.Config, subnetID string) (*Engine, error) {
}
if config.CreateVNET {
cs.ContainerService.Properties.MasterProfile.VnetSubnetID = subnetID
if isVMSS {
cs.ContainerService.Properties.MasterProfile.VnetSubnetID = masterSubnetID
cs.ContainerService.Properties.MasterProfile.AgentVnetSubnetID = agentSubnetID
for _, p := range cs.ContainerService.Properties.AgentPoolProfiles {
p.VnetSubnetID = subnetID
p.VnetSubnetID = agentSubnetID
}
} else {
cs.ContainerService.Properties.MasterProfile.VnetSubnetID = masterSubnetID
for _, p := range cs.ContainerService.Properties.AgentPoolProfiles {
p.VnetSubnetID = masterSubnetID
}
}
}

68
test/e2e/kubernetes/kubernetes_test.go
Просмотреть файл

@ -32,6 +32,7 @@ import (
const (
WorkloadDir = "workloads"
PolicyDir = "workloads/policies"
SSHPort = "50001"
)
var (
@ -67,9 +68,17 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
Expect(err).NotTo(HaveOccurred())
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
sshKeyPath := cfg.GetSSHKeyPath()
masterNodes, err := node.GetByPrefix("k8s-master")
Expect(err).NotTo(HaveOccurred())
masterName := masterNodes[0].Metadata.Name
lsbReleaseCmd := fmt.Sprintf("lsb_release -a && uname -r")
cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd)
var cmd *exec.Cmd
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd)
}
util.PrintCommand(cmd)
out, err := cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -78,7 +87,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
}
kernelVerCmd := fmt.Sprintf("cat /proc/version")
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd)
}
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -186,8 +199,17 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
sshKeyPath := cfg.GetSSHKeyPath()
masterNodes, err := node.GetByPrefix("k8s-master")
Expect(err).NotTo(HaveOccurred())
masterName := masterNodes[0].Metadata.Name
ifconfigCmd := fmt.Sprintf("ifconfig -a -v")
cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
var cmd *exec.Cmd
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
}
util.PrintCommand(cmd)
out, err := cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -196,7 +218,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
}
resolvCmd := fmt.Sprintf("cat /etc/resolv.conf")
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
}
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -206,7 +232,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
By("Ensuring that we have a valid connection to our resolver")
digCmd := fmt.Sprintf("dig +short +search +answer `hostname`")
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
}
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -219,7 +249,12 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
for _, node := range nodeList.Nodes {
By("Ensuring that we get a DNS lookup answer response for each node hostname")
digCmd := fmt.Sprintf("dig +short +search +answer %s | grep -v -e '^$'", node.Metadata.Name)
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
}
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -231,14 +266,22 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
By("Ensuring that we get a DNS lookup answer response for external names")
digCmd = fmt.Sprintf("dig +short +search www.bing.com | grep -v -e '^$'")
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
}
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
if err != nil {
log.Printf("Error while querying DNS: %s\n", out)
}
digCmd = fmt.Sprintf("dig +short +search google.com | grep -v -e '^$'")
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
}
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -248,7 +291,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
By("Ensuring that we get a DNS lookup answer response for external names using external resolver")
digCmd = fmt.Sprintf("dig +short +search www.bing.com @8.8.8.8 | grep -v -e '^$'")
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
}
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -256,7 +303,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
log.Printf("Error while querying DNS: %s\n", err)
}
digCmd = fmt.Sprintf("dig +short +search google.com @8.8.8.8 | grep -v -e '^$'")
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
}
util.PrintCommand(cmd)
out, err = cmd.CombinedOutput()
log.Printf("%s\n", out)
@ -403,7 +454,9 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
sshKeyPath := cfg.GetSSHKeyPath()
masterNodes, err := node.GetByPrefix("k8s-master")
Expect(err).NotTo(HaveOccurred())
masterName := masterNodes[0].Metadata.Name
if dashboardPort == 80 {
By("Ensuring that we can connect via HTTP to the dashboard on any one node")
} else {
@ -421,7 +474,12 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
Expect(address).NotTo(Equal(nil))
dashboardURL := fmt.Sprintf("http://%s:%v", address.Address, port)
curlCMD := fmt.Sprintf("curl --max-time 60 %s", dashboardURL)
cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD)
var cmd *exec.Cmd
if strings.Contains(masterName, "vmss") {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD)
} else {
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD)
}
util.PrintCommand(cmd)
out, err := cmd.CombinedOutput()
if err == nil {

31
test/e2e/runner/cli_provisioner.go
Просмотреть файл

@ -31,6 +31,7 @@ type CLIProvisioner struct {
ClusterDefinition string `envconfig:"CLUSTER_DEFINITION" required:"true" default:"examples/kubernetes.json"` // ClusterDefinition is the path on disk to the json template these are normally located in examples/
ProvisionRetries int `envconfig:"PROVISION_RETRIES" default:"0"`
CreateVNET bool `envconfig:"CREATE_VNET" default:"false"`
MasterVMSS bool `envconfig:"MASTER_VMSS" default:"false"`
Config *config.Config
Account *azure.Account
Point *metrics.Point
@ -110,16 +111,44 @@ func (cli *CLIProvisioner) provision() error {
subnetID := ""
vnetName := fmt.Sprintf("%sCustomVnet", cli.Config.Name)
subnetName := fmt.Sprintf("%sCustomSubnet", cli.Config.Name)
masterSubnetID := ""
agentSubnetID := ""
if cli.CreateVNET {
if cli.MasterVMSS {
masterSubnetName := fmt.Sprintf("%sCustomSubnetMaster", cli.Config.Name)
agentSubnetName := fmt.Sprintf("%sCustomSubnetAgent", cli.Config.Name)
err = cli.Account.CreateVnet(vnetName, "10.239.0.0/16", masterSubnetName, "10.239.0.0/17")
if err != nil {
return errors.Errorf("Error trying to create vnet:%s", err.Error())
}
masterSubnetID = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", cli.Account.SubscriptionID, cli.Account.ResourceGroup.Name, vnetName, masterSubnetName)
err = cli.Account.CreateSubnet(vnetName, agentSubnetName, "10.239.128.0/17")
if err != nil {
return errors.Errorf("Error trying to create subnet in vnet:%s", err.Error())
}
agentSubnetID = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", cli.Account.SubscriptionID, cli.Account.ResourceGroup.Name, vnetName, agentSubnetName)
} else {
err = cli.Account.CreateVnet(vnetName, "10.239.0.0/16", subnetName, "10.239.0.0/16")
if err != nil {
return errors.Errorf("Error trying to create vnet:%s", err.Error())
}
subnetID = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", cli.Account.SubscriptionID, cli.Account.ResourceGroup.Name, vnetName, subnetName)
}
}
// Lets modify our template and call acs-engine generate on it
eng, err := engine.Build(cli.Config, subnetID)
var eng *engine.Engine
if cli.CreateVNET && cli.MasterVMSS {
eng, err = engine.Build(cli.Config, masterSubnetID, agentSubnetID, true)
} else {
eng, err = engine.Build(cli.Config, subnetID, subnetID, false)
}
if err != nil {
return errors.Wrap(err, "Error while trying to build cluster definition")
}