зеркало из https://github.com/Azure/acs-engine.git
Add VMSS support for k8s master (#3769)
This commit is contained in:
Rita Zhang
Jack Francis
Родитель
40800b91cf
Коммит
c83e1028ba
|
|
@ -314,6 +314,9 @@ func autofillApimodel(dc *deployCmd) error {
|
|||
useManagedIdentity := k8sConfig != nil &&
|
||||
k8sConfig.UseManagedIdentity
|
||||
|
||||
if dc.containerService.Properties.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
k8sConfig.UserAssignedID = acsengine.DefaultUserAssignedID
|
||||
}
|
||||
userAssignedID := k8sConfig != nil &&
|
||||
k8sConfig.UseManagedIdentity &&
|
||||
k8sConfig.UserAssignedID != ""
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ To learn more about supported orchestrators and versions, run the orchestrators
|
|||
| schedulerConfig | no | Configure various runtime configuration for scheduler. See `schedulerConfig` [below](#feat-scheduler-config) |
|
||||
| serviceCidr | no | IP range for Service IPs, Default is "10.0.0.0/16". This range is never routed outside of a node so does not need to lie within clusterSubnet or the VNET |
|
||||
| useInstanceMetadata | no | Use the Azure cloudprovider instance metadata service for appropriate resource discovery operations. Default is `true` |
|
||||
| useManagedIdentity | no | Includes and uses MSI identities for all interactions with the Azure Resource Manager (ARM) API. Instead of using a static service principal written to /etc/kubernetes/azure.json, Kubernetes will use a dynamic, time-limited token fetched from the MSI extension running on master and agent nodes. This support is currently alpha and requires Kubernetes v1.9.1 or newer. (boolean - default == false) |
|
||||
| useManagedIdentity | no | Includes and uses MSI identities for all interactions with the Azure Resource Manager (ARM) API. Instead of using a static service principal written to /etc/kubernetes/azure.json, Kubernetes will use a dynamic, time-limited token fetched from the MSI extension running on master and agent nodes. This support is currently alpha and requires Kubernetes v1.9.1 or newer. (boolean - default == false). When MasterProfile is using `VirtualMachineScaleSets`, this feature requires Kubernetes v1.12 or newer as we default to using user assigned identity. |
|
||||
|
||||
#### addons
|
||||
|
||||
|
|
@ -487,16 +487,18 @@ We consider `kubeletConfig`, `controllerManagerConfig`, `apiServerConfig`, and `
|
|||
| count | yes | Masters have count value of 1, 3, or 5 masters |
|
||||
| dnsPrefix | yes | The dns prefix for the master FQDN. The master FQDN is used for SSH or commandline access. This must be a unique name. ([bring your own VNET examples](../examples/vnet)) |
|
||||
| subjectAltNames | no | An array of fully qualified domain names using which a user can reach API server. These domains are added as Subject Alternative Names to the generated API server certificate. **NOTE**: These domains **will not** be automatically provisioned. |
|
||||
| firstConsecutiveStaticIP | only required when vnetSubnetId specified | The IP address of the first master. IP Addresses will be assigned consecutively to additional master nodes |
|
||||
| firstConsecutiveStaticIP | only required when vnetSubnetId specified and when MasterProfile is not `VirtualMachineScaleSets` | The IP address of the first master. IP Addresses will be assigned consecutively to additional master nodes. When MasterProfile is using `VirtualMachineScaleSets`, this value will be determined by an offset from the first IP in the `vnetCidr`. For example, if `vnetCidr` is `10.239.0.0/16`, then `firstConsecutiveStaticIP` will be `10.239.0.4` |
|
||||
| vmsize | yes | Describes a valid [Azure VM Sizes](https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes/). These are restricted to machines with at least 2 cores and 100GB of ephemeral disk space |
|
||||
| osDiskSizeGB | no | Describes the OS Disk Size in GB |
|
||||
| vnetSubnetId | no | Specifies the Id of an alternate VNET subnet. The subnet id must specify a valid VNET ID owned by the same subscription. ([bring your own VNET examples](../examples/vnet)) |
|
||||
| vnetSubnetId | only required when using custom VNET | Specifies the Id of an alternate VNET subnet. The subnet id must specify a valid VNET ID owned by the same subscription. ([bring your own VNET examples](../examples/vnet)). When MasterProfile is set to `VirtualMachineScaleSets`, this value should be the subnetId of the master subnet. When MasterProfile is set to `AvailabilitySet`, this value should be the subnetId shared by both master and agent nodes. |
|
||||
| extensions | no | This is an array of extensions. This indicates that the extension be run on a single master. The name in the extensions array must exactly match the extension name in the extensionProfiles |
|
||||
| vnetCidr | no | Specifies the VNET cidr when using a custom VNET ([bring your own VNET examples](../examples/vnet)) |
|
||||
| vnetCidr | no | Specifies the VNET cidr when using a custom VNET ([bring your own VNET examples](../examples/vnet)). This VNET cidr should include both the master and the agent subnets. |
|
||||
| imageReference.name | no | The name of the Linux OS image. Needs to be used in conjunction with resourceGroup, below |
|
||||
| imageReference.resourceGroup | no | Resource group that contains the Linux OS image. Needs to be used in conjunction with name, above |
|
||||
| distro | no | Select Master(s) Operating System (Linux only). Currently supported values are: `ubuntu`, `aks` and `coreos` (CoreOS support is currently experimental). Defaults to `aks` if undefined. `aks` is a custom image based on `ubuntu` that comes with pre-installed software necessary for Kubernetes deployments. Currently supported OS and orchestrator configurations -- `ubuntu` and `aks`: DCOS, Docker Swarm, Kubernetes; `RHEL`: OpenShift; `coreos`: Kubernetes. [Example of CoreOS Master with CoreOS Agents](../examples/coreos/kubernetes-coreos.json) |
|
||||
| customFiles | no | The custom files to be provisioned to the master nodes. Defined as an array of json objects with each defined as `"source":"absolute-local-path", "dest":"absolute-path-on-masternodes"`.[See examples](../examples/customfiles) |
|
||||
| availabilityProfile | no | Supported values are `AvailabilitySet` (default) and `VirtualMachineScaleSets` (requires Kubernetes clusters version 1.10+ and agent pool availabilityProfile must also be `VirtualMachineScaleSets`). |
|
||||
| agentVnetSubnetId | only required when using custom VNET and when MasterProfile is using `VirtualMachineScaleSets` | Specifies the Id of an alternate VNET subnet for all the agent pool nodes. The subnet id must specify a valid VNET ID owned by the same subscription. ([bring your own VNET examples](../examples/vnet)). When MasterProfile is using `VirtualMachineScaleSets`, this value should be the subnetId of the subnet for all agent pool nodes. |
|
||||
|
||||
### agentPoolProfiles
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
{
|
||||
"apiVersion": "vlabs",
|
||||
"properties": {
|
||||
"orchestratorProfile": {
|
||||
"orchestratorType": "Kubernetes",
|
||||
"kubernetesConfig": {
|
||||
"clusterSubnet": "10.239.0.0/16",
|
||||
"addons": [
|
||||
{
|
||||
"name": "tiller",
|
||||
"enabled": true,
|
||||
"config": {
|
||||
"max-history": "10"
|
||||
},
|
||||
"containers": [
|
||||
{
|
||||
"name": "tiller",
|
||||
"cpuRequests": "1",
|
||||
"memoryRequests": "1Gi",
|
||||
"cpuLimits": "1",
|
||||
"memoryLimits": "1Gi"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "kubernetes-dashboard",
|
||||
"enabled": true,
|
||||
"containers": [
|
||||
{
|
||||
"name": "kubernetes-dashboard",
|
||||
"cpuRequests": "50m",
|
||||
"memoryRequests": "512Mi",
|
||||
"cpuLimits": "50m",
|
||||
"memoryLimits": "512Mi"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "rescheduler",
|
||||
"enabled": true,
|
||||
"containers": [
|
||||
{
|
||||
"name": "rescheduler",
|
||||
"cpuRequests": "20m",
|
||||
"memoryRequests": "200Mi",
|
||||
"cpuLimits": "20m",
|
||||
"memoryLimits": "200Mi"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"masterProfile": {
|
||||
"count": 3,
|
||||
"dnsPrefix": "",
|
||||
"vmSize": "Standard_DS2_v2",
|
||||
"OSDiskSizeGB": 200,
|
||||
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
|
||||
"agentVnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
|
||||
"vnetCidr": "10.239.0.0/16",
|
||||
"availabilityProfile": "VirtualMachineScaleSets"
|
||||
},
|
||||
"agentPoolProfiles": [
|
||||
{
|
||||
"name": "agentmd",
|
||||
"count": 3,
|
||||
"vmSize": "Standard_DS2_v2",
|
||||
"OSDiskSizeGB": 200,
|
||||
"storageProfile": "ManagedDisks",
|
||||
"diskSizesGB": [
|
||||
128,
|
||||
128,
|
||||
128,
|
||||
128
|
||||
],
|
||||
"availabilityProfile": "VirtualMachineScaleSets",
|
||||
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
|
||||
},
|
||||
{
|
||||
"name": "agentsa",
|
||||
"count": 3,
|
||||
"vmSize": "Standard_DS2_v2",
|
||||
"OSDiskSizeGB": 200,
|
||||
"storageProfile": "ManagedDisks",
|
||||
"diskSizesGB": [
|
||||
128
|
||||
],
|
||||
"availabilityProfile": "VirtualMachineScaleSets",
|
||||
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
|
||||
}
|
||||
],
|
||||
"linuxProfile": {
|
||||
"adminUsername": "azureuser",
|
||||
"ssh": {
|
||||
"publicKeys": [
|
||||
{
|
||||
"keyData": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"servicePrincipalProfile": {
|
||||
"clientId": "",
|
||||
"secret": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"apiVersion": "vlabs",
|
||||
"properties": {
|
||||
"orchestratorProfile": {
|
||||
"orchestratorType": "Kubernetes",
|
||||
"orchestratorRelease": "1.10"
|
||||
},
|
||||
"masterProfile": {
|
||||
"count": 1,
|
||||
"dnsPrefix": "",
|
||||
"vmSize": "Standard_D2_v2",
|
||||
"availabilityProfile": "VirtualMachineScaleSets"
|
||||
},
|
||||
"agentPoolProfiles": [
|
||||
{
|
||||
"name": "agentpool1",
|
||||
"count": 3,
|
||||
"vmSize": "Standard_D2_v2",
|
||||
"availabilityProfile": "VirtualMachineScaleSets",
|
||||
"storageProfile": "ManagedDisks"
|
||||
}
|
||||
],
|
||||
"linuxProfile": {
|
||||
"adminUsername": "azureuser",
|
||||
"ssh": {
|
||||
"publicKeys": [
|
||||
{
|
||||
"keyData": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
{
|
||||
"apiVersion": "vlabs",
|
||||
"properties": {
|
||||
"orchestratorProfile": {
|
||||
"orchestratorType": "Kubernetes",
|
||||
"orchestratorRelease": "1.10"
|
||||
},
|
||||
"masterProfile": {
|
||||
"count": 1,
|
||||
"dnsPrefix": "",
|
||||
"vmSize": "Standard_D2_v2",
|
||||
"availabilityProfile": "VirtualMachineScaleSets"
|
||||
},
|
||||
"agentPoolProfiles": [
|
||||
{
|
||||
"name": "linuxpool1",
|
||||
"count": 1,
|
||||
"vmSize": "Standard_D2_v2",
|
||||
"availabilityProfile": "VirtualMachineScaleSets"
|
||||
},
|
||||
{
|
||||
"name": "agentwin",
|
||||
"count": 2,
|
||||
"vmSize": "Standard_D2_v2",
|
||||
"availabilityProfile": "VirtualMachineScaleSets",
|
||||
"osType": "Windows"
|
||||
}
|
||||
],
|
||||
"windowsProfile": {
|
||||
"adminUsername": "azureuser",
|
||||
"adminPassword": "replacepassword1234$"
|
||||
},
|
||||
"linuxProfile": {
|
||||
"adminUsername": "azureuser",
|
||||
"ssh": {
|
||||
"publicKeys": [
|
||||
{
|
||||
"keyData": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"servicePrincipalProfile": {
|
||||
"clientId": "",
|
||||
"secret": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
{
|
||||
"apiVersion": "vlabs",
|
||||
"properties": {
|
||||
"orchestratorProfile": {
|
||||
"orchestratorType": "Kubernetes",
|
||||
"kubernetesConfig": {
|
||||
"clusterSubnet": "10.239.0.0/16",
|
||||
"addons": [
|
||||
{
|
||||
"name": "tiller",
|
||||
"enabled": true,
|
||||
"config": {
|
||||
"max-history": "10"
|
||||
},
|
||||
"containers": [
|
||||
{
|
||||
"name": "tiller",
|
||||
"cpuRequests": "1",
|
||||
"memoryRequests": "1Gi",
|
||||
"cpuLimits": "1",
|
||||
"memoryLimits": "1Gi"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "kubernetes-dashboard",
|
||||
"enabled": true,
|
||||
"containers": [
|
||||
{
|
||||
"name": "kubernetes-dashboard",
|
||||
"cpuRequests": "50m",
|
||||
"memoryRequests": "512Mi",
|
||||
"cpuLimits": "50m",
|
||||
"memoryLimits": "512Mi"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "rescheduler",
|
||||
"enabled": true,
|
||||
"containers": [
|
||||
{
|
||||
"name": "rescheduler",
|
||||
"cpuRequests": "20m",
|
||||
"memoryRequests": "200Mi",
|
||||
"cpuLimits": "20m",
|
||||
"memoryLimits": "200Mi"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"masterProfile": {
|
||||
"count": 3,
|
||||
"dnsPrefix": "",
|
||||
"vmSize": "Standard_DS2_v2",
|
||||
"OSDiskSizeGB": 200,
|
||||
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
|
||||
"agentVnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
|
||||
"vnetCidr": "10.239.0.0/16",
|
||||
"availabilityProfile": "VirtualMachineScaleSets"
|
||||
},
|
||||
"agentPoolProfiles": [
|
||||
{
|
||||
"name": "agentmd",
|
||||
"count": 3,
|
||||
"vmSize": "Standard_DS2_v2",
|
||||
"OSDiskSizeGB": 200,
|
||||
"storageProfile": "ManagedDisks",
|
||||
"diskSizesGB": [
|
||||
128,
|
||||
128,
|
||||
128,
|
||||
128
|
||||
],
|
||||
"availabilityProfile": "VirtualMachineScaleSets",
|
||||
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
|
||||
},
|
||||
{
|
||||
"name": "agentsa",
|
||||
"count": 3,
|
||||
"vmSize": "Standard_DS2_v2",
|
||||
"OSDiskSizeGB": 200,
|
||||
"storageProfile": "ManagedDisks",
|
||||
"diskSizesGB": [
|
||||
128
|
||||
],
|
||||
"availabilityProfile": "VirtualMachineScaleSets",
|
||||
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
|
||||
}
|
||||
],
|
||||
"linuxProfile": {
|
||||
"adminUsername": "azureuser",
|
||||
"ssh": {
|
||||
"publicKeys": [
|
||||
{
|
||||
"keyData": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"servicePrincipalProfile": {
|
||||
"clientId": "",
|
||||
"secret": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -49,7 +49,11 @@
|
|||
"{{.Name}}AccountName": "[concat(variables('storageAccountBaseName'), 'agnt{{$index}}')]",
|
||||
{{end}}
|
||||
{{end}}
|
||||
{{template "k8s/kubernetesmastervars.t" .}}
|
||||
{{if IsMasterVirtualMachineScaleSets}}
|
||||
{{template "k8s/kubernetesmastervarsvmss.t" .}}
|
||||
{{else}}
|
||||
{{template "k8s/kubernetesmastervars.t" .}}
|
||||
{{end}}
|
||||
},
|
||||
"resources": [
|
||||
{{if UserAssignedIDEnabled}}
|
||||
|
|
@ -80,7 +84,11 @@
|
|||
{{end}}
|
||||
{{end}}
|
||||
{{if not IsHostedMaster}}
|
||||
,{{template "k8s/kubernetesmasterresources.t" .}}
|
||||
{{if IsMasterVirtualMachineScaleSets}}
|
||||
,{{template "k8s/kubernetesmasterresourcesvmss.t" .}}
|
||||
{{else}}
|
||||
,{{template "k8s/kubernetesmasterresources.t" .}}
|
||||
{{end}}
|
||||
{{else}}
|
||||
{{if not IsCustomVNET}}
|
||||
,{
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
#!/bin/bash
|
||||
NODE_INDEX=$(hostname | tail -c 2)
|
||||
NODE_NAME=$(hostname)
|
||||
PRIVATE_IP=$(hostname -I | cut -d' ' -f1)
|
||||
ETCD_PEER_URL="https://${PRIVATE_IP}:2380"
|
||||
ETCD_CLIENT_URL="https://${PRIVATE_IP}:2379"
|
||||
|
||||
function systemctlEnableAndStart() {
|
||||
systemctl_restart 100 5 30 $1
|
||||
|
|
@ -41,7 +46,7 @@ function configureEtcd() {
|
|||
chmod 0600 "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
chown root:root "${ETCD_CLIENT_PRIVATE_KEY_PATH}"
|
||||
|
||||
ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.key"
|
||||
ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${NODE_INDEX}.key"
|
||||
touch "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
chmod 0600 "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
chown etcd:etcd "${ETCD_PEER_PRIVATE_KEY_PATH}"
|
||||
|
|
@ -56,7 +61,7 @@ function configureEtcd() {
|
|||
chmod 0644 "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
chown root:root "${ETCD_CLIENT_CERTIFICATE_PATH}"
|
||||
|
||||
ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.crt"
|
||||
ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${NODE_INDEX}.crt"
|
||||
touch "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
chmod 0644 "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
chown root:root "${ETCD_PEER_CERTIFICATE_PATH}"
|
||||
|
|
@ -85,7 +90,7 @@ function configureEtcd() {
|
|||
$MOUNT_ETCD_FILE || exit $ERR_ETCD_VOL_MOUNT_FAIL
|
||||
systemctlEnableAndStart etcd || exit $ERR_ETCD_START_TIMEOUT
|
||||
for i in $(seq 1 600); do
|
||||
MEMBER="$(sudo etcdctl member list | grep -E ${MASTER_VM_NAME} | cut -d':' -f 1)"
|
||||
MEMBER="$(sudo etcdctl member list | grep -E ${NODE_NAME} | cut -d':' -f 1)"
|
||||
if [ "$MEMBER" != "" ]; then
|
||||
break
|
||||
else
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ source /opt/azure/containers/provision_configs.sh
|
|||
CUSTOM_SEARCH_DOMAIN_SCRIPT=/opt/azure/containers/setup-custom-search-domains.sh
|
||||
|
||||
set +x
|
||||
ETCD_PEER_CERT=$(echo ${ETCD_PEER_CERTIFICATES} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${MASTER_INDEX}+1)))
|
||||
ETCD_PEER_KEY=$(echo ${ETCD_PEER_PRIVATE_KEYS} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${MASTER_INDEX}+1)))
|
||||
ETCD_PEER_CERT=$(echo ${ETCD_PEER_CERTIFICATES} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${NODE_INDEX}+1)))
|
||||
ETCD_PEER_KEY=$(echo ${ETCD_PEER_PRIVATE_KEYS} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${NODE_INDEX}+1)))
|
||||
set -x
|
||||
|
||||
if [[ $OS == $COREOS_OS_NAME ]]; then
|
||||
|
|
|
|||
|
|
@ -0,0 +1,540 @@
|
|||
#cloud-config
|
||||
|
||||
{{if not .MasterProfile.IsCoreOS}}
|
||||
packages:
|
||||
- jq
|
||||
- traceroute
|
||||
{{end}}
|
||||
|
||||
write_files:
|
||||
- path: "/opt/azure/containers/provision_source.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "provisionSource"}}
|
||||
|
||||
- path: "/etc/ssh/sshd_config"
|
||||
permissions: "0644"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "sshdConfig"}}
|
||||
|
||||
- path: "/opt/azure/containers/provision_installs.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "provisionInstalls"}}
|
||||
|
||||
- path: "/opt/azure/containers/provision_configs.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "provisionConfigs"}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}}
|
||||
{{if not .MasterProfile.IsCoreOS}}
|
||||
- path: "/etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
content: |
|
||||
[Service]
|
||||
MountFlags=shared
|
||||
{{end}}
|
||||
|
||||
- path: "/etc/systemd/system/docker.service.d/exec_start.conf"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
content: |
|
||||
[Service]
|
||||
ExecStart=
|
||||
{{if .MasterProfile.IsCoreOS}}
|
||||
ExecStart=/usr/bin/env PATH=${TORCX_BINDIR}:${PATH} ${TORCX_BINDIR}/dockerd --host=fd:// --containerd=/var/run/docker/libcontainerd/docker-containerd.sock --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} $DOCKER_SELINUX $DOCKER_OPTS $DOCKER_CGROUPS $DOCKER_OPT_BIP $DOCKER_OPT_MTU $DOCKER_OPT_IPMASQ
|
||||
{{else}}
|
||||
ExecStart=/usr/bin/docker daemon -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}}
|
||||
{{end}}
|
||||
- path: "/etc/docker/daemon.json"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
content: |
|
||||
{
|
||||
"live-restore": true,
|
||||
"log-driver": "json-file",
|
||||
"log-opts": {
|
||||
"max-size": "50m",
|
||||
"max-file": "5"
|
||||
}
|
||||
}
|
||||
|
||||
- path: "/usr/local/bin/health-monitor.sh"
|
||||
permissions: "0544"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "healthMonitorScript"}}
|
||||
|
||||
- path: "/etc/systemd/system/docker-monitor.service"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=a script that checks docker health and restarts if needed
|
||||
After=docker.service
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/bin/health-monitor.sh container-runtime
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
- path: "/etc/systemd/system/kubelet-monitor.service"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=a script that checks kubelet health and restarts if needed
|
||||
After=kubelet.service
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/bin/health-monitor.sh kubelet
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
{{end}}
|
||||
|
||||
- path: "/etc/kubernetes/certs/ca.crt"
|
||||
permissions: "0644"
|
||||
encoding: "base64"
|
||||
owner: "root"
|
||||
content: |
|
||||
{{WrapAsParameter "caCertificate"}}
|
||||
|
||||
- path: "/etc/kubernetes/certs/client.crt"
|
||||
permissions: "0644"
|
||||
encoding: "base64"
|
||||
owner: "root"
|
||||
content: |
|
||||
{{WrapAsParameter "clientCertificate"}}
|
||||
|
||||
{{if EnableAggregatedAPIs}}
|
||||
- path: "/etc/kubernetes/generate-proxy-certs.sh"
|
||||
permissions: "0744"
|
||||
encoding: "gzip"
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "generateProxyCertsScript"}}
|
||||
{{end}}
|
||||
|
||||
{{if HasCustomSearchDomain}}
|
||||
- path: "/opt/azure/containers/setup-custom-search-domains.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "customSearchDomainsScript"}}
|
||||
{{end}}
|
||||
|
||||
- path: "/var/lib/kubelet/kubeconfig"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
content: |
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- name: localcluster
|
||||
cluster:
|
||||
certificate-authority: /etc/kubernetes/certs/ca.crt
|
||||
server: <SERVERIP>
|
||||
users:
|
||||
- name: client
|
||||
user:
|
||||
client-certificate: /etc/kubernetes/certs/client.crt
|
||||
client-key: /etc/kubernetes/certs/client.key
|
||||
contexts:
|
||||
- context:
|
||||
cluster: localcluster
|
||||
user: client
|
||||
name: localclustercontext
|
||||
current-context: localclustercontext
|
||||
|
||||
{{if EnableDataEncryptionAtRest}}
|
||||
- path: "/etc/kubernetes/encryption-config.yaml"
|
||||
permissions: "0600"
|
||||
owner: "root"
|
||||
content: |
|
||||
kind: EncryptionConfig
|
||||
apiVersion: v1
|
||||
resources:
|
||||
- resources:
|
||||
- secrets
|
||||
providers:
|
||||
- aescbc:
|
||||
keys:
|
||||
- name: key1
|
||||
secret: <etcdEncryptionSecret>
|
||||
- identity: {}
|
||||
{{end}}
|
||||
|
||||
{{if EnableEncryptionWithExternalKms}}
|
||||
- path: "/etc/kubernetes/encryption-config.yaml"
|
||||
permissions: "0444"
|
||||
owner: "root"
|
||||
content: |
|
||||
kind: EncryptionConfig
|
||||
apiVersion: v1
|
||||
resources:
|
||||
- resources:
|
||||
- secrets
|
||||
providers:
|
||||
- kms:
|
||||
name: azurekmsprovider
|
||||
endpoint: unix:///opt/azurekms.socket
|
||||
cachesize: 0
|
||||
- identity: {}
|
||||
{{end}}
|
||||
|
||||
MASTER_MANIFESTS_CONFIG_PLACEHOLDER
|
||||
|
||||
MASTER_ADDONS_CONFIG_PLACEHOLDER
|
||||
|
||||
MASTER_CUSTOM_FILES_PLACEHOLDER
|
||||
|
||||
- path: "/etc/default/kubelet"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
content: |
|
||||
{{if IsKubernetesVersionLt "1.8.0"}}
|
||||
KUBELET_OPTS=--require-kubeconfig
|
||||
{{else}}
|
||||
KUBELET_OPTS=
|
||||
{{end}}
|
||||
KUBELET_CONFIG={{GetKubeletConfigKeyVals .MasterProfile.KubernetesConfig}}
|
||||
KUBELET_IMAGE={{WrapAsParameter "kubernetesHyperkubeSpec"}}
|
||||
KUBELET_NODE_LABELS={{GetMasterKubernetesLabels "',variables('labelResourceGroup'),'"}}
|
||||
{{if IsKubernetesVersionGe "1.6.0"}}
|
||||
{{if HasLinuxAgents}}
|
||||
KUBELET_REGISTER_NODE=--register-node=true
|
||||
KUBELET_REGISTER_WITH_TAINTS=--register-with-taints=node-role.kubernetes.io/master=true:NoSchedule
|
||||
{{end}}
|
||||
{{else}}
|
||||
KUBELET_REGISTER_SCHEDULABLE={{WrapAsVariable "registerSchedulable"}}
|
||||
{{end}}
|
||||
|
||||
MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
|
||||
|
||||
- path: "/opt/azure/containers/kubelet.sh"
|
||||
permissions: "0755"
|
||||
owner: "root"
|
||||
content: |
|
||||
#!/bin/bash
|
||||
set -e
|
||||
PRIVATE_IP=$(hostname -I | cut -d" " -f1)
|
||||
ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
|
||||
{{if gt .MasterProfile.Count 1}}
|
||||
# Azure does not support two LoadBalancers(LB) sharing the same nic and backend port.
|
||||
# As a workaround, the Internal LB(ILB) listens for apiserver traffic on port 4443 and the External LB(ELB) on port 443
|
||||
# This IPTable rule then redirects ILB traffic to port 443 in the prerouting chain
|
||||
iptables -t nat -A PREROUTING -p tcp --dport 4443 -j REDIRECT --to-port 443
|
||||
{{end}}
|
||||
|
||||
{{if IsAzureCNI}}
|
||||
# SNAT outbound traffic from pods to destinations outside of VNET.
|
||||
iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsParameter "vnetCidr"}} -j MASQUERADE
|
||||
sed -i "s|<azureCNINetworkMonitorImage>|{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml"
|
||||
{{end}}
|
||||
sed -i "s|<kubernetesAddonManagerSpec>|{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml"
|
||||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
|
||||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml"
|
||||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
|
||||
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
|
||||
sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsParameter "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsParameter "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsParameter "kubernetesExecHealthzSpec"}}|g; s|<kubernetesDNSSidecarSpec>|{{WrapAsParameter "kubernetesDNSSidecarSpec"}}|g; s|<kubernetesKubeletClusterDomain>|{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s|<kubeDNSServiceIP>|{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
|
||||
sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}}
|
||||
sed -i "s|<kubernetesDashboardSpec>|{{WrapAsParameter "kubernetesDashboardSpec"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
|
||||
sed -i "s|<kubernetesDashboardCPURequests>|{{WrapAsParameter "kubernetesDashboardCPURequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
|
||||
sed -i "s|<kubernetesDashboardMemoryRequests>|{{WrapAsParameter "kubernetesDashboardMemoryRequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
|
||||
sed -i "s|<kubernetesDashboardCPULimit>|{{WrapAsParameter "kubernetesDashboardCPULimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
|
||||
sed -i "s|<kubernetesDashboardMemoryLimit>|{{WrapAsParameter "kubernetesDashboardMemoryLimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsTillerEnabled}}
|
||||
sed -i "s|<kubernetesTillerSpec>|{{WrapAsParameter "kubernetesTillerSpec"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
|
||||
sed -i "s|<kubernetesTillerCPURequests>|{{WrapAsParameter "kubernetesTillerCPURequests"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
|
||||
sed -i "s|<kubernetesTillerMemoryRequests>|{{WrapAsParameter "kubernetesTillerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
|
||||
sed -i "s|<kubernetesTillerCPULimit>|{{WrapAsParameter "kubernetesTillerCPULimit"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
|
||||
sed -i "s|<kubernetesTillerMemoryLimit>|{{WrapAsParameter "kubernetesTillerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
|
||||
sed -i "s|<kubernetesTillerMaxHistory>|{{WrapAsParameter "kubernetesTillerMaxHistory"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if AdminGroupID }}
|
||||
sed -i "s|<aadAdminGroupId>|{{WrapAsParameter "aadAdminGroupId"}}|g" "/etc/kubernetes/addons/aad-default-admin-group-rbac.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled}}
|
||||
sed -i "s|<kubernetesACIConnectorSpec>|{{WrapAsParameter "kubernetesACIConnectorSpec"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorNodeName>|{{WrapAsParameter "kubernetesACIConnectorNodeName"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorOS>|{{WrapAsParameter "kubernetesACIConnectorOS"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorTaint>|{{WrapAsParameter "kubernetesACIConnectorTaint"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorRegion>|{{WrapAsParameter "kubernetesACIConnectorRegion"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorCPURequests>|{{WrapAsParameter "kubernetesACIConnectorCPURequests"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorMemoryRequests>|{{WrapAsParameter "kubernetesACIConnectorMemoryRequests"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorCPULimit>|{{WrapAsParameter "kubernetesACIConnectorCPULimit"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
sed -i "s|<kubernetesACIConnectorMemoryLimit>|{{WrapAsParameter "kubernetesACIConnectorMemoryLimit"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}}
|
||||
sed -i "s|<kubernetesClusterAutoscalerAzureCloud>|{{WrapAsParameter "kubernetesClusterAutoscalerAzureCloud"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerSpec>|{{WrapAsParameter "kubernetesClusterAutoscalerSpec"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerCPULimit>|{{WrapAsParameter "kubernetesClusterAutoscalerCPULimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerMemoryLimit>|{{WrapAsParameter "kubernetesClusterAutoscalerMemoryLimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerCPURequests>|{{WrapAsParameter "kubernetesClusterAutoscalerCPURequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerMemoryRequests>|{{WrapAsParameter "kubernetesClusterAutoscalerMemoryRequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerMinNodes>|{{WrapAsParameter "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerMaxNodes>|{{WrapAsParameter "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
sed -i "s|<kubernetesClusterAutoscalerUseManagedIdentity>|{{WrapAsParameter "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if eq .OrchestratorProfile.KubernetesConfig.LoadBalancerSku "Standard"}}
|
||||
sed -i "s|<kuberneteselbsvcname>|{{WrapAsParameter "kuberneteselbsvcname"}}|g" "/etc/kubernetes/addons/elb-svc.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsBlobfuseFlexVolumeEnabled}}
|
||||
sed -i "s|<kubernetesBlobfuseFlexVolumeInstallerCPURequests>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesBlobfuseFlexVolumeInstallerMemoryRequests>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesBlobfuseFlexVolumeInstallerCPULimit>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesBlobfuseFlexVolumeInstallerMemoryLimit>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsSMBFlexVolumeEnabled}}
|
||||
sed -i "s|<kubernetesSMBFlexVolumeInstallerCPURequests>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesSMBFlexVolumeInstallerMemoryRequests>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesSMBFlexVolumeInstallerCPULimit>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesSMBFlexVolumeInstallerMemoryLimit>|{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled}}
|
||||
sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerCPURequests>|{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerMemoryRequests>|{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerCPULimit>|{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
|
||||
sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerMemoryLimit>|{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}}
|
||||
sed -i "s|<kubernetesReschedulerSpec>|{{WrapAsParameter "kubernetesReschedulerSpec"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
|
||||
sed -i "s|<kubernetesReschedulerCPURequests>|{{WrapAsParameter "kubernetesReschedulerCPURequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
|
||||
sed -i "s|<kubernetesReschedulerMemoryRequests>|{{WrapAsParameter "kubernetesReschedulerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
|
||||
sed -i "s|<kubernetesReschedulerCPULimit>|{{WrapAsParameter "kubernetesReschedulerCPULimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
|
||||
sed -i "s|<kubernetesReschedulerMemoryLimit>|{{WrapAsParameter "kubernetesReschedulerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if .OrchestratorProfile.IsMetricsServerEnabled}}
|
||||
sed -i "s|<kubernetesMetricsServerSpec>|{{WrapAsParameter "kubernetesMetricsServerSpec"}}|g" "/etc/kubernetes/addons/kube-metrics-server-deployment.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if IsNVIDIADevicePluginEnabled}}
|
||||
sed -i "s|<kubernetesNVIDIADevicePluginSpec>|{{WrapAsParameter "kubernetesNVIDIADevicePluginSpec"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
|
||||
sed -i "s|<kubernetesNVIDIADevicePluginCPURequests>|{{WrapAsParameter "kubernetesNVIDIADevicePluginCPURequests"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
|
||||
sed -i "s|<kubernetesNVIDIADevicePluginMemoryRequests>|{{WrapAsParameter "kubernetesNVIDIADevicePluginMemoryRequests"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
|
||||
sed -i "s|<kubernetesNVIDIADevicePluginCPULimit>|{{WrapAsParameter "kubernetesNVIDIADevicePluginCPULimit"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
|
||||
sed -i "s|<kubernetesNVIDIADevicePluginMemoryLimit>|{{WrapAsParameter "kubernetesNVIDIADevicePluginMemoryLimit"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if EnableDataEncryptionAtRest }}
|
||||
sed -i "s|<etcdEncryptionSecret>|\"{{WrapAsParameter "etcdEncryptionKey"}}\"|g" "/etc/kubernetes/encryption-config.yaml"
|
||||
{{end}}
|
||||
|
||||
{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}}
|
||||
# If Calico Policy enabled then update Cluster Cidr
|
||||
sed -i "s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml"
|
||||
{{end}}
|
||||
{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPlugin "flannel"}}
|
||||
# If Flannel is enabled then update Cluster Cidr
|
||||
sed -i "s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/flannel-daemonset.yaml"
|
||||
{{end}}
|
||||
{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "cilium"}}
|
||||
# If Cilium Policy enabled then update the etcd certs and address
|
||||
sed -i "s|<ETCD_URL>|"https://$PRIVATE_IP:$ETCD_CLIENT_PORT"|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
|
||||
sed -i "s|<ETCD_CA>|$(base64 -w 0 /etc/kubernetes/certs/ca.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
|
||||
sed -i "s|<ETCD_CLIENT_KEY>|$(base64 -w 0 /etc/kubernetes/certs/etcdclient.key)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
|
||||
sed -i "s|<ETCD_CLIENT_CERT>|$(base64 -w 0 /etc/kubernetes/certs/etcdclient.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
|
||||
{{end}}
|
||||
{{if UseCloudControllerManager }}
|
||||
sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsParameter "kubernetesCcmImageSpec"}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml"
|
||||
sed -i "s|<kubernetesCloudControllerManagerConfig>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml"
|
||||
{{end}}
|
||||
sed -i "s|<kubernetesControllerManagerConfig>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.ControllerManagerConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml"
|
||||
sed -i "s|<kubernetesAPIServerConfig>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.APIServerConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
|
||||
sed -i "s|<kubernetesSchedulerConfig>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.SchedulerConfig}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
|
||||
sed -i "s|<kubernetesAPIServerIP>|{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
|
||||
{{if not EnablePodSecurityPolicy}}
|
||||
sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service"
|
||||
{{end}}
|
||||
{{if EnableEncryptionWithExternalKms}}
|
||||
sed -i "s|# Required|Requires=kms.service|g" "/etc/systemd/system/kubelet.service"
|
||||
{{end}}
|
||||
{{if HasCustomSearchDomain}}
|
||||
sed -i "s|<searchDomainName>|{{WrapAsParameter "searchDomainName"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh"
|
||||
sed -i "s|<searchDomainRealmUser>|{{WrapAsParameter "searchDomainRealmUser"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh"
|
||||
sed -i "s|<searchDomainRealmPassword>|{{WrapAsParameter "searchDomainRealmPassword"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh"
|
||||
{{end}}
|
||||
{{if .OrchestratorProfile.KubernetesConfig.IsContainerMonitoringEnabled}}
|
||||
sed -i "s|<omsAgentVersion>|{{WrapAsParameter "omsAgentVersion"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
sed -i "s|<dockerProviderVersion>|{{WrapAsParameter "omsAgentDockerProviderVersion"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
sed -i "s|<kubernetesContainerMonitoringSpec>|{{WrapAsParameter "omsAgentImage"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
sed -i "s|<workspaceGuid>|{{WrapAsParameter "omsAgentWorkspaceGuid"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
sed -i "s|<workspaceKey>|{{WrapAsParameter "omsAgentWorkspaceKey"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
sed -i "s|<kubernetesOMSAgentCPURequests>|{{WrapAsParameter "kubernetesOMSAgentCPURequests"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
sed -i "s|<kubernetesOMSAgentMemoryRequests>|{{WrapAsParameter "kubernetesOMSAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
sed -i "s|<kubernetesOMSAgentCPULimit>|{{WrapAsParameter "kubernetesOMSAgentCPULimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
sed -i "s|<kubernetesOMSAgentMemoryLimit>|{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml"
|
||||
{{end}}
|
||||
|
||||
- path: "/opt/azure/containers/provision.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "provisionScript"}}
|
||||
|
||||
- path: "/opt/azure/containers/mountetcd.sh"
|
||||
permissions: "0744"
|
||||
encoding: gzip
|
||||
owner: "root"
|
||||
content: !!binary |
|
||||
{{WrapAsVariable "mountetcdScript"}}
|
||||
|
||||
- path: "/etc/systemd/system/etcd.service"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=etcd - highly-available key value store
|
||||
Documentation=https://github.com/coreos/etcd
|
||||
Documentation=man:etcd
|
||||
After=network.target
|
||||
Wants=network-online.target
|
||||
[Service]
|
||||
Environment=DAEMON_ARGS=
|
||||
Environment=ETCD_NAME=%H
|
||||
Environment=ETCD_DATA_DIR=
|
||||
EnvironmentFile=-/etc/default/%p
|
||||
Type=notify
|
||||
User=etcd
|
||||
PermissionsStartOnly=true
|
||||
ExecStart=/usr/bin/etcd $DAEMON_ARGS
|
||||
Restart=always
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
- path: "/opt/azure/containers/setup-etcd.sh"
|
||||
permissions: "0744"
|
||||
owner: "root"
|
||||
content: |
|
||||
#!/bin/bash
|
||||
set -x
|
||||
MASTER_VM_NAME=$(hostname)
|
||||
MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//")
|
||||
MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}}
|
||||
MASTER_INDEX=$(hostname | tail -c 2)
|
||||
PRIVATE_IP=$(hostname -I | cut -d" " -f1)
|
||||
PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3)
|
||||
MASTER_COUNT={{WrapAsVariable "masterCount"}}
|
||||
IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}}
|
||||
echo $IPADDRESS_COUNT
|
||||
ETCD_SERVER_PORT={{WrapAsVariable "masterEtcdServerPort"}}
|
||||
ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
|
||||
MASTER_URLS=""
|
||||
index=0
|
||||
while [ $index -lt $MASTER_COUNT ]
|
||||
do
|
||||
echo $index
|
||||
offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4`
|
||||
echo $offset
|
||||
MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT,"
|
||||
index=`expr $index + 1`
|
||||
done
|
||||
MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//")
|
||||
echo $MASTER_URLS
|
||||
sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:$ETCD_CLIENT_PORT" /etc/environment
|
||||
sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment
|
||||
sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment
|
||||
sudo sed -i "1iETCDCTL_CERT_FILE={{WrapAsVariable "etcdClientCertFilepath"}}" /etc/environment
|
||||
sudo sed -i "s|<SERVERIP>|https://$PRIVATE_IP:443|g" "/var/lib/kubelet/kubeconfig"
|
||||
/bin/echo DAEMON_ARGS=--name $MASTER_VM_NAME --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.crt --peer-key-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.key --initial-advertise-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --listen-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT" --listen-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT,https://127.0.0.1:$ETCD_CLIENT_PORT" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster $MASTER_URLS --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd
|
||||
|
||||
{{if .MasterProfile.IsCoreOS}}
|
||||
- path: "/opt/azure/containers/provision-setup.sh"
|
||||
permissions: "0755"
|
||||
owner: "root"
|
||||
content: |
|
||||
#!/bin/bash
|
||||
# the first arg is the number of retries, the second arg is the wait duration between two retries and the rest of the args are the cmd to run
|
||||
source /opt/azure/containers/provision_source.sh
|
||||
set -x
|
||||
MASTER_VM_NAME=$(hostname)
|
||||
MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//")
|
||||
MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}}
|
||||
MASTER_INDEX=$(hostname | tail -c 2)
|
||||
PRIVATE_IP=$(hostname -I | cut -d" " -f1)
|
||||
PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3)
|
||||
MASTER_COUNT={{WrapAsVariable "masterCount"}}
|
||||
IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}}
|
||||
echo $IPADDRESS_COUNT
|
||||
ETCD_SERVER_PORT={{WrapAsVariable "masterEtcdServerPort"}}
|
||||
ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
|
||||
MASTER_URLS=""
|
||||
index=0
|
||||
while [ $index -lt $MASTER_COUNT ]
|
||||
do
|
||||
echo $index
|
||||
offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4`
|
||||
echo $offset
|
||||
MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT,"
|
||||
index=`expr $index + 1`
|
||||
done
|
||||
MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//")
|
||||
echo $MASTER_URLS
|
||||
|
||||
/bin/echo DAEMON_ARGS=--name $MASTER_VM_NAME --initial-advertise-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --listen-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --advertise-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT" --listen-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT,https://127.0.0.1:$ETCD_CLIENT_PORT" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster $MASTER_URLS --data-dir "/var/lib/etcddisk"" --initial-cluster-state "new" | tee -a /etc/default/etcd
|
||||
/opt/azure/containers/mountetcd.sh
|
||||
sudo /bin/chown -R etcd:etcd /var/lib/etcddisk
|
||||
systemctl stop etcd-member
|
||||
sudo /bin/sed -i s/Restart=on-failure/Restart=always/g /lib/systemd/system/etcd-member.service
|
||||
systemctl daemon-reload
|
||||
systemctl restart etcd-member
|
||||
retrycmd_if_failure 5 5 10 curl --retry 5 --retry-delay 10 --retry-max-time 10 --max-time 60 http://127.0.0.1:2379/v2/machines
|
||||
mkdir -p /etc/kubernetes/manifests
|
||||
|
||||
{{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}}
|
||||
usermod -aG docker {{WrapAsParameter "linuxAdminUsername"}}
|
||||
{{end}}
|
||||
|
||||
{{if EnableAggregatedAPIs}}
|
||||
sudo bash /etc/kubernetes/generate-proxy-certs.sh
|
||||
{{end}}
|
||||
|
||||
touch /opt/azure/containers/runcmd.complete
|
||||
|
||||
coreos:
|
||||
units:
|
||||
- name: start-provision-setup.service
|
||||
command: "start"
|
||||
content: |
|
||||
# Note: Initiated as a service since there is no runcmd within CoreOS on cloud-config/Ignition
|
||||
[Unit]
|
||||
Description=Start provision setup service
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/azure/containers/provision-setup.sh
|
||||
{{else}}
|
||||
runcmd:
|
||||
- set -x
|
||||
- timeout 10 apt-mark hold walinuxagent{{GetKubernetesMasterPreprovisionYaml}}
|
||||
- timeout 10 apt-mark unhold walinuxagent
|
||||
{{end}}
|
||||
|
|
@ -924,7 +924,7 @@
|
|||
{{if IsOpenShift}}
|
||||
"script": "{{ Base64 OpenShiftGetMasterSh }}"
|
||||
{{else}}
|
||||
"commandToExecute": "[concat('for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' MASTER_INDEX=',copyIndex(variables('masterOffset')),' /usr/bin/nohup /bin/bash -c \"stat /opt/azure/containers/provision.complete > /dev/null 2>&1 || /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]"
|
||||
"commandToExecute": "[concat('for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"stat /opt/azure/containers/provision.complete > /dev/null 2>&1 || /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]"
|
||||
{{end}}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,468 @@
|
|||
{{if and UseManagedIdentity (not UserAssignedIDEnabled)}}
|
||||
{
|
||||
"apiVersion": "2014-10-01-preview",
|
||||
"name": "[guid(concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmidentity'))]",
|
||||
"type": "Microsoft.Authorization/roleAssignments",
|
||||
"properties": {
|
||||
"roleDefinitionId": "[variables('contributorRoleDefinitionId')]",
|
||||
"principalId": "[reference(concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmss'), '2017-03-30', 'Full').identity.principalId]"
|
||||
}
|
||||
},
|
||||
{{end}}
|
||||
{{if EnableEncryptionWithExternalKms}}
|
||||
{
|
||||
"type": "Microsoft.Storage/storageAccounts",
|
||||
"name": "[variables('clusterKeyVaultName')]",
|
||||
"apiVersion": "[variables('apiVersionStorage')]",
|
||||
"location": "[variables('location')]",
|
||||
"properties": {
|
||||
"accountType": "Standard_LRS"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.KeyVault/vaults",
|
||||
"name": "[variables('clusterKeyVaultName')]",
|
||||
"apiVersion": "[variables('apiVersionKeyVault')]",
|
||||
"location": "[variables('location')]",
|
||||
{{ if UseManagedIdentity}}
|
||||
"dependsOn":
|
||||
[
|
||||
"[concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmss')]",
|
||||
"[concat('Microsoft.Authorization/roleAssignments/', guid(concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmidentity')))]",
|
||||
],
|
||||
{{end}}
|
||||
"properties": {
|
||||
"enabledForDeployment": "false",
|
||||
"enabledForDiskEncryption": "false",
|
||||
"enabledForTemplateDeployment": "false",
|
||||
"tenantId": "[variables('tenantID')]",
|
||||
{{if not UseManagedIdentity}}
|
||||
"accessPolicies":
|
||||
[
|
||||
{
|
||||
"tenantId": "[variables('tenantID')]",
|
||||
"objectId": "[parameters('servicePrincipalObjectId')]",
|
||||
"permissions": {
|
||||
"keys": ["create", "encrypt", "decrypt", "get", "list"]
|
||||
}
|
||||
}
|
||||
],
|
||||
{{else}}
|
||||
"accessPolicies":
|
||||
[
|
||||
{
|
||||
"objectId": "[reference(concat('Microsoft.Compute/virtualMachineScaleSets/', variables('masterVMNamePrefix'), 'vmss'), '2017-03-30', 'Full').identity.principalId]",
|
||||
"permissions": {
|
||||
"keys": [
|
||||
"create",
|
||||
"encrypt",
|
||||
"decrypt",
|
||||
"get",
|
||||
"list"
|
||||
]
|
||||
},
|
||||
"tenantId": "[variables('tenantID')]"
|
||||
},
|
||||
],
|
||||
{{end}}
|
||||
"sku": {
|
||||
"name": "[parameters('clusterKeyVaultSku')]",
|
||||
"family": "A"
|
||||
}
|
||||
}
|
||||
},
|
||||
{{end}}
|
||||
{
|
||||
"apiVersion": "[variables('apiVersionDefault')]",
|
||||
"location": "[variables('location')]",
|
||||
"name": "[variables('nsgName')]",
|
||||
"properties": {
|
||||
"securityRules": [
|
||||
{{if .HasWindows}}
|
||||
{
|
||||
"name": "allow_rdp",
|
||||
"properties": {
|
||||
"access": "Allow",
|
||||
"description": "Allow RDP traffic to master",
|
||||
"destinationAddressPrefix": "*",
|
||||
"destinationPortRange": "3389-3389",
|
||||
"direction": "Inbound",
|
||||
"priority": 102,
|
||||
"protocol": "Tcp",
|
||||
"sourceAddressPrefix": "*",
|
||||
"sourcePortRange": "*"
|
||||
}
|
||||
},
|
||||
{{end}}
|
||||
{
|
||||
"name": "allow_ssh",
|
||||
"properties": {
|
||||
"access": "Allow",
|
||||
"description": "Allow SSH traffic to master",
|
||||
"destinationAddressPrefix": "*",
|
||||
"destinationPortRange": "22-22",
|
||||
"direction": "Inbound",
|
||||
"priority": 101,
|
||||
"protocol": "Tcp",
|
||||
"sourceAddressPrefix": "*",
|
||||
"sourcePortRange": "*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "allow_kube_tls",
|
||||
"properties": {
|
||||
"access": "Allow",
|
||||
"description": "Allow kube-apiserver (tls) traffic to master",
|
||||
"destinationAddressPrefix": "*",
|
||||
"destinationPortRange":"443-443",
|
||||
"direction": "Inbound",
|
||||
"priority": 100,
|
||||
"protocol": "Tcp",
|
||||
"sourceAddressPrefix": "*",
|
||||
"sourcePortRange": "*"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"type": "Microsoft.Network/networkSecurityGroups"
|
||||
},
|
||||
{{if RequireRouteTable}}
|
||||
{
|
||||
"apiVersion": "[variables('apiVersionDefault')]",
|
||||
"location": "[variables('location')]",
|
||||
"name": "[variables('routeTableName')]",
|
||||
"type": "Microsoft.Network/routeTables"
|
||||
},
|
||||
{{end}}
|
||||
{{if not .MasterProfile.IsCustomVNET}}
|
||||
{
|
||||
"apiVersion": "[variables('apiVersionDefault')]",
|
||||
"dependsOn": [
|
||||
{{if RequireRouteTable}}
|
||||
"[concat('Microsoft.Network/routeTables/', variables('routeTableName'))]",
|
||||
{{end}}
|
||||
"[concat('Microsoft.Network/networkSecurityGroups/', variables('nsgName'))]"
|
||||
],
|
||||
"location": "[variables('location')]",
|
||||
"name": "[variables('virtualNetworkName')]",
|
||||
"properties": {
|
||||
"addressSpace": {
|
||||
"addressPrefixes": [
|
||||
"[parameters('vnetCidr')]"
|
||||
]
|
||||
},
|
||||
"subnets": [
|
||||
{
|
||||
"name": "subnetmaster",
|
||||
"properties": {
|
||||
"addressPrefix": "[parameters('masterSubnet')]"
|
||||
,"networkSecurityGroup": {
|
||||
"id": "[variables('nsgID')]"
|
||||
}
|
||||
{{if RequireRouteTable}}
|
||||
,"routeTable": {
|
||||
"id": "[variables('routeTableID')]"
|
||||
}
|
||||
{{end}}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name":"subnetagent",
|
||||
"properties":{
|
||||
"addressPrefix": "[parameters('agentSubnet')]",
|
||||
"networkSecurityGroup": {
|
||||
"id": "[variables('nsgID')]"
|
||||
}
|
||||
{{if RequireRouteTable}}
|
||||
,"routeTable": {
|
||||
"id": "[variables('routeTableID')]"
|
||||
}
|
||||
{{end}}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"type": "Microsoft.Network/virtualNetworks"
|
||||
},
|
||||
{{end}}
|
||||
{
|
||||
"apiVersion": "[variables('apiVersionDefault')]",
|
||||
"location": "[variables('location')]",
|
||||
"name": "[variables('masterPublicIPAddressName')]",
|
||||
"properties": {
|
||||
"dnsSettings": {
|
||||
"domainNameLabel": "[variables('masterFqdnPrefix')]"
|
||||
},
|
||||
"publicIPAllocationMethod": "Dynamic"
|
||||
},
|
||||
"type": "Microsoft.Network/publicIPAddresses"
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Network/loadBalancers",
|
||||
"name": "[variables('masterLbName')]",
|
||||
"location": "[variables('location')]",
|
||||
"apiVersion": "[variables('apiVersionDefault')]",
|
||||
"dependsOn": [
|
||||
"[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]"
|
||||
],
|
||||
"properties": {
|
||||
"frontendIPConfigurations": [
|
||||
{
|
||||
"name": "[variables('masterLbIPConfigName')]",
|
||||
"properties": {
|
||||
"publicIPAddress": {
|
||||
"id": "[resourceId('Microsoft.Network/publicIpAddresses', variables('masterPublicIPAddressName'))]"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"backendAddressPools": [
|
||||
{
|
||||
"name": "[variables('masterLbBackendPoolName')]"
|
||||
}
|
||||
],
|
||||
"probes": [
|
||||
{
|
||||
"name": "tcpHTTPSProbe",
|
||||
"properties": {
|
||||
"protocol": "tcp",
|
||||
"port": 443,
|
||||
"intervalInSeconds": 5,
|
||||
"numberOfProbes": 2
|
||||
}
|
||||
}
|
||||
],
|
||||
"inboundNatPools": [
|
||||
{
|
||||
"name": "[concat('SSH-', variables('masterVMNamePrefix'), 'natpools')]",
|
||||
"properties": {
|
||||
"frontendIPConfiguration": {
|
||||
"id": "[variables('masterLbIPConfigID')]"
|
||||
},
|
||||
"protocol": "tcp",
|
||||
"backendPort": "22",
|
||||
"frontendPortRangeStart": "50001",
|
||||
"frontendPortRangeEnd": "50119",
|
||||
"enableFloatingIP": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"loadBalancingRules": [
|
||||
{
|
||||
"name": "LBRuleHTTPS",
|
||||
"properties": {
|
||||
"frontendIPConfiguration": {
|
||||
"id": "[variables('masterLbIPConfigID')]"
|
||||
},
|
||||
"backendAddressPool": {
|
||||
"id": "[concat(variables('masterLbID'), '/backendAddressPools/', variables('masterLbBackendPoolName'))]"
|
||||
},
|
||||
"protocol": "tcp",
|
||||
"frontendPort": 443,
|
||||
"backendPort": 443,
|
||||
"enableFloatingIP": false,
|
||||
"idleTimeoutInMinutes": 5,
|
||||
"loadDistribution": "Default",
|
||||
"probe": {
|
||||
"id": "[concat(variables('masterLbID'),'/probes/tcpHTTPSProbe')]"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "[variables('apiVersionVirtualMachineScaleSets')]",
|
||||
"dependsOn": [
|
||||
{{if .MasterProfile.IsCustomVNET}}
|
||||
"[variables('nsgID')]"
|
||||
{{else}}
|
||||
"[variables('vnetID')]",
|
||||
"[variables('masterLbID')]"
|
||||
{{end}}
|
||||
],
|
||||
"tags":
|
||||
{
|
||||
"creationSource": "[concat(parameters('generatorCode'), '-', variables('masterVMNamePrefix'), 'vmss')]",
|
||||
"resourceNameSuffix": "[parameters('nameSuffix')]",
|
||||
"orchestrator": "[variables('orchestratorNameVersionTag')]",
|
||||
"acsengineVersion" : "[parameters('acsengineVersion')]",
|
||||
"poolName": "master"
|
||||
},
|
||||
"location": "[variables('location')]",
|
||||
"name": "[concat(variables('masterVMNamePrefix'), 'vmss')]",
|
||||
{{if UseManagedIdentity}}
|
||||
{{if UserAssignedIDEnabled}}
|
||||
"identity": {
|
||||
"type": "userAssigned",
|
||||
"identityIds": [
|
||||
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('userAssignedID'))]"
|
||||
]
|
||||
},
|
||||
{{else}}
|
||||
"identity": {
|
||||
"type": "systemAssigned"
|
||||
},
|
||||
{{end}}
|
||||
{{end}}
|
||||
"sku": {
|
||||
"tier": "Standard",
|
||||
"capacity": "[variables('masterCount')]",
|
||||
"name": "[parameters('masterVMSize')]"
|
||||
},
|
||||
"properties": {
|
||||
"overprovision": false,
|
||||
"upgradePolicy": {
|
||||
"mode": "Manual"
|
||||
},
|
||||
"virtualMachineProfile": {
|
||||
"networkProfile": {
|
||||
"networkInterfaceConfigurations": [
|
||||
{
|
||||
"name": "[concat(variables('masterVMNamePrefix'), 'netintconfig')]",
|
||||
"properties": {
|
||||
"primary": true,
|
||||
{{if .MasterProfile.IsCustomVNET}}
|
||||
"networkSecurityGroup": {
|
||||
"id": "[variables('nsgID')]"
|
||||
},
|
||||
{{end}}
|
||||
"ipConfigurations": [
|
||||
{{range $seq := loop 1 .MasterProfile.IPAddressCount}}
|
||||
{
|
||||
"name": "ipconfig{{$seq}}",
|
||||
"properties": {
|
||||
{{if eq $seq 1}}
|
||||
"loadBalancerBackendAddressPools": [
|
||||
{
|
||||
"id": "[concat(variables('masterLbID'), '/backendAddressPools/', variables('masterLbBackendPoolName'))]"
|
||||
}
|
||||
],
|
||||
"loadBalancerInboundNatPools": [
|
||||
{
|
||||
"id": "[concat(variables('masterLbID'),'/inboundNatPools/SSH-', variables('masterVMNamePrefix'), 'natpools')]"
|
||||
}
|
||||
],
|
||||
"primary": true,
|
||||
{{else}}
|
||||
"primary": false,
|
||||
{{end}}
|
||||
"subnet": {
|
||||
"id": "[variables('vnetSubnetIDMaster')]"
|
||||
}
|
||||
}
|
||||
}
|
||||
{{if lt $seq $.MasterProfile.IPAddressCount}},{{end}}
|
||||
{{end}}
|
||||
]
|
||||
{{if HasCustomNodesDNS}}
|
||||
,"dnsSettings": {
|
||||
"dnsServers": [
|
||||
"[parameters('dnsServer')]"
|
||||
]
|
||||
}
|
||||
{{end}}
|
||||
{{if not IsAzureCNI}}
|
||||
,"enableIPForwarding": true
|
||||
{{end}}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"osProfile": {
|
||||
"adminUsername": "[parameters('linuxAdminUsername')]",
|
||||
"computerNamePrefix": "[concat(variables('masterVMNamePrefix'), 'vmss')]",
|
||||
{{GetKubernetesMasterCustomDataVMSS .}}
|
||||
"linuxConfiguration": {
|
||||
"disablePasswordAuthentication": "true",
|
||||
"ssh": {
|
||||
"publicKeys": [
|
||||
{
|
||||
"keyData": "[parameters('sshRSAPublicKey')]",
|
||||
"path": "[variables('sshKeyPath')]"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
{{if .LinuxProfile.HasSecrets}}
|
||||
,
|
||||
"secrets": "[variables('linuxProfileSecrets')]"
|
||||
{{end}}
|
||||
},
|
||||
"storageProfile": {
|
||||
{{if not UseMasterCustomImage }}
|
||||
"dataDisks": [
|
||||
{
|
||||
"createOption": "Empty",
|
||||
"diskSizeGB": "[parameters('etcdDiskSizeGB')]",
|
||||
"lun": 0
|
||||
}
|
||||
],
|
||||
{{end}}
|
||||
"imageReference": {
|
||||
{{if UseMasterCustomImage}}
|
||||
"id": "[resourceId(parameters('osImageResourceGroup'), 'Microsoft.Compute/images', parameters('osImageName'))]"
|
||||
{{else}}
|
||||
"offer": "[parameters('osImageOffer')]",
|
||||
"publisher": "[parameters('osImagePublisher')]",
|
||||
"sku": "[parameters('osImageSku')]",
|
||||
"version": "[parameters('osImageVersion')]"
|
||||
{{end}}
|
||||
},
|
||||
"osDisk": {
|
||||
"caching": "ReadWrite"
|
||||
,"createOption": "FromImage"
|
||||
{{if ne .MasterProfile.OSDiskSizeGB 0}}
|
||||
,"diskSizeGB": {{.MasterProfile.OSDiskSizeGB}}
|
||||
{{end}}
|
||||
}
|
||||
},
|
||||
"extensionProfile": {
|
||||
"extensions": [
|
||||
{{if UseManagedIdentity}}
|
||||
{
|
||||
"name": "[concat(variables('masterVMNamePrefix'), 'vmss-ManagedIdentityExtension')]",
|
||||
"properties": {
|
||||
"publisher": "Microsoft.ManagedIdentity",
|
||||
"type": "ManagedIdentityExtensionForLinux",
|
||||
"typeHandlerVersion": "1.0",
|
||||
"autoUpgradeMinorVersion": true,
|
||||
"settings": {
|
||||
"port": 50343
|
||||
},
|
||||
"protectedSettings": {}
|
||||
}
|
||||
},
|
||||
{{end}}
|
||||
{
|
||||
"name": "[concat(variables('masterVMNamePrefix'), 'vmssCSE')]",
|
||||
"properties": {
|
||||
"publisher": "Microsoft.Azure.Extensions",
|
||||
"type": "CustomScript",
|
||||
"typeHandlerVersion": "2.0",
|
||||
"autoUpgradeMinorVersion": true,
|
||||
"settings": {},
|
||||
"protectedSettings": {
|
||||
"commandToExecute": "[concat('for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"stat /opt/azure/containers/provision.complete > /dev/null 2>&1 || /bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]"
|
||||
}
|
||||
}
|
||||
}
|
||||
{{if UseAksExtension}}
|
||||
,{
|
||||
"name": "[concat(variables('masterVMNamePrefix'), 'vmss-computeAksLinuxBilling')]",
|
||||
"location": "[variables('location')]",
|
||||
"properties": {
|
||||
"publisher": "Microsoft.AKS",
|
||||
"type": "Compute.AKS-Engine.Linux.Billing",
|
||||
"typeHandlerVersion": "1.0",
|
||||
"autoUpgradeMinorVersion": true,
|
||||
"settings": {}
|
||||
}
|
||||
}
|
||||
{{end}}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"type": "Microsoft.Compute/virtualMachineScaleSets"
|
||||
}
|
||||
|
|
@ -0,0 +1,286 @@
|
|||
"maxVMsPerPool": 100,
|
||||
{{ if not IsHostedMaster }}
|
||||
{{if eq .MasterProfile.Count 1}}
|
||||
"etcdPeerPrivateKeys": [
|
||||
"[parameters('etcdPeerPrivateKey0')]"
|
||||
],
|
||||
"etcdPeerCertificates": [
|
||||
"[parameters('etcdPeerCertificate0')]"
|
||||
],
|
||||
{{end}}
|
||||
{{if eq .MasterProfile.Count 3}}
|
||||
"etcdPeerPrivateKeys": [
|
||||
"[parameters('etcdPeerPrivateKey0')]",
|
||||
"[parameters('etcdPeerPrivateKey1')]",
|
||||
"[parameters('etcdPeerPrivateKey2')]"
|
||||
],
|
||||
"etcdPeerCertificates": [
|
||||
"[parameters('etcdPeerCertificate0')]",
|
||||
"[parameters('etcdPeerCertificate1')]",
|
||||
"[parameters('etcdPeerCertificate2')]"
|
||||
],
|
||||
{{end}}
|
||||
{{if eq .MasterProfile.Count 5}}
|
||||
"etcdPeerPrivateKeys": [
|
||||
"[parameters('etcdPeerPrivateKey0')]",
|
||||
"[parameters('etcdPeerPrivateKey1')]",
|
||||
"[parameters('etcdPeerPrivateKey2')]",
|
||||
"[parameters('etcdPeerPrivateKey3')]",
|
||||
"[parameters('etcdPeerPrivateKey4')]"
|
||||
],
|
||||
"etcdPeerCertificates": [
|
||||
"[parameters('etcdPeerCertificate0')]",
|
||||
"[parameters('etcdPeerCertificate1')]",
|
||||
"[parameters('etcdPeerCertificate2')]",
|
||||
"[parameters('etcdPeerCertificate3')]",
|
||||
"[parameters('etcdPeerCertificate4')]"
|
||||
],
|
||||
{{end}}
|
||||
"etcdPeerCertFilepath":[
|
||||
"/etc/kubernetes/certs/etcdpeer0.crt",
|
||||
"/etc/kubernetes/certs/etcdpeer1.crt",
|
||||
"/etc/kubernetes/certs/etcdpeer2.crt",
|
||||
"/etc/kubernetes/certs/etcdpeer3.crt",
|
||||
"/etc/kubernetes/certs/etcdpeer4.crt"
|
||||
],
|
||||
"etcdPeerKeyFilepath":[
|
||||
"/etc/kubernetes/certs/etcdpeer0.key",
|
||||
"/etc/kubernetes/certs/etcdpeer1.key",
|
||||
"/etc/kubernetes/certs/etcdpeer2.key",
|
||||
"/etc/kubernetes/certs/etcdpeer3.key",
|
||||
"/etc/kubernetes/certs/etcdpeer4.key"
|
||||
],
|
||||
"etcdCaFilepath": "/etc/kubernetes/certs/ca.crt",
|
||||
"etcdClientCertFilepath": "/etc/kubernetes/certs/etcdclient.crt",
|
||||
"etcdClientKeyFilepath": "/etc/kubernetes/certs/etcdclient.key",
|
||||
"etcdServerCertFilepath": "/etc/kubernetes/certs/etcdserver.crt",
|
||||
"etcdServerKeyFilepath": "/etc/kubernetes/certs/etcdserver.key",
|
||||
{{end}}
|
||||
"useManagedIdentityExtension": "{{ UseManagedIdentity }}",
|
||||
"userAssignedID": "{{UserAssignedID}}",
|
||||
"userAssignedClientID": "{{UserAssignedClientID}}",
|
||||
"useInstanceMetadata": "{{ UseInstanceMetadata }}",
|
||||
"loadBalancerSku": "{{ LoadBalancerSku }}",
|
||||
"excludeMasterFromStandardLB": "{{ ExcludeMasterFromStandardLB }}",
|
||||
{{ if UseManagedIdentity }}
|
||||
"servicePrincipalClientId": "msi",
|
||||
"servicePrincipalClientSecret": "msi",
|
||||
{{ else }}
|
||||
"servicePrincipalClientId": "[parameters('servicePrincipalClientId')]",
|
||||
"servicePrincipalClientSecret": "[parameters('servicePrincipalClientSecret')]",
|
||||
{{ end }}
|
||||
"masterFqdnPrefix": "[tolower(parameters('masterEndpointDNSNamePrefix'))]",
|
||||
{{if not IsHostedMaster}}
|
||||
"masterCount": {{.MasterProfile.Count}},
|
||||
"masterOffset": "",
|
||||
"masterIpAddressCount": {{.MasterProfile.IPAddressCount}},
|
||||
{{end}}
|
||||
"apiVersionDefault": "2016-03-30",
|
||||
"apiVersionLinkDefault": "2015-01-01",
|
||||
"locations": [
|
||||
"[resourceGroup().location]",
|
||||
"[parameters('location')]"
|
||||
],
|
||||
"location": "[variables('locations')[mod(add(2,length(parameters('location'))),add(1,length(parameters('location'))))]]",
|
||||
"resourceGroup": "[resourceGroup().name]",
|
||||
"truncatedResourceGroup": "[take(replace(replace(resourceGroup().name, '(', '-'), ')', '-'), 63)]",
|
||||
"labelResourceGroup": "[if(or(or(endsWith(variables('truncatedResourceGroup'), '-'), endsWith(variables('truncatedResourceGroup'), '_')), endsWith(variables('truncatedResourceGroup'), '.')), concat(take(variables('truncatedResourceGroup'), 62), 'z'), variables('truncatedResourceGroup'))]",
|
||||
{{if not IsHostedMaster}}
|
||||
"routeTableName": "[concat(variables('masterVMNamePrefix'),'routetable')]",
|
||||
{{else}}
|
||||
"routeTableName": "[concat(variables('agentNamePrefix'), 'routetable')]",
|
||||
{{end}}
|
||||
"routeTableID": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]",
|
||||
"sshNatPorts": [22,2201,2202,2203,2204],
|
||||
"sshKeyPath": "[concat('/home/',parameters('linuxAdminUsername'),'/.ssh/authorized_keys')]",
|
||||
|
||||
{{if .HasStorageAccountDisks}}
|
||||
"apiVersionStorage": "2015-06-15",
|
||||
"maxVMsPerStorageAccount": 20,
|
||||
"maxStorageAccountsPerAgent": "[div(variables('maxVMsPerPool'),variables('maxVMsPerStorageAccount'))]",
|
||||
"dataStorageAccountPrefixSeed": 97,
|
||||
"storageAccountPrefixes": [ "0", "6", "c", "i", "o", "u", "1", "7", "d", "j", "p", "v", "2", "8", "e", "k", "q", "w", "3", "9", "f", "l", "r", "x", "4", "a", "g", "m", "s", "y", "5", "b", "h", "n", "t", "z" ],
|
||||
"storageAccountPrefixesCount": "[length(variables('storageAccountPrefixes'))]",
|
||||
"vmsPerStorageAccount": 20,
|
||||
"storageAccountBaseName": "[uniqueString(concat(variables('masterFqdnPrefix'),variables('location')))]",
|
||||
{{GetSizeMap}},
|
||||
{{else}}
|
||||
"storageAccountPrefixes": [],
|
||||
"storageAccountBaseName": "",
|
||||
{{end}}
|
||||
{{if UserAssignedIDEnabled}}
|
||||
"apiVersionUserMSI": "2018-06-01",
|
||||
{{end}}
|
||||
{{if .HasManagedDisks}}
|
||||
"apiVersionStorageManagedDisks": "2016-04-30-preview",
|
||||
{{end}}
|
||||
{{if .HasVirtualMachineScaleSets}}
|
||||
"apiVersionVirtualMachineScaleSets": "2017-12-01",
|
||||
{{end}}
|
||||
{{if not IsHostedMaster}}
|
||||
{{if .MasterProfile.IsStorageAccount}}
|
||||
"masterStorageAccountName": "[concat(variables('storageAccountBaseName'), 'mstr0')]",
|
||||
{{end}}
|
||||
{{end}}
|
||||
"provisionScript": "{{GetKubernetesB64Provision}}",
|
||||
"provisionSource": "{{GetKubernetesB64ProvisionSource}}",
|
||||
"healthMonitorScript": "{{GetKubernetesB64HealthMonitorScript}}",
|
||||
"provisionInstalls": "{{GetKubernetesB64Installs}}",
|
||||
"provisionConfigs": "{{GetKubernetesB64Configs}}",
|
||||
"mountetcdScript": "{{GetKubernetesB64Mountetcd}}",
|
||||
"customSearchDomainsScript": "{{GetKubernetesB64CustomSearchDomainsScript}}",
|
||||
"sshdConfig": "{{GetB64sshdConfig}}",
|
||||
"provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]",
|
||||
{{if not IsHostedMaster}}
|
||||
"provisionScriptParametersMaster": "[concat('MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]",
|
||||
{{end}}
|
||||
"generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}",
|
||||
"orchestratorNameVersionTag": "{{.OrchestratorProfile.OrchestratorType}}:{{.OrchestratorProfile.OrchestratorVersion}}",
|
||||
|
||||
{{if IsAzureCNI}}
|
||||
"allocateNodeCidrs": false,
|
||||
{{else}}
|
||||
"allocateNodeCidrs": true,
|
||||
{{end}}
|
||||
{{if not IsHostedMaster}}
|
||||
{{if .MasterProfile.IsCustomVNET}}
|
||||
"vnetSubnetID": "[parameters('agentVnetSubnetID')]",
|
||||
"vnetSubnetIDMaster": "[parameters('masterVnetSubnetID')]",
|
||||
"subnetNameResourceSegmentIndex": 10,
|
||||
"subnetName": "[split(parameters('masterVnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]",
|
||||
"vnetNameResourceSegmentIndex": 8,
|
||||
"virtualNetworkName": "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]",
|
||||
"vnetResourceGroupNameResourceSegmentIndex": 4,
|
||||
"virtualNetworkResourceGroupName": "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]",
|
||||
{{else}}
|
||||
"subnetName": "subnetmaster",
|
||||
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]",
|
||||
"vnetSubnetID": "[concat(variables('vnetID'),'/subnets/subnetagent')]",
|
||||
"vnetSubnetIDMaster": "[concat(variables('vnetID'),'/subnets/subnetmaster')]",
|
||||
"virtualNetworkName": "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]",
|
||||
"virtualNetworkResourceGroupName": "''",
|
||||
{{end}}
|
||||
{{else}}
|
||||
{{if IsCustomVNET}}
|
||||
"vnetSubnetID": "[parameters('{{ (index .AgentPoolProfiles 0).Name }}VnetSubnetID')]",
|
||||
"subnetNameResourceSegmentIndex": 10,
|
||||
"subnetName": "[split(variables('vnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]",
|
||||
"vnetNameResourceSegmentIndex": 8,
|
||||
"virtualNetworkName": "[split(variables('vnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]",
|
||||
"vnetResourceGroupNameResourceSegmentIndex": 4,
|
||||
"virtualNetworkResourceGroupName": "[split(variables('vnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]",
|
||||
{{else}}
|
||||
"subnetName": "[concat(parameters('orchestratorName'), '-subnet')]",
|
||||
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]",
|
||||
"vnetSubnetID": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]",
|
||||
"virtualNetworkName": "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]",
|
||||
"virtualNetworkResourceGroupName": "",
|
||||
{{end}}
|
||||
{{end}}
|
||||
{{if not IsHostedMaster }}
|
||||
"nsgName": "[concat(variables('masterVMNamePrefix'), 'nsg')]",
|
||||
{{else}}
|
||||
"nsgName": "[concat(variables('agentNamePrefix'), 'nsg')]",
|
||||
{{end}}
|
||||
"nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]",
|
||||
{{if not AnyAgentUsesVirtualMachineScaleSets}}
|
||||
"primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',parameters('nameSuffix'))]",
|
||||
"primaryScaleSetName": "",
|
||||
"vmType": "standard",
|
||||
{{else}}
|
||||
"primaryScaleSetName": "[concat(parameters('orchestratorName'), '-{{ (index .AgentPoolProfiles 0).Name }}-',parameters('nameSuffix'), '-vmss')]",
|
||||
"primaryAvailabilitySetName": "",
|
||||
"vmType": "vmss",
|
||||
{{end}}
|
||||
{{if not IsHostedMaster }}
|
||||
{{if IsPrivateCluster}}
|
||||
"kubeconfigServer": "[concat('https://', variables('kubernetesAPIServerIP'), ':443')]",
|
||||
{{if ProvisionJumpbox}}
|
||||
"jumpboxOSDiskName": "[concat(parameters('jumpboxVMName'), '-osdisk')]",
|
||||
"jumpboxPublicIpAddressName": "[concat(parameters('jumpboxVMName'), '-ip')]",
|
||||
"jumpboxNetworkInterfaceName": "[concat(parameters('jumpboxVMName'), '-nic')]",
|
||||
"jumpboxNetworkSecurityGroupName": "[concat(parameters('jumpboxVMName'), '-nsg')]",
|
||||
"kubeconfig": "{{GetKubeConfig}}",
|
||||
{{if not JumpboxIsManagedDisks}}
|
||||
"jumpboxStorageAccountName": "[concat(variables('storageAccountBaseName'), 'jb')]",
|
||||
{{end}}
|
||||
{{if not .HasStorageAccountDisks}}
|
||||
{{GetSizeMap}},
|
||||
{{end}}
|
||||
{{end}}
|
||||
{{else}}
|
||||
"masterPublicIPAddressName": "[concat(parameters('orchestratorName'), '-master-ip-', variables('masterFqdnPrefix'), '-', parameters('nameSuffix'))]",
|
||||
"masterLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterLbName'))]",
|
||||
"masterLbIPConfigID": "[concat(variables('masterLbID'),'/frontendIPConfigurations/', variables('masterLbIPConfigName'))]",
|
||||
"masterLbIPConfigName": "[concat(parameters('orchestratorName'), '-master-lbFrontEnd-', parameters('nameSuffix'))]",
|
||||
"masterLbName": "[concat(parameters('orchestratorName'), '-master-lb-', parameters('nameSuffix'))]",
|
||||
"kubeconfigServer": "[concat('https://', variables('masterFqdnPrefix'), '.', variables('location'), '.', parameters('fqdnEndpointSuffix'))]",
|
||||
{{end}}
|
||||
{{if gt .MasterProfile.Count 1}}
|
||||
"masterInternalLbName": "[concat(parameters('orchestratorName'), '-master-internal-lb-', parameters('nameSuffix'))]",
|
||||
"masterInternalLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterInternalLbName'))]",
|
||||
"masterInternalLbIPConfigName": "[concat(parameters('orchestratorName'), '-master-internal-lbFrontEnd-', parameters('nameSuffix'))]",
|
||||
"masterInternalLbIPConfigID": "[concat(variables('masterInternalLbID'),'/frontendIPConfigurations/', variables('masterInternalLbIPConfigName'))]",
|
||||
"masterInternalLbIPOffset": {{GetDefaultInternalLbStaticIPOffset}},
|
||||
"kubernetesAPIServerIP": "[parameters('firstConsecutiveStaticIP')]",
|
||||
{{else}}
|
||||
"kubernetesAPIServerIP": "[parameters('firstConsecutiveStaticIP')]",
|
||||
{{end}}
|
||||
"masterLbBackendPoolName": "[concat(parameters('orchestratorName'), '-master-pool-', parameters('nameSuffix'))]",
|
||||
"masterFirstAddrComment": "these MasterFirstAddrComment are used to place multiple masters consecutively in the address space",
|
||||
"masterFirstAddrOctets": "[split(parameters('firstConsecutiveStaticIP'),'.')]",
|
||||
"masterFirstAddrOctet4": "[variables('masterFirstAddrOctets')[3]]",
|
||||
"masterFirstAddrPrefix": "[concat(variables('masterFirstAddrOctets')[0],'.',variables('masterFirstAddrOctets')[1],'.',variables('masterFirstAddrOctets')[2],'.')]",
|
||||
"masterVMNamePrefix": "[concat(parameters('orchestratorName'), '-master-', parameters('nameSuffix'), '-')]",
|
||||
"masterVMNames": [
|
||||
"[concat(variables('masterVMNamePrefix'), 'vmss000000')]",
|
||||
"[concat(variables('masterVMNamePrefix'), 'vmss000001')]",
|
||||
"[concat(variables('masterVMNamePrefix'), 'vmss000002')]",
|
||||
"[concat(variables('masterVMNamePrefix'), 'vmss000003')]",
|
||||
"[concat(variables('masterVMNamePrefix'), 'vmss000004')]"
|
||||
],
|
||||
"masterEtcdServerPort": {{GetMasterEtcdServerPort}},
|
||||
"masterEtcdClientPort": {{GetMasterEtcdClientPort}},
|
||||
{{else}}
|
||||
"kubernetesAPIServerIP": "[parameters('kubernetesEndpoint')]",
|
||||
"agentNamePrefix": "[concat(parameters('orchestratorName'), '-agentpool-', parameters('nameSuffix'), '-')]",
|
||||
{{end}}
|
||||
"subscriptionId": "[subscription().subscriptionId]",
|
||||
"contributorRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]",
|
||||
"readerRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]",
|
||||
"scope": "[resourceGroup().id]",
|
||||
"tenantId": "[subscription().tenantId]",
|
||||
"singleQuote": "'"
|
||||
{{if .LinuxProfile.HasSecrets}}
|
||||
, "linuxProfileSecrets" :
|
||||
[
|
||||
{{range $vIndex, $vault := .LinuxProfile.Secrets}}
|
||||
{{if $vIndex}} , {{end}}
|
||||
{
|
||||
"sourceVault":{
|
||||
"id":"[parameters('linuxKeyVaultID{{$vIndex}}')]"
|
||||
},
|
||||
"vaultCertificates":[
|
||||
{{range $cIndex, $cert := $vault.VaultCertificates}}
|
||||
{{if $cIndex}} , {{end}}
|
||||
{
|
||||
"certificateUrl" :"[parameters('linuxKeyVaultID{{$vIndex}}CertificateURL{{$cIndex}}')]"
|
||||
}
|
||||
{{end}}
|
||||
]
|
||||
}
|
||||
{{end}}
|
||||
]
|
||||
{{end}}
|
||||
{{if .HasWindows}}
|
||||
, "windowsCustomScriptSuffix": " $inputFile = '%SYSTEMDRIVE%\\AzureData\\CustomData.bin' ; $outputFile = '%SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.ps1' ; Copy-Item $inputFile $outputFile ; Invoke-Expression('{0} {1}' -f $outputFile, $arguments) ; "
|
||||
{{end}}
|
||||
{{if EnableEncryptionWithExternalKms}}
|
||||
,"apiVersionKeyVault": "2016-10-01",
|
||||
{{if not .HasStorageAccountDisks}}
|
||||
"apiVersionStorage": "2015-06-15",
|
||||
{{end}}
|
||||
"clusterKeyVaultName": "[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]"
|
||||
{{else}}
|
||||
,"clusterKeyVaultName": ""
|
||||
{{end}}
|
||||
|
|
@ -32,6 +32,14 @@
|
|||
},
|
||||
"type": "string"
|
||||
},
|
||||
{{if .MasterProfile.IsVirtualMachineScaleSets}}
|
||||
"agentVnetSubnetID": {
|
||||
"metadata": {
|
||||
"description": "Sets the vnet subnet of the agent."
|
||||
},
|
||||
"type": "string"
|
||||
},
|
||||
{{end}}
|
||||
{{else}}
|
||||
"masterSubnet": {
|
||||
"defaultValue": "{{.MasterProfile.Subnet}}",
|
||||
|
|
@ -40,6 +48,13 @@
|
|||
},
|
||||
"type": "string"
|
||||
},
|
||||
"agentSubnet": {
|
||||
"defaultValue": "{{.MasterProfile.AgentSubnet}}",
|
||||
"metadata": {
|
||||
"description": "Sets the subnet of the agent node(s)."
|
||||
},
|
||||
"type": "string"
|
||||
},
|
||||
{{end}}
|
||||
{{end}}
|
||||
{{if IsHostedMaster}}
|
||||
|
|
|
|||
|
|
@ -20,21 +20,17 @@ const (
|
|||
// DefaultDCOSBootstrapStaticIP specifies the static IP address on bootstrap for a DCOS cluster
|
||||
DefaultDCOSBootstrapStaticIP = "192.168.255.240"
|
||||
// DefaultKubernetesMasterSubnet specifies the default subnet for masters and agents.
|
||||
// Except when master VMSS is used, this specifies the default subnet for masters.
|
||||
DefaultKubernetesMasterSubnet = "10.240.0.0/16"
|
||||
// DefaultKubernetesClusterSubnet specifies the default subnet for pods.
|
||||
DefaultKubernetesClusterSubnet = "10.244.0.0/16"
|
||||
// DefaultDockerBridgeSubnet specifies the default subnet for the docker bridge network for masters and agents.
|
||||
DefaultDockerBridgeSubnet = "172.17.0.1/16"
|
||||
// DefaultFirstConsecutiveKubernetesStaticIP specifies the static IP address on Kubernetes master 0
|
||||
DefaultFirstConsecutiveKubernetesStaticIP = "10.240.255.5"
|
||||
// DefaultAgentSubnetTemplate specifies a default agent subnet
|
||||
DefaultAgentSubnetTemplate = "10.%d.0.0/16"
|
||||
// DefaultKubernetesSubnet specifies the default subnet used for all masters, agents and pods
|
||||
// when VNET integration is enabled.
|
||||
DefaultKubernetesSubnet = "10.240.0.0/12"
|
||||
// DefaultKubernetesFirstConsecutiveStaticIPOffset specifies the IP address offset of master 0
|
||||
// when VNET integration is enabled.
|
||||
DefaultKubernetesFirstConsecutiveStaticIPOffset = 5
|
||||
// DefaultKubernetesMaxPods is the maximum number of pods to run on a node.
|
||||
DefaultKubernetesMaxPods = 110
|
||||
// DefaultKubernetesMaxPodsVNETIntegrated is the maximum number of pods to run on a node when VNET integration is enabled.
|
||||
|
|
@ -196,6 +192,10 @@ const (
|
|||
DefaultJumpboxUsername = "azureuser"
|
||||
// DefaultKubeletPodMaxPIDs specifies the default max pid authorized by pods
|
||||
DefaultKubeletPodMaxPIDs = 100
|
||||
// DefaultKubernetesAgentSubnetVMSS specifies the default subnet for agents when master is VMSS
|
||||
DefaultKubernetesAgentSubnetVMSS = "10.248.0.0/13"
|
||||
// DefaultUserAssignedID specifies the default name for the user assigned identity
|
||||
DefaultUserAssignedID = "acsenginetestid"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -229,6 +229,7 @@ const (
|
|||
|
||||
const (
|
||||
kubernetesMasterCustomDataYaml = "k8s/kubernetesmastercustomdata.yml"
|
||||
kubernetesMasterCustomDataVMSSYaml = "k8s/kubernetesmastercustomdatavmss.yml"
|
||||
kubernetesCustomScript = "k8s/kubernetescustomscript.sh"
|
||||
kubernetesProvisionSourceScript = "k8s/kubernetesprovisionsource.sh"
|
||||
kubernetesHealthMonitorScript = "k8s/health-monitor.sh"
|
||||
|
|
@ -294,8 +295,10 @@ const (
|
|||
kubernetesAgentResourcesVMAS = "k8s/kubernetesagentresourcesvmas.t"
|
||||
kubernetesAgentResourcesVMSS = "k8s/kubernetesagentresourcesvmss.t"
|
||||
kubernetesAgentVars = "k8s/kubernetesagentvars.t"
|
||||
kubernetesMasterResources = "k8s/kubernetesmasterresources.t"
|
||||
kubernetesMasterVars = "k8s/kubernetesmastervars.t"
|
||||
kubernetesMasterResourcesVMAS = "k8s/kubernetesmasterresources.t"
|
||||
kubernetesMasterResourcesVMSS = "k8s/kubernetesmasterresourcesvmss.t"
|
||||
kubernetesMasterVarsVMAS = "k8s/kubernetesmastervars.t"
|
||||
kubernetesMasterVarsVMSS = "k8s/kubernetesmastervarsvmss.t"
|
||||
kubernetesParams = "k8s/kubernetesparams.t"
|
||||
kubernetesWinAgentVars = "k8s/kuberneteswinagentresourcesvmas.t"
|
||||
kubernetesWinAgentVarsVMSS = "k8s/kuberneteswinagentresourcesvmss.t"
|
||||
|
|
|
|||
|
|
@ -316,6 +316,7 @@ func setOrchestratorDefaults(cs *api.ContainerService, isUpdate bool) {
|
|||
if o.KubernetesConfig.ClusterSubnet == "" {
|
||||
if o.IsAzureCNI() {
|
||||
// When Azure CNI is enabled, all masters, agents and pods share the same large subnet.
|
||||
// Except when master is VMSS, then masters and agents have separate subnets within the same large subnet.
|
||||
o.KubernetesConfig.ClusterSubnet = DefaultKubernetesSubnet
|
||||
} else {
|
||||
o.KubernetesConfig.ClusterSubnet = DefaultKubernetesClusterSubnet
|
||||
|
|
@ -496,6 +497,10 @@ func setMasterProfileDefaults(a *api.Properties, isUpgrade bool) {
|
|||
a.MasterProfile.Distro = api.AKS
|
||||
}
|
||||
}
|
||||
// set default to VMAS for now
|
||||
if len(a.MasterProfile.AvailabilityProfile) == 0 {
|
||||
a.MasterProfile.AvailabilityProfile = api.AvailabilitySet
|
||||
}
|
||||
|
||||
if !a.MasterProfile.IsCustomVNET() {
|
||||
if a.OrchestratorProfile.OrchestratorType == api.Kubernetes {
|
||||
|
|
@ -504,13 +509,24 @@ func setMasterProfileDefaults(a *api.Properties, isUpgrade bool) {
|
|||
a.MasterProfile.Subnet = a.OrchestratorProfile.KubernetesConfig.ClusterSubnet
|
||||
// FirstConsecutiveStaticIP is not reset if it is upgrade and some value already exists
|
||||
if !isUpgrade || len(a.MasterProfile.FirstConsecutiveStaticIP) == 0 {
|
||||
a.MasterProfile.FirstConsecutiveStaticIP = getFirstConsecutiveStaticIPAddress(a.MasterProfile.Subnet)
|
||||
if a.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
a.MasterProfile.FirstConsecutiveStaticIP = api.DefaultFirstConsecutiveKubernetesStaticIPVMSS
|
||||
a.MasterProfile.Subnet = DefaultKubernetesMasterSubnet
|
||||
a.MasterProfile.AgentSubnet = DefaultKubernetesAgentSubnetVMSS
|
||||
} else {
|
||||
a.MasterProfile.FirstConsecutiveStaticIP = a.MasterProfile.GetFirstConsecutiveStaticIPAddress(a.MasterProfile.Subnet)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
a.MasterProfile.Subnet = DefaultKubernetesMasterSubnet
|
||||
// FirstConsecutiveStaticIP is not reset if it is upgrade and some value already exists
|
||||
if !isUpgrade || len(a.MasterProfile.FirstConsecutiveStaticIP) == 0 {
|
||||
a.MasterProfile.FirstConsecutiveStaticIP = DefaultFirstConsecutiveKubernetesStaticIP
|
||||
if a.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
a.MasterProfile.FirstConsecutiveStaticIP = api.DefaultFirstConsecutiveKubernetesStaticIPVMSS
|
||||
a.MasterProfile.AgentSubnet = DefaultKubernetesAgentSubnetVMSS
|
||||
} else {
|
||||
a.MasterProfile.FirstConsecutiveStaticIP = api.DefaultFirstConsecutiveKubernetesStaticIP
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if a.OrchestratorProfile.OrchestratorType == api.OpenShift {
|
||||
|
|
@ -544,6 +560,11 @@ func setMasterProfileDefaults(a *api.Properties, isUpgrade bool) {
|
|||
}
|
||||
}
|
||||
|
||||
if a.MasterProfile.IsCustomVNET() && a.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
if a.OrchestratorProfile.OrchestratorType == api.Kubernetes {
|
||||
a.MasterProfile.FirstConsecutiveStaticIP = a.MasterProfile.GetFirstConsecutiveStaticIPAddress(a.MasterProfile.VnetCidr)
|
||||
}
|
||||
}
|
||||
// Set the default number of IP addresses allocated for masters.
|
||||
if a.MasterProfile.IPAddressCount == 0 {
|
||||
// Allocate one IP address for the node.
|
||||
|
|
@ -592,7 +613,9 @@ func setAgentProfileDefaults(a *api.Properties, isUpgrade, isScale bool) {
|
|||
for _, profile := range a.AgentPoolProfiles {
|
||||
if a.OrchestratorProfile.OrchestratorType == api.Kubernetes ||
|
||||
a.OrchestratorProfile.OrchestratorType == api.OpenShift {
|
||||
profile.Subnet = a.MasterProfile.Subnet
|
||||
if !a.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
profile.Subnet = a.MasterProfile.Subnet
|
||||
}
|
||||
} else {
|
||||
profile.Subnet = fmt.Sprintf(DefaultAgentSubnetTemplate, subnetCounter)
|
||||
}
|
||||
|
|
@ -697,16 +720,23 @@ func setDefaultCerts(a *api.Properties) (bool, error) {
|
|||
}
|
||||
|
||||
ips := []net.IP{firstMasterIP}
|
||||
|
||||
// Add the Internal Loadbalancer IP which is always at at a known offset from the firstMasterIP
|
||||
ips = append(ips, net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(DefaultInternalLbStaticIPOffset)})
|
||||
|
||||
// Include the Internal load balancer as well
|
||||
for i := 1; i < a.MasterProfile.Count; i++ {
|
||||
ip := net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(i)}
|
||||
ips = append(ips, ip)
|
||||
}
|
||||
|
||||
if a.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
// Include the Internal load balancer as well
|
||||
for i := 1; i < a.MasterProfile.Count; i++ {
|
||||
offset := i * a.MasterProfile.IPAddressCount
|
||||
ip := net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(offset)}
|
||||
ips = append(ips, ip)
|
||||
}
|
||||
} else {
|
||||
for i := 1; i < a.MasterProfile.Count; i++ {
|
||||
ip := net.IP{firstMasterIP[0], firstMasterIP[1], firstMasterIP[2], firstMasterIP[3] + byte(i)}
|
||||
ips = append(ips, ip)
|
||||
}
|
||||
}
|
||||
if a.CertificateProfile == nil {
|
||||
a.CertificateProfile = &api.CertificateProfile{}
|
||||
}
|
||||
|
|
@ -803,31 +833,6 @@ func certsAlreadyPresent(c *api.CertificateProfile, m int) map[string]bool {
|
|||
return g
|
||||
}
|
||||
|
||||
// getFirstConsecutiveStaticIPAddress returns the first static IP address of the given subnet.
|
||||
func getFirstConsecutiveStaticIPAddress(subnetStr string) string {
|
||||
_, subnet, err := net.ParseCIDR(subnetStr)
|
||||
if err != nil {
|
||||
return DefaultFirstConsecutiveKubernetesStaticIP
|
||||
}
|
||||
|
||||
// Find the first and last octet of the host bits.
|
||||
ones, bits := subnet.Mask.Size()
|
||||
firstOctet := ones / 8
|
||||
lastOctet := bits/8 - 1
|
||||
|
||||
// Set the remaining host bits in the first octet.
|
||||
subnet.IP[firstOctet] |= (1 << byte((8 - (ones % 8)))) - 1
|
||||
|
||||
// Fill the intermediate octets with 1s and last octet with offset. This is done so to match
|
||||
// the existing behavior of allocating static IP addresses from the last /24 of the subnet.
|
||||
for i := firstOctet + 1; i < lastOctet; i++ {
|
||||
subnet.IP[i] = 255
|
||||
}
|
||||
subnet.IP[lastOctet] = DefaultKubernetesFirstConsecutiveStaticIPOffset
|
||||
|
||||
return subnet.IP.String()
|
||||
}
|
||||
|
||||
// combine user-provided --feature-gates vals with defaults
|
||||
// a minimum k8s version may be declared as required for defaults assignment
|
||||
func addDefaultFeatureGates(m map[string]string, version string, minVersion string, defaults string) {
|
||||
|
|
|
|||
|
|
@ -497,6 +497,132 @@ func TestStorageProfile(t *testing.T) {
|
|||
|
||||
}
|
||||
|
||||
// TestMasterProfileDefaults covers tests for setMasterProfileDefaults
|
||||
func TestMasterProfileDefaults(t *testing.T) {
|
||||
// this validates default masterProfile configuration
|
||||
mockCS := getMockBaseContainerService("1.10.3")
|
||||
properties := mockCS.Properties
|
||||
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
|
||||
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
|
||||
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure"
|
||||
properties.MasterProfile.AvailabilityProfile = ""
|
||||
properties.MasterProfile.Count = 3
|
||||
mockCS.Properties = properties
|
||||
setPropertiesDefaults(&mockCS, false, false)
|
||||
if properties.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
t.Fatalf("Master VMAS, AzureCNI: MasterProfile AvailabilityProfile did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.AvailabilityProfile, api.AvailabilitySet)
|
||||
}
|
||||
if properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet != DefaultKubernetesSubnet {
|
||||
t.Fatalf("Master VMAS, AzureCNI: MasterProfile ClusterSubnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet, DefaultKubernetesSubnet)
|
||||
}
|
||||
if properties.MasterProfile.Subnet != properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet {
|
||||
t.Fatalf("Master VMAS, AzureCNI: MasterProfile Subnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.Subnet, properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet)
|
||||
}
|
||||
if properties.AgentPoolProfiles[0].Subnet != properties.MasterProfile.Subnet {
|
||||
t.Fatalf("Master VMAS, AzureCNI: AgentPoolProfiles Subnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.AgentPoolProfiles[0].Subnet, properties.MasterProfile.Subnet)
|
||||
}
|
||||
if properties.MasterProfile.FirstConsecutiveStaticIP != "10.255.255.5" {
|
||||
t.Fatalf("Master VMAS, AzureCNI: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.FirstConsecutiveStaticIP, "10.255.255.5")
|
||||
}
|
||||
|
||||
// this validates default vmss masterProfile configuration
|
||||
mockCS = getMockBaseContainerService("1.10.3")
|
||||
properties = mockCS.Properties
|
||||
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
|
||||
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure"
|
||||
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
|
||||
properties.MasterProfile.AvailabilityProfile = api.VirtualMachineScaleSets
|
||||
setPropertiesDefaults(&mockCS, false, true)
|
||||
if !properties.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
t.Fatalf("Master VMSS, AzureCNI: MasterProfile AvailabilityProfile did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.AvailabilityProfile, api.VirtualMachineScaleSets)
|
||||
}
|
||||
if properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet != DefaultKubernetesSubnet {
|
||||
t.Fatalf("Master VMSS, AzureCNI: MasterProfile ClusterSubnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet, DefaultKubernetesSubnet)
|
||||
}
|
||||
if properties.MasterProfile.FirstConsecutiveStaticIP != api.DefaultFirstConsecutiveKubernetesStaticIPVMSS {
|
||||
t.Fatalf("Master VMSS, AzureCNI: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.FirstConsecutiveStaticIP, api.DefaultFirstConsecutiveKubernetesStaticIPVMSS)
|
||||
}
|
||||
if properties.MasterProfile.Subnet != DefaultKubernetesMasterSubnet {
|
||||
t.Fatalf("Master VMSS, AzureCNI: MasterProfile Subnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.Subnet, DefaultKubernetesMasterSubnet)
|
||||
}
|
||||
if properties.MasterProfile.AgentSubnet != DefaultKubernetesAgentSubnetVMSS {
|
||||
t.Fatalf("Master VMSS, AzureCNI: MasterProfile AgentSubnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.AgentSubnet, DefaultKubernetesAgentSubnetVMSS)
|
||||
}
|
||||
|
||||
// this validates default masterProfile configuration and kubenet
|
||||
mockCS = getMockBaseContainerService("1.10.3")
|
||||
properties = mockCS.Properties
|
||||
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
|
||||
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
|
||||
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "kubenet"
|
||||
properties.MasterProfile.AvailabilityProfile = api.VirtualMachineScaleSets
|
||||
setPropertiesDefaults(&mockCS, false, true)
|
||||
if properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet != DefaultKubernetesClusterSubnet {
|
||||
t.Fatalf("Master VMSS, kubenet: MasterProfile ClusterSubnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet, DefaultKubernetesClusterSubnet)
|
||||
}
|
||||
if properties.MasterProfile.Subnet != DefaultKubernetesMasterSubnet {
|
||||
t.Fatalf("Master VMSS, kubenet: MasterProfile Subnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.Subnet, DefaultKubernetesMasterSubnet)
|
||||
}
|
||||
if properties.MasterProfile.FirstConsecutiveStaticIP != api.DefaultFirstConsecutiveKubernetesStaticIPVMSS {
|
||||
t.Fatalf("Master VMSS, kubenet: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.FirstConsecutiveStaticIP, api.DefaultFirstConsecutiveKubernetesStaticIPVMSS)
|
||||
}
|
||||
if properties.MasterProfile.AgentSubnet != DefaultKubernetesAgentSubnetVMSS {
|
||||
t.Fatalf("Master VMSS, kubenet: MasterProfile AgentSubnet did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.AgentSubnet, DefaultKubernetesAgentSubnetVMSS)
|
||||
}
|
||||
properties.MasterProfile.AvailabilityProfile = api.AvailabilitySet
|
||||
setPropertiesDefaults(&mockCS, false, true)
|
||||
if properties.MasterProfile.FirstConsecutiveStaticIP != api.DefaultFirstConsecutiveKubernetesStaticIP {
|
||||
t.Fatalf("Master VMAS, kubenet: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.FirstConsecutiveStaticIP, api.DefaultFirstConsecutiveKubernetesStaticIP)
|
||||
}
|
||||
|
||||
// this validates default vmas masterProfile configuration, AzureCNI, and custom vnet
|
||||
mockCS = getMockBaseContainerService("1.10.3")
|
||||
properties = mockCS.Properties
|
||||
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
|
||||
properties.MasterProfile.VnetSubnetID = "/subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/Microsoft.Network/virtualNetworks/ExampleCustomVNET/subnets/ExampleMasterSubnet"
|
||||
properties.MasterProfile.VnetCidr = "10.239.0.0/16"
|
||||
properties.MasterProfile.FirstConsecutiveStaticIP = "10.239.255.239"
|
||||
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
|
||||
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure"
|
||||
properties.MasterProfile.AvailabilityProfile = api.AvailabilitySet
|
||||
setPropertiesDefaults(&mockCS, false, true)
|
||||
if properties.MasterProfile.FirstConsecutiveStaticIP != "10.239.255.239" {
|
||||
t.Fatalf("Master VMAS, AzureCNI, customvnet: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.FirstConsecutiveStaticIP, "10.239.255.239")
|
||||
}
|
||||
|
||||
// this validates default vmss masterProfile configuration, AzureCNI, and custom vnet
|
||||
mockCS = getMockBaseContainerService("1.10.3")
|
||||
properties = mockCS.Properties
|
||||
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
|
||||
properties.MasterProfile.VnetSubnetID = "/subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/Microsoft.Network/virtualNetworks/ExampleCustomVNET/subnets/ExampleMasterSubnet"
|
||||
properties.MasterProfile.VnetCidr = "10.239.0.0/16"
|
||||
properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet = ""
|
||||
properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure"
|
||||
properties.MasterProfile.AvailabilityProfile = api.VirtualMachineScaleSets
|
||||
setPropertiesDefaults(&mockCS, false, true)
|
||||
if properties.MasterProfile.FirstConsecutiveStaticIP != "10.239.0.4" {
|
||||
t.Fatalf("Master VMSS, AzureCNI, customvnet: MasterProfile FirstConsecutiveStaticIP did not have the expected default configuration, got %s, expected %s",
|
||||
properties.MasterProfile.FirstConsecutiveStaticIP, "10.239.0.4")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestAgentPoolProfile(t *testing.T) {
|
||||
mockCS := getMockBaseContainerService("1.10")
|
||||
properties := mockCS.Properties
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import (
|
|||
var commonTemplateFiles = []string{agentOutputs, agentParams, masterOutputs, iaasOutputs, masterParams, windowsParams}
|
||||
var dcosTemplateFiles = []string{dcosBaseFile, dcosAgentResourcesVMAS, dcosAgentResourcesVMSS, dcosAgentVars, dcosMasterResources, dcosMasterVars, dcosParams, dcosWindowsAgentResourcesVMAS, dcosWindowsAgentResourcesVMSS}
|
||||
var dcos2TemplateFiles = []string{dcos2BaseFile, dcosAgentResourcesVMAS, dcosAgentResourcesVMSS, dcosAgentVars, dcos2MasterResources, dcos2BootstrapResources, dcos2MasterVars, dcosParams, dcosWindowsAgentResourcesVMAS, dcosWindowsAgentResourcesVMSS, dcos2BootstrapVars, dcos2BootstrapParams}
|
||||
var kubernetesTemplateFiles = []string{kubernetesBaseFile, kubernetesAgentResourcesVMAS, kubernetesAgentResourcesVMSS, kubernetesAgentVars, kubernetesMasterResources, kubernetesMasterVars, kubernetesParams, kubernetesWinAgentVars, kubernetesWinAgentVarsVMSS}
|
||||
var kubernetesTemplateFiles = []string{kubernetesBaseFile, kubernetesAgentResourcesVMAS, kubernetesAgentResourcesVMSS, kubernetesAgentVars, kubernetesMasterResourcesVMAS, kubernetesMasterResourcesVMSS, kubernetesMasterVarsVMAS, kubernetesMasterVarsVMSS, kubernetesParams, kubernetesWinAgentVars, kubernetesWinAgentVarsVMSS}
|
||||
var swarmTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentResourcesVMAS, swarmAgentVars, swarmAgentResourcesVMSS, swarmBaseFile, swarmMasterResources, swarmMasterVars, swarmWinAgentResourcesVMAS, swarmWinAgentResourcesVMSS}
|
||||
var swarmModeTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentResourcesVMAS, swarmAgentVars, swarmAgentResourcesVMSS, swarmBaseFile, swarmMasterResources, swarmMasterVars, swarmWinAgentResourcesVMAS, swarmWinAgentResourcesVMSS}
|
||||
var openshiftTemplateFiles = append(
|
||||
|
|
|
|||
|
|
@ -57,11 +57,15 @@ func getParameters(cs *api.ContainerService, generatorCode string, acsengineVers
|
|||
if properties.MasterProfile != nil {
|
||||
if properties.MasterProfile.IsCustomVNET() {
|
||||
addValue(parametersMap, "masterVnetSubnetID", properties.MasterProfile.VnetSubnetID)
|
||||
if properties.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
addValue(parametersMap, "agentVnetSubnetID", properties.MasterProfile.AgentVnetSubnetID)
|
||||
}
|
||||
if properties.OrchestratorProfile.IsKubernetes() || properties.OrchestratorProfile.IsOpenShift() {
|
||||
addValue(parametersMap, "vnetCidr", properties.MasterProfile.VnetCidr)
|
||||
}
|
||||
} else {
|
||||
addValue(parametersMap, "masterSubnet", properties.MasterProfile.Subnet)
|
||||
addValue(parametersMap, "agentSubnet", properties.MasterProfile.AgentSubnet)
|
||||
}
|
||||
addValue(parametersMap, "firstConsecutiveStaticIP", properties.MasterProfile.FirstConsecutiveStaticIP)
|
||||
addValue(parametersMap, "masterVMSize", properties.MasterProfile.VMSize)
|
||||
|
|
|
|||
|
|
@ -156,9 +156,55 @@ func (t *TemplateGenerator) prepareTemplateFiles(properties *api.Properties) ([]
|
|||
return files, baseFile, nil
|
||||
}
|
||||
|
||||
func (t *TemplateGenerator) getMasterCustomData(cs *api.ContainerService, textFilename string, profile *api.Properties) string {
|
||||
str, e := t.getSingleLineForTemplate(textFilename, cs, profile)
|
||||
if e != nil {
|
||||
panic(e)
|
||||
}
|
||||
|
||||
// add manifests
|
||||
str = substituteConfigString(str,
|
||||
kubernetesManifestSettingsInit(profile),
|
||||
"k8s/manifests",
|
||||
"/etc/kubernetes/manifests",
|
||||
"MASTER_MANIFESTS_CONFIG_PLACEHOLDER",
|
||||
profile.OrchestratorProfile.OrchestratorVersion)
|
||||
|
||||
// add artifacts
|
||||
str = substituteConfigString(str,
|
||||
kubernetesArtifactSettingsInitMaster(profile),
|
||||
"k8s/artifacts",
|
||||
"/etc/systemd/system",
|
||||
"MASTER_ARTIFACTS_CONFIG_PLACEHOLDER",
|
||||
profile.OrchestratorProfile.OrchestratorVersion)
|
||||
|
||||
// add addons
|
||||
str = substituteAddonConfigString(str,
|
||||
kubernetesAddonSettingsInit(profile),
|
||||
"k8s/addons",
|
||||
"/etc/kubernetes/addons",
|
||||
"MASTER_ADDONS_CONFIG_PLACEHOLDER",
|
||||
profile.OrchestratorProfile.OrchestratorVersion)
|
||||
|
||||
// add custom files
|
||||
customFilesReader, err := customfilesIntoReaders(masterCustomFiles(profile))
|
||||
if err != nil {
|
||||
log.Fatalf("Could not read custom files: %s", err.Error())
|
||||
}
|
||||
str = substituteConfigStringCustomFiles(str,
|
||||
customFilesReader,
|
||||
"MASTER_CUSTOM_FILES_PLACEHOLDER")
|
||||
|
||||
// return the custom data
|
||||
return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str)
|
||||
}
|
||||
|
||||
// getTemplateFuncMap returns all functions used in template generation
|
||||
func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) template.FuncMap {
|
||||
return template.FuncMap{
|
||||
"IsMasterVirtualMachineScaleSets": func() bool {
|
||||
return cs.Properties.MasterProfile != nil && cs.Properties.MasterProfile.IsVirtualMachineScaleSets()
|
||||
},
|
||||
"IsHostedMaster": func() bool {
|
||||
return cs.Properties.HostedMasterProfile != nil
|
||||
},
|
||||
|
|
@ -498,46 +544,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
|
|||
return DefaultInternalLbStaticIPOffset
|
||||
},
|
||||
"GetKubernetesMasterCustomData": func(profile *api.Properties) string {
|
||||
str, e := t.getSingleLineForTemplate(kubernetesMasterCustomDataYaml, cs, profile)
|
||||
if e != nil {
|
||||
panic(e)
|
||||
}
|
||||
|
||||
// add manifests
|
||||
str = substituteConfigString(str,
|
||||
kubernetesManifestSettingsInit(profile),
|
||||
"k8s/manifests",
|
||||
"/etc/kubernetes/manifests",
|
||||
"MASTER_MANIFESTS_CONFIG_PLACEHOLDER",
|
||||
profile.OrchestratorProfile.OrchestratorVersion)
|
||||
|
||||
// add artifacts
|
||||
str = substituteConfigString(str,
|
||||
kubernetesArtifactSettingsInitMaster(profile),
|
||||
"k8s/artifacts",
|
||||
"/etc/systemd/system",
|
||||
"MASTER_ARTIFACTS_CONFIG_PLACEHOLDER",
|
||||
profile.OrchestratorProfile.OrchestratorVersion)
|
||||
|
||||
// add addons
|
||||
str = substituteAddonConfigString(str,
|
||||
kubernetesAddonSettingsInit(profile),
|
||||
"k8s/addons",
|
||||
"/etc/kubernetes/addons",
|
||||
"MASTER_ADDONS_CONFIG_PLACEHOLDER",
|
||||
profile.OrchestratorProfile.OrchestratorVersion)
|
||||
|
||||
// add custom files
|
||||
customFilesReader, err := customfilesIntoReaders(masterCustomFiles(profile))
|
||||
if err != nil {
|
||||
log.Fatalf("Could not read custom files: %s", err.Error())
|
||||
}
|
||||
str = substituteConfigStringCustomFiles(str,
|
||||
customFilesReader,
|
||||
"MASTER_CUSTOM_FILES_PLACEHOLDER")
|
||||
|
||||
// return the custom data
|
||||
return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str)
|
||||
str := t.getMasterCustomData(cs, kubernetesMasterCustomDataYaml, profile)
|
||||
return str
|
||||
},
|
||||
"GetKubernetesMasterCustomDataVMSS": func(profile *api.Properties) string {
|
||||
str := t.getMasterCustomData(cs, kubernetesMasterCustomDataVMSSYaml, profile)
|
||||
return str
|
||||
},
|
||||
"GetKubernetesAgentCustomData": func(profile *api.AgentPoolProfile) string {
|
||||
str, e := t.getSingleLineForTemplate(kubernetesAgentCustomDataYaml, cs, profile)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,16 @@ const (
|
|||
const (
|
||||
// AvailabilitySet means that the vms are in an availability set
|
||||
AvailabilitySet = "AvailabilitySet"
|
||||
// DefaultFirstConsecutiveKubernetesStaticIP specifies the static IP address on Kubernetes master 0
|
||||
DefaultFirstConsecutiveKubernetesStaticIP = "10.240.255.5"
|
||||
// DefaultFirstConsecutiveKubernetesStaticIPVMSS specifies the static IP address on Kubernetes master 0 of VMSS
|
||||
DefaultFirstConsecutiveKubernetesStaticIPVMSS = "10.240.0.4"
|
||||
// DefaultKubernetesFirstConsecutiveStaticIPOffset specifies the IP address offset of master 0
|
||||
// when VNET integration is enabled.
|
||||
DefaultKubernetesFirstConsecutiveStaticIPOffset = 5
|
||||
// DefaultKubernetesFirstConsecutiveStaticIPOffsetVMSS specifies the IP address offset of master 0 in VMSS
|
||||
// when VNET integration is enabled.
|
||||
DefaultKubernetesFirstConsecutiveStaticIPOffsetVMSS = 4
|
||||
// VirtualMachineScaleSets means that the vms are in a virtual machine scaleset
|
||||
VirtualMachineScaleSets = "VirtualMachineScaleSets"
|
||||
// ScaleSetPriorityRegular is the default ScaleSet Priority
|
||||
|
|
|
|||
|
|
@ -894,6 +894,7 @@ func convertMasterProfileToVLabs(api *MasterProfile, vlabsProfile *vlabs.MasterP
|
|||
vlabsProfile.VMSize = api.VMSize
|
||||
vlabsProfile.OSDiskSizeGB = api.OSDiskSizeGB
|
||||
vlabsProfile.VnetSubnetID = api.VnetSubnetID
|
||||
vlabsProfile.AgentVnetSubnetID = api.AgentVnetSubnetID
|
||||
vlabsProfile.FirstConsecutiveStaticIP = api.FirstConsecutiveStaticIP
|
||||
vlabsProfile.VnetCidr = api.VnetCidr
|
||||
vlabsProfile.SetSubnet(api.Subnet)
|
||||
|
|
@ -920,7 +921,8 @@ func convertMasterProfileToVLabs(api *MasterProfile, vlabsProfile *vlabs.MasterP
|
|||
vlabsProfile.ImageRef.Name = api.ImageRef.Name
|
||||
vlabsProfile.ImageRef.ResourceGroup = api.ImageRef.ResourceGroup
|
||||
}
|
||||
|
||||
vlabsProfile.AvailabilityProfile = api.AvailabilityProfile
|
||||
vlabsProfile.AgentSubnet = api.AgentSubnet
|
||||
convertCustomFilesToVlabs(api, vlabsProfile)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -893,6 +893,7 @@ func convertVLabsMasterProfile(vlabs *vlabs.MasterProfile, api *MasterProfile) {
|
|||
api.VMSize = vlabs.VMSize
|
||||
api.OSDiskSizeGB = vlabs.OSDiskSizeGB
|
||||
api.VnetSubnetID = vlabs.VnetSubnetID
|
||||
api.AgentVnetSubnetID = vlabs.AgentVnetSubnetID
|
||||
api.FirstConsecutiveStaticIP = vlabs.FirstConsecutiveStaticIP
|
||||
api.VnetCidr = vlabs.VnetCidr
|
||||
api.Subnet = vlabs.GetSubnet()
|
||||
|
|
@ -930,6 +931,8 @@ func convertVLabsMasterProfile(vlabs *vlabs.MasterProfile, api *MasterProfile) {
|
|||
api.ImageRef.ResourceGroup = vlabs.ImageRef.ResourceGroup
|
||||
}
|
||||
|
||||
api.AvailabilityProfile = vlabs.AvailabilityProfile
|
||||
api.AgentSubnet = vlabs.AgentSubnet
|
||||
convertCustomFilesToAPI(vlabs, api)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"net"
|
||||
neturl "net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
|
@ -395,6 +396,7 @@ type MasterProfile struct {
|
|||
OSDiskSizeGB int `json:"osDiskSizeGB,omitempty"`
|
||||
VnetSubnetID string `json:"vnetSubnetID,omitempty"`
|
||||
VnetCidr string `json:"vnetCidr,omitempty"`
|
||||
AgentVnetSubnetID string `json:"agentVnetSubnetID,omitempty"`
|
||||
FirstConsecutiveStaticIP string `json:"firstConsecutiveStaticIP,omitempty"`
|
||||
Subnet string `json:"subnet"`
|
||||
IPAddressCount int `json:"ipAddressCount,omitempty"`
|
||||
|
|
@ -407,6 +409,8 @@ type MasterProfile struct {
|
|||
KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"`
|
||||
ImageRef *ImageReference `json:"imageReference,omitempty"`
|
||||
CustomFiles *[]CustomFile `json:"customFiles,omitempty"`
|
||||
AvailabilityProfile string `json:"availabilityProfile"`
|
||||
AgentSubnet string `json:"agentSubnet,omitempty"`
|
||||
|
||||
// Master LB public endpoint/FQDN with port
|
||||
// The format will be FQDN:2376
|
||||
|
|
@ -736,6 +740,40 @@ func (m *MasterProfile) IsCoreOS() bool {
|
|||
return m.Distro == CoreOS
|
||||
}
|
||||
|
||||
// IsVirtualMachineScaleSets returns true if the master availability profile is VMSS
|
||||
func (m *MasterProfile) IsVirtualMachineScaleSets() bool {
|
||||
return m.AvailabilityProfile == VirtualMachineScaleSets
|
||||
}
|
||||
|
||||
// GetFirstConsecutiveStaticIPAddress returns the first static IP address of the given subnet.
|
||||
func (m *MasterProfile) GetFirstConsecutiveStaticIPAddress(subnetStr string) string {
|
||||
_, subnet, err := net.ParseCIDR(subnetStr)
|
||||
if err != nil {
|
||||
return DefaultFirstConsecutiveKubernetesStaticIP
|
||||
}
|
||||
|
||||
// Find the first and last octet of the host bits.
|
||||
ones, bits := subnet.Mask.Size()
|
||||
firstOctet := ones / 8
|
||||
lastOctet := bits/8 - 1
|
||||
|
||||
if m.IsVirtualMachineScaleSets() {
|
||||
subnet.IP[lastOctet] = DefaultKubernetesFirstConsecutiveStaticIPOffsetVMSS
|
||||
} else {
|
||||
// Set the remaining host bits in the first octet.
|
||||
subnet.IP[firstOctet] |= (1 << byte((8 - (ones % 8)))) - 1
|
||||
|
||||
// Fill the intermediate octets with 1s and last octet with offset. This is done so to match
|
||||
// the existing behavior of allocating static IP addresses from the last /24 of the subnet.
|
||||
for i := firstOctet + 1; i < lastOctet; i++ {
|
||||
subnet.IP[i] = 255
|
||||
}
|
||||
subnet.IP[lastOctet] = DefaultKubernetesFirstConsecutiveStaticIPOffset
|
||||
}
|
||||
|
||||
return subnet.IP.String()
|
||||
}
|
||||
|
||||
// IsCustomVNET returns true if the customer brought their own VNET
|
||||
func (a *AgentPoolProfile) IsCustomVNET() bool {
|
||||
return len(a.VnetSubnetID) > 0
|
||||
|
|
|
|||
|
|
@ -392,7 +392,41 @@ func TestTotalNodes(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
func TestMasterAvailabilityProfile(t *testing.T) {
|
||||
cases := []struct {
|
||||
p Properties
|
||||
expectedISVMSS bool
|
||||
}{
|
||||
{
|
||||
p: Properties{
|
||||
MasterProfile: &MasterProfile{},
|
||||
},
|
||||
expectedISVMSS: false,
|
||||
},
|
||||
{
|
||||
p: Properties{
|
||||
MasterProfile: &MasterProfile{
|
||||
AvailabilityProfile: AvailabilitySet,
|
||||
},
|
||||
},
|
||||
expectedISVMSS: false,
|
||||
},
|
||||
{
|
||||
p: Properties{
|
||||
MasterProfile: &MasterProfile{
|
||||
AvailabilityProfile: VirtualMachineScaleSets,
|
||||
},
|
||||
},
|
||||
expectedISVMSS: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
if c.p.MasterProfile.IsVirtualMachineScaleSets() != c.expectedISVMSS {
|
||||
t.Fatalf("expected MasterProfile.IsVirtualMachineScaleSets() to return %t but instead returned %t", c.expectedISVMSS, c.p.MasterProfile.IsVirtualMachineScaleSets())
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestAvailabilityProfile(t *testing.T) {
|
||||
cases := []struct {
|
||||
p Properties
|
||||
|
|
|
|||
|
|
@ -365,6 +365,7 @@ type MasterProfile struct {
|
|||
OSDiskSizeGB int `json:"osDiskSizeGB,omitempty" validate:"min=0,max=1023"`
|
||||
VnetSubnetID string `json:"vnetSubnetID,omitempty"`
|
||||
VnetCidr string `json:"vnetCidr,omitempty"`
|
||||
AgentVnetSubnetID string `json:"agentVnetSubnetID,omitempty"`
|
||||
FirstConsecutiveStaticIP string `json:"firstConsecutiveStaticIP,omitempty"`
|
||||
IPAddressCount int `json:"ipAddressCount,omitempty" validate:"min=0,max=256"`
|
||||
StorageProfile string `json:"storageProfile,omitempty" validate:"eq=StorageAccount|eq=ManagedDisks|len=0"`
|
||||
|
|
@ -376,6 +377,8 @@ type MasterProfile struct {
|
|||
KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"`
|
||||
ImageRef *ImageReference `json:"imageReference,omitempty"`
|
||||
CustomFiles *[]CustomFile `json:"customFiles,omitempty"`
|
||||
AvailabilityProfile string `json:"availabilityProfile"`
|
||||
AgentSubnet string `json:"agentSubnet,omitempty"`
|
||||
|
||||
// subnet is internal
|
||||
subnet string
|
||||
|
|
@ -547,6 +550,25 @@ func (m *MasterProfile) IsCoreOS() bool {
|
|||
return m.Distro == CoreOS
|
||||
}
|
||||
|
||||
// IsVirtualMachineScaleSets returns true if the master availability profile is VMSS
|
||||
func (m *MasterProfile) IsVirtualMachineScaleSets() bool {
|
||||
return m.AvailabilityProfile == VirtualMachineScaleSets
|
||||
}
|
||||
|
||||
// IsAllVirtualMachineScaleSets returns true if the cluster contains only Virtual Machine Scale Sets
|
||||
func (p *Properties) IsAllVirtualMachineScaleSets() bool {
|
||||
isAll := p.MasterProfile.IsVirtualMachineScaleSets()
|
||||
if isAll {
|
||||
for _, agentPoolProfile := range p.AgentPoolProfiles {
|
||||
if agentPoolProfile.AvailabilityProfile != VirtualMachineScaleSets {
|
||||
isAll = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return isAll
|
||||
}
|
||||
|
||||
// IsCustomVNET returns true if the customer brought their own VNET
|
||||
func (a *AgentPoolProfile) IsCustomVNET() bool {
|
||||
return len(a.VnetSubnetID) > 0
|
||||
|
|
|
|||
|
|
@ -81,6 +81,64 @@ func TestOrchestratorProfile(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestMasterProfile(t *testing.T) {
|
||||
// With osType not specified
|
||||
MasterProfileText := `{"count" : 0, "storageProfile" : "StorageAccount", "vnetSubnetID" : "1234", "agentVnetSubnetID" : "5678"}`
|
||||
mp := &MasterProfile{}
|
||||
if e := json.Unmarshal([]byte(MasterProfileText), mp); e != nil {
|
||||
t.Fatalf("unexpectedly detected unmarshal failure for MasterProfile, %+v", e)
|
||||
}
|
||||
|
||||
if mp.Count != 0 {
|
||||
t.Fatalf("unexpectedly detected MasterProfile.Count != 1 after unmarshal")
|
||||
}
|
||||
|
||||
if !mp.IsCustomVNET() {
|
||||
t.Fatalf("unexpectedly detected nil MasterProfile.VNetSubNetID after unmarshal")
|
||||
}
|
||||
|
||||
if !mp.IsStorageAccount() {
|
||||
t.Fatalf("unexpectedly detected MasterProfile.StorageProfile != ManagedDisks after unmarshal")
|
||||
}
|
||||
|
||||
// With vmas
|
||||
MasterProfileText = `{ "count": 1, "vmSize": "Standard_D2_v2", "storageProfile" : "ManagedDisks", "diskSizesGB" : [750, 250, 600, 1000] }`
|
||||
mp = &MasterProfile{}
|
||||
if e := json.Unmarshal([]byte(MasterProfileText), mp); e != nil {
|
||||
t.Fatalf("unexpectedly detected unmarshal failure for MasterProfile, %+v", e)
|
||||
}
|
||||
|
||||
if mp.Count != 1 {
|
||||
t.Fatalf("unexpectedly detected MasterProfile.Count != 1 after unmarshal")
|
||||
}
|
||||
|
||||
if !mp.IsManagedDisks() {
|
||||
t.Fatalf("unexpectedly detected MasterProfile.StorageProfile != ManagedDisks after unmarshal")
|
||||
}
|
||||
|
||||
if mp.IsVirtualMachineScaleSets() {
|
||||
t.Fatalf("unexpectedly detected MasterProfile.AvailabilitySets == VirtualMachineScaleSets after unmarshal")
|
||||
}
|
||||
|
||||
// With vmss
|
||||
MasterProfileText = `{ "count": 3, "vmSize": "Standard_D2_v2", "availabilityProfile": "VirtualMachineScaleSets", "storageProfile" : "ManagedDisks", "diskSizesGB" : [750, 250, 600, 1000] }`
|
||||
mp = &MasterProfile{}
|
||||
if e := json.Unmarshal([]byte(MasterProfileText), mp); e != nil {
|
||||
t.Fatalf("unexpectedly detected unmarshal failure for MasterProfile, %+v", e)
|
||||
}
|
||||
|
||||
if mp.Count != 3 {
|
||||
t.Fatalf("unexpectedly detected MasterProfile.Count != 3 after unmarshal")
|
||||
}
|
||||
|
||||
if !mp.IsManagedDisks() {
|
||||
t.Fatalf("unexpectedly detected MasterProfile.StorageProfile != ManagedDisks after unmarshal")
|
||||
}
|
||||
|
||||
if !mp.IsVirtualMachineScaleSets() {
|
||||
t.Fatalf("unexpectedly detected MasterProfile.AvailabilitySets != VirtualMachineScaleSets after unmarshal")
|
||||
}
|
||||
}
|
||||
func TestAgentPoolProfile(t *testing.T) {
|
||||
// With osType not specified
|
||||
AgentPoolProfileText := `{"count" : 0, "storageProfile" : "StorageAccount", "vnetSubnetID" : "1234"}`
|
||||
|
|
|
|||
|
|
@ -129,6 +129,10 @@ func (a *Properties) Validate(isUpdate bool) error {
|
|||
return e
|
||||
}
|
||||
|
||||
if e := a.validateManagedIdentity(); e != nil {
|
||||
return e
|
||||
}
|
||||
|
||||
if e := a.validateAADProfile(); e != nil {
|
||||
return e
|
||||
}
|
||||
|
|
@ -354,6 +358,17 @@ func (a *Properties) validateMasterProfile() error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if m.IsVirtualMachineScaleSets() && a.OrchestratorProfile.OrchestratorType == Kubernetes {
|
||||
e := validateVMSS(a.OrchestratorProfile, false, m.StorageProfile)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if !a.IsAllVirtualMachineScaleSets() {
|
||||
return errors.New("VirtualMachineScaleSets for master profile must be used together with virtualMachineScaleSets for agent profiles. Set \"availabilityProfile\" to \"VirtualMachineScaleSets\" for agent profiles")
|
||||
}
|
||||
}
|
||||
|
||||
return common.ValidateDNSPrefix(m.DNSPrefix)
|
||||
}
|
||||
|
||||
|
|
@ -406,8 +421,11 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error {
|
|||
return e
|
||||
}
|
||||
|
||||
if e := agentPoolProfile.validateVMSS(a.OrchestratorProfile, isUpdate); agentPoolProfile.AvailabilityProfile == VirtualMachineScaleSets && e != nil {
|
||||
return e
|
||||
if agentPoolProfile.AvailabilityProfile == VirtualMachineScaleSets {
|
||||
e := validateVMSS(a.OrchestratorProfile, isUpdate, agentPoolProfile.StorageProfile)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
}
|
||||
|
||||
if a.OrchestratorProfile.OrchestratorType == Kubernetes {
|
||||
|
|
@ -550,6 +568,10 @@ func (a *Properties) validateVNET() error {
|
|||
}
|
||||
}
|
||||
if isCustomVNET {
|
||||
if a.MasterProfile.IsVirtualMachineScaleSets() && a.MasterProfile.AgentVnetSubnetID == "" {
|
||||
return errors.New("when master profile is using VirtualMachineScaleSets and is custom vnet, set \"vnetsubnetid\" and \"agentVnetSubnetID\" for master profile")
|
||||
}
|
||||
|
||||
subscription, resourcegroup, vnetname, _, e := common.GetVNETSubnetIDComponents(a.MasterProfile.VnetSubnetID)
|
||||
if e != nil {
|
||||
return e
|
||||
|
|
@ -568,7 +590,7 @@ func (a *Properties) validateVNET() error {
|
|||
}
|
||||
|
||||
masterFirstIP := net.ParseIP(a.MasterProfile.FirstConsecutiveStaticIP)
|
||||
if masterFirstIP == nil {
|
||||
if masterFirstIP == nil && !a.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
return errors.Errorf("MasterProfile.FirstConsecutiveStaticIP (with VNET Subnet specification) '%s' is an invalid IP address", a.MasterProfile.FirstConsecutiveStaticIP)
|
||||
}
|
||||
|
||||
|
|
@ -619,6 +641,43 @@ func (a *Properties) validateServicePrincipalProfile() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (a *Properties) validateManagedIdentity() error {
|
||||
if a.OrchestratorProfile.OrchestratorType == Kubernetes {
|
||||
useManagedIdentity := a.OrchestratorProfile.KubernetesConfig != nil &&
|
||||
a.OrchestratorProfile.KubernetesConfig.UseManagedIdentity
|
||||
|
||||
if useManagedIdentity {
|
||||
version := common.RationalizeReleaseAndVersion(
|
||||
a.OrchestratorProfile.OrchestratorType,
|
||||
a.OrchestratorProfile.OrchestratorRelease,
|
||||
a.OrchestratorProfile.OrchestratorVersion,
|
||||
false,
|
||||
false)
|
||||
if version == "" {
|
||||
return errors.Errorf("the following user supplied OrchestratorProfile configuration is not supported: OrchestratorType: %s, OrchestratorRelease: %s, OrchestratorVersion: %s. Please check supported Release or Version for this build of acs-engine", a.OrchestratorProfile.OrchestratorType, a.OrchestratorProfile.OrchestratorRelease, a.OrchestratorProfile.OrchestratorVersion)
|
||||
}
|
||||
sv, err := semver.Make(version)
|
||||
if err != nil {
|
||||
return errors.Errorf("could not validate version %s", version)
|
||||
}
|
||||
minVersion, err := semver.Make("1.12.0-beta.0")
|
||||
if err != nil {
|
||||
return errors.New("could not validate version")
|
||||
}
|
||||
|
||||
if a.MasterProfile.IsVirtualMachineScaleSets() {
|
||||
if sv.LT(minVersion) {
|
||||
return errors.New("managed identity and VMSS masters can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"")
|
||||
}
|
||||
} else if a.OrchestratorProfile.KubernetesConfig.UserAssignedID != "" && sv.LT(minVersion) {
|
||||
return errors.New("user assigned identity can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Properties) validateAADProfile() error {
|
||||
if profile := a.AADProfile; profile != nil {
|
||||
if a.OrchestratorProfile.OrchestratorType != Kubernetes {
|
||||
|
|
@ -744,7 +803,7 @@ func (a *AgentPoolProfile) validateCustomNodeLabels(orchestratorType string) err
|
|||
return nil
|
||||
}
|
||||
|
||||
func (a *AgentPoolProfile) validateVMSS(o *OrchestratorProfile, isUpdate bool) error {
|
||||
func validateVMSS(o *OrchestratorProfile, isUpdate bool, storageProfile string) error {
|
||||
if o.OrchestratorType == Kubernetes {
|
||||
version := common.RationalizeReleaseAndVersion(
|
||||
o.OrchestratorType,
|
||||
|
|
@ -777,7 +836,7 @@ func (a *AgentPoolProfile) validateVMSS(o *OrchestratorProfile, isUpdate bool) e
|
|||
return errors.Errorf("VirtualMachineScaleSets with instance metadata is supported for Kubernetes version %s or greater. Please set \"useInstanceMetadata\": false in \"kubernetesConfig\" or set \"orchestratorVersion\" to %s or above", minVersion.String(), minVersion.String())
|
||||
}
|
||||
}
|
||||
if (a.AvailabilityProfile == VirtualMachineScaleSets || len(a.AvailabilityProfile) == 0) && a.StorageProfile == StorageAccount {
|
||||
if storageProfile == StorageAccount {
|
||||
return errors.Errorf("VirtualMachineScaleSets does not support %s disks. Please specify \"storageProfile\": \"%s\" (recommended) or \"availabilityProfile\": \"%s\"", StorageAccount, ManagedDisks, AvailabilitySet)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1377,13 +1377,104 @@ func TestValidateImageNameAndGroup(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
func TestProperties_ValidateManagedIdentity(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
orchestratorRelease string
|
||||
useManagedIdentity bool
|
||||
userAssignedID string
|
||||
masterProfile MasterProfile
|
||||
expectedErr string
|
||||
agentPoolProfiles []*AgentPoolProfile
|
||||
}{
|
||||
{
|
||||
name: "use managed identity with master vmss",
|
||||
orchestratorRelease: "1.11",
|
||||
useManagedIdentity: true,
|
||||
masterProfile: MasterProfile{
|
||||
DNSPrefix: "dummy",
|
||||
Count: 3,
|
||||
AvailabilityProfile: VirtualMachineScaleSets,
|
||||
},
|
||||
agentPoolProfiles: []*AgentPoolProfile{
|
||||
{
|
||||
Name: "agentpool",
|
||||
VMSize: "Standard_DS2_v2",
|
||||
Count: 1,
|
||||
AvailabilityProfile: VirtualMachineScaleSets,
|
||||
},
|
||||
},
|
||||
expectedErr: "managed identity and VMSS masters can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"",
|
||||
},
|
||||
{
|
||||
name: "use managed identity with master vmas",
|
||||
orchestratorRelease: "1.11",
|
||||
useManagedIdentity: true,
|
||||
masterProfile: MasterProfile{
|
||||
DNSPrefix: "dummy",
|
||||
Count: 3,
|
||||
},
|
||||
agentPoolProfiles: []*AgentPoolProfile{
|
||||
{
|
||||
Name: "agentpool",
|
||||
VMSize: "Standard_DS2_v2",
|
||||
Count: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "use user assigned identity with master vmas",
|
||||
orchestratorRelease: "1.11",
|
||||
useManagedIdentity: true,
|
||||
userAssignedID: "acsenginetestid",
|
||||
masterProfile: MasterProfile{
|
||||
DNSPrefix: "dummy",
|
||||
Count: 3,
|
||||
},
|
||||
agentPoolProfiles: []*AgentPoolProfile{
|
||||
{
|
||||
Name: "agentpool",
|
||||
VMSize: "Standard_DS2_v2",
|
||||
Count: 1,
|
||||
},
|
||||
},
|
||||
expectedErr: "user assigned identity can only be used with Kubernetes 1.12.0-beta.0 or above. Please specify \"orchestratorRelease\": \"1.12\"",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
properties := getK8sDefaultProperties(true)
|
||||
properties.MasterProfile = &test.masterProfile
|
||||
properties.MasterProfile.VMSize = "Standard_DS2_v2"
|
||||
properties.OrchestratorProfile = &OrchestratorProfile{
|
||||
OrchestratorRelease: test.orchestratorRelease,
|
||||
OrchestratorType: Kubernetes,
|
||||
KubernetesConfig: &KubernetesConfig{
|
||||
UseManagedIdentity: test.useManagedIdentity,
|
||||
UserAssignedID: test.userAssignedID,
|
||||
},
|
||||
}
|
||||
properties.AgentPoolProfiles = test.agentPoolProfiles
|
||||
err := properties.Validate(false)
|
||||
if test.expectedErr == "" && err != nil ||
|
||||
test.expectedErr != "" && (err == nil || test.expectedErr != err.Error()) {
|
||||
t.Errorf("test %s: unexpected error %q\n", test.name, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMasterProfileValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
orchestratorType string
|
||||
masterProfile MasterProfile
|
||||
expectedErr string
|
||||
name string
|
||||
orchestratorType string
|
||||
orchestratorVersion string
|
||||
orchestratorRelease string
|
||||
useInstanceMetadata bool
|
||||
masterProfile MasterProfile
|
||||
agentPoolProfiles []*AgentPoolProfile
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "Master Profile with Invalid DNS Prefix",
|
||||
|
|
@ -1410,8 +1501,9 @@ func TestMasterProfileValidate(t *testing.T) {
|
|||
name: "Master Profile with valid DNS Prefix 3",
|
||||
orchestratorType: OpenShift,
|
||||
masterProfile: MasterProfile{
|
||||
DNSPrefix: "dummy",
|
||||
Count: 1,
|
||||
DNSPrefix: "dummy",
|
||||
Count: 1,
|
||||
StorageProfile: ManagedDisks,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -1427,8 +1519,9 @@ func TestMasterProfileValidate(t *testing.T) {
|
|||
name: "Master Profile with empty firstconsecutivestaticip and non-empty vnetsubnetid",
|
||||
orchestratorType: OpenShift,
|
||||
masterProfile: MasterProfile{
|
||||
VnetSubnetID: "testvnetstring",
|
||||
Count: 1,
|
||||
VnetSubnetID: "testvnetstring",
|
||||
Count: 1,
|
||||
StorageProfile: ManagedDisks,
|
||||
},
|
||||
expectedErr: "when specifying a vnetsubnetid the firstconsecutivestaticip is required",
|
||||
},
|
||||
|
|
@ -1440,6 +1533,7 @@ func TestMasterProfileValidate(t *testing.T) {
|
|||
VnetSubnetID: "testvnetstring",
|
||||
FirstConsecutiveStaticIP: "10.0.0.1",
|
||||
Count: 1,
|
||||
StorageProfile: ManagedDisks,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -1455,17 +1549,65 @@ func TestMasterProfileValidate(t *testing.T) {
|
|||
},
|
||||
expectedErr: "imageName needs to be specified when imageResourceGroup is provided",
|
||||
},
|
||||
{
|
||||
name: "Master Profile with VMSS and Kubernetes v1.9.6",
|
||||
orchestratorType: Kubernetes,
|
||||
orchestratorRelease: "1.9",
|
||||
masterProfile: MasterProfile{
|
||||
DNSPrefix: "dummy",
|
||||
Count: 3,
|
||||
AvailabilityProfile: VirtualMachineScaleSets,
|
||||
},
|
||||
expectedErr: "VirtualMachineScaleSets are only available in Kubernetes version 1.10.0 or greater. Please set \"orchestratorVersion\" to 1.10.0 or above",
|
||||
},
|
||||
{
|
||||
name: "Master Profile with VMSS and storage account",
|
||||
orchestratorType: Kubernetes,
|
||||
orchestratorRelease: "1.10",
|
||||
masterProfile: MasterProfile{
|
||||
DNSPrefix: "dummy",
|
||||
Count: 3,
|
||||
AvailabilityProfile: VirtualMachineScaleSets,
|
||||
StorageProfile: StorageAccount,
|
||||
},
|
||||
expectedErr: "VirtualMachineScaleSets does not support StorageAccount disks. Please specify \"storageProfile\": \"ManagedDisks\" (recommended) or \"availabilityProfile\": \"AvailabilitySet\"",
|
||||
},
|
||||
{
|
||||
name: "Master Profile with VMSS and agent profiles with VMAS",
|
||||
orchestratorType: Kubernetes,
|
||||
orchestratorRelease: "1.10",
|
||||
masterProfile: MasterProfile{
|
||||
DNSPrefix: "dummy",
|
||||
Count: 3,
|
||||
AvailabilityProfile: VirtualMachineScaleSets,
|
||||
},
|
||||
agentPoolProfiles: []*AgentPoolProfile{
|
||||
{
|
||||
Name: "agentpool",
|
||||
VMSize: "Standard_DS2_v2",
|
||||
Count: 1,
|
||||
AvailabilityProfile: AvailabilitySet,
|
||||
},
|
||||
},
|
||||
expectedErr: "VirtualMachineScaleSets for master profile must be used together with virtualMachineScaleSets for agent profiles. Set \"availabilityProfile\" to \"VirtualMachineScaleSets\" for agent profiles",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
properties := &Properties{}
|
||||
properties := getK8sDefaultProperties(true)
|
||||
properties.MasterProfile = &test.masterProfile
|
||||
properties.MasterProfile.StorageProfile = ManagedDisks
|
||||
properties.MasterProfile.StorageProfile = test.masterProfile.StorageProfile
|
||||
properties.OrchestratorProfile = &OrchestratorProfile{
|
||||
OrchestratorType: test.orchestratorType,
|
||||
OrchestratorType: test.orchestratorType,
|
||||
OrchestratorVersion: test.orchestratorVersion,
|
||||
OrchestratorRelease: test.orchestratorRelease,
|
||||
KubernetesConfig: &KubernetesConfig{
|
||||
UseInstanceMetadata: helpers.PointerToBool(test.useInstanceMetadata),
|
||||
},
|
||||
}
|
||||
properties.AgentPoolProfiles = test.agentPoolProfiles
|
||||
err := properties.validateMasterProfile()
|
||||
if test.expectedErr == "" && err != nil ||
|
||||
test.expectedErr != "" && (err == nil || test.expectedErr != err.Error()) {
|
||||
|
|
@ -1670,7 +1812,7 @@ func TestProperties_ValidateVNET(t *testing.T) {
|
|||
expectedMsg: "Multiple VNETS specified. The master profile and each agent pool must reference the same VNET (but it is ok to reference different subnets on that VNET)",
|
||||
},
|
||||
{
|
||||
name: "Invalid MasterProfile FirstConsecutiveStaticIP",
|
||||
name: "Invalid MasterProfile FirstConsecutiveStaticIP when master is VMAS",
|
||||
masterProfile: &MasterProfile{
|
||||
VnetSubnetID: validVNetSubnetID,
|
||||
Count: 1,
|
||||
|
|
@ -1689,6 +1831,26 @@ func TestProperties_ValidateVNET(t *testing.T) {
|
|||
},
|
||||
expectedMsg: "MasterProfile.FirstConsecutiveStaticIP (with VNET Subnet specification) '10.0.0.invalid' is an invalid IP address",
|
||||
},
|
||||
{
|
||||
name: "Empty MasterProfile FirstConsecutiveStaticIP and empty agentVnetSubnetID when master is VMSS",
|
||||
masterProfile: &MasterProfile{
|
||||
VnetSubnetID: validVNetSubnetID,
|
||||
Count: 1,
|
||||
DNSPrefix: "foo",
|
||||
VMSize: "Standard_DS2_v2",
|
||||
AvailabilityProfile: VirtualMachineScaleSets,
|
||||
},
|
||||
agentPoolProfiles: []*AgentPoolProfile{
|
||||
{
|
||||
Name: "agentpool",
|
||||
VMSize: "Standard_D2_v2",
|
||||
Count: 1,
|
||||
AvailabilityProfile: VirtualMachineScaleSets,
|
||||
VnetSubnetID: validVNetSubnetID,
|
||||
},
|
||||
},
|
||||
expectedMsg: "when master profile is using VirtualMachineScaleSets and is custom vnet, set \"vnetsubnetid\" and \"agentVnetSubnetID\" for master profile",
|
||||
},
|
||||
{
|
||||
name: "Invalid vnetcidr",
|
||||
masterProfile: &MasterProfile{
|
||||
|
|
@ -1716,6 +1878,7 @@ func TestProperties_ValidateVNET(t *testing.T) {
|
|||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
p := getK8sDefaultProperties(true)
|
||||
p.OrchestratorProfile.OrchestratorRelease = "1.10"
|
||||
p.MasterProfile = test.masterProfile
|
||||
p.AgentPoolProfiles = test.agentPoolProfiles
|
||||
err := p.Validate(true)
|
||||
|
|
|
|||
|
|
@ -243,6 +243,25 @@ func (a *Account) CreateVnet(vnet, addressPrefixes, subnetName, subnetPrefix str
|
|||
return nil
|
||||
}
|
||||
|
||||
// CreateSubnet will create a subnet in a vnet in a resource group
|
||||
func (a *Account) CreateSubnet(vnet, subnetName, subnetPrefix string) error {
|
||||
var cmd *exec.Cmd
|
||||
if a.TimeoutCommands {
|
||||
cmd = exec.Command("timeout", "60", "az", "network", "vnet", "subnet", "create", "-g",
|
||||
a.ResourceGroup.Name, "--vnet-name", vnet, "--name", subnetName, "--address-prefix", subnetPrefix)
|
||||
} else {
|
||||
cmd = exec.Command("az", "network", "vnet", "subnet", "create", "-g",
|
||||
a.ResourceGroup.Name, "--vnet-name", vnet, "--name", subnetName, "--address-prefix", subnetPrefix)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Printf("Error while trying to create subnet in vnet with the following command:\n az network vnet subnet create -g %s --vnet-name %s --name %s --address-prefix %s \n Output:%s\n", a.ResourceGroup.Name, vnet, subnetName, subnetPrefix, out)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RouteTable holds information from running az network route-table list
|
||||
type RouteTable struct {
|
||||
ID string `json:"id"`
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ func ParseConfig(cwd, clusterDefinition, name string) (*Config, error) {
|
|||
|
||||
// Build takes a template path and will inject values based on provided environment variables
|
||||
// it will then serialize the structs back into json and save it to outputPath
|
||||
func Build(cfg *config.Config, subnetID string) (*Engine, error) {
|
||||
func Build(cfg *config.Config, masterSubnetID string, agentSubnetID string, isVMSS bool) (*Engine, error) {
|
||||
config, err := ParseConfig(cfg.CurrentWorkingDir, cfg.ClusterDefinition, cfg.Name)
|
||||
if err != nil {
|
||||
log.Printf("Error while trying to build Engine Configuration:%s\n", err)
|
||||
|
|
@ -168,9 +168,17 @@ func Build(cfg *config.Config, subnetID string) (*Engine, error) {
|
|||
}
|
||||
|
||||
if config.CreateVNET {
|
||||
cs.ContainerService.Properties.MasterProfile.VnetSubnetID = subnetID
|
||||
for _, p := range cs.ContainerService.Properties.AgentPoolProfiles {
|
||||
p.VnetSubnetID = subnetID
|
||||
if isVMSS {
|
||||
cs.ContainerService.Properties.MasterProfile.VnetSubnetID = masterSubnetID
|
||||
cs.ContainerService.Properties.MasterProfile.AgentVnetSubnetID = agentSubnetID
|
||||
for _, p := range cs.ContainerService.Properties.AgentPoolProfiles {
|
||||
p.VnetSubnetID = agentSubnetID
|
||||
}
|
||||
} else {
|
||||
cs.ContainerService.Properties.MasterProfile.VnetSubnetID = masterSubnetID
|
||||
for _, p := range cs.ContainerService.Properties.AgentPoolProfiles {
|
||||
p.VnetSubnetID = masterSubnetID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import (
|
|||
const (
|
||||
WorkloadDir = "workloads"
|
||||
PolicyDir = "workloads/policies"
|
||||
SSHPort = "50001"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -67,9 +68,17 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
Expect(err).NotTo(HaveOccurred())
|
||||
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
|
||||
sshKeyPath := cfg.GetSSHKeyPath()
|
||||
|
||||
masterNodes, err := node.GetByPrefix("k8s-master")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
masterName := masterNodes[0].Metadata.Name
|
||||
lsbReleaseCmd := fmt.Sprintf("lsb_release -a && uname -r")
|
||||
cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd)
|
||||
|
||||
var cmd *exec.Cmd
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, lsbReleaseCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err := cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -78,7 +87,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
}
|
||||
|
||||
kernelVerCmd := fmt.Sprintf("cat /proc/version")
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd)
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, kernelVerCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err = cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -186,8 +199,17 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
|
||||
sshKeyPath := cfg.GetSSHKeyPath()
|
||||
|
||||
masterNodes, err := node.GetByPrefix("k8s-master")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
masterName := masterNodes[0].Metadata.Name
|
||||
|
||||
ifconfigCmd := fmt.Sprintf("ifconfig -a -v")
|
||||
cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
|
||||
var cmd *exec.Cmd
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, ifconfigCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err := cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -196,7 +218,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
}
|
||||
|
||||
resolvCmd := fmt.Sprintf("cat /etc/resolv.conf")
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, resolvCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err = cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -206,7 +232,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
|
||||
By("Ensuring that we have a valid connection to our resolver")
|
||||
digCmd := fmt.Sprintf("dig +short +search +answer `hostname`")
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err = cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -219,7 +249,12 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
for _, node := range nodeList.Nodes {
|
||||
By("Ensuring that we get a DNS lookup answer response for each node hostname")
|
||||
digCmd := fmt.Sprintf("dig +short +search +answer %s | grep -v -e '^$'", node.Metadata.Name)
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err = cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -231,14 +266,22 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
|
||||
By("Ensuring that we get a DNS lookup answer response for external names")
|
||||
digCmd = fmt.Sprintf("dig +short +search www.bing.com | grep -v -e '^$'")
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Printf("Error while querying DNS: %s\n", out)
|
||||
}
|
||||
digCmd = fmt.Sprintf("dig +short +search google.com | grep -v -e '^$'")
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err = cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -248,7 +291,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
|
||||
By("Ensuring that we get a DNS lookup answer response for external names using external resolver")
|
||||
digCmd = fmt.Sprintf("dig +short +search www.bing.com @8.8.8.8 | grep -v -e '^$'")
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err = cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -256,7 +303,11 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
log.Printf("Error while querying DNS: %s\n", err)
|
||||
}
|
||||
digCmd = fmt.Sprintf("dig +short +search google.com @8.8.8.8 | grep -v -e '^$'")
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, digCmd)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err = cmd.CombinedOutput()
|
||||
log.Printf("%s\n", out)
|
||||
|
|
@ -403,7 +454,9 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
|
||||
|
||||
sshKeyPath := cfg.GetSSHKeyPath()
|
||||
|
||||
masterNodes, err := node.GetByPrefix("k8s-master")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
masterName := masterNodes[0].Metadata.Name
|
||||
if dashboardPort == 80 {
|
||||
By("Ensuring that we can connect via HTTP to the dashboard on any one node")
|
||||
} else {
|
||||
|
|
@ -421,7 +474,12 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
|
|||
Expect(address).NotTo(Equal(nil))
|
||||
dashboardURL := fmt.Sprintf("http://%s:%v", address.Address, port)
|
||||
curlCMD := fmt.Sprintf("curl --max-time 60 %s", dashboardURL)
|
||||
cmd := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD)
|
||||
var cmd *exec.Cmd
|
||||
if strings.Contains(masterName, "vmss") {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-p", SSHPort, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD)
|
||||
} else {
|
||||
cmd = exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD)
|
||||
}
|
||||
util.PrintCommand(cmd)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err == nil {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ type CLIProvisioner struct {
|
|||
ClusterDefinition string `envconfig:"CLUSTER_DEFINITION" required:"true" default:"examples/kubernetes.json"` // ClusterDefinition is the path on disk to the json template these are normally located in examples/
|
||||
ProvisionRetries int `envconfig:"PROVISION_RETRIES" default:"0"`
|
||||
CreateVNET bool `envconfig:"CREATE_VNET" default:"false"`
|
||||
MasterVMSS bool `envconfig:"MASTER_VMSS" default:"false"`
|
||||
Config *config.Config
|
||||
Account *azure.Account
|
||||
Point *metrics.Point
|
||||
|
|
@ -110,16 +111,44 @@ func (cli *CLIProvisioner) provision() error {
|
|||
subnetID := ""
|
||||
vnetName := fmt.Sprintf("%sCustomVnet", cli.Config.Name)
|
||||
subnetName := fmt.Sprintf("%sCustomSubnet", cli.Config.Name)
|
||||
masterSubnetID := ""
|
||||
agentSubnetID := ""
|
||||
|
||||
if cli.CreateVNET {
|
||||
err = cli.Account.CreateVnet(vnetName, "10.239.0.0/16", subnetName, "10.239.0.0/16")
|
||||
if err != nil {
|
||||
return errors.Errorf("Error trying to create vnet:%s", err.Error())
|
||||
if cli.MasterVMSS {
|
||||
masterSubnetName := fmt.Sprintf("%sCustomSubnetMaster", cli.Config.Name)
|
||||
agentSubnetName := fmt.Sprintf("%sCustomSubnetAgent", cli.Config.Name)
|
||||
err = cli.Account.CreateVnet(vnetName, "10.239.0.0/16", masterSubnetName, "10.239.0.0/17")
|
||||
if err != nil {
|
||||
return errors.Errorf("Error trying to create vnet:%s", err.Error())
|
||||
}
|
||||
|
||||
masterSubnetID = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", cli.Account.SubscriptionID, cli.Account.ResourceGroup.Name, vnetName, masterSubnetName)
|
||||
|
||||
err = cli.Account.CreateSubnet(vnetName, agentSubnetName, "10.239.128.0/17")
|
||||
if err != nil {
|
||||
return errors.Errorf("Error trying to create subnet in vnet:%s", err.Error())
|
||||
}
|
||||
|
||||
agentSubnetID = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", cli.Account.SubscriptionID, cli.Account.ResourceGroup.Name, vnetName, agentSubnetName)
|
||||
} else {
|
||||
err = cli.Account.CreateVnet(vnetName, "10.239.0.0/16", subnetName, "10.239.0.0/16")
|
||||
if err != nil {
|
||||
return errors.Errorf("Error trying to create vnet:%s", err.Error())
|
||||
}
|
||||
subnetID = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", cli.Account.SubscriptionID, cli.Account.ResourceGroup.Name, vnetName, subnetName)
|
||||
}
|
||||
subnetID = fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", cli.Account.SubscriptionID, cli.Account.ResourceGroup.Name, vnetName, subnetName)
|
||||
}
|
||||
|
||||
// Lets modify our template and call acs-engine generate on it
|
||||
eng, err := engine.Build(cli.Config, subnetID)
|
||||
var eng *engine.Engine
|
||||
|
||||
if cli.CreateVNET && cli.MasterVMSS {
|
||||
eng, err = engine.Build(cli.Config, masterSubnetID, agentSubnetID, true)
|
||||
} else {
|
||||
eng, err = engine.Build(cli.Config, subnetID, subnetID, false)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Error while trying to build cluster definition")
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче