From 1ade2f387a770697ed196d1d254456cac87499b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serta=C3=A7=20=C3=96zercan?= Date: Tue, 14 Aug 2018 17:20:26 -0700 Subject: [PATCH] add loadBalancerSku and excludeMasterFromStandardLB (#3515) --- docs/clusterdefinition.md | 3 +- .../kubernetes-standardlb.json | 40 +++++++++++++++++++ .../kubernetesmasteraddons-elb-svc.yaml | 15 +++++++ parts/k8s/kubernetesconfigs.sh | 2 + parts/k8s/kubernetescustomscript.sh | 2 +- parts/k8s/kubernetesmastercustomdata.yml | 4 ++ parts/k8s/kubernetesmastervars.t | 6 ++- parts/k8s/kubernetesparams.t | 7 ++++ parts/k8s/kuberneteswindowssetup.ps1 | 18 +++++---- pkg/acsengine/artifacts.go | 5 +++ pkg/acsengine/const.go | 2 + pkg/acsengine/defaults.go | 8 ++++ pkg/acsengine/params_k8s.go | 9 +++++ pkg/acsengine/template_generator.go | 6 +++ pkg/api/const.go | 4 ++ pkg/api/converterfromapi.go | 2 + pkg/api/convertertoapi.go | 2 + pkg/api/types.go | 2 + pkg/api/vlabs/types.go | 2 + pkg/api/vlabs/validate.go | 11 +++++ pkg/api/vlabs/validate_test.go | 12 ++++++ 21 files changed, 151 insertions(+), 11 deletions(-) create mode 100644 examples/kubernetes-config/kubernetes-standardlb.json create mode 100644 parts/k8s/addons/kubernetesmasteraddons-elb-svc.yaml diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index ab4fbd108..53c506280 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -53,6 +53,7 @@ Here are the valid values for the orchestrator types: | gcLowThreshold | no | Sets the --image-gc-low-threshold value on the kublet configuration. Default is 80. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) | | kubeletConfig | no | Configure various runtime configuration for kubelet. See `kubeletConfig` [below](#feat-kubelet-config) | | kubernetesImageBase | no | Specifies the base URL (everything preceding the actual image filename) of the kubernetes hyperkube image to use for cluster deployment, e.g., `k8s.gcr.io/` | +| loadBalancerSku | no | Sku of Load Balancer and Public IP. Candidate values are: `basic` and `standard`. If not set, it will be default to basic. Requires Kubernetes 1.11 or newer. NOTE: VMs behind ILB standard SKU will not be able to access the internet without ELB configured with at least one frontend IP as described in the [standard loadbalancer outbound connectivity doc](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-overview#control-outbound-connectivity). For Kubernetes 1.11, We have created an external loadbalancer service in the kube-system namespace as a workaround to this issue. Starting k8s 1.12, instead of creating an ELB service, we will setup outbound rules in ARM template once the API is available. | | networkPlugin | no | Specifies the network plugin implementation for the cluster. Valid values are:
`"azure"` (default), which provides an Azure native networking experience
`"kubenet"` for k8s software networking implementation.
`"flannel"` for using CoreOS Flannel
`"cilium"` for using the default Cilium CNI IPAM | | networkPolicy | no | Specifies the network policy enforcement tool for the cluster (currently Linux-only). Valid values are:
`"calico"` for Calico network policy.
`"cilium"` for cilium network policy (Lin), and `"azure"` (experimental) for Azure CNI-compliant network policy (note: Azure CNI-compliant network policy requires explicit `"networkPlugin": "azure"` configuration as well).
See [network policy examples](../examples/networkpolicy) for more information. | | privateCluster | no | Build a cluster without public addresses assigned. See `privateClusters` [below](#feat-private-cluster). | @@ -626,4 +627,4 @@ A cluster can have 0 - N extensions in extension profiles. Extension profiles al | extensionParameters | optional | Extension parameters may be required by extensions. The format of the parameters is also extension dependant | | rootURL | optional | URL to the root location of extensions. The rootURL must have an extensions child folder that follows the extensions convention. The rootURL is mainly used for testing purposes | -You can find more information, as well as a list of extensions on the [extensions documentation](extensions.md). +You can find more information, as well as a list of extensions on the [extensions documentation](extensions.md). \ No newline at end of file diff --git a/examples/kubernetes-config/kubernetes-standardlb.json b/examples/kubernetes-config/kubernetes-standardlb.json new file mode 100644 index 000000000..d774f9515 --- /dev/null +++ b/examples/kubernetes-config/kubernetes-standardlb.json @@ -0,0 +1,40 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorRelease": "1.11", + "kubernetesConfig": { + "loadBalancerSku": "Standard", + "excludeMasterFromStandardLB": true + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool1", + "count": 1, + "vmSize": "Standard_D2_v2", + "availabilityProfile": "VirtualMachineScaleSets" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/parts/k8s/addons/kubernetesmasteraddons-elb-svc.yaml b/parts/k8s/addons/kubernetesmasteraddons-elb-svc.yaml new file mode 100644 index 000000000..d8442a2fd --- /dev/null +++ b/parts/k8s/addons/kubernetesmasteraddons-elb-svc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" + name: elb + namespace: kube-system +spec: + ports: + - port: 8765 + targetPort: 9376 + selector: + app: "" + type: LoadBalancer \ No newline at end of file diff --git a/parts/k8s/kubernetesconfigs.sh b/parts/k8s/kubernetesconfigs.sh index 9373e9e66..41b71e731 100644 --- a/parts/k8s/kubernetesconfigs.sh +++ b/parts/k8s/kubernetesconfigs.sh @@ -150,6 +150,8 @@ function configureK8s() { "cloudProviderRateLimitBucket": ${CLOUDPROVIDER_RATELIMIT_BUCKET}, "useManagedIdentityExtension": ${USE_MANAGED_IDENTITY_EXTENSION}, "useInstanceMetadata": ${USE_INSTANCE_METADATA}, + "loadBalancerSku": "${LOAD_BALANCER_SKU}", + "excludeMasterFromStandardLB": ${EXCLUDE_MASTER_FROM_STANDARD_LB}, "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", "providerKeyName": "k8s", "providerKeyVersion": "" diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 07cbd4c7e..e257d36c4 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -129,4 +129,4 @@ if $REBOOTREQUIRED; then /bin/bash -c "shutdown -r 1 &" else runAptDaily & -fi +fi \ No newline at end of file diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 1a7a25b79..368f6eb54 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -274,6 +274,10 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" {{end}} +{{if eq .OrchestratorProfile.KubernetesConfig.LoadBalancerSku "Standard"}} + sed -i "s||{{WrapAsParameter "kuberneteselbsvcname"}}|g" "/etc/kubernetes/addons/elb-svc.yaml" +{{end}} + {{if .OrchestratorProfile.KubernetesConfig.IsBlobfuseFlexVolumeEnabled}} sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml" sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml" diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index f83ff884f..88a9d0ddd 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -67,6 +67,8 @@ {{end}} "useManagedIdentityExtension": "{{ UseManagedIdentity }}", "useInstanceMetadata": "{{ UseInstanceMetadata }}", + "loadBalancerSku": "{{ LoadBalancerSku }}", + "excludeMasterFromStandardLB": "{{ ExcludeMasterFromStandardLB }}", {{ if UseManagedIdentity }} "servicePrincipalClientId": "msi", "servicePrincipalClientSecret": "msi", @@ -132,7 +134,7 @@ "customSearchDomainsScript": "{{GetKubernetesB64CustomSearchDomainsScript}}", "sshdConfig": "{{GetB64sshdConfig}}", {{if not IsOpenShift}} - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", + "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_VERSION=',parameters('dockerEngineVersion'),' DOCKER_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',parameters('cloudproviderConfig').cloudProviderBackoff,' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',parameters('cloudproviderConfig').cloudProviderRatelimit,' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", {{if not IsHostedMaster}} "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{end}} @@ -310,4 +312,4 @@ "clusterKeyVaultName": "[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]" {{else}} ,"clusterKeyVaultName": "" -{{end}} +{{end}} \ No newline at end of file diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index cec5b7b01..f07ca0755 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -708,6 +708,13 @@ }, "type": "int" }, + "kuberneteselbsvcname": { + "defaultValue": "", + "metadata": { + "description": "elb service for standard lb" + }, + "type": "string" + }, {{if .OrchestratorProfile.KubernetesConfig.IsContainerMonitoringEnabled}} "omsAgentVersion": { "defaultValue": "", diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index cee31ac0b..11479adfe 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -65,6 +65,8 @@ $global:KubeDnsSearchPath = "svc.cluster.local" $global:UseManagedIdentityExtension = "{{WrapAsVariable "useManagedIdentityExtension"}}" $global:UseInstanceMetadata = "{{WrapAsVariable "useInstanceMetadata"}}" +$global:LoadBalancerSku = "{{WrapAsVariable "loadBalancerSku"}}" +$global:ExcludeMasterFromStandardLB = "{{WrapAsVariable "excludeMasterFromStandardLB"}}" $global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") $global:NetworkMode = "L2Bridge" @@ -183,7 +185,9 @@ Write-AzureConfig() "primaryAvailabilitySetName": "$global:PrimaryAvailabilitySetName", "primaryScaleSetName": "$global:PrimaryScaleSetName", "useManagedIdentityExtension": $global:UseManagedIdentityExtension, - "useInstanceMetadata": $global:UseInstanceMetadata + "useInstanceMetadata": $global:UseInstanceMetadata, + "loadBalancerSku": "$global:LoadBalancerSku", + "excludeMasterFromStandardLB": $global:ExcludeMasterFromStandardLB } "@ @@ -227,14 +231,14 @@ New-InfraContainer() $computerInfo = Get-ComputerInfo $windowsBase = if ($computerInfo.WindowsVersion -eq "1709") { "microsoft/nanoserver:1709" - } elseif ( ($computerInfo.WindowsVersion -eq "1803") -and ($computerInfo.WindowsBuildLabEx.StartsWith("17134")) ) { + } elseif ( ($computerInfo.WindowsVersion -eq "1803") -and ($computerInfo.WindowsBuildLabEx.StartsWith("17134")) ) { "microsoft/nanoserver:1803" - } else { + } else { # This is a temporary workaround. As of May 2018, Windows Server Insider builds still report 1803 which is wrong. # Once that is fixed, add another elseif ( -eq "nnnn") instead and remove the StartsWith("17134") above "microsoft/nanoserver-insider" } - + "FROM $($windowsBase)" | Out-File -encoding ascii -FilePath Dockerfile "CMD cmd /c ping -t localhost" | Out-File -encoding ascii -FilePath Dockerfile -Append docker build -t kubletwin/pause . @@ -284,7 +288,7 @@ Set-AzureCNIConfig() $fileName = [Io.path]::Combine("$global:AzureCNIConfDir", "10-azure.conflist") $configJson = Get-Content $fileName | ConvertFrom-Json $configJson.plugins.dns.Nameservers[0] = $KubeDnsServiceIp - $configJson.plugins.dns.Search[0] = $global:KubeDnsSearchPath + $configJson.plugins.dns.Search[0] = $global:KubeDnsSearchPath $configJson.plugins.AdditionalArgs[0].Value.ExceptionList[0] = $global:KubeClusterCIDR $configJson.plugins.AdditionalArgs[0].Value.ExceptionList[1] = $global:MasterSubnet $configJson.plugins.AdditionalArgs[1].Value.DestinationPrefix = $global:KubeServiceCIDR @@ -307,8 +311,8 @@ Set-NetworkConfig function Write-KubernetesStartFiles($podCIDR) { - mkdir $global:VolumePluginDir - $KubeletArgList = @(" --node-labels=`$global:KubeletNodeLabels --hostname-override=`$global:AzureHostname","--pod-infra-container-image=kubletwin/pause","--resolv-conf=""""""""","--kubeconfig=c:\k\config","--cloud-provider=azure","--cloud-config=c:\k\azure.json") + mkdir $global:VolumePluginDir + $KubeletArgList = @(" --node-labels=`$global:KubeletNodeLabels --hostname-override=`$global:AzureHostname","--pod-infra-container-image=kubletwin/pause","--resolv-conf=""""""""","--kubeconfig=c:\k\config","--cloud-provider=azure","--cloud-config=c:\k\azure.json") $KubeletCommandLine = @" c:\k\kubelet.exe --hostname-override=`$env:computername --pod-infra-container-image=kubletwin/pause --resolv-conf="" --allow-privileged=true --enable-debugging-handlers --cluster-dns=`$global:KubeDnsServiceIp --cluster-domain=cluster.local --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge --v=2 --azure-container-registry-config=c:\k\azure.json --runtime-request-timeout=10m --cloud-provider=azure --cloud-config=c:\k\azure.json "@ diff --git a/pkg/acsengine/artifacts.go b/pkg/acsengine/artifacts.go index 9a6317f07..d9c6bc14e 100644 --- a/pkg/acsengine/artifacts.go +++ b/pkg/acsengine/artifacts.go @@ -142,6 +142,11 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet "keyvault-flexvolume-installer.yaml", profile.OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled(), }, + { + "kubernetesmasteraddons-elb-svc.yaml", + "elb-svc.yaml", + profile.OrchestratorProfile.KubernetesConfig.LoadBalancerSku == "Standard", + }, } } diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 2510d86a7..0fbe4b79e 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -116,6 +116,8 @@ const ( DefaultSMBFlexVolumeAddonName = "smb-flexvolume" // DefaultKeyVaultFlexVolumeAddonName is the name of the keyvault flexvolume addon deployment DefaultKeyVaultFlexVolumeAddonName = "keyvault-flexvolume" + // DefaultELBSVCAddonName is the name of the elb service addon deployment + DefaultELBSVCAddonName = "elb-svc" // DefaultKubernetesDNSServiceIP specifies the IP address that kube-dns // listens on by default. must by in the default Service CIDR range. DefaultKubernetesDNSServiceIP = "10.0.0.10" diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index be8b933d5..a894e9cff 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -399,6 +399,14 @@ func setOrchestratorDefaults(cs *api.ContainerService) { a.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata = helpers.PointerToBool(api.DefaultUseInstanceMetadata) } + if a.OrchestratorProfile.KubernetesConfig.LoadBalancerSku == "" { + a.OrchestratorProfile.KubernetesConfig.LoadBalancerSku = api.DefaultLoadBalancerSku + } + + if common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.11.0") && a.OrchestratorProfile.KubernetesConfig.LoadBalancerSku == "Standard" { + a.OrchestratorProfile.KubernetesConfig.ExcludeMasterFromStandardLB = helpers.PointerToBool(api.DefaultExcludeMasterFromStandardLB) + } + // Configure addons setAddonsConfig(cs) // Configure kubelet diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index 97ed6b7b1..d2350b30b 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -1,8 +1,11 @@ package acsengine import ( + "fmt" + "math/rand" "strconv" "strings" + "time" "github.com/Azure/acs-engine/pkg/api" "github.com/Azure/acs-engine/pkg/helpers" @@ -211,6 +214,12 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params } } } + if properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku == "Standard" { + random := rand.New(rand.NewSource(time.Now().UnixNano())) + elbsvcName := random.Int() + addValue(parametersMap, "kuberneteselbsvcname", fmt.Sprintf("%d", elbsvcName)) + } + if properties.OrchestratorProfile.IsAzureCNI() { azureCNINetworkmonitorAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, AzureCNINetworkMonitoringAddonName) c = getAddonContainersIndexByName(azureCNINetworkmonitorAddon.Containers, AzureCNINetworkMonitoringAddonName) diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index e5df92340..8680d1065 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -289,6 +289,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat "UseInstanceMetadata": func() bool { return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata) }, + "LoadBalancerSku": func() string { + return cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku + }, + "ExcludeMasterFromStandardLB": func() bool { + return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.ExcludeMasterFromStandardLB) + }, "GetVNETSubnetDependencies": func() string { return getVNETSubnetDependencies(cs.Properties) }, diff --git a/pkg/api/const.go b/pkg/api/const.go index 3daa00457..7e3c02762 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -107,6 +107,10 @@ const ( DefaultRBACEnabled = true // DefaultUseInstanceMetadata determines the acs-engine provided default for enabling Azure cloudprovider instance metadata service DefaultUseInstanceMetadata = true + // DefaultLoadBalancerSku determines the acs-engine provided default for enabling Azure cloudprovider load balancer SKU + DefaultLoadBalancerSku = "Basic" + // DefaultExcludeMasterFromStandardLB determines the acs-engine provided default for excluding master nodes from standard load balancer. + DefaultExcludeMasterFromStandardLB = true // DefaultSecureKubeletEnabled determines the acs-engine provided default for securing kubelet communications DefaultSecureKubeletEnabled = true // DefaultMetricsServerAddonEnabled determines the acs-engine provided default for enabling kubernetes metrics-server addon diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 9b62a2d78..6f2e8db30 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -734,6 +734,8 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet vlabs.UseCloudControllerManager = api.UseCloudControllerManager vlabs.CustomWindowsPackageURL = api.CustomWindowsPackageURL vlabs.UseInstanceMetadata = api.UseInstanceMetadata + vlabs.LoadBalancerSku = api.LoadBalancerSku + vlabs.ExcludeMasterFromStandardLB = api.ExcludeMasterFromStandardLB vlabs.EnableRbac = api.EnableRbac vlabs.EnableSecureKubelet = api.EnableSecureKubelet vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 66c189430..f74f5a59f 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -691,6 +691,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.UseCloudControllerManager = vlabs.UseCloudControllerManager api.CustomWindowsPackageURL = vlabs.CustomWindowsPackageURL api.UseInstanceMetadata = vlabs.UseInstanceMetadata + api.LoadBalancerSku = vlabs.LoadBalancerSku + api.ExcludeMasterFromStandardLB = vlabs.ExcludeMasterFromStandardLB api.EnableRbac = vlabs.EnableRbac api.EnableSecureKubelet = vlabs.EnableSecureKubelet api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs diff --git a/pkg/api/types.go b/pkg/api/types.go index 560122f20..ed5f9d5b6 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -325,6 +325,8 @@ type KubernetesConfig struct { CtrlMgrNodeMonitorGracePeriod string `json:"ctrlMgrNodeMonitorGracePeriod,omitempty"` CtrlMgrPodEvictionTimeout string `json:"ctrlMgrPodEvictionTimeout,omitempty"` CtrlMgrRouteReconciliationPeriod string `json:"ctrlMgrRouteReconciliationPeriod,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` } // CustomFile has source as the full absolute source path to a file and dest diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 49dc46eb6..a2c8f39bb 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -305,6 +305,8 @@ type KubernetesConfig struct { CloudProviderRateLimit bool `json:"cloudProviderRateLimit,omitempty"` CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` } // CustomFile has source as the full absolute source path to a file and dest diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 7493582eb..8effb811f 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -245,6 +245,17 @@ func (a *Properties) validateOrchestratorProfile(isUpdate bool) error { minVersion.String(), version) } } + + if o.KubernetesConfig.LoadBalancerSku == "Standard" { + minVersion, err := semver.Make("1.11.0") + if err != nil { + return errors.Errorf("could not validate version") + } + if sv.LT(minVersion) { + return errors.Errorf("loadBalancerSku is only available in Kubernetes version %s or greater; unable to validate for Kubernetes version %s", + minVersion.String(), o.OrchestratorVersion) + } + } } case OpenShift: // TODO: add appropriate additional validation logic diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index a1064ebab..fb41fb025 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -119,6 +119,18 @@ func Test_OrchestratorProfile_Validate(t *testing.T) { }, expectedError: "enableEncryptionWithExternalKms is only available in Kubernetes version 1.10.0 or greater; unable to validate for Kubernetes version 1.6.6", }, + "should error when KubernetesConfig has Standard loadBalancerSku with invalid version": { + properties: &Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: "Kubernetes", + OrchestratorVersion: "1.6.6", + KubernetesConfig: &KubernetesConfig{ + LoadBalancerSku: "Standard", + }, + }, + }, + expectedError: "loadBalancerSku is only available in Kubernetes version 1.11.0 or greater; unable to validate for Kubernetes version 1.6.6", + }, "should error when KubernetesConfig has enablePodSecurity enabled with invalid settings": { properties: &Properties{ OrchestratorProfile: &OrchestratorProfile{