feat: dualstack phase2 changes (#1929)

examples/dualstack/kubernetes.json

@@ -9,6 +9,9 @@
             "orchestratorRelease": "1.16",
             "kubernetesConfig": {
                 "clusterSubnet": "10.244.0.0/16,fd00:101::/8",
+                "serviceCidr": "10.0.0.0/16,fe80:20d::/112",
+                "dnsServiceIP": "10.0.0.10",
+                "kubeProxyMode": "ipvs",
                 "networkPlugin": "kubenet",
                 "apiServerConfig": {
                     "--feature-gates": "IPv6DualStack=true"
@@ -49,4 +52,4 @@
             "secret": ""
         }
     }
-}
+}

parts/k8s/addons/1.16/kubernetesmasteraddons-kube-proxy-daemonset.yaml

@@ -9,6 +9,8 @@ data:
       kubeconfig: /var/lib/kubelet/kubeconfig
     clusterCIDR: "<CIDR>"
     mode: "<kubeProxyMode>"
+    featureGates:
+      <IPv6DualStackFeature>
 metadata:
   name: kube-proxy-config
   namespace: kube-system

parts/k8s/cloud-init/masternodecustomdata.yml

@@ -354,9 +354,9 @@ MASTER_CONTAINER_ADDONS_PLACEHOLDER
     sed -i "s|<args>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.ControllerManagerConfig}}|g" /etc/kubernetes/manifests/kube-controller-manager.yaml
     sed -i "s|<args>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.SchedulerConfig}}|g" /etc/kubernetes/manifests/kube-scheduler.yaml
     {{ if IsIPv6DualStackFeatureEnabled }}
-    sed -i "s|<img>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<CIDR>|',first(split(parameters('kubeClusterCidr'),',')),'|g; s|<kubeProxyMode>|{{ .OrchestratorProfile.KubernetesConfig.ProxyMode}}|g" /etc/kubernetes/addons/kube-proxy-daemonset.yaml
+    sed -i "s|<img>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<CIDR>|{{WrapAsParameter "kubeClusterCidr"}}|g; s|<kubeProxyMode>|{{ .OrchestratorProfile.KubernetesConfig.ProxyMode}}|g; s|<IPv6DualStackFeature>|IPv6DualStack: true|g" /etc/kubernetes/addons/kube-proxy-daemonset.yaml
     {{ else }}
-    sed -i "s|<img>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<CIDR>|{{WrapAsParameter "kubeClusterCidr"}}|g; s|<kubeProxyMode>|{{ .OrchestratorProfile.KubernetesConfig.ProxyMode}}|g" /etc/kubernetes/addons/kube-proxy-daemonset.yaml
+    sed -i "s|<img>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<CIDR>|{{WrapAsParameter "kubeClusterCidr"}}|g; s|<kubeProxyMode>|{{ .OrchestratorProfile.KubernetesConfig.ProxyMode}}|g; s|<IPv6DualStackFeature>|{}|g" /etc/kubernetes/addons/kube-proxy-daemonset.yaml
     {{ end }}
     KUBEDNS=/etc/kubernetes/addons/kube-dns-deployment.yaml
 {{if NeedsKubeDNSWithExecHealthz}}

pkg/api/defaults.go

@@ -856,7 +856,17 @@ func (cs *ContainerService) SetDefaultCerts(params DefaultCertParams) (bool, []n
 		p.CertificateProfile.CaPrivateKey = caPair.PrivateKeyPem
 	}
 
-	cidrFirstIP, err := common.CidrStringFirstIP(p.OrchestratorProfile.KubernetesConfig.ServiceCIDR)
+	serviceCIDR := p.OrchestratorProfile.KubernetesConfig.ServiceCIDR
+
+	// all validation for dual stack done with primary service cidr as that is considered
+	// the default ip family for cluster.
+	if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") {
+		// split service cidrs
+		serviceCIDRs := strings.Split(serviceCIDR, ",")
+		serviceCIDR = serviceCIDRs[0]
+	}
+
+	cidrFirstIP, err := common.CidrStringFirstIP(serviceCIDR)
 	if err != nil {
 		return false, ips, err
 	}

pkg/api/vlabs/validate.go

@@ -1144,9 +1144,22 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows, ipv6DualStack
 	// number of minimum retries allowed for kubelet to post node status
 	const minKubeletRetries = 4
 
-	// ipv6 dual stack feature is currently only supported with kubenet
-	if ipv6DualStackEnabled && k.NetworkPlugin != "kubenet" {
-		return errors.Errorf("OrchestratorProfile.KubernetesConfig.NetworkPlugin '%s' is invalid. IPv6 dual stack supported only with kubenet.", k.NetworkPlugin)
+	if ipv6DualStackEnabled {
+		sv, err := semver.Make(k8sVersion)
+		if err != nil {
+			return errors.Errorf("could not validate version %s", k8sVersion)
+		}
+		minVersion, err := semver.Make("1.16.0")
+		if err != nil {
+			return errors.New("could not validate version")
+		}
+		if sv.LT(minVersion) {
+			return errors.Errorf("IPv6 dual stack not available in kubernetes version %s", k8sVersion)
+		}
+		// ipv6 dual stack feature is currently only supported with kubenet
+		if k.NetworkPlugin != "kubenet" {
+			return errors.Errorf("OrchestratorProfile.KubernetesConfig.NetworkPlugin '%s' is invalid. IPv6 dual stack supported only with kubenet.", k.NetworkPlugin)
+		}
 	}
 
 	if k.ClusterSubnet != "" {
@@ -1249,26 +1262,45 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows, ipv6DualStack
 			return errors.Errorf("OrchestratorProfile.KubernetesConfig.DNSServiceIP '%s' is an invalid IP address", k.DNSServiceIP)
 		}
 
-		_, serviceCidr, err := net.ParseCIDR(k.ServiceCidr)
+		primaryServiceCIDR := k.ServiceCidr
+		if ipv6DualStackEnabled {
+			// split the service cidr to see if there are multiple cidrs
+			serviceCidrs := strings.Split(k.ServiceCidr, ",")
+			if len(serviceCidrs) > 2 {
+				return errors.Errorf("OrchestratorProfile.KubernetesConfig.ServiceCidr '%s' is an invalid CIDR subnet. More than 2 CIDRs not allowed for dualstack", k.ServiceCidr)
+			}
+			if len(serviceCidrs) == 2 {
+				firstServiceCIDR, secondServiceCIDR := serviceCidrs[0], serviceCidrs[1]
+				_, _, err := net.ParseCIDR(secondServiceCIDR)
+				if err != nil {
+					return errors.Errorf("OrchestratorProfile.KubernetesConfig.ServiceCidr '%s' is an invalid CIDR subnet", secondServiceCIDR)
+				}
+				// use the primary service cidr for further validation
+				primaryServiceCIDR = firstServiceCIDR
+			}
+			// if # of service cidrs is 1, then continues with the default validation
+		}
+
+		_, serviceCidr, err := net.ParseCIDR(primaryServiceCIDR)
 		if err != nil {
-			return errors.Errorf("OrchestratorProfile.KubernetesConfig.ServiceCidr '%s' is an invalid CIDR subnet", k.ServiceCidr)
+			return errors.Errorf("OrchestratorProfile.KubernetesConfig.ServiceCidr '%s' is an invalid CIDR subnet", primaryServiceCIDR)
 		}
 
 		// Finally validate that the DNS ip is within the subnet
 		if !serviceCidr.Contains(dnsIP) {
-			return errors.Errorf("OrchestratorProfile.KubernetesConfig.DNSServiceIP '%s' is not within the ServiceCidr '%s'", k.DNSServiceIP, k.ServiceCidr)
+			return errors.Errorf("OrchestratorProfile.KubernetesConfig.DNSServiceIP '%s' is not within the ServiceCidr '%s'", k.DNSServiceIP, primaryServiceCIDR)
 		}
 
 		// and that the DNS IP is _not_ the subnet broadcast address
 		broadcast := common.IP4BroadcastAddress(serviceCidr)
 		if dnsIP.Equal(broadcast) {
-			return errors.Errorf("OrchestratorProfile.KubernetesConfig.DNSServiceIP '%s' cannot be the broadcast address of ServiceCidr '%s'", k.DNSServiceIP, k.ServiceCidr)
+			return errors.Errorf("OrchestratorProfile.KubernetesConfig.DNSServiceIP '%s' cannot be the broadcast address of ServiceCidr '%s'", k.DNSServiceIP, primaryServiceCIDR)
 		}
 
 		// and that the DNS IP is _not_ the first IP in the service subnet
 		firstServiceIP := common.CidrFirstIP(serviceCidr.IP)
 		if firstServiceIP.Equal(dnsIP) {
-			return errors.Errorf("OrchestratorProfile.KubernetesConfig.DNSServiceIP '%s' cannot be the first IP of ServiceCidr '%s'", k.DNSServiceIP, k.ServiceCidr)
+			return errors.Errorf("OrchestratorProfile.KubernetesConfig.DNSServiceIP '%s' cannot be the first IP of ServiceCidr '%s'", k.DNSServiceIP, primaryServiceCIDR)
 		}
 	}
 
@@ -1276,6 +1308,11 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows, ipv6DualStack
 		return errors.Errorf("Invalid KubeProxyMode %v. Allowed modes are %v and %v", k.ProxyMode, KubeProxyModeIPTables, KubeProxyModeIPVS)
 	}
 
+	// dualstack is currently supported only with ipvs proxy mode
+	if ipv6DualStackEnabled && k.ProxyMode != KubeProxyModeIPVS {
+		return errors.Errorf("Invalid KubeProxyMode %v. Dualstack supported currently only with %v mode", k.ProxyMode, KubeProxyModeIPVS)
+	}
+
 	// Validate that we have a valid etcd version
 	if e := validateEtcdVersion(k.EtcdVersion); e != nil {
 		return e

pkg/api/vlabs/validate_test.go

@@ -566,8 +566,32 @@ func Test_KubernetesConfig_Validate(t *testing.T) {
 				t.Error("should error when ProxyMode has a valid string value")
 			}
 		}
+	}
 
-		c = KubernetesConfig{
+	// Tests that apply to 1.6 and later releases
+	for _, k8sVersion := range common.GetAllSupportedKubernetesVersions(false, false) {
+		c := KubernetesConfig{
+			CloudProviderBackoff:   to.BoolPtr(true),
+			CloudProviderRateLimit: to.BoolPtr(true),
+		}
+		if err := c.Validate(k8sVersion, false, false); err != nil {
+			t.Error("should not error when basic backoff and rate limiting are set to true with no options")
+		}
+	}
+
+	// Tests that apply to 1.8 and later releases
+	for _, k8sVersion := range common.GetVersionsGt(common.GetAllSupportedKubernetesVersions(true, false), "1.8.0", true, true) {
+		c := KubernetesConfig{
+			UseCloudControllerManager: to.BoolPtr(true),
+		}
+		if err := c.Validate(k8sVersion, false, false); err != nil {
+			t.Error("should not error because UseCloudControllerManager is available since v1.8")
+		}
+	}
+
+	// Tests that apply to dualstack with 1.16 and later releases
+	for _, k8sVersion := range common.GetVersionsGt(common.GetAllSupportedKubernetesVersions(false, false), "1.16.0", true, true) {
+		c := KubernetesConfig{
 			NetworkPlugin: "kubenet",
 			ClusterSubnet: "10.244.0.0/16,ace:cab:deca::/8",
 		}
@@ -593,26 +617,58 @@ func Test_KubernetesConfig_Validate(t *testing.T) {
 		if err := c.Validate(k8sVersion, false, true); err == nil {
 			t.Error("should error when more than 2 cluster subnets provided")
 		}
-	}
 
-	// Tests that apply to 1.6 and later releases
-	for _, k8sVersion := range common.GetAllSupportedKubernetesVersions(false, false) {
-		c := KubernetesConfig{
-			CloudProviderBackoff:   to.BoolPtr(true),
-			CloudProviderRateLimit: to.BoolPtr(true),
+		c = KubernetesConfig{
+			NetworkPlugin: "kubenet",
+			ClusterSubnet: "10.244.0.0/16,ace:cab:deca::/8",
 		}
-		if err := c.Validate(k8sVersion, false, false); err != nil {
-			t.Error("should not error when basic backoff and rate limiting are set to true with no options")
-		}
-	}
 
-	// Tests that apply to 1.8 and later releases
-	for _, k8sVersion := range common.GetVersionsGt(common.GetAllSupportedKubernetesVersions(true, false), "1.8.0", true, true) {
-		c := KubernetesConfig{
-			UseCloudControllerManager: to.BoolPtr(true),
+		if err := c.Validate(k8sVersion, false, true); err == nil {
+			t.Error("should error when proxy mode is not set to ipvs")
 		}
-		if err := c.Validate(k8sVersion, false, false); err != nil {
-			t.Error("should not error because UseCloudControllerManager is available since v1.8")
+
+		c = KubernetesConfig{
+			ServiceCidr: "10.0.0.0/16,fe80:20d::/112",
+		}
+
+		if err := c.Validate(k8sVersion, false, false); err == nil {
+			t.Error("should error when more than 1 service cidr provided with ipv6dualstack feature disabled")
+		}
+
+		c = KubernetesConfig{
+			NetworkPlugin: "kubenet",
+			ClusterSubnet: "10.244.0.0/16,ace:cab:deca::/8",
+			ProxyMode:     "ipvs",
+			ServiceCidr:   "10.0.0.0/16,fe80:20d::/112,fec0::/7",
+			DNSServiceIP:  "10.0.0.10",
+		}
+
+		if err := c.Validate(k8sVersion, false, true); err == nil {
+			t.Error("should error when more than 2 service cidr provided with ipv6dualstack feature enabled")
+		}
+
+		c = KubernetesConfig{
+			NetworkPlugin: "kubenet",
+			ClusterSubnet: "10.244.0.0/16,ace:cab:deca::/8",
+			ProxyMode:     "ipvs",
+			ServiceCidr:   "10.0.0.0/16,2001:db8::/129",
+			DNSServiceIP:  "10.0.0.10",
+		}
+
+		if err := c.Validate(k8sVersion, false, true); err == nil {
+			t.Error("should error when secondary cidr is invalid with ipv6dualstack feature enabled")
+		}
+
+		c = KubernetesConfig{
+			NetworkPlugin: "kubenet",
+			ClusterSubnet: "10.244.0.0/16,ace:cab:deca::/8",
+			ProxyMode:     "ipvs",
+			ServiceCidr:   "10.0.0.0/16,fe80:20d::/112",
+			DNSServiceIP:  "10.0.0.10",
+		}
+
+		if err := c.Validate(k8sVersion, false, true); err != nil {
+			t.Error("shouldn't have errored with ipv6 dual stack feature enabled")
 		}
 	}
 }

pkg/engine/ipaddresses.go

@@ -105,27 +105,3 @@ func CreateClusterPublicIPAddress() PublicIPAddressARM {
 		},
 	}
 }
-
-// CreateClusterPublicIPv6Address returns public ipv6 address resource for cluster
-// ipv6 fe is required to make egress work for ipv6 dual stack. This place holder can
-// be removed in the future once changes are incorporated in the platform.
-// TODO (aramase)
-func CreateClusterPublicIPv6Address() PublicIPAddressARM {
-	return PublicIPAddressARM{
-		ARMResource: ARMResource{
-			APIVersion: "[variables('apiVersionNetwork')]",
-		},
-		PublicIPAddress: network.PublicIPAddress{
-			Location: to.StringPtr("[variables('location')]"),
-			Name:     to.StringPtr("fee-ipv6"),
-			PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
-				PublicIPAllocationMethod: network.Dynamic,
-				PublicIPAddressVersion:   "IPv6",
-			},
-			Sku: &network.PublicIPAddressSku{
-				Name: "[variables('loadBalancerSku')]",
-			},
-			Type: to.StringPtr("Microsoft.Network/publicIPAddresses"),
-		},
-	}
-}

pkg/engine/ipaddresses_test.go

@@ -125,34 +125,6 @@ func TestCreateClusterPublicIPAddress(t *testing.T) {
 	}
 }
 
-func TestCreateClusterPublicIPv6Address(t *testing.T) {
-	expected := PublicIPAddressARM{
-		ARMResource: ARMResource{
-			APIVersion: "[variables('apiVersionNetwork')]",
-		},
-		PublicIPAddress: network.PublicIPAddress{
-			Location: to.StringPtr("[variables('location')]"),
-			Name:     to.StringPtr("fee-ipv6"),
-			PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
-				PublicIPAllocationMethod: network.Dynamic,
-				PublicIPAddressVersion:   "IPv6",
-			},
-			Sku: &network.PublicIPAddressSku{
-				Name: "[variables('loadBalancerSku')]",
-			},
-			Type: to.StringPtr("Microsoft.Network/publicIPAddresses"),
-		},
-	}
-
-	actual := CreateClusterPublicIPv6Address()
-
-	diff := cmp.Diff(actual, expected)
-
-	if diff != "" {
-		t.Errorf("unexpected diff while expecting equal structs: %s", diff)
-	}
-}
-
 func TestCreateAppGwPublicIPAddress(t *testing.T) {
 	expected := PublicIPAddressARM{
 		ARMResource: ARMResource{

pkg/engine/loadbalancers.go

@@ -23,7 +23,6 @@ func CreateClusterLoadBalancerForIPv6() LoadBalancerARM {
 			APIVersion: "[variables('apiVersionNetwork')]",
 			DependsOn: []string{
 				"[concat('Microsoft.Network/publicIPAddresses/', 'fee-ipv4')]",
-				"[concat('Microsoft.Network/publicIPAddresses/', 'fee-ipv6')]",
 			},
 		},
 		LoadBalancer: network.LoadBalancer{
@@ -35,9 +34,6 @@ func CreateClusterLoadBalancerForIPv6() LoadBalancerARM {
 						// cluster name used as backend addr pool name for ipv4 to ensure backward compat
 						Name: to.StringPtr("[parameters('masterEndpointDNSNamePrefix')]"),
 					},
-					{
-						Name: to.StringPtr("[concat(parameters('masterEndpointDNSNamePrefix'), '-ipv6')]"),
-					},
 				},
 				FrontendIPConfigurations: &[]network.FrontendIPConfiguration{
 					{
@@ -48,30 +44,8 @@ func CreateClusterLoadBalancerForIPv6() LoadBalancerARM {
 							},
 						},
 					},
-					{
-						Name: to.StringPtr("LBFE-v6"),
-						FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
-							PublicIPAddress: &network.PublicIPAddress{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/publicIpAddresses', 'fee-ipv6')]"),
-							},
-						},
-					},
 				},
 				LoadBalancingRules: &[]network.LoadBalancingRule{
-					{
-						Name: to.StringPtr("LBRuleIPv6"),
-						LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-							FrontendIPConfiguration: &network.SubResource{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIpConfigurations', parameters('masterEndpointDNSNamePrefix'), 'LBFE-v6')]"),
-							},
-							BackendAddressPool: &network.SubResource{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('masterEndpointDNSNamePrefix'), concat(parameters('masterEndpointDNSNamePrefix'), '-ipv6'))]"),
-							},
-							Protocol:     network.TransportProtocolTCP,
-							FrontendPort: to.Int32Ptr(9090),
-							BackendPort:  to.Int32Ptr(9090),
-						},
-					},
 					{
 						Name: to.StringPtr("LBRuleIPv4"),
 						LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{

pkg/engine/loadbalancers_test.go

@@ -571,7 +571,6 @@ func TestCreateClusterLoadBalancerForIPv6(t *testing.T) {
 			APIVersion: "[variables('apiVersionNetwork')]",
 			DependsOn: []string{
 				"[concat('Microsoft.Network/publicIPAddresses/', 'fee-ipv4')]",
-				"[concat('Microsoft.Network/publicIPAddresses/', 'fee-ipv6')]",
 			},
 		},
 		LoadBalancer: network.LoadBalancer{
@@ -582,9 +581,6 @@ func TestCreateClusterLoadBalancerForIPv6(t *testing.T) {
 					{
 						Name: to.StringPtr("[parameters('masterEndpointDNSNamePrefix')]"),
 					},
-					{
-						Name: to.StringPtr("[concat(parameters('masterEndpointDNSNamePrefix'), '-ipv6')]"),
-					},
 				},
 				FrontendIPConfigurations: &[]network.FrontendIPConfiguration{
 					{
@@ -595,30 +591,8 @@ func TestCreateClusterLoadBalancerForIPv6(t *testing.T) {
 							},
 						},
 					},
-					{
-						Name: to.StringPtr("LBFE-v6"),
-						FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
-							PublicIPAddress: &network.PublicIPAddress{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/publicIpAddresses', 'fee-ipv6')]"),
-							},
-						},
-					},
 				},
 				LoadBalancingRules: &[]network.LoadBalancingRule{
-					{
-						Name: to.StringPtr("LBRuleIPv6"),
-						LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
-							FrontendIPConfiguration: &network.SubResource{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/frontendIpConfigurations', parameters('masterEndpointDNSNamePrefix'), 'LBFE-v6')]"),
-							},
-							BackendAddressPool: &network.SubResource{
-								ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('masterEndpointDNSNamePrefix'), concat(parameters('masterEndpointDNSNamePrefix'), '-ipv6'))]"),
-							},
-							Protocol:     network.TransportProtocolTCP,
-							FrontendPort: to.Int32Ptr(9090),
-							BackendPort:  to.Int32Ptr(9090),
-						},
-					},
 					{
 						Name: to.StringPtr("LBRuleIPv4"),
 						LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{

pkg/engine/masterarmresources.go

@@ -92,11 +92,13 @@ func createKubernetesMasterResourcesVMAS(cs *api.ContainerService) []interface{}
 	}
 
 	if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") {
-		clusterIPv4PublicIPAddress := CreateClusterPublicIPAddress()
-		clusterIPv6PublicIPAddress := CreateClusterPublicIPv6Address()
-		clusterLB := CreateClusterLoadBalancerForIPv6()
+		// for standard lb sku, the loadbalancer and ipv4 FE is already created
+		if cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku != api.StandardLoadBalancerSku {
+			clusterIPv4PublicIPAddress := CreateClusterPublicIPAddress()
+			clusterLB := CreateClusterLoadBalancerForIPv6()
 
-		masterResources = append(masterResources, clusterIPv4PublicIPAddress, clusterIPv6PublicIPAddress, clusterLB)
+			masterResources = append(masterResources, clusterIPv4PublicIPAddress, clusterLB)
+		}
 	}
 
 	masterVM := CreateMasterVM(cs)
@@ -174,11 +176,13 @@ func createKubernetesMasterResourcesVMSS(cs *api.ContainerService) []interface{}
 	}
 
 	if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") {
-		clusterIPv4PublicIPAddress := CreateClusterPublicIPAddress()
-		clusterIPv6PublicIPAddress := CreateClusterPublicIPv6Address()
-		clusterLB := CreateClusterLoadBalancerForIPv6()
+		// for standard lb sku, the loadbalancer and ipv4 FE is already created
+		if cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku != api.StandardLoadBalancerSku {
+			clusterIPv4PublicIPAddress := CreateClusterPublicIPAddress()
+			clusterLB := CreateClusterLoadBalancerForIPv6()
 
-		masterResources = append(masterResources, clusterIPv4PublicIPAddress, clusterIPv6PublicIPAddress, clusterLB)
+			masterResources = append(masterResources, clusterIPv4PublicIPAddress, clusterLB)
+		}
 	}
 
 	masterVmss := CreateMasterVMSS(cs)

pkg/engine/networkinterfaces.go

@@ -378,14 +378,16 @@ func createAgentVMASNetworkInterface(cs *api.ContainerService, profile *api.Agen
 		}
 
 		if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") {
-			var backendPools []network.BackendAddressPool
-			if ipConfig.LoadBalancerBackendAddressPools != nil {
-				backendPools = *ipConfig.LoadBalancerBackendAddressPools
+			if cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku != api.StandardLoadBalancerSku {
+				var backendPools []network.BackendAddressPool
+				if ipConfig.LoadBalancerBackendAddressPools != nil {
+					backendPools = *ipConfig.LoadBalancerBackendAddressPools
+				}
+				backendPools = append(backendPools, network.BackendAddressPool{
+					ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'))]"),
+				})
+				ipConfig.LoadBalancerBackendAddressPools = &backendPools
 			}
-			backendPools = append(backendPools, network.BackendAddressPool{
-				ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'))]"),
-			})
-			ipConfig.LoadBalancerBackendAddressPools = &backendPools
 		}
 		ipConfigurations = append(ipConfigurations, ipConfig)
 	}
@@ -400,11 +402,6 @@ func createAgentVMASNetworkInterface(cs *api.ContainerService, profile *api.Agen
 				Subnet: &network.Subnet{
 					ID: to.StringPtr(fmt.Sprintf("[variables('%sVnetSubnetID')]", profile.Name)),
 				},
-				LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
-					{
-						ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'), '-ipv6')]"),
-					},
-				},
 			},
 		}
 		ipConfigurations = append(ipConfigurations, ipv6Config)

pkg/engine/networkinterfaces_test.go

@@ -1139,11 +1139,6 @@ func TestCreateAgentVMASNICWithIPv6DualStackFeature(t *testing.T) {
 				Subnet: &network.Subnet{
 					ID: to.StringPtr(fmt.Sprintf("[variables('%sVnetSubnetID')]", profile.Name)),
 				},
-				LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{
-					{
-						ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'), '-ipv6')]"),
-					},
-				},
 			},
 		},
 	}

pkg/engine/templates_generated.go

@@ -8301,6 +8301,8 @@ data:
       kubeconfig: /var/lib/kubelet/kubeconfig
     clusterCIDR: "<CIDR>"
     mode: "<kubeProxyMode>"
+    featureGates:
+      <IPv6DualStackFeature>
 metadata:
   name: kube-proxy-config
   namespace: kube-system
@@ -15578,9 +15580,9 @@ MASTER_CONTAINER_ADDONS_PLACEHOLDER
     sed -i "s|<args>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.ControllerManagerConfig}}|g" /etc/kubernetes/manifests/kube-controller-manager.yaml
     sed -i "s|<args>|{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.SchedulerConfig}}|g" /etc/kubernetes/manifests/kube-scheduler.yaml
     {{ if IsIPv6DualStackFeatureEnabled }}
-    sed -i "s|<img>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<CIDR>|',first(split(parameters('kubeClusterCidr'),',')),'|g; s|<kubeProxyMode>|{{ .OrchestratorProfile.KubernetesConfig.ProxyMode}}|g" /etc/kubernetes/addons/kube-proxy-daemonset.yaml
+    sed -i "s|<img>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<CIDR>|{{WrapAsParameter "kubeClusterCidr"}}|g; s|<kubeProxyMode>|{{ .OrchestratorProfile.KubernetesConfig.ProxyMode}}|g; s|<IPv6DualStackFeature>|IPv6DualStack: true|g" /etc/kubernetes/addons/kube-proxy-daemonset.yaml
     {{ else }}
-    sed -i "s|<img>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<CIDR>|{{WrapAsParameter "kubeClusterCidr"}}|g; s|<kubeProxyMode>|{{ .OrchestratorProfile.KubernetesConfig.ProxyMode}}|g" /etc/kubernetes/addons/kube-proxy-daemonset.yaml
+    sed -i "s|<img>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<CIDR>|{{WrapAsParameter "kubeClusterCidr"}}|g; s|<kubeProxyMode>|{{ .OrchestratorProfile.KubernetesConfig.ProxyMode}}|g; s|<IPv6DualStackFeature>|{}|g" /etc/kubernetes/addons/kube-proxy-daemonset.yaml
     {{ end }}
     KUBEDNS=/etc/kubernetes/addons/kube-dns-deployment.yaml
 {{if NeedsKubeDNSWithExecHealthz}}

pkg/engine/virtualmachinescalesets.go

@@ -514,15 +514,17 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
 
 			ipConfigProps.LoadBalancerBackendAddressPools = &backendAddressPools
 			if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") {
-				defaultIPv4BackendPool := compute.SubResource{
-					ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'))]"),
+				if cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku != StandardLoadBalancerSku {
+					defaultIPv4BackendPool := compute.SubResource{
+						ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'))]"),
+					}
+					backendPools := make([]compute.SubResource, 0)
+					if ipConfigProps.LoadBalancerBackendAddressPools != nil {
+						backendPools = *ipConfigProps.LoadBalancerBackendAddressPools
+					}
+					backendPools = append(backendPools, defaultIPv4BackendPool)
+					ipConfigProps.LoadBalancerBackendAddressPools = &backendPools
 				}
-				backendPools := make([]compute.SubResource, 0)
-				if ipConfigProps.LoadBalancerBackendAddressPools != nil {
-					backendPools = *ipConfigProps.LoadBalancerBackendAddressPools
-				}
-				backendPools = append(backendPools, defaultIPv4BackendPool)
-				ipConfigProps.LoadBalancerBackendAddressPools = &backendPools
 			}
 
 			// Set VMSS node public IP if requested
@@ -550,14 +552,6 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
 					PrivateIPAddressVersion: "IPv6",
 				},
 			}
-			if i == 1 {
-				backendPools := make([]compute.SubResource, 0)
-				defaultIPv6BackendPool := compute.SubResource{
-					ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'), '-ipv6')]"),
-				}
-				backendPools = append(backendPools, defaultIPv6BackendPool)
-				ipconfigv6.LoadBalancerBackendAddressPools = &backendPools
-			}
 			ipConfigurations = append(ipConfigurations, ipconfigv6)
 		}
 	}

pkg/engine/virtualmachinescalesets_test.go

@@ -526,6 +526,29 @@ func TestCreateAgentVMSS(t *testing.T) {
 	// Test with ipv6 dual stack enabled
 	cs.Properties.FeatureFlags = &api.FeatureFlags{EnableIPv6DualStack: true}
 	cs.Properties.AgentPoolProfiles[0].OSType = "Linux"
+
+	actual = CreateAgentVMSS(cs, cs.Properties.AgentPoolProfiles[0])
+
+	expected.VirtualMachineProfile.NetworkProfile = &compute.VirtualMachineScaleSetNetworkProfile{
+		NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{
+			{
+				Name: to.StringPtr("[variables('agentpool1VMNamePrefix')]"),
+				VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{
+					Primary:                     to.BoolPtr(true),
+					EnableAcceleratedNetworking: to.BoolPtr(true),
+					IPConfigurations:            getIPConfigs(to.StringPtr("/subscriptions/123/resourceGroups/rg/providers/Microsoft.Network/loadBalancers/mySLB/backendAddressPools/mySLBBEPool"), false, true),
+				},
+			},
+		},
+	}
+
+	diff = cmp.Diff(actual.VirtualMachineProfile.NetworkProfile, expected.VirtualMachineProfile.NetworkProfile)
+
+	if diff != "" {
+		t.Errorf("unexpected diff while expecting equal structs: %s", diff)
+	}
+
+	cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku = api.StandardLoadBalancerSku
 	actual = CreateAgentVMSS(cs, cs.Properties.AgentPoolProfiles[0])
 
 	expected.VirtualMachineProfile.NetworkProfile = &compute.VirtualMachineScaleSetNetworkProfile{
@@ -541,8 +564,6 @@ func TestCreateAgentVMSS(t *testing.T) {
 		},
 	}
 
-	diff = cmp.Diff(actual.VirtualMachineProfile.NetworkProfile, expected.VirtualMachineProfile.NetworkProfile)
-
 	if diff != "" {
 		t.Errorf("unexpected diff while expecting equal structs: %s", diff)
 	}
@@ -816,7 +837,7 @@ func getIPConfigs(lbBackendAddresPoolID *string, isStandardLB, ipv6DualStackEnab
 				}
 			}
 			ipconfig.LoadBalancerBackendAddressPools = &backendAddressPools
-			if ipv6DualStackEnabled {
+			if ipv6DualStackEnabled && !isStandardLB {
 				defaultIPv4BackendPool := compute.SubResource{
 					ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'))]"),
 				}
@@ -842,14 +863,6 @@ func getIPConfigs(lbBackendAddresPoolID *string, isStandardLB, ipv6DualStackEnab
 					PrivateIPAddressVersion: "IPv6",
 				},
 			}
-			if i == 1 {
-				backendPools := make([]compute.SubResource, 0)
-				defaultIPv6BackendPool := compute.SubResource{
-					ID: to.StringPtr("[concat(resourceId('Microsoft.Network/loadBalancers',parameters('masterEndpointDNSNamePrefix')), '/backendAddressPools/', parameters('masterEndpointDNSNamePrefix'), '-ipv6')]"),
-				}
-				backendPools = append(backendPools, defaultIPv6BackendPool)
-				ipconfigv6.LoadBalancerBackendAddressPools = &backendPools
-			}
 			ipConfigs = append(ipConfigs, ipconfigv6)
 		}
 	}