feat: External cloud provider support for Azure Stack Cloud (#4635)

parts/k8s/addons/azure-cloud-provider.yaml

@@ -152,6 +152,7 @@ allowedTopologies:
   {{else}}
 volumeBindingMode: Immediate
   {{- end}}
+  {{- if not IsAzureStackCloud}}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
@@ -165,6 +166,7 @@ parameters:
 reclaimPolicy: Delete
 allowVolumeExpansion: true
 volumeBindingMode: Immediate
+  {{- end}}
 {{else}}
   {{- if NeedsStorageAccountStorageClasses}}
 ---

parts/k8s/addons/cloud-node-manager.yaml

@@ -88,11 +88,22 @@ spec:
         command:
         - cloud-node-manager
         - --node-name=$(NODE_NAME)
+        {{- if IsAzureStackCloud}}
+        - --use-instance-metadata=false
+        - --cloud-config=/etc/kubernetes/azure.json
+        - --kubeconfig=/var/lib/kubelet/kubeconfig
+        {{end}}
         env:
         - name: NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        {{- if IsAzureStackCloud}}
+        - name: AZURE_ENVIRONMENT_FILEPATH
+          value: /etc/kubernetes/azurestackcloud.json
+        - name: AZURE_GO_SDK_LOG_LEVEL
+          value: INFO
+        {{end}}
         resources:
           requests:
             cpu: 50m
@@ -100,6 +111,29 @@ spec:
           limits:
             cpu: 2000m
             memory: 512Mi
+        {{- if IsAzureStackCloud}}
+        volumeMounts:
+        - name: etc-kubernetes
+          mountPath: /etc/kubernetes
+          readOnly: true
+        - name: etc-ssl
+          mountPath: /etc/ssl
+          readOnly: true
+        - name: path-kubeconfig
+          mountPath: /var/lib/kubelet/kubeconfig
+          readOnly: true
+      volumes:
+        - name: etc-kubernetes
+          hostPath:
+            path: /etc/kubernetes
+        - name: etc-ssl
+          hostPath:
+            path: /etc/ssl
+        - name: path-kubeconfig
+          hostPath:
+            path: /var/lib/kubelet/kubeconfig
+            type: FileOrCreate
+        {{end}}
 {{- if and HasWindows (IsKubernetesVersionGe "1.18.0")}}
 ---
 apiVersion: apps/v1
@@ -148,11 +182,27 @@ spec:
         command:
         - /cloud-node-manager.exe
         - --node-name=$(NODE_NAME)
+        - --kubeconfig=C:\k\config
+        {{- if IsAzureStackCloud}}
+        - --use-instance-metadata=false
+        - --cloud-config=C:\k\azure.json
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - C:\k\addazsroot.bat
+        {{end}}
         env:
         - name: NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        {{- if IsAzureStackCloud}}
+        - name: AZURE_ENVIRONMENT_FILEPATH
+          value: C:\k\azurestackcloud.json
+        - name: AZURE_GO_SDK_LOG_LEVEL
+          value: INFO
+        {{end}}
         resources:
           requests:
             cpu: 50m
@@ -160,4 +210,12 @@ spec:
           limits:
             cpu: 2000m
             memory: 512Mi
-{{end}}
+        volumeMounts:
+        - name: azure-config
+          mountPath: C:\k
+      volumes:
+        - name: azure-config
+          hostPath:
+            path: C:\k
+            type: Directory
+{{end}}

parts/k8s/kuberneteswindowssetup.ps1

@@ -448,6 +448,44 @@ try
         Register-NodeResetScriptTask
         Update-DefenderPreferences
 
+        {{if IsAzureStackCloud}}
+            {{if UseCloudControllerManager}}
+            # Export the Azure Stack root cert for use in cloud node manager container setup.
+            $azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json")
+            if (Test-Path -Path $azsConfigFile) {
+                $azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
+                if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
+                    $azsARMUri = [System.Uri]$azsJson.managementPortalURL
+                    $azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
+                    if ($null -ne $azsRootCert) {
+                        $azsRootCertFilePath =  [io.path]::Combine($global:KubeDir, "azsroot.cer")
+                        Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT
+                    } else {
+                        throw "$azsRootCert is null, cannot export Azure Stack root cert"
+                    }
+                } else {
+                    throw "managementPortalURL is null or empty in $azsConfigFile, cannot get Azure Stack ARM uri"
+                }
+            } else {
+                throw "$azsConfigFile does not exist, cannot export Azure Stack root cert"
+            }
+
+            # Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory
+            $certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe")
+            if (Test-Path -Path $certocSourcePath) {
+                Copy-Item -Path $certocSourcePath -Destination $global:KubeDir
+            }
+
+            # Create add cert script
+            $addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat")
+            if ($null -ne $azsRootCert) {
+                [io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
+            } else {
+                throw "$azsRootCertFilePath is null, cannot create add cert script"
+            }
+            {{end}}
+        {{end}}
+
         if (Test-Path $CacheDir)
         {
             Write-Log "Removing aks-engine bits cache directory"
@@ -476,4 +514,4 @@ catch
 
     Write-Error $_
     throw $_
-}
+}

parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml

@@ -13,6 +13,13 @@ spec:
     - name: cloud-controller-manager
       image: {{ContainerImage "cloud-controller-manager"}}
       imagePullPolicy: IfNotPresent
+      {{- if IsAzureStackCloud}}
+      env:
+      - name: AZURE_ENVIRONMENT_FILEPATH
+        value: /etc/kubernetes/azurestackcloud.json
+      - name: AZURE_GO_SDK_LOG_LEVEL
+        value: INFO
+      {{end}}
       command: [{{ContainerConfig "command"}}]
       args: [{{GetCloudControllerManagerArgs}}]
       resources:

pkg/api/addons.go

@@ -581,7 +581,7 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 
 	defaultAzureFileCSIDriverAddonsConfig := KubernetesAddon{
 		Name:    common.AzureFileCSIDriverAddonName,
-		Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName)),
+		Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName) && !cs.Properties.IsAzureStackCloud()),
 		Containers: []KubernetesContainerSpec{
 			{
 				Name:           common.CSIProvisionerContainerName,
@@ -1176,7 +1176,7 @@ func getCSISidecarComponent(csiDriverName, csiSidecarName string, k8sComponents
 // Otherwise, it returns empty string.
 // Azure Stack needs the '-azs' suffix so kube-proxy's manifests uses the custom hyperkube image present in the VHD
 func kubeProxyImageSuffix(cs ContainerService) string {
-	if cs.Properties.IsAzureStackCloud() {
+	if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
 		return common.AzureStackSuffix
 	}
 	return ""

pkg/api/components.go

@@ -307,7 +307,7 @@ func getComponentDefaultContainerImage(component string, cs *ContainerService) s
 
 // componentImageSuffix returns '-azs' if target cloud is Azure Stack. Otherwise, it returns empty string.
 func componentImageSuffix(cs ContainerService) string {
-	if cs.Properties.IsAzureStackCloud() {
+	if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
 		return common.AzureStackSuffix
 	}
 	return ""

pkg/api/k8s_versions.go

@@ -45,8 +45,8 @@ const (
 	csiSnapshotControllerImageReference               string = "oss/kubernetes-csi/snapshot-controller:v2.0.0"
 	csiAzureDiskImageReference                        string = "k8s/csi/azuredisk-csi:v0.7.0"
 	csiAzureFileImageReference                        string = "k8s/csi/azurefile-csi:v0.6.0"
-	azureCloudControllerManagerImageReference         string = "oss/kubernetes/azure-cloud-controller-manager:v0.5.1"
+	azureCloudControllerManagerImageReference         string = "oss/kubernetes/azure-cloud-controller-manager:v1.1.1"
-	azureCloudNodeManagerImageReference               string = "oss/kubernetes/azure-cloud-node-manager:v0.5.1"
+	azureCloudNodeManagerImageReference               string = "oss/kubernetes/azure-cloud-node-manager:v1.1.1"
 	dashboardImageReference                           string = "mcr.microsoft.com/oss/kubernetes/dashboard:v2.0.4" // deprecated
 	dashboardMetricsScraperImageReference             string = "mcr.microsoft.com/oss/kubernetes/metrics-scraper:v1.0.4"
 	kubeFlannelImageReference                         string = "quay.io/coreos/flannel:v0.8.0-amd64"
@@ -571,7 +571,7 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides
 			common.CloudControllerManagerComponentName:        azureCloudControllerManagerImageReference,
 			common.CloudNodeManagerAddonName:                  azureCloudNodeManagerImageReference,
 			common.WindowsArtifactComponentName:               "v" + version + "/windowszip/v" + version + "-1int.zip",
-			common.WindowsArtifactAzureStackComponentName:     "v" + version + common.AzureStackSuffix + "/windowszip/v" + version + common.AzureStackSuffix + "-1int.zip",
+			common.WindowsArtifactAzureStackComponentName:     "v" + version + "/windowszip/v" + version + "-1int.zip",
 			common.DashboardAddonName:                         dashboardImageReference,
 			common.DashboardMetricsScraperContainerName:       dashboardMetricsScraperImageReference,
 			common.ExecHealthZComponentName:                   getDefaultImage(common.ExecHealthZComponentName, kubernetesImageBaseType),
@@ -655,7 +655,7 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides
 			common.CloudControllerManagerComponentName:        azureCloudControllerManagerImageReference,
 			common.CloudNodeManagerAddonName:                  azureCloudNodeManagerImageReference,
 			common.WindowsArtifactComponentName:               "v" + version + "/windowszip/v" + version + "-1int.zip",
-			common.WindowsArtifactAzureStackComponentName:     "v" + version + common.AzureStackSuffix + "/windowszip/v" + version + common.AzureStackSuffix + "-1int.zip",
+			common.WindowsArtifactAzureStackComponentName:     "v" + version + "/windowszip/v" + version + "-1int.zip",
 			common.DashboardAddonName:                         dashboardImageReference,
 			common.DashboardMetricsScraperContainerName:       dashboardMetricsScraperImageReference,
 			common.ExecHealthZComponentName:                   getDefaultImage(common.ExecHealthZComponentName, kubernetesImageBaseType),

pkg/api/types.go

@@ -1919,7 +1919,7 @@ func (p *Properties) GetCustomCloudSourcesList() string {
 
 // GetKubernetesVersion returns the cluster Kubernetes version, with the Azure Stack suffix if Azure Stack Cloud.
 func (p *Properties) GetKubernetesVersion() string {
-	if p.IsAzureStackCloud() {
+	if p.IsAzureStackCloud() && !common.IsKubernetesVersionGe(p.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
 		return p.OrchestratorProfile.OrchestratorVersion + AzureStackSuffix
 	}
 	return p.OrchestratorProfile.OrchestratorVersion
@@ -1930,7 +1930,7 @@ func (p *Properties) GetKubernetesHyperkubeSpec() string {
 	var kubernetesHyperkubeSpec string
 	k8sComponents := GetK8sComponentsByVersionMap(p.OrchestratorProfile.KubernetesConfig)[p.OrchestratorProfile.OrchestratorVersion]
 	kubernetesHyperkubeSpec = p.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + k8sComponents["hyperkube"]
-	if p.IsAzureStackCloud() {
+	if p.IsAzureStackCloud() && !common.IsKubernetesVersionGe(p.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
 		kubernetesHyperkubeSpec = kubernetesHyperkubeSpec + AzureStackSuffix
 	}
 	if p.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {

pkg/engine/template_generator.go

@@ -698,7 +698,7 @@ version = 2
 			hyperkubeImageBase := cs.Properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase
 			k8sComponents := api.GetK8sComponentsByVersionMap(cs.Properties.OrchestratorProfile.KubernetesConfig)[cs.Properties.OrchestratorProfile.OrchestratorVersion]
 			hyperkubeImage := hyperkubeImageBase + k8sComponents[common.Hyperkube]
-			if cs.Properties.IsAzureStackCloud() {
+			if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
 				hyperkubeImage = hyperkubeImage + common.AzureStackSuffix
 			}
 			if cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {

pkg/engine/templates_generated.go

@@ -1848,6 +1848,7 @@ allowedTopologies:
   {{else}}
 volumeBindingMode: Immediate
   {{- end}}
+  {{- if not IsAzureStackCloud}}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
@@ -1861,6 +1862,7 @@ parameters:
 reclaimPolicy: Delete
 allowVolumeExpansion: true
 volumeBindingMode: Immediate
+  {{- end}}
 {{else}}
   {{- if NeedsStorageAccountStorageClasses}}
 ---
@@ -7231,11 +7233,22 @@ spec:
         command:
         - cloud-node-manager
         - --node-name=$(NODE_NAME)
+        {{- if IsAzureStackCloud}}
+        - --use-instance-metadata=false
+        - --cloud-config=/etc/kubernetes/azure.json
+        - --kubeconfig=/var/lib/kubelet/kubeconfig
+        {{end}}
         env:
         - name: NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        {{- if IsAzureStackCloud}}
+        - name: AZURE_ENVIRONMENT_FILEPATH
+          value: /etc/kubernetes/azurestackcloud.json
+        - name: AZURE_GO_SDK_LOG_LEVEL
+          value: INFO
+        {{end}}
         resources:
           requests:
             cpu: 50m
@@ -7243,6 +7256,29 @@ spec:
           limits:
             cpu: 2000m
             memory: 512Mi
+        {{- if IsAzureStackCloud}}
+        volumeMounts:
+        - name: etc-kubernetes
+          mountPath: /etc/kubernetes
+          readOnly: true
+        - name: etc-ssl
+          mountPath: /etc/ssl
+          readOnly: true
+        - name: path-kubeconfig
+          mountPath: /var/lib/kubelet/kubeconfig
+          readOnly: true
+      volumes:
+        - name: etc-kubernetes
+          hostPath:
+            path: /etc/kubernetes
+        - name: etc-ssl
+          hostPath:
+            path: /etc/ssl
+        - name: path-kubeconfig
+          hostPath:
+            path: /var/lib/kubelet/kubeconfig
+            type: FileOrCreate
+        {{end}}
 {{- if and HasWindows (IsKubernetesVersionGe "1.18.0")}}
 ---
 apiVersion: apps/v1
@@ -7291,11 +7327,27 @@ spec:
         command:
         - /cloud-node-manager.exe
         - --node-name=$(NODE_NAME)
+        - --kubeconfig=C:\k\config
+        {{- if IsAzureStackCloud}}
+        - --use-instance-metadata=false
+        - --cloud-config=C:\k\azure.json
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - C:\k\addazsroot.bat
+        {{end}}
         env:
         - name: NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
+        {{- if IsAzureStackCloud}}
+        - name: AZURE_ENVIRONMENT_FILEPATH
+          value: C:\k\azurestackcloud.json
+        - name: AZURE_GO_SDK_LOG_LEVEL
+          value: INFO
+        {{end}}
         resources:
           requests:
             cpu: 50m
@@ -7303,8 +7355,15 @@ spec:
           limits:
             cpu: 2000m
             memory: 512Mi
-{{end}}
+        volumeMounts:
-`)
+        - name: azure-config
+          mountPath: C:\k
+      volumes:
+        - name: azure-config
+          hostPath:
+            path: C:\k
+            type: Directory
+{{end}}`)
 
 func k8sAddonsCloudNodeManagerYamlBytes() ([]byte, error) {
 	return _k8sAddonsCloudNodeManagerYaml, nil
@@ -17602,6 +17661,44 @@ try
         Register-NodeResetScriptTask
         Update-DefenderPreferences
 
+        {{if IsAzureStackCloud}}
+            {{if UseCloudControllerManager}}
+            # Export the Azure Stack root cert for use in cloud node manager container setup.
+            $azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json")
+            if (Test-Path -Path $azsConfigFile) {
+                $azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
+                if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
+                    $azsARMUri = [System.Uri]$azsJson.managementPortalURL
+                    $azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
+                    if ($null -ne $azsRootCert) {
+                        $azsRootCertFilePath =  [io.path]::Combine($global:KubeDir, "azsroot.cer")
+                        Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT
+                    } else {
+                        throw "$azsRootCert is null, cannot export Azure Stack root cert"
+                    }
+                } else {
+                    throw "managementPortalURL is null or empty in $azsConfigFile, cannot get Azure Stack ARM uri"
+                }
+            } else {
+                throw "$azsConfigFile does not exist, cannot export Azure Stack root cert"
+            }
+
+            # Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory
+            $certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe")
+            if (Test-Path -Path $certocSourcePath) {
+                Copy-Item -Path $certocSourcePath -Destination $global:KubeDir
+            }
+
+            # Create add cert script
+            $addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat")
+            if ($null -ne $azsRootCert) {
+                [io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
+            } else {
+                throw "$azsRootCertFilePath is null, cannot create add cert script"
+            }
+            {{end}}
+        {{end}}
+
         if (Test-Path $CacheDir)
         {
             Write-Log "Removing aks-engine bits cache directory"
@@ -17630,8 +17727,7 @@ catch
 
     Write-Error $_
     throw $_
-}
+}`)
-`)
 
 func k8sKuberneteswindowssetupPs1Bytes() ([]byte, error) {
 	return _k8sKuberneteswindowssetupPs1, nil
@@ -17732,6 +17828,13 @@ spec:
     - name: cloud-controller-manager
       image: {{ContainerImage "cloud-controller-manager"}}
       imagePullPolicy: IfNotPresent
+      {{- if IsAzureStackCloud}}
+      env:
+      - name: AZURE_ENVIRONMENT_FILEPATH
+        value: /etc/kubernetes/azurestackcloud.json
+      - name: AZURE_GO_SDK_LOG_LEVEL
+        value: INFO
+      {{end}}
       command: [{{ContainerConfig "command"}}]
       args: [{{GetCloudControllerManagerArgs}}]
       resources:

vhd/packer/install-dependencies.sh

@@ -247,7 +247,7 @@ done
 
 # Starting with 1.16 we pull cloud-controller-manager and cloud-node-manager
 CLOUD_MANAGER_VERSIONS="
-0.5.1
+1.1.1
 "
 for CLOUD_MANAGER_VERSION in ${CLOUD_MANAGER_VERSIONS}; do
   for COMPONENT in azure-cloud-controller-manager azure-cloud-node-manager; do