chore: add support for Kubernetes v1.21.10 and v1.22.7 on Azure Stack Hub (#4846)

2022-02-24 12:34:37 -08:00 · 2022-02-24 12:34:37 -08:00 · 765e5de378
--- a/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml
+++ b/parts/k8s/addons/azuredisk-csi-driver-deployment.yaml
@ -357,6 +357,10 @@ spec:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
+            {{- if IsAzureStackCloud}}
+            - name: AZURE_ENVIRONMENT_FILEPATH
+              value: C:\k\azurestackcloud.json
+            {{end}}
          volumeMounts:
            - name: kubelet-dir
              mountPath: "C:\\var\\lib\\kubelet"
@ -534,6 +538,10 @@ spec:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
+            {{- if IsAzureStackCloud}}
+            - name: AZURE_ENVIRONMENT_FILEPATH
+              value: /etc/kubernetes/azurestackcloud.json
+            {{end}}
          securityContext:
            privileged: true
          volumeMounts:
@ -550,6 +558,11 @@ spec:
              name: sys-devices-dir
            - mountPath: /sys/class/scsi_host/
              name: scsi-host-dir
+            {{- if IsAzureStackCloud}}
+            - mountPath: /etc/ssl/certs
+              readOnly: true
+              name: ssl
+            {{end}}
          resources:
            limits:
              cpu: {{ContainerCPULimits "azuredisk-csi"}}
@ -586,6 +599,12 @@ spec:
            path: /sys/class/scsi_host/
            type: Directory
          name: scsi-host-dir
+        {{- if IsAzureStackCloud}}
+        - hostPath:
+            path: /etc/ssl/certs
+            type: Directory
+          name: ssl
+        {{end}}
 {{end}}
 ---
 # Source: azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml
@ -751,11 +770,20 @@ spec:
              value: "/etc/kubernetes/azure.json"
            - name: CSI_ENDPOINT
              value: unix:///csi/csi.sock
+            {{- if IsAzureStackCloud}}
+            - name: AZURE_ENVIRONMENT_FILEPATH
+              value: /etc/kubernetes/azurestackcloud.json
+             {{end}}
          volumeMounts:
            - mountPath: /csi
              name: socket-dir
            - mountPath: /etc/kubernetes/
              name: azure-cred
+            {{- if IsAzureStackCloud}}
+            - mountPath: /etc/ssl/certs
+              readOnly: true
+              name: ssl
+            {{end}}
          resources:
            limits:
              cpu: {{ContainerCPULimits "azuredisk-csi"}}
@ -770,6 +798,12 @@ spec:
          hostPath:
            path: /etc/kubernetes/
            type: DirectoryOrCreate
+        {{- if IsAzureStackCloud}}
+        - hostPath:
+            path: /etc/ssl/certs
+            type: Directory
+          name: ssl
+        {{end}}
 {{if ShouldEnableCSISnapshotFeature "azuredisk-csi-driver"}}
 ---
 # Source: azuredisk-csi-driver/templates/csi-snapshot-controller.yaml
--- a/parts/k8s/kuberneteswindowssetup.ps1
+++ b/parts/k8s/kuberneteswindowssetup.ps1
@ -452,37 +452,34 @@ try
            {{if UseCloudControllerManager}}
            # Export the Azure Stack root cert for use in cloud node manager container setup.
            $azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json")
-            if (Test-Path -Path $azsConfigFile) {
-                $azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
-                if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
-                    $azsARMUri = [System.Uri]$azsJson.managementPortalURL
-                    $azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
-                    if ($null -ne $azsRootCert) {
-                        $azsRootCertFilePath =  [io.path]::Combine($global:KubeDir, "azsroot.cer")
-                        Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT
-                    } else {
-                        throw "$azsRootCert is null, cannot export Azure Stack root cert"
-                    }
-                } else {
-                    throw "managementPortalURL is null or empty in $azsConfigFile, cannot get Azure Stack ARM uri"
-                }
-            } else {
+            if (-not (Test-Path -Path $azsConfigFile)) {
                throw "$azsConfigFile does not exist, cannot export Azure Stack root cert"
            }
+            $azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
+            if ([string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
+                throw "managementPortalURL is empty, cannot get Azure Stack ARM uri"
+            }
+            $azsARMUri = [System.Uri]$azsJson.managementPortalURL
+            $azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList.Unicode -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
+            if ($null -eq $azsRootCert) {
+                throw "$azsRootCert is null, cannot export Azure Stack root cert"
+            }
+            $azsRootCertFilePath =  [io.path]::Combine($global:KubeDir, "azsroot.cer")
+            Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT

            # Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory
            $certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe")
-            if (Test-Path -Path $certocSourcePath) {
-                Copy-Item -Path $certocSourcePath -Destination $global:KubeDir
+            if (-not (Test-Path -Path $certocSourcePath)) {
+                throw "$certocSourcePath does not exist, cannot export Azure Stack root cert"
            }
+            Copy-Item -Path $certocSourcePath -Destination $global:KubeDir

            # Create add cert script
            $addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat")
-            if ($null -ne $azsRootCert) {
-                [io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
-            } else {
+            if ($null -eq $azsRootCert) {
                throw "$azsRootCertFilePath is null, cannot create add cert script"
            }
+            [io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
            {{end}}
        {{end}}

--- a/pkg/api/common/const.go
+++ b/pkg/api/common/const.go
@ -61,9 +61,9 @@ const (
 	// KubernetesDefaultReleaseWindows is the default Kubernetes release for Windows
 	KubernetesDefaultReleaseWindows string = "1.19"
 	// KubernetesDefaultReleaseAzureStack is the default Kubernetes release on Azure Stack
-	KubernetesDefaultReleaseAzureStack string = "1.19"
+	KubernetesDefaultReleaseAzureStack string = "1.21"
 	// KubernetesDefaultReleaseWindowsAzureStack is the default Kubernetes release for Windows on Azure Stack
-	KubernetesDefaultReleaseWindowsAzureStack string = "1.19"
+	KubernetesDefaultReleaseWindowsAzureStack string = "1.21"
 )

 const LegacyControlPlaneVMPrefix string = "k8s-master"
--- a/pkg/api/common/versions.go
+++ b/pkg/api/common/versions.go
@ -347,9 +347,11 @@ var AllKubernetesSupportedVersionsAzureStack = map[string]bool{
 	"1.18.15": false,
 	"1.18.18": false,
 	"1.19.10": false,
-	"1.19.15": true,
+	"1.19.15": false,
 	"1.20.6":  false,
-	"1.20.11": true,
+	"1.20.11": false,
+	"1.21.10": true,
+	"1.22.7":  true,
 }

 // AllKubernetesWindowsSupportedVersionsAzureStack maintain a set of available k8s Windows versions in aks-engine on Azure Stack
@ -374,9 +376,11 @@ var AllKubernetesWindowsSupportedVersionsAzureStack = map[string]bool{
 	"1.18.15": false,
 	"1.18.18": false,
 	"1.19.10": false,
-	"1.19.15": true,
+	"1.19.15": false,
 	"1.20.6":  false,
-	"1.20.11": true,
+	"1.20.11": false,
+	"1.21.10": true,
+	"1.22.7":  true,
 }

 // GetDefaultKubernetesVersion returns the default Kubernetes version, that is the latest patch of the default release
--- a/pkg/api/components.go
+++ b/pkg/api/components.go
@ -270,7 +270,8 @@ func getComponentDefaultContainerImage(component string, cs *ContainerService) s
 	k8sComponents := GetK8sComponentsByVersionMap(kubernetesConfig)[cs.Properties.OrchestratorProfile.OrchestratorVersion]
 	hyperkubeImageBase := kubernetesImageBase
 	hyperkubeImage := hyperkubeImageBase + k8sComponents[common.Hyperkube]
-	if cs.Properties.IsAzureStackCloud() {
+	// For Azure Stack Hub clusters, no custom hyperkubeImage with "-azs" will be used for Kubernetes 1.21+
+	if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
 		hyperkubeImage = hyperkubeImage + common.AzureStackSuffix
 	}
 	controllerManagerBase := kubernetesImageBase
--- a/pkg/api/types.go
+++ b/pkg/api/types.go
@ -1232,6 +1232,10 @@ func (p *Properties) ShouldEnableAzureCloudAddon(addonName string) bool {
 	if !to.Bool(o.KubernetesConfig.UseCloudControllerManager) {
 		return false
 	}
+	// For Azure Stack Hub clusters, azuredisk-csi driver will not be enabled by default when cloud-controller-manager is enabled due to custom data oversize
+	if addonName == common.AzureDiskCSIDriverAddonName && p.IsAzureStackCloud() {
+		return false
+	}
 	if !p.HasWindows() {
 		switch addonName {
 		case common.AzureDiskCSIDriverAddonName, common.AzureFileCSIDriverAddonName:
--- a/pkg/api/vlabs/types.go
+++ b/pkg/api/vlabs/types.go
@ -809,6 +809,9 @@ func (p *Properties) ShouldEnableAzureCloudAddon(addonName string) bool {
 	if !to.Bool(o.KubernetesConfig.UseCloudControllerManager) {
 		return false
 	}
+	if addonName == common.AzureDiskCSIDriverAddonName && p.IsAzureStackCloud() {
+		return false
+	}
 	if !p.HasWindows() {
 		switch addonName {
 		case common.AzureDiskCSIDriverAddonName, common.AzureFileCSIDriverAddonName:
--- a/pkg/engine/params_k8s.go
+++ b/pkg/engine/params_k8s.go
@ -80,7 +80,7 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
 			// will be removed in future release as if gets phased out (https://github.com/Azure/aks-engine/issues/3851)
 			kubeBinariesSASURL := kubernetesConfig.CustomWindowsPackageURL
 			if kubeBinariesSASURL == "" {
-				if properties.IsAzureStackCloud() {
+				if properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
 					kubeBinariesSASURL = cloudSpecConfig.KubernetesSpecConfig.KubeBinariesSASURLBase + k8sComponents[common.WindowsArtifactAzureStackComponentName]
 				} else {
 					kubeBinariesSASURL = cloudSpecConfig.KubernetesSpecConfig.KubeBinariesSASURLBase + k8sComponents[common.WindowsArtifactComponentName]
--- a/pkg/engine/templates_generated.go
+++ b/pkg/engine/templates_generated.go
@ -7654,6 +7654,10 @@ spec:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
+            {{- if IsAzureStackCloud}}
+            - name: AZURE_ENVIRONMENT_FILEPATH
+              value: C:\k\azurestackcloud.json
+            {{end}}
          volumeMounts:
            - name: kubelet-dir
              mountPath: "C:\\var\\lib\\kubelet"
@ -7831,6 +7835,10 @@ spec:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
+            {{- if IsAzureStackCloud}}
+            - name: AZURE_ENVIRONMENT_FILEPATH
+              value: /etc/kubernetes/azurestackcloud.json
+            {{end}}
          securityContext:
            privileged: true
          volumeMounts:
@ -7847,6 +7855,11 @@ spec:
              name: sys-devices-dir
            - mountPath: /sys/class/scsi_host/
              name: scsi-host-dir
+            {{- if IsAzureStackCloud}}
+            - mountPath: /etc/ssl/certs
+              readOnly: true
+              name: ssl
+            {{end}}
          resources:
            limits:
              cpu: {{ContainerCPULimits "azuredisk-csi"}}
@ -7883,6 +7896,12 @@ spec:
            path: /sys/class/scsi_host/
            type: Directory
          name: scsi-host-dir
+        {{- if IsAzureStackCloud}}
+        - hostPath:
+            path: /etc/ssl/certs
+            type: Directory
+          name: ssl
+        {{end}}
 {{end}}
 ---
 # Source: azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml
@ -8048,11 +8067,20 @@ spec:
              value: "/etc/kubernetes/azure.json"
            - name: CSI_ENDPOINT
              value: unix:///csi/csi.sock
+            {{- if IsAzureStackCloud}}
+            - name: AZURE_ENVIRONMENT_FILEPATH
+              value: /etc/kubernetes/azurestackcloud.json
+             {{end}}
          volumeMounts:
            - mountPath: /csi
              name: socket-dir
            - mountPath: /etc/kubernetes/
              name: azure-cred
+            {{- if IsAzureStackCloud}}
+            - mountPath: /etc/ssl/certs
+              readOnly: true
+              name: ssl
+            {{end}}
          resources:
            limits:
              cpu: {{ContainerCPULimits "azuredisk-csi"}}
@ -8067,6 +8095,12 @@ spec:
          hostPath:
            path: /etc/kubernetes/
            type: DirectoryOrCreate
+        {{- if IsAzureStackCloud}}
+        - hostPath:
+            path: /etc/ssl/certs
+            type: Directory
+          name: ssl
+        {{end}}
 {{if ShouldEnableCSISnapshotFeature "azuredisk-csi-driver"}}
 ---
 # Source: azuredisk-csi-driver/templates/csi-snapshot-controller.yaml
@ -22114,37 +22148,34 @@ try
            {{if UseCloudControllerManager}}
            # Export the Azure Stack root cert for use in cloud node manager container setup.
            $azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json")
-            if (Test-Path -Path $azsConfigFile) {
-                $azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
-                if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
-                    $azsARMUri = [System.Uri]$azsJson.managementPortalURL
-                    $azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
-                    if ($null -ne $azsRootCert) {
-                        $azsRootCertFilePath =  [io.path]::Combine($global:KubeDir, "azsroot.cer")
-                        Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT
-                    } else {
-                        throw "$azsRootCert is null, cannot export Azure Stack root cert"
-                    }
-                } else {
-                    throw "managementPortalURL is null or empty in $azsConfigFile, cannot get Azure Stack ARM uri"
-                }
-            } else {
+            if (-not (Test-Path -Path $azsConfigFile)) {
                throw "$azsConfigFile does not exist, cannot export Azure Stack root cert"
            }
+            $azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
+            if ([string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
+                throw "managementPortalURL is empty, cannot get Azure Stack ARM uri"
+            }
+            $azsARMUri = [System.Uri]$azsJson.managementPortalURL
+            $azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList.Unicode -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
+            if ($null -eq $azsRootCert) {
+                throw "$azsRootCert is null, cannot export Azure Stack root cert"
+            }
+            $azsRootCertFilePath =  [io.path]::Combine($global:KubeDir, "azsroot.cer")
+            Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT

            # Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory
            $certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe")
-            if (Test-Path -Path $certocSourcePath) {
-                Copy-Item -Path $certocSourcePath -Destination $global:KubeDir
+            if (-not (Test-Path -Path $certocSourcePath)) {
+                throw "$certocSourcePath does not exist, cannot export Azure Stack root cert"
            }
+            Copy-Item -Path $certocSourcePath -Destination $global:KubeDir

            # Create add cert script
            $addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat")
-            if ($null -ne $azsRootCert) {
-                [io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
-            } else {
+            if ($null -eq $azsRootCert) {
                throw "$azsRootCertFilePath is null, cannot create add cert script"
            }
+            [io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
            {{end}}
        {{end}}

--- a/vhd/packer/configure-windows-vhd.ps1
+++ b/vhd/packer/configure-windows-vhd.ps1
@ -118,8 +118,6 @@ function Get-FilesToCacheOnVHD {
            "https://kubernetesartifacts.azureedge.net/csi-proxy/v0.2.2/binaries/csi-proxy-v0.2.2.tar.gz"
        );
        "c:\akse-cache\win-k8s\"      = @(
-            "https://kubernetesartifacts.azureedge.net/kubernetes/v1.19.15-azs/windowszip/v1.19.15-azs-1int.zip",
-            "https://kubernetesartifacts.azureedge.net/kubernetes/v1.20.11-azs/windowszip/v1.20.11-azs-1int.zip",
            "https://kubernetesartifacts.azureedge.net/kubernetes/v1.19.16/windowszip/v1.19.16-1int.zip",
            "https://kubernetesartifacts.azureedge.net/kubernetes/v1.20.15/windowszip/v1.20.15-1int.zip",
            "https://kubernetesartifacts.azureedge.net/kubernetes/v1.21.10/windowszip/v1.21.10-1int.zip",
--- a/vhd/packer/install-dependencies.sh
+++ b/vhd/packer/install-dependencies.sh
@ -224,9 +224,7 @@ K8S_VERSIONS="
 1.22.7
 1.21.10
 1.20.15
-1.20.11-azs
 1.19.16
-1.19.15-azs
 "
 for KUBERNETES_VERSION in ${K8S_VERSIONS}; do
  for component in kube-apiserver kube-controller-manager kube-proxy kube-scheduler; do