Azure
/
aks-engine
зеркало из https://github.com/Azure/aks-engine.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

feat: External cloud provider support for Azure Stack Cloud (#4635)

This commit is contained in:
haofan-ms 2021-09-03 10:03:30 -07:00 коммит произвёл GitHub
Родитель a3c4663274
Коммит d0da828ba3
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
11 изменённых файлов: 225 добавлений и 17 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
parts/k8s/addons/azure-cloud-provider.yaml
Просмотреть файл

@ -152,6 +152,7 @@ allowedTopologies:
{{else}}
volumeBindingMode: Immediate
{{- end}}
{{- if not IsAzureStackCloud}}
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
@ -165,6 +166,7 @@ parameters:
reclaimPolicy: Delete
allowVolumeExpansion: true
volumeBindingMode: Immediate
{{- end}}
{{else}}
{{- if NeedsStorageAccountStorageClasses}}
---

60
parts/k8s/addons/cloud-node-manager.yaml
Просмотреть файл

@ -88,11 +88,22 @@ spec:
command:
- cloud-node-manager
- --node-name=$(NODE_NAME)
{{- if IsAzureStackCloud}}
- --use-instance-metadata=false
- --cloud-config=/etc/kubernetes/azure.json
- --kubeconfig=/var/lib/kubelet/kubeconfig
{{end}}
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
{{- if IsAzureStackCloud}}
- name: AZURE_ENVIRONMENT_FILEPATH
value: /etc/kubernetes/azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: INFO
{{end}}
resources:
requests:
cpu: 50m
@ -100,6 +111,29 @@ spec:
limits:
cpu: 2000m
memory: 512Mi
{{- if IsAzureStackCloud}}
volumeMounts:
- name: etc-kubernetes
mountPath: /etc/kubernetes
readOnly: true
- name: etc-ssl
mountPath: /etc/ssl
readOnly: true
- name: path-kubeconfig
mountPath: /var/lib/kubelet/kubeconfig
readOnly: true
volumes:
- name: etc-kubernetes
hostPath:
path: /etc/kubernetes
- name: etc-ssl
hostPath:
path: /etc/ssl
- name: path-kubeconfig
hostPath:
path: /var/lib/kubelet/kubeconfig
type: FileOrCreate
{{end}}
{{- if and HasWindows (IsKubernetesVersionGe "1.18.0")}}
---
apiVersion: apps/v1
@ -148,11 +182,27 @@ spec:
command:
- /cloud-node-manager.exe
- --node-name=$(NODE_NAME)
- --kubeconfig=C:\k\config
{{- if IsAzureStackCloud}}
- --use-instance-metadata=false
- --cloud-config=C:\k\azure.json
lifecycle:
postStart:
exec:
command:
- C:\k\addazsroot.bat
{{end}}
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
{{- if IsAzureStackCloud}}
- name: AZURE_ENVIRONMENT_FILEPATH
value: C:\k\azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: INFO
{{end}}
resources:
requests:
cpu: 50m
@ -160,4 +210,12 @@ spec:
limits:
cpu: 2000m
memory: 512Mi
{{end}}
volumeMounts:
- name: azure-config
mountPath: C:\k
volumes:
- name: azure-config
hostPath:
path: C:\k
type: Directory
{{end}}

40
parts/k8s/kuberneteswindowssetup.ps1
Просмотреть файл

@ -448,6 +448,44 @@ try
Register-NodeResetScriptTask
Update-DefenderPreferences
{{if IsAzureStackCloud}}
{{if UseCloudControllerManager}}
# Export the Azure Stack root cert for use in cloud node manager container setup.
$azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json")
if (Test-Path -Path $azsConfigFile) {
$azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
$azsARMUri = [System.Uri]$azsJson.managementPortalURL
$azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
if ($null -ne $azsRootCert) {
$azsRootCertFilePath = [io.path]::Combine($global:KubeDir, "azsroot.cer")
Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT
} else {
throw "$azsRootCert is null, cannot export Azure Stack root cert"
}
} else {
throw "managementPortalURL is null or empty in $azsConfigFile, cannot get Azure Stack ARM uri"
}
} else {
throw "$azsConfigFile does not exist, cannot export Azure Stack root cert"
}
# Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory
$certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe")
if (Test-Path -Path $certocSourcePath) {
Copy-Item -Path $certocSourcePath -Destination $global:KubeDir
}
# Create add cert script
$addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat")
if ($null -ne $azsRootCert) {
[io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
} else {
throw "$azsRootCertFilePath is null, cannot create add cert script"
}
{{end}}
{{end}}
if (Test-Path $CacheDir)
{
Write-Log "Removing aks-engine bits cache directory"
@ -476,4 +514,4 @@ catch
Write-Error $_
throw $_
}
}

7
parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml
Просмотреть файл

@ -13,6 +13,13 @@ spec:
- name: cloud-controller-manager
image: {{ContainerImage "cloud-controller-manager"}}
imagePullPolicy: IfNotPresent
{{- if IsAzureStackCloud}}
env:
- name: AZURE_ENVIRONMENT_FILEPATH
value: /etc/kubernetes/azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: INFO
{{end}}
command: [{{ContainerConfig "command"}}]
args: [{{GetCloudControllerManagerArgs}}]
resources:

4
pkg/api/addons.go
Просмотреть файл

@ -581,7 +581,7 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
defaultAzureFileCSIDriverAddonsConfig := KubernetesAddon{
Name: common.AzureFileCSIDriverAddonName,
Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName)),
Enabled: to.BoolPtr(DefaultAzureFileCSIDriverAddonEnabled && cs.Properties.ShouldEnableAzureCloudAddon(common.AzureFileCSIDriverAddonName) && !cs.Properties.IsAzureStackCloud()),
Containers: []KubernetesContainerSpec{
{
Name: common.CSIProvisionerContainerName,
@ -1176,7 +1176,7 @@ func getCSISidecarComponent(csiDriverName, csiSidecarName string, k8sComponents
// Otherwise, it returns empty string.
// Azure Stack needs the '-azs' suffix so kube-proxy's manifests uses the custom hyperkube image present in the VHD
func kubeProxyImageSuffix(cs ContainerService) string {
if cs.Properties.IsAzureStackCloud() {
if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
return common.AzureStackSuffix
}
return ""

2
pkg/api/components.go
Просмотреть файл

@ -307,7 +307,7 @@ func getComponentDefaultContainerImage(component string, cs *ContainerService) s
// componentImageSuffix returns '-azs' if target cloud is Azure Stack. Otherwise, it returns empty string.
func componentImageSuffix(cs ContainerService) string {
if cs.Properties.IsAzureStackCloud() {
if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
return common.AzureStackSuffix
}
return ""

8
pkg/api/k8s_versions.go
Просмотреть файл

@ -45,8 +45,8 @@ const (
csiSnapshotControllerImageReference string = "oss/kubernetes-csi/snapshot-controller:v2.0.0"
csiAzureDiskImageReference string = "k8s/csi/azuredisk-csi:v0.7.0"
csiAzureFileImageReference string = "k8s/csi/azurefile-csi:v0.6.0"
azureCloudControllerManagerImageReference string = "oss/kubernetes/azure-cloud-controller-manager:v0.5.1"
azureCloudNodeManagerImageReference string = "oss/kubernetes/azure-cloud-node-manager:v0.5.1"
azureCloudControllerManagerImageReference string = "oss/kubernetes/azure-cloud-controller-manager:v1.1.1"
azureCloudNodeManagerImageReference string = "oss/kubernetes/azure-cloud-node-manager:v1.1.1"
dashboardImageReference string = "mcr.microsoft.com/oss/kubernetes/dashboard:v2.0.4" // deprecated
dashboardMetricsScraperImageReference string = "mcr.microsoft.com/oss/kubernetes/metrics-scraper:v1.0.4"
kubeFlannelImageReference string = "quay.io/coreos/flannel:v0.8.0-amd64"
@ -571,7 +571,7 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides
common.CloudControllerManagerComponentName: azureCloudControllerManagerImageReference,
common.CloudNodeManagerAddonName: azureCloudNodeManagerImageReference,
common.WindowsArtifactComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip",
common.WindowsArtifactAzureStackComponentName: "v" + version + common.AzureStackSuffix + "/windowszip/v" + version + common.AzureStackSuffix + "-1int.zip",
common.WindowsArtifactAzureStackComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip",
common.DashboardAddonName: dashboardImageReference,
common.DashboardMetricsScraperContainerName: dashboardMetricsScraperImageReference,
common.ExecHealthZComponentName: getDefaultImage(common.ExecHealthZComponentName, kubernetesImageBaseType),
@ -655,7 +655,7 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides
common.CloudControllerManagerComponentName: azureCloudControllerManagerImageReference,
common.CloudNodeManagerAddonName: azureCloudNodeManagerImageReference,
common.WindowsArtifactComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip",
common.WindowsArtifactAzureStackComponentName: "v" + version + common.AzureStackSuffix + "/windowszip/v" + version + common.AzureStackSuffix + "-1int.zip",
common.WindowsArtifactAzureStackComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip",
common.DashboardAddonName: dashboardImageReference,
common.DashboardMetricsScraperContainerName: dashboardMetricsScraperImageReference,
common.ExecHealthZComponentName: getDefaultImage(common.ExecHealthZComponentName, kubernetesImageBaseType),

4
pkg/api/types.go
Просмотреть файл

@ -1919,7 +1919,7 @@ func (p *Properties) GetCustomCloudSourcesList() string {
// GetKubernetesVersion returns the cluster Kubernetes version, with the Azure Stack suffix if Azure Stack Cloud.
func (p *Properties) GetKubernetesVersion() string {
if p.IsAzureStackCloud() {
if p.IsAzureStackCloud() && !common.IsKubernetesVersionGe(p.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
return p.OrchestratorProfile.OrchestratorVersion + AzureStackSuffix
}
return p.OrchestratorProfile.OrchestratorVersion
@ -1930,7 +1930,7 @@ func (p *Properties) GetKubernetesHyperkubeSpec() string {
var kubernetesHyperkubeSpec string
k8sComponents := GetK8sComponentsByVersionMap(p.OrchestratorProfile.KubernetesConfig)[p.OrchestratorProfile.OrchestratorVersion]
kubernetesHyperkubeSpec = p.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + k8sComponents["hyperkube"]
if p.IsAzureStackCloud() {
if p.IsAzureStackCloud() && !common.IsKubernetesVersionGe(p.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
kubernetesHyperkubeSpec = kubernetesHyperkubeSpec + AzureStackSuffix
}
if p.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {

2
pkg/engine/template_generator.go
Просмотреть файл

@ -698,7 +698,7 @@ version = 2
hyperkubeImageBase := cs.Properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase
k8sComponents := api.GetK8sComponentsByVersionMap(cs.Properties.OrchestratorProfile.KubernetesConfig)[cs.Properties.OrchestratorProfile.OrchestratorVersion]
hyperkubeImage := hyperkubeImageBase + k8sComponents[common.Hyperkube]
if cs.Properties.IsAzureStackCloud() {
if cs.Properties.IsAzureStackCloud() && !common.IsKubernetesVersionGe(cs.Properties.OrchestratorProfile.OrchestratorVersion, "1.21.0") {
hyperkubeImage = hyperkubeImage + common.AzureStackSuffix
}
if cs.Properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" {

111
pkg/engine/templates_generated.go
Просмотреть файл

@ -1848,6 +1848,7 @@ allowedTopologies:
{{else}}
volumeBindingMode: Immediate
{{- end}}
{{- if not IsAzureStackCloud}}
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
@ -1861,6 +1862,7 @@ parameters:
reclaimPolicy: Delete
allowVolumeExpansion: true
volumeBindingMode: Immediate
{{- end}}
{{else}}
{{- if NeedsStorageAccountStorageClasses}}
---
@ -7231,11 +7233,22 @@ spec:
command:
- cloud-node-manager
- --node-name=$(NODE_NAME)
{{- if IsAzureStackCloud}}
- --use-instance-metadata=false
- --cloud-config=/etc/kubernetes/azure.json
- --kubeconfig=/var/lib/kubelet/kubeconfig
{{end}}
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
{{- if IsAzureStackCloud}}
- name: AZURE_ENVIRONMENT_FILEPATH
value: /etc/kubernetes/azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: INFO
{{end}}
resources:
requests:
cpu: 50m
@ -7243,6 +7256,29 @@ spec:
limits:
cpu: 2000m
memory: 512Mi
{{- if IsAzureStackCloud}}
volumeMounts:
- name: etc-kubernetes
mountPath: /etc/kubernetes
readOnly: true
- name: etc-ssl
mountPath: /etc/ssl
readOnly: true
- name: path-kubeconfig
mountPath: /var/lib/kubelet/kubeconfig
readOnly: true
volumes:
- name: etc-kubernetes
hostPath:
path: /etc/kubernetes
- name: etc-ssl
hostPath:
path: /etc/ssl
- name: path-kubeconfig
hostPath:
path: /var/lib/kubelet/kubeconfig
type: FileOrCreate
{{end}}
{{- if and HasWindows (IsKubernetesVersionGe "1.18.0")}}
---
apiVersion: apps/v1
@ -7291,11 +7327,27 @@ spec:
command:
- /cloud-node-manager.exe
- --node-name=$(NODE_NAME)
- --kubeconfig=C:\k\config
{{- if IsAzureStackCloud}}
- --use-instance-metadata=false
- --cloud-config=C:\k\azure.json
lifecycle:
postStart:
exec:
command:
- C:\k\addazsroot.bat
{{end}}
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
{{- if IsAzureStackCloud}}
- name: AZURE_ENVIRONMENT_FILEPATH
value: C:\k\azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: INFO
{{end}}
resources:
requests:
cpu: 50m
@ -7303,8 +7355,15 @@ spec:
limits:
cpu: 2000m
memory: 512Mi
{{end}}
`)
volumeMounts:
- name: azure-config
mountPath: C:\k
volumes:
- name: azure-config
hostPath:
path: C:\k
type: Directory
{{end}}`)
func k8sAddonsCloudNodeManagerYamlBytes() ([]byte, error) {
return _k8sAddonsCloudNodeManagerYaml, nil
@ -17602,6 +17661,44 @@ try
Register-NodeResetScriptTask
Update-DefenderPreferences
{{if IsAzureStackCloud}}
{{if UseCloudControllerManager}}
# Export the Azure Stack root cert for use in cloud node manager container setup.
$azsConfigFile = [io.path]::Combine($global:KubeDir, "azurestackcloud.json")
if (Test-Path -Path $azsConfigFile) {
$azsJson = Get-Content -Raw -Path $azsConfigFile | ConvertFrom-Json
if (-not [string]::IsNullOrEmpty($azsJson.managementPortalURL)) {
$azsARMUri = [System.Uri]$azsJson.managementPortalURL
$azsRootCert = Get-ChildItem -Path Cert:\LocalMachine\Root | Where-Object {$_.DnsNameList -contains $azsARMUri.Host.Substring($azsARMUri.Host.IndexOf(".")).TrimStart(".")}
if ($null -ne $azsRootCert) {
$azsRootCertFilePath = [io.path]::Combine($global:KubeDir, "azsroot.cer")
Export-Certificate -Cert $azsRootCert -FilePath $azsRootCertFilePath -Type CERT
} else {
throw "$azsRootCert is null, cannot export Azure Stack root cert"
}
} else {
throw "managementPortalURL is null or empty in $azsConfigFile, cannot get Azure Stack ARM uri"
}
} else {
throw "$azsConfigFile does not exist, cannot export Azure Stack root cert"
}
# Copy certoc tool for use in cloud node manager container setup. [Environment]::SystemDirectory
$certocSourcePath = [io.path]::Combine([Environment]::SystemDirectory, "certoc.exe")
if (Test-Path -Path $certocSourcePath) {
Copy-Item -Path $certocSourcePath -Destination $global:KubeDir
}
# Create add cert script
$addRootCertFile = [io.path]::Combine($global:KubeDir, "addazsroot.bat")
if ($null -ne $azsRootCert) {
[io.file]::WriteAllText($addRootCertFile, "${global:KubeDir}\certoc.exe -addstore root ${azsRootCertFilePath}")
} else {
throw "$azsRootCertFilePath is null, cannot create add cert script"
}
{{end}}
{{end}}
if (Test-Path $CacheDir)
{
Write-Log "Removing aks-engine bits cache directory"
@ -17630,8 +17727,7 @@ catch
Write-Error $_
throw $_
}
`)
}`)
func k8sKuberneteswindowssetupPs1Bytes() ([]byte, error) {
return _k8sKuberneteswindowssetupPs1, nil
@ -17732,6 +17828,13 @@ spec:
- name: cloud-controller-manager
image: {{ContainerImage "cloud-controller-manager"}}
imagePullPolicy: IfNotPresent
{{- if IsAzureStackCloud}}
env:
- name: AZURE_ENVIRONMENT_FILEPATH
value: /etc/kubernetes/azurestackcloud.json
- name: AZURE_GO_SDK_LOG_LEVEL
value: INFO
{{end}}
command: [{{ContainerConfig "command"}}]
args: [{{GetCloudControllerManagerArgs}}]
resources:

2
vhd/packer/install-dependencies.sh
Просмотреть файл

@ -247,7 +247,7 @@ done
# Starting with 1.16 we pull cloud-controller-manager and cloud-node-manager
CLOUD_MANAGER_VERSIONS="
0.5.1
1.1.1
"
for CLOUD_MANAGER_VERSION in ${CLOUD_MANAGER_VERSIONS}; do
for COMPONENT in azure-cloud-controller-manager azure-cloud-node-manager; do