From b068aa700cde40c68e4bac2790f434537f2908b8 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Thu, 5 Mar 2020 12:40:01 -0800 Subject: [PATCH] feat: Experimental support for Windows+ContainerD (#1322) --- docs/topics/clusterdefinitions.md | 8 +- docs/topics/features.md | 45 +- .../windows/kubernetes-hybrid.containerd.json | 55 ++ parts/k8s/kuberneteswindowsfunctions.ps1 | 11 + parts/k8s/kuberneteswindowssetup.ps1 | 64 +- parts/k8s/windowscnifunc.ps1 | 15 + parts/k8s/windowsconfigfunc.ps1 | 35 ++ parts/k8s/windowscontainerdfunc.ps1 | 154 +++++ parts/k8s/windowskubeletfunc.ps1 | 247 +++++++- parts/windowsparams.t | 12 + pkg/api/converterfromapi.go | 2 + pkg/api/convertertoapi.go | 2 + pkg/api/convertertoapi_test.go | 16 + pkg/api/types.go | 2 + pkg/api/vlabs/types.go | 2 + pkg/api/vlabs/validate.go | 15 +- pkg/engine/const.go | 21 +- pkg/engine/params_k8s.go | 2 + pkg/engine/template_generator.go | 4 +- pkg/engine/templates_generated.go | 557 ++++++++++++++++-- .../windows/kubernetes-kubernetesconfig.json | 4 +- scripts/build-windows-containerd.sh | 71 +++ 22 files changed, 1246 insertions(+), 98 deletions(-) create mode 100644 examples/windows/kubernetes-hybrid.containerd.json create mode 100644 parts/k8s/windowscontainerdfunc.ps1 create mode 100755 scripts/build-windows-containerd.sh diff --git a/docs/topics/clusterdefinitions.md b/docs/topics/clusterdefinitions.md index 0557592c4..0ad91770e 100644 --- a/docs/topics/clusterdefinitions.md +++ b/docs/topics/clusterdefinitions.md @@ -1,5 +1,7 @@ # Cluster Definitions + + ## Cluster Defintions for apiVersion "vlabs" Here are the cluster definitions for apiVersion "vlabs": @@ -41,13 +43,15 @@ $ aks-engine get-versions | apiServerConfig | no | Configure various runtime configuration for apiserver. See `apiServerConfig` [below](#feat-apiserver-config) | | cloudControllerManagerConfig | no | Configure various runtime configuration for cloud-controller-manager. See `cloudControllerManagerConfig` [below](#feat-cloud-controller-manager-config) | | clusterSubnet | no | The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. With Azure CNI enabled, the default value is 10.240.0.0/12. Without Azure CNI, the default value is 10.244.0.0/16. | -| containerRuntime | no | The container runtime to use as a backend. The default is `docker`. The other options are `kata-containers`, and `containerd` | +| containerRuntime | no | The container runtime to use as a backend. The default is `docker`. The other options are `kata-containers`, and `containerd`. Windows support for `containerd` is **Experimental** - see [Windows ContainerD](features.md#windows-containerd) | | controllerManagerConfig | no | Configure various runtime configuration for controller-manager. See `controllerManagerConfig` [below](#feat-controller-manager-config) | | customWindowsPackageURL | no | Configure custom windows Kubernetes release package URL for deployment on Windows. The format of this file is a zip file with multiple items (binaries, cni, infra container) in it. This setting will be deprecated in a future release of aks-engine where the binaries will be pulled in the format of Kubernetes releases that only contain the kubernetes binaries. | | WindowsNodeBinariesURL | no | Windows Kubernetes Node binaries can be provided in the format of Kubernetes release (example: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md#node-binaries-1). This setting allows overriding the binaries for custom builds. | +| WindowsContainerdURL | no (for development only) | **Experimental** - see [Windows ContainerD](features.md#windows-containerd) | +| WindowsSdnPluginURL | no (for development only) | **Experimental** - see [Windows ContainerD](features.md#windows-containerd) | | dnsServiceIP | no | IP address for coredns or kube-dns to listen on. If specified must be in the range of `serviceCidr` | | mobyVersion | no (for development only) | Enables an explicit moby version, e.g. `3.0.3`. Default is `3.0.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `mobyVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of moby. | -| containerdVersion | no (for development only) | Enables an explicit containerd version, e.g. `1.1.4`. Default is `1.1.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `containerdVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of containerd. | +| containerdVersion | no (for development only) | Enables an explicit containerd version, e.g. `1.1.4`. Default is `1.1.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `containerdVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of containerd. This value is currently ignored for Windows. | | dockerBridgeSubnet | no | The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0) | | enableAggregatedAPIs | no | Enable [Kubernetes Aggregated APIs](https://kubernetes.io/docs/concepts/api-extension/apiserver-aggregation/). enableRbac must be set to true to use aggregated APIs. Aggregated API functionality is required by [Service Catalog](https://github.com/kubernetes-incubator/service-catalog/blob/master/README.md). (boolean - default is true) | | enableDataEncryptionAtRest | no | Enable [kubernetes data encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/).This is currently an alpha feature. (boolean - default == false) | diff --git a/docs/topics/features.md b/docs/topics/features.md index 314949a40..a2d377ae9 100644 --- a/docs/topics/features.md +++ b/docs/topics/features.md @@ -2,17 +2,17 @@ |Feature|Status|API Version|Example|Description| |---|---|---|---|---| -|Managed Disks|Beta|`vlabs`|[kubernetes-vmas.json](../../examples/disks-managed/kubernetes-vmas.json)|[Description](#feat-managed-disks)| +|Antrea Network Policy|Alpha|`vlabs`|[kubernetes-antrea.json](../../examples/networkpolicy/kubernetes-antrea.json)|[Description](#feat-antrea)| +|Azure Key Vault Encryption|Alpha|`vlabs`|[kubernetes-keyvault-encryption.json](../../examples/kubernetes-config/kubernetes-keyvault-encryption.json)|[Description](#feat-keyvault-encryption)| |Calico Network Policy|Alpha|`vlabs`|[kubernetes-calico.json](../../examples/networkpolicy/kubernetes-calico-azure.json)|[Description](#feat-calico)| |Cilium Network Policy|Alpha|`vlabs`|[kubernetes-cilium.json](../../examples/networkpolicy/kubernetes-cilium.json)|[Description](#feat-cilium)| -|Antrea Network Policy|Alpha|`vlabs`|[kubernetes-antrea.json](../../examples/networkpolicy/kubernetes-antrea.json)|[Description](#feat-antrea)| +|ContainerD Runtime for Windows|Experimental|`vlabs`|[kubernetes-hybrid.containerd.json](../../examples/windows/kubernetes-hybrid.containerd.json)|[Description](#windows-containerd)| |Custom VNET|Beta|`vlabs`|[kubernetesvnet-azure-cni.json](../../examples/vnet/kubernetesvnet-azure-cni.json)|[Description](#feat-custom-vnet)| -|Kata Containers Runtime|Alpha|`vlabs`|[kubernetes-kata-containers.json](../../examples/kubernetes-kata-containers.json)|[Description](#feat-kata-containers)| -|Private Cluster|Alpha|`vlabs`|[kubernetes-private-cluster.json](../../examples/kubernetes-config/kubernetes-private-cluster.json)|[Description](#feat-private-cluster)| -|Azure Key Vault Encryption|Alpha|`vlabs`|[kubernetes-keyvault-encryption.json](../../examples/kubernetes-config/kubernetes-keyvault-encryption.json)|[Description](#feat-keyvault-encryption)| -|Shared Image Gallery images|Alpha|`vlabs`|[custom-shared-image.json](../../examples/custom-shared-image.json)|[Description](#feat-shared-image-gallery)| |Ephemeral OS Disks|Experimental|`vlabs`|[ephmeral-disk.json](../../examples/disks-ephemeral/ephemeral-disks.json)|[Description](#ephemeral-os-disks)| - +|Kata Containers Runtime|Alpha|`vlabs`|[kubernetes-kata-containers.json](../../examples/kubernetes-kata-containers.json)|[Description](#feat-kata-containers)| +|Managed Disks|Beta|`vlabs`|[kubernetes-vmas.json](../../examples/disks-managed/kubernetes-vmas.json)|[Description](#feat-managed-disks)| +|Private Cluster|Alpha|`vlabs`|[kubernetes-private-cluster.json](../../examples/kubernetes-config/kubernetes-private-cluster.json)|[Description](#feat-private-cluster)| +|Shared Image Gallery images|Alpha|`vlabs`|[custom-shared-image.json](../../examples/custom-shared-image.json)|[Description](#feat-shared-image-gallery)| @@ -543,3 +543,34 @@ We are investigating possible risks & mitigations for when VMs are deprovisioned [Ephemeral OS Disks]: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/ephemeral-os-disks + + +## Windows ContainerD + +> This feature is currently experimental, and has open issues. + +Kubernetes 1.18 introduces alpha support for the ContainerD runtime on Windows Server 2019. This is still a work-in-progress tracked in [kubernetes/enhancements#1001](https://github.com/kubernetes/enhancements/issues/1001). This feature in AKS-Engine is for testing the in-development versions of ContainerD and Kubernetes, and is not for production use. Be sure to review [open issues](https://github.com/azure/aks-engine/issues?q=containerd+label%3Awindows+is%3Aopen) if you want to test or contribute to this effort. + +Currently it only supports the `kubenet` networking model, and requires URLs to custom ContainerD and CNI plugin builds. + +### Deploying multi-OS clusters with ContainerD + +If you want to test or develop with Windows & ContainerD in AKS-Engine, see this sample +[kubernetes-hybrid.containerd.json](../../examples/windows/kubernetes-hybrid.containerd.json) + +These parameters are all required. + +```json + "kubernetesConfig": { + "networkPlugin": "kubenet", + "containerRuntime": "containerd", + "windowsContainerdURL": "...", + "windowsSdnPluginURL": "..." + } +``` + +### Building ContainerD + +As of March 3, 2020, the ContainerD and network plugin repos don't have public builds available. This repo has a script that will build them from source and create two ZIP files: [build-windows-containerd.sh](../../scripts/build-windows-containerd.sh) + +Upload these ZIP files to a location that your cluster will be able to reach, then put those URLs in `windowsContainerdURL` and `windowsSdnPluginURL` in the AKS-Engine apimodel shown above. \ No newline at end of file diff --git a/examples/windows/kubernetes-hybrid.containerd.json b/examples/windows/kubernetes-hybrid.containerd.json new file mode 100644 index 000000000..6597e3079 --- /dev/null +++ b/examples/windows/kubernetes-hybrid.containerd.json @@ -0,0 +1,55 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorRelease": "1.18", + "kubernetesConfig": { + "networkPlugin": "kubenet", + "containerRuntime": "containerd", + "windowsContainerdURL": "https://aksenginee2etestimages.blob.core.windows.net/test-content/windows-cri-containerd.zip", + "windowsSdnPluginURL": "https://aksenginee2etestimages.blob.core.windows.net/test-content/windows-cni-containerd.zip" + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "linuxpool1", + "count": 2, + "vmSize": "Standard_D2_v2", + "availabilityProfile": "AvailabilitySet" + }, + { + "name": "windowspool2", + "count": 2, + "vmSize": "Standard_D2s_v3", + "availabilityProfile": "AvailabilitySet", + "osType": "Windows" + } + ], + "windowsProfile": { + "adminUsername": "azureuser", + "adminPassword": "replacepassword1234$", + "enableAutomaticUpdates": false, + "sshEnabled": true + }, + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/parts/k8s/kuberneteswindowsfunctions.ps1 b/parts/k8s/kuberneteswindowsfunctions.ps1 index 6c8c2ab53..7d7486ebb 100644 --- a/parts/k8s/kuberneteswindowsfunctions.ps1 +++ b/parts/k8s/kuberneteswindowsfunctions.ps1 @@ -220,3 +220,14 @@ function Register-NodeResetScriptTask { $definition = New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "k8s-restart-job" Register-ScheduledTask -TaskName "k8s-restart-job" -InputObject $definition } + +function Assert-FileExists { + Param( + [Parameter(Mandatory=$true,Position=0)][string] + $Filename + ) + + if (-Not (Test-Path $Filename)) { + throw "$Filename does not exist" + } +} diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index e6a5f3b3f..a9a301542 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -65,10 +65,15 @@ $global:AgentCertificate = "{{WrapAsParameter "clientCertificate"}}" $global:KubeBinariesPackageSASURL = "{{WrapAsParameter "kubeBinariesSASURL"}}" $global:WindowsKubeBinariesURL = "{{WrapAsParameter "windowsKubeBinariesURL"}}" $global:KubeBinariesVersion = "{{WrapAsParameter "kubeBinariesVersion"}}" +$global:ContainerdUrl = "{{WrapAsParameter "windowsContainerdURL"}}" +$global:ContainerdSdnPluginUrl = "{{WrapAsParameter "windowsSdnPluginURL"}}" ## Docker Version $global:DockerVersion = "{{WrapAsParameter "windowsDockerVersion"}}" +## ContainerD Usage +$global:ContainerRuntime = "{{WrapAsParameter "containerRuntime"}}" + ## VM configuration passed by Azure $global:WindowsTelemetryGUID = "{{WrapAsParameter "windowsTelemetryGUID"}}" {{if eq GetIdentitySystem "adfs"}} @@ -145,14 +150,9 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\k8s\windowscnifunc.ps1 . c:\AzureData\k8s\windowsazurecnifunc.ps1 . c:\AzureData\k8s\windowsinstallopensshfunc.ps1 +. c:\AzureData\k8s\windowscontainerdfunc.ps1 -function -Update-ServiceFailureActions() -{ - sc.exe failure "kubelet" actions= restart/60000/restart/60000/restart/60000 reset= 900 - sc.exe failure "kubeproxy" actions= restart/60000/restart/60000/restart/60000 reset= 900 - sc.exe failure "docker" actions= restart/60000/restart/60000/restart/60000 reset= 900 -} +$useContainerD = ($global:ContainerRuntime -eq "containerd") try { @@ -222,12 +222,22 @@ try Write-Log "Create required data directories as needed" Initialize-DataDirectories - Write-Log "Install docker" - $dockerTimer = [System.Diagnostics.Stopwatch]::StartNew() - Install-Docker -DockerVersion $global:DockerVersion - Set-DockerLogFileOptions - $dockerTimer.Stop() - $global:AppInsightsClient.TrackMetric("Install-Docker", $dockerTimer.Elapsed.TotalSeconds) + + if ($useContainerD) { + Write-Log "Installing ContainerD" + $containerdTimer = [System.Diagnostics.Stopwatch]::StartNew() + Install-Containerd -ContainerdUrl $global:ContainerdUrl + $containerdTimer.Stop() + $global:AppInsightsClient.TrackMetric("Install-ContainerD", $containerdTimer.Elapsed.TotalSeconds) + # TODO: disable/uninstall Docker later + } else { + Write-Log "Install docker" + $dockerTimer = [System.Diagnostics.Stopwatch]::StartNew() + Install-Docker -DockerVersion $global:DockerVersion + Set-DockerLogFileOptions + $dockerTimer.Stop() + $global:AppInsightsClient.TrackMetric("Install-Docker", $dockerTimer.Elapsed.TotalSeconds) + } Write-Log "Download kubelet binaries and unzip" Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL @@ -284,14 +294,19 @@ try Write-Log "Create the Pause Container kubletwin/pause" $infraContainerTimer = [System.Diagnostics.Stopwatch]::StartNew() - New-InfraContainer -KubeDir $global:KubeDir + New-InfraContainer -KubeDir $global:KubeDir -ContainerRuntime $global:ContainerRuntime $infraContainerTimer.Stop() $global:AppInsightsClient.TrackMetric("New-InfraContainer", $infraContainerTimer.Elapsed.TotalSeconds) - if (-not (Test-ContainerImageExists -Image "kubletwin/pause")) { + if (-not (Test-ContainerImageExists -Image "kubletwin/pause" -ContainerRuntime $global:ContainerRuntime)) { Write-Log "Could not find container with name kubletwin/pause" - $o = docker image list - Write-Log $o + if ($useContainerD) { + $o = ctr -n k8s.io image list + Write-Log $o + } else { + $o = docker image list + Write-Log $o + } throw "kubletwin/pause container does not exist!" } @@ -330,7 +345,13 @@ try } elseif ($global:NetworkPlugin -eq "kubenet") { Write-Log "Fetching additional files needed for kubenet" - Update-WinCNI -CNIPath $global:CNIPath + if ($useContainerD) { + # TODO: CNI may need to move to c:\program files\containerd\cni\bin with ContainerD + Install-SdnBridge -Url $global:ContainerdSdnPluginUrl -CNIPath $global:CNIPath + } else { + Update-WinCNI -CNIPath $global:CNIPath + } + Get-HnsPsm1 -HNSModule $global:HNSModule } New-ExternalHnsNetwork @@ -354,7 +375,8 @@ try -KubeClusterCIDR $global:KubeClusterCIDR ` -KubeServiceCIDR $global:KubeServiceCIDR ` -HNSModule $global:HNSModule ` - -KubeletNodeLabels $global:KubeletNodeLabels + -KubeletNodeLabels $global:KubeletNodeLabels ` + -UseContainerD $useContainerD Get-LogCollectionScripts @@ -368,7 +390,7 @@ try PREPROVISION_EXTENSION Write-Log "Update service failure actions" - Update-ServiceFailureActions + Update-ServiceFailureActions -ContainerRuntime $global:ContainerRuntime Adjust-DynamicPortRange Register-LogsCleanupScriptTask @@ -390,7 +412,7 @@ try else { # keep for debugging purposes - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AgentKey $AgentKey -AADClientId $AADClientId -AADClientSecret $AADClientSecret" + Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AgentKey $AgentKey -AADClientId $AADClientId -AADClientSecret $AADClientSecret -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" } } catch diff --git a/parts/k8s/windowscnifunc.ps1 b/parts/k8s/windowscnifunc.ps1 index 4010da2c0..7edd2e797 100644 --- a/parts/k8s/windowscnifunc.ps1 +++ b/parts/k8s/windowscnifunc.ps1 @@ -22,4 +22,19 @@ function Update-WinCNI DownloadFileOverHttp -Url $WinCniUrl -DestinationPath $wincniFile } +function Install-SdnBridge +{ + Param( + [Parameter(Mandatory=$true)][string] + $Url, + [Parameter(Mandatory=$true)][string] + $CNIPath + ) + + $cnizip = [Io.path]::Combine($CNIPath, "cni.zip") + DownloadFileOverHttp -Url $Url -DestinationPath $cnizip + Expand-Archive -path $cnizip -DestinationPath $CNIPath + del $cnizip +} + # TODO: Move the code that creates the wincni configuration file out of windowskubeletfunc.ps1 and put it here \ No newline at end of file diff --git a/parts/k8s/windowsconfigfunc.ps1 b/parts/k8s/windowsconfigfunc.ps1 index c7fa8574b..65b0f190e 100644 --- a/parts/k8s/windowsconfigfunc.ps1 +++ b/parts/k8s/windowsconfigfunc.ps1 @@ -143,4 +143,39 @@ function Adjust-DynamicPortRange() # Kube-proxy load balancing should be set to DSR mode when it releases with future versions of the OS Invoke-Executable -Executable "netsh.exe" -ArgList @("int", "ipv4", "set", "dynamicportrange", "tcp", "16385", "49151") +} + +# TODO: should this be in this PR? +# Service start actions. These should be split up later and included in each install step +function Update-ServiceFailureActions +{ + Param( + [Parameter(Mandatory = $true)][string] + $ContainerRuntime + ) + sc.exe failure "kubelet" actions= restart/60000/restart/60000/restart/60000 reset= 900 + sc.exe failure "kubeproxy" actions= restart/60000/restart/60000/restart/60000 reset= 900 + sc.exe failure $ContainerRuntime actions= restart/60000/restart/60000/restart/60000 reset= 900 +} + +function Add-SystemPathEntry +{ + Param( + [Parameter(Mandatory = $true)][string] + $Directory + ) + # update the path variable if it doesn't have the needed paths + $path = [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + $updated = $false + if(-not ($path -match $Directory.Replace("\","\\")+"(;|$)")) + { + $path += ";"+$Directory + $updated = $true + } + if($updated) + { + Write-Output "Updating path, added $Directory" + [Environment]::SetEnvironmentVariable("Path", $path, [EnvironmentVariableTarget]::Machine) + $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") + } } \ No newline at end of file diff --git a/parts/k8s/windowscontainerdfunc.ps1 b/parts/k8s/windowscontainerdfunc.ps1 new file mode 100644 index 000000000..2ea2c553a --- /dev/null +++ b/parts/k8s/windowscontainerdfunc.ps1 @@ -0,0 +1,154 @@ +# this is $global to persist across all functions since this is dot-sourced +$global:ContainerdInstallLocation = "$Env:ProgramFiles\containerd" + +function RegisterContainerDService +{ + Assert-FileExists (Join-Path $global:ContainerdInstallLocation containerd.exe) + + Write-Host "Registering containerd as a service" + $cdbinary = Join-Path $global:ContainerdInstallLocation containerd.exe + $svc = Get-Service -Name containerd -ErrorAction SilentlyContinue + if ($null -ne $svc) { + & $cdbinary --unregister-service + } + & $cdbinary --register-service + $svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue + if ($null -eq $svc) { + throw "containerd.exe did not get installed as a service correctly." + } + $svc | Start-Service + if ($svc.Status -ne "Running") { + throw "containerd service is not running" + } +} + + +function Install-Containerd +{ + Param( + [Parameter(Mandatory = $true)][string] + $ContainerdUrl + ) + $zipfile = [Io.path]::Combine($ENV:TEMP, "containerd.zip") + DownloadFileOverHttp -Url $ContainerdUrl -DestinationPath $zipfile + Expand-Archive -path $zipfile -DestinationPath $global:ContainerdInstallLocation + del $zipfile + + Add-SystemPathEntry $global:ContainerdInstallLocation + + # TODO: remove if the node comes up without this code + # $configDir = [Io.Path]::Combine($ENV:ProgramData, "containerd") + # if (-Not (Test-Path $configDir)) { + # mkdir $configDir + # } + + # TODO: call containerd.exe dump config, then modify instead of starting with hardcoded + $configFile = [Io.Path]::Combine($global:ContainerdInstallLocation, "config.toml") + @" +version = 2 +root = "C:\\ProgramData\\containerd\\root" +state = "C:\\ProgramData\\containerd\\state" +plugin_dir = "" +disabled_plugins = [] +required_plugins = [] +oom_score = 0 + +[grpc] + address = "\\\\.\\pipe\\containerd-containerd" + tcp_address = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[ttrpc] + address = "" + uid = 0 + gid = 0 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[plugins] + [plugins."io.containerd.gc.v1.scheduler"] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" + [plugins."io.containerd.grpc.v1.cri"] + disable_tcp_service = true + stream_server_address = "127.0.0.1" + stream_server_port = "0" + stream_idle_timeout = "4h0m0s" + enable_selinux = false + sandbox_image = "mcr.microsoft.com/k8s/core/pause:1.2.0" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + disable_cgroup = false + disable_apparmor = false + restrict_oom_score_adj = false + max_concurrent_downloads = 3 + disable_proc_mount = false + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "windows" + default_runtime_name = "runhcs-wcow-process" + no_pivot = false + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process] + runtime_type = "io.containerd.runhcs.v1" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "C:\\k\\cni" + conf_dir = "C:\\k\\cni\\config" + max_conf_num = 1 + conf_template = "" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + [plugins."io.containerd.runtime.v2.task"] + platforms = ["windows/amd64", "linux/amd64"] + [plugins."io.containerd.service.v1.diff-service"] + default = ["windows", "windows-lcow"] +"@ | Out-File -Encoding ascii $configFile + RegisterContainerDService + +} \ No newline at end of file diff --git a/parts/k8s/windowskubeletfunc.ps1 b/parts/k8s/windowskubeletfunc.ps1 index f32148740..895933ce4 100644 --- a/parts/k8s/windowskubeletfunc.ps1 +++ b/parts/k8s/windowskubeletfunc.ps1 @@ -41,7 +41,9 @@ Write-AzureConfig { [Parameter(Mandatory = $true)][string] $KubeDir, [Parameter(Mandatory = $true)][string] - $TargetEnvironment + $TargetEnvironment, + [Parameter(Mandatory = $false)][bool] + $UseContainerD = $false ) if ( -Not $PrimaryAvailabilitySetName -And -Not $PrimaryScaleSetName ) { @@ -133,17 +135,51 @@ users: $kubeConfig | Out-File -encoding ASCII -filepath "$kubeConfigFile" } +function +Test-ContainerImageExists { + Param( + [Parameter(Mandatory = $true)][string] + $Image, + [Parameter(Mandatory = $false)][string] + $Tag, + [Parameter(Mandatory = $false)][string] + $ContainerRuntime = "docker" + ) + + $target = $Image + if ($Tag) { + $target += ":$Tag" + } + + if ($ContainerRuntime -eq "docker") { + $images = docker image list $target --format "{{json .}}" + return $images.Count -gt 0 + } + else + { + return ( (ctr.exe -n k8s.io images list) | Select-String $target) -ne $Null + } +} + function Build-PauseContainer { Param( [Parameter(Mandatory = $true)][string] $WindowsBase, - $DestinationTag + $DestinationTag, + [Parameter(Mandatory = $false)][string] + $ContainerRuntime = "docker" ) # Future work: This needs to build wincat - see https://github.com/Azure/aks-engine/issues/1461 + # Otherwise, delete this code and require a prebuilt pause image (or override with one from an Azure Container Registry instance) + # ContainerD can't build, so doing the builds outside of node deployment is probably the right long-term solution. "FROM $($WindowsBase)" | Out-File -encoding ascii -FilePath Dockerfile "CMD cmd /c ping -t localhost" | Out-File -encoding ascii -FilePath Dockerfile -Append - Invoke-Executable -Executable "docker" -ArgList @("build", "-t", "$DestinationTag", ".") + if ($ContainerRuntime -eq "docker") { + Invoke-Executable -Executable "docker" -ArgList @("build", "-t", "$DestinationTag", ".") + } else { + throw "Cannot build pause container without Docker" + } } function @@ -151,7 +187,9 @@ New-InfraContainer { Param( [Parameter(Mandatory = $true)][string] $KubeDir, - $DestinationTag = "kubletwin/pause" + $DestinationTag = "kubletwin/pause", + [Parameter(Mandatory = $false)][string] + $ContainerRuntime = "docker" ) cd $KubeDir $computerInfo = Get-ComputerInfo @@ -164,14 +202,21 @@ New-InfraContainer { $pauseImageVersions = @("1803", "1809", "1903", "1909") if ($pauseImageVersions -icontains $computerInfo.WindowsVersion) { - $imageList = docker images $defaultPauseImage --format "{{.Repository}}:{{.Tag}}" - if (-not $imageList) { - Invoke-Executable -Executable "docker" -ArgList @("pull", "$defaultPauseImage") -Retries 5 -RetryDelaySeconds 30 + if ($ContainerRuntime -eq "docker") { + if (-not (Test-ContainerImageExists -Image $defaultPauseImage -ContainerRuntime $ContainerRuntime)) { + Invoke-Executable -Executable "docker" -ArgList @("pull", "$defaultPauseImage") -Retries 5 -RetryDelaySeconds 30 + } + Invoke-Executable -Executable "docker" -ArgList @("tag", "$defaultPauseImage", "$DestinationTag") + } else { + # containerd + if (-not (Test-ContainerImageExists -Image $defaultPauseImage -ContainerRuntime $ContainerRuntime)) { + Invoke-Executable -Executable "ctr" -ArgList @("-n", "k8s.io", "image", "pull", "$defaultPauseImage") -Retries 5 -RetryDelaySeconds 30 + } + Invoke-Executable -Executable "ctr" -ArgList @("-n", "k8s.io", "image", "tag", "$defaultPauseImage", "$DestinationTag") } - Invoke-Executable -Executable "docker" -ArgList @("tag", "$defaultPauseImage", "$DestinationTag") } else { - Build-PauseContainer -WindowsBase "mcr.microsoft.com/nanoserver-insider" -DestinationTag $DestinationTag + Build-PauseContainer -WindowsBase "mcr.microsoft.com/nanoserver-insider" -DestinationTag $DestinationTag -ContainerRuntime $ContainerRuntime } } @@ -181,7 +226,9 @@ Test-ContainerImageExists { [Parameter(Mandatory = $true)][string] $Image, [Parameter(Mandatory = $false)][string] - $Tag + $Tag, + [Parameter(Mandatory = $false)][string] + $ContainerRuntime = "docker" ) $target = $Image @@ -189,12 +236,16 @@ Test-ContainerImageExists { $target += ":$Tag" } - $images = docker image list $target --format "{{json .}}" - - return $images.Count -gt 0 + if ($ContainerRuntime -eq "docker") { + $images = docker image list $target --format "{{json .}}" + return $images.Count -gt 0 + } + else + { + return ( (ctr.exe -n k8s.io images list) | Select-String $target) -ne $Null + } } - # TODO: Deprecate this and replace with methods that get individual components instead of zip containing everything # This expects the ZIP file created by Azure Pipelines. function @@ -343,7 +394,9 @@ Install-KubernetesServices { [Parameter(Mandatory = $true)][string] $HNSModule, [Parameter(Mandatory = $true)][string] - $KubeletNodeLabels + $KubeletNodeLabels, + [Parameter(Mandatory = $false)][bool] + $UseContainerD = $false ) # Calculate some local paths @@ -381,6 +434,11 @@ Install-KubernetesServices { throw "Unknown network type $NetworkPlugin, can't configure kubelet" } + # Update args to use ContainerD if needed + if ($UseContainerD -eq $true) { + $KubeletArgList += @("--container-runtime=remote", "--container-runtime-endpoint=npipe://./pipe/containerd-containerd") + } + # Used in WinCNI version of kubeletstart.ps1 $KubeletArgListStr = "" $KubeletArgList | Foreach-Object { @@ -472,7 +530,8 @@ $KubeletCommandLine "@ } - else { + elseif (($NetworkPlugin -eq "kubenet" ) -and ($UseContainerD -eq $false)) + { # using WinCNI. TODO: If WinCNI support is removed, then delete this as dead code later $KubeNetwork = "l2bridge" $kubeStartStr += @" @@ -605,7 +664,161 @@ catch } "@ - } # end else using WinCNI. + } # end elseif using WinCNI and Docker. + elseif (($NetworkPlugin -eq "kubenet" ) -and ($UseContainerD -eq $true)) + { + # TODO: something is wrong with the CNI configuration + # Warning FailedCreatePodSandBox 2m16s (x222 over 50m) kubelet, 4068k8s011 (combined from similar events): Failed to create pod sandbox: + # rpc error: code = Unknown desc = failed to setup network for sandbox "922b1a200078edb15c7a5732612cbe19e5dadf7cf8a4622d72b376874522435d": error creating endpoint hcnCreateEndpoint failed in Win32: Invalid JSON document string. (0x803b001b) {"Success":false,"Error":"Invalid JSON document string. {{Policies.Type,UnknownEnumValue}}","ErrorCode":2151350299} : endpoint config &{ 922b1a200078edb15c7a5732612cbe19e5dadf7cf8a4622d72b376874522435d_l2bridge f94e9649-d4df-486e-bcd4-2721938d89f3 [{OutBoundNAT []} {ROUTE []}] [] { [default.svc.cluster.local] [10.0.0.10] []} [{10.240.0.1 0.0.0.0/0 0}] 0 {2 0}} + + + # using SDNBridge & Containerd + $KubeNetwork = "l2bridge" + $kubeStartStr += @" + +function +Get-DefaultGateway(`$CIDR) +{ + return `$CIDR.substring(0,`$CIDR.lastIndexOf(".")) + ".1" +} + +function +Get-PodCIDR() +{ + `$podCIDR = c:\k\kubectl.exe --kubeconfig=c:\k\config get nodes/`$(`$env:computername.ToLower()) -o custom-columns=podCidr:.spec.podCIDR --no-headers + return `$podCIDR +} + +function +Test-PodCIDR(`$podCIDR) +{ + return `$podCIDR.length -gt 0 +} + +function +Update-CNIConfig(`$podCIDR, `$masterSubnetGW) +{ + `$jsonSampleConfig = +"{ + ""cniVersion"": ""0.2.0"", + ""name"": """", + ""type"": ""sdnbridge.exe"", + ""master"": ""Ethernet"", + ""capabilities"": { ""portMappings"": true }, + ""ipam"": { + ""environment"": ""azure"", + ""subnet"":"""", + ""routes"": [{ + ""GW"":"""" + }] + }, + ""dns"" : { + ""Nameservers"" : [ """" ], + ""Search"" : [ """" ] + }, + ""AdditionalArgs"" : [ + { + ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""OutBoundNAT"", ""Settings"" : { ""Exceptions"": [ """", """" ] }} + }, + { + ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""SDNRoute"", ""Settings"" : { ""DestinationPrefix"": """", ""NeedEncap"" : true }} + } + ] +}" + + `$configJson = ConvertFrom-Json `$jsonSampleConfig + `$configJson.name = `$global:NetworkMode.ToLower() + `$configJson.ipam.subnet=`$podCIDR + `$configJson.ipam.routes[0].GW = `$masterSubnetGW + `$configJson.dns.Nameservers[0] = `$global:KubeDnsServiceIp + `$configJson.dns.Search[0] = `$global:KubeDnsSearchPath + + + `$configJson.AdditionalArgs[0].Value.Settings.Exceptions[0] = `$global:KubeClusterCIDR + `$configJson.AdditionalArgs[0].Value.Settings.Exceptions[1] = `$global:MasterSubnet + `$configJson.AdditionalArgs[1].Value.Settings.DestinationPrefix = `$global:KubeServiceCIDR + + if (Test-Path `$global:CNIConfig) + { + Clear-Content -Path `$global:CNIConfig + } + + Write-Host "Generated CNI Config [`$configJson]" + + Add-Content -Path `$global:CNIConfig -Value (ConvertTo-Json `$configJson -Depth 20) +} + +try +{ + `$masterSubnetGW = Get-DefaultGateway `$global:MasterSubnet + `$podCIDR=Get-PodCIDR + `$podCidrDiscovered=Test-PodCIDR(`$podCIDR) + + # if the podCIDR has not yet been assigned to this node, start the kubelet process to get the podCIDR, and then promptly kill it. + if (-not `$podCidrDiscovered) + { + `$argList = $KubeletArgListStr + + `$process = Start-Process -FilePath c:\k\kubelet.exe -PassThru -ArgumentList `$argList + + # run kubelet until podCidr is discovered + Write-Host "waiting to discover pod CIDR" + while (-not `$podCidrDiscovered) + { + Write-Host "Sleeping for 10s, and then waiting to discover pod CIDR" + Start-Sleep 10 + + `$podCIDR=Get-PodCIDR + `$podCidrDiscovered=Test-PodCIDR(`$podCIDR) + } + + # stop the kubelet process now that we have our CIDR, discard the process output + `$process | Stop-Process | Out-Null + } + + # Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) + netsh advfirewall set allprofiles state off + + # startup the service + `$hnsNetwork = Get-HnsNetwork | ? Name -EQ `$global:NetworkMode.ToLower() + + if (`$hnsNetwork) + { + # Kubelet has been restarted with existing network. + # Cleanup all containers + # TODO: convert this to ctr.exe -n k8s.io container list ; container rm + docker ps -q | foreach {docker rm `$_ -f} + # cleanup network + Write-Host "Cleaning up old HNS network found" + Remove-HnsNetwork `$hnsNetwork + Start-Sleep 10 + } + + Write-Host "Creating a new hns Network" + ipmo `$global:HNSModule + + `$hnsNetwork = New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix `$podCIDR -Gateway `$masterSubnetGW -Name `$global:NetworkMode.ToLower() -Verbose + # New network has been created, Kubeproxy service has to be restarted + Restart-Service Kubeproxy + + Start-Sleep 10 + # Add route to all other POD networks + Write-Host "Updating CNI config" + Update-CNIConfig `$podCIDR `$masterSubnetGW + + $KubeletCommandLine +} +catch +{ + Write-Error `$_ +} + +"@ + } # end elseif using sdnbridge and containerd. + else + { + throw "The combination of $NetworkPlugin and UseContainerD=$UseContainerD is not implemented" + } # Now that the script is generated, based on what CNI plugin and startup options are needed, write it to disk $kubeStartStr | Out-File -encoding ASCII -filepath $KubeletStartFile diff --git a/parts/windowsparams.t b/parts/windowsparams.t index b05dfde2e..258c8c5aa 100644 --- a/parts/windowsparams.t +++ b/parts/windowsparams.t @@ -17,6 +17,18 @@ }, "type": "string" }, + "windowsContainerdURL": { + "metadata": { + "description": "TODO: containerd - these binaries are not available yet" + }, + "type": "string" + }, + "windowsSdnPluginURL": { + "metadata": { + "description": "TODO: containerd - these binaries are not available yet" + }, + "type": "string" + }, "kubeServiceCidr": { "metadata": { "description": "Kubernetes service address space" diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index f90cd2474..0a245812b 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -297,6 +297,8 @@ func convertKubernetesConfigToVLabs(apiCfg *KubernetesConfig, vlabsCfg *vlabs.Ku vlabsCfg.UseCloudControllerManager = apiCfg.UseCloudControllerManager vlabsCfg.CustomWindowsPackageURL = apiCfg.CustomWindowsPackageURL vlabsCfg.WindowsNodeBinariesURL = apiCfg.WindowsNodeBinariesURL + vlabsCfg.WindowsContainerdURL = apiCfg.WindowsContainerdURL + vlabsCfg.WindowsSdnPluginURL = apiCfg.WindowsSdnPluginURL vlabsCfg.UseInstanceMetadata = apiCfg.UseInstanceMetadata vlabsCfg.LoadBalancerSku = apiCfg.LoadBalancerSku vlabsCfg.ExcludeMasterFromStandardLB = apiCfg.ExcludeMasterFromStandardLB diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 1d61ee027..c6d6c66d4 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -310,6 +310,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.UseCloudControllerManager = vlabs.UseCloudControllerManager api.CustomWindowsPackageURL = vlabs.CustomWindowsPackageURL api.WindowsNodeBinariesURL = vlabs.WindowsNodeBinariesURL + api.WindowsContainerdURL = vlabs.WindowsContainerdURL + api.WindowsSdnPluginURL = vlabs.WindowsSdnPluginURL api.UseInstanceMetadata = vlabs.UseInstanceMetadata api.LoadBalancerSku = vlabs.LoadBalancerSku api.ExcludeMasterFromStandardLB = vlabs.ExcludeMasterFromStandardLB diff --git a/pkg/api/convertertoapi_test.go b/pkg/api/convertertoapi_test.go index 9620a8c2e..b599db5c2 100644 --- a/pkg/api/convertertoapi_test.go +++ b/pkg/api/convertertoapi_test.go @@ -130,6 +130,22 @@ func TestConvertVLabsKubernetesConfigProfile(t *testing.T) { WindowsNodeBinariesURL: "http://test/test.tar.gz", }, }, + "WindowsContainerdURL": { + props: &vlabs.KubernetesConfig{ + WindowsContainerdURL: "http://test/testcontainerd.tar.gz", + }, + expect: &KubernetesConfig{ + WindowsContainerdURL: "http://test/testcontainerd.tar.gz", + }, + }, + "WindowsSdnPluginURL": { + props: &vlabs.KubernetesConfig{ + WindowsSdnPluginURL: "http://test/testsdnplugin.tar.gz", + }, + expect: &KubernetesConfig{ + WindowsSdnPluginURL: "http://test/testsdnplugin.tar.gz", + }, + }, } for name, test := range tests { diff --git a/pkg/api/types.go b/pkg/api/types.go index cfd4f80d7..fe2f0c144 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -446,6 +446,8 @@ type KubernetesConfig struct { UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` + WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` + WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` EnableRbac *bool `json:"enableRbac,omitempty"` EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 9850da522..4b8c7b0f8 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -328,6 +328,8 @@ type KubernetesConfig struct { UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` + WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` + WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` EnableRbac *bool `json:"enableRbac,omitempty"` EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index eb0d42166..53898cb18 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -1590,10 +1590,23 @@ func (a *Properties) validateContainerRuntime() error { } // Make sure we don't use unsupported container runtimes on windows. - if (containerRuntime == KataContainers || containerRuntime == Containerd) && a.HasWindows() { + if (containerRuntime == KataContainers) && a.HasWindows() { return errors.Errorf("containerRuntime %q is not supporting windows agents", containerRuntime) } + // TODO: These validations should be relaxed once ContainerD and CNI plugins are more readily available + if containerRuntime == Containerd && a.HasWindows() { + if a.OrchestratorProfile.KubernetesConfig.NetworkPlugin != "kubenet" { + return errors.Errorf("Windows only supports kubenet with containerd runtime. %q is not supported", a.OrchestratorProfile.KubernetesConfig.NetworkPlugin) + } + if a.OrchestratorProfile.KubernetesConfig.WindowsContainerdURL == "" { + return errors.Errorf("WindowsContainerdURL must be provided when using Windows with ContainerRuntime=containerd") + } + if a.OrchestratorProfile.KubernetesConfig.WindowsSdnPluginURL == "" { + return errors.Errorf("WindowsSdnPluginURL must be provided when using Windows with ContainerRuntime=containerd") + } + } + return nil } diff --git a/pkg/engine/const.go b/pkg/engine/const.go index 261812f28..4b6dd4476 100644 --- a/pkg/engine/const.go +++ b/pkg/engine/const.go @@ -76,16 +76,17 @@ const ( const ( kubeConfigJSON = "k8s/kubeconfig.json" - // Windows custom scripts - kubernetesWindowsAgentCustomDataPS1 = "k8s/kuberneteswindowssetup.ps1" - kubernetesWindowsAgentFunctionsPS1 = "k8s/kuberneteswindowsfunctions.ps1" - kubernetesWindowsConfigFunctionsPS1 = "k8s/windowsconfigfunc.ps1" - kubernetesWindowsKubeletFunctionsPS1 = "k8s/windowskubeletfunc.ps1" - kubernetesWindowsCniFunctionsPS1 = "k8s/windowscnifunc.ps1" - kubernetesWindowsAzureCniFunctionsPS1 = "k8s/windowsazurecnifunc.ps1" - kubernetesWindowsOpenSSHFunctionPS1 = "k8s/windowsinstallopensshfunc.ps1" - kubernetesWindowsLogsCleanupPS1 = "k8s/windowslogscleanup.ps1" - kubernetesWindowsNodeResetPS1 = "k8s/windowsnodereset.ps1" + // Windows custom scripts. These should all be listed in template_generator.go:func GetKubernetesWindowsAgentFunctions + kubernetesWindowsAgentCustomDataPS1 = "k8s/kuberneteswindowssetup.ps1" + kubernetesWindowsAgentFunctionsPS1 = "k8s/kuberneteswindowsfunctions.ps1" + kubernetesWindowsConfigFunctionsPS1 = "k8s/windowsconfigfunc.ps1" + kubernetesWindowsContainerdFunctionsPS1 = "k8s/windowscontainerdfunc.ps1" + kubernetesWindowsKubeletFunctionsPS1 = "k8s/windowskubeletfunc.ps1" + kubernetesWindowsCniFunctionsPS1 = "k8s/windowscnifunc.ps1" + kubernetesWindowsAzureCniFunctionsPS1 = "k8s/windowsazurecnifunc.ps1" + kubernetesWindowsOpenSSHFunctionPS1 = "k8s/windowsinstallopensshfunc.ps1" + kubernetesWindowsLogsCleanupPS1 = "k8s/windowslogscleanup.ps1" + kubernetesWindowsNodeResetPS1 = "k8s/windowsnodereset.ps1" ) // cloud-init (i.e. ARM customData) source file references diff --git a/pkg/engine/params_k8s.go b/pkg/engine/params_k8s.go index 84c891d68..c44bdd80f 100644 --- a/pkg/engine/params_k8s.go +++ b/pkg/engine/params_k8s.go @@ -112,6 +112,8 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params addValue(parametersMap, "kubeServiceCidr", kubernetesConfig.ServiceCIDR) addValue(parametersMap, "kubeBinariesVersion", k8sVersion) addValue(parametersMap, "windowsTelemetryGUID", cloudSpecConfig.KubernetesSpecConfig.WindowsTelemetryGUID) + addValue(parametersMap, "windowsContainerdURL", kubernetesConfig.WindowsContainerdURL) + addValue(parametersMap, "windowsSdnPluginURL", kubernetesConfig.WindowsSdnPluginURL) } } diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go index 8ed619888..15c7876ff 100644 --- a/pkg/engine/template_generator.go +++ b/pkg/engine/template_generator.go @@ -469,12 +469,14 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap { var parts = []string{ kubernetesWindowsAgentFunctionsPS1, kubernetesWindowsConfigFunctionsPS1, + kubernetesWindowsContainerdFunctionsPS1, kubernetesWindowsKubeletFunctionsPS1, kubernetesWindowsCniFunctionsPS1, kubernetesWindowsAzureCniFunctionsPS1, kubernetesWindowsLogsCleanupPS1, kubernetesWindowsNodeResetPS1, - kubernetesWindowsOpenSSHFunctionPS1} + kubernetesWindowsOpenSSHFunctionPS1, + } // Create a buffer, new zip buf := new(bytes.Buffer) diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 85d984446..acd73f29a 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -212,6 +212,7 @@ // ../../parts/k8s/windowsazurecnifunc.tests.ps1 // ../../parts/k8s/windowscnifunc.ps1 // ../../parts/k8s/windowsconfigfunc.ps1 +// ../../parts/k8s/windowscontainerdfunc.ps1 // ../../parts/k8s/windowsinstallopensshfunc.ps1 // ../../parts/k8s/windowskubeletfunc.ps1 // ../../parts/k8s/windowslogscleanup.ps1 @@ -39369,6 +39370,17 @@ function Register-NodeResetScriptTask { $definition = New-ScheduledTask -Action $action -Principal $principal -Trigger $trigger -Description "k8s-restart-job" Register-ScheduledTask -TaskName "k8s-restart-job" -InputObject $definition } + +function Assert-FileExists { + Param( + [Parameter(Mandatory=$true,Position=0)][string] + $Filename + ) + + if (-Not (Test-Path $Filename)) { + throw "$Filename does not exist" + } +} `) func k8sKuberneteswindowsfunctionsPs1Bytes() ([]byte, error) { @@ -39453,10 +39465,15 @@ $global:AgentCertificate = "{{WrapAsParameter "clientCertificate"}}" $global:KubeBinariesPackageSASURL = "{{WrapAsParameter "kubeBinariesSASURL"}}" $global:WindowsKubeBinariesURL = "{{WrapAsParameter "windowsKubeBinariesURL"}}" $global:KubeBinariesVersion = "{{WrapAsParameter "kubeBinariesVersion"}}" +$global:ContainerdUrl = "{{WrapAsParameter "windowsContainerdURL"}}" +$global:ContainerdSdnPluginUrl = "{{WrapAsParameter "windowsSdnPluginURL"}}" ## Docker Version $global:DockerVersion = "{{WrapAsParameter "windowsDockerVersion"}}" +## ContainerD Usage +$global:ContainerRuntime = "{{WrapAsParameter "containerRuntime"}}" + ## VM configuration passed by Azure $global:WindowsTelemetryGUID = "{{WrapAsParameter "windowsTelemetryGUID"}}" {{if eq GetIdentitySystem "adfs"}} @@ -39533,14 +39550,9 @@ Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" . c:\AzureData\k8s\windowscnifunc.ps1 . c:\AzureData\k8s\windowsazurecnifunc.ps1 . c:\AzureData\k8s\windowsinstallopensshfunc.ps1 +. c:\AzureData\k8s\windowscontainerdfunc.ps1 -function -Update-ServiceFailureActions() -{ - sc.exe failure "kubelet" actions= restart/60000/restart/60000/restart/60000 reset= 900 - sc.exe failure "kubeproxy" actions= restart/60000/restart/60000/restart/60000 reset= 900 - sc.exe failure "docker" actions= restart/60000/restart/60000/restart/60000 reset= 900 -} +$useContainerD = ($global:ContainerRuntime -eq "containerd") try { @@ -39610,12 +39622,22 @@ try Write-Log "Create required data directories as needed" Initialize-DataDirectories - Write-Log "Install docker" - $dockerTimer = [System.Diagnostics.Stopwatch]::StartNew() - Install-Docker -DockerVersion $global:DockerVersion - Set-DockerLogFileOptions - $dockerTimer.Stop() - $global:AppInsightsClient.TrackMetric("Install-Docker", $dockerTimer.Elapsed.TotalSeconds) + + if ($useContainerD) { + Write-Log "Installing ContainerD" + $containerdTimer = [System.Diagnostics.Stopwatch]::StartNew() + Install-Containerd -ContainerdUrl $global:ContainerdUrl + $containerdTimer.Stop() + $global:AppInsightsClient.TrackMetric("Install-ContainerD", $containerdTimer.Elapsed.TotalSeconds) + # TODO: disable/uninstall Docker later + } else { + Write-Log "Install docker" + $dockerTimer = [System.Diagnostics.Stopwatch]::StartNew() + Install-Docker -DockerVersion $global:DockerVersion + Set-DockerLogFileOptions + $dockerTimer.Stop() + $global:AppInsightsClient.TrackMetric("Install-Docker", $dockerTimer.Elapsed.TotalSeconds) + } Write-Log "Download kubelet binaries and unzip" Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL @@ -39672,14 +39694,19 @@ try Write-Log "Create the Pause Container kubletwin/pause" $infraContainerTimer = [System.Diagnostics.Stopwatch]::StartNew() - New-InfraContainer -KubeDir $global:KubeDir + New-InfraContainer -KubeDir $global:KubeDir -ContainerRuntime $global:ContainerRuntime $infraContainerTimer.Stop() $global:AppInsightsClient.TrackMetric("New-InfraContainer", $infraContainerTimer.Elapsed.TotalSeconds) - if (-not (Test-ContainerImageExists -Image "kubletwin/pause")) { + if (-not (Test-ContainerImageExists -Image "kubletwin/pause" -ContainerRuntime $global:ContainerRuntime)) { Write-Log "Could not find container with name kubletwin/pause" - $o = docker image list - Write-Log $o + if ($useContainerD) { + $o = ctr -n k8s.io image list + Write-Log $o + } else { + $o = docker image list + Write-Log $o + } throw "kubletwin/pause container does not exist!" } @@ -39718,7 +39745,13 @@ try } elseif ($global:NetworkPlugin -eq "kubenet") { Write-Log "Fetching additional files needed for kubenet" - Update-WinCNI -CNIPath $global:CNIPath + if ($useContainerD) { + # TODO: CNI may need to move to c:\program files\containerd\cni\bin with ContainerD + Install-SdnBridge -Url $global:ContainerdSdnPluginUrl -CNIPath $global:CNIPath + } else { + Update-WinCNI -CNIPath $global:CNIPath + } + Get-HnsPsm1 -HNSModule $global:HNSModule } New-ExternalHnsNetwork @@ -39742,7 +39775,8 @@ try -KubeClusterCIDR $global:KubeClusterCIDR ` + "`" + ` -KubeServiceCIDR $global:KubeServiceCIDR ` + "`" + ` -HNSModule $global:HNSModule ` + "`" + ` - -KubeletNodeLabels $global:KubeletNodeLabels + -KubeletNodeLabels $global:KubeletNodeLabels ` + "`" + ` + -UseContainerD $useContainerD Get-LogCollectionScripts @@ -39756,7 +39790,7 @@ try PREPROVISION_EXTENSION Write-Log "Update service failure actions" - Update-ServiceFailureActions + Update-ServiceFailureActions -ContainerRuntime $global:ContainerRuntime Adjust-DynamicPortRange Register-LogsCleanupScriptTask @@ -39778,7 +39812,7 @@ try else { # keep for debugging purposes - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AgentKey $AgentKey -AADClientId $AADClientId -AADClientSecret $AADClientSecret" + Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AgentKey $AgentKey -AADClientId $AADClientId -AADClientSecret $AADClientSecret -NetworkAPIVersion $NetworkAPIVersion -TargetEnvironment $TargetEnvironment" } } catch @@ -40474,6 +40508,21 @@ function Update-WinCNI DownloadFileOverHttp -Url $WinCniUrl -DestinationPath $wincniFile } +function Install-SdnBridge +{ + Param( + [Parameter(Mandatory=$true)][string] + $Url, + [Parameter(Mandatory=$true)][string] + $CNIPath + ) + + $cnizip = [Io.path]::Combine($CNIPath, "cni.zip") + DownloadFileOverHttp -Url $Url -DestinationPath $cnizip + Expand-Archive -path $cnizip -DestinationPath $CNIPath + del $cnizip +} + # TODO: Move the code that creates the wincni configuration file out of windowskubeletfunc.ps1 and put it here`) func k8sWindowscnifuncPs1Bytes() ([]byte, error) { @@ -40636,6 +40685,41 @@ function Adjust-DynamicPortRange() # Kube-proxy load balancing should be set to DSR mode when it releases with future versions of the OS Invoke-Executable -Executable "netsh.exe" -ArgList @("int", "ipv4", "set", "dynamicportrange", "tcp", "16385", "49151") +} + +# TODO: should this be in this PR? +# Service start actions. These should be split up later and included in each install step +function Update-ServiceFailureActions +{ + Param( + [Parameter(Mandatory = $true)][string] + $ContainerRuntime + ) + sc.exe failure "kubelet" actions= restart/60000/restart/60000/restart/60000 reset= 900 + sc.exe failure "kubeproxy" actions= restart/60000/restart/60000/restart/60000 reset= 900 + sc.exe failure $ContainerRuntime actions= restart/60000/restart/60000/restart/60000 reset= 900 +} + +function Add-SystemPathEntry +{ + Param( + [Parameter(Mandatory = $true)][string] + $Directory + ) + # update the path variable if it doesn't have the needed paths + $path = [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + $updated = $false + if(-not ($path -match $Directory.Replace("\","\\")+"(;|$)")) + { + $path += ";"+$Directory + $updated = $true + } + if($updated) + { + Write-Output "Updating path, added $Directory" + [Environment]::SetEnvironmentVariable("Path", $path, [EnvironmentVariableTarget]::Machine) + $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") + } }`) func k8sWindowsconfigfuncPs1Bytes() ([]byte, error) { @@ -40653,6 +40737,176 @@ func k8sWindowsconfigfuncPs1() (*asset, error) { return a, nil } +var _k8sWindowscontainerdfuncPs1 = []byte(`# this is $global to persist across all functions since this is dot-sourced +$global:ContainerdInstallLocation = "$Env:ProgramFiles\containerd" + +function RegisterContainerDService +{ + Assert-FileExists (Join-Path $global:ContainerdInstallLocation containerd.exe) + + Write-Host "Registering containerd as a service" + $cdbinary = Join-Path $global:ContainerdInstallLocation containerd.exe + $svc = Get-Service -Name containerd -ErrorAction SilentlyContinue + if ($null -ne $svc) { + & $cdbinary --unregister-service + } + & $cdbinary --register-service + $svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue + if ($null -eq $svc) { + throw "containerd.exe did not get installed as a service correctly." + } + $svc | Start-Service + if ($svc.Status -ne "Running") { + throw "containerd service is not running" + } +} + + +function Install-Containerd +{ + Param( + [Parameter(Mandatory = $true)][string] + $ContainerdUrl + ) + $zipfile = [Io.path]::Combine($ENV:TEMP, "containerd.zip") + DownloadFileOverHttp -Url $ContainerdUrl -DestinationPath $zipfile + Expand-Archive -path $zipfile -DestinationPath $global:ContainerdInstallLocation + del $zipfile + + Add-SystemPathEntry $global:ContainerdInstallLocation + + # TODO: remove if the node comes up without this code + # $configDir = [Io.Path]::Combine($ENV:ProgramData, "containerd") + # if (-Not (Test-Path $configDir)) { + # mkdir $configDir + # } + + # TODO: call containerd.exe dump config, then modify instead of starting with hardcoded + $configFile = [Io.Path]::Combine($global:ContainerdInstallLocation, "config.toml") + @" +version = 2 +root = "C:\\ProgramData\\containerd\\root" +state = "C:\\ProgramData\\containerd\\state" +plugin_dir = "" +disabled_plugins = [] +required_plugins = [] +oom_score = 0 + +[grpc] + address = "\\\\.\\pipe\\containerd-containerd" + tcp_address = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[ttrpc] + address = "" + uid = 0 + gid = 0 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[plugins] + [plugins."io.containerd.gc.v1.scheduler"] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" + [plugins."io.containerd.grpc.v1.cri"] + disable_tcp_service = true + stream_server_address = "127.0.0.1" + stream_server_port = "0" + stream_idle_timeout = "4h0m0s" + enable_selinux = false + sandbox_image = "mcr.microsoft.com/k8s/core/pause:1.2.0" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + disable_cgroup = false + disable_apparmor = false + restrict_oom_score_adj = false + max_concurrent_downloads = 3 + disable_proc_mount = false + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "windows" + default_runtime_name = "runhcs-wcow-process" + no_pivot = false + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process] + runtime_type = "io.containerd.runhcs.v1" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "C:\\k\\cni" + conf_dir = "C:\\k\\cni\\config" + max_conf_num = 1 + conf_template = "" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + [plugins."io.containerd.runtime.v2.task"] + platforms = ["windows/amd64", "linux/amd64"] + [plugins."io.containerd.service.v1.diff-service"] + default = ["windows", "windows-lcow"] +"@ | Out-File -Encoding ascii $configFile + RegisterContainerDService + +}`) + +func k8sWindowscontainerdfuncPs1Bytes() ([]byte, error) { + return _k8sWindowscontainerdfuncPs1, nil +} + +func k8sWindowscontainerdfuncPs1() (*asset, error) { + bytes, err := k8sWindowscontainerdfuncPs1Bytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "k8s/windowscontainerdfunc.ps1", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _k8sWindowsinstallopensshfuncPs1 = []byte(`function Install-OpenSSH { Param( @@ -40772,7 +41026,9 @@ Write-AzureConfig { [Parameter(Mandatory = $true)][string] $KubeDir, [Parameter(Mandatory = $true)][string] - $TargetEnvironment + $TargetEnvironment, + [Parameter(Mandatory = $false)][bool] + $UseContainerD = $false ) if ( -Not $PrimaryAvailabilitySetName -And -Not $PrimaryScaleSetName ) { @@ -40864,17 +41120,51 @@ users: $kubeConfig | Out-File -encoding ASCII -filepath "$kubeConfigFile" } +function +Test-ContainerImageExists { + Param( + [Parameter(Mandatory = $true)][string] + $Image, + [Parameter(Mandatory = $false)][string] + $Tag, + [Parameter(Mandatory = $false)][string] + $ContainerRuntime = "docker" + ) + + $target = $Image + if ($Tag) { + $target += ":$Tag" + } + + if ($ContainerRuntime -eq "docker") { + $images = docker image list $target --format "{{json .}}" + return $images.Count -gt 0 + } + else + { + return ( (ctr.exe -n k8s.io images list) | Select-String $target) -ne $Null + } +} + function Build-PauseContainer { Param( [Parameter(Mandatory = $true)][string] $WindowsBase, - $DestinationTag + $DestinationTag, + [Parameter(Mandatory = $false)][string] + $ContainerRuntime = "docker" ) # Future work: This needs to build wincat - see https://github.com/Azure/aks-engine/issues/1461 + # Otherwise, delete this code and require a prebuilt pause image (or override with one from an Azure Container Registry instance) + # ContainerD can't build, so doing the builds outside of node deployment is probably the right long-term solution. "FROM $($WindowsBase)" | Out-File -encoding ascii -FilePath Dockerfile "CMD cmd /c ping -t localhost" | Out-File -encoding ascii -FilePath Dockerfile -Append - Invoke-Executable -Executable "docker" -ArgList @("build", "-t", "$DestinationTag", ".") + if ($ContainerRuntime -eq "docker") { + Invoke-Executable -Executable "docker" -ArgList @("build", "-t", "$DestinationTag", ".") + } else { + throw "Cannot build pause container without Docker" + } } function @@ -40882,7 +41172,9 @@ New-InfraContainer { Param( [Parameter(Mandatory = $true)][string] $KubeDir, - $DestinationTag = "kubletwin/pause" + $DestinationTag = "kubletwin/pause", + [Parameter(Mandatory = $false)][string] + $ContainerRuntime = "docker" ) cd $KubeDir $computerInfo = Get-ComputerInfo @@ -40895,14 +41187,21 @@ New-InfraContainer { $pauseImageVersions = @("1803", "1809", "1903", "1909") if ($pauseImageVersions -icontains $computerInfo.WindowsVersion) { - $imageList = docker images $defaultPauseImage --format "{{.Repository}}:{{.Tag}}" - if (-not $imageList) { - Invoke-Executable -Executable "docker" -ArgList @("pull", "$defaultPauseImage") -Retries 5 -RetryDelaySeconds 30 + if ($ContainerRuntime -eq "docker") { + if (-not (Test-ContainerImageExists -Image $defaultPauseImage -ContainerRuntime $ContainerRuntime)) { + Invoke-Executable -Executable "docker" -ArgList @("pull", "$defaultPauseImage") -Retries 5 -RetryDelaySeconds 30 + } + Invoke-Executable -Executable "docker" -ArgList @("tag", "$defaultPauseImage", "$DestinationTag") + } else { + # containerd + if (-not (Test-ContainerImageExists -Image $defaultPauseImage -ContainerRuntime $ContainerRuntime)) { + Invoke-Executable -Executable "ctr" -ArgList @("-n", "k8s.io", "image", "pull", "$defaultPauseImage") -Retries 5 -RetryDelaySeconds 30 + } + Invoke-Executable -Executable "ctr" -ArgList @("-n", "k8s.io", "image", "tag", "$defaultPauseImage", "$DestinationTag") } - Invoke-Executable -Executable "docker" -ArgList @("tag", "$defaultPauseImage", "$DestinationTag") } else { - Build-PauseContainer -WindowsBase "mcr.microsoft.com/nanoserver-insider" -DestinationTag $DestinationTag + Build-PauseContainer -WindowsBase "mcr.microsoft.com/nanoserver-insider" -DestinationTag $DestinationTag -ContainerRuntime $ContainerRuntime } } @@ -40912,7 +41211,9 @@ Test-ContainerImageExists { [Parameter(Mandatory = $true)][string] $Image, [Parameter(Mandatory = $false)][string] - $Tag + $Tag, + [Parameter(Mandatory = $false)][string] + $ContainerRuntime = "docker" ) $target = $Image @@ -40920,12 +41221,16 @@ Test-ContainerImageExists { $target += ":$Tag" } - $images = docker image list $target --format "{{json .}}" - - return $images.Count -gt 0 + if ($ContainerRuntime -eq "docker") { + $images = docker image list $target --format "{{json .}}" + return $images.Count -gt 0 + } + else + { + return ( (ctr.exe -n k8s.io images list) | Select-String $target) -ne $Null + } } - # TODO: Deprecate this and replace with methods that get individual components instead of zip containing everything # This expects the ZIP file created by Azure Pipelines. function @@ -41074,7 +41379,9 @@ Install-KubernetesServices { [Parameter(Mandatory = $true)][string] $HNSModule, [Parameter(Mandatory = $true)][string] - $KubeletNodeLabels + $KubeletNodeLabels, + [Parameter(Mandatory = $false)][bool] + $UseContainerD = $false ) # Calculate some local paths @@ -41112,6 +41419,11 @@ Install-KubernetesServices { throw "Unknown network type $NetworkPlugin, can't configure kubelet" } + # Update args to use ContainerD if needed + if ($UseContainerD -eq $true) { + $KubeletArgList += @("--container-runtime=remote", "--container-runtime-endpoint=npipe://./pipe/containerd-containerd") + } + # Used in WinCNI version of kubeletstart.ps1 $KubeletArgListStr = "" $KubeletArgList | Foreach-Object { @@ -41203,7 +41515,8 @@ $KubeletCommandLine "@ } - else { + elseif (($NetworkPlugin -eq "kubenet" ) -and ($UseContainerD -eq $false)) + { # using WinCNI. TODO: If WinCNI support is removed, then delete this as dead code later $KubeNetwork = "l2bridge" $kubeStartStr += @" @@ -41336,7 +41649,161 @@ catch } "@ - } # end else using WinCNI. + } # end elseif using WinCNI and Docker. + elseif (($NetworkPlugin -eq "kubenet" ) -and ($UseContainerD -eq $true)) + { + # TODO: something is wrong with the CNI configuration + # Warning FailedCreatePodSandBox 2m16s (x222 over 50m) kubelet, 4068k8s011 (combined from similar events): Failed to create pod sandbox: + # rpc error: code = Unknown desc = failed to setup network for sandbox "922b1a200078edb15c7a5732612cbe19e5dadf7cf8a4622d72b376874522435d": error creating endpoint hcnCreateEndpoint failed in Win32: Invalid JSON document string. (0x803b001b) {"Success":false,"Error":"Invalid JSON document string. {{Policies.Type,UnknownEnumValue}}","ErrorCode":2151350299} : endpoint config &{ 922b1a200078edb15c7a5732612cbe19e5dadf7cf8a4622d72b376874522435d_l2bridge f94e9649-d4df-486e-bcd4-2721938d89f3 [{OutBoundNAT []} {ROUTE []}] [] { [default.svc.cluster.local] [10.0.0.10] []} [{10.240.0.1 0.0.0.0/0 0}] 0 {2 0}} + + + # using SDNBridge & Containerd + $KubeNetwork = "l2bridge" + $kubeStartStr += @" + +function +Get-DefaultGateway(` + "`" + `$CIDR) +{ + return ` + "`" + `$CIDR.substring(0,` + "`" + `$CIDR.lastIndexOf(".")) + ".1" +} + +function +Get-PodCIDR() +{ + ` + "`" + `$podCIDR = c:\k\kubectl.exe --kubeconfig=c:\k\config get nodes/` + "`" + `$(` + "`" + `$env:computername.ToLower()) -o custom-columns=podCidr:.spec.podCIDR --no-headers + return ` + "`" + `$podCIDR +} + +function +Test-PodCIDR(` + "`" + `$podCIDR) +{ + return ` + "`" + `$podCIDR.length -gt 0 +} + +function +Update-CNIConfig(` + "`" + `$podCIDR, ` + "`" + `$masterSubnetGW) +{ + ` + "`" + `$jsonSampleConfig = +"{ + ""cniVersion"": ""0.2.0"", + ""name"": """", + ""type"": ""sdnbridge.exe"", + ""master"": ""Ethernet"", + ""capabilities"": { ""portMappings"": true }, + ""ipam"": { + ""environment"": ""azure"", + ""subnet"":"""", + ""routes"": [{ + ""GW"":"""" + }] + }, + ""dns"" : { + ""Nameservers"" : [ """" ], + ""Search"" : [ """" ] + }, + ""AdditionalArgs"" : [ + { + ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""OutBoundNAT"", ""Settings"" : { ""Exceptions"": [ """", """" ] }} + }, + { + ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""SDNRoute"", ""Settings"" : { ""DestinationPrefix"": """", ""NeedEncap"" : true }} + } + ] +}" + + ` + "`" + `$configJson = ConvertFrom-Json ` + "`" + `$jsonSampleConfig + ` + "`" + `$configJson.name = ` + "`" + `$global:NetworkMode.ToLower() + ` + "`" + `$configJson.ipam.subnet=` + "`" + `$podCIDR + ` + "`" + `$configJson.ipam.routes[0].GW = ` + "`" + `$masterSubnetGW + ` + "`" + `$configJson.dns.Nameservers[0] = ` + "`" + `$global:KubeDnsServiceIp + ` + "`" + `$configJson.dns.Search[0] = ` + "`" + `$global:KubeDnsSearchPath + + + ` + "`" + `$configJson.AdditionalArgs[0].Value.Settings.Exceptions[0] = ` + "`" + `$global:KubeClusterCIDR + ` + "`" + `$configJson.AdditionalArgs[0].Value.Settings.Exceptions[1] = ` + "`" + `$global:MasterSubnet + ` + "`" + `$configJson.AdditionalArgs[1].Value.Settings.DestinationPrefix = ` + "`" + `$global:KubeServiceCIDR + + if (Test-Path ` + "`" + `$global:CNIConfig) + { + Clear-Content -Path ` + "`" + `$global:CNIConfig + } + + Write-Host "Generated CNI Config [` + "`" + `$configJson]" + + Add-Content -Path ` + "`" + `$global:CNIConfig -Value (ConvertTo-Json ` + "`" + `$configJson -Depth 20) +} + +try +{ + ` + "`" + `$masterSubnetGW = Get-DefaultGateway ` + "`" + `$global:MasterSubnet + ` + "`" + `$podCIDR=Get-PodCIDR + ` + "`" + `$podCidrDiscovered=Test-PodCIDR(` + "`" + `$podCIDR) + + # if the podCIDR has not yet been assigned to this node, start the kubelet process to get the podCIDR, and then promptly kill it. + if (-not ` + "`" + `$podCidrDiscovered) + { + ` + "`" + `$argList = $KubeletArgListStr + + ` + "`" + `$process = Start-Process -FilePath c:\k\kubelet.exe -PassThru -ArgumentList ` + "`" + `$argList + + # run kubelet until podCidr is discovered + Write-Host "waiting to discover pod CIDR" + while (-not ` + "`" + `$podCidrDiscovered) + { + Write-Host "Sleeping for 10s, and then waiting to discover pod CIDR" + Start-Sleep 10 + + ` + "`" + `$podCIDR=Get-PodCIDR + ` + "`" + `$podCidrDiscovered=Test-PodCIDR(` + "`" + `$podCIDR) + } + + # stop the kubelet process now that we have our CIDR, discard the process output + ` + "`" + `$process | Stop-Process | Out-Null + } + + # Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) + netsh advfirewall set allprofiles state off + + # startup the service + ` + "`" + `$hnsNetwork = Get-HnsNetwork | ? Name -EQ ` + "`" + `$global:NetworkMode.ToLower() + + if (` + "`" + `$hnsNetwork) + { + # Kubelet has been restarted with existing network. + # Cleanup all containers + # TODO: convert this to ctr.exe -n k8s.io container list ; container rm + docker ps -q | foreach {docker rm ` + "`" + `$_ -f} + # cleanup network + Write-Host "Cleaning up old HNS network found" + Remove-HnsNetwork ` + "`" + `$hnsNetwork + Start-Sleep 10 + } + + Write-Host "Creating a new hns Network" + ipmo ` + "`" + `$global:HNSModule + + ` + "`" + `$hnsNetwork = New-HNSNetwork -Type ` + "`" + `$global:NetworkMode -AddressPrefix ` + "`" + `$podCIDR -Gateway ` + "`" + `$masterSubnetGW -Name ` + "`" + `$global:NetworkMode.ToLower() -Verbose + # New network has been created, Kubeproxy service has to be restarted + Restart-Service Kubeproxy + + Start-Sleep 10 + # Add route to all other POD networks + Write-Host "Updating CNI config" + Update-CNIConfig ` + "`" + `$podCIDR ` + "`" + `$masterSubnetGW + + $KubeletCommandLine +} +catch +{ + Write-Error ` + "`" + `$_ +} + +"@ + } # end elseif using sdnbridge and containerd. + else + { + throw "The combination of $NetworkPlugin and UseContainerD=$UseContainerD is not implemented" + } # Now that the script is generated, based on what CNI plugin and startup options are needed, write it to disk $kubeStartStr | Out-File -encoding ASCII -filepath $KubeletStartFile @@ -44790,6 +45257,18 @@ var _windowsparamsT = []byte(` {{if IsKubernetes}} }, "type": "string" }, + "windowsContainerdURL": { + "metadata": { + "description": "TODO: containerd - these binaries are not available yet" + }, + "type": "string" + }, + "windowsSdnPluginURL": { + "metadata": { + "description": "TODO: containerd - these binaries are not available yet" + }, + "type": "string" + }, "kubeServiceCidr": { "metadata": { "description": "Kubernetes service address space" @@ -45152,6 +45631,7 @@ var _bindata = map[string]func() (*asset, error){ "k8s/windowsazurecnifunc.tests.ps1": k8sWindowsazurecnifuncTestsPs1, "k8s/windowscnifunc.ps1": k8sWindowscnifuncPs1, "k8s/windowsconfigfunc.ps1": k8sWindowsconfigfuncPs1, + "k8s/windowscontainerdfunc.ps1": k8sWindowscontainerdfuncPs1, "k8s/windowsinstallopensshfunc.ps1": k8sWindowsinstallopensshfuncPs1, "k8s/windowskubeletfunc.ps1": k8sWindowskubeletfuncPs1, "k8s/windowslogscleanup.ps1": k8sWindowslogscleanupPs1, @@ -45466,6 +45946,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "windowsazurecnifunc.tests.ps1": {k8sWindowsazurecnifuncTestsPs1, map[string]*bintree{}}, "windowscnifunc.ps1": {k8sWindowscnifuncPs1, map[string]*bintree{}}, "windowsconfigfunc.ps1": {k8sWindowsconfigfuncPs1, map[string]*bintree{}}, + "windowscontainerdfunc.ps1": {k8sWindowscontainerdfuncPs1, map[string]*bintree{}}, "windowsinstallopensshfunc.ps1": {k8sWindowsinstallopensshfuncPs1, map[string]*bintree{}}, "windowskubeletfunc.ps1": {k8sWindowskubeletfuncPs1, map[string]*bintree{}}, "windowslogscleanup.ps1": {k8sWindowslogscleanupPs1, map[string]*bintree{}}, diff --git a/pkg/engine/testdata/windows/kubernetes-kubernetesconfig.json b/pkg/engine/testdata/windows/kubernetes-kubernetesconfig.json index 4a49359ff..98bb4cc1e 100644 --- a/pkg/engine/testdata/windows/kubernetes-kubernetesconfig.json +++ b/pkg/engine/testdata/windows/kubernetes-kubernetesconfig.json @@ -6,7 +6,9 @@ "orchestratorRelease": "1.13", "kubernetesConfig": { "useInstanceMetadata": false, - "windowsNodeBinariesURL": "http://test/test.tar.gz" + "windowsNodeBinariesURL": "http://test/test.tar.gz", + "windowsContainerdURL": "http://test/testcontainerd.tar.gz", + "windowsSdnPluginURL": "http://test/testsdnplugin.tar.gz" } }, "masterProfile": { diff --git a/scripts/build-windows-containerd.sh b/scripts/build-windows-containerd.sh new file mode 100755 index 000000000..c4d08fd6b --- /dev/null +++ b/scripts/build-windows-containerd.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e -x -o pipefail + +GOTAG="1.13.7" +DOCKERARGS="--network=host" + +OUTDIR="$(pwd)/_output" +if [ ! -d $OUTDIR ]; then + mkdir $OUTDIR +fi + +cat < $OUTDIR/buildcri.sh +set -e -x -o pipefail +export GOOS=windows +export GOARCH=amd64 +go get github.com/Microsoft/hcsshim +cd src/github.com/Microsoft/hcsshim +git rev-parse HEAD > /output/hcsshim-revision.txt +cd \$GOPATH +go build -o /output/containerd-shim-runhcs-v1.exe github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1 +mkdir -p src/github.com/containerd +cd src/github.com/containerd +pwd +git clone https://github.com/containerd/containerd.git +cd containerd +git rev-parse HEAD > /output/containerd-revision.txt +make +# make cri-release # should work, but doesn't +cp bin/ctr.exe /output +#cp bin/containerd.exe /output # missing CRI plugin, so build from containerd/cri +cd \$GOPATH +cd src/github.com/containerd +git clone https://github.com/containerd/cri.git +cd cri +git rev-parse HEAD > /output/cri-revision.txt +make containerd +cp _output/containerd.exe /output +apt update +apt install -y zip +cd /output +zip windows-cri-containerd.zip *.exe *.txt +rm -f /output/*.exe +rm -f /output/*.txt +EOF +chmod +x $OUTDIR/buildcri.sh + +cat < $OUTDIR/buildcni.sh +set -e -x -o pipefail +export GOOS=windows +export GOARCH=amd64 +mkdir -p src/github.com/Microsoft +cd src/github.com/Microsoft +git clone https://github.com/Microsoft/windows-container-networking.git +cd windows-container-networking +git rev-parse HEAD > /output/cni-revision.txt +make all +mv out/*.exe /output +apt update +apt install -y zip +cd /output +zip windows-cni-containerd.zip *.exe *.txt +rm -f /output/*.exe +rm -f /output/*.txt +EOF +chmod +x $OUTDIR/buildcni.sh + + + +docker run $DOCKERARGS -v $OUTDIR:/output golang:$GOTAG /bin/bash -c /output/buildcri.sh +docker run $DOCKERARGS -v $OUTDIR:/output golang:$GOTAG /bin/bash -c /output/buildcni.sh