feat: Use NSSM for containerd and collect containerd logs (#4219)

* feat: add logging to see the containerd containers * respond to feedback
2021-03-25 13:19:07 -07:00 · 2021-03-25 13:19:07 -07:00 · eeab855fd9
--- a/parts/k8s/containerdtemplate.toml
+++ b/parts/k8s/containerdtemplate.toml
@ -11,7 +11,7 @@ state = "C:\\ProgramData\\containerd\\state"

 [debug]
  address = ""
-  level = "debug"
+  level = "info"

 [metrics]
  address = ""
@ -32,6 +32,7 @@ state = "C:\\ProgramData\\containerd\\state"
    max_container_log_line_size = 16384
    [plugins.cri.containerd]
      snapshotter = "windows"
+      discard_unpacked_layers = true
      no_pivot = false
      [plugins.cri.containerd.default_runtime]
        runtime_type = "io.containerd.runhcs.v1"
--- a/parts/k8s/kuberneteswindowssetup.ps1
+++ b/parts/k8s/kuberneteswindowssetup.ps1
@ -256,6 +256,17 @@ try

        Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp

+        Write-Log "Download kubelet binaries and unzip"
+        Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
+
+        # The custom package has a few files that are nessary for future steps (nssm.exe)
+        # this is a temporary work around to get the binaries until we depreciate
+        # custom package and nssm.exe as defined in #3851.
+        if ($global:WindowsKubeBinariesURL){
+            Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
+            Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
+        }
+
        if ($useContainerD) {
            Write-Log "Installing ContainerD"
            $containerdTimer = [System.Diagnostics.Stopwatch]::StartNew()
@ -265,7 +276,7 @@ try
                $cniBinPath = $global:CNIPath
                $cniConfigPath = $global:CNIConfigPath
            }
-            Install-Containerd -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath
+            Install-Containerd -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir
            $containerdTimer.Stop()
            $global:AppInsightsClient.TrackMetric("Install-ContainerD", $containerdTimer.Elapsed.TotalSeconds)
            # TODO: disable/uninstall Docker later
@ -278,18 +289,6 @@ try
            $global:AppInsightsClient.TrackMetric("Install-Docker", $dockerTimer.Elapsed.TotalSeconds)
        }

-        Write-Log "Download kubelet binaries and unzip"
-        Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
-
-        # this overwrite the binaries that are download from the custom packge with binaries
-        # The custom package has a few files that are nessary for future steps (nssm.exe)
-        # this is a temporary work around to get the binaries until we depreciate
-        # custom package and nssm.exe as defined in #3851.
-        if ($global:WindowsKubeBinariesURL){
-            Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-            Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-        }
-
        Write-Log "Write Azure cloud provider config"
        Write-AzureConfig `
            -KubeDir $global:KubeDir `
--- a/parts/k8s/windowscontainerdfunc.ps1
+++ b/parts/k8s/windowscontainerdfunc.ps1
@ -1,23 +1,41 @@
 # this is $global to persist across all functions since this is dot-sourced
 $global:ContainerdInstallLocation = "$Env:ProgramFiles\containerd"
+$global:Containerdbinary = (Join-Path $global:ContainerdInstallLocation containerd.exe)

 function RegisterContainerDService {
-  Assert-FileExists (Join-Path $global:ContainerdInstallLocation containerd.exe)
+  Param(
+    [Parameter(Mandatory = $true)][string]
+    $kubedir
+  )
+
+  Assert-FileExists $global:Containerdbinary
+
+  # in the past service was not installed via nssm so remove it in case
+  $svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue
+  if ($null -ne $svc) {
+    sc.exe delete containerd
+  }

  Write-Host "Registering containerd as a service"
-  $cdbinary = Join-Path $global:ContainerdInstallLocation containerd.exe
-  $svc = Get-Service -Name containerd -ErrorAction SilentlyContinue
-  if ($null -ne $svc) {
-    & $cdbinary --unregister-service
-  }
-  & $cdbinary --register-service
+  # setup containerd
+  & "$KubeDir\nssm.exe" install containerd $global:Containerdbinary | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppDirectory $KubeDir | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd DisplayName containerd | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd Description containerd | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd Start SERVICE_DEMAND_START | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd ObjectName LocalSystem | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd Type SERVICE_WIN32_OWN_PROCESS | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppThrottle 1500 | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppStdout "$KubeDir\containerd.log" | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppStderr "$KubeDir\containerd.err.log" | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppRotateFiles 1 | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppRotateOnline 1 | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppRotateSeconds 86400 | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppRotateBytes 10485760 | RemoveNulls
+
  $svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue
-  if ($null -eq $svc) {
-    throw "containerd.exe did not get installed as a service correctly."
-  }
-  $svc | Start-Service
  if ($svc.Status -ne "Running") {
-    throw "containerd service is not running"
+    Start-Service containerd
  }
 }

@ -94,9 +112,6 @@ function Enable-Logging {
    # !ContainerPlatformPersistent profile is made to work with long term and boot tracing
    & $diag -Start -ProfilePath "$global:ContainerdInstallLocation\ContainerPlatform.wprp!ContainerPlatformPersistent" -TempPath $logs
  }
-  else {
-    Write-Log "Containerd hyperv logging script not avalaible"
-  }
 }

 function Install-Containerd {
@ -106,7 +121,9 @@ function Install-Containerd {
    [Parameter(Mandatory = $true)][string]
    $CNIBinDir,
    [Parameter(Mandatory = $true)][string]
-    $CNIConfDir
+    $CNIConfDir,
+    [Parameter(Mandatory = $true)][string]
+    $KubeDir
  )

  $svc = Get-Service -Name containerd -ErrorAction SilentlyContinue
@ -137,7 +154,6 @@ function Install-Containerd {

  # get configuration options
  Add-SystemPathEntry $global:ContainerdInstallLocation
-  $cdbinary = Join-Path $global:ContainerdInstallLocation containerd.exe
  $configFile = [Io.Path]::Combine($global:ContainerdInstallLocation, "config.toml")
  $clusterConfig = ConvertFrom-Json ((Get-Content $global:KubeClusterConfigPath -ErrorAction Stop) | Out-String)
  $pauseImage = $clusterConfig.Cri.Images.Pause
@ -177,6 +193,6 @@ function Install-Containerd {
  Replace('{{currentversion}}', $windowsVersion) | `
    Out-File -FilePath "$configFile" -Encoding ascii

-  RegisterContainerDService
+  RegisterContainerDService -KubeDir $KubeDir
  Enable-Logging
 }
--- a/pkg/engine/templates_generated.go
+++ b/pkg/engine/templates_generated.go
@ -16167,7 +16167,7 @@ state = "C:\\ProgramData\\containerd\\state"

 [debug]
  address = ""
-  level = "debug"
+  level = "info"

 [metrics]
  address = ""
@ -16188,6 +16188,7 @@ state = "C:\\ProgramData\\containerd\\state"
    max_container_log_line_size = 16384
    [plugins.cri.containerd]
      snapshotter = "windows"
+      discard_unpacked_layers = true
      no_pivot = false
      [plugins.cri.containerd.default_runtime]
        runtime_type = "io.containerd.runhcs.v1"
@ -17372,6 +17373,17 @@ try

        Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp

+        Write-Log "Download kubelet binaries and unzip"
+        Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
+
+        # The custom package has a few files that are nessary for future steps (nssm.exe)
+        # this is a temporary work around to get the binaries until we depreciate
+        # custom package and nssm.exe as defined in #3851.
+        if ($global:WindowsKubeBinariesURL){
+            Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
+            Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
+        }
+
        if ($useContainerD) {
            Write-Log "Installing ContainerD"
            $containerdTimer = [System.Diagnostics.Stopwatch]::StartNew()
@ -17381,7 +17393,7 @@ try
                $cniBinPath = $global:CNIPath
                $cniConfigPath = $global:CNIConfigPath
            }
-            Install-Containerd -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath
+            Install-Containerd -ContainerdUrl $global:ContainerdUrl -CNIBinDir $cniBinPath -CNIConfDir $cniConfigPath -KubeDir $global:KubeDir
            $containerdTimer.Stop()
            $global:AppInsightsClient.TrackMetric("Install-ContainerD", $containerdTimer.Elapsed.TotalSeconds)
            # TODO: disable/uninstall Docker later
@ -17394,18 +17406,6 @@ try
            $global:AppInsightsClient.TrackMetric("Install-Docker", $dockerTimer.Elapsed.TotalSeconds)
        }

-        Write-Log "Download kubelet binaries and unzip"
-        Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL
-
-        # this overwrite the binaries that are download from the custom packge with binaries
-        # The custom package has a few files that are nessary for future steps (nssm.exe)
-        # this is a temporary work around to get the binaries until we depreciate
-        # custom package and nssm.exe as defined in #3851.
-        if ($global:WindowsKubeBinariesURL){
-            Write-Log "Overwriting kube node binaries from $global:WindowsKubeBinariesURL"
-            Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL
-        }
-
        Write-Log "Write Azure cloud provider config"
        Write-AzureConfig ` + "`" + `
            -KubeDir $global:KubeDir ` + "`" + `
@ -18854,24 +18854,42 @@ func k8sWindowsconfigfuncPs1() (*asset, error) {

 var _k8sWindowscontainerdfuncPs1 = []byte(`# this is $global to persist across all functions since this is dot-sourced
 $global:ContainerdInstallLocation = "$Env:ProgramFiles\containerd"
+$global:Containerdbinary = (Join-Path $global:ContainerdInstallLocation containerd.exe)

 function RegisterContainerDService {
-  Assert-FileExists (Join-Path $global:ContainerdInstallLocation containerd.exe)
+  Param(
+    [Parameter(Mandatory = $true)][string]
+    $kubedir
+  )
+
+  Assert-FileExists $global:Containerdbinary
+
+  # in the past service was not installed via nssm so remove it in case
+  $svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue
+  if ($null -ne $svc) {
+    sc.exe delete containerd
+  }

  Write-Host "Registering containerd as a service"
-  $cdbinary = Join-Path $global:ContainerdInstallLocation containerd.exe
-  $svc = Get-Service -Name containerd -ErrorAction SilentlyContinue
-  if ($null -ne $svc) {
-    & $cdbinary --unregister-service
-  }
-  & $cdbinary --register-service
+  # setup containerd
+  & "$KubeDir\nssm.exe" install containerd $global:Containerdbinary | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppDirectory $KubeDir | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd DisplayName containerd | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd Description containerd | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd Start SERVICE_DEMAND_START | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd ObjectName LocalSystem | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd Type SERVICE_WIN32_OWN_PROCESS | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppThrottle 1500 | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppStdout "$KubeDir\containerd.log" | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppStderr "$KubeDir\containerd.err.log" | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppRotateFiles 1 | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppRotateOnline 1 | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppRotateSeconds 86400 | RemoveNulls
+  & "$KubeDir\nssm.exe" set containerd AppRotateBytes 10485760 | RemoveNulls
+
  $svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue
-  if ($null -eq $svc) {
-    throw "containerd.exe did not get installed as a service correctly."
-  }
-  $svc | Start-Service
  if ($svc.Status -ne "Running") {
-    throw "containerd service is not running"
+    Start-Service containerd
  }
 }

@ -18948,9 +18966,6 @@ function Enable-Logging {
    # !ContainerPlatformPersistent profile is made to work with long term and boot tracing
    & $diag -Start -ProfilePath "$global:ContainerdInstallLocation\ContainerPlatform.wprp!ContainerPlatformPersistent" -TempPath $logs
  }
-  else {
-    Write-Log "Containerd hyperv logging script not avalaible"
-  }
 }

 function Install-Containerd {
@ -18960,7 +18975,9 @@ function Install-Containerd {
    [Parameter(Mandatory = $true)][string]
    $CNIBinDir,
    [Parameter(Mandatory = $true)][string]
-    $CNIConfDir
+    $CNIConfDir,
+    [Parameter(Mandatory = $true)][string]
+    $KubeDir
  )

  $svc = Get-Service -Name containerd -ErrorAction SilentlyContinue
@ -18991,7 +19008,6 @@ function Install-Containerd {

  # get configuration options
  Add-SystemPathEntry $global:ContainerdInstallLocation
-  $cdbinary = Join-Path $global:ContainerdInstallLocation containerd.exe
  $configFile = [Io.Path]::Combine($global:ContainerdInstallLocation, "config.toml")
  $clusterConfig = ConvertFrom-Json ((Get-Content $global:KubeClusterConfigPath -ErrorAction Stop) | Out-String)
  $pauseImage = $clusterConfig.Cri.Images.Pause
@ -19031,7 +19047,7 @@ function Install-Containerd {
  Replace('{{currentversion}}', $windowsVersion) | ` + "`" + `
    Out-File -FilePath "$configFile" -Encoding ascii

-  RegisterContainerDService
+  RegisterContainerDService -KubeDir $KubeDir
  Enable-Logging
 }
 `)
--- a/scripts/collect-windows-logs.ps1
+++ b/scripts/collect-windows-logs.ps1
@ -1,6 +1,19 @@
 $ProgressPreference = "SilentlyContinue"

-$lockedFiles = "kubelet.err.log", "kubelet.log", "kubeproxy.log", "kubeproxy.err.log", "azure-vnet-telemetry.log", "azure-vnet.log", "network-interfaces.json", "interfaces.json", "csi-proxy.log", "csi-proxy.err.log"
+$lockedFiles = @(
+  "kubelet.err.log", 
+  "kubelet.log", 
+  "kubeproxy.log", 
+  "kubeproxy.err.log", 
+  "azure-vnet-telemetry.log", 
+  "azure-vnet.log", 
+  "network-interfaces.json", 
+  "interfaces.json", 
+  "csi-proxy.log", 
+  "csi-proxy.err.log",
+  "containerd.log",
+  "containerd.err.log"
+)

 $timeStamp = get-date -format 'yyyyMMdd-hhmmss'
 $zipName = "$env:computername-$($timeStamp)_logs.zip"
@ -86,6 +99,16 @@ else {
  Write-Host "Containerd hyperv logs not avalaible"
 }

+# log containerd containers (this is done for docker via networking collectlogs.ps1)
+$res = Get-Command ctr.exe -ErrorAction SilentlyContinue
+if ($res)
+{ 
+  & ctr.exe -n k8s.io c ls > "$ENV:TEMP\$timeStamp-containerd-containers.txt"
+  & ctr.exe -n k8s.io t ls > "$ENV:TEMP\$timeStamp-containerd-tasks.txt"
+  $paths += "$ENV:TEMP\$timeStamp-containerd-containers.txt"
+  $paths += "$ENV:TEMP\$timeStamp-containerd-tasks.txt"
+}
+
 Write-Host "Compressing all logs to $zipName"
 $paths | Format-Table FullName, Length -AutoSize
 Compress-Archive -LiteralPath $paths -DestinationPath $zipName
--- a/staging/provisioning/windows/windowslogscleanup.ps1
+++ b/staging/provisioning/windows/windowslogscleanup.ps1
@ -13,7 +13,7 @@ function Write-Log ($message) {

 Write-Log "Entering windowslogscleanup.ps1"

-$logFilePrefixes = @("kubelet", "kubelet.err", "kubeproxy", "kubeproxy.err")
+$logFilePrefixes = @("kubelet", "kubelet.err", "kubeproxy", "kubeproxy.err", "containerd.err", "containerd")

 foreach ($logFilePrefix in $logFilePrefixes) {
    $oldLogs = [IO.Directory]::GetFiles("c:\k", "$($logFilePrefix)-*.log")
--- a/vhd/packer/configure-windows-vhd.ps1
+++ b/vhd/packer/configure-windows-vhd.ps1
@ -66,9 +66,14 @@ function Get-ContainerImages {


    if ($containerRuntime -eq 'containerd') {
+        # start containerd to pre-pull the images to disk on VHD
+        # CSE will configure and register containerd as a service at deployment time
+        Start-Job -Name containerd -ScriptBlock { containerd.exe }
        foreach ($image in $imagesToPull) {
            & ctr.exe -n k8s.io images pull $image
        }
+        Stop-Job  -Name containerd
+        Remove-Job -Name containerd
    }
    else {
        foreach ($image in $imagesToPull) {
@ -83,6 +88,7 @@ function Get-FilesToCacheOnVHD {
    $map = @{
        "c:\akse-cache\"              = @(
            "https://github.com/Azure/aks-engine/raw/master/scripts/collect-windows-logs.ps1",
+            "https://github.com/Azure/aks-engine/raw/master/scripts/containerd.wprp",
            "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/cni/win-bridge.exe",
            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/collectlogs.ps1",
            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/dumpVfpPolicies.ps1",
@ -159,19 +165,6 @@ function Install-ContainerD {
    $newPath = [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";$installDir"
    [Environment]::SetEnvironmentVariable("Path", $newPath, [EnvironmentVariableTarget]::Machine)
    $env:Path += ";$installDir"
-
-    Write-Log "Registering containerd as a service"
-    & containerd.exe --register-service
-    $svc = Get-Service -Name "containerd" -ErrorAction SilentlyContinue
-    if ($null -eq $svc) {
-        throw "containerd.exe did not get installed as a service correctly."
-    }
-
-    Write-Log "Starting containerd service"
-    $svc | Start-Service
-    if ($svc.Status -ne "Running") {
-        throw "containerd service is not running"
-    }
 }

 function Install-Docker {