Rework dump handling, add trace messages, fix process id for user mode dump (#3355)

Co-authored-by: Dhiren Vispute <Dhiren.Vispute@microsoft.com>
2024-03-21 10:40:56 -07:00 · 2024-03-21 10:40:56 -07:00 · b494058832
--- a/scripts/Run-Self-Hosted-Runner-Test.ps1
+++ b/scripts/Run-Self-Hosted-Runner-Test.ps1
@ -179,8 +179,16 @@ if (-not $WaitResult) {
    $DriveFreeSpaceGB = GetDriveFreeSpaceGB -DriveSpecification $Env:SystemDrive
    Write-Log "Current available disk space: $DriveFreeSpaceGB GB`n"

+    # $TestProcess refers to 'cmd.exe' which ends up running the real test application.
+    # (This is done so that we can capture the stdout and stderr outputs from the test application
+    # itself. I have not been able to get Powershell's 'built-in' redirection mechanisms for the
+    # 'Process' object to play well in our scenario).
+    # We therefore need pass the test application's id to procdump64.exe
+    $TestCommandId = (Get-Process -Name $TestCommand).Id
+    Write-Log "Test Command:$TestCommand, Id:$TestCommandId"
+
    Write-Log "Creating User mode dump @ $UserModeDumpFilePath"
-    $ProcDumpArguments = "-r -ma $($TestProcess.Id) $UserModeDumpFilePath"
+    $ProcDumpArguments = "-r -ma $($TestCommandId) $UserModeDumpFilePath"
    Write-Log "Dump Command: $ProcDumpBinaryPath $ProcDumpArguments"
    $ProcDumpProcess = Start-Process -NoNewWindow `
        -FilePath $ProcDumpBinaryPath `
--- a/scripts/config_test_vm.psm1
+++ b/scripts/config_test_vm.psm1
@ -210,10 +210,69 @@ function Export-BuildArtifactsToVMs
    Remove-Item -Force $tempFileName
 }

-#
-# Import test logs from VM.
-#
+function ArchiveKernelModeDumpOnVM
+{
+    param (
+        [Parameter(Mandatory = $True)] [System.Management.Automation.Runspaces.PSSession] $Session
+    )

+    Invoke-Command -Session $Session -ScriptBlock {
+
+        $KernelModeDumpFileSourcePath = "$Env:WinDir"
+        $KernelModeDumpFileDestinationPath = "$Env:SystemDrive\KernelDumps"
+
+        # Create the compressed dump folder if doesn't exist.
+        if (!(Test-Path $KernelModeDumpFileDestinationPath)) {
+            Write-Output "Creating $KernelModeDumpFileDestinationPath directory."
+            New-Item -ItemType Directory -Path $KernelModeDumpFileDestinationPath | Out-Null
+
+            # Make sure it was created
+            if (!(Test-Path $KernelModeDumpFileDestinationPath)) {
+                $ErrorMessage = `
+                    "*** ERROR *** Create compressed dump file directory failed: $KernelModeDumpFileDestinationPath`n"
+                Write-Output $ErrorMessage
+                Start-Sleep -seconds 3
+                Throw $ErrorMessage
+            }
+        }
+
+        if (Test-Path $KernelModeDumpFileSourcePath\*.dmp -PathType Leaf) {
+            Write-Output "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):"
+            $DumpFiles = get-childitem -Path $KernelModeDumpFileSourcePath\*.dmp
+            foreach ($DumpFile in $DumpFiles) {
+                Write-Output "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB"
+            }
+            Write-Output "`n"
+
+            Write-Output `
+                "Compressing kernel dump files: $KernelModeDumpFileSourcePath -> $KernelModeDumpFileDestinationPath"
+            Compress-Archive `
+                -Path $KernelModeDumpFileSourcePath\*.dmp `
+                -DestinationPath $KernelModeDumpFileDestinationPath\km_dumps.zip `
+                -CompressionLevel Fastest `
+                -Force
+
+            if (Test-Path $KernelModeDumpFileDestinationPath\km_dumps.zip -PathType Leaf) {
+                $CompressedDumpFile = get-childitem -Path $KernelModeDumpFileDestinationPath\km_dumps.zip
+                Write-Output "Found compressed kernel mode dump file in $($KernelModeDumpFileDestinationPath):"
+                Write-Output `
+                    "`tName:$($CompressedDumpFile.Name), Size:$((($CompressedDumpFile.Length) / 1MB).ToString("F2")) MB"
+            } else {
+                $ErrorMessage = "*** ERROR *** kernel mode dump compressed file not found.`n`n"
+                Write-Output $ErrorMessage
+                Start-Sleep -seconds 3
+                throw $ErrorMessage
+            }
+        } else {
+            Write-Output "`n"
+            Write-Output "No kernel mode dump(s) in $($KernelModeDumpFileSourcePath)."
+        }
+    }
+}
+
+#
+# Import test logs and dumps from VM.
+#
 function Import-ResultsFromVM
 {
    param([Parameter(Mandatory=$True)] $VMList,
@ -236,21 +295,37 @@ function Import-ResultsFromVM
        }
        $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive}

-        # Copy kernel crash dumps if any.
-        Invoke-Command -Session $VMSession -ScriptBlock {
-            if (!(Test-Path "$Env:SystemDrive\KernelDumps")) {
-                New-Item -ItemType Directory -Path "$Env:SystemDrive\KernelDumps"
-            }
+        # Archive and copy kernel crash dumps, if any.
+        Write-Log "Processing kernel mode dump (if any) on VM $VMName"
+        ArchiveKernelModeDumpOnVM -Session $VMSession

-            if (Test-Path $Env:WinDir\*.dmp -PathType Leaf) {
-                tar czf $Env:SystemDrive\KernelDumps\km_dumps.tgz -C $Env:WinDir *.dmp
-                Remove-Item -Path $Env:WinDir\*.dmp
-            }
+        $LocalKernelArchiveLocation = ".\TestLogs\$VMName\KernelDumps"
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\KernelDumps" `
+            -Destination $LocalKernelArchiveLocation `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
+
+        if (Test-Path $LocalKernelArchiveLocation\km_dumps.zip -PathType Leaf) {
+            $LocalFile = get-childitem -Path $LocalKernelArchiveLocation\km_dumps.zip
+            Write-Log "`n"
+            Write-Log "Local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $($VMName):"
+            Write-Log "`tName:$($LocalFile.Name), Size:$((($LocalFile.Length) / 1MB).ToString("F2")) MB"
+        } else {
+            Write-Log "`n"
+            Write-Log "No local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $VMName."
        }
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\KernelDumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log

        # Copy user mode crash dumps if any.
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\dumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\dumps" `
+            -Destination ".\TestLogs\$VMName" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log

        # Copy logs from Test VM.
        if (!(Test-Path ".\TestLogs\$VMName\Logs")) {
@ -259,10 +334,22 @@ function Import-ResultsFromVM

        $VMTemp = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:TEMP}
        Write-Log ("Copy $LogFileName from $VMTemp on $VMName to $pwd\TestLogs")
-        Copy-Item -FromSession $VMSession "$VMTemp\$LogFileName" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMTemp\$LogFileName" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log

        Write-Log ("Copy CodeCoverage from eBPF on $VMName to $pwd\..\..")
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\eBPF\ebpf_for_windows.xml" -Destination "$pwd\..\.." -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\ebpf_for_windows.xml" `
+            -Destination "$pwd\..\.." `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log

        # Copy kernel mode traces, if enabled.
        if ($KmTracing) {
@ -273,7 +360,11 @@ function Import-ResultsFromVM
                      [Parameter(Mandatory=$True)] [string] $LogFileName,
                      [Parameter(Mandatory=$True)] [string] $EtlFile)
                $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory"
-                Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
+                Import-Module `
+                    $WorkingDirectory\common.psm1 `
+                    -ArgumentList ($LogFileName) `
+                    -Force `
+                    -WarningAction SilentlyContinue

                Write-Log "Query KM ETL tracing status before trace stop"
                $ProcInfo = Start-Process -FilePath "wpr.exe" `
@ -304,12 +395,24 @@ function Import-ResultsFromVM

            # Copy ETL from Test VM.
            Write-Log ("Copy $WorkingDirectory\$EtlFile on $VMName to $pwd\TestLogs\$VMName\Logs")
-            Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\$EtlFile" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+            Copy-Item `
+                -FromSession $VMSession `
+                -Path "$VMSystemDrive\eBPF\$EtlFile" `
+                -Destination ".\TestLogs\$VMName\Logs" `
+                -Recurse `
+                -Force `
+                -ErrorAction Ignore 2>&1 | Write-Log
        }

        # Copy performance results from Test VM.
        Write-Log ("Copy performance results from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
-        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\*.csv" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\*.csv" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log

        # Compress and copy the performance profile if present.
        Invoke-Command -Session $VMSession -ScriptBlock {
@ -320,7 +423,13 @@ function Import-ResultsFromVM
            }
        }
        Write-Log ("Copy performance profile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
-        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
    }
    # Move runner test logs to TestLogs folder.
    Write-Host ("Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs")