diff --git a/scripts/Run-Self-Hosted-Runner-Test.ps1 b/scripts/Run-Self-Hosted-Runner-Test.ps1 index 080d97cf0..e890cb86d 100644 --- a/scripts/Run-Self-Hosted-Runner-Test.ps1 +++ b/scripts/Run-Self-Hosted-Runner-Test.ps1 @@ -179,8 +179,16 @@ if (-not $WaitResult) { $DriveFreeSpaceGB = GetDriveFreeSpaceGB -DriveSpecification $Env:SystemDrive Write-Log "Current available disk space: $DriveFreeSpaceGB GB`n" + # $TestProcess refers to 'cmd.exe' which ends up running the real test application. + # (This is done so that we can capture the stdout and stderr outputs from the test application + # itself. I have not been able to get Powershell's 'built-in' redirection mechanisms for the + # 'Process' object to play well in our scenario). + # We therefore need pass the test application's id to procdump64.exe + $TestCommandId = (Get-Process -Name $TestCommand).Id + Write-Log "Test Command:$TestCommand, Id:$TestCommandId" + Write-Log "Creating User mode dump @ $UserModeDumpFilePath" - $ProcDumpArguments = "-r -ma $($TestProcess.Id) $UserModeDumpFilePath" + $ProcDumpArguments = "-r -ma $($TestCommandId) $UserModeDumpFilePath" Write-Log "Dump Command: $ProcDumpBinaryPath $ProcDumpArguments" $ProcDumpProcess = Start-Process -NoNewWindow ` -FilePath $ProcDumpBinaryPath ` diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1 index 70a9c6607..9d098a4d7 100644 --- a/scripts/config_test_vm.psm1 +++ b/scripts/config_test_vm.psm1 @@ -210,10 +210,69 @@ function Export-BuildArtifactsToVMs Remove-Item -Force $tempFileName } -# -# Import test logs from VM. -# +function ArchiveKernelModeDumpOnVM +{ + param ( + [Parameter(Mandatory = $True)] [System.Management.Automation.Runspaces.PSSession] $Session + ) + Invoke-Command -Session $Session -ScriptBlock { + + $KernelModeDumpFileSourcePath = "$Env:WinDir" + $KernelModeDumpFileDestinationPath = "$Env:SystemDrive\KernelDumps" + + # Create the compressed dump folder if doesn't exist. + if (!(Test-Path $KernelModeDumpFileDestinationPath)) { + Write-Output "Creating $KernelModeDumpFileDestinationPath directory." + New-Item -ItemType Directory -Path $KernelModeDumpFileDestinationPath | Out-Null + + # Make sure it was created + if (!(Test-Path $KernelModeDumpFileDestinationPath)) { + $ErrorMessage = ` + "*** ERROR *** Create compressed dump file directory failed: $KernelModeDumpFileDestinationPath`n" + Write-Output $ErrorMessage + Start-Sleep -seconds 3 + Throw $ErrorMessage + } + } + + if (Test-Path $KernelModeDumpFileSourcePath\*.dmp -PathType Leaf) { + Write-Output "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):" + $DumpFiles = get-childitem -Path $KernelModeDumpFileSourcePath\*.dmp + foreach ($DumpFile in $DumpFiles) { + Write-Output "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB" + } + Write-Output "`n" + + Write-Output ` + "Compressing kernel dump files: $KernelModeDumpFileSourcePath -> $KernelModeDumpFileDestinationPath" + Compress-Archive ` + -Path $KernelModeDumpFileSourcePath\*.dmp ` + -DestinationPath $KernelModeDumpFileDestinationPath\km_dumps.zip ` + -CompressionLevel Fastest ` + -Force + + if (Test-Path $KernelModeDumpFileDestinationPath\km_dumps.zip -PathType Leaf) { + $CompressedDumpFile = get-childitem -Path $KernelModeDumpFileDestinationPath\km_dumps.zip + Write-Output "Found compressed kernel mode dump file in $($KernelModeDumpFileDestinationPath):" + Write-Output ` + "`tName:$($CompressedDumpFile.Name), Size:$((($CompressedDumpFile.Length) / 1MB).ToString("F2")) MB" + } else { + $ErrorMessage = "*** ERROR *** kernel mode dump compressed file not found.`n`n" + Write-Output $ErrorMessage + Start-Sleep -seconds 3 + throw $ErrorMessage + } + } else { + Write-Output "`n" + Write-Output "No kernel mode dump(s) in $($KernelModeDumpFileSourcePath)." + } + } +} + +# +# Import test logs and dumps from VM. +# function Import-ResultsFromVM { param([Parameter(Mandatory=$True)] $VMList, @@ -236,21 +295,37 @@ function Import-ResultsFromVM } $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive} - # Copy kernel crash dumps if any. - Invoke-Command -Session $VMSession -ScriptBlock { - if (!(Test-Path "$Env:SystemDrive\KernelDumps")) { - New-Item -ItemType Directory -Path "$Env:SystemDrive\KernelDumps" - } + # Archive and copy kernel crash dumps, if any. + Write-Log "Processing kernel mode dump (if any) on VM $VMName" + ArchiveKernelModeDumpOnVM -Session $VMSession - if (Test-Path $Env:WinDir\*.dmp -PathType Leaf) { - tar czf $Env:SystemDrive\KernelDumps\km_dumps.tgz -C $Env:WinDir *.dmp - Remove-Item -Path $Env:WinDir\*.dmp - } + $LocalKernelArchiveLocation = ".\TestLogs\$VMName\KernelDumps" + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\KernelDumps" ` + -Destination $LocalKernelArchiveLocation ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log + + if (Test-Path $LocalKernelArchiveLocation\km_dumps.zip -PathType Leaf) { + $LocalFile = get-childitem -Path $LocalKernelArchiveLocation\km_dumps.zip + Write-Log "`n" + Write-Log "Local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $($VMName):" + Write-Log "`tName:$($LocalFile.Name), Size:$((($LocalFile.Length) / 1MB).ToString("F2")) MB" + } else { + Write-Log "`n" + Write-Log "No local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $VMName." } - Copy-Item -FromSession $VMSession "$VMSystemDrive\KernelDumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log # Copy user mode crash dumps if any. - Copy-Item -FromSession $VMSession "$VMSystemDrive\dumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\dumps" ` + -Destination ".\TestLogs\$VMName" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Copy logs from Test VM. if (!(Test-Path ".\TestLogs\$VMName\Logs")) { @@ -259,10 +334,22 @@ function Import-ResultsFromVM $VMTemp = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:TEMP} Write-Log ("Copy $LogFileName from $VMTemp on $VMName to $pwd\TestLogs") - Copy-Item -FromSession $VMSession "$VMTemp\$LogFileName" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMTemp\$LogFileName" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log Write-Log ("Copy CodeCoverage from eBPF on $VMName to $pwd\..\..") - Copy-Item -FromSession $VMSession "$VMSystemDrive\eBPF\ebpf_for_windows.xml" -Destination "$pwd\..\.." -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\ebpf_for_windows.xml" ` + -Destination "$pwd\..\.." ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Copy kernel mode traces, if enabled. if ($KmTracing) { @@ -273,7 +360,11 @@ function Import-ResultsFromVM [Parameter(Mandatory=$True)] [string] $LogFileName, [Parameter(Mandatory=$True)] [string] $EtlFile) $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory" - Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + Import-Module ` + $WorkingDirectory\common.psm1 ` + -ArgumentList ($LogFileName) ` + -Force ` + -WarningAction SilentlyContinue Write-Log "Query KM ETL tracing status before trace stop" $ProcInfo = Start-Process -FilePath "wpr.exe" ` @@ -304,12 +395,24 @@ function Import-ResultsFromVM # Copy ETL from Test VM. Write-Log ("Copy $WorkingDirectory\$EtlFile on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\$EtlFile" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\$EtlFile" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log } # Copy performance results from Test VM. Write-Log ("Copy performance results from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\*.csv" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\*.csv" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Compress and copy the performance profile if present. Invoke-Command -Session $VMSession -ScriptBlock { @@ -320,7 +423,13 @@ function Import-ResultsFromVM } } Write-Log ("Copy performance profile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log } # Move runner test logs to TestLogs folder. Write-Host ("Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs")