зеркало из https://github.com/dotnet/razor.git
[Infrastructure] Install procdump and capture dumps on build hangs (#1146)
* Captures process dumps when the build hangs. * Captures process dumps when a child process hangs inside a test.
This commit is contained in:
Javier Calvarro Nelson
GitHub
Родитель
56727ea267
Коммит
a9b7e24791
|
|
@ -164,6 +164,11 @@ stages:
|
|||
inputs:
|
||||
command: custom
|
||||
arguments: 'locals all -clear'
|
||||
|
||||
- powershell: ./eng/scripts/InstallProcDump.ps1
|
||||
displayName: Install ProcDump
|
||||
- powershell: ./eng/scripts/StartDumpCollectionForHangingBuilds.ps1 $(ProcDumpPath)procdump.exe artifacts/log/$(_BuildConfig) (Get-Date).AddMinutes(25) dotnet, msbuild
|
||||
displayName: Start background dump collection
|
||||
- script: eng\common\cibuild.cmd
|
||||
-configuration $(_BuildConfig)
|
||||
-prepareMachine
|
||||
|
|
@ -172,6 +177,11 @@ stages:
|
|||
name: Build
|
||||
displayName: Build
|
||||
condition: succeeded()
|
||||
- powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/log/$(_BuildConfig)
|
||||
displayName: Finish background dump collection
|
||||
continueOnError: true
|
||||
condition: always()
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: Upload TestResults
|
||||
condition: always()
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ jobs:
|
|||
- ${{ if ne(variable.name, '') }}:
|
||||
- name: ${{ variable.name }}
|
||||
value: ${{ variable.value }}
|
||||
|
||||
|
||||
# handle variable groups
|
||||
- ${{ if ne(variable.group, '') }}:
|
||||
- group: ${{ variable.group }}
|
||||
|
|
@ -160,7 +160,7 @@ jobs:
|
|||
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
|
||||
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
|
||||
- task: MicroBuildCleanup@1
|
||||
displayName: Execute Microbuild cleanup tasks
|
||||
displayName: Execute Microbuild cleanup tasks
|
||||
condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
|
||||
continueOnError: ${{ parameters.continueOnError }}
|
||||
env:
|
||||
|
|
@ -188,11 +188,11 @@ jobs:
|
|||
displayName: Publish Test Results
|
||||
inputs:
|
||||
testResultsFormat: 'xUnit'
|
||||
testResultsFiles: '*.xml'
|
||||
testResultsFiles: '*.xml'
|
||||
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
|
||||
continueOnError: true
|
||||
condition: always()
|
||||
|
||||
|
||||
- ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
|
||||
- task: CopyFiles@2
|
||||
displayName: Gather Asset Manifests
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateNotNullOrEmpty()]
|
||||
[string]
|
||||
$ProcDumpOutputPath
|
||||
)
|
||||
|
||||
Write-Output "Finishing dump collection for hanging builds.";
|
||||
|
||||
$repoRoot = Resolve-Path "$PSScriptRoot\..\..";
|
||||
$ProcDumpOutputPath = Join-Path $repoRoot $ProcDumpOutputPath;
|
||||
|
||||
$sentinelFile = Join-Path $ProcDumpOutputPath "dump-sentinel.txt";
|
||||
if ((-not (Test-Path $sentinelFile))) {
|
||||
Write-Output "No sentinel file available in '$sentinelFile'. " +
|
||||
"StartDumpCollectionForHangingBuilds.ps1 has not been executed, is not correctly configured or failed before creating the sentinel file.";
|
||||
return;
|
||||
}
|
||||
|
||||
Get-Process "procdump" -ErrorAction SilentlyContinue | ForEach-Object { Write-Output "ProcDump with PID $($_.Id) is still running."; };
|
||||
|
||||
$capturedDumps = Get-ChildItem $ProcDumpOutputPath -Filter *.dmp;
|
||||
$capturedDumps | ForEach-Object { Write-Output "Found captured dump $_"; };
|
||||
|
||||
$JobName = (Get-Content $sentinelFile);
|
||||
|
||||
if ($JobName.Count -ne 1) {
|
||||
if ($JobName.Count -eq 0) {
|
||||
Write-Warning "No job name found. This is likely an error.";
|
||||
return;
|
||||
}
|
||||
else {
|
||||
Write-Output "Multiple job names found '$JobName'.";
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
$dumpCollectionJob = Get-Job -Name $JobName -ErrorAction SilentlyContinue;
|
||||
$registeredJob = Get-ScheduledJob -Name $JobName -ErrorAction SilentlyContinue;
|
||||
|
||||
if ($null -eq $dumpCollectionJob) {
|
||||
Write-Output "No job found for '$JobName'. It either didn't run or there is an issue with the job definition.";
|
||||
|
||||
if ($null -eq $registeredJob) {
|
||||
Write-Warning "Couldn't find a scheduled job '$JobName'.";
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
Write-Output "Listing existing jobs";
|
||||
Get-Job -Name CaptureDumps*
|
||||
|
||||
Write-Output "Listing existing scheduled jobs";
|
||||
Get-ScheduledJob -Name CaptureDumps*
|
||||
|
||||
Write-Output "Displaying job output";
|
||||
Receive-Job $dumpCollectionJob;
|
||||
|
||||
Write-Output "Waiting for current job to finish";
|
||||
Get-Job -ErrorAction SilentlyContinue | Wait-Job;
|
||||
|
||||
try {
|
||||
Write-Output "Removing collection job";
|
||||
Remove-Job $dumpCollectionJob;
|
||||
}
|
||||
catch {
|
||||
Write-Output "Failed to remove collection job";
|
||||
}
|
||||
|
||||
try {
|
||||
Write-Output "Unregistering scheduled job";
|
||||
Unregister-ScheduledJob $registeredJob;
|
||||
}
|
||||
catch {
|
||||
Write-Output "Failed to unregister $JobName";
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Installs ProcDump into a folder in this repo.
|
||||
.DESCRIPTION
|
||||
This script downloads and extracts the ProcDump.
|
||||
.PARAMETER Force
|
||||
Overwrite the existing installation
|
||||
#>
|
||||
param(
|
||||
[switch]$Force
|
||||
)
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
|
||||
|
||||
Set-StrictMode -Version 1
|
||||
|
||||
$repoRoot = Resolve-Path "$PSScriptRoot\..\.."
|
||||
$installDir = "$repoRoot\.tools\ProcDump\"
|
||||
$tempDir = "$repoRoot\obj"
|
||||
|
||||
if (Test-Path $installDir) {
|
||||
if ($Force) {
|
||||
Remove-Item -Force -Recurse $installDir
|
||||
}
|
||||
else {
|
||||
Write-Host "ProcDump already installed to $installDir. Exiting without action. Call this script again with -Force to overwrite."
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
|
||||
Remove-Item -Force -Recurse $tempDir -ErrorAction Ignore | out-null
|
||||
mkdir $tempDir -ea Ignore | out-null
|
||||
mkdir $installDir -ea Ignore | out-null
|
||||
Write-Host "Starting ProcDump download"
|
||||
Invoke-WebRequest -UseBasicParsing -Uri "https://download.sysinternals.com/files/Procdump.zip" -Out "$tempDir/ProcDump.zip"
|
||||
Write-Host "Done downloading ProcDump"
|
||||
Expand-Archive "$tempDir/ProcDump.zip" -d "$tempDir/ProcDump/"
|
||||
Write-Host "Expanded ProcDump to $tempDir"
|
||||
Write-Host "Installing ProcDump to $installDir"
|
||||
Move-Item "$tempDir/ProcDump/*" $installDir
|
||||
Write-Host "Done installing ProcDump to $installDir"
|
||||
|
||||
if ($env:TF_BUILD) {
|
||||
Write-Host "##vso[task.setvariable variable=ProcDumpPath]$installDir"
|
||||
Write-Host "##vso[task.prependpath]$installDir"
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateNotNullOrEmpty()]
|
||||
[string]
|
||||
$ProcDumpPath,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateNotNullOrEmpty()]
|
||||
[string]
|
||||
$ProcDumpOutputPath,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[datetime]
|
||||
$WakeTime,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateNotNullOrEmpty()]
|
||||
[string []]
|
||||
$CandidateProcessNames
|
||||
)
|
||||
|
||||
Write-Output "Setting up a scheduled job to capture process dumps.";
|
||||
|
||||
if ((-not (Test-Path $ProcDumpPath))) {
|
||||
Write-Warning "Can't find ProcDump at '$ProcDumpPath'.";
|
||||
}
|
||||
else {
|
||||
Write-Output "Using ProcDump from '$ProcDumpPath'.";
|
||||
}
|
||||
|
||||
try {
|
||||
$previousJobs = Get-Job -Name CaptureDumps* -ErrorAction SilentlyContinue;
|
||||
$previousScheduledJobs = Get-ScheduledJob CaptureDumps* -ErrorAction SilentlyContinue;
|
||||
|
||||
if ($previousJobs.Count -ne 0) {
|
||||
Write-Output "Found existing dump jobs.";
|
||||
}
|
||||
|
||||
if ($previousScheduledJobs.Count -ne 0) {
|
||||
Write-Output "Found existing dump jobs.";
|
||||
}
|
||||
|
||||
$previousJobs | Stop-Job -PassThru | Remove-Job;
|
||||
$previousScheduledJobs | Unregister-ScheduledJob;
|
||||
}
|
||||
catch {
|
||||
Write-Output "There was an error cleaning up previous jobs.";
|
||||
Write-Output $_.Exception.Message;
|
||||
}
|
||||
|
||||
$repoRoot = Resolve-Path "$PSScriptRoot\..\..";
|
||||
$ProcDumpOutputPath = Join-Path $repoRoot $ProcDumpOutputPath;
|
||||
|
||||
Write-Output "Dumps will be placed at '$ProcDumpOutputPath'.";
|
||||
Write-Output "Watching processes $($CandidateProcessNames -join ', ')";
|
||||
|
||||
# This script registers as a scheduled job. This scheduled job executes after $WakeTime.
|
||||
# When the scheduled job executes, it runs procdump on all alive processes whose name matches $CandidateProcessNames.
|
||||
# The dumps are placed in $ProcDumpOutputPath
|
||||
# If the build completes sucessfully in less than $WakeTime, a final step unregisters the job.
|
||||
|
||||
# Create a unique identifier for the job name
|
||||
$JobName = "CaptureDumps" + (New-Guid).ToString("N");
|
||||
|
||||
# Ensure that the dumps output path exists.
|
||||
if ((-not (Test-Path $ProcDumpOutputPath))) {
|
||||
New-Item -ItemType Directory $ProcDumpOutputPath | Out-Null;
|
||||
}
|
||||
|
||||
# We write a sentinel file that we use at the end of the build to
|
||||
# find the job we started and to determine the results from the sheduled
|
||||
# job (Whether it ran or not and to display the outputs form the job)
|
||||
$sentinelFile = Join-Path $ProcDumpOutputPath "dump-sentinel.txt";
|
||||
Out-File -FilePath $sentinelFile -InputObject $JobName | Out-Null;
|
||||
|
||||
[scriptblock] $ScriptCode = {
|
||||
param(
|
||||
$ProcDumpPath,
|
||||
$ProcDumpOutputPath,
|
||||
$CandidateProcessNames)
|
||||
|
||||
Write-Output "Waking up to capture process dumps. Determining hanging processes.";
|
||||
|
||||
[System.Diagnostics.Process []]$AliveProcesses = @();
|
||||
foreach ($candidate in $CandidateProcessNames) {
|
||||
try {
|
||||
$candidateProcesses = Get-Process $candidate;
|
||||
$candidateProcesses | ForEach-Object { Write-Output "Found candidate process $candidate with PID '$($_.Id)'." };
|
||||
$AliveProcesses += $candidateProcesses;
|
||||
}
|
||||
catch {
|
||||
Write-Output "No process found for $candidate";
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output "Starting process dump capture.";
|
||||
|
||||
$dumpFullPath = [System.IO.Path]::Combine($ProcDumpOutputPath, "hung_PROCESSNAME_PID_YYMMDD_HHMMSS.dmp");
|
||||
|
||||
Write-Output "Capturing output for $($AliveProcesses.Length) processes.";
|
||||
|
||||
foreach ($process in $AliveProcesses) {
|
||||
|
||||
$procDumpArgs = @("-accepteula", "-ma", $process.Id, $dumpFullPath);
|
||||
try {
|
||||
Write-Output "Capturing dump for dump for '$($process.Name)' with PID '$($process.Id)'.";
|
||||
Start-Process -FilePath $ProcDumpPath -ArgumentList $procDumpArgs -NoNewWindow -Wait;
|
||||
}
|
||||
catch {
|
||||
Write-Output "There was an error capturing a process dump for '$($process.Name)' with PID '$($process.Id)'."
|
||||
Write-Warning $_.Exception.Message;
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output "Done capturing process dumps.";
|
||||
}
|
||||
|
||||
$ScriptTrigger = New-JobTrigger -Once -At $WakeTime;
|
||||
|
||||
try {
|
||||
Register-ScheduledJob -Name $JobName -ScriptBlock $ScriptCode -Trigger $ScriptTrigger -ArgumentList $ProcDumpPath, $ProcDumpOutputPath, $CandidateProcessNames;
|
||||
}
|
||||
catch {
|
||||
Write-Warning "Failed to register scheduled job '$JobName'. Dumps will not be captured for build hangs.";
|
||||
Write-Warning $_.Exception.Message;
|
||||
}
|
||||
|
|
@ -83,22 +83,7 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests
|
|||
process.BeginOutputReadLine();
|
||||
process.BeginErrorReadLine();
|
||||
|
||||
var timeoutTask = Task.Delay(timeout.Value).ContinueWith<ProcessResult>((t) =>
|
||||
{
|
||||
// Don't timeout during debug sessions
|
||||
while (Debugger.IsAttached)
|
||||
{
|
||||
Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
|
||||
if (!process.HasExited)
|
||||
{
|
||||
// This is a timeout.
|
||||
process.Kill();
|
||||
}
|
||||
|
||||
throw new TimeoutException($"command '${process.StartInfo.FileName} {process.StartInfo.Arguments}' timed out after {timeout}. Output: {output.ToString()}");
|
||||
});
|
||||
var timeoutTask = GetTimeoutForProcess(process, timeout);
|
||||
|
||||
var waitTask = Task.Run(() =>
|
||||
{
|
||||
|
|
@ -142,6 +127,87 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests
|
|||
output.AppendLine(e.Data);
|
||||
}
|
||||
}
|
||||
|
||||
async Task<ProcessResult> GetTimeoutForProcess(Process process, TimeSpan? timeout)
|
||||
{
|
||||
await Task.Delay(timeout.Value);
|
||||
|
||||
// Don't timeout during debug sessions
|
||||
while (Debugger.IsAttached)
|
||||
{
|
||||
Thread.Sleep(TimeSpan.FromSeconds(1));
|
||||
}
|
||||
if (!process.HasExited)
|
||||
{
|
||||
var procDumpProcess = await CaptureDump(process);
|
||||
if (procDumpProcess != null && procDumpProcess.HasExited)
|
||||
{
|
||||
Console.WriteLine("ProcDump failed to run.");
|
||||
procDumpProcess.Kill();
|
||||
}
|
||||
|
||||
// This is a timeout.
|
||||
process.Kill();
|
||||
}
|
||||
|
||||
throw new TimeoutException($"command '${process.StartInfo.FileName} {process.StartInfo.Arguments}' timed out after {timeout}. Output: {output.ToString()}");
|
||||
}
|
||||
|
||||
async Task<Process> CaptureDump(Process process)
|
||||
{
|
||||
var metadataAttributes = Assembly.GetExecutingAssembly()
|
||||
.GetCustomAttributes<AssemblyMetadataAttribute>();
|
||||
|
||||
var procDumpPath = metadataAttributes
|
||||
.SingleOrDefault(ama => ama.Key == "ProcDumpPath")?.Value;
|
||||
|
||||
if (string.IsNullOrEmpty(procDumpPath))
|
||||
{
|
||||
Console.WriteLine("ProcDumpPath not defined.");
|
||||
return null;
|
||||
}
|
||||
var procDumpExePath = Path.Combine(procDumpPath, "procdump.exe");
|
||||
if (!File.Exists(procDumpExePath))
|
||||
{
|
||||
Console.WriteLine($"Can't find procdump.exe in '{procDumpPath}'.");
|
||||
return null;
|
||||
}
|
||||
|
||||
var dumpDirectory = metadataAttributes
|
||||
.SingleOrDefault(ama => ama.Key == "ArtifactsLogDir")?.Value;
|
||||
|
||||
if (string.IsNullOrEmpty(dumpDirectory))
|
||||
{
|
||||
Console.WriteLine("ArtifactsLogDir not defined.");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!Directory.Exists(dumpDirectory))
|
||||
{
|
||||
Console.WriteLine($"'{dumpDirectory}' does not exist.");
|
||||
return null;
|
||||
}
|
||||
|
||||
var procDumpPattern = Path.Combine(dumpDirectory, "HangingProcess_PROCESSNAME_PID_YYMMDD_HHMMSS.dmp");
|
||||
var procDumpStartInfo = new ProcessStartInfo(
|
||||
procDumpExePath,
|
||||
$"-accepteula -ma {process.Id} {procDumpPattern}")
|
||||
{
|
||||
CreateNoWindow = true,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true
|
||||
};
|
||||
|
||||
var procDumpProcess = Process.Start(procDumpStartInfo);
|
||||
var tcs = new TaskCompletionSource<object>();
|
||||
|
||||
procDumpProcess.Exited += (s, a) => tcs.TrySetResult(null);
|
||||
procDumpProcess.EnableRaisingEvents = true;
|
||||
|
||||
await Task.WhenAny(tcs.Task, Task.Delay(timeout ?? TimeSpan.FromSeconds(30)));
|
||||
return procDumpProcess;
|
||||
}
|
||||
}
|
||||
|
||||
internal class ProcessResult
|
||||
|
|
|
|||
|
|
@ -43,6 +43,16 @@
|
|||
<_Parameter2>$(MSBuildThisFileDirectory)..\testapps\TestPackageRestoreSource</_Parameter2>
|
||||
</AssemblyAttribute>
|
||||
|
||||
<AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute">
|
||||
<_Parameter1>ArtifactsLogDir</_Parameter1>
|
||||
<_Parameter2>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'log', '$(_BuildConfiguration)'))</_Parameter2>
|
||||
</AssemblyAttribute>
|
||||
|
||||
<AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute">
|
||||
<_Parameter1>ProcDumpToolPath</_Parameter1>
|
||||
<_Parameter2>$(ProcDumpPath)</_Parameter2>
|
||||
</AssemblyAttribute>
|
||||
|
||||
</ItemGroup>
|
||||
|
||||
<!-- The test projects rely on these binaries being available -->
|
||||
|
|
@ -112,4 +122,8 @@ namespace Microsoft.AspNetCore.Razor.Design.IntegrationTests
|
|||
<MSBuild Projects="..\testapps\RestoreTestProjects\RestoreTestProjects.csproj" Targets="Restore" Properties="MicrosoftNetCompilersToolsetPackageVersion=$(MicrosoftNetCompilersToolsetPackageVersion)" />
|
||||
</Target>
|
||||
|
||||
<Target Name="EnsureLogFolder" AfterTargets="Build">
|
||||
<MakeDir Directories="$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'log', '$(_BuildConfiguration)'))" />
|
||||
</Target>
|
||||
|
||||
</Project>
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче