Collect dumps and sequence.xml files on test hangs and crashes

This makes diagnosing test runner crashes and hangs on Azure Pipelines possible.
After such a failure, a testResults artifact is collected that includes the `Sequence_*.xml` and `testhost.*.dmp` files collected at the timeout or crash.
These files can then be downloaded from the Azure Pipeline artifacts for study.

azure-pipelines/Darwin.runsettings

@@ -0,0 +1 @@
+<RunSettings />

azure-pipelines/Linux.runsettings

@@ -0,0 +1 @@
+<RunSettings />

azure-pipelines/Windows_NT.runsettings

@@ -0,0 +1,14 @@
+<RunSettings>
+  <DataCollectionRunSettings>
+    <DataCollectors>
+      <DataCollector friendlyName="blame" enabled="True">
+        <Configuration>
+          <CollectDump DumpType="full" />
+          <CollectDumpOnTestSessionHang TestTimeout="30000" DumpType="full" />
+          <!-- This ResultsDirectory must exist, but it is totally ignored. -->
+          <ResultsDirectory>%BUILD_ARTIFACTSTAGINGDIRECTORY%</ResultsDirectory>
+        </Configuration>
+      </DataCollector>
+    </DataCollectors>
+  </DataCollectionRunSettings>
+</RunSettings>

azure-pipelines/artifacts/testResults.ps1

@@ -0,0 +1,12 @@
+if ($env:AGENT_TEMPDIRECTORY) {
+    # The DotNetCoreCLI uses an alternate location to publish these files
+    $guidRegex = '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$'
+    @{
+        $env:AGENT_TEMPDIRECTORY = (Get-ChildItem $env:AGENT_TEMPDIRECTORY -Directory |? { $_.Name -match $guidRegex } |% { Get-ChildItem "$($_.FullName)\testhost.*.dmp","$($_.FullName)\Sequence_*.xml" -Recurse });
+    }
+} else {
+    $srcRoot = Resolve-Path "$PSScriptRoot\..\..\src"
+    @{
+        $srcRoot = (Get-ChildItem "$srcRoot\TestResults" -Recurse -Directory | Get-ChildItem -Recurse -File);
+    }
+}

azure-pipelines/dotnet.yml

@@ -12,7 +12,7 @@ steps:
   displayName: dotnet test -f net472
   inputs:
     command: test
-    arguments: --no-build -c $(BuildConfiguration) -f net472 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true
+    arguments: --no-build -c $(BuildConfiguration) -f net472 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true --settings "$(Build.Repository.LocalPath)/azure-pipelines/$(Agent.OS).runsettings"
     testRunTitle: net472-$(Agent.JobName)
     workingDirectory: src
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
@@ -21,7 +21,7 @@ steps:
   displayName: dotnet test -f netcoreapp2.1
   inputs:
     command: test
-    arguments: --no-build -c $(BuildConfiguration) -f netcoreapp2.1 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true
+    arguments: --no-build -c $(BuildConfiguration) -f netcoreapp2.1 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true --settings "$(Build.Repository.LocalPath)/azure-pipelines/$(Agent.OS).runsettings"
     testRunTitle: netcoreapp2.1-$(Agent.JobName)
     workingDirectory: src
 
@@ -29,7 +29,7 @@ steps:
   displayName: dotnet test -f netcoreapp3.1
   inputs:
     command: test
-    arguments: --no-build -c $(BuildConfiguration) -f netcoreapp3.1 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true
+    arguments: --no-build -c $(BuildConfiguration) -f netcoreapp3.1 --filter "TestCategory!=FailsInCloudTest" -v n /p:CollectCoverage=true --settings "$(Build.Repository.LocalPath)/azure-pipelines/$(Agent.OS).runsettings"
     testRunTitle: netcoreapp3.1-$(Agent.JobName)
     workingDirectory: src
 

azure-pipelines/install-dependencies.yml

@@ -8,6 +8,13 @@ steps:
     dotnet --info
   displayName: Install prerequisites
 
+# The procdump tool and env var is required for dotnet test to collect hang/crash dumps of tests.
+- powershell: |
+    choco install procdump
+    Write-Host "##vso[task.setvariable variable=PROCDUMP_PATH;]$env:ProgramData\chocolatey\bin\"
+  displayName: Install procdump
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+
 - task: PowerShell@2
   inputs:
     filePath: azure-pipelines/variables/_pipelines.ps1