Add step to gather dumps for any process launched from working folder. (#2612)

* Add "Run-Test.ps1" that executes a test with a
timeout and captures a dump if it times out.

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

* PR feedback

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

* Increasing test timeout to 60 minutes and overall timeout to 90 minutes

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

* PR feedback

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

---------

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

.github/workflows/reusable-test.yml

@@ -63,7 +63,8 @@ permissions:
 
 jobs:
   run_test:
-    timeout-minutes: 60
+    # Due to app-verifier, unit tests take a lot longer to execute. Increase the timeout to 90 minutes.
+    timeout-minutes: 90
 
     strategy:
       matrix:
@@ -79,6 +80,8 @@ jobs:
       PRE_COMMAND: ${{inputs.pre_test}}
       POST_COMMAND: ${{inputs.post_test}}
       EBPF_MEMORY_LEAK_DETECTION: ${{inputs.leak_detection}}
+      DUMP_PATH: c:/dumps/x64/${{matrix.configurations}}
+      TEST_TIMEOUT: 3600 # 1 hour timeout for tests.
 
     steps:
     - id: skip_check
@@ -101,6 +104,13 @@ jobs:
       with:
         ref: ${{ github.event.workflow_run.head_branch }}
 
+    - name: Install ProcDump
+      id: install_procdump
+      if: (inputs.gather_dumps == true) && (steps.skip_check.outputs.should_skip != 'true')
+      run: |
+        choco install -y procdump
+        where procdump.exe
+
     - name: Set up OpenCppCoverage and add to PATH
       id: set_up_opencppcoverage
       if: (inputs.code_coverage == true) && (inputs.environment != 'ebpf_cicd_tests') && (steps.skip_check.outputs.should_skip != 'true')
@@ -112,10 +122,11 @@ jobs:
       id: configure_windows_error_reporting
       if: (inputs.gather_dumps == true) && (steps.skip_check.outputs.should_skip != 'true')
       run: |
-        mkdir c:/dumps/${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
+        mkdir ${{env.DUMP_PATH}}
         New-Item -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -ErrorAction SilentlyContinue
         New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpType" -Value 2 -PropertyType DWord -ErrorAction SilentlyContinue
-        New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpFolder" -Value "c:\dumps\${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}}" -PropertyType ExpandString -ErrorAction SilentlyContinue
+        $dump_path = "${{env.DUMP_PATH}}".Replace("/", "\")
+        New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpFolder" -Value "$dump_path" -PropertyType ExpandString -ErrorAction SilentlyContinue
 
     - name: Remove existing artifacts
       if: (inputs.environment == 'ebpf_cicd_tests') && (steps.skip_check.outputs.should_skip != 'true')
@@ -198,7 +209,7 @@ jobs:
       shell: cmd
       run: |
           set EBPF_ENABLE_WER_REPORT=yes
-          OpenCppCoverage.exe -q --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}}\${{env.TEST_COMMAND}}
+          OpenCppCoverage.exe -q --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- powershell .\Run-Test.ps1 ${{env.DUMP_PATH}} ${{env.TEST_TIMEOUT}} ${{env.TEST_COMMAND}}
 
     - name: Run test without Code Coverage
       if: (inputs.code_coverage == false) && (steps.skip_check.outputs.should_skip != 'true')
@@ -305,7 +316,7 @@ jobs:
       uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b
       id: check_dumps
       with:
-        files: c:/dumps/${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}/*.dmp
+        files: ${{env.DUMP_PATH}}/*.dmp
 
     - name: Upload any crash dumps
       # Upload crash dumps even if the workflow failed.
@@ -314,7 +325,7 @@ jobs:
       id: upload_crash_dumps
       with:
         name: Crash-Dumps-${{env.NAME}}-${{env.BUILD_PLATFORM}}-${{env.BUILD_CONFIGURATION}}
-        path: c:/dumps/${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
+        path: ${{env.DUMP_PATH}}
         retention-days: 5
 
     - name: Check for TestLogs

scripts/Run-Test.ps1

@@ -0,0 +1,33 @@
+# Copyright (c) Microsoft Corporation
+# SPDX-License-Identifier: MIT
+
+# This script executes the provided test command, waits for <timeout in seconds>
+# and then captures a dump of the test process if it is still running. The dump
+# is captured using the procdump tool from Sysinternals. The dump is saved to
+# the <output folder> with the name of the test executable and the current date
+# and time.
+
+if ($args.Count -eq 0) {
+    Write-Output "Usage: Run-Test.ps1 <output folder> <timeout in seconds> <test command> <test arguments>"
+    exit 1
+}
+
+$OutputFolder = $args[0]
+$args = $args[1..($args.Length - 1)]
+$Timeout = [int]$args[0]
+$args = $args[1..($args.Length - 1)]
+
+$process = Start-Process -PassThru -NoNewWindow -FilePath $args[0] -ArgumentList $args[1..($args.Length - 1)]
+
+if (!$process.WaitForExit($Timeout * 1000)) {
+    $dumpFileName = "$($process.ProcessName)_$(Get-Date -Format 'yyyy-MM-dd_HH-mm-ss').dmp"
+    $dumpFilePath = Join-Path $OutputFolder $dumpFileName
+    Write-Output "Capturing dump of $($process.ProcessName) to $dumpFilePath"
+    Start-Process -NoNewWindow -Wait -FilePath procdump -ArgumentList "-accepteula -ma $($process.Id) $dumpFilePath"
+    if (!$process.HasExited) {
+        Write-Output "Killing $($process.ProcessName)"
+        $process.Kill()
+    }
+}
+
+exit $process.ExitCode

scripts/setup_build/setup_build.vcxproj

@@ -149,6 +149,12 @@ popd
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='NativeOnlyRelease|x64'">true</DeploymentContent>
     </CopyFileToFolders>
+    <CopyFileToFolders Include="..\Run-Test.ps1">
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='NativeOnlyDebug|x64'">true</DeploymentContent>
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='NativeOnlyRelease|x64'">true</DeploymentContent>
+    </CopyFileToFolders>
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <VCProjectVersion>16.0</VCProjectVersion>

scripts/setup_build/setup_build.vcxproj.filters

@@ -68,6 +68,7 @@
       <Filter>Source Files</Filter>
     </CopyFileToFolders>
     <CopyFileToFolders Include="..\Test-FaultInjection.ps1" />
+    <CopyFileToFolders Include="..\Run-Test.ps1" />
   </ItemGroup>
   <ItemGroup>
     <None Include="..\pre-commit">