Timeout during fault injection (#2621)

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

.github/workflows/reusable-test.yml

@@ -192,7 +192,7 @@ jobs:
       shell: cmd
       run: |
           set EBPF_ENABLE_WER_REPORT=yes
-          OpenCppCoverage.exe -q --cover_children --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- powershell.exe .\Test-FaultInjection.ps1 ${{env.TEST_COMMAND}} 4
+          OpenCppCoverage.exe -q --cover_children --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- powershell.exe .\Test-FaultInjection.ps1 ${{env.DUMP_PATH}} ${{env.TEST_TIMEOUT}} ${{env.TEST_COMMAND}} 4
 
     - name: Run test with low resource simulation
       if: (inputs.code_coverage != true) && (inputs.fault_injection == true) && (steps.skip_check.outputs.should_skip != 'true')
@@ -201,7 +201,7 @@ jobs:
       working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
       run: |
           set EBPF_ENABLE_WER_REPORT=yes
-          powershell.exe .\Test-FaultInjection.ps1 ${{env.TEST_COMMAND}} 16
+          powershell.exe .\Test-FaultInjection.ps1 ${{env.DUMP_PATH}} ${{env.TEST_TIMEOUT}} ${{env.TEST_COMMAND}} 16
 
     - name: Run test with Code Coverage
       if: (inputs.code_coverage == true) && (inputs.vs_dev != true) && (inputs.fault_injection != true) && (steps.skip_check.outputs.should_skip != 'true')

scripts/Test-FaultInjection.ps1

@@ -9,7 +9,7 @@
 # Fifth it will check if all tests passed and exit.
 #
 
-param ($TestProgram, $StackDepth)
+param ($OutputFolder, $Timeout, $TestProgram, $StackDepth)
 
 # Gather list of all possible tests
 $tests = & $TestProgram "--list-tests" "--verbosity=quiet"
@@ -22,15 +22,6 @@ Set-Content -Path ($TestProgram + ".fault.log") ""
 
 $iteration = 0
 
-$timer = New-Object System.Timers.Timer
-# Set timer for 5 minutes
-$timer.Interval = 300000
-
-# When the watchdog timer fires, kill the test binary.
-Register-ObjectEvent -InputObject $timer -EventName Elapsed -Action {
-    Write-Error "Test binary exceeded time"
-    Get-Process -Name $TestProgram | Stop-Process -Force -ErrorAction SilentlyContinue
-}
 
 # Rerun failing tests until they pass
 while ($true) {
@@ -45,6 +36,7 @@ while ($true) {
 
     # If all the tests have passed, exit.
     if ($remaining_tests.Count -eq 0) {
+        Write-Host "Zero remaining tests"
         break
     }
 
@@ -55,18 +47,17 @@ while ($true) {
     write-host "Running iteration #" $iteration
     $remaining_tests | ForEach-Object { write-host "Running: $_" }
 
-    # Start the watchdog timer
-    $timer.Start()
-
     # Run the test binary with any remaining tests.
-    & $TestProgram "-d yes" "--verbosity=quiet" "-f remaining_tests.txt"
-
-    # Stop the watchdog timer
-    $timer.Stop()
-
-    if ($LASTEXITCODE -eq 0) {
-        write-host "All tests passed"
-        break
+    $process = Start-Process -NoNewWindow -FilePath $TestProgram -ArgumentList "-d yes", "--verbosity=quiet", "-f remaining_tests.txt" -PassThru
+    if (!$process.WaitForExit($Timeout * 1000)) {
+        $dumpFileName = "$($process.ProcessName)_$(Get-Date -Format 'yyyy-MM-dd_HH-mm-ss').dmp"
+        $dumpFilePath = Join-Path $OutputFolder $dumpFileName
+        Write-Output "Capturing dump of $($process.ProcessName) to $dumpFilePath"
+        Start-Process -NoNewWindow -Wait -FilePath procdump -ArgumentList "-accepteula -ma $($process.Id) $dumpFilePath"
+        if (!$process.HasExited) {
+            Write-Output "Killing $($process.ProcessName)"
+            $process.Kill()
+        }
     }
 }
 

tests/libs/common/passed_test_log.h

@@ -30,16 +30,16 @@ class _passed_test_log : public Catch::EventListenerBase
             GetModuleFileNameA(nullptr, process_name, MAX_PATH);
             std::string log_file = process_name;
             log_file += ".passed.log";
-            passed_tests.open(log_file, std::ios::app);
+            passed_tests = std::make_unique<std::ofstream>(log_file, std::ios::app);
         }
         if (testCaseStats.totals.assertions.failed == 0) {
-            passed_tests << testCaseStats.testInfo->name << std::endl;
-            passed_tests.flush();
+            *passed_tests << testCaseStats.testInfo->name << std::endl;
+            passed_tests->flush();
         }
     }
 
   private:
-    static std::ofstream passed_tests;
+    static std::unique_ptr<std::ofstream> passed_tests;
 };
 
-std::ofstream _passed_test_log::passed_tests;
+std::unique_ptr<std::ofstream> _passed_test_log::passed_tests;