Emulate driver verifier systematic low memory mode (#1512)

* Emulate driver verifier systematic low memory mode

Signed-off-by: Alan Jowett <alan.jowett@microsoft.com>

.github/workflows/cicd.yml

@@ -291,3 +291,17 @@ jobs:
     with:
       build_artifact: Build-x64-CodeQl
       build_codeql: true
+
+  # Run the low memory simulator in GitHub.
+  low_memory:
+    needs: regular
+    if: github.event_name == 'schedule' || github.event_name == 'pull_request'
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      name: low_memory
+      test_command: .\unit_tests.exe
+      build_artifact: Build-x64
+      environment: windows-2019
+      code_coverage: true
+      gather_dumps: true
+      low_memory: true

.github/workflows/reusable-test.yml

@@ -45,6 +45,9 @@ on:
       vs_dev:
         required: false
         type: boolean
+      low_memory:
+        required: false
+        type: boolean
 
 permissions:
   checks: read  # Required by fountainhead/action-wait-for-check to wait for another GitHub check to complete.
@@ -138,6 +141,7 @@ jobs:
       run: |
         ${{env.PRE_COMMAND}}
 
+    # TODO: Cleanup the combination of options: https://github.com/microsoft/ebpf-for-windows/issues/1590
     - name: Run test with Code Coverage in VS Dev environment
       if: (inputs.code_coverage == true) && (inputs.vs_dev == true)
       id: run_test_with_code_coverage_in_vs_dev
@@ -145,10 +149,27 @@ jobs:
       run: |
           call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat"
           set EBPF_ENABLE_WER_REPORT=yes
-          OpenCppCoverage.exe --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}}\${{env.TEST_COMMAND}}
+          OpenCppCoverage.exe --cover_children --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}}\${{env.TEST_COMMAND}}
+
+    - name: Run test with Code Coverage and low resource simulation
+      if: (inputs.code_coverage == true) && (inputs.low_memory == true)
+      id: run_test_with_code_coverage_in_low_memory
+      shell: cmd
+      run: |
+          set EBPF_ENABLE_WER_REPORT=yes
+          OpenCppCoverage.exe --cover_children --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- powershell.exe .\Test-LowMemory.ps1 ${{env.TEST_COMMAND}} 4
+
+    - name: Run test with low resource simulation
+      if: (inputs.code_coverage != true) && (inputs.low_memory == true)
+      id: run_test_with_low_memory
+      shell: cmd
+      working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
+      run: |
+          set EBPF_ENABLE_WER_REPORT=yes
+          powershell.exe .\Test-LowMemory.ps1 ${{env.TEST_COMMAND}} 16
 
     - name: Run test with Code Coverage
-      if: (inputs.code_coverage == true) && (inputs.vs_dev != true)
+      if: (inputs.code_coverage == true) && (inputs.vs_dev != true) && (inputs.low_memory != true)
       id: run_test_with_code_coverage
       shell: cmd
       run: |
@@ -259,7 +280,7 @@ jobs:
         files: c:/dumps/${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}/*.dmp
 
     - name: Upload any crash dumps
-      if: (failure()) && (inputs.gather_dumps == true)
+      if: always() && (steps.check_dumps.outputs.files_exists == 'true') && (inputs.gather_dumps == true)
       uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
       id: upload_crash_dumps
       with:

libs/platform/CMakeLists.txt

@@ -106,6 +106,7 @@ add_library("platform_user" STATIC
   ${platform_user_sources}
   user/framework.h
   user/ebpf_handle_user.c
+  user/ebpf_low_memory_test.cpp
   user/ebpf_platform_user.cpp
   user/ebpf_native_user.c
   user/kernel_um.cpp

libs/platform/user/ebpf_low_memory_test.cpp

@@ -0,0 +1,167 @@
+// Copyright (c) Microsoft Corporation
+// SPDX-License-Identifier: MIT
+
+#include "ebpf_low_memory_test.h"
+
+#include <DbgHelp.h>
+#include <sstream>
+#include <string>
+
+// Link with DbgHelp.lib
+#pragma comment(lib, "dbghelp.lib")
+
+/**
+ * @brief Approximate size in bytes of the image being tested.
+ *
+ */
+#define EBPF_MODULE_SIZE_IN_BYTES (10 * 1024 * 1024)
+
+/**
+ * @brief The number of stack frames to write to the human readable log.
+ */
+#define EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT 16
+
+/**
+ * @brief The number of stack frames to capture to uniquely identify an allocation stack.
+ *
+ */
+#define EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT_FOR_HASH 4
+
+#define EBPF_MODULE_SIZE_IN_BYTES (10 * 1024 * 1024)
+
+_ebpf_low_memory_test::_ebpf_low_memory_test(size_t stack_depth = EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT_FOR_HASH)
+    : _stack_depth(stack_depth)
+{
+    _base_address = (uintptr_t)(GetModuleHandle(nullptr));
+    load_allocation_log();
+}
+
+_ebpf_low_memory_test::~_ebpf_low_memory_test()
+{
+    _log_file.flush();
+    _log_file.close();
+}
+
+bool
+_ebpf_low_memory_test::fail_stack_allocation()
+{
+    std::unique_lock lock(_mutex);
+    return is_new_stack();
+}
+
+bool
+_ebpf_low_memory_test::is_new_stack()
+{
+    std::vector<uintptr_t> stack(EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT);
+    std::vector<uintptr_t> canonical_stack(_stack_depth);
+    DWORD hash;
+    // Capture EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT_FOR_HASH frames of the current stack trace.
+    if (CaptureStackBackTrace(
+            1, static_cast<unsigned int>(stack.size()), reinterpret_cast<void**>(stack.data()), &hash) > 0) {
+        // Form the canonical stack
+        for (size_t i = 0; i < _stack_depth; i++) {
+            uintptr_t frame = stack[i];
+            if (frame < _base_address || frame > (_base_address + EBPF_MODULE_SIZE_IN_BYTES)) {
+                frame = 0;
+            } else {
+                frame -= _base_address;
+            }
+            canonical_stack[i] = frame;
+        }
+
+        // Check if the stack trace is already in the hash.
+        if (_allocation_hash.contains(canonical_stack)) {
+            // Stack is already in the hash, allow the allocation.
+            return false;
+        } else {
+            // Stack is not in the hash, add it to the hash, write it to the log file and fail the allocation.
+            _allocation_hash.insert(canonical_stack);
+            log_stack_trace(canonical_stack, stack);
+            return true;
+        }
+    }
+    return false;
+}
+
+void
+_ebpf_low_memory_test::log_stack_trace(
+    const std::vector<uintptr_t>& canonical_stack, const std::vector<uintptr_t>& stack)
+{
+    for (auto i : canonical_stack) {
+        _log_file << std::hex << i << " ";
+    }
+    _log_file << std::endl;
+
+    std::vector<uint8_t> symbol_buffer(sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR));
+    SYMBOL_INFO* symbol = reinterpret_cast<SYMBOL_INFO*>(symbol_buffer.data());
+    IMAGEHLP_LINE64 line;
+    symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+    symbol->MaxNameLen = MAX_SYM_NAME;
+    for (auto frame : stack) {
+        if (frame == 0) {
+            break;
+        }
+        DWORD64 displacement = 0;
+        _log_file << "# ";
+        if (SymFromAddr(GetCurrentProcess(), frame, &displacement, symbol)) {
+            _log_file << std::hex << frame << " " << symbol->Name << " + " << displacement;
+            DWORD displacement32 = (DWORD)displacement;
+            if (SymGetLineFromAddr64(GetCurrentProcess(), frame, &displacement32, &line)) {
+                _log_file << " " << line.FileName << std::dec << " " << line.LineNumber;
+            }
+            _log_file << std::endl;
+        } else {
+            _log_file << std::hex << frame << std::endl;
+        }
+    }
+    _log_file << std::endl;
+    // Flush the file after every write to prevent loss on crash.
+    _log_file.flush();
+}
+
+void
+_ebpf_low_memory_test::load_allocation_log()
+{
+    // Get the path to the executable being run.
+    char process_name[MAX_PATH];
+    GetModuleFileNameA(nullptr, process_name, MAX_PATH);
+
+    // Read back the list of allocations that have been failed in the previous runs.
+    std::string allocation_log_file = process_name;
+    allocation_log_file += ".allocation.log";
+    {
+        std::ifstream allocation_log(allocation_log_file);
+        std::string line;
+        std::string frame;
+        while (std::getline(allocation_log, line)) {
+            // Count the iterations to correlate crashes with the last failed allocation.
+            if (line.starts_with("# Iteration: ")) {
+                _iteration++;
+                continue;
+            }
+            // Skip the stack trace.
+            if (line.starts_with("#")) {
+                continue;
+            }
+            // Parse the stack frame.
+            std::vector<uintptr_t> stack;
+            auto stream = std::istringstream(line);
+            while (std::getline(stream, frame, ' ')) {
+                stack.push_back(std::stoull(frame, nullptr, 16));
+            }
+            _allocation_hash.insert(stack);
+        }
+        allocation_log.close();
+    }
+
+    // Re-open the log file in append mode to record the allocations that are failed in this run.
+    _log_file.open(allocation_log_file, std::ios_base::app);
+
+    // Add the current iteration number to the log file.
+    _log_file << "# Iteration: " << ++_iteration << std::endl;
+
+    // Initialize DbgHelp.dll.
+    SymSetOptions(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS);
+    SymInitialize(GetCurrentProcess(), nullptr, TRUE);
+    SymSetOptions(SYMOPT_LOAD_LINES);
+}

libs/platform/user/ebpf_low_memory_test.h

@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft Corporation
+// SPDX-License-Identifier: MIT
+#pragma once
+
+#include <cstddef>
+#include <fstream>
+#include <mutex>
+#include <unordered_set>
+#include <vector>
+
+#include "ebpf_platform.h"
+
+/**
+ * @brief This class is used to track memory allocations and fail the first allocation for
+ * a specific stack. Increasing the number of stack frames examined will increase the
+ * accuracy of the test, but also increase the time it takes to run the test.
+ */
+typedef class _ebpf_low_memory_test
+{
+  public:
+    /**
+     * @brief Construct a new ebpf low memory test object.
+     * @param[in] stack_depth The number of stack frames to compare when tracking allocations.
+     */
+    _ebpf_low_memory_test(size_t stack_depth);
+
+    /**
+     * @brief Destroy the ebpf low memory test object.
+     *
+     */
+    ~_ebpf_low_memory_test();
+
+    /**
+     * @brief Test to see if the allocator should fail this allocation.
+     *
+     * @retval true Fail the allocation.
+     * @retval false Don't fail the allocation.
+     */
+    bool
+    fail_stack_allocation();
+
+  private:
+    /**
+     * @brief Compute a hash over the current stack.
+     */
+    struct _stack_hasher
+    {
+        size_t
+        operator()(const std::vector<uintptr_t>& key) const
+        {
+            size_t hash_value = 0;
+            for (const auto value : key) {
+                hash_value ^= std::hash<uintptr_t>{}(value);
+            }
+            return hash_value;
+        }
+    };
+
+    /**
+     * @brief Determine if this allocation path is new.
+     * If it is new, then fail the allocation, add it to the set of known
+     * allocation paths and return true.
+     */
+    bool
+    is_new_stack();
+
+    /**
+     * @brief Write the current stack to the log file.
+     */
+    void
+    log_stack_trace(const std::vector<uintptr_t>& canonical_stack, const std::vector<uintptr_t>& stack);
+
+    /**
+     * @brief Load the list of known allocation paths from the log file.
+     */
+    void
+    load_allocation_log();
+
+    /**
+     * @brief The base address of the current process.
+     */
+    uintptr_t _base_address = 0;
+
+    /**
+     * @brief The iteration number of the current test pass.
+     */
+    size_t _iteration = 0;
+
+    /**
+     * @brief The log file for allocations that have been failed.
+     */
+    std::ofstream _log_file;
+
+    /**
+     * @brief The set of known allocation paths.
+     */
+    std::unordered_set<std::vector<uintptr_t>, _stack_hasher> _allocation_hash;
+
+    /**
+     * @brief The mutex to protect the set of known allocation paths.
+     */
+    std::mutex _mutex;
+
+    size_t _stack_depth;
+} ebpf_low_memory_test_t;

libs/platform/user/ebpf_platform_user.cpp

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 #include "ebpf_platform.h"
-#include "ebpf_utilities.h"
+
 #include <intsafe.h>
 #include <functional>
 #include <map>
@@ -12,8 +12,12 @@
 #include <set>
 #include <stdbool.h>
 #include <stdint.h>
-#include <vector>
+#include <string>
 #include <TraceLoggingProvider.h>
+#include <vector>
+
+#include "ebpf_low_memory_test.h"
+#include "ebpf_utilities.h"
 
 // Global variables used to override behavior for testing.
 // Permit the test to simulate both Hyper-V Code Integrity.
@@ -24,6 +28,14 @@ bool _ebpf_platform_is_preemptible = true;
 extern "C" bool ebpf_fuzzing_enabled = false;
 extern "C" size_t ebfp_fuzzing_memory_limit = MAXSIZE_T;
 
+std::unique_ptr<ebpf_low_memory_test_t> _ebpf_low_memory_test_ptr;
+
+/**
+ * @brief Environment variable to enable low memory testing.
+ *
+ */
+#define EBPF_LOW_MEMORY_SIMULATION_ENVIRONMENT_VARIABLE_NAME "EBPF_LOW_MEMORY_SIMULATION"
+
 // Thread pool related globals.
 static TP_CALLBACK_ENVIRON _callback_environment;
 static PTP_POOL _pool = nullptr;
@@ -213,6 +225,20 @@ class _ebpf_emulated_dpc
     bool terminate;
 };
 
+static std::string
+_get_environment_variable(const std::string& name)
+{
+    std::string value;
+    size_t required_size = 0;
+    getenv_s(&required_size, nullptr, 0, name.c_str());
+    if (required_size > 0) {
+        value.resize(required_size);
+        getenv_s(&required_size, &value[0], required_size, name.c_str());
+        value.resize(required_size - 1);
+    }
+    return value;
+}
+
 ebpf_result_t
 ebpf_platform_initiate()
 {
@@ -220,6 +246,13 @@ ebpf_platform_initiate()
     try {
         _ebpf_platform_maximum_group_count = GetMaximumProcessorGroupCount();
         _ebpf_platform_maximum_processor_count = GetMaximumProcessorCount(ALL_PROCESSOR_GROUPS);
+        auto low_memory_stack_depth = _get_environment_variable(EBPF_LOW_MEMORY_SIMULATION_ENVIRONMENT_VARIABLE_NAME);
+        if (!low_memory_stack_depth.empty() && !_ebpf_low_memory_test_ptr) {
+            _ebpf_low_memory_test_ptr =
+                std::make_unique<ebpf_low_memory_test_t>(std::strtoul(low_memory_stack_depth.c_str(), nullptr, 10));
+            // Set flag to remove some asserts that fire from incorrect client behavior.
+            ebpf_fuzzing_enabled = true;
+        }
 
         for (size_t i = 0; i < ebpf_get_cpu_count(); i++) {
             _ebpf_emulated_dpcs.push_back(std::make_shared<_ebpf_emulated_dpc>(i));
@@ -259,6 +292,12 @@ ebpf_get_code_integrity_state(_Out_ ebpf_code_integrity_state_t* state)
     EBPF_RETURN_RESULT(EBPF_SUCCESS);
 }
 
+bool
+ebpf_low_memory_test_in_progress()
+{
+    return _ebpf_low_memory_test_ptr != nullptr;
+}
+
 __drv_allocatesMem(Mem) _Must_inspect_result_ _Ret_maybenull_
     _Post_writable_byte_size_(size) void* ebpf_allocate(size_t size)
 {
@@ -266,6 +305,11 @@ __drv_allocatesMem(Mem) _Must_inspect_result_ _Ret_maybenull_
     if (size > ebfp_fuzzing_memory_limit) {
         return nullptr;
     }
+
+    if (_ebpf_low_memory_test_ptr && _ebpf_low_memory_test_ptr->fail_stack_allocation()) {
+        return nullptr;
+    }
+
     void* memory;
     memory = calloc(size, 1);
     if (memory != nullptr)
@@ -281,6 +325,11 @@ __drv_allocatesMem(Mem) _Must_inspect_result_ _Ret_maybenull_
     if (new_size > ebfp_fuzzing_memory_limit) {
         return nullptr;
     }
+
+    if (_ebpf_low_memory_test_ptr && _ebpf_low_memory_test_ptr->fail_stack_allocation()) {
+        return nullptr;
+    }
+
     void* p = realloc(memory, new_size);
     if (p && (new_size > old_size))
         memset(((char*)p) + old_size, 0, new_size - old_size);

libs/platform/user/platform_user.vcxproj

@@ -39,6 +39,7 @@
     <ClCompile Include="..\ebpf_state.c" />
     <ClCompile Include="..\ebpf_trampoline.c" />
     <ClCompile Include="ebpf_handle_user.c" />
+    <ClCompile Include="ebpf_low_memory_test.cpp" />
     <ClCompile Include="ebpf_native_user.c" />
     <ClCompile Include="ebpf_platform_user.cpp" />
     <ClCompile Include="kernel_um.cpp" />
@@ -53,6 +54,7 @@
     <ClInclude Include="..\ebpf_platform.h" />
     <ClInclude Include="..\ebpf_ring_buffer.h" />
     <ClInclude Include="..\ebpf_state.h" />
+    <ClInclude Include="ebpf_low_memory_test.h" />
     <ClInclude Include="framework.h" />
     <ClInclude Include="kernel_um.h" />
     <ClInclude Include="nmr_impl.h" />

libs/platform/user/platform_user.vcxproj.filters

@@ -85,6 +85,9 @@
     <ClCompile Include="kernel_um.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="ebpf_low_memory_test.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <Midl Include="..\ebpf_program_types.idl">
@@ -133,5 +136,8 @@
     <ClInclude Include="kernel_um.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="ebpf_low_memory_test.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

scripts/Test-LowMemory.ps1

@@ -0,0 +1,49 @@
+# Copyright (c) Microsoft Corporation
+# SPDX-License-Identifier: MIT
+# This script is used as part of the systematic testing of the
+# low memory handling.
+# First it will create a list of all the tests in the test binary.
+# Second it will execute each test in the test binary.
+# Third it will check the output of each test to see if it passed or failed.
+# Fourth re-run the failed tests.
+# Fifth it will check if all tests passed and exit.
+#
+
+param ($TestProgram, $StackDepth)
+
+# Gather list of all possible tests
+$tests = & $TestProgram "--list-tests" "--verbosity=quiet"
+
+$env:EBPF_LOW_MEMORY_SIMULATION = $StackDepth
+$env:EBPF_ENABLE_WER_REPORT = "yes"
+
+Set-Content -Path ($TestProgram +".passed.log") ""
+
+# Rerun failing tests until they pass
+while ($true) {
+    $previous_passed_tests = $passed_tests
+    $passed_tests = Get-Content -Path ($TestProgram +".passed.log")
+
+    # Get the list of tests that have passed in the previous iteration.
+    $passed_tests | Where-Object { $_ -notin $previous_passed_tests } | ForEach-Object { Write-Host "Passed: $_" }
+
+    # Compute list of tests that haven't passed yet
+    $remaining_tests = $tests | Where-Object { $_ -notin $passed_tests }
+
+    # If all the tests have passed, exit.
+    if ($remaining_tests.Count -eq 0) {
+        break
+    }
+
+    # Write the list of tests that haven't passed yet to a file.
+    Set-Content -Path "remaining_tests.txt" -Value $remaining_tests
+
+    # Run the test binary with any remaining tests.
+    $log =(& $TestProgram "-d yes" "--verbosity=quiet" "-f remaining_tests.txt" 2>&1)
+    if ($LASTEXITCODE -eq 0) {
+        write-host "All tests passed"
+        break
+    }
+}
+
+Write-Host "All tests passed"

scripts/setup_build/setup_build.vcxproj

@@ -106,6 +106,10 @@ popd
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
     </CopyFileToFolders>
+    <CopyFileToFolders Include="..\Test-LowMemory.ps1">
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
+    </CopyFileToFolders>
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <VCProjectVersion>16.0</VCProjectVersion>

tests/end_to_end/end_to_end.cpp

@@ -22,6 +22,7 @@
 #include "helpers.h"
 #include "ioctl_helper.h"
 #include "mock.h"
+#include "passed_test_log.h"
 #include "platform.h"
 #include "program_helper.h"
 #include "sample_test_common.h"
@@ -34,6 +35,8 @@ namespace ebpf {
 #include "net/udp.h"
 }; // namespace ebpf
 
+CATCH_REGISTER_LISTENER(_passed_test_log)
+
 #define NATIVE_DRIVER_SERVICE_NAME L"test_service"
 #define NATIVE_DRIVER_SERVICE_NAME_2 L"test_service2"
 #define SERVICE_PATH_PREFIX L"\\Registry\\Machine\\System\\CurrentControlSet\\Services\\"
@@ -347,7 +350,9 @@ ebpf_program_load(
     if (error < 0) {
         if (log_buffer) {
             size_t log_buffer_size;
-            *log_buffer = _strdup(bpf_program__log_buf(program, &log_buffer_size));
+            if (program != nullptr) {
+                *log_buffer = _strdup(bpf_program__log_buf(program, &log_buffer_size));
+            }
         }
         bpf_object__close(new_object);
         return error;
@@ -2623,4 +2628,4 @@ TEST_CASE("test_ebpf_object_set_execution_type", "[end_to_end]")
     REQUIRE(ebpf_object_get_execution_type(jit_object) == EBPF_EXECUTION_INTERPRET);
 
     bpf_object__close(jit_object);
-}
+}

tests/end_to_end/test_helper.cpp

@@ -230,7 +230,7 @@ _unload_all_native_modules()
             ebpf_extension_unload(context->binding_context);
         }
         // The service should have been marked for deletion till now.
-        REQUIRE((context->delete_pending || _expect_native_module_load_failures));
+        REQUIRE((context->delete_pending || get_native_module_failures()));
         if (context->dll != nullptr) {
             FreeLibrary(context->dll);
         }
@@ -250,7 +250,7 @@ _preprocess_load_native_module(_Inout_ service_context_t* context)
     _is_platform_preemptible = !_is_platform_preemptible;
 
     context->dll = LoadLibraryW(context->file_path.c_str());
-    REQUIRE(((context->dll != nullptr) || (_expect_native_module_load_failures)));
+    REQUIRE(((context->dll != nullptr) || get_native_module_failures()));
 
     if (context->dll == nullptr) {
         return;
@@ -259,7 +259,7 @@ _preprocess_load_native_module(_Inout_ service_context_t* context)
     auto get_function =
         reinterpret_cast<decltype(&get_metadata_table)>(GetProcAddress(context->dll, "get_metadata_table"));
     if (get_function == nullptr) {
-        REQUIRE(_expect_native_module_load_failures);
+        REQUIRE(get_native_module_failures());
         return;
     }
 
@@ -285,7 +285,7 @@ _preprocess_load_native_module(_Inout_ service_context_t* context)
         &returned_provider_dispatch_table,
         nullptr);
 
-    REQUIRE((result == EBPF_SUCCESS || _expect_native_module_load_failures));
+    REQUIRE((result == EBPF_SUCCESS || get_native_module_failures()));
 
     context->loaded = true;
 }
@@ -308,7 +308,7 @@ _preprocess_ioctl(_In_ const ebpf_operation_header_t* user_request)
                 context->second->module_id = request->module_id;
 
                 if (context->second->loaded) {
-                    REQUIRE(_expect_native_module_load_failures);
+                    REQUIRE(get_native_module_failures());
                 } else {
                     _preprocess_load_native_module(context->second);
                 }
@@ -604,6 +604,15 @@ set_native_module_failures(bool expected)
     _expect_native_module_load_failures = expected;
 }
 
+extern bool
+ebpf_low_memory_test_in_progress();
+
+bool
+get_native_module_failures()
+{
+    return _expect_native_module_load_failures || ebpf_low_memory_test_in_progress();
+}
+
 ebpf_result_t
 get_service_details_for_file(
     _In_ const std::wstring& file_path, _Out_ const wchar_t** service_name, _Out_ GUID* provider_guid)

tests/end_to_end/test_helper.hpp

@@ -35,6 +35,9 @@ class _test_helper_libbpf
 void
 set_native_module_failures(bool expected);
 
+bool
+get_native_module_failures();
+
 ebpf_result_t
 get_service_details_for_file(
     _In_ const std::wstring& file_path, _Out_ const wchar_t** service_name, _Out_ GUID* provider_guid);

tests/libs/common/passed_test_log.h

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation
+// SPDX-License-Identifier: MIT
+
+#include <windows.h>
+
+#include <fstream>
+#include <iostream>
+#include <catch2/reporters/catch_reporter_event_listener.hpp>
+#include <catch2/reporters/catch_reporter_registrars.hpp>
+#include "catch_wrapper.hpp"
+
+/**
+ * @brief A Catch2 reporter that logs the name of each test that passes.
+ * This is used to generate a list of tests that passed in the last run in a
+ * file that can be used to filter the set of tests that are run in the next
+ * run. This is consumed by the Test-LowMemory.ps1 script.
+ */
+class _passed_test_log : public Catch::EventListenerBase
+{
+  public:
+    using Catch::EventListenerBase::EventListenerBase;
+
+    // Log failed tests.
+    void
+    testCaseEnded(Catch::TestCaseStats const& testCaseStats) override
+    {
+        if (!passed_tests) {
+            char process_name[MAX_PATH];
+            GetModuleFileNameA(nullptr, process_name, MAX_PATH);
+            std::string log_file = process_name;
+            log_file += ".passed.log";
+            passed_tests.open(log_file, std::ios::app);
+        }
+        if (testCaseStats.totals.assertions.failed == 0) {
+            passed_tests << testCaseStats.testInfo->name << std::endl;
+            passed_tests.flush();
+        }
+    }
+
+  private:
+    static std::ofstream passed_tests;
+};
+
+std::ofstream _passed_test_log::passed_tests;

tests/libs/util/wer_report.hpp

@@ -5,6 +5,7 @@
 
 #define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers
 #include <Windows.h>
+#include <csignal>
 #include <errorrep.h>
 #include <stdexcept>
 #include <string>
@@ -41,6 +42,8 @@ class _wer_report
         unsigned long guaranteed_stack_size = static_cast<unsigned long>(minimum_stack_size_for_wer);
         char* buffer = nullptr;
         size_t size = 0;
+
+        // Check if the EBPF_ENABLE_WER_REPORT is set to "yes".
         _dupenv_s(&buffer, &size, environment_variable_name);
         if (size == 0 || !buffer || _stricmp(environment_variable_value, buffer) != 0) {
             free(buffer);
@@ -48,9 +51,25 @@ class _wer_report
         }
         free(buffer);
 
+        // Redirect error output to STDERR so that it is captured in the console
+        // when running in CI/CD.
+        _set_error_mode(_OUT_TO_STDERR);
+
+        // Add a hook to generate WER report on failed assertions and other
+        //  failures raised by MSVC runtime.
+        _CrtSetReportHook(_terminate_hook);
+
+        // Add a hook to generate WER report on noexcept violations and other
+        // cases where the CRT calls std::abort().
+        signal(SIGABRT, signal_handler);
+
+        // Reserve stack space for WER report generation.
         if (!SetThreadStackGuarantee(&guaranteed_stack_size)) {
             throw std::runtime_error("SetThreadStackGuarantee failed");
         }
+
+        // Add a vectored exception handler to generate WER report on unhandled
+        // exceptions.
         vectored_exception_handler_handle = AddVectoredExceptionHandler(TRUE, _wer_report::vectored_exception_handler);
         enabled = true;
     }
@@ -65,6 +84,20 @@ class _wer_report
     static constexpr const char environment_variable_value[] = "yes";
     static constexpr const wchar_t wer_event_type[] = L"Test Application Crash";
 
+    static int __CRTDECL
+    _terminate_hook(int, char*, int*)
+    {
+        // Convert a CRT runtime error into a SEH exception.
+        RaiseException(STATUS_ASSERTION_FAILURE, 0, 0, NULL);
+        return 0;
+    }
+
+    static void __cdecl signal_handler(int)
+    {
+        // Convert a SIGABRT signal into a SEH exception.
+        RaiseException(STATUS_ASSERTION_FAILURE, 0, 0, NULL);
+    }
+
     static constexpr unsigned long fatal_exceptions[] = {
         STATUS_ACCESS_VIOLATION,
         STATUS_ASSERTION_FAILURE,

tests/unit/test.vcxproj

@@ -166,9 +166,6 @@
     <ProjectReference Include="..\..\libs\service\service.vcxproj">
       <Project>{af85c549-57cc-40a5-bdfc-dcf1998de80f}</Project>
     </ProjectReference>
-    <ProjectReference Include="..\..\libs\spec\spec.vcxproj">
-      <Project>{c3d2cd73-bf4c-47df-8808-2a9996124d5b}</Project>
-    </ProjectReference>
     <ProjectReference Include="..\..\libs\ubpf\user\ubpf_user.vcxproj">
       <Project>{245f0ec7-1ebc-4d68-8b1f-f758ea9196ae}</Project>
     </ProjectReference>