Refactor low memory test into generic fault injection (#2173)

* Refactor low memory test into generic fault injection

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

* Update number of frames to skip

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

* PR feedback

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

---------

Signed-off-by: Alan Jowett <alanjo@microsoft.com>

.github/workflows/cicd.yml

@@ -351,18 +351,18 @@ jobs:
       gather_dumps: true
       capture_etw: true
 
-  # Run the low memory simulator in GitHub.
-  low_memory:
+  # Run the fault injection simulator in GitHub.
+  fault_injection:
     needs: regular
     uses: ./.github/workflows/reusable-test.yml
     with:
-      name: low_memory
+      name: fault_injection
       test_command: .\unit_tests.exe
       build_artifact: Build-x64
       environment: windows-2022
       code_coverage: true
       gather_dumps: true
-      low_memory: true
+      fault_injection: true
 
   # Additional jobs to run on a schedule only (skip push and pull request).
   # ---------------------------------------------------------------------------
@@ -375,17 +375,17 @@ jobs:
       build_codeql: true
 
 
-  # Run the complete low memory simulator in GitHub.
+  # Run the complete fault injection simulator in GitHub.
   # Runs on a schedule as this takes a long time to run.
-  low_memory_full:
+  fault_injection_full:
     needs: regular
     if: github.event_name == 'schedule'
     uses: ./.github/workflows/reusable-test.yml
     with:
-      name: low_memory_full
+      name: fault_injection_full
       test_command: .\unit_tests.exe
       build_artifact: Build-x64
       environment: windows-2019
       code_coverage: false
       gather_dumps: true
-      low_memory: true
+      fault_injection: true

.github/workflows/reusable-test.yml

@@ -45,7 +45,7 @@ on:
       vs_dev:
         required: false
         type: boolean
-      low_memory:
+      fault_injection:
         required: false
         type: boolean
       leak_detection:
@@ -172,24 +172,24 @@ jobs:
           OpenCppCoverage.exe -q --cover_children --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}}\${{env.TEST_COMMAND}}
 
     - name: Run test with Code Coverage and low resource simulation
-      if: (inputs.code_coverage == true) && (inputs.low_memory == true) && (steps.skip_check.outputs.should_skip != 'true')
-      id: run_test_with_code_coverage_in_low_memory
+      if: (inputs.code_coverage == true) && (inputs.fault_injection == true) && (steps.skip_check.outputs.should_skip != 'true')
+      id: run_test_with_code_coverage_in_fault_injection
       shell: cmd
       run: |
           set EBPF_ENABLE_WER_REPORT=yes
-          OpenCppCoverage.exe -q --cover_children --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- powershell.exe .\Test-LowMemory.ps1 ${{env.TEST_COMMAND}} 4
+          OpenCppCoverage.exe -q --cover_children --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- powershell.exe .\Test-FaultInjection.ps1 ${{env.TEST_COMMAND}} 4
 
     - name: Run test with low resource simulation
-      if: (inputs.code_coverage != true) && (inputs.low_memory == true) && (steps.skip_check.outputs.should_skip != 'true')
-      id: run_test_with_low_memory
+      if: (inputs.code_coverage != true) && (inputs.fault_injection == true) && (steps.skip_check.outputs.should_skip != 'true')
+      id: run_test_with_fault_injection
       shell: cmd
       working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
       run: |
           set EBPF_ENABLE_WER_REPORT=yes
-          powershell.exe .\Test-LowMemory.ps1 ${{env.TEST_COMMAND}} 16
+          powershell.exe .\Test-FaultInjection.ps1 ${{env.TEST_COMMAND}} 16
 
     - name: Run test with Code Coverage
-      if: (inputs.code_coverage == true) && (inputs.vs_dev != true) && (inputs.low_memory != true) && (steps.skip_check.outputs.should_skip != 'true')
+      if: (inputs.code_coverage == true) && (inputs.vs_dev != true) && (inputs.fault_injection != true) && (steps.skip_check.outputs.should_skip != 'true')
       id: run_test_with_code_coverage
       shell: cmd
       run: |

docs/AutomatedTests.md

@@ -63,12 +63,11 @@ Tests in this category currently include:
 * bpftool_tests.exe: This tests app compat for scripts (and users) that invoke bpftool commands.
 * cilium_tests.exe: This tests that the Cilium L4LB eBPF programs can be verified.
 
-## Low memory tests
-Low memory tests use error injection to fail memory allocations in order to test behavior under low
-memory conditions.
+## Fault injection tests
+Fault injection tests inject faults in order to test behavior under fault conditions.
 
 Tests in this category currently include:
-* unit_tests.exe: The unit test discussed above, but run under low memory conditions.
+* unit_tests.exe: The unit test discussed above, but run under fault injection conditions.
 
 ## Performance tests
 Performance tests check for performance regressions across builds.

docs/SelfHostedRunnerSetup.md

@@ -29,7 +29,7 @@ This document discusses the steps to set up such a self-hosted actions-runner th
    2) `New-StoredCredential -Target `**`TEST_VM`**` -Username <VM Administrator> -Password <VM Administrator account password> -Persist LocalMachine`
    3) `New-StoredCredential -Target `**`TEST_VM_STANDARD`**` -Username <VM Standard User Name> -Password <VM Standard User account password> -Persist LocalMachine`
 
-10) Modify the environment of the VM as needed. Create new checkpoints using **Hyper-V**. Rename the new checkpoint as `baseline`, and remove the old baseline. 
+10) Modify the environment of the VM as needed. Create new checkpoints using **Hyper-V**. Rename the new checkpoint as `baseline`, and remove the old baseline.
 11) Set up Windows Error Reporting [Local Dump Collection](https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps) on the VMs with the following commands.
     ```New-Item -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -ErrorAction SilentlyContinue```
     ```New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpType" -Value 2 -PropertyType DWord -ErrorAction SilentlyContinue```

libs/ebpfnetsh/programs.cpp

@@ -78,7 +78,7 @@ struct _program_unloader
     ~_program_unloader() { bpf_object__close(object); }
 };
 
-// The following function uses windows specific input type to match 
+// The following function uses windows specific input type to match
 // definition of "FN_HANDLE_CMD" in public file of NetSh.h
 unsigned long
 handle_ebpf_add_program(
@@ -310,7 +310,7 @@ _find_object_with_program(ebpf_id_t id)
     return _ebpf_netsh_objects.end();
 }
 
-// The following function uses windows specific type to match 
+// The following function uses windows specific type to match
 // definition of "FN_HANDLE_CMD" in public file of NetSh.h
 unsigned long
 handle_ebpf_delete_program(
@@ -440,7 +440,7 @@ _ebpf_program_detach_by_id(ebpf_id_t program_id)
     return ERROR_NOT_FOUND;
 }
 
-// The following function uses windows specific type as an input to match 
+// The following function uses windows specific type as an input to match
 // definition of "FN_HANDLE_CMD" in public file of NetSh.h
 unsigned long
 handle_ebpf_set_program(
@@ -552,7 +552,7 @@ handle_ebpf_set_program(
     return ERROR_OKAY;
 }
 
-// The following function uses windows specific type as an input to match 
+// The following function uses windows specific type as an input to match
 // definition of "FN_HANDLE_CMD" in public file of NetSh.h
 unsigned long
 handle_ebpf_show_programs(

libs/platform/CMakeLists.txt

@@ -56,7 +56,7 @@ add_library("platform_user" STATIC
   user/framework.h
   user/ebpf_handle_user.c
   user/ebpf_leak_detector.cpp
-  user/ebpf_low_memory_test.cpp
+  user/ebpf_fault_injection.cpp
   user/ebpf_platform_user.cpp
   user/ebpf_native_user.c
   user/kernel_um.cpp

libs/platform/user/ebpf_fault_injection.cpp

@@ -0,0 +1,333 @@
+// Copyright (c) Microsoft Corporation
+// SPDX-License-Identifier: MIT
+
+#include "ebpf_fault_injection.h"
+#include "ebpf_symbol_decoder.h"
+
+#include <DbgHelp.h>
+#include <cstddef>
+#include <fstream>
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+/**
+ * @brief This class is used to track potential fault points and fail them in
+ * a deterministic manner. Increasing the number of stack frames examined will
+ * increase the accuracy of the test, but also increase the time it takes to run
+ * the test.
+ */
+typedef class _ebpf_fault_injection
+{
+  public:
+    /**
+     * @brief Construct a new ebpf fault injection object.
+     * @param[in] stack_depth The number of stack frames to compare when tracking faults.
+     */
+    _ebpf_fault_injection(size_t stack_depth);
+
+    /**
+     * @brief Destroy the ebpf fault injection object.
+     */
+    ~_ebpf_fault_injection();
+
+    bool
+    inject_fault();
+
+  private:
+    /**
+     * @brief Compute a hash over the current stack.
+     */
+    struct _stack_hasher
+    {
+        size_t
+        operator()(const std::vector<uintptr_t>& key) const
+        {
+            size_t hash_value = 0;
+            for (const auto value : key) {
+                hash_value ^= std::hash<uintptr_t>{}(value);
+            }
+            return hash_value;
+        }
+    };
+
+    /**
+     * @brief Determine if this path is new.
+     * If it is new, then inject the fault, add it to the set of known
+     * fault paths and return true.
+     */
+    bool
+    is_new_stack();
+
+    /**
+     * @brief Write the current stack to the log file.
+     */
+    void
+    log_stack_trace(const std::vector<uintptr_t>& canonical_stack, const std::vector<uintptr_t>& stack);
+
+    /**
+     * @brief Load the list of known faults from the log file.
+     */
+    void
+    load_fault_log();
+
+    /**
+     * @brief The base address of the current process.
+     */
+    uintptr_t _base_address = 0;
+
+    /**
+     * @brief The iteration number of the current test pass.
+     */
+    size_t _iteration = 0;
+
+    /**
+     * @brief The log file for faults that have been injected.
+     */
+    std::ofstream _log_file;
+
+    /**
+     * @brief The set of known fault paths.
+     */
+    std::unordered_set<std::vector<uintptr_t>, _stack_hasher> _fault_hash;
+
+    /**
+     * @brief The mutex to protect the set of known fault paths.
+     */
+    std::mutex _mutex;
+
+    size_t _stack_depth;
+    std::vector<std::string> _last_fault_stack;
+
+} ebpf_fault_injection_t;
+
+static std::unique_ptr<ebpf_fault_injection_t> _ebpf_fault_injection_singleton;
+
+// Link with DbgHelp.lib
+#pragma comment(lib, "dbghelp.lib")
+
+/**
+ * @brief Approximate size in bytes of the image being tested.
+ */
+#define EBPF_MODULE_SIZE_IN_BYTES (10 * 1024 * 1024)
+
+/**
+ * @brief The number of stack frames to write to the human readable log.
+ */
+#define EBPF_FAULT_STACK_CAPTURE_FRAME_COUNT 16
+
+/**
+ * @brief The number of stack frames to capture to uniquely identify an fault path.
+ */
+#define EBPF_FAULT_STACK_CAPTURE_FRAME_COUNT_FOR_HASH 4
+
+#define EBPF_MODULE_SIZE_IN_BYTES (10 * 1024 * 1024)
+
+#define EBPF_FAULT_STACK_CAPTURE_FRAMES_TO_SKIP 3
+
+/**
+ * @brief Thread local storage to track recursing from the fault injection callback.
+ */
+static thread_local int _ebpf_fault_injection_recursion = 0;
+
+/**
+ * @brief Class to automatically increment and decrement the recursion count.
+ */
+class ebpf_fault_injection_recursion_guard
+{
+  public:
+    ebpf_fault_injection_recursion_guard() { _ebpf_fault_injection_recursion++; }
+    ~ebpf_fault_injection_recursion_guard() { _ebpf_fault_injection_recursion--; }
+    /**
+     * @brief Return true if the current thread is recursing from the fault injection callback.
+     * @retval true The current thread is recursing from the fault injection callback.
+     * @retval false The current thread is not recursing from the fault injection callback.
+     */
+    bool
+    is_recursing()
+    {
+        return (_ebpf_fault_injection_recursion > 1);
+    }
+};
+
+_ebpf_fault_injection::_ebpf_fault_injection(size_t stack_depth = EBPF_FAULT_STACK_CAPTURE_FRAME_COUNT_FOR_HASH)
+    : _stack_depth(stack_depth)
+{
+    _base_address = (uintptr_t)(GetModuleHandle(nullptr));
+    load_fault_log();
+}
+
+_ebpf_fault_injection::~_ebpf_fault_injection()
+{
+    _log_file.flush();
+    _log_file.close();
+}
+
+bool
+_ebpf_fault_injection::inject_fault()
+{
+    std::unique_lock lock(_mutex);
+    return is_new_stack();
+}
+
+bool
+_ebpf_fault_injection::is_new_stack()
+{
+    // Prevent infinite recursion during fault injection.
+    ebpf_fault_injection_recursion_guard recursion_guard;
+    if (recursion_guard.is_recursing()) {
+        return false;
+    }
+    std::vector<uintptr_t> stack(EBPF_FAULT_STACK_CAPTURE_FRAME_COUNT);
+    std::vector<uintptr_t> canonical_stack(_stack_depth);
+
+    unsigned long hash;
+    // Capture EBPF_FAULT_STACK_CAPTURE_FRAME_COUNT_FOR_HASH frames of the current stack trace.
+    // The first EBPF_FAULT_STACK_CAPTURE_FRAMES_TO_SKIP frames are skipped to avoid
+    // capturing the fault injection code.
+    if (CaptureStackBackTrace(
+            EBPF_FAULT_STACK_CAPTURE_FRAMES_TO_SKIP,
+            static_cast<unsigned int>(stack.size()),
+            reinterpret_cast<void**>(stack.data()),
+            &hash) > 0) {
+        // Form the canonical stack.
+        for (size_t i = 0; i < _stack_depth; i++) {
+            uintptr_t frame = stack[i];
+            if (frame < _base_address || frame > (_base_address + EBPF_MODULE_SIZE_IN_BYTES)) {
+                frame = 0;
+            } else {
+                frame -= _base_address;
+            }
+            canonical_stack[i] = frame;
+        }
+
+        // Check if the stack trace is already in the hash.
+        if (_fault_hash.contains(canonical_stack)) {
+            // Stack is already in the hash, don't inject the fault.
+            return false;
+        } else {
+            // Stack is not in the hash, add it to the hash, write it to the log file and inject the fault.
+            _fault_hash.insert(canonical_stack);
+            log_stack_trace(canonical_stack, stack);
+            return true;
+        }
+    }
+    return false;
+}
+
+void
+_ebpf_fault_injection::log_stack_trace(
+    const std::vector<uintptr_t>& canonical_stack, const std::vector<uintptr_t>& stack)
+{
+    for (auto i : canonical_stack) {
+        _log_file << std::hex << i << " ";
+    }
+    _log_file << std::endl;
+
+    _last_fault_stack.resize(0);
+
+    for (auto frame : stack) {
+        std::string name;
+        std::string string_stack_frame;
+        uint64_t displacement;
+        std::optional<uint32_t> line_number;
+        std::optional<std::string> file_name;
+        if (frame == 0) {
+            break;
+        }
+        _log_file << "# ";
+        if (_ebpf_decode_symbol(frame, name, displacement, line_number, file_name) == EBPF_SUCCESS) {
+            _log_file << std::hex << frame << " " << name << " + " << displacement;
+            string_stack_frame = name + " + " + std::to_string(displacement);
+            if (line_number.has_value() && file_name.has_value()) {
+                _log_file << " " << file_name.value() << " " << line_number.value();
+                string_stack_frame += " " + file_name.value() + " " + std::to_string(line_number.value());
+            }
+        }
+        _log_file << std::endl;
+        _last_fault_stack.push_back(string_stack_frame);
+    }
+    _log_file << std::endl;
+    // Flush the file after every write to prevent loss on crash.
+    _log_file.flush();
+}
+
+void
+_ebpf_fault_injection::load_fault_log()
+{
+    // Get the path to the executable being run.
+    char process_name[MAX_PATH];
+    GetModuleFileNameA(nullptr, process_name, MAX_PATH);
+
+    // Read back the list of faults that have been failed in the previous runs.
+    std::string fault_log_file = process_name;
+    fault_log_file += ".fault.log";
+    {
+        std::ifstream fault_log(fault_log_file);
+        std::string line;
+        std::string frame;
+        while (std::getline(fault_log, line)) {
+            // Count the iterations to correlate crashes with the last failed fault.
+            if (line.starts_with("# Iteration: ")) {
+                _iteration++;
+                continue;
+            }
+            // Skip the stack trace.
+            if (line.starts_with("#")) {
+                continue;
+            }
+            // Parse the stack frame.
+            std::vector<uintptr_t> stack;
+            auto stream = std::istringstream(line);
+            while (std::getline(stream, frame, ' ')) {
+                stack.push_back(std::stoull(frame, nullptr, 16));
+            }
+            _fault_hash.insert(stack);
+        }
+        fault_log.close();
+    }
+
+    // Re-open the log file in append mode to record the faults that are failed in this run.
+    _log_file.open(fault_log_file, std::ios_base::app);
+
+    // Add the current iteration number to the log file.
+    _log_file << "# Iteration: " << ++_iteration << std::endl;
+}
+
+ebpf_result_t
+ebpf_fault_injection_initialize(size_t stack_depth) noexcept
+{
+    try {
+        _ebpf_fault_injection_singleton = std::make_unique<_ebpf_fault_injection>(stack_depth);
+    } catch (...) {
+        return EBPF_NO_MEMORY;
+    }
+    return EBPF_SUCCESS;
+}
+
+void
+ebpf_fault_injection_uninitialize() noexcept
+{
+    _ebpf_fault_injection_singleton.reset();
+}
+
+bool
+ebpf_fault_injection_inject_fault() noexcept
+{
+    try {
+        if (_ebpf_fault_injection_singleton) {
+            return _ebpf_fault_injection_singleton->inject_fault();
+        }
+        return false;
+    } catch (...) {
+        return false;
+    }
+}
+
+bool
+ebpf_fault_injection_is_enabled() noexcept
+{
+    return _ebpf_fault_injection_singleton != nullptr;
+}

libs/platform/user/ebpf_fault_injection.h

@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation
+// SPDX-License-Identifier: MIT
+#pragma once
+
+#include "ebpf_platform.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    /**
+     * @brief Initialize fault injection.  This must be called before any other
+     * fault injection functions. This function is not thread safe.
+     *
+     * @param[in] stack_depth Number of stack frames to capture when a fault is
+     * injected.
+     * @retval EBPF_SUCCESS The operation was successful.
+     * @retval EBPF_NO_MEMORY Operation failed due to memory allocation failure.
+     */
+    ebpf_result_t
+    ebpf_fault_injection_initialize(size_t stack_depth) noexcept;
+
+    /**
+     * @brief Uninitialize fault injection. This must be called after all other
+     * fault injection functions. This function is not thread safe.
+     */
+    void
+    ebpf_fault_injection_uninitialize() noexcept;
+
+    /**
+     * @brief Enable fault injection. This function is thread safe.
+     *
+     * @retval true Fault should be injected.
+     * @retval false Fault should not be injected.
+     */
+    bool
+    ebpf_fault_injection_inject_fault() noexcept;
+
+    /**
+     * @brief Test if fault injection is enabled. This function is thread safe.
+     *
+     * @retval true Fault injection is enabled.
+     * @retval false Fault injection is disabled.
+     */
+    bool
+    ebpf_fault_injection_is_enabled() noexcept;
+
+#ifdef __cplusplus
+}
+#endif

libs/platform/user/ebpf_low_memory_test.cpp

@@ -1,195 +0,0 @@
-// Copyright (c) Microsoft Corporation
-// SPDX-License-Identifier: MIT
-
-#include "ebpf_low_memory_test.h"
-#include "ebpf_symbol_decoder.h"
-
-#include <DbgHelp.h>
-#include <sstream>
-#include <string>
-
-// Link with DbgHelp.lib
-#pragma comment(lib, "dbghelp.lib")
-
-/**
- * @brief Approximate size in bytes of the image being tested.
- *
- */
-#define EBPF_MODULE_SIZE_IN_BYTES (10 * 1024 * 1024)
-
-/**
- * @brief The number of stack frames to write to the human readable log.
- */
-#define EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT 16
-
-/**
- * @brief The number of stack frames to capture to uniquely identify an allocation stack.
- *
- */
-#define EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT_FOR_HASH 4
-
-#define EBPF_MODULE_SIZE_IN_BYTES (10 * 1024 * 1024)
-
-/**
- * @brief Thread local storage to track recursing from the low memory callback.
- */
-static thread_local int _ebpf_low_memory_test_recursion = 0;
-
-/**
- * @brief Class to automatically increment and decrement the recursion count.
- */
-class ebpf_low_memory_test_recursion_guard
-{
-  public:
-    ebpf_low_memory_test_recursion_guard() { _ebpf_low_memory_test_recursion++; }
-    ~ebpf_low_memory_test_recursion_guard() { _ebpf_low_memory_test_recursion--; }
-    /**
-     * @brief Return true if the current thread is recursing from the low memory callback.
-     * @retval true
-     * @retval false
-     */
-    bool
-    is_recursing()
-    {
-        return (_ebpf_low_memory_test_recursion > 1);
-    }
-};
-
-_ebpf_low_memory_test::_ebpf_low_memory_test(size_t stack_depth = EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT_FOR_HASH)
-    : _stack_depth(stack_depth)
-{
-    _base_address = (uintptr_t)(GetModuleHandle(nullptr));
-    load_allocation_log();
-}
-
-_ebpf_low_memory_test::~_ebpf_low_memory_test()
-{
-    _log_file.flush();
-    _log_file.close();
-}
-
-bool
-_ebpf_low_memory_test::fail_stack_allocation()
-{
-    std::unique_lock lock(_mutex);
-    return is_new_stack();
-}
-
-bool
-_ebpf_low_memory_test::is_new_stack()
-{
-    // Prevent infinite recursion during allocation.
-    ebpf_low_memory_test_recursion_guard recursion_guard;
-    if (recursion_guard.is_recursing()) {
-        return false;
-    }
-    std::vector<uintptr_t> stack(EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT);
-    std::vector<uintptr_t> canonical_stack(_stack_depth);
-
-    unsigned long hash;
-    // Capture EBPF_ALLOCATION_STACK_CAPTURE_FRAME_COUNT_FOR_HASH frames of the current stack trace.
-    if (CaptureStackBackTrace(
-            1, static_cast<unsigned int>(stack.size()), reinterpret_cast<void**>(stack.data()), &hash) > 0) {
-        // Form the canonical stack
-        for (size_t i = 0; i < _stack_depth; i++) {
-            uintptr_t frame = stack[i];
-            if (frame < _base_address || frame > (_base_address + EBPF_MODULE_SIZE_IN_BYTES)) {
-                frame = 0;
-            } else {
-                frame -= _base_address;
-            }
-            canonical_stack[i] = frame;
-        }
-
-        // Check if the stack trace is already in the hash.
-        if (_allocation_hash.contains(canonical_stack)) {
-            // Stack is already in the hash, allow the allocation.
-            return false;
-        } else {
-            // Stack is not in the hash, add it to the hash, write it to the log file and fail the allocation.
-            _allocation_hash.insert(canonical_stack);
-            log_stack_trace(canonical_stack, stack);
-            return true;
-        }
-    }
-    return false;
-}
-
-void
-_ebpf_low_memory_test::log_stack_trace(
-    const std::vector<uintptr_t>& canonical_stack, const std::vector<uintptr_t>& stack)
-{
-    for (auto i : canonical_stack) {
-        _log_file << std::hex << i << " ";
-    }
-    _log_file << std::endl;
-
-    _last_failure_stack.resize(0);
-
-    for (auto frame : stack) {
-        std::string name;
-        std::string string_stack_frame;
-        uint64_t displacement;
-        std::optional<uint32_t> line_number;
-        std::optional<std::string> file_name;
-        if (frame == 0) {
-            break;
-        }
-        _log_file << "# ";
-        if (_ebpf_decode_symbol(frame, name, displacement, line_number, file_name) == EBPF_SUCCESS) {
-            _log_file << std::hex << frame << " " << name << " + " << displacement;
-            string_stack_frame = name + " + " + std::to_string(displacement);
-            if (line_number.has_value() && file_name.has_value()) {
-                _log_file << " " << file_name.value() << " " << line_number.value();
-                string_stack_frame += " " + file_name.value() + " " + std::to_string(line_number.value());
-            }
-        }
-        _log_file << std::endl;
-        _last_failure_stack.push_back(string_stack_frame);
-    }
-    _log_file << std::endl;
-    // Flush the file after every write to prevent loss on crash.
-    _log_file.flush();
-}
-
-void
-_ebpf_low_memory_test::load_allocation_log()
-{
-    // Get the path to the executable being run.
-    char process_name[MAX_PATH];
-    GetModuleFileNameA(nullptr, process_name, MAX_PATH);
-
-    // Read back the list of allocations that have been failed in the previous runs.
-    std::string allocation_log_file = process_name;
-    allocation_log_file += ".allocation.log";
-    {
-        std::ifstream allocation_log(allocation_log_file);
-        std::string line;
-        std::string frame;
-        while (std::getline(allocation_log, line)) {
-            // Count the iterations to correlate crashes with the last failed allocation.
-            if (line.starts_with("# Iteration: ")) {
-                _iteration++;
-                continue;
-            }
-            // Skip the stack trace.
-            if (line.starts_with("#")) {
-                continue;
-            }
-            // Parse the stack frame.
-            std::vector<uintptr_t> stack;
-            auto stream = std::istringstream(line);
-            while (std::getline(stream, frame, ' ')) {
-                stack.push_back(std::stoull(frame, nullptr, 16));
-            }
-            _allocation_hash.insert(stack);
-        }
-        allocation_log.close();
-    }
-
-    // Re-open the log file in append mode to record the allocations that are failed in this run.
-    _log_file.open(allocation_log_file, std::ios_base::app);
-
-    // Add the current iteration number to the log file.
-    _log_file << "# Iteration: " << ++_iteration << std::endl;
-}

libs/platform/user/ebpf_low_memory_test.h

@@ -1,106 +0,0 @@
-// Copyright (c) Microsoft Corporation
-// SPDX-License-Identifier: MIT
-#pragma once
-
-#include "ebpf_platform.h"
-
-#include <cstddef>
-#include <fstream>
-#include <mutex>
-#include <unordered_set>
-#include <vector>
-
-/**
- * @brief This class is used to track memory allocations and fail the first allocation for
- * a specific stack. Increasing the number of stack frames examined will increase the
- * accuracy of the test, but also increase the time it takes to run the test.
- */
-typedef class _ebpf_low_memory_test
-{
-  public:
-    /**
-     * @brief Construct a new ebpf low memory test object.
-     * @param[in] stack_depth The number of stack frames to compare when tracking allocations.
-     */
-    _ebpf_low_memory_test(size_t stack_depth);
-
-    /**
-     * @brief Destroy the ebpf low memory test object.
-     *
-     */
-    ~_ebpf_low_memory_test();
-
-    /**
-     * @brief Test to see if the allocator should fail this allocation.
-     *
-     * @retval true Fail the allocation.
-     * @retval false Don't fail the allocation.
-     */
-    bool
-    fail_stack_allocation();
-
-  private:
-    /**
-     * @brief Compute a hash over the current stack.
-     */
-    struct _stack_hasher
-    {
-        size_t
-        operator()(const std::vector<uintptr_t>& key) const
-        {
-            size_t hash_value = 0;
-            for (const auto value : key) {
-                hash_value ^= std::hash<uintptr_t>{}(value);
-            }
-            return hash_value;
-        }
-    };
-
-    /**
-     * @brief Determine if this allocation path is new.
-     * If it is new, then fail the allocation, add it to the set of known
-     * allocation paths and return true.
-     */
-    bool
-    is_new_stack();
-
-    /**
-     * @brief Write the current stack to the log file.
-     */
-    void
-    log_stack_trace(const std::vector<uintptr_t>& canonical_stack, const std::vector<uintptr_t>& stack);
-
-    /**
-     * @brief Load the list of known allocation paths from the log file.
-     */
-    void
-    load_allocation_log();
-
-    /**
-     * @brief The base address of the current process.
-     */
-    uintptr_t _base_address = 0;
-
-    /**
-     * @brief The iteration number of the current test pass.
-     */
-    size_t _iteration = 0;
-
-    /**
-     * @brief The log file for allocations that have been failed.
-     */
-    std::ofstream _log_file;
-
-    /**
-     * @brief The set of known allocation paths.
-     */
-    std::unordered_set<std::vector<uintptr_t>, _stack_hasher> _allocation_hash;
-
-    /**
-     * @brief The mutex to protect the set of known allocation paths.
-     */
-    std::mutex _mutex;
-
-    size_t _stack_depth;
-    std::vector<std::string> _last_failure_stack;
-} ebpf_low_memory_test_t;

libs/platform/user/ebpf_platform_user.cpp

@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft Corporation
 // SPDX-License-Identifier: MIT
 
+#include "ebpf_fault_injection.h"
 #include "ebpf_leak_detector.h"
-#include "ebpf_low_memory_test.h"
 #include "ebpf_symbol_decoder.h"
 #include "ebpf_utilities.h"
 
@@ -33,14 +33,13 @@ int32_t _ebpf_platform_initiate_count = 0;
 extern "C" bool ebpf_fuzzing_enabled = false;
 extern "C" size_t ebpf_fuzzing_memory_limit = MAXSIZE_T;
 
-std::unique_ptr<ebpf_low_memory_test_t> _ebpf_low_memory_test_ptr;
 ebpf_leak_detector_ptr _ebpf_leak_detector_ptr;
 
 /**
- * @brief Environment variable to enable low memory testing.
+ * @brief Environment variable to enable fault injection testing.
  *
  */
-#define EBPF_LOW_MEMORY_SIMULATION_ENVIRONMENT_VARIABLE_NAME "EBPF_LOW_MEMORY_SIMULATION"
+#define EBPF_FAULT_INJECTION_SIMULATION_ENVIRONMENT_VARIABLE_NAME "EBPF_FAULT_INJECTION_SIMULATION"
 #define EBPF_MEMORY_LEAK_DETECTION_ENVIRONMENT_VARIABLE_NAME "EBPF_MEMORY_LEAK_DETECTION"
 
 // Thread pool related globals.
@@ -310,14 +309,16 @@ ebpf_platform_initiate()
     try {
         _ebpf_platform_maximum_group_count = GetMaximumProcessorGroupCount();
         _ebpf_platform_maximum_processor_count = GetMaximumProcessorCount(ALL_PROCESSOR_GROUPS);
-        auto low_memory_stack_depth =
-            _get_environment_variable_as_size_t(EBPF_LOW_MEMORY_SIMULATION_ENVIRONMENT_VARIABLE_NAME);
+        auto fault_injection_stack_depth =
+            _get_environment_variable_as_size_t(EBPF_FAULT_INJECTION_SIMULATION_ENVIRONMENT_VARIABLE_NAME);
         auto leak_detector = _get_environment_variable_as_bool(EBPF_MEMORY_LEAK_DETECTION_ENVIRONMENT_VARIABLE_NAME);
-        if (low_memory_stack_depth || leak_detector) {
+        if (fault_injection_stack_depth || leak_detector) {
             _ebpf_symbol_decoder_initialize();
         }
-        if (low_memory_stack_depth && !_ebpf_low_memory_test_ptr) {
-            _ebpf_low_memory_test_ptr = std::make_unique<ebpf_low_memory_test_t>(low_memory_stack_depth);
+        if (fault_injection_stack_depth && !ebpf_fault_injection_is_enabled()) {
+            if (ebpf_fault_injection_initialize(fault_injection_stack_depth) != EBPF_SUCCESS) {
+                return EBPF_NO_MEMORY;
+            }
             // Set flag to remove some asserts that fire from incorrect client behavior.
             ebpf_fuzzing_enabled = true;
         }
@@ -373,12 +374,6 @@ ebpf_get_code_integrity_state(_Out_ ebpf_code_integrity_state_t* state)
     EBPF_RETURN_RESULT(EBPF_SUCCESS);
 }
 
-bool
-ebpf_low_memory_test_in_progress()
-{
-    return _ebpf_low_memory_test_ptr != nullptr;
-}
-
 __drv_allocatesMem(Mem) _Must_inspect_result_ _Ret_writes_maybenull_(size) void* ebpf_allocate(size_t size)
 {
     ebpf_assert(size);
@@ -386,9 +381,8 @@ __drv_allocatesMem(Mem) _Must_inspect_result_ _Ret_writes_maybenull_(size) void*
         return nullptr;
     }
 
-    if (_ebpf_low_memory_test_ptr && _ebpf_low_memory_test_ptr->fail_stack_allocation()) {
+    if (ebpf_fault_injection_inject_fault())
         return nullptr;
-    }
 
     void* memory;
     memory = calloc(size, 1);
@@ -418,9 +412,8 @@ __drv_allocatesMem(Mem) _Must_inspect_result_ _Ret_writes_maybenull_(new_size) v
         return nullptr;
     }
 
-    if (_ebpf_low_memory_test_ptr && _ebpf_low_memory_test_ptr->fail_stack_allocation()) {
+    if (ebpf_fault_injection_inject_fault())
         return nullptr;
-    }
 
     void* p = realloc(memory, new_size);
     if (p && (new_size > old_size))
@@ -458,9 +451,8 @@ __drv_allocatesMem(Mem) _Must_inspect_result_
         return nullptr;
     }
 
-    if (_ebpf_low_memory_test_ptr && _ebpf_low_memory_test_ptr->fail_stack_allocation()) {
+    if (ebpf_fault_injection_inject_fault())
         return nullptr;
-    }
 
     void* memory = _aligned_malloc(size, EBPF_CACHE_LINE_SIZE);
     if (memory) {

libs/platform/user/ebpf_symbol_decoder.h

@@ -6,6 +6,8 @@
 
 #include <DbgHelp.h>
 #include <optional>
+#include <string>
+#include <vector>
 
 inline ebpf_result_t
 _ebpf_symbol_decoder_initialize()

libs/platform/user/kernel_um.h

@@ -487,7 +487,7 @@ extern "C"
     VOID
     RtlMapGenericMask(_Inout_ PACCESS_MASK AccessMask, _In_ const GENERIC_MAPPING* GenericMapping);
 
-    unsigned long 
+    unsigned long
     RtlLengthSid(_In_ PSID Sid);
 
     NTSTATUS

libs/platform/user/platform_user.vcxproj

@@ -30,9 +30,9 @@
     <ClCompile Include="..\ebpf_serialize.c" />
     <ClCompile Include="..\ebpf_state.c" />
     <ClCompile Include="..\ebpf_trampoline.c" />
+    <ClCompile Include="ebpf_fault_injection.cpp" />
     <ClCompile Include="ebpf_handle_user.c" />
     <ClCompile Include="ebpf_leak_detector.cpp" />
-    <ClCompile Include="ebpf_low_memory_test.cpp" />
     <ClCompile Include="ebpf_native_user.c" />
     <ClCompile Include="ebpf_platform_user.cpp" />
     <ClCompile Include="kernel_um.cpp" />
@@ -47,8 +47,8 @@
     <ClInclude Include="..\ebpf_platform.h" />
     <ClInclude Include="..\ebpf_ring_buffer.h" />
     <ClInclude Include="..\ebpf_state.h" />
+    <ClInclude Include="ebpf_fault_injection.h" />
     <ClInclude Include="ebpf_leak_detector.h" />
-    <ClInclude Include="ebpf_low_memory_test.h" />
     <ClInclude Include="ebpf_rundown.h" />
     <ClInclude Include="ebpf_symbol_decoder.h" />
     <ClInclude Include="framework.h" />

libs/platform/user/platform_user.vcxproj.filters

@@ -82,9 +82,6 @@
     <ClCompile Include="kernel_um.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="ebpf_low_memory_test.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="ebpf_leak_detector.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -94,6 +91,9 @@
     <ClCompile Include="ebpf_leak_detector.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="ebpf_fault_injection.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\ebpf_epoch.h">
@@ -132,9 +132,6 @@
     <ClInclude Include="kernel_um.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="ebpf_low_memory_test.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
     <ClInclude Include="ebpf_leak_detector.h">
       <Filter>Header Files</Filter>
     </ClInclude>
@@ -144,5 +141,8 @@
     <ClInclude Include="ebpf_rundown.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="ebpf_fault_injection.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

netebpfext/user/fwp_um.cpp

@@ -228,7 +228,7 @@ void static _allocate_and_initialize_connection_request(
     ebpf_assert(_fwp_um_connect_request == nullptr);
     _fwp_um_connect_request = (FWPS_CONNECT_REQUEST0*)ebpf_allocate(sizeof(FWPS_CONNECT_REQUEST0));
     if (_fwp_um_connect_request == nullptr) {
-        // Most likely we are under low memory simulation. Return.
+        // Most likely we are under fault injection simulation. Return.
         return;
     }
 
@@ -357,7 +357,7 @@ _fwp_engine::test_cgroup_inet6_connect(_In_ fwp_classify_parameters_t* parameter
     incoming_value2[FWPS_FIELD_ALE_AUTH_CONNECT_V6_ALE_USER_ID].value.byteBlob = &parameters->user_id;
     incoming_value2[FWPS_FIELD_ALE_AUTH_CONNECT_V6_IP_LOCAL_INTERFACE].value.uint64 = &parameters->interface_luid;
     incoming_value2[FWPS_FIELD_ALE_AUTH_CONNECT_V6_FLAGS].value.uint32 = parameters->reauthorization_flag;
-    
+
     action = test_callout(
         FWPS_LAYER_ALE_AUTH_CONNECT_V6, FWPM_LAYER_ALE_AUTH_CONNECT_V6, EBPF_DEFAULT_SUBLAYER, incoming_value2);
 

scripts/Test-FaultInjection.ps1

@@ -1,7 +1,7 @@
 # Copyright (c) Microsoft Corporation
 # SPDX-License-Identifier: MIT
 # This script is used as part of the systematic testing of the
-# low memory handling.
+# failure handling.
 # First it will create a list of all the tests in the test binary.
 # Second it will execute each test in the test binary.
 # Third it will check the output of each test to see if it passed or failed.
@@ -14,7 +14,7 @@ param ($TestProgram, $StackDepth)
 # Gather list of all possible tests
 $tests = & $TestProgram "--list-tests" "--verbosity=quiet"
 
-$env:EBPF_LOW_MEMORY_SIMULATION = $StackDepth
+$env:EBPF_FAULT_INJECTION_SIMULATION = $StackDepth
 $env:EBPF_ENABLE_WER_REPORT = "yes"
 
 Set-Content -Path ($TestProgram +".passed.log") ""

scripts/setup_build/setup_build.vcxproj

@@ -95,7 +95,7 @@ popd
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
     </CopyFileToFolders>
-    <CopyFileToFolders Include="..\Test-LowMemory.ps1">
+    <CopyFileToFolders Include="..\Test-FaultInjection.ps1">
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
     </CopyFileToFolders>

scripts/setup_build/setup_build.vcxproj.filters

@@ -67,7 +67,7 @@
     <CopyFileToFolders Include="..\test_execution.json">
       <Filter>Source Files</Filter>
     </CopyFileToFolders>
-    <CopyFileToFolders Include="..\Test-LowMemory.ps1" />
+    <CopyFileToFolders Include="..\Test-FaultInjection.ps1" />
   </ItemGroup>
   <ItemGroup>
     <None Include="..\pre-commit">

tests/end_to_end/netsh_test.cpp

@@ -7,6 +7,7 @@
 #include "bpf/libbpf.h"
 #pragma warning(pop)
 #include "ebpf_epoch.h"
+#include "ebpf_fault_injection.h"
 #include "netsh_test_helper.h"
 #include "platform.h"
 #include "test_helper.hpp"
@@ -32,12 +33,9 @@ class _test_helper_netsh
 
 _test_helper_netsh::_test_helper_netsh() { _ebpf_netsh_objects.clear(); }
 
-extern bool
-ebpf_low_memory_test_in_progress();
-
 _test_helper_netsh::~_test_helper_netsh()
 {
-    if (ebpf_low_memory_test_in_progress()) {
+    if (ebpf_fault_injection_is_enabled()) {
         for (auto& object : _ebpf_netsh_objects) {
             bpf_object__close(object);
         }

tests/end_to_end/test_helper.cpp

@@ -8,6 +8,7 @@
 #include "catch_wrapper.hpp"
 #include "ebpf_async.h"
 #include "ebpf_core.h"
+#include "ebpf_fault_injection.h"
 #include "ebpf_platform.h"
 #include "hash.h"
 #include "helpers.h"
@@ -729,13 +730,10 @@ set_native_module_failures(bool expected)
     _expect_native_module_load_failures = expected;
 }
 
-extern bool
-ebpf_low_memory_test_in_progress();
-
 bool
 get_native_module_failures()
 {
-    return _expect_native_module_load_failures || ebpf_low_memory_test_in_progress();
+    return _expect_native_module_load_failures || ebpf_fault_injection_is_enabled();
 }
 
 _Must_inspect_result_ ebpf_result_t

tests/libs/common/passed_test_log.h

@@ -14,7 +14,7 @@
  * @brief A Catch2 reporter that logs the name of each test that passes.
  * This is used to generate a list of tests that passed in the last run in a
  * file that can be used to filter the set of tests that are run in the next
- * run. This is consumed by the Test-LowMemory.ps1 script.
+ * run. This is consumed by the Test-FaultInjection.ps1 script.
  */
 class _passed_test_log : public Catch::EventListenerBase
 {

tests/netebpfext_unit/netebpfext_unit.cpp

@@ -471,9 +471,9 @@ TEST_CASE("sock_addr_invoke", "[netebpfext]")
     // Classify operations that should be allowed.
     client_context.sock_addr_action = SOCK_ADDR_TEST_ACTION_PERMIT;
     client_context.validate_sock_addr_entries = true;
-    
+
     parameters.reauthorization_flag = FWP_CONDITION_FLAG_IS_REAUTHORIZE;
-    
+
     result = helper.test_cgroup_inet4_recv_accept(&parameters);
     REQUIRE(result == FWP_ACTION_PERMIT);
 

tests/unit/wer_report_test_wrapper.cpp

@@ -41,7 +41,7 @@ AddVectoredExceptionHandler_test(_In_ unsigned long first, _In_ PVECTORED_EXCEPT
 
 unsigned long SetThreadStackGuarantee_test_stack_size_in_bytes = 0;
 
-// Use BOOL to pass "SetThreadStackGuarantee" 
+// Use BOOL to pass "SetThreadStackGuarantee"
 // defined in windows "processthreadapi.h" file
 BOOL
 SetThreadStackGuarantee_test(_Inout_ unsigned long* stack_size_in_bytes)
@@ -50,8 +50,7 @@ SetThreadStackGuarantee_test(_Inout_ unsigned long* stack_size_in_bytes)
     return TRUE;
 }
 
-unsigned long
-WINAPI
+unsigned long WINAPI
 RemoveVectoredExceptionHandler_test(_In_ void* handle)
 {
     UNREFERENCED_PARAMETER(handle);