Bug 1865795 - Collect single-step data upon utility process LoadLibraryOrCrash failure. r=rkraesig

In order to investigate LoadLibraryW failures in bug 1851889, we make
LoadLibraryOrCrash collect ntdll.dll single step data in utility
processes. The data will be available in the stack of the crashing
thread in nightly and early beta builds.

Single-step data collection only occurs after a first failure to load
the library, when we would be about to crash anyway. This ensures that
the patch cannot introduce extra unstability to LoadLibraryOrCrash.

Single-stepping through all ntdll instructions would record too many
steps, so we add the capability to record only call and ret
instructions. We apply this new capability for the single-step recording
of LoadLibraryW. Thanks to the detection of error state changes, this
should help us identify what path leads to a 0x241 last error in
bug 1851889.

Depends on D194203

Differential Revision: https://phabricator.services.mozilla.com/D194204

ipc/glue/UtilityProcessImpl.cpp

@@ -10,6 +10,7 @@
 
 #if defined(XP_WIN)
 #  include "nsExceptionHandler.h"
+#  include "mozilla/WindowsDiagnostics.h"
 #endif
 
 #if defined(XP_WIN) && defined(MOZ_SANDBOX)
@@ -70,7 +71,29 @@ void UtilityProcessImpl::LoadLibraryOrCrash(LPCWSTR aLib) {
       break;
   }
 
+#  if defined(MOZ_DIAGNOSTIC_ASSERT_ENABLED) && defined(_M_X64)
+  static constexpr int kMaxStepsNtdll = 0x1800;
+  static constexpr int kMaxErrorStatesNtdll = 0x200;
+  using NtdllSingleStepData =
+      ModuleSingleStepData<kMaxStepsNtdll, kMaxErrorStatesNtdll>;
+
+  HMODULE module{};
+  nsresult rv =
+      CollectModuleSingleStepData<kMaxStepsNtdll, kMaxErrorStatesNtdll,
+                                  InstructionFilter::CALL_RET>(
+          L"ntdll.dll", [&module, aLib]() { module = ::LoadLibraryW(aLib); },
+          [&module](const NtdllSingleStepData& aData) {
+            // Crashing here gives access to the single step data on stack
+            MOZ_DIAGNOSTIC_ASSERT(
+                module, "Unable to preload module: collected single-step data");
+          });
+  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv),
+                        "Failed to collect single step data for LoadLibraryW");
+  // If we reach this point then somehow the single-stepped call succeeded and
+  // we can proceed
+#  else
   MOZ_CRASH_UNSAFE_PRINTF("Unable to preload module: 0x%lx", err);
+#  endif  // MOZ_DIAGNOSTIC_ASSERT_ENABLED && _M_X64
 }
 #endif  // defined(XP_WIN)
 

mozglue/misc/WindowsDiagnostics.h

@@ -149,6 +149,44 @@ struct ModuleSingleStepState {
         mData{} {}
 };
 
+enum class InstructionFilter {
+  ALL = 0,
+  CALL_RET = 1,
+};
+
+template <InstructionFilter Filter>
+inline bool ShouldRecordInstruction(const uint8_t* aInstructionPointer) {
+  if constexpr (Filter == InstructionFilter::ALL) {
+    return true;
+  }
+  // Note: This filter does *not* exhaustively identify all call/ret
+  //       instructions. For example, prefixed instructions are not
+  //       currently recognized.
+  else if constexpr (Filter == InstructionFilter::CALL_RET) {
+    auto firstByte = aInstructionPointer[0];
+    // E8: CALL rel. Call near, relative.
+    if (firstByte == 0xe8) {
+      return true;
+    }
+    // FF /2: CALL r.	Call near, absolute indirect.
+    else if (firstByte == 0xff) {
+      auto secondByte = aInstructionPointer[1];
+      if ((secondByte & 0x38) == 0x10) {
+        return true;
+      }
+    }
+    // C3: RET. Near return.
+    else if (firstByte == 0xc3) {
+      return true;
+    }
+    // C2: RET imm. Near return and pop imm bytes.
+    else if (firstByte == 0xc2) {
+      return true;
+    }
+  }
+  return false;
+}
+
 // This function runs aCallbackToRun instruction by instruction, recording
 // information about the paths taken within a specific module given by
 // aModulePath. It then calls aPostCollectionCallback with the collected data.
@@ -156,15 +194,17 @@ struct ModuleSingleStepState {
 // We store stores the collected data in stack, so that it is available in
 // crash reports in case we decide to crash from aPostCollectionCallback.
 // Remember to carefully estimate the stack usage when choosing NMaxSteps and
-// NMaxErrorStates.
+// NMaxErrorStates. Consider using an InstructionFilter if you need to reduce
+// the number of steps that get recorded.
 //
 // This function is typically useful on known-to-crash paths, where we can
 // replace the crash by a new single-stepped attempt at doing the operation
 // that just failed. If the operation fails while single-stepped, we'll be able
 // to produce a crash report that contains single step data, which may prove
 // useful to understand why the operation failed.
-template <int NMaxSteps, int NMaxErrorStates, typename CallbackToRun,
-          typename PostCollectionCallback>
+template <int NMaxSteps, int NMaxErrorStates,
+          InstructionFilter Filter = InstructionFilter::ALL,
+          typename CallbackToRun, typename PostCollectionCallback>
 nsresult CollectModuleSingleStepData(
     const wchar_t* aModulePath, CallbackToRun aCallbackToRun,
     PostCollectionCallback aPostCollectionCallback) {
@@ -192,27 +232,31 @@ nsresult CollectModuleSingleStepData(
         // Record data for the current step, if in module
         if (state.mModuleStart <= instructionPointer &&
             instructionPointer < state.mModuleEnd) {
-          // We record the instruction pointer
-          if (state.mSteps < NMaxSteps) {
-            state.mData.mStepsLog[state.mSteps] =
-                static_cast<uint32_t>(instructionPointer - state.mModuleStart);
-          }
-
-          // We record changes in the error state
-          auto currentErrorState{WinErrorState::Get()};
-          if (currentErrorState != state.mLastRecordedErrorState) {
-            state.mLastRecordedErrorState = currentErrorState;
-
-            if (state.mErrorStates < NMaxErrorStates) {
-              state.mData.mErrorStatesLog[state.mErrorStates] =
-                  currentErrorState;
-              state.mData.mStepsAtErrorState[state.mErrorStates] = state.mSteps;
+          if (ShouldRecordInstruction<Filter>(
+                  reinterpret_cast<uint8_t*>(instructionPointer))) {
+            // We record the instruction pointer
+            if (state.mSteps < NMaxSteps) {
+              state.mData.mStepsLog[state.mSteps] = static_cast<uint32_t>(
+                  instructionPointer - state.mModuleStart);
             }
 
-            ++state.mErrorStates;
-          }
+            // We record changes in the error state
+            auto currentErrorState{WinErrorState::Get()};
+            if (currentErrorState != state.mLastRecordedErrorState) {
+              state.mLastRecordedErrorState = currentErrorState;
 
-          ++state.mSteps;
+              if (state.mErrorStates < NMaxErrorStates) {
+                state.mData.mErrorStatesLog[state.mErrorStates] =
+                    currentErrorState;
+                state.mData.mStepsAtErrorState[state.mErrorStates] =
+                    state.mSteps;
+              }
+
+              ++state.mErrorStates;
+            }
+
+            ++state.mSteps;
+          }
         }
 
         // Continue single-stepping