Bug 1588245 - Collect the assembly pattern of a target function on detour failure. r=mhowell

Many instances of the launcher failure ping indicate hooking NtMapViewOfSection
or LdrLoadDll failed.  This is most likely caused by a third-party application
applying a hook onto the same target earlier than we do.

This patch is to add a new field "detour_orig_bytes" in the laucnher failure ping
to collect the first sixteen bytes of a detour target function.  With this,
we can know whether those detour failures were caused by a third-party hook or not,
and if yes, what was the actual binary pattern.

Differential Revision: https://phabricator.services.mozilla.com/D89836

browser/app/winlauncher/DllBlocklistInit.cpp

@@ -70,13 +70,13 @@ static LauncherVoidResultWithLineInfo InitializeDllBlocklistOOPInternal(
       aChildProcess, intcpt, "NtMapViewOfSection",
       &freestanding::patched_NtMapViewOfSection);
   if (!ok) {
-    return LAUNCHER_ERROR_GENERIC();
+    return LAUNCHER_ERROR_GENERIC_WITH_DETOUR_ERROR(intcpt.GetLastError());
   }
 
   ok = freestanding::stub_LdrLoadDll.SetDetour(
       aChildProcess, intcpt, "LdrLoadDll", &freestanding::patched_LdrLoadDll);
   if (!ok) {
-    return LAUNCHER_ERROR_GENERIC();
+    return LAUNCHER_ERROR_GENERIC_WITH_DETOUR_ERROR(intcpt.GetLastError());
   }
 
   // Because aChildProcess has just been created in a suspended state, its

browser/app/winlauncher/ErrorHandler.cpp

@@ -579,6 +579,21 @@ static bool PrepPing(const PingThreadContext& aContext, const std::wstring& aId,
 
   aJson.IntProperty("source_line", aContext.mLauncherError.mLine);
   aJson.IntProperty("hresult", aContext.mLauncherError.mError.AsHResult());
+
+#  if defined(NIGHTLY_BUILD)
+  if (aContext.mLauncherError.mDetourError.isSome()) {
+    static const char* kHexMap = "0123456789abcdef";
+    char hexStr[sizeof(mozilla::DetourError::mOrigBytes) * 2 + 1];
+    int cnt = 0;
+    for (uint8_t byte : aContext.mLauncherError.mDetourError->mOrigBytes) {
+      hexStr[cnt++] = kHexMap[(byte >> 4) & 0x0f];
+      hexStr[cnt++] = kHexMap[byte & 0x0f];
+    }
+    hexStr[cnt] = 0;
+    aJson.StringProperty("detour_orig_bytes", hexStr);
+  }
+#  endif  // defined(NIGHTLY_BUILD)
+
   aJson.EndObject();
 
 #  if !defined(__MINGW32__)

mozglue/misc/interceptor/PatcherDetour.h

@@ -128,6 +128,10 @@ class WindowsDllDetourPatcher final
   using PrimitiveT = WindowsDllDetourPatcherPrimitive<MMPolicyT>;
   Maybe<DetourFlags> mFlags;
 
+#if defined(NIGHTLY_BUILD)
+  Maybe<DetourError> mLastError;
+#endif  // defined(NIGHTLY_BUILD)
+
  public:
   template <typename... Args>
   explicit WindowsDllDetourPatcher(Args&&... aArgs)
@@ -140,6 +144,10 @@ class WindowsDllDetourPatcher final
   WindowsDllDetourPatcher& operator=(const WindowsDllDetourPatcher&) = delete;
   WindowsDllDetourPatcher& operator=(WindowsDllDetourPatcher&&) = delete;
 
+#if defined(NIGHTLY_BUILD)
+  const Maybe<DetourError>& GetLastError() const { return mLastError; }
+#endif  // defined(NIGHTLY_BUILD)
+
   void Clear() {
     if (!this->mVMPolicy.ShouldUnhookUponDestruction()) {
       return;
@@ -898,17 +906,29 @@ class WindowsDllDetourPatcher final
       return;
     }
 
-    auto clearInstanceOnFailure = MakeScopeExit([aOutTramp, &tramp]() -> void {
+    auto clearInstanceOnFailure = MakeScopeExit([this, aOutTramp, &tramp,
+                                                 &origBytes]() -> void {
       // *aOutTramp is not set until CreateTrampoline has completed
       // successfully, so we can use that to check for success.
       if (*aOutTramp) {
         return;
       }
 
-      // Clear the instance pointer so that we don't try to reset a nonexistent
+      // Clear the instance pointer so that we don't try to reset a
-      // hook.
+      // nonexistent hook.
       tramp.Rewind();
       tramp.WriteEncodedPointer(nullptr);
+
+#if defined(NIGHTLY_BUILD)
+      origBytes.Rewind();
+      mLastError = Some(DetourError());
+      size_t bytesToCapture = std::min(
+          ArrayLength(mLastError->mOrigBytes),
+          static_cast<size_t>(PrimitiveT::GetWorstCaseRequiredBytesToPatch()));
+      for (size_t i = 0; i < bytesToCapture; ++i) {
+        mLastError->mOrigBytes[i] = origBytes[i];
+      }
+#endif  // defined(NIGHTLY_BUILD)
     });
 
     tramp.WritePointer(origBytes.AsEncodedPtr());

mozglue/misc/nsWindowsDllInterceptor.h

@@ -361,6 +361,12 @@ class WindowsDllInterceptor final
     // NB: We intentionally leak mModule
   }
 
+#if defined(NIGHTLY_BUILD)
+  const Maybe<DetourError>& GetLastError() const {
+    return mDetourPatcher.GetLastError();
+  }
+#endif  // defined(NIGHTLY_BUILD)
+
   constexpr static uint32_t GetWorstCaseRequiredBytesToPatch() {
     return WindowsDllDetourPatcherPrimitive<
         typename VMPolicy::MMPolicyT>::GetWorstCaseRequiredBytesToPatch();

toolkit/components/telemetry/docs/data/launcher-process-failure-ping.rst

@@ -48,7 +48,10 @@ Structure:
         // The line number of the source file where the error was raised
         "source_line": <int>,
         // The HRESULT error code of the error that was raised
-        "hresult": <int>
+        "hresult": <int>,
+        // First sixteen bytes of a function that we failed to hook (Nightly-only).
+        // This field is added only on detour failures.
+        "detour_orig_bytes": <string>
       },
       "security": {
         // A list of names of installed antivirus products
@@ -87,6 +90,7 @@ Structure:
 Version History
 ~~~~~~~~~~~~~~~
 
+- Firefox 82: Added ``detour_orig_bytes`` (`bug 1588245 <https://bugzilla.mozilla.org/show_bug.cgi?id=1588245>`_).
 - Firefox 82: Added ``process_type`` (`bug 1630444 <https://bugzilla.mozilla.org/show_bug.cgi?id=1630444>`_).
 - Firefox 71: Added ``is_admin_without_uac`` (`bug 1567605 <https://bugzilla.mozilla.org/show_bug.cgi?id=1567605>`_).
 - Firefox 67: Initial release (`bug 1460433 <https://bugzilla.mozilla.org/show_bug.cgi?id=1460433>`_).

widget/windows/WinHeaderOnlyUtils.h

@@ -204,6 +204,15 @@ class WindowsError final {
   HRESULT mHResult;
 };
 
+#if defined(NIGHTLY_BUILD)
+struct DetourError {
+  // We have a 16-bytes buffer, but only minimum bytes to detour per
+  // architecture are copied.  See CreateTrampoline in PatcherDetour.h.
+  uint8_t mOrigBytes[16];
+  DetourError() : mOrigBytes{} {}
+};
+#endif  // defined(NIGHTLY_BUILD)
+
 template <typename T>
 using WindowsErrorResult = Result<T, WindowsError>;
 
@@ -211,9 +220,22 @@ struct LauncherError {
   LauncherError(const char* aFile, int aLine, WindowsError aWin32Error)
       : mFile(aFile), mLine(aLine), mError(aWin32Error) {}
 
+#if defined(NIGHTLY_BUILD)
+  template <typename... Args>
+  LauncherError(const char* aFile, int aLine, WindowsError aWin32Error,
+                const Maybe<DetourError>& aDetourError)
+      : mFile(aFile),
+        mLine(aLine),
+        mError(aWin32Error),
+        mDetourError(aDetourError) {}
+#endif  // defined(NIGHTLY_BUILD)
+
   const char* mFile;
   int mLine;
   WindowsError mError;
+#if defined(NIGHTLY_BUILD)
+  Maybe<DetourError> mDetourError;
+#endif  // defined(NIGHTLY_BUILD)
 
   bool operator==(const LauncherError& aOther) const {
     return mError == aOther.mError;
@@ -260,6 +282,16 @@ using LauncherVoidResultWithLineInfo = LauncherResultWithLineInfo<Ok>;
     ::mozilla::Err(::mozilla::LauncherError( \
         __FILE__, __LINE__, ::mozilla::WindowsError::CreateGeneric()))
 
+#  if defined(NIGHTLY_BUILD)
+#    define LAUNCHER_ERROR_GENERIC_WITH_DETOUR_ERROR(...)               \
+      ::mozilla::Err(::mozilla::LauncherError(                          \
+          __FILE__, __LINE__, ::mozilla::WindowsError::CreateGeneric(), \
+          __VA_ARGS__))
+#  else
+#    define LAUNCHER_ERROR_GENERIC_WITH_DETOUR_ERROR(...) \
+      LAUNCHER_ERROR_GENERIC()
+#  endif  // defined(NIGHTLY_BUILD)
+
 #  define LAUNCHER_ERROR_FROM_WIN32(err)     \
     ::mozilla::Err(::mozilla::LauncherError( \
         __FILE__, __LINE__, ::mozilla::WindowsError::FromWin32Error(err)))
@@ -285,6 +317,8 @@ using LauncherVoidResultWithLineInfo = LauncherResultWithLineInfo<Ok>;
 #  define LAUNCHER_ERROR_GENERIC() \
     ::mozilla::Err(::mozilla::WindowsError::CreateGeneric())
 
+#  define LAUNCHER_ERROR_GENERIC_WITH_DETOUR_ERROR(...) LAUNCHER_ERROR_GENERIC()
+
 #  define LAUNCHER_ERROR_FROM_WIN32(err) \
     ::mozilla::Err(::mozilla::WindowsError::FromWin32Error(err))