Bug 1349444: Suppress stack walking in LdrUnloadDll. r=mstange,aklotz,froydnj

mozglue/misc/StackWalk.cpp

@@ -191,6 +191,11 @@ StackWalkInitCriticalAddress()
 #include "mozilla/Atomics.h"
 #include "mozilla/StackWalk_windows.h"
 
+#ifdef MOZ_STATIC_JS // The standalone SM build lacks the interceptor headers.
+#include "nsWindowsDllInterceptor.h"
+#define STACKWALK_HAS_DLL_INTERCEPTOR
+#endif
+
 #include <imagehlp.h>
 // We need a way to know if we are building for WXP (or later), as if we are, we
 // need to use the newer 64-bit APIs. API_VERSION_NUMBER seems to fit the bill.
@@ -276,6 +281,22 @@ UnregisterJitCodeRegion(uint8_t* aStart, size_t aSize)
   sJitCodeRegionStart = nullptr;
   sJitCodeRegionSize = 0;
 }
+
+#ifdef STACKWALK_HAS_DLL_INTERCEPTOR
+static WindowsDllInterceptor NtDllInterceptor;
+
+typedef NTSTATUS (NTAPI *LdrUnloadDll_func)(HMODULE module);
+static LdrUnloadDll_func stub_LdrUnloadDll;
+
+static NTSTATUS NTAPI
+patched_LdrUnloadDll(HMODULE module)
+{
+  // Prevent the stack walker from suspending this thread when LdrUnloadDll
+  // holds the RtlLookupFunctionEntry lock.
+  AutoSuppressStackWalking suppress;
+  return stub_LdrUnloadDll(module);
+}
+#endif // STACKWALK_HAS_DLL_INTERCEPTOR
 #endif // _M_AMD64
 
 // Routine to print an error message to standard error.
@@ -362,6 +383,13 @@ EnsureWalkThreadReady()
   stackWalkThread = nullptr;
   readyEvent = nullptr;
 
+#if defined(_M_AMD64) && defined(STACKWALK_HAS_DLL_INTERCEPTOR)
+  NtDllInterceptor.Init("ntdll.dll");
+  NtDllInterceptor.AddHook("LdrUnloadDll",
+                           reinterpret_cast<intptr_t>(patched_LdrUnloadDll),
+                           (void**)&stub_LdrUnloadDll);
+#endif
+
   InitializeDbgHelpCriticalSection();
 
   return walkThreadReady = true;

toolkit/xre/test/win/TestDllInterceptor.cpp

@@ -236,6 +236,13 @@ bool TestNtQueryFullAttributesFile(void* aFunc)
   return patchedNtQueryFullAttributesFile(0, 0) != 0;
 }
 
+bool TestLdrUnloadDll(void* aFunc)
+{
+  typedef NTSTATUS (NTAPI *LdrUnloadDllType)(HMODULE);
+  auto patchedLdrUnloadDll = reinterpret_cast<LdrUnloadDllType>(aFunc);
+  return patchedLdrUnloadDll(0) != 0;
+}
+
 bool TestSetUnhandledExceptionFilter(void* aFunc)
 {
   auto patchedSetUnhandledExceptionFilter =
@@ -463,6 +470,7 @@ int main()
       TestHook(TestGetOpenFileNameW, "comdlg32.dll", "GetOpenFileNameW") &&
 #ifdef _M_X64
       TestHook(TestGetKeyState, "user32.dll", "GetKeyState") &&    // see Bug 1316415
+      TestHook(TestLdrUnloadDll, "ntdll.dll", "LdrUnloadDll") &&
 #endif
       MaybeTestHook(ShouldTestTipTsf(), TestProcessCaretEvents, "tiptsf.dll", "ProcessCaretEvents") &&
 #ifdef _M_IX86