Bug 575515 - Fix for deadlocks in windows modal loop message processing under heavy js load. r=matspal, a=betaN

widget/src/windows/nsAppShell.cpp

@@ -22,6 +22,7 @@
  * Contributor(s):
  *   Michael Lowe <michael.lowe@bigfoot.com>
  *   Darin Fisher <darin@meer.net>
+ *   Jim Mathies <jmathies@mozilla.com>
  *
  * Alternatively, the contents of this file may be used under the terms of
  * either the GNU General Public License Version 2 or later (the "GPL"), or
@@ -48,6 +49,9 @@
 #include <windows.h> 
 #include <tlhelp32.h> 
 
+const PRUnichar* kAppShellEventId = L"nsAppShell:EventID";
+const PRUnichar* kTaskbarButtonEventId = L"TaskbarButtonCreated";
+
 #ifdef WINCE
 BOOL WaitMessage(VOID)
 {
@@ -140,10 +144,10 @@ nsAppShell::Init()
 #endif
 
   if (!sMsgId)
-    sMsgId = RegisterWindowMessageW(L"nsAppShell:EventID");
+    sMsgId = RegisterWindowMessageW(kAppShellEventId);
 
 #if MOZ_WINSDK_TARGETVER >= MOZ_NTDDI_WIN7
-  sTaskbarButtonCreatedMsg = ::RegisterWindowMessageW(L"TaskbarButtonCreated");
+  sTaskbarButtonCreatedMsg = ::RegisterWindowMessageW(kTaskbarButtonEventId);
   NS_ASSERTION(sTaskbarButtonCreatedMsg, "Could not register taskbar button creation message");
 
   // Global app registration id for Win7 and up. See
@@ -176,7 +180,6 @@ nsAppShell::Init()
   return nsBaseAppShell::Init();
 }
 
-
 /**
  * This is some temporary code to keep track of where in memory dlls are
  * loaded. This is useful in case someone calls into a dll that has been
@@ -262,11 +265,45 @@ nsAppShell::Run(void)
 
 #endif
 
+void
+nsAppShell::DoProcessMoreGeckoEvents()
+{
+  // Called by nsBaseAppShell's NativeEventCallback() after it has finished
+  // processing pending gecko events and there are still gecko events pending
+  // for the thread. (This can happen if NS_ProcessPendingEvents reached it's
+  // starvation timeout limit.) The default behavior in nsBaseAppShell is to
+  // call ScheduleNativeEventCallback to post a follow up native event callback
+  // message. This triggers an additional call to NativeEventCallback for more
+  // gecko event processing.
+
+  // There's a deadlock risk here with certain internal Windows modal loops. In
+  // our dispatch code, we prioritize messages so that input is handled first.
+  // However Windows modal dispatch loops often prioritize posted messages. If
+  // we find ourselves in a tight gecko timer loop where NS_ProcessPendingEvents
+  // takes longer than the timer duration, NS_HasPendingEvents(thread) will
+  // always be true. ScheduleNativeEventCallback will be called on every
+  // NativeEventCallback callback, and in a Windows modal dispatch loop, the
+  // callback message will be processed first -> input gets starved, dead lock.
+  
+  // To avoid, don't post native callback messages from NativeEventCallback
+  // when we're in a modal loop. This gets us back into the Windows modal
+  // dispatch loop dispatching input messages. Once we drop out of the modal
+  // loop, we use mNativeCallbackPending to fire off a final NativeEventCallback
+  // if we need it, which insures NS_ProcessPendingEvents gets called and all
+  // gecko events get processed.
+  if (mEventloopNestingLevel < 2) {
+    OnDispatchedEvent(nsnull);
+    mNativeCallbackPending = PR_FALSE;
+  } else {
+    mNativeCallbackPending = PR_TRUE;
+  }
+}
+
 void
 nsAppShell::ScheduleNativeEventCallback()
 {
-  // post a message to the native event queue...
+  // Post a message to the hidden message window
-  NS_ADDREF_THIS();
+  NS_ADDREF_THIS(); // will be released when the event is processed
   ::PostMessage(mEventWnd, sMsgId, 0, reinterpret_cast<LPARAM>(this));
 }
 
@@ -303,5 +340,10 @@ nsAppShell::ProcessNextNativeEvent(PRBool mayWait)
     }
   } while (!gotMessage && mayWait);
 
+  // See DoProcessNextNativeEvent, mEventloopNestingLevel will be
+  // one when a modal loop unwinds.
+  if (mNativeCallbackPending && mEventloopNestingLevel == 1)
+    DoProcessMoreGeckoEvents();
+
   return gotMessage;
 }

widget/src/windows/nsAppShell.h

@@ -47,9 +47,13 @@
 class nsAppShell : public nsBaseAppShell
 {
 public:
-  nsAppShell() : mEventWnd(NULL) {}
+  nsAppShell() :
+    mEventWnd(NULL),
+    mNativeCallbackPending(PR_FALSE)
+  {}
 
   nsresult Init();
+  void DoProcessMoreGeckoEvents();
 
 #if MOZ_WINSDK_TARGETVER >= MOZ_NTDDI_WIN7
   static UINT GetTaskbarButtonCreatedMessage();
@@ -67,6 +71,7 @@ protected:
 
 protected:
   HWND mEventWnd;
+  PRBool mNativeCallbackPending;
 };
 
 #endif // nsAppShell_h__

widget/src/xpwidgets/nsBaseAppShell.cpp

@@ -93,6 +93,7 @@ nsBaseAppShell::Init()
   return NS_OK;
 }
 
+// Called by nsAppShell's native event callback
 void
 nsBaseAppShell::NativeEventCallback()
 {
@@ -136,11 +137,21 @@ nsBaseAppShell::NativeEventCallback()
   // Continue processing pending events later (we don't want to starve the
   // embedders event loop).
   if (NS_HasPendingEvents(thread))
-    OnDispatchedEvent(nsnull);
+    DoProcessMoreGeckoEvents();
 
   --mEventloopNestingLevel;
 }
 
+// Note, this is currently overidden on windows, see comments in nsAppShell for
+// details. 
+void
+nsBaseAppShell::DoProcessMoreGeckoEvents()
+{
+  OnDispatchedEvent(nsnull);
+}
+
+
+// Main thread via OnProcessNextEvent below
 PRBool
 nsBaseAppShell::DoProcessNextNativeEvent(PRBool mayWait)
 {
@@ -255,6 +266,7 @@ nsBaseAppShell::OnDispatchedEvent(nsIThreadInternal *thr)
   if (lastVal == 1)
     return NS_OK;
 
+  // Returns on the main thread in NativeEventCallback above
   ScheduleNativeEventCallback();
   return NS_OK;
 }

widget/src/xpwidgets/nsBaseAppShell.h

@@ -75,6 +75,13 @@ protected:
    */
   void NativeEventCallback();
 
+  /**
+   * Make a decision as to whether or not NativeEventCallback will
+   * trigger gecko event processing when there are pending gecko
+   * events.
+   */
+  virtual void DoProcessMoreGeckoEvents();
+
   /**
    * Implemented by subclasses.  Invoke NativeEventCallback from a native
    * event.  This method may be called on any thread.
@@ -96,6 +103,7 @@ protected:
   virtual PRBool ProcessNextNativeEvent(PRBool mayWait) = 0;
 
   PRInt32 mSuspendNativeCount;
+  PRUint32 mEventloopNestingLevel;
 
 private:
   PRBool DoProcessNextNativeEvent(PRBool mayWait);
@@ -115,7 +123,6 @@ private:
   PRBool *mBlockedWait;
   PRInt32 mFavorPerf;
   PRInt32 mNativeEventPending;
-  PRUint32 mEventloopNestingLevel;
   PRIntervalTime mStarvationDelay;
   PRIntervalTime mSwitchTime;
   PRIntervalTime mLastNativeEventTime;