Block native event processing in the appshell while processing gecko events from a nested native event loop. b=420148 r+sr=roc a=blocking1.9

widget/src/xpwidgets/nsBaseAppShell.cpp

@@ -60,6 +60,7 @@ nsBaseAppShell::nsBaseAppShell()
   , mEventloopNestingState(eEventloopNone)
   , mRunWasCalled(PR_FALSE)
   , mExiting(PR_FALSE)
+  , mBlockNativeEvent(PR_FALSE)
 {
 }
 
@@ -102,10 +103,22 @@ nsBaseAppShell::NativeEventCallback()
   // nsBaseAppShell::Run is not being used to pump events, so this may be
   // our only opportunity to process pending gecko events.
 
-  EventloopNestingState prevVal = mEventloopNestingState;
   nsIThread *thread = NS_GetCurrentThread();
+  PRBool prevBlockNativeEvent = mBlockNativeEvent;
+  if (mEventloopNestingState == eEventloopOther) {
+    if (!NS_HasPendingEvents(thread))
+      return;
+    // We're in a nested native event loop and have some gecko events to
+    // process.  While doing that we block processing native events from the
+    // appshell - instead, we want to get back to the nested native event
+    // loop ASAP (bug 420148).
+    mBlockNativeEvent = PR_TRUE;
+  }
+
+  EventloopNestingState prevVal = mEventloopNestingState;
   NS_ProcessPendingEvents(thread, THREAD_EVENT_STARVATION_LIMIT);
   mEventloopNestingState = prevVal;
+  mBlockNativeEvent = prevBlockNativeEvent;
 
   // Continue processing pending events later (we don't want to starve the
   // embedders event loop).
@@ -197,6 +210,9 @@ nsBaseAppShell::ResumeNative(void)
 NS_IMETHODIMP
 nsBaseAppShell::OnDispatchedEvent(nsIThreadInternal *thr)
 {
+  if (mBlockNativeEvent)
+    return NS_OK;
+
   PRInt32 lastVal = PR_AtomicSet(&mNativeEventPending, 1);
   if (lastVal == 1)
     return NS_OK;
@@ -210,6 +226,18 @@ NS_IMETHODIMP
 nsBaseAppShell::OnProcessNextEvent(nsIThreadInternal *thr, PRBool mayWait,
                                    PRUint32 recursionDepth)
 {
+  if (mBlockNativeEvent) {
+    if (!mayWait)
+      return NS_OK;
+    // Hmm, we're in a nested native event loop and would like to get
+    // back to it ASAP, but it seems a gecko event has caused us to
+    // spin up a nested XPCOM event loop (eg. modal window), so we
+    // really must start processing native events here again.
+    mBlockNativeEvent = PR_FALSE;
+    if (NS_HasPendingEvents(thr))
+      OnDispatchedEvent(thr); // in case we blocked it earlier
+  }
+
   PRIntervalTime start = PR_IntervalNow();
   PRIntervalTime limit = THREAD_EVENT_STARVATION_LIMIT;
 

widget/src/xpwidgets/nsBaseAppShell.h

@@ -120,6 +120,16 @@ private:
   EventloopNestingState mEventloopNestingState;
   PRPackedBool mRunWasCalled;
   PRPackedBool mExiting;
+  /**
+   * mBlockNativeEvent blocks the appshell from processing native events.
+   * It is set to PR_TRUE while a nested native event loop (eEventloopOther)
+   * is processing gecko events in NativeEventCallback(), thus queuing up
+   * native events until we return to that loop (bug 420148).
+   * We force mBlockNativeEvent to PR_FALSE in case handling one of the gecko
+   * events spins up a nested XPCOM event loop (eg. modal window) which would
+   * otherwise lead to a "deadlock" where native events aren't processed at all.
+   */
+  PRPackedBool mBlockNativeEvent;
 };
 
 #endif // nsBaseAppShell_h__