Bug 561817 part B - Install the nested event loop tracking mechanism for all RPC calls to plugins, not just specific handle-event calls, r=jimm

dom/plugins/PPluginInstance.ipdl

@@ -178,8 +178,6 @@ parent:
 parent:
   rpc PluginGotFocus();
 
-  async ProcessNativeEventsInRPCCall();
-
 child:
   rpc SetPluginFocus();
   rpc UpdateWindow();

dom/plugins/PPluginModule.ipdl

@@ -95,6 +95,10 @@ parent:
   // native events, then "livelock" and some other glitches can occur.
   rpc ProcessSomeEvents();
 
+  // Window-specific message which instructs the RPC mechanism to enter
+  // a nested event loop for the current RPC call.
+  async ProcessNativeEventsInRPCCall();
+
   sync AppendNotesToCrashReport(nsCString aNotes);
 };
 

dom/plugins/PluginInstanceChild.cpp

@@ -101,8 +101,6 @@ PluginInstanceChild::PluginInstanceChild(const NPPluginFuncs* aPluginIface,
     , mPluginWindowHWND(0)
     , mPluginWndProc(0)
     , mPluginParentHWND(0)
-    , mNestedEventHook(0)
-    , mNestedEventLevelDepth(0)
     , mCachedWinlessPluginHWND(0)
     , mWinlessPopupSurrogateHWND(0)
     , mWinlessThrottleOldWndProc(0)
@@ -1094,61 +1092,6 @@ PluginInstanceChild::PluginWindowProc(HWND hWnd,
     return res;
 }
 
-/* winless modal ui loop logic */
-
-// gTempChildPointer is only in use from the time we enter handle event, to the
-// point where ui might be created by that call. If ui isn't created, there's
-// no issue. If ui is created, the parent can't start processing messages in
-// spin loop until SendSetNestedEventState, at which point,
-// gTempChildPointer is no longer needed.
-static PluginInstanceChild* gTempChildPointer;
-
-LRESULT CALLBACK
-PluginInstanceChild::NestedInputEventHook(int nCode,
-                                          WPARAM wParam,
-                                          LPARAM lParam)
-{
-    if (!gTempChildPointer) {
-        return CallNextHookEx(NULL, nCode, wParam, lParam);
-    }
-
-    if (nCode >= 0) {
-        NS_ASSERTION(gTempChildPointer, "Never should be null here!");
-        gTempChildPointer->ResetNestedEventHook();
-        gTempChildPointer->SendProcessNativeEventsInRPCCall();
-
-        gTempChildPointer = NULL;
-    }
-    return CallNextHookEx(NULL, nCode, wParam, lParam);
-}
-
-void
-PluginInstanceChild::SetNestedInputEventHook()
-{
-    NS_ASSERTION(!mNestedEventHook,
-        "mNestedEventHook already setup in call to SetNestedInputEventHook?");
-
-    PLUGIN_LOG_DEBUG(("%s", FULLFUNCTION));
-
-    // WH_GETMESSAGE hooks are triggered by peek message calls in parent due to
-    // attached message queues, resulting in stomped in-process ipc calls.  So
-    // we use a filter hook specific to dialogs, menus, and scroll bars to kick
-    // things off.
-    mNestedEventHook = SetWindowsHookEx(WH_MSGFILTER,
-                                        NestedInputEventHook,
-                                        NULL,
-                                        GetCurrentThreadId());
-}
-
-void
-PluginInstanceChild::ResetNestedEventHook()
-{
-    PLUGIN_LOG_DEBUG(("%s", FULLFUNCTION));
-    if (mNestedEventHook)
-        UnhookWindowsHookEx(mNestedEventHook);
-    mNestedEventHook = NULL;
-}
-
 /* windowless track popup menu helpers */
 
 BOOL
@@ -1331,16 +1274,6 @@ PluginInstanceChild::WinlessHandleEvent(NPEvent& event)
     // special handling during delivery.
     int16_t handled;
 
-    mNestedEventLevelDepth++;
-    PLUGIN_LOG_DEBUG(("WinlessHandleEvent start depth: %i", mNestedEventLevelDepth));
-
-    // On the first, non-reentrant call, setup our modal ui detection hook.
-    if (mNestedEventLevelDepth == 1) {
-        NS_ASSERTION(!gTempChildPointer, "valid gTempChildPointer here?");
-        gTempChildPointer = this;
-        SetNestedInputEventHook();
-    }
-
     // TrackPopupMenu will fail if the parent window is not associated with
     // our ui thread. So we hook TrackPopupMenu so we can hand in a surrogate
     // parent created in the child process.
@@ -1350,21 +1283,10 @@ PluginInstanceChild::WinlessHandleEvent(NPEvent& event)
       sWinlessPopupSurrogateHWND = mWinlessPopupSurrogateHWND;
     }
 
-    bool old_state = MessageLoop::current()->NestableTasksAllowed();
-    MessageLoop::current()->SetNestableTasksAllowed(true);
     handled = mPluginIface->event(&mData, reinterpret_cast<void*>(&event));
-    MessageLoop::current()->SetNestableTasksAllowed(old_state);
 
-    gTempChildPointer = NULL;
     sWinlessPopupSurrogateHWND = NULL;
 
-    mNestedEventLevelDepth--;
-    PLUGIN_LOG_DEBUG(("WinlessHandleEvent end depth: %i", mNestedEventLevelDepth));
-
-    NS_ASSERTION(!(mNestedEventLevelDepth < 0), "mNestedEventLevelDepth < 0?");
-    if (mNestedEventLevelDepth <= 0) {
-        ResetNestedEventHook();
-    }
     return handled;
 }
 
@@ -2084,7 +2006,6 @@ PluginInstanceChild::AnswerNPP_Destroy(NPError* aResult)
 
 #if defined(OS_WIN)
     SharedSurfaceRelease();
-    ResetNestedEventHook();
     DestroyWinlessPopupSurrogate();
 #endif
 

dom/plugins/PluginInstanceChild.h

@@ -233,10 +233,6 @@ private:
     void ReparentPluginWindow(HWND hWndParent);
     void SizePluginWindow(int width, int height);
     int16_t WinlessHandleEvent(NPEvent& event);
-    void SetNestedInputEventHook();
-    void ResetNestedEventHook();
-    void SetNestedInputPumpHook();
-    void ResetPumpHooks();
     void CreateWinlessPopupSurrogate();
     void DestroyWinlessPopupSurrogate();
     void InitPopupMenuHook();
@@ -250,16 +246,6 @@ private:
                                              UINT message,
                                              WPARAM wParam,
                                              LPARAM lParam);
-    static VOID CALLBACK PumpTimerProc(HWND hwnd,
-                                       UINT uMsg,
-                                       UINT_PTR idEvent,
-                                       DWORD dwTime);
-    static LRESULT CALLBACK NestedInputEventHook(int code,
-                                                 WPARAM wParam,
-                                                 LPARAM lParam);
-    static LRESULT CALLBACK NestedInputPumpHook(int code,
-                                                WPARAM wParam,
-                                                LPARAM lParam);
     static BOOL WINAPI TrackPopupHookProc(HMENU hMenu,
                                           UINT uFlags,
                                           int x,
@@ -323,7 +309,6 @@ private:
     HWND mPluginWindowHWND;
     WNDPROC mPluginWndProc;
     HWND mPluginParentHWND;
-    HHOOK mNestedEventHook;
     int mNestedEventLevelDepth;
     HWND mCachedWinlessPluginHWND;
     HWND mWinlessPopupSurrogateHWND;

dom/plugins/PluginInstanceParent.cpp

@@ -1313,21 +1313,6 @@ PluginInstanceParent::AnswerPluginGotFocus()
 #endif
 }
 
-bool
-PluginInstanceParent::RecvProcessNativeEventsInRPCCall()
-{
-    PLUGIN_LOG_DEBUG(("%s", FULLFUNCTION));
-#if defined(OS_WIN)
-    static_cast<PluginModuleParent*>(Manager())
-        ->ProcessNativeEventsInRPCCall();
-    return true;
-#else
-    NS_NOTREACHED(
-        "PluginInstanceParent::AnswerSetNestedEventState not implemented!");
-    return false;
-#endif
-}
-
 #ifdef OS_MACOSX
 void
 PluginInstanceParent::Invalidate()

dom/plugins/PluginInstanceParent.h

@@ -246,9 +246,6 @@ public:
     virtual bool
     AnswerPluginGotFocus();
 
-    NS_OVERRIDE virtual bool
-    RecvProcessNativeEventsInRPCCall();
-
 #if defined(OS_MACOSX)
     void Invalidate();
 #endif // definied(OS_MACOSX)

dom/plugins/PluginModuleChild.cpp

@@ -80,6 +80,9 @@ PluginModuleChild::PluginModuleChild() :
 #elif defined(MOZ_WIDGET_GTK2)
   , mNestedLoopTimerId(0)
 #endif
+#ifdef OS_WIN
+  , mNestedEventHook(NULL)
+#endif
 {
     NS_ASSERTION(!gInstance, "Something terribly wrong here!");
     memset(&mFunctions, 0, sizeof(mFunctions));
@@ -484,6 +487,10 @@ PluginModuleChild::AnswerNP_Shutdown(NPError *rv)
     // weakly guard against re-entry after NP_Shutdown
     memset(&mFunctions, 0, sizeof(mFunctions));
 
+#ifdef OS_WIN
+    ResetNestedInputEventHook();
+#endif
+
     return true;
 }
 
@@ -1488,6 +1495,10 @@ PluginModuleChild::AnswerNP_Initialize(NativeThreadId* tid, NPError* _retval)
     *tid = 0;
 #endif
 
+#ifdef OS_WIN
+    SetNestedInputEventHook();
+#endif
+
 #if defined(OS_LINUX)
     *_retval = mInitializeFunc(&sBrowserFuncs, &mFunctions);
     return true;
@@ -1844,3 +1855,59 @@ PluginModuleChild::NPN_IntFromIdentifier(NPIdentifier aIdentifier)
     }
     return PR_INT32_MIN;
 }
+
+#ifdef OS_WIN
+void
+PluginModuleChild::EnteredCall()
+{
+    mIncallPumpingStack.AppendElement(false);
+}
+
+void
+PluginModuleChild::ExitedCall()
+{
+    NS_ASSERTION(mIncallPumpingStack.Length(), "mismatched entered/exited");
+    PRUint32 len = mIncallPumpingStack.Length() - 1;
+    mIncallPumpingStack.TruncateLength(len);
+}
+
+LRESULT CALLBACK
+PluginModuleChild::NestedInputEventHook(int nCode, WPARAM wParam, LPARAM lParam)
+{
+    PluginModuleChild* self = current();
+    PRUint32 len = self->mIncallPumpingStack.Length();
+    if (nCode >= 0 && len && !self->mIncallPumpingStack[len - 1]) {
+        self->SendProcessNativeEventsInRPCCall();
+        self->mIncallPumpingStack[len - 1] = true;
+    }
+
+    return CallNextHookEx(NULL, nCode, wParam, lParam);
+}
+
+void
+PluginModuleChild::SetNestedInputEventHook()
+{
+    NS_ASSERTION(!mNestedEventHook,
+        "mNestedEventHook already setup in call to SetNestedInputEventHook?");
+
+    PLUGIN_LOG_DEBUG(("%s", FULLFUNCTION));
+
+    // WH_GETMESSAGE hooks are triggered by peek message calls in parent due to
+    // attached message queues, resulting in stomped in-process ipc calls.  So
+    // we use a filter hook specific to dialogs, menus, and scroll bars to kick
+    // things off.
+    mNestedEventHook = SetWindowsHookEx(WH_MSGFILTER,
+                                        NestedInputEventHook,
+                                        NULL,
+                                        GetCurrentThreadId());
+}
+
+void
+PluginModuleChild::ResetNestedInputEventHook()
+{
+    PLUGIN_LOG_DEBUG(("%s", FULLFUNCTION));
+    if (mNestedEventHook)
+        UnhookWindowsHookEx(mNestedEventHook);
+    mNestedEventHook = NULL;
+}
+#endif

dom/plugins/PluginModuleChild.h

@@ -298,6 +298,24 @@ public: // called by PluginInstanceChild
 
 private:
     static PLDHashOperator CollectForInstance(NPObjectData* d, void* userArg);
+
+#if defined(OS_WIN)
+    NS_OVERRIDE
+    virtual void EnteredCall();
+    NS_OVERRIDE
+    virtual void ExitedCall();
+
+    // Entered/ExitedCall notifications keep track of whether the plugin has
+    // entered a nested event loop within this RPC call.
+    nsAutoTArray<bool, 8> mIncallPumpingStack;
+
+    static LRESULT CALLBACK NestedInputEventHook(int code,
+                                                 WPARAM wParam,
+                                                 LPARAM lParam);
+    void SetNestedInputEventHook();
+    void ResetNestedInputEventHook();
+    HHOOK mNestedEventHook;
+#endif
 };
 
 } /* namespace plugins */

dom/plugins/PluginModuleParent.cpp

@@ -784,6 +784,20 @@ PluginModuleParent::AnswerProcessSomeEvents()
 }
 #endif
 
+bool
+PluginModuleParent::RecvProcessNativeEventsInRPCCall()
+{
+    PLUGIN_LOG_DEBUG(("%s", FULLFUNCTION));
+#if defined(OS_WIN)
+    ProcessNativeEventsInRPCCall();
+    return true;
+#else
+    NS_NOTREACHED(
+        "PluginInstanceParent::AnswerSetNestedEventState not implemented!");
+    return false;
+#endif
+}
+
 #ifdef OS_MACOSX
 #define DEFAULT_REFRESH_MS 20 // CoreAnimation: 50 FPS
 void

dom/plugins/PluginModuleParent.h

@@ -165,6 +165,9 @@ protected:
     NS_OVERRIDE
     virtual bool AnswerProcessSomeEvents();
 
+    NS_OVERRIDE virtual bool
+    RecvProcessNativeEventsInRPCCall();
+
     virtual bool
     RecvAppendNotesToCrashReport(const nsCString& aNotes);