Bug 542337 - '[OOPP] hang in test_plugin_clipping2.xhtml'. r=jimm.

--HG--
extra : rebase_source : 22b97f9252c02f7343d12b60fa9cd93cfbe12a25

dom/plugins/PPluginInstance.ipdl

@@ -166,6 +166,7 @@ parent:
   rpc PluginGotFocus();
 child:
   rpc SetPluginFocus();
+  rpc UpdateWindow();
 };
 
 } // namespace plugins

dom/plugins/PluginInstanceChild.cpp

@@ -901,6 +901,21 @@ PluginInstanceChild::AnswerSetPluginFocus()
 #endif
 }
 
+bool
+PluginInstanceChild::AnswerUpdateWindow()
+{
+    PR_LOG(gPluginLog, PR_LOG_DEBUG, ("%s", FULLFUNCTION));
+
+#if defined(OS_WIN)
+    if (mPluginWindowHWND)
+      UpdateWindow(mPluginWindowHWND);
+    return true;
+#else
+    NS_NOTREACHED("PluginInstanceChild::AnswerUpdateWindow not implemented!");
+    return false;
+#endif
+}
+
 PPluginScriptableObjectChild*
 PluginInstanceChild::AllocPPluginScriptableObject()
 {

dom/plugins/PluginInstanceChild.h

@@ -148,6 +148,9 @@ protected:
     virtual bool
     AnswerSetPluginFocus();
 
+    virtual bool
+    AnswerUpdateWindow();
+
 public:
     PluginInstanceChild(const NPPluginFuncs* aPluginIface);
 

widget/src/windows/nsWindow.cpp

@@ -106,6 +106,8 @@
 
 #ifdef MOZ_IPC
 #include "mozilla/ipc/SyncChannel.h"
+#include "mozilla/plugins/PluginInstanceParent.h"
+using mozilla::plugins::PluginInstanceParent;
 #endif
 
 #include "nsWindow.h"
@@ -2107,7 +2109,7 @@ NS_METHOD nsWindow::Invalidate(PRBool aIsSynchronous)
     VERIFY(::InvalidateRect(mWnd, NULL, FALSE));
 
     if (aIsSynchronous) {
-      VERIFY(::UpdateWindow(mWnd));
+      UpdateWindowInternal(mWnd);
     }
   }
   return NS_OK;
@@ -2141,7 +2143,7 @@ NS_METHOD nsWindow::Invalidate(const nsIntRect & aRect, PRBool aIsSynchronous)
     VERIFY(::InvalidateRect(mWnd, &rect, FALSE));
 
     if (aIsSynchronous) {
-      VERIFY(::UpdateWindow(mWnd));
+      UpdateWindowInternal(mWnd);
     }
   }
   return NS_OK;
@@ -2185,7 +2187,7 @@ NS_IMETHODIMP nsWindow::Update()
   // updates can come through for windows no longer holding an mWnd during
   // deletes triggered by JavaScript in buttons with mouse feedback
   if (mWnd)
-    VERIFY(::UpdateWindow(mWnd));
+    UpdateWindowInternal(mWnd);
 
   return rv;
 }
@@ -3146,7 +3148,12 @@ BOOL CALLBACK nsWindow::DispatchStarvedPaints(HWND aWnd, LPARAM aMsg)
     // invalidated rect. If it does. Dispatch a synchronous
     // paint.
     if (GetUpdateRect(aWnd, NULL, FALSE)) {
-      VERIFY(::UpdateWindow(aWnd));
+      nsWindow* win = GetNSWindowPtr(aWnd);
+      if (win)
+        win->UpdateWindowInternal(aWnd);
+      else
+        // Bad, this could hang a plugin process. Is this possible?
+        VERIFY(::UpdateWindow(aWnd));
     }
   }
   return TRUE;
@@ -7110,6 +7117,31 @@ LPARAM nsWindow::lParamToClient(LPARAM lParam)
   return MAKELPARAM(pt.x, pt.y);
 }
 
+// If this window hosts a plugin window from another process then we cannot use
+// the windows UpdateWindow function to update it. Doing so sends a synchronous
+// WM_PAINT message to the plugin process which may call back to this process
+// in response. As this process is waiting for the UpdateWindow call to return
+// we deadlock. To work around this we issue an IPC call to the plugin process
+// to force the redraw since the IPC call handles reentrancy properly.
+void nsWindow::UpdateWindowInternal(HWND aWnd)
+{
+  if (aWnd) {
+#ifdef MOZ_IPC
+    if (mWindowType == eWindowType_plugin) {
+      PluginInstanceParent* instance = reinterpret_cast<PluginInstanceParent*>(
+        ::GetPropW(aWnd, L"PluginInstanceParentProperty"));
+      if (instance) {
+        if (!instance->CallUpdateWindow())
+          NS_ERROR("Failed to send message!");
+        ValidateWindow(aWnd, NULL);
+        return;
+      }
+    }
+#endif
+    VERIFY(::UpdateWindow(aWnd));
+  }
+}
+
 /**************************************************************
  **************************************************************
  **

widget/src/windows/nsWindow.h

@@ -403,6 +403,8 @@ protected:
   static STDMETHODIMP_(LRESULT) LresultFromObject(REFIID riid, WPARAM wParam, LPUNKNOWN pAcc);
 #endif // ACCESSIBILITY
 
+  void                    UpdateWindowInternal(HWND aWnd);
+
 protected:
   nsIntSize             mLastSize;
   nsIntPoint            mLastPoint;