Bug 558986 - Fix for crashes in TSF with Flash/Silverlight on tablets (OLE/COM nested event loops). r=cjones, jimm, ehsan.

dom/plugins/COMMessageFilter.cpp

@@ -0,0 +1,114 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Firefox.
+ *
+ * The Initial Developer of the Original Code is
+ * the Mozilla Foundation <http://www.mozilla.org/>.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#include "COMMessageFilter.h"
+#include "base/message_loop.h"
+#include "mozilla/plugins/PluginModuleChild.h"
+
+#include <stdio.h>
+
+namespace mozilla {
+namespace plugins {
+
+HRESULT
+COMMessageFilter::QueryInterface(REFIID riid, void** ppv)
+{
+  if (riid == IID_IUnknown || riid == IID_IMessageFilter) {
+    *ppv = static_cast<IMessageFilter*>(this);
+    AddRef();
+    return S_OK;
+  }
+  return E_NOINTERFACE;
+}
+
+DWORD COMMessageFilter::AddRef()
+{
+  ++mRefCnt;
+  return mRefCnt;
+}
+
+DWORD COMMessageFilter::Release()
+{
+  DWORD r = --mRefCnt;
+  if (0 == r)
+    delete this;
+  return r;
+}
+
+DWORD 
+COMMessageFilter::HandleInComingCall(DWORD dwCallType,
+				     HTASK htaskCaller,
+				     DWORD dwTickCount,
+				     LPINTERFACEINFO lpInterfaceInfo)
+{
+  if (mPreviousFilter)
+    return mPreviousFilter->HandleInComingCall(dwCallType, htaskCaller,
+					       dwTickCount, lpInterfaceInfo);
+  return SERVERCALL_ISHANDLED;
+}
+
+DWORD
+COMMessageFilter::RetryRejectedCall(HTASK htaskCallee,
+				    DWORD dwTickCount,
+				    DWORD dwRejectType)
+{
+  if (mPreviousFilter)
+    return mPreviousFilter->RetryRejectedCall(htaskCallee, dwTickCount,
+					      dwRejectType);
+  return -1;
+}
+
+DWORD
+COMMessageFilter::MessagePending(HTASK htaskCallee,
+				 DWORD dwTickCount,
+				 DWORD dwPendingType)
+{
+  mPlugin->FlushPendingRPCQueue();
+  if (mPreviousFilter)
+    return mPreviousFilter->MessagePending(htaskCallee, dwTickCount,
+					   dwPendingType);
+  return PENDINGMSG_WAITNOPROCESS;
+}
+
+void
+COMMessageFilter::Initialize(PluginModuleChild* module)
+{
+  nsRefPtr<COMMessageFilter> f = new COMMessageFilter(module);
+  ::CoRegisterMessageFilter(f, getter_AddRefs(f->mPreviousFilter));
+}
+
+} // namespace plugins
+} // namespace mozilla

dom/plugins/COMMessageFilter.h

@@ -0,0 +1,82 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Firefox.
+ *
+ * The Initial Developer of the Original Code is
+ * the Mozilla Foundation <http://www.mozilla.org/>.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef mozilla_plugins_COMMessageFilter_h
+#define mozilla_plugins_COMMessageFilter_h
+
+#include <objidl.h>
+#include "nsISupportsImpl.h"
+#include "nsAutoPtr.h"
+
+namespace mozilla {
+namespace plugins {
+
+class PluginModuleChild;
+
+class COMMessageFilter : public IMessageFilter
+{
+public:
+  static void Initialize(PluginModuleChild* plugin);
+
+  COMMessageFilter(PluginModuleChild* plugin)
+    : mPlugin(plugin)
+  { }
+
+  HRESULT WINAPI QueryInterface(REFIID riid, void** ppv);
+  DWORD WINAPI AddRef();
+  DWORD WINAPI Release();
+
+  DWORD WINAPI HandleInComingCall(DWORD dwCallType,
+                                  HTASK htaskCaller,
+                                  DWORD dwTickCount,
+                                  LPINTERFACEINFO lpInterfaceInfo);
+  DWORD WINAPI RetryRejectedCall(HTASK htaskCallee,
+                                 DWORD dwTickCount,
+                                 DWORD dwRejectType);
+  DWORD WINAPI MessagePending(HTASK htaskCallee,
+                              DWORD dwTickCount,
+                              DWORD dwPendingType);
+
+private:
+  nsAutoRefCnt mRefCnt;
+  PluginModuleChild* mPlugin;
+  nsRefPtr<IMessageFilter> mPreviousFilter;
+};
+
+} // namespace plugins
+} // namespace mozilla
+
+#endif // COMMessageFilter_h

dom/plugins/Makefile.in

@@ -113,6 +113,10 @@ CPPSRCS = \
   PluginStreamParent.cpp \
   $(NULL)
 
+ifeq (WINNT,$(OS_ARCH))
+CPPSRCS += COMMessageFilter.cpp
+endif
+
 LOCAL_INCLUDES = \
   -I$(topsrcdir)/modules/plugin/base/public/ \
   -I$(topsrcdir)/modules/plugin/base/src/ \

dom/plugins/PluginModuleChild.cpp

@@ -63,6 +63,10 @@
 
 #include "nsNPAPIPlugin.h"
 
+#ifdef XP_WIN
+#include "COMMessageFilter.h"
+#endif
+
 using namespace mozilla::plugins;
 
 #if defined(XP_WIN)
@@ -124,6 +128,8 @@ PluginModuleChild::Init(const std::string& aPluginFilename,
 {
     PLUGIN_LOG_DEBUG_METHOD;
 
+    COMMessageFilter::Initialize(this);
+
     NS_ASSERTION(aChannel, "need a channel");
 
     if (!mObjectMap.Init()) {

ipc/chromium/src/base/message_pump_win.cc

@@ -97,6 +97,10 @@ void MessagePumpForUI::ScheduleWork() {
 
   // Make sure the MessagePump does some work for us.
   PostMessage(message_hwnd_, kMsgHaveWork, reinterpret_cast<WPARAM>(this), 0);
+
+  // In order to wake up any cross-process COM calls which may currently be
+  // pending on the main thread, we also have to post a UI message.
+  PostMessage(message_hwnd_, WM_NULL, NULL, 0);
 }
 
 void MessagePumpForUI::ScheduleDelayedWork(const Time& delayed_work_time) {

ipc/glue/RPCChannel.cpp

@@ -315,14 +315,14 @@ RPCChannel::Call(Message* msg, Message* reply)
     return true;
 }
 
-void
+bool
 RPCChannel::MaybeProcessDeferredIncall()
 {
     AssertWorkerThread();
     mMutex.AssertCurrentThreadOwns();
 
     if (mDeferred.empty())
-        return;
+        return false;
 
     size_t stackDepth = StackDepth();
 
@@ -331,7 +331,7 @@ RPCChannel::MaybeProcessDeferredIncall()
                "fatal logic error");
 
     if (mDeferred.top().rpc_remote_stack_depth_guess() < stackDepth)
-        return;
+        return false;
 
     // time to process this message
     Message call = mDeferred.top();
@@ -348,6 +348,7 @@ RPCChannel::MaybeProcessDeferredIncall()
 
     CxxStackFrame f(*this, IN_MESSAGE, &call);
     Incall(call, stackDepth);
+    return true;
 }
 
 void
@@ -371,6 +372,28 @@ RPCChannel::EnqueuePendingMessages()
 }
 
 void
+RPCChannel::FlushPendingRPCQueue()
+{
+    AssertWorkerThread();
+    mMutex.AssertNotCurrentThreadOwns();
+
+    {
+        MutexAutoLock lock(mMutex);
+
+        if (mDeferred.empty()) {
+            if (mPending.empty())
+                return;
+
+            const Message& last = mPending.back();
+            if (!last.is_rpc() || last.is_reply())
+                return;
+        }
+    }
+
+    while (OnMaybeDequeueOne());
+}
+
+bool
 RPCChannel::OnMaybeDequeueOne()
 {
     // XXX performance tuning knob: could process all or k pending
@@ -385,14 +408,14 @@ RPCChannel::OnMaybeDequeueOne()
 
         if (!Connected()) {
             ReportConnectionError("RPCChannel");
-            return;
+            return false;
         }
 
         if (!mDeferred.empty())
             return MaybeProcessDeferredIncall();
 
         if (mPending.empty())
-            return;
+            return false;
 
         recvd = mPending.front();
         mPending.pop();
@@ -402,17 +425,19 @@ RPCChannel::OnMaybeDequeueOne()
         // We probably just received a reply in a nested loop for an
         // RPC call sent before entering that loop.
         mOutOfTurnReplies[recvd.seqno()] = recvd;
-        return;
+        return false;
     }
 
     CxxStackFrame f(*this, IN_MESSAGE, &recvd);
 
     if (recvd.is_rpc())
-        return Incall(recvd, 0);
+        Incall(recvd, 0);
     else if (recvd.is_sync())
-        return SyncChannel::OnDispatchMessage(recvd);
+        SyncChannel::OnDispatchMessage(recvd);
     else
-        return AsyncChannel::OnDispatchMessage(recvd);
+        AsyncChannel::OnDispatchMessage(recvd);
+
+    return true;
 }
 
 void

ipc/glue/RPCChannel.h

@@ -166,6 +166,14 @@ public:
     NS_OVERRIDE
     virtual void OnChannelError();
 
+    /**
+     * If there is a pending RPC message, process all pending messages.
+     *
+     * @note This method is used on Windows when we detect that an outbound
+     * OLE RPC call is being made to unblock the parent.
+     */
+    void FlushPendingRPCQueue();
+
 #ifdef OS_WIN
     void ProcessNativeEventsInRPCCall();
 
@@ -188,10 +196,15 @@ protected:
 
     bool EventOccurred() const;
 
-    void MaybeProcessDeferredIncall();
+    bool MaybeProcessDeferredIncall();
     void EnqueuePendingMessages();
 
-    void OnMaybeDequeueOne();
+    /**
+     * Process one deferred or pending message.
+     * @return true if a message was processed
+     */
+    bool OnMaybeDequeueOne();
+
     void Incall(const Message& call, size_t stackDepth);
     void DispatchIncall(const Message& call);
 

ipc/ipdl/ipdl/lower.py

@@ -2978,9 +2978,14 @@ class _GenerateProtocolActorCode(ipdl.ast.Visitor):
             onstack.addstmt(StmtReturn(ExprCall(
                 ExprSelect(p.channelVar(), '.', p.onCxxStackVar().name))))
 
+            # void ProcessIncomingRacingRPCCall
+            processincoming = MethodDefn(
+                MethodDecl('FlushPendingRPCQueue', ret=Type.VOID))
+            processincoming.addstmt(StmtExpr(ExprCall(ExprSelect(_actorChannel(ExprVar.THIS), '.', 'FlushPendingRPCQueue'))))
+
             self.cls.addstmts([ onentered, onexited,
                                 onenteredcall, onexitedcall,
-                                onstack, Whitespace.NL ])
+                                onstack, processincoming, Whitespace.NL ])
 
         # OnChannelClose()
         onclose = MethodDefn(MethodDecl('OnChannelClose'))