Bug 550026: Don't deliver NotifyError() when RPCChannel code is on the stack. r=bent

ipc/glue/AsyncChannel.cpp

@@ -260,6 +260,32 @@ AsyncChannel::SendSpecialMessage(Message* msg)
         NewRunnableMethod(this, &AsyncChannel::OnSend, msg));
 }
 
+void
+AsyncChannel::OnNotifyMaybeChannelError()
+{
+    AssertWorkerThread();
+    mMutex.AssertNotCurrentThreadOwns();
+
+    // OnChannelError holds mMutex when it posts this task and this
+    // task cannot be allowed to run until OnChannelError has
+    // exited. We enforce that order by grabbing the mutex here which
+    // should only continue once OnChannelError has completed.
+    {
+        MutexAutoLock lock(mMutex);
+        // nothing to do here
+    }
+
+    if (ShouldDeferNotifyMaybeError()) {
+        mChannelErrorTask =
+            NewRunnableMethod(this, &AsyncChannel::OnNotifyMaybeChannelError);
+        // 10 ms delay is completely arbitrary
+        mWorkerLoop->PostDelayedTask(FROM_HERE, mChannelErrorTask, 10);
+        return;
+    }
+
+    NotifyMaybeChannelError();
+}
+
 void
 AsyncChannel::NotifyChannelClosed()
 {
@@ -280,15 +306,6 @@ AsyncChannel::NotifyMaybeChannelError()
 {
     mMutex.AssertNotCurrentThreadOwns();
 
-    // OnChannelError holds mMutex when it posts this task and this task cannot
-    // be allowed to run until OnChannelError has exited. We enforce that order
-    // by grabbing the mutex here which should only continue once OnChannelError
-    // has completed.
-    {
-        MutexAutoLock lock(mMutex);
-        // Nothing to do here!
-    }
-
     // TODO sort out Close() on this side racing with Close() on the
     // other side
     if (ChannelClosing == mChannelState) {
@@ -436,7 +453,7 @@ AsyncChannel::OnChannelError()
 
     // This must be the last code that runs on this thread!
     mChannelErrorTask =
-        NewRunnableMethod(this, &AsyncChannel::NotifyMaybeChannelError);
+        NewRunnableMethod(this, &AsyncChannel::OnNotifyMaybeChannelError);
     mWorkerLoop->PostTask(FROM_HERE, mChannelErrorTask);
 }
 

ipc/glue/AsyncChannel.h

@@ -161,6 +161,10 @@ protected:
 
     // Run on the worker thread
 
+    void OnNotifyMaybeChannelError();
+    virtual bool ShouldDeferNotifyMaybeError() {
+        return false;
+    }
     void NotifyChannelClosed();
     void NotifyMaybeChannelError();
 

ipc/glue/RPCChannel.h

@@ -176,6 +176,11 @@ protected:
         return static_cast<RPCListener*>(mListener);
     }
 
+    NS_OVERRIDE
+    virtual bool ShouldDeferNotifyMaybeError() {
+        return 0 < mCxxStackFrames;
+    }
+
     bool EventOccurred();
 
     void MaybeProcessDeferredIncall();