Bug 545455, part 1: Track when RPCChannel code is first pushed on the C++ stack and last popped. r=bent

--HG--
extra : rebase_source : 1bf50aac0f5da1c4322347b42b24e72215543915

ipc/glue/AsyncChannel.h

@@ -112,7 +112,7 @@ public:
     void Close();
 
     // Asynchronously send a message to the other side of the channel
-    bool Send(Message* msg);
+    virtual bool Send(Message* msg);
 
     //
     // These methods are called on the "IO" thread

ipc/glue/RPCChannel.cpp

@@ -98,7 +98,8 @@ RPCChannel::RPCChannel(RPCListener* aListener,
     mDeferred(),
     mRemoteStackDepthGuess(0),
     mRacePolicy(aPolicy),
-    mBlockedOnParent(false)
+    mBlockedOnParent(false),
+    mCxxStackFrames(0)
 {
     MOZ_COUNT_CTOR(RPCChannel);
 }
@@ -106,7 +107,7 @@ RPCChannel::RPCChannel(RPCListener* aListener,
 RPCChannel::~RPCChannel()
 {
     MOZ_COUNT_DTOR(RPCChannel);
-    // FIXME/cjones: impl
+    RPC_ASSERT(0 == mCxxStackFrames, "mismatched CxxStackFrame ctor/dtors");
 }
 
 #ifdef OS_WIN
@@ -128,6 +129,20 @@ RPCChannel::EventOccurred()
              != mOutOfTurnReplies.end()));
 }
 
+bool
+RPCChannel::Send(Message* msg)
+{
+    CxxStackFrame f(*this);
+    return AsyncChannel::Send(msg);
+}
+
+bool
+RPCChannel::Send(Message* msg, Message* reply)
+{
+    CxxStackFrame f(*this);
+    return SyncChannel::Send(msg, reply);
+}
+
 bool
 RPCChannel::Call(Message* msg, Message* reply)
 {
@@ -137,6 +152,8 @@ RPCChannel::Call(Message* msg, Message* reply)
                "violation of sync handler invariant");
     RPC_ASSERT(msg->is_rpc(), "can only Call() RPC messages here");
 
+    CxxStackFrame f(*this);
+
     MutexAutoLock lock(mMutex);
 
     if (!Connected()) {
@@ -341,6 +358,8 @@ RPCChannel::OnMaybeDequeueOne()
         mPending.pop();
     }
 
+    CxxStackFrame f(*this);
+
     if (recvd.is_rpc())
         return Incall(recvd, 0);
     else if (recvd.is_sync())

ipc/glue/RPCChannel.h

@@ -51,6 +51,8 @@ namespace ipc {
 
 class RPCChannel : public SyncChannel
 {
+    friend class CxxStackFrame;
+
 public:
     class /*NS_INTERFACE_CLASS*/ RPCListener :
         public SyncChannel::SyncListener
@@ -82,6 +84,13 @@ public:
     // Make an RPC to the other side of the channel
     bool Call(Message* msg, Message* reply);
 
+    // RPCChannel overrides these so that the async and sync messages
+    // can be counted against mStackFrames
+    NS_OVERRIDE
+    virtual bool Send(Message* msg);
+    NS_OVERRIDE
+    virtual bool Send(Message* msg, Message* reply);
+
     // Asynchronously, send the child a message that puts it in such a
     // state that it can't send messages to the parent unless the
     // parent sends a message to it first.  The child stays in this
@@ -151,6 +160,55 @@ protected:
     void BlockOnParent();
     void UnblockFromParent();
 
+    // This helper class managed RPCChannel.mCxxStackDepth on behalf
+    // of RPCChannel.  When the stack depth is incremented from zero
+    // to non-zero, it invokes an RPCChannel callback, and similarly
+    // for when the depth goes from non-zero to zero;
+    void EnteredCxxStack()
+    {
+        // FIXME/bug 545455: call mListener hook
+        printf("[%s] +++ CXX STACK\n", mChild ? "child" : "parent");
+    }
+
+    void ExitedCxxStack()
+    {
+        // FIXME/bug 545455: call mListener hook
+        printf("[%s] --- CXX STACK\n", mChild ? "child" : "parent");
+    }
+
+    class NS_STACK_CLASS CxxStackFrame
+    {
+    public:
+        CxxStackFrame(RPCChannel& that) : mThat(that) {
+            NS_ABORT_IF_FALSE(0 <= mThat.mCxxStackFrames,
+                              "mismatched CxxStackFrame ctor/dtor");
+            mThat.AssertWorkerThread();
+
+            if (0 == mThat.mCxxStackFrames++)
+                mThat.EnteredCxxStack();
+        }
+
+        ~CxxStackFrame() {
+            bool exitingStack = (0 == --mThat.mCxxStackFrames);
+
+            // mListener could have gone away if Close() was called while
+            // RPCChannel code was still on the stack
+            if (!mThat.mListener)
+                return;
+
+            mThat.AssertWorkerThread();
+            if (exitingStack)
+                mThat.ExitedCxxStack();
+        }
+    private:
+        RPCChannel& mThat;
+
+        // disable harmful methods
+        CxxStackFrame();
+        CxxStackFrame(const CxxStackFrame&);
+        CxxStackFrame& operator=(const CxxStackFrame&);
+    };
+
     // Called from both threads
     size_t StackDepth() {
         mMutex.AssertCurrentThreadOwns();
@@ -256,6 +314,16 @@ protected:
 
     // True iff the parent has put us in a |BlockChild()| state.
     bool mBlockedOnParent;
+
+    // Approximation of number of Sync/RPCChannel-code frames on the
+    // C++ stack.  It can only be interpreted as the implication
+    //
+    //  mCxxStackDepth > 0 => RPCChannel code on C++ stack
+    //
+    // This member is only accessed on the worker thread, and so is
+    // not protected by mMutex.  It is managed exclusively by the
+    // helper |class CxxStackFrame|.
+    int mCxxStackFrames;
 };
 
 

ipc/glue/SyncChannel.h

@@ -71,12 +71,13 @@ public:
     SyncChannel(SyncListener* aListener);
     virtual ~SyncChannel();
 
-    bool Send(Message* msg) {
+    NS_OVERRIDE
+    virtual bool Send(Message* msg) {
         return AsyncChannel::Send(msg);
     }
 
     // Synchronously send |msg| (i.e., wait for |reply|)
-    bool Send(Message* msg, Message* reply);
+    virtual bool Send(Message* msg, Message* reply);
 
     void SetReplyTimeoutMs(int32 aTimeoutMs) {
         AssertWorkerThread();