Bug 1487249 - Part 1: Allow MessageChannel objects to be created within a thread, r=mccr8

To create a more generic interface for interacting both within the main thread
of the parent process and between the parent and child processes, it would be
nice to support IPDL actors within the main thread of the parent process. This
requires the underlying MessageChannel actor to support intra-thread links.

This change adds support for intra-thread links to the underlying MessageChannel
object using ThreadLink, and an extra boolean flag.

Differential Revision: https://phabricator.services.mozilla.com/D4620

ipc/glue/MessageChannel.cpp

@@ -634,7 +634,8 @@ MessageChannel::MessageChannel(const char* aName,
     mPeerPidSet(false),
     mPeerPid(-1),
     mIsPostponingSends(false),
-    mBuildIDsConfirmedMatch(false)
+    mBuildIDsConfirmedMatch(false),
+    mIsSameThreadChannel(false)
 {
     MOZ_COUNT_CTOR(ipc::MessageChannel);
 
@@ -966,6 +967,35 @@ MessageChannel::CommonThreadOpenInit(MessageChannel *aTargetChan, Side aSide)
     mSide = aSide;
 }
 
+bool
+MessageChannel::OpenOnSameThread(MessageChannel* aTargetChan,
+                                 mozilla::ipc::Side aSide)
+{
+    CommonThreadOpenInit(aTargetChan, aSide);
+
+    Side oppSide = UnknownSide;
+    switch (aSide) {
+      case ChildSide: oppSide = ParentSide; break;
+      case ParentSide: oppSide = ChildSide; break;
+      case UnknownSide: break;
+    }
+    mIsSameThreadChannel = true;
+
+    // XXX(nika): Avoid setting up a monitor for same thread channels? We
+    // shouldn't need it.
+    mMonitor = new RefCountedMonitor();
+
+    mChannelState = ChannelOpening;
+    aTargetChan->CommonThreadOpenInit(this, oppSide);
+
+    aTargetChan->mIsSameThreadChannel = true;
+    aTargetChan->mMonitor = mMonitor;
+
+    mChannelState = ChannelConnected;
+    aTargetChan->mChannelState = ChannelConnected;
+    return true;
+}
+
 bool
 MessageChannel::Echo(Message* aMsg)
 {
@@ -1475,6 +1505,8 @@ MessageChannel::Send(Message* aMsg, Message* aReply)
     // Sanity checks.
     AssertWorkerThread();
     mMonitor->AssertNotCurrentThreadOwns();
+    MOZ_RELEASE_ASSERT(!mIsSameThreadChannel,
+                       "sync send over same-thread channel will deadlock!");
 
 #ifdef OS_WIN
     SyncStackFrame frame(this, false);
@@ -1680,6 +1712,8 @@ MessageChannel::Call(Message* aMsg, Message* aReply)
     UniquePtr<Message> msg(aMsg);
     AssertWorkerThread();
     mMonitor->AssertNotCurrentThreadOwns();
+    MOZ_RELEASE_ASSERT(!mIsSameThreadChannel,
+                       "intr call send over same-thread channel will deadlock!");
 
 #ifdef OS_WIN
     SyncStackFrame frame(this, true);
@@ -2456,6 +2490,9 @@ MessageChannel::WaitForSyncNotify(bool /* aHandleWindowsMessages */)
     }
 #endif
 
+    MOZ_RELEASE_ASSERT(!mIsSameThreadChannel,
+                       "Wait on same-thread channel will deadlock!");
+
     TimeDuration timeout = (kNoTimeout == mTimeoutMs) ?
                            TimeDuration::Forever() :
                            TimeDuration::FromMilliseconds(mTimeoutMs);
@@ -2768,6 +2805,10 @@ MessageChannel::SynchronouslyClose()
     AssertWorkerThread();
     mMonitor->AssertCurrentThreadOwns();
     mLink->SendClose();
+
+    MOZ_RELEASE_ASSERT(!mIsSameThreadChannel || ChannelClosed == mChannelState,
+                       "same-thread channel failed to synchronously close?");
+
     while (ChannelClosed != mChannelState)
         mMonitor->Wait();
 }

ipc/glue/MessageChannel.h

@@ -180,6 +180,15 @@ private:
     // in MessageChannel.cpp.
     bool Open(MessageChannel *aTargetChan, nsIEventTarget *aEventTarget, Side aSide);
 
+    // "Open" a connection to an actor on the current thread.
+    //
+    // Returns true if the transport layer was successfully connected,
+    // i.e., mChannelState == ChannelConnected.
+    //
+    // Same-thread channels may not perform synchronous or blocking message
+    // sends, to avoid deadlocks.
+    bool OpenOnSameThread(MessageChannel* aTargetChan, Side aSide);
+
     // Close the underlying transport channel.
     void Close();
 
@@ -564,11 +573,20 @@ private:
                            "not on worker thread!");
     }
 
-    // The "link" thread is either the I/O thread (ProcessLink) or the
-    // other actor's work thread (ThreadLink).  In either case, it is
-    // NOT our worker thread.
+    // The "link" thread is either the I/O thread (ProcessLink), the other
+    // actor's work thread (ThreadLink), or the worker thread (same-thread
+    // channels).
     void AssertLinkThread() const
     {
+        if (mIsSameThreadChannel) {
+            // If we're a same-thread channel, we have to be on our worker
+            // thread.
+            AssertWorkerThread();
+            return;
+        }
+
+        // If we aren't a same-thread channel, our "link" thread is _not_ our
+        // worker thread!
         MOZ_ASSERT(mWorkerThread, "Channel hasn't been opened yet");
         MOZ_RELEASE_ASSERT(mWorkerThread != GetCurrentVirtualThread(),
                            "on worker thread but should not be!");
@@ -861,6 +879,10 @@ private:
     std::vector<UniquePtr<Message>> mPostponedSends;
 
     bool mBuildIDsConfirmedMatch;
+
+    // If this is true, both ends of this message channel have event targets
+    // on the same thread.
+    bool mIsSameThreadChannel;
 };
 
 void

ipc/glue/ProtocolUtils.cpp

@@ -780,6 +780,13 @@ IToplevelProtocol::OpenWithAsyncPid(mozilla::ipc::Transport* aTransport,
   return GetIPCChannel()->Open(aTransport, aThread, aSide);
 }
 
+bool
+IToplevelProtocol::OpenOnSameThread(MessageChannel* aChannel, Side aSide)
+{
+  SetOtherProcessId(base::GetCurrentProcId());
+  return GetIPCChannel()->OpenOnSameThread(aChannel, aSide);
+}
+
 void
 IToplevelProtocol::Close()
 {

ipc/glue/ProtocolUtils.h

@@ -514,6 +514,15 @@ public:
                           MessageLoop* aThread = nullptr,
                           mozilla::ipc::Side aSide = mozilla::ipc::UnknownSide);
 
+    // Open a toplevel actor such that both ends of the actor's channel are on
+    // the same thread. This method should be called on the thread to perform
+    // the link.
+    //
+    // WARNING: Attempting to send a sync or intr message on the same thread
+    // will crash.
+    bool OpenOnSameThread(MessageChannel* aChannel,
+                          mozilla::ipc::Side aSide = mozilla::ipc::UnknownSide);
+
     void Close();
 
     void SetReplyTimeoutMs(int32_t aTimeoutMs);