Fix recursive locking when unblocking an RPC channel for a sync message and add minimal error handling for sync messages.

ipc/glue/RPCChannel.cpp

@@ -238,11 +238,7 @@ RPCChannel::ProcessIncall(const Message& call, size_t stackDepth)
 
     switch (rv) {
     case MsgProcessed:
-        mIOLoop->PostTask(FROM_HERE,
-                          NewRunnableMethod(this,
-                                            &RPCChannel::OnSendReply,
-                                            reply));
-        return;
+        break;
 
     case MsgNotKnown:
     case MsgNotAllowed:
@@ -254,17 +250,19 @@ RPCChannel::ProcessIncall(const Message& call, size_t stackDepth)
         reply->set_rpc();
         reply->set_reply();
         reply->set_reply_error();
-        mIOLoop->PostTask(FROM_HERE,
-                          NewRunnableMethod(this,
-                                            &RPCChannel::OnSendReply,
-                                            reply));
         // FIXME/cjones: error handling; OnError()?
-        return;
+        break;
 
     default:
         NOTREACHED();
         return;
     }
+
+    mIOLoop->PostTask(FROM_HERE,
+                      NewRunnableMethod(this,
+                                        &RPCChannel::OnSendReply,
+                                        reply));
+
 }
 
 //
@@ -286,7 +284,7 @@ RPCChannel::OnMessageReceived(const Message& msg)
         // async messages, and the sync channel also will do error
         // checking wrt to its invariants
         if (!msg.is_rpc()) {
-            // unlocks mMutex
+            MutexAutoUnlock unlock(mMutex);
             return SyncChannel::OnMessageReceived(msg);
         }
 

ipc/glue/SyncChannel.cpp

@@ -78,8 +78,8 @@ SyncChannel::Send(Message* msg, Message* reply)
     // (NB: IPDL prevents the latter from occuring in actor code)
 
     // FIXME/cjones: real error handling
-    NS_ABORT_IF_FALSE(mRecvd.is_sync()
-                      && mRecvd.is_reply() && mPendingReply == mRecvd.type(),
+    NS_ABORT_IF_FALSE(mRecvd.is_sync() && mRecvd.is_reply() &&
+                      (mPendingReply == mRecvd.type() || mRecvd.is_reply_error()),
                       "unexpected sync message");
 
     mPendingReply = 0;
@@ -103,11 +103,7 @@ SyncChannel::OnDispatchMessage(const Message& msg)
 
     switch (rv) {
     case MsgProcessed:
-        mIOLoop->PostTask(FROM_HERE,
-                          NewRunnableMethod(this,
-                                            &SyncChannel::OnSendReply,
-                                            reply));
-        return;
+        break;
 
     case MsgNotKnown:
     case MsgNotAllowed:
@@ -115,12 +111,22 @@ SyncChannel::OnDispatchMessage(const Message& msg)
     case MsgRouteError:
     case MsgValueError:
         // FIXME/cjones: error handling; OnError()?
-        return;
+        delete reply;
+        reply = new Message();
+        reply->set_sync();
+        reply->set_reply();
+        reply->set_reply_error();
+        break;
 
     default:
         NOTREACHED();
         return;
     }
+
+    mIOLoop->PostTask(FROM_HERE,
+                      NewRunnableMethod(this,
+                                        &SyncChannel::OnSendReply,
+                                        reply));
 }
 
 //

ipc/ipdl/ipdl/lower.py

@@ -1823,6 +1823,12 @@ class GenerateProtocolActorHeader(Visitor):
             impl.addstmt(failif)
 
             if hasreply:
+                replyerror = cxx.ExprCall(
+                    cxx.ExprSelect(replyvar, '.', 'is_reply_error'))
+                failif2 = cxx.StmtIf(replyerror)
+                failif2.ifb.addstmt(cxx.StmtReturn(valueerrcode))
+                impl.addstmt(failif2)
+
                 # if this message has explicit actor returns, we need
                 # to convert them from Handle before returning to C++
                 for ret in md._cxx.returns: