Test for bug 538586.

ipc/ipdl/test/cxx/Makefile.in

@@ -61,6 +61,7 @@ EXPORT_LIBRARY = 1
 IPDLTESTS = \
   TestSanity  \
   TestRPCErrorCleanup \
+  TestCrashCleanup \
   TestLatency \
   TestRPCRaces \
   TestManyChildAllocs  \

ipc/ipdl/test/cxx/PTestCrashCleanup.ipdl

@@ -0,0 +1,22 @@
+// See bug 538586: if the top-level protocol's actor is deleted before
+// the "connection error" notification comes in from the IO thread,
+// IPDL teardown never occurs, even if Channel::Close() is called
+// after the error.
+
+namespace mozilla {
+namespace _ipdltest {
+
+// NB: needs to be RPC so that the parent blocks on the child's crash.
+rpc protocol PTestCrashCleanup {
+child:
+    rpc DIEDIEDIE();
+    __delete__();
+
+state ALIVE:
+    call DIEDIEDIE goto CRASH;
+state CRASH:
+    send __delete__;
+};
+
+}
+}

ipc/ipdl/test/cxx/TestCrashCleanup.cpp

@@ -0,0 +1,116 @@
+#include "TestCrashCleanup.h"
+
+#include "mozilla/CondVar.h"
+#include "mozilla/Mutex.h"
+
+#include "IPDLUnitTests.h"      // fail etc.
+#include "IPDLUnitTestSubprocess.h"
+
+using mozilla::CondVar;
+using mozilla::Mutex;
+using mozilla::MutexAutoLock;
+
+namespace mozilla {
+namespace _ipdltest {
+
+//-----------------------------------------------------------------------------
+// parent
+
+namespace {
+
+// NB: this test does its own shutdown, rather than going through
+// QuitParent(), because it's testing degenerate edge cases
+
+void DeleteSubprocess(Mutex* mutex, CondVar* cvar)
+{
+    MutexAutoLock lock(*mutex);
+
+    delete gSubprocess;
+    gSubprocess = NULL;
+
+    cvar->Notify();
+}
+
+void DeleteTheWorld()
+{
+    delete static_cast<TestCrashCleanupParent*>(gParentActor);
+    gParentActor = NULL;
+
+    // needs to be synchronous to avoid affecting event ordering on
+    // the main thread
+    Mutex mutex("TestCrashCleanup.DeleteTheWorld.mutex");
+    CondVar cvar(mutex, "TestCrashCleanup.DeleteTheWorld.cvar");
+
+    MutexAutoLock lock(mutex);
+
+    XRE_GetIOMessageLoop()->PostTask(
+      FROM_HERE,
+      NewRunnableFunction(DeleteSubprocess, &mutex, &cvar));
+
+    cvar.Wait();
+}
+
+void Done()
+{
+  static NS_DEFINE_CID(kAppShellCID, NS_APPSHELL_CID);
+  nsCOMPtr<nsIAppShell> appShell (do_GetService(kAppShellCID));
+  appShell->Exit();
+
+  passed(__FILE__);
+}
+
+} // namespace <anon>
+
+TestCrashCleanupParent::TestCrashCleanupParent() : mCleanedUp(false)
+{
+    MOZ_COUNT_CTOR(TestCrashCleanupParent);
+}
+
+TestCrashCleanupParent::~TestCrashCleanupParent()
+{
+    MOZ_COUNT_DTOR(TestCrashCleanupParent);
+
+    if (!mCleanedUp)
+        fail("should have been ActorDestroy()d!");
+}
+
+void
+TestCrashCleanupParent::Main()
+{
+    // NB: has to be enqueued before IO thread's error notification
+    MessageLoop::current()->PostTask(
+        FROM_HERE, NewRunnableFunction(DeleteTheWorld));
+
+    if (CallDIEDIEDIE())
+        fail("expected an error!");
+
+    Close();
+
+    MessageLoop::current()->PostTask(FROM_HERE, NewRunnableFunction(Done));
+}
+
+
+//-----------------------------------------------------------------------------
+// child
+
+TestCrashCleanupChild::TestCrashCleanupChild()
+{
+    MOZ_COUNT_CTOR(TestCrashCleanupChild);
+}
+
+TestCrashCleanupChild::~TestCrashCleanupChild()
+{
+    MOZ_COUNT_DTOR(TestCrashCleanupChild);
+}
+
+bool
+TestCrashCleanupChild::AnswerDIEDIEDIE()
+{
+    _exit(0);
+    NS_RUNTIMEABORT("unreached");
+    return false;
+}
+
+
+} // namespace _ipdltest
+} // namespace mozilla

ipc/ipdl/test/cxx/TestCrashCleanup.h

@@ -0,0 +1,58 @@
+#ifndef mozilla__ipdltest_TestCrashCleanup_h
+#define mozilla__ipdltest_TestCrashCleanup_h 1
+
+#include "mozilla/_ipdltest/IPDLUnitTests.h"
+
+#include "mozilla/_ipdltest/PTestCrashCleanupParent.h"
+#include "mozilla/_ipdltest/PTestCrashCleanupChild.h"
+
+namespace mozilla {
+namespace _ipdltest {
+
+
+class TestCrashCleanupParent :
+    public PTestCrashCleanupParent
+{
+public:
+    TestCrashCleanupParent();
+    virtual ~TestCrashCleanupParent();
+
+    void Main();
+
+protected:    
+    NS_OVERRIDE
+    virtual void ActorDestroy(ActorDestroyReason why)
+    {
+        if (AbnormalShutdown != why)
+            fail("unexpected destruction!");
+        mCleanedUp = true;
+    }
+
+    bool mCleanedUp;
+};
+
+
+class TestCrashCleanupChild :
+    public PTestCrashCleanupChild
+{
+public:
+    TestCrashCleanupChild();
+    virtual ~TestCrashCleanupChild();
+
+protected:
+    NS_OVERRIDE
+    virtual bool AnswerDIEDIEDIE();
+
+    NS_OVERRIDE
+    virtual void ActorDestroy(ActorDestroyReason why)
+    {
+        fail("should have 'crashed'!");
+    }
+};
+
+
+} // namespace _ipdltest
+} // namespace mozilla
+
+
+#endif // ifndef mozilla__ipdltest_TestCrashCleanup_h

ipc/ipdl/test/cxx/TestRPCErrorCleanup.cpp

@@ -16,6 +16,8 @@ namespace _ipdltest {
 //-----------------------------------------------------------------------------
 // parent
 
+namespace {
+
 // NB: this test does its own shutdown, rather than going through
 // QuitParent(), because it's testing degenerate edge cases
 
@@ -57,6 +59,8 @@ void Done()
   passed(__FILE__);
 }
 
+} // namespace <anon>
+
 TestRPCErrorCleanupParent::TestRPCErrorCleanupParent()
 {
     MOZ_COUNT_CTOR(TestRPCErrorCleanupParent);

ipc/ipdl/test/cxx/ipdl.mk

@@ -1,6 +1,7 @@
 IPDLSRCS =					\
   PTestArrays.ipdl				\
   PTestArraysSub.ipdl				\
+  PTestCrashCleanup.ipdl			\
   PTestDesc.ipdl				\
   PTestDescSub.ipdl				\
   PTestDescSubsub.ipdl				\