From 676a802cc703694528be40f594a6d203e9d487f5 Mon Sep 17 00:00:00 2001
From: Andrea Marchesini <amarchesini@mozilla.com>
Date: Tue, 23 Jun 2015 15:50:00 -0700
Subject: [PATCH] Bug 1176034 - MessagePort should force a close() if the
 structured clone algorithm fails, r=bent

---
 dom/base/PostMessageEvent.cpp                 |  8 ++-
 dom/messagechannel/MessagePort.cpp            | 51 +++++++++++++++++++
 dom/messagechannel/MessagePort.h              |  3 ++
 dom/messagechannel/MessagePortParent.cpp      | 14 +++++
 dom/messagechannel/MessagePortParent.h        |  4 ++
 dom/messagechannel/MessagePortService.cpp     | 50 ++++++++++++++++++
 dom/messagechannel/MessagePortService.h       |  5 ++
 dom/messagechannel/MessagePortUtils.cpp       | 18 ++++++-
 dom/messagechannel/tests/mochitest.ini        |  1 +
 .../tests/test_messageChannel_forceClose.html | 36 +++++++++++++
 dom/workers/WorkerPrivate.cpp                 |  9 +++-
 ipc/glue/BackgroundParentImpl.cpp             | 11 ++++
 ipc/glue/BackgroundParentImpl.h               |  5 ++
 ipc/glue/PBackground.ipdl                     |  2 +
 14 files changed, 213 insertions(+), 4 deletions(-)
 create mode 100644 dom/messagechannel/tests/test_messageChannel_forceClose.html

diff --git a/dom/base/PostMessageEvent.cpp b/dom/base/PostMessageEvent.cpp
index 534d389cda14..f375a672a725 100644
--- a/dom/base/PostMessageEvent.cpp
+++ b/dom/base/PostMessageEvent.cpp
@@ -238,7 +238,13 @@ PostMessageEvent::FreeTransferStructuredClone(uint32_t aTag,
                                               uint64_t aExtraData,
                                               void* aClosure)
 {
-  // Nothing to do.
+  if (aTag == SCTAG_DOM_MAP_MESSAGEPORT) {
+    MOZ_ASSERT(aClosure);
+    MOZ_ASSERT(!aContent);
+
+    StructuredCloneInfo* scInfo = static_cast<StructuredCloneInfo*>(aClosure);
+    MessagePort::ForceClose(scInfo->event->GetPortIdentifier(aExtraData));
+  }
 }
 
 PostMessageEvent::PostMessageEvent(nsGlobalWindow* aSource,
diff --git a/dom/messagechannel/MessagePort.cpp b/dom/messagechannel/MessagePort.cpp
index 6a328bb969ee..fa36218a744d 100644
--- a/dom/messagechannel/MessagePort.cpp
+++ b/dom/messagechannel/MessagePort.cpp
@@ -21,6 +21,7 @@
 #include "mozilla/dom/WorkerScope.h"
 #include "mozilla/ipc/BackgroundChild.h"
 #include "mozilla/ipc/PBackgroundChild.h"
+#include "mozilla/unused.h"
 #include "nsContentUtils.h"
 #include "nsGlobalWindow.h"
 #include "nsPresContext.h"
@@ -249,6 +250,50 @@ private:
   }
 };
 
+class ForceCloseHelper final : public nsIIPCBackgroundChildCreateCallback
+{
+public:
+  NS_DECL_ISUPPORTS
+
+  static void ForceClose(const MessagePortIdentifier& aIdentifier)
+  {
+    PBackgroundChild* actor =
+      mozilla::ipc::BackgroundChild::GetForCurrentThread();
+    if (actor) {
+      unused << actor->SendMessagePortForceClose(aIdentifier.uuid(),
+                                                 aIdentifier.destinationUuid(),
+                                                 aIdentifier.sequenceId());
+      return;
+    }
+
+    nsRefPtr<ForceCloseHelper> helper = new ForceCloseHelper(aIdentifier);
+    if (NS_WARN_IF(!mozilla::ipc::BackgroundChild::GetOrCreateForCurrentThread(helper))) {
+      MOZ_CRASH();
+    }
+  }
+
+private:
+  explicit ForceCloseHelper(const MessagePortIdentifier& aIdentifier)
+    : mIdentifier(aIdentifier)
+  {}
+
+  ~ForceCloseHelper() {}
+
+  void ActorFailed() override
+  {
+    MOZ_CRASH("Failed to create a PBackgroundChild actor!");
+  }
+
+  void ActorCreated(mozilla::ipc::PBackgroundChild* aActor)
+  {
+    ForceClose(mIdentifier);
+  }
+
+  const MessagePortIdentifier mIdentifier;
+};
+
+NS_IMPL_ISUPPORTS(ForceCloseHelper, nsIIPCBackgroundChildCreateCallback)
+
 } // anonymous namespace
 
 MessagePort::MessagePort(nsPIDOMWindow* aWindow)
@@ -863,5 +908,11 @@ MessagePort::RemoveDocFromBFCache()
   bfCacheEntry->RemoveFromBFCacheSync();
 }
 
+/* static */ void
+MessagePort::ForceClose(const MessagePortIdentifier& aIdentifier)
+{
+  ForceCloseHelper::ForceClose(aIdentifier);
+}
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/messagechannel/MessagePort.h b/dom/messagechannel/MessagePort.h
index 63081a083323..ede640ba6ae6 100644
--- a/dom/messagechannel/MessagePort.h
+++ b/dom/messagechannel/MessagePort.h
@@ -86,6 +86,9 @@ public:
   Create(nsPIDOMWindow* aWindow, const MessagePortIdentifier& aIdentifier,
          ErrorResult& aRv);
 
+  static void
+  ForceClose(const MessagePortIdentifier& aIdentifier);
+
   virtual JSObject*
   WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
 
diff --git a/dom/messagechannel/MessagePortParent.cpp b/dom/messagechannel/MessagePortParent.cpp
index 25dadafdab3b..917606bf09d5 100644
--- a/dom/messagechannel/MessagePortParent.cpp
+++ b/dom/messagechannel/MessagePortParent.cpp
@@ -159,5 +159,19 @@ MessagePortParent::Close()
   mEntangled = false;
 }
 
+/* static */ bool
+MessagePortParent::ForceClose(const nsID& aUUID,
+                              const nsID& aDestinationUUID,
+                              const uint32_t& aSequenceID)
+{
+  MessagePortService* service =  MessagePortService::Get();
+  if (!service) {
+    NS_WARNING("The service must exist if we want to close an existing MessagePort.");
+    return false;
+  }
+
+  return service->ForceClose(aUUID, aDestinationUUID, aSequenceID);
+}
+
 } // dom namespace
 } // mozilla namespace
diff --git a/dom/messagechannel/MessagePortParent.h b/dom/messagechannel/MessagePortParent.h
index 46dbb19ff0cc..1dcfb53e79f8 100644
--- a/dom/messagechannel/MessagePortParent.h
+++ b/dom/messagechannel/MessagePortParent.h
@@ -36,6 +36,10 @@ public:
     return mUUID;
   }
 
+  static bool ForceClose(const nsID& aUUID,
+                         const nsID& aDestinationUUID,
+                         const uint32_t& aSequenceID);
+
 private:
   virtual bool RecvPostMessages(nsTArray<MessagePortMessage>&& aMessages)
                                                                        override;
diff --git a/dom/messagechannel/MessagePortService.cpp b/dom/messagechannel/MessagePortService.cpp
index 0d75e6ba1951..afd5ab98af2b 100644
--- a/dom/messagechannel/MessagePortService.cpp
+++ b/dom/messagechannel/MessagePortService.cpp
@@ -6,16 +6,27 @@
 #include "MessagePortService.h"
 #include "MessagePortParent.h"
 #include "SharedMessagePortMessage.h"
+#include "mozilla/ipc/BackgroundParent.h"
 #include "mozilla/StaticPtr.h"
 #include "mozilla/unused.h"
 #include "nsDataHashtable.h"
 #include "nsTArray.h"
 
+using mozilla::ipc::AssertIsOnBackgroundThread;
+
 namespace mozilla {
 namespace dom {
 
 namespace {
+
 StaticRefPtr<MessagePortService> gInstance;
+
+void
+AssertIsInMainProcess()
+{
+  MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_Default);
+}
+
 } // anonymous namespace
 
 class MessagePortService::MessagePortServiceData final
@@ -53,9 +64,21 @@ public:
   FallibleTArray<nsRefPtr<SharedMessagePortMessage>> mMessages;
 };
 
+/* static */ MessagePortService*
+MessagePortService::Get()
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+
+  return gInstance;
+}
+
 /* static */ MessagePortService*
 MessagePortService::GetOrCreate()
 {
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+
   if (!gInstance) {
     gInstance = new MessagePortService();
   }
@@ -248,6 +271,12 @@ MessagePortService::CloseAll(const nsID& aUUID)
 
   CloseAll(destinationUUID);
 
+  // CloseAll calls itself recursively and it can happen that it deletes
+  // itself. Before continuing we must check if we are still alive.
+  if (!gInstance) {
+    return;
+  }
+
 #ifdef DEBUG
   mPorts.EnumerateRead(CloseAllDebugCheck, const_cast<nsID*>(&aUUID));
 #endif
@@ -324,5 +353,26 @@ MessagePortService::ParentDestroy(MessagePortParent* aParent)
   CloseAll(aParent->ID());
 }
 
+bool
+MessagePortService::ForceClose(const nsID& aUUID,
+                               const nsID& aDestinationUUID,
+                               const uint32_t& aSequenceID)
+{
+  MessagePortServiceData* data;
+  if (!mPorts.Get(aUUID, &data)) {
+    NS_WARNING("Unknown MessagePort in ForceClose()");
+    return true;
+  }
+
+  if (!data->mDestinationUUID.Equals(aDestinationUUID) ||
+      data->mSequenceID != aSequenceID) {
+    NS_WARNING("DestinationUUID and/or sequenceID do not match.");
+    return false;
+  }
+
+  CloseAll(aUUID);
+  return true;
+}
+
 } // dom namespace
 } // mozilla namespace
diff --git a/dom/messagechannel/MessagePortService.h b/dom/messagechannel/MessagePortService.h
index 437cf2393c11..faae81fcbc38 100644
--- a/dom/messagechannel/MessagePortService.h
+++ b/dom/messagechannel/MessagePortService.h
@@ -20,6 +20,7 @@ class MessagePortService final
 public:
   NS_INLINE_DECL_REFCOUNTING(MessagePortService)
 
+  static MessagePortService* Get();
   static MessagePortService* GetOrCreate();
 
   bool RequestEntangling(MessagePortParent* aParent,
@@ -38,6 +39,10 @@ public:
 
   void ParentDestroy(MessagePortParent* aParent);
 
+  bool ForceClose(const nsID& aUUID,
+                  const nsID& aDestinationUUID,
+                  const uint32_t& aSequenceID);
+
 private:
   ~MessagePortService() {}
 
diff --git a/dom/messagechannel/MessagePortUtils.cpp b/dom/messagechannel/MessagePortUtils.cpp
index 06b330b7f426..ccd3b13043e2 100644
--- a/dom/messagechannel/MessagePortUtils.cpp
+++ b/dom/messagechannel/MessagePortUtils.cpp
@@ -141,7 +141,7 @@ ReadTransfer(JSContext* aCx, JSStructuredCloneReader* aReader,
   ErrorResult rv;
   nsRefPtr<MessagePort> port =
     MessagePort::Create(closure->mWindow,
-                        closure->mClosure.mMessagePortIdentifiers[(uint32_t)aExtraData],
+                        closure->mClosure.mMessagePortIdentifiers[aExtraData],
                         rv);
   if (NS_WARN_IF(rv.Failed())) {
     return false;
@@ -195,13 +195,27 @@ WriteTransfer(JSContext* aCx, JS::Handle<JSObject*> aObj, void* aClosure,
   return true;
 }
 
+void
+FreeTransfer(uint32_t aTag, JS::TransferableOwnership aOwnership,
+             void* aContent, uint64_t aExtraData, void* aClosure)
+{
+  MOZ_ASSERT(aClosure);
+  auto* closure = static_cast<StructuredCloneClosureInternal*>(aClosure);
+
+  if (aTag == SCTAG_DOM_MAP_MESSAGEPORT) {
+    MOZ_ASSERT(!aContent);
+    MOZ_ASSERT(aExtraData < closure->mClosure.mMessagePortIdentifiers.Length());
+    MessagePort::ForceClose(closure->mClosure.mMessagePortIdentifiers[(uint32_t)aExtraData]);
+  }
+}
+
 const JSStructuredCloneCallbacks gCallbacks = {
   Read,
   Write,
   Error,
   ReadTransfer,
   WriteTransfer,
-  nullptr
+  FreeTransfer,
 };
 
 } // anonymous namespace
diff --git a/dom/messagechannel/tests/mochitest.ini b/dom/messagechannel/tests/mochitest.ini
index fc5cecd1304a..8633b3cdc9f2 100644
--- a/dom/messagechannel/tests/mochitest.ini
+++ b/dom/messagechannel/tests/mochitest.ini
@@ -23,3 +23,4 @@ support-files =
 [test_messageChannel_sharedWorker.html]
 [test_messageChannel_sharedWorker2.html]
 [test_messageChannel_any.html]
+[test_messageChannel_forceClose.html]
diff --git a/dom/messagechannel/tests/test_messageChannel_forceClose.html b/dom/messagechannel/tests/test_messageChannel_forceClose.html
new file mode 100644
index 000000000000..dae2292ab8d4
--- /dev/null
+++ b/dom/messagechannel/tests/test_messageChannel_forceClose.html
@@ -0,0 +1,36 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1176034
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test for Bug 1176034 - start/close</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1176034">Mozilla Bug 1176034</a>
+<div id="content"></div>
+<pre id="test">
+</pre>
+  <script type="application/javascript">
+
+  function runTests() {
+    var mc = new MessageChannel();
+
+    try {
+      postMessage(42, "*", [ mc.port1, window ]);
+      ok(false, "Something went wrong.");
+    } catch(e) {
+      ok(true, "PostMessage should fail and we should not leak.");
+    }
+
+    SimpleTest.finish();
+  }
+
+  SimpleTest.waitForExplicitFinish();
+  SpecialPowers.pushPrefEnv({"set": [["dom.messageChannel.enabled", true]]}, runTests);
+  </script>
+</body>
+</html>
diff --git a/dom/workers/WorkerPrivate.cpp b/dom/workers/WorkerPrivate.cpp
index 3ad44e905d37..43eac9e7b54a 100644
--- a/dom/workers/WorkerPrivate.cpp
+++ b/dom/workers/WorkerPrivate.cpp
@@ -763,7 +763,14 @@ struct WorkerStructuredCloneCallbacks
   FreeTransfer(uint32_t aTag, JS::TransferableOwnership aOwnership,
                void *aContent, uint64_t aExtraData, void* aClosure)
   {
-    // Nothing to do.
+    if (aTag == SCTAG_DOM_MAP_MESSAGEPORT) {
+      MOZ_ASSERT(aClosure);
+      MOZ_ASSERT(!aContent);
+      auto* closure = static_cast<WorkerStructuredCloneClosure*>(aClosure);
+
+      MOZ_ASSERT(aExtraData < closure->mMessagePortIdentifiers.Length());
+      dom::MessagePort::ForceClose(closure->mMessagePortIdentifiers[aExtraData]);
+    }
   }
 };
 
diff --git a/ipc/glue/BackgroundParentImpl.cpp b/ipc/glue/BackgroundParentImpl.cpp
index f29ed4fc6631..8956ab5d5095 100644
--- a/ipc/glue/BackgroundParentImpl.cpp
+++ b/ipc/glue/BackgroundParentImpl.cpp
@@ -588,6 +588,17 @@ BackgroundParentImpl::DeallocPMessagePortParent(PMessagePortParent* aActor)
   return true;
 }
 
+bool
+BackgroundParentImpl::RecvMessagePortForceClose(const nsID& aUUID,
+                                                const nsID& aDestinationUUID,
+                                                const uint32_t& aSequenceID)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+
+  return MessagePortParent::ForceClose(aUUID, aDestinationUUID, aSequenceID);
+}
+
 } // namespace ipc
 } // namespace mozilla
 
diff --git a/ipc/glue/BackgroundParentImpl.h b/ipc/glue/BackgroundParentImpl.h
index 6dd54f75f267..777303f8feec 100644
--- a/ipc/glue/BackgroundParentImpl.h
+++ b/ipc/glue/BackgroundParentImpl.h
@@ -137,6 +137,11 @@ protected:
 
   virtual bool
   DeallocPMessagePortParent(PMessagePortParent* aActor) override;
+
+  virtual bool
+  RecvMessagePortForceClose(const nsID& aUUID,
+                            const nsID& aDestinationUUID,
+                            const uint32_t& aSequenceID) override;
 };
 
 } // namespace ipc
diff --git a/ipc/glue/PBackground.ipdl b/ipc/glue/PBackground.ipdl
index 6b09ed079cbd..705b094ed062 100644
--- a/ipc/glue/PBackground.ipdl
+++ b/ipc/glue/PBackground.ipdl
@@ -60,6 +60,8 @@ parent:
 
   PMessagePort(nsID uuid, nsID destinationUuid, uint32_t sequenceId);
 
+  MessagePortForceClose(nsID uuid, nsID destinationUuid, uint32_t sequenceId);
+
 child:
   PCache();
   PCacheStreamControl();