Bug 1540136 - P5: Handle if ChromiumCDMProxy is shutdown in the middle of init. r=cpearce

- Watch for if a proxy shuts down during init and if so, shutdown the CDM parent
  that is being initialized.
- Make ChromiumCDMParent only store a pointer to a ChromiumCDMProxy when it has
  successfully initialized. This avoid the lopsided relationship where a if a
  ChromiumCDMParent fails to initialize it may keep a pointer to a proxy, but
  the proxy will never have a reference to that CDM parent.

Differential Revision: https://phabricator.services.mozilla.com/D26208

--HG--
extra : moz-landing-system : lando

dom/media/gmp/ChromiumCDMParent.cpp

@@ -61,7 +61,6 @@ RefPtr<ChromiumCDMParent::InitPromise> ChromiumCDMParent::Init(
                                     !aMainThread ? "true" : "false")),
         __func__);
   }
-  mCDMCallback = aCDMCallback;
 
   RefPtr<ChromiumCDMParent::InitPromise> promise =
       mInitPromise.Ensure(__func__);
@@ -69,7 +68,7 @@ RefPtr<ChromiumCDMParent::InitPromise> ChromiumCDMParent::Init(
   SendInit(aAllowDistinctiveIdentifier, aAllowPersistentState)
       ->Then(
           AbstractThread::GetCurrent(), __func__,
-          [self](bool aSuccess) {
+          [self, aCDMCallback](bool aSuccess) {
             if (!aSuccess) {
               GMP_LOG(
                   "ChromiumCDMParent::Init() failed with callback from "
@@ -83,6 +82,7 @@ RefPtr<ChromiumCDMParent::InitPromise> ChromiumCDMParent::Init(
             }
             GMP_LOG(
                 "ChromiumCDMParent::Init() succeeded with callback from child");
+            self->mCDMCallback = aCDMCallback;
             self->mInitPromise.ResolveIfExists(true /* unused */, __func__);
           },
           [self](ResponseRejectReason&& aReason) {

dom/media/gmp/ChromiumCDMProxy.cpp

@@ -102,6 +102,18 @@ void ChromiumCDMProxy::Init(PromiseId aPromiseId, const nsAString& aOrigin,
                           MutexAutoLock lock(self->mCDMMutex);
                           self->mCDM = cdm;
                         }
+                        if (self->mIsShutdown) {
+                          self->RejectPromise(
+                              aPromiseId, NS_ERROR_DOM_INVALID_STATE_ERR,
+                              NS_LITERAL_CSTRING(
+                                  "ChromiumCDMProxy shutdown during "
+                                  "ChromiumCDMProxy::Init"));
+                          // If shutdown happened while waiting to init, we
+                          // need to explicitly shutdown the CDM to avoid it
+                          // referencing this proxy which is on its way out.
+                          self->ShutdownCDMIfExists();
+                          return;
+                        }
                         self->OnCDMCreated(aPromiseId);
                       },
                       [self, aPromiseId](MediaResult aResult) {
@@ -139,6 +151,29 @@ void ChromiumCDMProxy::OnCDMCreated(uint32_t aPromiseId) {
   }
 }
 
+void ChromiumCDMProxy::ShutdownCDMIfExists() {
+  EME_LOG(
+      "ChromiumCDMProxy::ShutdownCDMIfExists(this=%p) mCDM=%p, mIsShutdown=%s",
+      this, mCDM.get(), mIsShutdown ? "true" : "false");
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(mGMPThread);
+  MOZ_ASSERT(mIsShutdown,
+             "Should only shutdown the CDM if the proxy is shutting down");
+  RefPtr<gmp::ChromiumCDMParent> cdm;
+  {
+    MutexAutoLock lock(mCDMMutex);
+    cdm.swap(mCDM);
+  }
+  if (cdm) {
+    // We need to keep this proxy alive until the parent has finished its
+    // Shutdown (as it may still try to use the proxy until then).
+    RefPtr<ChromiumCDMProxy> self(this);
+    nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
+        "ChromiumCDMProxy::Shutdown", [self, cdm]() { cdm->Shutdown(); });
+    mGMPThread->Dispatch(task.forget());
+  }
+}
+
 #ifdef DEBUG
 bool ChromiumCDMProxy::IsOnOwnerThread() {
   return mGMPThread->IsCurrentThreadIn();
@@ -298,21 +333,14 @@ void ChromiumCDMProxy::RemoveSession(const nsAString& aSessionId,
 
 void ChromiumCDMProxy::Shutdown() {
   MOZ_ASSERT(NS_IsMainThread());
-  EME_LOG("ChromiumCDMProxy::Shutdown(this=%p) mCDM=%p", this, mCDM.get());
+  EME_LOG("ChromiumCDMProxy::Shutdown(this=%p) mCDM=%p, mIsShutdown=%s", this,
+          mCDM.get(), mIsShutdown ? "true" : "false");
+  if (mIsShutdown) {
+    return;
+  }
+  mIsShutdown = true;
   mKeys.Clear();
-  RefPtr<gmp::ChromiumCDMParent> cdm;
-  {
-    MutexAutoLock lock(mCDMMutex);
-    cdm.swap(mCDM);
-  }
-  if (cdm) {
-    // We need to keep this proxy alive until the parent has finished its
-    // Shutdown (as it may still try to use the proxy until then).
-    RefPtr<ChromiumCDMProxy> self(this);
-    nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
-        "ChromiumCDMProxy::Shutdown", [self, cdm]() { cdm->Shutdown(); });
-    mGMPThread->Dispatch(task.forget());
-  }
+  ShutdownCDMIfExists();
 }
 
 void ChromiumCDMProxy::RejectPromise(PromiseId aId, nsresult aCode,

dom/media/gmp/ChromiumCDMProxy.h

@@ -7,8 +7,8 @@
 #ifndef ChromiumCDMProxy_h_
 #define ChromiumCDMProxy_h_
 
-#include "mozilla/CDMProxy.h"
 #include "mozilla/AbstractThread.h"
+#include "mozilla/CDMProxy.h"
 #include "ChromiumCDMParent.h"
 
 namespace mozilla {
@@ -108,9 +108,14 @@ class ChromiumCDMProxy : public CDMProxy {
 
  private:
   void OnCDMCreated(uint32_t aPromiseId);
+  void ShutdownCDMIfExists();
 
   ~ChromiumCDMProxy();
 
+  // True if Shutdown() has been called. Should only be read and written on
+  // main thread.
+  bool mIsShutdown = false;
+
   RefPtr<GMPCrashHelper> mCrashHelper;
 
   Mutex mCDMMutex;