Bug 1289378 - Fix inter-locking in PDMFactory+GMPDecoderModule - r=jya

Say PDMFactory::EnsureInit() runs on the main thread (effectively locking it)
and then tries to lock sMonitor; if another thread was also running
EnsureInit() and locked sMonitor, it will dead-lock when trying to sync-
dispatch the GMPDecoderModule creation to the main thread.

This can be fixed by ensuring that the actual PDMFactory instance creation is
always done on the main thread (and that we don't hold sMonitor before
dispatching to the main thread.)

Note that we can now simplify GMPDecoderModule::Init and assert we are already
on the main thread.

MozReview-Commit-ID: 8xHpoymw6po

--HG--
extra : rebase_source : aabc1f4768aebdd16873bbc1afdd1679d90aed6f

dom/media/platforms/PDMFactory.cpp

@@ -29,6 +29,7 @@
 #include "mozilla/ClearOnShutdown.h"
 #include "mozilla/SharedThreadPool.h"
 #include "mozilla/StaticPtr.h"
+#include "mozilla/SyncRunnable.h"
 #include "mozilla/TaskQueue.h"
 
 #include "MediaInfo.h"
@@ -87,17 +88,31 @@ PDMFactory::~PDMFactory()
 void
 PDMFactory::EnsureInit() const
 {
+  {
+    StaticMutexAutoLock mon(sMonitor);
+    if (sInstance) {
+      // Quick exit if we already have an instance.
+      return;
+    }
+    if (NS_IsMainThread()) {
+      // On the main thread and holding the lock -> Create instance.
+      sInstance = new PDMFactoryImpl();
+      ClearOnShutdown(&sInstance);
+      return;
+    }
+  }
+
+  // Not on the main thread -> Sync-dispatch creation to main thread.
+  nsCOMPtr<nsIThread> mainThread = do_GetMainThread();
+  nsCOMPtr<nsIRunnable> runnable =
+    NS_NewRunnableFunction([]() {
       StaticMutexAutoLock mon(sMonitor);
       if (!sInstance) {
         sInstance = new PDMFactoryImpl();
-    if (NS_IsMainThread()) {
         ClearOnShutdown(&sInstance);
-    } else {
-      nsCOMPtr<nsIRunnable> runnable =
-        NS_NewRunnableFunction([]() { ClearOnShutdown(&sInstance); });
-      NS_DispatchToMainThread(runnable);
-    }
       }
+    });
+  SyncRunnable::DispatchToThread(mainThread, runnable);
 }
 
 already_AddRefed<MediaDataDecoder>

dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp

@@ -13,7 +13,6 @@
 #include "mozIGeckoMediaPluginService.h"
 #include "nsServiceManagerUtils.h"
 #include "mozilla/StaticMutex.h"
-#include "mozilla/SyncRunnable.h"
 #include "gmp-audio-decode.h"
 #include "gmp-video-decode.h"
 #ifdef XP_WIN
@@ -169,13 +168,7 @@ GMPDecoderModule::UpdateUsableCodecs()
 void
 GMPDecoderModule::Init()
 {
-  if (!NS_IsMainThread()) {
+  MOZ_ASSERT(NS_IsMainThread());
-    nsCOMPtr<nsIThread> mainThread = do_GetMainThread();
-    nsCOMPtr<nsIRunnable> runnable =
-      NS_NewRunnableFunction([]() { Init(); });
-    SyncRunnable::DispatchToThread(mainThread, runnable);
-    return;
-  }
   // GMPService::HasPluginForAPI is main thread only, so to implement
   // SupportsMimeType() we build a table of the codecs which each whitelisted
   // GMP has and update it when any GMPs are removed or added at runtime.