Bug 1718210 - Release the compositor thread earlier r=mattwoodrow

While running SandboxTest on Windows/Debug setup we can encounter a GPU
process hanging at shutdown. This is due to a destruction race between
GPUParent and VideoBridgeParent, and we end up with one
CompositorThreadHolder reference that is not properly released, making
the GPU process waiting for it.

This change aims at doing the release earlier to prevent such a
destruction race.

Differential Revision: https://phabricator.services.mozilla.com/D121045

dom/media/ipc/RemoteDecoderManagerParent.cpp

@@ -20,6 +20,7 @@
 #include "mozilla/ipc/Endpoint.h"
 #include "mozilla/layers/ImageDataSerializer.h"
 #include "mozilla/layers/VideoBridgeChild.h"
+#include "mozilla/layers/VideoBridgeParent.h"
 
 namespace mozilla {
 
@@ -93,8 +94,10 @@ void RemoteDecoderManagerParent::ShutdownThreads() {
 void RemoteDecoderManagerParent::ShutdownVideoBridge() {
   if (sRemoteDecoderManagerParentThread) {
     RefPtr<Runnable> task = NS_NewRunnableFunction(
-        "RemoteDecoderManagerParent::ShutdownVideoBridge",
-        []() { VideoBridgeChild::Shutdown(); });
+        "RemoteDecoderManagerParent::ShutdownVideoBridge", []() {
+          VideoBridgeParent::Shutdown();
+          VideoBridgeChild::Shutdown();
+        });
     SyncRunnable::DispatchToThread(sRemoteDecoderManagerParentThread, task);
   }
 }

gfx/layers/ipc/VideoBridgeParent.cpp

@@ -89,8 +89,22 @@ void VideoBridgeParent::ActorDestroy(ActorDestroyReason aWhy) {
   mClosed = true;
 }
 
+/* static */
+void VideoBridgeParent::Shutdown() {
+  if (sVideoBridgeFromRddProcess) {
+    sVideoBridgeFromRddProcess->ReleaseCompositorThread();
+  } else if (sVideoBridgeFromGpuProcess) {
+    sVideoBridgeFromGpuProcess->ReleaseCompositorThread();
+  }
+}
+
+void VideoBridgeParent::ReleaseCompositorThread() {
+  mCompositorThreadHolder = nullptr;
+}
+
 void VideoBridgeParent::ActorDealloc() {
   mCompositorThreadHolder = nullptr;
+  ReleaseCompositorThread();
   mSelfRef = nullptr;
 }
 

gfx/layers/ipc/VideoBridgeParent.h

@@ -27,6 +27,7 @@ class VideoBridgeParent final : public PVideoBridgeParent,
 
   static void Open(Endpoint<PVideoBridgeParent>&& aEndpoint,
                    VideoBridgeSource aSource);
+  static void Shutdown();
 
   TextureHost* LookupTexture(uint64_t aSerial);
 
@@ -65,6 +66,7 @@ class VideoBridgeParent final : public PVideoBridgeParent,
   void Bind(Endpoint<PVideoBridgeParent>&& aEndpoint);
 
   void ActorDealloc() override;
+  void ReleaseCompositorThread();
 
   // This keeps us alive until ActorDestroy(), at which point we do a
   // deferred destruction of ourselves.