Bug 1812498 - Destroy RenderBufferTextureHosts that use VideoBridgeParent's Shmems in VideoBridgeParent::OnChannelError() r=lsalzman

Destroy RenderBufferTextureHosts that use VideoBridgeParent's Shmems before destroying all VideoBridgeParent's Shmems by PVideoBridgeParent::OnChannelError().

Differential Revision: https://phabricator.services.mozilla.com/D169796

gfx/layers/composite/TextureHost.h

@@ -64,6 +64,7 @@ class SurfaceDescriptor;
 class HostIPCAllocator;
 class ISurfaceAllocator;
 class MacIOSurfaceTextureHostOGL;
+class ShmemTextureHost;
 class SurfaceTextureHost;
 class TextureHostOGL;
 class TextureReadLock;
@@ -595,6 +596,7 @@ class TextureHost : public AtomicRefCountedWithFinalize<TextureHost> {
   TextureReadLock* GetReadLock() { return mReadLock; }
 
   virtual BufferTextureHost* AsBufferTextureHost() { return nullptr; }
+  virtual ShmemTextureHost* AsShmemTextureHost() { return nullptr; }
   virtual MacIOSurfaceTextureHostOGL* AsMacIOSurfaceTextureHost() {
     return nullptr;
   }
@@ -868,6 +870,8 @@ class ShmemTextureHost : public BufferTextureHost {
 
   void OnShutdown() override;
 
+  ShmemTextureHost* AsShmemTextureHost() override { return this; }
+
  protected:
   UniquePtr<mozilla::ipc::Shmem> mShmem;
   RefPtr<ISurfaceAllocator> mDeallocator;

gfx/layers/ipc/VideoBridgeParent.cpp

@@ -10,6 +10,7 @@
 #include "mozilla/ipc/Endpoint.h"
 #include "mozilla/layers/TextureHost.h"
 #include "mozilla/layers/VideoBridgeUtils.h"
+#include "mozilla/webrender/RenderThread.h"
 
 namespace mozilla {
 namespace layers {
@@ -200,5 +201,27 @@ bool VideoBridgeParent::IsSameProcess() const {
 void VideoBridgeParent::NotifyNotUsed(PTextureParent* aTexture,
                                       uint64_t aTransactionId) {}
 
+void VideoBridgeParent::OnChannelError() {
+  bool shutdown = sVideoBridgeParentShutDown;
+  if (!shutdown) {
+    // Destory RenderBufferTextureHosts. Shmems of ShmemTextureHosts are going
+    // to be destroyed
+    std::vector<wr::ExternalImageId> ids;
+    auto& ptextures = ManagedPTextureParent();
+    for (const auto& ptexture : ptextures) {
+      RefPtr<TextureHost> texture = TextureHost::AsTextureHost(ptexture);
+      if (texture && texture->AsShmemTextureHost() &&
+          texture->GetMaybeExternalImageId().isSome()) {
+        ids.emplace_back(texture->GetMaybeExternalImageId().ref());
+      }
+    }
+    if (!ids.empty()) {
+      wr::RenderThread::Get()->DestroyExternalImagesSyncWait(std::move(ids));
+    }
+  }
+
+  PVideoBridgeParent::OnChannelError();
+}
+
 }  // namespace layers
 }  // namespace mozilla

gfx/layers/ipc/VideoBridgeParent.h

@@ -60,6 +60,8 @@ class VideoBridgeParent final : public PVideoBridgeParent,
 
   bool DeallocShmem(ipc::Shmem& aShmem) override;
 
+  void OnChannelError() override;
+
  private:
   explicit VideoBridgeParent(VideoBridgeSource aSource);
   void Bind(Endpoint<PVideoBridgeParent>&& aEndpoint);

gfx/webrender_bindings/RenderBufferTextureHost.cpp

@@ -51,8 +51,10 @@ wr::WrExternalImage RenderBufferTextureHost::Lock(uint8_t aChannelIndex,
                                                   gl::GLContext* aGL) {
   if (!mLocked) {
     if (!GetBuffer()) {
-      // We hit some problems to get the shmem.
-      gfxCriticalNote << "GetBuffer Failed";
+      if (!mDestroyed) {
+        // We hit some problems to get the shmem.
+        gfxCriticalNote << "GetBuffer Failed";
+      }
       return InvalidToWrExternalImage();
     }
     if (mFormat != gfx::SurfaceFormat::YUV) {
@@ -196,8 +198,10 @@ bool RenderBufferTextureHost::MapPlane(RenderCompositor* aCompositor,
                                        uint8_t aChannelIndex,
                                        PlaneInfo& aPlaneInfo) {
   if (!mBuffer) {
-    // We hit some problems to get the shmem.
-    gfxCriticalNote << "GetBuffer Failed";
+    if (!mDestroyed) {
+      // We hit some problems to get the shmem.
+      gfxCriticalNote << "GetBuffer Failed";
+    }
     return false;
   }
 
@@ -241,5 +245,10 @@ bool RenderBufferTextureHost::MapPlane(RenderCompositor* aCompositor,
 
 void RenderBufferTextureHost::UnmapPlanes() {}
 
+void RenderBufferTextureHost::Destroy() {
+  mBuffer = nullptr;
+  mDestroyed = true;
+}
+
 }  // namespace wr
 }  // namespace mozilla

gfx/webrender_bindings/RenderBufferTextureHost.h

@@ -48,6 +48,8 @@ class RenderBufferTextureHost final : public RenderTextureHostSWGL {
 
   void UnmapPlanes() override;
 
+  void Destroy() override;
+
  private:
   virtual ~RenderBufferTextureHost();
 
@@ -69,6 +71,8 @@ class RenderBufferTextureHost final : public RenderTextureHostSWGL {
   gfx::DataSourceSurface::MappedSurface mCrMap;
 
   bool mLocked;
+
+  bool mDestroyed = false;
 };
 
 }  // namespace wr

gfx/webrender_bindings/RenderTextureHost.cpp

@@ -50,5 +50,9 @@ std::pair<gfx::Point, gfx::Point> RenderTextureHost::GetUvCoords(
                                    static_cast<float>(aTextureSize.height)));
 }
 
+void RenderTextureHost::Destroy() {
+  MOZ_ASSERT_UNREACHABLE("unexpected to be called");
+}
+
 }  // namespace wr
 }  // namespace mozilla

gfx/webrender_bindings/RenderTextureHost.h

@@ -104,6 +104,8 @@ class RenderTextureHost {
 
   virtual bool IsWrappingAsyncRemoteTexture() { return false; }
 
+  virtual void Destroy();
+
  protected:
   virtual ~RenderTextureHost();
 

gfx/webrender_bindings/RenderThread.cpp

@@ -24,6 +24,7 @@
 #include "mozilla/layers/WebRenderBridgeParent.h"
 #include "mozilla/layers/SharedSurfacesParent.h"
 #include "mozilla/layers/SurfacePool.h"
+#include "mozilla/layers/SynchronousTask.h"
 #include "mozilla/PerfStats.h"
 #include "mozilla/StaticPtr.h"
 #include "mozilla/Telemetry.h"
@@ -808,6 +809,50 @@ void RenderThread::UnregisterExternalImage(
   }
 }
 
+void RenderThread::DestroyExternalImagesSyncWait(
+    const std::vector<wr::ExternalImageId>&& aIds) {
+  if (!IsInRenderThread()) {
+    layers::SynchronousTask task("Destroy external images");
+
+    RefPtr<Runnable> runnable = NS_NewRunnableFunction(
+        "RenderThread::DestroyExternalImagesSyncWait::Runnable",
+        [&task, ids = std::move(aIds)]() {
+          layers::AutoCompleteTask complete(&task);
+          RenderThread::Get()->DestroyExternalImages(std::move(ids));
+        });
+
+    PostRunnable(runnable.forget());
+    task.Wait();
+    return;
+  }
+  DestroyExternalImages(std::move(aIds));
+}
+
+void RenderThread::DestroyExternalImages(
+    const std::vector<wr::ExternalImageId>&& aIds) {
+  MOZ_ASSERT(IsInRenderThread());
+
+  std::vector<RefPtr<RenderTextureHost>> hosts;
+  {
+    MutexAutoLock lock(mRenderTextureMapLock);
+    if (mHasShutdown) {
+      return;
+    }
+
+    for (auto& id : aIds) {
+      auto it = mRenderTextures.find(id);
+      if (it == mRenderTextures.end()) {
+        continue;
+      }
+      hosts.emplace_back(it->second);
+    }
+  }
+
+  for (auto& host : hosts) {
+    host->Destroy();
+  }
+}
+
 void RenderThread::PrepareForUse(const wr::ExternalImageId& aExternalImageId) {
   AddRenderTextureOp(RenderTextureOp::PrepareForUse, aExternalImageId);
 }

gfx/webrender_bindings/RenderThread.h

@@ -204,6 +204,10 @@ class RenderThread final {
   /// Can be called from any thread.
   void UnregisterExternalImage(const wr::ExternalImageId& aExternalImageId);
 
+  /// Can be called from any thread.
+  void DestroyExternalImagesSyncWait(
+      const std::vector<wr::ExternalImageId>&& aIds);
+
   /// Can be called from any thread.
   void PrepareForUse(const wr::ExternalImageId& aExternalImageId);
 
@@ -321,6 +325,8 @@ class RenderThread final {
 
   void CreateSingletonGL(nsACString& aError);
 
+  void DestroyExternalImages(const std::vector<wr::ExternalImageId>&& aIds);
+
   ~RenderThread();
 
   RefPtr<nsIThread> const mThread;