Bug 907463. Dealloc shmem on ActorDestroy. r=nical

gfx/layers/client/CanvasClient.cpp

@@ -104,6 +104,14 @@ CanvasClient2D::CreateBufferTextureClient(gfx::SurfaceFormat aFormat)
                                                        mTextureInfo.mTextureFlags);
 }
 
+void
+CanvasClient2D::OnActorDestroy()
+{
+  if (mBuffer) {
+    mBuffer->OnActorDestroy();
+  }
+}
+
 void
 DeprecatedCanvasClient2D::Updated()
 {
@@ -155,6 +163,14 @@ DeprecatedCanvasClient2D::Update(gfx::IntSize aSize, ClientCanvasLayer* aLayer)
   mDeprecatedTextureClient->Unlock();
 }
 
+void
+DeprecatedCanvasClient2D::OnActorDestroy()
+{
+  if (mDeprecatedTextureClient) {
+    mDeprecatedTextureClient->OnActorDestroy();
+  }
+}
+
 void
 DeprecatedCanvasClientSurfaceStream::Updated()
 {
@@ -223,5 +239,13 @@ DeprecatedCanvasClientSurfaceStream::Update(gfx::IntSize aSize, ClientCanvasLaye
   aLayer->Painted();
 }
 
+void
+DeprecatedCanvasClientSurfaceStream::OnActorDestroy()
+{
+  if (mDeprecatedTextureClient) {
+    mDeprecatedTextureClient->OnActorDestroy();
+  }
+}
+
 }
 }

gfx/layers/client/CanvasClient.h

@@ -90,9 +90,12 @@ public:
     mBuffer = nullptr;
   }
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 private:
   RefPtr<TextureClient> mBuffer;
 };
+
 class DeprecatedCanvasClient2D : public CanvasClient
 {
 public:
@@ -113,6 +116,8 @@ public:
     mDeprecatedTextureClient->SetDescriptorFromReply(aDescriptor);
   }
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 private:
   RefPtr<DeprecatedTextureClient> mDeprecatedTextureClient;
 };
@@ -139,6 +144,8 @@ public:
     mDeprecatedTextureClient->SetDescriptorFromReply(aDescriptor);
   }
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 private:
   RefPtr<DeprecatedTextureClient> mDeprecatedTextureClient;
 };

gfx/layers/client/CompositableClient.cpp

@@ -35,7 +35,6 @@ CompositableClient::~CompositableClient()
   Destroy();
 
   FlushTexturesToRemoveCallbacks();
-
   MOZ_ASSERT(mTexturesToRemove.Length() == 0, "would leak textures pending for deletion");
 }
 
@@ -87,6 +86,7 @@ CompositableClient::Destroy()
   if (!mCompositableChild) {
     return;
   }
+  mCompositableChild->SetClient(nullptr);
   mCompositableChild->Destroy();
   mCompositableChild = nullptr;
 }
@@ -263,5 +263,13 @@ CompositableClient::OnTransaction()
   mTexturesToRemove.Clear();
 }
 
+
+void
+CompositableChild::ActorDestroy(ActorDestroyReason why)
+{
+  if (mCompositableClient) {
+    mCompositableClient->OnActorDestroy();
+  }
+}
 } // namespace layers
 } // namespace mozilla

gfx/layers/client/CompositableClient.h

@@ -144,6 +144,12 @@ public:
   void OnReplyTextureRemoved(uint64_t aTextureID);
 
   void FlushTexturesToRemoveCallbacks();
+
+  /**
+   * Our IPDL actor is being destroyed, get rid of any shmem resources now.
+   */
+  virtual void OnActorDestroy() = 0;
+
 protected:
   struct TextureIDAndFlags {
     TextureIDAndFlags(uint64_t aID, TextureFlags aFlags)
@@ -190,6 +196,8 @@ public:
     return mCompositableClient;
   }
 
+  virtual void ActorDestroy(ActorDestroyReason why) MOZ_OVERRIDE;
+
   void SetAsyncID(uint64_t aID) { mID = aID; }
   uint64_t GetAsyncID() const
   {

gfx/layers/client/ContentClient.cpp

@@ -326,6 +326,21 @@ ContentClientRemoteBuffer::SwapBuffers(const nsIntRegion& aFrontUpdatedRegion)
   }
 }
 
+
+void
+ContentClientRemoteBuffer::OnActorDestroy()
+{
+  if (mDeprecatedTextureClient) {
+    mDeprecatedTextureClient->OnActorDestroy();
+  }
+  if (mDeprecatedTextureClientOnWhite) {
+    mDeprecatedTextureClientOnWhite->OnActorDestroy();
+  }
+  for (size_t i = 0; i < mOldTextures.Length(); ++i) {
+    mOldTextures[i]->OnActorDestroy();
+  }
+}
+
 ContentClientDoubleBuffered::~ContentClientDoubleBuffered()
 {
   if (mDeprecatedTextureClient) {
@@ -426,6 +441,26 @@ ContentClientDoubleBuffered::SwapBuffers(const nsIntRegion& aFrontUpdatedRegion)
   ContentClientRemoteBuffer::SwapBuffers(aFrontUpdatedRegion);
 }
 
+void
+ContentClientDoubleBuffered::OnActorDestroy()
+{
+  if (mDeprecatedTextureClient) {
+    mDeprecatedTextureClient->OnActorDestroy();
+  }
+  if (mDeprecatedTextureClientOnWhite) {
+    mDeprecatedTextureClientOnWhite->OnActorDestroy();
+  }
+  for (size_t i = 0; i < mOldTextures.Length(); ++i) {
+    mOldTextures[i]->OnActorDestroy();
+  }
+  if (mFrontClient) {
+    mFrontClient->OnActorDestroy();
+  }
+  if (mFrontClientOnWhite) {
+    mFrontClientOnWhite->OnActorDestroy();
+  }
+}
+
 struct AutoDeprecatedTextureClient {
   AutoDeprecatedTextureClient()
     : mTexture(nullptr)

gfx/layers/client/ContentClient.h

@@ -163,6 +163,8 @@ public:
     MOZ_CRASH("Should not be called on non-remote ContentClient");
   }
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE {}
+
 private:
   BasicLayerManager* mManager;
 };
@@ -247,6 +249,8 @@ public:
     return mTextureInfo;
   }
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 protected:
   virtual nsIntRegion GetUpdatedRegion(const nsIntRegion& aRegionToDraw,
                                        const nsIntRegion& aVisibleRegion,
@@ -310,6 +314,8 @@ protected:
   virtual void DestroyFrontBuffer() MOZ_OVERRIDE;
   virtual void LockFrontBuffer() MOZ_OVERRIDE;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 private:
   void UpdateDestinationFrom(const RotatedBuffer& aSource,
                              const nsIntRegion& aUpdateRegion);

gfx/layers/client/ImageClient.cpp

@@ -268,6 +268,25 @@ ImageClientBuffered::UpdateImage(ImageContainer* aContainer,
   return ImageClientSingle::UpdateImage(aContainer, aContentFlags);
 }
 
+void
+ImageClientSingle::OnActorDestroy()
+{
+  if (mFrontBuffer) {
+    mFrontBuffer->OnActorDestroy();
+  }
+}
+
+void
+ImageClientBuffered::OnActorDestroy()
+{
+  if (mFrontBuffer) {
+    mFrontBuffer->OnActorDestroy();
+  }
+  if (mBackBuffer) {
+    mBackBuffer->OnActorDestroy();
+  }
+}
+
 void
 ImageClientSingle::AddTextureClient(TextureClient* aTexture)
 {
@@ -460,6 +479,14 @@ ImageClientBridge::ImageClientBridge(CompositableForwarder* aFwd,
 {
 }
 
+void
+DeprecatedImageClientSingle::OnActorDestroy()
+{
+  if (mDeprecatedTextureClient) {
+    mDeprecatedTextureClient->OnActorDestroy();
+  }
+}
+
 bool
 ImageClientBridge::UpdateImage(ImageContainer* aContainer, uint32_t aContentFlags)
 {

gfx/layers/client/ImageClient.h

@@ -104,6 +104,8 @@ public:
 
   virtual void FlushImage() MOZ_OVERRIDE;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 protected:
   RefPtr<TextureClient> mFrontBuffer;
   // Some layers may want to enforce some flags to all their textures
@@ -127,6 +129,8 @@ public:
 
   virtual void FlushImage() MOZ_OVERRIDE;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 protected:
   RefPtr<TextureClient> mBackBuffer;
 };
@@ -171,6 +175,8 @@ public:
   virtual already_AddRefed<Image> CreateImage(const uint32_t *aFormats,
                                               uint32_t aNumFormats) MOZ_OVERRIDE;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 private:
   RefPtr<DeprecatedTextureClient> mDeprecatedTextureClient;
   TextureInfo mTextureInfo;
@@ -212,6 +218,8 @@ public:
     return nullptr;
   }
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE {}
+
 protected:
   uint64_t mAsyncContainerID;
   ShadowableLayer* mLayer;

gfx/layers/client/TextureClient.h

@@ -191,6 +191,10 @@ public:
 
   void MarkInvalid() { mValid = false; }
 
+  // If a texture client holds a reference to shmem, it should override this
+  // method to forget about the shmem _without_ releasing it.
+  virtual void OnActorDestroy() {}
+
 protected:
   void AddFlags(TextureFlags  aFlags)
   {
@@ -298,6 +302,11 @@ public:
 
   ipc::Shmem& GetShmem() { return mShmem; }
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE
+  {
+    mShmem = ipc::Shmem();
+  }
+
 protected:
   ipc::Shmem mShmem;
   ISurfaceAllocator* mAllocator;
@@ -474,6 +483,13 @@ public:
 
   virtual gfxContentType GetContentType() = 0;
 
+  void OnActorDestroy()
+  {
+    if (mDescriptor.type() == SurfaceDescriptor::TShmem) {
+      mDescriptor = SurfaceDescriptor();
+    }
+  }
+
 protected:
   DeprecatedTextureClient(CompositableForwarder* aForwarder,
                 const TextureInfo& aTextureInfo);

gfx/layers/client/TiledContentClient.h

@@ -201,6 +201,14 @@ public:
   static BasicTiledLayerBuffer OpenDescriptor(ISurfaceAllocator* aAllocator,
                                               const SurfaceDescriptorTiles& aDescriptor);
 
+  void OnActorDestroy()
+  {
+    for (size_t i = 0; i < mRetainedTiles.Length(); i++) {
+      if (mRetainedTiles[i].IsPlaceholderTile()) continue;
+      mRetainedTiles[i].mDeprecatedTextureClient->OnActorDestroy();
+    }
+  }
+
 protected:
   BasicTiledLayerTile ValidateTile(BasicTiledLayerTile aTile,
                                    const nsIntPoint& aTileRect,
@@ -290,6 +298,12 @@ public:
   };
   void LockCopyAndWrite(TiledBufferType aType);
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE
+  {
+    mTiledBuffer.OnActorDestroy();
+    mLowPrecisionTiledBuffer.OnActorDestroy();
+  }
+
 private:
   BasicTiledLayerBuffer mTiledBuffer;
   BasicTiledLayerBuffer mLowPrecisionTiledBuffer;

gfx/layers/composite/CompositableHost.cpp

@@ -234,6 +234,7 @@ void
 CompositableParent::ActorDestroy(ActorDestroyReason why)
 {
   if (mHost) {
+    mHost->OnActorDestroy();
     mHost->Detach();
   }
 }

gfx/layers/composite/CompositableHost.h

@@ -106,6 +106,12 @@ public:
     mQuirks = aQuirks;
   }
 
+  /**
+   * Our IPDL actor is being destroyed, get rid of any shmem resources now and
+   * don't worry about compositing anymore.
+   */
+  virtual void OnActorDestroy() = 0;
+
   // If base class overrides, it should still call the parent implementation
   virtual void SetCompositor(Compositor* aCompositor);
 

gfx/layers/composite/ContentHost.cpp

@@ -52,6 +52,23 @@ ContentHostBase::DestroyFrontHost()
   mDeprecatedTextureHostOnWhite = nullptr;
 }
 
+void
+ContentHostBase::OnActorDestroy()
+{
+  if (mDeprecatedTextureHost) {
+    mDeprecatedTextureHost->OnActorDestroy();
+  }
+  if (mDeprecatedTextureHostOnWhite) {
+    mDeprecatedTextureHostOnWhite->OnActorDestroy();
+  }
+  if (mNewFrontHost) {
+    mNewFrontHost->OnActorDestroy();
+  }
+  if (mNewFrontHostOnWhite) {
+    mNewFrontHostOnWhite->OnActorDestroy();
+  }
+}
+
 void
 ContentHostBase::Composite(EffectChain& aEffectChain,
                            float aOpacity,
@@ -404,19 +421,16 @@ ContentHostDoubleBuffered::DestroyTextures()
                "We won't be able to destroy our SurfaceDescriptor");
     mNewFrontHost = nullptr;
   }
-
   if (mNewFrontHostOnWhite) {
     MOZ_ASSERT(mNewFrontHostOnWhite->GetDeAllocator(),
                "We won't be able to destroy our SurfaceDescriptor");
     mNewFrontHostOnWhite = nullptr;
   }
-
   if (mBackHost) {
     MOZ_ASSERT(mBackHost->GetDeAllocator(),
                "We won't be able to destroy our SurfaceDescriptor");
     mBackHost = nullptr;
   }
-
   if (mBackHostOnWhite) {
     MOZ_ASSERT(mBackHostOnWhite->GetDeAllocator(),
                "We won't be able to destroy our SurfaceDescriptor");
@@ -426,6 +440,29 @@ ContentHostDoubleBuffered::DestroyTextures()
   // don't touch mDeprecatedTextureHost, we might need it for compositing
 }
 
+void
+ContentHostDoubleBuffered::OnActorDestroy()
+{
+  if (mDeprecatedTextureHost) {
+    mDeprecatedTextureHost->OnActorDestroy();
+  }
+  if (mDeprecatedTextureHostOnWhite) {
+    mDeprecatedTextureHostOnWhite->OnActorDestroy();
+  }
+  if (mNewFrontHost) {
+    mNewFrontHost->OnActorDestroy();
+  }
+  if (mNewFrontHostOnWhite) {
+    mNewFrontHostOnWhite->OnActorDestroy();
+  }
+  if (mBackHost) {
+    mBackHost->OnActorDestroy();
+  }
+  if (mBackHostOnWhite) {
+    mBackHostOnWhite->OnActorDestroy();
+  }
+}
+
 void
 ContentHostDoubleBuffered::UpdateThebes(const ThebesBufferData& aData,
                                         const nsIntRegion& aUpdated,

gfx/layers/composite/ContentHost.h

@@ -128,6 +128,8 @@ public:
   // destroy our front buffer so that we can continue to composite.
   virtual void DestroyTextures() = 0;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 protected:
   virtual nsIntPoint GetOriginOffset()
   {
@@ -178,6 +180,8 @@ public:
                                  const TextureInfo& aTextureInfo) MOZ_OVERRIDE;
   virtual void DestroyTextures() MOZ_OVERRIDE;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 #ifdef MOZ_DUMP_PAINTING
   virtual void Dump(FILE* aFile=nullptr,
                     const char* aPrefix="",

gfx/layers/composite/ImageHost.h

@@ -69,6 +69,13 @@ public:
 
   virtual LayerRenderState GetRenderState() MOZ_OVERRIDE;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE
+  {
+    if (mFrontBuffer) {
+      mFrontBuffer->OnActorDestroy();
+    }
+  }
+
 #ifdef MOZ_LAYERS_HAVE_LOG
   virtual void PrintInfo(nsACString& aTo, const char* aPrefix);
 #endif
@@ -132,6 +139,13 @@ public:
 
   virtual void SetCompositor(Compositor* aCompositor) MOZ_OVERRIDE;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE
+  {
+    if (mDeprecatedTextureHost) {
+      mDeprecatedTextureHost->OnActorDestroy();
+    }
+  }
+
 #ifdef MOZ_LAYERS_HAVE_LOG
   virtual void PrintInfo(nsACString& aTo, const char* aPrefix);
 #endif

gfx/layers/composite/TextureHost.cpp

@@ -244,6 +244,15 @@ DeprecatedTextureHost::SwapTextures(const SurfaceDescriptor& aImage,
   SetBuffer(mBuffer, mDeAllocator);
 }
 
+void
+DeprecatedTextureHost::OnActorDestroy()
+{
+  if (mBuffer && mBuffer->type() == SurfaceDescriptor::TShmem) {
+    *mBuffer = SurfaceDescriptor();
+    mBuffer = nullptr;
+  }
+}
+
 #ifdef MOZ_LAYERS_HAVE_LOG
 
 void
@@ -511,6 +520,13 @@ ShmemTextureHost::DeallocateSharedData()
   }
 }
 
+void
+ShmemTextureHost::OnActorDestroy()
+{
+  delete mShmem;
+  mShmem = nullptr;
+}
+
 uint8_t* ShmemTextureHost::GetBuffer()
 {
   return mShmem ? mShmem->get<uint8_t>() : nullptr;

gfx/layers/composite/TextureHost.h

@@ -384,6 +384,10 @@ public:
 
   virtual void SetCompositableQuirks(CompositableQuirks* aQuirks);
 
+  // If a texture host holds a reference to shmem, it should override this method
+  // to forget about the shmem _without_ releasing it.
+  virtual void OnActorDestroy() {}
+
 #ifdef MOZ_LAYERS_HAVE_LOG
   virtual const char *Name() { return "TextureHost"; }
   virtual void PrintInfo(nsACString& aTo, const char* aPrefix);
@@ -484,6 +488,8 @@ public:
   virtual const char *Name() MOZ_OVERRIDE { return "ShmemTextureHost"; }
 #endif
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE;
+
 protected:
   ipc::Shmem* mShmem;
   ISurfaceAllocator* mDeallocator;
@@ -718,6 +724,8 @@ public:
   // see bug 865908 about fixing this.
   virtual void ForgetBuffer() {}
 
+  void OnActorDestroy();
+
 protected:
   /**
    * Should be implemented by the backend-specific DeprecatedTextureHost classes

gfx/layers/composite/TiledContentHost.h

@@ -119,6 +119,16 @@ public:
     mCompositor = aCompositor;
   }
 
+  void OnActorDestroy()
+  {
+    Iterator end = TilesEnd();
+    for (Iterator it = TilesBegin(); it != end; ++it) {
+      if (it->mDeprecatedTextureHost) {
+        it->mDeprecatedTextureHost->OnActorDestroy();
+      }
+    }
+  }
+
 protected:
   TiledTexture ValidateTile(TiledTexture aTile,
                             const nsIntPoint& aTileRect,
@@ -241,6 +251,12 @@ public:
                       Compositor* aCompositor,
                       AttachFlags aFlags = NO_FLAGS) MOZ_OVERRIDE;
 
+  virtual void OnActorDestroy() MOZ_OVERRIDE
+  {
+    mVideoMemoryTiledBuffer.OnActorDestroy();
+    mLowPrecisionVideoMemoryTiledBuffer.OnActorDestroy();
+  }
+
 #ifdef MOZ_DUMP_PAINTING
   virtual void Dump(FILE* aFile=nullptr,
                     const char* aPrefix="",