Bug 1337761 - Part 2: Don't send external images to the parent side if a transaction is incomplete. r=nical

gfx/layers/wr/WebRenderBridgeChild.cpp

@@ -65,7 +65,6 @@ bool
 WebRenderBridgeChild::DPBegin(const gfx::IntSize& aSize)
 {
   MOZ_ASSERT(!mDestroyed);
-  MOZ_ASSERT(!mIsInTransaction);
 
   UpdateFwdTransactionId();
   this->SendDPBegin(aSize);
@@ -76,11 +75,8 @@ WebRenderBridgeChild::DPBegin(const gfx::IntSize& aSize)
 }
 
 void
-WebRenderBridgeChild::DPEnd(wr::DisplayListBuilder &aBuilder, const gfx::IntSize& aSize, bool aIsSync, uint64_t aTransactionId)
+WebRenderBridgeChild::ClearReadLocks()
 {
-  MOZ_ASSERT(!mDestroyed);
-  MOZ_ASSERT(mIsInTransaction);
-
   for (nsTArray<ReadLockInit>& locks : mReadLocks) {
     if (locks.Length()) {
       if (!SendInitReadLocks(locks)) {
@@ -90,6 +86,15 @@ WebRenderBridgeChild::DPEnd(wr::DisplayListBuilder &aBuilder, const gfx::IntSize
     }
   }
 
+  mReadLocks.Clear();
+}
+
+void
+WebRenderBridgeChild::DPEnd(wr::DisplayListBuilder &aBuilder, const gfx::IntSize& aSize, bool aIsSync, uint64_t aTransactionId)
+{
+  MOZ_ASSERT(!mDestroyed);
+  MOZ_ASSERT(mIsInTransaction);
+
   wr::BuiltDisplayList dl = aBuilder.Finalize();
   ByteBuffer dlData(Move(dl.dl));
   ByteBuffer auxData(Move(dl.aux));
@@ -104,7 +109,6 @@ WebRenderBridgeChild::DPEnd(wr::DisplayListBuilder &aBuilder, const gfx::IntSize
 
   mParentCommands.Clear();
   mDestroyedActors.Clear();
-  mReadLocks.Clear();
   mIsInTransaction = false;
 }
 

gfx/layers/wr/WebRenderBridgeChild.h

@@ -96,6 +96,7 @@ public:
   wr::FontKey GetFontKeyForScaledFont(gfx::ScaledFont* aScaledFont);
 
   void RemoveExpiredFontKeys();
+  void ClearReadLocks();
 
 private:
   friend class CompositorBridgeChild;

gfx/layers/wr/WebRenderLayerManager.cpp

@@ -335,6 +335,7 @@ WebRenderLayerManager::EndEmptyTransaction(EndTransactionFlags aFlags)
     return false;
   }
 
+  // We might used painted layer images so don't delete them yet.
   return EndTransactionInternal(nullptr, nullptr, aFlags);
 }
 
@@ -373,6 +374,15 @@ WebRenderLayerManager::EndTransactionInternal(DrawPaintedLayerCallback aCallback
 
   wr::DisplayListBuilder builder(WrBridge()->GetPipeline());
   WebRenderLayer::ToWebRenderLayer(mRoot)->RenderLayer(builder);
+  WrBridge()->ClearReadLocks();
+
+  // We can't finish this transaction so return. This usually
+  // happens in an empty transaction where we can't repaint a painted layer.
+  // In this case, leave the transaction open and let a full transaction happen.
+  if (mTransactionIncomplete) {
+    DiscardLocalImages();
+    return false;
+  }
 
   bool sync = mTarget != nullptr;
   mLatestTransactionId = mTransactionIdAllocator->GetTransactionId();
@@ -389,7 +399,7 @@ WebRenderLayerManager::EndTransactionInternal(DrawPaintedLayerCallback aCallback
   mKeepAlive.Clear();
   ClearMutatedLayers();
 
-  return !mTransactionIncomplete;
+  return true;
 }
 
 void
@@ -484,6 +494,16 @@ WebRenderLayerManager::DiscardCompositorAnimations()
   mDiscardedCompositorAnimationsIds.clear();
 }
 
+void
+WebRenderLayerManager::DiscardLocalImages()
+{
+  // Removes images but doesn't tell the parent side about them
+  // This is useful in empty / failed transactions where we created
+  // image keys but didn't tell the parent about them yet.
+  mImageKeys.clear();
+}
+
+void
 WebRenderLayerManager::Mutated(Layer* aLayer)
 {
   LayerManager::Mutated(aLayer);

gfx/layers/wr/WebRenderLayerManager.h

@@ -154,6 +154,7 @@ public:
   // transaction or destruction
   void AddImageKeyForDiscard(wr::ImageKey);
   void DiscardImages();
+  void DiscardLocalImages();
 
   // Before destroying a layer with animations, add its compositorAnimationsId
   // to a list of ids that will be discarded on the next transaction

gfx/layers/wr/WebRenderPaintedLayer.cpp

@@ -81,7 +81,7 @@ WebRenderPaintedLayer::UpdateImageClient()
   }
 
   if (!helper.UpdateImage()) {
-    return;
+    return false;
   }
 
   return true;