Bug 1245400 - P3. Report number of frames dropped by compositor back to VideoSink. r=nical

We report the number of frames dropped by the compositor because they were too late through:
ImageComposite -> ImageHost -> CompositableTransactionParent -> ImageBridgeParent -> IPDL -> ImageBridgeChild -> ImageContainerListener -> ImageContainer -> VideoSink

Differential Revision: https://phabricator.services.mozilla.com/D2177

dom/media/mediasink/VideoSink.cpp

@@ -47,6 +47,7 @@ VideoSink::VideoSink(AbstractThread* aThread,
   , mContainer(aContainer)
   , mProducerID(ImageContainer::AllocateProducerID())
   , mFrameStats(aFrameStats)
+  , mOldDroppedCount(0)
   , mHasVideo(false)
   , mUpdateScheduler(aThread)
   , mVideoQueueSendToCompositorSize(aVQueueSentToCompositerSize)
@@ -464,6 +465,13 @@ VideoSink::RenderVideoFrames(int32_t aMaxFrames,
 
   if (images.Length() > 0) {
     mContainer->SetCurrentFrames(frames[0]->mDisplay, images);
+    uint32_t droppedCount = mContainer->GetDroppedImageCount();
+    uint32_t dropped = droppedCount - mOldDroppedCount;
+    if (dropped > 0) {
+      mFrameStats.NotifyDecodedFrames({0, 0, dropped});
+      mOldDroppedCount = droppedCount;
+      VSINK_LOG_V("%u video frame discarded by compositor", dropped);
+    }
   }
 }
 

dom/media/mediasink/VideoSink.h

@@ -108,7 +108,8 @@ private:
     MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   }
 
-  MediaQueue<VideoData>& VideoQueue() const {
+  MediaQueue<VideoData>& VideoQueue() const
+  {
     return mVideoQueue;
   }
 
@@ -131,6 +132,8 @@ private:
   // The presentation end time of the last video frame which has been displayed.
   TimeUnit mVideoFrameEndTime;
 
+  uint32_t mOldDroppedCount;
+
   // Event listeners for VideoQueue
   MediaEventListener mPushListener;
   MediaEventListener mFinishListener;

gfx/layers/ImageContainer.cpp

@@ -121,6 +121,15 @@ ImageContainerListener::NotifyComposite(const ImageCompositeNotification& aNotif
   }
 }
 
+void
+ImageContainerListener::NotifyDropped(uint32_t aDropped)
+{
+  MutexAutoLock lock(mLock);
+  if (mImageContainer) {
+    mImageContainer->NotifyDropped(aDropped);
+  }
+}
+
 void
 ImageContainerListener::ClearImageContainer()
 {
@@ -262,7 +271,7 @@ ImageContainer::SetCurrentImageInternal(const nsTArray<NonOwningImage>& aImages)
       mCurrentProducerID = aImages[0].mProducerID;
     } else if (!aImages[0].mTimeStamp.IsNull()) {
       // Check for expired frames
-      for (auto& img : mCurrentImages) {
+      for (const auto& img : mCurrentImages) {
         if (img.mProducerID != aImages[0].mProducerID ||
             img.mTimeStamp.IsNull() ||
             img.mTimeStamp >= aImages[0].mTimeStamp) {
@@ -278,7 +287,6 @@ ImageContainer::SetCurrentImageInternal(const nsTArray<NonOwningImage>& aImages)
       const uint32_t maxFrames = 100;
       if (mFrameIDsNotYetComposited.Length() > maxFrames) {
         uint32_t dropFrames = mFrameIDsNotYetComposited.Length() - maxFrames;
-        mDroppedImageCount += dropFrames;
         mFrameIDsNotYetComposited.RemoveElementsAt(0, dropFrames);
       }
     }
@@ -303,7 +311,7 @@ ImageContainer::SetCurrentImageInternal(const nsTArray<NonOwningImage>& aImages)
     img->mTimeStamp = aImages[i].mTimeStamp;
     img->mFrameID = aImages[i].mFrameID;
     img->mProducerID = aImages[i].mProducerID;
-    for (auto& oldImg : mCurrentImages) {
+    for (const auto& oldImg : mCurrentImages) {
       if (oldImg.mFrameID == img->mFrameID &&
           oldImg.mProducerID == img->mProducerID) {
         img->mComposited = oldImg.mComposited;
@@ -441,11 +449,7 @@ ImageContainer::NotifyComposite(const ImageCompositeNotification& aNotification)
   if (aNotification.producerID() == mCurrentProducerID) {
     uint32_t i;
     for (i = 0; i < mFrameIDsNotYetComposited.Length(); ++i) {
-      if (mFrameIDsNotYetComposited[i] <= aNotification.frameID()) {
-        if (mFrameIDsNotYetComposited[i] < aNotification.frameID()) {
-          ++mDroppedImageCount;
-        }
-      } else {
+      if (mFrameIDsNotYetComposited[i] > aNotification.frameID()) {
         break;
       }
     }
@@ -458,11 +462,17 @@ ImageContainer::NotifyComposite(const ImageCompositeNotification& aNotification)
   }
 
   if (!aNotification.imageTimeStamp().IsNull()) {
-    mPaintDelay = aNotification.firstCompositeTimeStamp() -
-        aNotification.imageTimeStamp();
+    mPaintDelay =
+      aNotification.firstCompositeTimeStamp() - aNotification.imageTimeStamp();
   }
 }
 
+void
+ImageContainer::NotifyDropped(uint32_t aDropped)
+{
+  mDroppedImageCount += aDropped;
+}
+
 #ifdef XP_WIN
 D3D11YCbCrRecycleAllocator*
 ImageContainer::GetD3D11YCbCrRecycleAllocator(KnowsCompositor* aAllocator)

gfx/layers/ImageContainer.h

@@ -356,6 +356,7 @@ public:
   explicit ImageContainerListener(ImageContainer* aImageContainer);
 
   void NotifyComposite(const ImageCompositeNotification& aNotification);
+  void NotifyDropped(uint32_t aDropped);
   void ClearImageContainer();
   void DropImageClient();
 private:
@@ -613,11 +614,11 @@ public:
    */
   uint32_t GetDroppedImageCount()
   {
-    RecursiveMutexAutoLock lock(mRecursiveMutex);
     return mDroppedImageCount;
   }
 
   void NotifyComposite(const ImageCompositeNotification& aNotification);
+  void NotifyDropped(uint32_t aDropped);
 
   ImageContainerListener* GetImageContainerListener()
   {
@@ -675,8 +676,8 @@ private:
   // See GetPaintDelay. Accessed only with mRecursiveMutex held.
   TimeDuration mPaintDelay;
 
-  // See GetDroppedImageCount. Accessed only with mRecursiveMutex held.
-  uint32_t mDroppedImageCount;
+  // See GetDroppedImageCount.
+  mozilla::Atomic<uint32_t> mDroppedImageCount;
 
   // This is the image factory used by this container, layer managers using
   // this container can set an alternative image factory that will be used to

gfx/layers/composite/CompositableHost.h

@@ -245,6 +245,8 @@ public:
 
   virtual void BindTextureSource() {}
 
+  virtual uint32_t GetDroppedFrames() { return 0; }
+
 protected:
   HostLayerManager* GetLayerManager() const;
 

gfx/layers/composite/ImageComposite.h

@@ -40,6 +40,12 @@ public:
 
   int32_t GetLastFrameID() const { return mLastFrameID; }
   int32_t GetLastProducerID() const { return mLastProducerID; }
+  uint32_t GetDroppedFramesAndReset()
+  {
+    uint32_t dropped = mDroppedFrames;
+    mDroppedFrames = 0;
+    return dropped;
+  }
 
   enum Bias {
     // Don't apply bias to frame times

gfx/layers/composite/ImageHost.h

@@ -91,6 +91,11 @@ public:
 
   bool IsOpaque();
 
+  uint32_t GetDroppedFrames() override
+  {
+    return GetDroppedFramesAndReset();
+  }
+
   struct RenderInfo {
     int imageIndex;
     const TimedImage* img;

gfx/layers/ipc/CompositableTransactionParent.cpp

@@ -68,18 +68,26 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
   if (!compositable) {
     return false;
   }
-  if (TextureSourceProvider* provider = compositable->GetTextureSourceProvider()) {
+  return ReceiveCompositableUpdate(aEdit.detail(), WrapNotNull(compositable));
+}
+
+bool
+CompositableParentManager::ReceiveCompositableUpdate(
+  const CompositableOperationDetail& aDetail,
+  NotNull<CompositableHost*> aCompositable)
+{
+  if (TextureSourceProvider* provider = aCompositable->GetTextureSourceProvider()) {
     if (!provider->IsValid()) {
       return false;
     }
   }
 
-  switch (aEdit.detail().type()) {
+  switch (aDetail.type()) {
     case CompositableOperationDetail::TOpPaintTextureRegion: {
       MOZ_LAYERS_LOG(("[ParentSide] Paint PaintedLayer"));
 
-      const OpPaintTextureRegion& op = aEdit.detail().get_OpPaintTextureRegion();
-      Layer* layer = compositable->GetLayer();
+      const OpPaintTextureRegion& op = aDetail.get_OpPaintTextureRegion();
+      Layer* layer = aCompositable->GetLayer();
       if (!layer || layer->GetType() != Layer::TYPE_PAINTED) {
         return false;
       }
@@ -89,10 +97,9 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
 
       RenderTraceInvalidateStart(thebes, "FF00FF", op.updatedRegion().GetBounds());
 
-      if (!compositable->UpdateThebes(bufferData,
-                                      op.updatedRegion(),
-                                      thebes->GetValidRegion()))
-      {
+      if (!aCompositable->UpdateThebes(bufferData,
+                                       op.updatedRegion(),
+                                       thebes->GetValidRegion())) {
         return false;
       }
 
@@ -101,8 +108,8 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
     }
     case CompositableOperationDetail::TOpUseTiledLayerBuffer: {
       MOZ_LAYERS_LOG(("[ParentSide] Paint TiledLayerBuffer"));
-      const OpUseTiledLayerBuffer& op = aEdit.detail().get_OpUseTiledLayerBuffer();
-      TiledContentHost* tiledHost = compositable->AsTiledContentHost();
+      const OpUseTiledLayerBuffer& op = aDetail.get_OpUseTiledLayerBuffer();
+      TiledContentHost* tiledHost = aCompositable->AsTiledContentHost();
 
       NS_ASSERTION(tiledHost, "The compositable is not tiled");
 
@@ -140,16 +147,16 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
       break;
     }
     case CompositableOperationDetail::TOpRemoveTexture: {
-      const OpRemoveTexture& op = aEdit.detail().get_OpRemoveTexture();
+      const OpRemoveTexture& op = aDetail.get_OpRemoveTexture();
 
       RefPtr<TextureHost> tex = TextureHost::AsTextureHost(op.textureParent());
 
       MOZ_ASSERT(tex.get());
-      compositable->RemoveTextureHost(tex);
+      aCompositable->RemoveTextureHost(tex);
       break;
     }
     case CompositableOperationDetail::TOpUseTexture: {
-      const OpUseTexture& op = aEdit.detail().get_OpUseTexture();
+      const OpUseTexture& op = aDetail.get_OpUseTexture();
 
       AutoTArray<CompositableHost::TimedTexture,4> textures;
       for (auto& timedTexture : op.textures()) {
@@ -166,7 +173,7 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
         }
       }
       if (textures.Length() > 0) {
-        compositable->UseTextureHost(textures);
+        aCompositable->UseTextureHost(textures);
 
         for (auto& timedTexture : op.textures()) {
           RefPtr<TextureHost> texture = TextureHost::AsTextureHost(timedTexture.textureParent());
@@ -179,13 +186,13 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
         }
       }
 
-      if (UsesImageBridge() && compositable->GetLayer()) {
-        ScheduleComposition(compositable);
+      if (UsesImageBridge() && aCompositable->GetLayer()) {
+        ScheduleComposition(aCompositable);
       }
       break;
     }
     case CompositableOperationDetail::TOpUseComponentAlphaTextures: {
-      const OpUseComponentAlphaTextures& op = aEdit.detail().get_OpUseComponentAlphaTextures();
+      const OpUseComponentAlphaTextures& op = aDetail.get_OpUseComponentAlphaTextures();
       RefPtr<TextureHost> texOnBlack = TextureHost::AsTextureHost(op.textureOnBlackParent());
       RefPtr<TextureHost> texOnWhite = TextureHost::AsTextureHost(op.textureOnWhiteParent());
       if (op.readLockedBlack()) {
@@ -196,7 +203,7 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
       }
 
       MOZ_ASSERT(texOnBlack && texOnWhite);
-      compositable->UseComponentAlphaTextures(texOnBlack, texOnWhite);
+      aCompositable->UseComponentAlphaTextures(texOnBlack, texOnWhite);
 
       if (texOnBlack) {
         texOnBlack->SetLastFwdTransactionId(mFwdTransactionId);
@@ -213,7 +220,7 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
       }
 
       if (UsesImageBridge()) {
-        ScheduleComposition(compositable);
+        ScheduleComposition(aCompositable);
       }
       break;
     }

gfx/layers/ipc/CompositableTransactionParent.h

@@ -9,6 +9,7 @@
 
 #include <vector>                       // for vector
 #include "mozilla/Attributes.h"         // for override
+#include "mozilla/NotNull.h"
 #include "mozilla/layers/ISurfaceAllocator.h"  // for ISurfaceAllocator
 #include "mozilla/layers/LayersMessages.h"  // for EditReply, etc
 #include "mozilla/layers/TextureClient.h"
@@ -48,6 +49,8 @@ protected:
    * Handle the IPDL messages that affect PCompositable actors.
    */
   bool ReceiveCompositableUpdate(const CompositableOperation& aEdit);
+  bool ReceiveCompositableUpdate(const CompositableOperationDetail& aDetail,
+                                 NotNull<CompositableHost*> aCompositable);
 
   void ReleaseCompositable(const CompositableHandle& aHandle);
 

gfx/layers/ipc/ImageBridgeChild.cpp

@@ -997,18 +997,23 @@ ImageBridgeChild::RecvParentAsyncMessages(InfallibleTArray<AsyncParentMessageDat
   return IPC_OK();
 }
 
+RefPtr<ImageContainerListener>
+ImageBridgeChild::FindListener(const CompositableHandle& aHandle)
+{
+  RefPtr<ImageContainerListener> listener;
+  MutexAutoLock lock(mContainerMapLock);
+  auto it = mImageContainerListeners.find(aHandle.Value());
+  if (it != mImageContainerListeners.end()) {
+    listener = it->second;
+  }
+  return listener;
+}
+
 mozilla::ipc::IPCResult
 ImageBridgeChild::RecvDidComposite(InfallibleTArray<ImageCompositeNotification>&& aNotifications)
 {
   for (auto& n : aNotifications) {
-    RefPtr<ImageContainerListener> listener;
-    {
-      MutexAutoLock lock(mContainerMapLock);
-      auto it = mImageContainerListeners.find(n.compositable().Value());
-      if (it != mImageContainerListeners.end()) {
-        listener = it->second;
-      }
-    }
+    RefPtr<ImageContainerListener> listener = FindListener(n.compositable());
     if (listener) {
       listener->NotifyComposite(n);
     }
@@ -1016,6 +1021,17 @@ ImageBridgeChild::RecvDidComposite(InfallibleTArray<ImageCompositeNotification>&
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+ImageBridgeChild::RecvReportFramesDropped(const CompositableHandle& aHandle, const uint32_t& aFrames)
+{
+  RefPtr<ImageContainerListener> listener = FindListener(aHandle);
+  if (listener) {
+    listener->NotifyDropped(aFrames);
+  }
+
+  return IPC_OK();
+}
+
 PTextureChild*
 ImageBridgeChild::CreateTexture(const SurfaceDescriptor& aSharedData,
                                 const ReadLockDescriptor& aReadLock,

gfx/layers/ipc/ImageBridgeChild.h

@@ -95,7 +95,7 @@ bool InImageBridgeChildThread();
  *   - (B) Since the ImageContainer does not use ImageBridge, the image data is swaped.
  *
  * - During composition:
- *   - (A) The CompositableHost has an AsyncID, it looks up the ID in the 
+ *   - (A) The CompositableHost has an AsyncID, it looks up the ID in the
  *   global table to see if there is an image. If there is no image, nothing is rendered.
  *   - (B) The CompositableHost has image data rather than an ID (meaning it is not
  *   using ImageBridge), then it just composites the image data normally.
@@ -194,6 +194,9 @@ public:
   virtual mozilla::ipc::IPCResult
   RecvDidComposite(InfallibleTArray<ImageCompositeNotification>&& aNotifications) override;
 
+  virtual mozilla::ipc::IPCResult
+  RecvReportFramesDropped(const CompositableHandle& aHandle, const uint32_t& aFrames) override;
+
   // Create an ImageClient from any thread.
   RefPtr<ImageClient> CreateImageClient(
     CompositableType aType,
@@ -403,6 +406,7 @@ private:
    */
   Mutex mContainerMapLock;
   std::unordered_map<uint64_t, RefPtr<ImageContainerListener>> mImageContainerListeners;
+  RefPtr<ImageContainerListener> FindListener(const CompositableHandle& aHandle);
 
 #if defined(XP_WIN)
   /**

gfx/layers/ipc/ImageBridgeParent.cpp

@@ -198,10 +198,17 @@ ImageBridgeParent::RecvUpdate(EditArray&& aEdits, OpDestroyArray&& aToDestroy,
   AutoImageBridgeParentAsyncMessageSender autoAsyncMessageSender(this, &aToDestroy);
   UpdateFwdTransactionId(aFwdTransactionId);
 
-  for (EditArray::index_type i = 0; i < aEdits.Length(); ++i) {
-    if (!ReceiveCompositableUpdate(aEdits[i])) {
+  for (const auto& edit : aEdits) {
+    RefPtr<CompositableHost> compositable =
+      FindCompositable(edit.compositable());
+    if (!compositable ||
+        !ReceiveCompositableUpdate(edit.detail(), WrapNotNull(compositable))) {
       return IPC_FAIL_NO_REASON(this);
     }
+    uint32_t dropped = compositable->GetDroppedFrames();
+    if (dropped) {
+      Unused << SendReportFramesDropped(edit.compositable(), dropped);
+    }
   }
 
   if (!IsSameProcess()) {

gfx/layers/ipc/PImageBridge.ipdl

@@ -36,6 +36,9 @@ child:
 
   async DidComposite(ImageCompositeNotification[] aNotifications);
 
+  // Report the number of frames dropped for the given CompositableHost.
+  async ReportFramesDropped(CompositableHandle aHandle, uint32_t aFrames);
+
 parent:
   async Update(CompositableOperation[] ops, OpDestroy[] toDestroy, uint64_t fwdTransactionId);