From 70040ff272afee81545c9b7cce982580190da719 Mon Sep 17 00:00:00 2001 From: Jean-Yves Avenard Date: Mon, 16 Jul 2018 23:19:09 +0200 Subject: [PATCH] Bug 1245400 - P3. Report number of frames dropped by compositor back to VideoSink. r=nical We report the number of frames dropped by the compositor because they were too late through: ImageComposite -> ImageHost -> CompositableTransactionParent -> ImageBridgeParent -> IPDL -> ImageBridgeChild -> ImageContainerListener -> ImageContainer -> VideoSink Differential Revision: https://phabricator.services.mozilla.com/D2177 --- dom/media/mediasink/VideoSink.cpp | 8 ++++ dom/media/mediasink/VideoSink.h | 5 ++- gfx/layers/ImageContainer.cpp | 30 ++++++++----- gfx/layers/ImageContainer.h | 7 +-- gfx/layers/composite/CompositableHost.h | 2 + gfx/layers/composite/ImageComposite.h | 6 +++ gfx/layers/composite/ImageHost.h | 5 +++ .../ipc/CompositableTransactionParent.cpp | 45 +++++++++++-------- .../ipc/CompositableTransactionParent.h | 3 ++ gfx/layers/ipc/ImageBridgeChild.cpp | 32 +++++++++---- gfx/layers/ipc/ImageBridgeChild.h | 6 ++- gfx/layers/ipc/ImageBridgeParent.cpp | 11 ++++- gfx/layers/ipc/PImageBridge.ipdl | 3 ++ 13 files changed, 119 insertions(+), 44 deletions(-) diff --git a/dom/media/mediasink/VideoSink.cpp b/dom/media/mediasink/VideoSink.cpp index f77387f0f741..0f4cec6e005c 100644 --- a/dom/media/mediasink/VideoSink.cpp +++ b/dom/media/mediasink/VideoSink.cpp @@ -47,6 +47,7 @@ VideoSink::VideoSink(AbstractThread* aThread, , mContainer(aContainer) , mProducerID(ImageContainer::AllocateProducerID()) , mFrameStats(aFrameStats) + , mOldDroppedCount(0) , mHasVideo(false) , mUpdateScheduler(aThread) , mVideoQueueSendToCompositorSize(aVQueueSentToCompositerSize) @@ -464,6 +465,13 @@ VideoSink::RenderVideoFrames(int32_t aMaxFrames, if (images.Length() > 0) { mContainer->SetCurrentFrames(frames[0]->mDisplay, images); + uint32_t droppedCount = mContainer->GetDroppedImageCount(); + uint32_t dropped = droppedCount - mOldDroppedCount; + if (dropped > 0) { + mFrameStats.NotifyDecodedFrames({0, 0, dropped}); + mOldDroppedCount = droppedCount; + VSINK_LOG_V("%u video frame discarded by compositor", dropped); + } } } diff --git a/dom/media/mediasink/VideoSink.h b/dom/media/mediasink/VideoSink.h index 2bf74cc12b9b..bb20d2d56a28 100644 --- a/dom/media/mediasink/VideoSink.h +++ b/dom/media/mediasink/VideoSink.h @@ -108,7 +108,8 @@ private: MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn()); } - MediaQueue& VideoQueue() const { + MediaQueue& VideoQueue() const + { return mVideoQueue; } @@ -131,6 +132,8 @@ private: // The presentation end time of the last video frame which has been displayed. TimeUnit mVideoFrameEndTime; + uint32_t mOldDroppedCount; + // Event listeners for VideoQueue MediaEventListener mPushListener; MediaEventListener mFinishListener; diff --git a/gfx/layers/ImageContainer.cpp b/gfx/layers/ImageContainer.cpp index 7a9ce3c28a0e..db79b20794ba 100644 --- a/gfx/layers/ImageContainer.cpp +++ b/gfx/layers/ImageContainer.cpp @@ -121,6 +121,15 @@ ImageContainerListener::NotifyComposite(const ImageCompositeNotification& aNotif } } +void +ImageContainerListener::NotifyDropped(uint32_t aDropped) +{ + MutexAutoLock lock(mLock); + if (mImageContainer) { + mImageContainer->NotifyDropped(aDropped); + } +} + void ImageContainerListener::ClearImageContainer() { @@ -262,7 +271,7 @@ ImageContainer::SetCurrentImageInternal(const nsTArray& aImages) mCurrentProducerID = aImages[0].mProducerID; } else if (!aImages[0].mTimeStamp.IsNull()) { // Check for expired frames - for (auto& img : mCurrentImages) { + for (const auto& img : mCurrentImages) { if (img.mProducerID != aImages[0].mProducerID || img.mTimeStamp.IsNull() || img.mTimeStamp >= aImages[0].mTimeStamp) { @@ -278,7 +287,6 @@ ImageContainer::SetCurrentImageInternal(const nsTArray& aImages) const uint32_t maxFrames = 100; if (mFrameIDsNotYetComposited.Length() > maxFrames) { uint32_t dropFrames = mFrameIDsNotYetComposited.Length() - maxFrames; - mDroppedImageCount += dropFrames; mFrameIDsNotYetComposited.RemoveElementsAt(0, dropFrames); } } @@ -303,7 +311,7 @@ ImageContainer::SetCurrentImageInternal(const nsTArray& aImages) img->mTimeStamp = aImages[i].mTimeStamp; img->mFrameID = aImages[i].mFrameID; img->mProducerID = aImages[i].mProducerID; - for (auto& oldImg : mCurrentImages) { + for (const auto& oldImg : mCurrentImages) { if (oldImg.mFrameID == img->mFrameID && oldImg.mProducerID == img->mProducerID) { img->mComposited = oldImg.mComposited; @@ -441,11 +449,7 @@ ImageContainer::NotifyComposite(const ImageCompositeNotification& aNotification) if (aNotification.producerID() == mCurrentProducerID) { uint32_t i; for (i = 0; i < mFrameIDsNotYetComposited.Length(); ++i) { - if (mFrameIDsNotYetComposited[i] <= aNotification.frameID()) { - if (mFrameIDsNotYetComposited[i] < aNotification.frameID()) { - ++mDroppedImageCount; - } - } else { + if (mFrameIDsNotYetComposited[i] > aNotification.frameID()) { break; } } @@ -458,11 +462,17 @@ ImageContainer::NotifyComposite(const ImageCompositeNotification& aNotification) } if (!aNotification.imageTimeStamp().IsNull()) { - mPaintDelay = aNotification.firstCompositeTimeStamp() - - aNotification.imageTimeStamp(); + mPaintDelay = + aNotification.firstCompositeTimeStamp() - aNotification.imageTimeStamp(); } } +void +ImageContainer::NotifyDropped(uint32_t aDropped) +{ + mDroppedImageCount += aDropped; +} + #ifdef XP_WIN D3D11YCbCrRecycleAllocator* ImageContainer::GetD3D11YCbCrRecycleAllocator(KnowsCompositor* aAllocator) diff --git a/gfx/layers/ImageContainer.h b/gfx/layers/ImageContainer.h index b5e5a7f83b82..c9d221269d92 100644 --- a/gfx/layers/ImageContainer.h +++ b/gfx/layers/ImageContainer.h @@ -356,6 +356,7 @@ public: explicit ImageContainerListener(ImageContainer* aImageContainer); void NotifyComposite(const ImageCompositeNotification& aNotification); + void NotifyDropped(uint32_t aDropped); void ClearImageContainer(); void DropImageClient(); private: @@ -613,11 +614,11 @@ public: */ uint32_t GetDroppedImageCount() { - RecursiveMutexAutoLock lock(mRecursiveMutex); return mDroppedImageCount; } void NotifyComposite(const ImageCompositeNotification& aNotification); + void NotifyDropped(uint32_t aDropped); ImageContainerListener* GetImageContainerListener() { @@ -675,8 +676,8 @@ private: // See GetPaintDelay. Accessed only with mRecursiveMutex held. TimeDuration mPaintDelay; - // See GetDroppedImageCount. Accessed only with mRecursiveMutex held. - uint32_t mDroppedImageCount; + // See GetDroppedImageCount. + mozilla::Atomic mDroppedImageCount; // This is the image factory used by this container, layer managers using // this container can set an alternative image factory that will be used to diff --git a/gfx/layers/composite/CompositableHost.h b/gfx/layers/composite/CompositableHost.h index 18c8c8f2c1e9..5705d0e31d7d 100644 --- a/gfx/layers/composite/CompositableHost.h +++ b/gfx/layers/composite/CompositableHost.h @@ -245,6 +245,8 @@ public: virtual void BindTextureSource() {} + virtual uint32_t GetDroppedFrames() { return 0; } + protected: HostLayerManager* GetLayerManager() const; diff --git a/gfx/layers/composite/ImageComposite.h b/gfx/layers/composite/ImageComposite.h index bb9cee1eeb29..8011ed530f0a 100644 --- a/gfx/layers/composite/ImageComposite.h +++ b/gfx/layers/composite/ImageComposite.h @@ -40,6 +40,12 @@ public: int32_t GetLastFrameID() const { return mLastFrameID; } int32_t GetLastProducerID() const { return mLastProducerID; } + uint32_t GetDroppedFramesAndReset() + { + uint32_t dropped = mDroppedFrames; + mDroppedFrames = 0; + return dropped; + } enum Bias { // Don't apply bias to frame times diff --git a/gfx/layers/composite/ImageHost.h b/gfx/layers/composite/ImageHost.h index 5359550a023c..75513cc3753e 100644 --- a/gfx/layers/composite/ImageHost.h +++ b/gfx/layers/composite/ImageHost.h @@ -91,6 +91,11 @@ public: bool IsOpaque(); + uint32_t GetDroppedFrames() override + { + return GetDroppedFramesAndReset(); + } + struct RenderInfo { int imageIndex; const TimedImage* img; diff --git a/gfx/layers/ipc/CompositableTransactionParent.cpp b/gfx/layers/ipc/CompositableTransactionParent.cpp index 3a5f893799ef..33db232699c4 100644 --- a/gfx/layers/ipc/CompositableTransactionParent.cpp +++ b/gfx/layers/ipc/CompositableTransactionParent.cpp @@ -68,18 +68,26 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation if (!compositable) { return false; } - if (TextureSourceProvider* provider = compositable->GetTextureSourceProvider()) { + return ReceiveCompositableUpdate(aEdit.detail(), WrapNotNull(compositable)); +} + +bool +CompositableParentManager::ReceiveCompositableUpdate( + const CompositableOperationDetail& aDetail, + NotNull aCompositable) +{ + if (TextureSourceProvider* provider = aCompositable->GetTextureSourceProvider()) { if (!provider->IsValid()) { return false; } } - switch (aEdit.detail().type()) { + switch (aDetail.type()) { case CompositableOperationDetail::TOpPaintTextureRegion: { MOZ_LAYERS_LOG(("[ParentSide] Paint PaintedLayer")); - const OpPaintTextureRegion& op = aEdit.detail().get_OpPaintTextureRegion(); - Layer* layer = compositable->GetLayer(); + const OpPaintTextureRegion& op = aDetail.get_OpPaintTextureRegion(); + Layer* layer = aCompositable->GetLayer(); if (!layer || layer->GetType() != Layer::TYPE_PAINTED) { return false; } @@ -89,10 +97,9 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation RenderTraceInvalidateStart(thebes, "FF00FF", op.updatedRegion().GetBounds()); - if (!compositable->UpdateThebes(bufferData, - op.updatedRegion(), - thebes->GetValidRegion())) - { + if (!aCompositable->UpdateThebes(bufferData, + op.updatedRegion(), + thebes->GetValidRegion())) { return false; } @@ -101,8 +108,8 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation } case CompositableOperationDetail::TOpUseTiledLayerBuffer: { MOZ_LAYERS_LOG(("[ParentSide] Paint TiledLayerBuffer")); - const OpUseTiledLayerBuffer& op = aEdit.detail().get_OpUseTiledLayerBuffer(); - TiledContentHost* tiledHost = compositable->AsTiledContentHost(); + const OpUseTiledLayerBuffer& op = aDetail.get_OpUseTiledLayerBuffer(); + TiledContentHost* tiledHost = aCompositable->AsTiledContentHost(); NS_ASSERTION(tiledHost, "The compositable is not tiled"); @@ -140,16 +147,16 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation break; } case CompositableOperationDetail::TOpRemoveTexture: { - const OpRemoveTexture& op = aEdit.detail().get_OpRemoveTexture(); + const OpRemoveTexture& op = aDetail.get_OpRemoveTexture(); RefPtr tex = TextureHost::AsTextureHost(op.textureParent()); MOZ_ASSERT(tex.get()); - compositable->RemoveTextureHost(tex); + aCompositable->RemoveTextureHost(tex); break; } case CompositableOperationDetail::TOpUseTexture: { - const OpUseTexture& op = aEdit.detail().get_OpUseTexture(); + const OpUseTexture& op = aDetail.get_OpUseTexture(); AutoTArray textures; for (auto& timedTexture : op.textures()) { @@ -166,7 +173,7 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation } } if (textures.Length() > 0) { - compositable->UseTextureHost(textures); + aCompositable->UseTextureHost(textures); for (auto& timedTexture : op.textures()) { RefPtr texture = TextureHost::AsTextureHost(timedTexture.textureParent()); @@ -179,13 +186,13 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation } } - if (UsesImageBridge() && compositable->GetLayer()) { - ScheduleComposition(compositable); + if (UsesImageBridge() && aCompositable->GetLayer()) { + ScheduleComposition(aCompositable); } break; } case CompositableOperationDetail::TOpUseComponentAlphaTextures: { - const OpUseComponentAlphaTextures& op = aEdit.detail().get_OpUseComponentAlphaTextures(); + const OpUseComponentAlphaTextures& op = aDetail.get_OpUseComponentAlphaTextures(); RefPtr texOnBlack = TextureHost::AsTextureHost(op.textureOnBlackParent()); RefPtr texOnWhite = TextureHost::AsTextureHost(op.textureOnWhiteParent()); if (op.readLockedBlack()) { @@ -196,7 +203,7 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation } MOZ_ASSERT(texOnBlack && texOnWhite); - compositable->UseComponentAlphaTextures(texOnBlack, texOnWhite); + aCompositable->UseComponentAlphaTextures(texOnBlack, texOnWhite); if (texOnBlack) { texOnBlack->SetLastFwdTransactionId(mFwdTransactionId); @@ -213,7 +220,7 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation } if (UsesImageBridge()) { - ScheduleComposition(compositable); + ScheduleComposition(aCompositable); } break; } diff --git a/gfx/layers/ipc/CompositableTransactionParent.h b/gfx/layers/ipc/CompositableTransactionParent.h index 3728d44d7185..b7c0ccd2d961 100644 --- a/gfx/layers/ipc/CompositableTransactionParent.h +++ b/gfx/layers/ipc/CompositableTransactionParent.h @@ -9,6 +9,7 @@ #include // for vector #include "mozilla/Attributes.h" // for override +#include "mozilla/NotNull.h" #include "mozilla/layers/ISurfaceAllocator.h" // for ISurfaceAllocator #include "mozilla/layers/LayersMessages.h" // for EditReply, etc #include "mozilla/layers/TextureClient.h" @@ -48,6 +49,8 @@ protected: * Handle the IPDL messages that affect PCompositable actors. */ bool ReceiveCompositableUpdate(const CompositableOperation& aEdit); + bool ReceiveCompositableUpdate(const CompositableOperationDetail& aDetail, + NotNull aCompositable); void ReleaseCompositable(const CompositableHandle& aHandle); diff --git a/gfx/layers/ipc/ImageBridgeChild.cpp b/gfx/layers/ipc/ImageBridgeChild.cpp index a24a0d88dcde..a433aed23899 100644 --- a/gfx/layers/ipc/ImageBridgeChild.cpp +++ b/gfx/layers/ipc/ImageBridgeChild.cpp @@ -997,18 +997,23 @@ ImageBridgeChild::RecvParentAsyncMessages(InfallibleTArray +ImageBridgeChild::FindListener(const CompositableHandle& aHandle) +{ + RefPtr listener; + MutexAutoLock lock(mContainerMapLock); + auto it = mImageContainerListeners.find(aHandle.Value()); + if (it != mImageContainerListeners.end()) { + listener = it->second; + } + return listener; +} + mozilla::ipc::IPCResult ImageBridgeChild::RecvDidComposite(InfallibleTArray&& aNotifications) { for (auto& n : aNotifications) { - RefPtr listener; - { - MutexAutoLock lock(mContainerMapLock); - auto it = mImageContainerListeners.find(n.compositable().Value()); - if (it != mImageContainerListeners.end()) { - listener = it->second; - } - } + RefPtr listener = FindListener(n.compositable()); if (listener) { listener->NotifyComposite(n); } @@ -1016,6 +1021,17 @@ ImageBridgeChild::RecvDidComposite(InfallibleTArray& return IPC_OK(); } +mozilla::ipc::IPCResult +ImageBridgeChild::RecvReportFramesDropped(const CompositableHandle& aHandle, const uint32_t& aFrames) +{ + RefPtr listener = FindListener(aHandle); + if (listener) { + listener->NotifyDropped(aFrames); + } + + return IPC_OK(); +} + PTextureChild* ImageBridgeChild::CreateTexture(const SurfaceDescriptor& aSharedData, const ReadLockDescriptor& aReadLock, diff --git a/gfx/layers/ipc/ImageBridgeChild.h b/gfx/layers/ipc/ImageBridgeChild.h index 84a638ee4cb6..2898a90a2af9 100644 --- a/gfx/layers/ipc/ImageBridgeChild.h +++ b/gfx/layers/ipc/ImageBridgeChild.h @@ -95,7 +95,7 @@ bool InImageBridgeChildThread(); * - (B) Since the ImageContainer does not use ImageBridge, the image data is swaped. * * - During composition: - * - (A) The CompositableHost has an AsyncID, it looks up the ID in the + * - (A) The CompositableHost has an AsyncID, it looks up the ID in the * global table to see if there is an image. If there is no image, nothing is rendered. * - (B) The CompositableHost has image data rather than an ID (meaning it is not * using ImageBridge), then it just composites the image data normally. @@ -194,6 +194,9 @@ public: virtual mozilla::ipc::IPCResult RecvDidComposite(InfallibleTArray&& aNotifications) override; + virtual mozilla::ipc::IPCResult + RecvReportFramesDropped(const CompositableHandle& aHandle, const uint32_t& aFrames) override; + // Create an ImageClient from any thread. RefPtr CreateImageClient( CompositableType aType, @@ -403,6 +406,7 @@ private: */ Mutex mContainerMapLock; std::unordered_map> mImageContainerListeners; + RefPtr FindListener(const CompositableHandle& aHandle); #if defined(XP_WIN) /** diff --git a/gfx/layers/ipc/ImageBridgeParent.cpp b/gfx/layers/ipc/ImageBridgeParent.cpp index 1cd68ad143e6..d15d5537a6ed 100644 --- a/gfx/layers/ipc/ImageBridgeParent.cpp +++ b/gfx/layers/ipc/ImageBridgeParent.cpp @@ -198,10 +198,17 @@ ImageBridgeParent::RecvUpdate(EditArray&& aEdits, OpDestroyArray&& aToDestroy, AutoImageBridgeParentAsyncMessageSender autoAsyncMessageSender(this, &aToDestroy); UpdateFwdTransactionId(aFwdTransactionId); - for (EditArray::index_type i = 0; i < aEdits.Length(); ++i) { - if (!ReceiveCompositableUpdate(aEdits[i])) { + for (const auto& edit : aEdits) { + RefPtr compositable = + FindCompositable(edit.compositable()); + if (!compositable || + !ReceiveCompositableUpdate(edit.detail(), WrapNotNull(compositable))) { return IPC_FAIL_NO_REASON(this); } + uint32_t dropped = compositable->GetDroppedFrames(); + if (dropped) { + Unused << SendReportFramesDropped(edit.compositable(), dropped); + } } if (!IsSameProcess()) { diff --git a/gfx/layers/ipc/PImageBridge.ipdl b/gfx/layers/ipc/PImageBridge.ipdl index e074a7b07e88..704726c8f676 100644 --- a/gfx/layers/ipc/PImageBridge.ipdl +++ b/gfx/layers/ipc/PImageBridge.ipdl @@ -36,6 +36,9 @@ child: async DidComposite(ImageCompositeNotification[] aNotifications); + // Report the number of frames dropped for the given CompositableHost. + async ReportFramesDropped(CompositableHandle aHandle, uint32_t aFrames); + parent: async Update(CompositableOperation[] ops, OpDestroy[] toDestroy, uint64_t fwdTransactionId);