From 641c2b7fc8ce953049befe27bd5de3727ec24cc7 Mon Sep 17 00:00:00 2001 From: Alfredo Yang Date: Tue, 25 Nov 2014 15:35:28 -0800 Subject: [PATCH] Bug 1101374: create frame time table to filter the invalid pts frame. r=edwin --- .../fmp4/gonk/GonkAudioDecoderManager.cpp | 6 ++ dom/media/fmp4/gonk/GonkAudioDecoderManager.h | 2 + dom/media/fmp4/gonk/GonkMediaDataDecoder.cpp | 3 +- dom/media/fmp4/gonk/GonkMediaDataDecoder.h | 1 + .../fmp4/gonk/GonkVideoDecoderManager.cpp | 66 ++++++++++++++++++- dom/media/fmp4/gonk/GonkVideoDecoderManager.h | 22 +++++++ 6 files changed, 97 insertions(+), 3 deletions(-) diff --git a/dom/media/fmp4/gonk/GonkAudioDecoderManager.cpp b/dom/media/fmp4/gonk/GonkAudioDecoderManager.cpp index d0b0f81561ab..aa831c2302d1 100644 --- a/dom/media/fmp4/gonk/GonkAudioDecoderManager.cpp +++ b/dom/media/fmp4/gonk/GonkAudioDecoderManager.cpp @@ -204,6 +204,12 @@ GonkAudioDecoderManager::Output(int64_t aStreamOffset, return NS_OK; } +nsresult +GonkAudioDecoderManager::Flush() +{ + return NS_OK; +} + void GonkAudioDecoderManager::ReleaseAudioBuffer() { if (mAudioBuffer) { mDecoder->ReleaseMediaBuffer(mAudioBuffer); diff --git a/dom/media/fmp4/gonk/GonkAudioDecoderManager.h b/dom/media/fmp4/gonk/GonkAudioDecoderManager.h index 2b070ceda6cb..7a73054d5449 100644 --- a/dom/media/fmp4/gonk/GonkAudioDecoderManager.h +++ b/dom/media/fmp4/gonk/GonkAudioDecoderManager.h @@ -32,6 +32,8 @@ public: virtual nsresult Output(int64_t aStreamOffset, nsRefPtr& aOutput) MOZ_OVERRIDE; + + virtual nsresult Flush() MOZ_OVERRIDE; private: nsresult CreateAudioData(int64_t aStreamOffset, diff --git a/dom/media/fmp4/gonk/GonkMediaDataDecoder.cpp b/dom/media/fmp4/gonk/GonkMediaDataDecoder.cpp index eee5d412458c..e2a474639866 100644 --- a/dom/media/fmp4/gonk/GonkMediaDataDecoder.cpp +++ b/dom/media/fmp4/gonk/GonkMediaDataDecoder.cpp @@ -135,8 +135,7 @@ GonkMediaDataDecoder::Flush() // flushing. mTaskQueue->Flush(); - status_t err = mDecoder->flush(); - return err == OK ? NS_OK : NS_ERROR_FAILURE; + return mManager->Flush(); } void diff --git a/dom/media/fmp4/gonk/GonkMediaDataDecoder.h b/dom/media/fmp4/gonk/GonkMediaDataDecoder.h index 1e36cf971d06..1bcdf260a51b 100644 --- a/dom/media/fmp4/gonk/GonkMediaDataDecoder.h +++ b/dom/media/fmp4/gonk/GonkMediaDataDecoder.h @@ -33,6 +33,7 @@ public: virtual nsresult Input(mp4_demuxer::MP4Sample* aSample) = 0; virtual nsresult Output(int64_t aStreamOffset, nsRefPtr& aOutput) = 0; + virtual nsresult Flush() = 0; virtual void ReleaseMediaResources() {}; }; diff --git a/dom/media/fmp4/gonk/GonkVideoDecoderManager.cpp b/dom/media/fmp4/gonk/GonkVideoDecoderManager.cpp index 4dd211dd446e..3e8b03bf646b 100644 --- a/dom/media/fmp4/gonk/GonkVideoDecoderManager.cpp +++ b/dom/media/fmp4/gonk/GonkVideoDecoderManager.cpp @@ -54,6 +54,7 @@ GonkVideoDecoderManager::GonkVideoDecoderManager( const mp4_demuxer::VideoDecoderConfig& aConfig) : mImageContainer(aImageContainer) , mReaderCallback(nullptr) + , mMonitor("GonkVideoDecoderManager") , mColorConverterBufferSize(0) , mNativeWindow(nullptr) , mPendingVideoBuffersLock("GonkVideoDecoderManager::mPendingVideoBuffersLock") @@ -121,6 +122,50 @@ GonkVideoDecoderManager::Init(MediaDataDecoderCallback* aCallback) return mDecoder; } +void +GonkVideoDecoderManager::QueueFrameTimeIn(int64_t aPTS, int64_t aDuration) +{ + MonitorAutoLock mon(mMonitor); + FrameTimeInfo timeInfo = {aPTS, aDuration}; + mFrameTimeInfo.AppendElement(timeInfo); +} + +nsresult +GonkVideoDecoderManager::QueueFrameTimeOut(int64_t aPTS, int64_t& aDuration) +{ + MonitorAutoLock mon(mMonitor); + + // Set default to 1 here. + // During seeking, frames could still in MediaCodec and the mFrameTimeInfo could + // be cleared before these frames are out from MediaCodec. This is ok because + // these frames are old frame before seeking. + aDuration = 1; + for (uint32_t i = 0; i < mFrameTimeInfo.Length(); i++) { + const FrameTimeInfo& entry = mFrameTimeInfo.ElementAt(i); + if (i == 0) { + if (entry.pts > aPTS) { + // Codec sent a frame with rollbacked PTS time. It could + // be codec's problem. + ReleaseVideoBuffer(); + return NS_ERROR_NOT_AVAILABLE; + } + } + + // Ideally, the first entry in mFrameTimeInfo should be the one we are looking + // for. However, MediaCodec could dropped frame and the first entry doesn't + // match current decoded frame's PTS. + if (entry.pts == aPTS) { + aDuration = entry.duration; + if (i > 0) { + LOG("Frame could be dropped by MediaCodec, %d dropped frames.", i); + } + mFrameTimeInfo.RemoveElementsAt(0, i+1); + break; + } + } + return NS_OK; +} + nsresult GonkVideoDecoderManager::CreateVideoData(int64_t aStreamOffset, VideoData **v) { @@ -139,6 +184,10 @@ GonkVideoDecoderManager::CreateVideoData(int64_t aStreamOffset, VideoData **v) return NS_ERROR_UNEXPECTED; } + int64_t duration; + nsresult rv = QueueFrameTimeOut(timeUs, duration); + NS_ENSURE_SUCCESS(rv, rv); + if (mVideoBuffer->range_length() == 0) { // Some decoders may return spurious empty buffers that we just want to ignore // quoted from Android's AwesomePlayer.cpp @@ -178,7 +227,7 @@ GonkVideoDecoderManager::CreateVideoData(int64_t aStreamOffset, VideoData **v) mImageContainer, aStreamOffset, timeUs, - 1, // We don't know the duration. + duration, textureClient, keyFrame, -1, @@ -400,6 +449,8 @@ GonkVideoDecoderManager::Input(mp4_demuxer::MP4Sample* aSample) mp4_demuxer::AnnexB::ConvertSample(aSample); // Forward sample data to the decoder. + QueueFrameTimeIn(aSample->composition_timestamp, aSample->duration); + const uint8_t* data = reinterpret_cast(aSample->data); uint32_t length = aSample->size; rv = mDecoder->Input(data, length, aSample->composition_timestamp, 0); @@ -411,6 +462,19 @@ GonkVideoDecoderManager::Input(mp4_demuxer::MP4Sample* aSample) return (rv == OK) ? NS_OK : NS_ERROR_FAILURE; } +nsresult +GonkVideoDecoderManager::Flush() +{ + status_t err = mDecoder->flush(); + if (err != OK) { + return NS_ERROR_FAILURE; + } + + MonitorAutoLock mon(mMonitor); + mFrameTimeInfo.Clear(); + return NS_OK; +} + void GonkVideoDecoderManager::codecReserved() { diff --git a/dom/media/fmp4/gonk/GonkVideoDecoderManager.h b/dom/media/fmp4/gonk/GonkVideoDecoderManager.h index 2249b7421ca8..57d0f216d762 100644 --- a/dom/media/fmp4/gonk/GonkVideoDecoderManager.h +++ b/dom/media/fmp4/gonk/GonkVideoDecoderManager.h @@ -50,6 +50,8 @@ public: virtual nsresult Output(int64_t aStreamOffset, nsRefPtr& aOutput) MOZ_OVERRIDE; + virtual nsresult Flush() MOZ_OVERRIDE; + virtual void ReleaseMediaResources(); static void RecycleCallback(TextureClient* aClient, void* aClosure); @@ -104,6 +106,15 @@ private: }; friend class VideoResourceListener; + // FrameTimeInfo keeps the presentation time stamp (pts) and its duration. + // On MediaDecoderStateMachine, it needs pts and duration to display decoded + // frame correctly. But OMX can carry one field of time info (kKeyTime) so + // we use FrameTimeInfo to keep pts and duration. + struct FrameTimeInfo { + int64_t pts; // presentation time stamp of this frame. + int64_t duration; // the playback duration. + }; + bool SetVideoFormat(); nsresult CreateVideoData(int64_t aStreamOffset, VideoData** aOutData); @@ -118,6 +129,9 @@ private: void ReleaseAllPendingVideoBuffers(); void PostReleaseVideoBuffer(android::MediaBuffer *aBuffer); + void QueueFrameTimeIn(int64_t aPTS, int64_t aDuration); + nsresult QueueFrameTimeOut(int64_t aPTS, int64_t& aDuration); + uint32_t mVideoWidth; uint32_t mVideoHeight; uint32_t mDisplayWidth; @@ -139,6 +153,14 @@ private: android::sp mManagerLooper; FrameInfo mFrameInfo; + // It protects mFrameTimeInfo. + Monitor mMonitor; + // Array of FrameTimeInfo whose corresponding frames are sent to OMX. + // Ideally, it is a FIFO. Input() adds the entry to the end element and + // CreateVideoData() takes the first entry. However, there are exceptions + // due to MediaCodec error or seeking. + nsTArray mFrameTimeInfo; + // color converter android::I420ColorConverterHelper mColorConverter; nsAutoArrayPtr mColorConverterBuffer;