diff --git a/dom/media/MediaDecoderReader.h b/dom/media/MediaDecoderReader.h index aed78679db6b..c43cdb7c2fd1 100644 --- a/dom/media/MediaDecoderReader.h +++ b/dom/media/MediaDecoderReader.h @@ -194,11 +194,10 @@ public: virtual size_t SizeOfVideoQueueInFrames(); virtual size_t SizeOfAudioQueueInFrames(); - void NotifyDataArrived() + virtual void NotifyDataArrived() { MOZ_ASSERT(OnTaskQueue()); NS_ENSURE_TRUE_VOID(!mShutdown); - NotifyDataArrivedInternal(); UpdateBuffered(); } @@ -254,24 +253,6 @@ protected: // Recomputes mBuffered. virtual void UpdateBuffered(); - // Populates aBuffered with the time ranges which are buffered. This may only - // be called on the decode task queue, and should only be used internally by - // UpdateBuffered - mBuffered (or mirrors of it) should be used for everything - // else. - // - // This base implementation in MediaDecoderReader estimates the time ranges - // buffered by interpolating the cached byte ranges with the duration - // of the media. Reader subclasses should override this method if they - // can quickly calculate the buffered ranges more accurately. - // - // The primary advantage of this implementation in the reader base class - // is that it's a fast approximation, which does not perform any I/O. - // - // The OggReader relies on this base implementation not performing I/O, - // since in FirefoxOS we can't do I/O on the main thread, where this is - // called. - virtual media::TimeIntervals GetBuffered(); - RefPtr DecodeToFirstVideoData(); // Queue of audio frames. This queue is threadsafe, and is accessed from @@ -344,8 +325,6 @@ private: virtual void VisibilityChanged(); - virtual void NotifyDataArrivedInternal() {} - // Overrides of this function should decodes an unspecified amount of // audio data, enqueuing the audio data in mAudioQueue. Returns true // when there's more audio to decode, false if the audio is finished, @@ -366,6 +345,24 @@ private: return false; } + // Populates aBuffered with the time ranges which are buffered. This may only + // be called on the decode task queue, and should only be used internally by + // UpdateBuffered - mBuffered (or mirrors of it) should be used for everything + // else. + // + // This base implementation in MediaDecoderReader estimates the time ranges + // buffered by interpolating the cached byte ranges with the duration + // of the media. Reader subclasses should override this method if they + // can quickly calculate the buffered ranges more accurately. + // + // The primary advantage of this implementation in the reader base class + // is that it's a fast approximation, which does not perform any I/O. + // + // The OggReader relies on this base implementation not performing I/O, + // since in FirefoxOS we can't do I/O on the main thread, where this is + // called. + media::TimeIntervals GetBuffered(); + // Promises used only for the base-class (sync->async adapter) implementation // of Request{Audio,Video}Data. MozPromiseHolder mBaseAudioPromise; diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index 8f7de2bc8829..0a1cbdaa64dd 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -4,24 +4,27 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "mozilla/CDMProxy.h" -#include "mozilla/ClearOnShutdown.h" -#include "mozilla/dom/HTMLMediaElement.h" -#include "mozilla/Preferences.h" -#include "mozilla/Telemetry.h" -#include "nsContentUtils.h" -#include "nsPrintfCString.h" -#include "nsSize.h" +#include "AutoTaskQueue.h" #include "Layers.h" #include "MediaData.h" #include "MediaInfo.h" #include "MediaFormatReader.h" #include "MediaPrefs.h" #include "MediaResource.h" -#include "mozilla/SharedThreadPool.h" #include "VideoUtils.h" #include "VideoFrameContainer.h" +#include "mozilla/dom/HTMLMediaElement.h" #include "mozilla/layers/ShadowLayers.h" +#include "mozilla/CDMProxy.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/Preferences.h" +#include "mozilla/Telemetry.h" +#include "mozilla/Mutex.h" +#include "mozilla/SharedThreadPool.h" +#include "mozilla/SyncRunnable.h" +#include "nsContentUtils.h" +#include "nsPrintfCString.h" +#include "nsSize.h" #include #include @@ -427,6 +430,339 @@ MediaFormatReader::DecoderFactory::DoInitDecoder(TrackType aTrack) })); } +// DemuxerProxy ensures that the original main demuxer is only ever accessed +// via its own dedicated task queue. +// This ensure that the reader's taskqueue will never blocked while a demuxer +// is itself blocked attempting to access the MediaCache or the MediaResource. +class MediaFormatReader::DemuxerProxy +{ + using TrackType = TrackInfo::TrackType; + class Wrapper; + +public: + explicit DemuxerProxy(MediaDataDemuxer* aDemuxer) + : mTaskQueue(new AutoTaskQueue( + GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER))) + , mData(new Data(aDemuxer)) + { + MOZ_COUNT_CTOR(DemuxerProxy); + } + + ~DemuxerProxy() + { + MOZ_COUNT_DTOR(DemuxerProxy); + mData->mAudioDemuxer = nullptr; + mData->mVideoDemuxer = nullptr; + RefPtr data = mData.forget(); + mTaskQueue->Dispatch( + // We need to clear our reference to the demuxer now. So that in the event + // the init promise wasn't resolved, such as what can happen with the + // mediasource demuxer that is waiting on more data, it will force the + // init promise to be rejected. + NS_NewRunnableFunction([data]() { data->mDemuxer = nullptr; })); + } + + RefPtr Init(); + + Wrapper* + GetTrackDemuxer(TrackType aTrack, uint32_t aTrackNumber) + { + MOZ_RELEASE_ASSERT(mData && mData->mInitDone); + + switch (aTrack) { + case TrackInfo::kAudioTrack: + return mData->mAudioDemuxer; + case TrackInfo::kVideoTrack: + return mData->mVideoDemuxer; + default: + return nullptr; + } + } + + uint32_t GetNumberTracks(TrackType aTrack) const + { + MOZ_RELEASE_ASSERT(mData && mData->mInitDone); + + switch (aTrack) { + case TrackInfo::kAudioTrack: + return mData->mNumAudioTrack; + case TrackInfo::kVideoTrack: + return mData->mNumVideoTrack; + default: + return 0; + } + } + + bool IsSeekable() const + { + MOZ_RELEASE_ASSERT(mData && mData->mInitDone); + + return mData->mSeekable; + } + + bool IsSeekableOnlyInBufferedRanges() const + { + MOZ_RELEASE_ASSERT(mData && mData->mInitDone); + + return mData->mSeekableOnlyInBufferedRange; + } + + UniquePtr GetCrypto() const + { + MOZ_RELEASE_ASSERT(mData && mData->mInitDone); + + if (!mData->mCrypto) { + return nullptr; + } + auto crypto = MakeUnique(); + *crypto = *mData->mCrypto; + return crypto; + } + + RefPtr NotifyDataArrived(); + + bool ShouldComputeStartTime() const + { + MOZ_RELEASE_ASSERT(mData && mData->mInitDone); + + return mData->mShouldComputeStartTime; + } + +private: + const RefPtr mTaskQueue; + struct Data + { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data) + + explicit Data(MediaDataDemuxer* aDemuxer) + : mInitDone(false) + , mDemuxer(aDemuxer) + { } + + Atomic mInitDone; + // Only ever accessed over mTaskQueue once. + RefPtr mDemuxer; + // Only accessed once InitPromise has been resolved and immutable after. + // So we can safely access them without the use of the mutex. + uint32_t mNumAudioTrack = 0; + RefPtr mAudioDemuxer; + uint32_t mNumVideoTrack = 0; + RefPtr mVideoDemuxer; + bool mSeekable = false; + bool mSeekableOnlyInBufferedRange = false; + bool mShouldComputeStartTime = true; + UniquePtr mCrypto; + private: + ~Data() { } + }; + RefPtr mData; +}; + +class MediaFormatReader::DemuxerProxy::Wrapper : public MediaTrackDemuxer +{ +public: + Wrapper(MediaTrackDemuxer* aTrackDemuxer, AutoTaskQueue* aTaskQueue) + : mMutex("TrackDemuxer Mutex") + , mTaskQueue(aTaskQueue) + , mGetSamplesMayBlock(aTrackDemuxer->GetSamplesMayBlock()) + , mInfo(aTrackDemuxer->GetInfo()) + , mTrackDemuxer(aTrackDemuxer) + { } + + UniquePtr GetInfo() const override + { + if (!mInfo) { + return nullptr; + } + return mInfo->Clone(); + } + + RefPtr Seek(const media::TimeUnit& aTime) override + { + RefPtr self = this; + return InvokeAsync( + mTaskQueue, __func__, + [self, aTime]() { return self->mTrackDemuxer->Seek(aTime); }) + ->Then(mTaskQueue, __func__, + [self]() { self->UpdateRandomAccessPoint(); }, + [self]() { self->UpdateRandomAccessPoint(); }); + } + + RefPtr GetSamples(int32_t aNumSamples) override + { + RefPtr self = this; + return InvokeAsync(mTaskQueue, __func__, + [self, aNumSamples]() { + return self->mTrackDemuxer->GetSamples(aNumSamples); + }) + ->Then(mTaskQueue, __func__, + [self]() { self->UpdateRandomAccessPoint(); }, + [self]() { self->UpdateRandomAccessPoint(); }); + } + + bool GetSamplesMayBlock() const override + { + return mGetSamplesMayBlock; + } + + void Reset() override + { + RefPtr self = this; + mTaskQueue->Dispatch(NS_NewRunnableFunction([self]() { + self->mTrackDemuxer->Reset(); + })); + } + + nsresult GetNextRandomAccessPoint(TimeUnit* aTime) override + { + MutexAutoLock lock(mMutex); + if (NS_SUCCEEDED(mNextRandomAccessPointResult)) { + *aTime = mNextRandomAccessPoint; + } + return mNextRandomAccessPointResult; + } + + RefPtr + SkipToNextRandomAccessPoint(const media::TimeUnit& aTimeThreshold) override + { + RefPtr self = this; + return InvokeAsync( + mTaskQueue, __func__, + [self, aTimeThreshold]() { + return self->mTrackDemuxer->SkipToNextRandomAccessPoint( + aTimeThreshold); + }) + ->Then(mTaskQueue, __func__, + [self]() { self->UpdateRandomAccessPoint(); }, + [self]() { self->UpdateRandomAccessPoint(); }); + } + + TimeIntervals GetBuffered() override + { + MutexAutoLock lock(mMutex); + return mBuffered; + } + + void BreakCycles() override { } + +private: + Mutex mMutex; + const RefPtr mTaskQueue; + const bool mGetSamplesMayBlock; + const UniquePtr mInfo; + // mTrackDemuxer is only ever accessed on demuxer's task queue. + RefPtr mTrackDemuxer; + // All following members are protected by mMutex + nsresult mNextRandomAccessPointResult = NS_OK; + TimeUnit mNextRandomAccessPoint; + TimeIntervals mBuffered; + friend class DemuxerProxy; + + ~Wrapper() + { + RefPtr trackDemuxer = mTrackDemuxer.forget(); + mTaskQueue->Dispatch(NS_NewRunnableFunction( + [trackDemuxer]() { trackDemuxer->BreakCycles(); })); + } + + void UpdateRandomAccessPoint() + { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + if (!mTrackDemuxer) { + // Detached. + return; + } + MutexAutoLock lock(mMutex); + mNextRandomAccessPointResult = + mTrackDemuxer->GetNextRandomAccessPoint(&mNextRandomAccessPoint); + } + + void UpdateBuffered() + { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + if (!mTrackDemuxer) { + // Detached. + return; + } + MutexAutoLock lock(mMutex); + mBuffered = mTrackDemuxer->GetBuffered(); + } +}; + +RefPtr +MediaFormatReader::DemuxerProxy::Init() +{ + RefPtr data = mData; + RefPtr taskQueue = mTaskQueue; + return InvokeAsync(mTaskQueue, __func__, + [data, taskQueue]() { + if (!data->mDemuxer) { + return MediaDataDemuxer::InitPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_CANCELED, __func__); + } + return data->mDemuxer->Init(); + }) + ->Then(taskQueue, __func__, + [data, taskQueue]() { + if (!data->mDemuxer) { // Was shutdown. + return; + } + data->mNumAudioTrack = + data->mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack); + if (data->mNumAudioTrack) { + RefPtr d = + data->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); + if (d) { + RefPtr wrapper = + new DemuxerProxy::Wrapper(d, taskQueue); + wrapper->UpdateBuffered(); + data->mAudioDemuxer = wrapper; + } + } + data->mNumVideoTrack = + data->mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack); + if (data->mNumVideoTrack) { + RefPtr d = + data->mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0); + if (d) { + RefPtr wrapper = + new DemuxerProxy::Wrapper(d, taskQueue); + wrapper->UpdateBuffered(); + data->mVideoDemuxer = wrapper; + } + } + data->mCrypto = data->mDemuxer->GetCrypto(); + data->mSeekable = data->mDemuxer->IsSeekable(); + data->mSeekableOnlyInBufferedRange = + data->mDemuxer->IsSeekableOnlyInBufferedRanges(); + data->mShouldComputeStartTime = + data->mDemuxer->ShouldComputeStartTime(); + data->mInitDone = true; + }, + []() {}); +} + +RefPtr +MediaFormatReader::DemuxerProxy::NotifyDataArrived() +{ + RefPtr data = mData; + return InvokeAsync(mTaskQueue, __func__, [data]() { + if (!data->mDemuxer) { + // Was shutdown. + return NotifyDataArrivedPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_CANCELED, __func__); + } + data->mDemuxer->NotifyDataArrived(); + if (data->mAudioDemuxer) { + data->mAudioDemuxer->UpdateBuffered(); + } + if (data->mVideoDemuxer) { + data->mVideoDemuxer->UpdateBuffered(); + } + return NotifyDataArrivedPromise::CreateAndResolve(true, __func__); + }); +} + static const char* TrackTypeToStr(TrackInfo::TrackType aTrack) { @@ -453,7 +789,7 @@ MediaFormatReader::MediaFormatReader(AbstractMediaDecoder* aDecoder, Preferences::GetUint("media.audio-max-decode-error", 3)) , mVideo(this, MediaData::VIDEO_DATA, Preferences::GetUint("media.video-max-decode-error", 2)) - , mDemuxer(aDemuxer) + , mDemuxer(new DemuxerProxy(aDemuxer)) , mDemuxerInitDone(false) , mLastReportedNumDecodedFrames(0) , mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe) @@ -689,7 +1025,6 @@ MediaFormatReader::OnDemuxerInitDone(nsresult) } mVideo.mOriginalInfo = Move(videoInfo); mVideo.mCallback = new DecoderCallback(this, TrackInfo::kVideoTrack); - mVideo.mTimeRanges = mVideo.mTrackDemuxer->GetBuffered(); mTrackDemuxersMayBlock |= mVideo.mTrackDemuxer->GetSamplesMayBlock(); } else { mVideo.mTrackDemuxer->BreakCycles(); @@ -718,7 +1053,6 @@ MediaFormatReader::OnDemuxerInitDone(nsresult) } mAudio.mOriginalInfo = Move(audioInfo); mAudio.mCallback = new DecoderCallback(this, TrackInfo::kAudioTrack); - mAudio.mTimeRanges = mAudio.mTrackDemuxer->GetBuffered(); mTrackDemuxersMayBlock |= mAudio.mTrackDemuxer->GetSamplesMayBlock(); } else { mAudio.mTrackDemuxer->BreakCycles(); @@ -794,12 +1128,15 @@ MediaFormatReader::MaybeResolveMetadataPromise() mInfo.mStartTime = startTime; // mInfo.mStartTime is initialized to 0. } - mHasStartTime = true; - UpdateBuffered(); - RefPtr metadata = new MetadataHolder(); metadata->mInfo = mInfo; metadata->mTags = mTags->Count() ? mTags.release() : nullptr; + + // We now have all the informations required to calculate the initial buffered + // range. + mHasStartTime = true; + UpdateBuffered(); + mMetadataPromise.Resolve(metadata, __func__); } @@ -1155,9 +1492,6 @@ MediaFormatReader::UpdateReceivedNewData(TrackType aTrack) return false; } - // Update our cached TimeRange. - decoder.mTimeRanges = decoder.mTrackDemuxer->GetBuffered(); - // We do not want to clear mWaitingForData while there are pending // demuxing or seeking operations that could affect the value of this flag. // This is in order to ensure that we will retry once they complete as we may @@ -1185,16 +1519,6 @@ MediaFormatReader::UpdateReceivedNewData(TrackType aTrack) return false; } - bool hasLastEnd; - media::TimeUnit lastEnd = decoder.mTimeRanges.GetEnd(&hasLastEnd); - if (hasLastEnd) { - if (decoder.mLastTimeRangesEnd && decoder.mLastTimeRangesEnd.ref() < lastEnd) { - // New data was added after our previous end, we can clear the EOS flag. - decoder.mDemuxEOS = false; - } - decoder.mLastTimeRangesEnd = Some(lastEnd); - } - decoder.mReceivedNewData = false; if (decoder.mTimeThreshold) { decoder.mTimeThreshold.ref().mWaiting = false; @@ -2186,47 +2510,6 @@ MediaFormatReader::OnAudioSeekFailed(const MediaResult& aError) OnSeekFailed(TrackType::kAudioTrack, aError); } -media::TimeIntervals -MediaFormatReader::GetBuffered() -{ - MOZ_ASSERT(OnTaskQueue()); - media::TimeIntervals videoti; - media::TimeIntervals audioti; - media::TimeIntervals intervals; - - if (!mInitDone || !mHasStartTime) { - return intervals; - } - - // Ensure we have up to date buffered time range. - if (HasVideo()) { - UpdateReceivedNewData(TrackType::kVideoTrack); - } - if (HasAudio()) { - UpdateReceivedNewData(TrackType::kAudioTrack); - } - if (HasVideo()) { - videoti = mVideo.mTimeRanges; - } - if (HasAudio()) { - audioti = mAudio.mTimeRanges; - } - if (HasAudio() && HasVideo()) { - intervals = media::Intersection(Move(videoti), Move(audioti)); - } else if (HasAudio()) { - intervals = Move(audioti); - } else if (HasVideo()) { - intervals = Move(videoti); - } - - if (!intervals.Length() || - intervals.GetStart() == media::TimeUnit::FromMicroseconds(0)) { - // IntervalSet already starts at 0 or is empty, nothing to shift. - return intervals; - } - return intervals.Shift(media::TimeUnit() - mInfo.mStartTime); -} - void MediaFormatReader::ReleaseResources() { mVideo.ShutdownDecoder(); @@ -2240,22 +2523,16 @@ MediaFormatReader::VideoIsHardwareAccelerated() const } void -MediaFormatReader::NotifyDemuxer() +MediaFormatReader::NotifyTrackDemuxers() { MOZ_ASSERT(OnTaskQueue()); - if (mShutdown || !mDemuxer || - (!mDemuxerInitDone && !mDemuxerInitRequest.Exists())) { - return; - } - LOGV(""); - mDemuxer->NotifyDataArrived(); - if (!mInitDone) { return; } + if (HasVideo()) { mVideo.mReceivedNewData = true; ScheduleUpdate(TrackType::kVideoTrack); @@ -2267,10 +2544,85 @@ MediaFormatReader::NotifyDemuxer() } void -MediaFormatReader::NotifyDataArrivedInternal() +MediaFormatReader::NotifyDataArrived() { MOZ_ASSERT(OnTaskQueue()); - NotifyDemuxer(); + + if (mShutdown || !mDemuxer || + (!mDemuxerInitDone && !mDemuxerInitRequest.Exists())) { + return; + } + + RefPtr self = this; + mDemuxer->NotifyDataArrived()->Then( + OwnerThread(), __func__, + [self]() { + self->UpdateBuffered(); + self->NotifyTrackDemuxers(); + }, + []() {}); +} + +void +MediaFormatReader::UpdateBuffered() +{ + MOZ_ASSERT(OnTaskQueue()); + + if (mShutdown) { + return; + } + + if (!mInitDone || !mHasStartTime) { + mBuffered = TimeIntervals(); + return; + } + + if (HasVideo()) { + mVideo.mTimeRanges = mVideo.mTrackDemuxer->GetBuffered(); + bool hasLastEnd; + media::TimeUnit lastEnd = mVideo.mTimeRanges.GetEnd(&hasLastEnd); + if (hasLastEnd) { + if (mVideo.mLastTimeRangesEnd + && mVideo.mLastTimeRangesEnd.ref() < lastEnd) { + // New data was added after our previous end, we can clear the EOS flag. + mVideo.mDemuxEOS = false; + ScheduleUpdate(TrackInfo::kVideoTrack); + } + mVideo.mLastTimeRangesEnd = Some(lastEnd); + } + } + if (HasAudio()) { + mAudio.mTimeRanges = mAudio.mTrackDemuxer->GetBuffered(); + bool hasLastEnd; + media::TimeUnit lastEnd = mAudio.mTimeRanges.GetEnd(&hasLastEnd); + if (hasLastEnd) { + if (mAudio.mLastTimeRangesEnd + && mAudio.mLastTimeRangesEnd.ref() < lastEnd) { + // New data was added after our previous end, we can clear the EOS flag. + mAudio.mDemuxEOS = false; + ScheduleUpdate(TrackInfo::kAudioTrack); + } + mAudio.mLastTimeRangesEnd = Some(lastEnd); + } + } + + media::TimeIntervals intervals; + if (HasAudio() && HasVideo()) { + intervals = media::Intersection(mVideo.mTimeRanges, mAudio.mTimeRanges); + } else if (HasAudio()) { + intervals = mAudio.mTimeRanges; + } else if (HasVideo()) { + intervals = mVideo.mTimeRanges; + } + + if (!intervals.Length() || + intervals.GetStart() == media::TimeUnit::FromMicroseconds(0)) { + // IntervalSet already starts at 0 or is empty, nothing to shift. + mBuffered = intervals; + } else { + mBuffered = + intervals.Shift(media::TimeUnit() - mInfo.mStartTime); + } } bool diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h index 4c916e9a2ab2..c9f0d7ee7e3c 100644 --- a/dom/media/MediaFormatReader.h +++ b/dom/media/MediaFormatReader.h @@ -25,6 +25,7 @@ class CDMProxy; class MediaFormatReader final : public MediaDecoderReader { typedef TrackInfo::TrackType TrackType; + typedef MozPromise NotifyDataArrivedPromise; public: MediaFormatReader(AbstractMediaDecoder* aDecoder, @@ -49,11 +50,10 @@ public: Seek(const SeekTarget& aTarget, int64_t aUnused) override; protected: - void NotifyDataArrivedInternal() override; + void NotifyDataArrived() override; + void UpdateBuffered() override; public: - media::TimeIntervals GetBuffered() override; - bool ForceZeroStartTime() const override; // For Media Resource Management @@ -92,10 +92,8 @@ private: bool IsWaitingOnCDMResource(); bool InitDemuxer(); - // Notify the demuxer that new data has been received. - // The next queued task calling GetBuffered() is guaranteed to have up to date - // buffered ranges. - void NotifyDemuxer(); + // Notify the track demuxers that new data has been received. + void NotifyTrackDemuxers(); void ReturnOutput(MediaData* aData, TrackType aTrack); // Enqueues a task to call Update(aTrack) on the decoder task queue. @@ -482,7 +480,8 @@ private: DecoderData& GetDecoderData(TrackType aTrack); // Demuxer objects. - RefPtr mDemuxer; + class DemuxerProxy; + UniquePtr mDemuxer; bool mDemuxerInitDone; void OnDemuxerInitDone(nsresult); void OnDemuxerInitFailed(const MediaResult& aError);