From de3872af290dd2e538442448328579ad93e4627b Mon Sep 17 00:00:00 2001 From: "Carsten \"Tomcat\" Book" Date: Tue, 21 Jul 2015 08:42:54 +0200 Subject: [PATCH] Backed out changeset 1704ea727e81 (bug 1163486) for at least b2g bustage --- dom/media/DecoderTraits.cpp | 5 +- dom/media/fmp4/MP4Decoder.cpp | 11 +- dom/media/fmp4/MP4Demuxer.cpp | 8 - dom/media/fmp4/MP4Reader.cpp | 1136 +++++++++++++++++ dom/media/fmp4/MP4Reader.h | 280 ++++ dom/media/fmp4/MP4Stream.h | 2 +- dom/media/fmp4/moz.build | 2 + dom/media/gtest/moz.build | 1 + dom/media/mediasource/MediaSourceReader.cpp | 10 +- dom/media/platforms/apple/AppleATDecoder.cpp | 1 + dom/media/platforms/apple/AppleVDADecoder.h | 1 + .../platforms/ffmpeg/FFmpegDataDecoder.h | 1 + dom/media/platforms/wmf/WMFAudioMFTManager.h | 1 + dom/media/platforms/wmf/WMFMediaDataDecoder.h | 2 +- dom/media/platforms/wmf/WMFVideoMFTManager.h | 1 + dom/media/webm/IntelWebMVideoDecoder.cpp | 2 + dom/media/webm/IntelWebMVideoDecoder.h | 1 + media/libstagefright/binding/Box.cpp | 2 +- .../binding/MP4TrackDemuxer.cpp | 51 + .../include/mp4_demuxer/MP4TrackDemuxer.h | 38 + .../binding/include/mp4_demuxer/MoofParser.h | 5 +- .../binding/include/mp4_demuxer/mp4_demuxer.h | 84 ++ media/libstagefright/binding/mp4_demuxer.cpp | 245 ++++ media/libstagefright/moz.build | 4 + modules/libpref/init/all.js | 5 + 25 files changed, 1877 insertions(+), 22 deletions(-) create mode 100644 dom/media/fmp4/MP4Reader.cpp create mode 100644 dom/media/fmp4/MP4Reader.h create mode 100644 media/libstagefright/binding/MP4TrackDemuxer.cpp create mode 100644 media/libstagefright/binding/include/mp4_demuxer/MP4TrackDemuxer.h create mode 100644 media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h create mode 100644 media/libstagefright/binding/mp4_demuxer.cpp diff --git a/dom/media/DecoderTraits.cpp b/dom/media/DecoderTraits.cpp index deca205e28ef..0d46a19eefeb 100644 --- a/dom/media/DecoderTraits.cpp +++ b/dom/media/DecoderTraits.cpp @@ -66,6 +66,7 @@ #include "AppleMP3Reader.h" #endif #ifdef MOZ_FMP4 +#include "MP4Reader.h" #include "MP4Decoder.h" #include "MP4Demuxer.h" #endif @@ -653,7 +654,9 @@ MediaDecoderReader* DecoderTraits::CreateReader(const nsACString& aType, Abstrac } #ifdef MOZ_FMP4 if (IsMP4SupportedType(aType)) { - decoderReader = new MediaFormatReader(aDecoder, new MP4Demuxer(aDecoder->GetResource())); + decoderReader = Preferences::GetBool("media.format-reader.mp4", true) ? + static_cast(new MediaFormatReader(aDecoder, new MP4Demuxer(aDecoder->GetResource()))) : + static_cast(new MP4Reader(aDecoder)); } else #endif if (IsMP3SupportedType(aType)) { diff --git a/dom/media/fmp4/MP4Decoder.cpp b/dom/media/fmp4/MP4Decoder.cpp index dd282cb9cb5f..2679f6ed96e7 100644 --- a/dom/media/fmp4/MP4Decoder.cpp +++ b/dom/media/fmp4/MP4Decoder.cpp @@ -5,6 +5,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "MP4Decoder.h" +#include "MP4Reader.h" #include "MediaDecoderStateMachine.h" #include "MediaFormatReader.h" #include "MP4Demuxer.h" @@ -41,7 +42,11 @@ MP4Decoder::MP4Decoder() MediaDecoderStateMachine* MP4Decoder::CreateStateMachine() { - MediaDecoderReader* reader = new MediaFormatReader(this, new MP4Demuxer(GetResource())); + bool useFormatDecoder = + Preferences::GetBool("media.format-reader.mp4", true); + nsRefPtr reader = useFormatDecoder ? + static_cast(new MediaFormatReader(this, new MP4Demuxer(GetResource()))) : + static_cast(new MP4Reader(this)); return new MediaDecoderStateMachine(this, reader); } @@ -53,8 +58,8 @@ MP4Decoder::SetCDMProxy(CDMProxy* aProxy) nsresult rv = MediaDecoder::SetCDMProxy(aProxy); NS_ENSURE_SUCCESS(rv, rv); if (aProxy) { - // The MediaFormatReader can't decrypt EME content until it has a CDMProxy, - // and the CDMProxy knows the capabilities of the CDM. The MediaFormatReader + // The MP4Reader can't decrypt EME content until it has a CDMProxy, + // and the CDMProxy knows the capabilities of the CDM. The MP4Reader // remains in "waiting for resources" state until then. CDMCaps::AutoLock caps(aProxy->Capabilites()); nsCOMPtr task( diff --git a/dom/media/fmp4/MP4Demuxer.cpp b/dom/media/fmp4/MP4Demuxer.cpp index a21ff213941f..c6d3d75eb388 100644 --- a/dom/media/fmp4/MP4Demuxer.cpp +++ b/dom/media/fmp4/MP4Demuxer.cpp @@ -20,14 +20,6 @@ #include "mp4_demuxer/AnnexB.h" #include "mp4_demuxer/H264.h" -PRLogModuleInfo* GetDemuxerLog() { - static PRLogModuleInfo* log = nullptr; - if (!log) { - log = PR_NewLogModule("MP4Demuxer"); - } - return log; -} - namespace mozilla { // Returns true if no SPS was found and search for it should continue. diff --git a/dom/media/fmp4/MP4Reader.cpp b/dom/media/fmp4/MP4Reader.cpp new file mode 100644 index 000000000000..b31e1c0b1e1d --- /dev/null +++ b/dom/media/fmp4/MP4Reader.cpp @@ -0,0 +1,1136 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MP4Reader.h" +#include "MP4Stream.h" +#include "MediaData.h" +#include "MediaInfo.h" +#include "MediaResource.h" +#include "nsPrintfCString.h" +#include "nsSize.h" +#include "VideoUtils.h" +#include "mozilla/dom/HTMLMediaElement.h" +#include "ImageContainer.h" +#include "Layers.h" +#include "SharedThreadPool.h" +#include "mozilla/Preferences.h" +#include "mozilla/Telemetry.h" +#include "mp4_demuxer/AnnexB.h" +#include "mp4_demuxer/H264.h" +#include "SharedDecoderManager.h" +#include "mp4_demuxer/MP4TrackDemuxer.h" +#include + +#ifdef MOZ_EME +#include "mozilla/CDMProxy.h" +#endif + +using mozilla::layers::Image; +using mozilla::layers::LayerManager; +using mozilla::layers::ImageContainer; +using mozilla::layers::LayersBackend; +using mozilla::media::TimeUnit; + +PRLogModuleInfo* GetDemuxerLog() { + static PRLogModuleInfo* log = nullptr; + if (!log) { + log = PR_NewLogModule("MP4Demuxer"); + } + return log; +} +#define LOG(arg, ...) MOZ_LOG(GetDemuxerLog(), mozilla::LogLevel::Debug, ("MP4Reader(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) +#define VLOG(arg, ...) MOZ_LOG(GetDemuxerLog(), mozilla::LogLevel::Debug, ("MP4Reader(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +using namespace mp4_demuxer; + +namespace mozilla { + +// Uncomment to enable verbose per-sample logging. +//#define LOG_SAMPLE_DECODE 1 + +static const char* +TrackTypeToStr(TrackInfo::TrackType aTrack) +{ + MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack || + aTrack == TrackInfo::kVideoTrack); + switch (aTrack) { + case TrackInfo::kAudioTrack: + return "Audio"; + case TrackInfo::kVideoTrack: + return "Video"; + default: + return "Unknown"; + } +} + +extern bool AccumulateSPSTelemetry(const MediaByteBuffer* aExtradata); + +// MP4Demuxer wants to do various blocking reads, which cause deadlocks while +// mDemuxerMonitor is held. This stuff should really be redesigned, but we don't +// have time for that right now. So in order to get proper synchronization while +// keeping behavior as similar as possible, we do the following nasty hack: +// +// The demuxer has a Stream object with APIs to do both blocking and non-blocking +// reads. When it does a blocking read, MP4Stream actually redirects it to a non- +// blocking read, but records the parameters of the read on the MP4Stream itself. +// This means that, when the read failure bubbles up to MP4Reader.cpp, we can +// detect whether we in fact just needed to block, and do that while releasing the +// monitor. We distinguish these fake failures from bonafide EOS by tracking the +// previous failed read as well. If we ever do a blocking read on the same segment +// twice, we know we've hit EOS. +template +ReturnType +InvokeAndRetry(ThisType* aThisVal, ReturnType(ThisType::*aMethod)(), MP4Stream* aStream, Monitor* aMonitor) +{ + AutoPinned stream(aStream); + MP4Stream::ReadRecord prevFailure(-1, 0); + while (true) { + ReturnType result = ((*aThisVal).*aMethod)(); + if (result) { + return result; + } + MP4Stream::ReadRecord failure(-1, 0); + if (NS_WARN_IF(!stream->LastReadFailed(&failure))) { + return result; + } + stream->ClearFailedRead(); + + if (NS_WARN_IF(failure == prevFailure)) { + NS_WARNING(nsPrintfCString("Failed reading the same block twice: offset=%lld, count=%lu", + failure.mOffset, failure.mCount).get()); + return result; + } + + prevFailure = failure; + if (NS_WARN_IF(!stream->BlockingReadIntoCache(failure.mOffset, failure.mCount, aMonitor))) { + return result; + } + } +} + +MP4Reader::MP4Reader(AbstractMediaDecoder* aDecoder, TaskQueue* aBorrowedTaskQueue) + : MediaDecoderReader(aDecoder, aBorrowedTaskQueue) + , mAudio(MediaData::AUDIO_DATA, Preferences::GetUint("media.mp4-audio-decode-ahead", 2)) + , mVideo(MediaData::VIDEO_DATA, Preferences::GetUint("media.mp4-video-decode-ahead", 2)) + , mLastReportedNumDecodedFrames(0) + , mLayersBackendType(layers::LayersBackend::LAYERS_NONE) + , mDemuxerInitialized(false) + , mFoundSPSForTelemetry(false) + , mIsEncrypted(false) + , mIndexReady(false) + , mLastSeenEnd(-1) + , mDemuxerMonitor("MP4 Demuxer") +{ + MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); + MOZ_COUNT_CTOR(MP4Reader); +} + +MP4Reader::~MP4Reader() +{ + MOZ_COUNT_DTOR(MP4Reader); +} + +nsRefPtr +MP4Reader::Shutdown() +{ + MOZ_ASSERT(OnTaskQueue()); + + if (mAudio.mDecoder) { + Flush(TrackInfo::kAudioTrack); + mAudio.mDecoder->Shutdown(); + mAudio.mDecoder = nullptr; + } + if (mAudio.mTaskQueue) { + mAudio.mTaskQueue->BeginShutdown(); + mAudio.mTaskQueue->AwaitShutdownAndIdle(); + mAudio.mTaskQueue = nullptr; + } + mAudio.mPromise.SetMonitor(nullptr); + MOZ_ASSERT(mAudio.mPromise.IsEmpty()); + + if (mVideo.mDecoder) { + Flush(TrackInfo::kVideoTrack); + mVideo.mDecoder->Shutdown(); + mVideo.mDecoder = nullptr; + } + if (mVideo.mTaskQueue) { + mVideo.mTaskQueue->BeginShutdown(); + mVideo.mTaskQueue->AwaitShutdownAndIdle(); + mVideo.mTaskQueue = nullptr; + } + mVideo.mPromise.SetMonitor(nullptr); + MOZ_ASSERT(mVideo.mPromise.IsEmpty()); + // Dispose of the queued sample before shutting down the demuxer + mQueuedVideoSample = nullptr; + + mPlatform = nullptr; + + return MediaDecoderReader::Shutdown(); +} + +void +MP4Reader::InitLayersBackendType() +{ + if (!IsVideoContentType(mDecoder->GetResource()->GetContentType())) { + // Not playing video, we don't care about the layers backend type. + return; + } + // Extract the layer manager backend type so that platform decoders + // can determine whether it's worthwhile using hardware accelerated + // video decoding. + MediaDecoderOwner* owner = mDecoder->GetOwner(); + if (!owner) { + NS_WARNING("MP4Reader without a decoder owner, can't get HWAccel"); + return; + } + + dom::HTMLMediaElement* element = owner->GetMediaElement(); + NS_ENSURE_TRUE_VOID(element); + + nsRefPtr layerManager = + nsContentUtils::LayerManagerForDocument(element->OwnerDoc()); + NS_ENSURE_TRUE_VOID(layerManager); + + mLayersBackendType = layerManager->GetCompositorBackendType(); +} + +static bool sIsEMEEnabled = false; +static bool sDemuxSkipToNextKeyframe = true; + +nsresult +MP4Reader::Init(MediaDecoderReader* aCloneDonor) +{ + MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); + PlatformDecoderModule::Init(); + mStream = new MP4Stream(mDecoder->GetResource()); + + InitLayersBackendType(); + + mAudio.mTaskQueue = + new FlushableTaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); + NS_ENSURE_TRUE(mAudio.mTaskQueue, NS_ERROR_FAILURE); + + mVideo.mTaskQueue = + new FlushableTaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); + NS_ENSURE_TRUE(mVideo.mTaskQueue, NS_ERROR_FAILURE); + + static bool sSetupPrefCache = false; + if (!sSetupPrefCache) { + sSetupPrefCache = true; + Preferences::AddBoolVarCache(&sIsEMEEnabled, "media.eme.enabled", false); + Preferences::AddBoolVarCache(&sDemuxSkipToNextKeyframe, "media.fmp4.demux-skip", true); + } + + return NS_OK; +} + +#ifdef MOZ_EME +class DispatchKeyNeededEvent : public nsRunnable { +public: + DispatchKeyNeededEvent(AbstractMediaDecoder* aDecoder, + nsTArray& aInitData, + const nsString& aInitDataType) + : mDecoder(aDecoder) + , mInitData(aInitData) + , mInitDataType(aInitDataType) + { + } + NS_IMETHOD Run() { + // Note: Null check the owner, as the decoder could have been shutdown + // since this event was dispatched. + MediaDecoderOwner* owner = mDecoder->GetOwner(); + if (owner) { + owner->DispatchEncrypted(mInitData, mInitDataType); + } + mDecoder = nullptr; + return NS_OK; + } +private: + nsRefPtr mDecoder; + nsTArray mInitData; + nsString mInitDataType; +}; +#endif // MOZ_EME + +bool MP4Reader::IsWaitingOnCDMResource() { +#ifdef MOZ_EME + nsRefPtr proxy; + { + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); + if (!mIsEncrypted) { + // Not encrypted, no need to wait for CDMProxy. + return false; + } + proxy = mDecoder->GetCDMProxy(); + if (!proxy) { + // We're encrypted, we need a CDMProxy to decrypt file. + return true; + } + } + // We'll keep waiting if the CDM hasn't informed Gecko of its capabilities. + { + CDMCaps::AutoLock caps(proxy->Capabilites()); + LOG("capsKnown=%d", caps.AreCapsKnown()); + return !caps.AreCapsKnown(); + } +#else + return false; +#endif +} + +void +MP4Reader::ExtractCryptoInitData(nsTArray& aInitData) +{ + MOZ_ASSERT(mCrypto.valid); + const nsTArray& psshs = mCrypto.pssh; + for (uint32_t i = 0; i < psshs.Length(); i++) { + aInitData.AppendElements(psshs[i].data); + } +} + +bool +MP4Reader::IsSupportedAudioMimeType(const nsACString& aMimeType) +{ + return (aMimeType.EqualsLiteral("audio/mpeg") || + aMimeType.EqualsLiteral("audio/mp4a-latm") || + aMimeType.EqualsLiteral("audio/amr-wb") || + aMimeType.EqualsLiteral("audio/3gpp")) && + mPlatform->SupportsMimeType(aMimeType); +} + +bool +MP4Reader::IsSupportedVideoMimeType(const nsACString& aMimeType) +{ + return (aMimeType.EqualsLiteral("video/mp4") || + aMimeType.EqualsLiteral("video/mp4v-es") || + aMimeType.EqualsLiteral("video/avc") || + aMimeType.EqualsLiteral("video/x-vnd.on2.vp6")) && + mPlatform->SupportsMimeType(aMimeType); +} + +bool +MP4Reader::InitDemuxer() +{ + mDemuxer = new MP4Demuxer(mStream, &mDemuxerMonitor); + return mDemuxer->Init(); +} + +nsresult +MP4Reader::ReadMetadata(MediaInfo* aInfo, + MetadataTags** aTags) +{ + if (!mDemuxerInitialized) { + MonitorAutoLock mon(mDemuxerMonitor); + bool ok = InvokeAndRetry(this, &MP4Reader::InitDemuxer, mStream, &mDemuxerMonitor); + NS_ENSURE_TRUE(ok, NS_ERROR_FAILURE); + mIndexReady = true; + + // To decode, we need valid video and a place to put it. + mVideo.mActive = mDemuxer->HasValidVideo() && mDecoder->GetImageContainer(); + if (mVideo.mActive) { + mVideo.mTrackDemuxer = new MP4VideoDemuxer(mDemuxer); + } + + mAudio.mActive = mDemuxer->HasValidAudio(); + if (mAudio.mActive) { + mAudio.mTrackDemuxer = new MP4AudioDemuxer(mDemuxer); + } + mCrypto = mDemuxer->Crypto(); + + { + MonitorAutoUnlock unlock(mDemuxerMonitor); + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); + mIsEncrypted = mCrypto.valid; + } + + // Remember that we've initialized the demuxer, so that if we're decoding + // an encrypted stream and we need to wait for a CDM to be set, we don't + // need to reinit the demuxer. + mDemuxerInitialized = true; + } else if (mPlatform) { + *aInfo = mInfo; + *aTags = nullptr; + NS_ENSURE_TRUE(EnsureDecodersSetup(), NS_ERROR_FAILURE); + return NS_OK; + } + + if (HasAudio()) { + mInfo.mAudio = mDemuxer->AudioConfig(); + mAudio.mCallback = new DecoderCallback(this, TrackInfo::kAudioTrack); + } + + if (HasVideo()) { + mInfo.mVideo = mDemuxer->VideoConfig(); + mVideo.mCallback = new DecoderCallback(this, TrackInfo::kVideoTrack); + + // Collect telemetry from h264 AVCC SPS. + if (!mFoundSPSForTelemetry) { + mFoundSPSForTelemetry = AccumulateSPSTelemetry(mInfo.mVideo.mExtraData); + } + } + + if (mCrypto.valid) { + nsTArray initData; + ExtractCryptoInitData(initData); + if (initData.Length() == 0) { + return NS_ERROR_FAILURE; + } + +#ifdef MOZ_EME + // Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING. + NS_DispatchToMainThread( + new DispatchKeyNeededEvent(mDecoder, initData, NS_LITERAL_STRING("cenc"))); +#endif // MOZ_EME + // Add init data to info, will get sent from HTMLMediaElement::MetadataLoaded + // (i.e., when transitioning from HAVE_NOTHING to HAVE_METADATA). + mInfo.mCrypto.AddInitData(NS_LITERAL_STRING("cenc"), Move(initData)); + } + + // Get the duration, and report it to the decoder if we have it. + mp4_demuxer::Microseconds duration; + { + MonitorAutoLock lock(mDemuxerMonitor); + duration = mDemuxer->Duration(); + } + if (duration != -1) { + mInfo.mMetadataDuration = Some(TimeUnit::FromMicroseconds(duration)); + } + + *aInfo = mInfo; + *aTags = nullptr; + + if (!IsWaitingOnCDMResource()) { + NS_ENSURE_TRUE(EnsureDecodersSetup(), NS_ERROR_FAILURE); + } + + MonitorAutoLock mon(mDemuxerMonitor); + UpdateIndex(); + + return NS_OK; +} + +bool +MP4Reader::EnsureDecodersSetup() +{ + MOZ_ASSERT(mDemuxerInitialized); + + if (!mPlatform) { + if (mIsEncrypted) { +#ifdef MOZ_EME + // We have encrypted audio or video. We'll need a CDM to decrypt and + // possibly decode this. Wait until we've received a CDM from the + // JavaScript player app. Note: we still go through the motions here + // even if EME is disabled, so that if script tries and fails to create + // a CDM, we can detect that and notify chrome and show some UI + // explaining that we failed due to EME being disabled. + nsRefPtr proxy; + { + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); + proxy = mDecoder->GetCDMProxy(); + } + MOZ_ASSERT(proxy); + + mPlatform = PlatformDecoderModule::CreateCDMWrapper(proxy); + NS_ENSURE_TRUE(mPlatform, false); +#else + // EME not supported. + return false; +#endif + } else { + mPlatform = PlatformDecoderModule::Create(); + NS_ENSURE_TRUE(mPlatform, false); + } + } + + MOZ_ASSERT(mPlatform); + + if (HasAudio() && !mAudio.mDecoder) { + NS_ENSURE_TRUE(IsSupportedAudioMimeType(mDemuxer->AudioConfig().mMimeType), + false); + + mAudio.mDecoder = + mPlatform->CreateDecoder(mDemuxer->AudioConfig(), + mAudio.mTaskQueue, + mAudio.mCallback); + NS_ENSURE_TRUE(mAudio.mDecoder != nullptr, false); + nsresult rv = mAudio.mDecoder->Init(); + NS_ENSURE_SUCCESS(rv, false); + } + + if (HasVideo() && !mVideo.mDecoder) { + NS_ENSURE_TRUE(IsSupportedVideoMimeType(mDemuxer->VideoConfig().mMimeType), + false); + + if (mSharedDecoderManager && + mPlatform->SupportsSharedDecoders(mDemuxer->VideoConfig())) { + mVideo.mDecoder = + mSharedDecoderManager->CreateVideoDecoder(mPlatform, + mDemuxer->VideoConfig(), + mLayersBackendType, + mDecoder->GetImageContainer(), + mVideo.mTaskQueue, + mVideo.mCallback); + } else { + mVideo.mDecoder = + mPlatform->CreateDecoder(mDemuxer->VideoConfig(), + mVideo.mTaskQueue, + mVideo.mCallback, + mLayersBackendType, + mDecoder->GetImageContainer()); + } + NS_ENSURE_TRUE(mVideo.mDecoder != nullptr, false); + nsresult rv = mVideo.mDecoder->Init(); + NS_ENSURE_SUCCESS(rv, false); + } + + return true; +} + +void +MP4Reader::ReadUpdatedMetadata(MediaInfo* aInfo) +{ + *aInfo = mInfo; +} + +bool +MP4Reader::IsMediaSeekable() +{ + // Check Demuxer to see if this content is seekable or not. + MonitorAutoLock mon(mDemuxerMonitor); + return mDemuxer->CanSeek(); +} + +bool +MP4Reader::HasAudio() +{ + return mAudio.mActive; +} + +bool +MP4Reader::HasVideo() +{ + return mVideo.mActive; +} + +MP4Reader::DecoderData& +MP4Reader::GetDecoderData(TrackType aTrack) +{ + MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack || + aTrack == TrackInfo::kVideoTrack); + if (aTrack == TrackInfo::kAudioTrack) { + return mAudio; + } + return mVideo; +} + +mp4_demuxer::Microseconds +MP4Reader::GetNextKeyframeTime() +{ + MonitorAutoLock mon(mDemuxerMonitor); + return mVideo.mTrackDemuxer->GetNextKeyframeTime(); +} + +void +MP4Reader::DisableHardwareAcceleration() +{ + if (HasVideo() && mSharedDecoderManager) { + mSharedDecoderManager->DisableHardwareAcceleration(); + + const VideoInfo& video = mDemuxer->VideoConfig(); + if (!mSharedDecoderManager->Recreate(video)) { + MonitorAutoLock mon(mVideo.mMonitor); + mVideo.mError = true; + if (mVideo.HasPromise()) { + mVideo.RejectPromise(DECODE_ERROR, __func__); + } + } else { + MonitorAutoLock lock(mVideo.mMonitor); + ScheduleUpdate(TrackInfo::kVideoTrack); + } + } +} + +bool +MP4Reader::ShouldSkip(bool aSkipToNextKeyframe, int64_t aTimeThreshold) +{ + // The MP4Reader doesn't do normal skip-to-next-keyframe if the demuxer + // has exposes where the next keyframe is. We can then instead skip only + // if the time threshold (the current playback position) is after the next + // keyframe in the stream. This means we'll only skip frames that we have + // no hope of ever playing. + mp4_demuxer::Microseconds nextKeyframe = -1; + if (!sDemuxSkipToNextKeyframe || + (nextKeyframe = GetNextKeyframeTime()) == -1) { + return aSkipToNextKeyframe; + } + return nextKeyframe < aTimeThreshold && nextKeyframe >= 0; +} + +nsRefPtr +MP4Reader::RequestVideoData(bool aSkipToNextKeyframe, + int64_t aTimeThreshold, + bool aForceDecodeAhead) +{ + MOZ_ASSERT(OnTaskQueue()); + VLOG("skip=%d time=%lld", aSkipToNextKeyframe, aTimeThreshold); + + if (!EnsureDecodersSetup()) { + NS_WARNING("Error constructing MP4 decoders"); + return VideoDataPromise::CreateAndReject(DECODE_ERROR, __func__); + } + + if (mShutdown) { + NS_WARNING("RequestVideoData on shutdown MP4Reader!"); + return VideoDataPromise::CreateAndReject(CANCELED, __func__); + } + + MOZ_ASSERT(HasVideo() && mPlatform && mVideo.mDecoder); + + bool eos = false; + mVideo.mForceDecodeAhead = aForceDecodeAhead; + if (ShouldSkip(aSkipToNextKeyframe, aTimeThreshold)) { + uint32_t parsed = 0; + eos = !SkipVideoDemuxToNextKeyFrame(aTimeThreshold, parsed); + if (!eos && NS_FAILED(mVideo.mDecoder->Flush())) { + NS_WARNING("Failed to skip/flush video when skipping-to-next-keyframe."); + } + mDecoder->NotifyDecodedFrames(parsed, 0, parsed); + } + + MonitorAutoLock lock(mVideo.mMonitor); + nsRefPtr p = mVideo.mPromise.Ensure(__func__); + if (mVideo.mError) { + mVideo.mPromise.Reject(DECODE_ERROR, __func__); + } else if (eos) { + mVideo.mPromise.Reject(END_OF_STREAM, __func__); + } else { + ScheduleUpdate(TrackInfo::kVideoTrack); + } + + return p; +} + +nsRefPtr +MP4Reader::RequestAudioData() +{ + MOZ_ASSERT(OnTaskQueue()); + VLOG(""); + + if (!EnsureDecodersSetup()) { + NS_WARNING("Error constructing MP4 decoders"); + return AudioDataPromise::CreateAndReject(DECODE_ERROR, __func__); + } + + if (mShutdown) { + NS_WARNING("RequestAudioData on shutdown MP4Reader!"); + return AudioDataPromise::CreateAndReject(CANCELED, __func__); + } + + MonitorAutoLock lock(mAudio.mMonitor); + nsRefPtr p = mAudio.mPromise.Ensure(__func__); + ScheduleUpdate(TrackInfo::kAudioTrack); + return p; +} + +void +MP4Reader::ScheduleUpdate(TrackType aTrack) +{ + auto& decoder = GetDecoderData(aTrack); + decoder.mMonitor.AssertCurrentThreadOwns(); + if (decoder.mUpdateScheduled) { + return; + } + VLOG("SchedulingUpdate(%s)", TrackTypeToStr(aTrack)); + decoder.mUpdateScheduled = true; + RefPtr task( + NS_NewRunnableMethodWithArg(this, &MP4Reader::Update, aTrack)); + OwnerThread()->Dispatch(task.forget()); +} + +bool +MP4Reader::NeedInput(DecoderData& aDecoder) +{ + aDecoder.mMonitor.AssertCurrentThreadOwns(); + // We try to keep a few more compressed samples input than decoded samples + // have been output, provided the state machine has requested we send it a + // decoded sample. To account for H.264 streams which may require a longer + // run of input than we input, decoders fire an "input exhausted" callback, + // which overrides our "few more samples" threshold. + return + !aDecoder.mError && + !aDecoder.mDemuxEOS && + (aDecoder.HasPromise() || aDecoder.mForceDecodeAhead) && + aDecoder.mOutput.IsEmpty() && + (aDecoder.mInputExhausted || + aDecoder.mForceDecodeAhead || + aDecoder.mNumSamplesInput - aDecoder.mNumSamplesOutput < aDecoder.mDecodeAhead); +} + +void +MP4Reader::Update(TrackType aTrack) +{ + MOZ_ASSERT(OnTaskQueue()); + + if (mShutdown) { + return; + } + + // Record number of frames decoded and parsed. Automatically update the + // stats counters using the AutoNotifyDecoded stack-based class. + AbstractMediaDecoder::AutoNotifyDecoded a(mDecoder); + + bool needInput = false; + bool needOutput = false; + auto& decoder = GetDecoderData(aTrack); + { + MonitorAutoLock lock(decoder.mMonitor); + decoder.mUpdateScheduled = false; + if (NeedInput(decoder)) { + needInput = true; + decoder.mInputExhausted = false; + decoder.mNumSamplesInput++; + } + if (aTrack == TrackInfo::kVideoTrack) { + uint64_t delta = + decoder.mNumSamplesOutput - mLastReportedNumDecodedFrames; + a.mDecoded = static_cast(delta); + mLastReportedNumDecodedFrames = decoder.mNumSamplesOutput; + } + if (decoder.HasPromise()) { + needOutput = true; + if (!decoder.mOutput.IsEmpty()) { + nsRefPtr output = decoder.mOutput[0]; + decoder.mOutput.RemoveElementAt(0); + ReturnOutput(output, aTrack); + } else if (decoder.mDrainComplete) { + decoder.RejectPromise(END_OF_STREAM, __func__); + } + } + } + + VLOG("Update(%s) ni=%d no=%d iex=%d fl=%d", + TrackTypeToStr(aTrack), + needInput, + needOutput, + decoder.mInputExhausted, + decoder.mIsFlushing); + + if (needInput) { + nsRefPtr sample(PopSample(aTrack)); + + // Collect telemetry from h264 Annex B SPS. + if (!mFoundSPSForTelemetry && sample && AnnexB::HasSPS(sample)) { + nsRefPtr extradata = AnnexB::ExtractExtraData(sample); + mFoundSPSForTelemetry = AccumulateSPSTelemetry(extradata); + } + + if (sample) { + decoder.mDecoder->Input(sample); + if (aTrack == TrackInfo::kVideoTrack) { + a.mParsed++; + } + } else { + { + MonitorAutoLock lock(decoder.mMonitor); + MOZ_ASSERT(!decoder.mDemuxEOS); + decoder.mDemuxEOS = true; + } + // DrainComplete takes care of reporting EOS upwards + decoder.mDecoder->Drain(); + } + } +} + +void +MP4Reader::ReturnOutput(MediaData* aData, TrackType aTrack) +{ + auto& decoder = GetDecoderData(aTrack); + decoder.mMonitor.AssertCurrentThreadOwns(); + MOZ_ASSERT(decoder.HasPromise()); + if (decoder.mDiscontinuity) { + decoder.mDiscontinuity = false; + aData->mDiscontinuity = true; + } + + if (aTrack == TrackInfo::kAudioTrack) { + AudioData* audioData = static_cast(aData); + + if (audioData->mChannels != mInfo.mAudio.mChannels || + audioData->mRate != mInfo.mAudio.mRate) { + LOG("change of sampling rate:%d->%d", + mInfo.mAudio.mRate, audioData->mRate); + mInfo.mAudio.mRate = audioData->mRate; + mInfo.mAudio.mChannels = audioData->mChannels; + } + + mAudio.mPromise.Resolve(audioData, __func__); + } else if (aTrack == TrackInfo::kVideoTrack) { + mVideo.mPromise.Resolve(static_cast(aData), __func__); + } +} + +already_AddRefed +MP4Reader::PopSample(TrackType aTrack) +{ + MonitorAutoLock mon(mDemuxerMonitor); + return PopSampleLocked(aTrack); +} + +already_AddRefed +MP4Reader::PopSampleLocked(TrackType aTrack) +{ + mDemuxerMonitor.AssertCurrentThreadOwns(); + nsRefPtr sample; + switch (aTrack) { + case TrackInfo::kAudioTrack: + sample = + InvokeAndRetry(this, &MP4Reader::DemuxAudioSample, mStream, &mDemuxerMonitor); + return sample.forget(); + case TrackInfo::kVideoTrack: + if (mQueuedVideoSample) { + return mQueuedVideoSample.forget(); + } + sample = + InvokeAndRetry(this, &MP4Reader::DemuxVideoSample, mStream, &mDemuxerMonitor); + return sample.forget(); + default: + return nullptr; + } +} + +nsRefPtr +MP4Reader::DemuxAudioSample() +{ + nsRefPtr sample = mAudio.mTrackDemuxer->DemuxSample(); + return sample; +} + +nsRefPtr +MP4Reader::DemuxVideoSample() +{ + nsRefPtr sample = mVideo.mTrackDemuxer->DemuxSample(); + return sample; +} + +size_t +MP4Reader::SizeOfVideoQueueInFrames() +{ + return SizeOfQueue(TrackInfo::kVideoTrack); +} + +size_t +MP4Reader::SizeOfAudioQueueInFrames() +{ + return SizeOfQueue(TrackInfo::kAudioTrack); +} + +size_t +MP4Reader::SizeOfQueue(TrackType aTrack) +{ + auto& decoder = GetDecoderData(aTrack); + MonitorAutoLock lock(decoder.mMonitor); + return decoder.mOutput.Length() + (decoder.mNumSamplesInput - decoder.mNumSamplesOutput); +} + +nsresult +MP4Reader::ResetDecode() +{ + MOZ_ASSERT(OnTaskQueue()); + Flush(TrackInfo::kVideoTrack); + { + MonitorAutoLock mon(mDemuxerMonitor); + if (mVideo.mTrackDemuxer) { + mVideo.mTrackDemuxer->Seek(0); + } + } + Flush(TrackInfo::kAudioTrack); + { + MonitorAutoLock mon(mDemuxerMonitor); + if (mAudio.mTrackDemuxer) { + mAudio.mTrackDemuxer->Seek(0); + } + } + return MediaDecoderReader::ResetDecode(); +} + +void +MP4Reader::Output(TrackType aTrack, MediaData* aSample) +{ +#ifdef LOG_SAMPLE_DECODE + VLOG("Decoded %s sample time=%lld dur=%lld", + TrackTypeToStr(aTrack), aSample->mTime, aSample->mDuration); +#endif + + if (!aSample) { + NS_WARNING("MP4Reader::Output() passed a null sample"); + Error(aTrack); + return; + } + + auto& decoder = GetDecoderData(aTrack); + // Don't accept output while we're flushing. + MonitorAutoLock mon(decoder.mMonitor); + if (decoder.mIsFlushing) { + LOG("MP4Reader produced output while flushing, discarding."); + mon.NotifyAll(); + return; + } + + decoder.mOutput.AppendElement(aSample); + decoder.mNumSamplesOutput++; + if (NeedInput(decoder) || decoder.HasPromise()) { + ScheduleUpdate(aTrack); + } +} + +void +MP4Reader::DrainComplete(TrackType aTrack) +{ + DecoderData& data = GetDecoderData(aTrack); + MonitorAutoLock mon(data.mMonitor); + data.mDrainComplete = true; + ScheduleUpdate(aTrack); +} + +void +MP4Reader::InputExhausted(TrackType aTrack) +{ + DecoderData& data = GetDecoderData(aTrack); + MonitorAutoLock mon(data.mMonitor); + data.mInputExhausted = true; + ScheduleUpdate(aTrack); +} + +void +MP4Reader::Error(TrackType aTrack) +{ + DecoderData& data = GetDecoderData(aTrack); + { + MonitorAutoLock mon(data.mMonitor); + data.mError = true; + if (data.HasPromise()) { + data.RejectPromise(DECODE_ERROR, __func__); + } + } +} + +void +MP4Reader::Flush(TrackType aTrack) +{ + MOZ_ASSERT(OnTaskQueue()); + VLOG("Flush(%s) BEGIN", TrackTypeToStr(aTrack)); + DecoderData& data = GetDecoderData(aTrack); + if (!data.mDecoder) { + return; + } + // Purge the current decoder's state. + // Set a flag so that we ignore all output while we call + // MediaDataDecoder::Flush(). + { + MonitorAutoLock mon(data.mMonitor); + data.mIsFlushing = true; + data.mDemuxEOS = false; + } + data.mDecoder->Flush(); + { + MonitorAutoLock mon(data.mMonitor); + data.mForceDecodeAhead = false; + data.mIsFlushing = false; + data.mDrainComplete = false; + data.mOutput.Clear(); + data.mNumSamplesInput = 0; + data.mNumSamplesOutput = 0; + data.mInputExhausted = false; + if (data.HasPromise()) { + data.RejectPromise(CANCELED, __func__); + } + data.mDiscontinuity = true; + data.mUpdateScheduled = false; + } + if (aTrack == TrackInfo::kVideoTrack) { + mQueuedVideoSample = nullptr; + } + VLOG("Flush(%s) END", TrackTypeToStr(aTrack)); +} + +bool +MP4Reader::SkipVideoDemuxToNextKeyFrame(int64_t aTimeThreshold, uint32_t& parsed) +{ + MOZ_ASSERT(OnTaskQueue()); + + MOZ_ASSERT(mVideo.mDecoder); + + Flush(TrackInfo::kVideoTrack); + + // Loop until we reach the next keyframe after the threshold. + while (true) { + nsRefPtr compressed(PopSample(TrackInfo::kVideoTrack)); + if (!compressed) { + // EOS, or error. This code assumes EOS, which may or may not be right. + MonitorAutoLock mon(mVideo.mMonitor); + mVideo.mDemuxEOS = true; + return false; + } + parsed++; + if (!compressed->mKeyframe || + compressed->mTime < aTimeThreshold) { + continue; + } + mQueuedVideoSample = compressed; + break; + } + + return true; +} + +nsRefPtr +MP4Reader::Seek(int64_t aTime, int64_t aEndTime) +{ + LOG("aTime=(%lld)", aTime); + MOZ_ASSERT(OnTaskQueue()); + MonitorAutoLock mon(mDemuxerMonitor); + if (!mDemuxer->CanSeek()) { + VLOG("Seek() END (Unseekable)"); + return SeekPromise::CreateAndReject(NS_ERROR_FAILURE, __func__); + } + + int64_t seekTime = aTime; + mQueuedVideoSample = nullptr; + if (mDemuxer->HasValidVideo()) { + mVideo.mTrackDemuxer->Seek(seekTime); + mQueuedVideoSample = PopSampleLocked(TrackInfo::kVideoTrack); + if (mQueuedVideoSample) { + seekTime = mQueuedVideoSample->mTime; + } + } + if (mDemuxer->HasValidAudio()) { + mAudio.mTrackDemuxer->Seek(seekTime); + } + LOG("aTime=%lld exit", aTime); + return SeekPromise::CreateAndResolve(seekTime, __func__); +} + +void +MP4Reader::UpdateIndex() +{ + if (!mIndexReady) { + return; + } + + AutoPinned resource(mDecoder->GetResource()); + nsTArray ranges; + if (NS_SUCCEEDED(resource->GetCachedRanges(ranges))) { + mDemuxer->UpdateIndex(ranges); + } +} + +int64_t +MP4Reader::GetEvictionOffset(double aTime) +{ + MonitorAutoLock mon(mDemuxerMonitor); + if (!mIndexReady) { + return 0; + } + + return mDemuxer->GetEvictionOffset(aTime * 1000000.0); +} + +media::TimeIntervals +MP4Reader::GetBuffered() +{ + MOZ_ASSERT(OnTaskQueue()); + MonitorAutoLock mon(mDemuxerMonitor); + media::TimeIntervals buffered; + if (!mIndexReady) { + return buffered; + } + UpdateIndex(); + NS_ENSURE_TRUE(HaveStartTime(), media::TimeIntervals()); + + AutoPinned resource(mDecoder->GetResource()); + nsTArray ranges; + nsresult rv = resource->GetCachedRanges(ranges); + + if (NS_SUCCEEDED(rv)) { + nsTArray> timeRanges; + mDemuxer->ConvertByteRangesToTime(ranges, &timeRanges); + for (size_t i = 0; i < timeRanges.Length(); i++) { + buffered += media::TimeInterval( + media::TimeUnit::FromMicroseconds(timeRanges[i].start - StartTime()), + media::TimeUnit::FromMicroseconds(timeRanges[i].end - StartTime())); + } + } + + return buffered; +} + +void MP4Reader::ReleaseMediaResources() +{ + // Before freeing a video codec, all video buffers needed to be released + // even from graphics pipeline. + VideoFrameContainer* container = mDecoder->GetVideoFrameContainer(); + if (container) { + container->ClearCurrentFrame(); + } + if (mVideo.mDecoder) { + mVideo.mDecoder->Shutdown(); + mVideo.mDecoder = nullptr; + } +} + +void +MP4Reader::SetIdle() +{ + if (mSharedDecoderManager && mVideo.mDecoder) { + mSharedDecoderManager->SetIdle(mVideo.mDecoder); + } +} + +void +MP4Reader::SetSharedDecoderManager(SharedDecoderManager* aManager) +{ +#if !defined(MOZ_WIDGET_ANDROID) + mSharedDecoderManager = aManager; +#endif +} + +bool +MP4Reader::VideoIsHardwareAccelerated() const +{ + return mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(); +} + +void +MP4Reader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) +{ + MOZ_ASSERT(OnTaskQueue()); + if (mLastSeenEnd < 0) { + MonitorAutoLock mon(mDemuxerMonitor); + mLastSeenEnd = mDecoder->GetResource()->GetLength(); + if (mLastSeenEnd < 0) { + // We dont have a length. Demuxer would have been blocking already. + return; + } + } + int64_t end = aOffset + aLength; + if (end <= mLastSeenEnd) { + return; + } + mLastSeenEnd = end; + + if (HasVideo()) { + auto& decoder = GetDecoderData(TrackInfo::kVideoTrack); + MonitorAutoLock lock(decoder.mMonitor); + decoder.mDemuxEOS = false; + } + if (HasAudio()) { + auto& decoder = GetDecoderData(TrackInfo::kAudioTrack); + MonitorAutoLock lock(decoder.mMonitor); + decoder.mDemuxEOS = false; + } +} + +} // namespace mozilla diff --git a/dom/media/fmp4/MP4Reader.h b/dom/media/fmp4/MP4Reader.h new file mode 100644 index 000000000000..853a12c9e7d0 --- /dev/null +++ b/dom/media/fmp4/MP4Reader.h @@ -0,0 +1,280 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(MP4Reader_h_) +#define MP4Reader_h_ + +#include "mozilla/Monitor.h" +#include "mozilla/TaskQueue.h" + +#include "MediaDecoderReader.h" +#include "nsAutoPtr.h" +#include "PlatformDecoderModule.h" +#include "mp4_demuxer/mp4_demuxer.h" +#include "demuxer/TrackDemuxer.h" + +#include + +namespace mozilla { + +typedef std::deque> MediaSampleQueue; + +class MP4Stream; + +class MP4Reader final : public MediaDecoderReader +{ + typedef TrackInfo::TrackType TrackType; + +public: + explicit MP4Reader(AbstractMediaDecoder* aDecoder, TaskQueue* aBorrowedTaskQueue = nullptr); + + virtual ~MP4Reader(); + + virtual nsresult Init(MediaDecoderReader* aCloneDonor) override; + + virtual size_t SizeOfVideoQueueInFrames() override; + virtual size_t SizeOfAudioQueueInFrames() override; + + virtual nsRefPtr + RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead) override; + + virtual nsRefPtr RequestAudioData() override; + + virtual bool HasAudio() override; + virtual bool HasVideo() override; + + virtual nsresult ReadMetadata(MediaInfo* aInfo, + MetadataTags** aTags) override; + + virtual void ReadUpdatedMetadata(MediaInfo* aInfo) override; + + virtual nsRefPtr + Seek(int64_t aTime, int64_t aEndTime) override; + + virtual bool IsMediaSeekable() override; + + virtual int64_t GetEvictionOffset(double aTime) override; + +protected: + virtual void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) override; +public: + + virtual media::TimeIntervals GetBuffered() override; + + // For Media Resource Management + virtual void SetIdle() override; + virtual void ReleaseMediaResources() override; + virtual void SetSharedDecoderManager(SharedDecoderManager* aManager) + override; + + virtual nsresult ResetDecode() override; + + virtual nsRefPtr Shutdown() override; + + virtual bool IsAsync() const override { return true; } + + virtual bool VideoIsHardwareAccelerated() const override; + + virtual void DisableHardwareAcceleration() override; + +private: + + bool InitDemuxer(); + void ReturnOutput(MediaData* aData, TrackType aTrack); + + bool EnsureDecodersSetup(); + + // Sends input to decoder for aTrack, and output to the state machine, + // if necessary. + void Update(TrackType aTrack); + + // Enqueues a task to call Update(aTrack) on the decoder task queue. + // Lock for corresponding track must be held. + void ScheduleUpdate(TrackType aTrack); + + void ExtractCryptoInitData(nsTArray& aInitData); + + // Initializes mLayersBackendType if possible. + void InitLayersBackendType(); + + // Blocks until the demuxer produces an sample of specified type. + // Returns nullptr on error on EOS. Caller must delete sample. + already_AddRefed PopSample(TrackType aTrack); + already_AddRefed PopSampleLocked(TrackType aTrack); + + bool SkipVideoDemuxToNextKeyFrame(int64_t aTimeThreshold, uint32_t& parsed); + + // DecoderCallback proxies the MediaDataDecoderCallback calls to these + // functions. + void Output(TrackType aType, MediaData* aSample); + void InputExhausted(TrackType aTrack); + void Error(TrackType aTrack); + void Flush(TrackType aTrack); + void DrainComplete(TrackType aTrack); + void UpdateIndex(); + bool IsSupportedAudioMimeType(const nsACString& aMimeType); + bool IsSupportedVideoMimeType(const nsACString& aMimeType); + virtual bool IsWaitingOnCDMResource() override; + + mp4_demuxer::Microseconds GetNextKeyframeTime(); + bool ShouldSkip(bool aSkipToNextKeyframe, int64_t aTimeThreshold); + + size_t SizeOfQueue(TrackType aTrack); + + nsRefPtr mStream; + nsRefPtr mDemuxer; + nsRefPtr mPlatform; + mp4_demuxer::CryptoFile mCrypto; + + class DecoderCallback : public MediaDataDecoderCallback { + public: + DecoderCallback(MP4Reader* aReader, TrackType aType) + : mReader(aReader) + , mType(aType) + { + } + virtual void Output(MediaData* aSample) override { + mReader->Output(mType, aSample); + } + virtual void InputExhausted() override { + mReader->InputExhausted(mType); + } + virtual void Error() override { + mReader->Error(mType); + } + virtual void DrainComplete() override { + mReader->DrainComplete(mType); + } + virtual void ReleaseMediaResources() override { + mReader->ReleaseMediaResources(); + } + virtual bool OnReaderTaskQueue() override { + return mReader->OnTaskQueue(); + } + private: + MP4Reader* mReader; + TrackType mType; + }; + + struct DecoderData { + DecoderData(MediaData::Type aType, + uint32_t aDecodeAhead) + : mType(aType) + , mMonitor(aType == MediaData::AUDIO_DATA ? "MP4 audio decoder data" + : "MP4 video decoder data") + , mNumSamplesInput(0) + , mNumSamplesOutput(0) + , mDecodeAhead(aDecodeAhead) + , mForceDecodeAhead(false) + , mActive(false) + , mInputExhausted(false) + , mError(false) + , mIsFlushing(false) + , mUpdateScheduled(false) + , mDemuxEOS(false) + , mDrainComplete(false) + , mDiscontinuity(false) + { + } + + nsAutoPtr mTrackDemuxer; + // The platform decoder. + nsRefPtr mDecoder; + // TaskQueue on which decoder can choose to decode. + // Only non-null up until the decoder is created. + nsRefPtr mTaskQueue; + // Callback that receives output and error notifications from the decoder. + nsAutoPtr mCallback; + // Decoded samples returned my mDecoder awaiting being returned to + // state machine upon request. + nsTArray > mOutput; + // Disambiguate Audio vs Video. + MediaData::Type mType; + + // These get overriden in the templated concrete class. + virtual bool HasPromise() = 0; + virtual void RejectPromise(MediaDecoderReader::NotDecodedReason aReason, + const char* aMethodName) = 0; + + // Monitor that protects all non-threadsafe state; the primitives + // that follow. + Monitor mMonitor; + uint64_t mNumSamplesInput; + uint64_t mNumSamplesOutput; + uint32_t mDecodeAhead; + bool mForceDecodeAhead; + // Whether this stream exists in the media. + bool mActive; + bool mInputExhausted; + bool mError; + bool mIsFlushing; + bool mUpdateScheduled; + bool mDemuxEOS; + bool mDrainComplete; + bool mDiscontinuity; + }; + + template + struct DecoderDataWithPromise : public DecoderData { + DecoderDataWithPromise(MediaData::Type aType, uint32_t aDecodeAhead) : + DecoderData(aType, aDecodeAhead) + { + mPromise.SetMonitor(&mMonitor); + } + + MozPromiseHolder mPromise; + + bool HasPromise() override { return !mPromise.IsEmpty(); } + void RejectPromise(MediaDecoderReader::NotDecodedReason aReason, + const char* aMethodName) override + { + mPromise.Reject(aReason, aMethodName); + } + }; + + DecoderDataWithPromise mAudio; + DecoderDataWithPromise mVideo; + + // Queued samples extracted by the demuxer, but not yet sent to the platform + // decoder. + nsRefPtr mQueuedVideoSample; + + // Returns true when the decoder for this track needs input. + // aDecoder.mMonitor must be locked. + bool NeedInput(DecoderData& aDecoder); + + // The last number of decoded output frames that we've reported to + // MediaDecoder::NotifyDecoded(). We diff the number of output video + // frames every time that DecodeVideoData() is called, and report the + // delta there. + uint64_t mLastReportedNumDecodedFrames; + + DecoderData& GetDecoderData(TrackType aTrack); + + layers::LayersBackend mLayersBackendType; + + // For use with InvokeAndRetry as an already_refed can't be converted to bool + nsRefPtr DemuxVideoSample(); + nsRefPtr DemuxAudioSample(); + + // True if we've read the streams' metadata. + bool mDemuxerInitialized; + + // True if we've gathered telemetry from an SPS. + bool mFoundSPSForTelemetry; + + // Synchronized by decoder monitor. + bool mIsEncrypted; + + bool mIndexReady; + int64_t mLastSeenEnd; + Monitor mDemuxerMonitor; + nsRefPtr mSharedDecoderManager; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/fmp4/MP4Stream.h b/dom/media/fmp4/MP4Stream.h index 43049d0394f7..d0b2bfb825f9 100644 --- a/dom/media/fmp4/MP4Stream.h +++ b/dom/media/fmp4/MP4Stream.h @@ -7,7 +7,7 @@ #ifndef MP4_STREAM_H_ #define MP4_STREAM_H_ -#include "mp4_demuxer/Stream.h" +#include "mp4_demuxer/mp4_demuxer.h" #include "MediaResource.h" diff --git a/dom/media/fmp4/moz.build b/dom/media/fmp4/moz.build index 54166c5647c1..77ecd6e6585d 100644 --- a/dom/media/fmp4/moz.build +++ b/dom/media/fmp4/moz.build @@ -7,6 +7,7 @@ EXPORTS += [ 'MP4Decoder.h', 'MP4Demuxer.h', + 'MP4Reader.h', 'MP4Stream.h', ] @@ -17,6 +18,7 @@ UNIFIED_SOURCES += [ SOURCES += [ 'MP4Demuxer.cpp', + 'MP4Reader.cpp', ] FINAL_LIBRARY = 'xul' diff --git a/dom/media/gtest/moz.build b/dom/media/gtest/moz.build index 8cb5223d8bfc..2e8db6947801 100644 --- a/dom/media/gtest/moz.build +++ b/dom/media/gtest/moz.build @@ -12,6 +12,7 @@ UNIFIED_SOURCES += [ 'TestIntervalSet.cpp', 'TestMozPromise.cpp', 'TestMP3Demuxer.cpp', + 'TestMP4Demuxer.cpp', # 'TestMP4Reader.cpp', disabled so we can turn check tests back on (bug 1175752) 'TestTrackEncoder.cpp', 'TestVideoSegment.cpp', diff --git a/dom/media/mediasource/MediaSourceReader.cpp b/dom/media/mediasource/MediaSourceReader.cpp index 666433f78d9e..fae257e27831 100644 --- a/dom/media/mediasource/MediaSourceReader.cpp +++ b/dom/media/mediasource/MediaSourceReader.cpp @@ -20,6 +20,7 @@ #include "SharedDecoderManager.h" #include "MP4Decoder.h" #include "MP4Demuxer.h" +#include "MP4Reader.h" #endif #ifdef MOZ_WEBM @@ -700,15 +701,18 @@ CreateReaderForType(const nsACString& aType, AbstractMediaDecoder* aDecoder, TaskQueue* aBorrowedTaskQueue) { #ifdef MOZ_FMP4 - // The MediaFormatReader that supports fragmented MP4 and uses + // The MP4Reader that supports fragmented MP4 and uses // PlatformDecoderModules is hidden behind prefs for regular video // elements, but we always want to use it for MSE, so instantiate it // directly here. if ((aType.LowerCaseEqualsLiteral("video/mp4") || aType.LowerCaseEqualsLiteral("audio/mp4")) && MP4Decoder::IsEnabled() && aDecoder) { - MediaDecoderReader* reader = - new MediaFormatReader(aDecoder, new MP4Demuxer(aDecoder->GetResource()), aBorrowedTaskQueue); + bool useFormatDecoder = + Preferences::GetBool("media.mediasource.format-reader.mp4", true); + MediaDecoderReader* reader = useFormatDecoder ? + static_cast(new MediaFormatReader(aDecoder, new MP4Demuxer(aDecoder->GetResource()), aBorrowedTaskQueue)) : + static_cast(new MP4Reader(aDecoder, aBorrowedTaskQueue)); return reader; } #endif diff --git a/dom/media/platforms/apple/AppleATDecoder.cpp b/dom/media/platforms/apple/AppleATDecoder.cpp index 17eb8df1609e..062ab39ec1ba 100644 --- a/dom/media/platforms/apple/AppleATDecoder.cpp +++ b/dom/media/platforms/apple/AppleATDecoder.cpp @@ -5,6 +5,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "AppleUtils.h" +#include "MP4Reader.h" #include "MP4Decoder.h" #include "mp4_demuxer/Adts.h" #include "MediaInfo.h" diff --git a/dom/media/platforms/apple/AppleVDADecoder.h b/dom/media/platforms/apple/AppleVDADecoder.h index 4c76503eefed..d7898e899b0f 100644 --- a/dom/media/platforms/apple/AppleVDADecoder.h +++ b/dom/media/platforms/apple/AppleVDADecoder.h @@ -9,6 +9,7 @@ #include "PlatformDecoderModule.h" #include "mozilla/ReentrantMonitor.h" +#include "MP4Reader.h" #include "MP4Decoder.h" #include "nsIThread.h" #include "ReorderQueue.h" diff --git a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h index 01c7beeddc2f..55065a2f799f 100644 --- a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h @@ -10,6 +10,7 @@ #include "PlatformDecoderModule.h" #include "FFmpegLibs.h" #include "mozilla/StaticMutex.h" +#include "mp4_demuxer/mp4_demuxer.h" namespace mozilla { diff --git a/dom/media/platforms/wmf/WMFAudioMFTManager.h b/dom/media/platforms/wmf/WMFAudioMFTManager.h index 95d6aab469f3..82abf4d24b96 100644 --- a/dom/media/platforms/wmf/WMFAudioMFTManager.h +++ b/dom/media/platforms/wmf/WMFAudioMFTManager.h @@ -8,6 +8,7 @@ #define WMFAudioOutputSource_h_ #include "WMF.h" +#include "MP4Reader.h" #include "MFTDecoder.h" #include "mozilla/RefPtr.h" #include "WMFMediaDataDecoder.h" diff --git a/dom/media/platforms/wmf/WMFMediaDataDecoder.h b/dom/media/platforms/wmf/WMFMediaDataDecoder.h index 4a0b1de51ed2..746f5398b162 100644 --- a/dom/media/platforms/wmf/WMFMediaDataDecoder.h +++ b/dom/media/platforms/wmf/WMFMediaDataDecoder.h @@ -9,9 +9,9 @@ #include "WMF.h" +#include "MP4Reader.h" #include "MFTDecoder.h" #include "mozilla/RefPtr.h" -#include "PlatformDecoderModule.h" namespace mozilla { diff --git a/dom/media/platforms/wmf/WMFVideoMFTManager.h b/dom/media/platforms/wmf/WMFVideoMFTManager.h index aad1fa3c9ba2..1477467b4b33 100644 --- a/dom/media/platforms/wmf/WMFVideoMFTManager.h +++ b/dom/media/platforms/wmf/WMFVideoMFTManager.h @@ -8,6 +8,7 @@ #define WMFVideoMFTManager_h_ #include "WMF.h" +#include "MP4Reader.h" #include "MFTDecoder.h" #include "nsRect.h" #include "WMFMediaDataDecoder.h" diff --git a/dom/media/webm/IntelWebMVideoDecoder.cpp b/dom/media/webm/IntelWebMVideoDecoder.cpp index 844f7006c18b..e500c4a346d7 100644 --- a/dom/media/webm/IntelWebMVideoDecoder.cpp +++ b/dom/media/webm/IntelWebMVideoDecoder.cpp @@ -24,6 +24,8 @@ PRLogModuleInfo* GetDemuxerLog(); #define LOG(...) MOZ_LOG(GetDemuxerLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) +using namespace mp4_demuxer; + namespace mozilla { using layers::Image; diff --git a/dom/media/webm/IntelWebMVideoDecoder.h b/dom/media/webm/IntelWebMVideoDecoder.h index cb7c1466eaab..4b4f34a50e76 100644 --- a/dom/media/webm/IntelWebMVideoDecoder.h +++ b/dom/media/webm/IntelWebMVideoDecoder.h @@ -13,6 +13,7 @@ #include "PlatformDecoderModule.h" #include "mozilla/Monitor.h" +#include "mp4_demuxer/mp4_demuxer.h" #include "MediaInfo.h" #include "MediaData.h" diff --git a/media/libstagefright/binding/Box.cpp b/media/libstagefright/binding/Box.cpp index 4b78d825966c..e73fc0a6cdad 100644 --- a/media/libstagefright/binding/Box.cpp +++ b/media/libstagefright/binding/Box.cpp @@ -5,7 +5,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mp4_demuxer/Box.h" -#include "mp4_demuxer/Stream.h" +#include "mp4_demuxer/mp4_demuxer.h" #include "mozilla/Endian.h" #include diff --git a/media/libstagefright/binding/MP4TrackDemuxer.cpp b/media/libstagefright/binding/MP4TrackDemuxer.cpp new file mode 100644 index 000000000000..f2258bcf5dba --- /dev/null +++ b/media/libstagefright/binding/MP4TrackDemuxer.cpp @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mp4_demuxer/MP4TrackDemuxer.h" + +using namespace mozilla; + +namespace mp4_demuxer { + +void +MP4AudioDemuxer::Seek(Microseconds aTime) +{ + mDemuxer->SeekAudio(aTime); +} + +already_AddRefed +MP4AudioDemuxer::DemuxSample() +{ + nsRefPtr sample(mDemuxer->DemuxAudioSample()); + return sample.forget(); +} + +Microseconds +MP4AudioDemuxer::GetNextKeyframeTime() +{ + return -1; +} + +void +MP4VideoDemuxer::Seek(Microseconds aTime) +{ + mDemuxer->SeekVideo(aTime); +} + +already_AddRefed +MP4VideoDemuxer::DemuxSample() +{ + nsRefPtr sample(mDemuxer->DemuxVideoSample()); + return sample.forget(); +} + +Microseconds +MP4VideoDemuxer::GetNextKeyframeTime() +{ + return mDemuxer->GetNextKeyframeTime(); +} + +} diff --git a/media/libstagefright/binding/include/mp4_demuxer/MP4TrackDemuxer.h b/media/libstagefright/binding/include/mp4_demuxer/MP4TrackDemuxer.h new file mode 100644 index 000000000000..d9ebb6055480 --- /dev/null +++ b/media/libstagefright/binding/include/mp4_demuxer/MP4TrackDemuxer.h @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MP4_TRACK_DEMUXER_H_ +#define MP4_TRACK_DEMUXER_H_ + +#include "mozilla/Attributes.h" +#include "demuxer/TrackDemuxer.h" + +namespace mp4_demuxer +{ + +class MP4AudioDemuxer : public mozilla::TrackDemuxer { +public: + explicit MP4AudioDemuxer(MP4Demuxer* aDemuxer) : mDemuxer(aDemuxer) {} + virtual void Seek(Microseconds aTime) override; + virtual already_AddRefed DemuxSample() override; + virtual Microseconds GetNextKeyframeTime() override; + +private: + nsRefPtr mDemuxer; +}; + +class MP4VideoDemuxer : public mozilla::TrackDemuxer { +public: + explicit MP4VideoDemuxer(MP4Demuxer* aDemuxer) : mDemuxer(aDemuxer) {} + virtual void Seek(Microseconds aTime) override; + virtual already_AddRefed DemuxSample() override; + virtual Microseconds GetNextKeyframeTime() override; + +private: + nsRefPtr mDemuxer; +}; + +} + +#endif diff --git a/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h b/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h index 465c1f6b7424..32f698fa3187 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h +++ b/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h @@ -5,16 +5,13 @@ #ifndef MOOF_PARSER_H_ #define MOOF_PARSER_H_ -#include "mozilla/Monitor.h" #include "mp4_demuxer/Atom.h" #include "mp4_demuxer/AtomType.h" +#include "mp4_demuxer/mp4_demuxer.h" #include "mp4_demuxer/SinfParser.h" -#include "mp4_demuxer/Interval.h" #include "MediaResource.h" namespace mp4_demuxer { -using mozilla::Monitor; -typedef int64_t Microseconds; class Stream; class Box; diff --git a/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h b/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h new file mode 100644 index 000000000000..91c1facf0ee1 --- /dev/null +++ b/media/libstagefright/binding/include/mp4_demuxer/mp4_demuxer.h @@ -0,0 +1,84 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MP4_DEMUXER_H_ +#define MP4_DEMUXER_H_ + +#include "MediaInfo.h" +#include "MediaResource.h" +#include "mozilla/Monitor.h" +#include "mozilla/UniquePtr.h" +#include "mp4_demuxer/DecoderData.h" +#include "mp4_demuxer/Interval.h" +#include "mp4_demuxer/Stream.h" +#include "nsISupportsImpl.h" +#include "nsTArray.h" + +namespace mp4_demuxer +{ +class Index; +class MP4Metadata; +class SampleIterator; +using mozilla::Monitor; +typedef int64_t Microseconds; + +class MP4Demuxer +{ +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MP4Demuxer) + + explicit MP4Demuxer(Stream* aSource, Monitor* aMonitor); + + bool Init(); + Microseconds Duration(); + bool CanSeek(); + + bool HasValidAudio(); + bool HasValidVideo(); + + void SeekAudio(Microseconds aTime); + void SeekVideo(Microseconds aTime); + + // DemuxAudioSample and DemuxVideoSample functions + // return nullptr on end of stream or error. + already_AddRefed DemuxAudioSample(); + already_AddRefed DemuxVideoSample(); + + const CryptoFile& Crypto() const; + const mozilla::AudioInfo& AudioConfig() const { return *mAudioConfig->GetAsAudioInfo(); } + const mozilla::VideoInfo& VideoConfig() const { return *mVideoConfig->GetAsVideoInfo(); } + + void UpdateIndex(const nsTArray& aByteRanges); + + void ConvertByteRangesToTime( + const nsTArray& aByteRanges, + nsTArray>* aIntervals); + + int64_t GetEvictionOffset(Microseconds aTime); + + // Returns timestamp of next keyframe, or -1 if demuxer can't + // report this. + Microseconds GetNextKeyframeTime(); + +protected: + ~MP4Demuxer(); + +private: + mozilla::UniquePtr mAudioConfig; + mozilla::UniquePtr mVideoConfig; + + nsRefPtr mSource; + nsTArray mCachedByteRanges; + nsTArray> mCachedTimeRanges; + Monitor* mMonitor; + Microseconds mNextKeyframeTime; + mozilla::UniquePtr mMetadata; + mozilla::UniquePtr mAudioIterator; + mozilla::UniquePtr mVideoIterator; + nsTArray> mIndexes; +}; + +} // namespace mp4_demuxer + +#endif // MP4_DEMUXER_H_ diff --git a/media/libstagefright/binding/mp4_demuxer.cpp b/media/libstagefright/binding/mp4_demuxer.cpp new file mode 100644 index 000000000000..b9ecd1323659 --- /dev/null +++ b/media/libstagefright/binding/mp4_demuxer.cpp @@ -0,0 +1,245 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mp4_demuxer/Index.h" +#include "mp4_demuxer/MP4Metadata.h" +#include "mp4_demuxer/mp4_demuxer.h" + +#include +#include +#include + +namespace mp4_demuxer +{ + +MP4Demuxer::MP4Demuxer(Stream* source, Monitor* aMonitor) + : mSource(source) + , mMonitor(aMonitor) + , mNextKeyframeTime(-1) +{ +} + +MP4Demuxer::~MP4Demuxer() +{ +} + +bool +MP4Demuxer::Init() +{ + mMonitor->AssertCurrentThreadOwns(); + + // Check that we have an entire moov before attempting any new reads to make + // the retry system work. + if (!MP4Metadata::HasCompleteMetadata(mSource)) { + return false; + } + + mMetadata = mozilla::MakeUnique(mSource); + + if (!mMetadata->GetNumberTracks(mozilla::TrackInfo::kAudioTrack) && + !mMetadata->GetNumberTracks(mozilla::TrackInfo::kVideoTrack)) { + return false; + } + + auto audioInfo = mMetadata->GetTrackInfo(mozilla::TrackInfo::kAudioTrack, 0); + if (audioInfo) { + mAudioConfig = mozilla::Move(audioInfo); + FallibleTArray indices; + if (!mMetadata->ReadTrackIndex(indices, mAudioConfig->mTrackId)) { + return false; + } + nsRefPtr index = + new Index(indices, + mSource, + mAudioConfig->mTrackId, + /* aIsAudio = */ true, + mMonitor); + mIndexes.AppendElement(index); + mAudioIterator = mozilla::MakeUnique(index); + } + + auto videoInfo = mMetadata->GetTrackInfo(mozilla::TrackInfo::kVideoTrack, 0); + if (videoInfo) { + mVideoConfig = mozilla::Move(videoInfo); + FallibleTArray indices; + if (!mMetadata->ReadTrackIndex(indices, mVideoConfig->mTrackId)) { + return false; + } + + nsRefPtr index = + new Index(indices, + mSource, + mVideoConfig->mTrackId, + /* aIsAudio = */ false, + mMonitor); + mIndexes.AppendElement(index); + mVideoIterator = mozilla::MakeUnique(index); + } + + return mAudioIterator || mVideoIterator; +} + +bool +MP4Demuxer::HasValidAudio() +{ + mMonitor->AssertCurrentThreadOwns(); + return mAudioIterator && mAudioConfig && mAudioConfig->IsValid(); +} + +bool +MP4Demuxer::HasValidVideo() +{ + mMonitor->AssertCurrentThreadOwns(); + return mVideoIterator && mVideoConfig && mVideoConfig->IsValid(); +} + +Microseconds +MP4Demuxer::Duration() +{ + mMonitor->AssertCurrentThreadOwns(); + int64_t videoDuration = mVideoConfig ? mVideoConfig->mDuration : 0; + int64_t audioDuration = mAudioConfig ? mAudioConfig->mDuration : 0; + return std::max(videoDuration, audioDuration); +} + +bool +MP4Demuxer::CanSeek() +{ + mMonitor->AssertCurrentThreadOwns(); + return mMetadata->CanSeek(); +} + +void +MP4Demuxer::SeekAudio(Microseconds aTime) +{ + mMonitor->AssertCurrentThreadOwns(); + MOZ_ASSERT(mAudioIterator); + mAudioIterator->Seek(aTime); +} + +void +MP4Demuxer::SeekVideo(Microseconds aTime) +{ + mMonitor->AssertCurrentThreadOwns(); + MOZ_ASSERT(mVideoIterator); + mVideoIterator->Seek(aTime); +} + +already_AddRefed +MP4Demuxer::DemuxAudioSample() +{ + mMonitor->AssertCurrentThreadOwns(); + MOZ_ASSERT(mAudioIterator); + nsRefPtr sample(mAudioIterator->GetNext()); + if (sample) { + if (sample->mCrypto.mValid) { + nsAutoPtr writer(sample->CreateWriter()); + writer->mCrypto.mMode = mAudioConfig->mCrypto.mMode; + writer->mCrypto.mIVSize = mAudioConfig->mCrypto.mIVSize; + writer->mCrypto.mKeyId.AppendElements(mAudioConfig->mCrypto.mKeyId); + } + } + return sample.forget(); +} + +already_AddRefed +MP4Demuxer::DemuxVideoSample() +{ + mMonitor->AssertCurrentThreadOwns(); + MOZ_ASSERT(mVideoIterator); + nsRefPtr sample(mVideoIterator->GetNext()); + if (sample) { + sample->mExtraData = mVideoConfig->GetAsVideoInfo()->mExtraData; + if (sample->mCrypto.mValid) { + nsAutoPtr writer(sample->CreateWriter()); + writer->mCrypto.mMode = mVideoConfig->mCrypto.mMode; + writer->mCrypto.mKeyId.AppendElements(mVideoConfig->mCrypto.mKeyId); + } + if (sample->mTime >= mNextKeyframeTime) { + mNextKeyframeTime = mVideoIterator->GetNextKeyframeTime(); + } + } + return sample.forget(); +} + +void +MP4Demuxer::UpdateIndex(const nsTArray& aByteRanges) +{ + mMonitor->AssertCurrentThreadOwns(); + for (int i = 0; i < mIndexes.Length(); i++) { + mIndexes[i]->UpdateMoofIndex(aByteRanges); + } +} + +void +MP4Demuxer::ConvertByteRangesToTime( + const nsTArray& aByteRanges, + nsTArray>* aIntervals) +{ + mMonitor->AssertCurrentThreadOwns(); + if (mIndexes.IsEmpty()) { + return; + } + + Microseconds lastComposition = 0; + nsTArray endCompositions; + for (int i = 0; i < mIndexes.Length(); i++) { + Microseconds endComposition = + mIndexes[i]->GetEndCompositionIfBuffered(aByteRanges); + endCompositions.AppendElement(endComposition); + lastComposition = std::max(lastComposition, endComposition); + } + + if (aByteRanges != mCachedByteRanges) { + mCachedByteRanges = aByteRanges; + mCachedTimeRanges.Clear(); + for (int i = 0; i < mIndexes.Length(); i++) { + nsTArray> ranges; + mIndexes[i]->ConvertByteRangesToTimeRanges(aByteRanges, &ranges); + if (lastComposition && endCompositions[i]) { + Interval::SemiNormalAppend( + ranges, Interval(endCompositions[i], lastComposition)); + } + + if (i) { + nsTArray> intersection; + Interval::Intersection(mCachedTimeRanges, ranges, &intersection); + mCachedTimeRanges = intersection; + } else { + mCachedTimeRanges = ranges; + } + } + } + aIntervals->AppendElements(mCachedTimeRanges); +} + +int64_t +MP4Demuxer::GetEvictionOffset(Microseconds aTime) +{ + mMonitor->AssertCurrentThreadOwns(); + if (mIndexes.IsEmpty()) { + return 0; + } + + uint64_t offset = std::numeric_limits::max(); + for (int i = 0; i < mIndexes.Length(); i++) { + offset = std::min(offset, mIndexes[i]->GetEvictionOffset(aTime)); + } + return offset == std::numeric_limits::max() ? 0 : offset; +} + +Microseconds +MP4Demuxer::GetNextKeyframeTime() +{ + mMonitor->AssertCurrentThreadOwns(); + return mNextKeyframeTime; +} + +const CryptoFile& +MP4Demuxer::Crypto() const +{ + return mMetadata->Crypto(); +} + +} // namespace mp4_demuxer diff --git a/media/libstagefright/moz.build b/media/libstagefright/moz.build index 1b6dc0a96dbc..b6fa60f58663 100644 --- a/media/libstagefright/moz.build +++ b/media/libstagefright/moz.build @@ -60,7 +60,9 @@ EXPORTS.mp4_demuxer += [ 'binding/include/mp4_demuxer/Index.h', 'binding/include/mp4_demuxer/Interval.h', 'binding/include/mp4_demuxer/MoofParser.h', + 'binding/include/mp4_demuxer/mp4_demuxer.h', 'binding/include/mp4_demuxer/MP4Metadata.h', + 'binding/include/mp4_demuxer/MP4TrackDemuxer.h', 'binding/include/mp4_demuxer/ResourceStream.h', 'binding/include/mp4_demuxer/SinfParser.h', 'binding/include/mp4_demuxer/Stream.h', @@ -93,7 +95,9 @@ UNIFIED_SOURCES += [ 'binding/H264.cpp', 'binding/Index.cpp', 'binding/MoofParser.cpp', + 'binding/mp4_demuxer.cpp', 'binding/MP4Metadata.cpp', + 'binding/MP4TrackDemuxer.cpp', 'binding/ResourceStream.cpp', 'binding/SinfParser.cpp', 'frameworks/av/media/libstagefright/DataSource.cpp', diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index b00357970b1e..d5d35ab1b0ac 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -474,6 +474,11 @@ pref("media.mediasource.webm.enabled", false); // Enable new MediaSource architecture. pref("media.mediasource.format-reader", false); +// Enable new MediaFormatReader architecture for mp4 in MSE +pref("media.mediasource.format-reader.mp4", true); +// Enable new MediaFormatReader architecture for plain mp4. +pref("media.format-reader.mp4", true); + // Enable new MediaFormatReader architecture for webm in MSE pref("media.mediasource.format-reader.webm", false); // Enable new MediaFormatReader architecture for plain webm.