From 0d1a10cc38ad8138b7f30349bdbf23b6894fede3 Mon Sep 17 00:00:00 2001 From: Andreas Pehrson Date: Fri, 10 Oct 2014 11:34:57 +0200 Subject: [PATCH] Bug 879717: Part 1: Delay HAVE_CURRENT_DATA state until a video frame has been stored in the image container. r=roc --- .../html/content/public/HTMLMediaElement.h | 11 ++ content/html/content/src/HTMLMediaElement.cpp | 129 ++++++++++++++---- 2 files changed, 114 insertions(+), 26 deletions(-) diff --git a/content/html/content/public/HTMLMediaElement.h b/content/html/content/public/HTMLMediaElement.h index 5086bbf3ebe7..9914bd9410c2 100755 --- a/content/html/content/public/HTMLMediaElement.h +++ b/content/html/content/public/HTMLMediaElement.h @@ -285,6 +285,13 @@ public: void NotifyMediaTrackEnabled(MediaTrack* aTrack); + /** + * Called by a DOMMediaStream when it has tracks available. + * This allows us to setup audio and video outputs after the stream + * has already reported that playback started, in case they are added late. + */ + void NotifyMediaStreamTracksAvailable(DOMMediaStream* aStream); + virtual bool IsNodeOfType(uint32_t aFlags) const MOZ_OVERRIDE; /** @@ -616,6 +623,7 @@ protected: class MediaLoadListener; class StreamListener; + class MediaStreamTracksAvailableCallback; virtual void GetItemValueText(nsAString& text) MOZ_OVERRIDE; virtual void SetItemValueText(const nsAString& text) MOZ_OVERRIDE; @@ -1018,6 +1026,9 @@ protected: nsMediaNetworkState mNetworkState; nsMediaReadyState mReadyState; + // Last value passed from codec or stream source to UpdateReadyStateForData. + NextFrameStatus mLastNextFrameStatus; + enum LoadAlgorithmState { // No load algorithm instance is waiting for a source to be added to the // media in order to continue loading. diff --git a/content/html/content/src/HTMLMediaElement.cpp b/content/html/content/src/HTMLMediaElement.cpp index 1d125bfd3d8f..92113feb98a0 100755 --- a/content/html/content/src/HTMLMediaElement.cpp +++ b/content/html/content/src/HTMLMediaElement.cpp @@ -71,6 +71,8 @@ #include "mozilla/dom/MediaSource.h" #include "MediaMetadataManager.h" #include "MediaSourceDecoder.h" +#include "AudioStreamTrack.h" +#include "VideoStreamTrack.h" #include "AudioChannelService.h" @@ -664,7 +666,10 @@ void HTMLMediaElement::AbortExistingLoads() mHaveQueuedSelectResource = false; mSuspendedForPreloadNone = false; mDownloadSuspendedByCache = false; + mHasAudio = false; + mHasVideo = false; mSourcePointer = nullptr; + mLastNextFrameStatus = NEXT_FRAME_UNINITIALIZED; mTags = nullptr; @@ -898,6 +903,39 @@ void HTMLMediaElement::NotifyMediaTrackEnabled(MediaTrack* aTrack) } } +void HTMLMediaElement::NotifyMediaStreamTracksAvailable(DOMMediaStream* aStream) +{ + if (!mSrcStream || mSrcStream != aStream) { + return; + } + + bool oldHasAudio = mHasAudio; + bool oldHasVideo = mHasVideo; + + nsAutoTArray,1> audioTracks; + aStream->GetAudioTracks(audioTracks); + nsAutoTArray,1> videoTracks; + aStream->GetVideoTracks(videoTracks); + + mHasAudio = !audioTracks.IsEmpty(); + mHasVideo = !videoTracks.IsEmpty(); + + if (!oldHasAudio && mHasAudio) { + GetSrcMediaStream()->AddAudioOutput(this); + GetSrcMediaStream()->SetAudioOutputVolume(this, float(mMuted ? 0.0 : mVolume)); + } + if (!oldHasVideo && mHasVideo ) { + VideoFrameContainer* container = GetVideoFrameContainer(); + if (container) { + GetSrcMediaStream()->AddVideoOutput(container); + } + // mHasVideo changed so make sure the screen wakelock is updated + NotifyOwnerDocumentActivityChanged(); + } + + CheckAutoplayDataReady(); +} + void HTMLMediaElement::LoadFromSourceChildren() { NS_ASSERTION(mDelayingLoadEvent, @@ -1987,6 +2025,7 @@ HTMLMediaElement::HTMLMediaElement(already_AddRefed& aNo mCurrentLoadID(0), mNetworkState(nsIDOMHTMLMediaElement::NETWORK_EMPTY), mReadyState(nsIDOMHTMLMediaElement::HAVE_NOTHING), + mLastNextFrameStatus(NEXT_FRAME_UNINITIALIZED), mLoadWaitStatus(NOT_WAITING), mVolume(1.0), mPreloadAction(PRELOAD_UNDEFINED), @@ -2807,6 +2846,28 @@ private: bool mPendingNotifyOutput; }; +class HTMLMediaElement::MediaStreamTracksAvailableCallback: + public DOMMediaStream::OnTracksAvailableCallback +{ +public: + explicit MediaStreamTracksAvailableCallback(HTMLMediaElement* aElement, + DOMMediaStream::TrackTypeHints aExpectedTracks = 0): + DOMMediaStream::OnTracksAvailableCallback(aExpectedTracks), + mElement(aElement) + {} + virtual void NotifyTracksAvailable(DOMMediaStream* aStream) + { + NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); + + if (!mElement) { + return; + } + mElement->NotifyMediaStreamTracksAvailable(aStream); + } +private: + HTMLMediaElement* mElement; +}; + void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) { NS_ASSERTION(!mSrcStream && !mSrcStreamListener, "Should have been ended already"); @@ -2828,22 +2889,24 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) if (mPausedForInactiveDocumentOrChannel) { GetSrcMediaStream()->ChangeExplicitBlockerCount(1); } + + mSrcStream->OnTracksAvailable(new MediaStreamTracksAvailableCallback(this, DOMMediaStream::HINT_CONTENTS_AUDIO)); + mSrcStream->OnTracksAvailable(new MediaStreamTracksAvailableCallback(this, DOMMediaStream::HINT_CONTENTS_VIDEO)); + + MediaInfo mediaInfo; + mediaInfo.mAudio.mHasAudio = mHasAudio; + mediaInfo.mVideo.mHasVideo = mHasVideo; + MetadataLoaded(&mediaInfo, nullptr); + + DispatchAsyncEvent(NS_LITERAL_STRING("suspend")); + mNetworkState = nsIDOMHTMLMediaElement::NETWORK_IDLE; + ChangeDelayLoadStatus(false); - GetSrcMediaStream()->AddAudioOutput(this); - GetSrcMediaStream()->SetAudioOutputVolume(this, float(mMuted ? 0.0 : mVolume)); - VideoFrameContainer* container = GetVideoFrameContainer(); - if (container) { - GetSrcMediaStream()->AddVideoOutput(container); - } // Note: we must call DisconnectTrackListListeners(...) before dropping // mSrcStream mSrcStream->ConstructMediaTracks(AudioTracks(), VideoTracks()); - ChangeReadyState(nsIDOMHTMLMediaElement::HAVE_METADATA); - DispatchAsyncEvent(NS_LITERAL_STRING("durationchange")); - DispatchAsyncEvent(NS_LITERAL_STRING("loadedmetadata")); - ChangeNetworkState(nsIDOMHTMLMediaElement::NETWORK_IDLE); AddRemoveSelfReference(); // FirstFrameLoaded() will be called when the stream has current data. } @@ -2859,12 +2922,12 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() // Kill its reference to this element mSrcStreamListener->Forget(); mSrcStreamListener = nullptr; - if (stream) { + if (stream && mHasAudio) { stream->RemoveAudioOutput(this); } VideoFrameContainer* container = GetVideoFrameContainer(); if (container) { - if (stream) { + if (stream && mHasVideo) { stream->RemoveVideoOutput(container); } container->ClearCurrentFrame(); @@ -2916,6 +2979,11 @@ void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo, mVideoFrameContainer->ForgetElement(); mVideoFrameContainer = nullptr; } + + if (IsVideo()) { + // Update the screen wakelock in case mHasVideo changed + NotifyOwnerDocumentActivityChanged(); + } } void HTMLMediaElement::FirstFrameLoaded() @@ -2923,6 +2991,7 @@ void HTMLMediaElement::FirstFrameLoaded() NS_ASSERTION(!mSuspendedAfterFirstFrame, "Should not have already suspended"); ChangeDelayLoadStatus(false); + UpdateReadyStateForData(NEXT_FRAME_UNAVAILABLE); if (mDecoder && mAllowSuspendAfterFirstFrame && mPaused && !HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay) && @@ -3085,10 +3154,18 @@ bool HTMLMediaElement::ShouldCheckAllowOrigin() void HTMLMediaElement::UpdateReadyStateForData(MediaDecoderOwner::NextFrameStatus aNextFrame) { + mLastNextFrameStatus = aNextFrame; + if (mReadyState < nsIDOMHTMLMediaElement::HAVE_METADATA) { // aNextFrame might have a next frame because the decoder can advance // on its own thread before MetadataLoaded gets a chance to run. // The arrival of more data can't change us out of this readyState. + + return; + } + + if (!mHasAudio && !mHasVideo) { + // No tracks available yet, don't advance from HAVE_METADATA return; } @@ -3111,6 +3188,14 @@ void HTMLMediaElement::UpdateReadyStateForData(MediaDecoderOwner::NextFrameStatu return; } + if (mReadyState < nsIDOMHTMLMediaElement::HAVE_CURRENT_DATA && mHasVideo) { + VideoFrameContainer* container = GetVideoFrameContainer(); + if (container && mMediaSize == nsIntSize(-1,-1)) { + // No frame has been set yet. Don't advance. + return; + } + } + if (aNextFrame != MediaDecoderOwner::NEXT_FRAME_AVAILABLE) { ChangeReadyState(nsIDOMHTMLMediaElement::HAVE_CURRENT_DATA); if (!mWaitingFired && aNextFrame == MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING) { @@ -3251,14 +3336,15 @@ void HTMLMediaElement::ChangeNetworkState(nsMediaNetworkState aState) bool HTMLMediaElement::CanActivateAutoplay() { - // For stream inputs, we activate autoplay on HAVE_CURRENT_DATA because + // For stream inputs, we activate autoplay on HAVE_METADATA because // this element itself might be blocking the stream from making progress by // being paused. return !mPausedForInactiveDocumentOrChannel && mAutoplaying && mPaused && ((mDecoder && mReadyState >= nsIDOMHTMLMediaElement::HAVE_ENOUGH_DATA) || - (mSrcStream && mReadyState >= nsIDOMHTMLMediaElement::HAVE_CURRENT_DATA)) && + (mSrcStream && mReadyState >= nsIDOMHTMLMediaElement::HAVE_METADATA)) && + (mHasAudio || mHasVideo) && HasAttr(kNameSpaceID_None, nsGkAtoms::autoplay) && mAutoplayEnabled && !IsEditable(); @@ -3287,24 +3373,14 @@ void HTMLMediaElement::CheckAutoplayDataReady() VideoFrameContainer* HTMLMediaElement::GetVideoFrameContainer() { - // If we have loaded the metadata, and the size of the video is still - // (-1, -1), the media has no video. Don't go a create a video frame - // container. - if (mReadyState >= nsIDOMHTMLMediaElement::HAVE_METADATA && - mMediaSize == nsIntSize(-1, -1)) { - return nullptr; - } + if (mVideoFrameContainer) + return mVideoFrameContainer; // Only video frames need an image container. if (!IsVideo()) { return nullptr; } - mHasVideo = true; - - if (mVideoFrameContainer) - return mVideoFrameContainer; - mVideoFrameContainer = new VideoFrameContainer(this, LayerManager::CreateAsynchronousImageContainer()); @@ -3417,6 +3493,7 @@ void HTMLMediaElement::NotifyDecoderPrincipalChanged() void HTMLMediaElement::UpdateMediaSize(nsIntSize size) { mMediaSize = size; + UpdateReadyStateForData(mLastNextFrameStatus); } void HTMLMediaElement::SuspendOrResumeElement(bool aPauseElement, bool aSuspendEvents)