diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index 1a41d0bc05ee..f9a03cbea61d 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -273,6 +273,56 @@ public: } }; +/** + * This listener observes the first video frame to arrive with a non-empty size, + * and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size. + */ +class HTMLMediaElement::StreamSizeListener : public MediaStreamTrackDirectListener { +public: + explicit StreamSizeListener(HTMLMediaElement* aElement) : + mElement(aElement), + mInitialSizeFound(false) + {} + void Forget() { mElement = nullptr; } + + void ReceivedSize(gfx::IntSize aSize) + { + if (!mElement) { + return; + } + RefPtr deathGrip = mElement; + mElement->UpdateInitialMediaSize(aSize); + } + + void NotifyRealtimeTrackData(MediaStreamGraph* aGraph, + StreamTime aTrackOffset, + const MediaSegment& aMedia) override + { + if (mInitialSizeFound || aMedia.GetType() != MediaSegment::VIDEO) { + return; + } + const VideoSegment& video = static_cast(aMedia); + for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) { + if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) { + mInitialSizeFound = true; + nsCOMPtr event = + NewRunnableMethod( + this, &StreamSizeListener::ReceivedSize, + c->mFrame.GetIntrinsicSize()); + aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget()); + return; + } + } + } + +private: + // These fields may only be accessed on the main thread + HTMLMediaElement* mElement; + + // These fields may only be accessed on the MSG thread + bool mInitialSizeFound; +}; + /** * There is a reference cycle involving this class: MediaLoadListener * holds a reference to the HTMLMediaElement, which holds a reference @@ -484,6 +534,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTM #ifdef MOZ_EME NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys) #endif + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLElement) @@ -510,6 +561,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLE #ifdef MOZ_EME NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys) #endif + NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack) NS_IMPL_CYCLE_COLLECTION_UNLINK_END NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(HTMLMediaElement) @@ -724,6 +776,14 @@ void HTMLMediaElement::AbortExistingLoads() bool fireTimeUpdate = false; + // We need to remove StreamSizeListener before VideoTracks get emptied. + if (mMediaStreamSizeListener) { + mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); + mSelectedVideoStreamTrack = nullptr; + mMediaStreamSizeListener->Forget(); + mMediaStreamSizeListener = nullptr; + } + // When aborting the existing loads, empty the objects in audio track list and // video track list, no events (in particular, no removetrack events) are // fired as part of this. Ending MediaStream sends track ended notifications, @@ -3216,59 +3276,6 @@ private: bool mPendingNotifyOutput; }; -/** - * This listener observes the first video frame to arrive with a non-empty size, - * and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size. - */ -class HTMLMediaElement::StreamSizeListener : public MediaStreamListener { -public: - explicit StreamSizeListener(HTMLMediaElement* aElement) : - mElement(aElement), - mInitialSizeFound(false) - {} - void Forget() { mElement = nullptr; } - - void ReceivedSize(gfx::IntSize aSize) - { - if (!mElement) { - return; - } - RefPtr deathGrip = mElement; - mElement->UpdateInitialMediaSize(aSize); - } - - void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, - StreamTime aTrackOffset, - uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia, - MediaStream* aInputStream, - TrackID aInputTrackID) override - { - if (mInitialSizeFound || aQueuedMedia.GetType() != MediaSegment::VIDEO) { - return; - } - const VideoSegment& video = static_cast(aQueuedMedia); - for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) { - if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) { - mInitialSizeFound = true; - nsCOMPtr event = - NewRunnableMethod( - this, &StreamSizeListener::ReceivedSize, - c->mFrame.GetIntrinsicSize()); - aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget()); - return; - } - } - } - -private: - // These fields may only be accessed on the main thread - HTMLMediaElement* mElement; - - // These fields may only be accessed on the MSG thread - bool mInitialSizeFound; -}; - class HTMLMediaElement::MediaStreamTracksAvailableCallback: public OnTracksAvailableCallback { @@ -3387,9 +3394,6 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) RefPtr stream = GetSrcMediaStream(); if (stream) { stream->SetAudioChannelType(mAudioChannel); - - mMediaStreamSizeListener = new StreamSizeListener(this); - stream->AddListener(mMediaStreamSizeListener); } UpdateSrcMediaStreamPlaying(); @@ -3420,10 +3424,8 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM); if (mMediaStreamSizeListener) { - RefPtr stream = GetSrcMediaStream(); - if (stream) { - stream->RemoveListener(mMediaStreamSizeListener); - } + mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); + mSelectedVideoStreamTrack = nullptr; mMediaStreamSizeListener->Forget(); mMediaStreamSizeListener = nullptr; } @@ -3459,7 +3461,8 @@ CreateVideoTrack(VideoStreamTrack* aStreamTrack) aStreamTrack->GetLabel(label); return MediaTrackList::CreateVideoTrack(id, NS_LITERAL_STRING("main"), - label, EmptyString()); + label, EmptyString(), + aStreamTrack); } void HTMLMediaElement::ConstructMediaTracks() @@ -3491,6 +3494,11 @@ void HTMLMediaElement::ConstructMediaTracks() // must be selected. int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0; (*VideoTracks())[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS); + VideoTrack* track = (*VideoTracks())[index]; + VideoStreamTrack* streamTrack = track->GetVideoStreamTrack(); + mMediaStreamSizeListener = new StreamSizeListener(this); + streamTrack->AddDirectListener(mMediaStreamSizeListener); + mSelectedVideoStreamTrack = streamTrack; } } @@ -3511,8 +3519,20 @@ HTMLMediaElement::NotifyMediaStreamTrackAdded(const RefPtr& aT RefPtr audioTrack = CreateAudioTrack(t); AudioTracks()->AddTrack(audioTrack); } else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) { + // TODO: Fix this per the spec on bug 1273443. + int32_t selectedIndex = VideoTracks()->SelectedIndex(); RefPtr videoTrack = CreateVideoTrack(t); VideoTracks()->AddTrack(videoTrack); + // New MediaStreamTrack added, set the new added video track as selected + // video track when there is no selected track. + if (selectedIndex == -1) { + MOZ_ASSERT(!mSelectedVideoStreamTrack); + videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS); + mMediaStreamSizeListener = new StreamSizeListener(this); + t->AddDirectListener(mMediaStreamSizeListener); + mSelectedVideoStreamTrack = t; + } + } } @@ -3531,6 +3551,47 @@ HTMLMediaElement::NotifyMediaStreamTrackRemoved(const RefPtr& AudioTracks()->RemoveTrack(t); } else if (MediaTrack* t = VideoTracks()->GetTrackById(id)) { VideoTracks()->RemoveTrack(t); + // TODO: Fix this per the spec on bug 1273443. + // If the removed media stream track is selected video track and there are + // still video tracks, change the selected video track to the first + // remaining track. + if (aTrack == mSelectedVideoStreamTrack) { + // The mMediaStreamSizeListener might already reset to nullptr. + if (mMediaStreamSizeListener) { + mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); + } + mSelectedVideoStreamTrack = nullptr; + MOZ_ASSERT(mSrcStream); + nsTArray> tracks; + mSrcStream->GetVideoTracks(tracks); + + for (const RefPtr& track : tracks) { + if (track->Ended() || !track->Enabled()) { + continue; + } + + if (track->Enabled()) { + nsAutoString trackId; + track->GetId(trackId); + MediaTrack* videoTrack = VideoTracks()->GetTrackById(trackId); + MOZ_ASSERT(videoTrack); + + videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS); + if (mMediaStreamSizeListener) { + track->AddDirectListener(mMediaStreamSizeListener); + } + mSelectedVideoStreamTrack = track; + return; + } + } + + // There is no enabled video track existing, clean the + // mMediaStreamSizeListener. + if (mMediaStreamSizeListener) { + mMediaStreamSizeListener->Forget(); + mMediaStreamSizeListener = nullptr; + } + } } else { // XXX (bug 1208328) Uncomment this when DOMMediaStream doesn't call // NotifyTrackRemoved multiple times for the same track, i.e., when it @@ -4445,10 +4506,7 @@ void HTMLMediaElement::UpdateInitialMediaSize(const nsIntSize& aSize) if (!mMediaStreamSizeListener) { return; } - RefPtr stream = GetSrcMediaStream(); - if (stream) { - stream->RemoveListener(mMediaStreamSizeListener); - } + mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener); mMediaStreamSizeListener->Forget(); mMediaStreamSizeListener = nullptr; } diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index b6eb9107677a..528696a25c9a 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -1239,6 +1239,8 @@ protected: // Holds a reference to the size-getting MediaStreamListener attached to // mSrcStream. RefPtr mMediaStreamSizeListener; + // The selected video stream track which contained mMediaStreamSizeListener. + RefPtr mSelectedVideoStreamTrack; // Holds a reference to the MediaSource, if any, referenced by the src // attribute on the media element. diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp index 0c79de0bd232..f375d35adde8 100644 --- a/dom/media/MediaStreamGraph.cpp +++ b/dom/media/MediaStreamGraph.cpp @@ -2927,15 +2927,20 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed listener = aListener; STREAM_LOG(LogLevel::Debug, ("Adding direct track listener %p bound to track %d to source stream %p", listener.get(), aTrackID, this)); + { MutexAutoLock lock(mMutex); data = FindDataForTrack(aTrackID); found = !!data; - isAudio = found && data->mData->GetType() == MediaSegment::AUDIO; - if (found && isAudio) { + if (found) { + isAudio = data->mData->GetType() == MediaSegment::AUDIO; + isVideo = data->mData->GetType() == MediaSegment::VIDEO; + } + if (found && (isAudio || isVideo)) { TrackBound* sourceListener = mDirectTrackListeners.AppendElement(); sourceListener->mListener = listener; @@ -2949,11 +2954,11 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefedNotifyDirectListenerInstalled( - MediaStreamTrackDirectListener::InstallationResult::TRACK_TYPE_NOT_SUPPORTED); + // It is not a video or audio track. + MOZ_ASSERT(true); return; } STREAM_LOG(LogLevel::Debug, ("Added direct track listener %p", listener.get())); diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h index 238257d7d80c..ad7213051b3b 100644 --- a/dom/media/MediaStreamGraph.h +++ b/dom/media/MediaStreamGraph.h @@ -344,7 +344,8 @@ public: * didn't exist. This should only happen if you try to install the listener * directly to a SourceMediaStream that doesn't contain the given TrackID. * TRACK_TYPE_NOT_SUPPORTED - * This is the failure when you install the listener to a non-audio track. + * This is the failure when you install the listener to a + * non-(audio or video) track. * STREAM_NOT_SUPPORTED * While looking for the data source of this track, we found a MediaStream * that is not a SourceMediaStream or a TrackUnionStream. @@ -354,7 +355,6 @@ public: */ enum class InstallationResult { TRACK_NOT_FOUND_AT_SOURCE, - TRACK_TYPE_NOT_SUPPORTED, STREAM_NOT_SUPPORTED, SUCCESS }; @@ -370,6 +370,15 @@ protected: aTo.AppendNullData(aFrom.GetDuration()); } + void MirrorAndDisableSegment(VideoSegment& aFrom, VideoSegment& aTo) + { + aTo.Clear(); + for (VideoSegment::ChunkIterator it(aFrom); !it.IsEnded(); it.Next()) { + aTo.AppendFrame(do_AddRef(it->mFrame.GetImage()), it->GetDuration(), + it->mFrame.GetIntrinsicSize(), it->GetPrincipalHandle(), true); + } + } + void NotifyRealtimeTrackDataAndApplyTrackDisabling(MediaStreamGraph* aGraph, StreamTime aTrackOffset, MediaSegment& aMedia) @@ -385,6 +394,9 @@ protected: if (aMedia.GetType() == MediaSegment::AUDIO) { MirrorAndDisableSegment(static_cast(aMedia), static_cast(*mMedia)); + } else if (aMedia.GetType() == MediaSegment::VIDEO) { + MirrorAndDisableSegment(static_cast(aMedia), + static_cast(*mMedia)); } else { MOZ_CRASH("Unsupported media type"); } diff --git a/dom/media/MediaTrackList.cpp b/dom/media/MediaTrackList.cpp index b9d8ab67a49a..348032987e73 100644 --- a/dom/media/MediaTrackList.cpp +++ b/dom/media/MediaTrackList.cpp @@ -9,6 +9,7 @@ #include "mozilla/AsyncEventDispatcher.h" #include "mozilla/dom/HTMLMediaElement.h" #include "mozilla/dom/AudioTrack.h" +#include "mozilla/dom/VideoStreamTrack.h" #include "mozilla/dom/VideoTrack.h" #include "mozilla/dom/TrackEvent.h" #include "nsThreadUtils.h" @@ -106,9 +107,10 @@ already_AddRefed MediaTrackList::CreateVideoTrack(const nsAString& aId, const nsAString& aKind, const nsAString& aLabel, - const nsAString& aLanguage) + const nsAString& aLanguage, + VideoStreamTrack* aVideoTrack) { - RefPtr track = new VideoTrack(aId, aKind, aLabel, aLanguage); + RefPtr track = new VideoTrack(aId, aKind, aLabel, aLanguage, aVideoTrack); return track.forget(); } diff --git a/dom/media/MediaTrackList.h b/dom/media/MediaTrackList.h index 7b165219e470..a78f6c1fe277 100644 --- a/dom/media/MediaTrackList.h +++ b/dom/media/MediaTrackList.h @@ -20,6 +20,7 @@ class AudioTrackList; class VideoTrackList; class AudioTrack; class VideoTrack; +class VideoStreamTrack; /** * Base class of AudioTrackList and VideoTrackList. The AudioTrackList and @@ -58,11 +59,14 @@ public: const nsAString& aLanguage, bool aEnabled); + // For the case of src of HTMLMediaElement is non-MediaStream, leave the + // aVideoTrack as default(nullptr). static already_AddRefed CreateVideoTrack(const nsAString& aId, const nsAString& aKind, const nsAString& aLabel, - const nsAString& aLanguage); + const nsAString& aLanguage, + VideoStreamTrack* aVideoTrack = nullptr); virtual void EmptyTracks(); diff --git a/dom/media/VideoTrack.cpp b/dom/media/VideoTrack.cpp index 80f36ffb845b..45784abff296 100644 --- a/dom/media/VideoTrack.cpp +++ b/dom/media/VideoTrack.cpp @@ -5,6 +5,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/dom/HTMLMediaElement.h" +#include "mozilla/dom/VideoStreamTrack.h" #include "mozilla/dom/VideoTrack.h" #include "mozilla/dom/VideoTrackBinding.h" #include "mozilla/dom/VideoTrackList.h" @@ -15,12 +16,25 @@ namespace dom { VideoTrack::VideoTrack(const nsAString& aId, const nsAString& aKind, const nsAString& aLabel, - const nsAString& aLanguage) + const nsAString& aLanguage, + VideoStreamTrack* aStreamTarck) : MediaTrack(aId, aKind, aLabel, aLanguage) , mSelected(false) + , mVideoStreamTrack(aStreamTarck) { } +VideoTrack::~VideoTrack() +{ +} + +NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoTrack, MediaTrack, mVideoStreamTrack) + +NS_IMPL_ADDREF_INHERITED(VideoTrack, MediaTrack) +NS_IMPL_RELEASE_INHERITED(VideoTrack, MediaTrack) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(VideoTrack) +NS_INTERFACE_MAP_END_INHERITING(MediaTrack) + JSObject* VideoTrack::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { diff --git a/dom/media/VideoTrack.h b/dom/media/VideoTrack.h index 94f5b362f1f5..8675f72849b0 100644 --- a/dom/media/VideoTrack.h +++ b/dom/media/VideoTrack.h @@ -13,6 +13,7 @@ namespace mozilla { namespace dom { class VideoTrackList; +class VideoStreamTrack; class VideoTrack : public MediaTrack { @@ -20,7 +21,11 @@ public: VideoTrack(const nsAString& aId, const nsAString& aKind, const nsAString& aLabel, - const nsAString& aLanguage); + const nsAString& aLanguage, + VideoStreamTrack* aStreamTarck = nullptr); + + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoTrack, MediaTrack) JSObject* WrapObject(JSContext* aCx, JS::Handle aGivenProto) override; @@ -36,6 +41,11 @@ public: // aFlags contains FIRE_NO_EVENTS because no events are fired in such cases. void SetEnabledInternal(bool aEnabled, int aFlags) override; + // Get associated video stream track when the video track comes from + // MediaStream. This might be nullptr when the src of owning HTMLMediaElement + // is not MediaStream. + VideoStreamTrack* GetVideoStreamTrack() { return mVideoStreamTrack; } + // WebIDL bool Selected() const { @@ -48,7 +58,10 @@ public: void SetSelected(bool aSelected); private: + virtual ~VideoTrack(); + bool mSelected; + RefPtr mVideoStreamTrack; }; } // namespace dom diff --git a/dom/media/VideoTrackList.cpp b/dom/media/VideoTrackList.cpp index 547a392c4c26..3205076a82ed 100644 --- a/dom/media/VideoTrackList.cpp +++ b/dom/media/VideoTrackList.cpp @@ -31,18 +31,18 @@ VideoTrackList::RemoveTrack(const RefPtr& aTrack) // need to be done after RemoveTrack. Also the call of // |MediaTrackList::RemoveTrack| is necessary even when mSelectedIndex = -1. bool found; - VideoTrack* videoTrack = IndexedGetter(mSelectedIndex, found); + VideoTrack* selectedVideoTrack = IndexedGetter(mSelectedIndex, found); MediaTrackList::RemoveTrack(aTrack); if (mSelectedIndex == -1) { // There was no selected track and we don't select another track on removal. return; } MOZ_ASSERT(found, "When mSelectedIndex is set it should point to a track"); - MOZ_ASSERT(videoTrack, "The mSelectedIndex should be set to video track only"); + MOZ_ASSERT(selectedVideoTrack, "The mSelectedIndex should be set to video track only"); // Let the caller of RemoveTrack deal with choosing the new selected track if // it removes the currently-selected track. - if (aTrack == videoTrack) { + if (aTrack == selectedVideoTrack) { mSelectedIndex = -1; return; } @@ -51,7 +51,7 @@ VideoTrackList::RemoveTrack(const RefPtr& aTrack) // currently-selected video track. We need to find the new location of the // selected track. for (size_t ix = 0; ix < mTracks.Length(); ix++) { - if (mTracks[ix] == videoTrack) { + if (mTracks[ix] == selectedVideoTrack) { mSelectedIndex = ix; return; }