diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp
index 1a41d0bc05ee..f9a03cbea61d 100644
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -273,6 +273,56 @@ public:
}
};
+/**
+ * This listener observes the first video frame to arrive with a non-empty size,
+ * and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
+ */
+class HTMLMediaElement::StreamSizeListener : public MediaStreamTrackDirectListener {
+public:
+ explicit StreamSizeListener(HTMLMediaElement* aElement) :
+ mElement(aElement),
+ mInitialSizeFound(false)
+ {}
+ void Forget() { mElement = nullptr; }
+
+ void ReceivedSize(gfx::IntSize aSize)
+ {
+ if (!mElement) {
+ return;
+ }
+ RefPtr deathGrip = mElement;
+ mElement->UpdateInitialMediaSize(aSize);
+ }
+
+ void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+ StreamTime aTrackOffset,
+ const MediaSegment& aMedia) override
+ {
+ if (mInitialSizeFound || aMedia.GetType() != MediaSegment::VIDEO) {
+ return;
+ }
+ const VideoSegment& video = static_cast(aMedia);
+ for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
+ if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) {
+ mInitialSizeFound = true;
+ nsCOMPtr event =
+ NewRunnableMethod(
+ this, &StreamSizeListener::ReceivedSize,
+ c->mFrame.GetIntrinsicSize());
+ aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
+ return;
+ }
+ }
+ }
+
+private:
+ // These fields may only be accessed on the main thread
+ HTMLMediaElement* mElement;
+
+ // These fields may only be accessed on the MSG thread
+ bool mInitialSizeFound;
+};
+
/**
* There is a reference cycle involving this class: MediaLoadListener
* holds a reference to the HTMLMediaElement, which holds a reference
@@ -484,6 +534,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTM
#ifdef MOZ_EME
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys)
#endif
+ NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLElement)
@@ -510,6 +561,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLE
#ifdef MOZ_EME
NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys)
#endif
+ NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack)
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(HTMLMediaElement)
@@ -724,6 +776,14 @@ void HTMLMediaElement::AbortExistingLoads()
bool fireTimeUpdate = false;
+ // We need to remove StreamSizeListener before VideoTracks get emptied.
+ if (mMediaStreamSizeListener) {
+ mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
+ mSelectedVideoStreamTrack = nullptr;
+ mMediaStreamSizeListener->Forget();
+ mMediaStreamSizeListener = nullptr;
+ }
+
// When aborting the existing loads, empty the objects in audio track list and
// video track list, no events (in particular, no removetrack events) are
// fired as part of this. Ending MediaStream sends track ended notifications,
@@ -3216,59 +3276,6 @@ private:
bool mPendingNotifyOutput;
};
-/**
- * This listener observes the first video frame to arrive with a non-empty size,
- * and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
- */
-class HTMLMediaElement::StreamSizeListener : public MediaStreamListener {
-public:
- explicit StreamSizeListener(HTMLMediaElement* aElement) :
- mElement(aElement),
- mInitialSizeFound(false)
- {}
- void Forget() { mElement = nullptr; }
-
- void ReceivedSize(gfx::IntSize aSize)
- {
- if (!mElement) {
- return;
- }
- RefPtr deathGrip = mElement;
- mElement->UpdateInitialMediaSize(aSize);
- }
-
- void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
- StreamTime aTrackOffset,
- uint32_t aTrackEvents,
- const MediaSegment& aQueuedMedia,
- MediaStream* aInputStream,
- TrackID aInputTrackID) override
- {
- if (mInitialSizeFound || aQueuedMedia.GetType() != MediaSegment::VIDEO) {
- return;
- }
- const VideoSegment& video = static_cast(aQueuedMedia);
- for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
- if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) {
- mInitialSizeFound = true;
- nsCOMPtr event =
- NewRunnableMethod(
- this, &StreamSizeListener::ReceivedSize,
- c->mFrame.GetIntrinsicSize());
- aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
- return;
- }
- }
- }
-
-private:
- // These fields may only be accessed on the main thread
- HTMLMediaElement* mElement;
-
- // These fields may only be accessed on the MSG thread
- bool mInitialSizeFound;
-};
-
class HTMLMediaElement::MediaStreamTracksAvailableCallback:
public OnTracksAvailableCallback
{
@@ -3387,9 +3394,6 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream)
RefPtr stream = GetSrcMediaStream();
if (stream) {
stream->SetAudioChannelType(mAudioChannel);
-
- mMediaStreamSizeListener = new StreamSizeListener(this);
- stream->AddListener(mMediaStreamSizeListener);
}
UpdateSrcMediaStreamPlaying();
@@ -3420,10 +3424,8 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback()
UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
if (mMediaStreamSizeListener) {
- RefPtr stream = GetSrcMediaStream();
- if (stream) {
- stream->RemoveListener(mMediaStreamSizeListener);
- }
+ mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
+ mSelectedVideoStreamTrack = nullptr;
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}
@@ -3459,7 +3461,8 @@ CreateVideoTrack(VideoStreamTrack* aStreamTrack)
aStreamTrack->GetLabel(label);
return MediaTrackList::CreateVideoTrack(id, NS_LITERAL_STRING("main"),
- label, EmptyString());
+ label, EmptyString(),
+ aStreamTrack);
}
void HTMLMediaElement::ConstructMediaTracks()
@@ -3491,6 +3494,11 @@ void HTMLMediaElement::ConstructMediaTracks()
// must be selected.
int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0;
(*VideoTracks())[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
+ VideoTrack* track = (*VideoTracks())[index];
+ VideoStreamTrack* streamTrack = track->GetVideoStreamTrack();
+ mMediaStreamSizeListener = new StreamSizeListener(this);
+ streamTrack->AddDirectListener(mMediaStreamSizeListener);
+ mSelectedVideoStreamTrack = streamTrack;
}
}
@@ -3511,8 +3519,20 @@ HTMLMediaElement::NotifyMediaStreamTrackAdded(const RefPtr& aT
RefPtr audioTrack = CreateAudioTrack(t);
AudioTracks()->AddTrack(audioTrack);
} else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) {
+ // TODO: Fix this per the spec on bug 1273443.
+ int32_t selectedIndex = VideoTracks()->SelectedIndex();
RefPtr videoTrack = CreateVideoTrack(t);
VideoTracks()->AddTrack(videoTrack);
+ // New MediaStreamTrack added, set the new added video track as selected
+ // video track when there is no selected track.
+ if (selectedIndex == -1) {
+ MOZ_ASSERT(!mSelectedVideoStreamTrack);
+ videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
+ mMediaStreamSizeListener = new StreamSizeListener(this);
+ t->AddDirectListener(mMediaStreamSizeListener);
+ mSelectedVideoStreamTrack = t;
+ }
+
}
}
@@ -3531,6 +3551,47 @@ HTMLMediaElement::NotifyMediaStreamTrackRemoved(const RefPtr&
AudioTracks()->RemoveTrack(t);
} else if (MediaTrack* t = VideoTracks()->GetTrackById(id)) {
VideoTracks()->RemoveTrack(t);
+ // TODO: Fix this per the spec on bug 1273443.
+ // If the removed media stream track is selected video track and there are
+ // still video tracks, change the selected video track to the first
+ // remaining track.
+ if (aTrack == mSelectedVideoStreamTrack) {
+ // The mMediaStreamSizeListener might already reset to nullptr.
+ if (mMediaStreamSizeListener) {
+ mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
+ }
+ mSelectedVideoStreamTrack = nullptr;
+ MOZ_ASSERT(mSrcStream);
+ nsTArray> tracks;
+ mSrcStream->GetVideoTracks(tracks);
+
+ for (const RefPtr& track : tracks) {
+ if (track->Ended() || !track->Enabled()) {
+ continue;
+ }
+
+ if (track->Enabled()) {
+ nsAutoString trackId;
+ track->GetId(trackId);
+ MediaTrack* videoTrack = VideoTracks()->GetTrackById(trackId);
+ MOZ_ASSERT(videoTrack);
+
+ videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
+ if (mMediaStreamSizeListener) {
+ track->AddDirectListener(mMediaStreamSizeListener);
+ }
+ mSelectedVideoStreamTrack = track;
+ return;
+ }
+ }
+
+ // There is no enabled video track existing, clean the
+ // mMediaStreamSizeListener.
+ if (mMediaStreamSizeListener) {
+ mMediaStreamSizeListener->Forget();
+ mMediaStreamSizeListener = nullptr;
+ }
+ }
} else {
// XXX (bug 1208328) Uncomment this when DOMMediaStream doesn't call
// NotifyTrackRemoved multiple times for the same track, i.e., when it
@@ -4445,10 +4506,7 @@ void HTMLMediaElement::UpdateInitialMediaSize(const nsIntSize& aSize)
if (!mMediaStreamSizeListener) {
return;
}
- RefPtr stream = GetSrcMediaStream();
- if (stream) {
- stream->RemoveListener(mMediaStreamSizeListener);
- }
+ mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}
diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h
index b6eb9107677a..528696a25c9a 100644
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@@ -1239,6 +1239,8 @@ protected:
// Holds a reference to the size-getting MediaStreamListener attached to
// mSrcStream.
RefPtr mMediaStreamSizeListener;
+ // The selected video stream track which contained mMediaStreamSizeListener.
+ RefPtr mSelectedVideoStreamTrack;
// Holds a reference to the MediaSource, if any, referenced by the src
// attribute on the media element.
diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp
index 0c79de0bd232..f375d35adde8 100644
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -2927,15 +2927,20 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed listener = aListener;
STREAM_LOG(LogLevel::Debug, ("Adding direct track listener %p bound to track %d to source stream %p",
listener.get(), aTrackID, this));
+
{
MutexAutoLock lock(mMutex);
data = FindDataForTrack(aTrackID);
found = !!data;
- isAudio = found && data->mData->GetType() == MediaSegment::AUDIO;
- if (found && isAudio) {
+ if (found) {
+ isAudio = data->mData->GetType() == MediaSegment::AUDIO;
+ isVideo = data->mData->GetType() == MediaSegment::VIDEO;
+ }
+ if (found && (isAudio || isVideo)) {
TrackBound* sourceListener =
mDirectTrackListeners.AppendElement();
sourceListener->mListener = listener;
@@ -2949,11 +2954,11 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefedNotifyDirectListenerInstalled(
- MediaStreamTrackDirectListener::InstallationResult::TRACK_TYPE_NOT_SUPPORTED);
+ // It is not a video or audio track.
+ MOZ_ASSERT(true);
return;
}
STREAM_LOG(LogLevel::Debug, ("Added direct track listener %p", listener.get()));
diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h
index 238257d7d80c..ad7213051b3b 100644
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -344,7 +344,8 @@ public:
* didn't exist. This should only happen if you try to install the listener
* directly to a SourceMediaStream that doesn't contain the given TrackID.
* TRACK_TYPE_NOT_SUPPORTED
- * This is the failure when you install the listener to a non-audio track.
+ * This is the failure when you install the listener to a
+ * non-(audio or video) track.
* STREAM_NOT_SUPPORTED
* While looking for the data source of this track, we found a MediaStream
* that is not a SourceMediaStream or a TrackUnionStream.
@@ -354,7 +355,6 @@ public:
*/
enum class InstallationResult {
TRACK_NOT_FOUND_AT_SOURCE,
- TRACK_TYPE_NOT_SUPPORTED,
STREAM_NOT_SUPPORTED,
SUCCESS
};
@@ -370,6 +370,15 @@ protected:
aTo.AppendNullData(aFrom.GetDuration());
}
+ void MirrorAndDisableSegment(VideoSegment& aFrom, VideoSegment& aTo)
+ {
+ aTo.Clear();
+ for (VideoSegment::ChunkIterator it(aFrom); !it.IsEnded(); it.Next()) {
+ aTo.AppendFrame(do_AddRef(it->mFrame.GetImage()), it->GetDuration(),
+ it->mFrame.GetIntrinsicSize(), it->GetPrincipalHandle(), true);
+ }
+ }
+
void NotifyRealtimeTrackDataAndApplyTrackDisabling(MediaStreamGraph* aGraph,
StreamTime aTrackOffset,
MediaSegment& aMedia)
@@ -385,6 +394,9 @@ protected:
if (aMedia.GetType() == MediaSegment::AUDIO) {
MirrorAndDisableSegment(static_cast(aMedia),
static_cast(*mMedia));
+ } else if (aMedia.GetType() == MediaSegment::VIDEO) {
+ MirrorAndDisableSegment(static_cast(aMedia),
+ static_cast(*mMedia));
} else {
MOZ_CRASH("Unsupported media type");
}
diff --git a/dom/media/MediaTrackList.cpp b/dom/media/MediaTrackList.cpp
index b9d8ab67a49a..348032987e73 100644
--- a/dom/media/MediaTrackList.cpp
+++ b/dom/media/MediaTrackList.cpp
@@ -9,6 +9,7 @@
#include "mozilla/AsyncEventDispatcher.h"
#include "mozilla/dom/HTMLMediaElement.h"
#include "mozilla/dom/AudioTrack.h"
+#include "mozilla/dom/VideoStreamTrack.h"
#include "mozilla/dom/VideoTrack.h"
#include "mozilla/dom/TrackEvent.h"
#include "nsThreadUtils.h"
@@ -106,9 +107,10 @@ already_AddRefed
MediaTrackList::CreateVideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
- const nsAString& aLanguage)
+ const nsAString& aLanguage,
+ VideoStreamTrack* aVideoTrack)
{
- RefPtr track = new VideoTrack(aId, aKind, aLabel, aLanguage);
+ RefPtr track = new VideoTrack(aId, aKind, aLabel, aLanguage, aVideoTrack);
return track.forget();
}
diff --git a/dom/media/MediaTrackList.h b/dom/media/MediaTrackList.h
index 7b165219e470..a78f6c1fe277 100644
--- a/dom/media/MediaTrackList.h
+++ b/dom/media/MediaTrackList.h
@@ -20,6 +20,7 @@ class AudioTrackList;
class VideoTrackList;
class AudioTrack;
class VideoTrack;
+class VideoStreamTrack;
/**
* Base class of AudioTrackList and VideoTrackList. The AudioTrackList and
@@ -58,11 +59,14 @@ public:
const nsAString& aLanguage,
bool aEnabled);
+ // For the case of src of HTMLMediaElement is non-MediaStream, leave the
+ // aVideoTrack as default(nullptr).
static already_AddRefed
CreateVideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
- const nsAString& aLanguage);
+ const nsAString& aLanguage,
+ VideoStreamTrack* aVideoTrack = nullptr);
virtual void EmptyTracks();
diff --git a/dom/media/VideoTrack.cpp b/dom/media/VideoTrack.cpp
index 80f36ffb845b..45784abff296 100644
--- a/dom/media/VideoTrack.cpp
+++ b/dom/media/VideoTrack.cpp
@@ -5,6 +5,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "mozilla/dom/HTMLMediaElement.h"
+#include "mozilla/dom/VideoStreamTrack.h"
#include "mozilla/dom/VideoTrack.h"
#include "mozilla/dom/VideoTrackBinding.h"
#include "mozilla/dom/VideoTrackList.h"
@@ -15,12 +16,25 @@ namespace dom {
VideoTrack::VideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
- const nsAString& aLanguage)
+ const nsAString& aLanguage,
+ VideoStreamTrack* aStreamTarck)
: MediaTrack(aId, aKind, aLabel, aLanguage)
, mSelected(false)
+ , mVideoStreamTrack(aStreamTarck)
{
}
+VideoTrack::~VideoTrack()
+{
+}
+
+NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoTrack, MediaTrack, mVideoStreamTrack)
+
+NS_IMPL_ADDREF_INHERITED(VideoTrack, MediaTrack)
+NS_IMPL_RELEASE_INHERITED(VideoTrack, MediaTrack)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(VideoTrack)
+NS_INTERFACE_MAP_END_INHERITING(MediaTrack)
+
JSObject*
VideoTrack::WrapObject(JSContext* aCx, JS::Handle aGivenProto)
{
diff --git a/dom/media/VideoTrack.h b/dom/media/VideoTrack.h
index 94f5b362f1f5..8675f72849b0 100644
--- a/dom/media/VideoTrack.h
+++ b/dom/media/VideoTrack.h
@@ -13,6 +13,7 @@ namespace mozilla {
namespace dom {
class VideoTrackList;
+class VideoStreamTrack;
class VideoTrack : public MediaTrack
{
@@ -20,7 +21,11 @@ public:
VideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
- const nsAString& aLanguage);
+ const nsAString& aLanguage,
+ VideoStreamTrack* aStreamTarck = nullptr);
+
+ NS_DECL_ISUPPORTS_INHERITED
+ NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoTrack, MediaTrack)
JSObject* WrapObject(JSContext* aCx, JS::Handle aGivenProto) override;
@@ -36,6 +41,11 @@ public:
// aFlags contains FIRE_NO_EVENTS because no events are fired in such cases.
void SetEnabledInternal(bool aEnabled, int aFlags) override;
+ // Get associated video stream track when the video track comes from
+ // MediaStream. This might be nullptr when the src of owning HTMLMediaElement
+ // is not MediaStream.
+ VideoStreamTrack* GetVideoStreamTrack() { return mVideoStreamTrack; }
+
// WebIDL
bool Selected() const
{
@@ -48,7 +58,10 @@ public:
void SetSelected(bool aSelected);
private:
+ virtual ~VideoTrack();
+
bool mSelected;
+ RefPtr mVideoStreamTrack;
};
} // namespace dom
diff --git a/dom/media/VideoTrackList.cpp b/dom/media/VideoTrackList.cpp
index 547a392c4c26..3205076a82ed 100644
--- a/dom/media/VideoTrackList.cpp
+++ b/dom/media/VideoTrackList.cpp
@@ -31,18 +31,18 @@ VideoTrackList::RemoveTrack(const RefPtr& aTrack)
// need to be done after RemoveTrack. Also the call of
// |MediaTrackList::RemoveTrack| is necessary even when mSelectedIndex = -1.
bool found;
- VideoTrack* videoTrack = IndexedGetter(mSelectedIndex, found);
+ VideoTrack* selectedVideoTrack = IndexedGetter(mSelectedIndex, found);
MediaTrackList::RemoveTrack(aTrack);
if (mSelectedIndex == -1) {
// There was no selected track and we don't select another track on removal.
return;
}
MOZ_ASSERT(found, "When mSelectedIndex is set it should point to a track");
- MOZ_ASSERT(videoTrack, "The mSelectedIndex should be set to video track only");
+ MOZ_ASSERT(selectedVideoTrack, "The mSelectedIndex should be set to video track only");
// Let the caller of RemoveTrack deal with choosing the new selected track if
// it removes the currently-selected track.
- if (aTrack == videoTrack) {
+ if (aTrack == selectedVideoTrack) {
mSelectedIndex = -1;
return;
}
@@ -51,7 +51,7 @@ VideoTrackList::RemoveTrack(const RefPtr& aTrack)
// currently-selected video track. We need to find the new location of the
// selected track.
for (size_t ix = 0; ix < mTracks.Length(); ix++) {
- if (mTracks[ix] == videoTrack) {
+ if (mTracks[ix] == selectedVideoTrack) {
mSelectedIndex = ix;
return;
}