Bug 1266646 - Change HTMLMediaElement::StreamSizeListerner to inherit MediaStreamTrackDirectListener. r=pehrsons

MozReview-Commit-ID: HnNv9BnlbDy

--HG--
extra : transplant_source : i0%FA%08%16%A6/%96%13%C5%F22%5E%EC%8C%2B%C6%B1%F3_
This commit is contained in:
ctai 2016-05-10 17:02:15 +08:00
Родитель fd36b32d94
Коммит ff32ed223d
11 изменённых файлов: 209 добавлений и 85 удалений

Просмотреть файл

@ -274,6 +274,56 @@ public:
}
};
/**
* This listener observes the first video frame to arrive with a non-empty size,
* and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
*/
class HTMLMediaElement::StreamSizeListener : public DirectMediaStreamTrackListener {
public:
explicit StreamSizeListener(HTMLMediaElement* aElement) :
mElement(aElement),
mInitialSizeFound(false)
{}
void Forget() { mElement = nullptr; }
void ReceivedSize(gfx::IntSize aSize)
{
if (!mElement) {
return;
}
RefPtr<HTMLMediaElement> deathGrip = mElement;
mElement->UpdateInitialMediaSize(aSize);
}
void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
StreamTime aTrackOffset,
const MediaSegment& aMedia) override
{
if (mInitialSizeFound || aMedia.GetType() != MediaSegment::VIDEO) {
return;
}
const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) {
mInitialSizeFound = true;
nsCOMPtr<nsIRunnable> event =
NewRunnableMethod<gfx::IntSize>(
this, &StreamSizeListener::ReceivedSize,
c->mFrame.GetIntrinsicSize());
aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
return;
}
}
}
private:
// These fields may only be accessed on the main thread
HTMLMediaElement* mElement;
// These fields may only be accessed on the MSG thread
bool mInitialSizeFound;
};
/**
* There is a reference cycle involving this class: MediaLoadListener
* holds a reference to the HTMLMediaElement, which holds a reference
@ -657,6 +707,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTM
#ifdef MOZ_EME
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys)
#endif
NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack)
NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLElement)
@ -683,6 +734,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLE
#ifdef MOZ_EME
NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys)
#endif
NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack)
NS_IMPL_CYCLE_COLLECTION_UNLINK_END
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(HTMLMediaElement)
@ -881,6 +933,14 @@ void HTMLMediaElement::AbortExistingLoads()
bool fireTimeUpdate = false;
// We need to remove StreamSizeListener before VideoTracks get emptied.
if (mMediaStreamSizeListener) {
mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
mSelectedVideoStreamTrack = nullptr;
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}
// When aborting the existing loads, empty the objects in audio track list and
// video track list, no events (in particular, no removetrack events) are
// fired as part of this. Ending MediaStream sends track ended notifications,
@ -3373,59 +3433,6 @@ private:
bool mPendingNotifyOutput;
};
/**
* This listener observes the first video frame to arrive with a non-empty size,
* and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
*/
class HTMLMediaElement::StreamSizeListener : public MediaStreamListener {
public:
explicit StreamSizeListener(HTMLMediaElement* aElement) :
mElement(aElement),
mInitialSizeFound(false)
{}
void Forget() { mElement = nullptr; }
void ReceivedSize(gfx::IntSize aSize)
{
if (!mElement) {
return;
}
RefPtr<HTMLMediaElement> deathGrip = mElement;
mElement->UpdateInitialMediaSize(aSize);
}
void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
StreamTime aTrackOffset,
TrackEventCommand aTrackEvents,
const MediaSegment& aQueuedMedia,
MediaStream* aInputStream,
TrackID aInputTrackID) override
{
if (mInitialSizeFound || aQueuedMedia.GetType() != MediaSegment::VIDEO) {
return;
}
const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia);
for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) {
mInitialSizeFound = true;
nsCOMPtr<nsIRunnable> event =
NewRunnableMethod<gfx::IntSize>(
this, &StreamSizeListener::ReceivedSize,
c->mFrame.GetIntrinsicSize());
aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
return;
}
}
}
private:
// These fields may only be accessed on the main thread
HTMLMediaElement* mElement;
// These fields may only be accessed on the MSG thread
bool mInitialSizeFound;
};
class HTMLMediaElement::MediaStreamTracksAvailableCallback:
public OnTracksAvailableCallback
{
@ -3544,9 +3551,6 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream)
RefPtr<MediaStream> stream = GetSrcMediaStream();
if (stream) {
stream->SetAudioChannelType(mAudioChannel);
mMediaStreamSizeListener = new StreamSizeListener(this);
stream->AddListener(mMediaStreamSizeListener);
}
UpdateSrcMediaStreamPlaying();
@ -3577,10 +3581,8 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback()
UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
if (mMediaStreamSizeListener) {
RefPtr<MediaStream> stream = GetSrcMediaStream();
if (stream) {
stream->RemoveListener(mMediaStreamSizeListener);
}
mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
mSelectedVideoStreamTrack = nullptr;
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}
@ -3616,7 +3618,8 @@ CreateVideoTrack(VideoStreamTrack* aStreamTrack)
aStreamTrack->GetLabel(label);
return MediaTrackList::CreateVideoTrack(id, NS_LITERAL_STRING("main"),
label, EmptyString());
label, EmptyString(),
aStreamTrack);
}
void HTMLMediaElement::ConstructMediaTracks()
@ -3648,6 +3651,11 @@ void HTMLMediaElement::ConstructMediaTracks()
// must be selected.
int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0;
(*VideoTracks())[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
VideoTrack* track = (*VideoTracks())[index];
VideoStreamTrack* streamTrack = track->GetVideoStreamTrack();
mMediaStreamSizeListener = new StreamSizeListener(this);
streamTrack->AddDirectListener(mMediaStreamSizeListener);
mSelectedVideoStreamTrack = streamTrack;
}
}
@ -3668,8 +3676,20 @@ HTMLMediaElement::NotifyMediaStreamTrackAdded(const RefPtr<MediaStreamTrack>& aT
RefPtr<AudioTrack> audioTrack = CreateAudioTrack(t);
AudioTracks()->AddTrack(audioTrack);
} else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) {
// TODO: Fix this per the spec on bug 1273443.
int32_t selectedIndex = VideoTracks()->SelectedIndex();
RefPtr<VideoTrack> videoTrack = CreateVideoTrack(t);
VideoTracks()->AddTrack(videoTrack);
// New MediaStreamTrack added, set the new added video track as selected
// video track when there is no selected track.
if (selectedIndex == -1) {
MOZ_ASSERT(!mSelectedVideoStreamTrack);
videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
mMediaStreamSizeListener = new StreamSizeListener(this);
t->AddDirectListener(mMediaStreamSizeListener);
mSelectedVideoStreamTrack = t;
}
}
}
@ -3688,6 +3708,48 @@ HTMLMediaElement::NotifyMediaStreamTrackRemoved(const RefPtr<MediaStreamTrack>&
AudioTracks()->RemoveTrack(t);
} else if (MediaTrack* t = VideoTracks()->GetTrackById(id)) {
VideoTracks()->RemoveTrack(t);
// TODO: Fix this per the spec on bug 1273443.
// If the removed media stream track is selected video track and there are
// still video tracks, change the selected video track to the first
// remaining track.
if (aTrack == mSelectedVideoStreamTrack) {
// The mMediaStreamSizeListener might already reset to nullptr.
if (mMediaStreamSizeListener) {
mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
}
mSelectedVideoStreamTrack = nullptr;
MOZ_ASSERT(mSrcStream);
nsTArray<RefPtr<VideoStreamTrack>> tracks;
mSrcStream->GetVideoTracks(tracks);
for (const RefPtr<VideoStreamTrack>& track : tracks) {
if (track->Ended()) {
continue;
}
if (!track->Enabled()) {
continue;
}
nsAutoString trackId;
track->GetId(trackId);
MediaTrack* videoTrack = VideoTracks()->GetTrackById(trackId);
MOZ_ASSERT(videoTrack);
videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
if (mMediaStreamSizeListener) {
track->AddDirectListener(mMediaStreamSizeListener);
}
mSelectedVideoStreamTrack = track;
return;
}
// There is no enabled video track existing, clean the
// mMediaStreamSizeListener.
if (mMediaStreamSizeListener) {
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}
}
} else {
// XXX (bug 1208328) Uncomment this when DOMMediaStream doesn't call
// NotifyTrackRemoved multiple times for the same track, i.e., when it
@ -4606,10 +4668,7 @@ void HTMLMediaElement::UpdateInitialMediaSize(const nsIntSize& aSize)
if (!mMediaStreamSizeListener) {
return;
}
RefPtr<MediaStream> stream = GetSrcMediaStream();
if (stream) {
stream->RemoveListener(mMediaStreamSizeListener);
}
mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
mMediaStreamSizeListener->Forget();
mMediaStreamSizeListener = nullptr;
}

Просмотреть файл

@ -1244,6 +1244,8 @@ protected:
// Holds a reference to the size-getting MediaStreamListener attached to
// mSrcStream.
RefPtr<StreamSizeListener> mMediaStreamSizeListener;
// The selected video stream track which contained mMediaStreamSizeListener.
RefPtr<VideoStreamTrack> mSelectedVideoStreamTrack;
const RefPtr<ShutdownObserver> mShutdownObserver;

Просмотреть файл

@ -2944,15 +2944,20 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStream
TrackData* data;
bool found;
bool isAudio;
bool isVideo;
RefPtr<DirectMediaStreamTrackListener> listener = aListener;
STREAM_LOG(LogLevel::Debug, ("Adding direct track listener %p bound to track %d to source stream %p",
listener.get(), aTrackID, this));
{
MutexAutoLock lock(mMutex);
data = FindDataForTrack(aTrackID);
found = !!data;
isAudio = found && data->mData->GetType() == MediaSegment::AUDIO;
if (found && isAudio) {
if (found) {
isAudio = data->mData->GetType() == MediaSegment::AUDIO;
isVideo = data->mData->GetType() == MediaSegment::VIDEO;
}
if (found && (isAudio || isVideo)) {
TrackBound<DirectMediaStreamTrackListener>* sourceListener =
mDirectTrackListeners.AppendElement();
sourceListener->mListener = listener;
@ -2966,11 +2971,11 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStream
DirectMediaStreamTrackListener::InstallationResult::TRACK_NOT_FOUND_AT_SOURCE);
return;
}
if (!isAudio) {
STREAM_LOG(LogLevel::Warning, ("Source track for direct track listener %p is not audio",
if (!isAudio && !isVideo) {
STREAM_LOG(LogLevel::Warning, ("Source track for direct track listener %p is unknown",
listener.get()));
listener->NotifyDirectListenerInstalled(
DirectMediaStreamTrackListener::InstallationResult::TRACK_TYPE_NOT_SUPPORTED);
// It is not a video or audio track.
MOZ_ASSERT(true);
return;
}
STREAM_LOG(LogLevel::Debug, ("Added direct track listener %p", listener.get()));

Просмотреть файл

@ -238,9 +238,6 @@ public:
* We found the source stream of media data for this track, but the track
* didn't exist. This should only happen if you try to install the listener
* directly to a SourceMediaStream that doesn't contain the given TrackID.
* TRACK_TYPE_NOT_SUPPORTED
* This is the failure when you install the listener to a
* non-(audio or video) track.
* STREAM_NOT_SUPPORTED
* While looking for the data source of this track, we found a MediaStream
* that is not a SourceMediaStream or a TrackUnionStream.

Просмотреть файл

@ -9,6 +9,7 @@
#include "MediaStreamGraph.h"
#include "nsIUUIDGenerator.h"
#include "nsServiceManagerUtils.h"
#include "MediaStreamListener.h"
#ifdef LOG
#undef LOG
@ -161,6 +162,12 @@ MediaStreamTrack::Destroy()
mPrincipalHandleListener->Forget();
mPrincipalHandleListener = nullptr;
}
for (auto l : mTrackListeners) {
RemoveListener(l);
}
for (auto l : mDirectTrackListeners) {
RemoveDirectListener(l);
}
}
NS_IMPL_CYCLE_COLLECTION_CLASS(MediaStreamTrack)
@ -387,6 +394,11 @@ MediaStreamTrack::GetInputStream()
ProcessedMediaStream*
MediaStreamTrack::GetOwnedStream()
{
if (!mOwningStream)
{
return nullptr;
}
return mOwningStream->GetOwnedStream();
}
@ -395,8 +407,10 @@ MediaStreamTrack::AddListener(MediaStreamTrackListener* aListener)
{
LOG(LogLevel::Debug, ("MediaStreamTrack %p adding listener %p",
this, aListener));
MOZ_ASSERT(GetOwnedStream());
GetOwnedStream()->AddTrackListener(aListener, mTrackID);
mTrackListeners.AppendElement(aListener);
}
void
@ -405,7 +419,10 @@ MediaStreamTrack::RemoveListener(MediaStreamTrackListener* aListener)
LOG(LogLevel::Debug, ("MediaStreamTrack %p removing listener %p",
this, aListener));
GetOwnedStream()->RemoveTrackListener(aListener, mTrackID);
if (GetOwnedStream()) {
GetOwnedStream()->RemoveTrackListener(aListener, mTrackID);
mTrackListeners.RemoveElement(aListener);
}
}
void
@ -415,8 +432,10 @@ MediaStreamTrack::AddDirectListener(DirectMediaStreamTrackListener *aListener)
"stream %p, track %d",
this, AsAudioStreamTrack() ? "audio" : "video",
aListener, GetOwnedStream(), mTrackID));
MOZ_ASSERT(GetOwnedStream());
GetOwnedStream()->AddDirectTrackListener(aListener, mTrackID);
mDirectTrackListeners.AppendElement(aListener);
}
void
@ -425,7 +444,10 @@ MediaStreamTrack::RemoveDirectListener(DirectMediaStreamTrackListener *aListener
LOG(LogLevel::Debug, ("MediaStreamTrack %p removing direct listener %p from stream %p",
this, aListener, GetOwnedStream()));
GetOwnedStream()->RemoveDirectTrackListener(aListener, mTrackID);
if (GetOwnedStream()) {
GetOwnedStream()->RemoveDirectTrackListener(aListener, mTrackID);
mDirectTrackListeners.RemoveElement(aListener);
}
}
already_AddRefed<MediaInputPort>

Просмотреть файл

@ -378,6 +378,8 @@ public:
*/
bool IsForwardedThrough(MediaInputPort* aPort);
void SetMediaStreamSizeListener(DirectMediaStreamTrackListener* aListener);
protected:
virtual ~MediaStreamTrack();
@ -416,6 +418,10 @@ protected:
nsCOMPtr<nsIPrincipal> mPrincipal;
nsCOMPtr<nsIPrincipal> mPendingPrincipal;
RefPtr<PrincipalHandleListener> mPrincipalHandleListener;
// Keep tracking MediaStreamTrackListener and DirectMediaStreamTrackListener,
// so we can remove them in |Destory|.
nsTArray<RefPtr<MediaStreamTrackListener>> mTrackListeners;
nsTArray<RefPtr<DirectMediaStreamTrackListener>> mDirectTrackListeners;
nsString mID;
MediaStreamTrackState mReadyState;
bool mEnabled;

Просмотреть файл

@ -9,6 +9,7 @@
#include "mozilla/AsyncEventDispatcher.h"
#include "mozilla/dom/HTMLMediaElement.h"
#include "mozilla/dom/AudioTrack.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "mozilla/dom/VideoTrack.h"
#include "mozilla/dom/TrackEvent.h"
#include "nsThreadUtils.h"
@ -106,9 +107,10 @@ already_AddRefed<VideoTrack>
MediaTrackList::CreateVideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
const nsAString& aLanguage)
const nsAString& aLanguage,
VideoStreamTrack* aVideoTrack)
{
RefPtr<VideoTrack> track = new VideoTrack(aId, aKind, aLabel, aLanguage);
RefPtr<VideoTrack> track = new VideoTrack(aId, aKind, aLabel, aLanguage, aVideoTrack);
return track.forget();
}

Просмотреть файл

@ -20,6 +20,7 @@ class AudioTrackList;
class VideoTrackList;
class AudioTrack;
class VideoTrack;
class VideoStreamTrack;
/**
* Base class of AudioTrackList and VideoTrackList. The AudioTrackList and
@ -58,11 +59,14 @@ public:
const nsAString& aLanguage,
bool aEnabled);
// For the case of src of HTMLMediaElement is non-MediaStream, leave the
// aVideoTrack as default(nullptr).
static already_AddRefed<VideoTrack>
CreateVideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
const nsAString& aLanguage);
const nsAString& aLanguage,
VideoStreamTrack* aVideoTrack = nullptr);
virtual void EmptyTracks();

Просмотреть файл

@ -5,6 +5,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "mozilla/dom/HTMLMediaElement.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "mozilla/dom/VideoTrack.h"
#include "mozilla/dom/VideoTrackBinding.h"
#include "mozilla/dom/VideoTrackList.h"
@ -15,12 +16,25 @@ namespace dom {
VideoTrack::VideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
const nsAString& aLanguage)
const nsAString& aLanguage,
VideoStreamTrack* aStreamTarck)
: MediaTrack(aId, aKind, aLabel, aLanguage)
, mSelected(false)
, mVideoStreamTrack(aStreamTarck)
{
}
VideoTrack::~VideoTrack()
{
}
NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoTrack, MediaTrack, mVideoStreamTrack)
NS_IMPL_ADDREF_INHERITED(VideoTrack, MediaTrack)
NS_IMPL_RELEASE_INHERITED(VideoTrack, MediaTrack)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(VideoTrack)
NS_INTERFACE_MAP_END_INHERITING(MediaTrack)
JSObject*
VideoTrack::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto)
{

Просмотреть файл

@ -13,6 +13,7 @@ namespace mozilla {
namespace dom {
class VideoTrackList;
class VideoStreamTrack;
class VideoTrack : public MediaTrack
{
@ -20,7 +21,11 @@ public:
VideoTrack(const nsAString& aId,
const nsAString& aKind,
const nsAString& aLabel,
const nsAString& aLanguage);
const nsAString& aLanguage,
VideoStreamTrack* aStreamTarck = nullptr);
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoTrack, MediaTrack)
JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
@ -36,6 +41,11 @@ public:
// aFlags contains FIRE_NO_EVENTS because no events are fired in such cases.
void SetEnabledInternal(bool aEnabled, int aFlags) override;
// Get associated video stream track when the video track comes from
// MediaStream. This might be nullptr when the src of owning HTMLMediaElement
// is not MediaStream.
VideoStreamTrack* GetVideoStreamTrack() { return mVideoStreamTrack; }
// WebIDL
bool Selected() const
{
@ -48,7 +58,10 @@ public:
void SetSelected(bool aSelected);
private:
virtual ~VideoTrack();
bool mSelected;
RefPtr<VideoStreamTrack> mVideoStreamTrack;
};
} // namespace dom

Просмотреть файл

@ -31,18 +31,18 @@ VideoTrackList::RemoveTrack(const RefPtr<MediaTrack>& aTrack)
// need to be done after RemoveTrack. Also the call of
// |MediaTrackList::RemoveTrack| is necessary even when mSelectedIndex = -1.
bool found;
VideoTrack* videoTrack = IndexedGetter(mSelectedIndex, found);
VideoTrack* selectedVideoTrack = IndexedGetter(mSelectedIndex, found);
MediaTrackList::RemoveTrack(aTrack);
if (mSelectedIndex == -1) {
// There was no selected track and we don't select another track on removal.
return;
}
MOZ_ASSERT(found, "When mSelectedIndex is set it should point to a track");
MOZ_ASSERT(videoTrack, "The mSelectedIndex should be set to video track only");
MOZ_ASSERT(selectedVideoTrack, "The mSelectedIndex should be set to video track only");
// Let the caller of RemoveTrack deal with choosing the new selected track if
// it removes the currently-selected track.
if (aTrack == videoTrack) {
if (aTrack == selectedVideoTrack) {
mSelectedIndex = -1;
return;
}
@ -51,7 +51,7 @@ VideoTrackList::RemoveTrack(const RefPtr<MediaTrack>& aTrack)
// currently-selected video track. We need to find the new location of the
// selected track.
for (size_t ix = 0; ix < mTracks.Length(); ix++) {
if (mTracks[ix] == videoTrack) {
if (mTracks[ix] == selectedVideoTrack) {
mSelectedIndex = ix;
return;
}