Bug 1266646 - Change HTMLMediaElement::StreamSizeListerner to inherit MediaStreamTrackDirectListener. r=jesup, r=pehrsons

MozReview-Commit-ID: HnNv9BnlbDy

--HG--
extra : rebase_source : 7d04886895910b00964c2ce36a337873195b12fa

dom/html/HTMLMediaElement.cpp

@@ -273,6 +273,56 @@ public:
   }
 };
 
+/**
+ * This listener observes the first video frame to arrive with a non-empty size,
+ * and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
+ */
+class HTMLMediaElement::StreamSizeListener : public MediaStreamTrackDirectListener {
+public:
+  explicit StreamSizeListener(HTMLMediaElement* aElement) :
+    mElement(aElement),
+    mInitialSizeFound(false)
+  {}
+  void Forget() { mElement = nullptr; }
+
+  void ReceivedSize(gfx::IntSize aSize)
+  {
+    if (!mElement) {
+      return;
+    }
+    RefPtr<HTMLMediaElement> deathGrip = mElement;
+    mElement->UpdateInitialMediaSize(aSize);
+  }
+
+  void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+                               StreamTime aTrackOffset,
+                               const MediaSegment& aMedia) override
+  {
+    if (mInitialSizeFound || aMedia.GetType() != MediaSegment::VIDEO) {
+      return;
+    }
+    const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
+    for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
+      if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) {
+        mInitialSizeFound = true;
+        nsCOMPtr<nsIRunnable> event =
+          NewRunnableMethod<gfx::IntSize>(
+              this, &StreamSizeListener::ReceivedSize,
+              c->mFrame.GetIntrinsicSize());
+        aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
+        return;
+      }
+    }
+  }
+
+private:
+  // These fields may only be accessed on the main thread
+  HTMLMediaElement* mElement;
+
+  // These fields may only be accessed on the MSG thread
+  bool mInitialSizeFound;
+};
+
 /**
  * There is a reference cycle involving this class: MediaLoadListener
  * holds a reference to the HTMLMediaElement, which holds a reference
@@ -484,6 +534,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTM
 #ifdef MOZ_EME
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaKeys)
 #endif
+  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSelectedVideoStreamTrack)
 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
 
 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLElement)
@@ -510,6 +561,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTMLE
 #ifdef MOZ_EME
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mMediaKeys)
 #endif
+  NS_IMPL_CYCLE_COLLECTION_UNLINK(mSelectedVideoStreamTrack)
 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
 
 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(HTMLMediaElement)
@@ -724,6 +776,14 @@ void HTMLMediaElement::AbortExistingLoads()
 
   bool fireTimeUpdate = false;
 
+  // We need to remove StreamSizeListener before VideoTracks get emptied.
+  if (mMediaStreamSizeListener) {
+    mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
+    mSelectedVideoStreamTrack = nullptr;
+    mMediaStreamSizeListener->Forget();
+    mMediaStreamSizeListener = nullptr;
+  }
+
   // When aborting the existing loads, empty the objects in audio track list and
   // video track list, no events (in particular, no removetrack events) are
   // fired as part of this. Ending MediaStream sends track ended notifications,
@@ -3216,59 +3276,6 @@ private:
   bool mPendingNotifyOutput;
 };
 
-/**
- * This listener observes the first video frame to arrive with a non-empty size,
- * and calls HTMLMediaElement::ReceivedMediaStreamInitialSize() with that size.
- */
-class HTMLMediaElement::StreamSizeListener : public MediaStreamListener {
-public:
-  explicit StreamSizeListener(HTMLMediaElement* aElement) :
-    mElement(aElement),
-    mInitialSizeFound(false)
-  {}
-  void Forget() { mElement = nullptr; }
-
-  void ReceivedSize(gfx::IntSize aSize)
-  {
-    if (!mElement) {
-      return;
-    }
-    RefPtr<HTMLMediaElement> deathGrip = mElement;
-    mElement->UpdateInitialMediaSize(aSize);
-  }
-
-  void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
-                                StreamTime aTrackOffset,
-                                uint32_t aTrackEvents,
-                                const MediaSegment& aQueuedMedia,
-                                MediaStream* aInputStream,
-                                TrackID aInputTrackID) override
-  {
-    if (mInitialSizeFound || aQueuedMedia.GetType() != MediaSegment::VIDEO) {
-      return;
-    }
-    const VideoSegment& video = static_cast<const VideoSegment&>(aQueuedMedia);
-    for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
-      if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0,0)) {
-        mInitialSizeFound = true;
-        nsCOMPtr<nsIRunnable> event =
-          NewRunnableMethod<gfx::IntSize>(
-              this, &StreamSizeListener::ReceivedSize,
-              c->mFrame.GetIntrinsicSize());
-        aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
-        return;
-      }
-    }
-  }
-
-private:
-  // These fields may only be accessed on the main thread
-  HTMLMediaElement* mElement;
-
-  // These fields may only be accessed on the MSG thread
-  bool mInitialSizeFound;
-};
-
 class HTMLMediaElement::MediaStreamTracksAvailableCallback:
   public OnTracksAvailableCallback
 {
@@ -3387,9 +3394,6 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream)
   RefPtr<MediaStream> stream = GetSrcMediaStream();
   if (stream) {
     stream->SetAudioChannelType(mAudioChannel);
-
-    mMediaStreamSizeListener = new StreamSizeListener(this);
-    stream->AddListener(mMediaStreamSizeListener);
   }
 
   UpdateSrcMediaStreamPlaying();
@@ -3420,10 +3424,8 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback()
   UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
 
   if (mMediaStreamSizeListener) {
-    RefPtr<MediaStream> stream = GetSrcMediaStream();
-    if (stream) {
-      stream->RemoveListener(mMediaStreamSizeListener);
-    }
+    mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
+    mSelectedVideoStreamTrack = nullptr;
     mMediaStreamSizeListener->Forget();
     mMediaStreamSizeListener = nullptr;
   }
@@ -3459,7 +3461,8 @@ CreateVideoTrack(VideoStreamTrack* aStreamTrack)
   aStreamTrack->GetLabel(label);
 
   return MediaTrackList::CreateVideoTrack(id, NS_LITERAL_STRING("main"),
-                                          label, EmptyString());
+                                          label, EmptyString(),
+                                          aStreamTrack);
 }
 
 void HTMLMediaElement::ConstructMediaTracks()
@@ -3491,6 +3494,11 @@ void HTMLMediaElement::ConstructMediaTracks()
     // must be selected.
     int index = firstEnabledVideo >= 0 ? firstEnabledVideo : 0;
     (*VideoTracks())[index]->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
+    VideoTrack* track = (*VideoTracks())[index];
+    VideoStreamTrack* streamTrack = track->GetVideoStreamTrack();
+    mMediaStreamSizeListener = new StreamSizeListener(this);
+    streamTrack->AddDirectListener(mMediaStreamSizeListener);
+    mSelectedVideoStreamTrack = streamTrack;
   }
 }
 
@@ -3511,8 +3519,20 @@ HTMLMediaElement::NotifyMediaStreamTrackAdded(const RefPtr<MediaStreamTrack>& aT
     RefPtr<AudioTrack> audioTrack = CreateAudioTrack(t);
     AudioTracks()->AddTrack(audioTrack);
   } else if (VideoStreamTrack* t = aTrack->AsVideoStreamTrack()) {
+    // TODO: Fix this per the spec on bug 1273443.
+    int32_t selectedIndex = VideoTracks()->SelectedIndex();
     RefPtr<VideoTrack> videoTrack = CreateVideoTrack(t);
     VideoTracks()->AddTrack(videoTrack);
+    // New MediaStreamTrack added, set the new added video track as selected
+    // video track when there is no selected track.
+    if (selectedIndex == -1) {
+      MOZ_ASSERT(!mSelectedVideoStreamTrack);
+      videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
+      mMediaStreamSizeListener = new StreamSizeListener(this);
+      t->AddDirectListener(mMediaStreamSizeListener);
+      mSelectedVideoStreamTrack = t;
+    }
+
   }
 }
 
@@ -3531,6 +3551,47 @@ HTMLMediaElement::NotifyMediaStreamTrackRemoved(const RefPtr<MediaStreamTrack>&
     AudioTracks()->RemoveTrack(t);
   } else if (MediaTrack* t = VideoTracks()->GetTrackById(id)) {
     VideoTracks()->RemoveTrack(t);
+    // TODO: Fix this per the spec on bug 1273443.
+    // If the removed media stream track is selected video track and there are
+    // still video tracks, change the selected video track to the first
+    // remaining track.
+    if (aTrack == mSelectedVideoStreamTrack) {
+      // The mMediaStreamSizeListener might already reset to nullptr.
+      if (mMediaStreamSizeListener) {
+        mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
+      }
+      mSelectedVideoStreamTrack = nullptr;
+      MOZ_ASSERT(mSrcStream);
+      nsTArray<RefPtr<VideoStreamTrack>> tracks;
+      mSrcStream->GetVideoTracks(tracks);
+
+      for (const RefPtr<VideoStreamTrack>& track : tracks) {
+        if (track->Ended() || !track->Enabled()) {
+          continue;
+        }
+
+        if (track->Enabled()) {
+          nsAutoString trackId;
+          track->GetId(trackId);
+          MediaTrack* videoTrack = VideoTracks()->GetTrackById(trackId);
+          MOZ_ASSERT(videoTrack);
+
+          videoTrack->SetEnabledInternal(true, MediaTrack::FIRE_NO_EVENTS);
+          if (mMediaStreamSizeListener) {
+            track->AddDirectListener(mMediaStreamSizeListener);
+          }
+          mSelectedVideoStreamTrack = track;
+          return;
+        }
+      }
+
+      // There is no enabled video track existing, clean the
+      // mMediaStreamSizeListener.
+      if (mMediaStreamSizeListener) {
+        mMediaStreamSizeListener->Forget();
+        mMediaStreamSizeListener = nullptr;
+      }
+    }
   } else {
     // XXX (bug 1208328) Uncomment this when DOMMediaStream doesn't call
     // NotifyTrackRemoved multiple times for the same track, i.e., when it
@@ -4445,10 +4506,7 @@ void HTMLMediaElement::UpdateInitialMediaSize(const nsIntSize& aSize)
   if (!mMediaStreamSizeListener) {
     return;
   }
-  RefPtr<MediaStream> stream = GetSrcMediaStream();
-  if (stream) {
-    stream->RemoveListener(mMediaStreamSizeListener);
-  }
+  mSelectedVideoStreamTrack->RemoveDirectListener(mMediaStreamSizeListener);
   mMediaStreamSizeListener->Forget();
   mMediaStreamSizeListener = nullptr;
 }

dom/html/HTMLMediaElement.h

@@ -1239,6 +1239,8 @@ protected:
   // Holds a reference to the size-getting MediaStreamListener attached to
   // mSrcStream.
   RefPtr<StreamSizeListener> mMediaStreamSizeListener;
+  // The selected video stream track which contained mMediaStreamSizeListener.
+  RefPtr<VideoStreamTrack> mSelectedVideoStreamTrack;
 
   // Holds a reference to the MediaSource, if any, referenced by the src
   // attribute on the media element.

dom/media/MediaStreamGraph.cpp

@@ -2927,15 +2927,20 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackD
   TrackData* data;
   bool found;
   bool isAudio;
+  bool isVideo;
   RefPtr<MediaStreamTrackDirectListener> listener = aListener;
   STREAM_LOG(LogLevel::Debug, ("Adding direct track listener %p bound to track %d to source stream %p",
              listener.get(), aTrackID, this));
+
   {
     MutexAutoLock lock(mMutex);
     data = FindDataForTrack(aTrackID);
     found = !!data;
-    isAudio = found && data->mData->GetType() == MediaSegment::AUDIO;
-    if (found && isAudio) {
+    if (found) {
+      isAudio = data->mData->GetType() == MediaSegment::AUDIO;
+      isVideo = data->mData->GetType() == MediaSegment::VIDEO;
+    }
+    if (found && (isAudio || isVideo)) {
       TrackBound<MediaStreamTrackDirectListener>* sourceListener =
         mDirectTrackListeners.AppendElement();
       sourceListener->mListener = listener;
@@ -2949,11 +2954,11 @@ SourceMediaStream::AddDirectTrackListenerImpl(already_AddRefed<MediaStreamTrackD
       MediaStreamTrackDirectListener::InstallationResult::TRACK_NOT_FOUND_AT_SOURCE);
     return;
   }
-  if (!isAudio) {
-    STREAM_LOG(LogLevel::Warning, ("Source track for direct track listener %p is not audio",
+  if (!isAudio && !isVideo) {
+    STREAM_LOG(LogLevel::Warning, ("Source track for direct track listener %p is unknown",
                                    listener.get()));
-    listener->NotifyDirectListenerInstalled(
-      MediaStreamTrackDirectListener::InstallationResult::TRACK_TYPE_NOT_SUPPORTED);
+    // It is not a video or audio track.
+    MOZ_ASSERT(true);
     return;
   }
   STREAM_LOG(LogLevel::Debug, ("Added direct track listener %p", listener.get()));

dom/media/MediaStreamGraph.h

@@ -344,7 +344,8 @@ public:
    *    didn't exist. This should only happen if you try to install the listener
    *    directly to a SourceMediaStream that doesn't contain the given TrackID.
    * TRACK_TYPE_NOT_SUPPORTED
-   *    This is the failure when you install the listener to a non-audio track.
+   *    This is the failure when you install the listener to a
+   *    non-(audio or video) track.
    * STREAM_NOT_SUPPORTED
    *    While looking for the data source of this track, we found a MediaStream
    *    that is not a SourceMediaStream or a TrackUnionStream.
@@ -354,7 +355,6 @@ public:
    */
   enum class InstallationResult {
     TRACK_NOT_FOUND_AT_SOURCE,
-    TRACK_TYPE_NOT_SUPPORTED,
     STREAM_NOT_SUPPORTED,
     SUCCESS
   };
@@ -370,6 +370,15 @@ protected:
     aTo.AppendNullData(aFrom.GetDuration());
   }
 
+  void MirrorAndDisableSegment(VideoSegment& aFrom, VideoSegment& aTo)
+  {
+    aTo.Clear();
+    for (VideoSegment::ChunkIterator it(aFrom); !it.IsEnded(); it.Next()) {
+      aTo.AppendFrame(do_AddRef(it->mFrame.GetImage()), it->GetDuration(),
+                      it->mFrame.GetIntrinsicSize(), it->GetPrincipalHandle(), true);
+    }
+  }
+
   void NotifyRealtimeTrackDataAndApplyTrackDisabling(MediaStreamGraph* aGraph,
                                                      StreamTime aTrackOffset,
                                                      MediaSegment& aMedia)
@@ -385,6 +394,9 @@ protected:
     if (aMedia.GetType() == MediaSegment::AUDIO) {
       MirrorAndDisableSegment(static_cast<AudioSegment&>(aMedia),
                               static_cast<AudioSegment&>(*mMedia));
+    } else if (aMedia.GetType() == MediaSegment::VIDEO) {
+      MirrorAndDisableSegment(static_cast<VideoSegment&>(aMedia),
+                              static_cast<VideoSegment&>(*mMedia));
     } else {
       MOZ_CRASH("Unsupported media type");
     }

dom/media/MediaTrackList.cpp

@@ -9,6 +9,7 @@
 #include "mozilla/AsyncEventDispatcher.h"
 #include "mozilla/dom/HTMLMediaElement.h"
 #include "mozilla/dom/AudioTrack.h"
+#include "mozilla/dom/VideoStreamTrack.h"
 #include "mozilla/dom/VideoTrack.h"
 #include "mozilla/dom/TrackEvent.h"
 #include "nsThreadUtils.h"
@@ -106,9 +107,10 @@ already_AddRefed<VideoTrack>
 MediaTrackList::CreateVideoTrack(const nsAString& aId,
                                  const nsAString& aKind,
                                  const nsAString& aLabel,
-                                 const nsAString& aLanguage)
+                                 const nsAString& aLanguage,
+                                 VideoStreamTrack* aVideoTrack)
 {
-  RefPtr<VideoTrack> track = new VideoTrack(aId, aKind, aLabel, aLanguage);
+  RefPtr<VideoTrack> track = new VideoTrack(aId, aKind, aLabel, aLanguage, aVideoTrack);
   return track.forget();
 }
 

dom/media/MediaTrackList.h

@@ -20,6 +20,7 @@ class AudioTrackList;
 class VideoTrackList;
 class AudioTrack;
 class VideoTrack;
+class VideoStreamTrack;
 
 /**
  * Base class of AudioTrackList and VideoTrackList. The AudioTrackList and
@@ -58,11 +59,14 @@ public:
                    const nsAString& aLanguage,
                    bool aEnabled);
 
+  // For the case of src of HTMLMediaElement is non-MediaStream, leave the
+  // aVideoTrack as default(nullptr).
   static already_AddRefed<VideoTrack>
   CreateVideoTrack(const nsAString& aId,
                    const nsAString& aKind,
                    const nsAString& aLabel,
-                   const nsAString& aLanguage);
+                   const nsAString& aLanguage,
+                   VideoStreamTrack* aVideoTrack = nullptr);
 
   virtual void EmptyTracks();
 

dom/media/VideoTrack.cpp

@@ -5,6 +5,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "mozilla/dom/HTMLMediaElement.h"
+#include "mozilla/dom/VideoStreamTrack.h"
 #include "mozilla/dom/VideoTrack.h"
 #include "mozilla/dom/VideoTrackBinding.h"
 #include "mozilla/dom/VideoTrackList.h"
@@ -15,12 +16,25 @@ namespace dom {
 VideoTrack::VideoTrack(const nsAString& aId,
                        const nsAString& aKind,
                        const nsAString& aLabel,
-                       const nsAString& aLanguage)
+                       const nsAString& aLanguage,
+                       VideoStreamTrack* aStreamTarck)
   : MediaTrack(aId, aKind, aLabel, aLanguage)
   , mSelected(false)
+  , mVideoStreamTrack(aStreamTarck)
 {
 }
 
+VideoTrack::~VideoTrack()
+{
+}
+
+NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoTrack, MediaTrack, mVideoStreamTrack)
+
+NS_IMPL_ADDREF_INHERITED(VideoTrack, MediaTrack)
+NS_IMPL_RELEASE_INHERITED(VideoTrack, MediaTrack)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(VideoTrack)
+NS_INTERFACE_MAP_END_INHERITING(MediaTrack)
+
 JSObject*
 VideoTrack::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto)
 {

dom/media/VideoTrack.h

@@ -13,6 +13,7 @@ namespace mozilla {
 namespace dom {
 
 class VideoTrackList;
+class VideoStreamTrack;
 
 class VideoTrack : public MediaTrack
 {
@@ -20,7 +21,11 @@ public:
   VideoTrack(const nsAString& aId,
              const nsAString& aKind,
              const nsAString& aLabel,
-             const nsAString& aLanguage);
+             const nsAString& aLanguage,
+             VideoStreamTrack* aStreamTarck = nullptr);
+
+  NS_DECL_ISUPPORTS_INHERITED
+  NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoTrack, MediaTrack)
 
   JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override;
 
@@ -36,6 +41,11 @@ public:
   // aFlags contains FIRE_NO_EVENTS because no events are fired in such cases.
   void SetEnabledInternal(bool aEnabled, int aFlags) override;
 
+  // Get associated video stream track when the video track comes from
+  // MediaStream. This might be nullptr when the src of owning HTMLMediaElement
+  // is not MediaStream.
+  VideoStreamTrack* GetVideoStreamTrack() { return mVideoStreamTrack; }
+
   // WebIDL
   bool Selected() const
   {
@@ -48,7 +58,10 @@ public:
   void SetSelected(bool aSelected);
 
 private:
+  virtual ~VideoTrack();
+
   bool mSelected;
+  RefPtr<VideoStreamTrack> mVideoStreamTrack;
 };
 
 } // namespace dom

dom/media/VideoTrackList.cpp

@@ -31,18 +31,18 @@ VideoTrackList::RemoveTrack(const RefPtr<MediaTrack>& aTrack)
   // need to be done after RemoveTrack. Also the call of
   // |MediaTrackList::RemoveTrack| is necessary even when mSelectedIndex = -1.
   bool found;
-  VideoTrack* videoTrack = IndexedGetter(mSelectedIndex, found);
+  VideoTrack* selectedVideoTrack = IndexedGetter(mSelectedIndex, found);
   MediaTrackList::RemoveTrack(aTrack);
   if (mSelectedIndex == -1) {
     // There was no selected track and we don't select another track on removal.
     return;
   }
   MOZ_ASSERT(found, "When mSelectedIndex is set it should point to a track");
-  MOZ_ASSERT(videoTrack, "The mSelectedIndex should be set to video track only");
+  MOZ_ASSERT(selectedVideoTrack, "The mSelectedIndex should be set to video track only");
 
   // Let the caller of RemoveTrack deal with choosing the new selected track if
   // it removes the currently-selected track.
-  if (aTrack == videoTrack) {
+  if (aTrack == selectedVideoTrack) {
     mSelectedIndex = -1;
     return;
   }
@@ -51,7 +51,7 @@ VideoTrackList::RemoveTrack(const RefPtr<MediaTrack>& aTrack)
   // currently-selected video track. We need to find the new location of the
   // selected track.
   for (size_t ix = 0; ix < mTracks.Length(); ix++) {
-    if (mTracks[ix] == videoTrack) {
+    if (mTracks[ix] == selectedVideoTrack) {
       mSelectedIndex = ix;
       return;
     }