Bug 1201363 - Let MediaStreamVideoSink bind with particular video track. r=jesup

MozReview-Commit-ID: FcjnmDKuRQI

--HG--
extra : transplant_source : %C3%D3%08%E5%C7%CA%9E%7C%F9L%C0w%C2o%11%16%877%04%21

dom/camera/CameraPreviewMediaStream.cpp

@@ -60,18 +60,18 @@ CameraPreviewMediaStream::RemoveAudioOutput(void* aKey)
 }
 
 void
-CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   MutexAutoLock lock(mMutex);
   RefPtr<MediaStreamVideoSink> sink = aSink;
-  AddVideoOutputImpl(sink.forget());
+  AddVideoOutputImpl(sink.forget(), aID);
 }
 
 void
-CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   MutexAutoLock lock(mMutex);
-  RemoveVideoOutputImpl(aSink);
+  RemoveVideoOutputImpl(aSink, aID);
 }
 
 void
@@ -126,8 +126,8 @@ CameraPreviewMediaStream::Invalidate()
 {
   MutexAutoLock lock(mMutex);
   --mInvalidatePending;
-  for (MediaStreamVideoSink* sink : mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }
@@ -168,8 +168,8 @@ CameraPreviewMediaStream::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Im
     mDiscardedFrames = 0;
 
     TimeStamp now = TimeStamp::Now();
-    for (MediaStreamVideoSink* sink : mVideoOutputs) {
-      VideoFrameContainer* output = sink->AsVideoFrameContainer();
+    for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+      VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
       if (!output) {
         continue;
       }
@@ -187,8 +187,8 @@ CameraPreviewMediaStream::ClearCurrentFrame()
 {
   MutexAutoLock lock(mMutex);
 
-  for (MediaStreamVideoSink* sink : mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }

dom/camera/CameraPreviewMediaStream.h

@@ -43,16 +43,16 @@ class CameraPreviewMediaStream : public ProcessedMediaStream
 public:
   CameraPreviewMediaStream();
 
-  virtual void AddAudioOutput(void* aKey) override;
-  virtual void SetAudioOutputVolume(void* aKey, float aVolume) override;
-  virtual void RemoveAudioOutput(void* aKey) override;
-  virtual void AddVideoOutput(MediaStreamVideoSink* aSink) override;
-  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink) override;
-  virtual void Suspend() override {}
-  virtual void Resume() override {}
-  virtual void AddListener(MediaStreamListener* aListener) override;
-  virtual void RemoveListener(MediaStreamListener* aListener) override;
-  virtual void Destroy() override;
+  void AddAudioOutput(void* aKey) override;
+  void SetAudioOutputVolume(void* aKey, float aVolume) override;
+  void RemoveAudioOutput(void* aKey) override;
+  void AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+  void RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID) override;
+  void Suspend() override {}
+  void Resume() override {}
+  void AddListener(MediaStreamListener* aListener) override;
+  void RemoveListener(MediaStreamListener* aListener) override;
+  void Destroy() override;
   void OnPreviewStateChange(bool aActive);
 
   void Invalidate();

dom/media/MediaStreamGraph.cpp

@@ -1025,10 +1025,10 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream)
         // "Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack."
         // of this bug.
         // This is a temp workaround to pass the build and test.
-        if (!aStream->mVideoOutputs[0]->AsVideoFrameContainer()) {
+        if (!aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()) {
           return;
         }
-        blackImage = aStream->mVideoOutputs[0]->AsVideoFrameContainer()->
+        blackImage = aStream->mVideoOutputs[0].mListener->AsVideoFrameContainer()->
           GetImageContainer()->CreatePlanarYCbCrImage();
         if (blackImage) {
           // Sets the image to a single black pixel, which will be scaled to
@@ -1053,8 +1053,8 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream)
   AutoTArray<ImageContainer::NonOwningImage,4> images;
   bool haveMultipleImages = false;
 
-  for (MediaStreamVideoSink* sink : aStream->mVideoOutputs) {
-    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+  for (const TrackBound<MediaStreamVideoSink>& sink : aStream->mVideoOutputs) {
+    VideoFrameContainer* output = sink.mListener->AsVideoFrameContainer();
     if (!output) {
       continue;
     }
@@ -2271,55 +2271,76 @@ MediaStream::RemoveAudioOutput(void* aKey)
 }
 
 void
-MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink)
+MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+                                TrackID aID)
 {
   RefPtr<MediaStreamVideoSink> sink = aSink;
   STREAM_LOG(LogLevel::Info, ("MediaStream %p Adding MediaStreamVideoSink %p as output",
                               this, sink.get()));
-  *mVideoOutputs.AppendElement() = sink.forget();
+  MOZ_ASSERT(aID != TRACK_NONE);
+   for (auto entry : mVideoOutputs) {
+     if (entry.mListener == sink &&
+         (entry.mTrackID == TRACK_ANY || entry.mTrackID == aID)) {
+       return;
+     }
+   }
+   TrackBound<MediaStreamVideoSink>* l = mVideoOutputs.AppendElement();
+   l->mListener = sink;
+   l->mTrackID = aID;
 }
 
 void
-MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink,
+                                   TrackID aID)
 {
   STREAM_LOG(LogLevel::Info, ("MediaStream %p Removing MediaStreamVideoSink %p as output",
                               this, aSink));
+  MOZ_ASSERT(aID != TRACK_NONE);
+
   // Ensure that any frames currently queued for playback by the compositor
   // are removed.
   aSink->ClearFrames();
-  mVideoOutputs.RemoveElement(aSink);
+  for (size_t i = 0; i < mVideoOutputs.Length(); ++i) {
+    if (mVideoOutputs[i].mListener == aSink &&
+        (mVideoOutputs[i].mTrackID == TRACK_ANY ||
+         mVideoOutputs[i].mTrackID == aID)) {
+      mVideoOutputs.RemoveElementAt(i);
+    }
+  }
 }
 
 void
-MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   class Message : public ControlMessage {
   public:
-    Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
-      ControlMessage(aStream), mSink(aSink) {}
+    Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+      ControlMessage(aStream), mSink(aSink), mID(aID) {}
     void Run() override
     {
-      mStream->AddVideoOutputImpl(mSink.forget());
+      mStream->AddVideoOutputImpl(mSink.forget(), mID);
     }
     RefPtr<MediaStreamVideoSink> mSink;
+    TrackID mID;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
 }
 
 void
-MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
+MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink, TrackID aID)
 {
   class Message : public ControlMessage {
   public:
-    Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
-      ControlMessage(aStream), mSink(aSink) {}
+    Message(MediaStream* aStream, MediaStreamVideoSink* aSink, TrackID aID) :
+      ControlMessage(aStream), mSink(aSink), mID(aID) {}
     void Run() override
     {
-      mStream->RemoveVideoOutputImpl(mSink);
+      mStream->RemoveVideoOutputImpl(mSink, mID);
     }
     RefPtr<MediaStreamVideoSink> mSink;
+    TrackID mID;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink, aID));
 }
 
 void

dom/media/MediaStreamGraph.h

@@ -17,7 +17,6 @@
 #include "nsIRunnable.h"
 #include "VideoSegment.h"
 #include "StreamTracks.h"
-#include "MediaStreamVideoSink.h"
 #include "MainThreadUtils.h"
 #include "StreamTracks.h"
 #include "nsAutoPtr.h"
@@ -167,8 +166,10 @@ class MediaInputPort;
 class MediaStreamGraphImpl;
 class MediaStreamListener;
 class MediaStreamTrackListener;
+class MediaStreamVideoSink;
 class ProcessedMediaStream;
 class SourceMediaStream;
+class TrackUnionStream;
 
 enum MediaStreamGraphEvent : uint32_t;
 enum TrackEventCommand : uint32_t;
@@ -294,8 +295,10 @@ public:
   // Since a stream can be played multiple ways, we need to be able to
   // play to multiple MediaStreamVideoSinks.
   // Only the first enabled video track is played.
-  virtual void AddVideoOutput(MediaStreamVideoSink* aSink);
-  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink);
+  virtual void AddVideoOutput(MediaStreamVideoSink* aSink,
+                              TrackID aID = TRACK_ANY);
+  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink,
+                                 TrackID aID = TRACK_ANY);
   // Explicitly suspend. Useful for example if a media element is pausing
   // and we need to stop its stream emitting its buffered data. As soon as the
   // Suspend message reaches the graph, the stream stops processing. It
@@ -404,6 +407,7 @@ public:
   virtual SourceMediaStream* AsSourceStream() { return nullptr; }
   virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
   virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
+  virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; }
 
   // These Impl methods perform the core functionality of the control methods
   // above, on the media graph thread.
@@ -424,8 +428,9 @@ public:
     return !mAudioOutputs.IsEmpty();
   }
   void RemoveAudioOutputImpl(void* aKey);
-  void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink);
-  void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink);
+  void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
+                          TrackID aID);
+  void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID);
   void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
   void RemoveListenerImpl(MediaStreamListener* aListener);
   void RemoveAllListenersImpl();
@@ -584,7 +589,7 @@ protected:
     float mVolume;
   };
   nsTArray<AudioOutput> mAudioOutputs;
-  nsTArray<RefPtr<MediaStreamVideoSink>> mVideoOutputs;
+  nsTArray<TrackBound<MediaStreamVideoSink>> mVideoOutputs;
   // We record the last played video frame to avoid playing the frame again
   // with a different frame id.
   VideoFrame mLastPlayedVideoFrame;

dom/media/TrackUnionStream.h

@@ -19,6 +19,9 @@ class TrackUnionStream : public ProcessedMediaStream {
 public:
   explicit TrackUnionStream();
 
+  virtual TrackUnionStream* AsTrackUnionStream() override { return this; }
+  friend class DOMMediaStream;
+
   void RemoveInput(MediaInputPort* aPort) override;
   void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override;