Bug 1201363 - Replace VideoFrameContainer with MediaStreamVideoSink in MSG. r=jesup

Replace the pointer of VideoFrameContainer with the pointer of MediaStreamVideoSink.

MozReview-Commit-ID: 5bqEMpemwuR

--HG--
extra : amend_source : 7eb1e87fdcbc61f2f9831fa3a6d803cc50306604

dom/camera/CameraPreviewMediaStream.cpp

@@ -6,6 +6,7 @@
 #include "CameraPreviewMediaStream.h"
 #include "CameraCommon.h"
 #include "MediaStreamListener.h"
+#include "VideoFrameContainer.h"
 
 /**
  * Maximum number of outstanding invalidates before we start to drop frames;
@@ -59,18 +60,18 @@ CameraPreviewMediaStream::RemoveAudioOutput(void* aKey)
 }
 
 void
-CameraPreviewMediaStream::AddVideoOutput(VideoFrameContainer* aContainer)
+CameraPreviewMediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
 {
   MutexAutoLock lock(mMutex);
-  RefPtr<VideoFrameContainer> container = aContainer;
-  AddVideoOutputImpl(container.forget());
+  RefPtr<MediaStreamVideoSink> sink = aSink;
+  AddVideoOutputImpl(sink.forget());
 }
 
 void
-CameraPreviewMediaStream::RemoveVideoOutput(VideoFrameContainer* aContainer)
+CameraPreviewMediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
 {
   MutexAutoLock lock(mMutex);
-  RemoveVideoOutputImpl(aContainer);
+  RemoveVideoOutputImpl(aSink);
 }
 
 void
@@ -125,8 +126,11 @@ CameraPreviewMediaStream::Invalidate()
 {
   MutexAutoLock lock(mMutex);
   --mInvalidatePending;
-  for (nsTArray<RefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
-    VideoFrameContainer* output = mVideoOutputs[i];
+  for (MediaStreamVideoSink* sink : mVideoOutputs) {
+    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+    if (!output) {
+      continue;
+    }
     output->Invalidate();
   }
 }
@@ -164,8 +168,11 @@ CameraPreviewMediaStream::SetCurrentFrame(const gfx::IntSize& aIntrinsicSize, Im
     mDiscardedFrames = 0;
 
     TimeStamp now = TimeStamp::Now();
-    for (nsTArray<RefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
-      VideoFrameContainer* output = mVideoOutputs[i];
+    for (MediaStreamVideoSink* sink : mVideoOutputs) {
+      VideoFrameContainer* output = sink->AsVideoFrameContainer();
+      if (!output) {
+        continue;
+      }
       output->SetCurrentFrame(aIntrinsicSize, aImage, now);
     }
 
@@ -180,8 +187,11 @@ CameraPreviewMediaStream::ClearCurrentFrame()
 {
   MutexAutoLock lock(mMutex);
 
-  for (nsTArray<RefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
-    VideoFrameContainer* output = mVideoOutputs[i];
+  for (MediaStreamVideoSink* sink : mVideoOutputs) {
+    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+    if (!output) {
+      continue;
+    }
     output->ClearCurrentFrame();
     NS_DispatchToMainThread(NewRunnableMethod(output, &VideoFrameContainer::Invalidate));
   }

dom/camera/CameraPreviewMediaStream.h

@@ -5,12 +5,13 @@
 #ifndef DOM_CAMERA_CAMERAPREVIEWMEDIASTREAM_H
 #define DOM_CAMERA_CAMERAPREVIEWMEDIASTREAM_H
 
-#include "VideoFrameContainer.h"
 #include "MediaStreamGraph.h"
 #include "mozilla/Mutex.h"
 
 namespace mozilla {
 
+class MediaStreamVideoSink;
+
 class FakeMediaStreamGraph : public MediaStreamGraph
 {
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FakeMediaStreamGraph)
@@ -45,8 +46,8 @@ public:
   virtual void AddAudioOutput(void* aKey) override;
   virtual void SetAudioOutputVolume(void* aKey, float aVolume) override;
   virtual void RemoveAudioOutput(void* aKey) override;
-  virtual void AddVideoOutput(VideoFrameContainer* aContainer) override;
-  virtual void RemoveVideoOutput(VideoFrameContainer* aContainer) override;
+  virtual void AddVideoOutput(MediaStreamVideoSink* aSink) override;
+  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink) override;
   virtual void Suspend() override {}
   virtual void Resume() override {}
   virtual void AddListener(MediaStreamListener* aListener) override;

dom/html/HTMLMediaElement.cpp

@@ -75,6 +75,7 @@
 #include "VideoStreamTrack.h"
 #include "MediaTrackList.h"
 #include "MediaStreamError.h"
+#include "VideoFrameContainer.h"
 
 #include "AudioChannelService.h"
 

dom/html/HTMLVideoElement.cpp

@@ -17,6 +17,7 @@
 #include "prlock.h"
 #include "nsThreadUtils.h"
 #include "ImageContainer.h"
+#include "VideoFrameContainer.h"
 
 #include "nsIScriptSecurityManager.h"
 #include "nsIXPConnect.h"

dom/media/MediaDecoder.cpp

@@ -14,6 +14,7 @@
 #include "MediaDecoderStateMachine.h"
 #include "ImageContainer.h"
 #include "MediaResource.h"
+#include "VideoFrameContainer.h"
 #include "nsError.h"
 #include "mozilla/Preferences.h"
 #include "mozilla/StaticPtr.h"

dom/media/MediaFormatReader.cpp

@@ -17,6 +17,7 @@
 #include "MediaResource.h"
 #include "mozilla/SharedThreadPool.h"
 #include "VideoUtils.h"
+#include "VideoFrameContainer.h"
 
 #include <algorithm>
 

dom/media/MediaStreamGraph.cpp

@@ -27,6 +27,7 @@
 #include "mozilla/media/MediaUtils.h"
 #include <algorithm>
 #include "GeckoProfiler.h"
+#include "VideoFrameContainer.h"
 #include "mozilla/unused.h"
 #include "mozilla/media/MediaUtils.h"
 #ifdef MOZ_WEBRTC
@@ -1020,7 +1021,15 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream)
 
     if (frame->GetForceBlack()) {
       if (!blackImage) {
-        blackImage = aStream->mVideoOutputs[0]->GetImageContainer()->CreatePlanarYCbCrImage();
+        // Fixme: PlayVideo will be replaced in latter changeset
+        // "Call MediaStreamVideoSink::setCurrentFrames in SourceMediaStream::AppendToTrack."
+        // of this bug.
+        // This is a temp workaround to pass the build and test.
+        if (!aStream->mVideoOutputs[0]->AsVideoFrameContainer()) {
+          return;
+        }
+        blackImage = aStream->mVideoOutputs[0]->AsVideoFrameContainer()->
+          GetImageContainer()->CreatePlanarYCbCrImage();
         if (blackImage) {
           // Sets the image to a single black pixel, which will be scaled to
           // fill the rendered size.
@@ -1044,8 +1053,11 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream)
   AutoTArray<ImageContainer::NonOwningImage,4> images;
   bool haveMultipleImages = false;
 
-  for (uint32_t i = 0; i < aStream->mVideoOutputs.Length(); ++i) {
-    VideoFrameContainer* output = aStream->mVideoOutputs[i];
+  for (MediaStreamVideoSink* sink : aStream->mVideoOutputs) {
+    VideoFrameContainer* output = sink->AsVideoFrameContainer();
+    if (!output) {
+      continue;
+    }
 
     bool principalHandleChanged =
       lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
@@ -2259,55 +2271,55 @@ MediaStream::RemoveAudioOutput(void* aKey)
 }
 
 void
-MediaStream::AddVideoOutputImpl(already_AddRefed<VideoFrameContainer> aContainer)
+MediaStream::AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink)
 {
-  RefPtr<VideoFrameContainer> container = aContainer;
-  STREAM_LOG(LogLevel::Info, ("MediaStream %p Adding VideoFrameContainer %p as output",
-                              this, container.get()));
-  *mVideoOutputs.AppendElement() = container.forget();
+  RefPtr<MediaStreamVideoSink> sink = aSink;
+  STREAM_LOG(LogLevel::Info, ("MediaStream %p Adding MediaStreamVideoSink %p as output",
+                              this, sink.get()));
+  *mVideoOutputs.AppendElement() = sink.forget();
 }
 
 void
-MediaStream::RemoveVideoOutputImpl(VideoFrameContainer* aContainer)
+MediaStream::RemoveVideoOutputImpl(MediaStreamVideoSink* aSink)
 {
-  STREAM_LOG(LogLevel::Info, ("MediaStream %p Removing VideoFrameContainer %p as output",
-                              this, aContainer));
+  STREAM_LOG(LogLevel::Info, ("MediaStream %p Removing MediaStreamVideoSink %p as output",
+                              this, aSink));
   // Ensure that any frames currently queued for playback by the compositor
   // are removed.
-  aContainer->ClearFutureFrames();
-  mVideoOutputs.RemoveElement(aContainer);
+  aSink->ClearFrames();
+  mVideoOutputs.RemoveElement(aSink);
 }
 
 void
-MediaStream::AddVideoOutput(VideoFrameContainer* aContainer)
+MediaStream::AddVideoOutput(MediaStreamVideoSink* aSink)
 {
   class Message : public ControlMessage {
   public:
-    Message(MediaStream* aStream, VideoFrameContainer* aContainer) :
-      ControlMessage(aStream), mContainer(aContainer) {}
+    Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
+      ControlMessage(aStream), mSink(aSink) {}
     void Run() override
     {
-      mStream->AddVideoOutputImpl(mContainer.forget());
+      mStream->AddVideoOutputImpl(mSink.forget());
     }
-    RefPtr<VideoFrameContainer> mContainer;
+    RefPtr<MediaStreamVideoSink> mSink;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aContainer));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
 }
 
 void
-MediaStream::RemoveVideoOutput(VideoFrameContainer* aContainer)
+MediaStream::RemoveVideoOutput(MediaStreamVideoSink* aSink)
 {
   class Message : public ControlMessage {
   public:
-    Message(MediaStream* aStream, VideoFrameContainer* aContainer) :
-      ControlMessage(aStream), mContainer(aContainer) {}
+    Message(MediaStream* aStream, MediaStreamVideoSink* aSink) :
+      ControlMessage(aStream), mSink(aSink) {}
     void Run() override
     {
-      mStream->RemoveVideoOutputImpl(mContainer);
+      mStream->RemoveVideoOutputImpl(mSink);
     }
-    RefPtr<VideoFrameContainer> mContainer;
+    RefPtr<MediaStreamVideoSink> mSink;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aContainer));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aSink));
 }
 
 void

dom/media/MediaStreamGraph.h

@@ -15,8 +15,9 @@
 #include "AudioStream.h"
 #include "nsTArray.h"
 #include "nsIRunnable.h"
-#include "VideoFrameContainer.h"
 #include "VideoSegment.h"
+#include "StreamTracks.h"
+#include "MediaStreamVideoSink.h"
 #include "MainThreadUtils.h"
 #include "StreamTracks.h"
 #include "nsAutoPtr.h"
@@ -209,7 +210,7 @@ struct TrackBound
  *
  * Any stream can have its audio and video playing when requested. The media
  * stream graph plays audio by constructing audio output streams as necessary.
- * Video is played by setting video frames into an VideoFrameContainer at the right
+ * Video is played by setting video frames into an MediaStreamVideoSink at the right
  * time. To ensure video plays in sync with audio, make sure that the same
  * stream is playing both the audio and video.
  *
@@ -291,10 +292,10 @@ public:
   virtual void SetAudioOutputVolume(void* aKey, float aVolume);
   virtual void RemoveAudioOutput(void* aKey);
   // Since a stream can be played multiple ways, we need to be able to
-  // play to multiple VideoFrameContainers.
+  // play to multiple MediaStreamVideoSinks.
   // Only the first enabled video track is played.
-  virtual void AddVideoOutput(VideoFrameContainer* aContainer);
-  virtual void RemoveVideoOutput(VideoFrameContainer* aContainer);
+  virtual void AddVideoOutput(MediaStreamVideoSink* aSink);
+  virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink);
   // Explicitly suspend. Useful for example if a media element is pausing
   // and we need to stop its stream emitting its buffered data. As soon as the
   // Suspend message reaches the graph, the stream stops processing. It
@@ -423,8 +424,8 @@ public:
     return !mAudioOutputs.IsEmpty();
   }
   void RemoveAudioOutputImpl(void* aKey);
-  void AddVideoOutputImpl(already_AddRefed<VideoFrameContainer> aContainer);
-  void RemoveVideoOutputImpl(VideoFrameContainer* aContainer);
+  void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink);
+  void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink);
   void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
   void RemoveListenerImpl(MediaStreamListener* aListener);
   void RemoveAllListenersImpl();
@@ -583,7 +584,7 @@ protected:
     float mVolume;
   };
   nsTArray<AudioOutput> mAudioOutputs;
-  nsTArray<RefPtr<VideoFrameContainer>> mVideoOutputs;
+  nsTArray<RefPtr<MediaStreamVideoSink>> mVideoOutputs;
   // We record the last played video frame to avoid playing the frame again
   // with a different frame id.
   VideoFrame mLastPlayedVideoFrame;

dom/media/android/AndroidMediaReader.cpp

@@ -14,6 +14,7 @@
 #include "ImageContainer.h"
 #include "AbstractMediaDecoder.h"
 #include "gfx2DGlue.h"
+#include "VideoFrameContainer.h"
 
 namespace mozilla {
 

dom/media/gtest/TestMediaFormatReader.cpp

@@ -14,6 +14,7 @@
 #include "MP4Decoder.h"
 #include "MockMediaDecoderOwner.h"
 #include "MockMediaResource.h"
+#include "VideoFrameContainer.h"
 
 using namespace mozilla;
 using namespace mozilla::dom;

dom/media/ogg/OggReader.cpp

@@ -25,6 +25,7 @@ extern "C" {
 #include "gfx2DGlue.h"
 #include "mozilla/Telemetry.h"
 #include "nsPrintfCString.h"
+#include "VideoFrameContainer.h"
 
 using namespace mozilla::gfx;
 using namespace mozilla::media;

dom/media/omx/MediaOmxReader.cpp

@@ -17,6 +17,7 @@
 #include "MPAPI.h"
 #include "gfx2DGlue.h"
 #include "MediaStreamSource.h"
+#include "VideoFrameContainer.h"
 
 #define MAX_DROPPED_FRAMES 25
 // Try not to spend more than this much time in a single call to DecodeVideoFrame.