Bug 1306999 - Move HTMLMediaElement's VideoFrameListener to a VideoOutput. r=jib

This allows it to intercept frames in the rendering pipe, so that we don't have
to duplicate the logic for converting VideoChunks to NonOwningImages.

Differential Revision: https://phabricator.services.mozilla.com/D33292

--HG--
extra : moz-landing-system : lando

dom/html/HTMLMediaElement.cpp

@@ -44,6 +44,7 @@
 #include "SVGObserverUtils.h"
 #include "TimeRanges.h"
 #include "VideoFrameContainer.h"
+#include "VideoOutput.h"
 #include "VideoStreamTrack.h"
 #include "base/basictypes.h"
 #include "jsapi.h"
@@ -374,11 +375,12 @@ class nsSourceErrorEventRunner : public nsMediaEvent {
  * This listener observes the first video frame to arrive with a non-empty size,
  * and calls HTMLMediaElement::UpdateInitialMediaSize() with that size.
  */
-class HTMLMediaElement::VideoFrameListener
-    : public DirectMediaStreamTrackListener {
+class HTMLMediaElement::VideoFrameListener : public VideoOutput {
  public:
-  explicit VideoFrameListener(HTMLMediaElement* aElement)
-      : mElement(aElement),
+  VideoFrameListener(HTMLMediaElement* aElement,
+                     VideoFrameContainer* aContainer)
+      : VideoOutput(aContainer, aElement->AbstractMainThread()),
+        mElement(aElement),
         mMainThreadEventTarget(aElement->MainThreadEventTarget()),
         mInitialSizeFound(false) {
     MOZ_ASSERT(NS_IsMainThread());
@@ -401,6 +403,9 @@ class HTMLMediaElement::VideoFrameListener
     mElement->UpdateInitialMediaSize(aSize);
   }
 
+  // NB that this overrides VideoOutput::NotifyRealtimeTrackData, so we can
+  // filter out all frames but the first one with a real size. This allows us to
+  // later re-use the logic in VideoOutput for rendering that frame.
   void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
                                StreamTime aTrackOffset,
                                const MediaSegment& aMedia) override {
@@ -409,7 +414,7 @@ class HTMLMediaElement::VideoFrameListener
     }
 
     if (aMedia.GetType() != MediaSegment::VIDEO) {
-      MOZ_ASSERT(false, "Should only lock on to a video track");
+      MOZ_ASSERT_UNREACHABLE("Should only lock on to a video track");
       return;
     }
 
@@ -417,6 +422,7 @@ class HTMLMediaElement::VideoFrameListener
     for (VideoSegment::ConstChunkIterator c(video); !c.IsEnded(); c.Next()) {
       if (c->mFrame.GetIntrinsicSize() != gfx::IntSize(0, 0)) {
         mInitialSizeFound = true;
+
         // This is fine to dispatch straight to main thread (instead of via
         // ...AfterStreamUpdate()) since it reflects state of the element,
         // not the stream. Events reflecting stream or track state should be
@@ -1752,7 +1758,7 @@ void HTMLMediaElement::AbortExistingLoads() {
 
   // We need to remove VideoFrameListener before VideoTracks get emptied.
   if (mVideoFrameListener) {
-    mSelectedVideoStreamTrack->RemoveDirectListener(mVideoFrameListener);
+    mSelectedVideoStreamTrack->RemoveVideoOutput(mVideoFrameListener);
     mVideoFrameListener->Forget();
     mVideoFrameListener = nullptr;
   }
@@ -2149,10 +2155,10 @@ void HTMLMediaElement::NotifyMediaTrackEnabled(MediaTrack* aTrack) {
       HTMLVideoElement* self = static_cast<HTMLVideoElement*>(this);
       if (self->VideoWidth() <= 1 && self->VideoHeight() <= 1) {
         // MediaInfo uses dummy values of 1 for width and height to
-        // mark video as valid. We need a new stream size listener
+        // mark video as valid. We need a new video frame listener
         // if size is 0x0 or 1x1.
-        mVideoFrameListener = new VideoFrameListener(this);
-        mSelectedVideoStreamTrack->AddDirectListener(mVideoFrameListener);
+        mVideoFrameListener = new VideoFrameListener(this, container);
+        mSelectedVideoStreamTrack->AddVideoOutput(mVideoFrameListener);
       }
     }
 
@@ -2207,7 +2213,7 @@ void HTMLMediaElement::NotifyMediaTrackDisabled(MediaTrack* aTrack) {
     if (mSrcStream) {
       MOZ_ASSERT(mSelectedVideoStreamTrack);
       if (mSelectedVideoStreamTrack && mVideoFrameListener) {
-        mSelectedVideoStreamTrack->RemoveDirectListener(mVideoFrameListener);
+        mSelectedVideoStreamTrack->RemoveVideoOutput(mVideoFrameListener);
         mVideoFrameListener->Forget();
         mVideoFrameListener = nullptr;
       }
@@ -4753,7 +4759,7 @@ void HTMLMediaElement::EndSrcMediaStreamPlayback() {
   if (mVideoFrameListener) {
     MOZ_ASSERT(mSelectedVideoStreamTrack);
     if (mSelectedVideoStreamTrack) {
-      mSelectedVideoStreamTrack->RemoveDirectListener(mVideoFrameListener);
+      mSelectedVideoStreamTrack->RemoveVideoOutput(mVideoFrameListener);
     }
     mVideoFrameListener->Forget();
   }
@@ -5944,7 +5950,7 @@ void HTMLMediaElement::UpdateInitialMediaSize(const nsIntSize& aSize) {
     return;
   }
 
-  mSelectedVideoStreamTrack->RemoveDirectListener(mVideoFrameListener);
+  mSelectedVideoStreamTrack->RemoveVideoOutput(mVideoFrameListener);
   mVideoFrameListener->Forget();
   mVideoFrameListener = nullptr;
 }

dom/media/VideoOutput.h

@@ -0,0 +1,205 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef VideoOutput_h
+#define VideoOutput_h
+
+#include "MediaStreamListener.h"
+#include "VideoFrameContainer.h"
+
+namespace mozilla {
+
+using layers::Image;
+using layers::ImageContainer;
+using layers::PlanarYCbCrData;
+using layers::PlanarYCbCrImage;
+
+static bool SetImageToBlackPixel(PlanarYCbCrImage* aImage) {
+  uint8_t blackPixel[] = {0x10, 0x80, 0x80};
+
+  PlanarYCbCrData data;
+  data.mYChannel = blackPixel;
+  data.mCbChannel = blackPixel + 1;
+  data.mCrChannel = blackPixel + 2;
+  data.mYStride = data.mCbCrStride = 1;
+  data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1);
+  return aImage->CopyData(data);
+}
+
+class VideoOutput : public DirectMediaStreamTrackListener {
+ protected:
+  virtual ~VideoOutput() = default;
+
+  void DropPastFrames() {
+    TimeStamp now = TimeStamp::Now();
+    size_t nrChunksInPast = 0;
+    for (const auto& idChunkPair : mFrames) {
+      const VideoChunk& chunk = idChunkPair.second();
+      if (chunk.mTimeStamp > now) {
+        break;
+      }
+      ++nrChunksInPast;
+    }
+    if (nrChunksInPast > 1) {
+      // We need to keep one frame that starts in the past, because it only ends
+      // when the next frame starts (which also needs to be in the past for it
+      // to drop).
+      mFrames.RemoveElementsAt(0, nrChunksInPast - 1);
+    }
+  }
+
+  void SendFramesEnsureLocked() {
+    mMutex.AssertCurrentThreadOwns();
+    SendFrames();
+  }
+
+  void SendFrames() {
+    DropPastFrames();
+
+    if (mFrames.IsEmpty()) {
+      return;
+    }
+
+    // Collect any new frames produced in this iteration.
+    AutoTArray<ImageContainer::NonOwningImage, 16> images;
+    PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
+
+    for (const auto& idChunkPair : mFrames) {
+      ImageContainer::FrameID frameId = idChunkPair.first();
+      const VideoChunk& chunk = idChunkPair.second();
+      const VideoFrame& frame = chunk.mFrame;
+      Image* image = frame.GetImage();
+      if (frame.GetForceBlack() || !mEnabled) {
+        if (!mBlackImage) {
+          RefPtr<Image> blackImage = mVideoFrameContainer->GetImageContainer()
+                                         ->CreatePlanarYCbCrImage();
+          if (blackImage) {
+            // Sets the image to a single black pixel, which will be scaled to
+            // fill the rendered size.
+            if (SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage())) {
+              mBlackImage = blackImage;
+            }
+          }
+        }
+        if (mBlackImage) {
+          image = mBlackImage;
+        }
+      }
+      if (!image) {
+        // We ignore null images.
+        continue;
+      }
+      images.AppendElement(
+          ImageContainer::NonOwningImage(image, chunk.mTimeStamp, frameId));
+
+      lastPrincipalHandle = chunk.GetPrincipalHandle();
+    }
+
+    if (images.IsEmpty()) {
+      // This could happen if the only images in mFrames are null. We leave the
+      // container at the current frame in this case.
+      mVideoFrameContainer->ClearFutureFrames();
+      return;
+    }
+
+    bool principalHandleChanged =
+        lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
+        lastPrincipalHandle != mVideoFrameContainer->GetLastPrincipalHandle();
+
+    if (principalHandleChanged) {
+      mVideoFrameContainer->UpdatePrincipalHandleForFrameID(
+          lastPrincipalHandle, images.LastElement().mFrameID);
+    }
+
+    mVideoFrameContainer->SetCurrentFrames(
+        mFrames[0].second().mFrame.GetIntrinsicSize(), images);
+    mMainThread->Dispatch(NewRunnableMethod("VideoFrameContainer::Invalidate",
+                                            mVideoFrameContainer,
+                                            &VideoFrameContainer::Invalidate));
+  }
+
+ public:
+  VideoOutput(VideoFrameContainer* aContainer, AbstractThread* aMainThread)
+      : mMutex("VideoOutput::mMutex"),
+        mVideoFrameContainer(aContainer),
+        mMainThread(aMainThread) {}
+  void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
+                               StreamTime aTrackOffset,
+                               const MediaSegment& aMedia) override {
+    MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
+    const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
+    MutexAutoLock lock(mMutex);
+    for (VideoSegment::ConstChunkIterator i(video); !i.IsEnded(); i.Next()) {
+      if (!mLastFrameTime.IsNull() && i->mTimeStamp < mLastFrameTime) {
+        // Time can go backwards if the source is a captured MediaDecoder and
+        // it seeks, as the previously buffered frames would stretch into the
+        // future. If this happens, we clear the buffered frames and start over.
+        mFrames.ClearAndRetainStorage();
+      }
+      mFrames.AppendElement(MakePair(mVideoFrameContainer->NewFrameID(), *i));
+      mLastFrameTime = i->mTimeStamp;
+    }
+
+    SendFramesEnsureLocked();
+  }
+  void NotifyRemoved() override {
+    // Doesn't need locking by mMutex, since the direct listener is removed from
+    // the track before we get notified.
+    if (mFrames.Length() <= 1) {
+      // The compositor has already received the last frame.
+      mFrames.ClearAndRetainStorage();
+      mVideoFrameContainer->ClearFutureFrames();
+      return;
+    }
+
+    // The compositor has multiple frames. ClearFutureFrames() would only retain
+    // the first as that's normally the current one. We however stop doing
+    // SetCurrentFrames() once we've received the last frame in a track, so
+    // there might be old frames lingering. We'll find the current one and
+    // re-send that.
+    DropPastFrames();
+    mFrames.RemoveElementsAt(1, mFrames.Length() - 1);
+    SendFrames();
+    mFrames.ClearAndRetainStorage();
+  }
+  void NotifyEnded() override {
+    // Doesn't need locking by mMutex, since for the track to end, it must have
+    // been ended by the source, meaning that the source won't append more data.
+    if (mFrames.IsEmpty()) {
+      return;
+    }
+
+    // Re-send only the last one to the compositor.
+    mFrames.RemoveElementsAt(0, mFrames.Length() - 1);
+    SendFrames();
+    mFrames.ClearAndRetainStorage();
+  }
+  void NotifyEnabledStateChanged(bool aEnabled) override {
+    MutexAutoLock lock(mMutex);
+    mEnabled = aEnabled;
+    // Since mEnabled will affect whether frames are real, or black, we assign
+    // new FrameIDs whenever this changes.
+    for (auto& idChunkPair : mFrames) {
+      idChunkPair.first() = mVideoFrameContainer->NewFrameID();
+    }
+    SendFramesEnsureLocked();
+  }
+
+  Mutex mMutex;
+  TimeStamp mLastFrameTime;
+  // Once the frame is forced to black, we initialize mBlackImage for use in any
+  // following forced-black frames.
+  RefPtr<Image> mBlackImage;
+  bool mEnabled = true;
+  // This array is accessed from both the direct video thread, and the graph
+  // thread. Protected by mMutex.
+  nsTArray<Pair<ImageContainer::FrameID, VideoChunk>> mFrames;
+  const RefPtr<VideoFrameContainer> mVideoFrameContainer;
+  const RefPtr<AbstractThread> mMainThread;
+};
+
+}  // namespace mozilla
+
+#endif  // VideoOutput_h

dom/media/VideoStreamTrack.cpp

@@ -9,199 +9,9 @@
 #include "MediaStreamListener.h"
 #include "nsContentUtils.h"
 #include "nsGlobalWindowInner.h"
-#include "VideoFrameContainer.h"
+#include "VideoOutput.h"
 
 namespace mozilla {
-
-using layers::Image;
-using layers::ImageContainer;
-using layers::PlanarYCbCrData;
-using layers::PlanarYCbCrImage;
-
-static bool SetImageToBlackPixel(PlanarYCbCrImage* aImage) {
-  uint8_t blackPixel[] = {0x10, 0x80, 0x80};
-
-  PlanarYCbCrData data;
-  data.mYChannel = blackPixel;
-  data.mCbChannel = blackPixel + 1;
-  data.mCrChannel = blackPixel + 2;
-  data.mYStride = data.mCbCrStride = 1;
-  data.mPicSize = data.mYSize = data.mCbCrSize = gfx::IntSize(1, 1);
-  return aImage->CopyData(data);
-}
-
-class VideoOutput : public DirectMediaStreamTrackListener {
- protected:
-  virtual ~VideoOutput() = default;
-
-  void DropPastFrames() {
-    TimeStamp now = TimeStamp::Now();
-    size_t nrChunksInPast = 0;
-    for (const auto& idChunkPair : mFrames) {
-      const VideoChunk& chunk = idChunkPair.second();
-      if (chunk.mTimeStamp > now) {
-        break;
-      }
-      ++nrChunksInPast;
-    }
-    if (nrChunksInPast > 1) {
-      // We need to keep one frame that starts in the past, because it only ends
-      // when the next frame starts (which also needs to be in the past for it
-      // to drop).
-      mFrames.RemoveElementsAt(0, nrChunksInPast - 1);
-    }
-  }
-
-  void SendFramesEnsureLocked() {
-    mMutex.AssertCurrentThreadOwns();
-    SendFrames();
-  }
-
-  void SendFrames() {
-    DropPastFrames();
-
-    if (mFrames.IsEmpty()) {
-      return;
-    }
-
-    // Collect any new frames produced in this iteration.
-    AutoTArray<ImageContainer::NonOwningImage, 16> images;
-    PrincipalHandle lastPrincipalHandle = PRINCIPAL_HANDLE_NONE;
-
-    for (const auto& idChunkPair : mFrames) {
-      ImageContainer::FrameID frameId = idChunkPair.first();
-      const VideoChunk& chunk = idChunkPair.second();
-      const VideoFrame& frame = chunk.mFrame;
-      Image* image = frame.GetImage();
-      if (frame.GetForceBlack() || !mEnabled) {
-        if (!mBlackImage) {
-          RefPtr<Image> blackImage = mVideoFrameContainer->GetImageContainer()
-                                         ->CreatePlanarYCbCrImage();
-          if (blackImage) {
-            // Sets the image to a single black pixel, which will be scaled to
-            // fill the rendered size.
-            if (SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage())) {
-              mBlackImage = blackImage;
-            }
-          }
-        }
-        if (mBlackImage) {
-          image = mBlackImage;
-        }
-      }
-      if (!image) {
-        // We ignore null images.
-        continue;
-      }
-      images.AppendElement(
-          ImageContainer::NonOwningImage(image, chunk.mTimeStamp, frameId));
-
-      lastPrincipalHandle = chunk.GetPrincipalHandle();
-    }
-
-    if (images.IsEmpty()) {
-      // This could happen if the only images in mFrames are null. We leave the
-      // container at the current frame in this case.
-      mVideoFrameContainer->ClearFutureFrames();
-      return;
-    }
-
-    bool principalHandleChanged =
-        lastPrincipalHandle != PRINCIPAL_HANDLE_NONE &&
-        lastPrincipalHandle != mVideoFrameContainer->GetLastPrincipalHandle();
-
-    if (principalHandleChanged) {
-      mVideoFrameContainer->UpdatePrincipalHandleForFrameID(
-          lastPrincipalHandle, images.LastElement().mFrameID);
-    }
-
-    mVideoFrameContainer->SetCurrentFrames(
-        mFrames[0].second().mFrame.GetIntrinsicSize(), images);
-    mMainThread->Dispatch(NewRunnableMethod("VideoFrameContainer::Invalidate",
-                                            mVideoFrameContainer,
-                                            &VideoFrameContainer::Invalidate));
-  }
-
- public:
-  VideoOutput(VideoFrameContainer* aContainer, AbstractThread* aMainThread)
-      : mMutex("VideoOutput::mMutex"),
-        mVideoFrameContainer(aContainer),
-        mMainThread(aMainThread) {}
-  void NotifyRealtimeTrackData(MediaStreamGraph* aGraph,
-                               StreamTime aTrackOffset,
-                               const MediaSegment& aMedia) override {
-    MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
-    const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
-    MutexAutoLock lock(mMutex);
-    for (VideoSegment::ConstChunkIterator i(video); !i.IsEnded(); i.Next()) {
-      if (!mLastFrameTime.IsNull() && i->mTimeStamp < mLastFrameTime) {
-        // Time can go backwards if the source is a captured MediaDecoder and
-        // it seeks, as the previously buffered frames would stretch into the
-        // future. If this happens, we clear the buffered frames and start over.
-        mFrames.ClearAndRetainStorage();
-      }
-      mFrames.AppendElement(MakePair(mVideoFrameContainer->NewFrameID(), *i));
-      mLastFrameTime = i->mTimeStamp;
-    }
-
-    SendFramesEnsureLocked();
-  }
-  void NotifyRemoved() override {
-    // Doesn't need locking by mMutex, since the direct listener is removed from
-    // the track before we get notified.
-    if (mFrames.Length() <= 1) {
-      // The compositor has already received the last frame.
-      mFrames.ClearAndRetainStorage();
-      mVideoFrameContainer->ClearFutureFrames();
-      return;
-    }
-
-    // The compositor has multiple frames. ClearFutureFrames() would only retain
-    // the first as that's normally the current one. We however stop doing
-    // SetCurrentFrames() once we've received the last frame in a track, so
-    // there might be old frames lingering. We'll find the current one and
-    // re-send that.
-    DropPastFrames();
-    mFrames.RemoveElementsAt(1, mFrames.Length() - 1);
-    SendFrames();
-    mFrames.ClearAndRetainStorage();
-  }
-  void NotifyEnded() override {
-    // Doesn't need locking by mMutex, since for the track to end, it must have
-    // been ended by the source, meaning that the source won't append more data.
-    if (mFrames.IsEmpty()) {
-      return;
-    }
-
-    // Re-send only the last one to the compositor.
-    mFrames.RemoveElementsAt(0, mFrames.Length() - 1);
-    SendFrames();
-    mFrames.ClearAndRetainStorage();
-  }
-  void NotifyEnabledStateChanged(bool aEnabled) override {
-    MutexAutoLock lock(mMutex);
-    mEnabled = aEnabled;
-    // Since mEnabled will affect whether frames are real, or black, we assign
-    // new FrameIDs whenever this changes.
-    for (auto& idChunkPair : mFrames) {
-      idChunkPair.first() = mVideoFrameContainer->NewFrameID();
-    }
-    SendFramesEnsureLocked();
-  }
-
-  Mutex mMutex;
-  TimeStamp mLastFrameTime;
-  // Once the frame is forced to black, we initialize mBlackImage for use in any
-  // following forced-black frames.
-  RefPtr<Image> mBlackImage;
-  bool mEnabled = true;
-  // This array is accessed from both the direct video thread, and the graph
-  // thread. Protected by mMutex.
-  nsTArray<Pair<ImageContainer::FrameID, VideoChunk>> mFrames;
-  const RefPtr<VideoFrameContainer> mVideoFrameContainer;
-  const RefPtr<AbstractThread> mMainThread;
-};
-
 namespace dom {
 
 VideoStreamTrack::VideoStreamTrack(DOMMediaStream* aStream, TrackID aTrackID,
@@ -217,18 +27,22 @@ void VideoStreamTrack::Destroy() {
 }
 
 void VideoStreamTrack::AddVideoOutput(VideoFrameContainer* aSink) {
+  auto output = MakeRefPtr<VideoOutput>(
+      aSink, nsGlobalWindowInner::Cast(GetParentObject())
+                 ->AbstractMainThreadFor(TaskCategory::Other));
+  AddVideoOutput(output);
+}
+
+void VideoStreamTrack::AddVideoOutput(VideoOutput* aOutput) {
   for (const auto& output : mVideoOutputs) {
-    if (output->mVideoFrameContainer == aSink) {
-      MOZ_ASSERT_UNREACHABLE("A VideoFrameContainer was already added");
+    if (output == aOutput) {
+      MOZ_ASSERT_UNREACHABLE("A VideoOutput was already added");
       return;
     }
   }
-  RefPtr<VideoOutput>& output =
-      *mVideoOutputs.AppendElement(MakeRefPtr<VideoOutput>(
-          aSink, nsGlobalWindowInner::Cast(GetParentObject())
-                     ->AbstractMainThreadFor(TaskCategory::Other)));
-  AddDirectListener(output);
-  AddListener(output);
+  mVideoOutputs.AppendElement(aOutput);
+  AddDirectListener(aOutput);
+  AddListener(aOutput);
 }
 
 void VideoStreamTrack::RemoveVideoOutput(VideoFrameContainer* aSink) {
@@ -241,6 +55,16 @@ void VideoStreamTrack::RemoveVideoOutput(VideoFrameContainer* aSink) {
   }
 }
 
+void VideoStreamTrack::RemoveVideoOutput(VideoOutput* aOutput) {
+  for (const auto& output : nsTArray<RefPtr<VideoOutput>>(mVideoOutputs)) {
+    if (output == aOutput) {
+      mVideoOutputs.RemoveElement(aOutput);
+      RemoveDirectListener(aOutput);
+      RemoveListener(aOutput);
+    }
+  }
+}
+
 void VideoStreamTrack::GetLabel(nsAString& aLabel, CallerType aCallerType) {
   if (nsContentUtils::ResistFingerprinting(aCallerType)) {
     aLabel.AssignLiteral("Internal Camera");

dom/media/VideoStreamTrack.h

@@ -29,7 +29,9 @@ class VideoStreamTrack : public MediaStreamTrack {
   const VideoStreamTrack* AsVideoStreamTrack() const override { return this; }
 
   void AddVideoOutput(VideoFrameContainer* aSink);
+  void AddVideoOutput(VideoOutput* aOutput);
   void RemoveVideoOutput(VideoFrameContainer* aSink);
+  void RemoveVideoOutput(VideoOutput* aOutput);
 
   // WebIDL
   void GetKind(nsAString& aKind) override { aKind.AssignLiteral("video"); }