Bug 1800882 - Implement HTMLVideoElement.requestVideoFrameCallback. r=webidl,media-playback-reviewers,emilio,ErichDonGubler,padenot

See https://wicg.github.io/video-rvfc/ for standard details.

Differential Revision: https://phabricator.services.mozilla.com/D216159

dom/base/AnimationFrameProvider.cpp

@@ -1,53 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "mozilla/dom/AnimationFrameProvider.h"
-#include "nsThreadUtils.h"
-
-namespace mozilla::dom {
-
-FrameRequest::FrameRequest(FrameRequestCallback& aCallback, uint32_t aHandle)
-    : mCallback(&aCallback), mHandle(aHandle) {
-  LogFrameRequestCallback::LogDispatch(mCallback);
-}
-
-FrameRequest::~FrameRequest() = default;
-
-nsresult FrameRequestManager::Schedule(FrameRequestCallback& aCallback,
-                                       uint32_t* aHandle) {
-  if (mCallbackCounter == UINT32_MAX) {
-    // Can't increment without overflowing; bail out
-    return NS_ERROR_NOT_AVAILABLE;
-  }
-  int32_t newHandle = ++mCallbackCounter;
-
-  mCallbacks.AppendElement(FrameRequest(aCallback, newHandle));
-
-  *aHandle = newHandle;
-  return NS_OK;
-}
-
-bool FrameRequestManager::Cancel(uint32_t aHandle) {
-  // mCallbacks is stored sorted by handle
-  if (mCallbacks.RemoveElementSorted(aHandle)) {
-    return true;
-  }
-
-  Unused << mCanceledCallbacks.put(aHandle);
-  return false;
-}
-
-void FrameRequestManager::Unlink() { mCallbacks.Clear(); }
-
-void FrameRequestManager::Traverse(nsCycleCollectionTraversalCallback& aCB) {
-  for (auto& i : mCallbacks) {
-    NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(aCB,
-                                       "FrameRequestManager::mCallbacks[i]");
-    aCB.NoteXPCOMChild(i.mCallback);
-  }
-}
-
-}  // namespace mozilla::dom

dom/base/AnimationFrameProvider.h

@@ -7,72 +7,63 @@
 #ifndef mozilla_dom_AnimationFrameProvider_h
 #define mozilla_dom_AnimationFrameProvider_h
 
+#include "MainThreadUtils.h"
+#include "mozilla/Assertions.h"
 #include "mozilla/dom/AnimationFrameProviderBinding.h"
-#include "mozilla/HashTable.h"
-#include "mozilla/RefPtr.h"
-#include "nsTArray.h"
+#include "mozilla/dom/HTMLVideoElement.h"
+#include "mozilla/dom/RequestCallbackManager.h"
 
 namespace mozilla::dom {
 
-struct FrameRequest {
-  FrameRequest(FrameRequestCallback& aCallback, uint32_t aHandle);
-  ~FrameRequest();
+using FrameRequest = RequestCallbackEntry<FrameRequestCallback>;
+using FrameRequestManagerBase = RequestCallbackManager<FrameRequestCallback>;
 
-  // Comparator operators to allow RemoveElementSorted with an
-  // integer argument on arrays of FrameRequest
-  bool operator==(uint32_t aHandle) const { return mHandle == aHandle; }
-  bool operator<(uint32_t aHandle) const { return mHandle < aHandle; }
-
-  RefPtr<FrameRequestCallback> mCallback;
-  uint32_t mHandle;
-};
-
-class FrameRequestManager {
+class FrameRequestManager final : public FrameRequestManagerBase {
  public:
   FrameRequestManager() = default;
   ~FrameRequestManager() = default;
 
-  nsresult Schedule(FrameRequestCallback& aCallback, uint32_t* aHandle);
-  bool Cancel(uint32_t aHandle);
+  using FrameRequestManagerBase::Cancel;
+  using FrameRequestManagerBase::Schedule;
+  using FrameRequestManagerBase::Take;
 
-  bool IsEmpty() const { return mCallbacks.IsEmpty(); }
-
-  bool IsCanceled(uint32_t aHandle) const {
-    return !mCanceledCallbacks.empty() && mCanceledCallbacks.has(aHandle);
+  void Schedule(HTMLVideoElement* aElement) {
+    if (!mVideoCallbacks.Contains(aElement)) {
+      mVideoCallbacks.AppendElement(aElement);
+    }
   }
 
-  void Take(nsTArray<FrameRequest>& aCallbacks) {
-    aCallbacks = std::move(mCallbacks);
-    mCanceledCallbacks.clear();
+  bool Cancel(HTMLVideoElement* aElement) {
+    return mVideoCallbacks.RemoveElement(aElement);
   }
 
-  void Unlink();
+  bool IsEmpty() const {
+    return FrameRequestManagerBase::IsEmpty() && mVideoCallbacks.IsEmpty();
+  }
 
-  void Traverse(nsCycleCollectionTraversalCallback& aCB);
+  void Take(nsTArray<RefPtr<HTMLVideoElement>>& aVideoCallbacks) {
+    MOZ_ASSERT(NS_IsMainThread());
+    aVideoCallbacks = std::move(mVideoCallbacks);
+  }
+
+  void Unlink() {
+    FrameRequestManagerBase::Unlink();
+    mVideoCallbacks.Clear();
+  }
+
+  void Traverse(nsCycleCollectionTraversalCallback& aCB) {
+    FrameRequestManagerBase::Traverse(aCB);
+    for (auto& i : mVideoCallbacks) {
+      NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(
+          aCB, "FrameRequestManager::mVideoCallbacks[i]");
+      aCB.NoteXPCOMChild(ToSupports(i));
+    }
+  }
 
  private:
-  nsTArray<FrameRequest> mCallbacks;
-
-  // The set of frame request callbacks that were canceled but which we failed
-  // to find in mFrameRequestCallbacks.
-  HashSet<uint32_t> mCanceledCallbacks;
-
-  /**
-   * The current frame request callback handle
-   */
-  uint32_t mCallbackCounter = 0;
+  nsTArray<RefPtr<HTMLVideoElement>> mVideoCallbacks;
 };
 
-inline void ImplCycleCollectionUnlink(FrameRequestManager& aField) {
-  aField.Unlink();
-}
-
-inline void ImplCycleCollectionTraverse(
-    nsCycleCollectionTraversalCallback& aCallback, FrameRequestManager& aField,
-    const char* aName, uint32_t aFlags) {
-  aField.Traverse(aCallback);
-}
-
 }  // namespace mozilla::dom
 
 #endif

dom/base/Document.cpp

@@ -7159,6 +7159,12 @@ void Document::MaybeScheduleFrameRequestCallbacks() {
   rd->EnsureFrameRequestCallbacksHappen();
 }
 
+void Document::TakeVideoFrameRequestCallbacks(
+    nsTArray<RefPtr<HTMLVideoElement>>& aVideoCallbacks) {
+  MOZ_ASSERT(aVideoCallbacks.IsEmpty());
+  mFrameRequestManager.Take(aVideoCallbacks);
+}
+
 void Document::TakeFrameRequestCallbacks(nsTArray<FrameRequest>& aCallbacks) {
   MOZ_ASSERT(aCallbacks.IsEmpty());
   mFrameRequestManager.Take(aCallbacks);
@@ -13692,6 +13698,18 @@ bool Document::IsCanceledFrameRequestCallback(uint32_t aHandle) const {
   return mFrameRequestManager.IsCanceled(aHandle);
 }
 
+void Document::ScheduleVideoFrameCallbacks(HTMLVideoElement* aElement) {
+  const bool wasEmpty = mFrameRequestManager.IsEmpty();
+  mFrameRequestManager.Schedule(aElement);
+  if (wasEmpty) {
+    MaybeScheduleFrameRequestCallbacks();
+  }
+}
+
+void Document::CancelVideoFrameCallbacks(HTMLVideoElement* aElement) {
+  mFrameRequestManager.Cancel(aElement);
+}
+
 nsresult Document::GetStateObject(JS::MutableHandle<JS::Value> aState) {
   // Get the document's current state object. This is the object backing both
   // history.state and popStateEvent.state.

dom/base/Document.h

@@ -254,6 +254,7 @@ class HTMLInputElement;
 class HTMLMetaElement;
 class HTMLDialogElement;
 class HTMLSharedElement;
+class HTMLVideoElement;
 class HTMLImageElement;
 struct LifecycleCallbackArgs;
 class Link;
@@ -3047,6 +3048,9 @@ class Document : public nsINode,
                                         uint32_t* aHandle);
   void CancelFrameRequestCallback(uint32_t aHandle);
 
+  void ScheduleVideoFrameCallbacks(HTMLVideoElement* aElement);
+  void CancelVideoFrameCallbacks(HTMLVideoElement* aElement);
+
   /**
    * Returns true if the handle refers to a callback that was canceled that
    * we did not find in our list of callbacks (e.g. because it is one of those
@@ -3054,6 +3058,13 @@ class Document : public nsINode,
    */
   bool IsCanceledFrameRequestCallback(uint32_t aHandle) const;
 
+  /**
+   * Put this document's video frame request callbacks into the provided
+   * list, and forget about them.
+   */
+  void TakeVideoFrameRequestCallbacks(
+      nsTArray<RefPtr<HTMLVideoElement>>& aVideoCallbacks);
+
   /**
    * Put this document's frame request callbacks into the provided
    * list, and forget about them.

dom/base/RequestCallbackManager.h

@@ -0,0 +1,115 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_RequestCallbackManager_h
+#define mozilla_dom_RequestCallbackManager_h
+
+#include <limits>
+#include "mozilla/HashTable.h"
+#include "mozilla/RefPtr.h"
+#include "nsTArray.h"
+#include "nsThreadUtils.h"
+
+namespace mozilla::dom {
+
+template <typename RequestCallback>
+struct RequestCallbackEntry {
+  RequestCallbackEntry(RequestCallback& aCallback, uint32_t aHandle)
+      : mCallback(&aCallback), mHandle(aHandle) {
+    LogTaskBase<RequestCallback>::LogDispatch(mCallback);
+  }
+
+  ~RequestCallbackEntry() = default;
+
+  // Comparator operators to allow RemoveElementSorted with an
+  // integer argument on arrays of RequestCallback
+  bool operator==(uint32_t aHandle) const { return mHandle == aHandle; }
+  bool operator<(uint32_t aHandle) const { return mHandle < aHandle; }
+
+  RefPtr<RequestCallback> mCallback;
+  uint32_t mHandle;
+};
+
+template <typename RequestCallback>
+class RequestCallbackManager {
+ public:
+  RequestCallbackManager() = default;
+  ~RequestCallbackManager() = default;
+
+  nsresult Schedule(RequestCallback& aCallback, uint32_t* aHandle) {
+    if (mCallbackCounter == std::numeric_limits<uint32_t>::max()) {
+      // Can't increment without overflowing; bail out
+      return NS_ERROR_NOT_AVAILABLE;
+    }
+    uint32_t newHandle = ++mCallbackCounter;
+
+    mCallbacks.AppendElement(RequestCallbackEntry(aCallback, newHandle));
+
+    *aHandle = newHandle;
+    return NS_OK;
+  }
+
+  bool Cancel(uint32_t aHandle) {
+    // mCallbacks is stored sorted by handle
+    if (mCallbacks.RemoveElementSorted(aHandle)) {
+      return true;
+    }
+
+    Unused << mCanceledCallbacks.put(aHandle);
+    return false;
+  }
+
+  bool IsEmpty() const { return mCallbacks.IsEmpty(); }
+
+  bool IsCanceled(uint32_t aHandle) const {
+    return !mCanceledCallbacks.empty() && mCanceledCallbacks.has(aHandle);
+  }
+
+  void Take(nsTArray<RequestCallbackEntry<RequestCallback>>& aCallbacks) {
+    aCallbacks = std::move(mCallbacks);
+    mCanceledCallbacks.clear();
+  }
+
+  void Unlink() { mCallbacks.Clear(); }
+
+  void Traverse(nsCycleCollectionTraversalCallback& aCB) {
+    for (auto& i : mCallbacks) {
+      NS_CYCLE_COLLECTION_NOTE_EDGE_NAME(
+          aCB, "RequestCallbackManager::mCallbacks[i]");
+      aCB.NoteXPCOMChild(i.mCallback);
+    }
+  }
+
+ private:
+  nsTArray<RequestCallbackEntry<RequestCallback>> mCallbacks;
+
+  // The set of frame request callbacks that were canceled but which we failed
+  // to find in mRequestCallbacks.
+  HashSet<uint32_t> mCanceledCallbacks;
+
+  /**
+   * The current frame request callback handle
+   */
+  uint32_t mCallbackCounter = 0;
+};
+
+template <class RequestCallback>
+inline void ImplCycleCollectionUnlink(
+    RequestCallbackManager<RequestCallback>& aField) {
+  aField.Unlink();
+}
+
+template <class RequestCallback>
+inline void ImplCycleCollectionTraverse(
+    nsCycleCollectionTraversalCallback& aCallback,
+    RequestCallbackManager<RequestCallback>& aField, const char* aName,
+    uint32_t aFlags) {
+  aField.Traverse(aCallback);
+}
+
+}  // namespace mozilla::dom
+
+#endif

dom/base/VideoFrameProvider.h

@@ -0,0 +1,28 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_VideoFrameProvider_h
+#define mozilla_dom_VideoFrameProvider_h
+
+#include "mozilla/dom/VideoFrameBinding.h"
+#include "mozilla/dom/HTMLVideoElementBinding.h"
+#include "mozilla/dom/RequestCallbackManager.h"
+
+namespace mozilla::dom {
+
+using VideoFrameRequest = RequestCallbackEntry<VideoFrameRequestCallback>;
+
+using VideoFrameRequestManager =
+    RequestCallbackManager<VideoFrameRequestCallback>;
+
+// Force instantiation.
+template void ImplCycleCollectionUnlink(VideoFrameRequestManager& aField);
+template void ImplCycleCollectionTraverse(
+    nsCycleCollectionTraversalCallback& aCallback,
+    VideoFrameRequestManager& aField, const char* aName, uint32_t aFlags);
+
+}  // namespace mozilla::dom
+#endif  // mozilla_dom_VideoFrameProvider_h

dom/base/moz.build

@@ -258,6 +258,7 @@ EXPORTS.mozilla.dom += [
     "PostMessageEvent.h",
     "ProcessMessageManager.h",
     "RadioGroupContainer.h",
+    "RequestCallbackManager.h",
     "ResizeObserver.h",
     "ResponsiveImageSelector.h",
     "SameProcessMessageQueue.h",
@@ -289,6 +290,7 @@ EXPORTS.mozilla.dom += [
     "UnbindContext.h",
     "UseCounterMetrics.h",
     "UserActivation.h",
+    "VideoFrameProvider.h",
     "ViewportMetaData.h",
     "VisualViewport.h",
     "WindowFeatures.h",
@@ -320,7 +322,6 @@ if CONFIG["COMPILE_ENVIRONMENT"]:
 UNIFIED_SOURCES += [
     "!UseCounterMetrics.cpp",
     "AbstractRange.cpp",
-    "AnimationFrameProvider.cpp",
     "AnonymousContent.cpp",
     "Attr.cpp",
     "AttrArray.cpp",

dom/canvas/test/webgl-conf/mochi-single.html

@@ -69,6 +69,7 @@ function runTest() {
 
 SpecialPowers.pushPrefEnv({"set": [
   ['gfx.offscreencanvas.enabled', true],
+  ['media.rvfc.enabled', true],
   ['webgl.force-enabled', true],
   ['webgl.enable-draft-extensions', true],
 ]}, runTest);

dom/html/HTMLMediaElement.h

@@ -1117,7 +1117,7 @@ class HTMLMediaElement : public nsGenericHTMLElement,
    * When loading a new source on an existing media element, make sure to reset
    * everything that is accessible using the media element API.
    */
-  void ResetState();
+  virtual void ResetState();
 
   /**
    * The resource-fetch algorithm step of the load algorithm.
@@ -1938,4 +1938,8 @@ bool HasDebuggerOrTabsPrivilege(JSContext* aCx, JSObject* aObj);
 
 }  // namespace mozilla::dom
 
+inline nsISupports* ToSupports(mozilla::dom::HTMLMediaElement* aElement) {
+  return static_cast<mozilla::dom::EventTarget*>(aElement);
+}
+
 #endif  // mozilla_dom_HTMLMediaElement_h

dom/html/HTMLVideoElement.cpp

@@ -79,6 +79,7 @@ NS_IMPL_ISUPPORTS_CYCLE_COLLECTION_INHERITED_0(HTMLVideoElement,
 NS_IMPL_CYCLE_COLLECTION_CLASS(HTMLVideoElement)
 
 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(HTMLVideoElement)
+  NS_IMPL_CYCLE_COLLECTION_UNLINK(mVideoFrameRequestManager)
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneTarget)
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneTargetPromise)
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mVisualCloneSource)
@@ -87,6 +88,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(HTMLMediaElement)
 
 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLVideoElement,
                                                   HTMLMediaElement)
+  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVideoFrameRequestManager)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVisualCloneTarget)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVisualCloneTargetPromise)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mVisualCloneSource)
@@ -153,6 +155,16 @@ void HTMLVideoElement::Invalidate(ImageSizeChanged aImageSizeChanged,
       container->Invalidate();
     }
   }
+
+  if (mVideoFrameRequestManager.IsEmpty()) {
+    return;
+  }
+
+  if (RefPtr<ImageContainer> imageContainer = GetImageContainer()) {
+    if (imageContainer->HasCurrentImage()) {
+      OwnerDoc()->ScheduleVideoFrameCallbacks(this);
+    }
+  }
 }
 
 bool HTMLVideoElement::ParseAttribute(int32_t aNamespaceID, nsAtom* aAttribute,
@@ -677,6 +689,109 @@ void HTMLVideoElement::OnVisibilityChange(Visibility aNewVisibility) {
   }
 }
 
+void HTMLVideoElement::ResetState() {
+  HTMLMediaElement::ResetState();
+  mLastPresentedFrameID = layers::kContainerFrameID_Invalid;
+}
+
+void HTMLVideoElement::TakeVideoFrameRequestCallbacks(
+    const TimeStamp& aNowTime, const Maybe<TimeStamp>& aNextTickTime,
+    VideoFrameCallbackMetadata& aMd, nsTArray<VideoFrameRequest>& aCallbacks) {
+  MOZ_ASSERT(aCallbacks.IsEmpty());
+
+  // Attempt to find the next image to be presented on this tick. Note that
+  // composited will be accurate only if the element is visible.
+  AutoTArray<ImageContainer::OwningImage, 4> images;
+  if (RefPtr<layers::ImageContainer> container = GetImageContainer()) {
+    container->GetCurrentImages(&images);
+  }
+
+  // If we did not find any current images, we must have fired too early, or we
+  // are in the process of shutting down. Wait for the next invalidation.
+  if (images.IsEmpty()) {
+    return;
+  }
+
+  gfx::IntSize frameSize;
+  ImageContainer::FrameID frameID = layers::kContainerFrameID_Invalid;
+  bool composited = false;
+
+  // We are guaranteed that the images are in timestamp order. It is possible we
+  // are already behind if the compositor notifications have not been processed
+  // yet, so as per the standard, this is a best effort attempt at synchronizing
+  // with the state of the GPU process.
+  for (const auto& image : images) {
+    if (image.mTimeStamp <= aNowTime) {
+      // Image should already have been composited. Because we might not be in
+      // the display list, we cannot rely upon its mComposited status, and
+      // should just assume it has indeed been composited.
+      frameSize = image.mImage->GetSize();
+      frameID = image.mFrameID;
+      composited = true;
+    } else if (!aNextTickTime || image.mTimeStamp <= aNextTickTime.ref()) {
+      // Image should be the next to be composited. mComposited will be false
+      // if the compositor hasn't rendered the frame yet or notified us of the
+      // render yet, but it is in progress. If it is true, then we know the
+      // next vsync will display the frame.
+      frameSize = image.mImage->GetSize();
+      frameID = image.mFrameID;
+      composited = false;
+    } else {
+      // Image is for a future composition.
+      break;
+    }
+  }
+
+  // If all of the available images are for future compositions, we must have
+  // fired too early. Wait for the next invalidation.
+  if (frameID == layers::kContainerFrameID_Invalid ||
+      frameID == mLastPresentedFrameID) {
+    return;
+  }
+
+  // If we have already displayed the expected frame, we need to make the
+  // display time match the presentation time to indicate it is already
+  // complete.
+  if (composited) {
+    aMd.mExpectedDisplayTime = aMd.mPresentationTime;
+  }
+
+  MOZ_ASSERT(!frameSize.IsEmpty());
+
+  aMd.mWidth = frameSize.width;
+  aMd.mHeight = frameSize.height;
+  aMd.mMediaTime = CurrentTime();
+
+  // Presented frames is a bit of a misnomer from a rendering perspective,
+  // because we still need to advance regardless of composition. Video elements
+  // that are outside of the DOM, or are not visible, still advance the video in
+  // the background, and presumably the caller still needs some way to know how
+  // many frames we have advanced.
+  aMd.mPresentedFrames = frameID;
+
+  // TODO(Bug 1908246): We should set processingDuration.
+  // TODO(Bug 1908245): We should set captureTime, receiveTime and rtpTimestamp
+  // for WebRTC.
+
+  mLastPresentedFrameID = frameID;
+  mVideoFrameRequestManager.Take(aCallbacks);
+}
+
+uint32_t HTMLVideoElement::RequestVideoFrameCallback(
+    VideoFrameRequestCallback& aCallback, ErrorResult& aRv) {
+  uint32_t handle = 0;
+  aRv = mVideoFrameRequestManager.Schedule(aCallback, &handle);
+  return handle;
+}
+
+bool HTMLVideoElement::IsVideoFrameCallbackCancelled(uint32_t aHandle) {
+  return mVideoFrameRequestManager.IsCanceled(aHandle);
+}
+
+void HTMLVideoElement::CancelVideoFrameCallback(uint32_t aHandle) {
+  mVideoFrameRequestManager.Cancel(aHandle);
+}
+
 }  // namespace mozilla::dom
 
 #undef LOG

dom/html/HTMLVideoElement.h

@@ -10,7 +10,9 @@
 #include "mozilla/Attributes.h"
 #include "mozilla/ErrorResult.h"
 #include "mozilla/dom/HTMLMediaElement.h"
+#include "mozilla/dom/VideoFrameProvider.h"
 #include "mozilla/StaticPrefs_media.h"
+#include "ImageTypes.h"
 #include "Units.h"
 
 namespace mozilla {
@@ -188,6 +190,24 @@ class HTMLVideoElement final : public HTMLMediaElement {
   // SetVisualCloneTarget() instead.
   RefPtr<HTMLVideoElement> mVisualCloneSource;
 
+ private:
+  void ResetState() override;
+
+  VideoFrameRequestManager mVideoFrameRequestManager;
+  layers::ContainerFrameID mLastPresentedFrameID =
+      layers::kContainerFrameID_Invalid;
+
+ public:
+  uint32_t RequestVideoFrameCallback(VideoFrameRequestCallback& aCallback,
+                                     ErrorResult& aRv);
+  void CancelVideoFrameCallback(uint32_t aHandle);
+  void TakeVideoFrameRequestCallbacks(const TimeStamp& aNowTime,
+                                      const Maybe<TimeStamp>& aNextTickTime,
+                                      VideoFrameCallbackMetadata& aMd,
+                                      nsTArray<VideoFrameRequest>& aCallbacks);
+  bool IsVideoFrameCallbackCancelled(uint32_t aHandle);
+
+ private:
   static void MapAttributesIntoRule(MappedDeclarationsBuilder&);
 
   static bool IsVideoStatsEnabled();

dom/media/MediaManager.h

@@ -147,7 +147,7 @@ class LocalMediaDevice final : public nsIMediaDevice {
   nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
                     const MediaEnginePrefs& aPrefs, uint64_t aWindowId,
                     const char** aOutBadConstraint);
-  void SetTrack(const RefPtr<MediaTrack>& aTrack,
+  void SetTrack(const RefPtr<mozilla::MediaTrack>& aTrack,
                 const nsMainThreadPtrHandle<nsIPrincipal>& aPrincipal);
   nsresult Start();
   nsresult Reconfigure(const dom::MediaTrackConstraints& aConstraints,

dom/webidl/HTMLVideoElement.webidl

@@ -11,13 +11,33 @@
  * and create derivative works of this document.
  */
 
+dictionary VideoFrameCallbackMetadata {
+  required DOMHighResTimeStamp presentationTime;
+  required DOMHighResTimeStamp expectedDisplayTime;
+
+  required unsigned long width;
+  required unsigned long height;
+  required double mediaTime;
+
+  required unsigned long presentedFrames;
+
+  //TODO(Bug 1908246)
+  //double processingDuration;
+
+  //TODO(Bug 1908245)
+  //DOMHighResTimeStamp captureTime;
+  //DOMHighResTimeStamp receiveTime;
+  //unsigned long rtpTimestamp;
+};
+
+callback VideoFrameRequestCallback =
+    undefined(DOMHighResTimeStamp now, VideoFrameCallbackMetadata metadata);
+
 [Exposed=Window,
- InstrumentedProps=(cancelVideoFrameCallback,
-                    onenterpictureinpicture,
+ InstrumentedProps=(onenterpictureinpicture,
                     onleavepictureinpicture,
                     playsInline,
-                    requestPictureInPicture,
-                    requestVideoFrameCallback)]
+                    requestPictureInPicture)]
 interface HTMLVideoElement : HTMLMediaElement {
   [HTMLConstructor] constructor();
 
@@ -85,3 +105,12 @@ partial interface HTMLVideoElement {
 partial interface HTMLVideoElement {
   [CEReactions, SetterThrows] attribute boolean disablePictureInPicture;
 };
+
+// https://wicg.github.io/video-rvfc
+partial interface HTMLVideoElement {
+  [Pref="media.rvfc.enabled", Throws]
+  unsigned long requestVideoFrameCallback(VideoFrameRequestCallback callback);
+
+  [Pref="media.rvfc.enabled"]
+  undefined cancelVideoFrameCallback(unsigned long handle);
+};

layout/base/nsRefreshDriver.cpp

@@ -52,6 +52,7 @@
 #include "mozilla/dom/Document.h"
 #include "mozilla/dom/DocumentTimeline.h"
 #include "mozilla/dom/DocumentInlines.h"
+#include "mozilla/dom/HTMLVideoElement.h"
 #include "nsIXULRuntime.h"
 #include "jsapi.h"
 #include "nsContentUtils.h"
@@ -2360,7 +2361,7 @@ void nsRefreshDriver::UpdateAnimationsAndSendEvents() {
   }
 }
 
-void nsRefreshDriver::RunFrameRequestCallbacks(TimeStamp aNowTime) {
+void nsRefreshDriver::RunVideoAndFrameRequestCallbacks(TimeStamp aNowTime) {
   if (!mNeedToRunFrameRequestCallbacks) {
     return;
   }
@@ -2383,17 +2384,19 @@ void nsRefreshDriver::RunFrameRequestCallbacks(TimeStamp aNowTime) {
   if (NS_WARN_IF(!mPresContext)) {
     return;
   }
-  // Grab all of our frame request callbacks up front.
+  // Grab all of our documents that can fire frame request callbacks up front.
   AutoTArray<RefPtr<Document>, 8> docs;
   auto ShouldCollect = [](const Document* aDoc) {
-    return aDoc->HasFrameRequestCallbacks() &&
-           aDoc->ShouldFireFrameRequestCallbacks();
+    return aDoc->ShouldFireFrameRequestCallbacks();
   };
   if (ShouldCollect(mPresContext->Document())) {
     docs.AppendElement(mPresContext->Document());
   }
   mPresContext->Document()->CollectDescendantDocuments(docs, ShouldCollect);
 
+  // First check for and run video frame callbacks. These can trigger new frame
+  // request callbacks that we need to handle in the following pass.
+  Maybe<TimeStamp> nextTickHint;
   for (Document* doc : docs) {
     if (!tickThrottledFrameRequests && doc->ShouldThrottleFrameRequests()) {
       // Skip throttled docs if it's not time to un-throttle them yet.
@@ -2404,10 +2407,75 @@ void nsRefreshDriver::RunFrameRequestCallbacks(TimeStamp aNowTime) {
       mNeedToRunFrameRequestCallbacks = true;
       continue;
     }
+    nsTArray<RefPtr<HTMLVideoElement>> videoElms;
+    doc->TakeVideoFrameRequestCallbacks(videoElms);
+    if (videoElms.IsEmpty()) {
+      continue;
+    }
+    if (!nextTickHint) {
+      nextTickHint = GetNextTickHint();
+    }
+    AUTO_PROFILER_TRACING_MARKER_INNERWINDOWID(
+        "Paint", "requestVideoFrame callbacks", GRAPHICS, doc->InnerWindowID());
+    DOMHighResTimeStamp timeStamp = 0;
+    DOMHighResTimeStamp nextTickTimeStamp = 0;
+    if (nsPIDOMWindowInner* innerWindow = doc->GetInnerWindow()) {
+      if (Performance* perf = innerWindow->GetPerformance()) {
+        timeStamp = perf->TimeStampToDOMHighResForRendering(aNowTime);
+        nextTickTimeStamp =
+            nextTickHint
+                ? perf->TimeStampToDOMHighResForRendering(*nextTickHint)
+                : timeStamp;
+      }
+      // else window is partially torn down already
+    }
+    for (const auto& videoElm : videoElms) {
+      nsTArray<VideoFrameRequest> callbacks;
+      VideoFrameCallbackMetadata metadata;
 
+      // Presentation time is our best estimate of when the video frame was
+      // submitted for compositing. Given that we decode frames in advance,
+      // this can be most closely estimated as the vsync time (aNowTime), as
+      // that is when the compositor samples the ImageHost to get the next
+      // frame to present.
+      metadata.mPresentationTime = timeStamp;
+
+      // Expected display time is our best estimate of when the video frame we
+      // are submitting for compositing this cycle is shown to the user's eye.
+      // This will generally be when the next vsync triggers, assuming we do
+      // not fall behind on compositing.
+      metadata.mExpectedDisplayTime = nextTickTimeStamp;
+
+      // TakeVideoFrameRequestCallbacks is responsible for populating the rest
+      // of the metadata fields. If it is not ready, or there has been no
+      // change, it will not populate metadata nor yield any callbacks.
+      videoElm->TakeVideoFrameRequestCallbacks(aNowTime, nextTickHint, metadata,
+                                               callbacks);
+
+      for (auto& callback : callbacks) {
+        if (videoElm->IsVideoFrameCallbackCancelled(callback.mHandle)) {
+          continue;
+        }
+
+        // MOZ_KnownLive is OK, because the stack array frameRequestCallbacks
+        // keeps callback alive and the mCallback strong reference can't be
+        // mutated by the call.
+        LogVideoFrameRequestCallback::Run run(callback.mCallback);
+        MOZ_KnownLive(callback.mCallback)->Call(timeStamp, metadata);
+      }
+    }
+  }
+
+  // Next check for and run frame request callbacks.
+  for (Document* doc : docs) {
+    if (!tickThrottledFrameRequests && doc->ShouldThrottleFrameRequests()) {
+      // Skip throttled docs if it's not time to un-throttle them yet.
+      MOZ_ASSERT(mNeedToRunFrameRequestCallbacks);
+      continue;
+    }
     AutoTArray<FrameRequest, 8> callbacks;
     doc->TakeFrameRequestCallbacks(callbacks);
-    if (NS_WARN_IF(callbacks.IsEmpty())) {
+    if (callbacks.IsEmpty()) {
       continue;
     }
     AUTO_PROFILER_TRACING_MARKER_INNERWINDOWID(
@@ -2666,8 +2734,9 @@ void nsRefreshDriver::Tick(VsyncId aId, TimeStamp aNowTime,
   // Step 12. For each doc of docs, run the fullscreen steps for doc.
   RunFullscreenSteps();
 
-  // Step 14. For each doc of docs, run the animation frame callbacks for doc.
-  RunFrameRequestCallbacks(aNowTime);
+  // Step 14. For each doc of docs, run the video frame callbacks and animation
+  // frame callbacks for doc.
+  RunVideoAndFrameRequestCallbacks(aNowTime);
   MaybeIncreaseMeasuredTicksSinceLoading();
 
   // Step 17. For each doc of docs, if the focused area of doc is not a

layout/base/nsRefreshDriver.h

@@ -488,7 +488,7 @@ class nsRefreshDriver final : public mozilla::layers::TransactionIdAllocator,
   void UpdateAnimationsAndSendEvents();
 
   MOZ_CAN_RUN_SCRIPT
-  void RunFrameRequestCallbacks(mozilla::TimeStamp aNowTime);
+  void RunVideoAndFrameRequestCallbacks(mozilla::TimeStamp aNowTime);
   void UpdateIntersectionObservations(mozilla::TimeStamp aNowTime);
   void UpdateRelevancyOfContentVisibilityAutoFrames();
   MOZ_CAN_RUN_SCRIPT void

modules/libpref/init/StaticPrefList.yaml

@@ -10224,6 +10224,12 @@
   value: false
   mirror: always
 
+# Whether to enable experimental requestVideoFrameCallback support
+- name: media.rvfc.enabled
+  type: bool
+  value: @IS_NIGHTLY_BUILD@
+  mirror: always
+
 # VideoSink
 - name: media.ruin-av-sync.enabled
   type: RelaxedAtomicBool

testing/web-platform/meta/html/canvas/element/manual/imagebitmap/canvas-createImageBitmap-video-resize.html.ini

@@ -1,3 +1,5 @@
+prefs: [media.rvfc.enabled:true]
+
 [canvas-createImageBitmap-video-resize.html]
   expected:
     if (os == "android") and fission: [OK, TIMEOUT]

testing/web-platform/meta/video-rvfc/__dir__.ini

@@ -1 +1,2 @@
+prefs: [media.rvfc.enabled:true]
 leak-threshold: [default:3020800]

testing/web-platform/meta/video-rvfc/idlharness.window.js.ini

@@ -1,19 +0,0 @@
-[idlharness.window.html]
-  [HTMLVideoElement interface: calling requestVideoFrameCallback(VideoFrameRequestCallback) on video with too few arguments must throw TypeError]
-    expected: FAIL
-
-  [HTMLVideoElement interface: calling cancelVideoFrameCallback(unsigned long) on video with too few arguments must throw TypeError]
-    expected: FAIL
-
-  [HTMLVideoElement interface: video must inherit property "requestVideoFrameCallback(VideoFrameRequestCallback)" with the proper type]
-    expected: FAIL
-
-  [HTMLVideoElement interface: operation requestVideoFrameCallback(VideoFrameRequestCallback)]
-    expected: FAIL
-
-  [HTMLVideoElement interface: video must inherit property "cancelVideoFrameCallback(unsigned long)" with the proper type]
-    expected: FAIL
-
-  [HTMLVideoElement interface: operation cancelVideoFrameCallback(unsigned long)]
-    expected: FAIL
-

testing/web-platform/meta/video-rvfc/request-video-frame-callback-before-xr-session.https.html.ini

@@ -1,2 +1,12 @@
 [request-video-frame-callback-before-xr-session.https.html]
-  expected: ERROR
+  [Make sure video.rVFC works during a non-immersive session - webgl]
+    expected: FAIL
+
+  [Make sure video.rVFC works during a non-immersive session - webgl2]
+    expected: FAIL
+
+  [Make sure video.rVFC works during an immersive session - webgl]
+    expected: FAIL
+
+  [Make sure video.rVFC works during an immersive session - webgl2]
+    expected: FAIL

testing/web-platform/meta/video-rvfc/request-video-frame-callback-dom.html.ini

@@ -1,11 +0,0 @@
-[request-video-frame-callback-dom.html]
-  expected:
-    if (os == "android") and not debug: [OK, CRASH]
-  [Test video.rVFC works with "display:none".]
-    expected: FAIL
-
-  [Test video.rVFC works with "visibility:hidden".]
-    expected: FAIL
-
-  [Test a video outside of the DOM can still use video.rVFC.]
-    expected: FAIL

testing/web-platform/meta/video-rvfc/request-video-frame-callback-during-xr-session.https.html.ini

@@ -1,7 +1,4 @@
 [request-video-frame-callback-during-xr-session.https.html]
-  expected:
-    if (os == "android") and debug and fission: [OK, TIMEOUT]
-    if (os == "android") and debug and not fission: [OK, TIMEOUT]
   [Make sure video.rVFC callbacks started during an immersive session continue after it ends - webgl2]
     expected: FAIL
 

testing/web-platform/meta/video-rvfc/request-video-frame-callback-parallel.html.ini

@@ -1,8 +0,0 @@
-[request-video-frame-callback-parallel.html]
-  expected:
-    if (os == "android") and fission: [TIMEOUT, OK]
-  [Test we can cancel callbacks from callbacks.]
-    expected: FAIL
-
-  [Test callbacks get the same information.]
-    expected: FAIL

testing/web-platform/meta/video-rvfc/request-video-frame-callback-repeating.html.ini

@@ -1,7 +0,0 @@
-[request-video-frame-callback-repeating.html]
-  [Test new callbacks are only called on the next frame.]
-    expected: FAIL
-
-  [Test chaining calls to video.rVFC, and verify the required parameters.]
-    expected: FAIL
-

testing/web-platform/meta/video-rvfc/request-video-frame-callback-webrtc.https.html.ini

@@ -1,7 +1,3 @@
 [request-video-frame-callback-webrtc.https.html]
-  expected:
-    if (os == "win") and not swgl and not debug and (processor == "x86_64"): [ERROR, TIMEOUT]
-    if (os == "win") and swgl: CRASH
-    ERROR
   [Test video.requestVideoFrameCallback() parameters for WebRTC applications.]
-    expected: TIMEOUT
+    expected: FAIL

testing/web-platform/meta/video-rvfc/request-video-frame-callback.html.ini

@@ -1,18 +0,0 @@
-[request-video-frame-callback.html]
-  expected:
-    if (os == "win") and debug: [OK, TIMEOUT]
-    if (os == "android") and not debug: [OK, CRASH]
-  [Test we can register a video.rVFC callback.]
-    expected: FAIL
-
-  [Test we can cancel a video.rVFC request.]
-    expected: FAIL
-
-  [Test invalid calls to the video.rVFC API.]
-    expected: FAIL
-
-  [Test video.rVFC callbacks run before window.rAF callbacks.]
-    expected: FAIL
-
-  [Test video.rVFC does not stop when switching sources.]
-    expected: FAIL

testing/web-platform/meta/webcodecs/__dir__.ini

@@ -1,4 +1,4 @@
-prefs: [dom.media.webcodecs.enabled:true, dom.media.webcodecs.image-decoder.enabled:true, media.ffmpeg.encoder.enabled:true]
+prefs: [dom.media.webcodecs.enabled:true, dom.media.webcodecs.image-decoder.enabled:true, media.ffmpeg.encoder.enabled:true, media.rvfc.enabled:true]
 tags: [webcodecs]
 disabled:
   if (os == "linux") and (bits == 32): Not implemented

testing/web-platform/meta/webcodecs/videoFrame-canvasImageSource.html.ini

@@ -1,7 +1,3 @@
 [videoFrame-canvasImageSource.html]
-  prefs: [dom.media.webcodecs.enabled:true]
-  [<video> and VideoFrame constructed VideoFrame]
-    expected: FAIL
-
   [CSSImageValue constructed VideoFrame]
     expected: FAIL

testing/web-platform/mozilla/meta/webgpu/__dir__.ini

@@ -1,7 +1,7 @@
 prefs:
-  if os == "mac": [dom.webgpu.enabled:true, dom.webgpu.workers.enabled:true, dom.webgpu.testing.assert-hardware-adapter:true]
-  if os == "windows": [dom.webgpu.enabled:true, dom.webgpu.workers.enabled:true, dom.webgpu.testing.assert-hardware-adapter:true]
-  [dom.webgpu.enabled:true, dom.webgpu.workers.enabled:true]
+  if os == "mac": [dom.webgpu.enabled:true, dom.webgpu.workers.enabled:true, dom.webgpu.testing.assert-hardware-adapter:true, media.rvfc.enabled:true]
+  if os == "windows": [dom.webgpu.enabled:true, dom.webgpu.workers.enabled:true, dom.webgpu.testing.assert-hardware-adapter:true, media.rvfc.enabled:true]
+  [dom.webgpu.enabled:true, dom.webgpu.workers.enabled:true, media.rvfc.enabled:true]
 tags: [webgpu]
 disabled:
   if release_or_beta: https://mozilla-hub.atlassian.net/browse/FFXP-223

testing/web-platform/mozilla/meta/webgpu/cts/webgpu/api/validation/gpu_external_texture_expiration/cts.https.html.ini

@@ -1,28 +1,37 @@
 [cts.https.html?q=webgpu:api,validation,gpu_external_texture_expiration:import_and_use_in_different_microtask:*]
   [:sourceType="VideoElement"]
+    expected: FAIL
 
   [:sourceType="VideoFrame"]
+    expected: FAIL
 
 
 [cts.https.html?q=webgpu:api,validation,gpu_external_texture_expiration:import_and_use_in_different_task:*]
   [:sourceType="VideoElement"]
+    expected: FAIL
 
   [:sourceType="VideoFrame"]
+    expected: FAIL
 
 
 [cts.https.html?q=webgpu:api,validation,gpu_external_texture_expiration:import_from_different_video_frame:*]
   [:]
+    expected: FAIL
 
 
 [cts.https.html?q=webgpu:api,validation,gpu_external_texture_expiration:import_multiple_times_in_same_task_scope:*]
   [:sourceType="VideoElement"]
+    expected: FAIL
 
   [:sourceType="VideoFrame"]
+    expected: FAIL
 
 
 [cts.https.html?q=webgpu:api,validation,gpu_external_texture_expiration:use_import_to_refresh:*]
   [:]
+    expected: FAIL
 
 
 [cts.https.html?q=webgpu:api,validation,gpu_external_texture_expiration:webcodec_video_frame_close_expire_immediately:*]
   [:]
+    expected: FAIL

testing/web-platform/mozilla/meta/webgpu/cts/webgpu/api/validation/state/device_lost/destroy/cts.https.html.ini

@@ -6802,12 +6802,16 @@
 
 [cts.https.html?q=webgpu:api,validation,state,device_lost,destroy:importExternalTexture:*]
   [:sourceType="VideoElement";awaitLost=false]
+    expected: FAIL
 
   [:sourceType="VideoElement";awaitLost=true]
+    expected: FAIL
 
   [:sourceType="VideoFrame";awaitLost=false]
+    expected: FAIL
 
   [:sourceType="VideoFrame";awaitLost=true]
+    expected: FAIL
 
 
 [cts.https.html?q=webgpu:api,validation,state,device_lost,destroy:queue,copyExternalImageToTexture,canvas:*]

xpcom/threads/nsThreadUtils.cpp

@@ -675,6 +675,7 @@ template class LogTaskBase<nsTimerImpl>;
 template class LogTaskBase<Task>;
 template class LogTaskBase<PresShell>;
 template class LogTaskBase<dom::FrameRequestCallback>;
+template class LogTaskBase<dom::VideoFrameRequestCallback>;
 
 MOZ_THREAD_LOCAL(nsISerialEventTarget*)
 SerialEventTargetGuard::sCurrentThreadTLS;

xpcom/threads/nsThreadUtils.h

@@ -1816,6 +1816,7 @@ class Task;  // TaskController
 class PresShell;
 namespace dom {
 class FrameRequestCallback;
+class VideoFrameRequestCallback;
 }  // namespace dom
 
 // Specialized methods must be explicitly predeclared.
@@ -1838,6 +1839,8 @@ typedef LogTaskBase<nsTimerImpl> LogTimerEvent;
 typedef LogTaskBase<Task> LogTask;
 typedef LogTaskBase<PresShell> LogPresShellObserver;
 typedef LogTaskBase<dom::FrameRequestCallback> LogFrameRequestCallback;
+typedef LogTaskBase<dom::VideoFrameRequestCallback>
+    LogVideoFrameRequestCallback;
 // If you add new types don't forget to add:
 // `template class LogTaskBase<YourType>;` to nsThreadUtils.cpp