Bug 1774306 - Implement [Transferable] for VideoFrame r=smaug

This patch implements `Custom{Read, Write, Free}TransferHandler` for
`VideoFrame` so `VideoFrame` can be *transferred*

Differential Revision: https://phabricator.services.mozilla.com/D154677

dom/base/StructuredCloneHolder.cpp

@@ -1279,6 +1279,25 @@ StructuredCloneHolder::CustomReadTransferHandler(
                                             aReturnObject);
   }
 
+  if (StaticPrefs::dom_media_webcodecs_enabled() &&
+      aTag == SCTAG_DOM_VIDEOFRAME &&
+      CloneScope() == StructuredCloneScope::SameProcess) {
+    MOZ_ASSERT(aContent);
+    VideoFrame::TransferredData* data =
+        static_cast<VideoFrame::TransferredData*>(aContent);
+    nsCOMPtr<nsIGlobalObject> global = mGlobal;
+    RefPtr<VideoFrame> frame = VideoFrame::FromTransferred(global.get(), data);
+    delete data;
+
+    JS::Rooted<JS::Value> value(aCx);
+    if (!GetOrCreateDOMReflector(aCx, frame, &value)) {
+      JS_ClearPendingException(aCx);
+      return false;
+    }
+    aReturnObject.set(&value.toObject());
+    return true;
+  }
+
   return false;
 }
 
@@ -1357,6 +1376,27 @@ StructuredCloneHolder::CustomWriteTransferHandler(
 
         return true;
       }
+
+      if (StaticPrefs::dom_media_webcodecs_enabled()) {
+        VideoFrame* videoFrame = nullptr;
+        rv = UNWRAP_OBJECT(VideoFrame, &obj, videoFrame);
+        if (NS_SUCCEEDED(rv)) {
+          MOZ_ASSERT(videoFrame);
+
+          *aExtraData = 0;
+          *aTag = SCTAG_DOM_VIDEOFRAME;
+          *aOwnership = JS::SCTAG_TMO_CUSTOM;
+          *aContent = nullptr;
+
+          UniquePtr<VideoFrame::TransferredData> data = videoFrame->Transfer();
+          if (!data) {
+            return false;
+          }
+          *aContent = data.release();
+          MOZ_ASSERT(*aContent);
+          return true;
+        }
+      }
     }
 
     if (StaticPrefs::dom_streams_transferable_enabled()) {
@@ -1485,6 +1525,16 @@ void StructuredCloneHolder::CustomFreeTransferHandler(
     MessagePort::ForceClose(mPortIdentifiers[aExtraData + 1]);
     return;
   }
+
+  if (StaticPrefs::dom_media_webcodecs_enabled() &&
+      aTag == SCTAG_DOM_VIDEOFRAME &&
+      CloneScope() == StructuredCloneScope::SameProcess) {
+    MOZ_ASSERT(aContent);
+    VideoFrame::TransferredData* data =
+        static_cast<VideoFrame::TransferredData*>(aContent);
+    delete data;
+    return;
+  }
 }
 
 bool StructuredCloneHolder::CustomCanTransferHandler(
@@ -1560,6 +1610,15 @@ bool StructuredCloneHolder::CustomCanTransferHandler(
     }
   }
 
+  if (StaticPrefs::dom_media_webcodecs_enabled()) {
+    VideoFrame* videoframe = nullptr;
+    nsresult rv = UNWRAP_OBJECT(VideoFrame, &obj, videoframe);
+    if (NS_SUCCEEDED(rv)) {
+      SameProcessScopeRequired(aSameProcessScopeRequired);
+      return CloneScope() == StructuredCloneScope::SameProcess;
+    }
+  }
+
   return false;
 }
 

dom/media/webcodecs/VideoFrame.cpp

@@ -1142,29 +1142,9 @@ InitializeFrameWithResourceAndSize(
 }
 
 // https://w3c.github.io/webcodecs/#videoframe-initialize-frame-from-other-frame
-struct VideoFrameData {
-  VideoFrameData(layers::Image* aImage, const VideoPixelFormat& aFormat,
-                 gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize,
-                 Maybe<uint64_t> aDuration, int64_t aTimestamp,
-                 const VideoColorSpaceInit& aColorSpace)
-      : mImage(aImage),
-        mFormat(aFormat),
-        mVisibleRect(aVisibleRect),
-        mDisplaySize(aDisplaySize),
-        mDuration(aDuration),
-        mTimestamp(aTimestamp),
-        mColorSpace(aColorSpace) {}
-
-  RefPtr<layers::Image> mImage;
-  VideoFrame::Format mFormat;
-  const gfx::IntRect mVisibleRect;
-  const gfx::IntSize mDisplaySize;
-  const Maybe<uint64_t> mDuration;
-  const int64_t mTimestamp;
-  const VideoColorSpaceInit mColorSpace;
-};
 static Result<already_AddRefed<VideoFrame>, nsCString>
-InitializeFrameFromOtherFrame(nsIGlobalObject* aGlobal, VideoFrameData&& aData,
+InitializeFrameFromOtherFrame(nsIGlobalObject* aGlobal,
+                              VideoFrame::FrameData&& aData,
                               const VideoFrameInit& aInit) {
   MOZ_ASSERT(aGlobal);
   MOZ_ASSERT(aData.mImage);
@@ -1453,9 +1433,9 @@ already_AddRefed<VideoFrame> VideoFrame::Constructor(
   // TODO: Retrive/infer the duration, and colorspace.
   auto r = InitializeFrameFromOtherFrame(
       global.get(),
-      VideoFrameData(image.get(), format.ref(), image->GetPictureRect(),
-                     image->GetSize(), Nothing(),
-                     static_cast<int64_t>(aVideoElement.CurrentTime()), {}),
+      FrameData(image.get(), format.ref(), image->GetPictureRect(),
+                image->GetSize(), Nothing(),
+                static_cast<int64_t>(aVideoElement.CurrentTime()), {}),
       aInit);
   if (r.isErr()) {
     aRv.ThrowTypeError(r.unwrapErr());
@@ -1591,11 +1571,11 @@ already_AddRefed<VideoFrame> VideoFrame::Constructor(
 
   auto r = InitializeFrameFromOtherFrame(
       global.get(),
-      VideoFrameData(aVideoFrame.mResource->mImage.get(),
-                     aVideoFrame.mResource->mFormat.PixelFormat(),
-                     aVideoFrame.mVisibleRect, aVideoFrame.mDisplaySize,
-                     aVideoFrame.mDuration, aVideoFrame.mTimestamp,
-                     aVideoFrame.mColorSpace),
+      FrameData(aVideoFrame.mResource->mImage.get(),
+                aVideoFrame.mResource->mFormat.PixelFormat(),
+                aVideoFrame.mVisibleRect, aVideoFrame.mDisplaySize,
+                aVideoFrame.mDuration, aVideoFrame.mTimestamp,
+                aVideoFrame.mColorSpace),
       aInit);
   if (r.isErr()) {
     aRv.ThrowTypeError(r.unwrapErr());
@@ -2018,6 +1998,36 @@ bool VideoFrame::WriteStructuredClone(JSStructuredCloneWriter* aWriter,
       NS_WARN_IF(!JS_WriteBytes(aWriter, &colorSpaceTransfer, 1)));
 }
 
+// https://w3c.github.io/webcodecs/#ref-for-transfer-steps%E2%91%A0
+UniquePtr<VideoFrame::TransferredData> VideoFrame::Transfer() {
+  AssertIsOnOwningThread();
+
+  // TODO: Throw error if this is _detached_ instead of checking resource (bug
+  // 1774306).
+  if (!mResource) {
+    return nullptr;
+  }
+
+  Resource r = mResource.extract();
+  auto frame = MakeUnique<TransferredData>(
+      r.mImage.get(), r.mFormat.PixelFormat(), mVisibleRect, mDisplaySize,
+      mDuration, mTimestamp, mColorSpace);
+  Close();
+  return frame;
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-transfer-receiving-steps%E2%91%A0
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::FromTransferred(
+    nsIGlobalObject* aGlobal, TransferredData* aData) {
+  MOZ_ASSERT(aData);
+
+  return MakeAndAddRef<VideoFrame>(
+      aGlobal, aData->mImage, aData->mFormat.PixelFormat(),
+      aData->mImage->GetSize(), aData->mVisibleRect, aData->mDisplaySize,
+      aData->mDuration.take(), aData->mTimestamp, aData->mColorSpace);
+}
+
 /*
  * VideoFrame::Format
  *

dom/media/webcodecs/VideoFrame.h

@@ -151,6 +151,15 @@ class VideoFrame final : public nsISupports, public nsWrapperCache {
   bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
                             StructuredCloneHolder* aHolder) const;
 
+  // [Transferable] implementations: Transfer, FromTransferred
+  struct FrameData;
+  using TransferredData = FrameData;
+
+  UniquePtr<TransferredData> Transfer();
+
+  static already_AddRefed<VideoFrame> FromTransferred(nsIGlobalObject* aGlobal,
+                                                      TransferredData* aData);
+
  public:
   // A VideoPixelFormat wrapper providing utilities for VideoFrame.
   class Format final {
@@ -176,6 +185,28 @@ class VideoFrame final : public nsISupports, public nsWrapperCache {
     VideoPixelFormat mFormat;
   };
 
+  struct FrameData {
+    FrameData(layers::Image* aImage, const VideoPixelFormat& aFormat,
+              gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize,
+              Maybe<uint64_t> aDuration, int64_t aTimestamp,
+              const VideoColorSpaceInit& aColorSpace)
+        : mImage(aImage),
+          mFormat(aFormat),
+          mVisibleRect(aVisibleRect),
+          mDisplaySize(aDisplaySize),
+          mDuration(aDuration),
+          mTimestamp(aTimestamp),
+          mColorSpace(aColorSpace) {}
+
+    RefPtr<layers::Image> mImage;
+    VideoFrame::Format mFormat;
+    const gfx::IntRect mVisibleRect;
+    const gfx::IntSize mDisplaySize;
+    Maybe<uint64_t> mDuration;
+    int64_t mTimestamp;
+    const VideoColorSpaceInit mColorSpace;
+  };
+
  private:
   // VideoFrame can run on either main thread or worker thread.
   void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(VideoFrame); }

dom/webidl/VideoFrame.webidl

@@ -12,8 +12,8 @@ enum AlphaOption {
   "discard",
 };
 
-// [Serializable] is implemented without adding attribute here.
-[Exposed=(Window,DedicatedWorker) /*, Transferable (bug 1774306) */, Pref="dom.media.webcodecs.enabled"]
+// [Serializable, Transferable] are implemented without adding attributes here.
+[Exposed=(Window,DedicatedWorker), Pref="dom.media.webcodecs.enabled"]
 interface VideoFrame {
   // The constructors should be shorten to:
   //   ```

testing/web-platform/meta/webcodecs/video-frame-serialization.any.js.ini

@@ -2,8 +2,6 @@
   prefs: [dom.media.webcodecs.enabled:true]
   expected:
     if (os == "android") and fission: [OK, TIMEOUT]
-  [Verify transferring frames closes them.]
-    expected: FAIL
 
   [Verify closing a frame doesn't affect its clones.]
     expected: FAIL
@@ -13,8 +11,6 @@
   prefs: [dom.media.webcodecs.enabled:true]
   expected:
     if (os == "android") and fission: [OK, TIMEOUT]
-  [Verify transferring frames closes them.]
-    expected: FAIL
 
   [Verify closing a frame doesn't affect its clones.]
     expected: FAIL