зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1774302 - Implement [Serializable] for VideoFrame r=padenot,smaug
This patch implements `{Read, Write}StructuredClone` for `VideoFrame` so
`VideoFrame` can be *{de,}serialize*d.
Since VideoFrame serialization requires to serialize a member RefPtr
instance, the standard [Serializable] implementation is not possible.
The serialized data can be deserialized any number of times, including
zero. As a result, that RefPtr instance should be able to share its
reference and increase the ref-count any time it needs. Therefore, this
patch implements the [Serializable] functions in a custom fashion, which
storing the RefPtr instance in StructuredCloneHolder when serializing
the VideoFrame.
Depends on D153685
Differential Revision: https://phabricator.services.mozilla.com/D153686
This commit is contained in:
Chun-Min Chang
Родитель
fd67bc7796
Коммит
20c5b5125b
|
|
@ -55,6 +55,8 @@
|
|||
#include "mozilla/dom/ToJSValue.h"
|
||||
#include "mozilla/dom/TransformStream.h"
|
||||
#include "mozilla/dom/TransformStreamBinding.h"
|
||||
#include "mozilla/dom/VideoFrame.h"
|
||||
#include "mozilla/dom/VideoFrameBinding.h"
|
||||
#include "mozilla/dom/WebIDLSerializable.h"
|
||||
#include "mozilla/dom/WritableStream.h"
|
||||
#include "mozilla/dom/WritableStreamBinding.h"
|
||||
|
|
@ -397,6 +399,7 @@ void StructuredCloneHolder::Read(nsIGlobalObject* aGlobal, JSContext* aCx,
|
|||
mWasmModuleArray.Clear();
|
||||
mClonedSurfaces.Clear();
|
||||
mInputStreamArray.Clear();
|
||||
mImages.Clear();
|
||||
Clear();
|
||||
}
|
||||
}
|
||||
|
|
@ -1022,6 +1025,13 @@ JSObject* StructuredCloneHolder::CustomReadHandler(
|
|||
return ClonedErrorHolder::ReadStructuredClone(aCx, aReader, this);
|
||||
}
|
||||
|
||||
if (StaticPrefs::dom_media_webcodecs_enabled() &&
|
||||
aTag == SCTAG_DOM_VIDEOFRAME &&
|
||||
CloneScope() == StructuredCloneScope::SameProcess) {
|
||||
return VideoFrame::ReadStructuredClone(aCx, mGlobal, aReader,
|
||||
Images()[aIndex]);
|
||||
}
|
||||
|
||||
return ReadFullySerializableObjects(aCx, aReader, aTag);
|
||||
}
|
||||
|
||||
|
|
@ -1118,6 +1128,17 @@ bool StructuredCloneHolder::CustomWriteHandler(
|
|||
return false;
|
||||
}
|
||||
|
||||
// See if this is a VideoFrame object.
|
||||
if (StaticPrefs::dom_media_webcodecs_enabled()) {
|
||||
VideoFrame* videoFrame = nullptr;
|
||||
if (NS_SUCCEEDED(UNWRAP_OBJECT(VideoFrame, &obj, videoFrame))) {
|
||||
SameProcessScopeRequired(aSameProcessScopeRequired);
|
||||
return CloneScope() == StructuredCloneScope::SameProcess
|
||||
? videoFrame->WriteStructuredClone(aWriter, this)
|
||||
: false;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// We only care about streams, so ReflectorToISupportsStatic is fine.
|
||||
nsCOMPtr<nsISupports> base = xpc::ReflectorToISupportsStatic(aObj);
|
||||
|
|
|
|||
|
|
@ -208,7 +208,8 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
|
|||
// Call this method to know if this object is keeping some DOM object alive.
|
||||
bool HasClonedDOMObjects() const {
|
||||
return !mBlobImplArray.IsEmpty() || !mWasmModuleArray.IsEmpty() ||
|
||||
!mClonedSurfaces.IsEmpty() || !mInputStreamArray.IsEmpty();
|
||||
!mClonedSurfaces.IsEmpty() || !mInputStreamArray.IsEmpty() ||
|
||||
!mImages.IsEmpty();
|
||||
}
|
||||
|
||||
nsTArray<RefPtr<BlobImpl>>& BlobImpls() {
|
||||
|
|
@ -264,6 +265,8 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
|
|||
return mClonedSurfaces;
|
||||
}
|
||||
|
||||
nsTArray<RefPtr<layers::Image>>& Images() { return mImages; }
|
||||
|
||||
// Implementations of the virtual methods to allow cloning of objects which
|
||||
// JS engine itself doesn't clone.
|
||||
|
||||
|
|
@ -361,6 +364,9 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
|
|||
// instance, so no race condition will occur.
|
||||
nsTArray<RefPtr<gfx::DataSourceSurface>> mClonedSurfaces;
|
||||
|
||||
// Used for cloning VideoFrame in the structured cloning algorithm.
|
||||
nsTArray<RefPtr<layers::Image>> mImages;
|
||||
|
||||
// This raw pointer is only set within ::Read() and is unset by the end.
|
||||
nsIGlobalObject* MOZ_NON_OWNING_REF mGlobal;
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
|
||||
#include "ImageContainer.h"
|
||||
#include "VideoColorSpace.h"
|
||||
#include "js/StructuredClone.h"
|
||||
#include "mozilla/Maybe.h"
|
||||
#include "mozilla/Result.h"
|
||||
#include "mozilla/ResultVariant.h"
|
||||
|
|
@ -28,6 +29,8 @@
|
|||
#include "mozilla/dom/OffscreenCanvas.h"
|
||||
#include "mozilla/dom/Promise.h"
|
||||
#include "mozilla/dom/SVGImageElement.h"
|
||||
#include "mozilla/dom/StructuredCloneHolder.h"
|
||||
#include "mozilla/dom/StructuredCloneTags.h"
|
||||
#include "mozilla/dom/UnionTypes.h"
|
||||
#include "mozilla/gfx/2D.h"
|
||||
#include "mozilla/gfx/Swizzle.h"
|
||||
|
|
@ -1795,21 +1798,168 @@ void VideoFrame::Close() {
|
|||
|
||||
// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A0
|
||||
/* static */
|
||||
already_AddRefed<VideoFrame> VideoFrame::ReadStructuredClone(
|
||||
JSContext* aCx, nsIGlobalObject* aGlobal,
|
||||
JSStructuredCloneReader* aReader) {
|
||||
return nullptr;
|
||||
JSObject* VideoFrame::ReadStructuredClone(JSContext* aCx,
|
||||
nsIGlobalObject* aGlobal,
|
||||
JSStructuredCloneReader* aReader,
|
||||
RefPtr<layers::Image>& aImage) {
|
||||
VideoPixelFormat format;
|
||||
if (NS_WARN_IF(!JS_ReadBytes(aReader, &format, 1))) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t codedWidth = 0;
|
||||
uint32_t codedHeight = 0;
|
||||
if (NS_WARN_IF(!JS_ReadUint32Pair(aReader, &codedWidth, &codedHeight))) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t visibleX = 0;
|
||||
uint32_t visibleY = 0;
|
||||
uint32_t visibleWidth = 0;
|
||||
uint32_t visibleHeight = 0;
|
||||
if (NS_WARN_IF(!JS_ReadUint32Pair(aReader, &visibleX, &visibleY)) ||
|
||||
NS_WARN_IF(!JS_ReadUint32Pair(aReader, &visibleWidth, &visibleHeight))) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t displayWidth = 0;
|
||||
uint32_t displayHeight = 0;
|
||||
if (NS_WARN_IF(!JS_ReadUint32Pair(aReader, &displayWidth, &displayHeight))) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint8_t hasDuration = 0;
|
||||
uint32_t durationLow = 0;
|
||||
uint32_t durationHigh = 0;
|
||||
if (NS_WARN_IF(!JS_ReadBytes(aReader, &hasDuration, 1)) ||
|
||||
NS_WARN_IF(!JS_ReadUint32Pair(aReader, &durationLow, &durationHigh))) {
|
||||
return nullptr;
|
||||
}
|
||||
Maybe<uint64_t> duration =
|
||||
hasDuration ? Some(uint64_t(durationHigh) << 32 | durationLow)
|
||||
: Nothing();
|
||||
|
||||
uint8_t hasTimestamp = 0;
|
||||
uint32_t timestampLow = 0;
|
||||
uint32_t timestampHigh = 0;
|
||||
if (NS_WARN_IF(!JS_ReadBytes(aReader, &hasTimestamp, 1)) ||
|
||||
NS_WARN_IF(!JS_ReadUint32Pair(aReader, ×tampLow, ×tampHigh))) {
|
||||
return nullptr;
|
||||
}
|
||||
Maybe<uint64_t> timestamp =
|
||||
hasTimestamp ? Some(uint64_t(timestampHigh) << 32 | timestampLow)
|
||||
: Nothing();
|
||||
|
||||
uint8_t colorSpaceFullRange = 0;
|
||||
uint8_t colorSpaceMatrix = 0;
|
||||
uint8_t colorSpacePrimaries = 0;
|
||||
uint8_t colorSpaceTransfer = 0;
|
||||
if (NS_WARN_IF(!JS_ReadBytes(aReader, &colorSpaceFullRange, 1)) ||
|
||||
NS_WARN_IF(!JS_ReadBytes(aReader, &colorSpaceMatrix, 1)) ||
|
||||
NS_WARN_IF(!JS_ReadBytes(aReader, &colorSpacePrimaries, 1)) ||
|
||||
NS_WARN_IF(!JS_ReadBytes(aReader, &colorSpaceTransfer, 1))) {
|
||||
return nullptr;
|
||||
}
|
||||
VideoColorSpaceInit colorSpace{};
|
||||
if (colorSpaceFullRange < 2) {
|
||||
colorSpace.mFullRange.Construct(colorSpaceFullRange > 0);
|
||||
}
|
||||
if (colorSpaceMatrix <
|
||||
static_cast<uint8_t>(VideoMatrixCoefficients::EndGuard_)) {
|
||||
colorSpace.mMatrix.Construct(
|
||||
static_cast<VideoMatrixCoefficients>(colorSpaceMatrix));
|
||||
}
|
||||
if (colorSpacePrimaries <
|
||||
static_cast<uint8_t>(VideoColorPrimaries::EndGuard_)) {
|
||||
colorSpace.mPrimaries.Construct(
|
||||
static_cast<VideoColorPrimaries>(colorSpacePrimaries));
|
||||
}
|
||||
if (colorSpaceTransfer <
|
||||
static_cast<uint8_t>(VideoTransferCharacteristics::EndGuard_)) {
|
||||
colorSpace.mTransfer.Construct(
|
||||
static_cast<VideoTransferCharacteristics>(colorSpaceTransfer));
|
||||
}
|
||||
|
||||
RefPtr<VideoFrame> frame = MakeAndAddRef<VideoFrame>(
|
||||
aGlobal, aImage, format, gfx::IntSize(codedWidth, codedHeight),
|
||||
gfx::IntRect(visibleX, visibleY, visibleWidth, visibleHeight),
|
||||
gfx::IntSize(displayWidth, displayHeight), std::move(duration),
|
||||
std::move(timestamp), colorSpace);
|
||||
|
||||
JS::Rooted<JS::Value> value(aCx, JS::NullValue());
|
||||
if (!GetOrCreateDOMReflector(aCx, frame, &value)) {
|
||||
return nullptr;
|
||||
}
|
||||
return value.toObjectOrNull();
|
||||
}
|
||||
|
||||
// https://w3c.github.io/webcodecs/#ref-for-serialization-steps%E2%91%A0
|
||||
bool VideoFrame::WriteStructuredClone(JSContext* aCx,
|
||||
JSStructuredCloneWriter* aWriter) const {
|
||||
bool VideoFrame::WriteStructuredClone(JSStructuredCloneWriter* aWriter,
|
||||
StructuredCloneHolder* aHolder) const {
|
||||
AssertIsOnOwningThread();
|
||||
|
||||
// TODO: Throw error if this is _detached_ instead of checking resource (bug
|
||||
// 1774306).
|
||||
if (!mResource) {
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
|
||||
const uint8_t format = BitwiseCast<uint8_t>(mResource->mFormat.PixelFormat());
|
||||
|
||||
const uint32_t codedWidth = BitwiseCast<uint32_t>(mCodedSize.Width());
|
||||
const uint32_t codedHeight = BitwiseCast<uint32_t>(mCodedSize.Height());
|
||||
|
||||
const uint32_t visibleX = BitwiseCast<uint32_t>(mVisibleRect.X());
|
||||
const uint32_t visibleY = BitwiseCast<uint32_t>(mVisibleRect.Y());
|
||||
const uint32_t visibleWidth = BitwiseCast<uint32_t>(mVisibleRect.Width());
|
||||
const uint32_t visibleHeight = BitwiseCast<uint32_t>(mVisibleRect.Height());
|
||||
|
||||
const uint32_t displayWidth = BitwiseCast<uint32_t>(mDisplaySize.Width());
|
||||
const uint32_t displayHeight = BitwiseCast<uint32_t>(mDisplaySize.Height());
|
||||
|
||||
const uint8_t hasDuration = mDuration ? 1 : 0;
|
||||
const uint32_t durationLow = mDuration ? uint32_t(*mDuration) : 0;
|
||||
const uint32_t durationHigh = mDuration ? uint32_t(*mDuration >> 32) : 0;
|
||||
|
||||
const uint8_t hasTimestamp = mTimestamp ? 1 : 0;
|
||||
const uint32_t timestampLow = mTimestamp ? uint32_t(*mTimestamp) : 0;
|
||||
const uint32_t timestampHigh = mTimestamp ? uint32_t(*mTimestamp >> 32) : 0;
|
||||
|
||||
const uint8_t colorSpaceFullRange =
|
||||
mColorSpace.mFullRange.WasPassed() ? mColorSpace.mFullRange.Value() : 2;
|
||||
const uint8_t colorSpaceMatrix = BitwiseCast<uint8_t>(
|
||||
mColorSpace.mMatrix.WasPassed() ? mColorSpace.mMatrix.Value()
|
||||
: VideoMatrixCoefficients::EndGuard_);
|
||||
const uint8_t colorSpacePrimaries = BitwiseCast<uint8_t>(
|
||||
mColorSpace.mPrimaries.WasPassed() ? mColorSpace.mPrimaries.Value()
|
||||
: VideoColorPrimaries::EndGuard_);
|
||||
const uint8_t colorSpaceTransfer =
|
||||
BitwiseCast<uint8_t>(mColorSpace.mTransfer.WasPassed()
|
||||
? mColorSpace.mTransfer.Value()
|
||||
: VideoTransferCharacteristics::EndGuard_);
|
||||
|
||||
// Indexing the image and send the index to the receiver.
|
||||
const uint32_t index = aHolder->Images().Length();
|
||||
RefPtr<layers::Image> image(mResource->mImage.get());
|
||||
// The serialization is limited to the same process scope so it's ok to
|
||||
// serialize a reference instead of a copy.
|
||||
aHolder->Images().AppendElement(image.forget());
|
||||
|
||||
return !(
|
||||
NS_WARN_IF(!JS_WriteUint32Pair(aWriter, SCTAG_DOM_VIDEOFRAME, index)) ||
|
||||
NS_WARN_IF(!JS_WriteBytes(aWriter, &format, 1)) ||
|
||||
NS_WARN_IF(!JS_WriteUint32Pair(aWriter, codedWidth, codedHeight)) ||
|
||||
NS_WARN_IF(!JS_WriteUint32Pair(aWriter, visibleX, visibleY)) ||
|
||||
NS_WARN_IF(!JS_WriteUint32Pair(aWriter, visibleWidth, visibleHeight)) ||
|
||||
NS_WARN_IF(!JS_WriteUint32Pair(aWriter, displayWidth, displayHeight)) ||
|
||||
NS_WARN_IF(!JS_WriteBytes(aWriter, &hasDuration, 1)) ||
|
||||
NS_WARN_IF(!JS_WriteUint32Pair(aWriter, durationLow, durationHigh)) ||
|
||||
NS_WARN_IF(!JS_WriteBytes(aWriter, &hasTimestamp, 1)) ||
|
||||
NS_WARN_IF(!JS_WriteUint32Pair(aWriter, timestampLow, timestampHigh)) ||
|
||||
NS_WARN_IF(!JS_WriteBytes(aWriter, &colorSpaceFullRange, 1)) ||
|
||||
NS_WARN_IF(!JS_WriteBytes(aWriter, &colorSpaceMatrix, 1)) ||
|
||||
NS_WARN_IF(!JS_WriteBytes(aWriter, &colorSpacePrimaries, 1)) ||
|
||||
NS_WARN_IF(!JS_WriteBytes(aWriter, &colorSpaceTransfer, 1)));
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ class OffscreenCanvas;
|
|||
class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
|
||||
class Promise;
|
||||
class SVGImageElement;
|
||||
class StructuredCloneHolder;
|
||||
class VideoColorSpace;
|
||||
class VideoFrame;
|
||||
enum class VideoPixelFormat : uint8_t;
|
||||
|
|
@ -137,12 +138,12 @@ class VideoFrame final : public nsISupports, public nsWrapperCache {
|
|||
void Close();
|
||||
|
||||
// [Serializable] implementations: {Read, Write}StructuredClone
|
||||
static already_AddRefed<VideoFrame> ReadStructuredClone(
|
||||
JSContext* aCx, nsIGlobalObject* aGlobal,
|
||||
JSStructuredCloneReader* aReader);
|
||||
static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
|
||||
JSStructuredCloneReader* aReader,
|
||||
RefPtr<layers::Image>& aImage);
|
||||
|
||||
bool WriteStructuredClone(JSContext* aCx,
|
||||
JSStructuredCloneWriter* aWriter) const;
|
||||
bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
|
||||
StructuredCloneHolder* aHolder) const;
|
||||
|
||||
public:
|
||||
// A VideoPixelFormat wrapper providing utilities for VideoFrame.
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ enum AlphaOption {
|
|||
"discard",
|
||||
};
|
||||
|
||||
[Exposed=(Window,DedicatedWorker), Serializable /* Transferable (bug 1774306) */, Pref="dom.media.webcodecs.enabled"]
|
||||
// [Serializable] is implemented without adding attribute here.
|
||||
[Exposed=(Window,DedicatedWorker) /*, Transferable (bug 1774306) */, Pref="dom.media.webcodecs.enabled"]
|
||||
interface VideoFrame {
|
||||
// The constructors should be shorten to:
|
||||
// ```
|
||||
|
|
|
|||
|
|
@ -1,12 +1,6 @@
|
|||
[video-frame-serialization.any.worker.html]
|
||||
prefs: [dom.media.webcodecs.enabled:true]
|
||||
|
||||
[Verify posting closed frames throws.]
|
||||
expected: FAIL
|
||||
|
||||
[Verify closing frames does not propagate accross contexts.]
|
||||
expected: FAIL
|
||||
|
||||
[Verify transferring frames closes them.]
|
||||
expected: FAIL
|
||||
|
||||
|
|
@ -16,12 +10,6 @@
|
|||
expected:
|
||||
if (os == "android") and fission: [OK, TIMEOUT]
|
||||
|
||||
[Verify posting closed frames throws.]
|
||||
expected: FAIL
|
||||
|
||||
[Verify closing frames does not propagate accross contexts.]
|
||||
expected: FAIL
|
||||
|
||||
[Verify transferring frames closes them.]
|
||||
expected: FAIL
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче