From 2df9204445c51368a776e86ec56c2b84ec77787e Mon Sep 17 00:00:00 2001 From: Andrew Osmond Date: Wed, 24 Jul 2024 03:16:17 +0000 Subject: [PATCH] Bug 1749048 - Part 2. Implement WebCodecs image decoding API. r=webidl,media-playback-reviewers,smaug,padenot This leverages the promise based image decoding API added in bug 1901078 to remap that onto the WebCodecs specification. Differential Revision: https://phabricator.services.mozilla.com/D212835 --- dom/media/webcodecs/ImageDecoder.cpp | 1004 +++++++++++++++++ dom/media/webcodecs/ImageDecoder.h | 159 +++ .../webcodecs/ImageDecoderReadRequest.cpp | 294 +++++ dom/media/webcodecs/ImageDecoderReadRequest.h | 66 ++ dom/media/webcodecs/ImageTrack.cpp | 107 ++ dom/media/webcodecs/ImageTrack.h | 100 ++ dom/media/webcodecs/ImageTrackList.cpp | 206 ++++ dom/media/webcodecs/ImageTrackList.h | 104 ++ dom/media/webcodecs/moz.build | 7 + .../mochitest/general/test_interfaces.js | 6 + dom/webidl/ImageDecoder.webidl | 73 ++ dom/webidl/moz.build | 4 + dom/workers/test/test_worker_interfaces.js | 6 + modules/libpref/init/StaticPrefList.yaml | 6 + netwerk/protocol/http/OpaqueResponseUtils.h | 2 +- .../web-platform/meta/webcodecs/__dir__.ini | 2 +- .../webcodecs/idlharness.https.any.js.ini | 611 +--------- ...nnect-readable-stream-crash.https.html.ini | 3 - ...oder-image-orientation-none.https.html.ini | 9 +- ...coder.crossOriginIsolated.https.any.js.ini | 18 - .../webcodecs/image-decoder.https.any.js.ini | 189 +--- .../webcodecs/transfering.https.any.js.ini | 8 +- 22 files changed, 2156 insertions(+), 828 deletions(-) create mode 100644 dom/media/webcodecs/ImageDecoder.cpp create mode 100644 dom/media/webcodecs/ImageDecoder.h create mode 100644 dom/media/webcodecs/ImageDecoderReadRequest.cpp create mode 100644 dom/media/webcodecs/ImageDecoderReadRequest.h create mode 100644 dom/media/webcodecs/ImageTrack.cpp create mode 100644 dom/media/webcodecs/ImageTrack.h create mode 100644 dom/media/webcodecs/ImageTrackList.cpp create mode 100644 dom/media/webcodecs/ImageTrackList.h create mode 100644 dom/webidl/ImageDecoder.webidl delete mode 100644 testing/web-platform/meta/webcodecs/image-decoder-disconnect-readable-stream-crash.https.html.ini delete mode 100644 testing/web-platform/meta/webcodecs/image-decoder.crossOriginIsolated.https.any.js.ini diff --git a/dom/media/webcodecs/ImageDecoder.cpp b/dom/media/webcodecs/ImageDecoder.cpp new file mode 100644 index 000000000000..345bf19be389 --- /dev/null +++ b/dom/media/webcodecs/ImageDecoder.cpp @@ -0,0 +1,1004 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/ImageDecoder.h" +#include +#include +#include "ImageContainer.h" +#include "ImageDecoderReadRequest.h" +#include "MediaResult.h" +#include "mozilla/dom/ImageTrack.h" +#include "mozilla/dom/ImageTrackList.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/ReadableStream.h" +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "mozilla/image/ImageUtils.h" +#include "mozilla/image/SourceBuffer.h" +#include "mozilla/Logging.h" +#include "mozilla/StaticPrefs_dom.h" +#include "nsComponentManagerUtils.h" +#include "nsTHashSet.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +class ImageDecoder::ControlMessage { + public: + ControlMessage() = default; + virtual ~ControlMessage() = default; + + virtual ConfigureMessage* AsConfigureMessage() { return nullptr; } + virtual DecodeMetadataMessage* AsDecodeMetadataMessage() { return nullptr; } + virtual DecodeFrameMessage* AsDecodeFrameMessage() { return nullptr; } + virtual SelectTrackMessage* AsSelectTrackMessage() { return nullptr; } +}; + +class ImageDecoder::ConfigureMessage final + : public ImageDecoder::ControlMessage { + public: + explicit ConfigureMessage(ColorSpaceConversion aColorSpaceConversion) + : mColorSpaceConversion(aColorSpaceConversion) {} + + ConfigureMessage* AsConfigureMessage() override { return this; } + + const ColorSpaceConversion mColorSpaceConversion; +}; + +class ImageDecoder::DecodeMetadataMessage final + : public ImageDecoder::ControlMessage { + public: + DecodeMetadataMessage* AsDecodeMetadataMessage() override { return this; } +}; + +class ImageDecoder::DecodeFrameMessage final + : public ImageDecoder::ControlMessage { + public: + DecodeFrameMessage* AsDecodeFrameMessage() override { return this; } +}; + +class ImageDecoder::SelectTrackMessage final + : public ImageDecoder::ControlMessage { + public: + explicit SelectTrackMessage(uint32_t aSelectedTrack) + : mSelectedTrack(aSelectedTrack) {} + + SelectTrackMessage* AsSelectTrackMessage() override { return this; } + + const uint32_t mSelectedTrack; +}; + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(ImageDecoder) + +NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(ImageDecoder) + tmp->Destroy(); + NS_IMPL_CYCLE_COLLECTION_UNLINK(mParent) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mTracks) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mReadRequest) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mCompletePromise) + NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutstandingDecodes) + NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER + NS_IMPL_CYCLE_COLLECTION_UNLINK_WEAK_PTR +NS_IMPL_CYCLE_COLLECTION_UNLINK_END + +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(ImageDecoder) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mParent) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTracks) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mReadRequest) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCompletePromise) + for (uint32_t i = 0; i < tmp->mOutstandingDecodes.Length(); ++i) { + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mOutstandingDecodes[i].mPromise); + } +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(ImageDecoder) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(ImageDecoder) +NS_IMPL_CYCLE_COLLECTING_RELEASE(ImageDecoder) + +ImageDecoder::ImageDecoder(nsCOMPtr&& aParent, + const nsAString& aType) + : mParent(std::move(aParent)), + mType(aType), + mFramesTimestamp(image::FrameTimeout::Zero()) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p ImageDecoder", this)); +} + +ImageDecoder::~ImageDecoder() { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p ~ImageDecoder", this)); + Destroy(); +} + +JSObject* ImageDecoder::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + return ImageDecoder_Binding::Wrap(aCx, this, aGivenProto); +} + +void ImageDecoder::Destroy() { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, ("ImageDecoder %p Destroy", this)); + + if (mReadRequest) { + mReadRequest->Destroy(); + mReadRequest = nullptr; + } + + if (mDecoder) { + mDecoder->Destroy(); + } + + if (mTracks) { + mTracks->Destroy(); + } + + mSourceBuffer = nullptr; + mDecoder = nullptr; + mParent = nullptr; +} + +void ImageDecoder::QueueConfigureMessage( + ColorSpaceConversion aColorSpaceConversion) { + mControlMessageQueue.push( + MakeUnique(aColorSpaceConversion)); +} + +void ImageDecoder::QueueDecodeMetadataMessage() { + mControlMessageQueue.push(MakeUnique()); +} + +void ImageDecoder::QueueDecodeFrameMessage() { + mControlMessageQueue.push(MakeUnique()); +} + +void ImageDecoder::QueueSelectTrackMessage(uint32_t aSelectedIndex) { + mControlMessageQueue.push(MakeUnique(aSelectedIndex)); +} + +void ImageDecoder::ResumeControlMessageQueue() { + MOZ_ASSERT(mMessageQueueBlocked); + mMessageQueueBlocked = false; + ProcessControlMessageQueue(); +} + +void ImageDecoder::ProcessControlMessageQueue() { + while (!mMessageQueueBlocked && !mControlMessageQueue.empty()) { + auto& msg = mControlMessageQueue.front(); + auto result = MessageProcessedResult::Processed; + if (auto* submsg = msg->AsConfigureMessage()) { + result = ProcessConfigureMessage(submsg); + } else if (auto* submsg = msg->AsDecodeMetadataMessage()) { + result = ProcessDecodeMetadataMessage(submsg); + } else if (auto* submsg = msg->AsDecodeFrameMessage()) { + result = ProcessDecodeFrameMessage(submsg); + } else if (auto* submsg = msg->AsSelectTrackMessage()) { + result = ProcessSelectTrackMessage(submsg); + } else { + MOZ_ASSERT_UNREACHABLE("Unhandled control message type!"); + } + + if (result == MessageProcessedResult::NotProcessed) { + break; + } + + mControlMessageQueue.pop(); + } +} + +MessageProcessedResult ImageDecoder::ProcessConfigureMessage( + ConfigureMessage* aMsg) { + // 10.2.2. Running a control message to configure the image decoder means + // running these steps: + + // 1. Let supported be the result of running the Check Type Support algorithm + // with init.type. + // + // 2. If supported is false, run the Close ImageDecoder algorithm with a + // NotSupportedError DOMException and return "processed". + NS_ConvertUTF16toUTF8 mimeType(mType); + image::DecoderType type = image::ImageUtils::GetDecoderType(mimeType); + if (NS_WARN_IF(type == image::DecoderType::UNKNOWN)) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Initialize -- unsupported mime type '%s'", this, + mimeType.get())); + Close(MediaResult(NS_ERROR_DOM_NOT_SUPPORTED_ERR, + "Unsupported mime type"_ns)); + return MessageProcessedResult::Processed; + } + + image::SurfaceFlags surfaceFlags = image::DefaultSurfaceFlags(); + switch (aMsg->mColorSpaceConversion) { + case ColorSpaceConversion::None: + surfaceFlags |= image::SurfaceFlags::NO_COLORSPACE_CONVERSION; + break; + case ColorSpaceConversion::Default: + break; + default: + MOZ_LOG( + gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Initialize -- unsupported colorspace conversion", + this)); + Close(MediaResult(NS_ERROR_DOM_NOT_SUPPORTED_ERR, + "Unsupported colorspace conversion"_ns)); + return MessageProcessedResult::Processed; + } + + // 3. Otherwise, assign the [[codec implementation]] internal slot with an + // implementation supporting init.type + mDecoder = + image::ImageUtils::CreateDecoder(mSourceBuffer, type, surfaceFlags); + if (NS_WARN_IF(!mDecoder)) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Initialize -- failed to create platform decoder", + this)); + Close(MediaResult(NS_ERROR_DOM_NOT_SUPPORTED_ERR, + "Failed to create platform decoder"_ns)); + return MessageProcessedResult::Processed; + } + + // 4. Assign true to [[message queue blocked]]. + mMessageQueueBlocked = true; + + NS_DispatchToCurrentThread(NS_NewCancelableRunnableFunction( + "ImageDecoder::ProcessConfigureMessage", [self = RefPtr{this}] { + // 5. Enqueue the following steps to the [[codec work queue]]: + // 5.1. Configure [[codec implementation]] in accordance with the values + // given for colorSpaceConversion, desiredWidth, and desiredHeight. + // 5.2. Assign false to [[message queue blocked]]. + // 5.3. Queue a task to Process the control message queue. + self->ResumeControlMessageQueue(); + })); + + // 6. Return "processed". + return MessageProcessedResult::Processed; +} + +MessageProcessedResult ImageDecoder::ProcessDecodeMetadataMessage( + DecodeMetadataMessage* aMsg) { + // 10.2.2. Running a control message to decode track metadata means running + // these steps: + + if (!mDecoder) { + return MessageProcessedResult::Processed; + } + + // 1. Enqueue the following steps to the [[codec work queue]]: + // 1.1. Run the Establish Tracks algorithm. + mDecoder->DecodeMetadata()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = WeakPtr{this}](const image::DecodeMetadataResult& aMetadata) { + if (self) { + self->OnMetadataSuccess(aMetadata); + } + }, + [self = WeakPtr{this}](const nsresult& aErr) { + if (self) { + self->OnMetadataFailed(aErr); + } + }); + return MessageProcessedResult::Processed; +} + +MessageProcessedResult ImageDecoder::ProcessDecodeFrameMessage( + DecodeFrameMessage* aMsg) { + // 10.4.2. Running a control message to decode the image means running these + // steps: + // + // 1. Enqueue the following steps to the [[codec work queue]]: + // 1.1. Wait for [[tracks established]] to become true. + // + // 1.2. If options.completeFramesOnly is false and the image is a + // Progressive Image for which the User Agent supports progressive + // decoding, run the Decode Progressive Frame algorithm with + // options.frameIndex and promise. + // + // 1.3. Otherwise, run the Decode Complete Frame algorithm with + // options.frameIndex and promise. + NS_DispatchToCurrentThread(NS_NewCancelableRunnableFunction( + "ImageDecoder::ProcessDecodeFrameMessage", + [self = RefPtr{this}] { self->CheckOutstandingDecodes(); })); + return MessageProcessedResult::Processed; +} + +MessageProcessedResult ImageDecoder::ProcessSelectTrackMessage( + SelectTrackMessage* aMsg) { + // 10.7.2. Running a control message to update the internal selected track + // index means running these steps: + // + // 1. Enqueue the following steps to [[ImageDecoder]]'s [[codec work queue]]: + // 1.1. Assign selectedIndex to [[internal selected track index]]. + // 1.2. Remove all entries from [[progressive frame generations]]. + // + // At this time, progressive images and multi-track images are not supported. + return MessageProcessedResult::Processed; +} + +void ImageDecoder::CheckOutstandingDecodes() { + // 10.2.5. Resolve Decode (with promise and result) + + // 1. If [[closed]], abort these steps. + if (mClosed || !mTracks) { + return; + } + + ImageTrack* track = mTracks->GetDefaultTrack(); + if (!track) { + return; + } + + const uint32_t decodedFrameCount = track->DecodedFrameCount(); + const uint32_t frameCount = track->FrameCount(); + const bool frameCountComplete = track->FrameCountComplete(); + + AutoTArray resolved; + AutoTArray rejected; + uint32_t minFrameIndex = UINT32_MAX; + + // 3. Remove promise from [[pending decode promises]]. + for (uint32_t i = 0; i < mOutstandingDecodes.Length();) { + auto& decode = mOutstandingDecodes[i]; + const auto frameIndex = decode.mFrameIndex; + if (frameIndex < decodedFrameCount) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p CheckOutstandingDecodes -- resolved index %u", + this, frameIndex)); + resolved.AppendElement(std::move(decode)); + mOutstandingDecodes.RemoveElementAt(i); + } else if (frameCountComplete && frameCount <= frameIndex) { + // We have gotten the last frame from the decoder, so we must reject any + // unfulfilled requests. + MOZ_LOG(gWebCodecsLog, LogLevel::Warning, + ("ImageDecoder %p CheckOutstandingDecodes -- rejected index %u " + "out-of-bounds", + this, frameIndex)); + rejected.AppendElement(std::move(decode)); + mOutstandingDecodes.RemoveElementAt(i); + } else { + // We haven't gotten the last frame yet, so we can advance to the next + // one. + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p CheckOutstandingDecodes -- pending index %u", + this, frameIndex)); + if (frameCount > frameIndex) { + minFrameIndex = std::min(minFrameIndex, frameIndex); + } + ++i; + } + } + + if (minFrameIndex < UINT32_MAX) { + RequestDecodeFrames(minFrameIndex + 1 - decodedFrameCount); + } + + // 4. Resolve promise with result. + for (const auto& i : resolved) { + ImageDecodeResult result; + result.mImage = track->GetDecodedFrame(i.mFrameIndex); + // TODO(aosmond): progressive images + result.mComplete = true; + i.mPromise->MaybeResolve(result); + } + + for (const auto& i : rejected) { + i.mPromise->MaybeRejectWithRangeError("No more frames available"_ns); + } +} + +/* static */ bool ImageDecoder::PrefEnabled(JSContext* aCx, JSObject* aObj) { + return StaticPrefs::dom_media_webcodecs_enabled() && + StaticPrefs::dom_media_webcodecs_image_decoder_enabled(); +} + +/* static */ already_AddRefed ImageDecoder::Constructor( + const GlobalObject& aGlobal, const ImageDecoderInit& aInit, + ErrorResult& aRv) { + // 10.2.2.1. If init is not valid ImageDecoderInit, throw a TypeError. + // 10.3.1. If type is not a valid image MIME type, return false. + const auto mimeType = Substring(aInit.mType, 0, 6); + if (!mimeType.Equals(u"image/"_ns)) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder Constructor -- bad mime type")); + aRv.ThrowTypeError("Invalid MIME type, must be 'image'"); + return nullptr; + } + + RefPtr readRequest; + + if (aInit.mData.IsReadableStream()) { + const auto& stream = aInit.mData.GetAsReadableStream(); + // 10.3.2. If data is of type ReadableStream and the ReadableStream is + // disturbed or locked, return false. + if (stream->Disturbed() || stream->Locked()) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder Constructor -- bad stream")); + aRv.ThrowTypeError("ReadableStream data is disturbed and/or locked"); + return nullptr; + } + } else { + // 10.3.3. If data is of type BufferSource: + bool empty; + if (aInit.mData.IsArrayBufferView()) { + const auto& view = aInit.mData.GetAsArrayBufferView(); + empty = view.ProcessData( + [](const Span& aData, JS::AutoCheckCannotGC&&) { + return aData.IsEmpty(); + }); + } else if (aInit.mData.IsArrayBuffer()) { + const auto& buffer = aInit.mData.GetAsArrayBuffer(); + empty = buffer.ProcessData( + [](const Span& aData, JS::AutoCheckCannotGC&&) { + return aData.IsEmpty(); + }); + } else { + MOZ_ASSERT_UNREACHABLE("Unsupported data type!"); + aRv.ThrowNotSupportedError("Unsupported data type"); + return nullptr; + } + + // 10.3.3.1. If data is [detached], return false. + // 10.3.3.2. If data is empty, return false. + if (empty) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder Constructor -- detached/empty BufferSource")); + aRv.ThrowTypeError("BufferSource is detached/empty"); + return nullptr; + } + } + + // 10.3.4. If desiredWidth exists and desiredHeight does not exist, return + // false. + // 10.3.5. If desiredHeight exists and desiredWidth does not exist, return + // false. + if (aInit.mDesiredHeight.WasPassed() != aInit.mDesiredWidth.WasPassed()) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder Constructor -- both/neither desiredHeight/width " + "needed")); + aRv.ThrowTypeError( + "Both or neither of desiredHeight and desiredWidth must be passed"); + return nullptr; + } + + nsTHashSet transferSet; + for (const auto& buffer : aInit.mTransfer) { + // 10.2.2.2. If init.transfer contains more than one reference to the same + // ArrayBuffer, then throw a DataCloneError DOMException. + if (transferSet.Contains(&buffer)) { + MOZ_LOG( + gWebCodecsLog, LogLevel::Error, + ("ImageDecoder Constructor -- duplicate transferred ArrayBuffer")); + aRv.ThrowDataCloneError( + "Transfer contains duplicate ArrayBuffer objects"); + return nullptr; + } + transferSet.Insert(&buffer); + // 10.2.2.3.1. If [[Detached]] internal slot is true, then throw a + // DataCloneError DOMException. + bool empty = buffer.ProcessData( + [&](const Span& aData, JS::AutoCheckCannotGC&&) { + return aData.IsEmpty(); + }); + if (empty) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder Constructor -- empty/detached transferred " + "ArrayBuffer")); + aRv.ThrowDataCloneError( + "Transfer contains empty/detached ArrayBuffer objects"); + return nullptr; + } + } + + // 10.2.2.4. Let d be a new ImageDecoder object. In the steps below, all + // mentions of ImageDecoder members apply to d unless stated + // otherwise. + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + auto imageDecoder = MakeRefPtr(std::move(global), aInit.mType); + imageDecoder->Initialize(aGlobal, aInit, aRv); + if (NS_WARN_IF(aRv.Failed())) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder Constructor -- initialize failed")); + return nullptr; + } + + // 10.2.2.19. For each transferable in init.transfer: + // 10.2.2.19.1. Perform DetachArrayBuffer on transferable + for (const auto& buffer : aInit.mTransfer) { + JS::Rooted obj(aGlobal.Context(), buffer.Obj()); + JS::DetachArrayBuffer(aGlobal.Context(), obj); + } + + // 10.2.2.20. return d. + return imageDecoder.forget(); +} + +/* static */ already_AddRefed ImageDecoder::IsTypeSupported( + const GlobalObject& aGlobal, const nsAString& aType, ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + RefPtr promise = Promise::Create(global, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + const auto subType = Substring(aType, 0, 6); + if (!subType.Equals(u"image/"_ns)) { + promise->MaybeRejectWithTypeError("Invalid MIME type, must be 'image'"_ns); + return promise.forget(); + } + + NS_ConvertUTF16toUTF8 mimeType(aType); + image::DecoderType type = image::ImageUtils::GetDecoderType(mimeType); + promise->MaybeResolve(type != image::DecoderType::UNKNOWN); + return promise.forget(); +} + +void ImageDecoder::Initialize(const GlobalObject& aGlobal, + const ImageDecoderInit& aInit, ErrorResult& aRv) { + mCompletePromise = Promise::Create(mParent, aRv); + if (NS_WARN_IF(aRv.Failed())) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Initialize -- create promise failed", this)); + return; + } + + // 10.2.2.8. Assign [[ImageTrackList]] a new ImageTrackList initialized as + // follows: + // 10.2.2.8.1. Assign a new list to [[track list]]. + mTracks = MakeAndAddRef(mParent, this); + mTracks->Initialize(aRv); + if (NS_WARN_IF(aRv.Failed())) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Initialize -- create tracks failed", this)); + return; + } + + mSourceBuffer = MakeRefPtr(); + + const auto fnSourceBufferFromSpan = [&](const Span& aData) { + nsresult rv = mSourceBuffer->ExpectLength(aData.Length()); + if (NS_WARN_IF(NS_FAILED(rv))) { + MOZ_LOG( + gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Initialize -- failed to pre-allocate source buffer", + this)); + aRv.ThrowRangeError("Could not allocate for encoded source buffer"); + return; + } + + // 10.2.2.18.3.2. Assign a copy of init.data to [[encoded data]]. + rv = mSourceBuffer->Append(reinterpret_cast(aData.Elements()), + aData.Length()); + if (NS_WARN_IF(NS_FAILED(rv))) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Initialize -- failed to append source buffer", + this)); + aRv.ThrowRangeError("Could not allocate for encoded source buffer"); + return; + } + + mSourceBuffer->Complete(NS_OK); + + // 10.2.2.18.4. Assign true to [[complete]]. + // 10.2.2.18.5. Resolve [[completed promise]]. + OnCompleteSuccess(); + }; + + if (aInit.mData.IsReadableStream()) { + // 10.2.2.17. If init’s data member is of type ReadableStream: + const auto& stream = aInit.mData.GetAsReadableStream(); + + // 10.2.2.17.2. Assign false to [[complete]] + MOZ_ASSERT(!mComplete); + + // 10.2.2.17.5. Let reader be the result of getting a reader for data. + // 10.2.2.17.6. In parallel, perform the Fetch Stream Data Loop on d with + // reader. + mReadRequest = MakeAndAddRef(mSourceBuffer); + if (NS_WARN_IF(!mReadRequest->Initialize(aGlobal, this, stream))) { + MOZ_LOG( + gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Initialize -- create read request failed", this)); + aRv.ThrowInvalidStateError("Could not create reader for ReadableStream"); + return; + } + } else if (aInit.mData.IsArrayBufferView()) { + // 10.2.2.18.3.1. Assert that init.data is of type BufferSource. + const auto& view = aInit.mData.GetAsArrayBufferView(); + view.ProcessFixedData(fnSourceBufferFromSpan); + if (aRv.Failed()) { + return; + } + } else if (aInit.mData.IsArrayBuffer()) { + // 10.2.2.18.3.1. Assert that init.data is of type BufferSource. + const auto& buffer = aInit.mData.GetAsArrayBuffer(); + buffer.ProcessFixedData(fnSourceBufferFromSpan); + if (aRv.Failed()) { + return; + } + } else { + MOZ_ASSERT_UNREACHABLE("Unsupported data type!"); + aRv.ThrowNotSupportedError("Unsupported data type"); + return; + } + + // 10.2.2.17.3 / 10.2.2.18.6. + // Queue a control message to configure the image decoder with init. + QueueConfigureMessage(aInit.mColorSpaceConversion); + + // 10.2.10.2.2.18.7. Queue a control message to decode track metadata. + // + // Note that for readable streams it doesn't ever say to decode the metadata, + // but we can reasonably assume it means to decode the metadata in parallel + // with the reading of the stream. + QueueDecodeMetadataMessage(); + + // 10.2.2.18.8. Process the control message queue. + ProcessControlMessageQueue(); +} + +void ImageDecoder::OnSourceBufferComplete(const MediaResult& aResult) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p OnSourceBufferComplete -- success %d", this, + NS_SUCCEEDED(aResult.Code()))); + + MOZ_ASSERT(mSourceBuffer->IsComplete()); + + if (NS_WARN_IF(NS_FAILED(aResult.Code()))) { + OnCompleteFailed(aResult); + return; + } + + OnCompleteSuccess(); +} + +void ImageDecoder::OnCompleteSuccess() { + if (mComplete) { + return; + } + + // There are two conditions we need to fulfill before we are complete: + // + // 10.2.1. Internal Slots - [[complete]] + // A boolean indicating whether [[encoded data]] is completely buffered. + // + // 10.6.1. Internal Slots - [[ready promise]] + // NOTE: ImageTrack frameCount can receive subsequent updates until complete + // is true. + if (!mSourceBuffer->IsComplete() || !mHasFrameCount) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p OnCompleteSuccess -- not complete yet; " + "sourceBuffer %d, hasFrameCount %d", + this, mSourceBuffer->IsComplete(), mHasFrameCount)); + return; + } + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p OnCompleteSuccess -- complete", this)); + mComplete = true; + mCompletePromise->MaybeResolveWithUndefined(); +} + +void ImageDecoder::OnCompleteFailed(const MediaResult& aResult) { + if (mComplete) { + return; + } + + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p OnCompleteFailed -- complete", this)); + mComplete = true; + aResult.RejectTo(mCompletePromise); +} + +void ImageDecoder::OnMetadataSuccess( + const image::DecodeMetadataResult& aMetadata) { + if (mClosed || !mTracks) { + return; + } + + // 10.2.5. Establish Tracks + + // 1. Assert [[tracks established]] is false. + MOZ_ASSERT(!mTracksEstablished); + + // 2. and 3. See ImageDecoder::OnMetadataFailed. + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p OnMetadataSuccess -- %dx%d, repetitions %d, " + "animated %d, frameCount %u, frameCountComplete %d", + this, aMetadata.mWidth, aMetadata.mHeight, aMetadata.mRepetitions, + aMetadata.mAnimated, aMetadata.mFrameCount, + aMetadata.mFrameCountComplete)); + + // 4. - 9., 11. See ImageTrackList::OnMetadataSuccess + mTracks->OnMetadataSuccess(aMetadata); + + // 10. Assign true to [[tracks established]]. + mTracksEstablished = true; + + // If our encoded data comes from a ReadableStream, we may not have reached + // the end of the stream yet. As such, our frame count may be incomplete. + OnFrameCountSuccess(image::DecodeFrameCountResult{ + aMetadata.mFrameCount, aMetadata.mFrameCountComplete}); +} + +void ImageDecoder::OnMetadataFailed(const nsresult& aErr) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p OnMetadataFailed 0x%08x", this, + static_cast(aErr))); + + // 10.2.5. Establish Tracks + + // 1. Assert [[tracks established]] is false. + MOZ_ASSERT(!mTracksEstablished); + + // 2. If [[encoded data]] does not contain enough data to determine the + // number of tracks: + // 2.1. If complete is true, queue a task to run the Close ImageDecoder + // algorithm. + // 2.2. Abort these steps. + // 3. If the number of tracks is found to be 0, queue a task to run the Close + // ImageDecoder algorithm and abort these steps. + Close(MediaResult(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR, + "Metadata decoding failed"_ns)); +} + +void ImageDecoder::RequestFrameCount(uint32_t aKnownFrameCount) { + MOZ_ASSERT(!mHasFrameCount); + + if (NS_WARN_IF(!mDecoder)) { + return; + } + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p RequestFrameCount -- knownFrameCount %u", this, + aKnownFrameCount)); + mDecoder->DecodeFrameCount(aKnownFrameCount) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [self = WeakPtr{this}](const image::DecodeFrameCountResult& aResult) { + if (self) { + self->OnFrameCountSuccess(aResult); + } + }, + [self = WeakPtr{this}](const nsresult& aErr) { + if (self) { + self->OnFrameCountFailed(aErr); + } + }); +} + +void ImageDecoder::RequestDecodeFrames(uint32_t aFramesToDecode) { + if (!mDecoder || mHasFramePending) { + return; + } + + mHasFramePending = true; + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p RequestDecodeFrames -- framesToDecode %u", this, + aFramesToDecode)); + + mDecoder->DecodeFrames(aFramesToDecode) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [self = WeakPtr{this}](const image::DecodeFramesResult& aResult) { + if (self) { + self->OnDecodeFramesSuccess(aResult); + } + }, + [self = WeakPtr{this}](const nsresult& aErr) { + if (self) { + self->OnDecodeFramesFailed(aErr); + } + }); +} + +void ImageDecoder::OnFrameCountSuccess( + const image::DecodeFrameCountResult& aResult) { + if (mClosed || !mTracks) { + return; + } + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p OnFrameCountSuccess -- frameCount %u, finished %d", + this, aResult.mFrameCount, aResult.mFinished)); + + // 10.2.5. Update Tracks. + + // 1. Assert [[tracks established]] is true. + MOZ_ASSERT(mTracksEstablished); + + // 2. - 6. See ImageTrackList::OnFrameCountSuccess. + mTracks->OnFrameCountSuccess(aResult); + + if (aResult.mFinished) { + mHasFrameCount = true; + OnCompleteSuccess(); + } else { + RequestFrameCount(aResult.mFrameCount); + } + + CheckOutstandingDecodes(); +} + +void ImageDecoder::OnFrameCountFailed(const nsresult& aErr) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p OnFrameCountFailed", this)); + Close(MediaResult(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR, + "Frame count decoding failed"_ns)); +} + +void ImageDecoder::GetType(nsAString& aType) const { aType.Assign(mType); } + +already_AddRefed ImageDecoder::Decode( + const ImageDecodeOptions& aOptions, ErrorResult& aRv) { + // 10.2.4. decode(options) + + // 4. Let promise be a new Promise. + RefPtr promise = Promise::Create(mParent, aRv); + if (NS_WARN_IF(aRv.Failed())) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Decode -- create promise failed", this)); + return nullptr; + } + + // NOTE: Calling decode() on the constructed ImageDecoder will trigger a + // NotSupportedError if the User Agent does not support type. This would have + // been set in Close by ProcessConfigureMessage. + if (mTypeNotSupported) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Decode -- not supported", this)); + promise->MaybeRejectWithNotSupportedError("Unsupported MIME type"_ns); + return promise.forget(); + } + + // 1. If [[closed]] is true, return a Promise rejected with an + // InvalidStateError DOMException. + if (mClosed || !mTracks || !mDecoder) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Decode -- closed", this)); + promise->MaybeRejectWithInvalidStateError("Closed decoder"_ns); + return promise.forget(); + } + + // 2. If [[ImageTrackList]]'s [[selected index]] is '-1', return a Promise + // rejected with an InvalidStateError DOMException. + // + // This must be balanced with the fact that we might get a decode call before + // the tracks are established and we are supposed to wait. + ImageTrack* track = mTracks->GetSelectedTrack(); + if (mTracksEstablished && !track) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p Decode -- no track selected", this)); + promise->MaybeRejectWithInvalidStateError("No track selected"_ns); + return promise.forget(); + } + + // 3. If options is undefined, assign a new ImageDecodeOptions to options. + // 5. Append promise to [[pending decode promises]]. + mOutstandingDecodes.AppendElement(OutstandingDecode{ + promise, aOptions.mFrameIndex, aOptions.mCompleteFramesOnly}); + + // 6. Queue a control message to decode the image with options, and promise. + QueueDecodeFrameMessage(); + + // 7. Process the control message queue. + ProcessControlMessageQueue(); + + // 8. Return promise. + return promise.forget(); +} + +void ImageDecoder::OnDecodeFramesSuccess( + const image::DecodeFramesResult& aResult) { + // 10.2.5. Decode Complete Frame (with frameIndex and promise) + MOZ_ASSERT(mHasFramePending); + mHasFramePending = false; + + // 1. Assert that [[tracks established]] is true. + MOZ_ASSERT(mTracksEstablished); + + if (mClosed || !mTracks) { + return; + } + + ImageTrack* track = mTracks->GetDefaultTrack(); + if (NS_WARN_IF(!track)) { + MOZ_ASSERT_UNREACHABLE("Must have default track!"); + return; + } + + track->OnDecodeFramesSuccess(aResult); + + CheckOutstandingDecodes(); +} + +void ImageDecoder::OnDecodeFramesFailed(const nsresult& aErr) { + MOZ_ASSERT(mHasFramePending); + mHasFramePending = false; + + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p OnDecodeFramesFailed", this)); + + AutoTArray rejected = std::move(mOutstandingDecodes); + for (const auto& i : rejected) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoder %p OnDecodeFramesFailed -- reject index %u", this, + i.mFrameIndex)); + i.mPromise->MaybeRejectWithRangeError("No more frames available"_ns); + } +} + +void ImageDecoder::Reset(const MediaResult& aResult) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, ("ImageDecoder %p Reset", this)); + // 10.2.5. Reset ImageDecoder (with exception) + + // 1. Signal [[codec implementation]] to abort any active decoding operation. + if (mDecoder) { + mDecoder->CancelDecodeFrames(); + } + + // 2. For each decodePromise in [[pending decode promises]]: + // 2.1. Reject decodePromise with exception. + // 2.3. Remove decodePromise from [[pending decode promises]]. + AutoTArray rejected = std::move(mOutstandingDecodes); + for (const auto& i : rejected) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoder %p Reset -- reject index %u", this, i.mFrameIndex)); + aResult.RejectTo(i.mPromise); + } +} + +void ImageDecoder::Close(const MediaResult& aResult) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, ("ImageDecoder %p Close", this)); + + // 10.2.5. Algorithms - Close ImageDecoder (with exception) + mClosed = true; + mTypeNotSupported = aResult.Code() == NS_ERROR_DOM_NOT_SUPPORTED_ERR; + + // 1. Run the Reset ImageDecoder algorithm with exception. + Reset(aResult); + + // 3. Clear [[codec implementation]] and release associated system resources. + if (mDecoder) { + mDecoder->Destroy(); + } + + if (mReadRequest) { + mReadRequest->Destroy(/* aCycleCollect */ false); + mReadRequest = nullptr; + } + + mSourceBuffer = nullptr; + mDecoder = nullptr; + mType = u""_ns; + + // 4. Remove all entries from [[ImageTrackList]]. + // 5. Assign -1 to [[ImageTrackList]]'s [[selected index]]. + if (mTracks) { + mTracks->MaybeRejectReady(aResult); + mTracks->Destroy(); + } + + if (!mComplete) { + aResult.RejectTo(mCompletePromise); + mComplete = true; + } +} + +void ImageDecoder::Reset() { + Reset(MediaResult(NS_ERROR_DOM_ABORT_ERR, "Reset decoder"_ns)); +} + +void ImageDecoder::Close() { + Close(MediaResult(NS_ERROR_DOM_ABORT_ERR, "Closed decoder"_ns)); +} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/ImageDecoder.h b/dom/media/webcodecs/ImageDecoder.h new file mode 100644 index 000000000000..4c8e2e13d157 --- /dev/null +++ b/dom/media/webcodecs/ImageDecoder.h @@ -0,0 +1,159 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_ImageDecoder_h +#define mozilla_dom_ImageDecoder_h + +#include "FrameTimeout.h" +#include "mozilla/Attributes.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/NotNull.h" +#include "mozilla/WeakPtr.h" +#include "mozilla/dom/ImageDecoderBinding.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { +class MediaResult; + +namespace image { +class AnonymousDecoder; +class SourceBuffer; +enum class DecoderType; +enum class SurfaceFlags : uint8_t; +struct DecodeFramesResult; +struct DecodeFrameCountResult; +struct DecodeMetadataResult; +} // namespace image + +namespace dom { +class Promise; +struct ImageDecoderReadRequest; + +class ImageDecoder final : public nsISupports, + public nsWrapperCache, + public SupportsWeakPtr { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(ImageDecoder) + + public: + ImageDecoder(nsCOMPtr&& aParent, const nsAString& aType); + + public: + nsIGlobalObject* GetParentObject() const { return mParent; } + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static bool PrefEnabled(JSContext* aCx, JSObject* aObj); + + static already_AddRefed Constructor( + const GlobalObject& aGlobal, const ImageDecoderInit& aInit, + ErrorResult& aRv); + + static already_AddRefed IsTypeSupported(const GlobalObject& aGlobal, + const nsAString& aType, + ErrorResult& aRv); + + void GetType(nsAString& aType) const; + + bool Complete() const { return mComplete; } + + Promise* Completed() const { return mCompletePromise; } + + ImageTrackList* Tracks() const { return mTracks; } + + already_AddRefed Decode(const ImageDecodeOptions& aOptions, + ErrorResult& aRv); + + void Reset(); + + void Close(); + + void OnSourceBufferComplete(const MediaResult& aResult); + + void QueueSelectTrackMessage(uint32_t aSelectedIndex); + void ProcessControlMessageQueue(); + + private: + ~ImageDecoder(); + + class ControlMessage; + class ConfigureMessage; + class DecodeMetadataMessage; + class DecodeFrameMessage; + class SelectTrackMessage; + + std::queue> mControlMessageQueue; + bool mMessageQueueBlocked = false; + bool mTracksEstablished = false; + + struct OutstandingDecode { + RefPtr mPromise; + uint32_t mFrameIndex = 0; + bool mCompleteFramesOnly = true; + }; + + // VideoFrame can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(ImageDecoder); } + + void Initialize(const GlobalObject& aGLobal, const ImageDecoderInit& aInit, + ErrorResult& aRv); + void Destroy(); + void Reset(const MediaResult& aResult); + void Close(const MediaResult& aResult); + + void QueueConfigureMessage(ColorSpaceConversion aColorSpaceConversion); + void QueueDecodeMetadataMessage(); + void QueueDecodeFrameMessage(); + + void ResumeControlMessageQueue(); + MessageProcessedResult ProcessConfigureMessage(ConfigureMessage* aMsg); + MessageProcessedResult ProcessDecodeMetadataMessage( + DecodeMetadataMessage* aMsg); + MessageProcessedResult ProcessDecodeFrameMessage(DecodeFrameMessage* aMsg); + MessageProcessedResult ProcessSelectTrackMessage(SelectTrackMessage* aMsg); + + void CheckOutstandingDecodes(); + + void OnCompleteSuccess(); + void OnCompleteFailed(const MediaResult& aResult); + + void OnMetadataSuccess(const image::DecodeMetadataResult& aMetadata); + void OnMetadataFailed(const nsresult& aErr); + + void RequestFrameCount(uint32_t aKnownFrameCount); + void OnFrameCountSuccess(const image::DecodeFrameCountResult& aResult); + void OnFrameCountFailed(const nsresult& aErr); + + void RequestDecodeFrames(uint32_t aFramesToDecode); + void OnDecodeFramesSuccess(const image::DecodeFramesResult& aResult); + void OnDecodeFramesFailed(const nsresult& aErr); + + nsCOMPtr mParent; + RefPtr mTracks; + RefPtr mReadRequest; + RefPtr mCompletePromise; + RefPtr mSourceBuffer; + RefPtr mDecoder; + AutoTArray mOutstandingDecodes; + nsAutoString mType; + image::FrameTimeout mFramesTimestamp; + bool mComplete = false; + bool mHasFrameCount = false; + bool mHasFramePending = false; + bool mTypeNotSupported = false; + bool mClosed = false; +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_ImageDecoder_h diff --git a/dom/media/webcodecs/ImageDecoderReadRequest.cpp b/dom/media/webcodecs/ImageDecoderReadRequest.cpp new file mode 100644 index 000000000000..5ce1c04fd277 --- /dev/null +++ b/dom/media/webcodecs/ImageDecoderReadRequest.cpp @@ -0,0 +1,294 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ImageDecoderReadRequest.h" +#include "MediaResult.h" +#include "mozilla/CycleCollectedJSContext.h" +#include "mozilla/dom/ImageDecoder.h" +#include "mozilla/dom/ReadableStream.h" +#include "mozilla/dom/ReadableStreamDefaultReader.h" +#include "mozilla/dom/WorkerCommon.h" +#include "mozilla/dom/WorkerRef.h" +#include "mozilla/image/SourceBuffer.h" +#include "mozilla/Logging.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_INHERITED(ImageDecoderReadRequest, ReadRequest, + mDecoder, mReader) +NS_IMPL_ADDREF_INHERITED(ImageDecoderReadRequest, ReadRequest) +NS_IMPL_RELEASE_INHERITED(ImageDecoderReadRequest, ReadRequest) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(ImageDecoderReadRequest) +NS_INTERFACE_MAP_END_INHERITING(ReadRequest) + +ImageDecoderReadRequest::ImageDecoderReadRequest( + image::SourceBuffer* aSourceBuffer) + : mSourceBuffer(std::move(aSourceBuffer)) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p ImageDecoderReadRequest", this)); +} + +ImageDecoderReadRequest::~ImageDecoderReadRequest() { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p ~ImageDecoderReadRequest", this)); +} + +bool ImageDecoderReadRequest::Initialize(const GlobalObject& aGlobal, + ImageDecoder* aDecoder, + ReadableStream& aStream) { + if (WorkerPrivate* wp = GetCurrentThreadWorkerPrivate()) { + mWorkerRef = WeakWorkerRef::Create(wp, [self = RefPtr{this}]() { + self->Destroy(/* aCycleCollect */ false); + }); + if (NS_WARN_IF(!mWorkerRef)) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoderReadRequest %p Initialize -- cannot get worker ref", + this)); + mSourceBuffer->Complete(NS_ERROR_FAILURE); + Destroy(/* aCycleCollect */ false); + return false; + } + } + + IgnoredErrorResult rv; + mReader = aStream.GetReader(rv); + if (NS_WARN_IF(rv.Failed())) { + MOZ_LOG( + gWebCodecsLog, LogLevel::Error, + ("ImageDecoderReadRequest %p Initialize -- cannot get stream reader", + this)); + mSourceBuffer->Complete(NS_ERROR_FAILURE); + Destroy(/* aCycleCollect */ false); + return false; + } + + mDecoder = aDecoder; + QueueRead(); + return true; +} + +void ImageDecoderReadRequest::Destroy(bool aCycleCollect /* = true */) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p Destroy", this)); + + if (!aCycleCollect) { + // Ensure we stop reading from the ReadableStream. + Cancel(); + } + + if (mSourceBuffer) { + if (!mSourceBuffer->IsComplete()) { + mSourceBuffer->Complete(NS_ERROR_ABORT); + } + mSourceBuffer = nullptr; + } + + mDecoder = nullptr; + mReader = nullptr; +} + +void ImageDecoderReadRequest::QueueRead() { + class ReadRunnable final : public CancelableRunnable { + public: + explicit ReadRunnable(ImageDecoderReadRequest* aOwner) + : CancelableRunnable( + "mozilla::dom::ImageDecoderReadRequest::QueueRead"), + mOwner(aOwner) {} + + NS_IMETHODIMP Run() override { + mOwner->Read(); + mOwner = nullptr; + return NS_OK; + } + + nsresult Cancel() override { + mOwner->Complete( + MediaResult(NS_ERROR_DOM_MEDIA_ABORT_ERR, "Read cancelled"_ns)); + mOwner = nullptr; + return NS_OK; + } + + private: + virtual ~ReadRunnable() { + if (mOwner) { + Cancel(); + } + } + + RefPtr mOwner; + }; + + if (!mReader) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p QueueRead -- destroyed", this)); + return; + } + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p QueueRead -- queue", this)); + auto task = MakeRefPtr(this); + NS_DispatchToCurrentThread(task.forget()); +} + +void ImageDecoderReadRequest::Read() { + if (!mReader || !mDecoder) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p Read -- destroyed", this)); + return; + } + + AutoJSAPI jsapi; + if (!jsapi.Init(mDecoder->GetParentObject())) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p Read -- no jsapi", this)); + Complete(MediaResult(NS_ERROR_DOM_FILE_NOT_READABLE_ERR, + "Reader cannot init jsapi"_ns)); + return; + } + + RefPtr self(this); + RefPtr reader(mReader); + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p Read -- begin read chunk", this)); + + IgnoredErrorResult err; + reader->ReadChunk(jsapi.cx(), *self, err); + if (NS_WARN_IF(err.Failed())) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoderReadRequest %p Read -- read chunk failed", this)); + Complete(MediaResult(NS_ERROR_DOM_FILE_NOT_READABLE_ERR, + "Reader cannot read chunk from stream"_ns)); + } + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p Read -- end read chunk", this)); +} + +void ImageDecoderReadRequest::Cancel() { + RefPtr reader = std::move(mReader); + if (!reader || !mDecoder) { + return; + } + + RefPtr self(this); + + AutoJSAPI jsapi; + if (!jsapi.Init(mDecoder->GetParentObject())) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p Cancel -- no jsapi", this)); + return; + } + + ErrorResult rv; + rv.ThrowAbortError("ImageDecoderReadRequest destroyed"); + + JS::Rooted errorValue(jsapi.cx()); + if (ToJSValue(jsapi.cx(), std::move(rv), &errorValue)) { + IgnoredErrorResult ignoredRv; + if (RefPtr p = reader->Cancel(jsapi.cx(), errorValue, ignoredRv)) { + MOZ_ALWAYS_TRUE(p->SetAnyPromiseIsHandled()); + } + } + + jsapi.ClearException(); +} + +void ImageDecoderReadRequest::Complete(const MediaResult& aResult) { + if (!mReader) { + return; + } + + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p Read -- complete, success %d", this, + NS_SUCCEEDED(aResult.Code()))); + + if (mSourceBuffer && !mSourceBuffer->IsComplete()) { + mSourceBuffer->Complete(aResult.Code()); + } + + if (mDecoder) { + mDecoder->OnSourceBufferComplete(aResult); + } + + Destroy(/* aCycleComplete */ false); +} + +void ImageDecoderReadRequest::ChunkSteps(JSContext* aCx, + JS::Handle aChunk, + ErrorResult& aRv) { + // 10.2.5. Fetch Stream Data Loop (with reader) - chunk steps + + // 1. If [[closed]] is true, abort these steps. + if (!mSourceBuffer) { + return; + } + + // 2. If chunk is not a Uint8Array object, queue a task to run the Close + // ImageDecoder algorithm with a DataError DOMException and abort these steps. + RootedSpiderMonkeyInterface chunk(aCx); + if (!aChunk.isObject() || !chunk.Init(&aChunk.toObject())) { + MOZ_LOG(gWebCodecsLog, LogLevel::Error, + ("ImageDecoderReadRequest %p ChunkSteps -- bad chunk", this)); + Complete(MediaResult(NS_ERROR_DOM_DATA_ERR, + "Reader cannot read chunk from stream"_ns)); + return; + } + + chunk.ProcessFixedData([&](const Span& aData) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p ChunkSteps -- write %zu bytes", this, + aData.Length())); + + // 3. Let bytes be the byte sequence represented by the Uint8Array object. + // 4. Append bytes to the [[encoded data]] internal slot. + nsresult rv = mSourceBuffer->Append( + reinterpret_cast(aData.Elements()), aData.Length()); + if (NS_WARN_IF(NS_FAILED(rv))) { + MOZ_LOG( + gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p ChunkSteps -- failed to append", this)); + Complete(MediaResult(NS_ERROR_DOM_UNKNOWN_ERR, + "Reader cannot allocate storage for chunk"_ns)); + } + + // 5. If [[tracks established]] is false, run the Establish Tracks + // algorithm. + // 6. Otherwise, run the Update Tracks algorithm. + // + // Note that these steps will be triggered by the decoder promise callbacks. + }); + + // 7. Run the Fetch Stream Data Loop algorithm with reader. + QueueRead(); +} + +void ImageDecoderReadRequest::CloseSteps(JSContext* aCx, ErrorResult& aRv) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p CloseSteps", this)); + + // 10.2.5. Fetch Stream Data Loop (with reader) - close steps + // 1. Assign true to [[complete]] + // 2. Resolve [[completed promise]]. + Complete(MediaResult(NS_OK)); +} + +void ImageDecoderReadRequest::ErrorSteps(JSContext* aCx, + JS::Handle aError, + ErrorResult& aRv) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageDecoderReadRequest %p ErrorSteps", this)); + + // 10.2.5. Fetch Stream Data Loop (with reader) - error steps + // 1. Queue a task to run the Close ImageDecoder algorithm with a + // NotReadableError DOMException + Complete(MediaResult(NS_ERROR_DOM_FILE_NOT_READABLE_ERR, + "Reader failed while waiting for chunk from stream"_ns)); +} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/ImageDecoderReadRequest.h b/dom/media/webcodecs/ImageDecoderReadRequest.h new file mode 100644 index 000000000000..02b65ecca747 --- /dev/null +++ b/dom/media/webcodecs/ImageDecoderReadRequest.h @@ -0,0 +1,66 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_ImageDecoderReadRequest_h +#define mozilla_dom_ImageDecoderReadRequest_h + +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/ReadRequest.h" + +namespace mozilla { +class MediaResult; + +namespace image { +class SourceBuffer; +} + +namespace dom { +class ImageDecoder; +class ReadableStream; +class ReadableStreamDefaultReader; +class WeakWorkerRef; + +struct ImageDecoderReadRequest final : public ReadRequest { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(ImageDecoderReadRequest, ReadRequest) + + public: + explicit ImageDecoderReadRequest(image::SourceBuffer* aSourceBuffer); + + bool Initialize(const GlobalObject& aGlobal, ImageDecoder* aDecoder, + ReadableStream& aStream); + void Destroy(bool aCycleCollect = true); + + MOZ_CAN_RUN_SCRIPT_BOUNDARY void ChunkSteps(JSContext* aCx, + JS::Handle aChunk, + ErrorResult& aRv) override; + + MOZ_CAN_RUN_SCRIPT_BOUNDARY void CloseSteps(JSContext* aCx, + ErrorResult& aRv) override; + + MOZ_CAN_RUN_SCRIPT_BOUNDARY void ErrorSteps(JSContext* aCx, + JS::Handle aError, + ErrorResult& aRv) override; + + private: + ~ImageDecoderReadRequest() override; + + void QueueRead(); + MOZ_CAN_RUN_SCRIPT_BOUNDARY void Read(); + MOZ_CAN_RUN_SCRIPT_BOUNDARY void Cancel(); + void Complete(const MediaResult& aResult); + + RefPtr mWorkerRef; + RefPtr mDecoder; + RefPtr mReader; + RefPtr mSourceBuffer; +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_ImageDecoderReadRequest_h diff --git a/dom/media/webcodecs/ImageTrack.cpp b/dom/media/webcodecs/ImageTrack.cpp new file mode 100644 index 000000000000..e7610f62ec3b --- /dev/null +++ b/dom/media/webcodecs/ImageTrack.cpp @@ -0,0 +1,107 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/ImageTrack.h" +#include "ImageContainer.h" +#include "mozilla/dom/ImageTrackList.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "mozilla/gfx/2D.h" +#include "mozilla/image/ImageUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(ImageTrack, mParent, mTrackList, + mDecodedFrames) +NS_IMPL_CYCLE_COLLECTING_ADDREF(ImageTrack) +NS_IMPL_CYCLE_COLLECTING_RELEASE(ImageTrack) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(ImageTrack) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +ImageTrack::ImageTrack(ImageTrackList* aTrackList, int32_t aIndex, + bool aSelected, bool aAnimated, uint32_t aFrameCount, + bool aFrameCountComplete, float aRepetitionCount) + : mParent(aTrackList->GetParentObject()), + mTrackList(aTrackList), + mFramesTimestamp(image::FrameTimeout::Zero()), + mIndex(aIndex), + mRepetitionCount(aRepetitionCount), + mFrameCount(aFrameCount), + mFrameCountComplete(aFrameCountComplete), + mAnimated(aAnimated), + mSelected(aSelected) {} + +ImageTrack::~ImageTrack() = default; + +void ImageTrack::Destroy() { mTrackList = nullptr; } + +JSObject* ImageTrack::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + return ImageTrack_Binding::Wrap(aCx, this, aGivenProto); +} + +void ImageTrack::SetSelected(bool aSelected) { + if (mTrackList) { + mTrackList->SetSelectedIndex(mIndex, aSelected); + } +} + +void ImageTrack::OnFrameCountSuccess( + const image::DecodeFrameCountResult& aResult) { + MOZ_ASSERT_IF(mFrameCountComplete, mFrameCount == aResult.mFrameCount); + MOZ_ASSERT_IF(!aResult.mFinished, !mFrameCountComplete); + MOZ_ASSERT_IF(!mAnimated, aResult.mFrameCount <= 1); + MOZ_ASSERT(aResult.mFrameCount >= mFrameCount); + mFrameCount = aResult.mFrameCount; + mFrameCountComplete = aResult.mFinished; +} + +void ImageTrack::OnDecodeFramesSuccess( + const image::DecodeFramesResult& aResult) { + MOZ_LOG(gWebCodecsLog, LogLevel::Debug, + ("ImageTrack %p OnDecodeFramesSuccess -- decoded %zu frames, %zu " + "total, finished %d", + this, aResult.mFrames.Length(), mDecodedFrames.Length(), + aResult.mFinished)); + + mDecodedFrames.SetCapacity(mDecodedFrames.Length() + + aResult.mFrames.Length()); + + for (const auto& f : aResult.mFrames) { + VideoColorSpaceInit colorSpace; + gfx::IntSize size = f.mSurface->GetSize(); + gfx::IntRect rect(gfx::IntPoint(0, 0), size); + + Maybe format = + SurfaceFormatToVideoPixelFormat(f.mSurface->GetFormat()); + MOZ_ASSERT(format, "Unexpected format for image!"); + + Maybe duration; + if (f.mTimeout != image::FrameTimeout::Forever()) { + duration = + Some(static_cast(f.mTimeout.AsMilliseconds()) * 1000); + } + + uint64_t timestamp = UINT64_MAX; + if (mFramesTimestamp != image::FrameTimeout::Forever()) { + timestamp = + static_cast(mFramesTimestamp.AsMilliseconds()) * 1000; + } + + mFramesTimestamp += f.mTimeout; + + auto image = MakeRefPtr(size, f.mSurface); + auto frame = MakeRefPtr(mParent, image, format, size, rect, + size, duration, timestamp, colorSpace); + mDecodedFrames.AppendElement(std::move(frame)); + } +} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/ImageTrack.h b/dom/media/webcodecs/ImageTrack.h new file mode 100644 index 000000000000..7efebe800209 --- /dev/null +++ b/dom/media/webcodecs/ImageTrack.h @@ -0,0 +1,100 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_ImageTrack_h +#define mozilla_dom_ImageTrack_h + +#include "FrameTimeout.h" +#include "mozilla/Attributes.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/NotNull.h" +#include "mozilla/dom/ImageDecoderBinding.h" +#include "nsCycleCollectionParticipant.h" +#include "nsTArray.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { +namespace image { +struct DecodeFrameCountResult; +struct DecodeFramesResult; +} // namespace image + +namespace dom { +class ImageTrackList; +class VideoFrame; + +class ImageTrack final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(ImageTrack) + + public: + ImageTrack(ImageTrackList* aTrackList, int32_t aIndex, bool aSelected, + bool aAnimated, uint32_t aFrameCount, bool aFrameCountComplete, + float aRepetitionCount); + + protected: + ~ImageTrack(); + + public: + nsIGlobalObject* GetParentObject() const { return mParent; } + + void Destroy(); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + int32_t Index() const { return mIndex; } + + bool Animated() const { return mAnimated; } + + uint32_t FrameCount() const { return mFrameCount; } + + bool FrameCountComplete() const { return mFrameCountComplete; } + + float RepetitionCount() const { return mRepetitionCount; } + + bool Selected() const { return mSelected; } + + void SetSelected(bool aSelected); + + void ClearSelected() { mSelected = false; } + void MarkSelected() { mSelected = true; } + + size_t DecodedFrameCount() const { return mDecodedFrames.Length(); } + + VideoFrame* GetDecodedFrame(uint32_t aIndex) const { + if (mDecodedFrames.Length() <= aIndex) { + return nullptr; + } + return mDecodedFrames[aIndex]; + } + + void OnFrameCountSuccess(const image::DecodeFrameCountResult& aResult); + void OnDecodeFramesSuccess(const image::DecodeFramesResult& aResult); + + private: + // ImageTrack can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(ImageTrack); } + + nsCOMPtr mParent; + RefPtr mTrackList; + AutoTArray, 1> mDecodedFrames; + image::FrameTimeout mFramesTimestamp; + int32_t mIndex = 0; + float mRepetitionCount = 0.0f; + uint32_t mFrameCount = 0; + bool mFrameCountComplete = false; + bool mAnimated = false; + bool mSelected = false; +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_ImageTrack_h diff --git a/dom/media/webcodecs/ImageTrackList.cpp b/dom/media/webcodecs/ImageTrackList.cpp new file mode 100644 index 000000000000..a7581a182889 --- /dev/null +++ b/dom/media/webcodecs/ImageTrackList.cpp @@ -0,0 +1,206 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/ImageTrackList.h" +#include "MediaResult.h" +#include "mozilla/dom/ImageDecoder.h" +#include "mozilla/dom/ImageTrack.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/image/ImageUtils.h" + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(ImageTrackList, mParent, mDecoder, + mReadyPromise, mTracks) +NS_IMPL_CYCLE_COLLECTING_ADDREF(ImageTrackList) +NS_IMPL_CYCLE_COLLECTING_RELEASE(ImageTrackList) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(ImageTrackList) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +ImageTrackList::ImageTrackList(nsIGlobalObject* aParent, ImageDecoder* aDecoder) + : mParent(aParent), mDecoder(aDecoder) {} + +ImageTrackList::~ImageTrackList() = default; + +JSObject* ImageTrackList::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + return ImageTrackList_Binding::Wrap(aCx, this, aGivenProto); +} + +void ImageTrackList::Initialize(ErrorResult& aRv) { + mReadyPromise = Promise::Create(mParent, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } +} + +void ImageTrackList::Destroy() { + if (!mIsReady && mReadyPromise && mReadyPromise->PromiseObj()) { + mReadyPromise->MaybeRejectWithAbortError("ImageTrackList destroyed"); + mIsReady = true; + } + + for (auto& track : mTracks) { + track->Destroy(); + } + mTracks.Clear(); + + mDecoder = nullptr; + mSelectedIndex = -1; +} + +void ImageTrackList::MaybeRejectReady(const MediaResult& aResult) { + if (mIsReady || !mReadyPromise || !mReadyPromise->PromiseObj()) { + return; + } + aResult.RejectTo(mReadyPromise); + mIsReady = true; +} + +void ImageTrackList::OnMetadataSuccess( + const image::DecodeMetadataResult& aMetadata) { + // 10.2.5. Establish Tracks + // + // Note that our implementation only supports one track, so many of these + // steps are simplified. + + // 4. Let newTrackList be a new list. + MOZ_ASSERT(mTracks.IsEmpty()); + + // 5. For each image track found in [[encoded data]]: + // 5.1. Let newTrack be a new ImageTrack, initialized as follows: + // 5.1.1. Assign this to [[ImageDecoder]]. + // 5.1.2. Assign tracks to [[ImageTrackList]]. + // 5.1.3. If image track is found to be animated, assign true to newTrack's + // [[animated]] internal slot. Otherwise, assign false. + // 5.1.4. If image track is found to describe a frame count, assign that + // count to newTrack's [[frame count]] internal slot. Otherwise, assign + // 0. + // 5.1.5. If image track is found to describe a repetition count, assign that + // count to [[repetition count]] internal slot. Otherwise, assign 0. + // 5.1.6. Assign false to newTrack’s [[selected]] internal slot. + // 5.2. Append newTrack to newTrackList. + // 6. Let selectedTrackIndex be the result of running the Get Default Selected + // Track Index algorithm with newTrackList. + // 7. Let selectedTrack be the track at position selectedTrackIndex within + // newTrackList. + // 8. Assign true to selectedTrack’s [[selected]] internal slot. + // 9. Assign selectedTrackIndex to [[internal selected track index]]. + const float repetitions = aMetadata.mRepetitions < 0 + ? std::numeric_limits::infinity() + : static_cast(aMetadata.mRepetitions); + auto track = MakeRefPtr( + this, /* aIndex */ 0, /* aSelected */ true, aMetadata.mAnimated, + aMetadata.mFrameCount, aMetadata.mFrameCountComplete, repetitions); + + // 11. Queue a task to perform the following steps: + // + // Note that we were already dispatched by the image decoder. + + // 11.1. Assign newTrackList to the tracks [[track list]] internal slot. + mTracks.AppendElement(std::move(track)); + + // 11.2. Assign selectedTrackIndex to tracks [[selected index]]. + mSelectedIndex = 0; + + // 11.3. Resolve [[ready promise]]. + MOZ_ASSERT(!mIsReady); + mReadyPromise->MaybeResolveWithUndefined(); + mIsReady = true; +} + +void ImageTrackList::OnFrameCountSuccess( + const image::DecodeFrameCountResult& aResult) { + if (mTracks.IsEmpty()) { + return; + } + + // 10.2.5. Update Tracks + // + // Note that we were already dispatched from the decoding threads. + + // 3. Let trackList be a copy of tracks' [[track list]]. + // 4. For each track in trackList: + // 4.1. Let trackIndex be the position of track in trackList. + // 4.2. Let latestFrameCount be the frame count as indicated by + // [[encoded data]] for the track corresponding to track. + // 4.3. Assert that latestFrameCount is greater than or equal to + // track.frameCount. + // 4.4. If latestFrameCount is greater than track.frameCount: + // 4.4.1. Let change be a track update struct whose track index is trackIndex + // and frame count is latestFrameCount. + // 4.4.2. Append change to tracksChanges. + // 5. If tracksChanges is empty, abort these steps. + // 6. Queue a task to perform the following steps: + // 6.1. For each update in trackChanges: + // 6.1.1. Let updateTrack be the ImageTrack at position update.trackIndex + // within tracks' [[track list]]. + // 6.1.2. Assign update.frameCount to updateTrack’s [[frame count]]. + mTracks.LastElement()->OnFrameCountSuccess(aResult); +} + +void ImageTrackList::SetSelectedIndex(int32_t aIndex, bool aSelected) { + MOZ_ASSERT(aIndex >= 0); + MOZ_ASSERT(uint32_t(aIndex) < mTracks.Length()); + + // 10.7.2. Attributes - selected, of type boolean + + // 1. If [[ImageDecoder]]'s [[closed]] slot is true, abort these steps. + if (!mDecoder) { + return; + } + + // 2. Let newValue be the given value. + // 3. If newValue equals [[selected]], abort these steps. + // 4. Assign newValue to [[selected]]. + // 5. Let parentTrackList be [[ImageTrackList]] + // 6. Let oldSelectedIndex be the value of parentTrackList [[selected index]]. + // 7. If oldSelectedIndex is not -1: + // 7.1. Let oldSelectedTrack be the ImageTrack in parentTrackList + // [[track list]] at the position of oldSelectedIndex. + // 7.2. Assign false to oldSelectedTrack [[selected]] + // 8. If newValue is true, let selectedIndex be the index of this ImageTrack + // within parentTrackList's [[track list]]. Otherwise, let selectedIndex be + // -1. + // 9. Assign selectedIndex to parentTrackList [[selected index]]. + if (aSelected) { + if (mSelectedIndex == -1) { + MOZ_ASSERT(!mTracks[aIndex]->Selected()); + mTracks[aIndex]->MarkSelected(); + mSelectedIndex = aIndex; + } else if (mSelectedIndex != aIndex) { + MOZ_ASSERT(mTracks[mSelectedIndex]->Selected()); + MOZ_ASSERT(!mTracks[aIndex]->Selected()); + mTracks[mSelectedIndex]->ClearSelected(); + mTracks[aIndex]->MarkSelected(); + mSelectedIndex = aIndex; + } else { + MOZ_ASSERT(mTracks[mSelectedIndex]->Selected()); + return; + } + } else if (mSelectedIndex == aIndex) { + mTracks[mSelectedIndex]->ClearSelected(); + mSelectedIndex = -1; + } else { + MOZ_ASSERT(!mTracks[aIndex]->Selected()); + return; + } + + // 10. Run the Reset ImageDecoder algorithm on [[ImageDecoder]]. + mDecoder->Reset(); + + // 11. Queue a control message to [[ImageDecoder]]'s control message queue to + // update the internal selected track index with selectedIndex. + mDecoder->QueueSelectTrackMessage(mSelectedIndex); + + // 12. Process the control message queue belonging to [[ImageDecoder]]. + mDecoder->ProcessControlMessageQueue(); +} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/ImageTrackList.h b/dom/media/webcodecs/ImageTrackList.h new file mode 100644 index 000000000000..bff8d95576c1 --- /dev/null +++ b/dom/media/webcodecs/ImageTrackList.h @@ -0,0 +1,104 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_ImageTrackList_h +#define mozilla_dom_ImageTrackList_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/ImageDecoderBinding.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { +class MediaResult; + +namespace image { +struct DecodeFrameCountResult; +struct DecodeMetadataResult; +} // namespace image + +namespace dom { + +class ImageTrack; +class Promise; + +class ImageTrackList final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(ImageTrackList) + + public: + ImageTrackList(nsIGlobalObject* aParent, ImageDecoder* aDecoder); + + void Initialize(ErrorResult& aRv); + void MaybeRejectReady(const MediaResult& aResult); + void Destroy(); + void OnMetadataSuccess(const image::DecodeMetadataResult& aMetadata); + void OnFrameCountSuccess(const image::DecodeFrameCountResult& aResult); + void SetSelectedIndex(int32_t aIndex, bool aSelected); + + protected: + ~ImageTrackList(); + + public: + nsIGlobalObject* GetParentObject() const { return mParent; } + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + Promise* Ready() const { return mReadyPromise; } + + bool IsReady() const { return mIsReady; } + + uint32_t Length() const { return mTracks.Length(); } + + int32_t SelectedIndex() const { return mSelectedIndex; } + + ImageTrack* GetSelectedTrack() const { + if (mSelectedIndex < 0) { + return nullptr; + } + return mTracks[mSelectedIndex]; + } + + ImageTrack* GetDefaultTrack() const { + if (mTracks.IsEmpty()) { + return nullptr; + } + return mTracks[0]; + } + + ImageTrack* IndexedGetter(uint32_t aIndex, bool& aFound) const { + if (aIndex >= mTracks.Length()) { + aFound = false; + return nullptr; + } + + MOZ_ASSERT(mTracks[aIndex]); + aFound = true; + return mTracks[aIndex]; + } + + private: + // ImageTrackList can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(ImageTrackList); + } + + nsCOMPtr mParent; + RefPtr mDecoder; + AutoTArray, 1> mTracks; + RefPtr mReadyPromise; + int32_t mSelectedIndex = -1; + bool mIsReady = false; +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_ImageTrackList_h diff --git a/dom/media/webcodecs/moz.build b/dom/media/webcodecs/moz.build index 1c398439a3ec..c94f509f83fd 100644 --- a/dom/media/webcodecs/moz.build +++ b/dom/media/webcodecs/moz.build @@ -31,6 +31,9 @@ EXPORTS.mozilla.dom += [ "EncoderAgent.h", "EncoderTemplate.h", "EncoderTypes.h", + "ImageDecoder.h", + "ImageTrack.h", + "ImageTrackList.h", "VideoColorSpace.h", "VideoDecoder.h", "VideoEncoder.h", @@ -48,6 +51,10 @@ UNIFIED_SOURCES += [ "EncodedVideoChunk.cpp", "EncoderAgent.cpp", "EncoderTemplate.cpp", + "ImageDecoder.cpp", + "ImageDecoderReadRequest.cpp", + "ImageTrack.cpp", + "ImageTrackList.cpp", "VideoColorSpace.cpp", "VideoDecoder.cpp", "VideoEncoder.cpp", diff --git a/dom/tests/mochitest/general/test_interfaces.js b/dom/tests/mochitest/general/test_interfaces.js index b7d7042a00b0..abd0cabfb391 100644 --- a/dom/tests/mochitest/general/test_interfaces.js +++ b/dom/tests/mochitest/general/test_interfaces.js @@ -770,6 +770,12 @@ let interfaceNamesInGlobalScope = [ // IMPORTANT: Do not change this list without review from a DOM peer! { name: "ImageData", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "ImageDecoder", nightly: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "ImageTrack", nightly: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "ImageTrackList", nightly: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! { name: "InputEvent", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! { diff --git a/dom/webidl/ImageDecoder.webidl b/dom/webidl/ImageDecoder.webidl new file mode 100644 index 000000000000..d19899b14fb9 --- /dev/null +++ b/dom/webidl/ImageDecoder.webidl @@ -0,0 +1,73 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. + * + * The origin of this IDL file is + * https://w3c.github.io/webcodecs/#image-decoding + */ + +// Bug 1696216: Should be AllowSharedBufferSource or ReadableStream +typedef ([AllowShared] ArrayBufferView or [AllowShared] ArrayBuffer or ReadableStream) ImageBufferSource; +dictionary ImageDecoderInit { + required DOMString type; + required ImageBufferSource data; + ColorSpaceConversion colorSpaceConversion = "default"; + [EnforceRange] unsigned long desiredWidth; + [EnforceRange] unsigned long desiredHeight; + boolean preferAnimation; + sequence transfer = []; +}; + +dictionary ImageDecodeOptions { + [EnforceRange] unsigned long frameIndex = 0; + boolean completeFramesOnly = true; +}; + +dictionary ImageDecodeResult { + required VideoFrame image; + required boolean complete; +}; + +[Exposed=(Window,DedicatedWorker), + SecureContext, + Func="mozilla::dom::ImageDecoder::PrefEnabled"] +interface ImageTrack { + readonly attribute boolean animated; + readonly attribute unsigned long frameCount; + readonly attribute unrestricted float repetitionCount; + attribute boolean selected; +}; + +[Exposed=(Window,DedicatedWorker), + SecureContext, + Func="mozilla::dom::ImageDecoder::PrefEnabled"] +interface ImageTrackList { + getter ImageTrack (unsigned long index); + + readonly attribute Promise ready; + readonly attribute unsigned long length; + readonly attribute long selectedIndex; + readonly attribute ImageTrack? selectedTrack; +}; + +[Exposed=(Window,DedicatedWorker), + SecureContext, + Func="mozilla::dom::ImageDecoder::PrefEnabled"] +interface ImageDecoder { + [Throws] + constructor(ImageDecoderInit init); + + readonly attribute DOMString type; + readonly attribute boolean complete; + readonly attribute Promise completed; + readonly attribute ImageTrackList tracks; + + [Throws] + Promise decode(optional ImageDecodeOptions options = {}); + undefined reset(); + undefined close(); + + [Throws] + static Promise isTypeSupported(DOMString type); +}; diff --git a/dom/webidl/moz.build b/dom/webidl/moz.build index 79018fe0f83d..156ee9543cdf 100644 --- a/dom/webidl/moz.build +++ b/dom/webidl/moz.build @@ -166,6 +166,9 @@ with Files("ImageBitmap*"): with Files("ImageCapture*"): BUG_COMPONENT = ("Core", "Audio/Video") +with Files("ImageDecoder.webidl"): + BUG_COMPONENT = ("Core", "Audio/Video: Web Codecs") + with Files("InputEvent.webidl"): BUG_COMPONENT = ("Core", "DOM: UI Events & Focus Handling") @@ -692,6 +695,7 @@ WEBIDL_FILES = [ "ImageBitmapRenderingContext.webidl", "ImageCapture.webidl", "ImageData.webidl", + "ImageDecoder.webidl", "ImageDocument.webidl", "InputEvent.webidl", "IntersectionObserver.webidl", diff --git a/dom/workers/test/test_worker_interfaces.js b/dom/workers/test/test_worker_interfaces.js index f2e35a5d5a86..645d02265432 100644 --- a/dom/workers/test/test_worker_interfaces.js +++ b/dom/workers/test/test_worker_interfaces.js @@ -259,6 +259,12 @@ let interfaceNamesInGlobalScope = [ // IMPORTANT: Do not change this list without review from a DOM peer! { name: "ImageData", insecureContext: true }, // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "ImageDecoder", nightly: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "ImageTrack", nightly: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! + { name: "ImageTrackList", nightly: true }, + // IMPORTANT: Do not change this list without review from a DOM peer! "Lock", // IMPORTANT: Do not change this list without review from a DOM peer! "LockManager", diff --git a/modules/libpref/init/StaticPrefList.yaml b/modules/libpref/init/StaticPrefList.yaml index 392f9235e1f4..2a28966a28cc 100644 --- a/modules/libpref/init/StaticPrefList.yaml +++ b/modules/libpref/init/StaticPrefList.yaml @@ -3149,6 +3149,12 @@ value: @IS_NIGHTLY_BUILD@ mirror: always +# WebCodecs API - Image decoder +- name: dom.media.webcodecs.image-decoder.enabled + type: RelaxedAtomicBool + value: @IS_NIGHTLY_BUILD@ + mirror: always + # Number of seconds of very quiet or silent audio before considering the audio # inaudible. - name: dom.media.silence_duration_for_audibility diff --git a/netwerk/protocol/http/OpaqueResponseUtils.h b/netwerk/protocol/http/OpaqueResponseUtils.h index 7b37095b0180..35fed3e29915 100644 --- a/netwerk/protocol/http/OpaqueResponseUtils.h +++ b/netwerk/protocol/http/OpaqueResponseUtils.h @@ -131,7 +131,7 @@ class OpaqueResponseBlocker final : public nsIStreamListener { nsILoadInfo* aLoadInfo); void ResolveAndProcessData(HttpBaseChannel* aChannel, bool aAllowed, - Maybe& aSharedData); + Maybe& aSharedData); void MaybeRunOnStopRequest(HttpBaseChannel* aChannel); diff --git a/testing/web-platform/meta/webcodecs/__dir__.ini b/testing/web-platform/meta/webcodecs/__dir__.ini index 7c7c7dab75fc..ee0aad70ed98 100644 --- a/testing/web-platform/meta/webcodecs/__dir__.ini +++ b/testing/web-platform/meta/webcodecs/__dir__.ini @@ -1,4 +1,4 @@ -prefs: [dom.media.webcodecs.enabled:true, media.ffmpeg.encoder.enabled:true] +prefs: [dom.media.webcodecs.enabled:true, dom.media.webcodecs.image-decoder.enabled:true, media.ffmpeg.encoder.enabled:true] tags: [webcodecs] disabled: if (os == "linux") and (bits == 32): Not implemented diff --git a/testing/web-platform/meta/webcodecs/idlharness.https.any.js.ini b/testing/web-platform/meta/webcodecs/idlharness.https.any.js.ini index d67dc8f953c4..c5ae81987a47 100644 --- a/testing/web-platform/meta/webcodecs/idlharness.https.any.js.ini +++ b/testing/web-platform/meta/webcodecs/idlharness.https.any.js.ini @@ -1,619 +1,14 @@ -prefs: [dom.media.webcodecs.enabled:true] +prefs: [dom.media.webcodecs.enabled:true, dom.media.webcodecs.image-decoder.enabled:true] [idlharness.https.any.html] [VideoFrame interface: operation metadata()] expected: FAIL - [ImageDecoder interface: existence and properties of interface object] + [VideoFrame interface: new VideoFrame(makeImageBitmap(32, 16), {timestamp: 100, duration: 33}) must inherit property "metadata()" with the proper type] expected: FAIL - [ImageDecoder interface object length] - expected: FAIL - - [ImageDecoder interface object name] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageDecoder interface: attribute type] - expected: FAIL - - [ImageDecoder interface: attribute complete] - expected: FAIL - - [ImageDecoder interface: attribute completed] - expected: FAIL - - [ImageDecoder interface: attribute tracks] - expected: FAIL - - [ImageDecoder interface: operation decode(optional ImageDecodeOptions)] - expected: FAIL - - [ImageDecoder interface: operation reset()] - expected: FAIL - - [ImageDecoder interface: operation close()] - expected: FAIL - - [ImageDecoder interface: operation isTypeSupported(DOMString)] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface object] - expected: FAIL - - [ImageTrackList interface object length] - expected: FAIL - - [ImageTrackList interface object name] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageTrackList interface: attribute ready] - expected: FAIL - - [ImageTrackList interface: attribute length] - expected: FAIL - - [ImageTrackList interface: attribute selectedIndex] - expected: FAIL - - [ImageTrackList interface: attribute selectedTrack] - expected: FAIL - - [ImageTrack interface: existence and properties of interface object] - expected: FAIL - - [ImageTrack interface object length] - expected: FAIL - - [ImageTrack interface object name] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageTrack interface: attribute animated] - expected: FAIL - - [ImageTrack interface: attribute frameCount] - expected: FAIL - - [ImageTrack interface: attribute repetitionCount] - expected: FAIL - - [ImageTrack interface: attribute selected] - expected: FAIL - - [idl_test setup] - expected: FAIL - - [idl_test setup] - expected: FAIL - - -[idlharness.https.any.worker.html] - [VideoEncoder interface: existence and properties of interface object] - expected: FAIL - - [VideoEncoder interface object length] - expected: FAIL - - [VideoEncoder interface object name] - expected: FAIL - - [VideoEncoder interface: existence and properties of interface prototype object] - expected: FAIL - - [VideoEncoder interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [VideoEncoder interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [VideoEncoder interface: attribute state] - expected: FAIL - - [VideoEncoder interface: attribute encodeQueueSize] - expected: FAIL - - [VideoEncoder interface: attribute ondequeue] - expected: FAIL - - [VideoEncoder interface: operation configure(VideoEncoderConfig)] - expected: FAIL - - [VideoEncoder interface: operation encode(VideoFrame, optional VideoEncoderEncodeOptions)] - expected: FAIL - - [VideoEncoder interface: operation flush()] - expected: FAIL - - [VideoEncoder interface: operation reset()] - expected: FAIL - - [VideoEncoder interface: operation close()] - expected: FAIL - - [VideoEncoder interface: operation isConfigSupported(VideoEncoderConfig)] - expected: FAIL - - [VideoFrame interface: operation metadata()] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface object] - expected: FAIL - - [ImageDecoder interface object length] - expected: FAIL - - [ImageDecoder interface object name] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageDecoder interface: attribute type] - expected: FAIL - - [ImageDecoder interface: attribute complete] - expected: FAIL - - [ImageDecoder interface: attribute completed] - expected: FAIL - - [ImageDecoder interface: attribute tracks] - expected: FAIL - - [ImageDecoder interface: operation decode(optional ImageDecodeOptions)] - expected: FAIL - - [ImageDecoder interface: operation reset()] - expected: FAIL - - [ImageDecoder interface: operation close()] - expected: FAIL - - [ImageDecoder interface: operation isTypeSupported(DOMString)] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface object] - expected: FAIL - - [ImageTrackList interface object length] - expected: FAIL - - [ImageTrackList interface object name] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageTrackList interface: attribute ready] - expected: FAIL - - [ImageTrackList interface: attribute length] - expected: FAIL - - [ImageTrackList interface: attribute selectedIndex] - expected: FAIL - - [ImageTrackList interface: attribute selectedTrack] - expected: FAIL - - [ImageTrack interface: existence and properties of interface object] - expected: FAIL - - [ImageTrack interface object length] - expected: FAIL - - [ImageTrack interface object name] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageTrack interface: attribute animated] - expected: FAIL - - [ImageTrack interface: attribute frameCount] - expected: FAIL - - [ImageTrack interface: attribute repetitionCount] - expected: FAIL - - [ImageTrack interface: attribute selected] - expected: FAIL - - [idl_test setup] - expected: FAIL - - [idl_test setup] - expected: FAIL - - -[idlharness.https.any.worker.html] - [AudioDecoder interface: existence and properties of interface object] - expected: FAIL - - [AudioDecoder interface object length] - expected: FAIL - - [AudioDecoder interface object name] - expected: FAIL - - [AudioDecoder interface: existence and properties of interface prototype object] - expected: FAIL - - [AudioDecoder interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [AudioDecoder interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [AudioDecoder interface: attribute state] - expected: FAIL - - [AudioDecoder interface: attribute decodeQueueSize] - expected: FAIL - - [AudioDecoder interface: attribute ondequeue] - expected: FAIL - - [AudioDecoder interface: operation configure(AudioDecoderConfig)] - expected: FAIL - - [AudioDecoder interface: operation decode(EncodedAudioChunk)] - expected: FAIL - - [AudioDecoder interface: operation flush()] - expected: FAIL - - [AudioDecoder interface: operation reset()] - expected: FAIL - - [AudioDecoder interface: operation close()] - expected: FAIL - - [AudioDecoder interface: operation isConfigSupported(AudioDecoderConfig)] - expected: FAIL - - [EncodedAudioChunk interface: existence and properties of interface object] - expected: FAIL - - [EncodedAudioChunk interface object length] - expected: FAIL - - [EncodedAudioChunk interface object name] - expected: FAIL - - [EncodedAudioChunk interface: existence and properties of interface prototype object] - expected: FAIL - - [EncodedAudioChunk interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [EncodedAudioChunk interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [EncodedAudioChunk interface: attribute type] - expected: FAIL - - [EncodedAudioChunk interface: attribute timestamp] - expected: FAIL - - [EncodedAudioChunk interface: attribute duration] - expected: FAIL - - [EncodedAudioChunk interface: attribute byteLength] - expected: FAIL - - [AudioData interface: existence and properties of interface object] - expected: FAIL - - [AudioData interface object length] - expected: FAIL - - [AudioData interface object name] - expected: FAIL - - [AudioData interface: existence and properties of interface prototype object] - expected: FAIL - - [AudioData interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [AudioData interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [AudioData interface: attribute format] - expected: FAIL - - [AudioData interface: attribute sampleRate] - expected: FAIL - - [AudioData interface: attribute numberOfFrames] - expected: FAIL - - [AudioData interface: attribute numberOfChannels] - expected: FAIL - - [AudioData interface: attribute duration] - expected: FAIL - - [AudioData interface: attribute timestamp] - expected: FAIL - - [AudioData interface: operation allocationSize(AudioDataCopyToOptions)] - expected: FAIL - - [AudioData interface: operation clone()] - expected: FAIL - - [AudioData interface: operation close()] - expected: FAIL - - [VideoFrame interface: operation metadata()] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface object] - expected: FAIL - - [ImageDecoder interface object length] - expected: FAIL - - [ImageDecoder interface object name] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageDecoder interface: attribute type] - expected: FAIL - - [ImageDecoder interface: attribute complete] - expected: FAIL - - [ImageDecoder interface: attribute completed] - expected: FAIL - - [ImageDecoder interface: attribute tracks] - expected: FAIL - - [ImageDecoder interface: operation decode(optional ImageDecodeOptions)] - expected: FAIL - - [ImageDecoder interface: operation reset()] - expected: FAIL - - [ImageDecoder interface: operation close()] - expected: FAIL - - [ImageDecoder interface: operation isTypeSupported(DOMString)] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface object] - expected: FAIL - - [ImageTrackList interface object length] - expected: FAIL - - [ImageTrackList interface object name] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageTrackList interface: attribute ready] - expected: FAIL - - [ImageTrackList interface: attribute length] - expected: FAIL - - [ImageTrackList interface: attribute selectedIndex] - expected: FAIL - - [ImageTrackList interface: attribute selectedTrack] - expected: FAIL - - [ImageTrack interface: existence and properties of interface object] - expected: FAIL - - [ImageTrack interface object length] - expected: FAIL - - [ImageTrack interface object name] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageTrack interface: attribute animated] - expected: FAIL - - [ImageTrack interface: attribute frameCount] - expected: FAIL - - [ImageTrack interface: attribute repetitionCount] - expected: FAIL - - [ImageTrack interface: attribute selected] - expected: FAIL - - [idl_test setup] - expected: FAIL - - [idl_test setup] - expected: FAIL - - [EncodedAudioChunk interface: operation copyTo(AllowSharedBufferSource)] - expected: FAIL - - [AudioData interface: operation copyTo(AllowSharedBufferSource, AudioDataCopyToOptions)] - expected: FAIL - - [idlharness.https.any.worker.html] [VideoFrame interface: operation metadata()] expected: FAIL - [ImageDecoder interface: existence and properties of interface object] - expected: FAIL - - [ImageDecoder interface object length] - expected: FAIL - - [ImageDecoder interface object name] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageDecoder interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageDecoder interface: attribute type] - expected: FAIL - - [ImageDecoder interface: attribute complete] - expected: FAIL - - [ImageDecoder interface: attribute completed] - expected: FAIL - - [ImageDecoder interface: attribute tracks] - expected: FAIL - - [ImageDecoder interface: operation decode(optional ImageDecodeOptions)] - expected: FAIL - - [ImageDecoder interface: operation reset()] - expected: FAIL - - [ImageDecoder interface: operation close()] - expected: FAIL - - [ImageDecoder interface: operation isTypeSupported(DOMString)] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface object] - expected: FAIL - - [ImageTrackList interface object length] - expected: FAIL - - [ImageTrackList interface object name] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageTrackList interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageTrackList interface: attribute ready] - expected: FAIL - - [ImageTrackList interface: attribute length] - expected: FAIL - - [ImageTrackList interface: attribute selectedIndex] - expected: FAIL - - [ImageTrackList interface: attribute selectedTrack] - expected: FAIL - - [ImageTrack interface: existence and properties of interface object] - expected: FAIL - - [ImageTrack interface object length] - expected: FAIL - - [ImageTrack interface object name] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - - [ImageTrack interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - - [ImageTrack interface: attribute animated] - expected: FAIL - - [ImageTrack interface: attribute frameCount] - expected: FAIL - - [ImageTrack interface: attribute repetitionCount] - expected: FAIL - - [ImageTrack interface: attribute selected] - expected: FAIL - - [idl_test setup] + [VideoFrame interface: new VideoFrame(makeImageBitmap(32, 16), {timestamp: 100, duration: 33}) must inherit property "metadata()" with the proper type] expected: FAIL diff --git a/testing/web-platform/meta/webcodecs/image-decoder-disconnect-readable-stream-crash.https.html.ini b/testing/web-platform/meta/webcodecs/image-decoder-disconnect-readable-stream-crash.https.html.ini deleted file mode 100644 index 2dd82c0a5373..000000000000 --- a/testing/web-platform/meta/webcodecs/image-decoder-disconnect-readable-stream-crash.https.html.ini +++ /dev/null @@ -1,3 +0,0 @@ -[image-decoder-disconnect-readable-stream-crash.https.html] - expected: - if (os == "win") and not debug and (processor == "x86_64"): [PASS, TIMEOUT] diff --git a/testing/web-platform/meta/webcodecs/image-decoder-image-orientation-none.https.html.ini b/testing/web-platform/meta/webcodecs/image-decoder-image-orientation-none.https.html.ini index 06504a120f77..b1d83cd55939 100644 --- a/testing/web-platform/meta/webcodecs/image-decoder-image-orientation-none.https.html.ini +++ b/testing/web-platform/meta/webcodecs/image-decoder-image-orientation-none.https.html.ini @@ -1,9 +1,5 @@ +prefs: [dom.media.webcodecs.enabled:true, dom.media.webcodecs.image-decoder.enabled:true] [image-decoder-image-orientation-none.https.html] - expected: - if (os == "android") and fission: [OK, TIMEOUT] - [Test JPEG w/ EXIF orientation top-left on canvas w/o orientation] - expected: FAIL - [Test JPEG w/ EXIF orientation top-right on canvas w/o orientation.] expected: FAIL @@ -25,9 +21,6 @@ [Test JPEG w/ EXIF orientation left-bottom on canvas w/o orientation.] expected: FAIL - [Test 4:2:0 JPEG w/ EXIF orientation top-left on canvas w/o orientation] - expected: FAIL - [Test 4:2:0 JPEG w/ EXIF orientation top-right on canvas w/o orientation.] expected: FAIL diff --git a/testing/web-platform/meta/webcodecs/image-decoder.crossOriginIsolated.https.any.js.ini b/testing/web-platform/meta/webcodecs/image-decoder.crossOriginIsolated.https.any.js.ini deleted file mode 100644 index 324199dfd1c3..000000000000 --- a/testing/web-platform/meta/webcodecs/image-decoder.crossOriginIsolated.https.any.js.ini +++ /dev/null @@ -1,18 +0,0 @@ -[image-decoder.crossOriginIsolated.https.any.html] - expected: - if (os == "android") and debug: [OK, TIMEOUT] - [Test ImageDecoder decoding with a SharedArrayBuffer source] - expected: FAIL - - [Test ImageDecoder decoding with a Uint8Array(SharedArrayBuffer) source] - expected: FAIL - - -[image-decoder.crossOriginIsolated.https.any.worker.html] - expected: - if (os == "android") and debug: [OK, TIMEOUT] - [Test ImageDecoder decoding with a SharedArrayBuffer source] - expected: FAIL - - [Test ImageDecoder decoding with a Uint8Array(SharedArrayBuffer) source] - expected: FAIL diff --git a/testing/web-platform/meta/webcodecs/image-decoder.https.any.js.ini b/testing/web-platform/meta/webcodecs/image-decoder.https.any.js.ini index a391295c89cb..8b8f68f15dd3 100644 --- a/testing/web-platform/meta/webcodecs/image-decoder.https.any.js.ini +++ b/testing/web-platform/meta/webcodecs/image-decoder.https.any.js.ini @@ -1,41 +1,8 @@ +prefs: [dom.media.webcodecs.enabled:true, dom.media.webcodecs.image-decoder.enabled:true] [image-decoder.https.any.worker.html] - expected: - if (os == "android") and fission: [OK, TIMEOUT] - [Test JPEG image decoding.] - expected: FAIL - - [Test JPEG w/ EXIF orientation top-left.] - expected: FAIL - - [Test JPEG w/ EXIF orientation top-right.] - expected: FAIL - - [Test JPEG w/ EXIF orientation bottom-right.] - expected: FAIL - - [Test JPEG w/ EXIF orientation bottom-left.] - expected: FAIL - - [Test JPEG w/ EXIF orientation left-top.] - expected: FAIL - - [Test JPEG w/ EXIF orientation right-top.] - expected: FAIL - - [Test JPEG w/ EXIF orientation right-bottom.] - expected: FAIL - - [Test JPEG w/ EXIF orientation left-bottom.] - expected: FAIL - - [Test PNG image decoding.] - expected: FAIL - - [Test AVIF image decoding.] - expected: FAIL - - [Test high bit depth HDR AVIF image decoding.] - expected: FAIL + # 'Test invalid mime type rejects ...' variants don't catch us rejecting all + # of the promises. + expected: ERROR [Test multi-track AVIF image decoding w/ preferAnimation=false.] expected: FAIL @@ -43,12 +10,6 @@ [Test multi-track AVIF image decoding w/ preferAnimation=true.] expected: FAIL - [Test WEBP image decoding.] - expected: FAIL - - [Test GIF image decoding.] - expected: FAIL - [Test JPEG image YUV 4:2:0 decoding.] expected: FAIL @@ -70,99 +31,17 @@ [Test invalid mime type rejects decodeMetadata() requests] expected: FAIL - [Test out of range index returns RangeError] - expected: FAIL - [Test partial decoding without a frame results in an error] expected: FAIL - [Test completed property on fully buffered decode] - expected: FAIL - - [Test decode, decodeMetadata after no track selected.] - expected: FAIL - [Test track selection in multi track image.] expected: FAIL - [Test ReadableStream of gif] - expected: FAIL - - [Test that decode requests are serialized.] - expected: FAIL - - [Test ReadableStream aborts promises on track change] - expected: FAIL - - [Test ReadableStream aborts completed on close] - expected: FAIL - - [Test ReadableStream resolves completed] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation top-left.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation top-right.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation bottom-right.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation bottom-left.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation left-top.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation right-top.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation right-bottom.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation left-bottom.] - expected: FAIL - [image-decoder.https.any.html] - expected: - if (os == "android") and fission: [OK, TIMEOUT] - [Test JPEG image decoding.] - expected: FAIL - - [Test JPEG w/ EXIF orientation top-left.] - expected: FAIL - - [Test JPEG w/ EXIF orientation top-right.] - expected: FAIL - - [Test JPEG w/ EXIF orientation bottom-right.] - expected: FAIL - - [Test JPEG w/ EXIF orientation bottom-left.] - expected: FAIL - - [Test JPEG w/ EXIF orientation left-top.] - expected: FAIL - - [Test JPEG w/ EXIF orientation right-top.] - expected: FAIL - - [Test JPEG w/ EXIF orientation right-bottom.] - expected: FAIL - - [Test JPEG w/ EXIF orientation left-bottom.] - expected: FAIL - - [Test PNG image decoding.] - expected: FAIL - - [Test AVIF image decoding.] - expected: FAIL - - [Test high bit depth HDR AVIF image decoding.] - expected: FAIL + # 'Test invalid mime type rejects ...' variants don't catch us rejecting all + # of the promises. + expected: ERROR [Test multi-track AVIF image decoding w/ preferAnimation=false.] expected: FAIL @@ -170,12 +49,6 @@ [Test multi-track AVIF image decoding w/ preferAnimation=true.] expected: FAIL - [Test WEBP image decoding.] - expected: FAIL - - [Test GIF image decoding.] - expected: FAIL - [Test JPEG image YUV 4:2:0 decoding.] expected: FAIL @@ -197,56 +70,8 @@ [Test invalid mime type rejects decodeMetadata() requests] expected: FAIL - [Test out of range index returns RangeError] - expected: FAIL - [Test partial decoding without a frame results in an error] expected: FAIL - [Test completed property on fully buffered decode] - expected: FAIL - - [Test decode, decodeMetadata after no track selected.] - expected: FAIL - [Test track selection in multi track image.] expected: FAIL - - [Test ReadableStream of gif] - expected: FAIL - - [Test that decode requests are serialized.] - expected: FAIL - - [Test ReadableStream aborts promises on track change] - expected: FAIL - - [Test ReadableStream aborts completed on close] - expected: FAIL - - [Test ReadableStream resolves completed] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation top-left.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation top-right.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation bottom-right.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation bottom-left.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation left-top.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation right-top.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation right-bottom.] - expected: FAIL - - [Test 4:2:0 JPEG w/ EXIF orientation left-bottom.] - expected: FAIL diff --git a/testing/web-platform/meta/webcodecs/transfering.https.any.js.ini b/testing/web-platform/meta/webcodecs/transfering.https.any.js.ini index 5ff19ca29e4f..cf5ea605a7e9 100644 --- a/testing/web-platform/meta/webcodecs/transfering.https.any.js.ini +++ b/testing/web-platform/meta/webcodecs/transfering.https.any.js.ini @@ -1,4 +1,4 @@ -prefs: [dom.media.webcodecs.enabled:true] +prefs: [dom.media.webcodecs.enabled:true, dom.media.webcodecs.image-decoder.enabled:true] [transfering.https.any.html] [Test transfering ArrayBuffer to VideoFrame] expected: FAIL @@ -18,9 +18,6 @@ prefs: [dom.media.webcodecs.enabled:true] [Encoding from AudioData with transferred buffer] expected: FAIL - [Test transfering ArrayBuffer to ImageDecoder.] - expected: FAIL - [transfering.https.any.worker.html] [Test transfering ArrayBuffer to VideoFrame] @@ -40,6 +37,3 @@ prefs: [dom.media.webcodecs.enabled:true] [Encoding from AudioData with transferred buffer] expected: FAIL - - [Test transfering ArrayBuffer to ImageDecoder.] - expected: FAIL