/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/dom/ImageBitmap.h" #include "mozilla/CheckedInt.h" #include "mozilla/dom/ImageBitmapBinding.h" #include "mozilla/dom/Promise.h" #include "mozilla/dom/StructuredCloneTags.h" #include "mozilla/dom/WorkerPrivate.h" #include "mozilla/dom/WorkerRunnable.h" #include "mozilla/gfx/2D.h" #include "mozilla/gfx/Swizzle.h" #include "mozilla/Mutex.h" #include "ImageBitmapColorUtils.h" #include "ImageBitmapUtils.h" #include "ImageUtils.h" #include "imgTools.h" using namespace mozilla::gfx; using namespace mozilla::layers; namespace mozilla { namespace dom { using namespace workers; NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(ImageBitmap, mParent) NS_IMPL_CYCLE_COLLECTING_ADDREF(ImageBitmap) NS_IMPL_CYCLE_COLLECTING_RELEASE(ImageBitmap) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(ImageBitmap) NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY NS_INTERFACE_MAP_ENTRY(nsISupports) NS_INTERFACE_MAP_END /* * If either aRect.width or aRect.height are negative, then return a new IntRect * which represents the same rectangle as the aRect does but with positive width * and height. */ static IntRect FixUpNegativeDimension(const IntRect& aRect, ErrorResult& aRv) { gfx::IntRect rect = aRect; // fix up negative dimensions if (rect.width < 0) { CheckedInt32 checkedX = CheckedInt32(rect.x) + rect.width; if (!checkedX.isValid()) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return rect; } rect.x = checkedX.value(); rect.width = -(rect.width); } if (rect.height < 0) { CheckedInt32 checkedY = CheckedInt32(rect.y) + rect.height; if (!checkedY.isValid()) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return rect; } rect.y = checkedY.value(); rect.height = -(rect.height); } return rect; } /* * This helper function copies the data of the given DataSourceSurface, * _aSurface_, in the given area, _aCropRect_, into a new DataSourceSurface. * This might return null if it can not create a new SourceSurface or it cannot * read data from the given _aSurface_. * * Warning: Even though the area of _aCropRect_ is just the same as the size of * _aSurface_, this function still copy data into a new * DataSourceSurface. */ static already_AddRefed CropAndCopyDataSourceSurface(DataSourceSurface* aSurface, const IntRect& aCropRect) { MOZ_ASSERT(aSurface); // Check the aCropRect ErrorResult error; const IntRect positiveCropRect = FixUpNegativeDimension(aCropRect, error); if (NS_WARN_IF(error.Failed())) { error.SuppressException(); return nullptr; } // Calculate the size of the new SourceSurface. // We cannot keep using aSurface->GetFormat() to create new DataSourceSurface, // since it might be SurfaceFormat::B8G8R8X8 which does not handle opacity, // however the specification explicitly define that "If any of the pixels on // this rectangle are outside the area where the input bitmap was placed, then // they will be transparent black in output." // So, instead, we force the output format to be SurfaceFormat::B8G8R8A8. const SurfaceFormat format = SurfaceFormat::B8G8R8A8; const int bytesPerPixel = BytesPerPixel(format); const IntSize dstSize = IntSize(positiveCropRect.width, positiveCropRect.height); const uint32_t dstStride = dstSize.width * bytesPerPixel; // Create a new SourceSurface. RefPtr dstDataSurface = Factory::CreateDataSourceSurfaceWithStride(dstSize, format, dstStride, true); if (NS_WARN_IF(!dstDataSurface)) { return nullptr; } // Only do copying and cropping when the positiveCropRect intersects with // the size of aSurface. const IntRect surfRect(IntPoint(0, 0), aSurface->GetSize()); if (surfRect.Intersects(positiveCropRect)) { const IntRect surfPortion = surfRect.Intersect(positiveCropRect); const IntPoint dest(std::max(0, surfPortion.X() - positiveCropRect.X()), std::max(0, surfPortion.Y() - positiveCropRect.Y())); // Copy the raw data into the newly created DataSourceSurface. DataSourceSurface::ScopedMap srcMap(aSurface, DataSourceSurface::READ); DataSourceSurface::ScopedMap dstMap(dstDataSurface, DataSourceSurface::WRITE); if (NS_WARN_IF(!srcMap.IsMapped()) || NS_WARN_IF(!dstMap.IsMapped())) { return nullptr; } uint8_t* srcBufferPtr = srcMap.GetData() + surfPortion.y * srcMap.GetStride() + surfPortion.x * bytesPerPixel; uint8_t* dstBufferPtr = dstMap.GetData() + dest.y * dstMap.GetStride() + dest.x * bytesPerPixel; CheckedInt copiedBytesPerRaw = CheckedInt(surfPortion.width) * bytesPerPixel; if (!copiedBytesPerRaw.isValid()) { return nullptr; } for (int i = 0; i < surfPortion.height; ++i) { memcpy(dstBufferPtr, srcBufferPtr, copiedBytesPerRaw.value()); srcBufferPtr += srcMap.GetStride(); dstBufferPtr += dstMap.GetStride(); } } return dstDataSurface.forget(); } /* * Encapsulate the given _aSurface_ into a layers::SourceSurfaceImage. */ static already_AddRefed CreateImageFromSurface(SourceSurface* aSurface) { MOZ_ASSERT(aSurface); RefPtr image = new layers::SourceSurfaceImage(aSurface->GetSize(), aSurface); return image.forget(); } /* * CreateImageFromRawData(), CreateSurfaceFromRawData() and * CreateImageFromRawDataInMainThreadSyncTask are helpers for * create-from-ImageData case */ static already_AddRefed CreateSurfaceFromRawData(const gfx::IntSize& aSize, uint32_t aStride, gfx::SurfaceFormat aFormat, uint8_t* aBuffer, uint32_t aBufferLength, const Maybe& aCropRect) { MOZ_ASSERT(!aSize.IsEmpty()); MOZ_ASSERT(aBuffer); // Wrap the source buffer into a SourceSurface. RefPtr dataSurface = Factory::CreateWrappingDataSourceSurface(aBuffer, aStride, aSize, aFormat); if (NS_WARN_IF(!dataSurface)) { return nullptr; } // The temporary cropRect variable is equal to the size of source buffer if we // do not need to crop, or it equals to the given cropping size. const IntRect cropRect = aCropRect.valueOr(IntRect(0, 0, aSize.width, aSize.height)); // Copy the source buffer in the _cropRect_ area into a new SourceSurface. RefPtr result = CropAndCopyDataSourceSurface(dataSurface, cropRect); if (NS_WARN_IF(!result)) { return nullptr; } return result.forget(); } static already_AddRefed CreateImageFromRawData(const gfx::IntSize& aSize, uint32_t aStride, gfx::SurfaceFormat aFormat, uint8_t* aBuffer, uint32_t aBufferLength, const Maybe& aCropRect) { MOZ_ASSERT(NS_IsMainThread()); // Copy and crop the source buffer into a SourceSurface. RefPtr rgbaSurface = CreateSurfaceFromRawData(aSize, aStride, aFormat, aBuffer, aBufferLength, aCropRect); if (NS_WARN_IF(!rgbaSurface)) { return nullptr; } // Convert RGBA to BGRA DataSourceSurface::MappedSurface rgbaMap; RefPtr rgbaDataSurface = rgbaSurface->GetDataSurface(); if (NS_WARN_IF(!rgbaDataSurface->Map(DataSourceSurface::MapType::READ, &rgbaMap))) { return nullptr; } RefPtr bgraDataSurface = Factory::CreateDataSourceSurfaceWithStride(rgbaDataSurface->GetSize(), SurfaceFormat::B8G8R8A8, rgbaMap.mStride); DataSourceSurface::MappedSurface bgraMap; if (NS_WARN_IF(!bgraDataSurface->Map(DataSourceSurface::MapType::WRITE, &bgraMap))) { return nullptr; } SwizzleData(rgbaMap.mData, rgbaMap.mStride, SurfaceFormat::R8G8B8A8, bgraMap.mData, bgraMap.mStride, SurfaceFormat::B8G8R8A8, bgraDataSurface->GetSize()); rgbaDataSurface->Unmap(); bgraDataSurface->Unmap(); // Create an Image from the BGRA SourceSurface. RefPtr image = CreateImageFromSurface(bgraDataSurface); if (NS_WARN_IF(!image)) { return nullptr; } return image.forget(); } /* * This is a synchronous task. * This class is used to create a layers::SourceSurfaceImage from raw data in the main * thread. While creating an ImageBitmap from an ImageData, we need to create * a SouceSurface from the ImageData's raw data and then set the SourceSurface * into a layers::SourceSurfaceImage. However, the layers::SourceSurfaceImage asserts the * setting operation in the main thread, so if we are going to create an * ImageBitmap from an ImageData off the main thread, we post an event to the * main thread to create a layers::SourceSurfaceImage from an ImageData's raw data. */ class CreateImageFromRawDataInMainThreadSyncTask final : public WorkerMainThreadRunnable { public: CreateImageFromRawDataInMainThreadSyncTask(uint8_t* aBuffer, uint32_t aBufferLength, uint32_t aStride, gfx::SurfaceFormat aFormat, const gfx::IntSize& aSize, const Maybe& aCropRect, layers::Image** aImage) : WorkerMainThreadRunnable(GetCurrentThreadWorkerPrivate(), NS_LITERAL_CSTRING("ImageBitmap :: Create Image from Raw Data")) , mImage(aImage) , mBuffer(aBuffer) , mBufferLength(aBufferLength) , mStride(aStride) , mFormat(aFormat) , mSize(aSize) , mCropRect(aCropRect) { MOZ_ASSERT(!(*aImage), "Don't pass an existing Image into CreateImageFromRawDataInMainThreadSyncTask."); } bool MainThreadRun() override { RefPtr image = CreateImageFromRawData(mSize, mStride, mFormat, mBuffer, mBufferLength, mCropRect); if (NS_WARN_IF(!image)) { return false; } image.forget(mImage); return true; } private: layers::Image** mImage; uint8_t* mBuffer; uint32_t mBufferLength; uint32_t mStride; gfx::SurfaceFormat mFormat; gfx::IntSize mSize; const Maybe& mCropRect; }; static bool CheckSecurityForHTMLElements(bool aIsWriteOnly, bool aCORSUsed, nsIPrincipal* aPrincipal) { MOZ_ASSERT(aPrincipal); if (aIsWriteOnly) { return false; } if (!aCORSUsed) { nsIGlobalObject* incumbentSettingsObject = GetIncumbentGlobal(); if (NS_WARN_IF(!incumbentSettingsObject)) { return false; } nsIPrincipal* principal = incumbentSettingsObject->PrincipalOrNull(); if (NS_WARN_IF(!principal) || !(principal->Subsumes(aPrincipal))) { return false; } } return true; } static bool CheckSecurityForHTMLElements(const nsLayoutUtils::SurfaceFromElementResult& aRes) { return CheckSecurityForHTMLElements(aRes.mIsWriteOnly, aRes.mCORSUsed, aRes.mPrincipal); } /* * A wrapper to the nsLayoutUtils::SurfaceFromElement() function followed by the * security checking. */ template static already_AddRefed GetSurfaceFromElement(nsIGlobalObject* aGlobal, HTMLElementType& aElement, ErrorResult& aRv) { nsLayoutUtils::SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromElement(&aElement, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE); // check origin-clean if (!CheckSecurityForHTMLElements(res)) { aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); return nullptr; } RefPtr surface = res.GetSourceSurface(); if (NS_WARN_IF(!surface)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } return surface.forget(); } /* * The specification doesn't allow to create an ImegeBitmap from a vector image. * This function is used to check if the given HTMLImageElement contains a * raster image. */ static bool HasRasterImage(HTMLImageElement& aImageEl) { nsresult rv; nsCOMPtr imgRequest; rv = aImageEl.GetRequest(nsIImageLoadingContent::CURRENT_REQUEST, getter_AddRefs(imgRequest)); if (NS_SUCCEEDED(rv) && imgRequest) { nsCOMPtr imgContainer; rv = imgRequest->GetImage(getter_AddRefs(imgContainer)); if (NS_SUCCEEDED(rv) && imgContainer && imgContainer->GetType() == imgIContainer::TYPE_RASTER) { return true; } } return false; } ImageBitmap::ImageBitmap(nsIGlobalObject* aGlobal, layers::Image* aData, gfxAlphaType aAlphaType) : mParent(aGlobal) , mData(aData) , mSurface(nullptr) , mDataWrapper(new ImageUtils(mData)) , mPictureRect(0, 0, aData->GetSize().width, aData->GetSize().height) , mAlphaType(aAlphaType) , mIsCroppingAreaOutSideOfSourceImage(false) , mAllocatedImageData(false) { MOZ_ASSERT(aData, "aData is null in ImageBitmap constructor."); } ImageBitmap::~ImageBitmap() { } JSObject* ImageBitmap::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { return ImageBitmapBinding::Wrap(aCx, this, aGivenProto); } void ImageBitmap::Close() { mData = nullptr; mSurface = nullptr; mPictureRect.SetEmpty(); } void ImageBitmap::SetPictureRect(const IntRect& aRect, ErrorResult& aRv) { mPictureRect = FixUpNegativeDimension(aRect, aRv); } void ImageBitmap::SetIsCroppingAreaOutSideOfSourceImage(const IntSize& aSourceSize, const Maybe& aCroppingRect) { // No cropping at all. if (aCroppingRect.isNothing()) { mIsCroppingAreaOutSideOfSourceImage = false; return; } if (aCroppingRect->X() < 0 || aCroppingRect->Y() < 0 || aCroppingRect->Width() > aSourceSize.width || aCroppingRect->Height() > aSourceSize.height) { mIsCroppingAreaOutSideOfSourceImage = true; } } static already_AddRefed ConvertColorFormatIfNeeded(RefPtr aSurface) { const SurfaceFormat srcFormat = aSurface->GetFormat(); if (srcFormat == SurfaceFormat::R8G8B8A8 || srcFormat == SurfaceFormat::B8G8R8A8 || srcFormat == SurfaceFormat::R8G8B8X8 || srcFormat == SurfaceFormat::B8G8R8X8 || srcFormat == SurfaceFormat::A8R8G8B8 || srcFormat == SurfaceFormat::X8R8G8B8) { return aSurface.forget(); } if (srcFormat == SurfaceFormat::A8 || srcFormat == SurfaceFormat::Depth) { return nullptr; } const int bytesPerPixel = BytesPerPixel(SurfaceFormat::B8G8R8A8); const IntSize dstSize = aSurface->GetSize(); const uint32_t dstStride = dstSize.width * bytesPerPixel; RefPtr dstDataSurface = Factory::CreateDataSourceSurfaceWithStride(dstSize, SurfaceFormat::B8G8R8A8, dstStride); RefPtr srcDataSurface = aSurface->GetDataSurface(); if (NS_WARN_IF(!srcDataSurface)) { return nullptr; } DataSourceSurface::ScopedMap srcMap(srcDataSurface, DataSourceSurface::READ); DataSourceSurface::ScopedMap dstMap(dstDataSurface, DataSourceSurface::WRITE); if (NS_WARN_IF(!srcMap.IsMapped()) || NS_WARN_IF(!dstMap.IsMapped())) { return nullptr; } int rv = 0; if (srcFormat == SurfaceFormat::R8G8B8) { rv = RGB24ToBGRA32(srcMap.GetData(), srcMap.GetStride(), dstMap.GetData(), dstMap.GetStride(), dstSize.width, dstSize.height); } else if (srcFormat == SurfaceFormat::B8G8R8) { rv = BGR24ToBGRA32(srcMap.GetData(), srcMap.GetStride(), dstMap.GetData(), dstMap.GetStride(), dstSize.width, dstSize.height); } else if (srcFormat == SurfaceFormat::HSV) { rv = HSVToBGRA32((const float*)srcMap.GetData(), srcMap.GetStride(), dstMap.GetData(), dstMap.GetStride(), dstSize.width, dstSize.height); } else if (srcFormat == SurfaceFormat::Lab) { rv = LabToBGRA32((const float*)srcMap.GetData(), srcMap.GetStride(), dstMap.GetData(), dstMap.GetStride(), dstSize.width, dstSize.height); } if (NS_WARN_IF(rv != 0)) { return nullptr; } return dstDataSurface.forget(); } /* * The functionality of PrepareForDrawTarget method: * (1) Get a SourceSurface from the mData (which is a layers::Image). * (2) Convert the SourceSurface to format B8G8R8A8 if the original format is * R8G8B8, B8G8R8, HSV or Lab. * Note: if the original format is A8 or Depth, then return null directly. * (3) Do cropping if the size of SourceSurface does not equal to the * mPictureRect. * (4) Pre-multiply alpha if needed. */ already_AddRefed ImageBitmap::PrepareForDrawTarget(gfx::DrawTarget* aTarget) { MOZ_ASSERT(aTarget); if (!mData) { return nullptr; } if (!mSurface) { mSurface = mData->GetAsSourceSurface(); if (!mSurface) { return nullptr; } } // Check if we need to convert the format. // Convert R8G8B8/B8G8R8/HSV/Lab to B8G8R8A8. // Return null if the original format is A8 or Depth. mSurface = ConvertColorFormatIfNeeded(mSurface); if (NS_WARN_IF(!mSurface)) { return nullptr; } RefPtr target = aTarget; IntRect surfRect(0, 0, mSurface->GetSize().width, mSurface->GetSize().height); // Check if we still need to crop our surface if (!mPictureRect.IsEqualEdges(surfRect)) { IntRect surfPortion = surfRect.Intersect(mPictureRect); // the crop lies entirely outside the surface area, nothing to draw if (surfPortion.IsEmpty()) { mSurface = nullptr; RefPtr surface(mSurface); return surface.forget(); } IntPoint dest(std::max(0, surfPortion.X() - mPictureRect.X()), std::max(0, surfPortion.Y() - mPictureRect.Y())); // We must initialize this target with mPictureRect.Size() because the // specification states that if the cropping area is given, then return an // ImageBitmap with the size equals to the cropping area. target = target->CreateSimilarDrawTarget(mPictureRect.Size(), target->GetFormat()); if (!target) { mSurface = nullptr; RefPtr surface(mSurface); return surface.forget(); } target->CopySurface(mSurface, surfPortion, dest); mSurface = target->Snapshot(); // Make mCropRect match new surface we've cropped to mPictureRect.MoveTo(0, 0); } // Pre-multiply alpha here. // Ignore this step if the source surface does not have alpha channel; this // kind of source surfaces might come form layers::PlanarYCbCrImage. if (mAlphaType == gfxAlphaType::NonPremult && !IsOpaque(mSurface->GetFormat())) { MOZ_ASSERT(mSurface->GetFormat() == SurfaceFormat::R8G8B8A8 || mSurface->GetFormat() == SurfaceFormat::B8G8R8A8 || mSurface->GetFormat() == SurfaceFormat::A8R8G8B8); RefPtr dstSurface = mSurface->GetDataSurface(); MOZ_ASSERT(dstSurface); RefPtr srcSurface; DataSourceSurface::MappedSurface srcMap; DataSourceSurface::MappedSurface dstMap; if (dstSurface->Map(DataSourceSurface::MapType::READ_WRITE, &dstMap)) { srcMap = dstMap; } else { srcSurface = dstSurface; if (!srcSurface->Map(DataSourceSurface::READ, &srcMap)) { gfxCriticalError() << "Failed to map source surface for premultiplying alpha."; return nullptr; } dstSurface = Factory::CreateDataSourceSurface(srcSurface->GetSize(), srcSurface->GetFormat()); if (!dstSurface || !dstSurface->Map(DataSourceSurface::MapType::WRITE, &dstMap)) { gfxCriticalError() << "Failed to map destination surface for premultiplying alpha."; srcSurface->Unmap(); return nullptr; } } PremultiplyData(srcMap.mData, srcMap.mStride, mSurface->GetFormat(), dstMap.mData, dstMap.mStride, mSurface->GetFormat(), dstSurface->GetSize()); dstSurface->Unmap(); if (srcSurface) { srcSurface->Unmap(); } mSurface = dstSurface; } // Replace our surface with one optimized for the target we're about to draw // to, under the assumption it'll likely be drawn again to that target. // This call should be a no-op for already-optimized surfaces mSurface = target->OptimizeSourceSurface(mSurface); RefPtr surface(mSurface); return surface.forget(); } already_AddRefed ImageBitmap::TransferAsImage() { RefPtr image = mData; Close(); return image.forget(); } UniquePtr ImageBitmap::ToCloneData() const { UniquePtr result(new ImageBitmapCloneData()); result->mPictureRect = mPictureRect; result->mAlphaType = mAlphaType; result->mIsCroppingAreaOutSideOfSourceImage = mIsCroppingAreaOutSideOfSourceImage; RefPtr surface = mData->GetAsSourceSurface(); result->mSurface = surface->GetDataSurface(); MOZ_ASSERT(result->mSurface); return Move(result); } /* static */ already_AddRefed ImageBitmap::CreateFromCloneData(nsIGlobalObject* aGlobal, ImageBitmapCloneData* aData) { RefPtr data = CreateImageFromSurface(aData->mSurface); RefPtr ret = new ImageBitmap(aGlobal, data, aData->mAlphaType); ret->mAllocatedImageData = true; ret->mIsCroppingAreaOutSideOfSourceImage = aData->mIsCroppingAreaOutSideOfSourceImage; ErrorResult rv; ret->SetPictureRect(aData->mPictureRect, rv); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateFromOffscreenCanvas(nsIGlobalObject* aGlobal, OffscreenCanvas& aOffscreenCanvas, ErrorResult& aRv) { // Check origin-clean. if (aOffscreenCanvas.IsWriteOnly()) { aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); return nullptr; } nsLayoutUtils::SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromOffscreenCanvas(&aOffscreenCanvas, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE); RefPtr surface = res.GetSourceSurface(); if (NS_WARN_IF(!surface)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr data = CreateImageFromSurface(surface); RefPtr ret = new ImageBitmap(aGlobal, data); ret->mAllocatedImageData = true; return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, HTMLImageElement& aImageEl, const Maybe& aCropRect, ErrorResult& aRv) { // Check if the image element is completely available or not. if (!aImageEl.Complete()) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Check if the image element is a bitmap (e.g. it's a vector graphic) or not. if (!HasRasterImage(aImageEl)) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Get the SourceSurface out from the image element and then do security // checking. RefPtr surface = GetSurfaceFromElement(aGlobal, aImageEl, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } // Create ImageBitmap. RefPtr data = CreateImageFromSurface(surface); if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr ret = new ImageBitmap(aGlobal, data); // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(aCropRect.ref(), aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(surface->GetSize(), aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, HTMLVideoElement& aVideoEl, const Maybe& aCropRect, ErrorResult& aRv) { aVideoEl.MarkAsContentSource(mozilla::dom::HTMLVideoElement::CallerAPI::CREATE_IMAGEBITMAP); // Check network state. if (aVideoEl.NetworkState() == HTMLMediaElement::NETWORK_EMPTY) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Check ready state. // Cannot be HTMLMediaElement::HAVE_NOTHING or HTMLMediaElement::HAVE_METADATA. if (aVideoEl.ReadyState() <= HTMLMediaElement::HAVE_METADATA) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Check security. nsCOMPtr principal = aVideoEl.GetCurrentVideoPrincipal(); bool CORSUsed = aVideoEl.GetCORSMode() != CORS_NONE; if (!CheckSecurityForHTMLElements(false, CORSUsed, principal)) { aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); return nullptr; } // Create ImageBitmap. RefPtr data = aVideoEl.GetCurrentImage(); if (!data) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr ret = new ImageBitmap(aGlobal, data); // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(aCropRect.ref(), aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(data->GetSize(), aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, HTMLCanvasElement& aCanvasEl, const Maybe& aCropRect, ErrorResult& aRv) { if (aCanvasEl.Width() == 0 || aCanvasEl.Height() == 0) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } RefPtr surface = GetSurfaceFromElement(aGlobal, aCanvasEl, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } // Crop the source surface if needed. RefPtr croppedSurface; IntRect cropRect = aCropRect.valueOr(IntRect()); // If the HTMLCanvasElement's rendering context is WebGL, then the snapshot // we got from the HTMLCanvasElement is a DataSourceSurface which is a copy // of the rendering context. We handle cropping in this case. bool needToReportMemoryAllocation = false; if ((aCanvasEl.GetCurrentContextType() == CanvasContextType::WebGL1 || aCanvasEl.GetCurrentContextType() == CanvasContextType::WebGL2) && aCropRect.isSome()) { RefPtr dataSurface = surface->GetDataSurface(); croppedSurface = CropAndCopyDataSourceSurface(dataSurface, cropRect); cropRect.MoveTo(0, 0); needToReportMemoryAllocation = true; } else { croppedSurface = surface; } if (NS_WARN_IF(!croppedSurface)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } // Create an Image from the SourceSurface. RefPtr data = CreateImageFromSurface(croppedSurface); if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr ret = new ImageBitmap(aGlobal, data); if (needToReportMemoryAllocation) { ret->mAllocatedImageData = true; } // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(cropRect, aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(surface->GetSize(), aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, ImageData& aImageData, const Maybe& aCropRect, ErrorResult& aRv) { // Copy data into SourceSurface. dom::Uint8ClampedArray array; DebugOnly inited = array.Init(aImageData.GetDataObject()); MOZ_ASSERT(inited); array.ComputeLengthAndData(); const SurfaceFormat FORMAT = SurfaceFormat::R8G8B8A8; // ImageData's underlying data is not alpha-premultiplied. const auto alphaType = gfxAlphaType::NonPremult; const uint32_t BYTES_PER_PIXEL = BytesPerPixel(FORMAT); const uint32_t imageWidth = aImageData.Width(); const uint32_t imageHeight = aImageData.Height(); const uint32_t imageStride = imageWidth * BYTES_PER_PIXEL; const uint32_t dataLength = array.Length(); const gfx::IntSize imageSize(imageWidth, imageHeight); // Check the ImageData is neutered or not. if (imageWidth == 0 || imageHeight == 0 || (imageWidth * imageHeight * BYTES_PER_PIXEL) != dataLength) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } // Create and Crop the raw data into a layers::Image RefPtr data; if (NS_IsMainThread()) { data = CreateImageFromRawData(imageSize, imageStride, FORMAT, array.Data(), dataLength, aCropRect); } else { RefPtr task = new CreateImageFromRawDataInMainThreadSyncTask(array.Data(), dataLength, imageStride, FORMAT, imageSize, aCropRect, getter_AddRefs(data)); task->Dispatch(Terminating, aRv); } if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } // Create an ImageBimtap. RefPtr ret = new ImageBitmap(aGlobal, data, alphaType); ret->mAllocatedImageData = true; // The cropping information has been handled in the CreateImageFromRawData() // function. // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(imageSize, aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, CanvasRenderingContext2D& aCanvasCtx, const Maybe& aCropRect, ErrorResult& aRv) { // Check origin-clean. if (aCanvasCtx.GetCanvas()->IsWriteOnly()) { aRv.Throw(NS_ERROR_DOM_SECURITY_ERR); return nullptr; } RefPtr surface = aCanvasCtx.GetSurfaceSnapshot(); if (NS_WARN_IF(!surface)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } const IntSize surfaceSize = surface->GetSize(); if (surfaceSize.width == 0 || surfaceSize.height == 0) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return nullptr; } RefPtr data = CreateImageFromSurface(surface); if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr ret = new ImageBitmap(aGlobal, data); ret->mAllocatedImageData = true; // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(aCropRect.ref(), aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. ret->SetIsCroppingAreaOutSideOfSourceImage(surface->GetSize(), aCropRect); return ret.forget(); } /* static */ already_AddRefed ImageBitmap::CreateInternal(nsIGlobalObject* aGlobal, ImageBitmap& aImageBitmap, const Maybe& aCropRect, ErrorResult& aRv) { if (!aImageBitmap.mData) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return nullptr; } RefPtr data = aImageBitmap.mData; RefPtr ret = new ImageBitmap(aGlobal, data, aImageBitmap.mAlphaType); // Set the picture rectangle. if (ret && aCropRect.isSome()) { ret->SetPictureRect(aCropRect.ref(), aRv); } // Set mIsCroppingAreaOutSideOfSourceImage. if (aImageBitmap.mIsCroppingAreaOutSideOfSourceImage == true) { ret->mIsCroppingAreaOutSideOfSourceImage = true; } else { ret->SetIsCroppingAreaOutSideOfSourceImage(aImageBitmap.mPictureRect.Size(), aCropRect); } return ret.forget(); } class FulfillImageBitmapPromise { protected: FulfillImageBitmapPromise(Promise* aPromise, ImageBitmap* aImageBitmap) : mPromise(aPromise) , mImageBitmap(aImageBitmap) { MOZ_ASSERT(aPromise); } void DoFulfillImageBitmapPromise() { mPromise->MaybeResolve(mImageBitmap); } private: RefPtr mPromise; RefPtr mImageBitmap; }; class FulfillImageBitmapPromiseTask final : public Runnable, public FulfillImageBitmapPromise { public: FulfillImageBitmapPromiseTask(Promise* aPromise, ImageBitmap* aImageBitmap) : Runnable("dom::FulfillImageBitmapPromiseTask") , FulfillImageBitmapPromise(aPromise, aImageBitmap) { } NS_IMETHOD Run() override { DoFulfillImageBitmapPromise(); return NS_OK; } }; class FulfillImageBitmapPromiseWorkerTask final : public WorkerSameThreadRunnable, public FulfillImageBitmapPromise { public: FulfillImageBitmapPromiseWorkerTask(Promise* aPromise, ImageBitmap* aImageBitmap) : WorkerSameThreadRunnable(GetCurrentThreadWorkerPrivate()), FulfillImageBitmapPromise(aPromise, aImageBitmap) { } bool WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override { DoFulfillImageBitmapPromise(); return true; } }; static void AsyncFulfillImageBitmapPromise(Promise* aPromise, ImageBitmap* aImageBitmap) { if (NS_IsMainThread()) { nsCOMPtr task = new FulfillImageBitmapPromiseTask(aPromise, aImageBitmap); NS_DispatchToCurrentThread(task); // Actually, to the main-thread. } else { RefPtr task = new FulfillImageBitmapPromiseWorkerTask(aPromise, aImageBitmap); task->Dispatch(); // Actually, to the current worker-thread. } } class CreateImageBitmapFromBlobRunnable; class CreateImageBitmapFromBlobHolder; class CreateImageBitmapFromBlob final : public CancelableRunnable , public imgIContainerCallback { friend class CreateImageBitmapFromBlobRunnable; public: NS_DECL_ISUPPORTS_INHERITED NS_DECL_IMGICONTAINERCALLBACK static already_AddRefed Create(Promise* aPromise, nsIGlobalObject* aGlobal, Blob& aBlob, const Maybe& aCropRect, nsIEventTarget* aMainThreadEventTarget); NS_IMETHOD Run() override { MOZ_ASSERT(IsCurrentThread()); nsresult rv = StartDecodeAndCropBlob(); if (NS_WARN_IF(NS_FAILED(rv))) { DecodeAndCropBlobCompletedMainThread(nullptr, rv); } return NS_OK; } // Called by the WorkerHolder. void WorkerShuttingDown(); private: CreateImageBitmapFromBlob(Promise* aPromise, nsIGlobalObject* aGlobal, already_AddRefed aInputStream, const nsACString& aMimeType, const Maybe& aCropRect, nsIEventTarget* aMainThreadEventTarget) : CancelableRunnable("dom::CreateImageBitmapFromBlob") , mMutex("dom::CreateImageBitmapFromBlob::mMutex") , mPromise(aPromise) , mGlobalObject(aGlobal) , mInputStream(Move(aInputStream)) , mMimeType(aMimeType) , mCropRect(aCropRect) , mOriginalCropRect(aCropRect) , mMainThreadEventTarget(aMainThreadEventTarget) , mThread(GetCurrentVirtualThread()) { } virtual ~CreateImageBitmapFromBlob() { } bool IsCurrentThread() const { return mThread == GetCurrentVirtualThread(); } // Called on the owning thread. nsresult StartDecodeAndCropBlob(); // Will be called when the decoding + cropping is completed on the // main-thread. This could the not the owning thread! void DecodeAndCropBlobCompletedMainThread(layers::Image* aImage, nsresult aStatus); // Will be called when the decoding + cropping is completed on the owning // thread. void DecodeAndCropBlobCompletedOwningThread(layers::Image* aImage, nsresult aStatus); // This is called on the main-thread only. nsresult DecodeAndCropBlob(); Mutex mMutex; // The access to this object is protected by mutex but is always nullified on // the owning thread. UniquePtr mWorkerHolder; // Touched only on the owning thread. RefPtr mPromise; // Touched only on the owning thread. nsCOMPtr mGlobalObject; nsCOMPtr mInputStream; nsCString mMimeType; Maybe mCropRect; Maybe mOriginalCropRect; IntSize mSourceSize; nsCOMPtr mMainThreadEventTarget; void* mThread; }; NS_IMPL_ISUPPORTS_INHERITED(CreateImageBitmapFromBlob, CancelableRunnable, imgIContainerCallback) class CreateImageBitmapFromBlobRunnable : public WorkerRunnable { public: explicit CreateImageBitmapFromBlobRunnable(WorkerPrivate* aWorkerPrivate, CreateImageBitmapFromBlob* aTask, layers::Image* aImage, nsresult aStatus) : WorkerRunnable(aWorkerPrivate) , mTask(aTask) , mImage(aImage) , mStatus(aStatus) {} bool WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override { mTask->DecodeAndCropBlobCompletedOwningThread(mImage, mStatus); return true; } private: RefPtr mTask; RefPtr mImage; nsresult mStatus; }; // This class keeps the worker alive and it informs CreateImageBitmapFromBlob // when it goes away. class CreateImageBitmapFromBlobHolder final : public WorkerHolder { public: CreateImageBitmapFromBlobHolder(WorkerPrivate* aWorkerPrivate, CreateImageBitmapFromBlob* aTask) : WorkerHolder("CreateImageBitmapFromBlobHolder") , mWorkerPrivate(aWorkerPrivate) , mTask(aTask) , mNotified(false) {} bool Notify(Status aStatus) override { if (!mNotified) { mNotified = true; mTask->WorkerShuttingDown(); } return true; } WorkerPrivate* GetWorkerPrivate() const { return mWorkerPrivate; } private: WorkerPrivate* mWorkerPrivate; RefPtr mTask; bool mNotified; }; static void AsyncCreateImageBitmapFromBlob(Promise* aPromise, nsIGlobalObject* aGlobal, Blob& aBlob, const Maybe& aCropRect) { // Let's identify the main-thread event target. nsCOMPtr mainThreadEventTarget; if (NS_IsMainThread()) { mainThreadEventTarget = aGlobal->EventTargetFor(TaskCategory::Other); } else { WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate(); MOZ_ASSERT(workerPrivate); mainThreadEventTarget = workerPrivate->MainThreadEventTarget(); } RefPtr task = CreateImageBitmapFromBlob::Create(aPromise, aGlobal, aBlob, aCropRect, mainThreadEventTarget); if (NS_WARN_IF(!task)) { aPromise->MaybeReject(NS_ERROR_DOM_INVALID_STATE_ERR); return; } NS_DispatchToCurrentThread(task); } /* static */ already_AddRefed ImageBitmap::Create(nsIGlobalObject* aGlobal, const ImageBitmapSource& aSrc, const Maybe& aCropRect, ErrorResult& aRv) { MOZ_ASSERT(aGlobal); RefPtr promise = Promise::Create(aGlobal, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } if (aCropRect.isSome() && (aCropRect->Width() == 0 || aCropRect->Height() == 0)) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return promise.forget(); } RefPtr imageBitmap; if (aSrc.IsHTMLImageElement()) { MOZ_ASSERT(NS_IsMainThread(), "Creating ImageBitmap from HTMLImageElement off the main thread."); imageBitmap = CreateInternal(aGlobal, aSrc.GetAsHTMLImageElement(), aCropRect, aRv); } else if (aSrc.IsHTMLVideoElement()) { MOZ_ASSERT(NS_IsMainThread(), "Creating ImageBitmap from HTMLVideoElement off the main thread."); imageBitmap = CreateInternal(aGlobal, aSrc.GetAsHTMLVideoElement(), aCropRect, aRv); } else if (aSrc.IsHTMLCanvasElement()) { MOZ_ASSERT(NS_IsMainThread(), "Creating ImageBitmap from HTMLCanvasElement off the main thread."); imageBitmap = CreateInternal(aGlobal, aSrc.GetAsHTMLCanvasElement(), aCropRect, aRv); } else if (aSrc.IsImageData()) { imageBitmap = CreateInternal(aGlobal, aSrc.GetAsImageData(), aCropRect, aRv); } else if (aSrc.IsCanvasRenderingContext2D()) { MOZ_ASSERT(NS_IsMainThread(), "Creating ImageBitmap from CanvasRenderingContext2D off the main thread."); imageBitmap = CreateInternal(aGlobal, aSrc.GetAsCanvasRenderingContext2D(), aCropRect, aRv); } else if (aSrc.IsImageBitmap()) { imageBitmap = CreateInternal(aGlobal, aSrc.GetAsImageBitmap(), aCropRect, aRv); } else if (aSrc.IsBlob()) { AsyncCreateImageBitmapFromBlob(promise, aGlobal, aSrc.GetAsBlob(), aCropRect); return promise.forget(); } else { aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); return nullptr; } if (!aRv.Failed()) { AsyncFulfillImageBitmapPromise(promise, imageBitmap); } return promise.forget(); } /*static*/ JSObject* ImageBitmap::ReadStructuredClone(JSContext* aCx, JSStructuredCloneReader* aReader, nsIGlobalObject* aParent, const nsTArray>& aClonedSurfaces, uint32_t aIndex) { MOZ_ASSERT(aCx); MOZ_ASSERT(aReader); // aParent might be null. uint32_t picRectX_; uint32_t picRectY_; uint32_t picRectWidth_; uint32_t picRectHeight_; uint32_t alphaType_; uint32_t isCroppingAreaOutSideOfSourceImage_; if (!JS_ReadUint32Pair(aReader, &picRectX_, &picRectY_) || !JS_ReadUint32Pair(aReader, &picRectWidth_, &picRectHeight_) || !JS_ReadUint32Pair(aReader, &alphaType_, &isCroppingAreaOutSideOfSourceImage_)) { return nullptr; } int32_t picRectX = BitwiseCast(picRectX_); int32_t picRectY = BitwiseCast(picRectY_); int32_t picRectWidth = BitwiseCast(picRectWidth_); int32_t picRectHeight = BitwiseCast(picRectHeight_); const auto alphaType = BitwiseCast(alphaType_); // Create a new ImageBitmap. MOZ_ASSERT(!aClonedSurfaces.IsEmpty()); MOZ_ASSERT(aIndex < aClonedSurfaces.Length()); // RefPtr needs to go out of scope before toObjectOrNull() is // called because the static analysis thinks dereferencing XPCOM objects // can GC (because in some cases it can!), and a return statement with a // JSObject* type means that JSObject* is on the stack as a raw pointer // while destructors are running. JS::Rooted value(aCx); { RefPtr img = CreateImageFromSurface(aClonedSurfaces[aIndex]); RefPtr imageBitmap = new ImageBitmap(aParent, img, alphaType); imageBitmap->mIsCroppingAreaOutSideOfSourceImage = isCroppingAreaOutSideOfSourceImage_; ErrorResult error; imageBitmap->SetPictureRect(IntRect(picRectX, picRectY, picRectWidth, picRectHeight), error); if (NS_WARN_IF(error.Failed())) { error.SuppressException(); return nullptr; } if (!GetOrCreateDOMReflector(aCx, imageBitmap, &value)) { return nullptr; } imageBitmap->mAllocatedImageData = true; } return &(value.toObject()); } /*static*/ bool ImageBitmap::WriteStructuredClone(JSStructuredCloneWriter* aWriter, nsTArray>& aClonedSurfaces, ImageBitmap* aImageBitmap) { MOZ_ASSERT(aWriter); MOZ_ASSERT(aImageBitmap); const uint32_t picRectX = BitwiseCast(aImageBitmap->mPictureRect.x); const uint32_t picRectY = BitwiseCast(aImageBitmap->mPictureRect.y); const uint32_t picRectWidth = BitwiseCast(aImageBitmap->mPictureRect.width); const uint32_t picRectHeight = BitwiseCast(aImageBitmap->mPictureRect.height); const uint32_t alphaType = BitwiseCast(aImageBitmap->mAlphaType); const uint32_t isCroppingAreaOutSideOfSourceImage = aImageBitmap->mIsCroppingAreaOutSideOfSourceImage ? 1 : 0; // Indexing the cloned surfaces and send the index to the receiver. uint32_t index = aClonedSurfaces.Length(); if (NS_WARN_IF(!JS_WriteUint32Pair(aWriter, SCTAG_DOM_IMAGEBITMAP, index)) || NS_WARN_IF(!JS_WriteUint32Pair(aWriter, picRectX, picRectY)) || NS_WARN_IF(!JS_WriteUint32Pair(aWriter, picRectWidth, picRectHeight)) || NS_WARN_IF(!JS_WriteUint32Pair(aWriter, alphaType, isCroppingAreaOutSideOfSourceImage))) { return false; } RefPtr surface = aImageBitmap->mData->GetAsSourceSurface(); RefPtr snapshot = surface->GetDataSurface(); RefPtr dstDataSurface; { // DataSourceSurfaceD2D1::GetStride() will call EnsureMapped implicitly and // won't Unmap after exiting function. So instead calling GetStride() // directly, using ScopedMap to get stride. DataSourceSurface::ScopedMap map(snapshot, DataSourceSurface::READ); dstDataSurface = Factory::CreateDataSourceSurfaceWithStride(snapshot->GetSize(), snapshot->GetFormat(), map.GetStride(), true); } MOZ_ASSERT(dstDataSurface); Factory::CopyDataSourceSurface(snapshot, dstDataSurface); aClonedSurfaces.AppendElement(dstDataSurface); return true; } /*static*/ bool ImageBitmap::ExtensionsEnabled(JSContext* aCx, JSObject*) { if (NS_IsMainThread()) { return Preferences::GetBool("canvas.imagebitmap_extensions.enabled"); } else { WorkerPrivate* workerPrivate = GetWorkerPrivateFromContext(aCx); MOZ_ASSERT(workerPrivate); return workerPrivate->ImageBitmapExtensionsEnabled(); } } // ImageBitmap extensions. ImageBitmapFormat ImageBitmap::FindOptimalFormat(const Optional>& aPossibleFormats, ErrorResult& aRv) { MOZ_ASSERT(mDataWrapper, "No ImageBitmapFormatUtils functionalities."); ImageBitmapFormat platformFormat = mDataWrapper->GetFormat(); if (!aPossibleFormats.WasPassed() || aPossibleFormats.Value().Contains(platformFormat)) { return platformFormat; } else { // If no matching is found, FindBestMatchingFromat() returns // ImageBitmapFormat::EndGuard_ and we throw an exception. ImageBitmapFormat optimalFormat = FindBestMatchingFromat(platformFormat, aPossibleFormats.Value()); if (optimalFormat == ImageBitmapFormat::EndGuard_) { aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); } return optimalFormat; } } int32_t ImageBitmap::MappedDataLength(ImageBitmapFormat aFormat, ErrorResult& aRv) { MOZ_ASSERT(mDataWrapper, "No ImageBitmapFormatUtils functionalities."); if (aFormat == mDataWrapper->GetFormat()) { return mDataWrapper->GetBufferLength(); } else { return CalculateImageBufferSize(aFormat, Width(), Height()); } } template class MapDataIntoBufferSource { protected: MapDataIntoBufferSource(JSContext* aCx, Promise *aPromise, ImageBitmap *aImageBitmap, const T& aBuffer, int32_t aOffset, ImageBitmapFormat aFormat) : mPromise(aPromise) , mImageBitmap(aImageBitmap) , mBuffer(aCx, aBuffer.Obj()) , mOffset(aOffset) , mFormat(aFormat) { MOZ_ASSERT(mPromise); MOZ_ASSERT(JS_IsArrayBufferObject(mBuffer) || JS_IsArrayBufferViewObject(mBuffer)); } virtual ~MapDataIntoBufferSource() = default; void DoMapDataIntoBufferSource() { ErrorResult error; // Prepare destination buffer. uint8_t* bufferData = nullptr; uint32_t bufferLength = 0; bool isSharedMemory = false; if (JS_IsArrayBufferObject(mBuffer)) { js::GetArrayBufferLengthAndData(mBuffer, &bufferLength, &isSharedMemory, &bufferData); } else if (JS_IsArrayBufferViewObject(mBuffer)) { js::GetArrayBufferViewLengthAndData(mBuffer, &bufferLength, &isSharedMemory, &bufferData); } else { error.ThrowWithCustomCleanup(NS_ERROR_NOT_IMPLEMENTED); mPromise->MaybeReject(error); return; } if (NS_WARN_IF(!bufferData) || NS_WARN_IF(!bufferLength)) { error.ThrowWithCustomCleanup(NS_ERROR_NOT_AVAILABLE); mPromise->MaybeReject(error); return; } // Check length. const int32_t neededBufferLength = mImageBitmap->MappedDataLength(mFormat, error); if (((int32_t)bufferLength - mOffset) < neededBufferLength) { error.ThrowWithCustomCleanup(NS_ERROR_DOM_INDEX_SIZE_ERR); mPromise->MaybeReject(error); return; } // Call ImageBitmapFormatUtils. UniquePtr layout = mImageBitmap->mDataWrapper->MapDataInto(bufferData, mOffset, bufferLength, mFormat, error); if (NS_WARN_IF(!layout)) { mPromise->MaybeReject(error); return; } mPromise->MaybeResolve(*layout); } RefPtr mPromise; RefPtr mImageBitmap; JS::PersistentRooted mBuffer; int32_t mOffset; ImageBitmapFormat mFormat; }; template class MapDataIntoBufferSourceTask final : public Runnable, public MapDataIntoBufferSource { public: MapDataIntoBufferSourceTask(JSContext* aCx, Promise* aPromise, ImageBitmap* aImageBitmap, const T& aBuffer, int32_t aOffset, ImageBitmapFormat aFormat) : Runnable("dom::MapDataIntoBufferSourceTask") , MapDataIntoBufferSource(aCx, aPromise, aImageBitmap, aBuffer, aOffset, aFormat) { } virtual ~MapDataIntoBufferSourceTask() = default; NS_IMETHOD Run() override { MapDataIntoBufferSource::DoMapDataIntoBufferSource(); return NS_OK; } }; template class MapDataIntoBufferSourceWorkerTask final : public WorkerSameThreadRunnable, public MapDataIntoBufferSource { public: MapDataIntoBufferSourceWorkerTask(JSContext* aCx, Promise *aPromise, ImageBitmap *aImageBitmap, const T& aBuffer, int32_t aOffset, ImageBitmapFormat aFormat) : WorkerSameThreadRunnable(GetCurrentThreadWorkerPrivate()), MapDataIntoBufferSource(aCx, aPromise, aImageBitmap, aBuffer, aOffset, aFormat) { } virtual ~MapDataIntoBufferSourceWorkerTask() = default; bool WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override { MapDataIntoBufferSource::DoMapDataIntoBufferSource(); return true; } }; void AsyncMapDataIntoBufferSource(JSContext* aCx, Promise *aPromise, ImageBitmap *aImageBitmap, const ArrayBufferViewOrArrayBuffer& aBuffer, int32_t aOffset, ImageBitmapFormat aFormat) { MOZ_ASSERT(aCx); MOZ_ASSERT(aPromise); MOZ_ASSERT(aImageBitmap); if (NS_IsMainThread()) { nsCOMPtr task; if (aBuffer.IsArrayBuffer()) { const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer(); task = new MapDataIntoBufferSourceTask(aCx, aPromise, aImageBitmap, buffer, aOffset, aFormat); } else if (aBuffer.IsArrayBufferView()) { const ArrayBufferView& bufferView = aBuffer.GetAsArrayBufferView(); task = new MapDataIntoBufferSourceTask(aCx, aPromise, aImageBitmap, bufferView, aOffset, aFormat); } NS_DispatchToCurrentThread(task); // Actually, to the main-thread. } else { RefPtr task; if (aBuffer.IsArrayBuffer()) { const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer(); task = new MapDataIntoBufferSourceWorkerTask(aCx, aPromise, aImageBitmap, buffer, aOffset, aFormat); } else if (aBuffer.IsArrayBufferView()) { const ArrayBufferView& bufferView = aBuffer.GetAsArrayBufferView(); task = new MapDataIntoBufferSourceWorkerTask(aCx, aPromise, aImageBitmap, bufferView, aOffset, aFormat); } task->Dispatch(); // Actually, to the current worker-thread. } } already_AddRefed ImageBitmap::MapDataInto(JSContext* aCx, ImageBitmapFormat aFormat, const ArrayBufferViewOrArrayBuffer& aBuffer, int32_t aOffset, ErrorResult& aRv) { MOZ_ASSERT(mDataWrapper, "No ImageBitmapFormatUtils functionalities."); MOZ_ASSERT(aCx, "No JSContext while calling ImageBitmap::MapDataInto()."); RefPtr promise = Promise::Create(mParent, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } // Check for cases that should throws. // Case 1: // If image bitmap was cropped to the source rectangle so that it contains any // transparent black pixels (cropping area is outside of the source image), // then reject promise with IndexSizeError and abort these steps. if (mIsCroppingAreaOutSideOfSourceImage) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return promise.forget(); } // Case 2: // If the image bitmap is going to be accessed in YUV422/YUV422 series with a // cropping area starts at an odd x or y coordinate. if (aFormat == ImageBitmapFormat::YUV422P || aFormat == ImageBitmapFormat::YUV420P || aFormat == ImageBitmapFormat::YUV420SP_NV12 || aFormat == ImageBitmapFormat::YUV420SP_NV21) { if ((mPictureRect.x & 1) || (mPictureRect.y & 1)) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return promise.forget(); } } AsyncMapDataIntoBufferSource(aCx, promise, this, aBuffer, aOffset, aFormat); return promise.forget(); } // ImageBitmapFactories extensions. static SurfaceFormat ImageFormatToSurfaceFromat(mozilla::dom::ImageBitmapFormat aFormat) { switch(aFormat) { case ImageBitmapFormat::RGBA32: return SurfaceFormat::R8G8B8A8; case ImageBitmapFormat::BGRA32: return SurfaceFormat::B8G8R8A8; case ImageBitmapFormat::RGB24: return SurfaceFormat::R8G8B8; case ImageBitmapFormat::BGR24: return SurfaceFormat::B8G8R8; case ImageBitmapFormat::GRAY8: return SurfaceFormat::A8; case ImageBitmapFormat::HSV: return SurfaceFormat::HSV; case ImageBitmapFormat::Lab: return SurfaceFormat::Lab; case ImageBitmapFormat::DEPTH: return SurfaceFormat::Depth; default: return SurfaceFormat::UNKNOWN; } } static already_AddRefed CreateImageFromBufferSourceRawData(const uint8_t*aBufferData, uint32_t aBufferLength, mozilla::dom::ImageBitmapFormat aFormat, const Sequence& aLayout) { MOZ_ASSERT(aBufferData); MOZ_ASSERT(aBufferLength > 0); switch(aFormat) { case ImageBitmapFormat::RGBA32: case ImageBitmapFormat::BGRA32: case ImageBitmapFormat::RGB24: case ImageBitmapFormat::BGR24: case ImageBitmapFormat::GRAY8: case ImageBitmapFormat::HSV: case ImageBitmapFormat::Lab: case ImageBitmapFormat::DEPTH: { const nsTArray& channels = aLayout; MOZ_ASSERT(channels.Length() != 0, "Empty Channels."); const SurfaceFormat srcFormat = ImageFormatToSurfaceFromat(aFormat); const uint32_t srcStride = channels[0].mStride; const IntSize srcSize(channels[0].mWidth, channels[0].mHeight); RefPtr dstDataSurface = Factory::CreateDataSourceSurfaceWithStride(srcSize, srcFormat, srcStride); if (NS_WARN_IF(!dstDataSurface)) { return nullptr; } // Copy the raw data into the newly created DataSourceSurface. DataSourceSurface::ScopedMap dstMap(dstDataSurface, DataSourceSurface::WRITE); if (NS_WARN_IF(!dstMap.IsMapped())) { return nullptr; } const uint8_t* srcBufferPtr = aBufferData; uint8_t* dstBufferPtr = dstMap.GetData(); for (int i = 0; i < srcSize.height; ++i) { memcpy(dstBufferPtr, srcBufferPtr, srcStride); srcBufferPtr += srcStride; dstBufferPtr += dstMap.GetStride(); } // Create an Image from the BGRA SourceSurface. RefPtr surface = dstDataSurface; RefPtr image = CreateImageFromSurface(surface); if (NS_WARN_IF(!image)) { return nullptr; } return image.forget(); } case ImageBitmapFormat::YUV444P: case ImageBitmapFormat::YUV422P: case ImageBitmapFormat::YUV420P: case ImageBitmapFormat::YUV420SP_NV12: case ImageBitmapFormat::YUV420SP_NV21: { // Prepare the PlanarYCbCrData. const ChannelPixelLayout& yLayout = aLayout[0]; const ChannelPixelLayout& uLayout = aFormat != ImageBitmapFormat::YUV420SP_NV21 ? aLayout[1] : aLayout[2]; const ChannelPixelLayout& vLayout = aFormat != ImageBitmapFormat::YUV420SP_NV21 ? aLayout[2] : aLayout[1]; layers::PlanarYCbCrData data; // Luminance buffer data.mYChannel = const_cast(aBufferData + yLayout.mOffset); data.mYStride = yLayout.mStride; data.mYSize = gfx::IntSize(yLayout.mWidth, yLayout.mHeight); data.mYSkip = yLayout.mSkip; // Chroma buffers data.mCbChannel = const_cast(aBufferData + uLayout.mOffset); data.mCrChannel = const_cast(aBufferData + vLayout.mOffset); data.mCbCrStride = uLayout.mStride; data.mCbCrSize = gfx::IntSize(uLayout.mWidth, uLayout.mHeight); data.mCbSkip = uLayout.mSkip; data.mCrSkip = vLayout.mSkip; // Picture rectangle. // We set the picture rectangle to exactly the size of the source image to // keep the full original data. data.mPicX = 0; data.mPicY = 0; data.mPicSize = data.mYSize; // Create a layers::Image and set data. if (aFormat == ImageBitmapFormat::YUV444P || aFormat == ImageBitmapFormat::YUV422P || aFormat == ImageBitmapFormat::YUV420P) { RefPtr image = new layers::RecyclingPlanarYCbCrImage(new layers::BufferRecycleBin()); if (NS_WARN_IF(!image)) { return nullptr; } // Set Data. if (NS_WARN_IF(!image->CopyData(data))) { return nullptr; } return image.forget(); } else { RefPtrimage = new layers::NVImage(); if (NS_WARN_IF(!image)) { return nullptr; } // Set Data. if (NS_WARN_IF(!image->SetData(data))) { return nullptr; } return image.forget(); } } default: return nullptr; } } /* * This is a synchronous task. * This class is used to create a layers::CairoImage from raw data in the main * thread. While creating an ImageBitmap from an BufferSource, we need to create * a SouceSurface from the BufferSource raw data and then set the SourceSurface * into a layers::CairoImage. However, the layers::CairoImage asserts the * setting operation in the main thread, so if we are going to create an * ImageBitmap from an BufferSource off the main thread, we post an event to the * main thread to create a layers::CairoImage from an BufferSource raw data. * * TODO: Once the layers::CairoImage is constructible off the main thread, which * means the SouceSurface could be released anywhere, we do not need this * task anymore. */ class CreateImageFromBufferSourceRawDataInMainThreadSyncTask final : public WorkerMainThreadRunnable { public: CreateImageFromBufferSourceRawDataInMainThreadSyncTask(const uint8_t* aBuffer, uint32_t aBufferLength, mozilla::dom::ImageBitmapFormat aFormat, const Sequence& aLayout, /*output*/ layers::Image** aImage) : WorkerMainThreadRunnable(GetCurrentThreadWorkerPrivate(), NS_LITERAL_CSTRING("ImageBitmap-extensions :: Create Image from BufferSource Raw Data")) , mImage(aImage) , mBuffer(aBuffer) , mBufferLength(aBufferLength) , mFormat(aFormat) , mLayout(aLayout) { MOZ_ASSERT(!(*aImage), "Don't pass an existing Image into CreateImageFromBufferSourceRawDataInMainThreadSyncTask."); } bool MainThreadRun() override { RefPtr image = CreateImageFromBufferSourceRawData(mBuffer, mBufferLength, mFormat, mLayout); if (NS_WARN_IF(!image)) { return true; } image.forget(mImage); return true; } private: layers::Image** mImage; const uint8_t* mBuffer; uint32_t mBufferLength; mozilla::dom::ImageBitmapFormat mFormat; const Sequence& mLayout; }; /*static*/ already_AddRefed ImageBitmap::Create(nsIGlobalObject* aGlobal, const ImageBitmapSource& aBuffer, int32_t aOffset, int32_t aLength, mozilla::dom::ImageBitmapFormat aFormat, const Sequence& aLayout, ErrorResult& aRv) { MOZ_ASSERT(aGlobal); RefPtr promise = Promise::Create(aGlobal, aRv); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } uint8_t* bufferData = nullptr; uint32_t bufferLength = 0; if (aBuffer.IsArrayBuffer()) { const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer(); buffer.ComputeLengthAndData(); bufferData = buffer.Data(); bufferLength = buffer.Length(); } else if (aBuffer.IsArrayBufferView()) { const ArrayBufferView& bufferView = aBuffer.GetAsArrayBufferView(); bufferView.ComputeLengthAndData(); bufferData = bufferView.Data(); bufferLength = bufferView.Length(); } else { aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); return promise.forget(); } MOZ_ASSERT(bufferData && bufferLength > 0, "Cannot read data from BufferSource."); // Check the buffer. if (((uint32_t)(aOffset + aLength) > bufferLength)) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return promise.forget(); } // Create and Crop the raw data into a layers::Image RefPtr data; if (NS_IsMainThread()) { data = CreateImageFromBufferSourceRawData(bufferData + aOffset, bufferLength, aFormat, aLayout); } else { RefPtr task = new CreateImageFromBufferSourceRawDataInMainThreadSyncTask(bufferData + aOffset, bufferLength, aFormat, aLayout, getter_AddRefs(data)); task->Dispatch(Terminating, aRv); if (aRv.Failed()) { return promise.forget(); } } if (NS_WARN_IF(!data)) { aRv.Throw(NS_ERROR_NOT_AVAILABLE); return promise.forget(); } // Create an ImageBimtap. // Assume the data from an external buffer is not alpha-premultiplied. RefPtr imageBitmap = new ImageBitmap(aGlobal, data, gfxAlphaType::NonPremult); imageBitmap->mAllocatedImageData = true; // We don't need to call SetPictureRect() here because there is no cropping // supported and the ImageBitmap's mPictureRect is the size of the source // image in default // We don't need to set mIsCroppingAreaOutSideOfSourceImage here because there // is no cropping supported and the mIsCroppingAreaOutSideOfSourceImage is // false in default. AsyncFulfillImageBitmapPromise(promise, imageBitmap); return promise.forget(); } size_t ImageBitmap::GetAllocatedSize() const { if (!mAllocatedImageData) { return 0; } // Calculate how many bytes are used. if (mData->GetFormat() == mozilla::ImageFormat::PLANAR_YCBCR) { return mData->AsPlanarYCbCrImage()->GetDataSize(); } if (mData->GetFormat() == mozilla::ImageFormat::NV_IMAGE) { return mData->AsNVImage()->GetBufferSize(); } RefPtr surface = mData->GetAsSourceSurface(); const int bytesPerPixel = BytesPerPixel(surface->GetFormat()); return surface->GetSize().height * surface->GetSize().width * bytesPerPixel; } size_t BindingJSObjectMallocBytes(ImageBitmap* aBitmap) { return aBitmap->GetAllocatedSize(); } /* static */ already_AddRefed CreateImageBitmapFromBlob::Create(Promise* aPromise, nsIGlobalObject* aGlobal, Blob& aBlob, const Maybe& aCropRect, nsIEventTarget* aMainThreadEventTarget) { // Get the internal stream of the blob. nsCOMPtr stream; ErrorResult error; aBlob.Impl()->CreateInputStream(getter_AddRefs(stream), error); if (NS_WARN_IF(error.Failed())) { return nullptr; } // Get the MIME type string of the blob. // The type will be checked in the DecodeImageAsync() method. nsAutoString mimeTypeUTF16; aBlob.Impl()->GetType(mimeTypeUTF16); NS_ConvertUTF16toUTF8 mimeType(mimeTypeUTF16); RefPtr task = new CreateImageBitmapFromBlob(aPromise, aGlobal, stream.forget(), mimeType, aCropRect, aMainThreadEventTarget); // Nothing to do for the main-thread. if (NS_IsMainThread()) { return task.forget(); } // Let's use a WorkerHolder to keep the worker alive if this is not the // main-thread. WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate(); MOZ_ASSERT(workerPrivate); UniquePtr holder( new CreateImageBitmapFromBlobHolder(workerPrivate, task)); if (!holder->HoldWorker(workerPrivate, Terminating)) { return nullptr; } task->mWorkerHolder = Move(holder); return task.forget(); } nsresult CreateImageBitmapFromBlob::StartDecodeAndCropBlob() { MOZ_ASSERT(IsCurrentThread()); // Workers. if (!NS_IsMainThread()) { RefPtr self = this; nsCOMPtr r = NS_NewRunnableFunction( "CreateImageBitmapFromBlob::DecodeAndCropBlob", [self]() { nsresult rv = self->DecodeAndCropBlob(); if (NS_WARN_IF(NS_FAILED(rv))) { self->DecodeAndCropBlobCompletedMainThread(nullptr, rv); } }); return mMainThreadEventTarget->Dispatch(r.forget()); } // Main-thread. return DecodeAndCropBlob(); } nsresult CreateImageBitmapFromBlob::DecodeAndCropBlob() { MOZ_ASSERT(NS_IsMainThread()); // Get the Component object. nsCOMPtr imgtool = do_GetService(NS_IMGTOOLS_CID); if (NS_WARN_IF(!imgtool)) { return NS_ERROR_FAILURE; } // Decode image. nsCOMPtr imgContainer; nsresult rv = imgtool->DecodeImageAsync(mInputStream, mMimeType, this, mMainThreadEventTarget); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } return NS_OK; } NS_IMETHODIMP CreateImageBitmapFromBlob::OnImageReady(imgIContainer* aImgContainer, nsresult aStatus) { MOZ_ASSERT(NS_IsMainThread()); if (NS_FAILED(aStatus)) { DecodeAndCropBlobCompletedMainThread(nullptr, aStatus); return NS_OK; } MOZ_ASSERT(aImgContainer); // Get the surface out. uint32_t frameFlags = imgIContainer::FLAG_SYNC_DECODE | imgIContainer::FLAG_WANT_DATA_SURFACE; uint32_t whichFrame = imgIContainer::FRAME_FIRST; RefPtr surface = aImgContainer->GetFrame(whichFrame, frameFlags); if (NS_WARN_IF(!surface)) { DecodeAndCropBlobCompletedMainThread(nullptr, NS_ERROR_FAILURE); return NS_OK; } // Store the sourceSize value for the DecodeAndCropBlobCompletedMainThread call. mSourceSize = surface->GetSize(); // Crop the source surface if needed. RefPtr croppedSurface = surface; if (mCropRect.isSome()) { // The blob is just decoded into a RasterImage and not optimized yet, so the // _surface_ we get is a DataSourceSurface which wraps the RasterImage's // raw buffer. // // The _surface_ might already be optimized so that its type is not // SurfaceType::DATA. However, we could keep using the generic cropping and // copying since the decoded buffer is only used in this ImageBitmap so we // should crop it to save memory usage. // // TODO: Bug1189632 is going to refactor this create-from-blob part to // decode the blob off the main thread. Re-check if we should do // cropping at this moment again there. RefPtr dataSurface = surface->GetDataSurface(); croppedSurface = CropAndCopyDataSourceSurface(dataSurface, mCropRect.ref()); mCropRect->MoveTo(0, 0); } if (NS_WARN_IF(!croppedSurface)) { DecodeAndCropBlobCompletedMainThread(nullptr, NS_ERROR_FAILURE); return NS_OK; } // Create an Image from the source surface. RefPtr image = CreateImageFromSurface(croppedSurface); if (NS_WARN_IF(!image)) { DecodeAndCropBlobCompletedMainThread(nullptr, NS_ERROR_FAILURE); return NS_OK; } DecodeAndCropBlobCompletedMainThread(image, NS_OK); return NS_OK; } void CreateImageBitmapFromBlob::DecodeAndCropBlobCompletedMainThread(layers::Image* aImage, nsresult aStatus) { MOZ_ASSERT(NS_IsMainThread()); if (!IsCurrentThread()) { MutexAutoLock lock(mMutex); if (!mWorkerHolder) { // The worker is already gone. return; } RefPtr r = new CreateImageBitmapFromBlobRunnable(mWorkerHolder->GetWorkerPrivate(), this, aImage, aStatus); r->Dispatch(); return; } DecodeAndCropBlobCompletedOwningThread(aImage, aStatus); } void CreateImageBitmapFromBlob::DecodeAndCropBlobCompletedOwningThread(layers::Image* aImage, nsresult aStatus) { MOZ_ASSERT(IsCurrentThread()); if (!mPromise) { // The worker is going to be released soon. No needs to continue. return; } // Let's release what has to be released on the owning thread. auto raii = MakeScopeExit([&] { // Doing this we also release the worker. mWorkerHolder = nullptr; mPromise = nullptr; mGlobalObject = nullptr; }); if (NS_WARN_IF(NS_FAILED(aStatus))) { mPromise->MaybeReject(NS_ERROR_DOM_INVALID_STATE_ERR); return; } // Create ImageBitmap object. RefPtr imageBitmap = new ImageBitmap(mGlobalObject, aImage); // Set mIsCroppingAreaOutSideOfSourceImage. imageBitmap->SetIsCroppingAreaOutSideOfSourceImage(mSourceSize, mOriginalCropRect); if (mCropRect.isSome()) { ErrorResult rv; imageBitmap->SetPictureRect(mCropRect.ref(), rv); if (rv.Failed()) { mPromise->MaybeReject(rv); return; } } imageBitmap->mAllocatedImageData = true; mPromise->MaybeResolve(imageBitmap); } void CreateImageBitmapFromBlob::WorkerShuttingDown() { MOZ_ASSERT(IsCurrentThread()); MutexAutoLock lock(mMutex); // Let's release all the non-thread-safe objects now. mWorkerHolder = nullptr; mPromise = nullptr; mGlobalObject = nullptr; } } // namespace dom } // namespace mozilla