diff --git a/dom/media/MediaData.cpp b/dom/media/MediaData.cpp index dd8d94ffb1fb..53de1aefd0a8 100644 --- a/dom/media/MediaData.cpp +++ b/dom/media/MediaData.cpp @@ -19,6 +19,8 @@ #ifdef XP_WIN # include "mozilla/WindowsVersion.h" # include "mozilla/layers/D3D11YCbCrImage.h" +#elif XP_MACOSX +# include "MacIOSurfaceImage.h" #endif namespace mozilla { @@ -348,6 +350,14 @@ already_AddRefed VideoData::CreateAndCopyData( return v.forget(); } } +#elif XP_MACOSX + RefPtr ioImage = + new layers::MacIOSurfaceImage(nullptr); + PlanarYCbCrData data = ConstructPlanarYCbCrData(aInfo, aBuffer, aPicture); + if (ioImage->SetData(aContainer, data)) { + v->mImage = ioImage; + return v.forget(); + } #endif if (!v->mImage) { v->mImage = aContainer->CreatePlanarYCbCrImage(); diff --git a/gfx/2d/MacIOSurface.cpp b/gfx/2d/MacIOSurface.cpp index 90d9a40ec2c8..9fd1a53496a6 100644 --- a/gfx/2d/MacIOSurface.cpp +++ b/gfx/2d/MacIOSurface.cpp @@ -74,6 +74,106 @@ already_AddRefed MacIOSurface::CreateIOSurface( return ioSurface.forget(); } +void AddDictionaryInt(const CFTypeRefPtr& aDict, + const void* aType, uint32_t aValue) { + auto cfValue = CFTypeRefPtr::WrapUnderCreateRule( + ::CFNumberCreate(nullptr, kCFNumberSInt32Type, &aValue)); + ::CFDictionaryAddValue(aDict.get(), aType, cfValue.get()); +} + +size_t CreatePlaneDictionary(CFTypeRefPtr& aDict, + const gfx::IntSize& aSize, size_t aOffset, + size_t aBytesPerPixel) { + size_t bytesPerRow = IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, + aSize.width * aBytesPerPixel); + size_t totalBytes = + IOSurfaceAlignProperty(kIOSurfaceAllocSize, aSize.height * bytesPerRow); + + aDict = CFTypeRefPtr::WrapUnderCreateRule( + ::CFDictionaryCreateMutable(kCFAllocatorDefault, 4, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks)); + + AddDictionaryInt(aDict, kIOSurfacePlaneWidth, aSize.width); + AddDictionaryInt(aDict, kIOSurfacePlaneHeight, aSize.height); + AddDictionaryInt(aDict, kIOSurfacePlaneBytesPerRow, bytesPerRow); + AddDictionaryInt(aDict, kIOSurfacePlaneOffset, aOffset); + AddDictionaryInt(aDict, kIOSurfacePlaneSize, totalBytes); + AddDictionaryInt(aDict, kIOSurfaceBytesPerElement, aBytesPerPixel); + + return totalBytes; +} + +/* static */ +already_AddRefed MacIOSurface::CreateNV12Surface( + const IntSize& aYSize, const IntSize& aCbCrSize, YUVColorSpace aColorSpace, + ColorRange aColorRange) { + MOZ_ASSERT(aColorSpace == YUVColorSpace::BT601 || + aColorSpace == YUVColorSpace::BT709); + MOZ_ASSERT(aColorRange == ColorRange::LIMITED || + aColorRange == ColorRange::FULL); + + auto props = CFTypeRefPtr::WrapUnderCreateRule( + ::CFDictionaryCreateMutable(kCFAllocatorDefault, 4, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks)); + if (!props) return nullptr; + + MOZ_ASSERT((size_t)aYSize.width <= GetMaxWidth()); + MOZ_ASSERT((size_t)aYSize.height <= GetMaxHeight()); + + AddDictionaryInt(props, kIOSurfaceWidth, aYSize.width); + AddDictionaryInt(props, kIOSurfaceHeight, aYSize.height); + ::CFDictionaryAddValue(props.get(), kIOSurfaceIsGlobal, kCFBooleanTrue); + + if (aColorRange == ColorRange::LIMITED) { + AddDictionaryInt(props, kIOSurfacePixelFormat, + (uint32_t)kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + } else { + AddDictionaryInt(props, kIOSurfacePixelFormat, + (uint32_t)kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); + } + + CFTypeRefPtr planeProps[2]; + size_t planeTotalBytes = CreatePlaneDictionary(planeProps[0], aYSize, 0, 1); + planeTotalBytes += + CreatePlaneDictionary(planeProps[1], aCbCrSize, planeTotalBytes, 2); + + AddDictionaryInt(props, kIOSurfaceAllocSize, planeTotalBytes); + + auto array = CFTypeRefPtr::WrapUnderCreateRule( + CFArrayCreate(kCFAllocatorDefault, (const void**)planeProps, 2, + &kCFTypeArrayCallBacks)); + ::CFDictionaryAddValue(props.get(), kIOSurfacePlaneInfo, array.get()); + + CFTypeRefPtr surfaceRef = + CFTypeRefPtr::WrapUnderCreateRule( + ::IOSurfaceCreate(props.get())); + + if (!surfaceRef) { + return nullptr; + } + + CFTypeRefPtr colorSpace; + if (aColorSpace == YUVColorSpace::BT601) { + colorSpace = CFTypeRefPtr::WrapUnderCreateRule( + CGColorSpaceCreateWithName(kCGColorSpaceSRGB)); + + } else { + colorSpace = CFTypeRefPtr::WrapUnderCreateRule( + CGColorSpaceCreateWithName(kCGColorSpaceITUR_709)); + } + auto colorData = CFTypeRefPtr::WrapUnderCreateRule( + CGColorSpaceCopyICCProfile(colorSpace.get())); + IOSurfaceSetValue(surfaceRef.get(), CFSTR("IOSurfaceColorSpace"), + colorData.get()); + + RefPtr ioSurface = + new MacIOSurface(std::move(surfaceRef), 1.0, false, aColorSpace); + + return ioSurface.forget(); +} + /* static */ already_AddRefed MacIOSurface::LookupSurface( IOSurfaceID aIOSurfaceID, double aContentsScaleFactor, bool aHasAlpha, diff --git a/gfx/2d/MacIOSurface.h b/gfx/2d/MacIOSurface.h index f459cf326050..2e460d2f29ba 100644 --- a/gfx/2d/MacIOSurface.h +++ b/gfx/2d/MacIOSurface.h @@ -48,6 +48,9 @@ class MacIOSurface final typedef mozilla::gfx::SourceSurface SourceSurface; typedef mozilla::gfx::DrawTarget DrawTarget; typedef mozilla::gfx::BackendType BackendType; + typedef mozilla::gfx::IntSize IntSize; + typedef mozilla::gfx::YUVColorSpace YUVColorSpace; + typedef mozilla::gfx::ColorRange ColorRange; // The usage count of the IOSurface is increased by 1 during the lifetime // of the MacIOSurface instance. @@ -56,6 +59,9 @@ class MacIOSurface final static already_AddRefed CreateIOSurface( int aWidth, int aHeight, double aContentsScaleFactor = 1.0, bool aHasAlpha = true); + static already_AddRefed CreateNV12Surface( + const IntSize& aYSize, const IntSize& aCbCrSize, + YUVColorSpace aColorSpace, ColorRange aColorRange); static void ReleaseIOSurface(MacIOSurface* aIOSurface); static already_AddRefed LookupSurface( IOSurfaceID aSurfaceID, double aContentsScaleFactor = 1.0, @@ -68,6 +74,7 @@ class MacIOSurface final bool aHasAlpha = true, mozilla::gfx::YUVColorSpace aColorSpace = mozilla::gfx::YUVColorSpace::UNKNOWN); + ~MacIOSurface(); IOSurfaceID GetIOSurfaceID() const; void* GetBaseAddress() const; @@ -80,6 +87,9 @@ class MacIOSurface final // device pixel. Use GetDevicePixel**() to get device pixels. size_t GetWidth(size_t plane = 0) const; size_t GetHeight(size_t plane = 0) const; + IntSize GetSize(size_t plane = 0) const { + return IntSize(GetWidth(plane), GetHeight(plane)); + } double GetContentsScaleFactor() const { return mContentsScaleFactor; } size_t GetDevicePixelWidth(size_t plane = 0) const; size_t GetDevicePixelHeight(size_t plane = 0) const; @@ -95,10 +105,10 @@ class MacIOSurface final // This would be better suited on MacIOSurfaceImage type, however due to the // current data structure, this is not possible as only the IOSurfaceRef is // being used across. - void SetYUVColorSpace(mozilla::gfx::YUVColorSpace aColorSpace) { + void SetYUVColorSpace(YUVColorSpace aColorSpace) { mColorSpace = aColorSpace; } - mozilla::gfx::YUVColorSpace GetYUVColorSpace() const { return mColorSpace; } + YUVColorSpace GetYUVColorSpace() const { return mColorSpace; } bool IsFullRange() const { return GetPixelFormat() == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; } @@ -132,8 +142,7 @@ class MacIOSurface final double mContentsScaleFactor; bool mHasAlpha; bool mIsLocked = false; - mozilla::gfx::YUVColorSpace mColorSpace = - mozilla::gfx::YUVColorSpace::UNKNOWN; + YUVColorSpace mColorSpace = YUVColorSpace::UNKNOWN; }; #endif diff --git a/gfx/layers/ImageContainer.cpp b/gfx/layers/ImageContainer.cpp index 1e04581d657f..47c484a6b26a 100644 --- a/gfx/layers/ImageContainer.cpp +++ b/gfx/layers/ImageContainer.cpp @@ -32,6 +32,7 @@ #ifdef XP_MACOSX # include "mozilla/gfx/QuartzSupport.h" +# include "MacIOSurfaceImage.h" #endif #ifdef XP_WIN @@ -462,6 +463,17 @@ D3D11YCbCrRecycleAllocator* ImageContainer::GetD3D11YCbCrRecycleAllocator( } #endif +#ifdef XP_MACOSX +MacIOSurfaceRecycleAllocator* +ImageContainer::GetMacIOSurfaceRecycleAllocator() { + if (!mMacIOSurfaceRecycleAllocator) { + mMacIOSurfaceRecycleAllocator = new MacIOSurfaceRecycleAllocator(); + } + + return mMacIOSurfaceRecycleAllocator; +} +#endif + PlanarYCbCrImage::PlanarYCbCrImage() : Image(nullptr, ImageFormat::PLANAR_YCBCR), mOffscreenFormat(SurfaceFormat::UNKNOWN), diff --git a/gfx/layers/ImageContainer.h b/gfx/layers/ImageContainer.h index 740e004bef18..a6ae277427ff 100644 --- a/gfx/layers/ImageContainer.h +++ b/gfx/layers/ImageContainer.h @@ -153,6 +153,9 @@ class NVImage; #ifdef XP_WIN class D3D11YCbCrRecycleAllocator; #endif +#ifdef XP_MACOSX +class MacIOSurfaceRecycleAllocator; +#endif class SurfaceDescriptorBuffer; struct ImageBackendData { @@ -555,6 +558,10 @@ class ImageContainer final : public SupportsWeakPtr { KnowsCompositor* aKnowsCompositor); #endif +#ifdef XP_MACOSX + MacIOSurfaceRecycleAllocator* GetMacIOSurfaceRecycleAllocator(); +#endif + /** * Returns the delay between the last composited image's presentation * timestamp and when it was first composited. It's possible for the delay @@ -639,6 +646,9 @@ class ImageContainer final : public SupportsWeakPtr { #ifdef XP_WIN RefPtr mD3D11YCbCrRecycleAllocator; #endif +#ifdef XP_MACOSX + RefPtr mMacIOSurfaceRecycleAllocator; +#endif nsTArray mCurrentImages; diff --git a/gfx/layers/MacIOSurfaceImage.cpp b/gfx/layers/MacIOSurfaceImage.cpp index 9abf2d168a06..8a320280363c 100644 --- a/gfx/layers/MacIOSurfaceImage.cpp +++ b/gfx/layers/MacIOSurfaceImage.cpp @@ -10,7 +10,9 @@ #include "mozilla/layers/CompositableClient.h" #include "mozilla/layers/CompositableForwarder.h" #include "mozilla/layers/MacIOSurfaceTextureClientOGL.h" +#include "mozilla/StaticPrefs_layers.h" #include "mozilla/UniquePtr.h" +#include "YCbCrUtils.h" using namespace mozilla; using namespace mozilla::layers; @@ -30,3 +32,105 @@ TextureClient* MacIOSurfaceImage::GetTextureClient( already_AddRefed MacIOSurfaceImage::GetAsSourceSurface() { return CreateSourceSurfaceFromMacIOSurface(mSurface); } + +bool MacIOSurfaceImage::SetData(ImageContainer* aContainer, + const PlanarYCbCrData& aData) { + MOZ_ASSERT(!mSurface); + + // The RDD sandbox won't allow us to create an IOSurface. + if (XRE_IsRDDProcess()) { + return false; + } + + if (aData.mYSkip != 0 || aData.mCbSkip != 0 || aData.mCrSkip != 0 || + !(aData.mYUVColorSpace == YUVColorSpace::BT601 || + aData.mYUVColorSpace == YUVColorSpace::BT709) || + !(aData.mColorRange == ColorRange::FULL || + aData.mColorRange == ColorRange::LIMITED) || + aData.mColorDepth != ColorDepth::COLOR_8) { + return false; + } + + RefPtr allocator = + aContainer->GetMacIOSurfaceRecycleAllocator(); + + RefPtr surf = allocator->Allocate( + aData.mYSize, aData.mCbCrSize, aData.mYUVColorSpace, aData.mColorRange); + + surf->Lock(false); + + MOZ_ASSERT(aData.mYSize.height > 0); + uint8_t* dst = (uint8_t*)surf->GetBaseAddressOfPlane(0); + size_t stride = surf->GetBytesPerRow(0); + for (size_t i = 0; i < (size_t)aData.mYSize.height; i++) { + uint8_t* rowSrc = aData.mYChannel + aData.mYStride * i; + uint8_t* rowDst = dst + stride * i; + memcpy(rowDst, rowSrc, aData.mYSize.width); + } + + // CoreAnimation doesn't appear to support planar YCbCr formats, so we + // allocated an NV12 surface and now we need to copy and interleave the Cb and + // Cr channels. + MOZ_ASSERT(aData.mCbCrSize.height > 0); + dst = (uint8_t*)surf->GetBaseAddressOfPlane(1); + stride = surf->GetBytesPerRow(1); + for (size_t i = 0; i < (size_t)aData.mCbCrSize.height; i++) { + uint8_t* rowCbSrc = aData.mCbChannel + aData.mCbCrStride * i; + uint8_t* rowCrSrc = aData.mCrChannel + aData.mCbCrStride * i; + uint8_t* rowDst = dst + stride * i; + + for (size_t j = 0; j < (size_t)aData.mCbCrSize.width; j++) { + *rowDst = *rowCbSrc; + rowDst++; + rowCbSrc++; + *rowDst = *rowCrSrc; + rowDst++; + rowCrSrc++; + } + } + + surf->Unlock(false); + mSurface = surf; + mPictureRect = aData.GetPictureRect(); + return true; +} + +already_AddRefed MacIOSurfaceRecycleAllocator::Allocate( + const gfx::IntSize aYSize, const gfx::IntSize& aCbCrSize, + gfx::YUVColorSpace aYUVColorSpace, gfx::ColorRange aColorRange) { + nsTArray> surfaces = std::move(mSurfaces); + RefPtr result; + for (auto& surf : surfaces) { + MOZ_ASSERT(::IOSurfaceGetPlaneCount(surf.get()) == 2); + + // If the surface size has changed, then discard any surfaces of the old + // size. + if (::IOSurfaceGetWidthOfPlane(surf.get(), 0) != (size_t)aYSize.width || + ::IOSurfaceGetHeightOfPlane(surf.get(), 0) != (size_t)aYSize.height || + ::IOSurfaceGetWidthOfPlane(surf.get(), 1) != (size_t)aCbCrSize.width || + ::IOSurfaceGetHeightOfPlane(surf.get(), 1) != + (size_t)aCbCrSize.height) { + continue; + } + + // Only construct a MacIOSurface object when we find one that isn't + // in-use, since the constructor adds a usage ref. + if (!result && !::IOSurfaceIsInUse(surf.get())) { + result = new MacIOSurface(surf, 1.0, false, aYUVColorSpace); + } + + mSurfaces.AppendElement(surf); + } + + if (!result) { + result = MacIOSurface::CreateNV12Surface(aYSize, aCbCrSize, aYUVColorSpace, + aColorRange); + + if (mSurfaces.Length() < + StaticPrefs::layers_iosurfaceimage_recycle_limit()) { + mSurfaces.AppendElement(result->GetIOSurfaceRef()); + } + } + + return result.forget(); +} diff --git a/gfx/layers/MacIOSurfaceImage.h b/gfx/layers/MacIOSurfaceImage.h index 3713f877f440..3419bcf9c3ba 100644 --- a/gfx/layers/MacIOSurfaceImage.h +++ b/gfx/layers/MacIOSurfaceImage.h @@ -19,7 +19,13 @@ namespace layers { class MacIOSurfaceImage : public Image { public: explicit MacIOSurfaceImage(MacIOSurface* aSurface) - : Image(nullptr, ImageFormat::MAC_IOSURFACE), mSurface(aSurface) {} + : Image(nullptr, ImageFormat::MAC_IOSURFACE), mSurface(aSurface) { + if (aSurface) { + mPictureRect = gfx::IntRect(gfx::IntPoint{}, aSurface->GetSize(0)); + } + } + + bool SetData(ImageContainer* aContainer, const PlanarYCbCrData& aData); MacIOSurface* GetSurface() { return mSurface; } @@ -34,9 +40,27 @@ class MacIOSurfaceImage : public Image { MacIOSurfaceImage* AsMacIOSurfaceImage() override { return this; } + gfx::IntRect GetPictureRect() const override { return mPictureRect; } + private: RefPtr mSurface; RefPtr mTextureClient; + gfx::IntRect mPictureRect; +}; + +class MacIOSurfaceRecycleAllocator { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MacIOSurfaceRecycleAllocator) + + already_AddRefed Allocate(const gfx::IntSize aYSize, + const gfx::IntSize& aCbCrSize, + gfx::YUVColorSpace aYUVColorSpace, + gfx::ColorRange aColorRange); + + private: + ~MacIOSurfaceRecycleAllocator() = default; + + nsTArray> mSurfaces; }; } // namespace layers diff --git a/modules/libpref/init/StaticPrefList.yaml b/modules/libpref/init/StaticPrefList.yaml index e6bcaf23c8c4..392d54143dcc 100644 --- a/modules/libpref/init/StaticPrefList.yaml +++ b/modules/libpref/init/StaticPrefList.yaml @@ -5550,6 +5550,11 @@ value: true mirror: once +- name: layers.iosurfaceimage.recycle-limit + type: RelaxedAtomicUint32 + value: 15 + mirror: always + #--------------------------------------------------------------------------- # Prefs starting with "layout." #---------------------------------------------------------------------------