Bug 1731136 Part 4: Make macOS native compositor and NativeLayerCA handle backdrop layers. r=mstange

This makes the macOS native compositor claim the ability to handle color
layers and makes NativeLayerCA actually do it. Color layers have a different
structure than other layers. Color layers apply the color to the
wrappingCALayer, and have no contentCALayer at all. That means that the
color layers are always sized to the layer's clip rect.

This also contains a drive-by fix to handle the case where an mOpaquenessTintLayer
exists when mMutatedSpecializeVideo is set to true. Before this change, in such a
case, the opaqueness layer will not be associated with the recreated wrapping layer.

Differential Revision: https://phabricator.services.mozilla.com/D128545

gfx/layers/NativeLayer.h

@@ -54,6 +54,10 @@ class NativeLayerRoot {
       SurfacePoolHandle* aSurfacePoolHandle) = 0;
   virtual already_AddRefed<NativeLayer> CreateLayerForExternalTexture(
       bool aIsOpaque) = 0;
+  virtual already_AddRefed<NativeLayer> CreateLayerForColor(
+      gfx::DeviceColor aColor) {
+    return nullptr;
+  }
 
   virtual void AppendLayer(NativeLayer* aLayer) = 0;
   virtual void RemoveLayer(NativeLayer* aLayer) = 0;

gfx/layers/NativeLayerCA.h

@@ -118,6 +118,8 @@ class NativeLayerRootCA : public NativeLayerRoot {
 
   already_AddRefed<NativeLayer> CreateLayerForExternalTexture(
       bool aIsOpaque) override;
+  already_AddRefed<NativeLayer> CreateLayerForColor(
+      gfx::DeviceColor aColor) override;
 
   void SetWindowIsFullscreen(bool aFullscreen);
   void NoteMouseMoveAtTime(const TimeStamp& aTime);
@@ -259,6 +261,7 @@ class NativeLayerCA : public NativeLayer {
   NativeLayerCA(const gfx::IntSize& aSize, bool aIsOpaque,
                 SurfacePoolHandleCA* aSurfacePoolHandle);
   explicit NativeLayerCA(bool aIsOpaque);
+  explicit NativeLayerCA(gfx::DeviceColor aColor);
   ~NativeLayerCA() override;
 
   // Gets the next surface for drawing from our swap chain and stores it in
@@ -348,7 +351,8 @@ class NativeLayerCA : public NativeLayer {
                       bool aSurfaceIsFlipped,
                       gfx::SamplingFilter aSamplingFilter,
                       bool aSpecializeVideo,
-                      CFTypeRefPtr<IOSurfaceRef> aFrontSurface);
+                      CFTypeRefPtr<IOSurfaceRef> aFrontSurface,
+                      CFTypeRefPtr<CGColorRef> aColor);
 
     // Return whether any aspects of this layer representation have been mutated
     // since the last call to ApplyChanges, i.e. whether ApplyChanges needs to
@@ -447,6 +451,7 @@ class NativeLayerCA : public NativeLayer {
   gfx::SamplingFilter mSamplingFilter = gfx::SamplingFilter::POINT;
   float mBackingScale = 1.0f;
   bool mSurfaceIsFlipped = false;
+  CFTypeRefPtr<CGColorRef> mColor;
   const bool mIsOpaque = false;
   bool mRootWindowIsFullscreen = false;
   bool mSpecializeVideo = false;

gfx/layers/NativeLayerCA.mm

@@ -156,6 +156,11 @@ already_AddRefed<NativeLayer> NativeLayerRootCA::CreateLayerForExternalTexture(b
   return layer.forget();
 }
 
+already_AddRefed<NativeLayer> NativeLayerRootCA::CreateLayerForColor(gfx::DeviceColor aColor) {
+  RefPtr<NativeLayer> layer = new NativeLayerCA(aColor);
+  return layer.forget();
+}
+
 void NativeLayerRootCA::AppendLayer(NativeLayer* aLayer) {
   MutexAutoLock lock(mMutex);
 
@@ -751,6 +756,36 @@ NativeLayerCA::NativeLayerCA(const IntSize& aSize, bool aIsOpaque,
 NativeLayerCA::NativeLayerCA(bool aIsOpaque)
     : mMutex("NativeLayerCA"), mSurfacePoolHandle(nullptr), mIsOpaque(aIsOpaque) {}
 
+CGColorRef CGColorCreateForDeviceColor(gfx::DeviceColor aColor) {
+  if (StaticPrefs::gfx_color_management_native_srgb()) {
+    // Use CGColorCreateSRGB if it's available, otherwise use older macOS API methods,
+    // which unfortunately allocate additional memory for the colorSpace object.
+    if (@available(macOS 10.15, iOS 13.0, *)) {
+      // Even if it is available, we have to address the function dynamically, to keep
+      // compiler happy when building with earlier versions of the SDK.
+      static auto CGColorCreateSRGBPtr = (CGColorRef(*)(CGFloat, CGFloat, CGFloat, CGFloat))dlsym(
+          RTLD_DEFAULT, "CGColorCreateSRGB");
+      if (CGColorCreateSRGBPtr) {
+        return CGColorCreateSRGBPtr(aColor.r, aColor.g, aColor.b, aColor.a);
+      }
+    }
+
+    CGColorSpaceRef colorSpace = CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
+    CGFloat components[] = {aColor.r, aColor.g, aColor.b, aColor.a};
+    CGColorRef color = CGColorCreate(colorSpace, components);
+    CFRelease(colorSpace);
+    return color;
+  }
+
+  return CGColorCreateGenericRGB(aColor.r, aColor.g, aColor.b, aColor.a);
+}
+
+NativeLayerCA::NativeLayerCA(gfx::DeviceColor aColor)
+    : mMutex("NativeLayerCA"), mSurfacePoolHandle(nullptr), mIsOpaque(aColor.a >= 1.0f) {
+  MOZ_ASSERT(aColor.a > 0.0f, "Can't handle a fully transparent backdrop.");
+  mColor.AssignUnderCreateRule(CGColorCreateForDeviceColor(aColor));
+}
+
 NativeLayerCA::~NativeLayerCA() {
   if (mInProgressLockedIOSurface) {
     mInProgressLockedIOSurface->Unlock(false);
@@ -988,6 +1023,16 @@ void NativeLayerCA::DumpLayer(std::ostream& aOutputStream) {
     aOutputStream << "height: " << wrappingDivSize.height << "px; ";
   }
 
+  if (mColor) {
+    const CGFloat* components = CGColorGetComponents(mColor.get());
+    aOutputStream << "background: rgb(" << components[0] * 255.0f << " " << components[1] * 255.0f
+                  << " " << components[2] * 255.0f << "); opacity: " << components[3] << "; ";
+
+    // That's all we need for color layers. We don't need to specify an image.
+    aOutputStream << "\"/></div>\n";
+    return;
+  }
+
   Matrix4x4 transform = mTransform;
   transform.PreTranslate(mPosition.x, mPosition.y, 0);
   transform.PostTranslate(clipToLayerOffset.x, clipToLayerOffset.y, 0);
@@ -1034,9 +1079,11 @@ void NativeLayerCA::DumpLayer(std::ostream& aOutputStream) {
   aOutputStream << "src=\"";
 
   if (surface) {
+    // Attempt to render the surface as a PNG. Skia can do this for RGB surfaces.
     RefPtr<MacIOSurface> surf = new MacIOSurface(surface);
     surf->Lock(true);
-    {
+    SurfaceFormat format = surf->GetFormat();
+    if (format == SurfaceFormat::B8G8R8A8 || format == SurfaceFormat::B8G8R8X8) {
       RefPtr<gfx::DrawTarget> dt = surf->GetAsDrawTargetLocked(gfx::BackendType::SKIA);
       if (dt) {
         RefPtr<gfx::SourceSurface> sourceSurf = dt->Snapshot();
@@ -1280,7 +1327,8 @@ bool NativeLayerCA::ApplyChanges(WhichRepresentation aRepresentation,
   }
   return GetRepresentation(aRepresentation)
       .ApplyChanges(aUpdate, mSize, mIsOpaque, mPosition, mTransform, mDisplayRect, mClipRect,
-                    mBackingScale, mSurfaceIsFlipped, mSamplingFilter, mSpecializeVideo, surface);
+                    mBackingScale, mSurfaceIsFlipped, mSamplingFilter, mSpecializeVideo, surface,
+                    mColor);
 }
 
 CALayer* NativeLayerCA::UnderlyingCALayer(WhichRepresentation aRepresentation) {
@@ -1431,7 +1479,7 @@ bool NativeLayerCA::Representation::ApplyChanges(
     const IntPoint& aPosition, const Matrix4x4& aTransform, const IntRect& aDisplayRect,
     const Maybe<IntRect>& aClipRect, float aBackingScale, bool aSurfaceIsFlipped,
     gfx::SamplingFilter aSamplingFilter, bool aSpecializeVideo,
-    CFTypeRefPtr<IOSurfaceRef> aFrontSurface) {
+    CFTypeRefPtr<IOSurfaceRef> aFrontSurface, CFTypeRefPtr<CGColorRef> aColor) {
   // If we have an OnlyVideo update, handle it and early exit.
   if (aUpdate == UpdateType::OnlyVideo) {
     // If we don't have any updates to do, exit early with success. This is
@@ -1461,9 +1509,11 @@ bool NativeLayerCA::Representation::ApplyChanges(
 
   if (mWrappingCALayer && mMutatedSpecializeVideo) {
     // Since specialize video changes the way we construct our wrapping and content layers,
-    // we have to scrap them if this value has changed. We can leave mOpaquenessTintLayer alone.
+    // we have to scrap them if this value has changed.
     [mContentCALayer release];
     mContentCALayer = nil;
+    [mOpaquenessTintLayer release];
+    mOpaquenessTintLayer = nil;
     [mWrappingCALayer removeFromSuperlayer];
     [mWrappingCALayer release];
     mWrappingCALayer = nil;
@@ -1478,29 +1528,36 @@ bool NativeLayerCA::Representation::ApplyChanges(
     mWrappingCALayer.anchorPoint = NSZeroPoint;
     mWrappingCALayer.contentsGravity = kCAGravityTopLeft;
     mWrappingCALayer.edgeAntialiasingMask = 0;
-    if (aSpecializeVideo) {
+
-      mContentCALayer = [[AVSampleBufferDisplayLayer layer] retain];
+    if (aColor) {
-      CMTimebaseRef timebase;
+      // Color layers set a color on the wrapping layer and don't get a content layer.
-      CMTimebaseCreateWithMasterClock(kCFAllocatorDefault, CMClockGetHostTimeClock(), &timebase);
+      mWrappingCALayer.backgroundColor = aColor.get();
-      CMTimebaseSetRate(timebase, 1.0f);
-      [(AVSampleBufferDisplayLayer*)mContentCALayer setControlTimebase:timebase];
-      CFRelease(timebase);
     } else {
-      mContentCALayer = [[CALayer layer] retain];
+      if (aSpecializeVideo) {
+        mContentCALayer = [[AVSampleBufferDisplayLayer layer] retain];
+        CMTimebaseRef timebase;
+        CMTimebaseCreateWithMasterClock(kCFAllocatorDefault, CMClockGetHostTimeClock(), &timebase);
+        CMTimebaseSetRate(timebase, 1.0f);
+        [(AVSampleBufferDisplayLayer*)mContentCALayer setControlTimebase:timebase];
+        CFRelease(timebase);
+      } else {
+        mContentCALayer = [[CALayer layer] retain];
+      }
+      mContentCALayer.position = NSZeroPoint;
+      mContentCALayer.anchorPoint = NSZeroPoint;
+      mContentCALayer.contentsGravity = kCAGravityTopLeft;
+      mContentCALayer.contentsScale = 1;
+      mContentCALayer.bounds = CGRectMake(0, 0, aSize.width, aSize.height);
+      mContentCALayer.edgeAntialiasingMask = 0;
+      mContentCALayer.opaque = aIsOpaque;
+      if ([mContentCALayer respondsToSelector:@selector(setContentsOpaque:)]) {
+        // The opaque property seems to not be enough when using IOSurface contents.
+        // Additionally, call the private method setContentsOpaque.
+        [mContentCALayer setContentsOpaque:aIsOpaque];
+      }
+
+      [mWrappingCALayer addSublayer:mContentCALayer];
     }
-    mContentCALayer.position = NSZeroPoint;
-    mContentCALayer.anchorPoint = NSZeroPoint;
-    mContentCALayer.contentsGravity = kCAGravityTopLeft;
-    mContentCALayer.contentsScale = 1;
-    mContentCALayer.bounds = CGRectMake(0, 0, aSize.width, aSize.height);
-    mContentCALayer.edgeAntialiasingMask = 0;
-    mContentCALayer.opaque = aIsOpaque;
-    if ([mContentCALayer respondsToSelector:@selector(setContentsOpaque:)]) {
-      // The opaque property seems to not be enough when using IOSurface contents.
-      // Additionally, call the private method setContentsOpaque.
-      [mContentCALayer setContentsOpaque:aIsOpaque];
-    }
-    [mWrappingCALayer addSublayer:mContentCALayer];
   }
 
   bool shouldTintOpaqueness = StaticPrefs::gfx_core_animation_tint_opaque();
@@ -1534,7 +1591,7 @@ bool NativeLayerCA::Representation::ApplyChanges(
   //  Important: Always use integral numbers for the width and height of your layer.
   // We hope that this refers to integral physical pixels, and not to integral logical coordinates.
 
-  if (mMutatedBackingScale || mMutatedSize || layerNeedsInitialization) {
+  if (mContentCALayer && (mMutatedBackingScale || mMutatedSize || layerNeedsInitialization)) {
     mContentCALayer.bounds =
         CGRectMake(0, 0, aSize.width / aBackingScale, aSize.height / aBackingScale);
     if (mOpaquenessTintLayer) {
@@ -1571,33 +1628,35 @@ bool NativeLayerCA::Representation::ApplyChanges(
       mWrappingCALayer.masksToBounds = NO;
     }
 
-    Matrix4x4 transform = aTransform;
+    if (mContentCALayer) {
-    transform.PreTranslate(aPosition.x, aPosition.y, 0);
+      Matrix4x4 transform = aTransform;
-    transform.PostTranslate(clipToLayerOffset.x, clipToLayerOffset.y, 0);
+      transform.PreTranslate(aPosition.x, aPosition.y, 0);
+      transform.PostTranslate(clipToLayerOffset.x, clipToLayerOffset.y, 0);
 
-    if (aSurfaceIsFlipped) {
+      if (aSurfaceIsFlipped) {
-      transform.PreTranslate(0, aSize.height, 0).PreScale(1, -1, 1);
+        transform.PreTranslate(0, aSize.height, 0).PreScale(1, -1, 1);
-    }
+      }
 
-    CATransform3D transformCA{transform._11,
+      CATransform3D transformCA{transform._11,
-                              transform._12,
+                                transform._12,
-                              transform._13,
+                                transform._13,
-                              transform._14,
+                                transform._14,
-                              transform._21,
+                                transform._21,
-                              transform._22,
+                                transform._22,
-                              transform._23,
+                                transform._23,
-                              transform._24,
+                                transform._24,
-                              transform._31,
+                                transform._31,
-                              transform._32,
+                                transform._32,
-                              transform._33,
+                                transform._33,
-                              transform._34,
+                                transform._34,
-                              transform._41 / aBackingScale,
+                                transform._41 / aBackingScale,
-                              transform._42 / aBackingScale,
+                                transform._42 / aBackingScale,
-                              transform._43,
+                                transform._43,
-                              transform._44};
+                                transform._44};
-    mContentCALayer.transform = transformCA;
+      mContentCALayer.transform = transformCA;
-    if (mOpaquenessTintLayer) {
+      if (mOpaquenessTintLayer) {
-      mOpaquenessTintLayer.transform = mContentCALayer.transform;
+        mOpaquenessTintLayer.transform = mContentCALayer.transform;
+      }
     }
   }
 
@@ -1614,7 +1673,7 @@ bool NativeLayerCA::Representation::ApplyChanges(
     }
   }
 
-  if (mMutatedSamplingFilter || layerNeedsInitialization) {
+  if (mContentCALayer && (mMutatedSamplingFilter || layerNeedsInitialization)) {
     if (aSamplingFilter == gfx::SamplingFilter::POINT) {
       mContentCALayer.minificationFilter = kCAFilterNearest;
       mContentCALayer.magnificationFilter = kCAFilterNearest;

gfx/webrender_bindings/RenderCompositorNative.cpp

@@ -123,6 +123,9 @@ bool RenderCompositorNative::ShouldUseNativeCompositor() {
 void RenderCompositorNative::GetCompositorCapabilities(
     CompositorCapabilities* aCaps) {
   RenderCompositor::GetCompositorCapabilities(aCaps);
+#if defined(XP_MACOSX)
+  aCaps->supports_surface_for_backdrop = !gfx::gfxVars::UseSoftwareWebRender();
+#endif
 }
 
 bool RenderCompositorNative::MaybeReadback(
@@ -311,6 +314,20 @@ void RenderCompositorNative::CreateExternalSurface(wr::NativeSurfaceId aId,
   mSurfaces.insert({aId, std::move(surface)});
 }
 
+void RenderCompositorNative::CreateBackdropSurface(wr::NativeSurfaceId aId,
+                                                   wr::ColorF aColor) {
+  MOZ_RELEASE_ASSERT(mSurfaces.find(aId) == mSurfaces.end());
+
+  gfx::DeviceColor color(aColor.r, aColor.g, aColor.b, aColor.a);
+  RefPtr<layers::NativeLayer> layer =
+      mNativeLayerRoot->CreateLayerForColor(color);
+
+  Surface surface{DeviceIntSize{}, (aColor.a >= 1.0f)};
+  surface.mNativeLayers.insert({TileKey(0, 0), layer});
+
+  mSurfaces.insert({aId, std::move(surface)});
+}
+
 void RenderCompositorNative::AttachExternalImage(
     wr::NativeSurfaceId aId, wr::ExternalImageId aExternalImage) {
   RenderTextureHost* image =

gfx/webrender_bindings/RenderCompositorNative.h

@@ -63,6 +63,8 @@ class RenderCompositorNative : public RenderCompositor {
   void CreateSurface(wr::NativeSurfaceId aId, wr::DeviceIntPoint aVirtualOffset,
                      wr::DeviceIntSize aTileSize, bool aIsOpaque) override;
   void CreateExternalSurface(wr::NativeSurfaceId aId, bool aIsOpaque) override;
+  void CreateBackdropSurface(wr::NativeSurfaceId aId,
+                             wr::ColorF aColor) override;
   void DestroySurface(NativeSurfaceId aId) override;
   void CreateTile(wr::NativeSurfaceId aId, int32_t aX, int32_t aY) override;
   void DestroyTile(wr::NativeSurfaceId aId, int32_t aX, int32_t aY) override;