Bug 1469711 - P1. Add 10/12 bits YUV support to D3D11 compositor. r=mattwoodrow

This change is for D3D11 with Advanced Layers enabled.

Differential Revision: https://phabricator.services.mozilla.com/D6492

--HG--
extra : moz-landing-system : lando

gfx/2d/Tools.h

@@ -122,6 +122,31 @@ SurfaceFormatForAlphaBitDepth(uint32_t aBitDepth)
   return SurfaceFormat::UNKNOWN;
 }
 
+static inline ColorDepth
+ColorDepthForAlphaBitDepth(uint32_t aBitDepth)
+{
+  switch (aBitDepth) {
+  case 8: return ColorDepth::COLOR_8;
+  case 10: return ColorDepth::COLOR_10;
+  case 12: return ColorDepth::COLOR_12;
+  default:
+    MOZ_ASSERT_UNREACHABLE("Unsupported alpha bit depth");
+    return ColorDepth::COLOR_8;
+  }
+}
+
+// 10 and 12 bits color depth image are using 16 bits integers for storage
+// As such we need to rescale the value from 8,10 or 12 to 16.
+static inline uint32_t
+RescalingFactorForAlphaBitDepth(uint32_t aBitDepth)
+{
+  MOZ_ASSERT(aBitDepth == 8 || aBitDepth == 10 || aBitDepth == 12);
+  uint32_t pixelBits =
+    8 * BytesPerPixel(SurfaceFormatForAlphaBitDepth(aBitDepth));
+  uint32_t paddingBits = pixelBits - aBitDepth;
+  return pow(2, paddingBits);
+}
+
 static inline bool
 IsOpaqueFormat(SurfaceFormat aFormat) {
   switch (aFormat) {

gfx/2d/Types.h

@@ -102,6 +102,13 @@ inline bool IsOpaque(SurfaceFormat aFormat)
   }
 }
 
+enum class ColorDepth : uint8_t {
+  COLOR_8,
+  COLOR_10,
+  COLOR_12,
+  MAX
+};
+
 enum class FilterType : int8_t {
   BLEND = 0,
   TRANSFORM,

gfx/layers/D3D11YCbCrImage.cpp

@@ -35,6 +35,7 @@ D3D11YCbCrImage::SetData(KnowsCompositor* aAllocator,
     aData.mPicX, aData.mPicY, aData.mPicSize.width, aData.mPicSize.height);
   mYSize = aData.mYSize;
   mCbCrSize = aData.mCbCrSize;
+  mBitDepth = aData.mBitDepth;
   mColorSpace = aData.mYUVColorSpace;
 
   D3D11YCbCrRecycleAllocator* allocator =
@@ -249,6 +250,7 @@ D3D11YCbCrImage::GetAsSourceSurface()
   data.mPicY = mPictureRect.Y();
   data.mPicSize = mPictureRect.Size();
   data.mStereoMode = StereoMode::MONO;
+  data.mBitDepth = mBitDepth;
   data.mYUVColorSpace = mColorSpace;
   data.mYSkip = data.mCbSkip = data.mCrSkip = 0;
   data.mYSize = mYSize;
@@ -353,6 +355,7 @@ DXGIYCbCrTextureAllocationHelper::IsCompatible(TextureClient* aTextureClient)
       aTextureClient->GetSize() != mData.mYSize ||
       dxgiData->GetYSize() != mData.mYSize ||
       dxgiData->GetCbCrSize() != mData.mCbCrSize ||
+      dxgiData->GetBitDepth() != mData.mBitDepth ||
       dxgiData->GetYUVColorSpace() != mData.mYUVColorSpace) {
     return false;
   }
@@ -386,8 +389,12 @@ DXGIYCbCrTextureAllocationHelper::IsCompatible(TextureClient* aTextureClient)
 already_AddRefed<TextureClient>
 DXGIYCbCrTextureAllocationHelper::Allocate(KnowsCompositor* aAllocator)
 {
-  CD3D11_TEXTURE2D_DESC newDesc(DXGI_FORMAT_R8_UNORM, mData.mYSize.width, mData.mYSize.height,
-                                1, 1);
+  CD3D11_TEXTURE2D_DESC newDesc(
+    mData.mBitDepth == 8 ? DXGI_FORMAT_R8_UNORM : DXGI_FORMAT_R16_UNORM,
+    mData.mYSize.width,
+    mData.mYSize.height,
+    1,
+    1);
   // WebRender requests keyed mutex
   if (mDevice == gfx::DeviceManagerDx::Get()->GetCompositorDevice() &&
       !gfxVars::UseWebRender()) {
@@ -437,6 +444,7 @@ DXGIYCbCrTextureAllocationHelper::Allocate(KnowsCompositor* aAllocator)
       mData.mYSize,
       mData.mYSize,
       mData.mCbCrSize,
+      mData.mBitDepth,
       mData.mYUVColorSpace),
     mTextureFlags,
     forwarder);

gfx/layers/D3D11YCbCrImage.h

@@ -84,6 +84,7 @@ private:
   gfx::IntSize mYSize;
   gfx::IntSize mCbCrSize;
   gfx::IntRect mPictureRect;
+  uint32_t mBitDepth;
   YUVColorSpace mColorSpace;
   RefPtr<TextureClient> mTextureClient;
 };

gfx/layers/d3d11/TextureD3D11.cpp

@@ -92,6 +92,8 @@ SurfaceFormatToDXGIFormat(gfx::SurfaceFormat aFormat)
       return DXGI_FORMAT_R8G8B8A8_UNORM;
     case SurfaceFormat::A8:
       return DXGI_FORMAT_R8_UNORM;
+    case SurfaceFormat::A16:
+      return DXGI_FORMAT_R16_UNORM;
     default:
       MOZ_ASSERT(false, "unsupported format");
       return DXGI_FORMAT_UNKNOWN;
@@ -624,6 +626,7 @@ DXGIYCbCrTextureData::Create(IDirect3DTexture9* aTextureY,
                              const gfx::IntSize& aSize,
                              const gfx::IntSize& aSizeY,
                              const gfx::IntSize& aSizeCbCr,
+                             uint32_t aBitDepth,
                              YUVColorSpace aYUVColorSpace)
 {
   if (!aHandleY || !aHandleCb || !aHandleCr ||
@@ -641,6 +644,7 @@ DXGIYCbCrTextureData::Create(IDirect3DTexture9* aTextureY,
   texture->mSize = aSize;
   texture->mSizeY = aSizeY;
   texture->mSizeCbCr = aSizeCbCr;
+  texture->mBitDepth = aBitDepth;
   texture->mYUVColorSpace = aYUVColorSpace;
 
   return texture;
@@ -653,6 +657,7 @@ DXGIYCbCrTextureData::Create(ID3D11Texture2D* aTextureY,
                              const gfx::IntSize& aSize,
                              const gfx::IntSize& aSizeY,
                              const gfx::IntSize& aSizeCbCr,
+                             uint32_t aBitDepth,
                              YUVColorSpace aYUVColorSpace)
 {
   if (!aTextureY || !aTextureCb || !aTextureCr) {
@@ -701,6 +706,7 @@ DXGIYCbCrTextureData::Create(ID3D11Texture2D* aTextureY,
   texture->mSize = aSize;
   texture->mSizeY = aSizeY;
   texture->mSizeCbCr = aSizeCbCr;
+  texture->mBitDepth = aBitDepth;
   texture->mYUVColorSpace = aYUVColorSpace;
 
   return texture;
@@ -721,7 +727,7 @@ DXGIYCbCrTextureData::SerializeSpecific(SurfaceDescriptorDXGIYCbCr* const aOutDe
 {
   *aOutDesc = SurfaceDescriptorDXGIYCbCr(
     (WindowsHandle)mHandles[0], (WindowsHandle)mHandles[1], (WindowsHandle)mHandles[2],
-    mSize, mSizeY, mSizeCbCr, mYUVColorSpace
+    mSize, mSizeY, mSizeCbCr, mBitDepth, mYUVColorSpace
   );
 }
 
@@ -1116,6 +1122,8 @@ DXGITextureHostD3D11::PushResourceUpdates(wr::TransactionBuilder& aResources,
       MOZ_ASSERT(mSize.width % 2 == 0);
       MOZ_ASSERT(mSize.height % 2 == 0);
 
+      // For now, no software decoder can output 10/12 bits NV12 images
+      // So forcing A8 is okay.
       wr::ImageDescriptor descriptor0(mSize, gfx::SurfaceFormat::A8);
       wr::ImageDescriptor descriptor1(mSize / 2, gfx::SurfaceFormat::R8G8);
       auto bufferType = wr::WrExternalImageBufferType::TextureExternalHandle;
@@ -1168,6 +1176,7 @@ DXGIYCbCrTextureHostD3D11::DXGIYCbCrTextureHostD3D11(TextureFlags aFlags,
   , mSize(aDescriptor.size())
   , mSizeCbCr(aDescriptor.sizeCbCr())
   , mIsLocked(false)
+  , mBitDepth(aDescriptor.bitDepth())
   , mYUVColorSpace(aDescriptor.yUVColorSpace())
 {
   mHandles[0] = aDescriptor.handleY();

gfx/layers/d3d11/TextureD3D11.h

@@ -145,6 +145,7 @@ public:
          const gfx::IntSize& aSize,
          const gfx::IntSize& aSizeY,
          const gfx::IntSize& aSizeCbCr,
+         uint32_t aBitDepth,
          YUVColorSpace aYUVColorSpace);
 
   static DXGIYCbCrTextureData*
@@ -154,6 +155,7 @@ public:
          const gfx::IntSize& aSize,
          const gfx::IntSize& aSizeY,
          const gfx::IntSize& aSizeCbCr,
+         uint32_t aBitDepth,
          YUVColorSpace aYUVColorSpace);
 
   virtual bool Lock(OpenMode) override { return true; }
@@ -191,6 +193,11 @@ public:
     return mSizeCbCr;
   }
 
+  uint32_t GetBitDepth() const
+  {
+    return mBitDepth;
+  }
+
   YUVColorSpace GetYUVColorSpace() const
   {
     return mYUVColorSpace;
@@ -205,6 +212,7 @@ protected:
    gfx::IntSize mSize;
    gfx::IntSize mSizeY;
    gfx::IntSize mSizeCbCr;
+   uint32_t mBitDepth;
    YUVColorSpace mYUVColorSpace;
 };
 
@@ -409,6 +417,8 @@ public:
 
   virtual YUVColorSpace GetYUVColorSpace() const override { return mYUVColorSpace; }
 
+  virtual uint32_t GetBitDepth() const override { return mBitDepth; }
+
   virtual bool Lock() override;
 
   virtual void Unlock() override;
@@ -452,6 +462,7 @@ protected:
   gfx::IntSize mSizeCbCr;
   WindowsHandle mHandles[3];
   bool mIsLocked;
+  uint32_t mBitDepth;
   YUVColorSpace mYUVColorSpace;
 };
 

gfx/layers/d3d11/mlgshaders/ycbcr-ps.hlsl

@@ -13,6 +13,10 @@ cbuffer YCbCrBuffer : register(b1) {
   row_major float3x3 YuvColorMatrix;
 };
 
+cbuffer vCoefficientBuffer : register(b2) {
+  float vCoefficient;
+}
+
 /* From Rec601:
 [R]   [1.1643835616438356,  0.0,                 1.5960267857142858]      [ Y -  16]
 [G] = [1.1643835616438358, -0.3917622900949137, -0.8129676472377708]    x [Cb - 128]
@@ -50,14 +54,14 @@ float4 CalculateIMC4Color(const float2 aTexCoords)
     tY.Sample(sSampler, aTexCoords).r,
     tCb.Sample(sSampler, aTexCoords).r,
     tCr.Sample(sSampler, aTexCoords).r);
-  return CalculateYCbCrColor(yuv);
+  return CalculateYCbCrColor(yuv * vCoefficient);
 }
 
 float4 CalculateNV12Color(const float2 aTexCoords)
 {
   float y = tY.Sample(sSampler, aTexCoords).r;
   float2 cbcr = tCb.Sample(sSampler, aTexCoords).rg;
-  return CalculateYCbCrColor(float3(y, cbcr));
+  return CalculateYCbCrColor(float3(y, cbcr) * vCoefficient);
 }
 
 float4 TexturedQuadIMC4(const VS_SAMPLEOUTPUT_CLIPPED aInput) : SV_Target

gfx/layers/ipc/LayersSurfaces.ipdlh

@@ -52,6 +52,7 @@ struct SurfaceDescriptorDXGIYCbCr {
   IntSize size;
   IntSize sizeY;
   IntSize sizeCbCr;
+  uint32_t bitDepth;
   YUVColorSpace yUVColorSpace;
 };
 

gfx/layers/mlgpu/MLGDevice.cpp

@@ -294,6 +294,27 @@ MLGDevice::GetBufferForColorSpace(YUVColorSpace aColorSpace)
   return resource;
 }
 
+RefPtr<MLGBuffer>
+MLGDevice::GetBufferForBitDepthCoefficient(uint8_t aBitDepth)
+{
+  ColorDepth depth = ColorDepthForAlphaBitDepth(aBitDepth);
+  if (mColorDepthBuffers[depth]) {
+    return mColorDepthBuffers[depth];
+  }
+
+  YCbCrBitDepthConstants buffer;
+  buffer.coefficient = gfx::RescalingFactorForAlphaBitDepth(aBitDepth);
+
+  RefPtr<MLGBuffer> resource = CreateBuffer(
+    MLGBufferType::Constant, sizeof(buffer), MLGUsage::Immutable, &buffer);
+  if (!resource) {
+    return nullptr;
+  }
+
+  mColorDepthBuffers[depth] = resource;
+  return resource;
+}
+
 bool
 MLGDevice::Synchronize()
 {

gfx/layers/mlgpu/MLGDevice.h

@@ -360,6 +360,9 @@ public:
   // This creates or returns a previously created constant buffer, containing
   // a YCbCrShaderConstants instance.
   RefPtr<MLGBuffer> GetBufferForColorSpace(YUVColorSpace aColorSpace);
+  // This creates or returns a previously created constant buffer, containing
+  // a YCbCrBitDepthConstants instance.
+  RefPtr<MLGBuffer> GetBufferForBitDepthCoefficient(uint8_t aBitDepth);
 
   // A shared buffer that can be used to build VertexBufferSections.
   SharedVertexBuffer* GetSharedVertexBuffer() {
@@ -466,7 +469,7 @@ protected:
   template <typename... T>
   bool Fail(const char* aFailureId,
             const char* aMessage,
-            const T&... args) 
+            const T&... args)
   {
     nsCString failureId(aFailureId);
     nsPrintfCString message(aMessage, args...);
@@ -491,6 +494,8 @@ private:
 
   typedef EnumeratedArray<YUVColorSpace, YUVColorSpace::UNKNOWN, RefPtr<MLGBuffer>> ColorSpaceArray;
   ColorSpaceArray mColorSpaceBuffers;
+  typedef EnumeratedArray<gfx::ColorDepth, gfx::ColorDepth::MAX, RefPtr<MLGBuffer>> ColorDepthArray;
+  ColorDepthArray mColorDepthBuffers;
 
 protected:
   bool mIsValid;

gfx/layers/mlgpu/RenderPassMLGPU.cpp

@@ -818,6 +818,12 @@ VideoRenderPass::SetupPipeline()
     return;
   }
 
+  RefPtr<MLGBuffer> ps2 =
+    mDevice->GetBufferForBitDepthCoefficient(mHost->GetBitDepth());
+  if (!ps2) {
+    return;
+  }
+
   if (mGeometry == GeometryMode::UnitQuad) {
     mDevice->SetVertexShader(VertexShaderID::TexturedQuad);
   } else {
@@ -848,6 +854,7 @@ VideoRenderPass::SetupPipeline()
 
   mDevice->SetSamplerMode(kDefaultSamplerSlot, mSamplerMode);
   mDevice->SetPSConstantBuffer(1, ps1);
+  mDevice->SetPSConstantBuffer(2, ps2);
 }
 
 RenderViewPass::RenderViewPass(FrameBuilder* aBuilder, const ItemInfo& aItem)

gfx/layers/mlgpu/ShaderDefinitionsMLGPU.h

@@ -43,7 +43,7 @@ static const uint32_t kMaskSamplerSlot = 1;
 // happens above the max slot, since we try to clear buffer bindings at
 // the end of each frame.
 static const uint32_t kMaxVertexShaderConstantBuffers = 5;
-static const uint32_t kMaxPixelShaderConstantBuffers = 2;
+static const uint32_t kMaxPixelShaderConstantBuffers = 3;
 
 // Maximum depth in the depth buffer. This must match common-vs.hlsl.
 static const int32_t kDepthLimit = 1000000;
@@ -92,6 +92,11 @@ struct YCbCrShaderConstants {
   float yuvColorMatrix[3][4];
 };
 
+struct YCbCrBitDepthConstants {
+  float coefficient;
+  uint32_t padding[3];
+};
+
 struct BlendVertexShaderConstants {
   float backdropTransform[4][4];
 };

gfx/layers/opengl/CompositorOGL.cpp

@@ -986,15 +986,11 @@ CompositorOGL::GetShaderConfigFor(Effect *aEffect,
   case EffectTypes::YCBCR:
   {
     config.SetYCbCr(true);
-    EffectYCbCr* effectYCbCr =
-      static_cast<EffectYCbCr*>(aEffect);
-    uint32_t pixelBits = (8 * BytesPerPixel(SurfaceFormatForAlphaBitDepth(effectYCbCr->mBitDepth)));
-    uint32_t paddingBits = pixelBits - effectYCbCr->mBitDepth;
-    // OpenGL expects values between [0,255], this range needs to be adjusted
-    // according to the bit depth.
-    // So we will scale the YUV values by this amount.
-    config.SetColorMultiplier(pow(2, paddingBits));
-    config.SetTextureTarget(effectYCbCr->mTexture->AsSourceOGL()->GetTextureTarget());
+    EffectYCbCr* effectYCbCr = static_cast<EffectYCbCr*>(aEffect);
+    config.SetColorMultiplier(
+      RescalingFactorForAlphaBitDepth(effectYCbCr->mBitDepth));
+    config.SetTextureTarget(
+      effectYCbCr->mTexture->AsSourceOGL()->GetTextureTarget());
     break;
   }
   case EffectTypes::NV12:

gfx/tests/gtest/TextureHelper.h

@@ -32,7 +32,6 @@ namespace layers {
 static already_AddRefed<TextureClient>
 CreateYCbCrTextureClientWithBackend(LayersBackend aLayersBackend)
 {
-
   TextureData* data = nullptr;
   IntSize size = IntSize(200, 150);
   IntSize ySize = IntSize(400, 300);