Bug 1540581 - P13. Properly handle colorspace with D3D11 compositors. r=mattwoodrow

Also fix 10/12 bits images composition with old D3D11 compositor.

Differential Revision: https://phabricator.services.mozilla.com/D26468

--HG--
extra : moz-landing-system : lando

gfx/layers/d3d11/CompositorD3D11.cpp

@@ -952,12 +952,12 @@ void CompositorD3D11::DrawGeometry(const Geometry& aGeometry,
       SetSamplerForSamplingFilter(texturedEffect->mSamplingFilter);
     } break;
     case EffectTypes::NV12: {
-      TexturedEffect* texturedEffect =
-          static_cast<TexturedEffect*>(aEffectChain.mPrimaryEffect.get());
+      EffectNV12* effectNV12 =
+          static_cast<EffectNV12*>(aEffectChain.mPrimaryEffect.get());
 
-      pTexCoordRect = &texturedEffect->mTextureCoords;
+      pTexCoordRect = &effectNV12->mTextureCoords;
 
-      TextureSourceD3D11* source = texturedEffect->mTexture->AsSourceD3D11();
+      TextureSourceD3D11* source = effectNV12->mTexture->AsSourceD3D11();
       if (!source) {
         NS_WARNING("Missing texture source!");
         return;
@@ -994,14 +994,13 @@ void CompositorD3D11::DrawGeometry(const Geometry& aGeometry,
       mContext->PSSetShaderResources(TexSlot::Y, 2, views);
 
       const float* yuvToRgb =
-          gfxUtils::YuvToRgbMatrix4x3RowMajor(YUVColorSpace::BT601);
+          gfxUtils::YuvToRgbMatrix4x3RowMajor(effectNV12->mYUVColorSpace);
       memcpy(&mPSConstants.yuvColorMatrix, yuvToRgb,
              sizeof(mPSConstants.yuvColorMatrix));
-      // TOTO: need to handle color depth properly. this assumes data is always
-      // 8 or 16 bits.
-      mPSConstants.vCoefficient[0] = 1.0;
+      mPSConstants.vCoefficient[0] =
+          RescalingFactorForColorDepth(effectNV12->mColorDepth);
 
-      SetSamplerForSamplingFilter(texturedEffect->mSamplingFilter);
+      SetSamplerForSamplingFilter(effectNV12->mSamplingFilter);
     } break;
     case EffectTypes::YCBCR: {
       EffectYCbCr* ycbcrEffect =

gfx/layers/mlgpu/RenderPassMLGPU.cpp

@@ -705,16 +705,11 @@ bool VideoRenderPass::AddToPass(LayerMLGPU* aLayer, ItemInfo& aItem) {
 void VideoRenderPass::SetupPipeline() {
   YUVColorSpace colorSpace = YUVColorSpace::UNKNOWN;
   switch (mHost->GetReadFormat()) {
-    case SurfaceFormat::YUV: {
-      colorSpace = mHost->GetYUVColorSpace();
-      break;
-    }
+    case SurfaceFormat::YUV:
     case SurfaceFormat::NV12:
     case SurfaceFormat::P010:
     case SurfaceFormat::P016:
-      // TODO. BT601 is very unlikely to be the right value for high-def
-      // content.
-      colorSpace = YUVColorSpace::BT601;
+      colorSpace = mHost->GetYUVColorSpace();
       break;
     default:
       MOZ_ASSERT_UNREACHABLE("Unexpected surface format in VideoRenderPass");