Bug 1743577 [Linux] Support color ranges by VA-API, r=sotaro,jgilbert

Get frame color range from ffmpeg by av_frame_get_color_range() (we do the same for non VA-API path),
store to DMABufSurface and propagate to rendering thread.

Differential Revision: https://phabricator.services.mozilla.com/D132469

dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp

@@ -733,6 +733,13 @@ MediaResult FFmpegVideoDecoder<LIBAV_VER>::CreateImageVAAPI(
   surface->LockVAAPIData(mCodecContext, mFrame, mLib);
   surface->SetYUVColorSpace(GetFrameColorSpace());
 
+  if (mLib->av_frame_get_color_range) {
+    auto range = mLib->av_frame_get_color_range(mFrame);
+    surface->SetColorRange(range == AVCOL_RANGE_JPEG
+                               ? gfx::ColorRange::FULL
+                               : gfx::ColorRange::LIMITED);
+  }
+
   RefPtr<VideoData> vp = VideoData::CreateFromImage(
       mInfo.mDisplay, aOffset, TimeUnit::FromMicroseconds(aPts),
       TimeUnit::FromMicroseconds(aDuration), surface->GetAsImage(),
@@ -763,6 +770,13 @@ MediaResult FFmpegVideoDecoder<LIBAV_VER>::CreateImageDMABuf(
   }
   surface->SetYUVColorSpace(GetFrameColorSpace());
 
+  if (mLib->av_frame_get_color_range) {
+    auto range = mLib->av_frame_get_color_range(mFrame);
+    surface->SetColorRange(range == AVCOL_RANGE_JPEG
+                               ? gfx::ColorRange::FULL
+                               : gfx::ColorRange::LIMITED);
+  }
+
   RefPtr<VideoData> vp = VideoData::CreateFromImage(
       mInfo.mDisplay, aOffset, TimeUnit::FromMicroseconds(aPts),
       TimeUnit::FromMicroseconds(aDuration), surface->GetAsImage(),

dom/media/platforms/ffmpeg/FFmpegVideoFramePool.h

@@ -28,6 +28,7 @@ class VideoFrameSurface {
   virtual bool IsUsed() const = 0;
 
   virtual void SetYUVColorSpace(mozilla::gfx::YUVColorSpace aColorSpace) = 0;
+  virtual void SetColorRange(mozilla::gfx::ColorRange aColorRange) = 0;
 
   virtual RefPtr<DMABufSurfaceYUV> GetDMABufSurface() { return nullptr; };
 
@@ -57,6 +58,10 @@ class VideoFrameSurfaceDMABuf : public VideoFrameSurface {
     mSurface->GetAsDMABufSurfaceYUV()->SetYUVColorSpace(aColorSpace);
   }
 
+  void SetColorRange(mozilla::gfx::ColorRange aColorRange) {
+    mSurface->GetAsDMABufSurfaceYUV()->SetColorRange(aColorRange);
+  }
+
   RefPtr<DMABufSurfaceYUV> GetDMABufSurface() {
     return mSurface->GetAsDMABufSurfaceYUV();
   };

gfx/layers/ipc/LayersSurfaces.ipdlh

@@ -59,6 +59,7 @@ namespace layers {
   uint32_t[] strides;
   uint32_t[] offsets;
   YUVColorSpace yUVColorSpace;
+  ColorRange colorRange;
   FileDescriptor[] fence;
   uint32_t uid;
   FileDescriptor[] refCount;

widget/gtk/DMABufSurface.cpp

@@ -507,10 +507,10 @@ bool DMABufSurfaceRGBA::Serialize(
     refCountFDs.AppendElement(ipc::FileDescriptor(mGlobalRefCountFd));
   }
 
-  aOutDescriptor =
-      SurfaceDescriptorDMABuf(mSurfaceType, mBufferModifier, mGbmBufferFlags,
-                              fds, width, height, format, strides, offsets,
-                              GetYUVColorSpace(), fenceFDs, mUID, refCountFDs);
+  aOutDescriptor = SurfaceDescriptorDMABuf(
+      mSurfaceType, mBufferModifier, mGbmBufferFlags, fds, width, height,
+      format, strides, offsets, GetYUVColorSpace(), mColorRange, fenceFDs, mUID,
+      refCountFDs);
   return true;
 }
 
@@ -1044,6 +1044,7 @@ bool DMABufSurfaceYUV::ImportSurfaceDescriptor(
   mSurfaceType = (mBufferPlaneCount == 2) ? SURFACE_NV12 : SURFACE_YUV420;
   mBufferModifier = aDesc.modifier();
   mColorSpace = aDesc.yUVColorSpace();
+  mColorRange = aDesc.colorRange();
   mUID = aDesc.uid();
 
   LOGDMABUF(("DMABufSurfaceYUV::ImportSurfaceDescriptor() UID %d", mUID));
@@ -1120,7 +1121,7 @@ bool DMABufSurfaceYUV::Serialize(
 
   aOutDescriptor = SurfaceDescriptorDMABuf(
       mSurfaceType, mBufferModifier, 0, fds, width, height, format, strides,
-      offsets, GetYUVColorSpace(), fenceFDs, mUID, refCountFDs);
+      offsets, GetYUVColorSpace(), mColorRange, fenceFDs, mUID, refCountFDs);
   return true;
 }
 

widget/gtk/DMABufSurface.h

@@ -88,7 +88,11 @@ class DMABufSurface {
   virtual mozilla::gfx::YUVColorSpace GetYUVColorSpace() {
     return mozilla::gfx::YUVColorSpace::Default;
   };
-  virtual bool IsFullRange() { return false; };
+
+  bool IsFullRange() { return mColorRange == mozilla::gfx::ColorRange::FULL; };
+  void SetColorRange(mozilla::gfx::ColorRange aColorRange) {
+    mColorRange = aColorRange;
+  };
 
   void FenceSet();
   void FenceWait();
@@ -178,6 +182,8 @@ class DMABufSurface {
   int mGlobalRefCountFd;
   uint32_t mUID;
   mozilla::Mutex mSurfaceLock;
+
+  mozilla::gfx::ColorRange mColorRange = mozilla::gfx::ColorRange::LIMITED;
 };
 
 class DMABufSurfaceRGBA : public DMABufSurface {