Bug 1366502 - Update BufferTextureHost and RenderBufferTextureHost for video pipeline. r=sotaro

WR supports the planar-ycbcr image format. We turn to use the planar-ycbcr image to get rid of the software-ycbcr-to-rgb color format conversion(using libyuv) in gecko.

The BufferTextureHost will use 3 image keys for SurfaceFormat::YUV format.
The RenderBufferTextureHost will also use 3 DataSourceSurfaces to represent the 3 channel data in planar-ycbcr format.

MozReview-Commit-ID: 3mMreSzKnMv

gfx/layers/composite/TextureHost.cpp

@@ -556,31 +556,81 @@ BufferTextureHost::Unlock()
   mLocked = false;
 }
 
+void
+BufferTextureHost::GetWRImageKeys(nsTArray<wr::ImageKey>& aImageKeys,
+                                  const std::function<wr::ImageKey()>& aImageKeyAllocator)
+{
+  MOZ_ASSERT(aImageKeys.IsEmpty());
+
+  if (GetFormat() != gfx::SurfaceFormat::YUV) {
+    // 1 image key
+    aImageKeys.AppendElement(aImageKeyAllocator());
+    MOZ_ASSERT(aImageKeys.Length() == 1);
+  } else {
+    // 3 image key
+    aImageKeys.AppendElement(aImageKeyAllocator());
+    aImageKeys.AppendElement(aImageKeyAllocator());
+    aImageKeys.AppendElement(aImageKeyAllocator());
+    MOZ_ASSERT(aImageKeys.Length() == 3);
+  }
+}
+
 void
 BufferTextureHost::AddWRImage(wr::WebRenderAPI* aAPI,
                               Range<const wr::ImageKey>& aImageKeys,
                               const wr::ExternalImageId& aExtID)
 {
-  MOZ_ASSERT(aImageKeys.length() == 1);
-  // XXX handling YUV
-  gfx::SurfaceFormat wrFormat =
-      (GetFormat() == gfx::SurfaceFormat::YUV) ? gfx::SurfaceFormat::B8G8R8A8
-                                               : GetFormat();
-  gfx::SurfaceFormat format = GetFormat();
-  uint32_t wrStride = 0;
+  if (GetFormat() != gfx::SurfaceFormat::YUV) {
+    MOZ_ASSERT(aImageKeys.length() == 1);
 
-  if (format == gfx::SurfaceFormat::YUV) {
-    // XXX this stride is used until yuv image rendering by webrender is used.
-    // Software converted RGB buffers strides are aliened to 16
-    wrStride = gfx::GetAlignedStride<16>(GetSize().width, BytesPerPixel(gfx::SurfaceFormat::B8G8R8A8));
+    wr::ImageDescriptor descriptor(GetSize(),
+                                   ImageDataSerializer::ComputeRGBStride(GetFormat(), GetSize().width),
+                                   GetFormat());
+    aAPI->AddExternalImageBuffer(aImageKeys[0], descriptor, aExtID);
   } else {
-    wrStride = ImageDataSerializer::ComputeRGBStride(format, GetSize().width);
-  }
+    MOZ_ASSERT(aImageKeys.length() == 3);
 
-  wr::ImageDescriptor descriptor(GetSize(), wrStride, wrFormat);
-  aAPI->AddExternalImageBuffer(aImageKeys[0],
-                               descriptor,
-                               aExtID);
+    const layers::YCbCrDescriptor& desc = mDescriptor.get_YCbCrDescriptor();
+    wr::ImageDescriptor yDescriptor(desc.ySize(), desc.ySize().width, gfx::SurfaceFormat::A8);
+    wr::ImageDescriptor cbcrDescriptor(desc.cbCrSize(), desc.cbCrSize().width, gfx::SurfaceFormat::A8);
+    aAPI->AddExternalImage(aImageKeys[0],
+                           yDescriptor,
+                           aExtID,
+                           WrExternalImageBufferType::ExternalBuffer,
+                           0);
+    aAPI->AddExternalImage(aImageKeys[1],
+                           cbcrDescriptor,
+                           aExtID,
+                           WrExternalImageBufferType::ExternalBuffer,
+                           1);
+    aAPI->AddExternalImage(aImageKeys[2],
+                           cbcrDescriptor,
+                           aExtID,
+                           WrExternalImageBufferType::ExternalBuffer,
+                           2);
+  }
+}
+
+void
+BufferTextureHost::PushExternalImage(wr::DisplayListBuilder& aBuilder,
+                                     const WrRect& aBounds,
+                                     const WrClipRegionToken aClip,
+                                     wr::ImageRendering aFilter,
+                                     Range<const wr::ImageKey>& aImageKeys)
+{
+  if (GetFormat() != gfx::SurfaceFormat::YUV) {
+    MOZ_ASSERT(aImageKeys.length() == 1);
+    aBuilder.PushImage(aBounds, aClip, aFilter, aImageKeys[0]);
+  } else {
+    MOZ_ASSERT(aImageKeys.length() == 3);
+    aBuilder.PushYCbCrPlanarImage(aBounds,
+                                  aClip,
+                                  aImageKeys[0],
+                                  aImageKeys[1],
+                                  aImageKeys[2],
+                                  WrYuvColorSpace::Rec601,
+                                  aFilter);
+  }
 }
 
 void

gfx/layers/composite/TextureHost.h

@@ -716,10 +716,19 @@ public:
 
   const BufferDescriptor& GetBufferDescriptor() const { return mDescriptor; }
 
+  virtual void GetWRImageKeys(nsTArray<wr::ImageKey>& aImageKeys,
+                              const std::function<wr::ImageKey()>& aImageKeyAllocator) override;
+
   virtual void AddWRImage(wr::WebRenderAPI* aAPI,
                           Range<const wr::ImageKey>& aImageKeys,
                           const wr::ExternalImageId& aExtID) override;
 
+  virtual void PushExternalImage(wr::DisplayListBuilder& aBuilder,
+                                 const WrRect& aBounds,
+                                 const WrClipRegionToken aClip,
+                                 wr::ImageRendering aFilter,
+                                 Range<const wr::ImageKey>& aImageKeys) override;
+
 protected:
   bool Upload(nsIntRegion *aRegion = nullptr);
   bool MaybeUpload(nsIntRegion *aRegion = nullptr);

gfx/webrender_bindings/RenderBufferTextureHost.cpp

@@ -43,67 +43,100 @@ RenderBufferTextureHost::~RenderBufferTextureHost()
   MOZ_COUNT_DTOR_INHERITED(RenderBufferTextureHost, RenderTextureHost);
 }
 
-already_AddRefed<gfx::DataSourceSurface>
-RenderBufferTextureHost::GetAsSurface()
-{
-  RefPtr<gfx::DataSourceSurface> result;
-  if (mFormat == gfx::SurfaceFormat::YUV) {
-    result = layers::ImageDataSerializer::DataSourceSurfaceFromYCbCrDescriptor(
-      GetBuffer(), mDescriptor.get_YCbCrDescriptor());
-    if (NS_WARN_IF(!result)) {
-      return nullptr;
-    }
-  } else {
-    result =
-      gfx::Factory::CreateWrappingDataSourceSurface(GetBuffer(),
-          layers::ImageDataSerializer::GetRGBStride(mDescriptor.get_RGBDescriptor()),
-        mSize, mFormat);
-  }
-  return result.forget();
-}
-
 bool
 RenderBufferTextureHost::Lock()
 {
-  MOZ_ASSERT(!mLocked);
+  if (!mLocked) {
+    if (mFormat != gfx::SurfaceFormat::YUV) {
+      mSurface = gfx::Factory::CreateWrappingDataSourceSurface(GetBuffer(),
+                                                               layers::ImageDataSerializer::GetRGBStride(mDescriptor.get_RGBDescriptor()),
+                                                               mSize,
+                                                               mFormat);
+      if (NS_WARN_IF(!mSurface)) {
+        return false;
+      }
+      if (NS_WARN_IF(!mSurface->Map(gfx::DataSourceSurface::MapType::READ_WRITE, &mMap))) {
+        mSurface = nullptr;
+        return false;
+      }
+    } else {
+      const layers::YCbCrDescriptor& desc = mDescriptor.get_YCbCrDescriptor();
 
-  // XXX temporal workaround for YUV handling
-  if (!mSurface) {
-    mSurface = GetAsSurface();
-    if (!mSurface) {
-      return false;
+      mYSurface = gfx::Factory::CreateWrappingDataSourceSurface(layers::ImageDataSerializer::GetYChannel(GetBuffer(), desc),
+                                                                desc.ySize().width,
+                                                                desc.ySize(),
+                                                                gfx::SurfaceFormat::A8);
+      mCbSurface = gfx::Factory::CreateWrappingDataSourceSurface(layers::ImageDataSerializer::GetCbChannel(GetBuffer(), desc),
+                                                                 desc.cbCrSize().width,
+                                                                 desc.cbCrSize(),
+                                                                 gfx::SurfaceFormat::A8);
+      mCrSurface = gfx::Factory::CreateWrappingDataSourceSurface(layers::ImageDataSerializer::GetCrChannel(GetBuffer(), desc),
+                                                                 desc.cbCrSize().width,
+                                                                 desc.cbCrSize(),
+                                                                 gfx::SurfaceFormat::A8);
+      if (NS_WARN_IF(!mYSurface || !mCbSurface || !mCrSurface)) {
+        mYSurface = mCbSurface = mCrSurface = nullptr;
+        return false;
+      }
+      if (NS_WARN_IF(!mYSurface->Map(gfx::DataSourceSurface::MapType::READ_WRITE, &mYMap) ||
+                     !mCbSurface->Map(gfx::DataSourceSurface::MapType::READ_WRITE, &mCbMap) ||
+                     !mCrSurface->Map(gfx::DataSourceSurface::MapType::READ_WRITE, &mCrMap))) {
+        mYSurface = mCbSurface = mCrSurface = nullptr;
+        return false;
+      }
     }
+    mLocked = true;
   }
 
-  if (NS_WARN_IF(!mSurface->Map(gfx::DataSourceSurface::MapType::READ_WRITE, &mMap))) {
-    mSurface = nullptr;
-    return false;
-  }
-
-  mLocked = true;
   return true;
 }
 
 void
 RenderBufferTextureHost::Unlock()
 {
-  MOZ_ASSERT(mLocked);
-  mLocked = false;
-  if (mSurface) {
-    mSurface->Unmap();
+  if (mLocked) {
+    if (mSurface) {
+      mSurface->Unmap();
+      mSurface = nullptr;
+    } else if (mYSurface) {
+      mYSurface->Unmap();
+      mCbSurface->Unmap();
+      mCrSurface->Unmap();
+      mYSurface = mCbSurface = mCrSurface = nullptr;
+    }
+    mLocked = false;
   }
-  mSurface = nullptr;
 }
 
 RenderBufferTextureHost::RenderBufferData
 RenderBufferTextureHost::GetBufferDataForRender(uint8_t aChannelIndex)
 {
-  // TODO: handle multiple channel bufferTextureHost(e.g. yuv textureHost)
-  MOZ_ASSERT(aChannelIndex < 1);
-
+  MOZ_ASSERT(mFormat != gfx::SurfaceFormat::YUV || aChannelIndex < 3);
+  MOZ_ASSERT(mFormat == gfx::SurfaceFormat::YUV || aChannelIndex < 1);
   MOZ_ASSERT(mLocked);
-  MOZ_ASSERT(mSurface);
-  return RenderBufferData(mMap.mData, mMap.mStride * mSurface->GetSize().height);
+
+  if (mFormat != gfx::SurfaceFormat::YUV) {
+    MOZ_ASSERT(mSurface);
+
+    return RenderBufferData(mMap.mData, mMap.mStride * mSurface->GetSize().height);
+  } else {
+    MOZ_ASSERT(mYSurface && mCbSurface && mCrSurface);
+
+    switch (aChannelIndex) {
+      case 0:
+        return RenderBufferData(mYMap.mData, mYMap.mStride * mYSurface->GetSize().height);
+        break;
+      case 1:
+        return RenderBufferData(mCbMap.mData, mCbMap.mStride * mCbSurface->GetSize().height);
+        break;
+      case 2:
+        return RenderBufferData(mCrMap.mData, mCrMap.mStride * mCrSurface->GetSize().height);
+        break;
+      default:
+        MOZ_ASSERT_UNREACHABLE("unexpected to be called");
+        return RenderBufferData(nullptr, 0);
+    }
+  }
 }
 
 } // namespace wr

gfx/webrender_bindings/RenderBufferTextureHost.h

@@ -42,7 +42,6 @@ public:
 private:
   virtual ~RenderBufferTextureHost();
 
-  already_AddRefed<gfx::DataSourceSurface> GetAsSurface();
   uint8_t* GetBuffer() const
   {
     return mBuffer;
@@ -52,8 +51,17 @@ private:
   layers::BufferDescriptor mDescriptor;
   gfx::IntSize mSize;
   gfx::SurfaceFormat mFormat;
+
   RefPtr<gfx::DataSourceSurface> mSurface;
   gfx::DataSourceSurface::MappedSurface mMap;
+
+  RefPtr<gfx::DataSourceSurface> mYSurface;
+  RefPtr<gfx::DataSourceSurface> mCbSurface;
+  RefPtr<gfx::DataSourceSurface> mCrSurface;
+  gfx::DataSourceSurface::MappedSurface mYMap;
+  gfx::DataSourceSurface::MappedSurface mCbMap;
+  gfx::DataSourceSurface::MappedSurface mCrMap;
+
   bool mLocked;
 };