Bug 1595994 - P16. Handle NullData object potentially returned by the video decoders. r=mattwoodrow

Depends on D91692

Differential Revision: https://phabricator.services.mozilla.com/D91693

dom/media/ipc/RemoteImageHolder.cpp

@@ -21,24 +21,19 @@ RemoteImageHolder::RemoteImageHolder(layers::IGPUVideoSurfaceManager* aManager,
                                      layers::VideoBridgeSource aSource,
                                      const gfx::IntSize& aSize,
                                      const layers::SurfaceDescriptor& aSD)
-    : mSource(aSource),
-      mSize(aSize),
-      mSD(aSD),
-      mManager(aManager),
-      mEmpty(false) {}
+    : mSource(aSource), mSize(aSize), mSD(Some(aSD)), mManager(aManager) {}
 RemoteImageHolder::RemoteImageHolder(RemoteImageHolder&& aOther)
     : mSource(aOther.mSource),
       mSize(aOther.mSize),
-      mSD(aOther.mSD),
-      mManager(aOther.mManager),
-      mEmpty(aOther.mEmpty) {
-  aOther.mEmpty = true;
+      mSD(std::move(aOther.mSD)),
+      mManager(aOther.mManager) {
+  aOther.mSD = Nothing();
 }
 
 already_AddRefed<Image> RemoteImageHolder::DeserializeImage(
     layers::BufferRecycleBin* aBufferRecycleBin) {
-  MOZ_ASSERT(mSD.type() == SurfaceDescriptor::TSurfaceDescriptorBuffer);
-  const SurfaceDescriptorBuffer& sdBuffer = mSD.get_SurfaceDescriptorBuffer();
+  MOZ_ASSERT(mSD && mSD->type() == SurfaceDescriptor::TSurfaceDescriptorBuffer);
+  const SurfaceDescriptorBuffer& sdBuffer = mSD->get_SurfaceDescriptorBuffer();
   MOZ_ASSERT(sdBuffer.desc().type() == BufferDescriptor::TYCbCrDescriptor);
   if (sdBuffer.desc().type() != BufferDescriptor::TYCbCrDescriptor ||
       !aBufferRecycleBin) {
@@ -105,22 +100,22 @@ already_AddRefed<Image> RemoteImageHolder::DeserializeImage(
 
 already_AddRefed<layers::Image> RemoteImageHolder::TransferToImage(
     layers::BufferRecycleBin* aBufferRecycleBin) {
-  if (mEmpty) {
+  if (IsEmpty()) {
     return nullptr;
   }
   RefPtr<Image> image;
-  if (mSD.type() == SurfaceDescriptor::TSurfaceDescriptorBuffer) {
+  if (mSD->type() == SurfaceDescriptor::TSurfaceDescriptorBuffer) {
     image = DeserializeImage(aBufferRecycleBin);
   } else {
     // The Image here creates a TextureData object that takes ownership
     // of the SurfaceDescriptor, and is responsible for making sure that
     // it gets deallocated.
     SurfaceDescriptorRemoteDecoder remoteSD =
-        static_cast<const SurfaceDescriptorGPUVideo&>(mSD);
+        static_cast<const SurfaceDescriptorGPUVideo&>(*mSD);
     remoteSD.source() = Some(mSource);
     image = new GPUVideoImage(mManager, remoteSD, mSize);
   }
-  mEmpty = true;
+  mSD = Nothing();
   mManager = nullptr;
 
   return image.forget();
@@ -131,10 +126,10 @@ RemoteImageHolder::~RemoteImageHolder() {
   // this image holder (the decoder could have been flushed). We don't need to
   // worry about Shmem based image as the Shmem will be automatically re-used
   // once the decoder is used again.
-  if (!mEmpty && mManager &&
-      mSD.type() != SurfaceDescriptor::TSurfaceDescriptorBuffer) {
+  if (!IsEmpty() && mManager &&
+      mSD->type() != SurfaceDescriptor::TSurfaceDescriptorBuffer) {
     SurfaceDescriptorRemoteDecoder remoteSD =
-        static_cast<const SurfaceDescriptorGPUVideo&>(mSD);
+        static_cast<const SurfaceDescriptorGPUVideo&>(*mSD);
     mManager->DeallocateSurfaceDescriptor(remoteSD);
   }
 }
@@ -144,9 +139,8 @@ RemoteImageHolder::~RemoteImageHolder() {
   WriteIPDLParam(aMsg, aActor, aParam.mSource);
   WriteIPDLParam(aMsg, aActor, aParam.mSize);
   WriteIPDLParam(aMsg, aActor, aParam.mSD);
-  WriteIPDLParam(aMsg, aActor, aParam.mEmpty);
   // Empty this holder.
-  aParam.mEmpty = true;
+  aParam.mSD = Nothing();
   aParam.mManager = nullptr;
 }
 
@@ -155,11 +149,10 @@ RemoteImageHolder::~RemoteImageHolder() {
     RemoteImageHolder* aResult) {
   if (!ReadIPDLParam(aMsg, aIter, aActor, &aResult->mSource) ||
       !ReadIPDLParam(aMsg, aIter, aActor, &aResult->mSize) ||
-      !ReadIPDLParam(aMsg, aIter, aActor, &aResult->mSD) ||
-      !ReadIPDLParam(aMsg, aIter, aActor, &aResult->mEmpty)) {
+      !ReadIPDLParam(aMsg, aIter, aActor, &aResult->mSD)) {
     return false;
   }
-  if (!aResult->mEmpty) {
+  if (!aResult->IsEmpty()) {
     aResult->mManager = RemoteDecoderManagerChild::GetSingleton(
         aResult->mSource == VideoBridgeSource::GpuProcess
             ? RemoteDecodeIn::GpuProcess

dom/media/ipc/RemoteImageHolder.h

@@ -9,6 +9,7 @@
 
 #include "MediaData.h"
 #include "ipc/IPCMessageUtils.h"
+#include "mozilla/Maybe.h"
 #include "mozilla/RefPtr.h"
 #include "mozilla/layers/LayersSurfaces.h"
 #include "mozilla/layers/VideoBridgeUtils.h"
@@ -17,13 +18,13 @@ namespace mozilla {
 namespace layers {
 class BufferRecycleBin;
 class IGPUVideoSurfaceManager;
+class SurfaceDescriptor;
 }  // namespace layers
 class RemoteImageHolder final {
   friend struct ipc::IPDLParamTraits<RemoteImageHolder>;
 
  public:
-  RemoteImageHolder();  // only required for the IPDL binding code,
-                        // not used in practice.
+  RemoteImageHolder();
   RemoteImageHolder(layers::IGPUVideoSurfaceManager* aManager,
                     layers::VideoBridgeSource aSource,
                     const gfx::IntSize& aSize,
@@ -33,6 +34,8 @@ class RemoteImageHolder final {
   RemoteImageHolder(const RemoteImageHolder& aOther) = delete;
   RemoteImageHolder& operator=(const RemoteImageHolder& aOther) = delete;
   ~RemoteImageHolder();
+
+  bool IsEmpty() const { return mSD.isNothing(); }
   // Move content of RemoteImageHolder into a usable Image. Ownership is
   // transfered to that Image.
   already_AddRefed<layers::Image> TransferToImage(
@@ -44,9 +47,8 @@ class RemoteImageHolder final {
   // We need a default for the default constructor, never used in practice.
   layers::VideoBridgeSource mSource = layers::VideoBridgeSource::GpuProcess;
   gfx::IntSize mSize;
-  layers::SurfaceDescriptor mSD;
+  Maybe<layers::SurfaceDescriptor> mSD;
   RefPtr<layers::IGPUVideoSurfaceManager> mManager;
-  bool mEmpty = true;
 };
 
 template <>

dom/media/ipc/RemoteVideoDecoder.cpp

@@ -75,7 +75,14 @@ MediaResult RemoteVideoDecoderChild::ProcessOutput(
       aDecodedData.get_ArrayOfRemoteVideoData()->Array();
 
   for (auto&& data : arrayData) {
+    if (data.image().IsEmpty()) {
+      // This is a NullData object.
+      mDecodedData.AppendElement(MakeRefPtr<NullData>(
+          data.base().offset(), data.base().time(), data.base().duration()));
+      continue;
+    }
     RefPtr<Image> image = data.image().TransferToImage(mBufferRecycleBin);
+
     RefPtr<VideoData> video = VideoData::CreateFromImage(
         data.display(), data.base().offset(), data.base().time(),
         data.base().duration(), image, data.base().keyframe(),
@@ -193,8 +200,18 @@ MediaResult RemoteVideoDecoderParent::ProcessDecodedData(
   nsTArray<RemoteVideoData> array;
 
   for (const auto& data : aData) {
-    MOZ_ASSERT(data->mType == MediaData::Type::VIDEO_DATA,
+    MOZ_ASSERT(data->mType == MediaData::Type::VIDEO_DATA ||
+                   data->mType == MediaData::Type::NULL_DATA,
                "Can only decode videos using RemoteDecoderParent!");
+    if (data->mType == MediaData::Type::NULL_DATA) {
+      RemoteVideoData output(
+          MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode,
+                        data->mDuration, data->mKeyframe),
+          IntSize(), RemoteImageHolder(), -1);
+
+      array.AppendElement(std::move(output));
+      continue;
+    }
     VideoData* video = static_cast<VideoData*>(data.get());
 
     MOZ_ASSERT(video->mImage,