Bug 1315850 - Implement video decoding through CDM. r=jya

At this stage, I store video frames in memory in nsTArrays rather than in
shmems just so we can get this working. Once this is working, I'll follow up
with patches to switch to storing all large buffer traffic between the CDM and
other processes in shmems.

I'm not planning on preffing this new CDM path on until that's in place.

MozReview-Commit-ID: LSTb42msWQS

--HG--
extra : rebase_source : b7f162515a1a32b2c344c11d0fa5c7004cec2e15

dom/media/gmp/ChromiumCDMChild.cpp

@@ -6,6 +6,7 @@
 #include "ChromiumCDMChild.h"
 #include "GMPContentChild.h"
 #include "WidevineUtils.h"
+#include "WidevineVideoFrame.h"
 #include "GMPLog.h"
 #include "GMPPlatform.h"
 #include "mozilla/Unused.h"
@@ -467,6 +468,54 @@ ChromiumCDMChild::RecvDecryptAndDecodeFrame(const CDMInputBuffer& aBuffer)
   MOZ_ASSERT(IsOnMessageLoopThread());
   GMP_LOG("ChromiumCDMChild::RecvDecryptAndDecodeFrame()");
   MOZ_ASSERT(mDecoderInitialized);
+
+  // The output frame may not have the same timestamp as the frame we put in.
+  // We may need to input a number of frames before we receive output. The
+  // CDM's decoder reorders to ensure frames output are in presentation order.
+  // So we need to store the durations of the frames input, and retrieve them
+  // on output.
+  mFrameDurations.Insert(aBuffer.mTimestamp(), aBuffer.mDuration());
+
+  cdm::InputBuffer input;
+  nsTArray<cdm::SubsampleEntry> subsamples;
+  InitInputBuffer(aBuffer, subsamples, input);
+
+  WidevineVideoFrame frame;
+  cdm::Status rv = mCDM->DecryptAndDecodeFrame(input, &frame);
+  GMP_LOG("WidevineVideoDecoder::Decode(timestamp=%" PRId64 ") rv=%d",
+          input.timestamp,
+          rv);
+
+  if (rv == cdm::kSuccess) {
+    // TODO: WidevineBuffers should hold a shmem instead of a array, and we can
+    // send the handle instead of copying the array here.
+
+    gmp::CDMVideoFrame output;
+    output.mFormat() = static_cast<cdm::VideoFormat>(frame.Format());
+    output.mImageWidth() = frame.Size().width;
+    output.mImageHeight() = frame.Size().height;
+    output.mData() = Move(
+      reinterpret_cast<WidevineBuffer*>(frame.FrameBuffer())->ExtractBuffer());
+    output.mYPlane() = { frame.PlaneOffset(cdm::VideoFrame::kYPlane),
+                         frame.Stride(cdm::VideoFrame::kYPlane) };
+    output.mUPlane() = { frame.PlaneOffset(cdm::VideoFrame::kUPlane),
+                         frame.Stride(cdm::VideoFrame::kUPlane) };
+    output.mVPlane() = { frame.PlaneOffset(cdm::VideoFrame::kVPlane),
+                         frame.Stride(cdm::VideoFrame::kVPlane) };
+    output.mTimestamp() = frame.Timestamp();
+
+    uint64_t duration = 0;
+    if (mFrameDurations.Find(frame.Timestamp(), duration)) {
+      output.mDuration() = duration;
+    }
+
+    Unused << SendDecoded(output);
+  } else if (rv == cdm::kNeedMoreData) {
+    Unused << SendDecoded(gmp::CDMVideoFrame());
+  } else {
+    Unused << SendDecodeFailed(rv);
+  }
+
   return IPC_OK();
 }
 

dom/media/gmp/ChromiumCDMChild.h

@@ -8,6 +8,7 @@
 
 #include "mozilla/gmp/PChromiumCDMChild.h"
 #include "content_decryption_module.h"
+#include "SimpleMap.h"
 
 namespace mozilla {
 namespace gmp {
@@ -108,6 +109,9 @@ protected:
   GMPContentChild* mPlugin = nullptr;
   cdm::ContentDecryptionModule_8* mCDM = nullptr;
 
+  typedef SimpleMap<uint64_t> DurationMap;
+  DurationMap mFrameDurations;
+
   bool mDecoderInitialized = false;
 };
 

dom/media/gmp/ChromiumCDMParent.cpp

@@ -445,12 +445,67 @@ ChromiumCDMParent::RecvDecrypted(const uint32_t& aId,
 ipc::IPCResult
 ChromiumCDMParent::RecvDecoded(const CDMVideoFrame& aFrame)
 {
+  VideoData::YCbCrBuffer b;
+  nsTArray<uint8_t> data;
+  data = aFrame.mData();
+
+  if (data.IsEmpty()) {
+    mDecodePromise.ResolveIfExists(nsTArray<RefPtr<MediaData>>(), __func__);
+    return IPC_OK();
+  }
+
+  b.mPlanes[0].mData = data.Elements();
+  b.mPlanes[0].mWidth = aFrame.mImageWidth();
+  b.mPlanes[0].mHeight = aFrame.mImageHeight();
+  b.mPlanes[0].mStride = aFrame.mYPlane().mStride();
+  b.mPlanes[0].mOffset = aFrame.mYPlane().mPlaneOffset();
+  b.mPlanes[0].mSkip = 0;
+
+  b.mPlanes[1].mData = data.Elements();
+  b.mPlanes[1].mWidth = (aFrame.mImageWidth() + 1) / 2;
+  b.mPlanes[1].mHeight = (aFrame.mImageHeight() + 1) / 2;
+  b.mPlanes[1].mStride = aFrame.mUPlane().mStride();
+  b.mPlanes[1].mOffset = aFrame.mUPlane().mPlaneOffset();
+  b.mPlanes[1].mSkip = 0;
+
+  b.mPlanes[2].mData = data.Elements();
+  b.mPlanes[2].mWidth = (aFrame.mImageWidth() + 1) / 2;
+  b.mPlanes[2].mHeight = (aFrame.mImageHeight() + 1) / 2;
+  b.mPlanes[2].mStride = aFrame.mVPlane().mStride();
+  b.mPlanes[2].mOffset = aFrame.mVPlane().mPlaneOffset();
+  b.mPlanes[2].mSkip = 0;
+
+  gfx::IntRect pictureRegion(0, 0, aFrame.mImageWidth(), aFrame.mImageHeight());
+  RefPtr<VideoData> v = VideoData::CreateAndCopyData(mVideoInfo,
+                                                     mImageContainer,
+                                                     mLastStreamOffset,
+                                                     aFrame.mTimestamp(),
+                                                     aFrame.mDuration(),
+                                                     b,
+                                                     false,
+                                                     -1,
+                                                     pictureRegion);
+
+  RefPtr<ChromiumCDMParent> self = this;
+  if (v) {
+    mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+  } else {
+    mDecodePromise.RejectIfExists(
+      MediaResult(NS_ERROR_OUT_OF_MEMORY,
+                  RESULT_DETAIL("CallBack::CreateAndCopyData")),
+      __func__);
+  }
+
   return IPC_OK();
 }
 
 ipc::IPCResult
 ChromiumCDMParent::RecvDecodeFailed(const uint32_t& aStatus)
 {
+  mDecodePromise.RejectIfExists(
+    MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                RESULT_DETAIL("ChromiumCDMParent::RecvDecodeFailed")),
+    __func__);
   return IPC_OK();
 }
 
@@ -470,7 +525,9 @@ ChromiumCDMParent::ActorDestroy(ActorDestroyReason aWhy)
 
 RefPtr<MediaDataDecoder::InitPromise>
 ChromiumCDMParent::InitializeVideoDecoder(
-  const gmp::CDMVideoDecoderConfig& aConfig)
+  const gmp::CDMVideoDecoderConfig& aConfig,
+  const VideoInfo& aInfo,
+  RefPtr<layers::ImageContainer> aImageContainer)
 {
   if (!SendInitializeVideoDecoder(aConfig)) {
     return MediaDataDecoder::InitPromise::CreateAndReject(
@@ -479,6 +536,9 @@ ChromiumCDMParent::InitializeVideoDecoder(
       __func__);
   }
 
+  mImageContainer = aImageContainer;
+  mVideoInfo = aInfo;
+
   return mInitVideoDecoderPromise.Ensure(__func__);
 }
 
@@ -500,5 +560,31 @@ ChromiumCDMParent::RecvOnDecoderInitDone(const uint32_t& aStatus)
   return IPC_OK();
 }
 
+RefPtr<MediaDataDecoder::DecodePromise>
+ChromiumCDMParent::DecryptAndDecodeFrame(MediaRawData* aSample)
+{
+  CDMInputBuffer buffer;
+
+  if (!InitCDMInputBuffer(buffer, aSample)) {
+    return MediaDataDecoder::DecodePromise::CreateAndReject(
+      MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to init CDM buffer."),
+      __func__);
+  }
+
+  mLastStreamOffset = aSample->mOffset;
+
+  if (!SendDecryptAndDecodeFrame(buffer)) {
+    GMP_LOG(
+      "ChromiumCDMParent::Decrypt(this=%p) failed to send decrypt message.",
+      this);
+    return MediaDataDecoder::DecodePromise::CreateAndReject(
+      MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                  "Failed to send decrypt to CDM process."),
+      __func__);
+  }
+
+  return mDecodePromise.Ensure(__func__);
+}
+
 } // namespace gmp
 } // namespace mozilla

dom/media/gmp/ChromiumCDMParent.h

@@ -14,6 +14,7 @@
 #include "mozilla/RefPtr.h"
 #include "nsDataHashtable.h"
 #include "PlatformDecoderModule.h"
+#include "ImageContainer.h"
 
 namespace mozilla {
 
@@ -61,7 +62,12 @@ public:
   // TODO: Add functions for clients to send data to CDM, and
   // a Close() function.
   RefPtr<MediaDataDecoder::InitPromise> InitializeVideoDecoder(
-    const gmp::CDMVideoDecoderConfig& aConfig);
+    const gmp::CDMVideoDecoderConfig& aConfig,
+    const VideoInfo& aInfo,
+    RefPtr<layers::ImageContainer> aImageContainer);
+
+  RefPtr<MediaDataDecoder::DecodePromise> DecryptAndDecodeFrame(
+    MediaRawData* aSample);
 
 protected:
   ~ChromiumCDMParent() {}
@@ -114,6 +120,11 @@ protected:
   nsTArray<RefPtr<DecryptJob>> mDecrypts;
 
   MozPromiseHolder<MediaDataDecoder::InitPromise> mInitVideoDecoderPromise;
+  MozPromiseHolder<MediaDataDecoder::DecodePromise> mDecodePromise;
+
+  RefPtr<layers::ImageContainer> mImageContainer;
+  VideoInfo mVideoInfo;
+  uint64_t mLastStreamOffset = 0;
 };
 
 } // namespace gmp

dom/media/gmp/widevine-adapter/moz.build

@@ -16,7 +16,8 @@ SOURCES += [
 
 EXPORTS += [
     'WidevineDecryptor.h',
-    'WidevineUtils.h'
+    'WidevineUtils.h',
+    'WidevineVideoFrame.h'
 ]
 
 FINAL_LIBRARY = 'xul'

dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp

@@ -68,6 +68,7 @@ ChromiumCDMVideoDecoder::Init()
     config.mProfile() =
       ToCDMH264Profile(mConfig.mExtraData->SafeElementAt(1, 0));
     config.mExtraData() = *mConfig.mExtraData;
+    mConvertToAnnexB = true;
   } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
     config.mCodec() = cdm::VideoDecoderConfig::kCodecVp8;
     config.mProfile() = cdm::VideoDecoderConfig::kProfileNotNeeded;
@@ -82,17 +83,35 @@ ChromiumCDMVideoDecoder::Init()
   config.mImageHeight() = mConfig.mImage.height;
 
   RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
-  return InvokeAsync(mGMPThread, __func__, [cdm, config]() {
-    return cdm->InitializeVideoDecoder(config);
-  });
+  VideoInfo info = mConfig;
+  RefPtr<layers::ImageContainer> imageContainer = mImageContainer;
+  return InvokeAsync(
+    mGMPThread, __func__, [cdm, config, info, imageContainer]() {
+      return cdm->InitializeVideoDecoder(config, info, imageContainer);
+    });
+}
+
+const char*
+ChromiumCDMVideoDecoder::GetDescriptionName() const
+{
+  return "Chromium CDM video decoder";
+}
+
+MediaDataDecoder::ConversionRequired
+ChromiumCDMVideoDecoder::NeedsConversion() const
+{
+  return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB
+                          : ConversionRequired::kNeedNone;
 }
 
 RefPtr<MediaDataDecoder::DecodePromise>
 ChromiumCDMVideoDecoder::Decode(MediaRawData* aSample)
 {
-  return DecodePromise::CreateAndReject(
-    MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Unimplemented")),
-    __func__);
+  RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
+  RefPtr<MediaRawData> sample = aSample;
+  return InvokeAsync(mGMPThread, __func__, [cdm, sample]() {
+    return cdm->DecryptAndDecodeFrame(sample);
+  });
 }
 
 RefPtr<MediaDataDecoder::FlushPromise>

dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h

@@ -26,10 +26,8 @@ public:
   RefPtr<FlushPromise> Flush() override;
   RefPtr<DecodePromise> Drain() override;
   RefPtr<ShutdownPromise> Shutdown() override;
-  const char* GetDescriptionName() const override
-  {
-    return "Chromium CDM video decoder";
-  }
+  const char* GetDescriptionName() const override;
+  ConversionRequired NeedsConversion() const override;
 
 private:
   ~ChromiumCDMVideoDecoder();
@@ -40,6 +38,7 @@ private:
   RefPtr<AbstractThread> mGMPThread;
   RefPtr<layers::ImageContainer> mImageContainer;
   MozPromiseHolder<InitPromise> mInitPromise;
+  bool mConvertToAnnexB = false;
 };
 
 } // mozilla