Bug 1372080 - Reorder frames decoded by Widevine CDM. r=jya

The next version of the Widevine CDM (970) has a new H.264 decoder and it does
not appear to be outputing frames it decodes in presentation order, so we need
to reorder the frames output by the CDM.


MozReview-Commit-ID: HMsQVN3NCIU

--HG--
extra : rebase_source : 68ef406556087434fa12b72ae5ed5c2e1bce2b64

dom/media/gmp/ChromiumCDMParent.cpp

@@ -4,17 +4,20 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "ChromiumCDMParent.h"
-#include "mozilla/gmp/GMPTypes.h"
+
+#include "ChromiumCDMProxy.h"
+#include "content_decryption_module.h"
 #include "GMPContentChild.h"
 #include "GMPContentParent.h"
-#include "mozilla/Unused.h"
-#include "ChromiumCDMProxy.h"
-#include "mozilla/dom/MediaKeyMessageEventBinding.h"
-#include "mozilla/Telemetry.h"
-#include "content_decryption_module.h"
 #include "GMPLog.h"
-#include "MediaPrefs.h"
 #include "GMPUtils.h"
+#include "MediaPrefs.h"
+#include "mozilla/dom/MediaKeyMessageEventBinding.h"
+#include "mozilla/gmp/GMPTypes.h"
+#include "mozilla/Telemetry.h"
+#include "mozilla/Unused.h"
+#include "mp4_demuxer/AnnexB.h"
+#include "mp4_demuxer/H264.h"
 
 namespace mozilla {
 namespace gmp {
@@ -731,7 +734,7 @@ ChromiumCDMParent::RecvDecodedData(const CDMVideoFrame& aFrame,
     return IPC_OK();
   }
 
-  mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+  ReorderAndReturnOutput(Move(v));
 
   return IPC_OK();
 }
@@ -740,7 +743,11 @@ ipc::IPCResult
 ChromiumCDMParent::RecvDecodedShmem(const CDMVideoFrame& aFrame,
                                     ipc::Shmem&& aShmem)
 {
-  GMP_LOG("ChromiumCDMParent::RecvDecodedShmem(this=%p)", this);
+  GMP_LOG("ChromiumCDMParent::RecvDecodedShmem(this=%p) time=%" PRId64
+          " duration=%" PRId64,
+          this,
+          aFrame.mTimestamp(),
+          aFrame.mDuration());
 
   // On failure we need to deallocate the shmem we're to return to the
   // CDM. On success we return it to the CDM to be reused.
@@ -775,11 +782,26 @@ ChromiumCDMParent::RecvDecodedShmem(const CDMVideoFrame& aFrame,
   // for it again.
   autoDeallocateShmem.release();
 
-  mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+  ReorderAndReturnOutput(Move(v));
 
   return IPC_OK();
 }
 
+void
+ChromiumCDMParent::ReorderAndReturnOutput(RefPtr<VideoData>&& aFrame)
+{
+  if (mMaxRefFrames == 0) {
+    mDecodePromise.ResolveIfExists({ Move(aFrame) }, __func__);
+    return;
+  }
+  mReorderQueue.Push(Move(aFrame));
+  MediaDataDecoder::DecodedData results;
+  while (mReorderQueue.Length() > mMaxRefFrames) {
+    results.AppendElement(mReorderQueue.Pop());
+  }
+  mDecodePromise.Resolve(Move(results), __func__);
+}
+
 already_AddRefed<VideoData>
 ChromiumCDMParent::CreateVideoFrame(const CDMVideoFrame& aFrame,
                                     Span<uint8_t> aData)
@@ -915,6 +937,13 @@ ChromiumCDMParent::InitializeVideoDecoder(
       __func__);
   }
 
+  mMaxRefFrames =
+    (aConfig.mCodec() == cdm::VideoDecoderConfig::kCodecH264)
+      ? mp4_demuxer::AnnexB::HasSPS(aInfo.mExtraData)
+          ? mp4_demuxer::H264::ComputeMaxRefFrames(aInfo.mExtraData)
+          : 16
+      : 0;
+
   mVideoDecoderInitialized = true;
   mImageContainer = aImageContainer;
   mVideoInfo = aInfo;
@@ -987,12 +1016,15 @@ RefPtr<MediaDataDecoder::FlushPromise>
 ChromiumCDMParent::FlushVideoDecoder()
 {
   if (mIsShutdown) {
+    MOZ_ASSERT(mReorderQueue.IsEmpty());
     return MediaDataDecoder::FlushPromise::CreateAndReject(
       MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
                   RESULT_DETAIL("ChromiumCDMParent is shutdown")),
       __func__);
   }
 
+  mReorderQueue.Clear();
+
   mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
   if (!SendResetVideoDecoder()) {
     return MediaDataDecoder::FlushPromise::CreateAndReject(
@@ -1005,6 +1037,7 @@ ChromiumCDMParent::FlushVideoDecoder()
 ipc::IPCResult
 ChromiumCDMParent::RecvResetVideoDecoderComplete()
 {
+  MOZ_ASSERT(mReorderQueue.IsEmpty());
   if (mIsShutdown) {
     MOZ_ASSERT(mFlushDecoderPromise.IsEmpty());
     return IPC_OK();
@@ -1038,7 +1071,13 @@ ChromiumCDMParent::RecvDrainComplete()
     MOZ_ASSERT(mDecodePromise.IsEmpty());
     return IPC_OK();
   }
-  mDecodePromise.ResolveIfExists(MediaDataDecoder::DecodedData(), __func__);
+
+  MediaDataDecoder::DecodedData samples;
+  while (!mReorderQueue.IsEmpty()) {
+    samples.AppendElement(Move(mReorderQueue.Pop()));
+  }
+
+  mDecodePromise.ResolveIfExists(Move(samples), __func__);
   return IPC_OK();
 }
 RefPtr<ShutdownPromise>
@@ -1097,6 +1136,8 @@ ChromiumCDMParent::Shutdown()
   // (including from an already-queued task, e.g.: ActorDestroy).
   mProxy = nullptr;
 
+  mReorderQueue.Clear();
+
   for (RefPtr<DecryptJob>& decrypt : mDecrypts) {
     decrypt->PostResult(eme::AbortedErr);
   }

dom/media/gmp/ChromiumCDMParent.h

@@ -16,6 +16,7 @@
 #include "PlatformDecoderModule.h"
 #include "ImageContainer.h"
 #include "mozilla/Span.h"
+#include "ReorderQueue.h"
 
 namespace mozilla {
 
@@ -128,6 +129,8 @@ protected:
   void ActorDestroy(ActorDestroyReason aWhy) override;
   bool SendBufferToCDM(uint32_t aSizeInBytes);
 
+  void ReorderAndReturnOutput(RefPtr<VideoData>&& aFrame);
+
   void RejectPromise(uint32_t aPromiseId,
                      nsresult aError,
                      const nsCString& aErrorMessage);
@@ -172,6 +175,15 @@ protected:
   bool mIsShutdown = false;
   bool mVideoDecoderInitialized = false;
   bool mActorDestroyed = false;
+
+  // The H.264 decoder in Widevine CDM versions 970 and later output in decode
+  // order rather than presentation order, so we reorder in presentation order
+  // before presenting. mMaxRefFrames is non-zero if we have an initialized
+  // decoder and we are decoding H.264. If so, it stores the maximum length of
+  // the reorder queue that we need. Note we may have multiple decoders for the
+  // life time of this object, but never more than one active at once.
+  uint32_t mMaxRefFrames = 0;
+  ReorderQueue mReorderQueue;
 };
 
 } // namespace gmp

dom/media/platforms/moz.build

@@ -14,6 +14,7 @@ EXPORTS += [
     'MediaTelemetryConstants.h',
     'PDMFactory.h',
     'PlatformDecoderModule.h',
+    'ReorderQueue.h',
     'SimpleMap.h',
     'wrappers/H264Converter.h',
     'wrappers/MediaDataDecoderProxy.h'