From 6f73601a0bcc31e7022ab70b13892fb6dbbcaf2e Mon Sep 17 00:00:00 2001 From: Chris Pearce Date: Thu, 9 Mar 2017 11:32:15 +1300 Subject: [PATCH] Bug 1315850 - Implement video decoding through CDM. r=jya At this stage, I store video frames in memory in nsTArrays rather than in shmems just so we can get this working. Once this is working, I'll follow up with patches to switch to storing all large buffer traffic between the CDM and other processes in shmems. I'm not planning on preffing this new CDM path on until that's in place. MozReview-Commit-ID: LSTb42msWQS --HG-- extra : rebase_source : b7f162515a1a32b2c344c11d0fa5c7004cec2e15 --- dom/media/gmp/ChromiumCDMChild.cpp | 49 +++++++++++ dom/media/gmp/ChromiumCDMChild.h | 4 + dom/media/gmp/ChromiumCDMParent.cpp | 88 ++++++++++++++++++- dom/media/gmp/ChromiumCDMParent.h | 13 ++- dom/media/gmp/widevine-adapter/moz.build | 3 +- .../agnostic/eme/ChromiumCDMVideoDecoder.cpp | 31 +++++-- .../agnostic/eme/ChromiumCDMVideoDecoder.h | 7 +- 7 files changed, 182 insertions(+), 13 deletions(-) diff --git a/dom/media/gmp/ChromiumCDMChild.cpp b/dom/media/gmp/ChromiumCDMChild.cpp index 0876ddec5f22..f95551c462ef 100644 --- a/dom/media/gmp/ChromiumCDMChild.cpp +++ b/dom/media/gmp/ChromiumCDMChild.cpp @@ -6,6 +6,7 @@ #include "ChromiumCDMChild.h" #include "GMPContentChild.h" #include "WidevineUtils.h" +#include "WidevineVideoFrame.h" #include "GMPLog.h" #include "GMPPlatform.h" #include "mozilla/Unused.h" @@ -467,6 +468,54 @@ ChromiumCDMChild::RecvDecryptAndDecodeFrame(const CDMInputBuffer& aBuffer) MOZ_ASSERT(IsOnMessageLoopThread()); GMP_LOG("ChromiumCDMChild::RecvDecryptAndDecodeFrame()"); MOZ_ASSERT(mDecoderInitialized); + + // The output frame may not have the same timestamp as the frame we put in. + // We may need to input a number of frames before we receive output. The + // CDM's decoder reorders to ensure frames output are in presentation order. + // So we need to store the durations of the frames input, and retrieve them + // on output. + mFrameDurations.Insert(aBuffer.mTimestamp(), aBuffer.mDuration()); + + cdm::InputBuffer input; + nsTArray subsamples; + InitInputBuffer(aBuffer, subsamples, input); + + WidevineVideoFrame frame; + cdm::Status rv = mCDM->DecryptAndDecodeFrame(input, &frame); + GMP_LOG("WidevineVideoDecoder::Decode(timestamp=%" PRId64 ") rv=%d", + input.timestamp, + rv); + + if (rv == cdm::kSuccess) { + // TODO: WidevineBuffers should hold a shmem instead of a array, and we can + // send the handle instead of copying the array here. + + gmp::CDMVideoFrame output; + output.mFormat() = static_cast(frame.Format()); + output.mImageWidth() = frame.Size().width; + output.mImageHeight() = frame.Size().height; + output.mData() = Move( + reinterpret_cast(frame.FrameBuffer())->ExtractBuffer()); + output.mYPlane() = { frame.PlaneOffset(cdm::VideoFrame::kYPlane), + frame.Stride(cdm::VideoFrame::kYPlane) }; + output.mUPlane() = { frame.PlaneOffset(cdm::VideoFrame::kUPlane), + frame.Stride(cdm::VideoFrame::kUPlane) }; + output.mVPlane() = { frame.PlaneOffset(cdm::VideoFrame::kVPlane), + frame.Stride(cdm::VideoFrame::kVPlane) }; + output.mTimestamp() = frame.Timestamp(); + + uint64_t duration = 0; + if (mFrameDurations.Find(frame.Timestamp(), duration)) { + output.mDuration() = duration; + } + + Unused << SendDecoded(output); + } else if (rv == cdm::kNeedMoreData) { + Unused << SendDecoded(gmp::CDMVideoFrame()); + } else { + Unused << SendDecodeFailed(rv); + } + return IPC_OK(); } diff --git a/dom/media/gmp/ChromiumCDMChild.h b/dom/media/gmp/ChromiumCDMChild.h index e039f746e7fc..ced29cfea013 100644 --- a/dom/media/gmp/ChromiumCDMChild.h +++ b/dom/media/gmp/ChromiumCDMChild.h @@ -8,6 +8,7 @@ #include "mozilla/gmp/PChromiumCDMChild.h" #include "content_decryption_module.h" +#include "SimpleMap.h" namespace mozilla { namespace gmp { @@ -108,6 +109,9 @@ protected: GMPContentChild* mPlugin = nullptr; cdm::ContentDecryptionModule_8* mCDM = nullptr; + typedef SimpleMap DurationMap; + DurationMap mFrameDurations; + bool mDecoderInitialized = false; }; diff --git a/dom/media/gmp/ChromiumCDMParent.cpp b/dom/media/gmp/ChromiumCDMParent.cpp index 56f05198e9ea..1abd1fa93cc2 100644 --- a/dom/media/gmp/ChromiumCDMParent.cpp +++ b/dom/media/gmp/ChromiumCDMParent.cpp @@ -445,12 +445,67 @@ ChromiumCDMParent::RecvDecrypted(const uint32_t& aId, ipc::IPCResult ChromiumCDMParent::RecvDecoded(const CDMVideoFrame& aFrame) { + VideoData::YCbCrBuffer b; + nsTArray data; + data = aFrame.mData(); + + if (data.IsEmpty()) { + mDecodePromise.ResolveIfExists(nsTArray>(), __func__); + return IPC_OK(); + } + + b.mPlanes[0].mData = data.Elements(); + b.mPlanes[0].mWidth = aFrame.mImageWidth(); + b.mPlanes[0].mHeight = aFrame.mImageHeight(); + b.mPlanes[0].mStride = aFrame.mYPlane().mStride(); + b.mPlanes[0].mOffset = aFrame.mYPlane().mPlaneOffset(); + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = data.Elements(); + b.mPlanes[1].mWidth = (aFrame.mImageWidth() + 1) / 2; + b.mPlanes[1].mHeight = (aFrame.mImageHeight() + 1) / 2; + b.mPlanes[1].mStride = aFrame.mUPlane().mStride(); + b.mPlanes[1].mOffset = aFrame.mUPlane().mPlaneOffset(); + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = data.Elements(); + b.mPlanes[2].mWidth = (aFrame.mImageWidth() + 1) / 2; + b.mPlanes[2].mHeight = (aFrame.mImageHeight() + 1) / 2; + b.mPlanes[2].mStride = aFrame.mVPlane().mStride(); + b.mPlanes[2].mOffset = aFrame.mVPlane().mPlaneOffset(); + b.mPlanes[2].mSkip = 0; + + gfx::IntRect pictureRegion(0, 0, aFrame.mImageWidth(), aFrame.mImageHeight()); + RefPtr v = VideoData::CreateAndCopyData(mVideoInfo, + mImageContainer, + mLastStreamOffset, + aFrame.mTimestamp(), + aFrame.mDuration(), + b, + false, + -1, + pictureRegion); + + RefPtr self = this; + if (v) { + mDecodePromise.ResolveIfExists({ Move(v) }, __func__); + } else { + mDecodePromise.RejectIfExists( + MediaResult(NS_ERROR_OUT_OF_MEMORY, + RESULT_DETAIL("CallBack::CreateAndCopyData")), + __func__); + } + return IPC_OK(); } ipc::IPCResult ChromiumCDMParent::RecvDecodeFailed(const uint32_t& aStatus) { + mDecodePromise.RejectIfExists( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("ChromiumCDMParent::RecvDecodeFailed")), + __func__); return IPC_OK(); } @@ -470,7 +525,9 @@ ChromiumCDMParent::ActorDestroy(ActorDestroyReason aWhy) RefPtr ChromiumCDMParent::InitializeVideoDecoder( - const gmp::CDMVideoDecoderConfig& aConfig) + const gmp::CDMVideoDecoderConfig& aConfig, + const VideoInfo& aInfo, + RefPtr aImageContainer) { if (!SendInitializeVideoDecoder(aConfig)) { return MediaDataDecoder::InitPromise::CreateAndReject( @@ -479,6 +536,9 @@ ChromiumCDMParent::InitializeVideoDecoder( __func__); } + mImageContainer = aImageContainer; + mVideoInfo = aInfo; + return mInitVideoDecoderPromise.Ensure(__func__); } @@ -500,5 +560,31 @@ ChromiumCDMParent::RecvOnDecoderInitDone(const uint32_t& aStatus) return IPC_OK(); } +RefPtr +ChromiumCDMParent::DecryptAndDecodeFrame(MediaRawData* aSample) +{ + CDMInputBuffer buffer; + + if (!InitCDMInputBuffer(buffer, aSample)) { + return MediaDataDecoder::DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to init CDM buffer."), + __func__); + } + + mLastStreamOffset = aSample->mOffset; + + if (!SendDecryptAndDecodeFrame(buffer)) { + GMP_LOG( + "ChromiumCDMParent::Decrypt(this=%p) failed to send decrypt message.", + this); + return MediaDataDecoder::DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Failed to send decrypt to CDM process."), + __func__); + } + + return mDecodePromise.Ensure(__func__); +} + } // namespace gmp } // namespace mozilla diff --git a/dom/media/gmp/ChromiumCDMParent.h b/dom/media/gmp/ChromiumCDMParent.h index 8d34bb81631a..fb8e2847cc44 100644 --- a/dom/media/gmp/ChromiumCDMParent.h +++ b/dom/media/gmp/ChromiumCDMParent.h @@ -14,6 +14,7 @@ #include "mozilla/RefPtr.h" #include "nsDataHashtable.h" #include "PlatformDecoderModule.h" +#include "ImageContainer.h" namespace mozilla { @@ -61,7 +62,12 @@ public: // TODO: Add functions for clients to send data to CDM, and // a Close() function. RefPtr InitializeVideoDecoder( - const gmp::CDMVideoDecoderConfig& aConfig); + const gmp::CDMVideoDecoderConfig& aConfig, + const VideoInfo& aInfo, + RefPtr aImageContainer); + + RefPtr DecryptAndDecodeFrame( + MediaRawData* aSample); protected: ~ChromiumCDMParent() {} @@ -114,6 +120,11 @@ protected: nsTArray> mDecrypts; MozPromiseHolder mInitVideoDecoderPromise; + MozPromiseHolder mDecodePromise; + + RefPtr mImageContainer; + VideoInfo mVideoInfo; + uint64_t mLastStreamOffset = 0; }; } // namespace gmp diff --git a/dom/media/gmp/widevine-adapter/moz.build b/dom/media/gmp/widevine-adapter/moz.build index 212d7f7794f8..9e30234e3fa7 100644 --- a/dom/media/gmp/widevine-adapter/moz.build +++ b/dom/media/gmp/widevine-adapter/moz.build @@ -16,7 +16,8 @@ SOURCES += [ EXPORTS += [ 'WidevineDecryptor.h', - 'WidevineUtils.h' + 'WidevineUtils.h', + 'WidevineVideoFrame.h' ] FINAL_LIBRARY = 'xul' diff --git a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp index 69d62fb734df..9f2ec56625da 100644 --- a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp +++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp @@ -68,6 +68,7 @@ ChromiumCDMVideoDecoder::Init() config.mProfile() = ToCDMH264Profile(mConfig.mExtraData->SafeElementAt(1, 0)); config.mExtraData() = *mConfig.mExtraData; + mConvertToAnnexB = true; } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { config.mCodec() = cdm::VideoDecoderConfig::kCodecVp8; config.mProfile() = cdm::VideoDecoderConfig::kProfileNotNeeded; @@ -82,17 +83,35 @@ ChromiumCDMVideoDecoder::Init() config.mImageHeight() = mConfig.mImage.height; RefPtr cdm = mCDMParent; - return InvokeAsync(mGMPThread, __func__, [cdm, config]() { - return cdm->InitializeVideoDecoder(config); - }); + VideoInfo info = mConfig; + RefPtr imageContainer = mImageContainer; + return InvokeAsync( + mGMPThread, __func__, [cdm, config, info, imageContainer]() { + return cdm->InitializeVideoDecoder(config, info, imageContainer); + }); +} + +const char* +ChromiumCDMVideoDecoder::GetDescriptionName() const +{ + return "Chromium CDM video decoder"; +} + +MediaDataDecoder::ConversionRequired +ChromiumCDMVideoDecoder::NeedsConversion() const +{ + return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB + : ConversionRequired::kNeedNone; } RefPtr ChromiumCDMVideoDecoder::Decode(MediaRawData* aSample) { - return DecodePromise::CreateAndReject( - MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Unimplemented")), - __func__); + RefPtr cdm = mCDMParent; + RefPtr sample = aSample; + return InvokeAsync(mGMPThread, __func__, [cdm, sample]() { + return cdm->DecryptAndDecodeFrame(sample); + }); } RefPtr diff --git a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h index 2c5ef0568987..bcb092629b67 100644 --- a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h +++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h @@ -26,10 +26,8 @@ public: RefPtr Flush() override; RefPtr Drain() override; RefPtr Shutdown() override; - const char* GetDescriptionName() const override - { - return "Chromium CDM video decoder"; - } + const char* GetDescriptionName() const override; + ConversionRequired NeedsConversion() const override; private: ~ChromiumCDMVideoDecoder(); @@ -40,6 +38,7 @@ private: RefPtr mGMPThread; RefPtr mImageContainer; MozPromiseHolder mInitPromise; + bool mConvertToAnnexB = false; }; } // mozilla