diff --git a/media/webrtc/signaling/src/media-conduit/MediaDataDecoderCodec.cpp b/media/webrtc/signaling/src/media-conduit/MediaDataDecoderCodec.cpp index cccc115e1060..d4c824897840 100644 --- a/media/webrtc/signaling/src/media-conduit/MediaDataDecoderCodec.cpp +++ b/media/webrtc/signaling/src/media-conduit/MediaDataDecoderCodec.cpp @@ -3,6 +3,8 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "MediaDataDecoderCodec.h" +#include "MediaPrefs.h" +#include "WebrtcMediaDataDecoderCodec.h" namespace mozilla { @@ -15,9 +17,21 @@ MediaDataDecoderCodec::CreateEncoder( /* static */ WebrtcVideoDecoder* MediaDataDecoderCodec::CreateDecoder( - webrtc::VideoCodecType aCodecbType) + webrtc::VideoCodecType aCodecType) { - return nullptr; + if (!MediaPrefs::MediaDataDecoderEnabled()) { + return nullptr; + } + + switch (aCodecType) { + case webrtc::VideoCodecType::kVideoCodecVP8: + case webrtc::VideoCodecType::kVideoCodecVP9: + case webrtc::VideoCodecType::kVideoCodecH264: + break; + default: + return nullptr; + } + return new WebrtcMediaDataDecoder(); } } // namespace mozilla \ No newline at end of file diff --git a/media/webrtc/signaling/src/media-conduit/MediaDataDecoderCodec.h b/media/webrtc/signaling/src/media-conduit/MediaDataDecoderCodec.h index 541c0e8e82bd..d10a7a778188 100644 --- a/media/webrtc/signaling/src/media-conduit/MediaDataDecoderCodec.h +++ b/media/webrtc/signaling/src/media-conduit/MediaDataDecoderCodec.h @@ -7,10 +7,11 @@ #include "MediaConduitInterface.h" #include "webrtc/common_types.h" -#include "webrtc/video_decoder.h" namespace mozilla { +class WebrtcVideoDecoder; +class WebrtcVideoEncoder; class MediaDataDecoderCodec { public: diff --git a/media/webrtc/signaling/src/media-conduit/WebrtcMediaDataDecoderCodec.cpp b/media/webrtc/signaling/src/media-conduit/WebrtcMediaDataDecoderCodec.cpp index fe570f6b2b4e..fb314b565ed8 100644 --- a/media/webrtc/signaling/src/media-conduit/WebrtcMediaDataDecoderCodec.cpp +++ b/media/webrtc/signaling/src/media-conduit/WebrtcMediaDataDecoderCodec.cpp @@ -3,49 +3,253 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "WebrtcMediaDataDecoderCodec.h" -#include "PlatformDecoderModule.h" +#include "ImageContainer.h" +#include "Layers.h" +#include "PDMFactory.h" +#include "VideoUtils.h" +#include "mozilla/layers/ImageBridgeChild.h" +#include "webrtc/base/keep_ref_until_done.h" namespace mozilla { -class MediaDataDecoder; - WebrtcMediaDataDecoder::WebrtcMediaDataDecoder() + : mTaskQueue( + new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), + "WebrtcMediaDataDecoder::mTaskQueue")) + , mImageContainer(layers::LayerManager::CreateImageContainer( + layers::ImageContainer::ASYNCHRONOUS)) + , mFactory(new PDMFactory()) + , mMonitor("WebrtcMediaDataDecoder") { } WebrtcMediaDataDecoder::~WebrtcMediaDataDecoder() { + mTaskQueue->BeginShutdown(); + mTaskQueue->AwaitShutdownAndIdle(); } int32_t -WebrtcMediaDataDecoder::InitDecode(const webrtc::VideoCodec* codecSettings, - int32_t numberOfCores) +WebrtcMediaDataDecoder::InitDecode(const webrtc::VideoCodec* aCodecSettings, + int32_t aNumberOfCores) { - return 0; + nsCString codec; + switch (aCodecSettings->codecType) { + case webrtc::VideoCodecType::kVideoCodecVP8: + codec = "video/webm; codecs=vp8"; + break; + case webrtc::VideoCodecType::kVideoCodecVP9: + codec = "video/webm; codecs=vp9"; + break; + case webrtc::VideoCodecType::kVideoCodecH264: + codec = "video/avc"; + break; + default: + return WEBRTC_VIDEO_CODEC_ERROR; + } + + mTrackType = TrackInfo::kVideoTrack; + + mInfo = VideoInfo(aCodecSettings->width, aCodecSettings->height); + mInfo.mMimeType = codec; + + RefPtr knowsCompositor = + layers::ImageBridgeChild::GetSingleton(); + + mDecoder = mFactory->CreateDecoder( + { mInfo, + mTaskQueue, + CreateDecoderParams::OptionSet(CreateDecoderParams::Option::LowLatency), + mTrackType, + mImageContainer, + knowsCompositor }); + + if (!mDecoder) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + MonitorAutoLock lock(mMonitor); + bool done = false; + mDecoder->Init()->Then(mTaskQueue, + __func__, + [&](TrackInfo::TrackType) { + MonitorAutoLock lock(mMonitor); + done = true; + mMonitor.Notify(); + }, + [&](const MediaResult& aError) { + MonitorAutoLock lock(mMonitor); + done = true; + mError = aError; + mMonitor.Notify(); + }); + + while (!done) { + mMonitor.Wait(); + } + + return NS_SUCCEEDED(mError) ? WEBRTC_VIDEO_CODEC_OK : WEBRTC_VIDEO_CODEC_ERROR; } int32_t WebrtcMediaDataDecoder::Decode( - const webrtc::EncodedImage& inputImage, - bool missingFrames, - const webrtc::RTPFragmentationHeader* fragmentation, - const webrtc::CodecSpecificInfo* codecSpecificInfo, - int64_t renderTimeMs) + const webrtc::EncodedImage& aInputImage, + bool aMissingFrames, + const webrtc::RTPFragmentationHeader* aFragmentation, + const webrtc::CodecSpecificInfo* aCodecSpecificInfo, + int64_t aRenderTimeMs) { - return 0; + if (!mCallback || !mDecoder) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + if (!aInputImage._buffer || !aInputImage._length) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + // Always start with a complete key frame. + if (mNeedKeyframe) { + if (aInputImage._frameType != webrtc::FrameType::kVideoFrameKey) + return WEBRTC_VIDEO_CODEC_ERROR; + // We have a key frame - is it complete? + if (aInputImage._completeFrame) { + mNeedKeyframe = false; + } else { + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + RefPtr compressedFrame = + new MediaRawData(aInputImage._buffer, aInputImage._length); + if (!compressedFrame->Data()) { + return WEBRTC_VIDEO_CODEC_MEMORY; + } + + compressedFrame->mTime = + media::TimeUnit::FromMicroseconds(aInputImage._timeStamp); + compressedFrame->mTimecode = + media::TimeUnit::FromMicroseconds(aRenderTimeMs * 1000); + compressedFrame->mKeyframe = + aInputImage._frameType == webrtc::FrameType::kVideoFrameKey; + { + MonitorAutoLock lock(mMonitor); + bool done = false; + mDecoder->Decode(compressedFrame)->Then( + mTaskQueue, + __func__, + [&](const MediaDataDecoder::DecodedData& aResults) { + MonitorAutoLock lock(mMonitor); + mResults = aResults; + done = true; + mMonitor.Notify(); + }, + [&](const MediaResult& aError) { + MonitorAutoLock lock(mMonitor); + mError = aError; + done = true; + mMonitor.Notify(); + }); + + while (!done) { + mMonitor.Wait(); + } + + for (auto& frame : mResults) { + MOZ_ASSERT(frame->mType == MediaData::VIDEO_DATA); + RefPtr video = frame->As(); + MOZ_ASSERT(video); + if (!video->mImage) { + // Nothing to display. + continue; + } + rtc::scoped_refptr image( + new rtc::RefCountedObject(Move(video->mImage))); + + webrtc::VideoFrame videoFrame(image, + frame->mTime.ToMicroseconds(), + frame->mDuration.ToMicroseconds() * 1000, + aInputImage.rotation_); + mCallback->Decoded(videoFrame); + } + mResults.Clear(); + } + return NS_SUCCEEDED(mError) ? WEBRTC_VIDEO_CODEC_OK + : WEBRTC_VIDEO_CODEC_ERROR; } int32_t WebrtcMediaDataDecoder::RegisterDecodeCompleteCallback( - webrtc::DecodedImageCallback* callback) + webrtc::DecodedImageCallback* aCallback) { - return 0; + mCallback = aCallback; + return WEBRTC_VIDEO_CODEC_OK; } int32_t WebrtcMediaDataDecoder::Release() { - return 0; + MonitorAutoLock lock(mMonitor); + bool done = false; + mDecoder->Flush() + ->Then(mTaskQueue, + __func__, + [this]() { return mDecoder->Shutdown(); }, + [this](const MediaResult& aError) { return mDecoder->Shutdown(); }) + ->Then(mTaskQueue, + __func__, + [&]() { + MonitorAutoLock lock(mMonitor); + done = true; + mMonitor.Notify(); + }, + []() { MOZ_ASSERT_UNREACHABLE("Shutdown promise always resolved"); }); + + while (!done) { + mMonitor.Wait(); + } + + mDecoder = nullptr; + mNeedKeyframe = true; + + return WEBRTC_VIDEO_CODEC_OK; +} + +bool +WebrtcMediaDataDecoder::OnTaskQueue() const +{ + return OwnerThread()->IsCurrentThreadIn(); +} + +ImageBuffer::ImageBuffer(RefPtr&& aImage) + : webrtc::NativeHandleBuffer(aImage, + aImage->GetSize().width, + aImage->GetSize().height) + , mImage(Move(aImage)) +{ +} + +rtc::scoped_refptr +ImageBuffer::NativeToI420Buffer() +{ + RefPtr image = mImage->AsPlanarYCbCrImage(); + if (!image) { + // TODO. YUV420 ReadBack, Image only provides a RGB readback. + return nullptr; + } + rtc::scoped_refptr refImage(image); + const layers::PlanarYCbCrData* data = image->GetData(); + rtc::scoped_refptr buf( + new rtc::RefCountedObject( + data->mPicSize.width, + data->mPicSize.height, + data->mYChannel, + data->mYStride, + data->mCbChannel, + data->mCbCrStride, + data->mCrChannel, + data->mCbCrStride, + rtc::KeepRefUntilDone(refImage))); + return buf; } } // namespace mozilla diff --git a/media/webrtc/signaling/src/media-conduit/WebrtcMediaDataDecoderCodec.h b/media/webrtc/signaling/src/media-conduit/WebrtcMediaDataDecoderCodec.h index 12593d7e7382..183ffd154a75 100644 --- a/media/webrtc/signaling/src/media-conduit/WebrtcMediaDataDecoderCodec.h +++ b/media/webrtc/signaling/src/media-conduit/WebrtcMediaDataDecoderCodec.h @@ -6,21 +6,39 @@ #define WebrtcMediaDataDecoderCodec_h__ #include "MediaConduitInterface.h" -#include "mozilla/RefPtr.h" - +#include "MediaInfo.h" +#include "MediaResult.h" +#include "PlatformDecoderModule.h" +#include "webrtc/common_video/include/video_frame_buffer.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" +namespace webrtc { + class DecodedImageCallback; +} namespace mozilla { +namespace layers { + class Image; + class ImageContainer; +} -class MediaDataDecoder; +class PDMFactory; +class TaskQueue; + +class ImageBuffer : public webrtc::NativeHandleBuffer +{ +public: + explicit ImageBuffer(RefPtr&& aImage); + rtc::scoped_refptr NativeToI420Buffer() override; + +private: + RefPtr mImage; +}; class WebrtcMediaDataDecoder : public WebrtcVideoDecoder { public: WebrtcMediaDataDecoder(); - virtual ~WebrtcMediaDataDecoder(); - // Implement VideoDecoder interface. uint64_t PluginID() const override { return 0; } @@ -37,6 +55,27 @@ public: webrtc::DecodedImageCallback* callback) override; int32_t Release() override; + +private: + ~WebrtcMediaDataDecoder(); + void QueueFrame(MediaRawData* aFrame); + AbstractThread* OwnerThread() const { return mTaskQueue; } + bool OnTaskQueue() const; + + const RefPtr mTaskQueue; + const RefPtr mImageContainer; + const RefPtr mFactory; + RefPtr mDecoder; + webrtc::DecodedImageCallback* mCallback = nullptr; + VideoInfo mInfo; + TrackInfo::TrackType mTrackType; + bool mNeedKeyframe = true; + MozPromiseRequestHolder mDecodeRequest; + + Monitor mMonitor; + // Members below are accessed via mMonitor + MediaResult mError = NS_OK; + MediaDataDecoder::DecodedData mResults; }; } // namespace mozilla diff --git a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp index 613d43ba1756..1842fe38f0b6 100644 --- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp +++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp @@ -2198,6 +2198,15 @@ public: uint32_t time_stamp, int64_t render_time) { + if (buffer.native_handle()) { + // We assume that only native handles are used with the + // WebrtcMediaDataDecoderCodec decoder. + RefPtr image = static_cast(buffer.native_handle()); + MutexAutoLock lock(mutex_); + image_ = image; + return; + } + MOZ_ASSERT(buffer.DataY()); // Create a video frame using |buffer|. RefPtr yuvImage =