From 05c11eb55bf5ad6962e4d96360eca81e2e445792 Mon Sep 17 00:00:00 2001 From: alwu Date: Wed, 11 May 2022 17:46:14 +0000 Subject: [PATCH] Bug 1756260 - part6 : create remote decoders based on the engine Id. r=jolin Implement a new decoder module which would use the media engine Id to create the decoder, which is implemented by WMF Media Engine API. The actual data flow would be like that, the encoded data would be passed to the RDD process by using the remote decoder, and remote decoder would feed the data into media engine decoder, such as MFMediaEngineVideoStream and MFMediaEngineAudioStream. Differential Revision: https://phabricator.services.mozilla.com/D143805 --- dom/media/ipc/MFMediaEngineParent.cpp | 13 +++ dom/media/ipc/MFMediaEngineParent.h | 3 + dom/media/platforms/PDMFactory.cpp | 8 ++ .../wmf/MFMediaEngineAudioStream.cpp | 18 ++++ .../platforms/wmf/MFMediaEngineAudioStream.h | 22 ++++ .../wmf/MFMediaEngineDecoderModule.cpp | 102 ++++++++++++++++++ .../wmf/MFMediaEngineDecoderModule.h | 41 +++++++ .../platforms/wmf/MFMediaEngineStream.cpp | 34 ++++++ dom/media/platforms/wmf/MFMediaEngineStream.h | 31 ++++++ .../wmf/MFMediaEngineVideoStream.cpp | 24 +++++ .../platforms/wmf/MFMediaEngineVideoStream.h | 23 ++++ dom/media/platforms/wmf/moz.build | 9 ++ 12 files changed, 328 insertions(+) create mode 100644 dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp create mode 100644 dom/media/platforms/wmf/MFMediaEngineAudioStream.h create mode 100644 dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp create mode 100644 dom/media/platforms/wmf/MFMediaEngineDecoderModule.h create mode 100644 dom/media/platforms/wmf/MFMediaEngineStream.cpp create mode 100644 dom/media/platforms/wmf/MFMediaEngineStream.h create mode 100644 dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp create mode 100644 dom/media/platforms/wmf/MFMediaEngineVideoStream.h diff --git a/dom/media/ipc/MFMediaEngineParent.cpp b/dom/media/ipc/MFMediaEngineParent.cpp index bbf22dc38e8d..4f2f39c8347a 100644 --- a/dom/media/ipc/MFMediaEngineParent.cpp +++ b/dom/media/ipc/MFMediaEngineParent.cpp @@ -4,7 +4,9 @@ #include "MFMediaEngineParent.h" +#include "MFMediaEngineAudioStream.h" #include "MFMediaEngineUtils.h" +#include "MFMediaEngineVideoStream.h" #include "RemoteDecoderManagerParent.h" #include "mozilla/ClearOnShutdown.h" #include "mozilla/StaticMutex.h" @@ -59,6 +61,17 @@ MFMediaEngineParent::~MFMediaEngineParent() { UnregisterMedieEngine(this); } +MFMediaEngineStream* MFMediaEngineParent::GetMediaEngineStream( + TrackType aType, const CreateDecoderParams& aParam) { + LOG("Create a media engine decoder for %s", TrackTypeToStr(aType)); + // TODO : make those streams associated with their media engine and source. + if (aType == TrackType::kAudioTrack) { + return new MFMediaEngineAudioStream(aParam); + } + MOZ_ASSERT(aType == TrackType::kVideoTrack); + return new MFMediaEngineVideoStream(aParam); +} + mozilla::ipc::IPCResult MFMediaEngineParent::RecvInitMediaEngine( const MediaEngineInfoIPDL& aInfo, InitMediaEngineResolver&& aResolver) { AssertOnManagerThread(); diff --git a/dom/media/ipc/MFMediaEngineParent.h b/dom/media/ipc/MFMediaEngineParent.h index 4e6066dfe937..7804371f5bdb 100644 --- a/dom/media/ipc/MFMediaEngineParent.h +++ b/dom/media/ipc/MFMediaEngineParent.h @@ -31,6 +31,9 @@ class MFMediaEngineParent final : public PMFMediaEngineParent { static MFMediaEngineParent* GetMediaEngineById(uint64_t aId); + MFMediaEngineStream* GetMediaEngineStream(TrackType aType, + const CreateDecoderParams& aParam); + uint64_t Id() const { return mMediaEngineId; } // Methods for PMFMediaEngineParent diff --git a/dom/media/platforms/PDMFactory.cpp b/dom/media/platforms/PDMFactory.cpp index 810358318c82..1892a44cf207 100644 --- a/dom/media/platforms/PDMFactory.cpp +++ b/dom/media/platforms/PDMFactory.cpp @@ -37,6 +37,7 @@ #include "nsPrintfCString.h" #ifdef XP_WIN +# include "MFMediaEngineDecoderModule.h" # include "WMFDecoderModule.h" # include "mozilla/WindowsVersion.h" #endif @@ -87,6 +88,10 @@ class PDMInitializer final { if (!IsWin7AndPre2000Compatible()) { WMFDecoderModule::Init(); } + + if (IsWin8OrLater() && StaticPrefs::media_wmf_media_engine_enabled()) { + MFMediaEngineDecoderModule::Init(); + } #endif #ifdef MOZ_APPLEMEDIA AppleDecoderModule::Init(); @@ -507,6 +512,9 @@ static DecoderDoctorDiagnostics::Flags GetFailureFlagBasedOnFFmpegStatus( void PDMFactory::CreateRddPDMs() { #ifdef XP_WIN + if (IsWin8OrLater() && StaticPrefs::media_wmf_media_engine_enabled()) { + CreateAndStartupPDM(); + } if (StaticPrefs::media_wmf_enabled() && StaticPrefs::media_rdd_wmf_enabled()) { CreateAndStartupPDM(); diff --git a/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp b/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp new file mode 100644 index 000000000000..e61b669daa49 --- /dev/null +++ b/dom/media/platforms/wmf/MFMediaEngineAudioStream.cpp @@ -0,0 +1,18 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MFMediaEngineAudioStream.h" + +namespace mozilla { + +RefPtr MFMediaEngineAudioStream::Init() { + // TODO : implement this by using MediaEngine API. + return InitPromise::CreateAndResolve(TrackType::kAudioTrack, __func__); +} + +nsCString MFMediaEngineAudioStream::GetDescriptionName() const { + return "media engine audio stream"_ns; +} + +} // namespace mozilla diff --git a/dom/media/platforms/wmf/MFMediaEngineAudioStream.h b/dom/media/platforms/wmf/MFMediaEngineAudioStream.h new file mode 100644 index 000000000000..6d40ccf91b60 --- /dev/null +++ b/dom/media/platforms/wmf/MFMediaEngineAudioStream.h @@ -0,0 +1,22 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEAUDIOSTREAM_H +#define DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEAUDIOSTREAM_H + +#include "MFMediaEngineStream.h" + +namespace mozilla { + +class MFMediaEngineAudioStream final : public MFMediaEngineStream { + public: + explicit MFMediaEngineAudioStream(const CreateDecoderParams& aParam) {} + + RefPtr Init() override; + nsCString GetDescriptionName() const override; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEAUDIOSTREAM_H diff --git a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp new file mode 100644 index 000000000000..e79513d4a766 --- /dev/null +++ b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.cpp @@ -0,0 +1,102 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MFMediaEngineDecoderModule.h" + +#include "VideoUtils.h" +#include "mozilla/MFMediaEngineParent.h" +#include "mozilla/MFMediaEngineUtils.h" +#include "mozilla/StaticPrefs_media.h" + +namespace mozilla { + +#define LOG(msg, ...) \ + MOZ_LOG(gMFMediaEngineLog, LogLevel::Debug, (msg, ##__VA_ARGS__)) + +/* static */ +void MFMediaEngineDecoderModule::Init() { + // TODO : Init any thing that media engine would need. Implement this when we + // start implementing media engine in following patches. +} + +/* static */ +already_AddRefed MFMediaEngineDecoderModule::Create() { + RefPtr module = new MFMediaEngineDecoderModule(); + return module.forget(); +} + +already_AddRefed +MFMediaEngineDecoderModule::CreateVideoDecoder( + const CreateDecoderParams& aParams) { + if (!aParams.mMediaEngineId || + !StaticPrefs::media_wmf_media_engine_enabled()) { + return nullptr; + } + RefPtr mediaEngine = + MFMediaEngineParent::GetMediaEngineById(*aParams.mMediaEngineId); + if (!mediaEngine) { + LOG("Can't find media engine %" PRIu64 " for video decoder", + *aParams.mMediaEngineId); + return nullptr; + } + LOG("MFMediaEngineDecoderModule, CreateVideoDecoder"); + RefPtr decoder = mediaEngine->GetMediaEngineStream( + TrackInfo::TrackType::kVideoTrack, aParams); + return decoder.forget(); +} + +already_AddRefed +MFMediaEngineDecoderModule::CreateAudioDecoder( + const CreateDecoderParams& aParams) { + if (!aParams.mMediaEngineId || + !StaticPrefs::media_wmf_media_engine_enabled()) { + return nullptr; + } + RefPtr mediaEngine = + MFMediaEngineParent::GetMediaEngineById(*aParams.mMediaEngineId); + if (!mediaEngine) { + LOG("Can't find media engine %" PRIu64 " for audio decoder", + *aParams.mMediaEngineId); + return nullptr; + } + LOG("MFMediaEngineDecoderModule, CreateAudioDecoder"); + RefPtr decoder = mediaEngine->GetMediaEngineStream( + TrackInfo::TrackType::kAudioTrack, aParams); + return decoder.forget(); +} + +media::DecodeSupportSet MFMediaEngineDecoderModule::SupportsMimeType( + const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const { + UniquePtr trackInfo = CreateTrackInfoWithMIMEType(aMimeType); + if (!trackInfo) { + return media::DecodeSupport::Unsupported; + } + return SupportInternal(SupportDecoderParams(*trackInfo), aDiagnostics); +} + +media::DecodeSupportSet MFMediaEngineDecoderModule::Supports( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const { + if (!aParams.mMediaEngineId) { + return media::DecodeSupport::Unsupported; + } + return SupportInternal(aParams, aDiagnostics); +} + +media::DecodeSupportSet MFMediaEngineDecoderModule::SupportInternal( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const { + if (!StaticPrefs::media_wmf_media_engine_enabled()) { + return media::DecodeSupport::Unsupported; + } + // TODO : ask media engine or simply ask WMFDecoderModule? I guess the + // internal implementation of MFMediaEngine should be using MFT, so they + // should be the same in term of support types. Implement this when we + // start implementing media engine in following patches. + return media::DecodeSupport::SoftwareDecode; +} + +#undef LOG + +} // namespace mozilla diff --git a/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h new file mode 100644 index 000000000000..83caff431205 --- /dev/null +++ b/dom/media/platforms/wmf/MFMediaEngineDecoderModule.h @@ -0,0 +1,41 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEDECODERMODULE_H +#define DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEDECODERMODULE_H + +#include "PlatformDecoderModule.h" + +namespace mozilla { + +class MFMediaEngineDecoderModule final : public PlatformDecoderModule { + public: + static void Init(); + + static already_AddRefed Create(); + + already_AddRefed CreateVideoDecoder( + const CreateDecoderParams& aParams) override; + + already_AddRefed CreateAudioDecoder( + const CreateDecoderParams& aParams) override; + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; + media::DecodeSupportSet Supports( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + private: + media::DecodeSupportSet SupportInternal( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const; + MFMediaEngineDecoderModule() = default; + ~MFMediaEngineDecoderModule() = default; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEDECODERMODULE_H diff --git a/dom/media/platforms/wmf/MFMediaEngineStream.cpp b/dom/media/platforms/wmf/MFMediaEngineStream.cpp new file mode 100644 index 000000000000..9730c8541ae6 --- /dev/null +++ b/dom/media/platforms/wmf/MFMediaEngineStream.cpp @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MFMediaEngineStream.h" + +namespace mozilla { + +RefPtr MFMediaEngineStream::Decode( + MediaRawData* aSample) { + // TODO : implement this by using MediaEngine API. + return DecodePromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DECODE_ERR, + __func__); +} + +RefPtr MFMediaEngineStream::Drain() { + // TODO : implement this by using MediaEngine API. + return DecodePromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DECODE_ERR, + __func__); +} + +RefPtr MFMediaEngineStream::Flush() { + // TODO : implement this by using MediaEngine API. + return MediaDataDecoder::FlushPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("NotImpl")), + __func__); +} + +RefPtr MFMediaEngineStream::Shutdown() { + // TODO : implement this by using MediaEngine API. + return ShutdownPromise::CreateAndReject(false, __func__); +} + +} // namespace mozilla diff --git a/dom/media/platforms/wmf/MFMediaEngineStream.h b/dom/media/platforms/wmf/MFMediaEngineStream.h new file mode 100644 index 000000000000..d7a978ed9bd5 --- /dev/null +++ b/dom/media/platforms/wmf/MFMediaEngineStream.h @@ -0,0 +1,31 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINESTREAM_H +#define DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINESTREAM_H + +#include "PlatformDecoderModule.h" + +namespace mozilla { + +/** + * MFMediaEngineStream represents a track which would be responsible to provide + * encoded data into the media engine. The media engine can access this stream + * by the presentation descriptor which was acquired from the custom media + * source. + */ +// TODO : Inherit IMFMediaStream in following patches +class MFMediaEngineStream : public MediaDataDecoder { + public: + MFMediaEngineStream() = default; + + RefPtr Decode(MediaRawData* aSample) override; + RefPtr Drain() override; + RefPtr Flush() override; + RefPtr Shutdown() override; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINESTREAM_H diff --git a/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp b/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp new file mode 100644 index 000000000000..158d38b50435 --- /dev/null +++ b/dom/media/platforms/wmf/MFMediaEngineVideoStream.cpp @@ -0,0 +1,24 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MFMediaEngineVideoStream.h" + +namespace mozilla { + +RefPtr MFMediaEngineVideoStream::Init() { + // TODO : implement this by using MediaEngine API. + return InitPromise::CreateAndResolve(TrackType::kVideoTrack, __func__); +} + +nsCString MFMediaEngineVideoStream::GetDescriptionName() const { + return "media engine video stream"_ns; +} + +MediaDataDecoder::ConversionRequired MFMediaEngineVideoStream::NeedsConversion() + const { + // TODO : check video type + return ConversionRequired::kNeedAnnexB; +} + +} // namespace mozilla diff --git a/dom/media/platforms/wmf/MFMediaEngineVideoStream.h b/dom/media/platforms/wmf/MFMediaEngineVideoStream.h new file mode 100644 index 000000000000..6b12832d3860 --- /dev/null +++ b/dom/media/platforms/wmf/MFMediaEngineVideoStream.h @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEVIDEOSTREAM_H +#define DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEVIDEOSTREAM_H + +#include "MFMediaEngineStream.h" + +namespace mozilla { + +class MFMediaEngineVideoStream final : public MFMediaEngineStream { + public: + explicit MFMediaEngineVideoStream(const CreateDecoderParams& aParam) {} + + RefPtr Init() override; + nsCString GetDescriptionName() const override; + ConversionRequired NeedsConversion() const override; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORM_WMF_MFMEDIAENGINEVIDEOSTREAM_H diff --git a/dom/media/platforms/wmf/moz.build b/dom/media/platforms/wmf/moz.build index 4ee4bc442f18..a32f2cbaa75a 100644 --- a/dom/media/platforms/wmf/moz.build +++ b/dom/media/platforms/wmf/moz.build @@ -6,6 +6,10 @@ EXPORTS += [ "DXVA2Manager.h", + "MFMediaEngineAudioStream.h", + "MFMediaEngineDecoderModule.h", + "MFMediaEngineStream.h", + "MFMediaEngineVideoStream.h", "MFTDecoder.h", "WMF.h", "WMFAudioMFTManager.h", @@ -16,8 +20,13 @@ EXPORTS += [ "WMFUtils.h", "WMFVideoMFTManager.h", ] + UNIFIED_SOURCES += [ "DXVA2Manager.cpp", + "MFMediaEngineAudioStream.cpp", + "MFMediaEngineDecoderModule.cpp", + "MFMediaEngineStream.cpp", + "MFMediaEngineVideoStream.cpp", "MFTDecoder.cpp", "MFTEncoder.cpp", "WMFAudioMFTManager.cpp",