/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "MediaCapabilities.h" #include "AllocationPolicy.h" #include "Benchmark.h" #include "DecoderTraits.h" #include "Layers.h" #include "MediaInfo.h" #include "MediaRecorder.h" #include "PDMFactory.h" #include "VPXDecoder.h" #include "mozilla/Move.h" #include "mozilla/StaticPrefs.h" #include "mozilla/TaskQueue.h" #include "mozilla/dom/DOMMozPromiseRequestHolder.h" #include "mozilla/dom/MediaCapabilitiesBinding.h" #include "mozilla/dom/MediaSource.h" #include "mozilla/dom/Promise.h" #include "mozilla/dom/WorkerPrivate.h" #include "mozilla/dom/WorkerRef.h" #include "mozilla/layers/KnowsCompositor.h" #include "nsContentUtils.h" #include static mozilla::LazyLogModule sMediaCapabilitiesLog("MediaCapabilities"); #define LOG(msg, ...) \ DDMOZ_LOG(sMediaCapabilitiesLog, LogLevel::Debug, msg, ##__VA_ARGS__) namespace mozilla { namespace dom { static nsCString VideoConfigurationToStr(const VideoConfiguration* aConfig) { if (!aConfig) { return nsCString(); } auto str = nsPrintfCString( "[contentType:%s width:%d height:%d bitrate:%" PRIu64 " framerate:%s]", NS_ConvertUTF16toUTF8(aConfig->mContentType).get(), aConfig->mWidth, aConfig->mHeight, aConfig->mBitrate, NS_ConvertUTF16toUTF8(aConfig->mFramerate).get()); return std::move(str); } static nsCString AudioConfigurationToStr(const AudioConfiguration* aConfig) { if (!aConfig) { return nsCString(); } auto str = nsPrintfCString( "[contentType:%s channels:%s bitrate:%" PRIu64 " samplerate:%d]", NS_ConvertUTF16toUTF8(aConfig->mContentType).get(), aConfig->mChannels.WasPassed() ? NS_ConvertUTF16toUTF8(aConfig->mChannels.Value()).get() : "?", aConfig->mBitrate.WasPassed() ? aConfig->mBitrate.Value() : 0, aConfig->mSamplerate.WasPassed() ? aConfig->mSamplerate.Value() : 0); return std::move(str); } static nsCString MediaCapabilitiesInfoToStr(const MediaCapabilitiesInfo* aInfo) { if (!aInfo) { return nsCString(); } auto str = nsPrintfCString("[supported:%s smooth:%s powerEfficient:%s]", aInfo->Supported() ? "true" : "false", aInfo->Smooth() ? "true" : "false", aInfo->PowerEfficient() ? "true" : "false"); return std::move(str); } static nsCString MediaDecodingConfigurationToStr(const MediaDecodingConfiguration& aConfig) { nsCString str; str += NS_LITERAL_CSTRING("["); if (aConfig.mVideo.WasPassed()) { str += NS_LITERAL_CSTRING("video:") + VideoConfigurationToStr(&aConfig.mVideo.Value()); if (aConfig.mAudio.WasPassed()) { str += NS_LITERAL_CSTRING(" "); } } if (aConfig.mAudio.WasPassed()) { str += NS_LITERAL_CSTRING("audio:") + AudioConfigurationToStr(&aConfig.mAudio.Value()); } str += NS_LITERAL_CSTRING("]"); return str; } MediaCapabilities::MediaCapabilities(nsIGlobalObject* aParent) : mParent(aParent) { } already_AddRefed MediaCapabilities::DecodingInfo( const MediaDecodingConfiguration& aConfiguration, ErrorResult& aRv) { RefPtr promise = Promise::Create(mParent, aRv); if (aRv.Failed()) { return nullptr; } // If configuration is not a valid MediaConfiguration, return a Promise // rejected with a TypeError. if (!aConfiguration.mVideo.WasPassed() && !aConfiguration.mAudio.WasPassed()) { aRv.ThrowTypeError( NS_LITERAL_STRING("'audio' or 'video'")); return nullptr; } LOG("Processing %s", MediaDecodingConfigurationToStr(aConfiguration).get()); bool supported = true; Maybe videoContainer; Maybe audioContainer; // If configuration.video is present and is not a valid video configuration, // return a Promise rejected with a TypeError. if (aConfiguration.mVideo.WasPassed()) { videoContainer = CheckVideoConfiguration(aConfiguration.mVideo.Value()); if (!videoContainer) { aRv.ThrowTypeError(); return nullptr; } // We have a video configuration and it is valid. Check if it is supported. supported &= aConfiguration.mType == MediaDecodingType::File ? CheckTypeForFile(aConfiguration.mVideo.Value().mContentType) : CheckTypeForMediaSource(aConfiguration.mVideo.Value().mContentType); } if (aConfiguration.mAudio.WasPassed()) { audioContainer = CheckAudioConfiguration(aConfiguration.mAudio.Value()); if (!audioContainer) { aRv.ThrowTypeError(); return nullptr; } // We have an audio configuration and it is valid. Check if it is supported. supported &= aConfiguration.mType == MediaDecodingType::File ? CheckTypeForFile(aConfiguration.mAudio.Value().mContentType) : CheckTypeForMediaSource(aConfiguration.mAudio.Value().mContentType); } if (!supported) { auto info = MakeUnique( false /* supported */, false /* smooth */, false /* power efficient */); LOG("%s -> %s", MediaDecodingConfigurationToStr(aConfiguration).get(), MediaCapabilitiesInfoToStr(info.get()).get()); promise->MaybeResolve(std::move(info)); return promise.forget(); } nsTArray> tracks; if (aConfiguration.mVideo.WasPassed()) { MOZ_ASSERT(videoContainer.isSome(), "configuration is valid and supported"); auto videoTracks = DecoderTraits::GetTracksInfo(*videoContainer); // If the MIME type does not imply a codec, the string MUST // also have one and only one parameter that is named codecs with a value // describing a single media codec. Otherwise, it MUST contain no // parameters. if (videoTracks.Length() != 1) { promise->MaybeReject(NS_ERROR_DOM_TYPE_ERR); return promise.forget(); } MOZ_DIAGNOSTIC_ASSERT(videoTracks.ElementAt(0), "must contain a valid trackinfo"); tracks.AppendElements(std::move(videoTracks)); } if (aConfiguration.mAudio.WasPassed()) { MOZ_ASSERT(audioContainer.isSome(), "configuration is valid and supported"); auto audioTracks = DecoderTraits::GetTracksInfo(*audioContainer); // If the MIME type does not imply a codec, the string MUST // also have one and only one parameter that is named codecs with a value // describing a single media codec. Otherwise, it MUST contain no // parameters. if (audioTracks.Length() != 1) { promise->MaybeReject(NS_ERROR_DOM_TYPE_ERR); return promise.forget(); } MOZ_DIAGNOSTIC_ASSERT(audioTracks.ElementAt(0), "must contain a valid trackinfo"); tracks.AppendElements(std::move(audioTracks)); } typedef MozPromise CapabilitiesPromise; nsTArray> promises; RefPtr taskQueue = new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), "MediaCapabilities::TaskQueue"); for (auto&& config : tracks) { TrackInfo::TrackType type = config->IsVideo() ? TrackInfo::kVideoTrack : TrackInfo::kAudioTrack; MOZ_ASSERT(type == TrackInfo::kAudioTrack || videoContainer->ExtendedType().GetFramerate().isSome(), "framerate is a required member of VideoConfiguration"); if (type == TrackInfo::kAudioTrack) { // There's no need to create an audio decoder has we only want to know if // such codec is supported RefPtr pdm = new PDMFactory(); if (!pdm->Supports(*config, nullptr /* decoder doctor */)) { auto info = MakeUnique(false /* supported */, false /* smooth */, false /* power efficient */); LOG("%s -> %s", MediaDecodingConfigurationToStr(aConfiguration).get(), MediaCapabilitiesInfoToStr(info.get()).get()); promise->MaybeResolve(std::move(info)); return promise.forget(); } // We can assume that if we could create the decoder, then we can play it. // We report that we can play it smoothly and in an efficient fashion. promises.AppendElement(CapabilitiesPromise::CreateAndResolve( MediaCapabilitiesInfo( true /* supported */, true /* smooth */, true /* power efficient */), __func__)); continue; } // On Windows, the MediaDataDecoder expects to be created on a thread // supporting MTA, which the main thread doesn't. So we use our task queue // to create such decoder and perform initialization. RefPtr compositor = GetCompositor(); double frameRate = videoContainer->ExtendedType().GetFramerate().ref(); promises.AppendElement(InvokeAsync( taskQueue, __func__, [taskQueue, frameRate, compositor, config = std::move(config)]() mutable -> RefPtr { // MediaDataDecoder keeps a reference to the config object, so we must // keep it alive until the decoder has been shutdown. CreateDecoderParams params{ *config, taskQueue, compositor, CreateDecoderParams::VideoFrameRate( frameRate), TrackInfo::kVideoTrack }; return AllocationWrapper::CreateDecoder(params)->Then( taskQueue, __func__, [taskQueue, frameRate, config = std::move(config)]( const AllocationWrapper::AllocateDecoderPromise:: ResolveOrRejectValue& aValue) mutable { if (aValue.IsReject()) { return CapabilitiesPromise::CreateAndReject(aValue.RejectValue(), __func__); } RefPtr decoder = aValue.ResolveValue(); // We now query the decoder to determine if it's power efficient. RefPtr p = decoder->Init()->Then( taskQueue, __func__, [taskQueue, decoder, frameRate, config = std::move(config)]( const MediaDataDecoder::InitPromise::ResolveOrRejectValue& aValue) mutable { RefPtr p; if (aValue.IsReject()) { p = CapabilitiesPromise::CreateAndReject(aValue.RejectValue(), __func__); } else { MOZ_ASSERT(config->IsVideo()); nsAutoCString reason; bool powerEfficient = true; bool smooth = true; if (config->GetAsVideoInfo()->mImage.height > 480) { // Assume that we can do stuff at 480p or less in a power // efficient manner and smoothly. If greater than 480p we // assume that if the video decoding is hardware accelerated // it will be smooth and power efficient, otherwise we use // the benchmark to estimate powerEfficient = decoder->IsHardwareAccelerated(reason); if (!powerEfficient && VPXDecoder::IsVP9(config->mMimeType)) { smooth = VP9Benchmark::IsVP9DecodeFast(true /* default */); uint32_t fps = VP9Benchmark::MediaBenchmarkVp9Fps(); if (!smooth && fps > 0) { // The VP9 estimizer decode a 1280x720 video. Let's // adjust the result for the resolution and frame rate // of what we actually want. If the result is twice that // we need we assume it will be smooth. const auto& videoConfig = *config->GetAsVideoInfo(); double needed = ((1280.0 * 720.0) / (videoConfig.mImage.width * videoConfig.mImage.height) * fps) / frameRate; smooth = needed > 2; } } } p = CapabilitiesPromise::CreateAndResolve( MediaCapabilitiesInfo( true /* supported */, smooth, powerEfficient), __func__); } MOZ_ASSERT(p.get(), "the promise has been created"); // Let's keep alive the decoder and the config object until the // decoder has shutdown. decoder->Shutdown()->Then( taskQueue, __func__, [taskQueue, decoder, config = std::move(config)]( const ShutdownPromise::ResolveOrRejectValue& aValue) {}); return p; }); return p; }); })); } auto holder = MakeRefPtr>( mParent); RefPtr targetThread; RefPtr workerRef; if (NS_IsMainThread()) { targetThread = mParent->AbstractMainThreadFor(TaskCategory::Other); } else { WorkerPrivate* wp = GetCurrentThreadWorkerPrivate(); MOZ_ASSERT(wp, "Must be called from a worker thread"); targetThread = wp->HybridEventTarget(); workerRef = StrongWorkerRef::Create( wp, "MediaCapabilities", [holder, targetThread]() { MOZ_ASSERT(targetThread->IsOnCurrentThread()); holder->DisconnectIfExists(); }); if (NS_WARN_IF(!workerRef)) { // The worker is shutting down. aRv.Throw(NS_ERROR_FAILURE); return nullptr; } } MOZ_ASSERT(targetThread); // this is only captured for use with the LOG macro. RefPtr self = this; CapabilitiesPromise::All(targetThread, promises) ->Then( targetThread, __func__, [promise, tracks = std::move(tracks), workerRef, holder, aConfiguration, self, this](const CapabilitiesPromise::AllPromiseType::ResolveOrRejectValue& aValue) { holder->Complete(); if (aValue.IsReject()) { auto info = MakeUnique(false /* supported */, false /* smooth */, false /* power efficient */); LOG("%s -> %s", MediaDecodingConfigurationToStr(aConfiguration).get(), MediaCapabilitiesInfoToStr(info.get()).get()); promise->MaybeResolve(std::move(info)); return; } bool powerEfficient = true; bool smooth = true; for (auto&& capability : aValue.ResolveValue()) { smooth &= capability.Smooth(); powerEfficient &= capability.PowerEfficient(); } auto info = MakeUnique( true /* supported */, smooth, powerEfficient); LOG("%s -> %s", MediaDecodingConfigurationToStr(aConfiguration).get(), MediaCapabilitiesInfoToStr(info.get()).get()); promise->MaybeResolve(std::move(info)); }) ->Track(*holder); return promise.forget(); } already_AddRefed MediaCapabilities::EncodingInfo( const MediaEncodingConfiguration& aConfiguration, ErrorResult& aRv) { RefPtr promise = Promise::Create(mParent, aRv); if (aRv.Failed()) { return nullptr; } // If configuration is not a valid MediaConfiguration, return a Promise // rejected with a TypeError. if (!aConfiguration.mVideo.WasPassed() && !aConfiguration.mAudio.WasPassed()) { aRv.ThrowTypeError( NS_LITERAL_STRING("'audio' or 'video'")); return nullptr; } bool supported = true; // If configuration.video is present and is not a valid video configuration, // return a Promise rejected with a TypeError. if (aConfiguration.mVideo.WasPassed()) { if (!CheckVideoConfiguration(aConfiguration.mVideo.Value())) { aRv.ThrowTypeError(); return nullptr; } // We have a video configuration and it is valid. Check if it is supported. supported &= CheckTypeForEncoder(aConfiguration.mVideo.Value().mContentType); } if (aConfiguration.mAudio.WasPassed()) { if (!CheckAudioConfiguration(aConfiguration.mAudio.Value())) { aRv.ThrowTypeError(); return nullptr; } // We have an audio configuration and it is valid. Check if it is supported. supported &= CheckTypeForEncoder(aConfiguration.mAudio.Value().mContentType); } auto info = MakeUnique(supported, supported, false); promise->MaybeResolve(std::move(info)); return promise.forget(); } Maybe MediaCapabilities::CheckVideoConfiguration( const VideoConfiguration& aConfig) const { Maybe container = MakeMediaExtendedMIMEType(aConfig); if (!container) { return Nothing(); } // A valid video MIME type is a string that is a valid media MIME type and for // which the type per [RFC7231] is either video or application. if (!container->Type().HasVideoMajorType() && !container->Type().HasApplicationMajorType()) { return Nothing(); } // If the MIME type does not imply a codec, the string MUST also have one and // only one parameter that is named codecs with a value describing a single // media codec. Otherwise, it MUST contain no parameters. // TODO (nsIMOMEHeaderParam doesn't provide backend to count number of // parameters) return Some(MediaContainerType(std::move(*container))); } Maybe MediaCapabilities::CheckAudioConfiguration( const AudioConfiguration& aConfig) const { Maybe container = MakeMediaExtendedMIMEType(aConfig); if (!container) { return Nothing(); } // A valid audio MIME type is a string that is valid media MIME type and for // which the type per [RFC7231] is either audio or application. if (!container->Type().HasAudioMajorType() && !container->Type().HasApplicationMajorType()) { return Nothing(); } // If the MIME type does not imply a codec, the string MUST also have one and // only one parameter that is named codecs with a value describing a single // media codec. Otherwise, it MUST contain no parameters. // TODO (nsIMOMEHeaderParam doesn't provide backend to count number of // parameters) return Some(MediaContainerType(std::move(*container))); } bool MediaCapabilities::CheckTypeForMediaSource(const nsAString& aType) { return NS_SUCCEEDED(MediaSource::IsTypeSupported( aType, nullptr /* DecoderDoctorDiagnostics */)); } bool MediaCapabilities::CheckTypeForFile(const nsAString& aType) { Maybe containerType = MakeMediaContainerType(aType); if (!containerType) { return false; } return DecoderTraits::CanHandleContainerType( *containerType, nullptr /* DecoderDoctorDiagnostics */) != CANPLAY_NO; } bool MediaCapabilities::CheckTypeForEncoder(const nsAString& aType) { return MediaRecorder::IsTypeSupported(aType); } already_AddRefed MediaCapabilities::GetCompositor() { nsCOMPtr window = do_QueryInterface(GetParentObject()); if (NS_WARN_IF(!window)) { return nullptr; } nsCOMPtr doc = window->GetExtantDoc(); if (NS_WARN_IF(!doc)) { return nullptr; } RefPtr layerManager = nsContentUtils::LayerManagerForDocument(doc); if (NS_WARN_IF(!layerManager)) { return nullptr; } RefPtr knows = layerManager->AsKnowsCompositor(); if (NS_WARN_IF(!knows)) { return nullptr; } return knows->GetForMedia().forget(); } bool MediaCapabilities::Enabled(JSContext* aCx, JSObject* aGlobal) { return StaticPrefs::MediaCapabilitiesEnabled(); } JSObject* MediaCapabilities::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { return MediaCapabilities_Binding::Wrap(aCx, this, aGivenProto); } NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(MediaCapabilities) NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY NS_INTERFACE_MAP_ENTRY(nsISupports) NS_INTERFACE_MAP_END NS_IMPL_CYCLE_COLLECTING_ADDREF(MediaCapabilities) NS_IMPL_CYCLE_COLLECTING_RELEASE(MediaCapabilities) NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(MediaCapabilities, mParent) // MediaCapabilitiesInfo bool MediaCapabilitiesInfo::WrapObject(JSContext* aCx, JS::Handle aGivenProto, JS::MutableHandle aReflector) { return MediaCapabilitiesInfo_Binding::Wrap( aCx, this, aGivenProto, aReflector); } } // namespace dom } // namespace mozilla