From 1781f8be7f6bfc7ff08b29f27aa5a05c206a4813 Mon Sep 17 00:00:00 2001
From: Jean-Yves Avenard <jyavenard@mozilla.com>
Date: Sun, 3 Jan 2021 23:20:25 +0000
Subject: [PATCH] Bug 1681043 - P2. Don't reject outright support for a GPU
 decoder if compositor isn't known. r=alwu,bryce

When calling PDM::SupportsMimeType not enough information is provided for the PDM to categorically state that a codec isn't supported.

Only  creating the decoder successfully matters and the PDMFactory only uses the value returned by Support as a hint to further query the PDM.

Without this change WMFDecoderModule::SupportsMimeType in the GPU process would always return false as it doesn't know yet the KnownCompositor.

Differential Revision: https://phabricator.services.mozilla.com/D100305
---
 dom/media/platforms/PlatformDecoderModule.h  |  2 ++
 dom/media/platforms/wmf/WMFDecoderModule.cpp | 30 ++++++++++++++------
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/dom/media/platforms/PlatformDecoderModule.h b/dom/media/platforms/PlatformDecoderModule.h
index 4dd62fad21c2..2d5ec7a54b1a 100644
--- a/dom/media/platforms/PlatformDecoderModule.h
+++ b/dom/media/platforms/PlatformDecoderModule.h
@@ -315,6 +315,8 @@ class PlatformDecoderModule {
   virtual nsresult Startup() { return NS_OK; }
 
   // Indicates if the PlatformDecoderModule supports decoding of aMimeType.
+  // The answer to both SupportsMimeType and Supports doesn't guarantee that
+  // creation of a decoder will actually succeed.
   virtual bool SupportsMimeType(
       const nsACString& aMimeType,
       DecoderDoctorDiagnostics* aDiagnostics) const = 0;
diff --git a/dom/media/platforms/wmf/WMFDecoderModule.cpp b/dom/media/platforms/wmf/WMFDecoderModule.cpp
index acd6f553617e..d4bd7b8ba3a1 100644
--- a/dom/media/platforms/wmf/WMFDecoderModule.cpp
+++ b/dom/media/platforms/wmf/WMFDecoderModule.cpp
@@ -68,15 +68,16 @@ WMFDecoderModule::~WMFDecoderModule() {
   }
 }
 
-static bool IsRemoteAcceleratedCompositor(const SupportDecoderParams& aParams) {
-  if (!aParams.mKnowsCompositor) {
+static bool IsRemoteAcceleratedCompositor(
+    layers::KnowsCompositor* aKnowsCompositor) {
+  if (!aKnowsCompositor) {
     return false;
   }
 
   TextureFactoryIdentifier ident =
-      aParams.mKnowsCompositor->GetTextureFactoryIdentifier();
+      aKnowsCompositor->GetTextureFactoryIdentifier();
   return ident.mParentBackend != LayersBackend::LAYERS_BASIC &&
-         !aParams.mKnowsCompositor->UsingSoftwareWebRender() &&
+         !aKnowsCompositor->UsingSoftwareWebRender() &&
          ident.mParentProcessType == GeckoProcessType_GPU;
 }
 
@@ -172,6 +173,13 @@ nsresult WMFDecoderModule::Startup() {
 
 already_AddRefed<MediaDataDecoder> WMFDecoderModule::CreateVideoDecoder(
     const CreateDecoderParams& aParams) {
+  // In GPU process, only support decoding if an accelerated compositor is
+  // known.
+  if (XRE_IsGPUProcess() &&
+      !IsRemoteAcceleratedCompositor(aParams.mKnowsCompositor)) {
+    return nullptr;
+  }
+
   UniquePtr<WMFVideoMFTManager> manager(new WMFVideoMFTManager(
       aParams.VideoConfig(), aParams.mKnowsCompositor, aParams.mImageContainer,
       aParams.mRate.mValue, aParams.mOptions, sDXVAEnabled));
@@ -201,6 +209,11 @@ already_AddRefed<MediaDataDecoder> WMFDecoderModule::CreateVideoDecoder(
 
 already_AddRefed<MediaDataDecoder> WMFDecoderModule::CreateAudioDecoder(
     const CreateDecoderParams& aParams) {
+  if (XRE_IsGPUProcess()) {
+    // Only allow video in the GPU process.
+    return nullptr;
+  }
+
   UniquePtr<WMFAudioMFTManager> manager(
       new WMFAudioMFTManager(aParams.AudioConfig()));
 
@@ -285,13 +298,14 @@ bool WMFDecoderModule::SupportsMimeType(
 
 bool WMFDecoderModule::Supports(const SupportDecoderParams& aParams,
                                 DecoderDoctorDiagnostics* aDiagnostics) const {
-  // In GPU process, only support decoding if an accelerated compositor is
-  // known.
-  if (XRE_IsGPUProcess() && !IsRemoteAcceleratedCompositor(aParams)) {
+  // In GPU process, only support decoding if video. This only gives a hint of
+  // what the GPU decoder *may* support. The actual check will occur in
+  // CreateVideoDecoder.
+  const auto& trackInfo = aParams.mConfig;
+  if (XRE_IsGPUProcess() && !trackInfo.GetAsVideoInfo()) {
     return false;
   }
 
-  const auto& trackInfo = aParams.mConfig;
   const auto* videoInfo = trackInfo.GetAsVideoInfo();
   // Temporary - forces use of VPXDecoder when alpha is present.
   // Bug 1263836 will handle alpha scenario once implemented. It will shift