Bug 1752493 Disable HW decoding in RDD process if we're asked so r=alwu

Differential Revision: https://phabricator.services.mozilla.com/D138000

dom/media/ipc/PRDD.ipdl

@@ -43,7 +43,8 @@ parent:
   async InitProfiler(Endpoint<PProfilerChild> endpoint);
 
   async NewContentRemoteDecoderManager(
-            Endpoint<PRemoteDecoderManagerParent> endpoint);
+            Endpoint<PRemoteDecoderManagerParent> endpoint,
+            bool allowHardwareDecoding);
 
   async RequestMemoryReport(uint32_t generation,
                             bool anonymize,

dom/media/ipc/RDDParent.cpp

@@ -181,8 +181,10 @@ mozilla::ipc::IPCResult RDDParent::RecvInitProfiler(
 }
 
 mozilla::ipc::IPCResult RDDParent::RecvNewContentRemoteDecoderManager(
-    Endpoint<PRemoteDecoderManagerParent>&& aEndpoint) {
-  if (!RemoteDecoderManagerParent::CreateForContent(std::move(aEndpoint))) {
+    Endpoint<PRemoteDecoderManagerParent>&& aEndpoint,
+    const bool& aAllowHardwareDecoding) {
+  if (!RemoteDecoderManagerParent::CreateForContent(std::move(aEndpoint),
+                                                    aAllowHardwareDecoding)) {
     return IPC_FAIL_NO_REASON(this);
   }
   return IPC_OK();

dom/media/ipc/RDDParent.h

@@ -34,7 +34,8 @@ class RDDParent final : public PRDDParent {
       Endpoint<PProfilerChild>&& aEndpoint);
 
   mozilla::ipc::IPCResult RecvNewContentRemoteDecoderManager(
-      Endpoint<PRemoteDecoderManagerParent>&& aEndpoint);
+      Endpoint<PRemoteDecoderManagerParent>&& aEndpoint,
+      const bool& aAllowHardwareDecoding);
   mozilla::ipc::IPCResult RecvInitVideoBridge(
       Endpoint<PVideoBridgeChild>&& aEndpoint,
       const bool& aCreateHardwareDevice,

dom/media/ipc/RDDProcessManager.cpp

@@ -294,7 +294,8 @@ bool RDDProcessManager::CreateContentBridge(
     return false;
   }
 
-  mRDDChild->SendNewContentRemoteDecoderManager(std::move(parentPipe));
+  mRDDChild->SendNewContentRemoteDecoderManager(std::move(parentPipe),
+      /* aAllowHardwareDecoding */ mNumUnexpectedCrashes == 0);
 
   *aOutRemoteDecoderManager = std::move(childPipe);
   return true;

dom/media/ipc/RemoteDecoderManagerParent.cpp

@@ -115,7 +115,8 @@ PDMFactory& RemoteDecoderManagerParent::EnsurePDMFactory() {
 }
 
 bool RemoteDecoderManagerParent::CreateForContent(
-    Endpoint<PRemoteDecoderManagerParent>&& aEndpoint) {
+    Endpoint<PRemoteDecoderManagerParent>&& aEndpoint,
+    const bool aAllowHardwareDecoding) {
   MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_RDD ||
              XRE_GetProcessType() == GeckoProcessType_GPU);
   MOZ_ASSERT(NS_IsMainThread());
@@ -125,8 +126,8 @@ bool RemoteDecoderManagerParent::CreateForContent(
   }
 
   RefPtr<RemoteDecoderManagerParent> parent =
-      new RemoteDecoderManagerParent(sRemoteDecoderManagerParentThread);
-
+      new RemoteDecoderManagerParent(sRemoteDecoderManagerParentThread,
+                                     aAllowHardwareDecoding);
   RefPtr<Runnable> task =
       NewRunnableMethod<Endpoint<PRemoteDecoderManagerParent>&&>(
           "dom::RemoteDecoderManagerParent::Open", parent,
@@ -156,8 +157,8 @@ bool RemoteDecoderManagerParent::CreateVideoBridgeToOtherProcess(
 }
 
 RemoteDecoderManagerParent::RemoteDecoderManagerParent(
-    nsISerialEventTarget* aThread)
-    : mThread(aThread) {
+    nsISerialEventTarget* aThread, const bool aAllowHardwareDecoding)
+    : mThread(aThread), mAllowHardwareDecoding(aAllowHardwareDecoding) {
   MOZ_COUNT_CTOR(RemoteDecoderManagerParent);
   auto& registrar = XRE_IsGPUProcess()
                         ? GPUParent::GetSingleton()->AsyncShutdownService()

dom/media/ipc/RemoteDecoderManagerParent.h

@@ -23,7 +23,8 @@ class RemoteDecoderManagerParent final
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RemoteDecoderManagerParent, override)
 
   static bool CreateForContent(
-      Endpoint<PRemoteDecoderManagerParent>&& aEndpoint);
+      Endpoint<PRemoteDecoderManagerParent>&& aEndpoint,
+      const bool aAllowHardwareDecoding);
 
   static bool CreateVideoBridgeToOtherProcess(
       Endpoint<layers::PVideoBridgeChild>&& aEndpoint);
@@ -50,6 +51,8 @@ class RemoteDecoderManagerParent final
 
   bool OnManagerThread();
 
+  bool AllowHardwareDecoding() const { return mAllowHardwareDecoding; };
+
   // Can be called from manager thread only
   PDMFactory& EnsurePDMFactory();
 
@@ -69,7 +72,8 @@ class RemoteDecoderManagerParent final
   void ActorDealloc() override;
 
  private:
-  explicit RemoteDecoderManagerParent(nsISerialEventTarget* aThread);
+  RemoteDecoderManagerParent(nsISerialEventTarget* aThread,
+                             const bool aAllowHardwareDecoding);
   ~RemoteDecoderManagerParent();
 
   void Open(Endpoint<PRemoteDecoderManagerParent>&& aEndpoint);
@@ -79,6 +83,8 @@ class RemoteDecoderManagerParent final
 
   nsCOMPtr<nsISerialEventTarget> mThread;
   RefPtr<PDMFactory> mPDMFactory;
+
+  const bool mAllowHardwareDecoding;
 };
 
 }  // namespace mozilla

dom/media/ipc/RemoteVideoDecoder.cpp

@@ -158,7 +158,9 @@ IPCResult RemoteVideoDecoderParent::RecvConstruct(
       imageContainer, CreateDecoderParams::VideoFrameRate(mFramerate),
       mOptions,       CreateDecoderParams::NoWrapper(true),
   };
-
+  if (!mParent->AllowHardwareDecoding()) {
+    params.mOptions += CreateDecoderParams::Option::HardwareDecoderNotAllowed;
+  }
   mParent->EnsurePDMFactory().CreateDecoder(params)->Then(
       GetCurrentSerialEventTarget(), __func__,
       [resolver = std::move(aResolver), self = RefPtr{this}](

gfx/ipc/GPUParent.cpp

@@ -541,7 +541,7 @@ mozilla::ipc::IPCResult GPUParent::RecvNewContentVRManager(
 
 mozilla::ipc::IPCResult GPUParent::RecvNewContentRemoteDecoderManager(
     Endpoint<PRemoteDecoderManagerParent>&& aEndpoint) {
-  if (!RemoteDecoderManagerParent::CreateForContent(std::move(aEndpoint))) {
+  if (!RemoteDecoderManagerParent::CreateForContent(std::move(aEndpoint), /* AllowHardwareDecoding */ true)) {
     return IPC_FAIL_NO_REASON(this);
   }
   return IPC_OK();