Bug 1676999 - P2. Disable windows hardware acceleration if the RDD crashed. r=mattwoodrow,mjf

Depends on D97753

Differential Revision: https://phabricator.services.mozilla.com/D97754

dom/media/ipc/PRDD.ipdl

@@ -45,6 +45,7 @@ parent:
   async UpdateVar(GfxVarUpdate var);
 
   async InitVideoBridge(Endpoint<PVideoBridgeChild> endpoint,
+                        bool createHardwareDevice,
                         ContentDeviceData contentDeviceData);
 
   async GetUntrustedModulesData() returns (UntrustedModulesData? data);

dom/media/ipc/RDDParent.cpp

@@ -168,6 +168,7 @@ mozilla::ipc::IPCResult RDDParent::RecvNewContentRemoteDecoderManager(
 
 mozilla::ipc::IPCResult RDDParent::RecvInitVideoBridge(
     Endpoint<PVideoBridgeChild>&& aEndpoint,
+    const bool& aCreateHardwareDevice,
     const ContentDeviceData& aContentDeviceData) {
   if (!RemoteDecoderManagerParent::CreateVideoBridgeToOtherProcess(
           std::move(aEndpoint))) {
@@ -189,7 +190,9 @@ mozilla::ipc::IPCResult RDDParent::RecvInitVideoBridge(
     auto* devmgr = DeviceManagerDx::Get();
     if (devmgr) {
       devmgr->ImportDeviceInfo(aContentDeviceData.d3d11());
-      devmgr->CreateContentDevices();
+      if (aCreateHardwareDevice) {
+        devmgr->CreateContentDevices();
+      }
     }
   }
 #endif

dom/media/ipc/RDDParent.h

@@ -37,6 +37,7 @@ class RDDParent final : public PRDDParent {
       Endpoint<PRemoteDecoderManagerParent>&& aEndpoint);
   mozilla::ipc::IPCResult RecvInitVideoBridge(
       Endpoint<PVideoBridgeChild>&& aEndpoint,
+      const bool& aCreateHardwareDevice,
       const ContentDeviceData& aContentDeviceData);
   mozilla::ipc::IPCResult RecvRequestMemoryReport(
       const uint32_t& generation, const bool& anonymize,

dom/media/ipc/RDDProcessManager.cpp

@@ -193,6 +193,8 @@ void RDDProcessManager::OnProcessUnexpectedShutdown(RDDProcessHost* aHost) {
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(mProcess && mProcess == aHost);
 
+  mNumUnexpectedCrashes++;
+
   DestroyProcess();
 }
 
@@ -283,7 +285,8 @@ bool RDDProcessManager::CreateVideoBridge() {
   ContentDeviceData contentDeviceData;
   gfxPlatform::GetPlatform()->BuildContentDeviceData(&contentDeviceData);
 
-  mRDDChild->SendInitVideoBridge(std::move(childPipe), contentDeviceData);
+  mRDDChild->SendInitVideoBridge(std::move(childPipe),
+                                 mNumUnexpectedCrashes == 0, contentDeviceData);
   if (gpuProcessPid != -1) {
     gpuManager->InitVideoBridge(std::move(parentPipe));
   } else {

dom/media/ipc/RDDProcessManager.h

@@ -94,6 +94,7 @@ class RDDProcessManager final : public RDDProcessHost::Listener {
   const RefPtr<Observer> mObserver;
   ipc::TaskFactory<RDDProcessManager> mTaskFactory;
   uint32_t mNumProcessAttempts = 0;
+  uint32_t mNumUnexpectedCrashes = 0;
 
   // Fields that are associated with the current RDD process.
   RDDProcessHost* mProcess = nullptr;

dom/media/platforms/wmf/WMFDecoderModule.cpp

@@ -107,9 +107,13 @@ void WMFDecoderModule::Init() {
     // means that we've given up on the GPU process (it's been crashing) so we
     // should disable DXVA
     sDXVAEnabled = !StaticPrefs::media_gpu_process_decoder();
-  } else if (XRE_IsGPUProcess() || XRE_IsRDDProcess()) {
-    // Always allow DXVA in the GPU or RDD process.
+  } else if (XRE_IsGPUProcess()) {
+    // Always allow DXVA in the GPU process.
     sDXVAEnabled = true;
+  } else if (XRE_IsRDDProcess()) {
+    // Only allows DXVA if we have an image device. We may have explicitly
+    // disabled its creation following an earlier RDD process crash.
+    sDXVAEnabled = !!DeviceManagerDx::Get()->GetImageDevice();
   } else {
     // Only allow DXVA in the UI process if we aren't in e10s Firefox
     sDXVAEnabled = !mozilla::BrowserTabsRemoteAutostart();