Bug 1777927 - Force-disable VAAPI if vaapitest() failed, r=gfx-reviewers,aosmond

`vaapitest()` is meant to be a sanity check. If it failed there's
likely something very broken about the driver and we log gfx
warnings accordingly, allowing to debug the problem.

Ensure to force-disable VAAPI in this case but still allow users
to enable the feature in blocklisted cases.

While on it add some additional fixes for issues encoutered while
testing:
- `InitVAAPIConfig()` was not run on X11-only builds, wrongly
  setting `sLayersSupportsHardwareVideoDecoding` to `true` there
  on allowlisted drivers. Thus replace `MOZ_WAYLAND` with
  `MOZ_WIDGET_GTK`, ensuring the "Wayland support missing" warning
  is shown in `about:support`.
- `UserForceEnable` must be run before `ForceDisable`, otherwise an
  assert is triggered on debug builds. Reorder this.
- `GetFeatureStatus` was run twice for
  `FEATURE_HARDWARE_VIDEO_DECODING`, once in `InitAcceleration()` in the
  common code path, the again in `InitVAAPIConfig()`. Untangle the common
  code path to only run `InitVAAPIConfig()`.
- Use the chance to turn `media.hardware-video-decoding.enabled` into a
  static pref so it matches `media.hardware-video-decoding.force-enabled`

Differential Revision: https://phabricator.services.mozilla.com/D150959

gfx/thebes/gfxPlatform.cpp

@@ -2351,9 +2351,11 @@ void gfxPlatform::InitAcceleration() {
   // explicit.
   MOZ_ASSERT(NS_IsMainThread(), "can only initialize prefs on the main thread");
 
+#ifndef MOZ_WIDGET_GTK
   nsCOMPtr<nsIGfxInfo> gfxInfo = components::GfxInfo::Service();
   nsCString discardFailureId;
   int32_t status;
+#endif
 
   if (XRE_IsParentProcess()) {
     gfxVars::SetBrowserTabsRemoteAutostart(BrowserTabsRemoteAutostart());
@@ -2378,27 +2380,29 @@ void gfxPlatform::InitAcceleration() {
 #endif
   }
 
-  if (Preferences::GetBool("media.hardware-video-decoding.enabled", false) &&
-#ifdef XP_WIN
-      Preferences::GetBool("media.wmf.dxva.enabled", true) &&
-#endif
-      NS_SUCCEEDED(
-          gfxInfo->GetFeatureStatus(nsIGfxInfo::FEATURE_HARDWARE_VIDEO_DECODING,
-                                    discardFailureId, &status))) {
-    if (status == nsIGfxInfo::FEATURE_STATUS_OK ||
-#ifdef MOZ_WAYLAND
-        StaticPrefs::media_ffmpeg_vaapi_enabled() ||
-#endif
-        StaticPrefs::media_hardware_video_decoding_force_enabled_AtStartup()) {
-      sLayersSupportsHardwareVideoDecoding = true;
+  if (StaticPrefs::media_hardware_video_decoding_enabled_AtStartup()) {
+#ifdef MOZ_WIDGET_GTK
+    sLayersSupportsHardwareVideoDecoding =
+        gfxPlatformGtk::GetPlatform()->InitVAAPIConfig(
+            StaticPrefs::
+                media_hardware_video_decoding_force_enabled_AtStartup() ||
+            StaticPrefs::media_ffmpeg_vaapi_enabled());
+#else
+    if (
+#  ifdef XP_WIN
+        Preferences::GetBool("media.wmf.dxva.enabled", true) &&
+#  endif
+        NS_SUCCEEDED(gfxInfo->GetFeatureStatus(
+            nsIGfxInfo::FEATURE_HARDWARE_VIDEO_DECODING, discardFailureId,
+            &status))) {
+      if (status == nsIGfxInfo::FEATURE_STATUS_OK ||
+          StaticPrefs::
+              media_hardware_video_decoding_force_enabled_AtStartup()) {
+        sLayersSupportsHardwareVideoDecoding = true;
+      }
     }
-  }
-
-#ifdef MOZ_WAYLAND
-  sLayersSupportsHardwareVideoDecoding =
-      gfxPlatformGtk::GetPlatform()->InitVAAPIConfig(
-          sLayersSupportsHardwareVideoDecoding);
 #endif
+  }
 
   sLayersAccelerationPrefsInitialized = true;
 

gfx/thebes/gfxPlatformGtk.cpp

@@ -225,12 +225,16 @@ void gfxPlatformGtk::InitDmabufConfig() {
 #endif
 }
 
-bool gfxPlatformGtk::InitVAAPIConfig(bool aEnabledByPlatform) {
+bool gfxPlatformGtk::InitVAAPIConfig(bool aForceEnabledByUser) {
   FeatureState& feature =
       gfxConfig::GetFeature(Feature::HARDWARE_VIDEO_DECODING);
 #ifdef MOZ_WAYLAND
   feature.EnableByDefault();
 
+  if (aForceEnabledByUser) {
+    feature.UserForceEnable("Force enabled by pref");
+  }
+
   nsCString failureId;
   int32_t status = nsIGfxInfo::FEATURE_STATUS_UNKNOWN;
   nsCOMPtr<nsIGfxInfo> gfxInfo = components::GfxInfo::Service();
@@ -238,16 +242,14 @@ bool gfxPlatformGtk::InitVAAPIConfig(bool aEnabledByPlatform) {
           nsIGfxInfo::FEATURE_HARDWARE_VIDEO_DECODING, failureId, &status))) {
     feature.Disable(FeatureStatus::BlockedNoGfxInfo, "gfxInfo is broken",
                     "FEATURE_FAILURE_NO_GFX_INFO"_ns);
+  } else if (status == nsIGfxInfo::FEATURE_BLOCKED_PLATFORM_TEST) {
+    feature.ForceDisable(FeatureStatus::Unavailable,
+                         "Force disabled by gfxInfo", failureId);
   } else if (status != nsIGfxInfo::FEATURE_STATUS_OK) {
     feature.Disable(FeatureStatus::Blocklisted, "Blocklisted by gfxInfo",
                     failureId);
   }
-  if (status != nsIGfxInfo::FEATURE_STATUS_OK && aEnabledByPlatform) {
-    feature.UserForceEnable("Force enabled by pref");
-  }
-  if (!aEnabledByPlatform) {
-    feature.Disable(FeatureStatus::Blocked, "Blocked by platform", failureId);
-  }
+
   if (!gfxVars::UseEGL()) {
     feature.ForceDisable(FeatureStatus::Unavailable, "Requires EGL",
                          "FEATURE_FAILURE_REQUIRES_EGL"_ns);

gfx/thebes/gfxPlatformGtk.h

@@ -68,7 +68,7 @@ class gfxPlatformGtk final : public gfxPlatform {
  protected:
   void InitX11EGLConfig();
   void InitDmabufConfig();
-  bool InitVAAPIConfig(bool aEnabledByPlatform);
+  bool InitVAAPIConfig(bool aForceEnabledByUser);
   void InitPlatformGPUProcessPrefs() override;
   void InitWebRenderConfig() override;
   bool CheckVariationFontSupport() override;

modules/libpref/init/StaticPrefList.yaml

@@ -9385,6 +9385,11 @@
   value: false
   mirror: always
 
+- name: media.hardware-video-decoding.enabled
+  type: bool
+  value: true
+  mirror: once
+
 - name: media.hardware-video-decoding.force-enabled
   type: bool
   value: false

modules/libpref/init/all.js

@@ -256,8 +256,6 @@ pref("media.volume_scale", "1.0");
 // opened as top-level documents, as opposed to inside a media element.
 pref("media.play-stand-alone", true);
 
-pref("media.hardware-video-decoding.enabled", true);
-
 #ifdef MOZ_WMF
   pref("media.wmf.dxva.enabled", true);
   pref("media.wmf.play-stand-alone", true);

toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_Version.js

@@ -161,7 +161,7 @@ async function run_test() {
       failureId
     );
     if (OS == "Linux" && status != Ci.nsIGfxInfo.FEATURE_STATUS_OK) {
-      Assert.equal(status, Ci.nsIGfxInfo.FEATURE_BLOCKED_DEVICE);
+      Assert.equal(status, Ci.nsIGfxInfo.FEATURE_BLOCKED_PLATFORM_TEST);
       Assert.equal(
         failureId.value,
         "FEATURE_FAILURE_VIDEO_DECODING_TEST_FAILED"

widget/gtk/GfxInfo.cpp

@@ -1008,7 +1008,7 @@ nsresult GfxInfo::GetFeatureStatusImpl(
 
   if (aFeature == nsIGfxInfo::FEATURE_HARDWARE_VIDEO_DECODING &&
       !mIsVAAPISupported) {
-    *aStatus = nsIGfxInfo::FEATURE_BLOCKED_DEVICE;
+    *aStatus = nsIGfxInfo::FEATURE_BLOCKED_PLATFORM_TEST;
     aFailureId = "FEATURE_FAILURE_VIDEO_DECODING_TEST_FAILED";
     return NS_OK;
   }

widget/nsIGfxInfo.idl

@@ -207,6 +207,8 @@ interface nsIGfxInfo : nsISupports
   const long FEATURE_ALLOW_ALWAYS = 9;
   /* This feature is safe to be on this device due to the allowlist, depending on qualified/experiment status. */
   const long FEATURE_ALLOW_QUALIFIED = 10;
+  /* This feature failed in a startup test, e.g. due to a crashing driver. */
+  const long FEATURE_BLOCKED_PLATFORM_TEST = 11;
 
   /**
    * Ask about a feature, and return the status of that feature.