Bug 1751710 [Linux] Allow VA-API decoding without GBM device and switch MOZ_WAYLAND_DRM_DEVICE to MOZ_DRM_DEVICE r=emilio,alwu,media-playback-reviewers

- Allow to use VA-API without GBM device as we need DRM device file descriptor only.
- Rename MOZ_WAYLAND_DRM_DEVICE to MOZ_DRM_DEVICE as it's used for both X11 and Wayland.
- Use gbm_device_destroy() to close opened GBM device on application quit.

Differential Revision: https://phabricator.services.mozilla.com/D136748

dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp

@@ -76,14 +76,16 @@ MediaResult FFmpegDataDecoder<LIBAV_VER>::InitDecoder() {
 
   AVCodec* codec = FindAVCodec(mLib, mCodecID);
   if (!codec) {
+    FFMPEG_LOG("  unable to find codec");
     return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
-                       RESULT_DETAIL("Couldn't find ffmpeg decoder"));
+                       RESULT_DETAIL("unable to find codec"));
   }
   FFMPEG_LOG("  codec %s : %s", codec->name, codec->long_name);
 
   StaticMutexAutoLock mon(sMutex);
 
   if (!(mCodecContext = mLib->avcodec_alloc_context3(codec))) {
+    FFMPEG_LOG("  couldn't init ffmpeg context");
     return MediaResult(NS_ERROR_OUT_OF_MEMORY,
                        RESULT_DETAIL("Couldn't init ffmpeg context"));
   }
@@ -100,6 +102,7 @@ MediaResult FFmpegDataDecoder<LIBAV_VER>::InitDecoder() {
   InitCodecContext();
   MediaResult ret = AllocateExtraData();
   if (NS_FAILED(ret)) {
+    FFMPEG_LOG("  failed to allocate extra data");
     mLib->av_freep(&mCodecContext);
     return ret;
   }
@@ -112,8 +115,9 @@ MediaResult FFmpegDataDecoder<LIBAV_VER>::InitDecoder() {
 
   if (mLib->avcodec_open2(mCodecContext, codec, nullptr) < 0) {
     mLib->av_freep(&mCodecContext);
+    FFMPEG_LOG("  Couldn't open avcodec");
     return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
-                       RESULT_DETAIL("Couldn't initialise ffmpeg decoder"));
+                       RESULT_DETAIL("Couldn't open avcodec"));
   }
 
   FFMPEG_LOG("  FFmpeg decoder init successful.");

dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp

@@ -200,8 +200,7 @@ bool FFmpegVideoDecoder<LIBAV_VER>::CreateVAAPIDeviceContext() {
   AVVAAPIDeviceContext* vactx = (AVVAAPIDeviceContext*)hwctx->hwctx;
 
   if (StaticPrefs::media_ffmpeg_vaapi_drm_display_enabled()) {
-    mDisplay =
-        mLib->vaGetDisplayDRM(widget::GetDMABufDevice()->GetGbmDeviceFd());
+    mDisplay = mLib->vaGetDisplayDRM(widget::GetDMABufDevice()->GetDRMFd());
     if (!mDisplay) {
       FFMPEG_LOG("  Can't get DRM VA-API display.");
       return false;

widget/gtk/DMABufLibWrapper.cpp

@@ -32,6 +32,7 @@ void* nsGbmLib::sGbmLibHandle = nullptr;
 void* nsGbmLib::sXf86DrmLibHandle = nullptr;
 bool nsGbmLib::sLibLoaded = false;
 CreateDeviceFunc nsGbmLib::sCreateDevice;
+DestroyDeviceFunc nsGbmLib::sDestroyDevice;
 CreateFunc nsGbmLib::sCreate;
 CreateWithModifiersFunc nsGbmLib::sCreateWithModifiers;
 GetModifierFunc nsGbmLib::sGetModifier;
@@ -48,10 +49,11 @@ DeviceIsFormatSupportedFunc nsGbmLib::sDeviceIsFormatSupported;
 DrmPrimeHandleToFDFunc nsGbmLib::sDrmPrimeHandleToFD;
 
 bool nsGbmLib::IsLoaded() {
-  return sCreateDevice != nullptr && sCreate != nullptr &&
-         sCreateWithModifiers != nullptr && sGetModifier != nullptr &&
-         sGetStride != nullptr && sGetFd != nullptr && sDestroy != nullptr &&
-         sMap != nullptr && sUnmap != nullptr && sGetPlaneCount != nullptr &&
+  return sCreateDevice != nullptr && sDestroyDevice != nullptr &&
+         sCreate != nullptr && sCreateWithModifiers != nullptr &&
+         sGetModifier != nullptr && sGetStride != nullptr &&
+         sGetFd != nullptr && sDestroy != nullptr && sMap != nullptr &&
+         sUnmap != nullptr && sGetPlaneCount != nullptr &&
          sGetHandleForPlane != nullptr && sGetStrideForPlane != nullptr &&
          sGetOffset != nullptr && sDeviceIsFormatSupported != nullptr &&
          sDrmPrimeHandleToFD != nullptr;
@@ -76,6 +78,8 @@ bool nsGbmLib::Load() {
     }
 
     sCreateDevice = (CreateDeviceFunc)dlsym(sGbmLibHandle, "gbm_create_device");
+    sDestroyDevice =
+        (DestroyDeviceFunc)dlsym(sGbmLibHandle, "gbm_device_destroy");
     sCreate = (CreateFunc)dlsym(sGbmLibHandle, "gbm_bo_create");
     sCreateWithModifiers = (CreateWithModifiersFunc)dlsym(
         sGbmLibHandle, "gbm_bo_create_with_modifiers");
@@ -114,8 +118,6 @@ gbm_device* nsDMABufDevice::GetGbmDevice() {
   return IsDMABufEnabled() ? mGbmDevice : nullptr;
 }
 
-int nsDMABufDevice::GetGbmDeviceFd() { return IsDMABufEnabled() ? mGbmFd : -1; }
-
 static void dmabuf_modifiers(void* data,
                              struct zwp_linux_dmabuf_v1* zwp_linux_dmabuf,
                              uint32_t format, uint32_t modifier_hi,
@@ -173,8 +175,8 @@ nsDMABufDevice::nsDMABufDevice()
     : mUseWebGLDmabufBackend(true),
       mXRGBFormat({true, false, GBM_FORMAT_XRGB8888, nullptr, 0}),
       mARGBFormat({true, true, GBM_FORMAT_ARGB8888, nullptr, 0}),
+      mDRMFd(-1),
       mGbmDevice(nullptr),
-      mGbmFd(-1),
       mInitialized(false) {
   if (GdkIsWaylandDisplay()) {
     wl_display* display = WaylandDisplayGetWLDisplay();
@@ -184,8 +186,37 @@ nsDMABufDevice::nsDMABufDevice()
     wl_display_roundtrip(display);
     wl_registry_destroy(registry);
   }
+
+  nsAutoCString drm_render_node(getenv("MOZ_DRM_DEVICE"));
+  if (drm_render_node.IsEmpty()) {
+    drm_render_node.Assign(gfx::gfxVars::DrmRenderDevice());
   }
 
+  if (!drm_render_node.IsEmpty()) {
+    LOGDMABUF(("Using DRM device %s", drm_render_node.get()));
+    mDRMFd = open(drm_render_node.get(), O_RDWR);
+    if (mDRMFd < 0) {
+      LOGDMABUF(("Failed to open drm render node %s error %s\n",
+                 drm_render_node.get(), strerror(errno)));
+    }
+  } else {
+    LOGDMABUF(("We're missing DRM render device!\n"));
+  }
+}
+
+nsDMABufDevice::~nsDMABufDevice() {
+  if (mGbmDevice) {
+    nsGbmLib::DestroyDevice(mGbmDevice);
+    mGbmDevice = nullptr;
+  }
+  if (mDRMFd != -1) {
+    close(mDRMFd);
+    mDRMFd = -1;
+  }
+}
+
+int nsDMABufDevice::GetDRMFd() { return mDRMFd; }
+
 bool nsDMABufDevice::Configure(nsACString& aFailureId) {
   LOGDMABUF(("nsDMABufDevice::Configure()"));
 
@@ -196,9 +227,7 @@ bool nsDMABufDevice::Configure(nsACString& aFailureId) {
 #ifdef NIGHTLY_BUILD
       StaticPrefs::widget_dmabuf_textures_enabled() ||
 #endif
-      StaticPrefs::widget_dmabuf_webgl_enabled() ||
-      StaticPrefs::media_ffmpeg_vaapi_enabled() ||
-      StaticPrefs::media_ffmpeg_vaapi_drm_display_enabled());
+      StaticPrefs::widget_dmabuf_webgl_enabled());
 
   if (!isDMABufUsed) {
     // Disabled by user, just quit.
@@ -213,36 +242,16 @@ bool nsDMABufDevice::Configure(nsACString& aFailureId) {
     return false;
   }
 
-  nsAutoCString drm_render_node(getenv("MOZ_WAYLAND_DRM_DEVICE"));
-  if (drm_render_node.IsEmpty()) {
-    drm_render_node.Assign(gfx::gfxVars::DrmRenderDevice());
-    if (drm_render_node.IsEmpty()) {
-      LOGDMABUF(("Failed: We're missing DRM render device!\n"));
-      aFailureId = "FEATURE_FAILURE_NO_DRM_RENDER_NODE";
-      return false;
-    }
-  }
-
-  mGbmFd = open(drm_render_node.get(), O_RDWR);
-  if (mGbmFd < 0) {
-    const char* error = strerror(errno);
-    LOGDMABUF(("Failed to open drm render node %s error %s\n",
-               drm_render_node.get(), error));
+  // fd passed to gbm_create_device() should be kept open until
+  // gbm_device_destroy() is called.
+  mGbmDevice = nsGbmLib::CreateDevice(GetDRMFd());
+  if (!mGbmDevice) {
+    LOGDMABUF(("Failed to create drm render device"));
     aFailureId = "FEATURE_FAILURE_BAD_DRM_RENDER_NODE";
     return false;
   }
 
-  mGbmDevice = nsGbmLib::CreateDevice(mGbmFd);
-  if (!mGbmDevice) {
-    LOGDMABUF(
-        ("Failed to create drm render device %s\n", drm_render_node.get()));
-    aFailureId = "FEATURE_FAILURE_NO_DRM_RENDER_DEVICE";
-    close(mGbmFd);
-    mGbmFd = -1;
-    return false;
-  }
-
-  LOGDMABUF(("DMABuf is enabled, using drm node %s", drm_render_node.get()));
+  LOGDMABUF(("DMABuf is enabled"));
   return true;
 }
 
@@ -274,14 +283,12 @@ bool nsDMABufDevice::IsDMABufVideoEnabled() {
 }
 bool nsDMABufDevice::IsDMABufVAAPIEnabled() {
   LOGDMABUF(
-      ("nsDMABufDevice::IsDMABufVAAPIEnabled: EGL %d DMABufEnabled %d  "
+      ("nsDMABufDevice::IsDMABufVAAPIEnabled: EGL %d "
        "media_ffmpeg_vaapi_enabled %d CanUseHardwareVideoDecoding %d "
        "XRE_IsRDDProcess %d\n",
-       gfx::gfxVars::UseEGL(), IsDMABufEnabled(),
-       StaticPrefs::media_ffmpeg_vaapi_enabled(),
+       gfx::gfxVars::UseEGL(), StaticPrefs::media_ffmpeg_vaapi_enabled(),
        gfx::gfxVars::CanUseHardwareVideoDecoding(), XRE_IsRDDProcess()));
   return StaticPrefs::media_ffmpeg_vaapi_enabled() && XRE_IsRDDProcess() &&
-         gfx::gfxVars::UseDMABuf() && IsDMABufEnabled() &&
          gfx::gfxVars::CanUseHardwareVideoDecoding();
 }
 bool nsDMABufDevice::IsDMABufWebGLEnabled() {

widget/gtk/DMABufLibWrapper.h

@@ -29,6 +29,7 @@ namespace mozilla {
 namespace widget {
 
 typedef struct gbm_device* (*CreateDeviceFunc)(int);
+typedef void (*DestroyDeviceFunc)(struct gbm_device*);
 typedef struct gbm_bo* (*CreateFunc)(struct gbm_device*, uint32_t, uint32_t,
                                      uint32_t, uint32_t);
 typedef struct gbm_bo* (*CreateWithModifiersFunc)(struct gbm_device*, uint32_t,
@@ -61,6 +62,10 @@ class nsGbmLib {
     StaticMutexAutoLock lockDRI(sDRILock);
     return sCreateDevice(fd);
   };
+  static void DestroyDevice(struct gbm_device* gdm) {
+    StaticMutexAutoLock lockDRI(sDRILock);
+    return sDestroyDevice(gdm);
+  };
   static struct gbm_bo* Create(struct gbm_device* gbm, uint32_t width,
                                uint32_t height, uint32_t format,
                                uint32_t flags) {
@@ -130,6 +135,7 @@ class nsGbmLib {
 
  private:
   static CreateDeviceFunc sCreateDevice;
+  static DestroyDeviceFunc sDestroyDevice;
   static CreateFunc sCreate;
   static CreateWithModifiersFunc sCreateWithModifiers;
   static GetModifierFunc sGetModifier;
@@ -162,10 +168,9 @@ struct GbmFormat {
 class nsDMABufDevice {
  public:
   nsDMABufDevice();
+  ~nsDMABufDevice();
 
   gbm_device* GetGbmDevice();
-  // Returns -1 if we fails to gbm device file descriptor.
-  int GetGbmDeviceFd();
 
   // Use dmabuf for WebRender general web content
   bool IsDMABufTexturesEnabled();
@@ -177,6 +182,7 @@ class nsDMABufDevice {
   bool IsDMABufWebGLEnabled();
   void DisableDMABufWebGL();
 
+  int GetDRMFd();
   GbmFormat* GetGbmFormat(bool aHasAlpha);
   GbmFormat* GetExactGbmFormat(int aFormat);
   void ResetFormatsModifiers();
@@ -193,8 +199,8 @@ class nsDMABufDevice {
   GbmFormat mXRGBFormat;
   GbmFormat mARGBFormat;
 
+  int mDRMFd;
   gbm_device* mGbmDevice;
-  int mGbmFd;
   bool mInitialized;
 };
 

widget/gtk/DMABufSurface.cpp

@@ -312,7 +312,7 @@ bool DMABufSurfaceRGBA::OpenFileDescriptorForPlane(
   } else {
     uint32_t handle =
         nsGbmLib::GetHandleForPlane(mGbmBufferObject[0], aPlane).u32;
-    int ret = nsGbmLib::DrmPrimeHandleToFD(GetDMABufDevice()->GetGbmDeviceFd(),
+    int ret = nsGbmLib::DrmPrimeHandleToFD(GetDMABufDevice()->GetDRMFd(),
                                            handle, 0, &mDmabufFds[aPlane]);
     if (ret < 0) {
       mDmabufFds[aPlane] = -1;