From 84655dbdb1f6e97a66d6c31d1b4ddcbed3c20587 Mon Sep 17 00:00:00 2001
From: Martin Stransky <stransky@redhat.com>
Date: Thu, 5 Nov 2020 07:38:26 +0000
Subject: [PATCH] Bug 1660336 Implement FFmpegLibWrapper::LinkVAAPILibs() to
 link VAAPI libraries and use at at FFmpegRuntimeLinker/FFVPXRuntimeLinker,
 r=jya

Differential Revision: https://phabricator.services.mozilla.com/D90556
---
 .../platforms/ffmpeg/FFmpegLibWrapper.cpp     | 49 +++++++++++++++++++
 dom/media/platforms/ffmpeg/FFmpegLibWrapper.h |  1 +
 .../platforms/ffmpeg/FFmpegRuntimeLinker.cpp  | 45 +----------------
 .../ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp       |  4 ++
 4 files changed, 55 insertions(+), 44 deletions(-)

diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp
index 1ecfd9234544..1800bb58088a 100644
--- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.cpp
@@ -11,6 +11,9 @@
 #include "mozilla/Types.h"
 #include "PlatformDecoderModule.h"
 #include "prlink.h"
+#ifdef MOZ_WAYLAND
+#  include "gfxPlatformGtk.h"
+#endif
 
 #define AV_LOG_DEBUG 48
 #define AV_LOG_INFO 32
@@ -253,6 +256,52 @@ void FFmpegLibWrapper::Unlink() {
   PodZero(this);
 }
 
+#ifdef MOZ_WAYLAND
+void FFmpegLibWrapper::LinkVAAPILibs() {
+  if (gfxPlatformGtk::GetPlatform()->UseHardwareVideoDecoding()) {
+    PRLibSpec lspec;
+    lspec.type = PR_LibSpec_Pathname;
+
+    if (gfxPlatformGtk::GetPlatform()->UseDRMVAAPIDisplay()) {
+      const char* libDrm = "libva-drm.so.2";
+      lspec.value.pathname = libDrm;
+      mVALibDrm = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
+      if (!mVALibDrm) {
+        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", libDrm);
+      }
+    } else {
+      if (gfxPlatformGtk::GetPlatform()->IsWaylandDisplay()) {
+        const char* libWayland = "libva-wayland.so.2";
+        lspec.value.pathname = libWayland;
+        mVALibWayland = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
+        if (!mVALibWayland) {
+          FFMPEG_LOG("VA-API support: Missing or old %s library.\n",
+                     libWayland);
+        }
+      } else {
+        FFMPEG_LOG("VA-API X11 display is not implemented.\n");
+      }
+    }
+
+    if (mVALibWayland || mVALibDrm) {
+      const char* lib = "libva.so.2";
+      lspec.value.pathname = lib;
+      mVALib = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
+      // Don't use libva when it's missing vaExportSurfaceHandle.
+      if (mVALib && !PR_FindSymbol(mVALib, "vaExportSurfaceHandle")) {
+        PR_UnloadLibrary(mVALib);
+        mVALib = nullptr;
+      }
+      if (!mVALib) {
+        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", lib);
+      }
+    }
+  } else {
+    FFMPEG_LOG("VA-API FFmpeg is disabled by platform");
+  }
+}
+#endif
+
 #ifdef MOZ_WAYLAND
 bool FFmpegLibWrapper::IsVAAPIAvailable() {
 #  define VA_FUNC_LOADED(func) (func != nullptr)
diff --git a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h
index 2a73ccabbc5e..22dc70f14059 100644
--- a/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h
+++ b/dom/media/platforms/ffmpeg/FFmpegLibWrapper.h
@@ -56,6 +56,7 @@ struct MOZ_ONLY_USED_TO_AVOID_STATIC_CONSTRUCTORS FFmpegLibWrapper {
 #ifdef MOZ_WAYLAND
   // Check if mVALib are available and we can use HW decode.
   bool IsVAAPIAvailable();
+  void LinkVAAPILibs();
 #endif
 
   // indicate the version of libavcodec linked to.
diff --git a/dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp b/dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp
index b32cdcb82ab2..e1514bdc0b4f 100644
--- a/dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegRuntimeLinker.cpp
@@ -57,50 +57,7 @@ bool FFmpegRuntimeLinker::Init() {
   }
 
 #ifdef MOZ_WAYLAND
-  if (gfxPlatformGtk::GetPlatform()->UseHardwareVideoDecoding()) {
-    PRLibSpec lspec;
-    lspec.type = PR_LibSpec_Pathname;
-
-    if (gfxPlatformGtk::GetPlatform()->UseDRMVAAPIDisplay()) {
-      const char* libDrm = "libva-drm.so.2";
-      lspec.value.pathname = libDrm;
-      sLibAV.mVALibDrm =
-          PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
-      if (!sLibAV.mVALibDrm) {
-        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", libDrm);
-      }
-    } else {
-      if (gfxPlatformGtk::GetPlatform()->IsWaylandDisplay()) {
-        const char* libWayland = "libva-wayland.so.2";
-        lspec.value.pathname = libWayland;
-        sLibAV.mVALibWayland =
-            PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
-        if (!sLibAV.mVALibWayland) {
-          FFMPEG_LOG("VA-API support: Missing or old %s library.\n",
-                     libWayland);
-        }
-      } else {
-        FFMPEG_LOG("VA-API X11 display is not implemented.\n");
-      }
-    }
-
-    if (sLibAV.mVALibWayland || sLibAV.mVALibDrm) {
-      const char* lib = "libva.so.2";
-      lspec.value.pathname = lib;
-      sLibAV.mVALib = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL);
-      // Don't use libva when it's missing vaExportSurfaceHandle.
-      if (sLibAV.mVALib &&
-          !PR_FindSymbol(sLibAV.mVALib, "vaExportSurfaceHandle")) {
-        PR_UnloadLibrary(sLibAV.mVALib);
-        sLibAV.mVALib = nullptr;
-      }
-      if (!sLibAV.mVALib) {
-        FFMPEG_LOG("VA-API support: Missing or old %s library.\n", lib);
-      }
-    }
-  } else {
-    FFMPEG_LOG("VA-API FFmpeg is disabled by platform");
-  }
+  sLibAV.LinkVAAPILibs();
 #endif
 
   // While going through all possible libs, this status will be updated with a
diff --git a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp
index ca301cb9db76..43480fcf8409 100644
--- a/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp
+++ b/dom/media/platforms/ffmpeg/ffvpx/FFVPXRuntimeLinker.cpp
@@ -64,6 +64,10 @@ bool FFVPXRuntimeLinker::Init() {
   MOZ_ASSERT(NS_IsMainThread());
   sLinkStatus = LinkStatus_FAILED;
 
+#ifdef MOZ_WAYLAND
+  sFFVPXLib.LinkVAAPILibs();
+#endif
+
   // We retrieve the path of the lgpllibs library as this is where mozavcodec
   // and mozavutil libs are located.
   PathString lgpllibsname = GetLibraryName(nullptr, "lgpllibs");