fix: Crash on the device that does not support NvEnc API . (#1009)

* check can open encode session

* fix clang format

* ignore missin field initializers

* clang formatted

* fix test compile error

Plugin~/WebRTCPlugin/Codec/CreateVideoCodecFactory.cpp

@@ -54,13 +54,16 @@ namespace webrtc
         if (impl == kNvCodecImpl)
         {
 #if CUDA_PLATFORM
-            if (gfxDevice && gfxDevice->IsCudaSupport() && NvEncoder::IsSupported())
+            if (gfxDevice && gfxDevice->IsCudaSupport())
             {
                 CUcontext context = gfxDevice->GetCUcontext();
-                NV_ENC_BUFFER_FORMAT format = gfxDevice->GetEncodeBufferFormat();
-                std::unique_ptr<VideoEncoderFactory> factory =
-                    std::make_unique<NvEncoderFactory>(context, format, profiler);
-                return CreateSimulcastEncoderFactory(std::move(factory));
+                if (NvEncoder::IsSupported(context))
+                {
+                    NV_ENC_BUFFER_FORMAT format = gfxDevice->GetEncodeBufferFormat();
+                    std::unique_ptr<VideoEncoderFactory> factory =
+                        std::make_unique<NvEncoderFactory>(context, format, profiler);
+                    return CreateSimulcastEncoderFactory(std::move(factory));
+                }
             }
 #endif
         }

Plugin~/WebRTCPlugin/Codec/NvCodec/NvCodec.cpp

@@ -146,15 +146,40 @@ namespace webrtc
         return std::make_unique<NvEncoderImpl>(codec, context, memoryType, format, profiler);
     }
 
-    bool NvEncoder::IsSupported()
+    bool NvEncoder::IsSupported(CUcontext context)
     {
         uint32_t version = 0;
         uint32_t currentVersion = (NVENCAPI_MAJOR_VERSION << 4) | NVENCAPI_MINOR_VERSION;
-        NVENC_API_CALL(NvEncodeAPIGetMaxSupportedVersion(&version));
-        if (currentVersion > version)
+        NVENCSTATUS result = NvEncodeAPIGetMaxSupportedVersion(&version);
+        if (result != NV_ENC_SUCCESS || currentVersion > version)
         {
             return false;
         }
+
+// Check if this device can get the function list of nvencoder API
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+        NV_ENCODE_API_FUNCTION_LIST funclist = { NV_ENCODE_API_FUNCTION_LIST_VER };
+        result = NvEncodeAPICreateInstance(&funclist);
+        if (result != NV_ENC_SUCCESS || funclist.nvEncOpenEncodeSession == nullptr)
+        {
+            return false;
+        }
+
+// Check if this device can open encode session
+#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+        NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS encodeSessionExParams = { NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER };
+        encodeSessionExParams.device = context;
+        encodeSessionExParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA;
+        encodeSessionExParams.apiVersion = NVENCAPI_VERSION;
+        void* hEncoder = nullptr;
+        result = funclist.nvEncOpenEncodeSessionEx(&encodeSessionExParams, &hEncoder);
+        if (result != NV_ENC_SUCCESS || hEncoder == nullptr)
+        {
+            return false;
+        }
+
+        funclist.nvEncDestroyEncoder(hEncoder);
+        hEncoder = nullptr;
         return true;
     }
 

Plugin~/WebRTCPlugin/Codec/NvCodec/NvCodec.h

@@ -35,7 +35,7 @@ namespace webrtc
             CUmemorytype memoryType,
             NV_ENC_BUFFER_FORMAT format,
             ProfilerMarkerFactory* profiler);
-        static bool IsSupported();
+        static bool IsSupported(CUcontext context);
         ~NvEncoder() override { }
     };
 

Plugin~/WebRTCPluginTest/NvCodec/NvCodecTest.cpp

@@ -36,11 +36,12 @@ namespace webrtc
                 GTEST_SKIP() << "The graphics driver is not installed on the device.";
             if (!device_->IsCudaSupport())
                 GTEST_SKIP() << "CUDA is not supported on this device.";
-            if (!NvEncoder::IsSupported())
-                GTEST_SKIP() << "Current Driver Version does not support this NvEncodeAPI version.";
 
             context_ = device_->GetCUcontext();
 
+            if (!NvEncoder::IsSupported(context_))
+                GTEST_SKIP() << "Current Driver Version does not support this NvEncodeAPI version.";
+
             VideoCodecTest::SetUp();
         }