Handle failures in InstallHooks and fix a random COMException when P/Invoking winrtact_Initialize (#784)

* remove "PreserveSig=true" from the P/Invoke signature of winrtact_Initialize as it returns Void.

* Add error checking for the Detour calls in InstallHooks.
Dllmain now returns False in case an error occured in InstallHooks or ExtRoLoadCatalog

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

Co-authored-by: Scott Jones <scottj1s@gmail.com>

src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

@@ -321,21 +321,23 @@ HRESULT WINAPI RoResolveNamespaceDetour(
     return hr;
 }
 
-void InstallHooks()
+HRESULT InstallHooks()
 {
-    if (DetourIsHelperProcess()) {
-        return;
-    }
+    // If this is loaded in a Detours helper process and not the actual process
+    // to be hooked, just return without performing any other operations.
+    if (DetourIsHelperProcess())
+        return S_OK;
 
     DetourRestoreAfterWith();
 
-    DetourTransactionBegin();
-    DetourUpdateThread(GetCurrentThread());
-    DetourAttach(&(PVOID&)TrueRoActivateInstance, RoActivateInstanceDetour);
-    DetourAttach(&(PVOID&)TrueRoGetActivationFactory, RoGetActivationFactoryDetour);
-    DetourAttach(&(PVOID&)TrueRoGetMetaDataFile, RoGetMetaDataFileDetour);
-    DetourAttach(&(PVOID&)TrueRoResolveNamespace, RoResolveNamespaceDetour);
-    DetourTransactionCommit();
+    RETURN_IF_WIN32_ERROR(DetourTransactionBegin());
+    RETURN_IF_WIN32_ERROR(DetourUpdateThread(GetCurrentThread()));
+    RETURN_IF_WIN32_ERROR(DetourAttach(&(PVOID&)TrueRoActivateInstance, RoActivateInstanceDetour));
+    RETURN_IF_WIN32_ERROR(DetourAttach(&(PVOID&)TrueRoGetActivationFactory, RoGetActivationFactoryDetour));
+    RETURN_IF_WIN32_ERROR(DetourAttach(&(PVOID&)TrueRoGetMetaDataFile, RoGetMetaDataFileDetour));
+    RETURN_IF_WIN32_ERROR(DetourAttach(&(PVOID&)TrueRoResolveNamespace, RoResolveNamespaceDetour));
+    RETURN_IF_WIN32_ERROR(DetourTransactionCommit());
+    return S_OK;
 }
 
 void RemoveHooks()
@@ -443,16 +445,17 @@ HRESULT ExtRoLoadCatalog()
 BOOL WINAPI DllMain(HINSTANCE hmodule, DWORD reason, LPVOID /*lpvReserved*/)
 {
     if (IsWindows1019H1OrGreater())
-    {
         return true;
-    }
+
     if (reason == DLL_PROCESS_ATTACH)
     {
+
         DisableThreadLibraryCalls(hmodule);
-        InstallHooks();
+
         try
         {
-            ExtRoLoadCatalog();
+            if (!SUCCEEDED(InstallHooks()) || !SUCCEEDED(ExtRoLoadCatalog()))
+                return false;
         }
         catch (...)
         {
@@ -460,10 +463,12 @@ BOOL WINAPI DllMain(HINSTANCE hmodule, DWORD reason, LPVOID /*lpvReserved*/)
             return false;
         }
     }
+
     if (reason == DLL_PROCESS_DETACH)
     {
         RemoveHooks();
     }
+
     return true;
 }
 

src/UndockedRegFreeWinRT/mwinrtact/mwinrtact.cs

@@ -4,7 +4,7 @@ namespace Microsoft.Windows
 {
     public static class UndockedRegFreeWinrt
     {
-        [DllImport("winrtact.dll", PreserveSig=true)]
+        [DllImport("winrtact.dll")]
         static extern void winrtact_Initialize();
 
         public static void Initialize()