From 952e2e6288d97fc5909db93ab10109ea4bf47dbf Mon Sep 17 00:00:00 2001
From: Eric Detoc <edetoc@users.noreply.github.com>
Date: Mon, 12 Dec 2022 20:41:35 +0100
Subject: [PATCH] Handle failures in InstallHooks and fix a random COMException
 when P/Invoking winrtact_Initialize  (#784)

* remove "PreserveSig=true" from the P/Invoke signature of winrtact_Initialize as it returns Void.

* Add error checking for the Detour calls in InstallHooks.
Dllmain now returns False in case an error occured in InstallHooks or ExtRoLoadCatalog

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

* Update src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp

Co-authored-by: Scott Jones <scottj1s@gmail.com>

Co-authored-by: Scott Jones <scottj1s@gmail.com>
---
 .../UndockedRegFreeWinRT/dllmain.cpp          | 35 +++++++++++--------
 .../mwinrtact/mwinrtact.cs                    |  2 +-
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp b/src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp
index 7f1cef35..1f8861f6 100644
--- a/src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp
+++ b/src/UndockedRegFreeWinRT/UndockedRegFreeWinRT/dllmain.cpp
@@ -321,21 +321,23 @@ HRESULT WINAPI RoResolveNamespaceDetour(
     return hr;
 }
 
-void InstallHooks()
+HRESULT InstallHooks()
 {
-    if (DetourIsHelperProcess()) {
-        return;
-    }
+    // If this is loaded in a Detours helper process and not the actual process
+    // to be hooked, just return without performing any other operations.
+    if (DetourIsHelperProcess())
+        return S_OK;
 
     DetourRestoreAfterWith();
 
-    DetourTransactionBegin();
-    DetourUpdateThread(GetCurrentThread());
-    DetourAttach(&(PVOID&)TrueRoActivateInstance, RoActivateInstanceDetour);
-    DetourAttach(&(PVOID&)TrueRoGetActivationFactory, RoGetActivationFactoryDetour);
-    DetourAttach(&(PVOID&)TrueRoGetMetaDataFile, RoGetMetaDataFileDetour);
-    DetourAttach(&(PVOID&)TrueRoResolveNamespace, RoResolveNamespaceDetour);
-    DetourTransactionCommit();
+    RETURN_IF_WIN32_ERROR(DetourTransactionBegin());
+    RETURN_IF_WIN32_ERROR(DetourUpdateThread(GetCurrentThread()));
+    RETURN_IF_WIN32_ERROR(DetourAttach(&(PVOID&)TrueRoActivateInstance, RoActivateInstanceDetour));
+    RETURN_IF_WIN32_ERROR(DetourAttach(&(PVOID&)TrueRoGetActivationFactory, RoGetActivationFactoryDetour));
+    RETURN_IF_WIN32_ERROR(DetourAttach(&(PVOID&)TrueRoGetMetaDataFile, RoGetMetaDataFileDetour));
+    RETURN_IF_WIN32_ERROR(DetourAttach(&(PVOID&)TrueRoResolveNamespace, RoResolveNamespaceDetour));
+    RETURN_IF_WIN32_ERROR(DetourTransactionCommit());
+    return S_OK;
 }
 
 void RemoveHooks()
@@ -443,16 +445,17 @@ HRESULT ExtRoLoadCatalog()
 BOOL WINAPI DllMain(HINSTANCE hmodule, DWORD reason, LPVOID /*lpvReserved*/)
 {
     if (IsWindows1019H1OrGreater())
-    {
         return true;
-    }
+
     if (reason == DLL_PROCESS_ATTACH)
     {
+
         DisableThreadLibraryCalls(hmodule);
-        InstallHooks();
+
         try
         {
-            ExtRoLoadCatalog();
+            if (!SUCCEEDED(InstallHooks()) || !SUCCEEDED(ExtRoLoadCatalog()))
+                return false;
         }
         catch (...)
         {
@@ -460,10 +463,12 @@ BOOL WINAPI DllMain(HINSTANCE hmodule, DWORD reason, LPVOID /*lpvReserved*/)
             return false;
         }
     }
+
     if (reason == DLL_PROCESS_DETACH)
     {
         RemoveHooks();
     }
+
     return true;
 }
 
diff --git a/src/UndockedRegFreeWinRT/mwinrtact/mwinrtact.cs b/src/UndockedRegFreeWinRT/mwinrtact/mwinrtact.cs
index f392d371..2526cc27 100644
--- a/src/UndockedRegFreeWinRT/mwinrtact/mwinrtact.cs
+++ b/src/UndockedRegFreeWinRT/mwinrtact/mwinrtact.cs
@@ -4,7 +4,7 @@ namespace Microsoft.Windows
 {
     public static class UndockedRegFreeWinrt
     {
-        [DllImport("winrtact.dll", PreserveSig=true)]
+        [DllImport("winrtact.dll")]
         static extern void winrtact_Initialize();
 
         public static void Initialize()