Bug 1615752: Require 10-byte detour for Win 8.0 x64 CreateFileA and DuplicateHandle r=aklotz

In the current Win 8.0, these functions both start with a RIP-relative JMP (6 bytes) followed by 6 nops (6-bytes), which does not give us the 13-bytes we need for a trampoline so we require the trampoline to fit into 10 bytes.

Differential Revision: https://phabricator.services.mozilla.com/D63260

--HG--
extra : moz-landing-system : lando

mozglue/misc/nsWindowsDllInterceptor.h

@@ -430,13 +430,23 @@ class WindowsDllInterceptor final
       // injected DLLs do the same and interfere with our stuff.
       bool needs10BytePatch = (mModule == ::GetModuleHandleW(L"ntdll.dll"));
 
-      // CloseHandle on Windows 8 only accomodates 10-byte patches.
       bool isWin8Or81 = IsWin8OrLater() && (!IsWin10OrLater());
-      needs10BytePatch |= isWin8Or81 &&
-                          (mModule == ::GetModuleHandleW(L"kernel32.dll")) &&
+      bool isWin8 = IsWin8OrLater() && (!IsWin8Point1OrLater());
+
+      bool isKernel32Dll = (mModule == ::GetModuleHandleW(L"kernel32.dll"));
+
+      // CloseHandle on Windows 8/8.1 only accomodates 10-byte patches.
+      needs10BytePatch |= isWin8Or81 && isKernel32Dll &&
                           (reinterpret_cast<void*>(aProc) ==
                            reinterpret_cast<void*>(&CloseHandle));
 
+      // CreateFileA and DuplicateHandle on Windows 8 require 10-byte patches.
+      needs10BytePatch |= isWin8 && isKernel32Dll &&
+                          ((reinterpret_cast<void*>(aProc) ==
+                            reinterpret_cast<void*>(&::CreateFileA)) ||
+                           (reinterpret_cast<void*>(aProc) ==
+                            reinterpret_cast<void*>(&::DuplicateHandle)));
+
       if (needs10BytePatch) {
         flags |= DetourFlags::eEnable10BytePatch;
       }