Bug 1240848: Adds additional instructions to x64 detour patcher and prevents register clobbering in jmp from trampoline; r=ehsan

MozReview-Commit-ID: 7DCQZc9eoQI

toolkit/xre/test/win/TestDllInterceptor.cpp

@@ -159,6 +159,7 @@ int main()
       TestHook("gdi32.dll", "CreateDIBSection") &&
       TestHook("kernel32.dll", "CreateFileW") &&
 #endif
+      TestDetour("user32.dll", "CreateWindowExW") &&
       TestHook("imm32.dll", "ImmGetContext") &&
       TestHook("imm32.dll", "ImmGetCompositionStringW") &&
       TestHook("imm32.dll", "ImmSetCandidateWindow") &&

xpcom/build/nsWindowsDllInterceptor.h

@@ -707,6 +707,31 @@ protected:
           // not support yet!
           return;
         }
+      } else if (origBytes[nBytes] == 0x66) {
+        // operand override prefix
+        nBytes += 1;
+        // This is the same as the x86 version
+        if (origBytes[nBytes] >= 0x88 && origBytes[nBytes] <= 0x8B) {
+          // various MOVs
+          unsigned char b = origBytes[nBytes + 1];
+          if (((b & 0xc0) == 0xc0) ||
+              (((b & 0xc0) == 0x00) &&
+               ((b & 0x07) != 0x04) && ((b & 0x07) != 0x05))) {
+            // REG=r, R/M=r or REG=r, R/M=[r]
+            nBytes += 2;
+          } else if ((b & 0xc0) == 0x40) {
+            if ((b & 0x07) == 0x04) {
+              // REG=r, R/M=[SIB + disp8]
+              nBytes += 4;
+            } else {
+              // REG=r, R/M=[r + disp8]
+              nBytes += 3;
+            }
+          } else {
+            // complex MOV, bail
+            return;
+          }
+        }
       } else if ((origBytes[nBytes] & 0xf0) == 0x50) {
         // 1-byte push/pop
         nBytes++;
@@ -734,6 +759,9 @@ protected:
       } else if (origBytes[nBytes] == 0xb8) {
         // MOV 0xB8: http://ref.x86asm.net/coder32.html#xB8
         nBytes += 5;
+      } else if (origBytes[nBytes] == 0x33) {
+        // xor r32, r/m32
+        nBytes += 2;
       } else if (origBytes[nBytes] == 0xf6) {
         // test r/m8, imm8 (used by ntdll on Windows 10 x64)
         // (no flags are affected by near jmp since there is no task switch,