Bug 1666571 - Part 1. Support more patterns of OpCode 0xFF. r=handyman

This patch optimizes our detour's code handling Opcode 0xFF, expanding
its coverage to INC and DEC reg64 as well as PUSH and CALL.
Testcases for these scenarios are of course included.

Differential Revision: https://phabricator.services.mozilla.com/D91154

mozglue/misc/interceptor/PatcherDetour.h

@@ -1464,36 +1464,38 @@ class WindowsDllDetourPatcher final
           return;
         }
       } else if (*origBytes == 0xff) {
-        if ((origBytes[1] & (kMaskMod | kMaskReg)) == 0xf0) {
-          // push r64
+        uint8_t mod = origBytes[1] & kMaskMod;
+        uint8_t reg = (origBytes[1] & kMaskReg) >> kRegFieldShift;
+        uint8_t rm = origBytes[1] & kMaskRm;
+        if (mod == kModReg && (reg == 0 || reg == 1 || reg == 2 || reg == 6)) {
+          // INC|DEC|CALL|PUSH r64
           COPY_CODES(2);
-        } else if (origBytes[1] == 0x25) {
-          // jmp absolute indirect m32
-          foundJmp = true;
+        } else if (reg == 4) {
+          // FF /4 (Opcode=ff, REG=4): JMP r/m
+          if (mod == kModNoRegDisp && rm == kRmNoRegDispDisp32) {
+            // FF 25    JMP [disp32]
+            foundJmp = true;
 
-          origBytes += 2;
+            origBytes += 2;
 
-          uintptr_t jmpDest = origBytes.ChasePointerFromDisp();
+            uintptr_t jmpDest = origBytes.ChasePointerFromDisp();
 
-          if (!GenerateJump(tramp, jmpDest, JumpType::Jmp)) {
-            return;
+            if (!GenerateJump(tramp, jmpDest, JumpType::Jmp)) {
+              return;
+            }
+          } else {
+            // JMP r/m except JMP [disp32]
+            int len = CountModRmSib(origBytes + 1);
+            if (len < 0) {
+              // RIP-relative not yet supported
+              MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
+              return;
+            }
+
+            COPY_CODES(len + 1);
+
+            foundJmp = true;
           }
-        } else if ((origBytes[1] & (kMaskMod | kMaskReg)) ==
-                   BuildModRmByte(kModReg, 2, 0)) {
-          // CALL reg (ff nn)
-          COPY_CODES(2);
-        } else if (((origBytes[1] & kMaskReg) >> kRegFieldShift) == 4) {
-          // JMP r/m
-          int len = CountModRmSib(origBytes + 1);
-          if (len < 0) {
-            // RIP-relative not yet supported
-            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
-            return;
-          }
-
-          COPY_CODES(len + 1);
-
-          foundJmp = true;
         } else {
           MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;

mozglue/tests/interceptor/AssemblyPayloads.h

@@ -87,6 +87,18 @@ __declspec(dllexport) __attribute__((naked)) void NearJump() {
       :
       : "i"(JumpDestination));
 }
+
+__declspec(dllexport) __attribute__((naked)) void OpcodeFF() {
+  // Skip PUSH (FF /6) because clang prefers Opcode 50+rd
+  // to translate PUSH r64 rather than Opcode FF.
+  asm volatile(
+      "incl %eax;"
+      "decl %ebx;"
+      "call *%rcx;"
+      "jmp *(%rip);"  // Indirect jump to 0xcccccccc`cccccccc
+      "int $3;int $3;int $3;int $3;"
+      "int $3;int $3;int $3;int $3;");
+}
 #  elif defined(_M_IX86)
 constexpr uintptr_t JumpDestination = 0x7fff0000;
 

mozglue/tests/interceptor/TestDllInterceptor.cpp

@@ -730,6 +730,7 @@ struct TestCase {
     // Passing NoStubAddressCheck as the following testcases return
     // a trampoline address instead of the original destination.
     TestCase("NearJump", NoStubAddressCheck),
+    TestCase("OpcodeFF", NoStubAddressCheck),
 #    elif defined(_M_IX86)
     // Skip the stub address check as we always generate a trampoline for x86.
     TestCase("PushRet", NoStubAddressCheck),