Bug 1642626 - Part2: Handle JAE and Opcode83 on x64 without REX Prefix. r=handyman

Depends on D81580

Differential Revision: https://phabricator.services.mozilla.com/D81581

mozglue/misc/interceptor/PatcherDetour.h

@@ -627,7 +627,7 @@ class WindowsDllDetourPatcher final
   }
 
 #  if defined(_M_X64)
-  enum class JumpType{Je, Jne, Jmp, Call};
+  enum class JumpType{Je, Jne, Jae, Jmp, Call};
 
   static bool GenerateJump(Trampoline<MMPolicyT>& aTramp,
                            uintptr_t aAbsTargetAddress, const JumpType aType) {
@@ -652,6 +652,10 @@ class WindowsDllDetourPatcher final
       // JE RIP+14
       aTramp.WriteByte(0x74);
       aTramp.WriteByte(14);
+    } else if (aType == JumpType::Jae) {
+      // JAE RIP+14
+      aTramp.WriteByte(0x73);
+      aTramp.WriteByte(14);
     }
 
     // Near jmp, absolute indirect, address given in r/m32
@@ -967,12 +971,12 @@ class WindowsDllDetourPatcher final
         // INC r32
         origBytes += 1;
       } else if (*origBytes == 0x83) {
-        uint8_t mod = static_cast<uint8_t>(origBytes[1]) >> 6;
+        uint8_t mod = static_cast<uint8_t>(origBytes[1]) & kMaskMod;
-        uint8_t rm = static_cast<uint8_t>(origBytes[1]) & 7;
+        uint8_t rm = static_cast<uint8_t>(origBytes[1]) & kMaskRm;
-        if (mod == 3) {
+        if (mod == kModReg) {
           // ADD|OR|ADC|SBB|AND|SUB|XOR|CMP r, imm8
           origBytes += 3;
-        } else if (mod == 1 && rm != 4) {
+        } else if (mod == kModDisp8 && rm != kRmNeedSib) {
           // ADD|OR|ADC|SBB|AND|SUB|XOR|CMP [r+disp8], imm8
           origBytes += 4;
         } else {
@@ -1382,6 +1386,9 @@ class WindowsDllDetourPatcher final
         // bit shifts/rotates : (SA|SH|RO|RC)(R|L) r32
         // (e.g. 0xd1 0xe0 is SAL, 0xd1 0xc8 is ROR)
         COPY_CODES(2);
+      } else if (*origBytes == 0x83 && (origBytes[1] & kMaskMod) == kModReg) {
+        // ADD|OR|ADC|SBB|AND|SUB|XOR|CMP r, imm8
+        COPY_CODES(3);
       } else if (*origBytes == 0xc3) {
         // ret
         COPY_CODES(1);
@@ -1397,13 +1404,14 @@ class WindowsDllDetourPatcher final
                           foundJmp ? JumpType::Jmp : JumpType::Call)) {
           return;
         }
-      } else if (*origBytes == 0x74 ||  // je rel8 (0x74)
+      } else if (*origBytes >= 0x73 && *origBytes <= 0x75) {
-                 *origBytes == 0x75) {  // jne rel8 (0x75)
+        // 73 cb    JAE rel8
+        // 74 cb    JE  rel8
+        // 75 cb    JNE rel8
+        const JumpType kJumpTypes[] = {JumpType::Jae, JumpType::Je,
+                                       JumpType::Jne};
+        auto jumpType = kJumpTypes[*origBytes - 0x73];
         uint8_t offset = origBytes[1];
-        auto jumpType = JumpType::Je;
-        if (*origBytes == 0x75) {
-          jumpType = JumpType::Jne;
-        }
 
         origBytes += 2;