Bug 681238 part 0 - Support a few more opcodes and fix x86 mov r, [r] test. r=vlad

toolkit/xre/nsWindowsDllInterceptor.h

@@ -188,6 +188,8 @@ protected:
     byteptr_t origBytes = (byteptr_t) origFunction;
 
     int nBytes = 0;
+    int pJmp32 = -1;
+
 #if defined(_M_IX86)
     while (nBytes < 5) {
       // Understand some simple instructions that might be found in a
@@ -196,17 +198,31 @@ protected:
       // Note!  If we ever need to understand jump instructions, we'll
       // need to rewrite the displacement argument.
       if (origBytes[nBytes] >= 0x88 && origBytes[nBytes] <= 0x8B) {
-        // various MOVs; but only handle the case where it truly is a 2-byte instruction
+        // various MOVs
         unsigned char b = origBytes[nBytes+1];
         if (((b & 0xc0) == 0xc0) ||
             (((b & 0xc0) == 0x00) &&
-             ((b & 0x38) != 0x20) && ((b & 0x38) != 0x28)))
+             ((b & 0x07) != 0x04) && ((b & 0x07) != 0x05)))
         {
+          // REG=r, R/M=r or REG=r, R/M=[r]
           nBytes += 2;
+        } else if (((b & 0xc0) == 0x40) && ((b & 0x38) != 0x20)) {
+          // REG=r, R/M=[r + disp8]
+          nBytes += 3;
         } else {
           // complex MOV, bail
           return 0;
         }
+      } else if (origBytes[nBytes] == 0x83) {
+        // ADD|ODR|ADC|SBB|AND|SUB|XOR|CMP r/m, imm8
+        unsigned char b = origBytes[nBytes+1];
+        if ((b & 0xc0) == 0xc0) {
+          // ADD|ODR|ADC|SBB|AND|SUB|XOR|CMP r, imm8
+          nBytes += 3;
+        } else {
+          // bail
+          return 0;
+        }
       } else if (origBytes[nBytes] == 0x68) {
         // PUSH with 4-byte operand
         nBytes += 5;
@@ -216,14 +232,16 @@ protected:
       } else if (origBytes[nBytes] == 0x6A) {
         // PUSH imm8
         nBytes += 2;
+      } else if (origBytes[nBytes] == 0xe9) {
+        pJmp32 = nBytes;
+        // jmp 32bit offset
+        nBytes += 5;
       } else {
         //printf ("Unknown x86 instruction byte 0x%02x, aborting trampoline\n", origBytes[nBytes]);
         return 0;
       }
     }
 #elif defined(_M_X64)
-    int pJmp32 = -1;
-
     while (nBytes < 13) {
 
       // if found JMP 32bit offset, next bytes must be NOP 
@@ -257,8 +275,8 @@ protected:
         } else {
           return 0;
         }
-      } else if (origBytes[nBytes] == 0x48) {
-        // REX.W
+      } else if ((origBytes[nBytes] & 0xfb) == 0x48) {
+        // REX.W | REX.WR
         nBytes++;
 
         if (origBytes[nBytes] == 0x81 && (origBytes[nBytes+1] & 0xf8) == 0xe8) {
@@ -272,32 +290,20 @@ protected:
                   (origBytes[nBytes+1] & 0xf8) == 0x60) {
           // and [r+d], imm8
           nBytes += 5;
-        } else if (origBytes[nBytes] == 0x89) {
-          // MOV r/m64, r64
+        } else if ((origBytes[nBytes] & 0xfd) == 0x89) {
+          // MOV r/m64, r64 | MOV r64, r/m64
           if ((origBytes[nBytes+1] & 0xc0) == 0x40) {
             if ((origBytes[nBytes+1] & 0x7) == 0x04) {
-              // mov [SIB+disp8], r64
+              // R/M=[SIB+disp8], REG=r64
               nBytes += 4;
             } else {
-              // mov [r64+disp8], r64
+              // R/M=[r64+disp8], REG=r64
               nBytes += 3;
             }
-          } else {
-            // complex mov
-            return 0;
-          }
-        } else if (origBytes[nBytes] == 0x8b) {
-          // mov r64, r/m64
-          if ((origBytes[nBytes+1] & 0xc0) == 0x40) {
-            if ((origBytes[nBytes+1] & 0x7) == 0x04) {
-              // mov r64, [SIB+disp8]
-              nBytes += 4;
-            } else {
-              // mov r64, [r64+disp8]
-              nBytes += 3;
-            }
-          } else if ((origBytes[nBytes+1] & 0xc0) == 0xc0) {
-            // MOV r64, r64
+          } else if (((origBytes[nBytes+1] & 0xc0) == 0xc0) ||
+                     (((origBytes[nBytes+1] & 0xc0) == 0x00) &&
+                      ((origBytes[nBytes+1] & 0x07) != 0x04) && ((origBytes[nBytes+1] & 0x07) != 0x05))) {
+            // REG=r64, R/M=r64 or REG=r64, R/M=[r64]
             nBytes += 2;
           } else {
             // complex MOV
@@ -349,8 +355,15 @@ protected:
     byteptr_t trampDest = origBytes + nBytes;
 
 #if defined(_M_IX86)
-    tramp[nBytes] = 0xE9; // jmp
-    *((intptr_t*)(tramp+nBytes+1)) = (intptr_t)trampDest - (intptr_t)(tramp+nBytes+5); // target displacement
+    if (pJmp32 >= 0) {
+      // Jump directly to the original target of the jump instead of jumping to the
+      // original function.
+      // Adjust jump target displacement to jump location in the trampoline.
+      *((intptr_t*)(tramp+pJmp32+1)) += origBytes + pJmp32 - tramp;
+    } else {
+      tramp[nBytes] = 0xE9; // jmp
+      *((intptr_t*)(tramp+nBytes+1)) = (intptr_t)trampDest - (intptr_t)(tramp+nBytes+5); // target displacement
+    }
 #elif defined(_M_X64)
     // If JMP32 opcode found, we don't insert to trampoline jump 
     if (pJmp32 >= 0) {