Backed out changeset 76e73aad0fab, changeset be03169f5635 and changeset 1aed3d723632 (bug 681238) because of Win debug orange

toolkit/xre/nsWindowsDllInterceptor.h

@@ -196,31 +196,17 @@ protected:
       // Note!  If we ever need to understand jump instructions, we'll
       // need to rewrite the displacement argument.
       if (origBytes[nBytes] >= 0x88 && origBytes[nBytes] <= 0x8B) {
-        // various MOVs
+        // various MOVs; but only handle the case where it truly is a 2-byte instruction
         unsigned char b = origBytes[nBytes+1];
         if (((b & 0xc0) == 0xc0) ||
             (((b & 0xc0) == 0x00) &&
-             ((b & 0x07) != 0x04) && ((b & 0x07) != 0x05)))
+             ((b & 0x38) != 0x20) && ((b & 0x38) != 0x28)))
         {
-          // REG=r, R/M=r or REG=r, R/M=[r]
           nBytes += 2;
-        } else if (((b & 0xc0) == 0x40) && ((b & 0x38) != 0x20)) {
-          // REG=r, R/M=[r + disp8]
-          nBytes += 3;
         } else {
           // complex MOV, bail
           return 0;
         }
-      } else if (origBytes[nBytes] == 0x83) {
-        // ADD|ODR|ADC|SBB|AND|SUB|XOR|CMP r/m, imm8
-        unsigned char b = origBytes[nBytes+1];
-        if ((b & 0xc0) == 0xc0) {
-          // ADD|ODR|ADC|SBB|AND|SUB|XOR|CMP r, imm8
-          nBytes += 3;
-        } else {
-          // bail
-          return 0;
-        }
       } else if (origBytes[nBytes] == 0x68) {
         // PUSH with 4-byte operand
         nBytes += 5;
@@ -271,8 +257,8 @@ protected:
         } else {
           return 0;
         }
-      } else if ((origBytes[nBytes] & 0xfb) == 0x48) {
+      } else if (origBytes[nBytes] == 0x48) {
-        // REX.W | REX.WR
+        // REX.W
         nBytes++;
 
         if (origBytes[nBytes] == 0x81 && (origBytes[nBytes+1] & 0xf8) == 0xe8) {
@@ -286,20 +272,32 @@ protected:
                   (origBytes[nBytes+1] & 0xf8) == 0x60) {
           // and [r+d], imm8
           nBytes += 5;
-        } else if ((origBytes[nBytes] & 0xfd) == 0x89) {
+        } else if (origBytes[nBytes] == 0x89) {
-          // MOV r/m64, r64 | MOV r64, r/m64
+          // MOV r/m64, r64
           if ((origBytes[nBytes+1] & 0xc0) == 0x40) {
             if ((origBytes[nBytes+1] & 0x7) == 0x04) {
-              // R/M=[SIB+disp8], REG=r64
+              // mov [SIB+disp8], r64
               nBytes += 4;
             } else {
-              // R/M=[r64+disp8], REG=r64
+              // mov [r64+disp8], r64
               nBytes += 3;
             }
-          } else if (((origBytes[nBytes+1] & 0xc0) == 0xc0) ||
+          } else {
-                     (((origBytes[nBytes+1] & 0xc0) == 0x00) &&
+            // complex mov
-                      ((origBytes[nBytes+1] & 0x07) != 0x04) && ((origBytes[nBytes+1] & 0x07) != 0x05))) {
+            return 0;
-            // REG=r64, R/M=r64 or REG=r64, R/M=[r64]
+          }
+        } else if (origBytes[nBytes] == 0x8b) {
+          // mov r64, r/m64
+          if ((origBytes[nBytes+1] & 0xc0) == 0x40) {
+            if ((origBytes[nBytes+1] & 0x7) == 0x04) {
+              // mov r64, [SIB+disp8]
+              nBytes += 4;
+            } else {
+              // mov r64, [r64+disp8]
+              nBytes += 3;
+            }
+          } else if ((origBytes[nBytes+1] & 0xc0) == 0xc0) {
+            // MOV r64, r64
             nBytes += 2;
           } else {
             // complex MOV

toolkit/xre/test/win/TestDllInterceptor.cpp

@@ -38,71 +38,47 @@
 #include <stdio.h>
 #include "nsWindowsDllInterceptor.h"
 
-struct payload {
-  UINT64 a;
-  UINT64 b;
-  UINT64 c;
-
-  bool operator==(const payload &other) const {
-    return (a == other.a &&
-            b == other.b &&
-            c == other.c);
-  }
-};
-
-extern "C" __declspec(dllexport,noinline) payload rotatePayload(payload p) {
-  UINT64 tmp = p.a;
-  p.a = p.b;
-  p.b = p.c;
-  p.c = tmp;
-  return p;
-}
-
 static bool patched_func_called = false;
 
-static payload (*orig_rotatePayload)(payload);
+static BOOL (WINAPI *orig_GetVersionExA)(LPOSVERSIONINFO);
 
-static payload
+static BOOL WINAPI
-patched_rotatePayload(payload p)
+patched_GetVersionExA(LPOSVERSIONINFO lpVersionInfo)
 {
   patched_func_called = true;
-  return orig_rotatePayload(p);
+  return orig_GetVersionExA(lpVersionInfo);
 }
 
-bool TestHook(const char *dll, const char *func)
+bool osvi_equal(OSVERSIONINFO &info0, OSVERSIONINFO &info1)
 {
-  void *orig_func;
+  return (info0.dwMajorVersion == info1.dwMajorVersion &&
-  WindowsDllInterceptor TestIntercept;
+          info0.dwMinorVersion == info1.dwMinorVersion &&
-  TestIntercept.Init(dll);
+          info0.dwBuildNumber == info1.dwBuildNumber &&
-  if (TestIntercept.AddHook(func, 0, &orig_func)) {
+          info0.dwPlatformId == info1.dwPlatformId &&
-    printf("TEST-PASS | WindowsDllInterceptor | Could hook %s from %s\n", func, dll);
+          !strncmp(info0.szCSDVersion, info1.szCSDVersion, sizeof(info0.szCSDVersion)));
-    return true;
-  } else {
-    printf("TEST-UNEXPECTED-FAIL | WindowsDllInterceptor | Failed to hook %s from %s\n", func, dll);
-    return false;
-  }
 }
 
 int main()
 {
-  payload initial = { 0x12345678, 0xfc4e9d31, 0x87654321 };
+  OSVERSIONINFO info0, info1;
-  payload p0, p1;
+  ZeroMemory(&info0, sizeof(OSVERSIONINFO));
-  ZeroMemory(&p0, sizeof(p0));
+  ZeroMemory(&info1, sizeof(OSVERSIONINFO));
-  ZeroMemory(&p1, sizeof(p1));
+  info0.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+  info1.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
 
-  p0 = rotatePayload(initial);
+  GetVersionExA(&info0);
 
   {
-    WindowsDllInterceptor ExeIntercept;
+    WindowsDllInterceptor Kernel32Intercept;
-    ExeIntercept.Init("TestDllInterceptor.exe");
+    Kernel32Intercept.Init("kernel32.dll");
-    if (ExeIntercept.AddHook("rotatePayload", reinterpret_cast<intptr_t>(patched_rotatePayload), (void**) &orig_rotatePayload)) {
+    if (Kernel32Intercept.AddHook("GetVersionExA", reinterpret_cast<intptr_t>(patched_GetVersionExA), (void**) &orig_GetVersionExA)) {
       printf("TEST-PASS | WindowsDllInterceptor | Hook added\n");
     } else {
       printf("TEST-UNEXPECTED-FAIL | WindowsDllInterceptor | Failed to add hook\n");
       return 1;
     }
 
-    p1 = rotatePayload(initial);
+    GetVersionExA(&info1);
 
     if (patched_func_called) {
       printf("TEST-PASS | WindowsDllInterceptor | Hook called\n");
@@ -111,7 +87,7 @@ int main()
       return 1;
     }
 
-    if (p0 == p1) {
+    if (osvi_equal(info0, info1)) {
       printf("TEST-PASS | WindowsDllInterceptor | Hook works properly\n");
     } else {
       printf("TEST-UNEXPECTED-FAIL | WindowsDllInterceptor | Hook didn't return the right information\n");
@@ -120,9 +96,8 @@ int main()
   }
 
   patched_func_called = false;
-  ZeroMemory(&p1, sizeof(p1));
 
-  p1 = rotatePayload(initial);
+  GetVersionExA(&info1);
 
   if (!patched_func_called) {
     printf("TEST-PASS | WindowsDllInterceptor | Hook was not called after unregistration\n");
@@ -131,26 +106,13 @@ int main()
     return 1;
   }
 
-  if (p0 == p1) {
+  if (osvi_equal(info0, info1)) {
     printf("TEST-PASS | WindowsDllInterceptor | Original function worked properly\n");
   } else {
     printf("TEST-UNEXPECTED-FAIL | WindowsDllInterceptor | Original function didn't return the right information\n");
     return 1;
   }
 
-  if (TestHook("user32.dll", "GetWindowInfo") &&
-#ifdef _WIN64
-      TestHook("user32.dll", "SetWindowLongPtrA") &&
-      TestHook("user32.dll", "SetWindowLongPtrW") &&
-#else
-      TestHook("user32.dll", "SetWindowLongA") &&
-      TestHook("user32.dll", "SetWindowLongW") &&
-#endif
-      TestHook("user32.dll", "TrackPopupMenu") &&
-      TestHook("ntdll.dll", "LdrLoadDll")) {
   printf("TEST-PASS | WindowsDllInterceptor | all checks passed\n");
   return 0;
 }
-
-  return 1;
-}