Bug 1284897 - Add missing hooked methods to TestDllInterceptor. r=aklotz

Added ASSERTions to nsWindowsDllInterceptor in case of a failed detour hook, with an exception for the RET opcode that appears in ImmReleaseContext. Added documentation about TestDllInterceptor.

--HG--
extra : rebase_source : a3c6fe0949f5503979a062bdaa5f35526ddee73b

toolkit/xre/test/win/TestDllInterceptor.cpp

@@ -209,15 +209,20 @@ int main()
       TestHook("kernel32.dll", "VirtualAlloc") &&
       TestHook("kernel32.dll", "MapViewOfFile") &&
       TestHook("gdi32.dll", "CreateDIBSection") &&
-      TestHook("kernel32.dll", "CreateFileW") &&
+      TestHook("kernel32.dll", "CreateFileW") &&    // see Bug 1316415
 #endif
+      TestHook("kernel32.dll", "CreateFileA") &&
       TestDetour("user32.dll", "CreateWindowExW") &&
       TestHook("user32.dll", "InSendMessageEx") &&
       TestHook("imm32.dll", "ImmGetContext") &&
+      // TestHook("imm32.dll", "ImmReleaseContext") &&    // see Bug 1316415
       TestHook("imm32.dll", "ImmGetCompositionStringW") &&
       TestHook("imm32.dll", "ImmSetCandidateWindow") &&
+      TestHook("imm32.dll", "ImmNotifyIME") &&
+      TestHook("comdlg32.dll", "GetSaveFileNameW") &&
+      TestHook("comdlg32.dll", "GetOpenFileNameW") &&
 #ifdef _M_X64
-      TestHook("user32.dll", "GetKeyState") &&
+      TestHook("user32.dll", "GetKeyState") &&    // see Bug 1316415
 #endif
       MaybeTestHook(ShouldTestTipTsf(), "tiptsf.dll", "ProcessCaretEvents") &&
 #ifdef _M_IX86

xpcom/build/nsWindowsDllInterceptor.h

@@ -700,6 +700,7 @@ protected:
       if (numPrefixBytes < 0 || (prefixGroups & (ePrefixGroup3 | ePrefixGroup4))) {
         // Either the prefix sequence was bad, or there are prefixes that
         // we don't currently support (groups 3 and 4)
+        MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
         return;
       }
       nOrigBytes += numPrefixBytes;
@@ -709,6 +710,7 @@ protected:
         ++nOrigBytes;
         int len = CountModRmSib(origBytes + nOrigBytes);
         if (len < 0) {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized MOV opcode sequence");
           return;
         }
         nOrigBytes += len;
@@ -733,6 +735,7 @@ protected:
           nOrigBytes += 3;
         } else {
           // bail
+          MOZ_ASSERT_UNREACHABLE("Unrecognized bit opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x68) {
@@ -759,7 +762,8 @@ protected:
 #endif
         return;
       } else {
-        //printf ("Unknown x86 instruction byte 0x%02x, aborting trampoline\n", origBytes[nBytes]);
+        //printf ("Unknown x86 instruction byte 0x%02x, aborting trampoline\n", origBytes[nOrigBytes]);
+        MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
         return;
       }
     }
@@ -786,6 +790,7 @@ protected:
           nOrigBytes++;
           continue;
         }
+        MOZ_ASSERT_UNREACHABLE("Opcode sequence includes commands after JMP");
         return;
       }
       if (origBytes[nOrigBytes] == 0x0f) {
@@ -797,6 +802,7 @@ protected:
               (origBytes[nOrigBytes] & 0x7) == 0x04) {
             COPY_CODES(3);
           } else {
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
         } else if (origBytes[nOrigBytes] == 0x05) {
@@ -811,6 +817,7 @@ protected:
           nTrampBytes = jump.GenerateJump(tramp);
           nOrigBytes += 5;
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x40 ||
@@ -824,6 +831,7 @@ protected:
           // mov r32, imm32
           COPY_CODES(5);
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x45) {
@@ -834,6 +842,7 @@ protected:
           // xor r32, r32
           COPY_CODES(2);
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if ((origBytes[nOrigBytes] & 0xfb) == 0x48) {
@@ -865,6 +874,7 @@ protected:
           if ((origBytes[nOrigBytes + 1] & 0xc0) == 0xc0) {
             COPY_CODES(2);
           } else {
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
         } else if ((origBytes[nOrigBytes] & 0xfd) == 0x89) {
@@ -872,6 +882,7 @@ protected:
           // MOV r/m64, r64 | MOV r64, r/m64
           int len = CountModRmSib(origBytes + nOrigBytes);
           if (len < 0) {
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
           COPY_CODES(len);
@@ -882,6 +893,7 @@ protected:
             // ModR/W + SIB + disp8 + imm32
             COPY_CODES(8);
           } else {
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
         } else if (origBytes[nOrigBytes] == 0xff) {
@@ -899,10 +911,12 @@ protected:
             foundJmp = true;
           } else {
             // not support yet!
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
         } else {
           // not support yet!
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x66) {
@@ -927,6 +941,7 @@ protected:
             }
           } else {
             // complex MOV, bail
+            MOZ_ASSERT_UNREACHABLE("Unrecognized MOV opcode sequence");
             return;
           }
         }
@@ -945,10 +960,12 @@ protected:
           int len = CountModRmSib(origBytes + nOrigBytes);
           if (len < 0) {
             // no way to support this yet.
+            MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
             return;
           }
           COPY_CODES(len);
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else if (origBytes[nOrigBytes] == 0x90) {
@@ -968,6 +985,7 @@ protected:
         int nModRmSibBytes = CountModRmSib(&origBytes[nOrigBytes + 1], &subOpcode);
         if (nModRmSibBytes < 0 || subOpcode != 0) {
           // Unsupported
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
         COPY_CODES(2 + nModRmSibBytes);
@@ -1007,9 +1025,11 @@ protected:
           nTrampBytes = jump.GenerateJump(tramp);
           nOrigBytes += 5;
         } else {
+          MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
           return;
         }
       } else {
+        MOZ_ASSERT_UNREACHABLE("Unrecognized opcode sequence");
         return;
       }
     }
@@ -1145,6 +1165,16 @@ public:
     }
   }
 
+  /**
+   * Hook/detour the method aName from the DLL we set in Init so that it calls
+   * aHookDest instead.  Returns the original method pointer in aOrigFunc
+   * and returns true if successful.
+   *
+   * IMPORTANT: If you use this method, please add your case to the
+   * TestDllInterceptor in order to detect future failures.  Even if this
+   * succeeds now, updates to the hooked DLL could cause it to fail in
+   * the future.
+   */
   bool AddHook(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     // Use a nop space patch if possible, otherwise fall back to a detour.
@@ -1161,6 +1191,16 @@ public:
     return AddDetour(aName, aHookDest, aOrigFunc);
   }
 
+  /**
+   * Detour the method aName from the DLL we set in Init so that it calls
+   * aHookDest instead.  Returns the original method pointer in aOrigFunc
+   * and returns true if successful.
+   *
+   * IMPORTANT: If you use this method, please add your case to the
+   * TestDllInterceptor in order to detect future failures.  Even if this
+   * succeeds now, updates to the detoured DLL could cause it to fail in
+   * the future.
+   */
   bool AddDetour(const char* aName, intptr_t aHookDest, void** aOrigFunc)
   {
     // Generally, code should not call this method directly. Use AddHook unless