Bug 1596930 - Add new patterns to detour. r=handyman

This patch adds the following pattern to our x64 detour so that we can hook APIs
even though a target is already detoured by another application.

```
mov   rax, imm64
push  rax
ret
```

We already have `PatchIfTargetIsRecognizedTrampoline` to detour the pattern
`mov; jmp`.  There is another variation using `push rax;ret` to jump.

Differential Revision: https://phabricator.services.mozilla.com/D53877

--HG--
extra : moz-landing-system : lando

mozglue/misc/interceptor/PatcherDetour.h

@@ -669,10 +669,17 @@ class WindowsDllDetourPatcher final : public WindowsDllPatcherBase<VMPolicy> {
       ReadOnlyTargetFunction<MMPolicyT>& aOriginalFn, intptr_t aDest,
       void** aOutTramp) {
 #if defined(_M_X64)
+    // Variation 1:
     // 48 b8 imm64  mov rax, imm64
     // ff e0        jmp rax
+    //
+    // Variation 2:
+    // 48 b8 imm64  mov rax, imm64
+    // 50           push rax
+    // c3           ret
     if ((aOriginalFn[0] == 0x48) && (aOriginalFn[1] == 0xB8) &&
-        (aOriginalFn[10] == 0xFF) && (aOriginalFn[11] == 0xE0)) {
+        ((aOriginalFn[10] == 0xFF && aOriginalFn[11] == 0xE0) ||
+         (aOriginalFn[10] == 0x50 && aOriginalFn[11] == 0xC3))) {
       uintptr_t originalTarget =
           (aOriginalFn + 2).template ChasePointer<uintptr_t>();
 

mozglue/tests/interceptor/AssemblyPayloads.h

@@ -0,0 +1,59 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+/* These assembly functions represent patterns that were already hooked by
+ * another application before our detour.
+ */
+
+#ifndef mozilla_AssemblyPayloads_h
+#define mozilla_AssemblyPayloads_h
+
+extern "C" {
+
+#if defined(__clang__)
+#  if defined(_M_X64)
+constexpr uintptr_t JumpDestination = 0x7fff00000000;
+
+__declspec(dllexport) __attribute__((naked)) void MovPushRet() {
+  asm volatile(
+      "mov %0, %%rax;"
+      "push %%rax;"
+      "ret;"
+      :
+      : "i"(JumpDestination));
+}
+
+__declspec(dllexport) __attribute__((naked)) void MovRaxJump() {
+  asm volatile(
+      "mov %0, %%rax;"
+      "jmpq *%%rax;"
+      :
+      : "i"(JumpDestination));
+}
+#  elif defined(_M_IX86)
+constexpr uintptr_t JumpDestination = 0x7fff0000;
+
+__declspec(dllexport) __attribute__((naked)) void PushRet() {
+  asm volatile(
+      "push %0;"
+      "ret;"
+      :
+      : "i"(JumpDestination));
+}
+
+__declspec(dllexport) __attribute__((naked)) void MovEaxJump() {
+  asm volatile(
+      "mov %0, %%eax;"
+      "jmp *%%eax;"
+      :
+      : "i"(JumpDestination));
+}
+#  endif
+#endif  // defined(__clang__)
+
+}  // extern "C"
+
+#endif  // mozilla_AssemblyPayloads_h

mozglue/tests/interceptor/TestDllInterceptor.cpp

@@ -11,6 +11,7 @@
 #include <schnlsp.h>
 #include <winternl.h>
 
+#include "AssemblyPayloads.h"
 #include "mozilla/DynamicallyLinkedFunctionPtr.h"
 #include "mozilla/TypeTraits.h"
 #include "mozilla/UniquePtr.h"
@@ -691,6 +692,75 @@ bool TestShortDetour() {
 #endif
 }
 
+template <typename InterceptorType>
+bool TestAssemblyFunctions() {
+  constexpr uintptr_t NoStubAddressCheck = 0;
+  struct TestCase {
+    const char* functionName;
+    uintptr_t expectedStub;
+    explicit TestCase(const char* aFunctionName, uintptr_t aExpectedStub)
+        : functionName(aFunctionName), expectedStub(aExpectedStub) {}
+  } testCases[] = {
+#if defined(__clang__)
+#  if defined(_M_X64)
+    // Since we have PatchIfTargetIsRecognizedTrampoline for x64, we expect the
+    // original jump destination is returned as a stub.
+    TestCase("MovPushRet", JumpDestination),
+    TestCase("MovRaxJump", JumpDestination),
+#  elif defined(_M_IX86)
+    // Skip the stub address check as we always generate a trampoline for x86.
+    TestCase("PushRet", NoStubAddressCheck),
+    TestCase("MovEaxJump", NoStubAddressCheck),
+#  endif
+#endif
+  };
+
+  static const auto patchedFunction = []() { patched_func_called = true; };
+
+  InterceptorType interceptor;
+  interceptor.Init("TestDllInterceptor.exe");
+
+  for (const auto& testCase : testCases) {
+    typename InterceptorType::template FuncHookType<void (*)()> hook;
+    bool result = hook.Set(interceptor, testCase.functionName, patchedFunction);
+    if (!result) {
+      printf(
+          "TEST-FAILED | WindowsDllInterceptor | "
+          "Failed to detour %s.\n",
+          testCase.functionName);
+      return false;
+    }
+
+    const auto actualStub = reinterpret_cast<uintptr_t>(hook.GetStub());
+    if (testCase.expectedStub != NoStubAddressCheck &&
+        actualStub != testCase.expectedStub) {
+      printf(
+          "TEST-FAILED | WindowsDllInterceptor | "
+          "Wrong stub was backed up for %s: %zx\n",
+          testCase.functionName, actualStub);
+      return false;
+    }
+
+    patched_func_called = false;
+
+    auto originalFunction = reinterpret_cast<void (*)()>(
+        GetProcAddress(GetModuleHandle(nullptr), testCase.functionName));
+    originalFunction();
+
+    if (!patched_func_called) {
+      printf(
+          "TEST-FAILED | WindowsDllInterceptor | "
+          "Hook from %s was not called\n",
+          testCase.functionName);
+      return false;
+    }
+
+    printf("TEST-PASS | WindowsDllInterceptor | %s\n", testCase.functionName);
+  }
+
+  return true;
+}
+
 extern "C" int wmain(int argc, wchar_t* argv[]) {
   LARGE_INTEGER start;
   QueryPerformanceCounter(&start);
@@ -787,6 +857,12 @@ extern "C" int wmain(int argc, wchar_t* argv[]) {
   // NB: These tests should be ordered such that lower-level APIs are tested
   // before higher-level APIs.
   if (TestShortDetour() &&
+  // Run <ShortInterceptor> first because <WindowsDllInterceptor>
+  // does not clean up hooks.
+#if defined(_M_X64)
+      TestAssemblyFunctions<ShortInterceptor>() &&
+#endif
+      TestAssemblyFunctions<WindowsDllInterceptor>() &&
 #ifdef _M_IX86
       // We keep this test to hook complex code on x86. (Bug 850957)
       TEST_HOOK("ntdll.dll", NtFlushBuffersFile, NotEquals, 0) &&