Bug 1477490: Merge upstream ASan patch to unpoison thread stacks. r=ted

browser/config/mozconfigs/win64/nightly-asan-reporter

@@ -35,8 +35,4 @@ ac_add_options --disable-sandbox
 # while the build otherwise identifies as "nightly" to receive its updates.
 export MOZ_TELEMETRY_REPORTING=1
 
-# Disable stack instrumentation until we can tackle bug 1477490
-export CFLAGS="-mllvm -asan-stack=0"
-export CXXFLAGS="-mllvm -asan-stack=0"
-
 . "$topsrcdir/build/mozconfig.common.override"

build/build-clang/clang-win64.json

@@ -15,6 +15,8 @@
     "ml": "ml64.exe",
     "patches": [
       "workaround-issue38586.patch",
+      "r342649-hotpatch-8-byte-nops.patch",
+      "r342652-unpoison-thread-stacks.patch",
       "loosen-msvc-detection.patch"
     ]
 }

build/build-clang/r342649-hotpatch-8-byte-nops.patch

@@ -0,0 +1,30 @@
+[winasan] Reduce hotpatch prefix check to 8 bytes
+
+Same idea as r310419: The 8 byte nop is a suffix of the 9 byte nop, and we need at most 6 bytes.
+
+Differential Revision: https://reviews.llvm.org/D51788
+
+--- a/compiler-rt/lib/interception/interception_win.cc	(revision 342648)
++++ b/compiler-rt/lib/interception/interception_win.cc	(revision 342649)
+@@ -223,8 +223,8 @@
+   return true;
+ }
+ 
+-static const u8 kHintNop9Bytes[] = {
+-  0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
++static const u8 kHintNop8Bytes[] = {
++  0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
+ };
+ 
+ template<class T>
+@@ -239,8 +239,8 @@
+ static bool FunctionHasPadding(uptr address, uptr size) {
+   if (IsMemoryPadding(address - size, size))
+     return true;
+-  if (size <= sizeof(kHintNop9Bytes) &&
+-      FunctionHasPrefix(address, kHintNop9Bytes))
++  if (size <= sizeof(kHintNop8Bytes) &&
++      FunctionHasPrefix(address, kHintNop8Bytes))
+     return true;
+   return false;
+ }

build/build-clang/r342652-unpoison-thread-stacks.patch

@@ -0,0 +1,34 @@
+[winasan] Unpoison the stack in NtTerminateThread
+
+In long-running builds we've seen some ASan complaints during thread creation that we suspect are due to leftover poisoning from previous threads whose stacks occupied that memory. This patch adds a hook that unpoisons the stack just before the NtTerminateThread syscall.
+
+Differential Revision: https://reviews.llvm.org/D52091
+
+--- a/compiler-rt/lib/asan/asan_win.cc	(revision 342651)
++++ b/compiler-rt/lib/asan/asan_win.cc	(revision 342652)
+@@ -154,6 +154,14 @@
+                             asan_thread_start, t, thr_flags, tid);
+ }
+ 
++INTERCEPTOR_WINAPI(void, NtTerminateThread, void *rcx) {
++  // Unpoison the terminating thread's stack because the memory may be re-used.
++  NT_TIB *tib = (NT_TIB *)NtCurrentTeb();
++  uptr stackSize = (uptr)tib->StackBase - (uptr)tib->StackLimit;
++  __asan_unpoison_memory_region(tib->StackLimit, stackSize);
++  return REAL(NtTerminateThread(rcx));
++}
++
+ // }}}
+ 
+ namespace __asan {
+@@ -161,7 +169,9 @@
+ void InitializePlatformInterceptors() {
+   ASAN_INTERCEPT_FUNC(CreateThread);
+   ASAN_INTERCEPT_FUNC(SetUnhandledExceptionFilter);
+-
++  CHECK(::__interception::OverrideFunction("NtTerminateThread",
++                                           (uptr)WRAP(NtTerminateThread),
++                                           (uptr *)&REAL(NtTerminateThread)));
+ #ifdef _WIN64
+   ASAN_INTERCEPT_FUNC(__C_specific_handler);
+ #else