Bug 1474488: add --enable-undefined-sanitizer with custom checks r=froydnj

build/autoconf/frameptr.m4

@@ -37,6 +37,7 @@ AC_DEFUN([MOZ_SET_FRAMEPTR_FLAGS], [
           -n "$MOZ_DEBUG" -o \
           -n "$MOZ_MSAN" -o \
           -n "$MOZ_ASAN" -o \
+          -n "$MOZ_UBSAN" -o \
           "$OS_ARCH:$CPU_ARCH" = "WINNT:x86" -o \
 	  "$OS_ARCH:$CPU_ARCH" = "WINNT:aarch64"; then
     MOZ_FRAMEPTR_FLAGS="$MOZ_ENABLE_FRAME_PTR"

build/autoconf/sanitize.m4

@@ -73,6 +73,24 @@ if test -n "$MOZ_TSAN"; then
 fi
 AC_SUBST(MOZ_TSAN)
 
+dnl ========================================================
+dnl = Use UndefinedBehavior Sanitizer (with custom checks)
+dnl ========================================================
+if test -n "$MOZ_UBSAN_CHECKS"; then
+    MOZ_UBSAN=1
+    UBSAN_TXT="$_objdir/ubsan_blacklist.txt"
+    cat $_topsrcdir/build/sanitizers/ubsan_*_blacklist.txt > $UBSAN_TXT
+    UBSAN_FLAGS="-fsanitize=$MOZ_UBSAN_CHECKS -fno-sanitize-recover=$MOZ_UBSAN_CHECKS -fsanitize-blacklist=$UBSAN_TXT"
+    CFLAGS="$UBSAN_FLAGS $CFLAGS"
+    CXXFLAGS="$UBSAN_FLAGS $CXXFLAGS"
+    if test -z "$CLANG_CL"; then
+        LDFLAGS="-fsanitize=undefined -rdynamic $LDFLAGS"
+    fi
+    AC_DEFINE(MOZ_UBSAN)
+    MOZ_PATH_PROG(LLVM_SYMBOLIZER, llvm-symbolizer)
+fi
+AC_SUBST(MOZ_UBSAN)
+
 dnl ========================================================
 dnl = Use UndefinedBehavior Sanitizer to find integer overflows
 dnl ========================================================

build/moz.configure/toolchain.configure

@@ -1464,6 +1464,26 @@ def asan(value, _):
 
 add_old_configure_assignment('MOZ_ASAN', asan)
 
+# UBSAN
+# ==============================================================
+
+js_option('--enable-undefined-sanitizer',
+          nargs='*',
+          help='Enable UndefinedBehavior Sanitizer')
+
+@depends_if('--enable-undefined-sanitizer')
+def ubsan(options):
+    default_checks = [
+        'bool',
+        'bounds',
+        'vla-bound',
+    ]
+
+    checks = options if len(options) else default_checks
+
+    return ','.join(checks)
+
+add_old_configure_assignment('MOZ_UBSAN_CHECKS', ubsan)
 
 # Security Hardening
 # ==============================================================

config/rules.mk

@@ -913,6 +913,7 @@ endif
 
 ifndef MOZ_ASAN
 ifndef MOZ_TSAN
+ifndef MOZ_UBSAN
 ifndef MOZ_CODE_COVERAGE
 # Pass the compilers and flags in use to cargo for use in build scripts.
 # * Don't do this for ASAN/TSAN builds because we don't pass our custom linker (see below)
@@ -946,6 +947,7 @@ cargo_c_compiler_envs := \
  $(NULL)
 endif # WINNT
 endif # MOZ_CODE_COVERAGE
+endif # MOZ_UBSAN
 endif # MOZ_TSAN
 endif # MOZ_ASAN
 
@@ -1008,6 +1010,7 @@ ifneq (WINNT,$(OS_ARCH))
 # some crates's build scripts (!), so disable it for now.
 ifndef MOZ_ASAN
 ifndef MOZ_TSAN
+ifndef MOZ_UBSAN
 # Cargo needs the same linker flags as the C/C++ compiler,
 # but not the final libraries. Filter those out because they
 # cause problems on macOS 10.7; see bug 1365993 for details.
@@ -1016,6 +1019,7 @@ target_cargo_env_vars := \
 	MOZ_CARGO_WRAP_LDFLAGS="$(filter-out -fsanitize=cfi% -framework Cocoa -lobjc AudioToolbox ExceptionHandling -fprofile-%,$(LDFLAGS))" \
 	MOZ_CARGO_WRAP_LD="$(CC)" \
 	$(cargo_linker_env_var)=$(topsrcdir)/build/cargo-linker
+endif # MOZ_UBSAN
 endif # MOZ_TSAN
 endif # MOZ_ASAN
 

mozglue/build/UbsanOptions.cpp

@@ -0,0 +1,14 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Attributes.h"
+
+#ifndef _MSC_VER // Not supported by clang-cl yet
+
+const char* __ubsan_default_options() {
+    return "print_stacktrace=1";
+}
+
+#endif

mozglue/build/moz.build

@@ -24,6 +24,11 @@ if CONFIG['MOZ_ASAN']:
         'AsanOptions.cpp',
     ]
 
+if CONFIG['MOZ_UBSAN']:
+    SOURCES += [
+        'UbsanOptions.cpp',
+    ]
+
 if CONFIG['OS_TARGET'] == 'WINNT':
     DEFFILE = 'mozglue.def'
     # We'll break the DLL blocklist if we immediately load user32.dll