Bug 1418052 Correctly process hardening flags for ASAN, --disable-hardening, and --disable-optimize r=decoder,glandium

Will also address Bug 1377553 and part of Bug 1419607

MozReview-Commit-ID: AUCqBxEGpAl

--HG--
extra : rebase_source : 5547e2c8fbf4e2e87182b8720d8352c131e4ec65

build/autoconf/sanitize.m4

@@ -7,10 +7,6 @@ AC_DEFUN([MOZ_CONFIG_SANITIZE], [
 dnl ========================================================
 dnl = Use Address Sanitizer
 dnl ========================================================
-MOZ_ARG_ENABLE_BOOL(address-sanitizer,
-[  --enable-address-sanitizer       Enable Address Sanitizer (default=no)],
-    MOZ_ASAN=1,
-    MOZ_ASAN= )
 if test -n "$MOZ_ASAN"; then
     MOZ_LLVM_HACKS=1
     if test -n "$CLANG_CL"; then

build/moz.configure/old.configure

@@ -170,7 +170,6 @@ def old_configure_options(*options):
     '--cache-file',
     '--datadir',
     '--enable-accessibility',
-    '--enable-address-sanitizer',
     '--enable-alsa',
     '--enable-bundled-fonts',
     '--enable-content-sandbox',

build/moz.configure/toolchain.configure

@@ -1297,19 +1297,65 @@ wine = check_prog('WINE', ['wine'], when=depends(target, host)
 # Security Hardening
 # ==============================================================
 
+js_option('--enable-address-sanitizer', help='Enable Address Sanitizer')
+
+
+@depends_if('--enable-address-sanitizer')
+def asan(value):
+    return True
+
+
+add_old_configure_assignment('MOZ_ASAN', asan)
+
+
 option('--enable-hardening', env='MOZ_SECURITY_HARDENING',
        help='Enables security hardening compiler options')
 
 
-@depends('--enable-hardening', c_compiler)
-def security_hardening_cflags(value, c_compiler):
-    if value and c_compiler.type in ['gcc', 'clang']:
-        return '-fstack-protector-strong'
+@depends('--enable-hardening', '--enable-address-sanitizer',
+         '--enable-optimize', c_compiler, target)
+def security_hardening_cflags(hardening_flag, asan, optimize, c_compiler, target):
+    compiler_is_gccish = c_compiler.type in ('gcc', 'clang')
+
+    flags = []
+    js_flags = []
+
+    # FORTIFY_SOURCE ------------------------------------
+    # If hardening is explicitly enabled, or not explicitly disabled
+    if hardening_flag.origin == "default" or hardening_flag:
+        # Require optimization for FORTIFY_SOURCE. See Bug 1417452
+        # Also, undefine it before defining it just in case a distro adds it, see Bug 1418398
+        if compiler_is_gccish and optimize and not asan:
+            # Don't enable FORTIFY_SOURCE on Android on the top-level, but do enable in js/
+            if target.os != 'Android':
+                flags.append("-U_FORTIFY_SOURCE")
+                flags.append("-D_FORTIFY_SOURCE=2")
+            js_flags.append("-U_FORTIFY_SOURCE")
+            js_flags.append("-D_FORTIFY_SOURCE=2")
+
+    # If ASAN _is_ on, undefine FOTIFY_SOURCE just to be safe
+    if asan:
+        flags.append("-U_FORTIFY_SOURCE")
+        js_flags.append("-U_FORTIFY_SOURCE")
+
+    # fstack-protector ------------------------------------
+    # Enable only if --enable-hardening is passed and ASAN is
+    # not on as ASAN will catch the crashes for us
+    if hardening_flag and compiler_is_gccish and not asan:
+        flags.append("-fstack-protector-strong")
+
+    return namespace(
+        flags=flags,
+        js_flags=js_flags,
+    )
 
 
-add_old_configure_assignment('HARDENING_CFLAGS', security_hardening_cflags)
+add_old_configure_assignment('MOZ_HARDENING_CFLAGS', security_hardening_cflags.flags)
+add_old_configure_assignment('MOZ_HARDENING_CFLAGS_JS', security_hardening_cflags.js_flags)
 imply_option('--enable-pie', depends_if('--enable-hardening')(lambda v: v))
 
+# ==============================================================
+
 option(env='RUSTFLAGS',
        nargs=1,
        help='Rust compiler flags')

js/src/old-configure.in

@@ -547,23 +547,12 @@ esac
 MOZ_DOING_LTO(lto_is_enabled)
 
 dnl ========================================================
-dnl Add optional and non-optional hardening flags
+dnl Add optional and non-optional hardening flags from toolchain.configure
 dnl ========================================================
 
-dnl In at least glibc-2.25, _FORTIFY_SOURCE requires compiling
-dnl with optimization (Bug 1417452)
-
-dnl Note that in the top-level old-configure.in, we don't enable
-dnl FORTIFY_SOURCE on Android. But in js/ we *can* enable it on
-dnl Android, so we do.
-
-if test -n "$MOZ_OPTIMIZE"; then
-   if test "$GNU_CC" -o -n "${CLANG_CC}${CLANG_CL}"; then
-      CFLAGS="$CFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2"
-      CPPFLAGS="$CPPFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2"
-      CXXFLAGS="$CXXFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2"
-   fi
-fi
+CFLAGS="$CFLAGS $MOZ_HARDENING_CFLAGS_JS"
+CPPFLAGS="$CPPFLAGS $MOZ_HARDENING_CFLAGS_JS"
+CXXFLAGS="$CXXFLAGS $MOZ_HARDENING_CFLAGS_JS"
 
 dnl ========================================================
 dnl System overrides of the defaults for target

old-configure.in

@@ -508,28 +508,10 @@ fi
 dnl ========================================================
 dnl Add optional and non-optional hardening flags
 dnl ========================================================
-CFLAGS="$CFLAGS $HARDENING_CFLAGS"
-CPPFLAGS="$CPPFLAGS $HARDENING_CFLAGS"
-CXXFLAGS="$CXXFLAGS $HARDENING_CFLAGS"
 
-dnl In at least glibc-2.25, _FORTIFY_SOURCE requires compiling
-dnl with optimization (Bug 1417452)
-if test -n "$MOZ_OPTIMIZE"; then
-   if test "$GNU_CC" -o -n "${CLANG_CC}${CLANG_CL}"; then
-      case $OS_TARGET in
-         Android)
-            dnl FORTIFY_SOURCE is not supported on Android on the
-            dnl top-level old-configure.in at this time.
-            dnl See Bug 1415595
-            ;;
-         *)
-            CFLAGS="$CFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2"
-            CPPFLAGS="$CPPFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2"
-            CXXFLAGS="$CXXFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2"
-            ;;
-      esac
-   fi
-fi
+CFLAGS="$CFLAGS $MOZ_HARDENING_CFLAGS"
+CPPFLAGS="$CPPFLAGS $MOZ_HARDENING_CFLAGS"
+CXXFLAGS="$CXXFLAGS $MOZ_HARDENING_CFLAGS"
 
 dnl ========================================================
 dnl GNU specific defaults