Bug 1450088 - Use Winchecksec instead of Binscope. r=dmajor

While almost rewriting autobinscope.py entirely (and renaming it),
switch to python 3.

Differential Revision: https://phabricator.services.mozilla.com/D68156

--HG--
rename : build/win32/autobinscope.py => build/win32/autowinchecksec.py
extra : moz-landing-system : lando

browser/config/mozconfigs/win32/common-win32

@@ -28,6 +28,12 @@ mk_add_options "export PATH=${VSPATH}/VC/bin/Hostx64/x86:${MOZ_FETCHES_DIR}/nsis
 
 unset VC_PATH
 
+else # ! Linux
+export WINCHECKSEC="${MOZ_FETCHES_DIR}/winchecksec/winchecksec.exe"
+if [ ! -f "${WINCHECKSEC}" ]; then
+    unset WINCHECKSEC
+fi
+
 fi
 
 export PDBSTR="${MOZ_FETCHES_DIR}/pdbstr/pdbstr.exe"

browser/config/mozconfigs/win64-aarch64/common-win64

@@ -29,6 +29,12 @@ mk_add_options "export WINEPATH=${VSPATH}/VC/bin/Hostx64/x64"
 
 unset VC_PATH
 
+else # ! Linux
+export WINCHECKSEC="${MOZ_FETCHES_DIR}/winchecksec/winchecksec.exe"
+if [ ! -f "${WINCHECKSEC}" ]; then
+    unset WINCHECKSEC
+fi
+
 fi
 
 export PDBSTR="${MOZ_FETCHES_DIR}/pdbstr/pdbstr.exe"

browser/config/mozconfigs/win64/common-win64

@@ -28,6 +28,12 @@ mk_add_options "export PATH=${VSPATH}/VC/bin/Hostx64/x64:${MOZ_FETCHES_DIR}/nsis
 
 unset VC_PATH
 
+else # ! Linux
+export WINCHECKSEC="${MOZ_FETCHES_DIR}/winchecksec/winchecksec.exe"
+if [ ! -f "${WINCHECKSEC}" ]; then
+    unset WINCHECKSEC
+fi
+
 fi
 
 export PDBSTR="${MOZ_FETCHES_DIR}/pdbstr/pdbstr.exe"

browser/config/mozconfigs/win64/plain-opt

@@ -11,6 +11,7 @@ CBINDGEN="${MOZ_FETCHES_DIR}/cbindgen/cbindgen"
 
 NASM="${MOZ_FETCHES_DIR}/nasm/nasm.exe"
 NODEJS="${MOZ_FETCHES_DIR}/node/node.exe"
+WINCHECKSEC="${MOZ_FETCHES_DIR}/winchecksec/winchecksec.exe"
 
 ac_add_options --target=x86_64-pc-mingw32
 

build/mozconfig.no-compile

@@ -50,3 +50,4 @@ unset WASM_CXX
 unset LUCETC
 unset WASI_SYSROOT
 unset PDBSTR
+unset WINCHECKSEC

build/win32/autobinscope.py

@@ -1,104 +0,0 @@
-#!/usr/bin/env python
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# run Microsoft's Binscope tool (https://www.microsoft.com/en-us/download/details.aspx?id=44995)
-# against a fresh Windows build. output a 'binscope.log' file with full details
-# of the run and appropriate strings to integrate with the buildbots
-
-# the symbol dir should point to the symbol dir hierarchy created
-# via running make buildsymbols in a windows build's objdir
-
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-
-# usage
-if len(sys.argv) != 3:
-    print("""usage : autobinscope.by path_to_binary path_to_symbols""")
-    sys.exit(0)
-
-binary_path = sys.argv[1]
-symbol_path = sys.argv[2]
-
-# execute binscope against the binary, using the BINSCOPE environment
-# variable as the path to binscope.exe
-try:
-    binscope_path = os.environ['BINSCOPE']
-except KeyError:
-    print("TEST-UNEXPECTED-FAIL | autobinscope.py | BINSCOPE environment variable is not set, "
-          "can't check DEP/ASLR etc. status.")
-    sys.exit(1)
-
-try:
-    tmpdir = tempfile.mkdtemp()
-    proc = subprocess.Popen([
-        binscope_path,
-        "/NoLogo",
-        "/OutDir", tmpdir,
-        "/Target", binary_path,
-        "/SymPath", symbol_path,
-        # ATLVersionCheck triggers a crash in msdia120: bug 1525113
-        "/SkippedChecks", "ATLVersionCheck",
-        "/Checks", "ATLVulnCheck",
-        # We do not ship in the Windows Store
-        "/SkippedChecks", "AppContainerCheck",
-        # The CompilerVersionCheck doesn't like clang-cl (we would need to set MinimumCompilerVersion)  # NOQA: E501
-        # But we check the compiler in our build system anyway, so this doesn't seem useful
-        "/SkippedChecks", "CompilerVersionCheck",
-        "/Checks", "DBCheck",
-        "/Checks", "DefaultGSCookieCheck",
-        "/Checks", "ExecutableImportsCheck",
-        # FunctonPointersCheck is disabled per bug 1014002
-        "/SkippedChecks", "FunctionPointersCheck",
-        # GSCheck doesn't know how to deal with Rust libs
-        "/SkippedChecks", "GSCheck",
-        "/Checks", "GSFriendlyInitCheck",
-        # We are not safebuffers-clean, bug 1449951
-        "/SkippedChecks", "GSFunctionSafeBuffersCheck",
-        "/Checks", "HighEntropyVACheck",
-        "/Checks", "NXCheck",
-        "/Checks", "RSA32Check",
-        "/Checks", "SafeSEHCheck",
-        "/Checks", "SharedSectionCheck",
-        # VB6Check triggers a crash in msdia120: bug 1525113
-        "/SkippedChecks", "VB6Check",
-        "/Checks", "WXCheck"
-    ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-except WindowsError, (errno, strerror):  # noqa
-    if errno != 2 and errno != 3:
-        print("TEST-UNEXPECTED-FAIL | autobinscope.py | Unexpected error %d : %s" (
-            errno, strerror))
-        sys.exit(1)
-    else:
-        print("TEST-UNEXPECTED-FAIL | autobinscope.py | Could not locate binscope at location : "
-              "%s\n" % binscope_path)
-        sys.exit(1)
-finally:
-    shutil.rmtree(tmpdir)
-
-proc.wait()
-
-output = proc.communicate()[1].decode('utf-8').splitlines()
-
-errors = 0
-for line in output:
-    print(line)
-    if 'error' in line:
-        errors += 1
-
-if proc.returncode != 0:
-    print("TEST-UNEXPECTED-FAIL | autobinscope.py | Binscope returned error code %d for file %s" %
-          (proc.returncode, binary_path))
-    sys.exit(1)
-elif errors != 0:
-    print("TEST-UNEXPECTED-FAIL | autobinscope.py | Binscope reported %d error(s) for file %s" % (
-        errors, binary_path))
-    sys.exit(1)
-else:
-    print("TEST-PASS | autobinscope.py | %s succeeded" % binary_path)

build/win32/autowinchecksec.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# run the Winchecksec tool (https://github.com/trailofbits/winchecksec)
+# against a given Windows binary.
+
+import buildconfig
+import json
+import subprocess
+import sys
+
+# usage
+if len(sys.argv) != 2:
+    print("""usage : autowinchecksec.by path_to_binary""")
+    sys.exit(0)
+
+binary_path = sys.argv[1]
+
+# execute winchecksec against the binary, using the WINCHECKSEC environment
+# variable as the path to winchecksec.exe
+try:
+    winchecksec_path = buildconfig.substs['WINCHECKSEC']
+except KeyError:
+    print("TEST-UNEXPECTED-FAIL | autowinchecksec.py | WINCHECKSEC environment variable is "
+          "not set, can't check DEP/ASLR etc. status.")
+    sys.exit(1)
+
+try:
+    result = subprocess.check_output([winchecksec_path, '-j', binary_path],
+                                     universal_newlines=True)
+
+except subprocess.CalledProcessError as e:
+    print("TEST-UNEXPECTED-FAIL | autowinchecksec.py | Winchecksec returned error code %d:\n%s" % (
+          e.returncode, e.output))
+    sys.exit(1)
+
+
+result = json.loads(result)
+
+checks = [
+    'aslr',
+    'cfg',
+    'dynamicBase',
+    'gs',
+    'isolation',
+    'nx',
+    'seh',
+]
+
+if buildconfig.substs['CPU_ARCH'] == 'x86':
+    checks += [
+        'safeSEH',
+    ]
+else:
+    checks += [
+        'highEntropyVA',
+    ]
+
+failed = [c for c in checks if result.get(c) is False]
+
+if failed:
+    print("TEST-UNEXPECTED-FAIL | autowinchecksec.py | Winchecksec reported %d error(s) for %s" %
+          (len(failed), binary_path))
+    print("TEST-UNEXPECTED-FAIL | autowinchecksec.py | The following check(s) failed: %s" %
+          (', '.join(failed)))
+    sys.exit(1)
+else:
+    print("TEST-PASS | autowinchecksec.py | %s succeeded" % binary_path)

config/rules.mk

@@ -704,9 +704,9 @@ $(2): $(1)
 ifdef MOZ_CRASHREPORTER
 	$$(call py3_action,dumpsymbols,$$(abspath $$<) $$(abspath $$@) $$(DUMP_SYMBOLS_FLAGS))
 ifeq ($(HOST_OS_ARCH),WINNT)
-ifdef BINSCOPE
-	$$(PYTHON) $$(topsrcdir)/build/win32/autobinscope.py $$< $$(DIST)/crashreporter-symbols
-endif # BINSCOPE
+ifdef WINCHECKSEC
+	$$(PYTHON3) $$(topsrcdir)/build/win32/autowinchecksec.py $$<
+endif # WINCHECKSEC
 endif # WINNT
 endif
 endef

taskcluster/ci/build/windows.yml

@@ -8,6 +8,7 @@ job-defaults:
         tooltool-downloads: internal
     fetches:
         toolchain:
+            - win64-winchecksec
             - win64-dump-syms
             - win64-pdbstr
 

testing/mozharness/configs/builds/taskcluster_base_windows.py

@@ -9,9 +9,6 @@ config = {
     'max_build_output_timeout': 60 * 80,
 
     'env': {
-        'BINSCOPE': os.path.join(
-            os.environ['ProgramFiles'], 'Microsoft BinScope 2014', 'Binscope.exe'
-        ),
         'HG_SHARE_BASE_DIR': os.path.join('y:', os.sep, 'hg-shared'),
         'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
         'MOZ_CRASHREPORTER_NO_REPORT': '1',

toolkit/moz.configure

@@ -1999,6 +1999,17 @@ check_prog('DUMP_SYMS', ['dump_syms'], allow_missing=True, paths=toolchain_searc
 check_prog('PDBSTR', ['pdbstr.exe'], allow_missing=True, paths=toolchain_search_path,
            when=compile_environment & target_is_windows)
 
+@depends('MOZ_AUTOMATION', c_compiler)
+def allow_missing_winchecksec(automation, c_compiler):
+    if not automation:
+        return True
+    if c_compiler and c_compiler.type != 'clang-cl':
+        return True
+
+check_prog('WINCHECKSEC', ['winchecksec.exe'], paths=toolchain_search_path,
+           allow_missing=allow_missing_winchecksec,
+           when=compile_environment & target_is_windows & host_is_windows)
+
 # Fork server
 @depends(target, build_project)
 def forkserver_default(target, build_project):