Bug 1410528 - When running via autospider.sh, make the shell generate a minidump on crashes, r=jonco,ted

--HG--
extra : rebase_source : 8ab4093145e666a5e73b5d2b5efd498d38823eea
extra : source : 30247bfa8807b0106860399ecbfd51050c9b1885

browser/config/tooltool-manifests/linux64/jsshell.manifest

@@ -0,0 +1,16 @@
+[
+  {
+    "size": 479280,
+    "digest": "dd7e9d67fc858f670502a16266963b15d8fb9a2fa4fc638ba1a27e17318b84ade5a9c1284783905dfbe113ce59690c9232c182e0466d53a37182e8aedea5530f",
+    "algorithm": "sha512",
+    "filename": "breakpad-tools.tar.xz",
+    "unpack": true
+  },
+  {
+    "size": 1737944,
+    "visibility": "public",
+    "digest": "76d704f0bfa110f5ea298b87200e34ec09d039b9d1a59ec819fc8e02b2cf073af32a4536dca33a3813f037a557fd0669b48a063c7a920f6308b307148029d41f",
+    "algorithm": "sha512",
+    "filename": "linux64-minidump_stackwalk"
+  }
+]

js/src/devtools/automation/README

@@ -0,0 +1,37 @@
+autospider.py is intended both as the harness for running automation builds, as
+well as a way to easily reproduce automation builds on a developer workstation.
+Some brave souls also use it as the basis for doing their normal local builds.
+
+Basic usage:
+
+  ./js/src/devtools/automation/autospider.py plain
+
+The script may be run from any directory. This will configure and build the
+source, then run a series of tests. See the --help message for many different
+ways of suppressing various actions or changing the output.
+
+The different possible build+test configurations are controlled mostly by JSON
+files in a variants/ directory (eg there is a .../variants/plain file for the
+above command).
+
+In automation, the test jobs will run with a dynamically loaded library that
+catches crashes and generates minidumps, so that autospider.py can produce a
+readable stack trace at the end of the run. Currently this library is only
+available on linux64, and is built via the following procedure:
+
+    % git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
+    % export PATH=$PATH:$(pwd)/depot_tools
+    % mkdir breakpad
+    % cd breakpad
+    % fetch breakpad
+    % cd src
+    % git fetch https://github.com/hotsphink/breakpad injector
+    % git checkout injector
+    % cd ..
+    % mkdir obj
+    % cd obj
+    % ../src/configure
+    % mkdir ../root
+    % make install DESTDIR=$(pwd)/../root
+
+The shared library will now be in root/usr/local/lib64/libbreakpadinjector.so

js/src/devtools/automation/autospider.py

@@ -23,7 +23,7 @@ def directories(pathmodule, cwd, fixup=lambda s: s):
     js_src = pathmodule.abspath(pathmodule.join(scripts, "..", ".."))
     source = pathmodule.abspath(pathmodule.join(js_src, "..", ".."))
     tooltool = pathmodule.abspath(env.get('TOOLTOOL_CHECKOUT',
-                                          pathmodule.join(source, "..")))
+                                          pathmodule.join(source, "..", "..")))
     return Dirs(scripts, js_src, source, tooltool)
 
 # Some scripts will be called with sh, which cannot use backslashed
@@ -92,7 +92,6 @@ POBJDIR = posixpath.join(PDIR.source, args.objdir)
 AUTOMATION = env.get('AUTOMATION', False)
 MAKE = env.get('MAKE', 'make')
 MAKEFLAGS = env.get('MAKEFLAGS', '-j6' + ('' if AUTOMATION else ' -s'))
-UNAME_M = subprocess.check_output(['uname', '-m']).strip()
 
 
 def set_vars_from_script(script, vars):
@@ -207,7 +206,7 @@ if word_bits is None and args.platform:
 
 # Fall back to the word size of the host.
 if word_bits is None:
-    word_bits = 64 if UNAME_M == 'x86_64' else 32
+    word_bits = 64 if platform.architecture()[0] == '64bit' else 32
 
 if 'compiler' in variant:
     compiler = variant['compiler']
@@ -256,11 +255,11 @@ if word_bits == 32:
     if platform.system() == 'Windows':
         CONFIGURE_ARGS += ' --target=i686-pc-mingw32 --host=i686-pc-mingw32'
     elif platform.system() == 'Linux':
-        if UNAME_M != 'arm':
+        if not platform.machine().startswith('arm'):
             CONFIGURE_ARGS += ' --target=i686-pc-linux --host=i686-pc-linux'
 
     # Add SSE2 support for x86/x64 architectures.
-    if UNAME_M != 'arm':
+    if not platform.machine().startswith('arm'):
         if platform.system() == 'Windows':
             sse_flags = '-arch:SSE2'
         else:
@@ -289,7 +288,8 @@ ensure_dir_exists(OUTDIR, clobber=not args.keep)
 
 
 def run_command(command, check=False, **kwargs):
-    proc = Popen(command, cwd=OBJDIR, **kwargs)
+    kwargs.setdefault('cwd', OBJDIR)
+    proc = Popen(command, **kwargs)
     ACTIVE_PROCESSES.add(proc)
     stdout, stderr = None, None
     try:
@@ -304,13 +304,34 @@ def run_command(command, check=False, **kwargs):
 # Add in environment variable settings for this variant. Normally used to
 # modify the flags passed to the shell or to set the GC zeal mode.
 for k, v in variant.get('env', {}).items():
-    env[k] = v.format(
+    env[k.encode('ascii')] = v.encode('ascii').format(
         DIR=DIR.scripts,
         TOOLTOOL_CHECKOUT=DIR.tooltool,
         MOZ_UPLOAD_DIR=env['MOZ_UPLOAD_DIR'],
         OUTDIR=OUTDIR,
     )
 
+if AUTOMATION:
+    # Currently only supported on linux64.
+    if platform.system() == 'Linux' and platform.machine() == 'x86_64':
+        use_minidump = variant.get('use_minidump', True)
+    else:
+        use_minidump = False
+else:
+    use_minidump = False
+
+if use_minidump:
+    env.setdefault('MINIDUMP_SAVE_PATH', env['MOZ_UPLOAD_DIR'])
+    injector_lib = None
+    if platform.system() == 'Linux':
+        injector_lib = os.path.join(DIR.tooltool, 'breakpad-tools', 'libbreakpadinjector.so')
+        env.setdefault('MINIDUMP_STACKWALK',
+                       os.path.join(DIR.tooltool, 'linux64-minidump_stackwalk'))
+    elif platform.system() == 'Darwin':
+        injector_lib = os.path.join(DIR.tooltool, 'breakpad-tools', 'breakpadinjector.dylib')
+    if not injector_lib or not os.path.exists(injector_lib):
+        use_minidump=False
+
 def need_updating_configure(configure):
     if not os.path.exists(configure):
         return True
@@ -342,10 +363,23 @@ if not args.nobuild:
     # Run make
     run_command('%s -w %s' % (MAKE, MAKEFLAGS), shell=True, check=True)
 
+    if use_minidump:
+        # Convert symbols to breakpad format.
+        hostdir = os.path.join(OBJDIR, "dist", "host", "bin")
+        os.makedirs(hostdir)
+        shutil.copy(os.path.join(DIR.tooltool, "breakpad-tools", "dump_syms"),
+                    os.path.join(hostdir, 'dump_syms'))
+        run_command([
+            'make',
+            'recurse_syms',
+            'MOZ_SOURCE_REPO=file://' + DIR.source,
+            'RUST_TARGET=0', 'RUSTC_COMMIT=0'
+        ], check=True)
+
 COMMAND_PREFIX = []
 # On Linux, disable ASLR to make shell builds a bit more reproducible.
 if subprocess.call("type setarch >/dev/null 2>&1", shell=True) == 0:
-    COMMAND_PREFIX.extend(['setarch', UNAME_M, '-R'])
+    COMMAND_PREFIX.extend(['setarch', platform.machine(), '-R'])
 
 
 def run_test_command(command, **kwargs):
@@ -392,6 +426,15 @@ if 'all' in args.skip_tests.split(","):
 if platform.system() == 'Windows':
     env['JITTEST_EXTRA_ARGS'] = "-j1 " + env.get('JITTEST_EXTRA_ARGS', '')
 
+if use_minidump:
+    # Set up later js invocations to run with the breakpad injector loaded.
+    # Originally, I intended for this to be used with LD_PRELOAD, but when
+    # cross-compiling from 64- to 32-bit, that will fail and produce stderr
+    # output when running any 64-bit commands, which breaks eg mozconfig
+    # processing. So use the --dll command line mechanism universally.
+    for v in ('JSTESTS_EXTRA_ARGS', 'JITTEST_EXTRA_ARGS'):
+        env[v] = "--args='--dll %s' %s" % (injector_lib, env.get(v, ''))
+
 # Always run all enabled tests, even if earlier ones failed. But return the
 # first failed status.
 results = []
@@ -406,7 +449,10 @@ if 'jittest' in test_suites:
     results.append(run_test_command([MAKE, 'check-jit-test']))
 if 'jsapitests' in test_suites:
     jsapi_test_binary = os.path.join(OBJDIR, 'dist', 'bin', 'jsapi-tests')
-    st = run_test_command([jsapi_test_binary])
+    test_env = env.copy()
+    if use_minidump and platform.system() == 'Linux':
+        test_env['LD_PRELOAD'] = injector_lib
+    st = run_test_command([jsapi_test_binary], env=test_env)
     if st < 0:
         print("PROCESS-CRASH | jsapi-tests | application crashed")
         print("Return code: {}".format(st))
@@ -498,6 +544,16 @@ if args.variant in ('tsan', 'msan'):
     command += files
     subprocess.call(command)
 
+# Generate stacks from minidumps.
+if use_minidump:
+    venv_python = os.path.join(OBJDIR, "_virtualenv", "bin", "python")
+    run_command([
+        venv_python,
+        os.path.join(DIR.source, "testing/mozbase/mozcrash/mozcrash/mozcrash.py"),
+        os.getenv("TMPDIR", "/tmp"),
+        os.path.join(OBJDIR, "dist/crashreporter-symbols"),
+    ])
+
 for st in results:
     if st != 0:
         sys.exit(st)

js/src/devtools/automation/variants/arm-sim

@@ -2,5 +2,6 @@
     "configure-args": "--enable-stdcxx-compat --enable-simulator=arm --target=i686-pc-linux --host=i686-pc-linux",
     "optimize": true,
     "debug": true,
-    "bits": 32
+    "bits": 32,
+    "use_minidump": false
 }

js/src/devtools/automation/variants/asan

@@ -5,5 +5,6 @@
     "compiler": "clang",
     "env": {
         "LLVM_SYMBOLIZER": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer"
-    }
+    },
+    "use_minidump": false
 }

js/src/devtools/automation/variants/fuzzing

@@ -7,5 +7,6 @@
         "JITTEST_EXTRA_ARGS": "--jitflags=none",
         "JSTESTS_EXTRA_ARGS": "--jitflags=none",
         "LLVM_SYMBOLIZER": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer"
-    }
+    },
+    "use_minidump": false
 }

js/src/make-source-package.sh

@@ -151,6 +151,10 @@ case $cmd in
         ${TOPSRCDIR}/toolkit/mozapps/installer/package-name.mk \
         ${TOPSRCDIR}/toolkit/mozapps/installer/upload-files.mk \
         ${tgtpath}/toolkit/mozapps/installer
+    ${MKDIR} -p ${tgtpath}/toolkit/crashreporter/tools
+    cp -pPR \
+        ${TOPSRCDIR}/toolkit/crashreporter/tools/symbolstore.py \
+        ${tgtpath}/toolkit/crashreporter/tools
     ${MKDIR} -p ${tgtpath}/mozglue
     cp -pPR \
         ${TOPSRCDIR}/mozglue/build \

taskcluster/ci/spidermonkey/kind.yml

@@ -76,7 +76,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             using: spidermonkey-package
             spidermonkey-variant: plain
@@ -92,7 +92,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             using: spidermonkey-mozjs-crate
             spidermonkey-variant: plain
@@ -109,7 +109,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             using: spidermonkey-rust-bindings
             spidermonkey-variant: plain
@@ -126,7 +126,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: plaindebug
 
@@ -155,7 +155,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+              TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: plain
 
@@ -200,7 +200,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: arm64-sim
 
@@ -214,7 +214,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: asan
 
@@ -229,7 +229,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: compacting
 
@@ -258,7 +258,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: msan
 
@@ -288,7 +288,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: rootanalysis
 
@@ -303,7 +303,7 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: nonunified
 
@@ -318,6 +318,6 @@ jobs:
             max-run-time: 36000
             docker-image: {in-tree: desktop-build}
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/jsshell.manifest"
         run:
             spidermonkey-variant: fuzzing