Bug 1151957 - Rewrite GDB extension for ignoring asm.js segmentation faults in Python, fixing some bugs and bringing it under test. r=jimb.

The GDB code being rewritten here did not work for users who build with -ggdb3
on Linux, because sa_handler is a macro; see bug 1151957 comment 1. Also, it
caused GDB to perform an invalid free in the JS shell debuggee process, on
startup, if the previous run received SIGSEGV.

--HG--
rename : js/src/shell/js-gdb.gdb => js/src/gdb/mozilla/asmjs.py
extra : rebase_source : ba061d1978a4cb40172e7a79a3597bc0f91e9232

js/src/gdb/gdb-tests-gdb.py.in

@@ -6,3 +6,6 @@ sys.path[0:0] = [os.path.join('@topsrcdir@', 'gdb')]
 
 import mozilla.autoload
 mozilla.autoload.register(gdb.current_objfile())
+
+import mozilla.asmjs
+mozilla.asmjs.install()

js/src/gdb/moz.build

@@ -8,6 +8,7 @@ GeckoProgram('gdb-tests', linkage=None)
 
 UNIFIED_SOURCES += [
     'gdb-tests.cpp',
+    'tests/test-asmjs.cpp',
     'tests/test-Interpreter.cpp',
     'tests/test-jsid.cpp',
     'tests/test-JSObject.cpp',

js/src/gdb/mozilla/asmjs.py

@@ -0,0 +1,40 @@
+"""
+In asm code, out-of-bounds heap accesses cause segfaults, which the engine
+handles internally. Make GDB ignore them.
+"""
+
+import gdb
+
+SIGSEGV = 11
+
+# A sigaction buffer for each inferior process.
+sigaction_buffers = {}
+
+def on_stop(event):
+    if isinstance(event, gdb.SignalEvent) and event.stop_signal == 'SIGSEGV':
+        # Allocate memory for sigaction, once per js shell process.
+        process = gdb.selected_inferior()
+        buf = sigaction_buffers.get(process)
+        if buf is None:
+            buf = gdb.parse_and_eval("(struct sigaction *) malloc(sizeof(struct sigaction))")
+            sigaction_buffers[process] = buf
+
+        # See if AsmJSFaultHandler is installed as the SIGSEGV signal action.
+        sigaction_fn = gdb.parse_and_eval('__sigaction')
+        sigaction_fn(SIGSEGV, 0, buf)
+        AsmJSFaultHandler = gdb.parse_and_eval("AsmJSFaultHandler")
+        if buf['__sigaction_handler']['sa_handler'] == AsmJSFaultHandler:
+            # Advise the user that magic is happening.
+            print "js/src/gdb/mozilla/asmjs.py: Allowing AsmJSFaultHandler to run."
+
+            # If AsmJSFaultHandler doesn't handle this segfault, it will unhook
+            # itself and re-raise.
+            gdb.execute("continue")
+
+def on_exited(event):
+    if event.inferior in sigaction_buffers:
+        del sigaction_buffers[event.inferior]
+
+def install():
+    gdb.events.stop.connect(on_stop)
+    gdb.events.exited.connect(on_exited)

js/src/gdb/tests/test-asmjs.cpp

@@ -0,0 +1,38 @@
+#include "gdb-tests.h"
+#include "jsapi.h"
+
+#include <string.h>
+
+FRAGMENT(asmjs, segfault) {
+    using namespace JS;
+
+    int line0 = __LINE__;
+    const char* bytes = "\n"
+        "function f(glob, ffi, heap) {\n"
+        "    \"use asm\";\n"
+        "    var f32 = new glob.Float32Array(heap);\n"
+        "    function g(n) {\n"
+        "        n = n | 0;\n"
+        "        return +f32[n>>2];\n"
+        "    }\n"
+        "    return g;\n"
+        "}\n"
+        "\n"
+        "var func = f(this, null, new ArrayBuffer(0x10000));\n"
+        "func(0x10000 << 2);\n"
+        "'ok'\n";
+
+    CompileOptions opts(cx);
+    opts.setFileAndLine(__FILE__, line0 + 1);
+    opts.asmJSOption = true;
+    RootedValue rval(cx);
+    bool ok;
+    ok = false;
+
+    ok = Evaluate(cx, opts, bytes, strlen(bytes), &rval);
+
+    breakpoint();
+
+    (void) ok;
+    (void) rval;
+}

js/src/gdb/tests/test-asmjs.py

@@ -0,0 +1,15 @@
+# Test for special asmjs SIGSEGV-handling.
+#
+# Expected behavior is for the asm.js code in the following fragment to trigger
+# SIGSEGV. The code in js/src/gdb/mozilla/asmjs.py should prevent GDB from
+# handling that signal.
+
+run_fragment('asmjs.segfault')
+
+# If SIGSEGV handling is broken, GDB would have stopped at the SIGSEGV signal.
+# The breakpoint would not have hit, and run_fragment would have thrown.
+#
+# So if we get here, and the asm.js code actually ran, we win.
+
+assert_pretty('ok', 'true')
+assert_pretty('rval', '$jsval("ok")')

js/src/shell/Makefile.in

@@ -24,10 +24,6 @@ INSTALL_TARGETS += SHELL_INSTALL_AUTOLOAD
 SHELL_INSTALL_AUTOLOAD_FILES := $(CURDIR)/js-gdb.py
 SHELL_INSTALL_AUTOLOAD_DEST := $(DIST)/bin
 
-INSTALL_TARGETS += SHELL_INSTALL_AUTOLOAD_SCRIPT
-SHELL_INSTALL_AUTOLOAD_SCRIPT_FILES := js-gdb.gdb
-SHELL_INSTALL_AUTOLOAD_SCRIPT_DEST := $(CURDIR)
-
 include $(topsrcdir)/config/rules.mk
 
 # People expect the js shell to wind up in the top-level JS dir.

js/src/shell/js-gdb.gdb

@@ -1,18 +0,0 @@
-define hookpost-run
-    if ($sigaction)
-        call free($sigaction)
-        set $sigaction = 0
-    end
-end
-
-catch signal SIGSEGV
-commands
-    if !$sigaction
-        set $sigaction = malloc(sizeof(sigaction))
-    end
-    set $ignored = __sigaction(11, 0, $sigaction)
-    set $handler = ((struct sigaction *)$sigaction)->__sigaction_handler.sa_handler
-    if $handler == AsmJSFaultHandler
-        continue
-    end
-end

js/src/shell/js-gdb.py.in

@@ -6,3 +6,6 @@ sys.path[0:0] = [os.path.join('@topsrcdir@', 'gdb')]
 
 import mozilla.autoload
 mozilla.autoload.register(gdb.current_objfile())
+
+import mozilla.asmjs
+mozilla.asmjs.install()