Bug 1106719 - Don't call onExceptionUnwind and onPop debugger hooks on OOM. (r=jimb)

js/src/doc/Debugger/Debugger.Frame.md

@@ -257,7 +257,9 @@ the compartment to which the handler method belongs.
     resumption value each handler returns establishes the completion value
     reported to the next handler.
 
-    This property is ignored on `"debugger"` frames.
+    This handler is not called on `"debugger"` frames. It is also not called
+    when unwinding a frame due to an over-recursion or out-of-memory
+    exception.
 
 `onResume`
 :   This property must be either `undefined` or a function. If it is a

js/src/doc/Debugger/Debugger.md

@@ -151,6 +151,9 @@ compartment.
     `continue`, or `break` statement, or a new exception. In those cases the
     old exception does not continue to propagate; it is discarded.)
 
+    This handler is not called when unwinding a frame due to an over-recursion
+    or out-of-memory exception.
+
 <code>sourceHandler(<i>ASuffusionOfYellow</i>)</code>
 :   This method is never called. If it is ever called, a contradiction has
     been proven, and the debugger is free to assume that everything is true.

js/src/jit-test/tests/debug/bug1106719.js

@@ -0,0 +1,11 @@
+// |jit-test| allow-oom
+
+g = newGlobal()
+g.parent = this
+g.eval("Debugger(parent).onExceptionUnwind=(function(){})")
+gcparam("maxBytes", gcparam("gcBytes"))
+function f() {
+    f()
+    y(arguments)
+}
+f()

js/src/vm/Debugger.cpp

@@ -600,11 +600,11 @@ Debugger::slowPathOnLeaveFrame(JSContext *cx, AbstractFramePtr frame, bool frame
     RootedValue value(cx);
     Debugger::resultToCompletion(cx, frameOk, frame.returnValue(), &status, &value);
 
-    // This path can be hit via unwinding the stack due to
-    // over-recursion. Only fire the frames' onPop handlers if we haven't
-    // over-recursed, because invoking more JS will only result in more
-    // over-recursion errors. See slowPathOnExceptionUnwind.
-    if (!cx->isThrowingOverRecursed()) {
+    // This path can be hit via unwinding the stack due to over-recursion or
+    // OOM. In those cases, don't fire the frames' onPop handlers, because
+    // invoking JS will only trigger the same condition. See
+    // slowPathOnExceptionUnwind.
+    if (!cx->isThrowingOverRecursed() && !cx->isThrowingOutOfMemory()) {
         /* Build a list of the recipients. */
         AutoObjectVector frames(cx);
         for (FrameRange r(frame, global); !r.empty(); r.popFront()) {
@@ -729,9 +729,9 @@ Debugger::slowPathOnDebuggerStatement(JSContext *cx, AbstractFramePtr frame)
 /* static */ JSTrapStatus
 Debugger::slowPathOnExceptionUnwind(JSContext *cx, AbstractFramePtr frame)
 {
-    // Invoking more JS on an over-recursed stack is only going to result in
-    // more over-recursion errors.
-    if (cx->isThrowingOverRecursed())
+    // Invoking more JS on an over-recursed stack or after OOM is only going
+    // to result in more of the same error.
+    if (cx->isThrowingOverRecursed() || cx->isThrowingOutOfMemory())
         return JSTRAP_CONTINUE;
 
     RootedValue rval(cx);