Bug 1251921 - Do not call debugger hooks with half-initialized frame if InterpeterFrame::prologue fails. (r=jorendorff)

js/src/jit-test/tests/debug/prologueFailure-01.js

@@ -0,0 +1,32 @@
+g = newGlobal();
+g.parent = this;
+
+function installHook() {
+    let calledTimes = 0;
+    function hook() {
+        calledTimes++;
+
+        // Allow the new.target.prototype get to throw.
+        if (calledTimes === 1)
+            return undefined;
+
+        return {
+            return: undefined
+        };
+    }
+
+    Debugger(parent).onExceptionUnwind = hook;
+}
+
+
+g.eval("(" + installHook + ")()");
+
+var handler = {
+    get(t, p) {
+        throw new TypeError;
+    }
+};
+
+
+var f = new Proxy(function(){}, handler);
+new f();

js/src/jit-test/tests/debug/prologueFailure-02.js

@@ -0,0 +1,49 @@
+g = newGlobal();
+g.parent = this;
+
+function installHook() {
+    let calledTimes = 0;
+    function hook(frame) {
+        calledTimes++;
+        switch (calledTimes) {
+          case 1:
+            // Proxy get trap
+            assertEq(frame.type, "call");
+            assertEq(frame.script.displayName.includes("get"), true);
+            break;
+          case 2:
+            // wrapper function. There is no entry for notRun
+            assertEq(frame.type, "call");
+            assertEq(frame.script.displayName.includes("wrapper"), true);
+            break;
+          case 3:
+            assertEq(frame.type, "global");
+            // Force the top-level to return cleanly, so that we can tell
+            // assertion failures from the intended throwing.
+            return { return: undefined };
+
+          default:
+            // that's the whole chain.
+            assertEq(false, true);
+        }
+    }
+
+    Debugger(parent).onExceptionUnwind = hook;
+}
+
+
+g.eval("(" + installHook + ")()");
+
+var handler = {
+    get(t, p) {
+        throw new TypeError;
+    }
+};
+
+function notRun() {}
+
+function wrapper() {
+    var f = new Proxy(notRun, handler);
+    new f();
+}
+wrapper();

js/src/vm/Interpreter.cpp

@@ -1665,9 +1665,10 @@ Interpret(JSContext* cx, RunState& state)
 
     /* State communicated between non-local jumps: */
     bool interpReturnOK;
+    bool frameHalfInitialized;
 
     if (!activation.entryFrame()->prologue(cx))
-        goto error;
+        goto prologue_error;
 
     switch (Debugger::onEnterFrame(cx, activation.entryFrame())) {
       case JSTRAP_CONTINUE:
@@ -1901,15 +1902,21 @@ CASE(JSOP_RETRVAL)
     interpReturnOK = true;
 
   return_continuation:
+    frameHalfInitialized = false;
+
+  prologue_return_continuation:
+
     if (activation.entryFrame() != REGS.fp()) {
         // Stop the engine. (No details about which engine exactly, could be
         // interpreter, Baseline or IonMonkey.)
         TraceLogStopEvent(logger, TraceLogger_Engine);
         TraceLogStopEvent(logger, TraceLogger_Scripts);
 
-        interpReturnOK = Debugger::onLeaveFrame(cx, REGS.fp(), REGS.pc, interpReturnOK);
+        if (MOZ_LIKELY(!frameHalfInitialized)) {
+            interpReturnOK = Debugger::onLeaveFrame(cx, REGS.fp(), REGS.pc, interpReturnOK);
 
-        REGS.fp()->epilogue(cx);
+            REGS.fp()->epilogue(cx);
+        }
 
   jit_return_pop_frame:
 
@@ -2872,7 +2879,7 @@ CASE(JSOP_FUNCALL)
     }
 
     if (!REGS.fp()->prologue(cx))
-        goto error;
+        goto prologue_error;
 
     switch (Debugger::onEnterFrame(cx, REGS.fp())) {
       case JSTRAP_CONTINUE:
@@ -3995,9 +4002,11 @@ DEFAULT()
     MOZ_CRASH("Invalid HandleError continuation");
 
   exit:
-    interpReturnOK = Debugger::onLeaveFrame(cx, REGS.fp(), REGS.pc, interpReturnOK);
+    if (MOZ_LIKELY(!frameHalfInitialized)) {
+        interpReturnOK = Debugger::onLeaveFrame(cx, REGS.fp(), REGS.pc, interpReturnOK);
 
-    REGS.fp()->epilogue(cx);
+        REGS.fp()->epilogue(cx);
+    }
 
     gc::MaybeVerifyBarriers(cx, true);
 
@@ -4014,6 +4023,11 @@ DEFAULT()
         state.setReturnValue(activation.entryFrame()->returnValue());
 
     return interpReturnOK;
+
+  prologue_error:
+    interpReturnOK = false;
+    frameHalfInitialized = true;
+    goto prologue_return_continuation;
 }
 
 bool