Bug 1576776 Part 2 - Add Debugger.onNativeCall hook, r=jimb.

Depends on D43542

Differential Revision: https://phabricator.services.mozilla.com/D43543

--HG--
extra : moz-landing-system : lando

js/src/debugger/DebugAPI-inl.h

@@ -131,6 +131,15 @@ ResumeMode DebugAPI::onResumeFrame(JSContext* cx, AbstractFramePtr frame) {
   return slowPathOnResumeFrame(cx, frame);
 }
 
+/* static */
+ResumeMode DebugAPI::onNativeCall(JSContext* cx, const CallArgs& args,
+                                  CallReason reason) {
+  if (!cx->realm()->isDebuggee()) {
+    return ResumeMode::Continue;
+  }
+  return slowPathOnNativeCall(cx, args, reason);
+}
+
 /* static */
 ResumeMode DebugAPI::onDebuggerStatement(JSContext* cx,
                                          AbstractFramePtr frame) {

js/src/debugger/DebugAPI.h

@@ -8,6 +8,7 @@
 #define debugger_DebugAPI_h
 
 #include "vm/GlobalObject.h"
+#include "vm/Interpreter.h"
 #include "vm/JSContext.h"
 
 namespace js {
@@ -213,6 +214,9 @@ class DebugAPI {
    */
   static inline ResumeMode onResumeFrame(JSContext* cx, AbstractFramePtr frame);
 
+  static inline ResumeMode onNativeCall(JSContext* cx, const CallArgs& args,
+                                        CallReason reason);
+
   /*
    * Announce to the debugger a |debugger;| statement on has been
    * encountered on the youngest JS frame on |cx|. Call whatever hooks have
@@ -377,6 +381,8 @@ class DebugAPI {
   static ResumeMode slowPathOnEnterFrame(JSContext* cx, AbstractFramePtr frame);
   static ResumeMode slowPathOnResumeFrame(JSContext* cx,
                                           AbstractFramePtr frame);
+  static ResumeMode slowPathOnNativeCall(JSContext* cx, const CallArgs& args,
+                                         CallReason reason);
   static ResumeMode slowPathOnDebuggerStatement(JSContext* cx,
                                                 AbstractFramePtr frame);
   static ResumeMode slowPathOnExceptionUnwind(JSContext* cx,

js/src/debugger/Debugger.cpp

@@ -842,6 +842,40 @@ ResumeMode DebugAPI::slowPathOnResumeFrame(JSContext* cx,
   return slowPathOnEnterFrame(cx, frame);
 }
 
+/* static */
+ResumeMode DebugAPI::slowPathOnNativeCall(JSContext* cx, const CallArgs& args,
+                                          CallReason reason) {
+  RootedValue rval(cx);
+  ResumeMode resumeMode = Debugger::dispatchHook(
+      cx,
+      [cx](Debugger* dbg) -> bool {
+        return dbg == cx->insideDebuggerEvaluationWithOnNativeCallHook &&
+          dbg->getHook(Debugger::OnNativeCall);
+      },
+      [&](Debugger* dbg) -> ResumeMode {
+        return dbg->fireNativeCall(cx, args, reason, &rval);
+      });
+
+  switch (resumeMode) {
+    case ResumeMode::Continue:
+      break;
+
+    case ResumeMode::Throw:
+      cx->setPendingExceptionAndCaptureStack(rval);
+      break;
+
+    case ResumeMode::Terminate:
+      cx->clearPendingException();
+      break;
+
+    case ResumeMode::Return:
+      args.rval().set(rval);
+      break;
+  }
+
+  return resumeMode;
+}
+
 /*
  * RAII class to mark a generator as "running" temporarily while running
  * debugger code.
@@ -1755,7 +1789,7 @@ ResumeMode Debugger::processHandlerResult(Maybe<AutoRealm>& ar, bool success,
 
   RootedValue thisv(cx);
   Maybe<HandleValue> maybeThisv;
-  if (!GetThisValueForCheck(cx, frame, pc, &thisv, maybeThisv)) {
+  if (frame && !GetThisValueForCheck(cx, frame, pc, &thisv, maybeThisv)) {
     ar.reset();
     return ResumeMode::Terminate;
   }
@@ -2156,6 +2190,44 @@ ResumeMode Debugger::fireEnterFrame(JSContext* cx, MutableHandleValue vp) {
                               vp);
 }
 
+ResumeMode Debugger::fireNativeCall(JSContext* cx, const CallArgs& args,
+                                    CallReason reason, MutableHandleValue vp) {
+  RootedObject hook(cx, getHook(OnNativeCall));
+  MOZ_ASSERT(hook);
+  MOZ_ASSERT(hook->isCallable());
+
+  Maybe<AutoRealm> ar;
+  ar.emplace(cx, object);
+
+  RootedValue fval(cx, ObjectValue(*hook));
+  RootedValue calleeval(cx, args.calleev());
+  if (!wrapDebuggeeValue(cx, &calleeval)) {
+    return reportUncaughtException(ar);
+  }
+
+  JSAtom* reasonAtom = nullptr;
+  switch (reason) {
+    case CallReason::Call:
+      reasonAtom = cx->names().call;
+      break;
+    case CallReason::Getter:
+      reasonAtom = cx->names().get;
+      break;
+    case CallReason::Setter:
+      reasonAtom = cx->names().set;
+      break;
+  }
+  cx->markAtom(reasonAtom);
+
+  RootedValue reasonval(cx, StringValue(reasonAtom));
+
+  RootedValue rv(cx);
+  bool ok = js::Call(cx, fval, object, calleeval, reasonval, &rv);
+
+  AbstractFramePtr frame;
+  return processHandlerResult(ar, ok, rv, frame, nullptr, vp);
+}
+
 void Debugger::fireNewScript(JSContext* cx,
                              Handle<DebuggerScriptReferent> scriptReferent) {
   RootedObject hook(cx, getHook(OnNewScript));
@@ -3314,6 +3386,13 @@ Debugger::IsObserving Debugger::observesCoverage() const {
   return NotObserving;
 }
 
+Debugger::IsObserving Debugger::observesNativeCalls() const {
+  if (getHook(Debugger::OnNativeCall)) {
+    return Observing;
+  }
+  return NotObserving;
+}
+
 // Toggle whether this Debugger's debuggees observe all execution. This is
 // called when a hook that observes all execution is set or unset. See
 // hookObservesAllExecution.
@@ -4093,6 +4172,18 @@ bool Debugger::setOnEnterFrame(JSContext* cx, unsigned argc, Value* vp) {
   return setHookImpl(cx, args, *dbg, OnEnterFrame);
 }
 
+/* static */
+bool Debugger::getOnNativeCall(JSContext* cx, unsigned argc, Value* vp) {
+  THIS_DEBUGGER(cx, argc, vp, "(get onNativeCall)", args, dbg);
+  return getHookImpl(cx, args, *dbg, OnNativeCall);
+}
+
+/* static */
+bool Debugger::setOnNativeCall(JSContext* cx, unsigned argc, Value* vp) {
+  THIS_DEBUGGER(cx, argc, vp, "(set onNativeCall)", args, dbg);
+  return setHookImpl(cx, args, *dbg, OnNativeCall);
+}
+
 /* static */
 bool Debugger::getOnNewGlobalObject(JSContext* cx, unsigned argc, Value* vp) {
   THIS_DEBUGGER(cx, argc, vp, "(get onNewGlobalObject)", args, dbg);
@@ -6094,6 +6185,8 @@ const JSPropertySpec Debugger::properties[] = {
             Debugger::setOnPromiseSettled, 0),
     JS_PSGS("onEnterFrame", Debugger::getOnEnterFrame,
             Debugger::setOnEnterFrame, 0),
+    JS_PSGS("onNativeCall", Debugger::getOnNativeCall,
+            Debugger::setOnNativeCall, 0),
     JS_PSGS("onNewGlobalObject", Debugger::getOnNewGlobalObject,
             Debugger::setOnNewGlobalObject, 0),
     JS_PSGS("uncaughtExceptionHook", Debugger::getUncaughtExceptionHook,

js/src/debugger/Debugger.h

@@ -463,6 +463,7 @@ class Debugger : private mozilla::LinkedListElement<Debugger> {
     OnExceptionUnwind,
     OnNewScript,
     OnEnterFrame,
+    OnNativeCall,
     OnNewGlobalObject,
     OnNewPromise,
     OnPromiseSettled,
@@ -853,6 +854,8 @@ class Debugger : private mozilla::LinkedListElement<Debugger> {
   static bool setOnNewScript(JSContext* cx, unsigned argc, Value* vp);
   static bool getOnEnterFrame(JSContext* cx, unsigned argc, Value* vp);
   static bool setOnEnterFrame(JSContext* cx, unsigned argc, Value* vp);
+  static bool getOnNativeCall(JSContext* cx, unsigned argc, Value* vp);
+  static bool setOnNativeCall(JSContext* cx, unsigned argc, Value* vp);
   static bool getOnNewGlobalObject(JSContext* cx, unsigned argc, Value* vp);
   static bool setOnNewGlobalObject(JSContext* cx, unsigned argc, Value* vp);
   static bool getOnNewPromise(JSContext* cx, unsigned argc, Value* vp);
@@ -942,6 +945,9 @@ class Debugger : private mozilla::LinkedListElement<Debugger> {
   // execution.
   IsObserving observesCoverage() const;
 
+  // Whether the Debugger instance needs to observe native call invocations.
+  IsObserving observesNativeCalls() const;
+
  private:
   static MOZ_MUST_USE bool ensureExecutionObservabilityOfFrame(
       JSContext* cx, AbstractFramePtr frame);
@@ -970,6 +976,8 @@ class Debugger : private mozilla::LinkedListElement<Debugger> {
   ResumeMode fireDebuggerStatement(JSContext* cx, MutableHandleValue vp);
   ResumeMode fireExceptionUnwind(JSContext* cx, MutableHandleValue vp);
   ResumeMode fireEnterFrame(JSContext* cx, MutableHandleValue vp);
+  ResumeMode fireNativeCall(JSContext* cx, const CallArgs& args,
+                            CallReason reason, MutableHandleValue vp);
   ResumeMode fireNewGlobalObject(JSContext* cx, Handle<GlobalObject*> global,
                                  MutableHandleValue vp);
   ResumeMode firePromiseHook(JSContext* cx, Hook hook, HandleObject promise,

js/src/debugger/Frame.cpp

@@ -913,6 +913,11 @@ Result<Completion> js::DebuggerGenericEval(
     env = newEnv;
   }
 
+  // Note whether we are in an evaluation that might invoke the OnNativeCall
+  // hook, so that the JITs will be disabled.
+  AutoNoteDebuggerEvaluationWithOnNativeCallHook noteEvaluation(
+      cx, dbg->observesNativeCalls() ? dbg : nullptr);
+
   // Run the code and produce the completion value.
   LeaveDebuggeeNoExecute nnx(cx);
   RootedValue rval(cx);

js/src/doc/Debugger/Debugger.md

@@ -162,6 +162,24 @@ compartment.
     SpiderMonkey only calls `onEnterFrame` to report
     [visible][vf], non-`"debugger"` frames.
 
+<code>onNativeCall(<i>callee</i>, <i>reason</i>)</code>
+:   A call to a native function is being made from a debuggee realm.
+    <i>callee</i> is a [`Debugger.Object`] for the function being called, and
+    <i>reason</i> is a string describing the reason the call was made, and
+    has one of the following values:
+
+    `get`: The native is the getter for a property which is being accessed.
+    `set`: The native is the setter for a property being written to.
+    `call`: Any call not fitting into the above categories.
+
+    This method should return a [resumption value][rv] specifying how the
+    debuggee's execution should proceed.
+
+    SpiderMonkey only calls `onNativeCall` hooks when execution is inside a
+    debugger evaluation associated with the debugger that has the `onNativeCall`
+    hook.  Such evaluation methods include `Debugger.Object.executeInGlobal`,
+    `Debugger.Frame.eval`, and associated methods.
+
 <code>onExceptionUnwind(<i>frame</i>, <i>value</i>)</code>
 :   The exception <i>value</i> has been thrown, and has propagated to
     <i>frame</i>; <i>frame</i> is the youngest remaining stack frame, and is a

js/src/jit-test/tests/debug/Debugger-onNativeCall-01.js

@@ -0,0 +1,64 @@
+// Test that the onNativeCall hook is called when expected.
+
+load(libdir + 'eqArrayHelper.js');
+
+var g = newGlobal({newCompartment: true});
+var dbg = Debugger(g);
+var gdbg = dbg.addDebuggee(g);
+
+g.eval(`
+const x = [];
+Object.defineProperty(x, "a", {
+  get: print,
+  set: print,
+});
+function f() {
+  x.a++;
+  x.push(4);
+}
+`);
+
+for (let i = 0; i < 5; i++) {
+  g.f();
+}
+
+const rv = [];
+dbg.onNativeCall = (callee, reason) => { rv.push(callee.name, reason); };
+
+var dbg2 = Debugger(g);
+var gdbg2 = dbg2.addDebuggee(g);
+
+const fscript = gdbg.getOwnPropertyDescriptor('f').value.script;
+
+for (let i = 0; i < 5; i++) {
+  // The onNativeCall hook is called when doing global evaluations.
+  rv.length = 0;
+  gdbg.executeInGlobal(`f()`);
+  assertEqArray(rv, ["print", "get", "print", "set", "push", "call"]);
+
+  // The onNativeCall hook is called when doing frame evaluations.
+  let handlerCalled = false;
+  const handler = {
+    hit(frame) {
+      fscript.clearBreakpoint(handler);
+      rv.length = 0;
+      frame.eval(`f()`);
+      assertEqArray(rv, ["print", "get", "print", "set", "push", "call"]);
+      handlerCalled = true;
+    },
+  };
+  fscript.setBreakpoint(fscript.mainOffset, handler);
+  g.f();
+  assertEq(handlerCalled, true);
+
+  // The onNativeCall hook is *not* called when not in a debugger evaluation.
+  rv.length = 0;
+  g.f();
+  assertEqArray(rv, []);
+
+  // The onNativeCall hook is *not* called when in a debugger evaluation
+  // associated with a different debugger.
+  rv.length = 0;
+  gdbg2.executeInGlobal(`f()`);
+  assertEqArray(rv, []);
+}

js/src/jit-test/tests/debug/Debugger-onNativeCall-02.js

@@ -0,0 +1,61 @@
+// Test that the onNativeCall hook can control the call's behavior.
+
+var g = newGlobal({newCompartment: true});
+var dbg = Debugger(g);
+var gdbg = dbg.addDebuggee(g);
+
+g.eval(`
+var x = [];
+Object.defineProperty(x, "a", {
+  get: print,
+  set: print,
+});
+var rv;
+function f() {
+  x.a++;
+  try {
+    rv = x.push(4);
+  } catch (e) {
+    throw "rethrowing";
+  }
+}
+`);
+
+for (let i = 0; i < 5; i++) {
+  g.f();
+}
+
+for (let i = 0; i < 5; i++) {
+  // Test terminating execution.
+  dbg.onNativeCall = (callee, reason) => {
+    return null;
+  };
+  const len = g.x.length;
+  let v = gdbg.executeInGlobal(`f()`);
+  assertEq(v, null);
+  assertEq(g.x.length, len);
+
+  // Test throwing an exception.
+  dbg.onNativeCall = (callee, reason) => {
+    return { throw: "throwing" };
+  };
+  v = gdbg.executeInGlobal(`f()`);
+  assertEq(v.throw, "throwing");
+
+  // Test throwing an exception #2.
+  dbg.onNativeCall = (callee, reason) => {
+    if (callee.name == "push") {
+      return { throw: "throwing" };
+    }
+  };
+  v = gdbg.executeInGlobal(`f()`);
+  assertEq(v.throw, "rethrowing");
+
+  // Test returning a different value from the native.
+  dbg.onNativeCall = (callee, reason) => {
+    return { return: "value" };
+  };
+  v = gdbg.executeInGlobal(`f()`);
+  assertEq(v.return, undefined);
+  assertEq(g.rv, "value");
+}

js/src/jit-test/tests/debug/Debugger-onNativeCall-03.js

@@ -0,0 +1,31 @@
+// Test onNativeCall's behavior when used with self-hosted functions.
+
+load(libdir + 'eqArrayHelper.js');
+
+var g = newGlobal({newCompartment: true});
+var dbg = Debugger(g);
+var gdbg = dbg.addDebuggee(g);
+
+const rv = [];
+
+dbg.onEnterFrame = f => {
+  rv.push("EnterFrame");
+};
+
+dbg.onNativeCall = f => {
+  rv.push(f.displayName);
+};
+
+gdbg.executeInGlobal(`
+  var x = [1,3,2];
+  x.sort((a, b) => {print(a)});
+`);
+
+// When running self-hosted code, we will see native calls to internal
+// self-hosted JS functions and intrinsic natives. Drop these from the result
+// array.
+const validNames = ["EnterFrame", "sort", "print"];
+const filtered = rv.filter(name => validNames.includes(name));
+
+assertEq(filtered.length < rv.length, true);
+assertEqArray(filtered, ["EnterFrame", "sort", "EnterFrame", "print", "EnterFrame", "print"]);

js/src/jit-test/tests/debug/Debugger-onNativeCall-04.js

@@ -0,0 +1,26 @@
+// Test that onNativeCall behaves correctly when a debugger eval might enter the
+// JIT via OSR.
+
+var g = newGlobal({newCompartment: true});
+var dbg = Debugger(g);
+var gdbg = dbg.addDebuggee(g);
+
+g.eval(`
+const x = [];
+function f() {
+  for (let i = 0; i < 5; i++) {
+    x.push(i);
+  }
+}
+`);
+
+let numCalls = 0;
+dbg.onNativeCall = callee => { assertEq(callee.name, "push"); numCalls++; };
+
+var dbg2 = Debugger(g);
+
+for (let i = 0; i < 5; i++) {
+  numCalls = 0;
+  gdbg.executeInGlobal(`f()`);
+  assertEq(numCalls, 5);
+}