Bug 1576776 Part 1 - VM support for DebugAPI::onNativeCall, r=jandem.

Differential Revision: https://phabricator.services.mozilla.com/D43542

--HG--
extra : moz-landing-system : lando

js/src/jit/BaselineJIT.cpp

@@ -359,6 +359,12 @@ MethodStatus jit::CanEnterBaselineInterpreterAtBranch(JSContext* cx,
     return Method_CantCompile;
   }
 
+  // JITs do not respect the debugger's OnNativeCall hook, so JIT execution is
+  // disabled if this hook might need to be called.
+  if (cx->insideDebuggerEvaluationWithOnNativeCallHook) {
+    return Method_CantCompile;
+  }
+
   return CanEnterBaselineInterpreter(cx, fp->script());
 }
 

js/src/jit/Jit.cpp

@@ -138,6 +138,12 @@ EnterJitStatus js::jit::MaybeEnterJit(JSContext* cx, RunState& state) {
     return EnterJitStatus::NotEntered;
   }
 
+  // JITs do not respect the debugger's OnNativeCall hook, so JIT execution is
+  // disabled if this hook might need to be called.
+  if (cx->insideDebuggerEvaluationWithOnNativeCallHook) {
+    return EnterJitStatus::NotEntered;
+  }
+
   JSScript* script = state.script();
 
   uint8_t* code = script->jitCodeRaw();

js/src/vm/Interpreter.cpp

@@ -429,6 +429,7 @@ bool js::RunScript(JSContext* cx, RunState& state) {
 
 STATIC_PRECONDITION_ASSUME(ubound(args.argv_) >= argc)
 MOZ_ALWAYS_INLINE bool CallJSNative(JSContext* cx, Native native,
+                                    CallReason reason,
                                     const CallArgs& args) {
   TraceLoggerThread* logger = TraceLoggerForCurrentThread(cx);
   AutoTraceLog traceLog(logger, TraceLogger_Call);
@@ -437,6 +438,16 @@ MOZ_ALWAYS_INLINE bool CallJSNative(JSContext* cx, Native native,
     return false;
   }
 
+  switch (DebugAPI::onNativeCall(cx, args, reason)) {
+    case ResumeMode::Continue:
+      break;
+    case ResumeMode::Throw:
+    case ResumeMode::Terminate:
+      return false;
+    case ResumeMode::Return:
+      return true;
+  }
+
 #ifdef DEBUG
   bool alreadyThrowing = cx->isExceptionPending();
 #endif
@@ -460,7 +471,7 @@ MOZ_ALWAYS_INLINE bool CallJSNativeConstructor(JSContext* cx, Native native,
 #endif
 
   MOZ_ASSERT(args.thisv().isMagic());
-  if (!CallJSNative(cx, native, args)) {
+  if (!CallJSNative(cx, native, CallReason::Call, args)) {
     return false;
   }
 
@@ -491,7 +502,8 @@ MOZ_ALWAYS_INLINE bool CallJSNativeConstructor(JSContext* cx, Native native,
  *       this step already!
  */
 bool js::InternalCallOrConstruct(JSContext* cx, const CallArgs& args,
-                                 MaybeConstruct construct) {
+                                 MaybeConstruct construct,
+                                 CallReason reason) {
   MOZ_ASSERT(args.length() <= ARGS_LENGTH_MAX);
   MOZ_ASSERT(!cx->zone()->types.activeAnalysis);
 
@@ -516,7 +528,7 @@ bool js::InternalCallOrConstruct(JSContext* cx, const CallArgs& args,
     JSNative call = args.callee().callHook();
     MOZ_ASSERT(call, "isCallable without a callHook?");
 
-    return CallJSNative(cx, call, args);
+    return CallJSNative(cx, call, reason, args);
   }
 
   /* Invoke native functions. */
@@ -536,7 +548,20 @@ bool js::InternalCallOrConstruct(JSContext* cx, const CallArgs& args,
         native = jitInfo->ignoresReturnValueMethod;
       }
     }
-    return CallJSNative(cx, native, args);
+    return CallJSNative(cx, native, reason, args);
+  }
+
+  // Self-hosted builtins are considered native by the onNativeCall hook.
+  if (fun->isSelfHostedBuiltin()) {
+    switch (DebugAPI::onNativeCall(cx, args, reason)) {
+      case ResumeMode::Continue:
+        break;
+      case ResumeMode::Throw:
+      case ResumeMode::Terminate:
+        return false;
+      case ResumeMode::Return:
+        return true;
+    }
   }
 
   if (!JSFunction::getOrCreateScript(cx, fun)) {
@@ -570,7 +595,8 @@ bool js::InternalCallOrConstruct(JSContext* cx, const CallArgs& args,
   return ok;
 }
 
-static bool InternalCall(JSContext* cx, const AnyInvokeArgs& args) {
+static bool InternalCall(JSContext* cx, const AnyInvokeArgs& args,
+                         CallReason reason = CallReason::Call) {
   MOZ_ASSERT(args.array() + args.length() == args.end(),
              "must pass calling arguments to a calling attempt");
 
@@ -591,7 +617,7 @@ static bool InternalCall(JSContext* cx, const AnyInvokeArgs& args) {
     }
   }
 
-  return InternalCallOrConstruct(cx, args, NO_CONSTRUCT);
+  return InternalCallOrConstruct(cx, args, NO_CONSTRUCT, reason);
 }
 
 bool js::CallFromStack(JSContext* cx, const CallArgs& args) {
@@ -601,13 +627,14 @@ bool js::CallFromStack(JSContext* cx, const CallArgs& args) {
 // ES7 rev 0c1bd3004329336774cbc90de727cd0cf5f11e93
 // 7.3.12 Call.
 bool js::Call(JSContext* cx, HandleValue fval, HandleValue thisv,
-              const AnyInvokeArgs& args, MutableHandleValue rval) {
+              const AnyInvokeArgs& args, MutableHandleValue rval,
+              CallReason reason) {
   // Explicitly qualify these methods to bypass AnyInvokeArgs's deliberate
   // shadowing.
   args.CallArgs::setCallee(fval);
   args.CallArgs::setThis(thisv);
 
-  if (!InternalCall(cx, args)) {
+  if (!InternalCall(cx, args, reason)) {
     return false;
   }
 
@@ -731,7 +758,7 @@ bool js::CallGetter(JSContext* cx, HandleValue thisv, HandleValue getter,
 
   FixedInvokeArgs<0> args(cx);
 
-  return Call(cx, getter, thisv, args, rval);
+  return Call(cx, getter, thisv, args, rval, CallReason::Getter);
 }
 
 bool js::CallSetter(JSContext* cx, HandleValue thisv, HandleValue setter,
@@ -745,7 +772,7 @@ bool js::CallSetter(JSContext* cx, HandleValue thisv, HandleValue setter,
   args[0].set(v);
 
   RootedValue ignored(cx);
-  return Call(cx, setter, thisv, args, &ignored);
+  return Call(cx, setter, thisv, args, &ignored, CallReason::Setter);
 }
 
 bool js::ExecuteKernel(JSContext* cx, HandleScript script,
@@ -3066,11 +3093,13 @@ static MOZ_NEVER_INLINE JS_HAZ_JSNATIVE_CALLER bool Interpret(JSContext* cx,
       JSFunction* maybeFun;
       bool isFunction = IsFunctionObject(args.calleev(), &maybeFun);
 
-      // Use the slow path if the callee is not an interpreted function or if we
-      // have to throw an exception.
+      // Use the slow path if the callee is not an interpreted function, if we
+      // have to throw an exception, or if we might have to invoke the
+      // OnNativeCall hook for a self-hosted builtin.
       if (!isFunction || !maybeFun->isInterpreted() ||
           (construct && !maybeFun->isConstructor()) ||
-          (!construct && maybeFun->isClassConstructor())) {
+          (!construct && maybeFun->isClassConstructor()) ||
+          cx->insideDebuggerEvaluationWithOnNativeCallHook) {
         if (construct) {
           if (!ConstructFromStack(cx, args)) {
             goto error;

js/src/vm/Interpreter.h

@@ -45,6 +45,13 @@ extern JSObject* ValueToCallable(JSContext* cx, HandleValue v,
                                  int numToSkip = -1,
                                  MaybeConstruct construct = NO_CONSTRUCT);
 
+// Reasons why a call could be performed, for passing onto the debugger.
+enum class CallReason {
+  Call,
+  Getter,
+  Setter
+};
+
 /*
  * Call or construct arguments that are stored in rooted memory.
  *
@@ -54,7 +61,8 @@ extern JSObject* ValueToCallable(JSContext* cx, HandleValue v,
  *       performed, use |Invoke|.
  */
 extern bool InternalCallOrConstruct(JSContext* cx, const CallArgs& args,
-                                    MaybeConstruct construct);
+                                    MaybeConstruct construct,
+                                    CallReason reason = CallReason::Call);
 
 /*
  * These helpers take care of the infinite-recursion check necessary for
@@ -76,7 +84,8 @@ extern bool CallSetter(JSContext* cx, HandleValue thisv, HandleValue setter,
 // |rval| is written to *only* after |fval| and |thisv| have been consumed, so
 // |rval| *may* alias either argument.
 extern bool Call(JSContext* cx, HandleValue fval, HandleValue thisv,
-                 const AnyInvokeArgs& args, MutableHandleValue rval);
+                 const AnyInvokeArgs& args, MutableHandleValue rval,
+                 CallReason reason = CallReason::Call);
 
 inline bool Call(JSContext* cx, HandleValue fval, HandleValue thisv,
                  MutableHandleValue rval) {

js/src/vm/JSContext.cpp

@@ -1294,11 +1294,11 @@ JSContext::JSContext(JSRuntime* runtime, const JS::ContextOptions& options)
       internalJobQueue(this),
       canSkipEnqueuingJobs(this, false),
       promiseRejectionTrackerCallback(this, nullptr),
-      promiseRejectionTrackerCallbackData(this, nullptr)
+      promiseRejectionTrackerCallbackData(this, nullptr),
 #ifdef JS_STRUCTURED_SPEW
-      ,
-      structuredSpewer_()
+      structuredSpewer_(),
 #endif
+      insideDebuggerEvaluationWithOnNativeCallHook(this, nullptr)
 {
   MOZ_ASSERT(static_cast<JS::RootingContext*>(this) ==
              JS::RootingContext::get(this));

js/src/vm/JSContext.h

@@ -998,6 +998,12 @@ struct JSContext : public JS::RootingContext,
   js::StructuredSpewer& spewer() { return structuredSpewer_.ref(); }
 #endif
 
+  // During debugger evaluations which need to observe native calls, JITs are
+  // completely disabled. This flag indicates whether we are in this state, and
+  // the debugger which initiated the evaluation. This debugger has other
+  // references on the stack and does not need to be traced.
+  js::ContextData<js::Debugger*> insideDebuggerEvaluationWithOnNativeCallHook;
+
 }; /* struct JSContext */
 
 inline JS::Result<> JSContext::boolToResult(bool ok) {
@@ -1243,6 +1249,20 @@ class MOZ_RAII AutoKeepAtoms {
   inline ~AutoKeepAtoms();
 };
 
+class MOZ_RAII AutoNoteDebuggerEvaluationWithOnNativeCallHook {
+  JSContext* cx;
+  Debugger* oldValue;
+ public:
+  AutoNoteDebuggerEvaluationWithOnNativeCallHook(JSContext* cx, Debugger* dbg)
+      : cx(cx), oldValue(cx->insideDebuggerEvaluationWithOnNativeCallHook) {
+    cx->insideDebuggerEvaluationWithOnNativeCallHook = dbg;
+  }
+
+  ~AutoNoteDebuggerEvaluationWithOnNativeCallHook() {
+    cx->insideDebuggerEvaluationWithOnNativeCallHook = oldValue;
+  }
+};
+
 enum UnsafeABIStrictness {
   NoExceptions,
   AllowPendingExceptions,