Bug 797131 part 3 - Add fast path for calling into Ion. r=dvander

js/src/ion/Bailouts.cpp

@@ -247,7 +247,7 @@ ConvertFrames(JSContext *cx, IonActivation *activation, IonBailoutIterator &it)
     activation->setBailout(br);
 
     StackFrame *fp;
-    if (it.isEntryJSFrame() && cx->fp()->runningInIon()) {
+    if (it.isEntryJSFrame() && cx->fp()->runningInIon() && activation->entryfp()) {
         // Avoid creating duplicate interpreter frames. This is necessary to
         // avoid blowing out the interpreter stack, and must be used in
         // conjunction with inline-OSR from within bailouts (since each Ion
@@ -257,6 +257,7 @@ ConvertFrames(JSContext *cx, IonActivation *activation, IonBailoutIterator &it)
         // Note: If the entry frame is a placeholder (a stub frame pushed for
         // JM -> Ion calls), then we cannot re-use it as it does not have
         // enough slots.
+        JS_ASSERT(cx->fp() == activation->entryfp());
         fp = cx->fp();
         cx->regs().sp = fp->base();
     } else {
@@ -410,12 +411,14 @@ ion::InvalidationBailout(InvalidationBailoutStack *sp, size_t *frameSizeOut)
         cx->regs().sp[-1] = cx->runtime->takeIonReturnOverride();
 
     if (retval != BAILOUT_RETURN_FATAL_ERROR) {
-        if (void *annotation = activation->entryfp()->annotation()) {
+        if (activation->entryfp()) {
-            // If the entry frame has an annotation, then we invalidated and have
+            if (void *annotation = activation->entryfp()->annotation()) {
-            // immediately returned into this bailout. Transfer the annotation to
+                // If the entry frame has an annotation, then we invalidated and have
-            // the new topmost frame.
+                // immediately returned into this bailout. Transfer the annotation to
-            activation->entryfp()->setAnnotation(NULL);
+                // the new topmost frame.
-            cx->fp()->setAnnotation(annotation);
+                activation->entryfp()->setAnnotation(NULL);
+                cx->fp()->setAnnotation(annotation);
+            }
         }
 
         // If invalidation was triggered inside a stub call, we may still have to

js/src/ion/Ion.cpp

@@ -1435,6 +1435,63 @@ ion::SideCannon(JSContext *cx, StackFrame *fp, jsbytecode *pc)
     return status;
 }
 
+IonExecStatus
+ion::FastInvoke(JSContext *cx, HandleFunction fun, CallArgs &args)
+{
+    JS_CHECK_RECURSION(cx, return IonExec_Error);
+
+    HandleScript script = fun->script();
+    IonScript *ion = script->ionScript();
+    IonCode *code = ion->method();
+    void *jitcode = code->raw();
+
+    JS_ASSERT(ion::IsEnabled(cx));
+    JS_ASSERT(!script->ion->bailoutExpected());
+
+    bool clearCallingIntoIon = false;
+    StackFrame *fp = cx->fp();
+
+    // Two cases we have to handle:
+    //
+    // (1) fp does not begin an Ion activation. This works exactly
+    //     like invoking Ion from JM: entryfp is set to fp and fp
+    //     has the callingIntoIon flag set.
+    //
+    // (2) fp already begins another IonActivation, for instance:
+    //        JM -> Ion -> array_sort -> Ion
+    //     In this cas we use an IonActivation with entryfp == NULL
+    //     and prevpc != NULL.
+    if (!fp->beginsIonActivation()) {
+        fp->setCallingIntoIon();
+        clearCallingIntoIon = true;
+        cx->runtime->ionActivation->setEntryFp(fp);
+    } else {
+        JS_ASSERT(!cx->runtime->ionActivation->entryfp());
+    }
+
+    cx->runtime->ionActivation->setPrevPc(cx->regs().pc);
+
+    EnterIonCode enter = cx->compartment->ionCompartment()->enterJITInfallible();
+    void *calleeToken = CalleeToToken(fun);
+
+    Value result = Int32Value(fun->nargs);
+    enter(jitcode, args.length() + 1, &args[0] - 1, fp, calleeToken, &result);
+
+    if (clearCallingIntoIon)
+        fp->clearCallingIntoIon();
+
+    cx->runtime->ionActivation->setEntryFp(NULL);
+    cx->runtime->ionActivation->setPrevPc(NULL);
+
+    JS_ASSERT(fp == cx->fp());
+    JS_ASSERT(!cx->runtime->hasIonReturnOverride());
+
+    args.rval().set(result);
+
+    JS_ASSERT_IF(result.isMagic(), result.isMagic(JS_ION_ERROR));
+    return result.isMagic() ? IonExec_Error : IonExec_Ok;
+}
+
 static void
 InvalidateActivation(FreeOp *fop, uint8 *ionTop, bool invalidateAll)
 {

js/src/ion/Ion.h

@@ -237,6 +237,9 @@ enum IonExecStatus
 IonExecStatus Cannon(JSContext *cx, StackFrame *fp);
 IonExecStatus SideCannon(JSContext *cx, StackFrame *fp, jsbytecode *pc);
 
+// Used to enter Ion from C++ natives like Array.map. Called from FastInvokeGuard.
+IonExecStatus FastInvoke(JSContext *cx, HandleFunction fun, CallArgs &args);
+
 // Walk the stack and invalidate active Ion frames for the invalid scripts.
 void Invalidate(types::TypeCompartment &types, FreeOp *fop,
                 const Vector<types::RecompileInfo> &invalid, bool resetUses = true);

js/src/ion/IonCompartment.h

@@ -196,9 +196,17 @@ class IonActivation
         return prevIonTop_;
     }
     jsbytecode *prevpc() const {
-        JS_ASSERT(entryfp_->callingIntoIon());
+        JS_ASSERT_IF(entryfp_, entryfp_->callingIntoIon());
         return prevpc_;
     }
+    void setEntryFp(StackFrame *fp) {
+        JS_ASSERT_IF(fp, !entryfp_);
+        entryfp_ = fp;
+    }
+    void setPrevPc(jsbytecode *pc) {
+        JS_ASSERT_IF(pc, !prevpc_);
+        prevpc_ = pc;
+    }
     void setBailout(BailoutClosure *bailout) {
         JS_ASSERT(!bailout_);
         bailout_ = bailout;
@@ -219,6 +227,12 @@ class IonActivation
     JSCompartment *compartment() const {
         return compartment_;
     }
+    bool empty() const {
+        // If we have an entryfp, this activation is active. However, if
+        // FastInvoke is used, entryfp may be NULL and a non-NULL prevpc
+        // indicates this activation is not empty.
+        return !entryfp_ && !prevpc_;
+    }
 
     static inline size_t offsetOfPrevPc() {
         return offsetof(IonActivation, prevpc_);

js/src/ion/IonFrames.cpp

@@ -359,7 +359,7 @@ ion::HandleException(ResumeFromException *rfe)
 void
 IonActivationIterator::settle()
 {
-    while (activation_ && !activation_->entryfp()) {
+    while (activation_ && activation_->empty()) {
         top_ = activation_->prevIonTop();
         activation_ = activation_->prev();
     }
@@ -1016,7 +1016,14 @@ IonFrameIterator::isConstructing() const
 
     JS_ASSERT(parent.done());
 
-    // JM ICs do not inline Ion constructor calls.
+    // If entryfp is not set, we entered Ion via a C++ native, like Array.map,
+    // using FastInvoke. FastInvoke is never used for constructor calls.
+    if (!activation_->entryfp())
+        return false;
+
+    // If callingIntoIon, we either entered Ion from JM or entered Ion from
+    // a C++ native using FastInvoke. In both of these cases we don't handle
+    // constructor calls.
     if (activation_->entryfp()->callingIntoIon())
         return false;
     JS_ASSERT(activation_->entryfp()->runningInIon());

js/src/jsinterpinlines.h

@@ -27,6 +27,7 @@
 #include "jstypedarrayinlines.h"
 
 #include "ion/Ion.h"
+#include "ion/IonCompartment.h"
 
 #include "vm/Stack-inl.h"
 
@@ -991,12 +992,22 @@ class FastInvokeGuard
 {
     InvokeArgsGuard args_;
     RootedFunction fun_;
+    RootedScript script_;
+#ifdef JS_ION
+    ion::IonContext ictx_;
+    ion::IonActivation activation_;
     bool useIon_;
+#endif
 
   public:
     FastInvokeGuard(JSContext *cx, const Value &fval)
       : fun_(cx),
+        script_(cx)
+#ifdef JS_ION
+        , ictx_(cx, cx->compartment, NULL),
+        activation_(cx, NULL),
         useIon_(ion::IsEnabled(cx))
+#endif
     {
         initFunction(fval);
     }
@@ -1004,8 +1015,10 @@ class FastInvokeGuard
     void initFunction(const Value &fval) {
         if (fval.isObject() && fval.toObject().isFunction()) {
             JSFunction *fun = fval.toObject().toFunction();
-            if (fun->isInterpreted())
+            if (fun->isInterpreted()) {
                 fun_ = fun;
+                script_ = fun->script();
+            }
         }
     }
 
@@ -1014,6 +1027,27 @@ class FastInvokeGuard
     }
 
     bool invoke(JSContext *cx) {
+#ifdef JS_ION
+        if (useIon_ && fun_) {
+            JS_ASSERT(fun_->script() == script_);
+
+            if (script_->hasIonScript() && !script_->ion->bailoutExpected()) {
+                ion::IonExecStatus result = ion::FastInvoke(cx, fun_, args_);
+                if (result == ion::IonExec_Error)
+                    return false;
+
+                JS_ASSERT(result == ion::IonExec_Ok);
+                return true;
+            }
+
+            if (script_->canIonCompile()) {
+                // This script is not yet hot. Since calling into Ion is much
+                // faster here, bump the use count a bit to account for this.
+                script_->incUseCount(5);
+            }
+        }
+#endif
+
         return Invoke(cx, args_);
     }
 

js/src/vm/Stack.cpp

@@ -1461,7 +1461,21 @@ StackIter::popIonFrame()
             ionInlineFrames_ = ion::InlineFrameIterator(&ionFrames_);
             pc_ = ionInlineFrames_.pc();
             script_ = ionInlineFrames_.script();
-        } else if (fp_->runningInIon()) {
+            return;
+        }
+
+        // The activation has no other frames. If entryfp is NULL, it was invoked
+        // by a native written in C++, using FastInvoke, on top of another activation.
+        ion::IonActivation *activation = ionActivations_.activation();
+        if (!activation->entryfp()) {
+            JS_ASSERT(activation->prevpc());
+            JS_ASSERT(fp_->beginsIonActivation());
+            ++ionActivations_;
+            settleOnNewState();
+            return;
+        }
+
+        if (fp_->runningInIon()) {
             ++ionActivations_;
             popFrame();
             settleOnNewState();