Bug 1416727 - Fix some problems with Baseline's Call IC. r=djvj

js/src/jit-test/tests/baseline/bug1416727.js

@@ -0,0 +1,16 @@
+// |jit-test| error: too much recursion
+g = newGlobal()
+g.parent = this
+g.eval("new Debugger(parent).onExceptionUnwind = function () {}")
+function test() {
+    function f(n) {
+        if (n != 0) {
+            f(n - 1);
+        }
+        try {
+            test();
+        } finally {}
+    }
+    f(100);
+}
+test();

js/src/jit/BaselineIC.cpp

@@ -2227,7 +2227,7 @@ TryAttachCallStub(JSContext* cx, ICCall_Fallback* stub, HandleScript script, jsb
             return true;
         }
 
-        if (stub->scriptedStubCount() >= ICCall_Fallback::MAX_SCRIPTED_STUBS) {
+        if (stub->state().mode() == ICState::Mode::Megamorphic) {
             // Create a Call_AnyScripted stub.
             JitSpew(JitSpew_BaselineIC, "  Generating Call_AnyScripted stub (cons=%s, spread=%s)",
                     constructing ? "yes" : "no", isSpread ? "yes" : "no");
@@ -2335,9 +2335,9 @@ TryAttachCallStub(JSContext* cx, ICCall_Fallback* stub, HandleScript script, jsb
                 return true;
         }
 
-        if (stub->nativeStubCount() >= ICCall_Fallback::MAX_NATIVE_STUBS) {
+        if (stub->state().mode() == ICState::Mode::Megamorphic) {
             JitSpew(JitSpew_BaselineIC,
-                    "  Too many Call_Native stubs. TODO: add Call_AnyNative!");
+                    "  Megamorphic Call_Native stubs. TODO: add Call_AnyNative!");
             return true;
         }
 
@@ -2518,18 +2518,16 @@ DoCallFallback(JSContext* cx, BaselineFrame* frame, ICCall_Fallback* stub_, uint
                     SetUpdateStubData(newStub->toCacheIR_Updated(), gen.typeCheckInfo());
             }
         }
-        if (!handled)
-            stub->state().trackNotAttached();
-    }
 
-    // Try attaching a regular call stub, but only if the CacheIR attempt didn't add
-    // any stubs.
-    if (!handled) {
-        bool createSingleton = ObjectGroup::useSingletonForNewObject(cx, script, pc);
-        if (!TryAttachCallStub(cx, stub, script, pc, op, argc, vp, constructing, false,
-                               createSingleton, &handled))
-        {
-            return false;
+        // Try attaching a regular call stub, but only if the CacheIR attempt didn't add
+        // any stubs.
+        if (!handled) {
+            bool createSingleton = ObjectGroup::useSingletonForNewObject(cx, script, pc);
+            if (!TryAttachCallStub(cx, stub, script, pc, op, argc, vp, constructing, false,
+                                   createSingleton, &handled))
+            {
+                return false;
+            }
         }
     }
 
@@ -2573,7 +2571,12 @@ DoCallFallback(JSContext* cx, BaselineFrame* frame, ICCall_Fallback* stub_, uint
     if (!stub->addMonitorStubForValue(cx, frame, types, res))
         return false;
 
-    if (!handled) {
+    // Try to transition again in case we called this IC recursively.
+    if (stub->state().maybeTransition())
+        stub->discardStubs(cx);
+    canAttachStub = stub->state().canAttachStub();
+
+    if (!handled && canAttachStub) {
         // If 'callee' is a potential Call_ConstStringSplit, try to attach an
         // optimized ConstStringSplit stub. Note that vp[0] now holds the return value
         // instead of the callee, so we pass the callee as well.
@@ -2581,8 +2584,11 @@ DoCallFallback(JSContext* cx, BaselineFrame* frame, ICCall_Fallback* stub_, uint
             return false;
     }
 
-    if (!handled)
+    if (!handled) {
         stub->noteUnoptimizableCall();
+        if (canAttachStub)
+            stub->state().trackNotAttached();
+    }
     return true;
 }
 

js/src/jit/BaselineIC.h

@@ -732,8 +732,6 @@ class ICCall_Fallback : public ICMonitoredFallbackStub
     static const unsigned UNOPTIMIZABLE_CALL_FLAG = 0x1;
 
     static const uint32_t MAX_OPTIMIZED_STUBS = 16;
-    static const uint32_t MAX_SCRIPTED_STUBS = 7;
-    static const uint32_t MAX_NATIVE_STUBS = 7;
 
   private:
     explicit ICCall_Fallback(JitCode* stubCode)
@@ -748,16 +746,9 @@ class ICCall_Fallback : public ICMonitoredFallbackStub
         return extra_ & UNOPTIMIZABLE_CALL_FLAG;
     }
 
-    unsigned scriptedStubCount() const {
-        return numStubsWithKind(Call_Scripted);
-    }
     bool scriptedStubsAreGeneralized() const {
         return hasStub(Call_AnyScripted);
     }
-
-    unsigned nativeStubCount() const {
-        return numStubsWithKind(Call_Native);
-    }
     bool nativeStubsAreGeneralized() const {
         // Return hasStub(Call_AnyNative) after Call_AnyNative stub is added.
         return false;