Bug 1416727 - Fix some problems with Baseline's Call IC. r=djvj

js/src/jit-test/tests/baseline/bug1416727.js

@@ -0,0 +1,19 @@
+g = newGlobal()
+g.parent = this
+g.eval("new Debugger(parent).onExceptionUnwind = function(){}");
+var depth = 0;
+function test() {
+    if (++depth > 50)
+        return;
+    function f(n) {
+        if (n != 0) {
+            f(n - 1);
+            return;
+        }
+        try {
+            test();
+        } finally {}
+    }
+    f(80);
+}
+test();

js/src/jit/BaselineIC.cpp

@@ -2230,7 +2230,7 @@ TryAttachCallStub(JSContext* cx, ICCall_Fallback* stub, HandleScript script, jsb
             return true;
         }
 
-        if (stub->scriptedStubCount() >= ICCall_Fallback::MAX_SCRIPTED_STUBS) {
+        if (stub->state().mode() == ICState::Mode::Megamorphic) {
             // Create a Call_AnyScripted stub.
             JitSpew(JitSpew_BaselineIC, "  Generating Call_AnyScripted stub (cons=%s, spread=%s)",
                     constructing ? "yes" : "no", isSpread ? "yes" : "no");
@@ -2338,9 +2338,9 @@ TryAttachCallStub(JSContext* cx, ICCall_Fallback* stub, HandleScript script, jsb
                 return true;
         }
 
-        if (stub->nativeStubCount() >= ICCall_Fallback::MAX_NATIVE_STUBS) {
+        if (stub->state().mode() == ICState::Mode::Megamorphic) {
             JitSpew(JitSpew_BaselineIC,
-                    "  Too many Call_Native stubs. TODO: add Call_AnyNative!");
+                    "  Megamorphic Call_Native stubs. TODO: add Call_AnyNative!");
             return true;
         }
 
@@ -2520,9 +2520,6 @@ DoCallFallback(JSContext* cx, BaselineFrame* frame, ICCall_Fallback* stub_, uint
                     SetUpdateStubData(newStub->toCacheIR_Updated(), gen.typeCheckInfo());
             }
         }
-        if (!handled)
-            stub->state().trackNotAttached();
-    }
 
         // Try attaching a regular call stub, but only if the CacheIR attempt didn't add
         // any stubs.
@@ -2534,6 +2531,7 @@ DoCallFallback(JSContext* cx, BaselineFrame* frame, ICCall_Fallback* stub_, uint
                 return false;
             }
         }
+    }
 
     if (constructing) {
         if (!ConstructFromStack(cx, callArgs))
@@ -2575,7 +2573,12 @@ DoCallFallback(JSContext* cx, BaselineFrame* frame, ICCall_Fallback* stub_, uint
     if (!stub->addMonitorStubForValue(cx, frame, types, res))
         return false;
 
-    if (!handled) {
+    // Try to transition again in case we called this IC recursively.
+    if (stub->state().maybeTransition())
+        stub->discardStubs(cx);
+    canAttachStub = stub->state().canAttachStub();
+
+    if (!handled && canAttachStub) {
         // If 'callee' is a potential Call_ConstStringSplit, try to attach an
         // optimized ConstStringSplit stub. Note that vp[0] now holds the return value
         // instead of the callee, so we pass the callee as well.
@@ -2583,8 +2586,11 @@ DoCallFallback(JSContext* cx, BaselineFrame* frame, ICCall_Fallback* stub_, uint
             return false;
     }
 
-    if (!handled)
+    if (!handled) {
         stub->noteUnoptimizableCall();
+        if (canAttachStub)
+            stub->state().trackNotAttached();
+    }
     return true;
 }
 

js/src/jit/BaselineIC.h

@@ -732,8 +732,6 @@ class ICCall_Fallback : public ICMonitoredFallbackStub
     static const unsigned UNOPTIMIZABLE_CALL_FLAG = 0x1;
 
     static const uint32_t MAX_OPTIMIZED_STUBS = 16;
-    static const uint32_t MAX_SCRIPTED_STUBS = 7;
-    static const uint32_t MAX_NATIVE_STUBS = 7;
 
   private:
     explicit ICCall_Fallback(JitCode* stubCode)
@@ -748,16 +746,9 @@ class ICCall_Fallback : public ICMonitoredFallbackStub
         return extra_ & UNOPTIMIZABLE_CALL_FLAG;
     }
 
-    unsigned scriptedStubCount() const {
-        return numStubsWithKind(Call_Scripted);
-    }
     bool scriptedStubsAreGeneralized() const {
         return hasStub(Call_AnyScripted);
     }
-
-    unsigned nativeStubCount() const {
-        return numStubsWithKind(Call_Native);
-    }
     bool nativeStubsAreGeneralized() const {
         // Return hasStub(Call_AnyNative) after Call_AnyNative stub is added.
         return false;