Cap JIT frames to the interpreter inline call count (bug 522136, r=brendan).

js/src/jsinterp.cpp

@@ -2490,8 +2490,6 @@ JS_STATIC_ASSERT(!CAN_DO_FAST_INC_DEC(INT_TO_JSVAL_CONSTEXPR(JSVAL_INT_MAX)));
 
 #endif
 
-#define MAX_INLINE_CALL_COUNT 3000
-
 /*
  * Threaded interpretation via computed goto appears to be well-supported by
  * GCC 3 and higher.  IBM's C compiler when run with the right options (e.g.,

js/src/jsinterp.h

@@ -587,6 +587,8 @@ js_GetUpvar(JSContext *cx, uintN level, uintN cookie);
 # endif
 #endif
 
+#define JS_MAX_INLINE_CALL_COUNT 3000
+
 #if !JS_LONE_INTERPRET
 # define JS_STATIC_INTERPRET    static
 #else

js/src/jsops.cpp

@@ -2078,7 +2078,7 @@ BEGIN_CASE(JSOP_APPLY)
             JSInterpreterHook hook;
 
             /* Restrict recursion of lightweight functions. */
-            if (inlineCallCount >= MAX_INLINE_CALL_COUNT) {
+            if (inlineCallCount >= JS_MAX_INLINE_CALL_COUNT) {
                 js_ReportOverRecursed(cx);
                 goto error;
             }

js/src/jstracer.cpp

@@ -6368,11 +6368,14 @@ ExecuteTree(JSContext* cx, Fragment* f, uintN& inlineCallCount,
     state->sp = stack_buffer + (ti->nativeStackBase/sizeof(double));
     state->eos = stack_buffer + MAX_NATIVE_STACK_SLOTS;
 
+    JS_ASSERT(JS_MAX_INLINE_CALL_COUNT > inlineCallCount);
+
     /* Set up the native call stack frame. */
     FrameInfo* callstack_buffer[MAX_CALL_STACK_ENTRIES];
     state->callstackBase = callstack_buffer;
     state->rp = callstack_buffer;
-    state->eor = callstack_buffer + MAX_CALL_STACK_ENTRIES;
+    state->eor = callstack_buffer +
+                 JS_MIN(MAX_CALL_STACK_ENTRIES, JS_MAX_INLINE_CALL_COUNT - inlineCallCount);
     state->sor = state->rp;
 
 #ifdef DEBUG

js/src/trace-test/tests/basic/bug520498.js

@@ -5,5 +5,5 @@ try {
 }
 
 if (Q == 100000)
-	assertEq(Q, "fail");
+   assertEq(Q, "fail");
 

js/src/trace-test/tests/basic/bug522136.js

@@ -0,0 +1,8 @@
+var Q = 0;
+try {
+   (function f(i) { Q = i; if (i == 100000) return; f(i+1); })(1)
+} catch (e) {
+}
+
+assertEq(Q, 3000);
+