Deep abort recorders outer recorders if we need to flush the JIT cache early (bug 463829, r=brendan,gal).

js/src/jscntxt.h

@@ -133,6 +133,9 @@ typedef struct JSTraceMonitor {
      * a distinct compiler but needs to be managed in exactly the same
      * way as the real tracing Fragmento. */
     CLS(nanojit::Fragmento) reFragmento;
+
+    /* Keep a list of recorders we need to abort on cache flush. */
+    CLS(TraceRecorder)      abortStack;
 } JSTraceMonitor;
 
 #ifdef JS_TRACER

js/src/jsinterp.cpp

@@ -2579,6 +2579,7 @@ js_Interpret(JSContext *cx)
         tr = TRACE_RECORDER(cx);
         SET_TRACE_RECORDER(cx, NULL);
         JS_TRACE_MONITOR(cx).onTrace = JS_FALSE;
+        tr->pushAbortStack();
     }
 #endif
 
@@ -7188,7 +7189,10 @@ js_Interpret(JSContext *cx)
     if (tr) {
         JS_TRACE_MONITOR(cx).onTrace = JS_TRUE;
         SET_TRACE_RECORDER(cx, tr);
-        tr->deepAbort();
+        if (!tr->wasDeepAborted()) {
+            tr->popAbortStack();
+            tr->deepAbort();
+        }
     }
 #endif
     return ok;

js/src/jstracer.cpp

@@ -1001,6 +1001,7 @@ TraceRecorder::TraceRecorder(JSContext* cx, VMSideExit* _anchor, Fragment* _frag
     this->global_dslots = this->globalObj->dslots;
     this->terminate = false;
     this->outerToBlacklist = outerToBlacklist;
+    this->wasRootFragment = _fragment == _fragment->root;
 
     debug_only_v(printf("recording starting from %s:%u@%u\n",
                         cx->fp->script->filename,
@@ -1057,13 +1058,18 @@ TreeInfo::~TreeInfo()
 
 TraceRecorder::~TraceRecorder()
 {
-    JS_ASSERT(treeInfo && fragment);
-    if (fragment == fragment->root && !fragment->root->code()) {
-        JS_ASSERT(!fragment->root->vmprivate);
+    JS_ASSERT(nextRecorderToAbort == NULL);
+    JS_ASSERT(treeInfo && (fragment || wasDeepAborted()));
+    if (fragment) {
+        if (wasRootFragment && !fragment->root->code()) {
+            JS_ASSERT(!fragment->root->vmprivate);
+            delete treeInfo;
+        }
+        if (trashTree)
+            js_TrashTree(cx, whichTreeToTrash);
+    } else if (wasRootFragment) {
         delete treeInfo;
     }
-    if (trashTree)
-        js_TrashTree(cx, whichTreeToTrash);
 #ifdef DEBUG
     delete verbose_filter;
 #endif
@@ -1076,6 +1082,11 @@ TraceRecorder::~TraceRecorder()
     delete lir_buf_writer;
 }
 
+void TraceRecorder::removeFragmentoReferences()
+{
+    fragment = NULL;
+}
+
 /* Add debug information to a LIR instruction as we emit it. */
 inline LIns*
 TraceRecorder::addName(LIns* ins, const char* name)
@@ -3931,6 +3942,28 @@ js_FinishJIT(JSTraceMonitor *tm)
     }
 }
 
+void
+TraceRecorder::pushAbortStack()
+{
+    JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
+
+    JS_ASSERT(tm->abortStack != this);
+
+    nextRecorderToAbort = tm->abortStack;
+    tm->abortStack = this;
+}
+
+void
+TraceRecorder::popAbortStack()
+{
+    JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
+
+    JS_ASSERT(tm->abortStack == this);
+
+    tm->abortStack = nextRecorderToAbort;
+    nextRecorderToAbort = NULL;
+}
+
 extern void
 js_FlushJITOracle(JSContext* cx)
 {
@@ -3948,6 +3981,12 @@ js_FlushJITCache(JSContext* cx)
     JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
     if (tm->recorder)
         js_AbortRecording(cx, "flush cache");
+    TraceRecorder* tr;
+    while ((tr = tm->abortStack) != NULL) {
+        tr->removeFragmentoReferences();
+        tr->deepAbort();
+        tr->popAbortStack();
+    }
     Fragmento* fragmento = tm->fragmento;
     if (fragmento) {
         fragmento->clearFrags();

js/src/jstracer.h

@@ -293,6 +293,8 @@ class TraceRecorder : public avmplus::GCObject {
     intptr_t                terminate_ip_adj;
     nanojit::Fragment*      outerToBlacklist;
     nanojit::Fragment*      promotedPeer;
+    TraceRecorder*          nextRecorderToAbort;
+    bool                    wasRootFragment;
 
     bool isGlobal(jsval* p) const;
     ptrdiff_t nativeGlobalOffset(jsval* p) const;
@@ -438,6 +440,9 @@ public:
     void prepareTreeCall(nanojit::Fragment* inner);
     void emitTreeCall(nanojit::Fragment* inner, VMSideExit* exit);
     unsigned getCallDepth() const;
+    void pushAbortStack();
+    void popAbortStack();
+    void removeFragmentoReferences();
 
     bool record_EnterFrame();
     bool record_LeaveFrame();