From c2478655ef4505e9eca8aca311f662013e917905 Mon Sep 17 00:00:00 2001
From: Igor Bukanov <igor@mir2.org>
Date: Fri, 16 Apr 2010 14:31:17 +0200
Subject: [PATCH] bug 557140 - avoid abort() on OOM with ill-lopping traced
 code. r=jorendorff

---
 js/src/jscntxt.cpp | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/js/src/jscntxt.cpp b/js/src/jscntxt.cpp
index 8af667477fad..72734f56d184 100644
--- a/js/src/jscntxt.cpp
+++ b/js/src/jscntxt.cpp
@@ -1844,11 +1844,27 @@ js_InvokeOperationCallback(JSContext *cx)
      * not yield. Operation callbacks are supposed to happen rarely (seconds,
      * not milliseconds) so it is acceptable to yield at every callback.
      */
-    if (cx->runtime->gcIsNeeded)
+    JSRuntime *rt = cx->runtime;
+    if (rt->gcIsNeeded) {
         js_GC(cx, GC_NORMAL);
+
+        /*
+         * On trace we can exceed the GC quota, see comments in NewGCArena. So
+         * we check the quota and report OOM here when we are off trace.
+         */
+        bool delayedOutOfMemory;
+        JS_LOCK_GC(rt);
+        delayedOutOfMemory = (rt->gcBytes > rt->gcMaxBytes);
+        JS_UNLOCK_GC(rt);
+        if (delayedOutOfMemory) {
+            js_ReportOutOfMemory(cx);
+            return false;
+        }
+    }
 #ifdef JS_THREADSAFE
-    else
+    else {
         JS_YieldRequest(cx);
+    }
 #endif
 
     JSOperationCallback cb = cx->operationCallback;