Bug 967589 - Add OOM test for runtime creation and fix failures r=billm

js/src/jsapi-tests/testOOM.cpp

@@ -25,3 +25,57 @@ virtual JSRuntime * createRuntime()
     return rt;
 }
 END_TEST(testOOM)
+
+#ifdef DEBUG  // OOM_maxAllocations is only available in debug builds.
+
+const uint32_t maxAllocsPerTest = 100;
+
+#define START_OOM_TEST(name)                                                  \
+    testName = name;                                                          \
+    printf("Test %s: started\n", testName);                                   \
+    for (oomAfter = 1; oomAfter < maxAllocsPerTest; ++oomAfter) {             \
+        setOOMAfter(oomAfter)
+
+#define OOM_TEST_FINISHED                                                     \
+    {                                                                         \
+        printf("Test %s: finished with %d allocations\n",                     \
+               testName, oomAfter - 1);                                       \
+        break;                                                                \
+    }
+
+#define END_OOM_TEST                                                          \
+    }                                                                         \
+    cancelOOMAfter();                                                         \
+    CHECK(oomAfter != maxAllocsPerTest)
+
+BEGIN_TEST(testNewRuntime)
+{
+    uninit(); // Get rid of test harness' original JSRuntime.
+
+    JSRuntime *rt;
+    START_OOM_TEST("new runtime");
+    rt = JS_NewRuntime(8L * 1024 * 1024, JS_USE_HELPER_THREADS);
+    if (rt)
+        OOM_TEST_FINISHED;
+    END_OOM_TEST;
+    JS_DestroyRuntime(rt);
+    return true;
+}
+
+const char* testName;
+uint32_t oomAfter;
+
+void
+setOOMAfter(uint32_t numAllocs)
+{
+    OOM_maxAllocations = OOM_counter + numAllocs;
+}
+
+void
+cancelOOMAfter()
+{
+    OOM_maxAllocations = UINT32_MAX;
+}
+END_TEST(testNewRuntime)
+
+#endif

js/src/jsgc.cpp

@@ -1150,23 +1150,28 @@ js_FinishGC(JSRuntime *rt)
 #endif
 
     /* Delete all remaining zones. */
-    for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
-        for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
-            js_delete(comp.get());
-        js_delete(zone.get());
+    if (rt->gcInitialized) {
+        for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+            for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
+                js_delete(comp.get());
+            js_delete(zone.get());
+        }
     }
 
     rt->zones.clear();
 
     rt->gcSystemAvailableChunkListHead = nullptr;
     rt->gcUserAvailableChunkListHead = nullptr;
-    for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront())
-        Chunk::release(rt, r.front());
-    rt->gcChunkSet.clear();
+    if (rt->gcChunkSet.initialized()) {
+        for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront())
+            Chunk::release(rt, r.front());
+        rt->gcChunkSet.clear();
+    }
 
     rt->gcChunkPool.expireAndFree(rt, true);
 
-    rt->gcRootsHash.clear();
+    if (rt->gcRootsHash.initialized())
+        rt->gcRootsHash.clear();
 
     rt->functionPersistentRooteds.clear();
     rt->idPersistentRooteds.clear();

js/src/vm/RegExpObject.cpp

@@ -650,8 +650,8 @@ RegExpCompartment::RegExpCompartment(JSRuntime *rt)
 
 RegExpCompartment::~RegExpCompartment()
 {
-    JS_ASSERT(map_.empty());
-    JS_ASSERT(inUse_.empty());
+    JS_ASSERT_IF(map_.initialized(), map_.empty());
+    JS_ASSERT_IF(inUse_.initialized(), inUse_.empty());
 }
 
 JSObject *

js/src/vm/Runtime.cpp

@@ -169,6 +169,7 @@ JSRuntime::JSRuntime(JSUseHelperThreads useHelperThreads)
     checkRequestDepth(0),
 # endif
 #endif
+    gcInitialized(false),
     gcSystemAvailableChunkListHead(nullptr),
     gcUserAvailableChunkListHead(nullptr),
     gcBytes(0),
@@ -404,17 +405,20 @@ JSRuntime::init(uint32_t maxbytes)
     if (!InitAtoms(this))
         return false;
 
-    if (!InitRuntimeNumberState(this))
-        return false;
-
-    dateTimeInfo.updateTimeZoneAdjustment();
-
     if (!scriptDataTable_.init())
         return false;
 
     if (!evalCache.init())
         return false;
 
+    /* The garbage collector depends on everything before this point being initialized. */
+    gcInitialized = true;
+
+    if (!InitRuntimeNumberState(this))
+        return false;
+
+    dateTimeInfo.updateTimeZoneAdjustment();
+
 #ifdef JS_ARM_SIMULATOR
     simulatorRuntime_ = js::jit::CreateSimulatorRuntime();
     if (!simulatorRuntime_)
@@ -435,44 +439,46 @@ JSRuntime::~JSRuntime()
 {
     JS_ASSERT(!isHeapBusy());
 
-    /* Free source hook early, as its destructor may want to delete roots. */
-    sourceHook = nullptr;
+    if (gcInitialized) {
+        /* Free source hook early, as its destructor may want to delete roots. */
+        sourceHook = nullptr;
 
-    /*
-     * Cancel any pending, in progress or completed Ion compilations and
-     * parse tasks. Waiting for AsmJS and compression tasks is done
-     * synchronously (on the main thread or during parse tasks), so no
-     * explicit canceling is needed for these.
-     */
-    for (CompartmentsIter comp(this, SkipAtoms); !comp.done(); comp.next())
-        CancelOffThreadIonCompile(comp, nullptr);
-    CancelOffThreadParses(this);
+        /*
+         * Cancel any pending, in progress or completed Ion compilations and
+         * parse tasks. Waiting for AsmJS and compression tasks is done
+         * synchronously (on the main thread or during parse tasks), so no
+         * explicit canceling is needed for these.
+         */
+        for (CompartmentsIter comp(this, SkipAtoms); !comp.done(); comp.next())
+            CancelOffThreadIonCompile(comp, nullptr);
+        CancelOffThreadParses(this);
 
-    /* Poison common names before final GC. */
-    FinishCommonNames(this);
+        /* Poison common names before final GC. */
+        FinishCommonNames(this);
 
-    /* Clear debugging state to remove GC roots. */
-    for (CompartmentsIter comp(this, SkipAtoms); !comp.done(); comp.next()) {
-        comp->clearTraps(defaultFreeOp());
-        if (WatchpointMap *wpmap = comp->watchpointMap)
-            wpmap->clear();
+        /* Clear debugging state to remove GC roots. */
+        for (CompartmentsIter comp(this, SkipAtoms); !comp.done(); comp.next()) {
+            comp->clearTraps(defaultFreeOp());
+            if (WatchpointMap *wpmap = comp->watchpointMap)
+                wpmap->clear();
+        }
+
+        /* Clear the statics table to remove GC roots. */
+        staticStrings.finish();
+
+        /*
+         * Flag us as being destroyed. This allows the GC to free things like
+         * interned atoms and Ion trampolines.
+         */
+        beingDestroyed_ = true;
+
+        /* Allow the GC to release scripts that were being profiled. */
+        profilingScripts = false;
+
+        JS::PrepareForFullGC(this);
+        GC(this, GC_NORMAL, JS::gcreason::DESTROY_RUNTIME);
     }
 
-    /* Clear the statics table to remove GC roots. */
-    staticStrings.finish();
-
-    /*
-     * Flag us as being destroyed. This allows the GC to free things like
-     * interned atoms and Ion trampolines.
-     */
-    beingDestroyed_ = true;
-
-    /* Allow the GC to release scripts that were being profiled. */
-    profilingScripts = false;
-
-    JS::PrepareForFullGC(this);
-    GC(this, GC_NORMAL, JS::gcreason::DESTROY_RUNTIME);
-
     /*
      * Clear the self-hosted global and delete self-hosted classes *after*
      * GC, as finalizers for objects check for clasp->finalize during GC.

js/src/vm/Runtime.h

@@ -1016,6 +1016,9 @@ struct JSRuntime : public JS::shadow::Runtime,
 
     /* Garbage collector state, used by jsgc.c. */
 
+    /* Garbase collector state has been sucessfully initialized. */
+    bool                gcInitialized;
+
     /*
      * Set of all GC chunks with at least one allocated thing. The
      * conservative GC uses it to quickly check if a possible GC thing points