Separate gcMaxBytes from gcMaxMallocBytes and unconstrain the former for Mozilla code (317865, r/sr=mrbkap/shaver).

js/src/jsapi.c

@@ -1889,6 +1889,19 @@ JS_IsAboutToBeFinalized(JSContext *cx, void *thing)
     return js_IsAboutToBeFinalized(cx, thing);
 }
 
+JS_PUBLIC_API(void)
+JS_SetGCParameter(JSRuntime *rt, JSGCParamKey key, uint32 value)
+{
+    switch (key) {
+      case JSGC_MAX_BYTES:
+        rt->gcMaxBytes = value;
+        break;
+      case JSGC_MAX_MALLOC_BYTES:
+        rt->gcMaxMallocBytes = value;
+        break;
+    }
+}
+
 JS_PUBLIC_API(intN)
 JS_AddExternalStringFinalizer(JSStringFinalizeOp finalizer)
 {

js/src/jsapi.h

@@ -814,6 +814,14 @@ JS_SetGCCallbackRT(JSRuntime *rt, JSGCCallback cb);
 extern JS_PUBLIC_API(JSBool)
 JS_IsAboutToBeFinalized(JSContext *cx, void *thing);
 
+typedef enum JSGCParamKey {
+    JSGC_MAX_BYTES        = 0,  /* maximum nominal heap before last ditch GC */
+    JSGC_MAX_MALLOC_BYTES = 1   /* # of JS_malloc bytes before last ditch GC */
+} JSGCParamKey;
+
+extern JS_PUBLIC_API(void)
+JS_SetGCParameter(JSRuntime *rt, JSGCParamKey key, uint32 value);
+
 /*
  * Add a finalizer for external strings created by JS_NewExternalString (see
  * below) using a type-code returned from this function, and that understands

js/src/jscntxt.h

@@ -83,6 +83,7 @@ struct JSRuntime {
     uint32              gcBytes;
     uint32              gcLastBytes;
     uint32              gcMaxBytes;
+    uint32              gcMaxMallocBytes;
     uint32              gcLevel;
     uint32              gcNumber;
     JSPackedBool        gcPoke;

js/src/jsgc.c

@@ -378,7 +378,12 @@ js_InitGC(JSRuntime *rt, uint32 maxbytes)
         return JS_FALSE;
     }
     rt->gcLocksHash = NULL;     /* create lazily */
-    rt->gcMaxBytes = maxbytes;
+
+    /*
+     * Separate gcMaxMallocBytes from gcMaxBytes but initialize to maxbytes
+     * for default backward API compatibility.
+     */
+    rt->gcMaxBytes = rt->gcMaxMallocBytes = maxbytes;
     return JS_TRUE;
 }
 
@@ -649,7 +654,7 @@ js_NewGCThing(JSContext *cx, uintN flags, size_t nbytes)
          * arenaList.
          */
         if (rt->gcBytes < rt->gcMaxBytes &&
-            (tried_gc || rt->gcMallocBytes < rt->gcMaxBytes)) {
+            (tried_gc || rt->gcMallocBytes < rt->gcMaxMallocBytes)) {
             if (!arenaList->last || arenaList->lastLimit == GC_THINGS_SIZE) {
                 /*
                  * The last arena (and the whole arenaList) is full, time

js/src/xpconnect/src/xpcruntimesvc.cpp

@@ -140,6 +140,14 @@ nsJSRuntimeServiceImpl::GetRuntime(JSRuntime **runtime)
         mRuntime = JS_NewRuntime(gGCSize);
         if(!mRuntime)
             return NS_ERROR_OUT_OF_MEMORY;
+
+        // Unconstrain the runtime's threshold on nominal heap size, to avoid
+        // triggering GC too often if operating continuously near an arbitrary
+        // finite threshold (0xffffffff is infinity for uint32 parameters).
+        // This leaves the maximum-JS_malloc-bytes threshold still in effect
+        // to cause period, and we hope hygienic, last-ditch GCs from within
+        // the GC's allocator.
+        JS_SetGCParameter(mRuntime, JSGC_MAX_BYTES, 0xffffffff);
     }
     *runtime = mRuntime;
     return NS_OK;