Bug 1215363 - Fix a couple of OOM handling issues and make JS_sprintf funcions crash when passed illegal format strings r=terrence

js/src/jit-test/tests/gc/bug-1215363-1.js

@@ -0,0 +1,4 @@
+if (!('oomTest' in this))
+    quit();
+
+oomTest(() => parseModule(10));

js/src/jit-test/tests/gc/bug-1215363-2.js

@@ -0,0 +1,7 @@
+if (!('oomTest' in this))
+    quit();
+
+var lfcode = new Array();
+oomTest((function(x) {
+    assertEq(...Object);
+}));

js/src/jit-test/tests/gc/bug-1215363-3.js

@@ -0,0 +1,5 @@
+if (!('oomTest' in this))
+    quit();
+
+var lfcode = new Array();
+oomTest(() => getBacktrace({}));

js/src/jscntxt.cpp

@@ -432,8 +432,10 @@ js::ReportErrorVA(JSContext* cx, unsigned flags, const char* format, va_list ap)
         return true;
 
     message = JS_vsmprintf(format, ap);
-    if (!message)
+    if (!message) {
+        ReportOutOfMemory(cx);
         return false;
+    }
     messagelen = strlen(message);
 
     report.flags = flags;

js/src/jsprf.cpp

@@ -375,11 +375,11 @@ BuildArgArray(const char* fmt, va_list ap, NumArgStateVector& nas)
             if (c > '9' || c < '0') {
                 if (c == '$') {         // numbered argument case
                     if (i > 0)
-                        return false;
+                        MOZ_CRASH("Bad format string");
                     number++;
                 } else {                // non-numbered argument case
                     if (number > 0)
-                        return false;
+                        MOZ_CRASH("Bad format string");
                     i = 1;
                 }
                 break;
@@ -417,7 +417,7 @@ BuildArgArray(const char* fmt, va_list ap, NumArgStateVector& nas)
         }
 
         if (!c || cn < 1 || cn > number)
-            return false;
+            MOZ_CRASH("Bad format string");
 
         // nas[cn] starts from 0, and make sure nas[cn].type is not assigned.
         cn--;
@@ -429,7 +429,7 @@ BuildArgArray(const char* fmt, va_list ap, NumArgStateVector& nas)
         // width
         if (c == '*') {
             // not supported feature, for the argument is not numbered
-            return false;
+            MOZ_CRASH("Bad format string");
         }
 
         while ((c >= '0') && (c <= '9')) {
@@ -441,7 +441,7 @@ BuildArgArray(const char* fmt, va_list ap, NumArgStateVector& nas)
             c = *p++;
             if (c == '*') {
                 // not supported feature, for the argument is not numbered
-                return false;
+                MOZ_CRASH("Bad format string");
             }
 
             while ((c >= '0') && (c <= '9')) {
@@ -527,7 +527,7 @@ BuildArgArray(const char* fmt, va_list ap, NumArgStateVector& nas)
 
         // get a legal para.
         if (nas[cn].type == TYPE_UNKNOWN)
-            return false;
+            MOZ_CRASH("Bad format string");
     }
 
 
@@ -557,7 +557,7 @@ BuildArgArray(const char* fmt, va_list ap, NumArgStateVector& nas)
         case TYPE_INTSTR:       (void) va_arg(ap, int*);        break;
         case TYPE_DOUBLE:       (void) va_arg(ap, double);      break;
 
-        default: return false;
+        default: MOZ_CRASH();
         }
 
         cn++;
@@ -599,8 +599,7 @@ dosprintf(SprintfState* ss, const char* fmt, va_list ap)
     NumArgStateVector nas;
     if (!BuildArgArray(fmt, ap, nas)) {
         // the fmt contains error Numbered Argument format, jliu@netscape.com
-        MOZ_ASSERT(0);
-        return false;
+        MOZ_CRASH("Bad format string");
     }
 
     while ((c = *fmt++) != 0) {
@@ -633,7 +632,7 @@ dosprintf(SprintfState* ss, const char* fmt, va_list ap)
             }
 
             if (nas[i - 1].type == TYPE_UNKNOWN)
-                return false;
+                MOZ_CRASH("Bad format string");
 
             ap = nas[i - 1].ap;
             dolPt = fmt;

js/src/vm/ObjectGroup.cpp

@@ -1447,8 +1447,10 @@ ObjectGroup::allocationSiteGroup(JSContext* cx, JSScript* script, jsbytecode* pc
             cx->recoverFromOutOfMemory();
     }
 
-    if (!table->add(p, key, res))
+    if (!table->add(p, key, res)) {
+        ReportOutOfMemory(cx);
         return nullptr;
+    }
 
     return res;
 }