Bug 1221747 - Fix OOM handling in IonScriptCounts r=nbp

js/src/jit-test/tests/gc/bug-1221747.js

@@ -0,0 +1,8 @@
+// |jit-test| --dump-bytecode
+if (!('oomTest' in this))
+    quit();
+
+function f() {
+    eval("(function() {})()");
+}
+oomTest(f);

js/src/jit/CodeGenerator.cpp

@@ -3849,7 +3849,7 @@ struct ScriptCountBlockState
 
   public:
     ScriptCountBlockState(IonBlockCounts* block, MacroAssembler* masm)
-      : block(*block), masm(*masm), printer(GetJitContext()->cx)
+      : block(*block), masm(*masm), printer(GetJitContext()->cx, false)
     {
     }
 
@@ -3883,7 +3883,8 @@ struct ScriptCountBlockState
     {
         masm.setPrinter(nullptr);
 
-        block.setCode(printer.string());
+        if (!printer.hadOutOfMemory())
+            block.setCode(printer.string());
     }
 };
 

js/src/jit/IonCode.h

@@ -669,7 +669,7 @@ struct IonBlockCounts
     }
 
     void setCode(const char* code) {
-        char* ncode = (char*) js_malloc(strlen(code) + 1);
+        char* ncode = js_pod_malloc<char>(strlen(code) + 1);
         if (ncode) {
             strcpy(ncode, code);
             code_ = ncode;
@@ -715,9 +715,12 @@ struct IonScriptCounts
     }
 
     bool init(size_t numBlocks) {
-        numBlocks_ = numBlocks;
         blocks_ = js_pod_calloc<IonBlockCounts>(numBlocks);
-        return blocks_ != nullptr;
+        if (!blocks_)
+            return false;
+
+        numBlocks_ = numBlocks;
+        return true;
     }
 
     size_t numBlocks() const {

js/src/vm/Printer.cpp

@@ -19,22 +19,22 @@
 namespace js {
 
 GenericPrinter::GenericPrinter()
-  : reportedOOM_(false)
+  : hadOOM_(false)
 {
 }
 
 void
 GenericPrinter::reportOutOfMemory()
 {
-    if (reportedOOM_)
+    if (hadOOM_)
         return;
-    reportedOOM_ = true;
+    hadOOM_ = true;
 }
 
 bool
 GenericPrinter::hadOutOfMemory() const
 {
-    return reportedOOM_;
+    return hadOOM_;
 }
 
 int
@@ -88,11 +88,12 @@ Sprinter::realloc_(size_t newSize)
     return true;
 }
 
-Sprinter::Sprinter(ExclusiveContext* cx)
+Sprinter::Sprinter(ExclusiveContext* cx, bool shouldReportOOM)
   : context(cx),
 #ifdef DEBUG
     initialized(false),
 #endif
+    shouldReportOOM(shouldReportOOM),
     base(nullptr), size(0), offset(0)
 { }
 
@@ -207,7 +208,7 @@ Sprinter::vprintf(const char* fmt, va_list ap)
     do {
         va_list aq;
         va_copy(aq, ap);
-        int i = vsnprintf(base + offset, size - offset, fmt, aq);
+        int i = JS_vsnprintf(base + offset, size - offset, fmt, aq);
         va_end(aq);
         if (i > -1 && (size_t) i < size - offset) {
             offset += i;
@@ -254,11 +255,11 @@ Sprinter::getOffset() const
 void
 Sprinter::reportOutOfMemory()
 {
-    if (reportedOOM_)
+    if (hadOOM_)
         return;
-    if (context)
+    if (context && shouldReportOOM)
         ReportOutOfMemory(context);
-    reportedOOM_ = true;
+    hadOOM_ = true;
 }
 
 ptrdiff_t
@@ -509,7 +510,7 @@ LSprinter::clear()
     head_ = nullptr;
     tail_ = nullptr;
     unused_ = 0;
-    reportedOOM_ = false;
+    hadOOM_ = false;
 }
 
 int
@@ -597,15 +598,15 @@ LSprinter::vprintf(const char* fmt, va_list ap)
 void
 LSprinter::reportOutOfMemory()
 {
-    if (reportedOOM_)
+    if (hadOOM_)
         return;
-    reportedOOM_ = true;
+    hadOOM_ = true;
 }
 
 bool
 LSprinter::hadOutOfMemory() const
 {
-    return reportedOOM_;
+    return hadOOM_;
 }
 
 } // namespace js

js/src/vm/Printer.h

@@ -26,7 +26,7 @@ class LifoAlloc;
 class GenericPrinter
 {
   protected:
-    bool                    reportedOOM_;   // record reported OOM.
+    bool                  hadOOM_;     // whether reportOutOfMemory() has been called.
 
     GenericPrinter();
 
@@ -66,21 +66,22 @@ class Sprinter final : public GenericPrinter
         }
     };
 
-    ExclusiveContext*       context;        // context executing the decompiler
+    ExclusiveContext*     context;          // context executing the decompiler
 
   private:
-    static const size_t     DefaultSize;
+    static const size_t   DefaultSize;
 #ifdef DEBUG
-    bool                    initialized;    // true if this is initialized, use for debug builds
+    bool                  initialized;      // true if this is initialized, use for debug builds
 #endif
-    char*                   base;           // malloc'd buffer address
-    size_t                  size;           // size of buffer allocated at base
-    ptrdiff_t               offset;         // offset of next free char in buffer
+    bool                  shouldReportOOM;  // whether to report OOM to the context
+    char*                 base;             // malloc'd buffer address
+    size_t                size;             // size of buffer allocated at base
+    ptrdiff_t             offset;           // offset of next free char in buffer
 
     bool realloc_(size_t newSize);
 
   public:
-    explicit Sprinter(ExclusiveContext* cx);
+    explicit Sprinter(ExclusiveContext* cx, bool shouldReportOOM = true);
     ~Sprinter();
 
     // Initialize this sprinter, returns false on error.