Bug 1264948 part 1 - Register if the LifoAlloc is supposed to be infallible or not. r=jonco,h4writer

This patch adds a new flag to the LifoAlloc structure, which would be used to
assert when we attempt to allocate a new chunk for the LifoAlloc.  This ensure
that we assert (in debug builds) if we attempt to allocate beyong the reserved
space of the ballast.

js/src/ds/LifoAlloc.h

@@ -162,6 +162,9 @@ class LifoAlloc
     size_t      defaultChunkSize_;
     size_t      curSize_;
     size_t      peakSize_;
+#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
+    bool        fallibleScope_;
+#endif
 
     void operator=(const LifoAlloc&) = delete;
     LifoAlloc(const LifoAlloc&) = delete;
@@ -231,6 +234,9 @@ class LifoAlloc
   public:
     explicit LifoAlloc(size_t defaultChunkSize)
       : peakSize_(0)
+#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
+      , fallibleScope_(true)
+#endif
     {
         reset(defaultChunkSize);
     }
@@ -270,19 +276,17 @@ class LifoAlloc
 
     MOZ_ALWAYS_INLINE
     void* alloc(size_t n) {
-        JS_OOM_POSSIBLY_FAIL();
+#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
+        // Only simulate OOMs when we are not using the LifoAlloc as an
+        // infallible allocator.
+        if (fallibleScope_)
+            JS_OOM_POSSIBLY_FAIL();
+#endif
         return allocImpl(n);
     }
 
     MOZ_ALWAYS_INLINE
-    void* allocInfallibleOrAssert(size_t n) {
-        void* result = allocImpl(n);
-        MOZ_RELEASE_ASSERT(result, "[OOM] Is it really infallible?");
-        return result;
-    }
-
-    MOZ_ALWAYS_INLINE
-    void* allocInfallibleOrCrash(size_t n) {
+    void* allocInfallible(size_t n) {
         AutoEnterOOMUnsafeRegion oomUnsafe;
         if (void* result = allocImpl(n))
             return result;
@@ -290,16 +294,12 @@ class LifoAlloc
         return nullptr;
     }
 
-    MOZ_ALWAYS_INLINE
-    void* allocInfallible(size_t n) {
-        return allocInfallibleOrCrash(n);
-    }
-
     // Ensures that enough space exists to satisfy N bytes worth of
     // allocation requests, not necessarily contiguous. Note that this does
     // not guarantee a successful single allocation of N bytes.
     MOZ_ALWAYS_INLINE
     MOZ_MUST_USE bool ensureUnusedApproximate(size_t n) {
+        AutoFallibleScope fallibleAllocator(this);
         size_t total = 0;
         for (BumpChunk* chunk = latest; chunk; chunk = chunk->next()) {
             total += chunk->unused();
@@ -314,6 +314,36 @@ class LifoAlloc
         return true;
     }
 
+    MOZ_ALWAYS_INLINE
+    void setAsInfallibleByDefault() {
+#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
+        fallibleScope_ = false;
+#endif
+    }
+
+    class MOZ_NON_TEMPORARY_CLASS AutoFallibleScope {
+#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
+        LifoAlloc* lifoAlloc_;
+        bool prevFallibleScope_;
+        MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER
+
+      public:
+        explicit AutoFallibleScope(LifoAlloc* lifoAlloc MOZ_GUARD_OBJECT_NOTIFIER_PARAM) {
+            MOZ_GUARD_OBJECT_NOTIFIER_INIT;
+            lifoAlloc_ = lifoAlloc;
+            prevFallibleScope_ = lifoAlloc->fallibleScope_;
+            lifoAlloc->fallibleScope_ = true;
+        }
+
+        ~AutoFallibleScope() {
+            lifoAlloc_->fallibleScope_ = prevFallibleScope_;
+        }
+#else
+      public:
+        explicit AutoFallibleScope(LifoAlloc*) {}
+#endif
+    };
+
     template <typename T>
     T* newArray(size_t count) {
         static_assert(mozilla::IsPod<T>::value,
@@ -499,6 +529,7 @@ class MOZ_NON_TEMPORARY_CLASS LifoAllocScope
 {
     LifoAlloc*      lifoAlloc;
     LifoAlloc::Mark mark;
+    LifoAlloc::AutoFallibleScope fallibleScope;
     bool            shouldRelease;
     MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER
 
@@ -507,6 +538,7 @@ class MOZ_NON_TEMPORARY_CLASS LifoAllocScope
                             MOZ_GUARD_OBJECT_NOTIFIER_PARAM)
       : lifoAlloc(lifoAlloc),
         mark(lifoAlloc->mark()),
+        fallibleScope(lifoAlloc),
         shouldRelease(true)
     {
         MOZ_GUARD_OBJECT_NOTIFIER_INIT;

js/src/jit-test/tests/ion/bug1264948.js

@@ -0,0 +1,21 @@
+if (!('oomTest' in this))
+    quit();
+
+loadFile(`
+  T = TypedObject
+  ObjectStruct = new T.StructType({f: T.Object})
+  var o = new ObjectStruct
+  function testGC(p) {
+    for (; i < 5; i++)
+        whatever.push;
+  }
+  testGC(o)
+  function writeObject()
+    o.f = v
+    writeObject({function() { } })
+  for (var i ; i < 5 ; ++i)
+    try {} catch (StringStruct) {}
+`);
+function loadFile(lfVarx) {
+  oomTest(Function(lfVarx));
+}

js/src/jit-test/tests/ion/bug1269756.js

@@ -0,0 +1,8 @@
+if (!('oomTest' in this))
+    quit();
+
+oomTest(function() {
+    m = parseModule(`while (x && NaN) prototype; let x`);
+    m.declarationInstantiation();
+    m.evaluation();
+})

js/src/jit/JitAllocPolicy.h

@@ -34,15 +34,18 @@ class TempAllocator
 
     explicit TempAllocator(LifoAlloc* lifoAlloc)
       : lifoScope_(lifoAlloc)
-    { }
+    {
+        lifoAlloc->setAsInfallibleByDefault();
+    }
 
     void* allocateInfallible(size_t bytes)
     {
-        return lifoScope_.alloc().allocInfallibleOrAssert(bytes);
+        return lifoScope_.alloc().allocInfallible(bytes);
     }
 
-    void* allocate(size_t bytes)
+    MOZ_MUST_USE void* allocate(size_t bytes)
     {
+        LifoAlloc::AutoFallibleScope fallibleAllocator(lifoAlloc());
         void* p = lifoScope_.alloc().alloc(bytes);
         if (!ensureBallast())
             return nullptr;
@@ -50,8 +53,9 @@ class TempAllocator
     }
 
     template <typename T>
-    T* allocateArray(size_t n)
+    MOZ_MUST_USE T* allocateArray(size_t n)
     {
+        LifoAlloc::AutoFallibleScope fallibleAllocator(lifoAlloc());
         size_t bytes;
         if (MOZ_UNLIKELY(!CalculateAllocSize<T>(n, &bytes)))
             return nullptr;
@@ -65,12 +69,12 @@ class TempAllocator
     struct Fallible { TempAllocator& alloc; };
     Fallible fallible() { return { *this }; }
 
-    LifoAlloc* lifoAlloc()
-    {
+    LifoAlloc* lifoAlloc() {
         return &lifoScope_.alloc();
     }
 
     MOZ_MUST_USE bool ensureBallast() {
+        JS_OOM_POSSIBLY_FAIL_BOOL();
         return lifoScope_.alloc().ensureUnusedApproximate(BallastSize);
     }
 };