Bug 1759396 part 2: Use fallible allocation when inserting into PresShell::mAllocatedPointers, to avert OOMs in builds with diagnostic assertions. r=emilio

If allocation fails, we'll now just discard the entire contents of
mAllocatedPointers and stop using it going forward.

See the documentation alongside the mAllocatedPointers declaration (in this
patch) for more details.

Differential Revision: https://phabricator.services.mozilla.com/D143416

layout/base/PresShell.cpp

@@ -897,7 +897,7 @@ PresShell::~PresShell() {
                    mLastCallbackEventRequest == nullptr,
                "post-reflow queues not empty.  This means we're leaking");
 
-  MOZ_ASSERT(mAllocatedPointers->IsEmpty(),
+  MOZ_ASSERT(!mAllocatedPointers || mAllocatedPointers->IsEmpty(),
              "Some pres arena objects were not freed");
 
   mFrameManager = nullptr;

layout/base/PresShell.h

@@ -1755,13 +1755,25 @@ class PresShell final : public nsStubDocumentObserver,
 
   void RecordAlloc(void* aPtr) {
 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
+    if (!mAllocatedPointers) {
+      return;  // Hash set was presumably freed to avert OOM.
+    }
     MOZ_DIAGNOSTIC_ASSERT(!mAllocatedPointers->Contains(aPtr));
-    mAllocatedPointers->Insert(aPtr);
+    if (!mAllocatedPointers->Insert(aPtr, fallible)) {
+      // Yikes! We're nearly out of memory, and this insertion would've pushed
+      // us over the ledge. At this point, we discard & stop using this set,
+      // since we don't have enough memory to keep it accurate from this point
+      // onwards. Hopefully this helps relieve the memory pressure a bit, too.
+      mAllocatedPointers = nullptr;
+    }
 #endif
   }
 
   void RecordFree(void* aPtr) {
 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
+    if (!mAllocatedPointers) {
+      return;  // Hash set was presumably freed to avert OOM.
+    }
     MOZ_DIAGNOSTIC_ASSERT(mAllocatedPointers->Contains(aPtr));
     mAllocatedPointers->Remove(aPtr);
 #endif
@@ -2858,8 +2870,13 @@ class PresShell final : public nsStubDocumentObserver,
   nsCOMPtr<nsITimer> mReflowContinueTimer;
 
 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
-  // We track allocated pointers in a debug-only hashtable to assert against
-  // missing/double frees.
+  // We track allocated pointers in a diagnostic hash set, to assert against
+  // missing/double frees. This set is allocated infallibly in the PresShell
+  // constructor's initialization list. The set can get quite large, so we use
+  // fallible allocation when inserting into it; and if these operations ever
+  // fail, then we just get rid of the set and stop using this diagnostic from
+  // that point on.  (There's not much else we can do, when the set grows
+  // larger than the available memory.)
   UniquePtr<nsTHashSet<void*>> mAllocatedPointers;
 #endif