Bug 1791975 - Don't sweep realms that were allocated during incremental GC r=jandem

When marking a BaseShape we mark its global, and we read the pointer to that
global from the realm. If a realm doesn't have a live global we can sweep the
realm but there may still be pointers to it in base shapes and these are left
dangling.

This happens when we hit OOM while creating a global during an incremental GC.
The BaseShape survives because it was allocated after the start of the GC. The
global itself is never successfully created and so the realm doesn't have a
live global and is swept. In this case, we trigger UAF when we try to compact
the heap and trace the base shape.

The patch adds an extra case for keeping a realm alive if it was created during
an incremental GC. This matches the way that GC things are not collected if
they are allocated after the start of a GC.

Differential Revision: https://phabricator.services.mozilla.com/D158022

js/src/gc/GC.cpp

@@ -2851,6 +2851,9 @@ void GCRuntime::finishCollection() {
   for (GCZonesIter zone(this); !zone.done(); zone.next()) {
     zone->changeGCState(Zone::Finished, Zone::NoGC);
     zone->notifyObservingDebuggers();
+    for (RealmsInZoneIter realm(zone); !realm.done(); realm.next()) {
+      realm->clearAllocatedDuringGC();
+    }
   }
 
 #ifdef JS_GC_ZEAL

js/src/vm/Realm-inl.h

@@ -35,9 +35,14 @@ inline bool JS::Realm::hasLiveGlobal() const {
 }
 
 inline bool JS::Realm::marked() const {
-  // Preserve this Realm if it has a live global or if it has been entered (to
-  // ensure we don't destroy the Realm while we're allocating its global).
-  return hasLiveGlobal() || hasBeenEnteredIgnoringJit();
+  // The Realm survives in the following cases:
+  //  - its global is live
+  //  - it has been entered (to ensure we don't destroy the Realm while we're
+  //    allocating its global)
+  //  - it was allocated after the start of an incremental GC (as there may be
+  //    pointers to it from other GC things)
+  return hasLiveGlobal() || hasBeenEnteredIgnoringJit() ||
+         allocatedDuringIncrementalGC_;
 }
 
 /* static */ inline js::ObjectRealm& js::ObjectRealm::get(const JSObject* obj) {

js/src/vm/Realm.cpp

@@ -55,6 +55,8 @@ Realm::Realm(Compartment* comp, const JS::RealmOptions& options)
       objects_(zone_),
       randomKeyGenerator_(runtime_->forkRandomKeyGenerator()),
       debuggers_(zone_),
+      allocatedDuringIncrementalGC_(zone_->isGCMarkingOrSweeping() ||
+                                    zone_->isGCFinished()),
       wasm(runtime_) {
   runtime_->numRealms++;
 }

js/src/vm/Realm.h

@@ -406,6 +406,7 @@ class JS::Realm : public JS::shadow::Realm {
   friend class js::AutoRestoreRealmDebugMode;
 
   bool isSystem_ = false;
+  bool allocatedDuringIncrementalGC_;
 
   js::UniquePtr<js::coverage::LCovRealm> lcovRealm_ = nullptr;
 
@@ -615,6 +616,7 @@ class JS::Realm : public JS::shadow::Realm {
   }
 
   inline bool marked() const;
+  void clearAllocatedDuringGC() { allocatedDuringIncrementalGC_ = false; }
 
   /*
    * The principals associated with this realm. Note that the same several