Bug 1710552 - pt 1. Prepare for GCs being denied r=smaug

This patch also:
 * adds an assertion to KillGCRunner() to ensure it's never killed if
   needed, now that there are more calls to KillGCRunner(), some calls have
   been moved eg in nsJSEnvironment so as not to kill the runner a little
   later and keep the assertions happy.

 * IdleSchedulerChild will decline a request for a GC if there's already a
   request in flight.

 * CCGCScheduler will check if a GC is already in progress when handling the
   parents' response to a GC request.

Differential Revision: https://phabricator.services.mozilla.com/D120830

dom/base/CCGCScheduler.cpp

@@ -34,6 +34,14 @@ void CCGCScheduler::NoteGCEnd() {
   mLikelyShortLivingObjectsNeedingGC = 0;
 }
 
+void CCGCScheduler::NoteWontGC() {
+  mReadyForMajorGC = false;
+  mMajorGCReason = JS::GCReason::NO_REASON;
+  mWantAtLeastRegularGC = false;
+  // Don't clear the WantFullGC state, we will do a full GC the next time a
+  // GC happens for any other reason.
+}
+
 bool CCGCScheduler::GCRunnerFired(TimeStamp aDeadline) {
   MOZ_ASSERT(!mDidShutdown, "GCRunner still alive during shutdown");
 
@@ -45,25 +53,61 @@ bool CCGCScheduler::GCRunnerFired(TimeStamp aDeadline) {
     case GCRunnerAction::WaitToMajorGC: {
       RefPtr<CCGCScheduler::MayGCPromise> mbPromise =
           CCGCScheduler::MayGCNow(step.mReason);
-      if (!mbPromise || mbPromise->IsResolved()) {
-        // Only use the promise if it's not resolved yet, otherwise fall through
-        // and begin the GC in the current idle time with our current deadline.
+      if (!mbPromise) {
+        // We can GC now.
         break;
       }
 
+      if (mbPromise->IsResolved()) {
+        // The promise is already resolved so if we are going to do a GC,
+        // begin it now in the current idle time.
+        mbPromise->Then(
+            GetMainThreadSerialEventTarget(), __func__,
+            [this, aDeadline, step](bool aMayGC) {
+              MOZ_ASSERT(!InIncrementalGC());
+              if (aMayGC) {
+                MOZ_ALWAYS_TRUE(NoteReadyForMajorGC());
+                GCRunnerFiredDoGC(aDeadline, step);
+              } else {
+                KillGCRunner();
+                NoteWontGC();
+              }
+            },
+            [this](mozilla::ipc::ResponseRejectReason r) {
+              if (!InIncrementalGC()) {
+                KillGCRunner();
+                NoteWontGC();
+              }
+            });
+        return true;
+      }
+
       KillGCRunner();
       mbPromise->Then(
           GetMainThreadSerialEventTarget(), __func__,
-          [this](bool aIgnored) {
-            if (!NoteReadyForMajorGC()) {
-              return;  // Another GC completed while waiting.
+          [this](bool aMayGC) {
+            if (aMayGC) {
+              if (!NoteReadyForMajorGC()) {
+                // Another GC started and maybe completed while waiting.
+                return;
+              }
+              // Recreate the GC runner with a 0 delay.  The new runner will
+              // continue in idle time.
+              KillGCRunner();
+              EnsureGCRunner(0);
+            } else if (!InIncrementalGC()) {
+              // We should kill the GC runner since we're done with it, but
+              // only if there's no incremental GC.
+              KillGCRunner();
+              NoteWontGC();
             }
-            // If a new runner was started, recreate it with a 0 delay. The new
-            // runner will continue in idle time.
-            KillGCRunner();
-            EnsureGCRunner(0);
           },
-          [](mozilla::ipc::ResponseRejectReason r) {});
+          [this](mozilla::ipc::ResponseRejectReason r) {
+            if (!InIncrementalGC()) {
+              KillGCRunner();
+              NoteWontGC();
+            }
+          });
 
       return true;
     }
@@ -73,9 +117,14 @@ bool CCGCScheduler::GCRunnerFired(TimeStamp aDeadline) {
       break;
   }
 
+  return GCRunnerFiredDoGC(aDeadline, step);
+}
+
+bool CCGCScheduler::GCRunnerFiredDoGC(TimeStamp aDeadline,
+                                      const GCRunnerStep& aStep) {
   // Run a GC slice, possibly the first one of a major GC.
   nsJSContext::IsShrinking is_shrinking = nsJSContext::NonShrinkingGC;
-  if (!InIncrementalGC() && step.mReason == JS::GCReason::USER_INACTIVE) {
+  if (!InIncrementalGC() && aStep.mReason == JS::GCReason::USER_INACTIVE) {
     if (!mUserIsActive) {
       mIsCompactingOnUserInactive = true;
       is_shrinking = nsJSContext::ShrinkingGC;
@@ -87,6 +136,7 @@ bool CCGCScheduler::GCRunnerFired(TimeStamp aDeadline) {
       if (child) {
         child->DoneGC();
       }
+      NoteWontGC();
       return true;
     }
   }
@@ -95,7 +145,7 @@ bool CCGCScheduler::GCRunnerFired(TimeStamp aDeadline) {
   TimeStamp startTimeStamp = TimeStamp::Now();
   TimeDuration budget = ComputeInterSliceGCBudget(aDeadline, startTimeStamp);
   TimeDuration duration = mGCUnnotifiedTotalTime;
-  nsJSContext::GarbageCollectNow(step.mReason, nsJSContext::IncrementalGC,
+  nsJSContext::GarbageCollectNow(aStep.mReason, nsJSContext::IncrementalGC,
                                  is_shrinking, budget.ToMilliseconds());
 
   mGCUnnotifiedTotalTime = TimeDuration();
@@ -206,6 +256,9 @@ void CCGCScheduler::PokeFullGC() {
         [](nsITimer* aTimer, void* aClosure) {
           CCGCScheduler* s = static_cast<CCGCScheduler*>(aClosure);
           s->KillFullGCTimer();
+
+          // Even if the GC is denied by the parent process, because we've
+          // set that we want a full GC we will get one eventually.
           s->SetNeedsFullGC();
           s->SetWantMajorGC(JS::GCReason::FULL_GC_TIMER);
           s->EnsureGCRunner(0);
@@ -304,6 +357,7 @@ void CCGCScheduler::KillFullGCTimer() {
 }
 
 void CCGCScheduler::KillGCRunner() {
+  MOZ_ASSERT(!(InIncrementalGC() && !mDidShutdown));
   if (mGCRunner) {
     mGCRunner->Cancel();
     mGCRunner = nullptr;

dom/base/CCGCScheduler.h

@@ -189,7 +189,7 @@ class CCGCScheduler {
   // Returns false if we started and finished a major GC while waiting for a
   // response.
   [[nodiscard]] bool NoteReadyForMajorGC() {
-    if (mMajorGCReason == JS::GCReason::NO_REASON) {
+    if (mMajorGCReason == JS::GCReason::NO_REASON || InIncrementalGC()) {
       return false;
     }
     mReadyForMajorGC = true;
@@ -198,6 +198,8 @@ class CCGCScheduler {
 
   void NoteGCBegin();
   void NoteGCEnd();
+  // A timer fired, but then decided not to run a GC.
+  void NoteWontGC();
 
   void NoteGCSliceEnd(TimeDuration aSliceDuration) {
     if (mMajorGCReason == JS::GCReason::NO_REASON) {
@@ -215,9 +217,10 @@ class CCGCScheduler {
   }
 
   bool GCRunnerFired(TimeStamp aDeadline);
+  bool GCRunnerFiredDoGC(TimeStamp aDeadline, const GCRunnerStep& aStep);
 
   using MayGCPromise =
-      MozPromise<bool /* aIgnored */, mozilla::ipc::ResponseRejectReason, true>;
+      MozPromise<bool, mozilla::ipc::ResponseRejectReason, true>;
 
   // Returns null if we shouldn't GC now (eg a GC is already running).
   static RefPtr<MayGCPromise> MayGCNow(JS::GCReason reason);

dom/base/nsJSEnvironment.cpp

@@ -1741,7 +1741,7 @@ static void DOMGCSliceCallback(JSContext* aCx, JS::GCProgress aProgress,
       sScheduler.NoteGCSliceEnd(aDesc.lastSliceEnd(aCx) -
                                 aDesc.lastSliceStart(aCx));
 
-      if (sShuttingDown || aDesc.isComplete_) {
+      if (sShuttingDown) {
         sScheduler.KillGCRunner();
       } else {
         // If incremental GC wasn't triggered by GCTimerFired, we may not have a

ipc/glue/IdleSchedulerChild.cpp

@@ -77,7 +77,7 @@ bool IdleSchedulerChild::SetPaused() {
 
 RefPtr<IdleSchedulerChild::MayGCPromise> IdleSchedulerChild::MayGCNow() {
   if (mIsRequestingGC || mIsDoingGC) {
-    return nullptr;
+    return MayGCPromise::CreateAndResolve(false, __func__);
   }
   TimeStamp wait_since = TimeStamp::Now();
 

ipc/glue/IdleSchedulerChild.h

@@ -37,8 +37,7 @@ class IdleSchedulerChild final : public PIdleSchedulerChild {
   // Returns true if activity state dropped below cpu count.
   bool SetPaused();
 
-  typedef MozPromise<bool /* aIgnored */, ResponseRejectReason, true>
-      MayGCPromise;
+  typedef MozPromise<bool, ResponseRejectReason, true> MayGCPromise;
 
   // Returns null if a GC or GC request is already in progress.
   RefPtr<MayGCPromise> MayGCNow();