Bug 1797755 - Part 1: Move testing mark queue to GCRuntime r=sfink

Parallel marking will use one GCMarker per thread. The testing mark queue is
really a per-runtime data structure, so this patch moves it to the GCRuntime.

Differential Revision: https://phabricator.services.mozilla.com/D160524

js/src/builtin/TestingFunctions.cpp

@@ -2393,7 +2393,7 @@ static bool CurrentGC(JSContext* cx, unsigned argc, Value* vp) {
   }
 
 #  ifdef DEBUG
-  val = Int32Value(gc.marker.queuePos);
+  val = Int32Value(gc.testMarkQueuePos());
   if (!JS_DefineProperty(cx, result, "queuePos", val, JSPROP_ENUMERATE)) {
     return false;
   }
@@ -6850,20 +6850,19 @@ static bool SetGCCallback(JSContext* cx, unsigned argc, Value* vp) {
 #ifdef DEBUG
 static bool EnqueueMark(JSContext* cx, unsigned argc, Value* vp) {
   CallArgs args = CallArgsFromVp(argc, vp);
-
-  auto& queue = cx->runtime()->gc.marker.markQueue;
+  gc::GCRuntime* gc = &cx->runtime()->gc;
 
   if (args.get(0).isString()) {
     RootedString val(cx, args[0].toString());
     if (!val->ensureLinear(cx)) {
       return false;
     }
-    if (!queue.append(StringValue(val))) {
+    if (!gc->appendTestMarkQueue(StringValue(val))) {
       JS_ReportOutOfMemory(cx);
       return false;
     }
   } else if (args.get(0).isObject()) {
-    if (!queue.append(args[0])) {
+    if (!gc->appendTestMarkQueue(args[0])) {
       JS_ReportOutOfMemory(cx);
       return false;
     }
@@ -6879,7 +6878,7 @@ static bool EnqueueMark(JSContext* cx, unsigned argc, Value* vp) {
 static bool GetMarkQueue(JSContext* cx, unsigned argc, Value* vp) {
   CallArgs args = CallArgsFromVp(argc, vp);
 
-  auto& queue = cx->runtime()->gc.marker.markQueue.get();
+  const auto& queue = cx->runtime()->gc.getTestMarkQueue();
 
   RootedObject result(cx, JS::NewArrayObject(cx, queue.length()));
   if (!result) {
@@ -6902,7 +6901,7 @@ static bool GetMarkQueue(JSContext* cx, unsigned argc, Value* vp) {
 static bool ClearMarkQueue(JSContext* cx, unsigned argc, Value* vp) {
   CallArgs args = CallArgsFromVp(argc, vp);
 
-  cx->runtime()->gc.marker.markQueue.clear();
+  cx->runtime()->gc.clearTestMarkQueue();
   args.rval().setUndefined();
   return true;
 }

js/src/gc/GC.cpp

@@ -428,6 +428,9 @@ GCRuntime::GCRuntime(JSRuntime* rt)
       sweepZone(nullptr),
       abortSweepAfterCurrentGroup(false),
       sweepMarkResult(IncrementalProgress::NotFinished),
+#ifdef DEBUG
+      testMarkQueue(rt),
+#endif
       startedCompacting(false),
       zonesCompacted(0),
 #ifdef DEBUG
@@ -2662,8 +2665,8 @@ void GCRuntime::endPreparePhase(JS::GCReason reason) {
       }
 
 #ifdef DEBUG
-      marker.markQueue.clear();
-      marker.queuePos = 0;
+      testMarkQueue.clear();
+      queuePos = 0;
 #endif
     }
   }
@@ -2712,6 +2715,11 @@ void GCRuntime::beginMarkPhase(AutoGCSession& session) {
   marker.start();
   MOZ_ASSERT(marker.isDrained());
 
+#ifdef DEBUG
+  queuePos = 0;
+  queueMarkColor.reset();
+#endif
+
   for (GCZonesIter zone(this); !zone.done(); zone.next()) {
     // Incremental marking barriers are enabled at this point.
     zone->changeGCState(Zone::Prepare, zone->initialMarkingState());
@@ -2826,7 +2834,7 @@ IncrementalProgress GCRuntime::markUntilBudgetExhausted(
   AutoSetThreadIsMarking threadIsMarking;
 #endif  // DEBUG
 
-  if (marker.processMarkQueue() == GCMarker::QueueYielded) {
+  if (processTestMarkQueue() == QueueYielded) {
     return NotFinished;
   }
 
@@ -2839,6 +2847,154 @@ void GCRuntime::drainMarkStack() {
   MOZ_RELEASE_ASSERT(marker.markUntilBudgetExhausted(unlimited));
 }
 
+#ifdef DEBUG
+
+const GCVector<HeapPtr<JS::Value>, 0, SystemAllocPolicy>&
+GCRuntime::getTestMarkQueue() const {
+  return testMarkQueue.get();
+}
+
+bool GCRuntime::appendTestMarkQueue(const JS::Value& value) {
+  return testMarkQueue.append(value);
+}
+
+void GCRuntime::clearTestMarkQueue() { testMarkQueue.clear(); }
+
+size_t GCRuntime::testMarkQueuePos() const { return queuePos; }
+
+#endif
+
+GCRuntime::MarkQueueProgress GCRuntime::processTestMarkQueue() {
+#ifdef DEBUG
+  if (testMarkQueue.empty()) {
+    return QueueComplete;
+  }
+
+  if (queueMarkColor == mozilla::Some(MarkColor::Gray) &&
+      state() != State::Sweep) {
+    return QueueSuspended;
+  }
+
+  // If the queue wants to be gray marking, but we've pushed a black object
+  // since set-color-gray was processed, then we can't switch to gray and must
+  // again wait until gray marking is possible.
+  //
+  // Remove this code if the restriction against marking gray during black is
+  // relaxed.
+  if (queueMarkColor == mozilla::Some(MarkColor::Gray) &&
+      marker.hasBlackEntries()) {
+    return QueueSuspended;
+  }
+
+  // If the queue wants to be marking a particular color, switch to that color.
+  // In any case, restore the mark color to whatever it was when we entered
+  // this function.
+  bool willRevertToGray = marker.markColor() == MarkColor::Gray;
+  AutoSetMarkColor autoRevertColor(marker,
+                                   queueMarkColor.valueOr(marker.markColor()));
+
+  // Process the mark queue by taking each object in turn, pushing it onto the
+  // mark stack, and processing just the top element with processMarkStackTop
+  // without recursing into reachable objects.
+  while (queuePos < testMarkQueue.length()) {
+    Value val = testMarkQueue[queuePos++].get();
+    if (val.isObject()) {
+      JSObject* obj = &val.toObject();
+      JS::Zone* zone = obj->zone();
+      if (!zone->isGCMarking() || obj->isMarkedAtLeast(marker.markColor())) {
+        continue;
+      }
+
+      // If we have started sweeping, obey sweep group ordering. But note that
+      // we will first be called during the initial sweep slice, when the sweep
+      // group indexes have not yet been computed. In that case, we can mark
+      // freely.
+      if (state() == State::Sweep && initialState != State::Sweep) {
+        if (zone->gcSweepGroupIndex < getCurrentSweepGroupIndex()) {
+          // Too late. This must have been added after we started collecting,
+          // and we've already processed its sweep group. Skip it.
+          continue;
+        }
+        if (zone->gcSweepGroupIndex > getCurrentSweepGroupIndex()) {
+          // Not ready yet. Wait until we reach the object's sweep group.
+          queuePos--;
+          return QueueSuspended;
+        }
+      }
+
+      if (marker.markColor() == MarkColor::Gray &&
+          zone->isGCMarkingBlackOnly()) {
+        // Have not yet reached the point where we can mark this object, so
+        // continue with the GC.
+        queuePos--;
+        return QueueSuspended;
+      }
+
+      if (marker.markColor() == MarkColor::Black && willRevertToGray) {
+        // If we put any black objects on the stack, we wouldn't be able to
+        // return to gray marking. So delay the marking until we're back to
+        // black marking.
+        queuePos--;
+        return QueueSuspended;
+      }
+
+      // Mark the object and push it onto the stack.
+      size_t oldPosition = marker.stack.position();
+      marker.markAndTraverse(obj);
+
+      // If we overflow the stack here and delay marking, then we won't be
+      // testing what we think we're testing.
+      if (marker.stack.position() == oldPosition) {
+        MOZ_ASSERT(obj->asTenured().arena()->onDelayedMarkingList());
+        AutoEnterOOMUnsafeRegion oomUnsafe;
+        oomUnsafe.crash("Overflowed stack while marking test queue");
+      }
+
+      SliceBudget unlimited = SliceBudget::unlimited();
+      marker.processMarkStackTop(unlimited);
+    } else if (val.isString()) {
+      JSLinearString* str = &val.toString()->asLinear();
+      if (js::StringEqualsLiteral(str, "yield") && isIncrementalGc()) {
+        return QueueYielded;
+      } else if (js::StringEqualsLiteral(str, "enter-weak-marking-mode") ||
+                 js::StringEqualsLiteral(str, "abort-weak-marking-mode")) {
+        if (marker.isRegularMarking()) {
+          // We can't enter weak marking mode at just any time, so instead
+          // we'll stop processing the queue and continue on with the GC. Once
+          // we enter weak marking mode, we can continue to the rest of the
+          // queue. Note that we will also suspend for aborting, and then abort
+          // the earliest following weak marking mode.
+          queuePos--;
+          return QueueSuspended;
+        }
+        if (js::StringEqualsLiteral(str, "abort-weak-marking-mode")) {
+          marker.abortLinearWeakMarking();
+        }
+      } else if (js::StringEqualsLiteral(str, "drain")) {
+        auto unlimited = SliceBudget::unlimited();
+        MOZ_RELEASE_ASSERT(marker.markUntilBudgetExhausted(
+            unlimited, GCMarker::DontReportMarkTime));
+      } else if (js::StringEqualsLiteral(str, "set-color-gray")) {
+        queueMarkColor = mozilla::Some(MarkColor::Gray);
+        if (state() != State::Sweep || marker.hasBlackEntries()) {
+          // Cannot mark gray yet, so continue with the GC.
+          queuePos--;
+          return QueueSuspended;
+        }
+        marker.setMarkColor(MarkColor::Gray);
+      } else if (js::StringEqualsLiteral(str, "set-color-black")) {
+        queueMarkColor = mozilla::Some(MarkColor::Black);
+        marker.setMarkColor(MarkColor::Black);
+      } else if (js::StringEqualsLiteral(str, "unset-color")) {
+        queueMarkColor.reset();
+      }
+    }
+  }
+#endif
+
+  return QueueComplete;
+}
+
 void GCRuntime::finishCollection() {
   assertBackgroundSweepingFinished();
 

js/src/gc/GCMarker.h

@@ -338,15 +338,6 @@ class GCMarker final : public GenericTracerImpl<GCMarker> {
 
   bool isDrained();
 
-  // The mark queue is a testing-only feature for controlling mark ordering and
-  // yield timing.
-  enum MarkQueueProgress {
-    QueueYielded,   // End this incremental GC slice, if possible
-    QueueComplete,  // Done with the queue
-    QueueSuspended  // Continue the GC without ending the slice
-  };
-  MarkQueueProgress processMarkQueue();
-
   enum ShouldReportMarkTime : bool {
     ReportMarkTime = true,
     DontReportMarkTime = false
@@ -379,8 +370,17 @@ class GCMarker final : public GenericTracerImpl<GCMarker> {
   template <typename T>
   void markImplicitEdges(T* oldThing);
 
+  bool isRegularMarking() const {
+    return state == MarkingState::RegularMarking;
+  }
   bool isWeakMarking() const { return state == MarkingState::WeakMarking; }
 
+  bool isMarkStackEmpty() { return stack.isEmpty(); }
+
+  bool hasBlackEntries() const { return stack.position() > grayPosition; }
+
+  bool hasGrayEntries() const { return grayPosition > 0 && !stack.isEmpty(); }
+
  private:
 #ifdef DEBUG
   void checkZone(void* p);
@@ -429,13 +429,8 @@ class GCMarker final : public GenericTracerImpl<GCMarker> {
   inline void pushValueRange(JSObject* obj, SlotsOrElementsKind kind,
                              size_t start, size_t end);
 
-  bool isMarkStackEmpty() { return stack.isEmpty(); }
-
-  bool hasBlackEntries() const { return stack.position() > grayPosition; }
-
-  bool hasGrayEntries() const { return grayPosition > 0 && !stack.isEmpty(); }
-
-  inline void processMarkStackTop(SliceBudget& budget);
+  void processMarkStackTop(SliceBudget& budget);
+  friend class gc::GCRuntime;
 
   void markDelayedChildren(gc::Arena* arena);
   void markAllDelayedChildren(ShouldReportMarkTime reportTime);
@@ -492,9 +487,6 @@ class GCMarker final : public GenericTracerImpl<GCMarker> {
    */
   MainThreadOrGCTaskData<bool> checkAtomMarking;
 
-  /* The test marking queue might want to be marking a particular color. */
-  mozilla::Maybe<js::gc::MarkColor> queueMarkColor;
-
   /*
    * If this is true, all marked objects must belong to a compartment being
    * GCed. This is used to look for compartment bugs.
@@ -509,21 +501,6 @@ class GCMarker final : public GenericTracerImpl<GCMarker> {
    */
   MainThreadOrGCTaskData<Compartment*> tracingCompartment;
   MainThreadOrGCTaskData<Zone*> tracingZone;
-
-  /*
-   * List of objects to mark at the beginning of a GC. May also contains string
-   * directives to change mark color or wait until different phases of the GC.
-   *
-   * This is a WeakCache because not everything in this list is guaranteed to
-   * end up marked (eg if you insert an object from an already-processed sweep
-   * group in the middle of an incremental GC). Also, the mark queue is not
-   * used during shutdown GCs. In either case, unmarked objects may need to be
-   * discarded.
-   */
-  JS::WeakCache<GCVector<HeapPtr<JS::Value>, 0, SystemAllocPolicy>> markQueue;
-
-  /* Position within the test mark queue. */
-  size_t queuePos;
 #endif  // DEBUG
 };
 

js/src/gc/GCRuntime.h

@@ -263,8 +263,6 @@ struct SweepingTracer final : public GenericTracerImpl<SweepingTracer> {
 };
 
 class GCRuntime {
-  friend GCMarker::MarkQueueProgress GCMarker::processMarkQueue();
-
  public:
   explicit GCRuntime(JSRuntime* rt);
   [[nodiscard]] bool init(uint32_t maxbytes);
@@ -638,6 +636,14 @@ class GCRuntime {
 
   void updateAllocationRates();
 
+#ifdef DEBUG
+  const GCVector<HeapPtr<JS::Value>, 0, SystemAllocPolicy>& getTestMarkQueue()
+      const;
+  [[nodiscard]] bool appendTestMarkQueue(const JS::Value& value);
+  void clearTestMarkQueue();
+  size_t testMarkQueuePos() const;
+#endif
+
  private:
   enum IncrementalResult { ResetIncremental = 0, Ok };
 
@@ -776,11 +782,21 @@ class GCRuntime {
   IncrementalProgress markAllWeakReferences();
   void markAllGrayReferences(gcstats::PhaseKind phase);
 
+  // The mark queue is a testing-only feature for controlling mark ordering and
+  // yield timing.
+  enum MarkQueueProgress {
+    QueueYielded,   // End this incremental GC slice, if possible
+    QueueComplete,  // Done with the queue
+    QueueSuspended  // Continue the GC without ending the slice
+  };
+  MarkQueueProgress processTestMarkQueue();
+
   // GC Sweeping. Implemented in Sweeping.cpp.
   void beginSweepPhase(JS::GCReason reason, AutoGCSession& session);
   void dropStringWrappers();
   void groupZonesForSweeping(JS::GCReason reason);
   [[nodiscard]] bool findSweepGroupEdges();
+  [[nodiscard]] bool addEdgesForMarkQueue();
   void getNextSweepGroup();
   void resetGrayList(Compartment* comp);
   IncrementalProgress beginMarkingSweepGroup(JS::GCContext* gcx,
@@ -1011,7 +1027,6 @@ class GCRuntime {
   mozilla::Atomic<uint32_t, mozilla::ReleaseAcquire> numArenasFreeCommitted;
   MainThreadData<VerifyPreTracer*> verifyPreData;
 
- private:
   MainThreadData<mozilla::TimeStamp> lastGCStartTime_;
   MainThreadData<mozilla::TimeStamp> lastGCEndTime_;
 
@@ -1032,7 +1047,6 @@ class GCRuntime {
 
   mozilla::Atomic<JS::GCReason, mozilla::ReleaseAcquire> majorGCTriggerReason;
 
- private:
   /* Incremented at the start of every minor GC. */
   MainThreadData<uint64_t> minorGCNumber;
 
@@ -1133,6 +1147,26 @@ class GCRuntime {
   MainThreadOrGCTaskData<IncrementalProgress> sweepMarkResult;
 
 #ifdef DEBUG
+  /*
+   * List of objects to mark at the beginning of a GC for testing purposes. May
+   * also contain string directives to change mark color or wait until different
+   * phases of the GC.
+   *
+   * This is a WeakCache because not everything in this list is guaranteed to
+   * end up marked (eg if you insert an object from an already-processed sweep
+   * group in the middle of an incremental GC). Also, the mark queue is not
+   * used during shutdown GCs. In either case, unmarked objects may need to be
+   * discarded.
+   */
+  JS::WeakCache<GCVector<HeapPtr<JS::Value>, 0, SystemAllocPolicy>>
+      testMarkQueue;
+
+  /* Position within the test mark queue. */
+  size_t queuePos;
+
+  /* The test marking queue might want to be marking a particular color. */
+  mozilla::Maybe<js::gc::MarkColor> queueMarkColor;
+
   // During gray marking, delay AssertCellIsNotGray checks by
   // recording the cell pointers here and checking after marking has
   // finished.
@@ -1273,7 +1307,6 @@ class GCRuntime {
    */
   MainThreadData<SortedArenaList> incrementalSweepList;
 
- private:
   MainThreadData<Nursery> nursery_;
 
   // The store buffer used to track tenured to nursery edges for generational

js/src/gc/Marking.cpp

@@ -1101,6 +1101,8 @@ void GCMarker::traverse(BaseScript* thing) {
 }
 }  // namespace js
 
+template void js::GCMarker::markAndTraverse<JSObject>(JSObject* thing);
+
 #ifdef DEBUG
 void GCMarker::setCheckAtomMarking(bool check) {
   MOZ_ASSERT(check != checkAtomMarking);
@@ -1188,137 +1190,6 @@ static inline void CallTraceHook(JSTracer* trc, JSObject* obj) {
   }
 }
 
-GCMarker::MarkQueueProgress GCMarker::processMarkQueue() {
-#ifdef DEBUG
-  if (markQueue.empty()) {
-    return QueueComplete;
-  }
-
-  GCRuntime& gcrt = runtime()->gc;
-  if (queueMarkColor == mozilla::Some(MarkColor::Gray) &&
-      gcrt.state() != State::Sweep) {
-    return QueueSuspended;
-  }
-
-  // If the queue wants to be gray marking, but we've pushed a black object
-  // since set-color-gray was processed, then we can't switch to gray and must
-  // again wait until gray marking is possible.
-  //
-  // Remove this code if the restriction against marking gray during black is
-  // relaxed.
-  if (queueMarkColor == mozilla::Some(MarkColor::Gray) && hasBlackEntries()) {
-    return QueueSuspended;
-  }
-
-  // If the queue wants to be marking a particular color, switch to that color.
-  // In any case, restore the mark color to whatever it was when we entered
-  // this function.
-  bool willRevertToGray = markColor() == MarkColor::Gray;
-  AutoSetMarkColor autoRevertColor(*this, queueMarkColor.valueOr(markColor()));
-
-  // Process the mark queue by taking each object in turn, pushing it onto the
-  // mark stack, and processing just the top element with processMarkStackTop
-  // without recursing into reachable objects.
-  while (queuePos < markQueue.length()) {
-    Value val = markQueue[queuePos++].get();
-    if (val.isObject()) {
-      JSObject* obj = &val.toObject();
-      JS::Zone* zone = obj->zone();
-      if (!zone->isGCMarking() || obj->isMarkedAtLeast(markColor())) {
-        continue;
-      }
-
-      // If we have started sweeping, obey sweep group ordering. But note that
-      // we will first be called during the initial sweep slice, when the sweep
-      // group indexes have not yet been computed. In that case, we can mark
-      // freely.
-      if (gcrt.state() == State::Sweep && gcrt.initialState != State::Sweep) {
-        if (zone->gcSweepGroupIndex < gcrt.getCurrentSweepGroupIndex()) {
-          // Too late. This must have been added after we started collecting,
-          // and we've already processed its sweep group. Skip it.
-          continue;
-        }
-        if (zone->gcSweepGroupIndex > gcrt.getCurrentSweepGroupIndex()) {
-          // Not ready yet. Wait until we reach the object's sweep group.
-          queuePos--;
-          return QueueSuspended;
-        }
-      }
-
-      if (markColor() == MarkColor::Gray && zone->isGCMarkingBlackOnly()) {
-        // Have not yet reached the point where we can mark this object, so
-        // continue with the GC.
-        queuePos--;
-        return QueueSuspended;
-      }
-
-      if (markColor() == MarkColor::Black && willRevertToGray) {
-        // If we put any black objects on the stack, we wouldn't be able to
-        // return to gray marking. So delay the marking until we're back to
-        // black marking.
-        queuePos--;
-        return QueueSuspended;
-      }
-
-      // Mark the object and push it onto the stack.
-      size_t oldPosition = stack.position();
-      markAndTraverse(obj);
-
-      // If we overflow the stack here and delay marking, then we won't be
-      // testing what we think we're testing.
-      if (stack.position() == oldPosition) {
-        MOZ_ASSERT(obj->asTenured().arena()->onDelayedMarkingList());
-        AutoEnterOOMUnsafeRegion oomUnsafe;
-        oomUnsafe.crash("Overflowed stack while marking test queue");
-      }
-
-      // Process just the one object that is now on top of the mark stack,
-      // possibly pushing more stuff onto the stack.
-      SliceBudget unlimited = SliceBudget::unlimited();
-      processMarkStackTop(unlimited);
-    } else if (val.isString()) {
-      JSLinearString* str = &val.toString()->asLinear();
-      if (js::StringEqualsLiteral(str, "yield") && gcrt.isIncrementalGc()) {
-        return QueueYielded;
-      } else if (js::StringEqualsLiteral(str, "enter-weak-marking-mode") ||
-                 js::StringEqualsLiteral(str, "abort-weak-marking-mode")) {
-        if (state == MarkingState::RegularMarking) {
-          // We can't enter weak marking mode at just any time, so instead
-          // we'll stop processing the queue and continue on with the GC. Once
-          // we enter weak marking mode, we can continue to the rest of the
-          // queue. Note that we will also suspend for aborting, and then abort
-          // the earliest following weak marking mode.
-          queuePos--;
-          return QueueSuspended;
-        }
-        if (js::StringEqualsLiteral(str, "abort-weak-marking-mode")) {
-          abortLinearWeakMarking();
-        }
-      } else if (js::StringEqualsLiteral(str, "drain")) {
-        auto unlimited = SliceBudget::unlimited();
-        MOZ_RELEASE_ASSERT(
-            markUntilBudgetExhausted(unlimited, DontReportMarkTime));
-      } else if (js::StringEqualsLiteral(str, "set-color-gray")) {
-        queueMarkColor = mozilla::Some(MarkColor::Gray);
-        if (gcrt.state() != State::Sweep || hasBlackEntries()) {
-          // Cannot mark gray yet, so continue with the GC.
-          queuePos--;
-          return QueueSuspended;
-        }
-        setMarkColor(MarkColor::Gray);
-      } else if (js::StringEqualsLiteral(str, "set-color-black")) {
-        queueMarkColor = mozilla::Some(MarkColor::Black);
-        setMarkColor(MarkColor::Black);
-      } else if (js::StringEqualsLiteral(str, "unset-color")) {
-        queueMarkColor.reset();
-      }
-    }
-  }
-#endif
-
-  return QueueComplete;
-}
-
 static gcstats::PhaseKind GrayMarkingPhaseForCurrentPhase(
     const gcstats::Statistics& stats) {
   using namespace gcstats;
@@ -1914,9 +1785,7 @@ GCMarker::GCMarker(JSRuntime* rt)
       ,
       markLaterArenas(0),
       checkAtomMarking(true),
-      strictCompartmentChecking(false),
-      markQueue(rt),
-      queuePos(0)
+      strictCompartmentChecking(false)
 #endif
 {
 }
@@ -1933,11 +1802,6 @@ void GCMarker::start() {
   state = MarkingState::RegularMarking;
   markColor_ = MarkColor::Black;
 
-#ifdef DEBUG
-  queuePos = 0;
-  queueMarkColor.reset();
-#endif
-
   MOZ_ASSERT(!delayedMarkingList);
   MOZ_ASSERT(markLaterArenas == 0);
 }
@@ -2067,12 +1931,6 @@ bool GCMarker::enterWeakMarkingMode() {
   // gcEphemeronEdges and marked according to ephemeron rules.
   state = MarkingState::WeakMarking;
 
-  // If there was an 'enter-weak-marking-mode' token in the queue, then it
-  // and everything after it will still be in the queue so we can process
-  // them now.
-  while (processMarkQueue() == QueueYielded) {
-  };
-
   return true;
 }
 

js/src/gc/Sweeping.cpp

@@ -499,6 +499,12 @@ IncrementalProgress GCRuntime::markWeakReferences(
       mozilla::MakeScopeExit([&] { marker.leaveWeakMarkingMode(); });
 
   if (marker.enterWeakMarkingMode()) {
+    // If there was an 'enter-weak-marking-mode' token in the queue, then it
+    // and everything after it will still be in the queue so we can process
+    // them now.
+    while (processTestMarkQueue() == QueueYielded) {
+    };
+
     // Do not rely on the information about not-yet-marked weak keys that have
     // been collected by barriers. Clear out the gcEphemeronEdges entries and
     // rebuild the full table. Note that this a cross-zone operation; delegate
@@ -664,7 +670,7 @@ bool Zone::findSweepGroupEdges(Zone* atomsZone) {
   return WeakMapBase::findSweepGroupEdgesForZone(this);
 }
 
-static bool AddEdgesForMarkQueue(GCMarker& marker) {
+bool GCRuntime::addEdgesForMarkQueue() {
 #ifdef DEBUG
   // For testing only.
   //
@@ -675,8 +681,8 @@ static bool AddEdgesForMarkQueue(GCMarker& marker) {
   // follow the sweep group ordering. These objects will wait until their sweep
   // group comes up, or will be skipped if their sweep group is already past.
   JS::Zone* prevZone = nullptr;
-  for (size_t i = 0; i < marker.markQueue.length(); i++) {
-    Value val = marker.markQueue[i].get();
+  for (size_t i = 0; i < testMarkQueue.length(); i++) {
+    Value val = testMarkQueue[i].get();
     if (!val.isObject()) {
       continue;
     }
@@ -703,7 +709,7 @@ bool GCRuntime::findSweepGroupEdges() {
     }
   }
 
-  if (!AddEdgesForMarkQueue(marker)) {
+  if (!addEdgesForMarkQueue()) {
     return false;
   }