Bug 1809141 - Use separate mutex when adding to the delayed marking list r=sfink

Currently this uses the GC lock, but that may already be held at this point if
we OOM during parallel marking. Also the helper thread mutex may be held if we
are marking helper thread state.

Adding to the delayed  marking list needs synchronization, but we only mark
this on a single thread (main or helper thread) so we don't need to lock there.

The patch adds a new mutex to use for this with mutex order higher that the
helper thread state mutex.

Differential Revision: https://phabricator.services.mozilla.com/D166325

js/src/gc/GC.cpp

@@ -458,6 +458,7 @@ GCRuntime::GCRuntime(JSRuntime* rt)
       alwaysPreserveCode(false),
       lowMemoryState(false),
       lock(mutexid::GCLock),
+      delayedMarkingLock(mutexid::GCDelayedMarkingLock),
       allocTask(this, emptyChunks_.ref()),
       unmarkTask(this),
       markTask(this),

js/src/gc/GCRuntime.h

@@ -625,8 +625,7 @@ class GCRuntime {
   static TenuredCell* refillFreeListInGC(Zone* zone, AllocKind thingKind);
 
   // Delayed marking.
-  void delayMarkingChildren(gc::Cell* cell, MarkColor color,
-                            const AutoLockGC& lock);
+  void delayMarkingChildren(gc::Cell* cell, MarkColor color);
   bool hasDelayedMarking() const;
   void markAllDelayedChildren(ShouldReportMarkTime reportTime);
 
@@ -1341,11 +1340,17 @@ class GCRuntime {
 
   MainThreadData<bool> lowMemoryState;
 
-  /* Synchronize GC heap access among GC helper threads and the main thread. */
+  /*
+   * General purpose GC lock, used for synchronising operations on
+   * arenas and during parallel marking.
+   */
   friend class js::AutoLockGC;
   friend class js::AutoLockGCBgAlloc;
   js::Mutex lock MOZ_UNANNOTATED;
 
+  /* Lock used to synchronise access to delayed marking state. */
+  js::Mutex delayedMarkingLock MOZ_UNANNOTATED;
+
   friend class BackgroundSweepTask;
   friend class BackgroundFreeTask;
 

js/src/gc/Marking.cpp

@@ -2122,8 +2122,7 @@ void GCMarker::abortLinearWeakMarking() {
 }
 
 MOZ_NEVER_INLINE void GCMarker::delayMarkingChildrenOnOOM(Cell* cell) {
-  AutoLockGC lock(runtime());
-  runtime()->gc.delayMarkingChildren(cell, markColor(), lock);
+  runtime()->gc.delayMarkingChildren(cell, markColor());
 }
 
 bool GCRuntime::hasDelayedMarking() const {
@@ -2132,8 +2131,10 @@ bool GCRuntime::hasDelayedMarking() const {
   return result;
 }
 
-void GCRuntime::delayMarkingChildren(Cell* cell, MarkColor color,
-                                     const AutoLockGC& lock) {
+void GCRuntime::delayMarkingChildren(Cell* cell, MarkColor color) {
+  // Synchronize access to delayed marking state during parallel marking.
+  LockGuard<Mutex> lock(delayedMarkingLock);
+
   Arena* arena = cell->asTenured().arena();
   if (!arena->onDelayedMarkingList()) {
     arena->setNextDelayedMarkingArena(delayedMarkingList);
@@ -2200,6 +2201,7 @@ void GCRuntime::processDelayedMarkingList(MarkColor color) {
 }
 
 void GCRuntime::markAllDelayedChildren(ShouldReportMarkTime reportTime) {
+  MOZ_ASSERT(CurrentThreadIsMainThread() || CurrentThreadIsPerformingGC());
   MOZ_ASSERT(marker().isDrained());
   MOZ_ASSERT(hasDelayedMarking());
 
@@ -2242,6 +2244,8 @@ void GCRuntime::rebuildDelayedMarkingList() {
 }
 
 void GCRuntime::resetDelayedMarking() {
+  MOZ_ASSERT(CurrentThreadIsMainThread());
+
   forEachDelayedMarkingArena([&](Arena* arena) {
     MOZ_ASSERT(arena->onDelayedMarkingList());
     arena->clearDelayedMarkingState();

js/src/vm/MutexIDs.h

@@ -59,6 +59,7 @@
   _(MemoryTracker, 500)               \
   _(StencilCache, 500)                \
   _(SourceCompression, 500)           \
+  _(GCDelayedMarkingLock, 500)        \
                                       \
   _(SharedImmutableStringsCache, 600) \
   _(IrregexpLazyStatic, 600)          \