Bug 1573458 - Leave the atoms zone when performing a GC r=tcampbell

Entering the atoms zone with AutoAllocInAtomsZone is a bit of a special case and we don't support entering another realm in this state. Unfortunately this can happen during GC in a couple of place. The patch temporarily leaves the atoms zone during GC so that callbacks can enter whatever zones they like.

Differential Revision: https://phabricator.services.mozilla.com/D42312

--HG--
extra : moz-landing-system : lando

js/src/builtin/TestingFunctions.cpp

@@ -4402,6 +4402,10 @@ static void minorGC(JSContext* cx, JSGCStatus status, void* data) {
 static MajorGC majorGCInfo;
 static MinorGC minorGCInfo;
 
+static void enterNullRealm(JSContext* cx, JSGCStatus status, void* data) {
+  JSAutoNullableRealm enterRealm(cx, nullptr);
+}
+
 } /* namespace gcCallback */
 
 static bool SetGCCallback(JSContext* cx, unsigned argc, Value* vp) {
@@ -4489,6 +4493,8 @@ static bool SetGCCallback(JSContext* cx, unsigned argc, Value* vp) {
     gcCallback::majorGCInfo.phases = phases;
     gcCallback::majorGCInfo.depth = depth;
     JS_SetGCCallback(cx, gcCallback::majorGC, &gcCallback::majorGCInfo);
+  } else if (StringEqualsAscii(action, "enterNullRealm")) {
+    JS_SetGCCallback(cx, gcCallback::enterNullRealm, nullptr);
   } else {
     JS_ReportErrorASCII(cx, "Unknown GC callback action");
     return false;

js/src/gc/GC.cpp

@@ -7724,6 +7724,7 @@ void GCRuntime::collect(bool nonincrementalByAPI, SliceBudget budget,
   AutoTraceLog logGC(TraceLoggerForCurrentThread(), TraceLogger_GC);
   AutoStopVerifyingBarriers av(rt, IsShutdownGC(reason));
   AutoEnqueuePendingParseTasksAfterGC aept(*this);
+  AutoMaybeLeaveAtomsZone leaveAtomsZone(rt->mainContextFromOwnThread());
 
 #ifdef DEBUG
   if (IsShutdownGC(reason)) {
@@ -7931,6 +7932,8 @@ void GCRuntime::minorGC(JS::GCReason reason, gcstats::PhaseKind phase) {
     return;
   }
 
+  AutoMaybeLeaveAtomsZone leaveAtomsZone(rt->mainContextFromOwnThread());
+
   // Note that we aren't collecting the updated alloc counts from any helper
   // threads.  We should be but I'm not sure where to add that
   // synchronisation.

js/src/gc/Nursery.cpp

@@ -1151,7 +1151,6 @@ float js::Nursery::doPretenuring(JSRuntime* rt, JS::GCReason reason,
       }
 
       ObjectGroup* group = entry.group;
-      AutoMaybeLeaveAtomsZone leaveAtomsZone(cx);
       AutoRealm ar(cx, group);
       AutoSweepObjectGroup sweep(group);
       if (group->canPreTenure(sweep)) {

js/src/jit-test/tests/gc/bug-1573458.js

@@ -0,0 +1,4 @@
+gczeal(0);
+setGCCallback({action: "enterNullRealm"});
+gczeal(2, 1);
+Symbol();

js/src/vm/Realm.h

@@ -892,8 +892,9 @@ class MOZ_RAII AutoAllocInAtomsZone {
   inline ~AutoAllocInAtomsZone();
 };
 
-// For the one place where we need to enter a realm when we may have been
-// allocating in the the atoms zone, this leaves the atoms zone temporarily.
+// During GC we sometimes need to enter a realm when we may have been allocating
+// in the the atoms zone. This leaves the atoms zone temporarily. This happens
+// in embedding callbacks and when we need to mark object groups as pretenured.
 class MOZ_RAII AutoMaybeLeaveAtomsZone {
   JSContext* const cx_;
   bool wasInAtomsZone_;