Bug 1536061 - Change the gray root trace hook to allow gray roots to be marked incrementally r=sfink,mccr8

This adds a slice budget parameter and boolean return value to indicate whether tracing has finished.

Differential Revision: https://phabricator.services.mozilla.com/D125558

js/public/GCAPI.h

@@ -26,6 +26,7 @@ namespace js {
 namespace gc {
 class GCRuntime;
 }  // namespace gc
+class JS_PUBLIC_API SliceBudget;
 namespace gcstats {
 struct Statistics;
 }  // namespace gcstats
@@ -434,6 +435,18 @@ typedef enum JSGCParamKey {
  */
 typedef void (*JSTraceDataOp)(JSTracer* trc, void* data);
 
+/*
+ * Trace hook used to trace gray roots incrementally.
+ *
+ * This should return whether tracing is finished. It will be called repeatedly
+ * in subsequent GC slices until it returns true.
+ *
+ * While tracing this should check the budget and return false if it has been
+ * exceeded. When passed an unlimited budget it should always return true.
+ */
+typedef bool (*JSGrayRootsTracer)(JSTracer* trc, js::SliceBudget& budget,
+                                  void* data);
+
 typedef enum JSGCStatus { JSGC_BEGIN, JSGC_END } JSGCStatus;
 
 typedef void (*JSObjectsTenuredCallback)(JSContext* cx, void* data);

js/src/gc/GC.cpp

@@ -1395,7 +1395,7 @@ void GCRuntime::removeBlackRootsTracer(JSTraceDataOp traceOp, void* data) {
   }
 }
 
-void GCRuntime::setGrayRootsTracer(JSTraceDataOp traceOp, void* data) {
+void GCRuntime::setGrayRootsTracer(JSGrayRootsTracer traceOp, void* data) {
   AssertHeapIsIdle();
   grayRootTracer.ref() = {traceOp, data};
 }

js/src/gc/GCRuntime.h

@@ -447,7 +447,7 @@ class GCRuntime {
 
   bool initSweepActions();
 
-  void setGrayRootsTracer(JSTraceDataOp traceOp, void* data);
+  void setGrayRootsTracer(JSGrayRootsTracer traceOp, void* data);
   [[nodiscard]] bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data);
   void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data);
   void clearBlackAndGrayRootTracers();
@@ -1191,7 +1191,7 @@ class GCRuntime {
    * collector.
    */
   MainThreadData<CallbackVector<JSTraceDataOp>> blackRootTracers;
-  MainThreadOrGCTaskData<Callback<JSTraceDataOp>> grayRootTracer;
+  MainThreadOrGCTaskData<Callback<JSGrayRootsTracer>> grayRootTracer;
 
   /* Always preserve JIT code during GCs, for testing. */
   MainThreadData<bool> alwaysPreserveCode;

js/src/gc/RootMarking.cpp

@@ -414,8 +414,9 @@ void GCRuntime::traceEmbeddingGrayRoots(JSTracer* trc) {
   JS::AutoSuppressGCAnalysis nogc;
 
   const auto& callback = grayRootTracer.ref();
-  if (JSTraceDataOp op = callback.op) {
-    (*op)(trc, callback.data);
+  if (JSGrayRootsTracer op = callback.op) {
+    SliceBudget budget = SliceBudget::unlimited();
+    MOZ_ALWAYS_TRUE((*op)(trc, budget, callback.data));
   }
 }
 

js/src/jsapi-tests/testGCGrayMarking.cpp

@@ -661,10 +661,11 @@ void RemoveGrayRootTracer() {
   JS_SetGrayGCRootsTracer(cx, nullptr, nullptr);
 }
 
-static void TraceGrayRoots(JSTracer* trc, void* data) {
+static bool TraceGrayRoots(JSTracer* trc, SliceBudget& budget, void* data) {
   auto grayRoots = static_cast<GrayRoots*>(data);
   TraceEdge(trc, &grayRoots->grayRoot1, "gray root 1");
   TraceEdge(trc, &grayRoots->grayRoot2, "gray root 2");
+  return true;
 }
 
 JSObject* AllocPlainObject() {

js/src/jsfriendapi.cpp

@@ -67,7 +67,8 @@ JS::RootingContext::RootingContext() : realm_(nullptr), zone_(nullptr) {
 #endif
 }
 
-JS_PUBLIC_API void JS_SetGrayGCRootsTracer(JSContext* cx, JSTraceDataOp traceOp,
+JS_PUBLIC_API void JS_SetGrayGCRootsTracer(JSContext* cx,
+                                           JSGrayRootsTracer traceOp,
                                            void* data) {
   cx->runtime()->gc.setGrayRootsTracer(traceOp, data);
 }

js/src/jsfriendapi.h

@@ -31,7 +31,7 @@ class JSJitInfo;
  * required.
  */
 extern JS_PUBLIC_API void JS_SetGrayGCRootsTracer(JSContext* cx,
-                                                  JSTraceDataOp traceOp,
+                                                  JSGrayRootsTracer traceOp,
                                                   void* data);
 
 extern JS_PUBLIC_API JSObject* JS_FindCompilationScope(JSContext* cx,

js/src/shell/js.cpp

@@ -787,7 +787,7 @@ ShellContext* js::shell::GetShellContext(JSContext* cx) {
   return sc;
 }
 
-static void TraceGrayRoots(JSTracer* trc, void* data) {
+static bool TraceGrayRoots(JSTracer* trc, SliceBudget& budget, void* data) {
   JSRuntime* rt = trc->runtime();
   for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) {
     for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
@@ -798,6 +798,8 @@ static void TraceGrayRoots(JSTracer* trc, void* data) {
       }
     }
   }
+
+  return true;
 }
 
 static mozilla::UniqueFreePtr<char[]> GetLine(FILE* file, const char* prompt) {

xpcom/base/CycleCollectedJSRuntime.cpp

@@ -984,7 +984,9 @@ void CycleCollectedJSRuntime::TraceBlackJS(JSTracer* aTracer, void* aData) {
 }
 
 /* static */
-void CycleCollectedJSRuntime::TraceGrayJS(JSTracer* aTracer, void* aData) {
+bool CycleCollectedJSRuntime::TraceGrayJS(JSTracer* aTracer,
+                                          js::SliceBudget& budget,
+                                          void* aData) {
   CycleCollectedJSRuntime* self = static_cast<CycleCollectedJSRuntime*>(aData);
 
   // Mark these roots as gray so the CC can walk them later.
@@ -996,6 +998,8 @@ void CycleCollectedJSRuntime::TraceGrayJS(JSTracer* aTracer, void* aData) {
   }
 
   self->TraceNativeGrayRoots(aTracer, which);
+
+  return true;
 }
 
 /* static */

xpcom/base/CycleCollectedJSRuntime.h

@@ -270,7 +270,12 @@ class CycleCollectedJSRuntime {
   void TraverseNativeRoots(nsCycleCollectionNoteRootCallback& aCb);
 
   static void TraceBlackJS(JSTracer* aTracer, void* aData);
-  static void TraceGrayJS(JSTracer* aTracer, void* aData);
+
+  // Trace gray JS roots until budget is exceeded and return whether we
+  // finished.
+  static bool TraceGrayJS(JSTracer* aTracer, js::SliceBudget& budget,
+                          void* aData);
+
   static void GCCallback(JSContext* aContext, JSGCStatus aStatus,
                          JS::GCReason aReason, void* aData);
   static void GCSliceCallback(JSContext* aContext, JS::GCProgress aProgress,