Bug 1272604 - Add a zeal mode to check the heap after a moving GC r=terrence

js/public/TracingAPI.h

@@ -71,19 +71,37 @@ class JS_PUBLIC_API(JSTracer)
     bool isTenuringTracer() const { return tag_ == TracerKindTag::Tenuring; }
     bool isCallbackTracer() const { return tag_ == TracerKindTag::Callback; }
     inline JS::CallbackTracer* asCallbackTracer();
+#ifdef DEBUG
+    bool checkEdges() { return checkEdges_; }
+#endif
 
   protected:
     JSTracer(JSRuntime* rt, TracerKindTag tag,
              WeakMapTraceKind weakTraceKind = TraceWeakMapValues)
-      : runtime_(rt), weakMapAction_(weakTraceKind), tag_(tag)
+      : runtime_(rt)
+      , weakMapAction_(weakTraceKind)
+#ifdef DEBUG
+      , checkEdges_(true)
+#endif
+      , tag_(tag)
     {}
 
+#ifdef DEBUG
+    // Set whether to check edges are valid in debug builds.
+    void setCheckEdges(bool check) {
+        checkEdges_ = check;
+    }
+#endif
+
   private:
-    JSRuntime*          runtime_;
-    WeakMapTraceKind    weakMapAction_;
+    JSRuntime* runtime_;
+    WeakMapTraceKind weakMapAction_;
+#ifdef DEBUG
+    bool checkEdges_;
+#endif
 
   protected:
-    TracerKindTag       tag_;
+    TracerKindTag tag_;
 };
 
 namespace JS {

js/src/gc/GCInternals.h

@@ -120,8 +120,8 @@ struct MOZ_RAII AutoStopVerifyingBarriers
 #endif /* JS_GC_ZEAL */
 
 #ifdef JSGC_HASH_TABLE_CHECKS
-void
-CheckHashTablesAfterMovingGC(JSRuntime* rt);
+void CheckHashTablesAfterMovingGC(JSRuntime* rt);
+void CheckHeapAfterMovingGC(JSRuntime* rt);
 #endif
 
 struct MovingTracer : JS::CallbackTracer

js/src/gc/Marking.cpp

@@ -181,6 +181,9 @@ js::CheckTracedThing(JSTracer* trc, T* thing)
     MOZ_ASSERT(trc);
     MOZ_ASSERT(thing);
 
+    if (!trc->checkEdges())
+        return;
+
     thing = MaybeForwarded(thing);
 
     /* This function uses data that's not available in the nursery. */

js/src/gc/Nursery.cpp

@@ -502,6 +502,13 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason, ObjectGroupList
 #endif
     TIME_END(checkHashTables);
 
+    TIME_START(checkHeap);
+#ifdef JS_GC_ZEAL
+    if (rt->hasZealMode(ZealMode::CheckHeapOnMovingGC))
+        CheckHeapAfterMovingGC(rt);
+#endif
+    TIME_END(checkHeap);
+
     // Resize the nursery.
     TIME_START(resize);
     double promotionRate = mover.tenuredSize / double(allocationEnd() - start());
@@ -552,6 +559,7 @@ js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason, ObjectGroupList
             {"mcWCll", TIME_TOTAL(traceWholeCells)},
             {"mkGnrc", TIME_TOTAL(traceGenericEntries)},
             {"ckTbls", TIME_TOTAL(checkHashTables)},
+            {"ckHeap", TIME_TOTAL(checkHeap)},
             {"mkRntm", TIME_TOTAL(markRuntime)},
             {"mkDbgr", TIME_TOTAL(markDebugger)},
             {"clrNOC", TIME_TOTAL(clearNewObjectCache)},

js/src/gc/Verifier.cpp

@@ -8,6 +8,8 @@
 # include <valgrind/memcheck.h>
 #endif
 
+#include "mozilla/IntegerPrintfMacros.h"
+
 #include "jscntxt.h"
 #include "jsgc.h"
 #include "jsprf.h"
@@ -413,3 +415,121 @@ js::gc::GCRuntime::finishVerifier()
 }
 
 #endif /* JS_GC_ZEAL */
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+
+class CheckHeapTracer : public JS::CallbackTracer
+{
+  public:
+    CheckHeapTracer(JSRuntime* rt);
+    bool init();
+    bool check();
+
+  private:
+    void onChild(const JS::GCCellPtr& thing) override;
+
+    struct WorkItem {
+        WorkItem(JS::GCCellPtr thing, const char* name, int parentIndex)
+          : thing(thing), name(name), parentIndex(parentIndex), processed(false)
+        {}
+
+        JS::GCCellPtr thing;
+        const char* name;
+        int parentIndex;
+        bool processed;
+    };
+
+    JSRuntime* rt;
+    bool oom;
+    size_t failures;
+    HashSet<Cell*, DefaultHasher<Cell*>, SystemAllocPolicy> visited;
+    Vector<WorkItem, 0, SystemAllocPolicy> stack;
+    int parentIndex;
+};
+
+CheckHeapTracer::CheckHeapTracer(JSRuntime* rt)
+  : CallbackTracer(rt, TraceWeakMapKeysValues),
+    rt(rt),
+    oom(false),
+    failures(0),
+    parentIndex(-1)
+{
+    setCheckEdges(false);
+}
+
+bool
+CheckHeapTracer::init()
+{
+    return visited.init();
+}
+
+void
+CheckHeapTracer::onChild(const JS::GCCellPtr& thing)
+{
+    Cell* cell = thing.asCell();
+    if (visited.lookup(cell))
+        return;
+
+    if (!visited.put(cell)) {
+        oom = true;
+        return;
+    }
+
+    if (!IsGCThingValidAfterMovingGC(cell)) {
+        failures++;
+        fprintf(stderr, "Stale pointer %p\n", cell);
+        const char* name = contextName();
+        for (int index = parentIndex; index != -1; index = stack[index].parentIndex) {
+            const WorkItem& parent = stack[index];
+            cell = parent.thing.asCell();
+            fprintf(stderr, "  from %s %p %s edge\n",
+                    GCTraceKindToAscii(cell->getTraceKind()), cell, name);
+            name = parent.name;
+        }
+        fprintf(stderr, "  from root %s\n", name);
+        return;
+    }
+
+    WorkItem item(thing, contextName(), parentIndex);
+    if (!stack.append(item))
+        oom = true;
+}
+
+bool
+CheckHeapTracer::check()
+{
+    rt->gc.markRuntime(this, GCRuntime::TraceRuntime);
+
+    while (!stack.empty()) {
+        WorkItem item = stack.back();
+        if (item.processed) {
+            stack.popBack();
+        } else {
+            parentIndex = stack.length() - 1;
+            TraceChildren(this, item.thing);
+            stack.back().processed = true;
+        }
+    }
+
+    if (oom)
+        return false;
+
+    if (failures) {
+        fprintf(stderr, "Heap check: %zu failure(s) out of %" PRIu32 " pointers checked\n",
+                failures, visited.count());
+    }
+    MOZ_RELEASE_ASSERT(failures == 0);
+
+    return true;
+}
+
+void
+js::gc::CheckHeapAfterMovingGC(JSRuntime* rt)
+{
+    MOZ_ASSERT(rt->isHeapCollecting());
+    CheckHeapTracer tracer(rt);
+    if (!tracer.init() || !tracer.check())
+        fprintf(stderr, "OOM checking heap\n");
+}
+
+#endif /* JSGC_HASH_TABLE_CHECKS */

js/src/jsgc.cpp

@@ -1214,7 +1214,8 @@ const char* gc::ZealModeHelpText =
     "   11: (IncrementalMarkingValidator) Verify incremental marking\n"
     "   12: (ElementsBarrier) Always use the individual element post-write barrier, regardless of elements size\n"
     "   13: (CheckHashTablesOnMinorGC) Check internal hashtables on minor GC\n"
-    "   14: (Compact) Perform a shrinking collection every N allocations\n";
+    "   14: (Compact) Perform a shrinking collection every N allocations\n"
+    "   15: (CheckHeapOnMovingGC) Walk the heap to check all pointers have been updated\n";
 
 void
 GCRuntime::setZeal(uint8_t zeal, uint32_t frequency)
@@ -5893,6 +5894,10 @@ GCRuntime::compactPhase(JS::gcreason::Reason reason, SliceBudget& sliceBudget)
 #ifdef DEBUG
     CheckHashTablesAfterMovingGC(rt);
 #endif
+#ifdef JS_GC_ZEAL
+    if (rt->hasZealMode(ZealMode::CheckHeapOnMovingGC))
+        CheckHeapAfterMovingGC(rt);
+#endif
 
     return zonesToMaybeCompact.isEmpty() ? Finished : NotFinished;
 }

js/src/jsgc.h

@@ -1185,14 +1185,19 @@ MaybeForwarded(T t)
 
 #ifdef JSGC_HASH_TABLE_CHECKS
 
+template <typename T>
+inline bool
+IsGCThingValidAfterMovingGC(T* t)
+{
+    return !IsInsideNursery(t) && !RelocationOverlay::isCellForwarded(t);
+}
+
 template <typename T>
 inline void
 CheckGCThingAfterMovingGC(T* t)
 {
-    if (t) {
-        MOZ_RELEASE_ASSERT(!IsInsideNursery(t));
-        MOZ_RELEASE_ASSERT(!RelocationOverlay::isCellForwarded(t));
-    }
+    if (t)
+        MOZ_RELEASE_ASSERT(IsGCThingValidAfterMovingGC(t));
 }
 
 template <typename T>
@@ -1228,13 +1233,14 @@ CheckValueAfterMovingGC(const JS::Value& value)
             D(IncrementalMarkingValidator, 11) \
             D(ElementsBarrier, 12)             \
             D(CheckHashTablesOnMinorGC, 13)    \
-            D(Compact, 14)
+            D(Compact, 14)                     \
+            D(CheckHeapOnMovingGC, 15)
 
 enum class ZealMode {
 #define ZEAL_MODE(name, value) name = value,
     JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE)
 #undef ZEAL_MODE
-    Limit = 14
+    Limit = 15
 };
 
 enum VerifierType {