diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index 895188e3869c..cbd001dc723b 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -700,6 +700,12 @@ class GCRuntime GCHelperState helperState; + /* + * During incremental sweeping, this field temporarily holds the arenas of + * the current AllocKind being swept in order of increasing free space. + */ + SortedArenaList incrementalSweepList; + ConservativeGCData conservativeGC; friend class js::GCHelperState; diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h index 1c1087cec734..b6e664f0da6e 100644 --- a/js/src/gc/Heap.h +++ b/js/src/gc/Heap.h @@ -39,6 +39,7 @@ namespace gc { struct Arena; class ArenaList; +class SortedArenaList; struct ArenaHeader; struct Chunk; @@ -597,7 +598,7 @@ struct Arena void setAsFullyUnused(AllocKind thingKind); template - bool finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize); + size_t finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize); }; static_assert(sizeof(Arena) == ArenaSize, "The hardcoded arena size must match the struct size."); @@ -828,7 +829,8 @@ struct Chunk ArenaHeader *allocateArena(JS::Zone *zone, AllocKind kind); void releaseArena(ArenaHeader *aheader); - void recycleArena(ArenaHeader *aheader, ArenaList &dest, AllocKind thingKind); + void recycleArena(ArenaHeader *aheader, SortedArenaList &dest, AllocKind thingKind, + size_t thingsPerArena); static Chunk *allocate(JSRuntime *rt); diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index dd2fa1517256..804b8a168ee8 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -175,6 +175,7 @@ #include "mozilla/ArrayUtils.h" #include "mozilla/DebugOnly.h" +#include "mozilla/MacroForEach.h" #include "mozilla/MemoryReporting.h" #include "mozilla/Move.h" @@ -255,7 +256,14 @@ const AllocKind gc::slotsToThingKind[] = { static_assert(JS_ARRAY_LENGTH(slotsToThingKind) == SLOTS_TO_THING_KIND_LIMIT, "We have defined a slot count for each kind."); -const uint32_t Arena::ThingSizes[] = { +// Assert that SortedArenaList::MinThingSize is <= the real minimum thing size. +#define CHECK_MIN_THING_SIZE_INNER(x_) \ + static_assert(x_ >= SortedArenaList::MinThingSize, \ + #x_ " is less than SortedArenaList::MinThingSize!"); +#define CHECK_MIN_THING_SIZE(...) { __VA_ARGS__ }; /* Define the array. */ \ + MOZ_FOR_EACH(CHECK_MIN_THING_SIZE_INNER, (), (__VA_ARGS__ UINT32_MAX)) + +const uint32_t Arena::ThingSizes[] = CHECK_MIN_THING_SIZE( sizeof(JSObject), /* FINALIZE_OBJECT0 */ sizeof(JSObject), /* FINALIZE_OBJECT0_BACKGROUND */ sizeof(JSObject_Slots2), /* FINALIZE_OBJECT2 */ @@ -278,7 +286,10 @@ const uint32_t Arena::ThingSizes[] = { sizeof(JSExternalString), /* FINALIZE_EXTERNAL_STRING */ sizeof(JS::Symbol), /* FINALIZE_SYMBOL */ sizeof(jit::JitCode), /* FINALIZE_JITCODE */ -}; +); + +#undef CHECK_MIN_THING_SIZE_INNER +#undef CHECK_MIN_THING_SIZE #define OFFSET(type) uint32_t(sizeof(ArenaHeader) + (ArenaSize - sizeof(ArenaHeader)) % sizeof(type)) @@ -450,7 +461,7 @@ Arena::setAsFullyUnused(AllocKind thingKind) } template -inline bool +inline size_t Arena::finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize) { /* Enforce requirements on size of T. */ @@ -496,7 +507,7 @@ Arena::finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize) // Do nothing. The caller will update the arena header appropriately. JS_ASSERT(newListTail == &newListHead); JS_EXTRA_POISON(data, JS_SWEPT_TENURED_PATTERN, sizeof(data)); - return true; + return nmarked; } JS_ASSERT(firstThingOrSuccessorOfLastMarkedThing != firstThing); @@ -517,20 +528,38 @@ Arena::finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize) JS_ASSERT(nfree + nmarked == thingsPerArena(thingSize)); #endif aheader.setFirstFreeSpan(&newListHead); - return false; + return nmarked; +} + +ArenaList +SortedArenaList::toArenaList() +{ + // Link the non-empty segment tails up to the non-empty segment heads. + size_t tailIndex = 0; + for (size_t headIndex = 1; headIndex <= thingsPerArena_; ++headIndex) { + if (headAt(headIndex)) { + segments[tailIndex].linkTo(headAt(headIndex)); + tailIndex = headIndex; + } + } + // Point the tail of the final non-empty segment at null. Note that if the + // list is empty, this will just set segments[0].head to null, a noop. + segments[tailIndex].linkTo(nullptr); + // Return an ArenaList representing the flattened list. + return ArenaList(segments[0]); } template static inline bool FinalizeTypedArenas(FreeOp *fop, ArenaHeader **src, - ArenaList &dest, + SortedArenaList &dest, AllocKind thingKind, SliceBudget &budget) { /* * Finalize arenas from src list, releasing empty arenas and inserting the - * others into dest in an appropriate position. + * others into the appropriate destination size bins. */ /* @@ -541,22 +570,24 @@ FinalizeTypedArenas(FreeOp *fop, bool releaseArenas = !InParallelSection(); size_t thingSize = Arena::thingSize(thingKind); + size_t thingsPerArena = Arena::thingsPerArena(thingSize); while (ArenaHeader *aheader = *src) { *src = aheader->next; - bool allClear = aheader->getArena()->finalize(fop, thingKind, thingSize); - if (!allClear) - dest.insertAtCursor(aheader); + size_t nmarked = aheader->getArena()->finalize(fop, thingKind, thingSize); + size_t nfree = thingsPerArena - nmarked; + + if (nmarked) + dest.insertAt(aheader, nfree); else if (releaseArenas) aheader->chunk()->releaseArena(aheader); else - aheader->chunk()->recycleArena(aheader, dest, thingKind); + aheader->chunk()->recycleArena(aheader, dest, thingKind, thingsPerArena); - budget.step(Arena::thingsPerArena(thingSize)); + budget.step(thingsPerArena); if (budget.isOverBudget()) return false; } - dest.deepCheck(); return true; } @@ -568,7 +599,7 @@ FinalizeTypedArenas(FreeOp *fop, static bool FinalizeArenas(FreeOp *fop, ArenaHeader **src, - ArenaList &dest, + SortedArenaList &dest, AllocKind thingKind, SliceBudget &budget) { @@ -965,10 +996,11 @@ Chunk::addArenaToFreeList(JSRuntime *rt, ArenaHeader *aheader) } void -Chunk::recycleArena(ArenaHeader *aheader, ArenaList &dest, AllocKind thingKind) +Chunk::recycleArena(ArenaHeader *aheader, SortedArenaList &dest, AllocKind thingKind, + size_t thingsPerArena) { aheader->getArena()->setAsFullyUnused(thingKind); - dest.insertAtCursor(aheader); + dest.insertAt(aheader, thingsPerArena); } void @@ -1893,10 +1925,6 @@ ArenaLists::allocateFromArenaInline(Zone *zone, AllocKind thingKind, * While we still hold the GC lock get an arena from some chunk, mark it * as full as its single free span is moved to the free lists, and insert * it to the list as a fully allocated arena. - * - * We add the arena before the the head, so that after the GC the most - * recently added arena will be used first for allocations. This improves - * cache locality. */ JS_ASSERT(al->isCursorAtEnd()); aheader = chunk->allocateArena(zone, thingKind); @@ -1905,7 +1933,7 @@ ArenaLists::allocateFromArenaInline(Zone *zone, AllocKind thingKind, if (MOZ_UNLIKELY(zone->wasGCStarted())) rt->gc.arenaAllocatedDuringGC(zone, aheader); - al->insertAtStart(aheader); + al->insertAtCursor(aheader); /* * Allocate from a newly allocated arena. The arena will have been set up @@ -1978,9 +2006,14 @@ ArenaLists::forceFinalizeNow(FreeOp *fop, AllocKind thingKind) ArenaHeader *arenas = arenaLists[thingKind].head(); arenaLists[thingKind].clear(); + size_t thingsPerArena = Arena::thingsPerArena(Arena::thingSize(thingKind)); + SortedArenaList finalizedSorted(thingsPerArena); + SliceBudget budget; - FinalizeArenas(fop, &arenas, arenaLists[thingKind], thingKind, budget); + FinalizeArenas(fop, &arenas, finalizedSorted, thingKind, budget); JS_ASSERT(!arenas); + + arenaLists[thingKind] = finalizedSorted.toArenaList(); } void @@ -2028,23 +2061,31 @@ ArenaLists::backgroundFinalize(FreeOp *fop, ArenaHeader *listHead, bool onBackgr AllocKind thingKind = listHead->getAllocKind(); Zone *zone = listHead->zone; - ArenaList finalized; + size_t thingsPerArena = Arena::thingsPerArena(Arena::thingSize(thingKind)); + SortedArenaList finalizedSorted(thingsPerArena); + SliceBudget budget; - FinalizeArenas(fop, &listHead, finalized, thingKind, budget); + FinalizeArenas(fop, &listHead, finalizedSorted, thingKind, budget); JS_ASSERT(!listHead); - // When arenas are queued for background finalization, all - // arenas are moved to arenaListsToSweep[], leaving the arenaLists[] empty. - // Then, if new arenas are allocated before background finalization - // finishes they are always added to the front of the list. Therefore, - // at this point, |al|'s cursor will always be at the end of its list. + // When arenas are queued for background finalization, all arenas are moved + // to arenaListsToSweep[], leaving the arenaLists[] empty. However, new + // arenas may be allocated before background finalization finishes; now that + // finalization is complete, we want to merge these lists back together. ArenaLists *lists = &zone->allocator.arenas; ArenaList *al = &lists->arenaLists[thingKind]; + // Flatten |finalizedSorted| into a regular ArenaList. + ArenaList finalized = finalizedSorted.toArenaList(); + + // Store this for later, since merging may change the state of |finalized|. + bool allClear = finalized.isEmpty(); + AutoLockGC lock(fop->runtime()); JS_ASSERT(lists->backgroundFinalizeState[thingKind] == BFS_RUN); - al->appendToListWithCursorAtEnd(finalized); + // Join |al| and |finalized| into a single list. + *al = finalized.insertListWithCursorAtEnd(*al); /* * We must set the state to BFS_JUST_FINISHED if we are running on the @@ -2055,7 +2096,7 @@ ArenaLists::backgroundFinalize(FreeOp *fop, ArenaHeader *listHead, bool onBackgr * allocating new arenas from the chunks we can set the state to BFS_DONE if * we have released all finalized arenas back to their chunks. */ - if (onBackgroundThread && !finalized.isEmpty()) + if (onBackgroundThread && !allClear) lists->backgroundFinalizeState[thingKind] = BFS_JUST_FINISHED; else lists->backgroundFinalizeState[thingKind] = BFS_DONE; @@ -4314,13 +4355,17 @@ GCRuntime::beginSweepPhase(bool lastGC) } bool -ArenaLists::foregroundFinalize(FreeOp *fop, AllocKind thingKind, SliceBudget &sliceBudget) +ArenaLists::foregroundFinalize(FreeOp *fop, AllocKind thingKind, SliceBudget &sliceBudget, + SortedArenaList &sweepList) { - if (!arenaListsToSweep[thingKind]) - return true; + if (!FinalizeArenas(fop, &arenaListsToSweep[thingKind], sweepList, thingKind, sliceBudget)) + return false; - ArenaList &dest = arenaLists[thingKind]; - return FinalizeArenas(fop, &arenaListsToSweep[thingKind], dest, thingKind, sliceBudget); + // Join |arenaLists[thingKind]| and |sweepList| into a single list. + ArenaList finalized = sweepList.toArenaList(); + arenaLists[thingKind] = finalized.insertListWithCursorAtEnd(arenaLists[thingKind]); + + return true; } bool @@ -4352,9 +4397,17 @@ GCRuntime::sweepPhase(SliceBudget &sliceBudget) while (sweepKindIndex < FinalizePhaseLength[finalizePhase]) { AllocKind kind = FinalizePhases[finalizePhase][sweepKindIndex]; - if (!zone->allocator.arenas.foregroundFinalize(&fop, kind, sliceBudget)) + /* Set the number of things per arena for this AllocKind. */ + size_t thingsPerArena = Arena::thingsPerArena(Arena::thingSize(kind)); + incrementalSweepList.setThingsPerArena(thingsPerArena); + + if (!zone->allocator.arenas.foregroundFinalize(&fop, kind, sliceBudget, + incrementalSweepList)) return false; /* Yield to the mutator. */ + /* Reset the slots of the sweep list that we used. */ + incrementalSweepList.reset(thingsPerArena); + ++sweepKindIndex; } sweepKindIndex = 0; @@ -5727,8 +5780,8 @@ ArenaLists::adoptArenas(JSRuntime *rt, ArenaLists *fromArenaLists) #endif ArenaList *fromList = &fromArenaLists->arenaLists[thingKind]; ArenaList *toList = &arenaLists[thingKind]; - fromList->deepCheck(); - toList->deepCheck(); + fromList->check(); + toList->check(); ArenaHeader *next; for (ArenaHeader *fromHeader = fromList->head(); fromHeader; fromHeader = next) { // Copy fromHeader->next before releasing/reinserting. @@ -5744,7 +5797,7 @@ ArenaLists::adoptArenas(JSRuntime *rt, ArenaLists *fromArenaLists) toList->insertAtCursor(fromHeader); } fromList->clear(); - toList->deepCheck(); + toList->check(); } } diff --git a/js/src/jsgc.h b/js/src/jsgc.h index 4a0a01a511d7..99e95615a6b8 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -383,6 +383,90 @@ GetGCKindSlots(AllocKind thingKind, const Class *clasp) // above locks is held. class AutoMaybeStartBackgroundAllocation; +// A single segment of a SortedArenaList. Each segment has a head and a tail, +// which track the start and end of a segment for O(1) append and concatenation. +struct SortedArenaListSegment +{ + ArenaHeader *head; + ArenaHeader **tailp; + + void clear() { + head = nullptr; + tailp = &head; + } + + bool isEmpty() const { + return tailp == &head; + } + + // Appends |aheader| to this segment. + void append(ArenaHeader *aheader) { + JS_ASSERT(aheader); + JS_ASSERT_IF(head, head->getAllocKind() == aheader->getAllocKind()); + *tailp = aheader; + tailp = &aheader->next; + } + + // Points the tail of this segment at |aheader|, which may be null. + void linkTo(ArenaHeader *aheader) { + *tailp = aheader; + } +}; + +// A class that holds arenas in sorted order by appending arenas to specific +// segments. SortedArenaLists can be flattened to a regular ArenaList. +class SortedArenaList +{ + public: + // The minimum size, in bytes, of a GC thing. + static const size_t MinThingSize = 16; + + static_assert(ArenaSize <= 4096, "When increasing the Arena size, please consider how"\ + " this will affect the size of a SortedArenaList."); + + static_assert(MinThingSize >= 16, "When decreasing the minimum thing size, please consider"\ + " how this will affect the size of a SortedArenaList."); + + private: + // The maximum number of GC things that an arena can hold. + static const size_t MaxThingsPerArena = (ArenaSize - sizeof(ArenaHeader)) / MinThingSize; + + size_t thingsPerArena_; + SortedArenaListSegment segments[MaxThingsPerArena + 1]; + + // Convenience functions to get the nth head and tail. + ArenaHeader *headAt(size_t n) { return segments[n].head; } + ArenaHeader **tailAt(size_t n) { return segments[n].tailp; } + + public: + explicit SortedArenaList(size_t thingsPerArena = MaxThingsPerArena) { + reset(thingsPerArena); + } + + void setThingsPerArena(size_t thingsPerArena) { + JS_ASSERT(thingsPerArena && thingsPerArena <= MaxThingsPerArena); + thingsPerArena_ = thingsPerArena; + } + + // Resets the first |thingsPerArena| segments of this list for further use. + void reset(size_t thingsPerArena = MaxThingsPerArena) { + setThingsPerArena(thingsPerArena); + // Initialize the segments. + for (size_t i = 0; i <= thingsPerArena; ++i) + segments[i].clear(); + } + + // Inserts a header, which has room for |nfree| more things, in its segment. + void insertAt(ArenaHeader *aheader, size_t nfree) { + JS_ASSERT(nfree <= thingsPerArena_); + segments[nfree].append(aheader); + } + + // Flattens the SortedArenaList into a regular ArenaList. Although we don't + // expect to do this more than once, note this operation is not destructive. + ArenaList toArenaList(); +}; + /* * Arena lists have a head and a cursor. The cursor conceptually lies on arena * boundaries, i.e. before the first arena, between two arenas, or after the @@ -425,11 +509,33 @@ class ArenaList { ArenaHeader *head_; ArenaHeader **cursorp_; + void copy(const ArenaList &other) { + other.check(); + head_ = other.head_; + cursorp_ = other.isCursorAtHead() ? &head_ : other.cursorp_; + check(); + } + public: ArenaList() { clear(); } + ArenaList(const ArenaList &other) { + copy(other); + } + + ArenaList &operator=(const ArenaList &other) { + copy(other); + return *this; + } + + ArenaList(const SortedArenaListSegment &segment) { + head_ = segment.head; + cursorp_ = segment.isEmpty() ? &head_ : segment.tailp; + check(); + } + // This does checking just of |head_| and |cursorp_|. void check() const { #ifdef DEBUG @@ -442,28 +548,6 @@ class ArenaList { #endif } - // This does checking involving all the arenas in the list. - void deepCheck() const { -#ifdef DEBUG - check(); - // All full arenas must precede all non-full arenas. - // - // XXX: this is currently commented out because it fails moderately - // often. I'm not sure if this is because (a) it's not true that all - // full arenas must precede all non-full arenas, or (b) we have some - // defective list-handling code. - // -// bool havePassedFullArenas = false; -// for (ArenaHeader *aheader = head_; aheader; aheader = aheader->next) { -// if (havePassedFullArenas) { -// JS_ASSERT(aheader->hasFreeThings()); -// } else if (aheader->hasFreeThings()) { -// havePassedFullArenas = true; -// } -// } -#endif - } - void clear() { head_ = nullptr; cursorp_ = &head_; @@ -481,6 +565,11 @@ class ArenaList { return head_; } + bool isCursorAtHead() const { + check(); + return cursorp_ == &head_; + } + bool isCursorAtEnd() const { check(); return !*cursorp_; @@ -515,33 +604,19 @@ class ArenaList { check(); } - // This inserts |a| at the start of the list, and doesn't change the - // cursor. - void insertAtStart(ArenaHeader *a) { + // This inserts |other|, which must be full, at the cursor of |this|. + ArenaList &insertListWithCursorAtEnd(const ArenaList &other) { check(); - a->next = head_; - if (isEmpty()) - cursorp_ = &a->next; // The cursor remains null. - head_ = a; + other.check(); + JS_ASSERT(other.isCursorAtEnd()); + if (other.isCursorAtHead()) + return *this; + // Insert the full arenas of |other| after those of |this|. + *other.cursorp_ = *cursorp_; + *cursorp_ = other.head_; + cursorp_ = other.cursorp_; check(); - } - - // Appends |list|. |this|'s cursor must be at the end. - void appendToListWithCursorAtEnd(ArenaList &other) { - JS_ASSERT(isCursorAtEnd()); - deepCheck(); - other.deepCheck(); - if (!other.isEmpty()) { - // Because |this|'s cursor is at the end, |cursorp_| points to the - // list-ending null. So this assignment appends |other| to |this|. - *cursorp_ = other.head_; - - // If |other|'s cursor isn't at the start of the list, then update - // |this|'s cursor accordingly. - if (other.cursorp_ != &other.head_) - cursorp_ = other.cursorp_; - } - deepCheck(); + return *this; } }; @@ -800,7 +875,8 @@ class ArenaLists void queueScriptsForSweep(FreeOp *fop); void queueJitCodeForSweep(FreeOp *fop); - bool foregroundFinalize(FreeOp *fop, AllocKind thingKind, SliceBudget &sliceBudget); + bool foregroundFinalize(FreeOp *fop, AllocKind thingKind, SliceBudget &sliceBudget, + SortedArenaList &sweepList); static void backgroundFinalize(FreeOp *fop, ArenaHeader *listHead, bool onBackgroundThread); void wipeDuringParallelExecution(JSRuntime *rt);