From f1b63963d7cd5af6e74006547881ae71357a803d Mon Sep 17 00:00:00 2001 From: Jeff Walden Date: Tue, 1 May 2012 20:04:36 -0700 Subject: [PATCH] Bug 751003 - Move Cell, Chunk, Arena, ArenaHeader, FreeSpan, ChunkInfo, ChunkBitmap, your mom into a new gc/Heap.h header that doesn't have any used-but-not-defined traps for the unwary #include-r. r=billm --HG-- extra : rebase_source : 5ef81fb9032ac5c13fb4cf2a6107a0f63343caa1 --- js/src/Makefile.in | 4 +- js/src/gc/Barrier.h | 2 +- js/src/gc/Heap.h | 984 +++++++++++++++++++++++++++++++++++++ js/src/jscell.h | 119 ----- js/src/jsgc.h | 865 +------------------------------- js/src/jsinfer.h | 2 +- js/src/jsobj.h | 2 +- js/src/jsstr.h | 1 - js/src/jsxml.h | 2 +- js/src/vm/ObjectImpl-inl.h | 2 +- js/src/vm/String.h | 2 +- 11 files changed, 993 insertions(+), 992 deletions(-) create mode 100644 js/src/gc/Heap.h delete mode 100644 js/src/jscell.h diff --git a/js/src/Makefile.in b/js/src/Makefile.in index bb458c88b04..fbe1ff05b48 100644 --- a/js/src/Makefile.in +++ b/js/src/Makefile.in @@ -196,7 +196,6 @@ INSTALLED_HEADERS = \ jsdhash.h \ jsfriendapi.h \ jsgc.h \ - jscell.h \ jshash.h \ jslock.h \ json.h \ @@ -223,9 +222,10 @@ EXPORTS_ds = \ $(NULL) EXPORTS_gc = \ - Statistics.h \ Barrier.h \ + Heap.h \ Root.h \ + Statistics.h \ $(NULL) ###################################################### diff --git a/js/src/gc/Barrier.h b/js/src/gc/Barrier.h index c0679e29f90..6bb104e9560 100644 --- a/js/src/gc/Barrier.h +++ b/js/src/gc/Barrier.h @@ -41,8 +41,8 @@ #define jsgc_barrier_h___ #include "jsapi.h" -#include "jscell.h" +#include "gc/Heap.h" #include "js/HashTable.h" /* diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h new file mode 100644 index 00000000000..bdab98fd330 --- /dev/null +++ b/js/src/gc/Heap.h @@ -0,0 +1,984 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_heap_h___ +#define gc_heap_h___ + +#include "mozilla/Attributes.h" +#include "mozilla/StandardInteger.h" + +#include + +#include "jstypes.h" +#include "jsutil.h" + +#include "ds/BitArray.h" + +struct JSCompartment; + +extern "C" { +struct JSRuntime; +} + +namespace js { + +class FreeOp; + +namespace gc { + +struct Arena; +struct ArenaHeader; +struct Chunk; + +/* + * Live objects are marked black. How many other additional colors are available + * depends on the size of the GCThing. Objects marked gray are eligible for + * cycle collection. + */ +static const uint32_t BLACK = 0; +static const uint32_t GRAY = 1; + +/* The GC allocation kinds. */ +enum AllocKind { + FINALIZE_OBJECT0, + FINALIZE_OBJECT0_BACKGROUND, + FINALIZE_OBJECT2, + FINALIZE_OBJECT2_BACKGROUND, + FINALIZE_OBJECT4, + FINALIZE_OBJECT4_BACKGROUND, + FINALIZE_OBJECT8, + FINALIZE_OBJECT8_BACKGROUND, + FINALIZE_OBJECT12, + FINALIZE_OBJECT12_BACKGROUND, + FINALIZE_OBJECT16, + FINALIZE_OBJECT16_BACKGROUND, + FINALIZE_OBJECT_LAST = FINALIZE_OBJECT16_BACKGROUND, + FINALIZE_SCRIPT, + FINALIZE_SHAPE, + FINALIZE_BASE_SHAPE, + FINALIZE_TYPE_OBJECT, +#if JS_HAS_XML_SUPPORT + FINALIZE_XML, +#endif + FINALIZE_SHORT_STRING, + FINALIZE_STRING, + FINALIZE_EXTERNAL_STRING, + FINALIZE_LAST = FINALIZE_EXTERNAL_STRING +}; + +static const unsigned FINALIZE_LIMIT = FINALIZE_LAST + 1; +static const unsigned FINALIZE_OBJECT_LIMIT = FINALIZE_OBJECT_LAST + 1; + +/* + * This must be an upper bound, but we do not need the least upper bound, so + * we just exclude non-background objects. + */ +static const size_t MAX_BACKGROUND_FINALIZE_KINDS = FINALIZE_LIMIT - FINALIZE_OBJECT_LIMIT / 2; + +/* + * A GC cell is the base class for all GC things. + */ +struct Cell +{ + static const size_t CellShift = 3; + static const size_t CellSize = size_t(1) << CellShift; + static const size_t CellMask = CellSize - 1; + + inline uintptr_t address() const; + inline ArenaHeader *arenaHeader() const; + inline Chunk *chunk() const; + inline AllocKind getAllocKind() const; + MOZ_ALWAYS_INLINE bool isMarked(uint32_t color = BLACK) const; + MOZ_ALWAYS_INLINE bool markIfUnmarked(uint32_t color = BLACK) const; + MOZ_ALWAYS_INLINE void unmark(uint32_t color) const; + + inline JSCompartment *compartment() const; + +#ifdef DEBUG + inline bool isAligned() const; +#endif +}; + +/* + * Page size is 4096 by default, except for SPARC, where it is 8192. + * Note: Do not use JS_CPU_SPARC here, this header is used outside JS. + * Bug 692267: Move page size definition to gc/Memory.h and include it + * directly once jsgc.h is no longer an installed header. + */ +#if defined(SOLARIS) && (defined(__sparc) || defined(__sparcv9)) +const size_t PageShift = 13; +#else +const size_t PageShift = 12; +#endif +const size_t PageSize = size_t(1) << PageShift; + +const size_t ChunkShift = 20; +const size_t ChunkSize = size_t(1) << ChunkShift; +const size_t ChunkMask = ChunkSize - 1; + +const size_t ArenaShift = PageShift; +const size_t ArenaSize = PageSize; +const size_t ArenaMask = ArenaSize - 1; + +/* + * This is the maximum number of arenas we allow in the FreeCommitted state + * before we trigger a GC_SHRINK to release free arenas to the OS. + */ +const static uint32_t FreeCommittedArenasThreshold = (32 << 20) / ArenaSize; + +/* + * The mark bitmap has one bit per each GC cell. For multi-cell GC things this + * wastes space but allows to avoid expensive devisions by thing's size when + * accessing the bitmap. In addition this allows to use some bits for colored + * marking during the cycle GC. + */ +const size_t ArenaCellCount = size_t(1) << (ArenaShift - Cell::CellShift); +const size_t ArenaBitmapBits = ArenaCellCount; +const size_t ArenaBitmapBytes = ArenaBitmapBits / 8; +const size_t ArenaBitmapWords = ArenaBitmapBits / JS_BITS_PER_WORD; + +/* + * A FreeSpan represents a contiguous sequence of free cells in an Arena. + * |first| is the address of the first free cell in the span. |last| is the + * address of the last free cell in the span. This last cell holds a FreeSpan + * data structure for the next span unless this is the last span on the list + * of spans in the arena. For this last span |last| points to the last byte of + * the last thing in the arena and no linkage is stored there, so + * |last| == arenaStart + ArenaSize - 1. If the space at the arena end is + * fully used this last span is empty and |first| == |last + 1|. + * + * Thus |first| < |last| implies that we have either the last span with at least + * one element or that the span is not the last and contains at least 2 + * elements. In both cases to allocate a thing from this span we need simply + * to increment |first| by the allocation size. + * + * |first| == |last| implies that we have a one element span that records the + * next span. So to allocate from it we need to update the span list head + * with a copy of the span stored at |last| address so the following + * allocations will use that span. + * + * |first| > |last| implies that we have an empty last span and the arena is + * fully used. + * + * Also only for the last span (|last| & 1)! = 0 as all allocation sizes are + * multiples of Cell::CellSize. + */ +struct FreeSpan +{ + uintptr_t first; + uintptr_t last; + + public: + FreeSpan() {} + + FreeSpan(uintptr_t first, uintptr_t last) + : first(first), last(last) { + checkSpan(); + } + + /* + * To minimize the size of the arena header the first span is encoded + * there as offsets from the arena start. + */ + static size_t encodeOffsets(size_t firstOffset, size_t lastOffset) { + /* Check that we can pack the offsets into uint16. */ + JS_STATIC_ASSERT(ArenaShift < 16); + JS_ASSERT(firstOffset <= ArenaSize); + JS_ASSERT(lastOffset < ArenaSize); + JS_ASSERT(firstOffset <= ((lastOffset + 1) & ~size_t(1))); + return firstOffset | (lastOffset << 16); + } + + /* + * Encoded offsets for a full arena when its first span is the last one + * and empty. + */ + static const size_t FullArenaOffsets = ArenaSize | ((ArenaSize - 1) << 16); + + static FreeSpan decodeOffsets(uintptr_t arenaAddr, size_t offsets) { + JS_ASSERT(!(arenaAddr & ArenaMask)); + + size_t firstOffset = offsets & 0xFFFF; + size_t lastOffset = offsets >> 16; + JS_ASSERT(firstOffset <= ArenaSize); + JS_ASSERT(lastOffset < ArenaSize); + + /* + * We must not use | when calculating first as firstOffset is + * ArenaMask + 1 for the empty span. + */ + return FreeSpan(arenaAddr + firstOffset, arenaAddr | lastOffset); + } + + void initAsEmpty(uintptr_t arenaAddr = 0) { + JS_ASSERT(!(arenaAddr & ArenaMask)); + first = arenaAddr + ArenaSize; + last = arenaAddr | (ArenaSize - 1); + JS_ASSERT(isEmpty()); + } + + bool isEmpty() const { + checkSpan(); + return first > last; + } + + bool hasNext() const { + checkSpan(); + return !(last & uintptr_t(1)); + } + + const FreeSpan *nextSpan() const { + JS_ASSERT(hasNext()); + return reinterpret_cast(last); + } + + FreeSpan *nextSpanUnchecked(size_t thingSize) const { +#ifdef DEBUG + uintptr_t lastOffset = last & ArenaMask; + JS_ASSERT(!(lastOffset & 1)); + JS_ASSERT((ArenaSize - lastOffset) % thingSize == 0); +#endif + return reinterpret_cast(last); + } + + uintptr_t arenaAddressUnchecked() const { + return last & ~ArenaMask; + } + + uintptr_t arenaAddress() const { + checkSpan(); + return arenaAddressUnchecked(); + } + + ArenaHeader *arenaHeader() const { + return reinterpret_cast(arenaAddress()); + } + + bool isSameNonEmptySpan(const FreeSpan *another) const { + JS_ASSERT(!isEmpty()); + JS_ASSERT(!another->isEmpty()); + return first == another->first && last == another->last; + } + + bool isWithinArena(uintptr_t arenaAddr) const { + JS_ASSERT(!(arenaAddr & ArenaMask)); + + /* Return true for the last empty span as well. */ + return arenaAddress() == arenaAddr; + } + + size_t encodeAsOffsets() const { + /* + * We must use first - arenaAddress(), not first & ArenaMask as + * first == ArenaMask + 1 for an empty span. + */ + uintptr_t arenaAddr = arenaAddress(); + return encodeOffsets(first - arenaAddr, last & ArenaMask); + } + + /* See comments before FreeSpan for details. */ + MOZ_ALWAYS_INLINE void *allocate(size_t thingSize) { + JS_ASSERT(thingSize % Cell::CellSize == 0); + checkSpan(); + uintptr_t thing = first; + if (thing < last) { + /* Bump-allocate from the current span. */ + first = thing + thingSize; + } else if (JS_LIKELY(thing == last)) { + /* + * Move to the next span. We use JS_LIKELY as without PGO + * compilers mis-predict == here as unlikely to succeed. + */ + *this = *reinterpret_cast(thing); + } else { + return NULL; + } + checkSpan(); + return reinterpret_cast(thing); + } + + /* A version of allocate when we know that the span is not empty. */ + MOZ_ALWAYS_INLINE void *infallibleAllocate(size_t thingSize) { + JS_ASSERT(thingSize % Cell::CellSize == 0); + checkSpan(); + uintptr_t thing = first; + if (thing < last) { + first = thing + thingSize; + } else { + JS_ASSERT(thing == last); + *this = *reinterpret_cast(thing); + } + checkSpan(); + return reinterpret_cast(thing); + } + + /* + * Allocate from a newly allocated arena. We do not move the free list + * from the arena. Rather we set the arena up as fully used during the + * initialization so to allocate we simply return the first thing in the + * arena and set the free list to point to the second. + */ + MOZ_ALWAYS_INLINE void *allocateFromNewArena(uintptr_t arenaAddr, size_t firstThingOffset, + size_t thingSize) { + JS_ASSERT(!(arenaAddr & ArenaMask)); + uintptr_t thing = arenaAddr | firstThingOffset; + first = thing + thingSize; + last = arenaAddr | ArenaMask; + checkSpan(); + return reinterpret_cast(thing); + } + + void checkSpan() const { +#ifdef DEBUG + /* We do not allow spans at the end of the address space. */ + JS_ASSERT(last != uintptr_t(-1)); + JS_ASSERT(first); + JS_ASSERT(last); + JS_ASSERT(first - 1 <= last); + uintptr_t arenaAddr = arenaAddressUnchecked(); + if (last & 1) { + /* The span is the last. */ + JS_ASSERT((last & ArenaMask) == ArenaMask); + + if (first - 1 == last) { + /* The span is last and empty. The above start != 0 check + * implies that we are not at the end of the address space. + */ + return; + } + size_t spanLength = last - first + 1; + JS_ASSERT(spanLength % Cell::CellSize == 0); + + /* Start and end must belong to the same arena. */ + JS_ASSERT((first & ~ArenaMask) == arenaAddr); + return; + } + + /* The span is not the last and we have more spans to follow. */ + JS_ASSERT(first <= last); + size_t spanLengthWithoutOneThing = last - first; + JS_ASSERT(spanLengthWithoutOneThing % Cell::CellSize == 0); + + JS_ASSERT((first & ~ArenaMask) == arenaAddr); + + /* + * If there is not enough space before the arena end to allocate one + * more thing, then the span must be marked as the last one to avoid + * storing useless empty span reference. + */ + size_t beforeTail = ArenaSize - (last & ArenaMask); + JS_ASSERT(beforeTail >= sizeof(FreeSpan) + Cell::CellSize); + + FreeSpan *next = reinterpret_cast(last); + + /* + * The GC things on the list of free spans come from one arena + * and the spans are linked in ascending address order with + * at least one non-free thing between spans. + */ + JS_ASSERT(last < next->first); + JS_ASSERT(arenaAddr == next->arenaAddressUnchecked()); + + if (next->first > next->last) { + /* + * The next span is the empty span that terminates the list for + * arenas that do not have any free things at the end. + */ + JS_ASSERT(next->first - 1 == next->last); + JS_ASSERT(arenaAddr + ArenaSize == next->first); + } +#endif + } + +}; + +/* Every arena has a header. */ +struct ArenaHeader +{ + friend struct FreeLists; + + JSCompartment *compartment; + + /* + * ArenaHeader::next has two purposes: when unallocated, it points to the + * next available Arena's header. When allocated, it points to the next + * arena of the same size class and compartment. + */ + ArenaHeader *next; + + private: + /* + * The first span of free things in the arena. We encode it as the start + * and end offsets within the arena, not as FreeSpan structure, to + * minimize the header size. + */ + size_t firstFreeSpanOffsets; + + /* + * One of AllocKind constants or FINALIZE_LIMIT when the arena does not + * contain any GC things and is on the list of empty arenas in the GC + * chunk. The latter allows to quickly check if the arena is allocated + * during the conservative GC scanning without searching the arena in the + * list. + */ + size_t allocKind : 8; + + /* + * When recursive marking uses too much stack the marking is delayed and + * the corresponding arenas are put into a stack using the following field + * as a linkage. To distinguish the bottom of the stack from the arenas + * not present in the stack we use an extra flag to tag arenas on the + * stack. + * + * Delayed marking is also used for arenas that we allocate into during an + * incremental GC. In this case, we intend to mark all the objects in the + * arena, and it's faster to do this marking in bulk. + * + * To minimize the ArenaHeader size we record the next delayed marking + * linkage as arenaAddress() >> ArenaShift and pack it with the allocKind + * field and hasDelayedMarking flag. We use 8 bits for the allocKind, not + * ArenaShift - 1, so the compiler can use byte-level memory instructions + * to access it. + */ + public: + size_t hasDelayedMarking : 1; + size_t allocatedDuringIncremental : 1; + size_t markOverflow : 1; + size_t nextDelayedMarking : JS_BITS_PER_WORD - 8 - 1 - 1 - 1; + + static void staticAsserts() { + /* We must be able to fit the allockind into uint8_t. */ + JS_STATIC_ASSERT(FINALIZE_LIMIT <= 255); + + /* + * nextDelayedMarkingpacking assumes that ArenaShift has enough bits + * to cover allocKind and hasDelayedMarking. + */ + JS_STATIC_ASSERT(ArenaShift >= 8 + 1 + 1 + 1); + } + + inline uintptr_t address() const; + inline Chunk *chunk() const; + + bool allocated() const { + JS_ASSERT(allocKind <= size_t(FINALIZE_LIMIT)); + return allocKind < size_t(FINALIZE_LIMIT); + } + + void init(JSCompartment *comp, AllocKind kind) { + JS_ASSERT(!allocated()); + JS_ASSERT(!markOverflow); + JS_ASSERT(!allocatedDuringIncremental); + JS_ASSERT(!hasDelayedMarking); + compartment = comp; + + JS_STATIC_ASSERT(FINALIZE_LIMIT <= 255); + allocKind = size_t(kind); + + /* See comments in FreeSpan::allocateFromNewArena. */ + firstFreeSpanOffsets = FreeSpan::FullArenaOffsets; + } + + void setAsNotAllocated() { + allocKind = size_t(FINALIZE_LIMIT); + markOverflow = 0; + allocatedDuringIncremental = 0; + hasDelayedMarking = 0; + nextDelayedMarking = 0; + } + + inline uintptr_t arenaAddress() const; + inline Arena *getArena(); + + AllocKind getAllocKind() const { + JS_ASSERT(allocated()); + return AllocKind(allocKind); + } + + inline size_t getThingSize() const; + + bool hasFreeThings() const { + return firstFreeSpanOffsets != FreeSpan::FullArenaOffsets; + } + + inline bool isEmpty() const; + + void setAsFullyUsed() { + firstFreeSpanOffsets = FreeSpan::FullArenaOffsets; + } + + inline FreeSpan getFirstFreeSpan() const; + inline void setFirstFreeSpan(const FreeSpan *span); + +#ifdef DEBUG + void checkSynchronizedWithFreeList() const; +#endif + + inline ArenaHeader *getNextDelayedMarking() const; + inline void setNextDelayedMarking(ArenaHeader *aheader); +}; + +struct Arena +{ + /* + * Layout of an arena: + * An arena is 4K in size and 4K-aligned. It starts with the ArenaHeader + * descriptor followed by some pad bytes. The remainder of the arena is + * filled with the array of T things. The pad bytes ensure that the thing + * array ends exactly at the end of the arena. + * + * +-------------+-----+----+----+-----+----+ + * | ArenaHeader | pad | T0 | T1 | ... | Tn | + * +-------------+-----+----+----+-----+----+ + * + * <----------------------------------------> = ArenaSize bytes + * <-------------------> = first thing offset + */ + ArenaHeader aheader; + uint8_t data[ArenaSize - sizeof(ArenaHeader)]; + + private: + static JS_FRIEND_DATA(const uint32_t) ThingSizes[]; + static JS_FRIEND_DATA(const uint32_t) FirstThingOffsets[]; + + public: + static void staticAsserts(); + + static size_t thingSize(AllocKind kind) { + return ThingSizes[kind]; + } + + static size_t firstThingOffset(AllocKind kind) { + return FirstThingOffsets[kind]; + } + + static size_t thingsPerArena(size_t thingSize) { + JS_ASSERT(thingSize % Cell::CellSize == 0); + + /* We should be able to fit FreeSpan in any GC thing. */ + JS_ASSERT(thingSize >= sizeof(FreeSpan)); + + return (ArenaSize - sizeof(ArenaHeader)) / thingSize; + } + + static size_t thingsSpan(size_t thingSize) { + return thingsPerArena(thingSize) * thingSize; + } + + static bool isAligned(uintptr_t thing, size_t thingSize) { + /* Things ends at the arena end. */ + uintptr_t tailOffset = (ArenaSize - thing) & ArenaMask; + return tailOffset % thingSize == 0; + } + + uintptr_t address() const { + return aheader.address(); + } + + uintptr_t thingsStart(AllocKind thingKind) { + return address() | firstThingOffset(thingKind); + } + + uintptr_t thingsEnd() { + return address() + ArenaSize; + } + + template + bool finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize); +}; + +inline size_t +ArenaHeader::getThingSize() const +{ + JS_ASSERT(allocated()); + return Arena::thingSize(getAllocKind()); +} + +/* The chunk header (located at the end of the chunk to preserve arena alignment). */ +struct ChunkInfo +{ + Chunk *next; + Chunk **prevp; + + /* Free arenas are linked together with aheader.next. */ + ArenaHeader *freeArenasHead; + + /* + * Decommitted arenas are tracked by a bitmap in the chunk header. We use + * this offset to start our search iteration close to a decommitted arena + * that we can allocate. + */ + uint32_t lastDecommittedArenaOffset; + + /* Number of free arenas, either committed or decommitted. */ + uint32_t numArenasFree; + + /* Number of free, committed arenas. */ + uint32_t numArenasFreeCommitted; + + /* Number of GC cycles this chunk has survived. */ + uint32_t age; +}; + +/* + * Calculating ArenasPerChunk: + * + * In order to figure out how many Arenas will fit in a chunk, we need to know + * how much extra space is available after we allocate the header data. This + * is a problem because the header size depends on the number of arenas in the + * chunk. The two dependent fields are bitmap and decommittedArenas. + * + * For the mark bitmap, we know that each arena will use a fixed number of full + * bytes: ArenaBitmapBytes. The full size of the header data is this number + * multiplied by the eventual number of arenas we have in the header. We, + * conceptually, distribute this header data among the individual arenas and do + * not include it in the header. This way we do not have to worry about its + * variable size: it gets attached to the variable number we are computing. + * + * For the decommitted arena bitmap, we only have 1 bit per arena, so this + * technique will not work. Instead, we observe that we do not have enough + * header info to fill 8 full arenas: it is currently 4 on 64bit, less on + * 32bit. Thus, with current numbers, we need 64 bytes for decommittedArenas. + * This will not become 63 bytes unless we double the data required in the + * header. Therefore, we just compute the number of bytes required to track + * every possible arena and do not worry about slop bits, since there are too + * few to usefully allocate. + * + * To actually compute the number of arenas we can allocate in a chunk, we + * divide the amount of available space less the header info (not including + * the mark bitmap which is distributed into the arena size) by the size of + * the arena (with the mark bitmap bytes it uses). + */ +const size_t BytesPerArenaWithHeader = ArenaSize + ArenaBitmapBytes; +const size_t ChunkDecommitBitmapBytes = ChunkSize / ArenaSize / JS_BITS_PER_BYTE; +const size_t ChunkBytesAvailable = ChunkSize - sizeof(ChunkInfo) - ChunkDecommitBitmapBytes; +const size_t ArenasPerChunk = ChunkBytesAvailable / BytesPerArenaWithHeader; + +/* A chunk bitmap contains enough mark bits for all the cells in a chunk. */ +struct ChunkBitmap +{ + uintptr_t bitmap[ArenaBitmapWords * ArenasPerChunk]; + + MOZ_ALWAYS_INLINE void getMarkWordAndMask(const Cell *cell, uint32_t color, + uintptr_t **wordp, uintptr_t *maskp); + + MOZ_ALWAYS_INLINE bool isMarked(const Cell *cell, uint32_t color) { + uintptr_t *word, mask; + getMarkWordAndMask(cell, color, &word, &mask); + return *word & mask; + } + + MOZ_ALWAYS_INLINE bool markIfUnmarked(const Cell *cell, uint32_t color) { + uintptr_t *word, mask; + getMarkWordAndMask(cell, BLACK, &word, &mask); + if (*word & mask) + return false; + *word |= mask; + if (color != BLACK) { + /* + * We use getMarkWordAndMask to recalculate both mask and word as + * doing just mask << color may overflow the mask. + */ + getMarkWordAndMask(cell, color, &word, &mask); + if (*word & mask) + return false; + *word |= mask; + } + return true; + } + + MOZ_ALWAYS_INLINE void unmark(const Cell *cell, uint32_t color) { + uintptr_t *word, mask; + getMarkWordAndMask(cell, color, &word, &mask); + *word &= ~mask; + } + + void clear() { + PodArrayZero(bitmap); + } + +#ifdef DEBUG + bool noBitsSet(ArenaHeader *aheader) { + /* + * We assume that the part of the bitmap corresponding to the arena + * has the exact number of words so we do not need to deal with a word + * that covers bits from two arenas. + */ + JS_STATIC_ASSERT(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD); + + uintptr_t *word, unused; + getMarkWordAndMask(reinterpret_cast(aheader->address()), BLACK, &word, &unused); + for (size_t i = 0; i != ArenaBitmapWords; i++) { + if (word[i]) + return false; + } + return true; + } +#endif +}; + +JS_STATIC_ASSERT(ArenaBitmapBytes * ArenasPerChunk == sizeof(ChunkBitmap)); + +typedef BitArray PerArenaBitmap; + +const size_t ChunkPadSize = ChunkSize + - (sizeof(Arena) * ArenasPerChunk) + - sizeof(ChunkBitmap) + - sizeof(PerArenaBitmap) + - sizeof(ChunkInfo); +JS_STATIC_ASSERT(ChunkPadSize < BytesPerArenaWithHeader); + +/* + * Chunks contain arenas and associated data structures (mark bitmap, delayed + * marking state). + */ +struct Chunk +{ + Arena arenas[ArenasPerChunk]; + + /* Pad to full size to ensure cache alignment of ChunkInfo. */ + uint8_t padding[ChunkPadSize]; + + ChunkBitmap bitmap; + PerArenaBitmap decommittedArenas; + ChunkInfo info; + + static Chunk *fromAddress(uintptr_t addr) { + addr &= ~ChunkMask; + return reinterpret_cast(addr); + } + + static bool withinArenasRange(uintptr_t addr) { + uintptr_t offset = addr & ChunkMask; + return offset < ArenasPerChunk * ArenaSize; + } + + static size_t arenaIndex(uintptr_t addr) { + JS_ASSERT(withinArenasRange(addr)); + return (addr & ChunkMask) >> ArenaShift; + } + + uintptr_t address() const { + uintptr_t addr = reinterpret_cast(this); + JS_ASSERT(!(addr & ChunkMask)); + return addr; + } + + bool unused() const { + return info.numArenasFree == ArenasPerChunk; + } + + bool hasAvailableArenas() const { + return info.numArenasFree != 0; + } + + inline void addToAvailableList(JSCompartment *compartment); + inline void insertToAvailableList(Chunk **insertPoint); + inline void removeFromAvailableList(); + + ArenaHeader *allocateArena(JSCompartment *comp, AllocKind kind); + + void releaseArena(ArenaHeader *aheader); + + static Chunk *allocate(JSRuntime *rt); + + /* Must be called with the GC lock taken. */ + static inline void release(JSRuntime *rt, Chunk *chunk); + static inline void releaseList(JSRuntime *rt, Chunk *chunkListHead); + + /* Must be called with the GC lock taken. */ + inline void prepareToBeFreed(JSRuntime *rt); + + /* + * Assuming that the info.prevp points to the next field of the previous + * chunk in a doubly-linked list, get that chunk. + */ + Chunk *getPrevious() { + JS_ASSERT(info.prevp); + return fromPointerToNext(info.prevp); + } + + /* Get the chunk from a pointer to its info.next field. */ + static Chunk *fromPointerToNext(Chunk **nextFieldPtr) { + uintptr_t addr = reinterpret_cast(nextFieldPtr); + JS_ASSERT((addr & ChunkMask) == offsetof(Chunk, info.next)); + return reinterpret_cast(addr - offsetof(Chunk, info.next)); + } + + private: + inline void init(); + + /* Search for a decommitted arena to allocate. */ + unsigned findDecommittedArenaOffset(); + ArenaHeader* fetchNextDecommittedArena(); + + public: + /* Unlink and return the freeArenasHead. */ + inline ArenaHeader* fetchNextFreeArena(JSRuntime *rt); + + inline void addArenaToFreeList(JSRuntime *rt, ArenaHeader *aheader); +}; + +JS_STATIC_ASSERT(sizeof(Chunk) == ChunkSize); + +inline uintptr_t +Cell::address() const +{ + uintptr_t addr = uintptr_t(this); + JS_ASSERT(addr % Cell::CellSize == 0); + JS_ASSERT(Chunk::withinArenasRange(addr)); + return addr; +} + +inline uintptr_t +ArenaHeader::address() const +{ + uintptr_t addr = reinterpret_cast(this); + JS_ASSERT(!(addr & ArenaMask)); + JS_ASSERT(Chunk::withinArenasRange(addr)); + return addr; +} + +inline Chunk * +ArenaHeader::chunk() const +{ + return Chunk::fromAddress(address()); +} + +inline uintptr_t +ArenaHeader::arenaAddress() const +{ + return address(); +} + +inline Arena * +ArenaHeader::getArena() +{ + return reinterpret_cast(arenaAddress()); +} + +inline bool +ArenaHeader::isEmpty() const +{ + /* Arena is empty if its first span covers the whole arena. */ + JS_ASSERT(allocated()); + size_t firstThingOffset = Arena::firstThingOffset(getAllocKind()); + return firstFreeSpanOffsets == FreeSpan::encodeOffsets(firstThingOffset, ArenaMask); +} + +FreeSpan +ArenaHeader::getFirstFreeSpan() const +{ +#ifdef DEBUG + checkSynchronizedWithFreeList(); +#endif + return FreeSpan::decodeOffsets(arenaAddress(), firstFreeSpanOffsets); +} + +void +ArenaHeader::setFirstFreeSpan(const FreeSpan *span) +{ + JS_ASSERT(span->isWithinArena(arenaAddress())); + firstFreeSpanOffsets = span->encodeAsOffsets(); +} + +inline ArenaHeader * +ArenaHeader::getNextDelayedMarking() const +{ + return &reinterpret_cast(nextDelayedMarking << ArenaShift)->aheader; +} + +inline void +ArenaHeader::setNextDelayedMarking(ArenaHeader *aheader) +{ + JS_ASSERT(!(uintptr_t(aheader) & ArenaMask)); + hasDelayedMarking = 1; + nextDelayedMarking = aheader->arenaAddress() >> ArenaShift; +} + +JS_ALWAYS_INLINE void +ChunkBitmap::getMarkWordAndMask(const Cell *cell, uint32_t color, + uintptr_t **wordp, uintptr_t *maskp) +{ + size_t bit = (cell->address() & ChunkMask) / Cell::CellSize + color; + JS_ASSERT(bit < ArenaBitmapBits * ArenasPerChunk); + *maskp = uintptr_t(1) << (bit % JS_BITS_PER_WORD); + *wordp = &bitmap[bit / JS_BITS_PER_WORD]; +} + +static void +AssertValidColor(const void *thing, uint32_t color) +{ +#ifdef DEBUG + ArenaHeader *aheader = reinterpret_cast(thing)->arenaHeader(); + JS_ASSERT_IF(color, color < aheader->getThingSize() / Cell::CellSize); +#endif +} + +inline ArenaHeader * +Cell::arenaHeader() const +{ + uintptr_t addr = address(); + addr &= ~ArenaMask; + return reinterpret_cast(addr); +} + +Chunk * +Cell::chunk() const +{ + uintptr_t addr = uintptr_t(this); + JS_ASSERT(addr % Cell::CellSize == 0); + addr &= ~(ChunkSize - 1); + return reinterpret_cast(addr); +} + +AllocKind +Cell::getAllocKind() const +{ + return arenaHeader()->getAllocKind(); +} + +bool +Cell::isMarked(uint32_t color /* = BLACK */) const +{ + AssertValidColor(this, color); + return chunk()->bitmap.isMarked(this, color); +} + +bool +Cell::markIfUnmarked(uint32_t color /* = BLACK */) const +{ + AssertValidColor(this, color); + return chunk()->bitmap.markIfUnmarked(this, color); +} + +void +Cell::unmark(uint32_t color) const +{ + JS_ASSERT(color != BLACK); + AssertValidColor(this, color); + chunk()->bitmap.unmark(this, color); +} + +JSCompartment * +Cell::compartment() const +{ + return arenaHeader()->compartment; +} + +#ifdef DEBUG +bool +Cell::isAligned() const +{ + return Arena::isAligned(address(), arenaHeader()->getThingSize()); +} +#endif + +} /* namespace gc */ + +} /* namespace js */ + +#endif /* gc_heap_h___ */ diff --git a/js/src/jscell.h b/js/src/jscell.h deleted file mode 100644 index b7f931671d8..00000000000 --- a/js/src/jscell.h +++ /dev/null @@ -1,119 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- - * - * ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is SpiderMonkey code. - * - * The Initial Developer of the Original Code is - * Mozilla Corporation. - * Portions created by the Initial Developer are Copyright (C) 2010 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Gregor Wagner - * - * Alternatively, the contents of this file may be used under the terms of - * either of the GNU General Public License Version 2 or later (the "GPL"), - * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef jscell_h___ -#define jscell_h___ - -#include "jspubtd.h" - -struct JSCompartment; - -namespace js { -namespace gc { - -struct ArenaHeader; -struct Chunk; - -/* The GC allocation kinds. */ -enum AllocKind { - FINALIZE_OBJECT0, - FINALIZE_OBJECT0_BACKGROUND, - FINALIZE_OBJECT2, - FINALIZE_OBJECT2_BACKGROUND, - FINALIZE_OBJECT4, - FINALIZE_OBJECT4_BACKGROUND, - FINALIZE_OBJECT8, - FINALIZE_OBJECT8_BACKGROUND, - FINALIZE_OBJECT12, - FINALIZE_OBJECT12_BACKGROUND, - FINALIZE_OBJECT16, - FINALIZE_OBJECT16_BACKGROUND, - FINALIZE_OBJECT_LAST = FINALIZE_OBJECT16_BACKGROUND, - FINALIZE_SCRIPT, - FINALIZE_SHAPE, - FINALIZE_BASE_SHAPE, - FINALIZE_TYPE_OBJECT, -#if JS_HAS_XML_SUPPORT - FINALIZE_XML, -#endif - FINALIZE_SHORT_STRING, - FINALIZE_STRING, - FINALIZE_EXTERNAL_STRING, - FINALIZE_LAST = FINALIZE_EXTERNAL_STRING -}; - -static const unsigned FINALIZE_LIMIT = FINALIZE_LAST + 1; -static const unsigned FINALIZE_OBJECT_LIMIT = FINALIZE_OBJECT_LAST + 1; - -/* - * Live objects are marked black. How many other additional colors are available - * depends on the size of the GCThing. Objects marked gray are eligible for - * cycle collection. - */ -static const uint32_t BLACK = 0; -static const uint32_t GRAY = 1; - -/* - * A GC cell is the base class for all GC things. - */ -struct Cell { - static const size_t CellShift = 3; - static const size_t CellSize = size_t(1) << CellShift; - static const size_t CellMask = CellSize - 1; - - inline uintptr_t address() const; - inline ArenaHeader *arenaHeader() const; - inline Chunk *chunk() const; - inline AllocKind getAllocKind() const; - - JS_ALWAYS_INLINE bool isMarked(uint32_t color = BLACK) const; - JS_ALWAYS_INLINE bool markIfUnmarked(uint32_t color = BLACK) const; - JS_ALWAYS_INLINE void unmark(uint32_t color) const; - - inline JSCompartment *compartment() const; - -#ifdef DEBUG - inline bool isAligned() const; -#endif -}; - -} /* namespace gc */ -} /* namespace js */ - -#endif /* jscell_h___ */ diff --git a/js/src/jsgc.h b/js/src/jsgc.h index da52099737b..56b404660e3 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -54,9 +54,9 @@ #include "jslock.h" #include "jsutil.h" #include "jsversion.h" -#include "jscell.h" #include "ds/BitArray.h" +#include "gc/Heap.h" #include "gc/Statistics.h" #include "js/HashTable.h" #include "js/Vector.h" @@ -87,737 +87,6 @@ enum State { INVALID }; -struct Arena; - -/* - * This must be an upper bound, but we do not need the least upper bound, so - * we just exclude non-background objects. - */ -const size_t MAX_BACKGROUND_FINALIZE_KINDS = FINALIZE_LIMIT - FINALIZE_OBJECT_LIMIT / 2; - -/* - * Page size is 4096 by default, except for SPARC, where it is 8192. - * Note: Do not use JS_CPU_SPARC here, this header is used outside JS. - * Bug 692267: Move page size definition to gc/Memory.h and include it - * directly once jsgc.h is no longer an installed header. - */ -#if defined(SOLARIS) && (defined(__sparc) || defined(__sparcv9)) -const size_t PageShift = 13; -#else -const size_t PageShift = 12; -#endif -const size_t PageSize = size_t(1) << PageShift; - -const size_t ChunkShift = 20; -const size_t ChunkSize = size_t(1) << ChunkShift; -const size_t ChunkMask = ChunkSize - 1; - -const size_t ArenaShift = PageShift; -const size_t ArenaSize = PageSize; -const size_t ArenaMask = ArenaSize - 1; - -/* - * This is the maximum number of arenas we allow in the FreeCommitted state - * before we trigger a GC_SHRINK to release free arenas to the OS. - */ -const static uint32_t FreeCommittedArenasThreshold = (32 << 20) / ArenaSize; - -/* - * The mark bitmap has one bit per each GC cell. For multi-cell GC things this - * wastes space but allows to avoid expensive devisions by thing's size when - * accessing the bitmap. In addition this allows to use some bits for colored - * marking during the cycle GC. - */ -const size_t ArenaCellCount = size_t(1) << (ArenaShift - Cell::CellShift); -const size_t ArenaBitmapBits = ArenaCellCount; -const size_t ArenaBitmapBytes = ArenaBitmapBits / 8; -const size_t ArenaBitmapWords = ArenaBitmapBits / JS_BITS_PER_WORD; - -/* - * A FreeSpan represents a contiguous sequence of free cells in an Arena. - * |first| is the address of the first free cell in the span. |last| is the - * address of the last free cell in the span. This last cell holds a FreeSpan - * data structure for the next span unless this is the last span on the list - * of spans in the arena. For this last span |last| points to the last byte of - * the last thing in the arena and no linkage is stored there, so - * |last| == arenaStart + ArenaSize - 1. If the space at the arena end is - * fully used this last span is empty and |first| == |last + 1|. - * - * Thus |first| < |last| implies that we have either the last span with at least - * one element or that the span is not the last and contains at least 2 - * elements. In both cases to allocate a thing from this span we need simply - * to increment |first| by the allocation size. - * - * |first| == |last| implies that we have a one element span that records the - * next span. So to allocate from it we need to update the span list head - * with a copy of the span stored at |last| address so the following - * allocations will use that span. - * - * |first| > |last| implies that we have an empty last span and the arena is - * fully used. - * - * Also only for the last span (|last| & 1)! = 0 as all allocation sizes are - * multiples of Cell::CellSize. - */ -struct FreeSpan { - uintptr_t first; - uintptr_t last; - - public: - FreeSpan() {} - - FreeSpan(uintptr_t first, uintptr_t last) - : first(first), last(last) { - checkSpan(); - } - - /* - * To minimize the size of the arena header the first span is encoded - * there as offsets from the arena start. - */ - static size_t encodeOffsets(size_t firstOffset, size_t lastOffset) { - /* Check that we can pack the offsets into uint16. */ - JS_STATIC_ASSERT(ArenaShift < 16); - JS_ASSERT(firstOffset <= ArenaSize); - JS_ASSERT(lastOffset < ArenaSize); - JS_ASSERT(firstOffset <= ((lastOffset + 1) & ~size_t(1))); - return firstOffset | (lastOffset << 16); - } - - /* - * Encoded offsets for a full arena when its first span is the last one - * and empty. - */ - static const size_t FullArenaOffsets = ArenaSize | ((ArenaSize - 1) << 16); - - static FreeSpan decodeOffsets(uintptr_t arenaAddr, size_t offsets) { - JS_ASSERT(!(arenaAddr & ArenaMask)); - - size_t firstOffset = offsets & 0xFFFF; - size_t lastOffset = offsets >> 16; - JS_ASSERT(firstOffset <= ArenaSize); - JS_ASSERT(lastOffset < ArenaSize); - - /* - * We must not use | when calculating first as firstOffset is - * ArenaMask + 1 for the empty span. - */ - return FreeSpan(arenaAddr + firstOffset, arenaAddr | lastOffset); - } - - void initAsEmpty(uintptr_t arenaAddr = 0) { - JS_ASSERT(!(arenaAddr & ArenaMask)); - first = arenaAddr + ArenaSize; - last = arenaAddr | (ArenaSize - 1); - JS_ASSERT(isEmpty()); - } - - bool isEmpty() const { - checkSpan(); - return first > last; - } - - bool hasNext() const { - checkSpan(); - return !(last & uintptr_t(1)); - } - - const FreeSpan *nextSpan() const { - JS_ASSERT(hasNext()); - return reinterpret_cast(last); - } - - FreeSpan *nextSpanUnchecked(size_t thingSize) const { -#ifdef DEBUG - uintptr_t lastOffset = last & ArenaMask; - JS_ASSERT(!(lastOffset & 1)); - JS_ASSERT((ArenaSize - lastOffset) % thingSize == 0); -#endif - return reinterpret_cast(last); - } - - uintptr_t arenaAddressUnchecked() const { - return last & ~ArenaMask; - } - - uintptr_t arenaAddress() const { - checkSpan(); - return arenaAddressUnchecked(); - } - - ArenaHeader *arenaHeader() const { - return reinterpret_cast(arenaAddress()); - } - - bool isSameNonEmptySpan(const FreeSpan *another) const { - JS_ASSERT(!isEmpty()); - JS_ASSERT(!another->isEmpty()); - return first == another->first && last == another->last; - } - - bool isWithinArena(uintptr_t arenaAddr) const { - JS_ASSERT(!(arenaAddr & ArenaMask)); - - /* Return true for the last empty span as well. */ - return arenaAddress() == arenaAddr; - } - - size_t encodeAsOffsets() const { - /* - * We must use first - arenaAddress(), not first & ArenaMask as - * first == ArenaMask + 1 for an empty span. - */ - uintptr_t arenaAddr = arenaAddress(); - return encodeOffsets(first - arenaAddr, last & ArenaMask); - } - - /* See comments before FreeSpan for details. */ - JS_ALWAYS_INLINE void *allocate(size_t thingSize) { - JS_ASSERT(thingSize % Cell::CellSize == 0); - checkSpan(); - uintptr_t thing = first; - if (thing < last) { - /* Bump-allocate from the current span. */ - first = thing + thingSize; - } else if (JS_LIKELY(thing == last)) { - /* - * Move to the next span. We use JS_LIKELY as without PGO - * compilers mis-predict == here as unlikely to succeed. - */ - *this = *reinterpret_cast(thing); - } else { - return NULL; - } - checkSpan(); - return reinterpret_cast(thing); - } - - /* A version of allocate when we know that the span is not empty. */ - JS_ALWAYS_INLINE void *infallibleAllocate(size_t thingSize) { - JS_ASSERT(thingSize % Cell::CellSize == 0); - checkSpan(); - uintptr_t thing = first; - if (thing < last) { - first = thing + thingSize; - } else { - JS_ASSERT(thing == last); - *this = *reinterpret_cast(thing); - } - checkSpan(); - return reinterpret_cast(thing); - } - - /* - * Allocate from a newly allocated arena. We do not move the free list - * from the arena. Rather we set the arena up as fully used during the - * initialization so to allocate we simply return the first thing in the - * arena and set the free list to point to the second. - */ - JS_ALWAYS_INLINE void *allocateFromNewArena(uintptr_t arenaAddr, size_t firstThingOffset, - size_t thingSize) { - JS_ASSERT(!(arenaAddr & ArenaMask)); - uintptr_t thing = arenaAddr | firstThingOffset; - first = thing + thingSize; - last = arenaAddr | ArenaMask; - checkSpan(); - return reinterpret_cast(thing); - } - - void checkSpan() const { -#ifdef DEBUG - /* We do not allow spans at the end of the address space. */ - JS_ASSERT(last != uintptr_t(-1)); - JS_ASSERT(first); - JS_ASSERT(last); - JS_ASSERT(first - 1 <= last); - uintptr_t arenaAddr = arenaAddressUnchecked(); - if (last & 1) { - /* The span is the last. */ - JS_ASSERT((last & ArenaMask) == ArenaMask); - - if (first - 1 == last) { - /* The span is last and empty. The above start != 0 check - * implies that we are not at the end of the address space. - */ - return; - } - size_t spanLength = last - first + 1; - JS_ASSERT(spanLength % Cell::CellSize == 0); - - /* Start and end must belong to the same arena. */ - JS_ASSERT((first & ~ArenaMask) == arenaAddr); - return; - } - - /* The span is not the last and we have more spans to follow. */ - JS_ASSERT(first <= last); - size_t spanLengthWithoutOneThing = last - first; - JS_ASSERT(spanLengthWithoutOneThing % Cell::CellSize == 0); - - JS_ASSERT((first & ~ArenaMask) == arenaAddr); - - /* - * If there is not enough space before the arena end to allocate one - * more thing, then the span must be marked as the last one to avoid - * storing useless empty span reference. - */ - size_t beforeTail = ArenaSize - (last & ArenaMask); - JS_ASSERT(beforeTail >= sizeof(FreeSpan) + Cell::CellSize); - - FreeSpan *next = reinterpret_cast(last); - - /* - * The GC things on the list of free spans come from one arena - * and the spans are linked in ascending address order with - * at least one non-free thing between spans. - */ - JS_ASSERT(last < next->first); - JS_ASSERT(arenaAddr == next->arenaAddressUnchecked()); - - if (next->first > next->last) { - /* - * The next span is the empty span that terminates the list for - * arenas that do not have any free things at the end. - */ - JS_ASSERT(next->first - 1 == next->last); - JS_ASSERT(arenaAddr + ArenaSize == next->first); - } -#endif - } - -}; - -/* Every arena has a header. */ -struct ArenaHeader { - friend struct FreeLists; - - JSCompartment *compartment; - - /* - * ArenaHeader::next has two purposes: when unallocated, it points to the - * next available Arena's header. When allocated, it points to the next - * arena of the same size class and compartment. - */ - ArenaHeader *next; - - private: - /* - * The first span of free things in the arena. We encode it as the start - * and end offsets within the arena, not as FreeSpan structure, to - * minimize the header size. - */ - size_t firstFreeSpanOffsets; - - /* - * One of AllocKind constants or FINALIZE_LIMIT when the arena does not - * contain any GC things and is on the list of empty arenas in the GC - * chunk. The latter allows to quickly check if the arena is allocated - * during the conservative GC scanning without searching the arena in the - * list. - */ - size_t allocKind : 8; - - /* - * When recursive marking uses too much stack the marking is delayed and - * the corresponding arenas are put into a stack using the following field - * as a linkage. To distinguish the bottom of the stack from the arenas - * not present in the stack we use an extra flag to tag arenas on the - * stack. - * - * Delayed marking is also used for arenas that we allocate into during an - * incremental GC. In this case, we intend to mark all the objects in the - * arena, and it's faster to do this marking in bulk. - * - * To minimize the ArenaHeader size we record the next delayed marking - * linkage as arenaAddress() >> ArenaShift and pack it with the allocKind - * field and hasDelayedMarking flag. We use 8 bits for the allocKind, not - * ArenaShift - 1, so the compiler can use byte-level memory instructions - * to access it. - */ - public: - size_t hasDelayedMarking : 1; - size_t allocatedDuringIncremental : 1; - size_t markOverflow : 1; - size_t nextDelayedMarking : JS_BITS_PER_WORD - 8 - 1 - 1 - 1; - - static void staticAsserts() { - /* We must be able to fit the allockind into uint8_t. */ - JS_STATIC_ASSERT(FINALIZE_LIMIT <= 255); - - /* - * nextDelayedMarkingpacking assumes that ArenaShift has enough bits - * to cover allocKind and hasDelayedMarking. - */ - JS_STATIC_ASSERT(ArenaShift >= 8 + 1 + 1 + 1); - } - - inline uintptr_t address() const; - inline Chunk *chunk() const; - - bool allocated() const { - JS_ASSERT(allocKind <= size_t(FINALIZE_LIMIT)); - return allocKind < size_t(FINALIZE_LIMIT); - } - - void init(JSCompartment *comp, AllocKind kind) { - JS_ASSERT(!allocated()); - JS_ASSERT(!markOverflow); - JS_ASSERT(!allocatedDuringIncremental); - JS_ASSERT(!hasDelayedMarking); - compartment = comp; - - JS_STATIC_ASSERT(FINALIZE_LIMIT <= 255); - allocKind = size_t(kind); - - /* See comments in FreeSpan::allocateFromNewArena. */ - firstFreeSpanOffsets = FreeSpan::FullArenaOffsets; - } - - void setAsNotAllocated() { - allocKind = size_t(FINALIZE_LIMIT); - markOverflow = 0; - allocatedDuringIncremental = 0; - hasDelayedMarking = 0; - nextDelayedMarking = 0; - } - - uintptr_t arenaAddress() const { - return address(); - } - - Arena *getArena() { - return reinterpret_cast(arenaAddress()); - } - - AllocKind getAllocKind() const { - JS_ASSERT(allocated()); - return AllocKind(allocKind); - } - - inline size_t getThingSize() const; - - bool hasFreeThings() const { - return firstFreeSpanOffsets != FreeSpan::FullArenaOffsets; - } - - inline bool isEmpty() const; - - void setAsFullyUsed() { - firstFreeSpanOffsets = FreeSpan::FullArenaOffsets; - } - - FreeSpan getFirstFreeSpan() const { -#ifdef DEBUG - checkSynchronizedWithFreeList(); -#endif - return FreeSpan::decodeOffsets(arenaAddress(), firstFreeSpanOffsets); - } - - void setFirstFreeSpan(const FreeSpan *span) { - JS_ASSERT(span->isWithinArena(arenaAddress())); - firstFreeSpanOffsets = span->encodeAsOffsets(); - } - -#ifdef DEBUG - void checkSynchronizedWithFreeList() const; -#endif - - inline ArenaHeader *getNextDelayedMarking() const; - inline void setNextDelayedMarking(ArenaHeader *aheader); -}; - -struct Arena { - /* - * Layout of an arena: - * An arena is 4K in size and 4K-aligned. It starts with the ArenaHeader - * descriptor followed by some pad bytes. The remainder of the arena is - * filled with the array of T things. The pad bytes ensure that the thing - * array ends exactly at the end of the arena. - * - * +-------------+-----+----+----+-----+----+ - * | ArenaHeader | pad | T0 | T1 | ... | Tn | - * +-------------+-----+----+----+-----+----+ - * - * <----------------------------------------> = ArenaSize bytes - * <-------------------> = first thing offset - */ - ArenaHeader aheader; - uint8_t data[ArenaSize - sizeof(ArenaHeader)]; - - private: - static JS_FRIEND_DATA(const uint32_t) ThingSizes[]; - static JS_FRIEND_DATA(const uint32_t) FirstThingOffsets[]; - - public: - static void staticAsserts(); - - static size_t thingSize(AllocKind kind) { - return ThingSizes[kind]; - } - - static size_t firstThingOffset(AllocKind kind) { - return FirstThingOffsets[kind]; - } - - static size_t thingsPerArena(size_t thingSize) { - JS_ASSERT(thingSize % Cell::CellSize == 0); - - /* We should be able to fit FreeSpan in any GC thing. */ - JS_ASSERT(thingSize >= sizeof(FreeSpan)); - - return (ArenaSize - sizeof(ArenaHeader)) / thingSize; - } - - static size_t thingsSpan(size_t thingSize) { - return thingsPerArena(thingSize) * thingSize; - } - - static bool isAligned(uintptr_t thing, size_t thingSize) { - /* Things ends at the arena end. */ - uintptr_t tailOffset = (ArenaSize - thing) & ArenaMask; - return tailOffset % thingSize == 0; - } - - uintptr_t address() const { - return aheader.address(); - } - - uintptr_t thingsStart(AllocKind thingKind) { - return address() | firstThingOffset(thingKind); - } - - uintptr_t thingsEnd() { - return address() + ArenaSize; - } - - template - bool finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize); -}; - -/* The chunk header (located at the end of the chunk to preserve arena alignment). */ -struct ChunkInfo { - Chunk *next; - Chunk **prevp; - - /* Free arenas are linked together with aheader.next. */ - ArenaHeader *freeArenasHead; - - /* - * Decommitted arenas are tracked by a bitmap in the chunk header. We use - * this offset to start our search iteration close to a decommitted arena - * that we can allocate. - */ - uint32_t lastDecommittedArenaOffset; - - /* Number of free arenas, either committed or decommitted. */ - uint32_t numArenasFree; - - /* Number of free, committed arenas. */ - uint32_t numArenasFreeCommitted; - - /* Number of GC cycles this chunk has survived. */ - uint32_t age; -}; - -/* - * Calculating ArenasPerChunk: - * - * In order to figure out how many Arenas will fit in a chunk, we need to know - * how much extra space is available after we allocate the header data. This - * is a problem because the header size depends on the number of arenas in the - * chunk. The two dependent fields are bitmap and decommittedArenas. - * - * For the mark bitmap, we know that each arena will use a fixed number of full - * bytes: ArenaBitmapBytes. The full size of the header data is this number - * multiplied by the eventual number of arenas we have in the header. We, - * conceptually, distribute this header data among the individual arenas and do - * not include it in the header. This way we do not have to worry about its - * variable size: it gets attached to the variable number we are computing. - * - * For the decommitted arena bitmap, we only have 1 bit per arena, so this - * technique will not work. Instead, we observe that we do not have enough - * header info to fill 8 full arenas: it is currently 4 on 64bit, less on - * 32bit. Thus, with current numbers, we need 64 bytes for decommittedArenas. - * This will not become 63 bytes unless we double the data required in the - * header. Therefore, we just compute the number of bytes required to track - * every possible arena and do not worry about slop bits, since there are too - * few to usefully allocate. - * - * To actually compute the number of arenas we can allocate in a chunk, we - * divide the amount of available space less the header info (not including - * the mark bitmap which is distributed into the arena size) by the size of - * the arena (with the mark bitmap bytes it uses). - */ -const size_t BytesPerArenaWithHeader = ArenaSize + ArenaBitmapBytes; -const size_t ChunkDecommitBitmapBytes = ChunkSize / ArenaSize / JS_BITS_PER_BYTE; -const size_t ChunkBytesAvailable = ChunkSize - sizeof(ChunkInfo) - ChunkDecommitBitmapBytes; -const size_t ArenasPerChunk = ChunkBytesAvailable / BytesPerArenaWithHeader; - -/* A chunk bitmap contains enough mark bits for all the cells in a chunk. */ -struct ChunkBitmap { - uintptr_t bitmap[ArenaBitmapWords * ArenasPerChunk]; - - JS_ALWAYS_INLINE void getMarkWordAndMask(const Cell *cell, uint32_t color, - uintptr_t **wordp, uintptr_t *maskp); - - JS_ALWAYS_INLINE bool isMarked(const Cell *cell, uint32_t color) { - uintptr_t *word, mask; - getMarkWordAndMask(cell, color, &word, &mask); - return *word & mask; - } - - JS_ALWAYS_INLINE bool markIfUnmarked(const Cell *cell, uint32_t color) { - uintptr_t *word, mask; - getMarkWordAndMask(cell, BLACK, &word, &mask); - if (*word & mask) - return false; - *word |= mask; - if (color != BLACK) { - /* - * We use getMarkWordAndMask to recalculate both mask and word as - * doing just mask << color may overflow the mask. - */ - getMarkWordAndMask(cell, color, &word, &mask); - if (*word & mask) - return false; - *word |= mask; - } - return true; - } - - JS_ALWAYS_INLINE void unmark(const Cell *cell, uint32_t color) { - uintptr_t *word, mask; - getMarkWordAndMask(cell, color, &word, &mask); - *word &= ~mask; - } - - void clear() { - PodArrayZero(bitmap); - } - -#ifdef DEBUG - bool noBitsSet(ArenaHeader *aheader) { - /* - * We assume that the part of the bitmap corresponding to the arena - * has the exact number of words so we do not need to deal with a word - * that covers bits from two arenas. - */ - JS_STATIC_ASSERT(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD); - - uintptr_t *word, unused; - getMarkWordAndMask(reinterpret_cast(aheader->address()), BLACK, &word, &unused); - for (size_t i = 0; i != ArenaBitmapWords; i++) { - if (word[i]) - return false; - } - return true; - } -#endif -}; - -JS_STATIC_ASSERT(ArenaBitmapBytes * ArenasPerChunk == sizeof(ChunkBitmap)); - -typedef BitArray PerArenaBitmap; - -const size_t ChunkPadSize = ChunkSize - - (sizeof(Arena) * ArenasPerChunk) - - sizeof(ChunkBitmap) - - sizeof(PerArenaBitmap) - - sizeof(ChunkInfo); -JS_STATIC_ASSERT(ChunkPadSize < BytesPerArenaWithHeader); - -/* - * Chunks contain arenas and associated data structures (mark bitmap, delayed - * marking state). - */ -struct Chunk { - Arena arenas[ArenasPerChunk]; - - /* Pad to full size to ensure cache alignment of ChunkInfo. */ - uint8_t padding[ChunkPadSize]; - - ChunkBitmap bitmap; - PerArenaBitmap decommittedArenas; - ChunkInfo info; - - static Chunk *fromAddress(uintptr_t addr) { - addr &= ~ChunkMask; - return reinterpret_cast(addr); - } - - static bool withinArenasRange(uintptr_t addr) { - uintptr_t offset = addr & ChunkMask; - return offset < ArenasPerChunk * ArenaSize; - } - - static size_t arenaIndex(uintptr_t addr) { - JS_ASSERT(withinArenasRange(addr)); - return (addr & ChunkMask) >> ArenaShift; - } - - uintptr_t address() const { - uintptr_t addr = reinterpret_cast(this); - JS_ASSERT(!(addr & ChunkMask)); - return addr; - } - - bool unused() const { - return info.numArenasFree == ArenasPerChunk; - } - - bool hasAvailableArenas() const { - return info.numArenasFree != 0; - } - - inline void addToAvailableList(JSCompartment *compartment); - inline void insertToAvailableList(Chunk **insertPoint); - inline void removeFromAvailableList(); - - ArenaHeader *allocateArena(JSCompartment *comp, AllocKind kind); - - void releaseArena(ArenaHeader *aheader); - - static Chunk *allocate(JSRuntime *rt); - - /* Must be called with the GC lock taken. */ - static inline void release(JSRuntime *rt, Chunk *chunk); - static inline void releaseList(JSRuntime *rt, Chunk *chunkListHead); - - /* Must be called with the GC lock taken. */ - inline void prepareToBeFreed(JSRuntime *rt); - - /* - * Assuming that the info.prevp points to the next field of the previous - * chunk in a doubly-linked list, get that chunk. - */ - Chunk *getPrevious() { - JS_ASSERT(info.prevp); - return fromPointerToNext(info.prevp); - } - - /* Get the chunk from a pointer to its info.next field. */ - static Chunk *fromPointerToNext(Chunk **nextFieldPtr) { - uintptr_t addr = reinterpret_cast(nextFieldPtr); - JS_ASSERT((addr & ChunkMask) == offsetof(Chunk, info.next)); - return reinterpret_cast(addr - offsetof(Chunk, info.next)); - } - - private: - inline void init(); - - /* Search for a decommitted arena to allocate. */ - unsigned findDecommittedArenaOffset(); - ArenaHeader* fetchNextDecommittedArena(); - - public: - /* Unlink and return the freeArenasHead. */ - inline ArenaHeader* fetchNextFreeArena(JSRuntime *rt); - - inline void addArenaToFreeList(JSRuntime *rt, ArenaHeader *aheader); -}; - -JS_STATIC_ASSERT(sizeof(Chunk) == ChunkSize); - class ChunkPool { Chunk *emptyChunkListHead; size_t emptyCount; @@ -852,138 +121,6 @@ class ChunkPool { JS_FRIEND_API(int64_t) countCleanDecommittedArenas(JSRuntime *rt); }; -inline uintptr_t -Cell::address() const -{ - uintptr_t addr = uintptr_t(this); - JS_ASSERT(addr % Cell::CellSize == 0); - JS_ASSERT(Chunk::withinArenasRange(addr)); - return addr; -} - -inline ArenaHeader * -Cell::arenaHeader() const -{ - uintptr_t addr = address(); - addr &= ~ArenaMask; - return reinterpret_cast(addr); -} - -Chunk * -Cell::chunk() const -{ - uintptr_t addr = uintptr_t(this); - JS_ASSERT(addr % Cell::CellSize == 0); - addr &= ~(ChunkSize - 1); - return reinterpret_cast(addr); -} - -AllocKind -Cell::getAllocKind() const -{ - return arenaHeader()->getAllocKind(); -} - -#ifdef DEBUG -inline bool -Cell::isAligned() const -{ - return Arena::isAligned(address(), arenaHeader()->getThingSize()); -} -#endif - -inline uintptr_t -ArenaHeader::address() const -{ - uintptr_t addr = reinterpret_cast(this); - JS_ASSERT(!(addr & ArenaMask)); - JS_ASSERT(Chunk::withinArenasRange(addr)); - return addr; -} - -inline Chunk * -ArenaHeader::chunk() const -{ - return Chunk::fromAddress(address()); -} - -inline bool -ArenaHeader::isEmpty() const -{ - /* Arena is empty if its first span covers the whole arena. */ - JS_ASSERT(allocated()); - size_t firstThingOffset = Arena::firstThingOffset(getAllocKind()); - return firstFreeSpanOffsets == FreeSpan::encodeOffsets(firstThingOffset, ArenaMask); -} - -inline size_t -ArenaHeader::getThingSize() const -{ - JS_ASSERT(allocated()); - return Arena::thingSize(getAllocKind()); -} - -inline ArenaHeader * -ArenaHeader::getNextDelayedMarking() const -{ - return &reinterpret_cast(nextDelayedMarking << ArenaShift)->aheader; -} - -inline void -ArenaHeader::setNextDelayedMarking(ArenaHeader *aheader) -{ - JS_ASSERT(!(uintptr_t(aheader) & ArenaMask)); - hasDelayedMarking = 1; - nextDelayedMarking = aheader->arenaAddress() >> ArenaShift; -} - -JS_ALWAYS_INLINE void -ChunkBitmap::getMarkWordAndMask(const Cell *cell, uint32_t color, - uintptr_t **wordp, uintptr_t *maskp) -{ - size_t bit = (cell->address() & ChunkMask) / Cell::CellSize + color; - JS_ASSERT(bit < ArenaBitmapBits * ArenasPerChunk); - *maskp = uintptr_t(1) << (bit % JS_BITS_PER_WORD); - *wordp = &bitmap[bit / JS_BITS_PER_WORD]; -} - -static void -AssertValidColor(const void *thing, uint32_t color) -{ -#ifdef DEBUG - ArenaHeader *aheader = reinterpret_cast(thing)->arenaHeader(); - JS_ASSERT_IF(color, color < aheader->getThingSize() / Cell::CellSize); -#endif -} - -inline bool -Cell::isMarked(uint32_t color) const -{ - AssertValidColor(this, color); - return chunk()->bitmap.isMarked(this, color); -} - -bool -Cell::markIfUnmarked(uint32_t color) const -{ - AssertValidColor(this, color); - return chunk()->bitmap.markIfUnmarked(this, color); -} - -void -Cell::unmark(uint32_t color) const -{ - JS_ASSERT(color != BLACK); - AssertValidColor(this, color); - chunk()->bitmap.unmark(this, color); -} - -JSCompartment * -Cell::compartment() const -{ - return arenaHeader()->compartment; -} - static inline JSGCTraceKind MapAllocToTraceKind(AllocKind thingKind) { diff --git a/js/src/jsinfer.h b/js/src/jsinfer.h index 30c134ea390..c20ac4a43fe 100644 --- a/js/src/jsinfer.h +++ b/js/src/jsinfer.h @@ -43,12 +43,12 @@ #define jsinfer_h___ #include "jsalloc.h" -#include "jscell.h" #include "jsfriendapi.h" #include "jsprvtd.h" #include "ds/LifoAlloc.h" #include "gc/Barrier.h" +#include "gc/Heap.h" #include "js/HashTable.h" namespace JS { diff --git a/js/src/jsobj.h b/js/src/jsobj.h index ce98a3f1766..aa72e312ef8 100644 --- a/js/src/jsobj.h +++ b/js/src/jsobj.h @@ -58,9 +58,9 @@ #include "jspubtd.h" #include "jsprvtd.h" #include "jslock.h" -#include "jscell.h" #include "gc/Barrier.h" +#include "gc/Heap.h" #include "vm/ObjectImpl.h" #include "vm/String.h" diff --git a/js/src/jsstr.h b/js/src/jsstr.h index 6a9f4e11822..4f48d5ce854 100644 --- a/js/src/jsstr.h +++ b/js/src/jsstr.h @@ -45,7 +45,6 @@ #include "jsatom.h" #include "jsprvtd.h" #include "jslock.h" -#include "jscell.h" #include "jsutil.h" #include "js/HashTable.h" diff --git a/js/src/jsxml.h b/js/src/jsxml.h index f9f3f3dfe1f..2b377563fe0 100644 --- a/js/src/jsxml.h +++ b/js/src/jsxml.h @@ -41,9 +41,9 @@ #include "jspubtd.h" #include "jsobj.h" -#include "jscell.h" #include "gc/Barrier.h" +#include "gc/Heap.h" extern const char js_AnyName_str[]; extern const char js_AttributeName_str[]; diff --git a/js/src/vm/ObjectImpl-inl.h b/js/src/vm/ObjectImpl-inl.h index 938f51fea48..bb7e46865a6 100644 --- a/js/src/vm/ObjectImpl-inl.h +++ b/js/src/vm/ObjectImpl-inl.h @@ -10,11 +10,11 @@ #include "mozilla/Assertions.h" -#include "jscell.h" #include "jscompartment.h" #include "jsgc.h" #include "jsinterp.h" +#include "gc/Heap.h" #include "gc/Marking.h" #include "js/TemplateLib.h" diff --git a/js/src/vm/String.h b/js/src/vm/String.h index fcaeea5d6c7..149b171d461 100644 --- a/js/src/vm/String.h +++ b/js/src/vm/String.h @@ -44,10 +44,10 @@ #include "mozilla/Attributes.h" #include "jsapi.h" -#include "jscell.h" #include "jsfriendapi.h" #include "gc/Barrier.h" +#include "gc/Heap.h" class JSString; class JSDependentString;