diff --git a/js/src/gc/GCMarker.h b/js/src/gc/GCMarker.h index 91642709fcbf..8ec467b4639b 100644 --- a/js/src/gc/GCMarker.h +++ b/js/src/gc/GCMarker.h @@ -27,6 +27,8 @@ static const size_t NON_INCREMENTAL_MARK_STACK_BASE_CAPACITY = 4096; static const size_t INCREMENTAL_MARK_STACK_BASE_CAPACITY = 32768; static const size_t SMALL_MARK_STACK_BASE_CAPACITY = 256; +enum class SlotsOrElementsKind { Elements, FixedSlots, DynamicSlots }; + namespace gc { enum IncrementalProgress { NotFinished = 0, Finished }; @@ -82,8 +84,7 @@ class MarkStack { * the context of push or pop operation. */ enum Tag { - SlotsRangeTag, - ElementsRangeTag, + SlotsOrElementsRangeTag, ObjectTag, GroupTag, JitCodeTag, @@ -111,19 +112,26 @@ class MarkStack { template T* as() const; - JSObject* asSlotsRangeObject() const; - JSObject* asElementsRangeObject() const; + JSObject* asRangeObject() const; JSRope* asTempRope() const; void assertValid() const; }; struct SlotsOrElementsRange { - SlotsOrElementsRange(Tag, JSObject* obj, size_t start); + SlotsOrElementsRange(SlotsOrElementsKind kind, JSObject* obj, size_t start); void assertValid() const; - size_t start; - TaggedPtr ptr; + SlotsOrElementsKind kind() const; + size_t start() const; + TaggedPtr ptr() const; + + static constexpr size_t StartShift = 2; + static constexpr size_t KindMask = (1 << StartShift) - 1; + + private: + uintptr_t startAndKind_; + TaggedPtr ptr_; }; explicit MarkStack(size_t maxCapacity = DefaultCapacity); @@ -147,7 +155,7 @@ class MarkStack { template MOZ_MUST_USE bool push(T* ptr); - MOZ_MUST_USE bool push(JSObject* obj, HeapSlot::Kind kind, size_t start); + MOZ_MUST_USE bool push(JSObject* obj, SlotsOrElementsKind kind, size_t start); MOZ_MUST_USE bool push(const SlotsOrElementsRange& array); // GCMarker::eagerlyMarkChildren uses unused marking stack as temporary @@ -407,10 +415,8 @@ class GCMarker : public JSTracer { template inline void pushTaggedPtr(T* ptr); - enum class RangeKind { Elements, FixedSlots, DynamicSlots }; - - inline void pushValueRange(JSObject* obj, RangeKind kind, size_t start, - size_t end); + inline void pushValueRange(JSObject* obj, SlotsOrElementsKind kind, + size_t start, size_t end); bool isMarkStackEmpty() { return stack.isEmpty() && auxStack.isEmpty(); } diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp index ae6ce9e757f1..d214a977a92d 100644 --- a/js/src/gc/Marking.cpp +++ b/js/src/gc/Marking.cpp @@ -1938,6 +1938,20 @@ static inline void CheckForCompartmentMismatch(JSObject* obj, JSObject* obj2) { #endif } +static inline size_t NumUsedFixedSlots(NativeObject* obj) { + return std::min(obj->numFixedSlots(), obj->slotSpan()); +} + +static inline size_t NumUsedDynamicSlots(NativeObject* obj) { + size_t nfixed = obj->numFixedSlots(); + size_t nslots = obj->slotSpan(); + if (nslots < nfixed) { + return 0; + } + + return nslots - nfixed; +} + inline void GCMarker::processMarkStackTop(SliceBudget& budget) { /* * This function uses explicit goto and scans objects directly. This allows us @@ -1950,7 +1964,7 @@ inline void GCMarker::processMarkStackTop(SliceBudget& budget) { */ JSObject* obj; // The object being scanned. - RangeKind kind; // The kind of slot range being scanned, if any. + SlotsOrElementsKind kind; // The kind of slot range being scanned, if any. HeapSlot* base; // Slot range base pointer. size_t index; // Index of the next slot to mark. size_t end; // End of slot range to mark. @@ -1958,38 +1972,37 @@ inline void GCMarker::processMarkStackTop(SliceBudget& budget) { gc::MarkStack& stack = currentStack(); switch (stack.peekTag()) { - case MarkStack::SlotsRangeTag: { + case MarkStack::SlotsOrElementsRangeTag: { auto range = stack.popSlotsOrElementsRange(); - obj = range.ptr.asSlotsRangeObject(); + obj = range.ptr().asRangeObject(); NativeObject* nobj = &obj->as(); - size_t nfixed = nobj->numFixedSlots(); - size_t nslots = nobj->slotSpan(); - if (range.start < nfixed) { - kind = RangeKind::FixedSlots; - base = nobj->fixedSlots(); - index = range.start; - end = std::min(nfixed, nslots); - } else { - kind = RangeKind::DynamicSlots; - base = nobj->slots_; - index = range.start - nfixed; - end = std::max(nslots, nfixed) - nfixed; + kind = range.kind(); + index = range.start(); + + switch (kind) { + case SlotsOrElementsKind::FixedSlots: { + base = nobj->fixedSlots(); + end = NumUsedFixedSlots(nobj); + break; + } + + case SlotsOrElementsKind::DynamicSlots: { + base = nobj->slots_; + end = NumUsedDynamicSlots(nobj); + break; + } + + case SlotsOrElementsKind::Elements: { + base = nobj->getDenseElementsAllowCopyOnWrite(); + + // Account for shifted elements. + size_t numShifted = nobj->getElementsHeader()->numShiftedElements(); + size_t initlen = nobj->getDenseInitializedLength(); + index = std::max(index, numShifted) - numShifted; + end = initlen; + break; + } } - goto scan_value_range; - } - - case MarkStack::ElementsRangeTag: { - auto range = stack.popSlotsOrElementsRange(); - obj = range.ptr.asElementsRangeObject(); - NativeObject* nobj = &obj->as(); - kind = RangeKind::Elements; - base = nobj->getDenseElementsAllowCopyOnWrite(); - - // Account for shifted elements. - size_t numShifted = nobj->getElementsHeader()->numShiftedElements(); - size_t initlen = nobj->getDenseInitializedLength(); - index = std::max(range.start, numShifted) - numShifted; - end = initlen; goto scan_value_range; } @@ -2108,7 +2121,7 @@ scan_obj : { } base = nobj->getDenseElementsAllowCopyOnWrite(); - kind = RangeKind::Elements; + kind = SlotsOrElementsKind::Elements; index = 0; end = nobj->getDenseInitializedLength(); @@ -2121,12 +2134,12 @@ scan_obj : { unsigned nfixed = nobj->numFixedSlots(); base = nobj->fixedSlots(); - kind = RangeKind::FixedSlots; + kind = SlotsOrElementsKind::FixedSlots; index = 0; if (nslots > nfixed) { pushValueRange(nobj, kind, index, nfixed); - kind = RangeKind::DynamicSlots; + kind = SlotsOrElementsKind::DynamicSlots; base = nobj->slots_; end = nslots - nfixed; goto scan_value_range; @@ -2170,7 +2183,7 @@ struct MapTypeToMarkStackTag { }; static inline bool TagIsRangeTag(MarkStack::Tag tag) { - return tag == MarkStack::SlotsRangeTag || tag == MarkStack::ElementsRangeTag; + return tag == MarkStack::SlotsOrElementsRangeTag; } inline MarkStack::TaggedPtr::TaggedPtr(Tag tag, Cell* ptr) @@ -2201,14 +2214,8 @@ inline T* MarkStack::TaggedPtr::as() const { return static_cast(ptr()); } -inline JSObject* MarkStack::TaggedPtr::asSlotsRangeObject() const { - MOZ_ASSERT(tag() == SlotsRangeTag); - MOZ_ASSERT(ptr()->isTenured()); - return ptr()->as(); -} - -inline JSObject* MarkStack::TaggedPtr::asElementsRangeObject() const { - MOZ_ASSERT(tag() == ElementsRangeTag); +inline JSObject* MarkStack::TaggedPtr::asRangeObject() const { + MOZ_ASSERT(TagIsRangeTag(tag())); MOZ_ASSERT(ptr()->isTenured()); return ptr()->as(); } @@ -2218,16 +2225,30 @@ inline JSRope* MarkStack::TaggedPtr::asTempRope() const { return &ptr()->as()->asRope(); } -inline MarkStack::SlotsOrElementsRange::SlotsOrElementsRange(Tag tag, - JSObject* obj, - size_t startArg) - : start(startArg), ptr(tag, obj) { +inline MarkStack::SlotsOrElementsRange::SlotsOrElementsRange( + SlotsOrElementsKind kindArg, JSObject* obj, size_t startArg) + : startAndKind_((startArg << StartShift) | size_t(kindArg)), + ptr_(SlotsOrElementsRangeTag, obj) { assertValid(); + MOZ_ASSERT(kind() == kindArg); + MOZ_ASSERT(start() == startArg); } inline void MarkStack::SlotsOrElementsRange::assertValid() const { - ptr.assertValid(); - MOZ_ASSERT(TagIsRangeTag(ptr.tag())); + ptr_.assertValid(); + MOZ_ASSERT(TagIsRangeTag(ptr_.tag())); +} + +inline SlotsOrElementsKind MarkStack::SlotsOrElementsRange::kind() const { + return SlotsOrElementsKind(startAndKind_ & KindMask); +} + +inline size_t MarkStack::SlotsOrElementsRange::start() const { + return startAndKind_ >> StartShift; +} + +inline MarkStack::TaggedPtr MarkStack::SlotsOrElementsRange::ptr() const { + return ptr_; } MarkStack::MarkStack(size_t maxCapacity) @@ -2316,12 +2337,9 @@ inline bool MarkStack::pushTempRope(JSRope* rope) { return pushTaggedPtr(TempRopeTag, rope); } -inline bool MarkStack::push(JSObject* obj, HeapSlot::Kind kind, size_t start) { - if (kind == HeapSlot::Slot) { - return push(SlotsOrElementsRange(SlotsRangeTag, obj, start)); - } - - return push(SlotsOrElementsRange(ElementsRangeTag, obj, start)); +inline bool MarkStack::push(JSObject* obj, SlotsOrElementsKind kind, + size_t start) { + return push(SlotsOrElementsRange(kind, obj, start)); } inline bool MarkStack::push(const SlotsOrElementsRange& array) { @@ -2580,8 +2598,8 @@ void GCMarker::pushTaggedPtr(T* ptr) { } } -void GCMarker::pushValueRange(JSObject* obj, RangeKind kind, size_t start, - size_t end) { +void GCMarker::pushValueRange(JSObject* obj, SlotsOrElementsKind kind, + size_t start, size_t end) { checkZone(obj); MOZ_ASSERT(obj->is()); MOZ_ASSERT(start <= end); @@ -2590,15 +2608,7 @@ void GCMarker::pushValueRange(JSObject* obj, RangeKind kind, size_t start, return; } - if (kind == RangeKind::DynamicSlots) { - uint32_t nfixed = obj->as().numFixedSlots(); - start += nfixed; - } - - HeapSlot::Kind slotsOrElements = - kind == RangeKind::Elements ? HeapSlot::Element : HeapSlot::Slot; - - if (!currentStack().push(obj, slotsOrElements, start)) { + if (!currentStack().push(obj, kind, start)) { delayMarkingChildren(obj); } }