diff --git a/js/public/TemplateLib.h b/js/public/TemplateLib.h index c94e57fdb5b6..fb5d2f9cd63f 100644 --- a/js/public/TemplateLib.h +++ b/js/public/TemplateLib.h @@ -87,7 +87,7 @@ template struct CeilingLog2 { /* Round up to the nearest power of 2. */ template struct RoundUpPow2 { - static const size_t result = 1u << CeilingLog2::result; + static const size_t result = size_t(1) << CeilingLog2::result; }; template <> struct RoundUpPow2<0> { static const size_t result = 1; diff --git a/js/src/jit-test/tests/basic/bug656261.js b/js/src/jit-test/tests/basic/bug656261.js index 68232f573a54..e62ab334a450 100644 --- a/js/src/jit-test/tests/basic/bug656261.js +++ b/js/src/jit-test/tests/basic/bug656261.js @@ -5,7 +5,7 @@ function build_getter(i) { function test() { - var N = internalConst("OBJECT_MARK_STACK_LENGTH") + 2; + var N = internalConst("MARK_STACK_LENGTH") + 2; var o = {}; var descriptor = { enumerable: true}; for (var i = 0; i != N; ++i) { diff --git a/js/src/jscntxt.h b/js/src/jscntxt.h index d323a51859a6..b50aaf050a84 100644 --- a/js/src/jscntxt.h +++ b/js/src/jscntxt.h @@ -437,11 +437,8 @@ struct JSRuntime /* The reason that an interrupt-triggered GC should be called. */ js::gcstats::Reason gcTriggerReason; - /* Pre-allocated space for the GC mark stacks. Pointer type ensures alignment. */ - void *gcMarkStackObjs[js::OBJECT_MARK_STACK_SIZE / sizeof(void *)]; - void *gcMarkStackTypes[js::TYPE_MARK_STACK_SIZE / sizeof(void *)]; - void *gcMarkStackXMLs[js::XML_MARK_STACK_SIZE / sizeof(void *)]; - void *gcMarkStackLarges[js::LARGE_MARK_STACK_SIZE / sizeof(void *)]; + /* Pre-allocated space for the GC mark stack. */ + uintptr_t gcMarkStackArray[js::MARK_STACK_LENGTH]; /* * Compartment that triggered GC. If more than one Compatment need GC, diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index 530791aa2b41..2187754ca230 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -1735,10 +1735,7 @@ namespace js { GCMarker::GCMarker(JSContext *cx) : color(BLACK), unmarkedArenaStackTop(NULL), - objStack(cx->runtime->gcMarkStackObjs, sizeof(cx->runtime->gcMarkStackObjs)), - typeStack(cx->runtime->gcMarkStackTypes, sizeof(cx->runtime->gcMarkStackTypes)), - xmlStack(cx->runtime->gcMarkStackXMLs, sizeof(cx->runtime->gcMarkStackXMLs)), - largeStack(cx->runtime->gcMarkStackLarges, sizeof(cx->runtime->gcMarkStackLarges)) + stack(cx->runtime->gcMarkStackArray) { JS_TRACER_INIT(this, cx, NULL); markLaterArenas = 0; @@ -1792,7 +1789,8 @@ MarkDelayedChildren(GCMarker *trc, Arena *a) void GCMarker::markDelayedChildren() { - while (unmarkedArenaStackTop) { + JS_ASSERT(unmarkedArenaStackTop); + do { /* * If marking gets delayed at the same arena again, we must repeat * marking of its things. For that we pop arena from the stack and @@ -1805,7 +1803,7 @@ GCMarker::markDelayedChildren() a->aheader.hasDelayedMarking = 0; markLaterArenas--; MarkDelayedChildren(this, a); - } + } while (unmarkedArenaStackTop); JS_ASSERT(!markLaterArenas); } @@ -3496,7 +3494,8 @@ EndVerifyBarriers(JSContext *cx) JS_ASSERT(trc->number == rt->gcNumber); - rt->gcIncrementalTracer->markDelayedChildren(); + if (rt->gcIncrementalTracer->hasDelayedChildren()) + rt->gcIncrementalTracer->markDelayedChildren(); rt->gcVerifyData = NULL; rt->gcIncrementalTracer = NULL; diff --git a/js/src/jsgc.h b/js/src/jsgc.h index 45277b3fed77..a2acda183e08 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -63,6 +63,7 @@ #include "gc/Statistics.h" #include "js/HashTable.h" #include "js/Vector.h" +#include "js/TemplateLib.h" struct JSCompartment; @@ -1593,49 +1594,68 @@ struct ConservativeGCThreadData { template struct MarkStack { T *stack; - uintN tos, limit; + T *tos; + T *limit; bool push(T item) { if (tos == limit) return false; - stack[tos++] = item; + *tos++ = item; return true; } - bool isEmpty() { return tos == 0; } + bool push(T item1, T item2) { + T *nextTos = tos + 2; + if (nextTos > limit) + return false; + tos[0] = item1; + tos[1] = item2; + tos = nextTos; + return true; + } + + bool isEmpty() const { + return tos == stack; + } T pop() { JS_ASSERT(!isEmpty()); - return stack[--tos]; + return *--tos; } - T &peek() { - JS_ASSERT(!isEmpty()); - return stack[tos-1]; - } - - MarkStack(void **buffer, size_t size) - { - tos = 0; - limit = size / sizeof(T) - 1; - stack = (T *)buffer; - } + template + MarkStack(T (&buffer)[N]) + : stack(buffer), + tos(buffer), + limit(buffer + N) { } }; -struct LargeMarkItem -{ - JSObject *obj; - uintN markpos; - - LargeMarkItem(JSObject *obj) : obj(obj), markpos(0) {} -}; - -static const size_t OBJECT_MARK_STACK_SIZE = 32768 * sizeof(JSObject *); -static const size_t XML_MARK_STACK_SIZE = 1024 * sizeof(JSXML *); -static const size_t TYPE_MARK_STACK_SIZE = 1024 * sizeof(types::TypeObject *); -static const size_t LARGE_MARK_STACK_SIZE = 64 * sizeof(LargeMarkItem); +static const size_t MARK_STACK_LENGTH = 32768; struct GCMarker : public JSTracer { + /* + * We use a common mark stack to mark GC things of different types and use + * the explicit tags to distinguish them when it cannot be deduced from + * the context of push or pop operation. + * + * Currently we need only 4 tags. However that can be extended to 8 if + * necessary. We tag either pointers to GC things or pointers to Value + * arrays. So the pointers are always at least 8-byte aligned. + */ + enum StackTag { + ValueArrayTag, + ObjectTag, + TypeTag, + XmlTag, + LastTag = XmlTag + }; + + static const uintptr_t StackTagMask = 3; + + static void staticAsserts() { + JS_STATIC_ASSERT(StackTagMask >= uintptr_t(LastTag)); + } + private: /* The color is only applied to objects, functions and xml. */ uint32 color; @@ -1653,10 +1673,7 @@ struct GCMarker : public JSTracer { void dumpConservativeRoots(); #endif - MarkStack objStack; - MarkStack typeStack; - MarkStack xmlStack; - MarkStack largeStack; + MarkStack stack; public: explicit GCMarker(JSContext *cx); @@ -1681,30 +1698,47 @@ struct GCMarker : public JSTracer { void delayMarkingChildren(const void *thing); + bool hasDelayedChildren() const { + return !!unmarkedArenaStackTop; + } + void markDelayedChildren(); bool isMarkStackEmpty() { - return objStack.isEmpty() && - typeStack.isEmpty() && - xmlStack.isEmpty() && - largeStack.isEmpty(); + return stack.isEmpty(); } void drainMarkStack(); + inline void processMarkStackTop(); + void pushObject(JSObject *obj) { - if (!objStack.push(obj)) - delayMarkingChildren(obj); + pushTaggedPtr(ObjectTag, obj); } void pushType(types::TypeObject *type) { - if (!typeStack.push(type)) - delayMarkingChildren(type); + pushTaggedPtr(TypeTag, type); } void pushXML(JSXML *xml) { - if (!xmlStack.push(xml)) - delayMarkingChildren(xml); + pushTaggedPtr(XmlTag, xml); + } + + void pushTaggedPtr(StackTag tag, void *ptr) { + uintptr_t addr = reinterpret_cast(ptr); + JS_ASSERT(!(addr & StackTagMask)); + if (!stack.push(addr | uintptr_t(tag))) + delayMarkingChildren(ptr); + } + + bool pushValueArray(void *start, void *end) { + JS_STATIC_ASSERT(ValueArrayTag == 0); + JS_ASSERT(start < end); + uintptr_t startAddr = reinterpret_cast(start); + uintptr_t endAddr = reinterpret_cast(end); + JS_ASSERT(!(startAddr & StackTagMask)); + JS_ASSERT(!(endAddr & StackTagMask)); + return stack.push(endAddr, startAddr); } }; diff --git a/js/src/jsgcmark.cpp b/js/src/jsgcmark.cpp index 607f12be479d..f06d602a0eaa 100644 --- a/js/src/jsgcmark.cpp +++ b/js/src/jsgcmark.cpp @@ -280,6 +280,9 @@ MarkXML(JSTracer *trc, const MarkablePtr &xml, const char *name) } #endif +#define JS_SAME_COMPARTMENT_ASSERT(thing1, thing2) \ + JS_ASSERT((thing1)->compartment() == (thing2)->compartment()) + #define JS_COMPARTMENT_ASSERT(rt, thing) \ JS_ASSERT_IF((rt)->gcCurrentCompartment, \ (thing)->compartment() == (rt)->gcCurrentCompartment); @@ -352,14 +355,13 @@ PushMarkStack(GCMarker *gcmarker, const Shape *thing) ScanShape(gcmarker, thing); } -static void +static inline void ScanBaseShape(GCMarker *gcmarker, BaseShape *base); void PushMarkStack(GCMarker *gcmarker, BaseShape *thing) { - JS_OPT_ASSERT_IF(gcmarker->context->runtime->gcCurrentCompartment, - thing->compartment() == gcmarker->context->runtime->gcCurrentCompartment); + JS_COMPARTMENT_ASSERT(gcmarker->runtime, thing); /* We mark base shapes directly rather than pushing on the stack. */ if (thing->markIfUnmarked(gcmarker->getMarkColor())) @@ -665,24 +667,10 @@ MarkRootRange(JSTracer *trc, size_t len, jsid *vec, const char *name) MarkIdRangeUnbarriered(trc, len, vec, name); } -static inline void -ScanValue(GCMarker *gcmarker, const Value &v) -{ - if (v.isMarkable()) { - JSGCTraceKind kind = v.gcKind(); - if (kind == JSTRACE_STRING) { - PushMarkStack(gcmarker, v.toString()); - } else { - JS_ASSERT(kind == JSTRACE_OBJECT); - PushMarkStack(gcmarker, &v.toObject()); - } - } -} - static void ScanShape(GCMarker *gcmarker, const Shape *shape) { -restart: + restart: PushMarkStack(gcmarker, shape->base()); jsid id = shape->maybePropid(); @@ -696,20 +684,33 @@ restart: goto restart; } -static void +static inline void ScanBaseShape(GCMarker *gcmarker, BaseShape *base) { - if (base->hasGetterObject()) - PushMarkStack(gcmarker, base->getterObject()); + for (;;) { + if (base->hasGetterObject()) + PushMarkStack(gcmarker, base->getterObject()); - if (base->hasSetterObject()) - PushMarkStack(gcmarker, base->setterObject()); + if (base->hasSetterObject()) + PushMarkStack(gcmarker, base->setterObject()); - if (base->isOwned()) - PushMarkStack(gcmarker, base->baseUnowned()); + if (JSObject *parent = base->getObjectParent()) + PushMarkStack(gcmarker, parent); - if (JSObject *parent = base->getObjectParent()) - PushMarkStack(gcmarker, parent); + if (base->isOwned()) { + /* + * Make sure that ScanBaseShape is not recursive so its inlining + * is possible. + */ + UnownedBaseShape *unowned = base->baseUnowned(); + JS_SAME_COMPARTMENT_ASSERT(base, unowned); + if (unowned->markIfUnmarked(gcmarker->getMarkColor())) { + base = unowned; + continue; + } + } + break; + } } static inline void @@ -736,15 +737,15 @@ ScanLinearString(GCMarker *gcmarker, JSLinearString *str) * The function tries to scan the whole rope tree using the marking stack as * temporary storage. If that becomes full, the unscanned ropes are added to * the delayed marking list. When the function returns, the marking stack is - * at the same depth as it was on entry. - * - * The function relies on the fact that a rope can only point to other ropes or - * linear strings, it cannot refer to other GC things of other types. + * at the same depth as it was on entry. This way we avoid using tags when + * pushing ropes to the stack as ropes never leaks to other users of the + * stack. This also assumes that a rope can only point to other ropes or + * linear strings, it cannot refer to GC things of other types. */ static void ScanRope(GCMarker *gcmarker, JSRope *rope) { - unsigned savedTos = gcmarker->objStack.tos; + uintptr_t *savedTos = gcmarker->stack.tos; for (;;) { JS_ASSERT(GetGCThingTraceKind(rope) == JSTRACE_STRING); JS_ASSERT(rope->JSString::isRope()); @@ -769,21 +770,21 @@ ScanRope(GCMarker *gcmarker, JSRope *rope) * When both children are ropes, set aside the right one to * scan it later. */ - if (next && !gcmarker->objStack.push(next)) + if (next && !gcmarker->stack.push(reinterpret_cast(next))) gcmarker->delayMarkingChildren(next); next = &left->asRope(); } } if (next) { rope = next; - } else if (savedTos != gcmarker->objStack.tos) { - JS_ASSERT(savedTos < gcmarker->objStack.tos); - rope = static_cast(gcmarker->objStack.pop()); + } else if (savedTos != gcmarker->stack.tos) { + JS_ASSERT(savedTos < gcmarker->stack.tos); + rope = reinterpret_cast(gcmarker->stack.pop()); } else { break; } } - JS_ASSERT(savedTos == gcmarker->objStack.tos); + JS_ASSERT(savedTos == gcmarker->stack.tos); } static inline void @@ -809,10 +810,40 @@ PushMarkStack(GCMarker *gcmarker, JSString *str) ScanString(gcmarker, str); } -static const uintN LARGE_OBJECT_CHUNK_SIZE = 2048; +static JS_NEVER_INLINE void +DelayMarkingValueArray(GCMarker *gcmarker, HeapValue *begin, HeapValue *end) +{ + for (HeapValue *vp = begin; vp != end; ++vp) { + const Value &v = *vp; + Cell *cell; + uint32 color; + if (v.isString()) { + cell = v.toString(); + color = BLACK; + } else if (v.isObject()) { + cell = &v.toObject(); + color = gcmarker->getMarkColor(); + } else { + continue; + } + if (cell->markIfUnmarked(color)) + gcmarker->delayMarkingChildren(cell); + } +} -static void -ScanObject(GCMarker *gcmarker, JSObject *obj) +static inline void +PushValueArray(GCMarker *gcmarker, HeapValue *array, size_t size) +{ + if (size != 0) { + JS_ASSERT(array); + HeapValue *end = array + size; + if (!gcmarker->pushValueArray(array, end)) + DelayMarkingValueArray(gcmarker, array, end); + } +} + +static JS_ALWAYS_INLINE bool +ScanObjectWithoutSlots(GCMarker *gcmarker, JSObject *obj) { types::TypeObject *type = obj->typeFromGC(); PushMarkStack(gcmarker, type); @@ -824,54 +855,15 @@ ScanObject(GCMarker *gcmarker, JSObject *obj) Class *clasp = shape->getObjectClass(); if (clasp->trace) { if (clasp == &ArrayClass) { - if (obj->getDenseArrayInitializedLength() > LARGE_OBJECT_CHUNK_SIZE) { - if (!gcmarker->largeStack.push(LargeMarkItem(obj))) - clasp->trace(gcmarker, obj); - } else { - clasp->trace(gcmarker, obj); - } + PushValueArray(gcmarker, + obj->getDenseArrayElements(), + obj->getDenseArrayInitializedLength()); } else { clasp->trace(gcmarker, obj); } } - if (shape->isNative()) { - uint32 nslots = obj->slotSpan(); - if (nslots > LARGE_OBJECT_CHUNK_SIZE) { - if (gcmarker->largeStack.push(LargeMarkItem(obj))) - return; - } - - obj->scanSlots(gcmarker); - } -} - -static bool -ScanLargeObject(GCMarker *gcmarker, LargeMarkItem &item) -{ - JSObject *obj = item.obj; - - uintN start = item.markpos; - uintN stop; - uint32 capacity; - if (obj->isDenseArray()) { - capacity = obj->getDenseArrayInitializedLength(); - stop = JS_MIN(start + LARGE_OBJECT_CHUNK_SIZE, capacity); - for (uintN i=stop; i>start; i--) - ScanValue(gcmarker, obj->getDenseArrayElement(i-1)); - } else { - JS_ASSERT(obj->isNative()); - capacity = obj->slotSpan(); - stop = JS_MIN(start + LARGE_OBJECT_CHUNK_SIZE, capacity); - for (uintN i=stop; i>start; i--) - ScanValue(gcmarker, obj->nativeGetSlot(i-1)); - } - - if (stop == capacity) - return true; - - item.markpos += LARGE_OBJECT_CHUNK_SIZE; - return false; + return shape->isNative(); } void @@ -1071,35 +1063,104 @@ MarkChildren(JSTracer *trc, JSXML *xml) } /* namespace gc */ +inline void +GCMarker::processMarkStackTop() +{ + /* + * The code uses explicit goto to eliminate the tail recursion that + * compilers cannot optimize on their own. + */ + HeapValue *vp, *end; + JSObject *obj; + + uintptr_t addr = stack.pop(); + uintptr_t tag = addr & StackTagMask; + if (tag == ValueArrayTag) { + /* + * We set ValueArrayTag to zero to avoid bit setting and clearing when + * pushing and poping tagged value array pointers. This is the most + * common stack operation as we push the array on the stack again when + * we find the next unmarked object in the array. + */ + JS_STATIC_ASSERT(ValueArrayTag == 0); + uintptr_t addr2 = stack.pop(); + JS_ASSERT(addr <= addr2); + JS_ASSERT((addr2 - addr) % sizeof(Value) == 0); + vp = reinterpret_cast(addr); + end = reinterpret_cast(addr2); + goto scan_value_array; + } + + addr &= ~StackTagMask; + if (tag == ObjectTag) { + obj = reinterpret_cast(addr); + goto scan_obj; + } else if (tag == TypeTag) { + ScanTypeObject(this, reinterpret_cast(addr)); + } else { + JS_ASSERT(tag == XmlTag); + MarkChildren(this, reinterpret_cast(addr)); + } + return; + + scan_value_array: + JS_ASSERT(vp < end); + do { + const Value &v = *vp++; + if (v.isString()) { + JSString *str = v.toString(); + if (str->markIfUnmarked()) + ScanString(this, str); + } else if (v.isObject()) { + obj = &v.toObject(); + if (obj->markIfUnmarked(getMarkColor())) { + if (vp != end && !pushValueArray(vp, end)) + DelayMarkingValueArray(this, vp, end); + goto scan_obj; + } + } + } while (vp != end); + return; + + scan_obj: + if (ScanObjectWithoutSlots(this, obj)) { + unsigned nslots = obj->slotSpan(); + vp = obj->fixedSlots(); + if (obj->slots) { + unsigned nfixed = obj->numFixedSlots(); + if (nslots > nfixed) { + PushValueArray(this, vp, nfixed); + vp = obj->slots; + end = vp + (nslots - nfixed); + goto scan_value_array; + } + } + if (nslots) { + end = vp + nslots; + goto scan_value_array; + } + } + return; +} + void GCMarker::drainMarkStack() { JSRuntime *rt = runtime; rt->gcCheckCompartment = rt->gcCurrentCompartment; - while (!isMarkStackEmpty()) { - while (!objStack.isEmpty()) - ScanObject(this, static_cast(objStack.pop())); + for (;;) { + while (!stack.isEmpty()) + processMarkStackTop(); + if (!hasDelayedChildren()) + break; - while (!typeStack.isEmpty()) - ScanTypeObject(this, typeStack.pop()); - - while (!xmlStack.isEmpty()) - MarkChildren(this, xmlStack.pop()); - - if (!largeStack.isEmpty()) { - LargeMarkItem &item = largeStack.peek(); - if (ScanLargeObject(this, item)) - largeStack.pop(); - } - - if (isMarkStackEmpty()) { - /* - * Mark children of things that caused too deep recursion during the above - * tracing. Don't do this until we're done with everything else. - */ - markDelayedChildren(); - } + /* + * Mark children of things that caused too deep recursion during the + * above tracing. Don't do this until we're done with everything + * else. + */ + markDelayedChildren(); } rt->gcCheckCompartment = NULL; @@ -1149,29 +1210,3 @@ CallTracer(JSTracer *trc, void *thing, JSGCTraceKind kind) } } /* namespace js */ - -inline void -JSObject::scanSlots(GCMarker *gcmarker) -{ - /* - * Scan the fixed slots and the dynamic slots separately, to avoid - * branching inside nativeGetSlot(). - */ - unsigned i, nslots = slotSpan(); - if (slots) { - unsigned nfixed = numFixedSlots(); - if (nslots > nfixed) { - HeapValue *vp = fixedSlots(); - for (i = 0; i < nfixed; i++, vp++) - ScanValue(gcmarker, *vp); - vp = slots; - for (; i < nslots; i++, vp++) - ScanValue(gcmarker, *vp); - return; - } - } - JS_ASSERT(nslots <= numFixedSlots()); - HeapValue *vp = fixedSlots(); - for (i = 0; i < nslots; i++, vp++) - ScanValue(gcmarker, *vp); -} diff --git a/js/src/jsobj.h b/js/src/jsobj.h index 67be223b9b78..50309e09479f 100644 --- a/js/src/jsobj.h +++ b/js/src/jsobj.h @@ -457,6 +457,7 @@ struct JSObject : js::gc::Cell { private: friend struct js::Shape; + friend struct js::GCMarker; /* * Shape of the object, encodes the layout of the object's properties and @@ -546,8 +547,6 @@ struct JSObject : js::gc::Cell inline bool hasClass(const js::Class *c) const; inline const js::ObjectOps *getOps() const; - inline void scanSlots(js::GCMarker *gcmarker); - /* * An object is a delegate if it is on another object's prototype or scope * chain, and therefore the delegate might be asked implicitly to get or diff --git a/js/src/shell/js.cpp b/js/src/shell/js.cpp index 1e526279f964..177a93cc8b7c 100644 --- a/js/src/shell/js.cpp +++ b/js/src/shell/js.cpp @@ -1309,8 +1309,8 @@ InternalConst(JSContext *cx, uintN argc, jsval *vp) if (!flat) return false; - if (JS_FlatStringEqualsAscii(flat, "OBJECT_MARK_STACK_LENGTH")) { - vp[0] = UINT_TO_JSVAL(js::OBJECT_MARK_STACK_SIZE / sizeof(JSObject *)); + if (JS_FlatStringEqualsAscii(flat, "MARK_STACK_LENGTH")) { + vp[0] = UINT_TO_JSVAL(js::MARK_STACK_LENGTH); } else { JS_ReportError(cx, "unknown const name"); return false;