From 3ec6cef3d29d0b73f37c205d0aeedfe43cc7c19d Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 20 Jun 2012 18:48:56 -0700 Subject: [PATCH] Bug 764962 - Add a verifier mode for GenerationalGC post barriers; r=billm This adds a store buffer which is enabled with --enable-gcgenerational and new verifier modes 11 & 12 corresponding to pre-barrier verifier modes 4 & 5 when enabled. --HG-- extra : rebase_source : c2ba68f1cb9eaf3375048b43b777f4ed6502cde1 --- js/src/Makefile.in | 2 + js/src/builtin/TestingFunctions.cpp | 35 +- js/src/gc/Barrier-inl.h | 2 +- js/src/gc/Barrier.h | 2 +- js/src/gc/StoreBuffer.cpp | 271 ++++++++++++++++ js/src/gc/StoreBuffer.h | 397 +++++++++++++++++++++++ js/src/jit-test/lib/prolog.js | 8 +- js/src/jit-test/tests/basic/bug729364.js | 2 +- js/src/jit-test/tests/basic/bug757199.js | 4 +- js/src/jsapi.cpp | 19 +- js/src/jsapi.h | 12 +- js/src/jscntxt.h | 3 +- js/src/jscompartment.cpp | 17 + js/src/jscompartment.h | 6 + js/src/jsfriendapi.cpp | 9 +- js/src/jsgc.cpp | 344 ++++++++++++++++---- js/src/jsgc.h | 46 ++- js/src/jsgcinlines.h | 11 + js/src/jsopcode.cpp | 2 +- js/src/methodjit/Compiler.cpp | 4 +- 20 files changed, 1102 insertions(+), 94 deletions(-) create mode 100644 js/src/gc/StoreBuffer.cpp create mode 100644 js/src/gc/StoreBuffer.h diff --git a/js/src/Makefile.in b/js/src/Makefile.in index 29e5695e5683..85133748995c 100644 --- a/js/src/Makefile.in +++ b/js/src/Makefile.in @@ -139,6 +139,7 @@ CPPSRCS = \ Marking.cpp \ Memory.cpp \ Statistics.cpp \ + StoreBuffer.cpp \ StringBuffer.cpp \ Unicode.cpp \ Xdr.cpp \ @@ -191,6 +192,7 @@ EXPORTS_gc = \ Heap.h \ Root.h \ Statistics.h \ + StoreBuffer.h \ $(NULL) ###################################################### diff --git a/js/src/builtin/TestingFunctions.cpp b/js/src/builtin/TestingFunctions.cpp index 8c1a5d09270d..79bf213febd7 100644 --- a/js/src/builtin/TestingFunctions.cpp +++ b/js/src/builtin/TestingFunctions.cpp @@ -247,13 +247,25 @@ SelectForGC(JSContext *cx, unsigned argc, jsval *vp) } static JSBool -VerifyBarriers(JSContext *cx, unsigned argc, jsval *vp) +VerifyPreBarriers(JSContext *cx, unsigned argc, jsval *vp) { if (argc) { ReportUsageError(cx, &JS_CALLEE(cx, vp).toObject(), "Too many arguments"); return JS_FALSE; } - gc::VerifyBarriers(cx->runtime); + gc::VerifyBarriers(cx->runtime, gc::PreBarrierVerifier); + *vp = JSVAL_VOID; + return JS_TRUE; +} + +static JSBool +VerifyPostBarriers(JSContext *cx, unsigned argc, jsval *vp) +{ + if (argc) { + ReportUsageError(cx, &JS_CALLEE(cx, vp).toObject(), "Too many arguments"); + return JS_FALSE; + } + gc::VerifyBarriers(cx->runtime, gc::PostBarrierVerifier); *vp = JSVAL_VOID; return JS_TRUE; } @@ -597,10 +609,15 @@ static JSFunctionSpecWithHelp TestingFunctions[] = { " 1: Collect when roots are added or removed\n" " 2: Collect when memory is allocated\n" " 3: Collect when the window paints (browser only)\n" -" 4: Verify write barriers between instructions\n" -" 5: Verify write barriers between paints\n" +" 4: Verify pre write barriers between instructions\n" +" 5: Verify pre write barriers between paints\n" " 6: Verify stack rooting (ignoring XML and Reflect)\n" " 7: Verify stack rooting (all roots)\n" +" 8: Incremental GC in two slices: 1) mark roots 2) finish collection\n" +" 9: Incremental GC in two slices: 1) mark all 2) new marking and finish\n" +" 10: Incremental GC in multiple slices\n" +" 11: Verify post write barriers between instructions\n" +" 12: Verify post write barriers between paints\n" " Period specifies that collection happens every n allocations.\n"), JS_FN_HELP("schedulegc", ScheduleGC, 1, 0, @@ -612,9 +629,13 @@ static JSFunctionSpecWithHelp TestingFunctions[] = { "selectforgc(obj1, obj2, ...)", " Schedule the given objects to be marked in the next GC slice."), - JS_FN_HELP("verifybarriers", VerifyBarriers, 0, 0, -"verifybarriers()", -" Start or end a run of the write barrier verifier."), + JS_FN_HELP("verifyprebarriers", VerifyPreBarriers, 0, 0, +"verifyprebarriers()", +" Start or end a run of the pre-write barrier verifier."), + + JS_FN_HELP("verifypostbarriers", VerifyPostBarriers, 0, 0, +"verifypostbarriers()", +" Start or end a run of the post-write barrier verifier."), JS_FN_HELP("gcslice", GCSlice, 1, 0, "gcslice(n)", diff --git a/js/src/gc/Barrier-inl.h b/js/src/gc/Barrier-inl.h index ab49fc1a9bea..05e4de2b9d95 100644 --- a/js/src/gc/Barrier-inl.h +++ b/js/src/gc/Barrier-inl.h @@ -296,7 +296,7 @@ HeapSlot::writeBarrierPost(JSObject *obj, uint32_t slot) } inline void -HeapSlot::writeBarrierPost(JSCompartment *comp, JSObject *obj, uint32_t slotno) +HeapSlot::writeBarrierPost(JSCompartment *comp, JSObject *obj, uint32_t slot) { } diff --git a/js/src/gc/Barrier.h b/js/src/gc/Barrier.h index 08c8de8f0d73..715a2f54f193 100644 --- a/js/src/gc/Barrier.h +++ b/js/src/gc/Barrier.h @@ -414,7 +414,7 @@ class HeapSlot : public EncapsulatedValue inline void set(JSCompartment *comp, JSObject *owner, uint32_t slot, const Value &v); static inline void writeBarrierPost(JSObject *obj, uint32_t slot); - static inline void writeBarrierPost(JSCompartment *comp, JSObject *obj, uint32_t slotno); + static inline void writeBarrierPost(JSCompartment *comp, JSObject *obj, uint32_t slot); private: inline void post(JSObject *owner, uint32_t slot); diff --git a/js/src/gc/StoreBuffer.cpp b/js/src/gc/StoreBuffer.cpp new file mode 100644 index 000000000000..3a8eeff2e9ca --- /dev/null +++ b/js/src/gc/StoreBuffer.cpp @@ -0,0 +1,271 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sw=4 et tw=78: + */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef JSGC_GENERATIONAL + +#include "jsgc.h" + +#include "gc/Barrier-inl.h" +#include "gc/StoreBuffer.h" +#include "vm/ObjectImpl-inl.h" + +namespace js { +namespace gc { + +/*** MonoTypeBuffer ***/ + +template +bool +StoreBuffer::MonoTypeBuffer::enable(uint8_t *region, size_t len) +{ + JS_ASSERT(len % sizeof(T) == 0); + base = pos = reinterpret_cast(region); + top = reinterpret_cast(region + len); + return true; +} + +template +void +StoreBuffer::MonoTypeBuffer::disable() +{ + base = pos = top = NULL; +} + +template +void +StoreBuffer::MonoTypeBuffer::compactNotInSet() +{ + T *insert = base; + for (T *v = base; v != pos; ++v) { + if (v->inRememberedSet(nursery)) + *insert++ = *v; + } + pos = insert; +} + +template +void +StoreBuffer::MonoTypeBuffer::compact() +{ + compactNotInSet(); +} + +template +void +StoreBuffer::MonoTypeBuffer::put(const T &v) +{ + /* Check if we have been enabled. */ + if (!pos) + return; + + /* + * Note: it is sometimes valid for a put to happen in the middle of a GC, + * e.g. a rekey of a Relocatable may end up here. In general, we do not + * care about these new entries or any overflows they cause. + */ + *pos++ = v; + if (isFull()) { + owner->setOverflowed(); + pos = base; + } +} + +template +bool +StoreBuffer::MonoTypeBuffer::accumulateEdges(EdgeSet &edges) +{ + compact(); + T *cursor = base; + while (cursor != pos) { + T edge = *cursor++; + + /* Note: the relocatable buffer is allowed to store pointers to NULL. */ + if (edge.isNullEdge()) + continue; + if (!edges.putNew(edge.location())) + return false; + } + return true; +} + +/*** RelocatableMonoTypeBuffer ***/ + +template +void +StoreBuffer::RelocatableMonoTypeBuffer::compactMoved() +{ + for (T *v = this->base; v != this->pos; ++v) { + if (v->isTagged()) { + T match = v->untagged(); + for (T *r = this->base; r != v; ++r) { + T check = r->untagged(); + if (check == match) + *r = NULL; + } + *v = NULL; + } + } + T *insert = this->base; + for (T *cursor = this->base; cursor != this->pos; ++cursor) { + if (*cursor != NULL) + *insert++ = *cursor; + } + this->pos = insert; +#ifdef DEBUG + for (T *cursor = this->base; cursor != this->pos; ++cursor) + JS_ASSERT(!cursor->isTagged()); +#endif +} + +template +void +StoreBuffer::RelocatableMonoTypeBuffer::compact() +{ + compactMoved(); + StoreBuffer::MonoTypeBuffer::compact(); +} + +template +void +StoreBuffer::RelocatableMonoTypeBuffer::unput(const T &v) +{ + put(v.tagged()); +} + +/*** GenericBuffer ***/ + +bool +StoreBuffer::GenericBuffer::enable(uint8_t *region, size_t len) +{ + base = pos = region; + top = region + len; + return true; +} + +void +StoreBuffer::GenericBuffer::disable() +{ + base = pos = top = NULL; +} + +bool +StoreBuffer::GenericBuffer::containsEdge(void *location) const +{ + uint8_t *p = base; + while (p < pos) { + unsigned size = *((unsigned *)p); + p += sizeof(unsigned); + + if (((BufferableRef *)p)->match(location)) + return true; + + p += size; + } + return false; +} + +/*** StoreBuffer ***/ + +bool +StoreBuffer::enable() +{ + buffer = js_malloc(TotalSize); + if (!buffer) + return false; + + /* Initialize the individual edge buffers in sub-regions. */ + uint8_t *asBytes = static_cast(buffer); + size_t offset = 0; + + if (!bufferVal.enable(&asBytes[offset], ValueBufferSize)) + return false; + offset += ValueBufferSize; + + if (!bufferCell.enable(&asBytes[offset], CellBufferSize)) + return false; + offset += CellBufferSize; + + if (!bufferSlot.enable(&asBytes[offset], SlotBufferSize)) + return false; + offset += SlotBufferSize; + + if (!bufferRelocVal.enable(&asBytes[offset], RelocValueBufferSize)) + return false; + offset += RelocValueBufferSize; + + if (!bufferRelocCell.enable(&asBytes[offset], RelocCellBufferSize)) + return false; + offset += RelocCellBufferSize; + + if (!bufferGeneric.enable(&asBytes[offset], GenericBufferSize)) + return false; + offset += GenericBufferSize; + + JS_ASSERT(offset == TotalSize); + + enabled = true; + return true; +} + +void +StoreBuffer::disable() +{ + bufferVal.disable(); + bufferCell.disable(); + bufferSlot.disable(); + bufferRelocVal.disable(); + bufferRelocCell.disable(); + bufferGeneric.disable(); + + js_free(buffer); + enabled = false; + overflowed = false; +} + +bool +StoreBuffer::coalesceForVerification() +{ + if (!edgeSet.initialized()) { + if (!edgeSet.init()) + return false; + } + JS_ASSERT(edgeSet.empty()); + if (!bufferVal.accumulateEdges(edgeSet)) + return false; + if (!bufferCell.accumulateEdges(edgeSet)) + return false; + if (!bufferSlot.accumulateEdges(edgeSet)) + return false; + if (!bufferRelocVal.accumulateEdges(edgeSet)) + return false; + if (!bufferRelocCell.accumulateEdges(edgeSet)) + return false; + return true; +} + +bool +StoreBuffer::containsEdgeAt(void *loc) const +{ + return edgeSet.has(loc) || bufferGeneric.containsEdge(loc); +} + +void +StoreBuffer::releaseVerificationData() +{ + edgeSet.finish(); +} + +template class StoreBuffer::MonoTypeBuffer; +template class StoreBuffer::MonoTypeBuffer; +template class StoreBuffer::MonoTypeBuffer; +template class StoreBuffer::RelocatableMonoTypeBuffer; +template class StoreBuffer::RelocatableMonoTypeBuffer; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* JSGC_GENERATIONAL */ diff --git a/js/src/gc/StoreBuffer.h b/js/src/gc/StoreBuffer.h new file mode 100644 index 000000000000..ace96153e490 --- /dev/null +++ b/js/src/gc/StoreBuffer.h @@ -0,0 +1,397 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sw=4 et tw=78: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef JSGC_GENERATIONAL +#ifndef jsgc_storebuffer_h___ +#define jsgc_storebuffer_h___ + +#include "jsgc.h" +#include "jsalloc.h" + +#include "gc/Marking.h" + +namespace js { +namespace gc { + +/* + * Note: this is a stub Nursery that does not actually contain a heap, just a + * set of pointers which are "inside" the nursery to implement verification. + */ +class Nursery +{ + HashSet, SystemAllocPolicy> nursery; + + public: + Nursery() : nursery() {} + + bool enable() { + if (!nursery.initialized()) + return nursery.init(); + return true; + } + + void disable() { + if (!nursery.initialized()) + return; + nursery.finish(); + } + + bool isInside(void *cell) const { + return nursery.initialized() && nursery.has(cell); + } + + void insertPointer(void *cell) { + nursery.putNew(cell); + } +}; + +/* + * BufferableRef represents an abstract reference for use in the generational + * GC's remembered set. Entries in the store buffer that cannot be represented + * with the simple pointer-to-a-pointer scheme must derive from this class and + * use the generic store buffer interface. + */ +class BufferableRef +{ + public: + virtual bool match(void *location) = 0; + virtual void mark(JSTracer *trc) = 0; +}; + +/* + * HashKeyRef represents a reference to a HashTable key. Manual HashTable + * barriers should should instantiate this template with their own table/key + * type to insert into the generic buffer with putGeneric. + */ +template +class HashKeyRef : public BufferableRef +{ + Map *map; + Key key; + + typedef typename Map::Ptr Ptr; + + public: + HashKeyRef(Map *m, const Key &k) : map(m), key(k) {} + + bool match(void *location) { + Ptr p = map->lookup(key); + if (!p) + return false; + return &p->key == location; + } + + void mark(JSTracer *trc) {} +}; + +/* + * The StoreBuffer observes all writes that occur in the system and performs + * efficient filtering of them to derive a remembered set for nursery GC. + */ +class StoreBuffer +{ + /* TODO: profile to find the ideal size for these. */ + static const size_t ValueBufferSize = 1 * 1024 * sizeof(Value *); + static const size_t CellBufferSize = 2 * 1024 * sizeof(Cell **); + static const size_t SlotBufferSize = 2 * 1024 * (sizeof(JSObject *) + sizeof(uint32_t)); + static const size_t RelocValueBufferSize = 1 * 1024 * sizeof(Value *); + static const size_t RelocCellBufferSize = 1 * 1024 * sizeof(Cell **); + static const size_t GenericBufferSize = 1 * 1024 * sizeof(int); + static const size_t TotalSize = ValueBufferSize + CellBufferSize + + SlotBufferSize + RelocValueBufferSize + RelocCellBufferSize + + GenericBufferSize; + + typedef HashSet, SystemAllocPolicy> EdgeSet; + + /* + * This buffer holds only a single type of edge. Using this buffer is more + * efficient than the generic buffer when many writes will be to the same + * type of edge: e.g. Value or Cell*. + */ + template + class MonoTypeBuffer + { + friend class StoreBuffer; + + StoreBuffer *owner; + Nursery *nursery; + + T *base; /* Pointer to the start of the buffer. */ + T *pos; /* Pointer to the current insertion position. */ + T *top; /* Pointer to one element after the end. */ + + MonoTypeBuffer(StoreBuffer *owner, Nursery *nursery) + : owner(owner), nursery(nursery), base(NULL), pos(NULL), top(NULL) + {} + + MonoTypeBuffer &operator=(const MonoTypeBuffer& other) MOZ_DELETE; + + bool enable(uint8_t *region, size_t len); + void disable(); + + bool isEmpty() const { return pos == base; } + bool isFull() const { JS_ASSERT(pos <= top); return pos == top; } + + /* Compaction algorithms. */ + void compactNotInSet(); + + /* + * Attempts to reduce the usage of the buffer by removing unnecessary + * entries. + */ + void compact(); + + /* Add one item to the buffer. */ + void put(const T &v); + + /* For verification. */ + bool accumulateEdges(EdgeSet &edges); + }; + + /* + * Overrides the MonoTypeBuffer to support pointers that may be moved in + * memory outside of the GC's control. + */ + template + class RelocatableMonoTypeBuffer : public MonoTypeBuffer + { + friend class StoreBuffer; + + RelocatableMonoTypeBuffer(StoreBuffer *owner, Nursery *nursery) + : MonoTypeBuffer(owner, nursery) + {} + + /* Override compaction to filter out removed items. */ + void compactMoved(); + void compact(); + + /* Record a removal from the buffer. */ + void unput(const T &v); + }; + + class GenericBuffer + { + friend class StoreBuffer; + + StoreBuffer *owner; + Nursery *nursery; + + uint8_t *base; /* Pointer to start of buffer. */ + uint8_t *pos; /* Pointer to current buffer position. */ + uint8_t *top; /* Pointer to one past the last entry. */ + + GenericBuffer(StoreBuffer *owner, Nursery *nursery) + : owner(owner), nursery(nursery) + {} + + GenericBuffer &operator=(const GenericBuffer& other) MOZ_DELETE; + + bool enable(uint8_t *region, size_t len); + void disable(); + + /* Check if a pointer is present in the buffer. */ + bool containsEdge(void *location) const; + + template + void put(const T &t) { + /* Check if we have been enabled. */ + if (!pos) + return; + + /* Check for overflow. */ + if (top - pos < (unsigned)(sizeof(unsigned) + sizeof(T))) { + owner->setOverflowed(); + return; + } + + *((unsigned *)pos) = sizeof(T); + pos += sizeof(unsigned); + + T *p = (T *)pos; + new (p) T(t); + pos += sizeof(T); + } + }; + + class CellPtrEdge + { + friend class StoreBuffer; + friend class StoreBuffer::MonoTypeBuffer; + friend class StoreBuffer::RelocatableMonoTypeBuffer; + + Cell **edge; + + CellPtrEdge(Cell **v) : edge(v) {} + bool operator==(const CellPtrEdge &other) const { return edge == other.edge; } + bool operator!=(const CellPtrEdge &other) const { return edge != other.edge; } + + void *location() const { return (void *)edge; } + + bool inRememberedSet(Nursery *n) { + return !n->isInside(edge) && n->isInside(*edge); + } + + bool isNullEdge() const { + return !*edge; + } + + CellPtrEdge tagged() const { return CellPtrEdge((Cell **)(uintptr_t(edge) | 1)); } + CellPtrEdge untagged() const { return CellPtrEdge((Cell **)(uintptr_t(edge) & ~1)); } + bool isTagged() const { return bool(uintptr_t(edge) & 1); } + }; + + class ValueEdge + { + friend class StoreBuffer; + friend class StoreBuffer::MonoTypeBuffer; + friend class StoreBuffer::RelocatableMonoTypeBuffer; + + Value *edge; + + ValueEdge(Value *v) : edge(v) {} + bool operator==(const ValueEdge &other) const { return edge == other.edge; } + bool operator!=(const ValueEdge &other) const { return edge != other.edge; } + + void *deref() const { return edge->isGCThing() ? edge->toGCThing() : NULL; } + void *location() const { return (void *)edge; } + + bool inRememberedSet(Nursery *n) { + return !n->isInside(edge) && n->isInside(deref()); + } + + bool isNullEdge() const { + return !deref(); + } + + ValueEdge tagged() const { return ValueEdge((Value *)(uintptr_t(edge) | 1)); } + ValueEdge untagged() const { return ValueEdge((Value *)(uintptr_t(edge) & ~1)); } + bool isTagged() const { return bool(uintptr_t(edge) & 1); } + }; + + struct SlotEdge + { + friend class StoreBuffer; + friend class StoreBuffer::MonoTypeBuffer; + + JSObject *object; + uint32_t offset; + + SlotEdge(JSObject *object, uint32_t offset) : object(object), offset(offset) {} + + bool operator==(const SlotEdge &other) const { + return object == other.object && offset == other.offset; + } + + bool operator!=(const SlotEdge &other) const { + return object != other.object || offset != other.offset; + } + + HeapSlot *slotLocation() const { + if (object->isDenseArray()) { + if (offset >= object->getDenseArrayInitializedLength()) + return NULL; + return (HeapSlot *)&object->getDenseArrayElement(offset); + } + if (offset >= object->slotSpan()) + return NULL; + return &object->getSlotRef(offset); + } + + void *deref() const { + HeapSlot *loc = slotLocation(); + return (loc && loc->isGCThing()) ? loc->toGCThing() : NULL; + } + + void *location() const { + return (void *)slotLocation(); + } + + bool inRememberedSet(Nursery *n) { + return !n->isInside(object) && n->isInside(deref()); + } + + bool isNullEdge() const { + return !deref(); + } + }; + + MonoTypeBuffer bufferVal; + MonoTypeBuffer bufferCell; + MonoTypeBuffer bufferSlot; + RelocatableMonoTypeBuffer bufferRelocVal; + RelocatableMonoTypeBuffer bufferRelocCell; + GenericBuffer bufferGeneric; + + Nursery *nursery; + + void *buffer; + + bool overflowed; + bool enabled; + + /* For the verifier. */ + EdgeSet edgeSet; + + /* For use by our owned buffers. */ + void setOverflowed() { overflowed = true; } + + public: + StoreBuffer(Nursery *n) + : bufferVal(this, n), bufferCell(this, n), bufferSlot(this, n), + bufferRelocVal(this, n), bufferRelocCell(this, n), bufferGeneric(this, n), + nursery(n), buffer(NULL), overflowed(false), enabled(false) + {} + + bool enable(); + void disable(); + bool isEnabled() { return enabled; } + + /* Get the overflowed status. */ + bool hasOverflowed() const { return overflowed; } + + /* Insert a single edge into the buffer/remembered set. */ + void putValue(Value *v) { + bufferVal.put(v); + } + void putCell(Cell **o) { + bufferCell.put(o); + } + void putSlot(JSObject *obj, uint32_t slot) { + bufferSlot.put(SlotEdge(obj, slot)); + } + + /* Insert or update a single edge in the Relocatable buffer. */ + void putRelocatableValue(Value *v) { + bufferRelocVal.put(v); + } + void putRelocatableCell(Cell **c) { + bufferRelocCell.put(c); + } + void removeRelocatableValue(Value *v) { + bufferRelocVal.unput(v); + } + void removeRelocatableCell(Cell **c) { + bufferRelocCell.unput(c); + } + + /* Insert an entry into the generic buffer. */ + template + void putGeneric(const T &t) { + bufferGeneric.put(t); + } + + /* For the verifier. */ + bool coalesceForVerification(); + void releaseVerificationData(); + bool containsEdgeAt(void *loc) const; +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* jsgc_storebuffer_h___ */ +#endif /* JSGC_GENERATIONAL */ diff --git a/js/src/jit-test/lib/prolog.js b/js/src/jit-test/lib/prolog.js index 8c2fb6de83d7..54ddabde597d 100644 --- a/js/src/jit-test/lib/prolog.js +++ b/js/src/jit-test/lib/prolog.js @@ -24,8 +24,12 @@ if (!("selectforgc" in this)) { selectforgc = function() { } } -if (!("verifybarriers" in this)) { - verifybarriers = function() { } +if (!("verifyprebarriers" in this)) { + verifyprebarriers = function() { } +} + +if (!("verifypostbarriers" in this)) { + verifypostbarriers = function() { } } if ("options" in this) diff --git a/js/src/jit-test/tests/basic/bug729364.js b/js/src/jit-test/tests/basic/bug729364.js index dc65d54f00dc..e9efdc59a587 100644 --- a/js/src/jit-test/tests/basic/bug729364.js +++ b/js/src/jit-test/tests/basic/bug729364.js @@ -17,4 +17,4 @@ function h(code, globalType) { } } function p()(function() function() {}) -g("print(let(x=verifybarriers(),q)((x(\"\",l('')))?(\"\"):(\"\")))()") +g("print(let(x=verifyprebarriers(),q)((x(\"\",l('')))?(\"\"):(\"\")))()") diff --git a/js/src/jit-test/tests/basic/bug757199.js b/js/src/jit-test/tests/basic/bug757199.js index c8c7e99d6033..73adacd8e22d 100644 --- a/js/src/jit-test/tests/basic/bug757199.js +++ b/js/src/jit-test/tests/basic/bug757199.js @@ -10,9 +10,9 @@ function f2(o) function f() { var x = new ctor(); - verifybarriers(); + verifyprebarriers(); f2(x); - verifybarriers(); + verifyprebarriers(); } f(); f(); diff --git a/js/src/jsapi.cpp b/js/src/jsapi.cpp index e0f27251b982..f82d49c73309 100644 --- a/js/src/jsapi.cpp +++ b/js/src/jsapi.cpp @@ -747,7 +747,8 @@ JSRuntime::JSRuntime() gcMaxBytes(0), gcMaxMallocBytes(0), gcNumArenasFreeCommitted(0), - gcVerifyData(NULL), + gcVerifyPreData(NULL), + gcVerifyPostData(NULL), gcChunkAllocationSinceLastGC(false), gcNextFullGCTime(0), gcLastGCTime(0), @@ -6748,21 +6749,27 @@ JS_SetGCZeal(JSContext *cx, uint8_t zeal, uint32_t frequency) " 1: additional GCs at common danger points\n" " 2: GC every F allocations (default: 100)\n" " 3: GC when the window paints (browser only)\n" - " 4: Verify write barriers between instructions\n" - " 5: Verify write barriers between paints\n" + " 4: Verify pre write barriers between instructions\n" + " 5: Verify pre write barriers between paints\n" " 6: Verify stack rooting (ignoring XML and Reflect)\n" " 7: Verify stack rooting (all roots)\n" " 8: Incremental GC in two slices: 1) mark roots 2) finish collection\n" " 9: Incremental GC in two slices: 1) mark all 2) new marking and finish\n" - " 10: Incremental GC in multiple slices\n"); + " 10: Incremental GC in multiple slices\n" + " 11: Verify post write barriers between instructions\n" + " 12: Verify post write barriers between paints\n"); } const char *p = strchr(env, ','); zeal = atoi(env); frequency = p ? atoi(p + 1) : JS_DEFAULT_ZEAL_FREQ; } - if (zeal == 0 && cx->runtime->gcVerifyData) - VerifyBarriers(cx->runtime); + if (zeal == 0) { + if (cx->runtime->gcVerifyPreData) + VerifyBarriers(cx->runtime, PreBarrierVerifier); + if (cx->runtime->gcVerifyPostData) + VerifyBarriers(cx->runtime, PostBarrierVerifier); + } bool schedule = zeal >= js::gc::ZealAllocValue; cx->runtime->gcZeal_ = zeal; diff --git a/js/src/jsapi.h b/js/src/jsapi.h index bc12480a1d96..6d0ece8e4456 100644 --- a/js/src/jsapi.h +++ b/js/src/jsapi.h @@ -859,6 +859,7 @@ class ValueOperations bool isMagic(JSWhyMagic why) const { return value()->isMagic(why); } bool isMarkable() const { return value()->isMarkable(); } bool isPrimitive() const { return value()->isPrimitive(); } + bool isGCThing() const { return value()->isGCThing(); } bool toBoolean() const { return value()->toBoolean(); } double toNumber() const { return value()->toNumber(); } @@ -3667,7 +3668,7 @@ struct JSTracer { const void *debugPrintArg; size_t debugPrintIndex; JSBool eagerlyTraceWeakMaps; -#ifdef DEBUG +#ifdef JS_GC_ZEAL void *realLocation; #endif }; @@ -3707,11 +3708,16 @@ JS_CallTracer(JSTracer *trc, void *thing, JSGCTraceKind kind); /* * Sets the real location for a marked reference, when passing the address * directly is not feasable. + * + * FIXME: This is currently overcomplicated by our need to nest calls for Values + * stored as keys in hash tables, but will get simplified once we can rekey + * in-place. */ -#ifdef DEBUG +#ifdef JS_GC_ZEAL # define JS_SET_TRACING_LOCATION(trc, location) \ JS_BEGIN_MACRO \ - (trc)->realLocation = (location); \ + if ((trc)->realLocation == NULL || (location) == NULL) \ + (trc)->realLocation = (location); \ JS_END_MACRO #else # define JS_SET_TRACING_LOCATION(trc, location) \ diff --git a/js/src/jscntxt.h b/js/src/jscntxt.h index b099e72cc3ab..4bf629fc7959 100644 --- a/js/src/jscntxt.h +++ b/js/src/jscntxt.h @@ -502,7 +502,8 @@ struct JSRuntime : js::RuntimeFriendFields */ volatile uint32_t gcNumArenasFreeCommitted; js::GCMarker gcMarker; - void *gcVerifyData; + void *gcVerifyPreData; + void *gcVerifyPostData; bool gcChunkAllocationSinceLastGC; int64_t gcNextFullGCTime; int64_t gcLastGCTime; diff --git a/js/src/jscompartment.cpp b/js/src/jscompartment.cpp index f2480a8e1dbc..c1a9a69d1f03 100644 --- a/js/src/jscompartment.cpp +++ b/js/src/jscompartment.cpp @@ -41,6 +41,9 @@ JSCompartment::JSCompartment(JSRuntime *rt) : rt(rt), principals(NULL), global_(NULL), +#ifdef JSGC_GENERATIONAL + gcStoreBuffer(&gcNursery), +#endif needsBarrier_(false), gcState(NoGCScheduled), gcPreserveCode(false), @@ -89,6 +92,20 @@ JSCompartment::init(JSContext *cx) if (!regExps.init(cx)) return false; +#ifdef JSGC_GENERATIONAL + /* + * If we are in the middle of post-barrier verification, we need to + * immediately begin collecting verification data on new compartments. + */ + if (rt->gcVerifyPostData) { + if (!gcNursery.enable()) + return false; + + if (!gcStoreBuffer.enable()) + return false; + } +#endif + return debuggees.init(); } diff --git a/js/src/jscompartment.h b/js/src/jscompartment.h index f1ea9c3196ee..452450214058 100644 --- a/js/src/jscompartment.h +++ b/js/src/jscompartment.h @@ -14,6 +14,7 @@ #include "jsobj.h" #include "jsscope.h" +#include "gc/StoreBuffer.h" #include "vm/GlobalObject.h" #include "vm/RegExpObject.h" @@ -144,6 +145,11 @@ struct JSCompartment public: js::gc::ArenaLists arenas; +#ifdef JSGC_GENERATIONAL + js::gc::Nursery gcNursery; + js::gc::StoreBuffer gcStoreBuffer; +#endif + private: bool needsBarrier_; public: diff --git a/js/src/jsfriendapi.cpp b/js/src/jsfriendapi.cpp index c0959b3bad76..974dff1261d1 100644 --- a/js/src/jsfriendapi.cpp +++ b/js/src/jsfriendapi.cpp @@ -776,8 +776,13 @@ GCDescription::formatJSON(JSRuntime *rt, uint64_t timestamp) const JS_FRIEND_API(void) NotifyDidPaint(JSRuntime *rt) { - if (rt->gcZeal() == gc::ZealFrameVerifierValue) { - gc::VerifyBarriers(rt); + if (rt->gcZeal() == gc::ZealFrameVerifierPreValue) { + gc::VerifyBarriers(rt, gc::PreBarrierVerifier); + return; + } + + if (rt->gcZeal() == gc::ZealFrameVerifierPostValue) { + gc::VerifyBarriers(rt, gc::PostBarrierVerifier); return; } diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index d93c9107a5ca..2f2a837ecd30 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -120,10 +120,16 @@ const int IGC_MARK_SLICE_MULTIPLIER = 2; #ifdef JS_GC_ZEAL static void -StartVerifyBarriers(JSRuntime *rt); +StartVerifyPreBarriers(JSRuntime *rt); static void -EndVerifyBarriers(JSRuntime *rt); +EndVerifyPreBarriers(JSRuntime *rt); + +static void +StartVerifyPostBarriers(JSRuntime *rt); + +static void +EndVerifyPostBarriers(JSRuntime *rt); void FinishVerifier(JSRuntime *rt); @@ -892,11 +898,11 @@ MarkExactStackRoots(JSTracer *trc) } else if (i == THING_ROOT_ID) { MarkIdRoot(trc, (jsid *)addr, "exact stackroot id"); } else if (i == THING_ROOT_PROPERTY_ID) { - MarkIdRoot(trc, ((PropertyId *)addr)->asId(), "exact stackroot property id"); + MarkIdRoot(trc, &((PropertyId *)addr)->asId(), "exact stackroot property id"); } else if (i == THING_ROOT_VALUE) { MarkValueRoot(trc, (Value *)addr, "exact stackroot value"); } else if (i == THING_ROOT_TYPE) { - MarkTypeRoot(trc, (Type)addr, "exact stackroot type"); + MarkTypeRoot(trc, (types::Type *)addr, "exact stackroot type"); } else if (i == THING_ROOT_SHAPE) { MarkShapeRoot(trc, (Shape **)addr, "exact stackroot shape"); } else if (i == THING_ROOT_BASE_SHAPE) { @@ -1393,7 +1399,6 @@ JSCompartment::setGCLastBytes(size_t lastBytes, size_t lastMallocBytes, JSGCInvo + rt->gcHighFrequencyHeapGrowthMax); JS_ASSERT(gcHeapGrowthFactor <= rt->gcHighFrequencyHeapGrowthMax && gcHeapGrowthFactor >= rt->gcHighFrequencyHeapGrowthMin); - } rt->gcHighFrequencyGC = true; } else { @@ -1802,6 +1807,9 @@ InitTracer(JSTracer *trc, JSRuntime *rt, JSTraceCallback callback) trc->debugPrintArg = NULL; trc->debugPrintIndex = size_t(-1); trc->eagerlyTraceWeakMaps = true; +#ifdef JS_GC_ZEAL + trc->realLocation = NULL; +#endif } /* static */ int64_t @@ -3940,25 +3948,28 @@ Collect(JSRuntime *rt, bool incremental, int64_t budget, JS_ASSERT_IF(!incremental || budget != SliceBudget::Unlimited, JSGC_INCREMENTAL); #ifdef JS_GC_ZEAL - bool restartVerify = rt->gcVerifyData && - rt->gcZeal() == ZealVerifierValue && - reason != gcreason::SHUTDOWN_CC && - rt->hasContexts(); - + bool isShutdown = reason == gcreason::SHUTDOWN_CC || !rt->hasContexts(); struct AutoVerifyBarriers { JSRuntime *runtime; - bool restart; - AutoVerifyBarriers(JSRuntime *rt, bool restart) - : runtime(rt), restart(restart) + bool restartPreVerifier; + bool restartPostVerifier; + AutoVerifyBarriers(JSRuntime *rt, bool isShutdown) + : runtime(rt) { - if (rt->gcVerifyData) - EndVerifyBarriers(rt); + restartPreVerifier = !isShutdown && rt->gcVerifyPreData; + restartPostVerifier = !isShutdown && rt->gcVerifyPostData; + if (rt->gcVerifyPreData) + EndVerifyPreBarriers(rt); + if (rt->gcVerifyPostData) + EndVerifyPostBarriers(rt); } ~AutoVerifyBarriers() { - if (restart) - StartVerifyBarriers(runtime); + if (restartPreVerifier) + StartVerifyPreBarriers(runtime); + if (restartPostVerifier) + StartVerifyPostBarriers(runtime); } - } av(rt, restartVerify); + } av(rt, isShutdown); #endif RecordNativeStackTopForGC(rt); @@ -4445,12 +4456,7 @@ namespace gc { /* * Write barrier verification * - * The next few functions are for incremental write barrier verification. When - * StartVerifyBarriers is called, a snapshot is taken of all objects in the GC - * heap and saved in an explicit graph data structure. Later, EndVerifyBarriers - * traverses the heap again. Any pointer values that were in the snapshot and - * are no longer found must be marked; otherwise an assertion triggers. Note - * that we must not GC in between starting and finishing a verification phase. + * The next few functions are for write barrier verification. * * The VerifyBarriers function is a shorthand. It checks if a verification phase * is currently running. If not, it starts one. Otherwise, it ends the current @@ -4459,6 +4465,21 @@ namespace gc { * The user can adjust the frequency of verifications, which causes * VerifyBarriers to be a no-op all but one out of N calls. However, if the * |always| parameter is true, it starts a new phase no matter what. + * + * Pre-Barrier Verifier: + * When StartVerifyBarriers is called, a snapshot is taken of all objects in + * the GC heap and saved in an explicit graph data structure. Later, + * EndVerifyBarriers traverses the heap again. Any pointer values that were in + * the snapshot and are no longer found must be marked; otherwise an assertion + * triggers. Note that we must not GC in between starting and finishing a + * verification phase. + * + * Post-Barrier Verifier: + * When StartVerifyBarriers is called, we create a virtual "Nursery Set" which + * future allocations are recorded in and turn on the StoreBuffer. Later, + * EndVerifyBarriers traverses the heap and ensures that the set of cross- + * generational pointers we find is a subset of the pointers recorded in our + * StoreBuffer. */ struct EdgeValue @@ -4491,12 +4512,12 @@ typedef HashMap, SystemAllocPolicy> * The nodemap field is a hashtable that maps from the address of the GC thing * to the VerifyNode that represents it. */ -struct VerifyTracer : JSTracer { +struct VerifyPreTracer : JSTracer { /* The gcNumber when the verification began. */ uint64_t number; - /* This counts up to JS_VERIFIER_FREQ to decide whether to verify. */ - uint32_t count; + /* This counts up to gcZealFrequency to decide whether to verify. */ + int count; /* This graph represents the initial GC "snapshot". */ VerifyNode *curnode; @@ -4505,8 +4526,8 @@ struct VerifyTracer : JSTracer { char *term; NodeMap nodemap; - VerifyTracer() : root(NULL) {} - ~VerifyTracer() { js_free(root); } + VerifyPreTracer() : root(NULL) {} + ~VerifyPreTracer() { js_free(root); } }; /* @@ -4516,7 +4537,7 @@ struct VerifyTracer : JSTracer { static void AccumulateEdge(JSTracer *jstrc, void **thingp, JSGCTraceKind kind) { - VerifyTracer *trc = (VerifyTracer *)jstrc; + VerifyPreTracer *trc = (VerifyPreTracer *)jstrc; trc->edgeptr += sizeof(EdgeValue); if (trc->edgeptr >= trc->term) { @@ -4534,7 +4555,7 @@ AccumulateEdge(JSTracer *jstrc, void **thingp, JSGCTraceKind kind) } static VerifyNode * -MakeNode(VerifyTracer *trc, void *thing, JSGCTraceKind kind) +MakeNode(VerifyPreTracer *trc, void *thing, JSGCTraceKind kind) { NodeMap::AddPtr p = trc->nodemap.lookupForAdd(thing); if (!p) { @@ -4566,9 +4587,9 @@ NextNode(VerifyNode *node) } static void -StartVerifyBarriers(JSRuntime *rt) +StartVerifyPreBarriers(JSRuntime *rt) { - if (rt->gcVerifyData || rt->gcIncrementalState != NO_INCREMENTAL) + if (rt->gcVerifyPreData || rt->gcIncrementalState != NO_INCREMENTAL) return; AutoTraceSession session(rt); @@ -4584,7 +4605,7 @@ StartVerifyBarriers(JSRuntime *rt) for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront()) r.front()->bitmap.clear(); - VerifyTracer *trc = new (js_malloc(sizeof(VerifyTracer))) VerifyTracer; + VerifyPreTracer *trc = new (js_malloc(sizeof(VerifyPreTracer))) VerifyPreTracer; rt->gcNumber++; trc->number = rt->gcNumber; @@ -4632,7 +4653,7 @@ StartVerifyBarriers(JSRuntime *rt) node = NextNode(node); } - rt->gcVerifyData = trc; + rt->gcVerifyPreData = trc; rt->gcIncrementalState = MARK; rt->gcMarker.start(rt); for (CompartmentsIter c(rt); !c.done(); c.next()) { @@ -4644,7 +4665,7 @@ StartVerifyBarriers(JSRuntime *rt) oom: rt->gcIncrementalState = NO_INCREMENTAL; - trc->~VerifyTracer(); + trc->~VerifyPreTracer(); js_free(trc); } @@ -4666,7 +4687,7 @@ const static uint32_t MAX_VERIFIER_EDGES = 1000; static void CheckEdge(JSTracer *jstrc, void **thingp, JSGCTraceKind kind) { - VerifyTracer *trc = (VerifyTracer *)jstrc; + VerifyPreTracer *trc = (VerifyPreTracer *)jstrc; VerifyNode *node = trc->curnode; /* Avoid n^2 behavior. */ @@ -4697,7 +4718,7 @@ AssertMarkedOrAllocated(const EdgeValue &edge) } static void -EndVerifyBarriers(JSRuntime *rt) +EndVerifyPreBarriers(JSRuntime *rt) { AutoTraceSession session(rt); @@ -4706,7 +4727,7 @@ EndVerifyBarriers(JSRuntime *rt) AutoCopyFreeListToArenas copy(rt); RecordNativeStackTopForGC(rt); - VerifyTracer *trc = (VerifyTracer *)rt->gcVerifyData; + VerifyPreTracer *trc = (VerifyPreTracer *)rt->gcVerifyPreData; if (!trc) return; @@ -4728,7 +4749,7 @@ EndVerifyBarriers(JSRuntime *rt) JS_ASSERT(trc->number == rt->gcNumber); rt->gcNumber++; - rt->gcVerifyData = NULL; + rt->gcVerifyPreData = NULL; rt->gcIncrementalState = NO_INCREMENTAL; if (!compartmentCreated && IsIncrementalGCSafe(rt)) { @@ -4752,45 +4773,244 @@ EndVerifyBarriers(JSRuntime *rt) rt->gcMarker.reset(); rt->gcMarker.stop(); - trc->~VerifyTracer(); + trc->~VerifyPreTracer(); js_free(trc); } -void -FinishVerifier(JSRuntime *rt) +/*** Post-Barrier Verifyier ***/ + +struct VerifyPostTracer : JSTracer { + /* The gcNumber when the verification began. */ + uint64_t number; + + /* This counts up to gcZealFrequency to decide whether to verify. */ + int count; +}; + +/* + * The post-barrier verifier runs the full store buffer and a fake nursery when + * running and when it stops, walks the full heap to ensure that all the + * important edges were inserted into the storebuffer. + */ +static void +StartVerifyPostBarriers(JSRuntime *rt) { - if (VerifyTracer *trc = (VerifyTracer *)rt->gcVerifyData) { - trc->~VerifyTracer(); - js_free(trc); +#ifdef JSGC_GENERATIONAL + if (!rt->gcExactScanningEnabled || + rt->gcVerifyPostData || + rt->gcIncrementalState != NO_INCREMENTAL) + { + return; } + VerifyPostTracer *trc = new (js_malloc(sizeof(VerifyPostTracer))) VerifyPostTracer; + rt->gcVerifyPostData = trc; + rt->gcNumber++; + trc->number = rt->gcNumber; + trc->count = 0; + for (CompartmentsIter c(rt); !c.done(); c.next()) { + if (IsAtomsCompartment(c)) + continue; + + if (!c->gcNursery.enable()) + goto oom; + + if (!c->gcStoreBuffer.enable()) + goto oom; + } + return; +oom: + trc->~VerifyPostTracer(); + js_free(trc); + rt->gcVerifyPostData = NULL; + for (CompartmentsIter c(rt); !c.done(); c.next()) { + c->gcNursery.disable(); + c->gcStoreBuffer.disable(); + } +#endif +} + +#ifdef JSGC_GENERATIONAL +static void +AssertStoreBufferContainsEdge(StoreBuffer *storebuf, void *loc, void *dst) +{ + if (storebuf->containsEdgeAt(loc)) + return; + + char msgbuf[1024]; + JS_snprintf(msgbuf, sizeof(msgbuf), "[post-barrier verifier] Missing edge @ %p to %p", + loc, dst); + MOZ_ReportAssertionFailure(msgbuf, __FILE__, __LINE__); + MOZ_CRASH(); +} + +static void +PostVerifierVisitEdge(JSTracer *jstrc, void **thingp, JSGCTraceKind kind) +{ + VerifyPostTracer *trc = (VerifyPostTracer *)jstrc; + Cell *dst = (Cell *)*thingp; + JSCompartment *comp = dst->compartment(); + + /* + * Note: watchpoint markAll will give us cross-compartment pointers into the + * atoms compartment. + */ + if (IsAtomsCompartment(comp)) + return; + + /* Filter out non cross-generational edges. */ + if (!comp->gcNursery.isInside(dst)) + return; + + /* + * Note: since Value travels through the stack to get Cell**, we need to use + * the annotated location in the tracer instead of the indirect location for + * these edges. + */ + Cell *loc = (Cell *)(trc->realLocation != NULL ? trc->realLocation : thingp); + + AssertStoreBufferContainsEdge(&comp->gcStoreBuffer, loc, dst); +} +#endif + +static void +EndVerifyPostBarriers(JSRuntime *rt) +{ +#ifdef JSGC_GENERATIONAL + AutoTraceSession session(rt); + + rt->gcHelperThread.waitBackgroundSweepOrAllocEnd(); + + AutoCopyFreeListToArenas copy(rt); + RecordNativeStackTopForGC(rt); + + VerifyPostTracer *trc = (VerifyPostTracer *)rt->gcVerifyPostData; + JS_TracerInit(trc, rt, PostVerifierVisitEdge); + trc->count = 0; + + if (!rt->gcExactScanningEnabled) + goto oom; + + for (CompartmentsIter c(rt); !c.done(); c.next()) { + if (!c->gcStoreBuffer.coalesceForVerification()) + goto oom; + } + + /* Walk the heap. */ + for (CompartmentsIter c(rt); !c.done(); c.next()) { + if (!c->gcStoreBuffer.isEnabled() || + c->gcStoreBuffer.hasOverflowed() || + IsAtomsCompartment(c)) + { + continue; + } + + if (c->watchpointMap) + c->watchpointMap->markAll(trc); + + for (size_t kind = 0; kind < FINALIZE_LIMIT; ++kind) { + for (CellIterUnderGC cells(c, AllocKind(kind)); !cells.done(); cells.next()) { + Cell *src = cells.getCell(); + if (!c->gcNursery.isInside(src)) + JS_TraceChildren(trc, src, MapAllocToTraceKind(AllocKind(kind))); + } + } + } + +oom: + trc->~VerifyPostTracer(); + js_free(trc); + rt->gcVerifyPostData = NULL; + for (CompartmentsIter c(rt); !c.done(); c.next()) { + c->gcNursery.disable(); + c->gcStoreBuffer.disable(); + c->gcStoreBuffer.releaseVerificationData(); + } +#endif +} + +/*** Barrier Verifier Scheduling ***/ + +static void +VerifyPreBarriers(JSRuntime *rt) +{ + if (rt->gcVerifyPreData) + EndVerifyPreBarriers(rt); + else + StartVerifyPreBarriers(rt); +} + +static void +VerifyPostBarriers(JSRuntime *rt) +{ + if (rt->gcVerifyPostData) + EndVerifyPostBarriers(rt); + else + StartVerifyPostBarriers(rt); } void -VerifyBarriers(JSRuntime *rt) +VerifyBarriers(JSRuntime *rt, VerifierType type) { - if (rt->gcVerifyData) - EndVerifyBarriers(rt); + if (type == PreBarrierVerifier) + VerifyPreBarriers(rt); else - StartVerifyBarriers(rt); + VerifyPostBarriers(rt); +} + +static void +MaybeVerifyPreBarriers(JSRuntime *rt, bool always) +{ + if (rt->gcZeal() != ZealVerifierPreValue) + return; + + if (VerifyPreTracer *trc = (VerifyPreTracer *)rt->gcVerifyPreData) { + if (++trc->count < rt->gcZealFrequency && !always) + return; + + EndVerifyPreBarriers(rt); + } + StartVerifyPreBarriers(rt); +} + +static void +MaybeVerifyPostBarriers(JSRuntime *rt, bool always) +{ + if (rt->gcZeal() != ZealVerifierPostValue) + return; + + if (VerifyPostTracer *trc = (VerifyPostTracer *)rt->gcVerifyPostData) { + if (++trc->count < rt->gcZealFrequency && !always) + return; + + EndVerifyPostBarriers(rt); + } + StartVerifyPostBarriers(rt); } void MaybeVerifyBarriers(JSContext *cx, bool always) { - JSRuntime *rt = cx->runtime; + MaybeVerifyPreBarriers(cx->runtime, always); + MaybeVerifyPostBarriers(cx->runtime, always); +} - if (rt->gcZeal() != ZealVerifierValue) - return; - - uint32_t freq = rt->gcZealFrequency; - - if (VerifyTracer *trc = (VerifyTracer *)rt->gcVerifyData) { - if (++trc->count < freq && !always) - return; - - EndVerifyBarriers(rt); +void +FinishVerifier(JSRuntime *rt) +{ + if (VerifyPreTracer *trc = (VerifyPreTracer *)rt->gcVerifyPreData) { + trc->~VerifyPreTracer(); + js_free(trc); } - StartVerifyBarriers(rt); +#ifdef JSGC_GENERATIONAL + if (VerifyPostTracer *trc = (VerifyPostTracer *)rt->gcVerifyPostData) { + trc->~VerifyPostTracer(); + js_free(trc); + for (CompartmentsIter c(rt); !c.done(); c.next()) { + c->gcNursery.disable(); + c->gcStoreBuffer.disable(); + } + } +#endif } #endif /* JS_GC_ZEAL */ diff --git a/js/src/jsgc.h b/js/src/jsgc.h index 0cb842505c57..8e5b44688718 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -112,6 +112,37 @@ MapAllocToTraceKind(AllocKind thingKind) return map[thingKind]; } +static inline bool +IsNurseryAllocable(AllocKind kind) +{ + JS_ASSERT(kind >= 0 && kind < FINALIZE_LIMIT); + static const bool map[FINALIZE_LIMIT] = { + false, /* FINALIZE_OBJECT0 */ + true, /* FINALIZE_OBJECT0_BACKGROUND */ + false, /* FINALIZE_OBJECT2 */ + true, /* FINALIZE_OBJECT2_BACKGROUND */ + false, /* FINALIZE_OBJECT4 */ + true, /* FINALIZE_OBJECT4_BACKGROUND */ + false, /* FINALIZE_OBJECT8 */ + true, /* FINALIZE_OBJECT8_BACKGROUND */ + false, /* FINALIZE_OBJECT12 */ + true, /* FINALIZE_OBJECT12_BACKGROUND */ + false, /* FINALIZE_OBJECT16 */ + true, /* FINALIZE_OBJECT16_BACKGROUND */ + false, /* FINALIZE_SCRIPT */ + false, /* FINALIZE_SHAPE */ + false, /* FINALIZE_BASE_SHAPE */ + false, /* FINALIZE_TYPE_OBJECT */ +#if JS_HAS_XML_SUPPORT + false, /* FINALIZE_XML */ +#endif + true, /* FINALIZE_SHORT_STRING */ + true, /* FINALIZE_STRING */ + false /* FINALIZE_EXTERNAL_STRING */ + }; + return map[kind]; +} + inline JSGCTraceKind GetGCThingTraceKind(const void *thing); @@ -1074,19 +1105,26 @@ SetDeterministicGC(JSContext *cx, bool enabled); const int ZealPokeValue = 1; const int ZealAllocValue = 2; const int ZealFrameGCValue = 3; -const int ZealVerifierValue = 4; -const int ZealFrameVerifierValue = 5; +const int ZealVerifierPreValue = 4; +const int ZealFrameVerifierPreValue = 5; const int ZealStackRootingSafeValue = 6; const int ZealStackRootingValue = 7; const int ZealIncrementalRootsThenFinish = 8; const int ZealIncrementalMarkAllThenFinish = 9; const int ZealIncrementalMultipleSlices = 10; +const int ZealVerifierPostValue = 11; +const int ZealFrameVerifierPostValue = 12; + +enum VerifierType { + PreBarrierVerifier, + PostBarrierVerifier +}; #ifdef JS_GC_ZEAL /* Check that write barriers have been used correctly. See jsgc.cpp. */ void -VerifyBarriers(JSRuntime *rt); +VerifyBarriers(JSRuntime *rt, VerifierType type); void MaybeVerifyBarriers(JSContext *cx, bool always = false); @@ -1094,7 +1132,7 @@ MaybeVerifyBarriers(JSContext *cx, bool always = false); #else static inline void -VerifyBarriers(JSRuntime *rt) +VerifyBarriers(JSRuntime *rt, VerifierType type) { } diff --git a/js/src/jsgcinlines.h b/js/src/jsgcinlines.h index 231d6abfc772..a0c6be533e45 100644 --- a/js/src/jsgcinlines.h +++ b/js/src/jsgcinlines.h @@ -420,6 +420,11 @@ NewGCThing(JSContext *cx, js::gc::AllocKind kind, size_t thingSize) JS_ASSERT_IF(t && comp->needsBarrier(), static_cast(t)->arenaHeader()->allocatedDuringIncremental); + +#if defined(JSGC_GENERATIONAL) && defined(JS_GC_ZEAL) + if (cx->runtime->gcVerifyPostData && IsNurseryAllocable(kind) && !IsAtomsCompartment(comp)) + comp->gcNursery.insertPointer(t); +#endif return static_cast(t); } @@ -442,6 +447,12 @@ TryNewGCThing(JSContext *cx, js::gc::AllocKind kind, size_t thingSize) void *t = cx->compartment->arenas.allocateFromFreeList(kind, thingSize); JS_ASSERT_IF(t && cx->compartment->needsBarrier(), static_cast(t)->arenaHeader()->allocatedDuringIncremental); + +#if defined(JSGC_GENERATIONAL) && defined(JS_GC_ZEAL) + JSCompartment *comp = cx->compartment; + if (cx->runtime->gcVerifyPostData && IsNurseryAllocable(kind) && !IsAtomsCompartment(comp)) + comp->gcNursery.insertPointer(t); +#endif return static_cast(t); } diff --git a/js/src/jsopcode.cpp b/js/src/jsopcode.cpp index ce2a428b7cb0..92d8bff6cca2 100644 --- a/js/src/jsopcode.cpp +++ b/js/src/jsopcode.cpp @@ -5097,7 +5097,7 @@ Decompile(SprintStack *ss, jsbytecode *pc, int nb) sn = js_GetSrcNote(jp->script, pc); /* Skip any #n= prefix to find the opening bracket. */ - for (xval = rval; *xval != '[' && *xval != '{'; xval++) + for (xval = rval; *xval != '[' && *xval != '{' && *xval; xval++) continue; inArray = (*xval == '['); if (inArray) diff --git a/js/src/methodjit/Compiler.cpp b/js/src/methodjit/Compiler.cpp index 8c82a38d0633..24f20369d834 100644 --- a/js/src/methodjit/Compiler.cpp +++ b/js/src/methodjit/Compiler.cpp @@ -3822,7 +3822,9 @@ void mjit::Compiler::interruptCheckHelper() { Jump jump; - if (cx->runtime->gcZeal() == js::gc::ZealVerifierValue) { + if (cx->runtime->gcZeal() == js::gc::ZealVerifierPreValue || + cx->runtime->gcZeal() == js::gc::ZealVerifierPostValue) + { /* For barrier verification, always take the interrupt so we can verify. */ jump = masm.jump(); } else {