Bug 888463 - Relocate hoisted elements/slots when their allocation moves during minor GC; r=billm

js/src/gc/Nursery.cpp

@@ -140,6 +140,7 @@ js::Nursery::allocateSlots(JSContext *cx, JSObject *obj, uint32_t nslots)
 ObjectElements *
 js::Nursery::allocateElements(JSContext *cx, JSObject *obj, uint32_t nelems)
 {
+    JS_ASSERT(nelems >= ObjectElements::VALUES_PER_HEADER);
     return reinterpret_cast<ObjectElements *>(allocateSlots(cx, obj, nelems));
 }
 
@@ -296,6 +297,56 @@ js::Nursery::allocateFromTenured(Zone *zone, AllocKind thingKind)
     return zone->allocator.arenas.allocateFromArena(zone, thingKind);
 }
 
+void
+js::Nursery::setSlotsForwardingPointer(HeapSlot *oldSlots, HeapSlot *newSlots, uint32_t nslots)
+{
+    JS_ASSERT(nslots > 0);
+    JS_ASSERT(isInside(oldSlots));
+    JS_ASSERT(!isInside(newSlots));
+    *reinterpret_cast<HeapSlot **>(oldSlots) = newSlots;
+}
+
+void
+js::Nursery::setElementsForwardingPointer(ObjectElements *oldHeader, ObjectElements *newHeader,
+                                          uint32_t nelems)
+{
+    /*
+     * If the JIT has hoisted a zero length pointer, then we do not need to
+     * relocate it because reads and writes to/from this pointer are invalid.
+     */
+    if (nelems - ObjectElements::VALUES_PER_HEADER < 1)
+        return;
+    JS_ASSERT(isInside(oldHeader));
+    JS_ASSERT(!isInside(newHeader));
+    *reinterpret_cast<HeapSlot **>(oldHeader->elements()) = newHeader->elements();
+}
+
+void
+js::Nursery::forwardBufferPointer(HeapSlot **pSlotsElems)
+{
+    HeapSlot *old = *pSlotsElems;
+    /*
+     * If the elements buffer is zero length, the "first" item could be inside
+     * of the next object or past the end of the allocable area.  However,
+     * since we always store the runtime as the last word in the nursery,
+     * isInside will still be true, even if this zero-size allocation abuts the
+     * end of the allocable area. Thus, it is always safe to read the first
+     * word of |old| here.
+     */
+    JS_ASSERT(isInside(old));
+    *pSlotsElems = *reinterpret_cast<HeapSlot **>(old);
+    JS_ASSERT(!isInside(*pSlotsElems));
+}
+
+void
+js::Nursery::forwardMovedBuffers(JSRuntime *rt)
+{
+    /*
+    for (current = rt->list; ; current = current->next)
+        forwardBufferPointer(current->item);
+    */
+}
+
 void *
 js::Nursery::moveToTenured(MinorCollectionTracer *trc, JSObject *src)
 {
@@ -355,6 +406,7 @@ js::Nursery::moveSlotsToTenured(JSObject *dst, JSObject *src, AllocKind dstKind)
     size_t count = src->numDynamicSlots();
     dst->slots = zone->pod_malloc<HeapSlot>(count);
     PodCopy(dst->slots, src->slots, count);
+    setSlotsForwardingPointer(src->slots, dst->slots, count);
     return count * sizeof(HeapSlot);
 }
 
@@ -387,6 +439,7 @@ js::Nursery::moveElementsToTenured(JSObject *dst, JSObject *src, AllocKind dstKi
             dstHeader = dst->getElementsHeader();
         }
         js_memcpy(dstHeader, srcHeader, nbytes);
+        setElementsForwardingPointer(srcHeader, dstHeader, nbytes / sizeof(HeapSlot));
         dst->elements = dstHeader->elements();
         return src->hasDynamicElements() ? nbytes : 0;
     }
@@ -398,14 +451,17 @@ js::Nursery::moveElementsToTenured(JSObject *dst, JSObject *src, AllocKind dstKi
         dst->setFixedElements();
         dstHeader = dst->getElementsHeader();
         js_memcpy(dstHeader, srcHeader, nslots * sizeof(HeapSlot));
+        setElementsForwardingPointer(srcHeader, dstHeader, nslots);
         return nslots * sizeof(HeapSlot);
     }
 
+    JS_ASSERT(nslots > 2);
     size_t nbytes = nslots * sizeof(HeapValue);
     dstHeader = static_cast<ObjectElements *>(zone->malloc_(nbytes));
     if (!dstHeader)
         MOZ_CRASH();
     js_memcpy(dstHeader, srcHeader, nslots * sizeof(HeapSlot));
+    setElementsForwardingPointer(srcHeader, dstHeader, nslots);
     dst->elements = dstHeader->elements();
     return nslots * sizeof(HeapSlot);
 }
@@ -548,6 +604,9 @@ js::Nursery::collect(JSRuntime *rt, JS::gcreason::Reason reason)
         JS_TraceChildren(&trc, obj, MapAllocToTraceKind(obj->tenuredGetAllocKind()));
     }
 
+    /* Forward any slot or elements pointers that were baked into jit code. */
+    forwardMovedBuffers(rt);
+
     /* Resize the nursery. */
     double promotionRate = trc.tenuredSize / double(allocationEnd() - start());
     if (promotionRate > 0.5)

js/src/gc/Nursery.h

@@ -208,6 +208,13 @@ class Nursery
     size_t moveElementsToTenured(JSObject *dst, JSObject *src, gc::AllocKind dstKind);
     size_t moveSlotsToTenured(JSObject *dst, JSObject *src, gc::AllocKind dstKind);
 
+    /* Handle relocation of slots/elements embedded in jit-code. */
+    void setSlotsForwardingPointer(HeapSlot *oldSlots, HeapSlot *newSlots, uint32_t nslots);
+    void setElementsForwardingPointer(ObjectElements *oldHeader, ObjectElements *newHeader,
+                                      uint32_t nelems);
+    void forwardBufferPointer(HeapSlot **pSlotsElems);
+    void forwardMovedBuffers(JSRuntime *rt);
+
     /* Handle fallback marking. See the comment in MarkStoreBuffer. */
     void markFallback(gc::Cell *cell);
     void moveFallbackToTenured(gc::MinorCollectionTracer *trc);