Bug 831507 - Add generational-gc post-write barriers to baseline compiler. r=jandem

js/src/gc/Nursery.h

@@ -27,6 +27,8 @@ class MinorCollectionTracer;
 namespace ion {
 class CodeGenerator;
 class MacroAssembler;
+class ICStubCompiler;
+class BaselineCompiler;
 }
 
 class Nursery
@@ -189,6 +191,8 @@ class Nursery
     friend class gc::MinorCollectionTracer;
     friend class ion::CodeGenerator;
     friend class ion::MacroAssembler;
+    friend class ion::ICStubCompiler;
+    friend class ion::BaselineCompiler;
 };
 
 } /* namespace js */

js/src/ion/BaselineCompiler.cpp

@@ -23,6 +23,9 @@ using namespace js::ion;
 BaselineCompiler::BaselineCompiler(JSContext *cx, HandleScript script)
   : BaselineCompilerSpecific(cx, script),
     return_(new HeapLabel())
+#ifdef JSGC_GENERATIONAL
+    , postBarrierSlot_(new HeapLabel())
+#endif
 {
 }
 
@@ -90,6 +93,11 @@ BaselineCompiler::compile()
     if (!emitEpilogue())
         return Method_Error;
 
+#ifdef JSGC_GENERATIONAL
+    if (!emitOutOfLinePostBarrierSlot())
+        return Method_Error;
+#endif
+
     if (masm.oom())
         return Method_Error;
 
@@ -264,6 +272,40 @@ BaselineCompiler::emitEpilogue()
     return true;
 }
 
+#ifdef JSGC_GENERATIONAL
+// On input:
+//  R2.scratchReg() contains object being written to.
+//  R1.scratchReg() contains slot index being written to.
+//  Otherwise, baseline stack will be synced, so all other registers are usable as scratch.
+// This calls:
+//    void PostWriteBarrier(JSRuntime *rt, JSObject *obj);
+bool
+BaselineCompiler::emitOutOfLinePostBarrierSlot()
+{
+    masm.bind(postBarrierSlot_);
+
+    Register objReg = R2.scratchReg();
+    GeneralRegisterSet regs(GeneralRegisterSet::All());
+    regs.take(objReg);
+    regs.take(BaselineFrameReg);
+    Register scratch = regs.takeAny();
+#if defined(JS_CPU_ARM)
+    // On ARM, save the link register before calling.  It contains the return
+    // address.  The |masm.ret()| later will pop this into |pc| to return.
+    masm.push(lr);
+#endif
+
+    masm.setupUnalignedABICall(2, scratch);
+    masm.movePtr(ImmWord(cx->runtime), scratch);
+    masm.passABIArg(scratch);
+    masm.passABIArg(objReg);
+    masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, PostWriteBarrier));
+
+    masm.ret();
+    return true;
+}
+#endif // JSGC_GENERATIONAL
+
 bool
 BaselineCompiler::emitIC(ICStub *stub, bool isForOp)
 {
@@ -1676,23 +1718,36 @@ BaselineCompiler::emit_JSOP_DELPROP()
     return true;
 }
 
-Address
-BaselineCompiler::getScopeCoordinateAddress(Register reg)
+void
+BaselineCompiler::getScopeCoordinateObject(Register reg)
 {
     ScopeCoordinate sc(pc);
 
     masm.loadPtr(frame.addressOfScopeChain(), reg);
     for (unsigned i = sc.hops; i; i--)
         masm.extractObject(Address(reg, ScopeObject::offsetOfEnclosingScope()), reg);
+}
 
+Address
+BaselineCompiler::getScopeCoordinateAddressFromObject(Register objReg, Register reg)
+{
+    ScopeCoordinate sc(pc);
     Shape *shape = ScopeCoordinateToStaticScopeShape(cx, script, pc);
+
     Address addr;
     if (shape->numFixedSlots() <= sc.slot) {
-        masm.loadPtr(Address(reg, JSObject::offsetOfSlots()), reg);
+        masm.loadPtr(Address(objReg, JSObject::offsetOfSlots()), reg);
         return Address(reg, (sc.slot - shape->numFixedSlots()) * sizeof(Value));
     }
 
-    return Address(reg, JSObject::getFixedSlotOffset(sc.slot));
+    return Address(objReg, JSObject::getFixedSlotOffset(sc.slot));
+}
+
+Address
+BaselineCompiler::getScopeCoordinateAddress(Register reg)
+{
+    getScopeCoordinateObject(reg);
+    return getScopeCoordinateAddressFromObject(reg, reg);
 }
 
 bool
@@ -1720,13 +1775,34 @@ BaselineCompiler::emit_JSOP_CALLALIASEDVAR()
 bool
 BaselineCompiler::emit_JSOP_SETALIASEDVAR()
 {
-    // Sync everything except the top value, so that we can use R0 as scratch
-    // (storeValue does not touch it if the top value is in R0).
-    frame.syncStack(1);
+    // Keep rvalue in R0.
+    frame.popRegsAndSync(1);
+    Register objReg = R2.scratchReg();
 
-    Address address = getScopeCoordinateAddress(R2.scratchReg());
+    getScopeCoordinateObject(objReg);
+    Address address = getScopeCoordinateAddressFromObject(objReg, R1.scratchReg());
     masm.patchableCallPreBarrier(address, MIRType_Value);
-    storeValue(frame.peek(-1), address, R0);
+    masm.storeValue(R0, address);
+    frame.push(R0);
+
+#ifdef JSGC_GENERATIONAL
+    // Fully sync the stack if post-barrier is needed.
+    // Scope coordinate object is already in R2.scratchReg().
+    frame.syncStack(0);
+
+    Nursery &nursery = cx->runtime->gcNursery;
+    Label skipBarrier;
+    Label isTenured;
+    masm.branchTestObject(Assembler::NotEqual, R0, &skipBarrier);
+    masm.branchPtr(Assembler::Below, objReg, ImmWord(nursery.start()), &isTenured);
+    masm.branchPtr(Assembler::Below, objReg, ImmWord(nursery.end()), &skipBarrier);
+
+    masm.bind(&isTenured);
+    masm.call(postBarrierSlot_);
+
+    masm.bind(&skipBarrier);
+#endif
+
     return true;
 }
 

js/src/ion/BaselineCompiler.h

@@ -171,6 +171,9 @@ class BaselineCompiler : public BaselineCompilerSpecific
 {
     FixedList<Label>            labels_;
     HeapLabel *                 return_;
+#ifdef JSGC_GENERATIONAL
+    HeapLabel *                 postBarrierSlot_;
+#endif
 
     // Native code offset right before the scope chain is initialized.
     CodeOffsetLabel prologueOffset_;
@@ -190,6 +193,9 @@ class BaselineCompiler : public BaselineCompilerSpecific
 
     bool emitPrologue();
     bool emitEpilogue();
+#ifdef JSGC_GENERATIONAL
+    bool emitOutOfLinePostBarrierSlot();
+#endif
     bool emitIC(ICStub *stub, bool isForOp);
     bool emitOpIC(ICStub *stub) {
         return emitIC(stub, true);
@@ -238,6 +244,8 @@ class BaselineCompiler : public BaselineCompilerSpecific
 
     bool addPCMappingEntry(bool addIndexEntry);
 
+    void getScopeCoordinateObject(Register reg);
+    Address getScopeCoordinateAddressFromObject(Register objReg, Register reg);
     Address getScopeCoordinateAddress(Register reg);
 };
 

js/src/ion/BaselineHelpers.h

@@ -11,8 +11,10 @@
 # include "x86/BaselineHelpers-x86.h"
 #elif defined(JS_CPU_X64)
 # include "x64/BaselineHelpers-x64.h"
-#else
+#elif defined(JS_CPU_ARM)
 # include "arm/BaselineHelpers-arm.h"
+#else
+# error "Unknown architecture!"
 #endif
 
 namespace js {

js/src/ion/BaselineIC.cpp

@@ -638,6 +638,36 @@ ICStubCompiler::guardProfilingEnabled(MacroAssembler &masm, Register scratch, La
     masm.branch32(Assembler::Equal, AbsoluteAddress(enabledAddr), Imm32(0), skip);
 }
 
+#ifdef JSGC_GENERATIONAL
+inline bool
+ICStubCompiler::emitPostWriteBarrierSlot(MacroAssembler &masm, Register obj, Register scratch,
+                                         GeneralRegisterSet saveRegs)
+{
+    Nursery &nursery = cx->runtime->gcNursery;
+
+    Label skipBarrier;
+    Label isTenured;
+    masm.branchPtr(Assembler::Below, obj, ImmWord(nursery.start()), &isTenured);
+    masm.branchPtr(Assembler::Below, obj, ImmWord(nursery.end()), &skipBarrier);
+    masm.bind(&isTenured);
+
+    // void PostWriteBarrier(JSRuntime *rt, JSObject *obj);
+#ifdef JS_CPU_ARM
+    saveRegs.add(BaselineTailCallReg);
+#endif
+    masm.PushRegsInMask(saveRegs);
+    masm.setupUnalignedABICall(2, scratch);
+    masm.movePtr(ImmWord(cx->runtime), scratch);
+    masm.passABIArg(scratch);
+    masm.passABIArg(obj);
+    masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, PostWriteBarrier));
+    masm.PopRegsInMask(saveRegs);
+
+    masm.bind(&skipBarrier);
+    return true;
+}
+#endif // JSGC_GENERATIONAL
+
 //
 // UseCount_Fallback
 //
@@ -4324,12 +4354,13 @@ ICSetElem_Dense::Compiler::generateStubCode(MacroAssembler &masm)
     regs = availableGeneralRegs(2);
     scratchReg = regs.takeAny();
 
+    // Unbox object and key.
+    obj = masm.extractObject(R0, ExtractTemp0);
+    Register key = masm.extractInt32(R1, ExtractTemp1);
+
     // Load obj->elements in scratchReg.
     masm.loadPtr(Address(obj, JSObject::offsetOfElements()), scratchReg);
 
-    // Unbox key.
-    Register key = masm.extractInt32(R1, ExtractTemp1);
-
     // Bounds check.
     Address initLength(scratchReg, ObjectElements::offsetOfInitializedLength());
     masm.branch32(Assembler::BelowOrEqual, initLength, key, &failure);
@@ -4338,6 +4369,12 @@ ICSetElem_Dense::Compiler::generateStubCode(MacroAssembler &masm)
     BaseIndex element(scratchReg, key, TimesEight);
     masm.branchTestMagic(Assembler::Equal, element, &failure);
 
+    // Failure is not possible now.  Free up registers.
+    regs.add(R0);
+    regs.add(R1);
+    regs.takeUnchecked(obj);
+    regs.takeUnchecked(key);
+
     // Convert int32 values to double if convertDoubleElements is set. In this
     // case the heap typeset is guaranteed to contain both int32 and double, so
     // it's okay to store a double.
@@ -4347,12 +4384,28 @@ ICSetElem_Dense::Compiler::generateStubCode(MacroAssembler &masm)
                       Imm32(ObjectElements::CONVERT_DOUBLE_ELEMENTS),
                       &convertDoubles);
     masm.bind(&convertDoublesDone);
+    // No longer need key.
 
-    // It's safe to overwrite R0 now.
+    // Don't overwrite R0 becuase |obj| might overlap with it, and it's needed
+    // for post-write barrier later.
     Address valueAddr(BaselineStackReg, ICStackValueOffset);
-    masm.loadValue(valueAddr, R0);
+    ValueOperand tmpVal = regs.takeAnyValue();
+    masm.loadValue(valueAddr, tmpVal);
     EmitPreBarrier(masm, element, MIRType_Value);
-    masm.storeValue(R0, element);
+    masm.storeValue(tmpVal, element);
+#ifdef JSGC_GENERATIONAL
+    Label skipBarrier;
+    masm.branchTestObject(Assembler::NotEqual, tmpVal, &skipBarrier);
+    regs.add(key);
+    regs.add(tmpVal);
+    {
+        Register r = regs.takeAny();
+        GeneralRegisterSet saveRegs;
+        emitPostWriteBarrierSlot(masm, obj, r, saveRegs);
+        regs.add(r);
+    }
+    masm.bind(&skipBarrier);
+#endif
     EmitReturnFromIC(masm);
 
     // Convert to double and jump back. Note that double arrays are only
@@ -4360,7 +4413,7 @@ ICSetElem_Dense::Compiler::generateStubCode(MacroAssembler &masm)
     // Ion is disabled and there should be no double arrays.
     masm.bind(&convertDoubles);
     if (cx->runtime->jitSupportsFloatingPoint)
-        masm.convertInt32ValueToDouble(valueAddr, R0.scratchReg(), &convertDoublesDone);
+        masm.convertInt32ValueToDouble(valueAddr, regs.getAny(), &convertDoublesDone);
     else
         masm.breakpoint();
     masm.jump(&convertDoublesDone);
@@ -4481,12 +4534,13 @@ ICSetElemDenseAddCompiler::generateStubCode(MacroAssembler &masm)
     regs = availableGeneralRegs(2);
     scratchReg = regs.takeAny();
 
+    // Unbox obj and key.
+    obj = masm.extractObject(R0, ExtractTemp0);
+    Register key = masm.extractInt32(R1, ExtractTemp1);
+
     // Load obj->elements in scratchReg.
     masm.loadPtr(Address(obj, JSObject::offsetOfElements()), scratchReg);
 
-    // Unbox key.
-    Register key = masm.extractInt32(R1, ExtractTemp1);
-
     // Bounds check (key == initLength)
     Address initLength(scratchReg, ObjectElements::offsetOfInitializedLength());
     masm.branch32(Assembler::NotEqual, initLength, key, &failure);
@@ -4495,6 +4549,12 @@ ICSetElemDenseAddCompiler::generateStubCode(MacroAssembler &masm)
     Address capacity(scratchReg, ObjectElements::offsetOfCapacity());
     masm.branch32(Assembler::BelowOrEqual, capacity, key, &failure);
 
+    // Failure is not possible now.  Free up registers.
+    regs.add(R0);
+    regs.add(R1);
+    regs.takeUnchecked(obj);
+    regs.takeUnchecked(key);
+
     // Increment initLength before write.
     masm.add32(Imm32(1), initLength);
 
@@ -4515,12 +4575,26 @@ ICSetElemDenseAddCompiler::generateStubCode(MacroAssembler &masm)
                       &convertDoubles);
     masm.bind(&convertDoublesDone);
 
-    // Write the value.  No need for write barrier since we're not overwriting an old value.
-    // It's safe to overwrite R0 now.
-    BaseIndex element(scratchReg, key, TimesEight);
     Address valueAddr(BaselineStackReg, ICStackValueOffset);
-    masm.loadValue(valueAddr, R0);
-    masm.storeValue(R0, element);
+
+    // Write the value.  No need for pre-barrier since we're not overwriting an old value.
+    ValueOperand tmpVal = regs.takeAnyValue();
+    BaseIndex element(scratchReg, key, TimesEight);
+    masm.loadValue(valueAddr, tmpVal);
+    masm.storeValue(tmpVal, element);
+#ifdef JSGC_GENERATIONAL
+    Label skipBarrier;
+    masm.branchTestObject(Assembler::NotEqual, tmpVal, &skipBarrier);
+    regs.add(key);
+    regs.add(tmpVal);
+    {
+        Register r = regs.takeAny();
+        GeneralRegisterSet saveRegs;
+        emitPostWriteBarrierSlot(masm, obj, r, saveRegs);
+        regs.add(r);
+    }
+    masm.bind(&skipBarrier);
+#endif
     EmitReturnFromIC(masm);
 
     // Convert to double and jump back. Note that double arrays are only
@@ -4528,7 +4602,7 @@ ICSetElemDenseAddCompiler::generateStubCode(MacroAssembler &masm)
     // Ion is disabled and there should be no double arrays.
     masm.bind(&convertDoubles);
     if (cx->runtime->jitSupportsFloatingPoint)
-        masm.convertInt32ValueToDouble(valueAddr, R0.scratchReg(), &convertDoublesDone);
+        masm.convertInt32ValueToDouble(valueAddr, regs.getAny(), &convertDoublesDone);
     else
         masm.breakpoint();
     masm.jump(&convertDoublesDone);
@@ -6299,13 +6373,35 @@ ICSetProp_Native::Compiler::generateStubCode(MacroAssembler &masm)
     // Unstow R0 and R1 (object and key)
     EmitUnstowICValues(masm, 2);
 
-    if (!isFixedSlot_)
-        masm.loadPtr(Address(objReg, JSObject::offsetOfSlots()), objReg);
+    regs.add(R0);
+    regs.takeUnchecked(objReg);
+
+    Register holderReg;
+    if (isFixedSlot_) {
+        holderReg = objReg;
+    } else {
+        holderReg = regs.takeAny();
+        masm.loadPtr(Address(objReg, JSObject::offsetOfSlots()), holderReg);
+    }
 
     // Perform the store.
     masm.load32(Address(BaselineStubReg, ICSetProp_Native::offsetOfOffset()), scratch);
-    EmitPreBarrier(masm, BaseIndex(objReg, scratch, TimesOne), MIRType_Value);
-    masm.storeValue(R1, BaseIndex(objReg, scratch, TimesOne));
+    EmitPreBarrier(masm, BaseIndex(holderReg, scratch, TimesOne), MIRType_Value);
+    masm.storeValue(R1, BaseIndex(holderReg, scratch, TimesOne));
+    if (holderReg != objReg)
+        regs.add(holderReg);
+#ifdef JSGC_GENERATIONAL
+    Label skipBarrier;
+    masm.branchTestObject(Assembler::NotEqual, R1, &skipBarrier);
+    {
+        Register scr = regs.takeAny();
+        GeneralRegisterSet saveRegs;
+        saveRegs.add(R1);
+        emitPostWriteBarrierSlot(masm, objReg, scr, saveRegs);
+        regs.add(scr);
+    }
+    masm.bind(&skipBarrier);
+#endif
 
     // The RHS has to be in R0.
     masm.moveValue(R1, R0);
@@ -6400,13 +6496,35 @@ ICSetPropNativeAddCompiler::generateStubCode(MacroAssembler &masm)
     masm.loadPtr(Address(BaselineStubReg, ICSetProp_NativeAdd::offsetOfNewShape()), scratch);
     masm.storePtr(scratch, shapeAddr);
 
-    if (!isFixedSlot_)
-        masm.loadPtr(Address(objReg, JSObject::offsetOfSlots()), objReg);
+    Register holderReg;
+    regs.add(R0);
+    regs.takeUnchecked(objReg);
+    if (isFixedSlot_) {
+        holderReg = objReg;
+    } else {
+        holderReg = regs.takeAny();
+        masm.loadPtr(Address(objReg, JSObject::offsetOfSlots()), holderReg);
+    }
 
     // Perform the store.  No write barrier required since this is a new
     // initialization.
     masm.load32(Address(BaselineStubReg, ICSetProp_NativeAdd::offsetOfOffset()), scratch);
-    masm.storeValue(R1, BaseIndex(objReg, scratch, TimesOne));
+    masm.storeValue(R1, BaseIndex(holderReg, scratch, TimesOne));
+
+    if (holderReg != objReg)
+        regs.add(holderReg);
+
+#ifdef JSGC_GENERATIONAL
+    Label skipBarrier;
+    masm.branchTestObject(Assembler::NotEqual, R1, &skipBarrier);
+    {
+        Register scr = regs.takeAny();
+        GeneralRegisterSet saveRegs;
+        saveRegs.add(R1);
+        emitPostWriteBarrierSlot(masm, objReg, scr, saveRegs);
+    }
+    masm.bind(&skipBarrier);
+#endif
 
     // The RHS has to be in R0.
     masm.moveValue(R1, R0);
@@ -8219,7 +8337,8 @@ ICSetProp_Native::Compiler::getStub(ICStubSpace *space)
 
 ICSetProp_NativeAdd::ICSetProp_NativeAdd(IonCode *stubCode, HandleTypeObject type,
                                          size_t protoChainDepth,
-                                         HandleShape newShape, uint32_t offset)
+                                         HandleShape newShape,
+                                         uint32_t offset)
   : ICUpdatedStub(SetProp_NativeAdd, stubCode),
     type_(type),
     newShape_(newShape),
@@ -8244,7 +8363,8 @@ ICSetProp_NativeAddImpl<ProtoChainDepth>::ICSetProp_NativeAddImpl(IonCode *stubC
 
 ICSetPropNativeAddCompiler::ICSetPropNativeAddCompiler(JSContext *cx, HandleObject obj,
                                                        HandleShape oldShape,
-                                                       size_t protoChainDepth, bool isFixedSlot,
+                                                       size_t protoChainDepth,
+                                                       bool isFixedSlot,
                                                        uint32_t offset)
   : ICStubCompiler(cx, ICStub::SetProp_NativeAdd),
     obj_(cx, obj),

js/src/ion/BaselineIC.h

@@ -744,6 +744,19 @@ class ICStub
         }
     }
 
+    static bool NeedsPostBarrier(ICStub::Kind kind) {
+        JS_ASSERT(IsValidKind(kind));
+        switch (kind) {
+          case SetProp_Native:
+          case SetProp_NativeAdd:
+          case SetElem_Dense:
+          case SetElem_DenseAdd:
+            return true;
+          default:
+            return false;
+        }
+    }
+
     // Optimized stubs get purged on GC.  But some stubs can be active on the
     // stack during GC - specifically the ones that can make calls.  To ensure
     // that these do not get purged, all stubs that can make calls are allocated
@@ -1046,6 +1059,11 @@ class ICStubCompiler
         return regs;
     }
 
+#ifdef JSGC_GENERATIONAL
+    inline bool emitPostWriteBarrierSlot(MacroAssembler &masm, Register obj, Register scratch,
+                                         GeneralRegisterSet saveRegs);
+#endif
+
   public:
     virtual ICStub *getStub(ICStubSpace *space) = 0;
 
@@ -4552,7 +4570,7 @@ class ICSetProp_Native : public ICUpdatedStub
     }
 
     class Compiler : public ICStubCompiler {
-        HandleObject obj_;
+        RootedObject obj_;
         bool isFixedSlot_;
         uint32_t offset_;
 
@@ -4566,7 +4584,7 @@ class ICSetProp_Native : public ICUpdatedStub
       public:
         Compiler(JSContext *cx, HandleObject obj, bool isFixedSlot, uint32_t offset)
           : ICStubCompiler(cx, ICStub::SetProp_Native),
-            obj_(obj),
+            obj_(cx, obj),
             isFixedSlot_(isFixedSlot),
             offset_(offset)
         {}