mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 903519 - Strings in the nursery: JIT, r=jandem 

--HG--
extra : rebase_source : 3e2c013d3085fa87f930bbf620c1cb5b46f8952e
extra : histedit_source : a07cbe1740f88d26a9a696cdcb5007181cc92ee8
This commit is contained in:
Steve Fink 2018-02-05 16:22:22 -08:00
Родитель ae2642fca6
Коммит 0742a54280
34 изменённых файлов: 723 добавлений и 173 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
js/public/HeapAPI.h
Просмотреть файл

@ -67,7 +67,7 @@ const size_t ChunkMarkBitmapBits = 129024;
const size_t ChunkRuntimeOffset = ChunkSize - sizeof(void*);
const size_t ChunkTrailerSize = 2 * sizeof(uintptr_t) + sizeof(uint64_t);
const size_t ChunkLocationOffset = ChunkSize - ChunkTrailerSize;
const size_t ChunkStoreBufferOffset = ChunkLocationOffset + sizeof(uint64_t);
const size_t ChunkStoreBufferOffset = ChunkSize - ChunkTrailerSize + sizeof(uint64_t);
const size_t ArenaZoneOffset = sizeof(size_t);
const size_t ArenaHeaderSize = sizeof(size_t) + 2 * sizeof(uintptr_t) +
sizeof(size_t) + sizeof(uintptr_t);

2
js/src/gc/Heap.h
Просмотреть файл

@ -887,6 +887,8 @@ InFreeList(Arena* arena, void* thing)
static const int32_t ChunkLocationOffsetFromLastByte =
int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask);
static const int32_t ChunkStoreBufferOffsetFromLastByte =
int32_t(gc::ChunkStoreBufferOffset) - int32_t(gc::ChunkMask);
} /* namespace gc */

12
js/src/gc/Marking.cpp
Просмотреть файл

@ -797,6 +797,18 @@ ShouldMark<JSObject*>(GCMarker* gcmarker, JSObject* obj)
return obj->asTenured().zone()->shouldMarkInZone();
}
// JSStrings can also be in the nursery. See ShouldMark<JSObject*> for comments.
template <>
bool
ShouldMark<JSString*>(GCMarker* gcmarker, JSString* str)
{
if (IsOwnedByOtherRuntime(gcmarker->runtime(), str))
return false;
if (IsInsideNursery(str))
return false;
return str->asTenured().zone()->shouldMarkInZone();
}
template <typename T>
void
DoMarking(GCMarker* gcmarker, T* thing)

3
js/src/gc/Nursery.h
Просмотреть файл

@ -508,9 +508,6 @@ class Nursery
JSRuntime* runtime() const { return runtime_; }
/* Allocates a new GC thing from the tenured generation during minor GC. */
gc::TenuredCell* allocateFromTenured(JS::Zone* zone, gc::AllocKind thingKind);
/* Common internal allocator function. */
void* allocate(size_t size);

3
js/src/jit/BaselineCacheIRCompiler.cpp
Просмотреть файл

@ -1303,9 +1303,8 @@ BaselineCacheIRCompiler::emitStoreTypedObjectReferenceProperty()
Address dest(scratch1, 0);
emitStoreTypedObjectReferenceProp(val, type, dest, scratch2);
emitPostBarrierSlot(obj, val, scratch1);
if (type != ReferenceTypeDescr::TYPE_STRING)
emitPostBarrierSlot(obj, val, scratch1);
return true;
}

4
js/src/jit/BaselineCompiler.cpp
Просмотреть файл

@ -2790,7 +2790,7 @@ BaselineCompiler::emit_JSOP_SETALIASEDVAR()
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, objReg, temp, &skipBarrier);
masm.branchValueIsNurseryObject(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.call(&postBarrierSlot_); // Won't clobber R0
@ -3214,7 +3214,7 @@ BaselineCompiler::emitFormalArgAccess(uint32_t arg, bool get)
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &skipBarrier);
masm.branchValueIsNurseryObject(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.call(&postBarrierSlot_);

13
js/src/jit/CacheIRCompiler.cpp
Просмотреть файл

@ -2483,6 +2483,8 @@ void
CacheIRCompiler::emitStoreTypedObjectReferenceProp(ValueOperand val, ReferenceTypeDescr::Type type,
const Address& dest, Register scratch)
{
// Callers will post-barrier this store.
switch (type) {
case ReferenceTypeDescr::TYPE_ANY:
EmitPreBarrier(masm, dest, MIRType::Value);
@ -2535,18 +2537,19 @@ CacheIRCompiler::emitPostBarrierShared(Register obj, const ConstantOrRegister& v
return;
if (val.constant()) {
MOZ_ASSERT_IF(val.value().isObject(), !IsInsideNursery(&val.value().toObject()));
MOZ_ASSERT_IF(val.value().isGCThing(), !IsInsideNursery(val.value().toGCThing()));
return;
}
TypedOrValueRegister reg = val.reg();
if (reg.hasTyped() && reg.type() != MIRType::Object)
return;
if (reg.hasTyped()) {
if (reg.type() != MIRType::Object && reg.type() != MIRType::String)
return;
}
Label skipBarrier;
if (reg.hasValue()) {
masm.branchValueIsNurseryObject(Assembler::NotEqual, reg.valueReg(), scratch,
&skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, reg.valueReg(), scratch, &skipBarrier);
} else {
masm.branchPtrInNurseryChunk(Assembler::NotEqual, reg.typedReg().gpr(), scratch,
&skipBarrier);

241
js/src/jit/CodeGenerator.cpp
Просмотреть файл

@ -1222,6 +1222,102 @@ CodeGenerator::visitValueToObjectOrNull(LValueToObjectOrNull* lir)
masm.bind(ool->rejoin());
}
enum class FieldToBarrier {
REGEXP_PENDING_INPUT,
REGEXP_MATCHES_INPUT,
DEPENDENT_STRING_BASE
};
static void
EmitStoreBufferMutation(MacroAssembler& masm, Register holder, FieldToBarrier field,
Register buffer,
LiveGeneralRegisterSet& liveVolatiles,
void (*fun)(js::gc::StoreBuffer*, js::gc::Cell**))
{
Label callVM;
Label exit;
// Call into the VM to barrier the write. The only registers that need to
// be preserved are those in liveVolatiles, so once they are saved on the
// stack all volatile registers are available for use.
masm.bind(&callVM);
masm.PushRegsInMask(liveVolatiles);
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
regs.takeUnchecked(buffer);
regs.takeUnchecked(holder);
Register addrReg = regs.takeAny();
switch (field) {
case FieldToBarrier::REGEXP_PENDING_INPUT:
masm.computeEffectiveAddress(Address(holder, RegExpStatics::offsetOfPendingInput()), addrReg);
break;
case FieldToBarrier::REGEXP_MATCHES_INPUT:
masm.computeEffectiveAddress(Address(holder, RegExpStatics::offsetOfMatchesInput()), addrReg);
break;
case FieldToBarrier::DEPENDENT_STRING_BASE:
masm.leaNewDependentStringBase(holder, addrReg);
break;
}
bool needExtraReg = !regs.hasAny<GeneralRegisterSet::DefaultType>();
if (needExtraReg) {
masm.push(holder);
masm.setupUnalignedABICall(holder);
} else {
masm.setupUnalignedABICall(regs.takeAny());
}
masm.passABIArg(buffer);
masm.passABIArg(addrReg);
masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, fun), MoveOp::GENERAL,
CheckUnsafeCallWithABI::DontCheckOther);
if (needExtraReg)
masm.pop(holder);
masm.PopRegsInMask(liveVolatiles);
masm.bind(&exit);
}
// Warning: this function modifies prev and next.
static void
EmitPostWriteBarrierS(MacroAssembler& masm,
Register string, FieldToBarrier field,
Register prev, Register next,
LiveGeneralRegisterSet& liveVolatiles)
{
Label exit;
Label checkRemove, putCell;
// if (next && (buffer = next->storeBuffer()))
// but we never pass in nullptr for next.
Register storebuffer = next;
masm.loadStoreBuffer(next, storebuffer);
masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &checkRemove);
// if (prev && prev->storeBuffer())
masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &putCell);
masm.loadStoreBuffer(prev, prev);
masm.branchPtr(Assembler::NotEqual, prev, ImmWord(0), &exit);
// buffer->putCell(cellp)
masm.bind(&putCell);
EmitStoreBufferMutation(masm, string, field, storebuffer, liveVolatiles,
JSString::addCellAddressToStoreBuffer);
masm.jump(&exit);
// if (prev && (buffer = prev->storeBuffer()))
masm.bind(&checkRemove);
masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &exit);
masm.loadStoreBuffer(prev, storebuffer);
masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &exit);
EmitStoreBufferMutation(masm, string, field, storebuffer, liveVolatiles,
JSString::removeCellAddressFromStoreBuffer);
masm.bind(&exit);
}
typedef JSObject* (*CloneRegExpObjectFn)(JSContext*, Handle<RegExpObject*>);
static const VMFunction CloneRegExpObjectInfo =
FunctionInfo<CloneRegExpObjectFn>(CloneRegExpObject, "CloneRegExpObject");
@ -1466,8 +1562,22 @@ PrepareAndExecuteRegExp(JSContext* cx, MacroAssembler& masm, Register regexp, Re
masm.guardedCallPreBarrier(matchesInputAddress, MIRType::String);
masm.guardedCallPreBarrier(lazySourceAddress, MIRType::String);
if (temp1.volatile_())
volatileRegs.add(temp1);
// Writing into RegExpStatics tenured memory; must post-barrier.
masm.loadPtr(pendingInputAddress, temp2);
masm.storePtr(input, pendingInputAddress);
masm.movePtr(input, temp3);
EmitPostWriteBarrierS(masm, temp1, FieldToBarrier::REGEXP_PENDING_INPUT,
temp2 /* prev */, temp3 /* next */, volatileRegs);
masm.loadPtr(matchesInputAddress, temp2);
masm.storePtr(input, matchesInputAddress);
masm.movePtr(input, temp3);
EmitPostWriteBarrierS(masm, temp1, FieldToBarrier::REGEXP_MATCHES_INPUT,
temp2 /* prev */, temp3 /* next */, volatileRegs);
masm.storePtr(lastIndex, Address(temp1, RegExpStatics::offsetOfLazyIndex()));
masm.store32(Imm32(1), Address(temp1, RegExpStatics::offsetOfPendingLazyEvaluation()));
@ -1510,6 +1620,7 @@ public:
bool latin1, Register string,
Register base, Register temp1, Register temp2,
BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
bool stringsCanBeInNursery,
Label* failure);
// Generate fallback path for creating DependentString.
@ -1521,6 +1632,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
bool latin1, Register string,
Register base, Register temp1, Register temp2,
BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
bool stringsCanBeInNursery,
Label* failure)
{
string_ = string;
@ -1558,7 +1670,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
masm.branch32(Assembler::Above, temp1, Imm32(maxThinInlineLength), &fatInline);
int32_t thinFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_THIN_INLINE_FLAGS;
masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString]);
masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString], stringsCanBeInNursery);
masm.bind(&joins_[FallbackKind::InlineString]);
masm.store32(Imm32(thinFlags), Address(string, JSString::offsetOfFlags()));
masm.jump(&stringAllocated);
@ -1566,7 +1678,7 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
masm.bind(&fatInline);
int32_t fatFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_FAT_INLINE_FLAGS;
masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString]);
masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString], stringsCanBeInNursery);
masm.bind(&joins_[FallbackKind::FatInlineString]);
masm.store32(Imm32(fatFlags), Address(string, JSString::offsetOfFlags()));
@ -1612,7 +1724,9 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
// Make a dependent string.
int32_t flags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::DEPENDENT_FLAGS;
masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString]);
masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString], stringsCanBeInNursery);
// Warning: string may be tenured (if the fallback case is hit), so
// stores into it must be post barriered.
masm.bind(&joins_[FallbackKind::NotInlineString]);
masm.store32(Imm32(flags), Address(string, JSString::offsetOfFlags()));
masm.store32(temp1, Address(string, JSString::offsetOfLength()));
@ -1626,17 +1740,29 @@ CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
masm.computeEffectiveAddress(BaseIndex(temp1, temp2, TimesTwo), temp1);
masm.storeNonInlineStringChars(temp1, string);
masm.storeDependentStringBase(base, string);
masm.movePtr(base, temp1);
// Follow any base pointer if the input is itself a dependent string.
// Watch for undepended strings, which have a base pointer but don't
// actually share their characters with it.
Label noBase;
masm.load32(Address(base, JSString::offsetOfFlags()), temp1);
masm.and32(Imm32(JSString::TYPE_FLAGS_MASK), temp1);
masm.branch32(Assembler::NotEqual, temp1, Imm32(JSString::DEPENDENT_FLAGS), &noBase);
masm.load32(Address(base, JSString::offsetOfFlags()), temp2);
masm.and32(Imm32(JSString::TYPE_FLAGS_MASK), temp2);
masm.branch32(Assembler::NotEqual, temp2, Imm32(JSString::DEPENDENT_FLAGS), &noBase);
masm.loadDependentStringBase(base, temp1);
masm.storeDependentStringBase(temp1, string);
masm.bind(&noBase);
// Post-barrier the base store, whether it was the direct or indirect
// base (both will end up in temp1 here).
masm.movePtr(ImmWord(0), temp2);
LiveGeneralRegisterSet saveRegs(GeneralRegisterSet::Volatile());
if (temp1.volatile_())
saveRegs.takeUnchecked(temp1);
if (temp2.volatile_())
saveRegs.takeUnchecked(temp2);
EmitPostWriteBarrierS(masm, string, FieldToBarrier::DEPENDENT_STRING_BASE,
temp2 /* prev */, temp1 /* next */, saveRegs);
}
masm.bind(&done);
@ -1859,10 +1985,12 @@ JitCompartment::generateRegExpMatcherStub(JSContext* cx)
masm.branch32(Assembler::LessThan, stringIndexAddress, Imm32(0), &isUndefined);
depStr[isLatin].generate(masm, cx->names(), isLatin, temp3, input, temp4, temp5,
stringIndexAddress, stringLimitAddress, failure);
stringIndexAddress, stringLimitAddress,
stringsCanBeInNursery,
failure);
masm.storeValue(JSVAL_TYPE_STRING, temp3, stringAddress);
// Storing into nursery-allocated results object's elements; no post barrier.
masm.jump(&storeDone);
masm.bind(&isUndefined);
@ -1907,7 +2035,9 @@ JitCompartment::generateRegExpMatcherStub(JSContext* cx)
masm.load32(pairsVectorAddress, temp3);
masm.storeValue(JSVAL_TYPE_INT32, temp3, Address(temp2, 0));
masm.storeValue(JSVAL_TYPE_STRING, input, Address(temp2, sizeof(Value)));
Address inputSlotAddress(temp2, sizeof(Value));
masm.storeValue(JSVAL_TYPE_STRING, input, inputSlotAddress);
// No post barrier needed (inputSlotAddress is within nursery object.)
// All done!
masm.tagValue(JSVAL_TYPE_OBJECT, object, result);
@ -2829,6 +2959,8 @@ CodeGenerator::emitLambdaInit(Register output, Register envChain,
masm.storePtr(ImmGCPtr(info.scriptOrLazyScript),
Address(output, JSFunction::offsetOfScriptOrLazyScript()));
masm.storePtr(envChain, Address(output, JSFunction::offsetOfEnvironment()));
// No post barrier needed because output is guaranteed to be allocated in
// the nursery.
masm.storePtr(ImmGCPtr(info.fun->displayAtom()), Address(output, JSFunction::offsetOfAtom()));
}
@ -3766,12 +3898,11 @@ class OutOfLineCallPostWriteBarrier : public OutOfLineCodeBase<CodeGenerator>
};
static void
EmitStoreBufferCheckForConstant(MacroAssembler& masm, JSObject* object,
EmitStoreBufferCheckForConstant(MacroAssembler& masm, const gc::TenuredCell* cell,
AllocatableGeneralRegisterSet& regs, Label* exit, Label* callVM)
{
Register temp = regs.takeAny();
const gc::TenuredCell* cell = &object->asTenured();
gc::Arena* arena = cell->arena();
Register cells = temp;
@ -3809,7 +3940,7 @@ EmitPostWriteBarrier(MacroAssembler& masm, CompileRuntime* runtime, Register obj
// We already have a fast path to check whether a global is in the store
// buffer.
if (!isGlobal && maybeConstant)
EmitStoreBufferCheckForConstant(masm, maybeConstant, regs, &exit, &callVM);
EmitStoreBufferCheckForConstant(masm, &maybeConstant->asTenured(), regs, &exit, &callVM);
// Call into the VM to barrier the write.
masm.bind(&callVM);
@ -3884,16 +4015,17 @@ CodeGenerator::maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal, OutOf
masm.branch32(Assembler::NotEqual, addr, Imm32(0), ool->rejoin());
}
template <class LPostBarrierType>
template <class LPostBarrierType, MIRType nurseryType>
void
CodeGenerator::visitPostWriteBarrierCommonO(LPostBarrierType* lir, OutOfLineCode* ool)
CodeGenerator::visitPostWriteBarrierCommon(LPostBarrierType* lir, OutOfLineCode* ool)
{
addOutOfLineCode(ool, lir->mir());
Register temp = ToTempRegisterOrInvalid(lir->temp());
if (lir->object()->isConstant()) {
// Constant nursery objects cannot appear here, see LIRGenerator::visitPostWriteElementBarrier.
// Constant nursery objects cannot appear here, see
// LIRGenerator::visitPostWriteElementBarrier.
MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()));
} else {
masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()), temp,
@ -3902,12 +4034,17 @@ CodeGenerator::visitPostWriteBarrierCommonO(LPostBarrierType* lir, OutOfLineCode
maybeEmitGlobalBarrierCheck(lir->object(), ool);
Register valueObj = ToRegister(lir->value());
if (lir->mir()->value()->type() == MIRType::ObjectOrNull)
masm.branchTestPtr(Assembler::Zero, valueObj, valueObj, ool->rejoin());
else
MOZ_ASSERT(lir->mir()->value()->type() == MIRType::Object);
masm.branchPtrInNurseryChunk(Assembler::Equal, valueObj, temp, ool->entry());
Register value = ToRegister(lir->value());
if (nurseryType == MIRType::Object) {
if (lir->mir()->value()->type() == MIRType::ObjectOrNull)
masm.branchTestPtr(Assembler::Zero, value, value, ool->rejoin());
else
MOZ_ASSERT(lir->mir()->value()->type() == MIRType::Object);
} else {
MOZ_ASSERT(nurseryType == MIRType::String);
MOZ_ASSERT(lir->mir()->value()->type() == MIRType::String);
}
masm.branchPtrInNurseryChunk(Assembler::Equal, value, temp, ool->entry());
masm.bind(ool->rejoin());
}
@ -3931,7 +4068,9 @@ CodeGenerator::visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode
maybeEmitGlobalBarrierCheck(lir->object(), ool);
ValueOperand value = ToValue(lir, LPostBarrierType::Input);
masm.branchValueIsNurseryObject(Assembler::Equal, value, temp, ool->entry());
// Bug 1386094 - most callers only need to check for object or string, not
// both.
masm.branchValueIsNurseryCell(Assembler::Equal, value, temp, ool->entry());
masm.bind(ool->rejoin());
}
@ -3940,7 +4079,14 @@ void
CodeGenerator::visitPostWriteBarrierO(LPostWriteBarrierO* lir)
{
auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
visitPostWriteBarrierCommonO(lir, ool);
visitPostWriteBarrierCommon<LPostWriteBarrierO, MIRType::Object>(lir, ool);
}
void
CodeGenerator::visitPostWriteBarrierS(LPostWriteBarrierS* lir)
{
auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
visitPostWriteBarrierCommon<LPostWriteBarrierS, MIRType::String>(lir, ool);
}
void
@ -4020,7 +4166,14 @@ void
CodeGenerator::visitPostWriteElementBarrierO(LPostWriteElementBarrierO* lir)
{
auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
visitPostWriteBarrierCommonO(lir, ool);
visitPostWriteBarrierCommon<LPostWriteElementBarrierO, MIRType::Object>(lir, ool);
}
void
CodeGenerator::visitPostWriteElementBarrierS(LPostWriteElementBarrierS* lir)
{
auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
visitPostWriteBarrierCommon<LPostWriteElementBarrierS, MIRType::String>(lir, ool);
}
void
@ -6757,20 +6910,16 @@ CodeGenerator::emitLoadIteratorValues<ValueMap>(Register result, Register temp,
masm.storeValue(keyAddress, keyElemAddress, temp);
masm.storeValue(valueAddress, valueElemAddress, temp);
Label keyIsNotObject, valueIsNotNurseryObject, emitBarrier;
masm.branchTestObject(Assembler::NotEqual, keyAddress, &keyIsNotObject);
masm.branchValueIsNurseryObject(Assembler::Equal, keyAddress, temp, &emitBarrier);
masm.bind(&keyIsNotObject);
masm.branchTestObject(Assembler::NotEqual, valueAddress, &valueIsNotNurseryObject);
masm.branchValueIsNurseryObject(Assembler::NotEqual, valueAddress, temp,
&valueIsNotNurseryObject);
Label emitBarrier, skipBarrier;
masm.branchValueIsNurseryCell(Assembler::Equal, keyAddress, temp, &emitBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, valueAddress, temp, &skipBarrier);
{
masm.bind(&emitBarrier);
saveVolatile(temp);
emitPostWriteBarrier(result);
restoreVolatile(temp);
}
masm.bind(&valueIsNotNurseryObject);
masm.bind(&skipBarrier);
}
template <>
@ -6784,15 +6933,14 @@ CodeGenerator::emitLoadIteratorValues<ValueSet>(Register result, Register temp,
masm.guardedCallPreBarrier(keyElemAddress, MIRType::Value);
masm.storeValue(keyAddress, keyElemAddress, temp);
Label keyIsNotObject;
masm.branchTestObject(Assembler::NotEqual, keyAddress, &keyIsNotObject);
masm.branchValueIsNurseryObject(Assembler::NotEqual, keyAddress, temp, &keyIsNotObject);
Label skipBarrier;
masm.branchValueIsNurseryCell(Assembler::NotEqual, keyAddress, temp, &skipBarrier);
{
saveVolatile(temp);
emitPostWriteBarrier(result);
restoreVolatile(temp);
}
masm.bind(&keyIsNotObject);
masm.bind(&skipBarrier);
}
template <class IteratorObject, class OrderedHashTable>
@ -7803,6 +7951,7 @@ CopyStringCharsMaybeInflate(MacroAssembler& masm, Register input, Register destC
static void
ConcatInlineString(MacroAssembler& masm, Register lhs, Register rhs, Register output,
Register temp1, Register temp2, Register temp3,
bool stringsCanBeInNursery,
Label* failure, Label* failurePopTemps, bool isTwoByte)
{
// State: result length in temp2.
@ -7824,7 +7973,7 @@ ConcatInlineString(MacroAssembler& masm, Register lhs, Register rhs, Register ou
uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
if (!isTwoByte)
flags |= JSString::LATIN1_CHARS_BIT;
masm.newGCString(output, temp1, failure);
masm.newGCString(output, temp1, failure, stringsCanBeInNursery);
masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
masm.jump(&allocDone);
}
@ -7833,7 +7982,7 @@ ConcatInlineString(MacroAssembler& masm, Register lhs, Register rhs, Register ou
uint32_t flags = JSString::INIT_FAT_INLINE_FLAGS;
if (!isTwoByte)
flags |= JSString::LATIN1_CHARS_BIT;
masm.newGCFatInlineString(output, temp1, failure);
masm.newGCFatInlineString(output, temp1, failure, stringsCanBeInNursery);
masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
}
masm.bind(&allocDone);
@ -7921,7 +8070,7 @@ CodeGenerator::visitSubstr(LSubstr* lir)
// Handle inlined strings by creating a FatInlineString.
masm.branchTest32(Assembler::Zero, stringFlags, Imm32(JSString::INLINE_CHARS_BIT), &notInline);
masm.newGCFatInlineString(output, temp, slowPath);
masm.newGCFatInlineString(output, temp, slowPath, stringsCanBeInNursery());
masm.store32(length, Address(output, JSString::offsetOfLength()));
masm.branchLatin1String(string, &isInlinedLatin1);
@ -7965,7 +8114,7 @@ CodeGenerator::visitSubstr(LSubstr* lir)
// Handle other cases with a DependentString.
masm.bind(&notInline);
masm.newGCString(output, temp, slowPath);
masm.newGCString(output, temp, slowPath, gen->stringsCanBeInNursery());
masm.store32(length, Address(output, JSString::offsetOfLength()));
masm.storeDependentStringBase(string, output);
@ -8045,8 +8194,8 @@ JitCompartment::generateStringConcatStub(JSContext* cx)
// Ensure result length <= JSString::MAX_LENGTH.
masm.branch32(Assembler::Above, temp2, Imm32(JSString::MAX_LENGTH), &failure);
// Allocate a new rope.
masm.newGCString(output, temp3, &failure);
// Allocate a new rope, guaranteed to be in the nursery.
masm.newGCString(output, temp3, &failure, stringsCanBeInNursery);
// Store rope length and flags. temp1 still holds the result of AND'ing the
// lhs and rhs flags, so we just have to clear the other flags and set
@ -8073,11 +8222,11 @@ JitCompartment::generateStringConcatStub(JSContext* cx)
masm.bind(&isFatInlineTwoByte);
ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
&failure, &failurePopTemps, true);
stringsCanBeInNursery, &failure, &failurePopTemps, true);
masm.bind(&isFatInlineLatin1);
ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
&failure, &failurePopTemps, false);
stringsCanBeInNursery, &failure, &failurePopTemps, false);
masm.bind(&failurePopTemps);
masm.pop(temp2);
@ -8345,7 +8494,7 @@ CodeGenerator::visitFromCodePoint(LFromCodePoint* lir)
"JSThinInlineString can hold a supplementary code point");
uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
masm.newGCString(output, temp1, ool->entry());
masm.newGCString(output, temp1, ool->entry(), gen->stringsCanBeInNursery());
masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
}
@ -10107,7 +10256,7 @@ CodeGenerator::link(JSContext* cx, CompilerConstraintList* constraints)
ionScript->copyConstants(vp);
for (size_t i = 0; i < graph.numConstants(); i++) {
const Value& v = vp[i];
if (v.isObject() && IsInsideNursery(&v.toObject())) {
if ((v.isObject() || v.isString()) && IsInsideNursery(v.toGCThing())) {
cx->zone()->group()->storeBuffer().putWholeCell(script);
break;
}

7
js/src/jit/CodeGenerator.h
Просмотреть файл

@ -164,14 +164,17 @@ class CodeGenerator final : public CodeGeneratorSpecific
void visitMonitorTypes(LMonitorTypes* lir) override;
void emitPostWriteBarrier(const LAllocation* obj);
void emitPostWriteBarrier(Register objreg);
template <class LPostBarrierType>
void visitPostWriteBarrierCommonO(LPostBarrierType* lir, OutOfLineCode* ool);
void emitPostWriteBarrierS(Address address, Register prev, Register next);
template <class LPostBarrierType, MIRType nurseryType>
void visitPostWriteBarrierCommon(LPostBarrierType* lir, OutOfLineCode* ool);
template <class LPostBarrierType>
void visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode* ool);
void visitPostWriteBarrierO(LPostWriteBarrierO* lir) override;
void visitPostWriteElementBarrierO(LPostWriteElementBarrierO* lir) override;
void visitPostWriteBarrierV(LPostWriteBarrierV* lir) override;
void visitPostWriteElementBarrierV(LPostWriteElementBarrierV* lir) override;
void visitPostWriteBarrierS(LPostWriteBarrierS* lir) override;
void visitPostWriteElementBarrierS(LPostWriteElementBarrierS* lir) override;
void visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool);
void visitOutOfLineCallPostWriteElementBarrier(OutOfLineCallPostWriteElementBarrier* ool);
void visitCallNative(LCallNative* call) override;

7
js/src/jit/CompileWrappers.cpp
Просмотреть файл

@ -188,10 +188,15 @@ CompileZone::addressOfNurseryCurrentEnd()
return zone()->runtimeFromAnyThread()->gc.addressOfNurseryCurrentEnd();
}
bool
CompileZone::canNurseryAllocateStrings()
{
return nurseryExists() && zone()->group()->nursery().canAllocateStrings();
}
bool
CompileZone::nurseryExists()
{
MOZ_ASSERT(CurrentThreadCanAccessZone(zone()));
return zone()->group()->nursery().exists();
}

1
js/src/jit/CompileWrappers.h
Просмотреть файл

@ -79,6 +79,7 @@ class CompileZone
const void* addressOfNurseryCurrentEnd();
bool nurseryExists();
bool canNurseryAllocateStrings();
void setMinorGCShouldCancelIonCompilations();
};

2
js/src/jit/Ion.cpp
Просмотреть файл

@ -455,6 +455,8 @@ JitCompartment::initialize(JSContext* cx)
return false;
}
stringsCanBeInNursery = cx->nursery().canAllocateStrings();
return true;
}

9
js/src/jit/IonBuilder.cpp
Просмотреть файл

@ -7407,9 +7407,14 @@ IonBuilder::loadStaticSlot(JSObject* staticObject, BarrierKind barrier, Temporar
bool
IonBuilder::needsPostBarrier(MDefinition* value)
{
if (!compartment->zone()->nurseryExists())
CompileZone* zone = compartment->zone();
if (!zone->nurseryExists())
return false;
return value->mightBeType(MIRType::Object);
if (value->mightBeType(MIRType::Object))
return true;
if (value->mightBeType(MIRType::String) && zone->canNurseryAllocateStrings())
return true;
return false;
}
AbortReasonOr<Ok>

2
js/src/jit/JitCompartment.h
Просмотреть файл

@ -644,6 +644,8 @@ class JitCompartment
}
size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
bool stringsCanBeInNursery;
};
// Called from JSCompartment::discardJitCode().

34
js/src/jit/Lowering.cpp
Просмотреть файл

@ -116,7 +116,7 @@ TryToUseImplicitInterruptCheck(MIRGraph& graph, MBasicBlock* backedge)
continue;
}
MOZ_ASSERT_IF(iter->isPostWriteBarrierO() || iter->isPostWriteBarrierV(),
MOZ_ASSERT_IF(iter->isPostWriteBarrierO() || iter->isPostWriteBarrierV() || iter->isPostWriteBarrierS(),
iter->safepoint());
if (iter->safepoint())
@ -2890,6 +2890,17 @@ LIRGenerator::visitPostWriteBarrier(MPostWriteBarrier* ins)
assignSafepoint(lir, ins);
break;
}
case MIRType::String: {
LDefinition tmp = needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
LPostWriteBarrierS* lir =
new(alloc()) LPostWriteBarrierS(useConstantObject
? useOrConstant(ins->object())
: useRegister(ins->object()),
useRegister(ins->value()), tmp);
add(lir, ins);
assignSafepoint(lir, ins);
break;
}
case MIRType::Value: {
LDefinition tmp = needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
LPostWriteBarrierV* lir =
@ -2903,8 +2914,8 @@ LIRGenerator::visitPostWriteBarrier(MPostWriteBarrier* ins)
break;
}
default:
// Currently, only objects can be in the nursery. Other instruction
// types cannot hold nursery pointers.
// Currently, only objects and strings can be in the nursery. Other
// instruction types cannot hold nursery pointers.
break;
}
}
@ -2938,6 +2949,19 @@ LIRGenerator::visitPostWriteElementBarrier(MPostWriteElementBarrier* ins)
assignSafepoint(lir, ins);
break;
}
case MIRType::String: {
LDefinition tmp = needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
LPostWriteElementBarrierS* lir =
new(alloc()) LPostWriteElementBarrierS(useConstantObject
? useOrConstant(ins->object())
: useRegister(ins->object()),
useRegister(ins->value()),
useRegister(ins->index()),
tmp);
add(lir, ins);
assignSafepoint(lir, ins);
break;
}
case MIRType::Value: {
LDefinition tmp = needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
LPostWriteElementBarrierV* lir =
@ -2952,8 +2976,8 @@ LIRGenerator::visitPostWriteElementBarrier(MPostWriteElementBarrier* ins)
break;
}
default:
// Currently, only objects can be in the nursery. Other instruction
// types cannot hold nursery pointers.
// Currently, only objects and strings can be in the nursery. Other
// instruction types cannot hold nursery pointers.
break;
}
}

5
js/src/jit/MIRGenerator.h
Просмотреть файл

@ -108,6 +108,10 @@ class MIRGenerator
!JitOptions.disableOptimizationTracking;
}
bool stringsCanBeInNursery() const {
return stringsCanBeInNursery_;
}
bool safeForMinorGC() const {
return safeForMinorGC_;
}
@ -197,6 +201,7 @@ class MIRGenerator
bool instrumentedProfiling_;
bool instrumentedProfilingIsCached_;
bool safeForMinorGC_;
bool stringsCanBeInNursery_;
void addAbortedPreliminaryGroup(ObjectGroup* group);

1
js/src/jit/MIRGraph.cpp
Просмотреть файл

@ -38,6 +38,7 @@ MIRGenerator::MIRGenerator(CompileCompartment* compartment, const JitCompileOpti
instrumentedProfiling_(false),
instrumentedProfilingIsCached_(false),
safeForMinorGC_(true),
stringsCanBeInNursery_(compartment ? compartment->zone()->canNurseryAllocateStrings() : false),
minWasmHeapLength_(0),
options(options),
gs_(alloc)

98
js/src/jit/MacroAssembler.cpp
Просмотреть файл

@ -729,7 +729,6 @@ MacroAssembler::checkAllocatorState(Label* fail)
jump(fail);
}
// Inline version of ShouldNurseryAllocate.
bool
MacroAssembler::shouldNurseryAllocate(gc::AllocKind allocKind, gc::InitialHeap initialHeap)
{
@ -744,11 +743,10 @@ MacroAssembler::shouldNurseryAllocate(gc::AllocKind allocKind, gc::InitialHeap i
// Inline version of Nursery::allocateObject. If the object has dynamic slots,
// this fills in the slots_ pointer.
void
MacroAssembler::nurseryAllocate(Register result, Register temp, gc::AllocKind allocKind,
size_t nDynamicSlots, gc::InitialHeap initialHeap, Label* fail)
MacroAssembler::nurseryAllocateObject(Register result, Register temp, gc::AllocKind allocKind,
size_t nDynamicSlots, Label* fail)
{
MOZ_ASSERT(IsNurseryAllocable(allocKind));
MOZ_ASSERT(initialHeap != gc::TenuredHeap);
// We still need to allocate in the nursery, per the comment in
// shouldNurseryAllocate; however, we need to insert into the
@ -868,8 +866,10 @@ MacroAssembler::allocateObject(Register result, Register temp, gc::AllocKind all
checkAllocatorState(fail);
if (shouldNurseryAllocate(allocKind, initialHeap))
return nurseryAllocate(result, temp, allocKind, nDynamicSlots, initialHeap, fail);
if (shouldNurseryAllocate(allocKind, initialHeap)) {
MOZ_ASSERT(initialHeap == gc::DefaultHeap);
return nurseryAllocateObject(result, temp, allocKind, nDynamicSlots, fail);
}
if (!nDynamicSlots)
return freeListAllocate(result, temp, allocKind, fail);
@ -931,16 +931,82 @@ MacroAssembler::allocateNonObject(Register result, Register temp, gc::AllocKind
freeListAllocate(result, temp, allocKind, fail);
}
// Inline version of Nursery::allocateString.
void
MacroAssembler::newGCString(Register result, Register temp, Label* fail)
MacroAssembler::nurseryAllocateString(Register result, Register temp, gc::AllocKind allocKind,
Label* fail)
{
allocateNonObject(result, temp, js::gc::AllocKind::STRING, fail);
MOZ_ASSERT(IsNurseryAllocable(allocKind));
// No explicit check for nursery.isEnabled() is needed, as the comparison
// with the nursery's end will always fail in such cases.
CompileZone* zone = GetJitContext()->compartment->zone();
int thingSize = int(gc::Arena::thingSize(allocKind));
int totalSize = js::Nursery::stringHeaderSize() + thingSize;
MOZ_ASSERT(totalSize % gc::CellAlignBytes == 0);
// The nursery position (allocation pointer) and the nursery end are stored
// very close to each other. In practice, the zone will probably be close
// (within 32 bits) as well. If so, use relative offsets between them, to
// avoid multiple 64-bit immediate loads.
auto nurseryPosAddr = intptr_t(zone->addressOfNurseryPosition());
auto nurseryEndAddr = intptr_t(zone->addressOfNurseryCurrentEnd());
auto zoneAddr = intptr_t(zone);
intptr_t maxOffset = std::max(std::abs(nurseryPosAddr - zoneAddr),
std::abs(nurseryEndAddr - zoneAddr));
if (maxOffset < (1 << 31)) {
movePtr(ImmPtr(zone), temp); // temp holds the Zone pointer from here on.
loadPtr(Address(temp, nurseryPosAddr - zoneAddr), result);
addPtr(Imm32(totalSize), result); // result points past this allocation.
branchPtr(Assembler::Below, Address(temp, nurseryEndAddr - zoneAddr), result, fail);
storePtr(result, Address(temp, nurseryPosAddr - zoneAddr)); // Update position.
subPtr(Imm32(thingSize), result); // Point result at Cell data.
storePtr(temp, Address(result, -js::Nursery::stringHeaderSize())); // Store Zone*
} else {
// Otherwise, the zone is far from the nursery pointers. But the
// nursery pos/end pointers are still near each other.
movePtr(ImmPtr(zone->addressOfNurseryPosition()), temp);
loadPtr(Address(temp, 0), result);
addPtr(Imm32(totalSize), result);
branchPtr(Assembler::Below, Address(temp, nurseryEndAddr - nurseryPosAddr), result, fail);
storePtr(result, Address(temp, 0));
subPtr(Imm32(thingSize), result);
storePtr(ImmPtr(zone), Address(result, -js::Nursery::stringHeaderSize()));
}
}
// Inlined equivalent of gc::AllocateString, jumping to fail if nursery
// allocation requested but unsuccessful.
void
MacroAssembler::allocateString(Register result, Register temp, gc::AllocKind allocKind,
gc::InitialHeap initialHeap, Label* fail)
{
MOZ_ASSERT(allocKind == gc::AllocKind::STRING || allocKind == gc::AllocKind::FAT_INLINE_STRING);
checkAllocatorState(fail);
if (shouldNurseryAllocate(allocKind, initialHeap)) {
MOZ_ASSERT(initialHeap == gc::DefaultHeap);
return nurseryAllocateString(result, temp, allocKind, fail);
}
freeListAllocate(result, temp, allocKind, fail);
}
void
MacroAssembler::newGCFatInlineString(Register result, Register temp, Label* fail)
MacroAssembler::newGCString(Register result, Register temp, Label* fail, bool attemptNursery)
{
allocateNonObject(result, temp, js::gc::AllocKind::FAT_INLINE_STRING, fail);
allocateString(result, temp, js::gc::AllocKind::STRING,
attemptNursery ? gc::DefaultHeap : gc::TenuredHeap, fail);
}
void
MacroAssembler::newGCFatInlineString(Register result, Register temp, Label* fail, bool attemptNursery)
{
allocateString(result, temp, js::gc::AllocKind::FAT_INLINE_STRING,
attemptNursery ? gc::DefaultHeap : gc::TenuredHeap, fail);
}
void
@ -1511,6 +1577,16 @@ MacroAssembler::loadDependentStringBase(Register str, Register dest)
loadPtr(Address(str, JSDependentString::offsetOfBase()), dest);
}
void
MacroAssembler::leaNewDependentStringBase(Register str, Register dest)
{
MOZ_ASSERT(str != dest);
// Spectre-safe because this is a newly allocated dependent string, thus we
// are certain of its type and the type of its base field.
computeEffectiveAddress(Address(str, JSDependentString::offsetOfBase()), dest);
}
void
MacroAssembler::storeDependentStringBase(Register base, Register str)
{
@ -3297,7 +3373,7 @@ MacroAssembler::emitPreBarrierFastPath(JSRuntime* rt, MIRType type, Register tem
andPtr(temp1, temp2);
// If the GC thing is in the nursery, we don't need to barrier it.
if (type == MIRType::Value || type == MIRType::Object) {
if (type == MIRType::Value || type == MIRType::Object || type == MIRType::String) {
branch32(Assembler::Equal, Address(temp2, gc::ChunkLocationOffset),
Imm32(int32_t(gc::ChunkLocation::Nursery)), noBarrier);
} else {

22
js/src/jit/MacroAssembler.h
Просмотреть файл

@ -1091,6 +1091,10 @@ class MacroAssembler : public MacroAssemblerSpecific
inline void branchPtr(Condition cond, wasm::SymbolicAddress lhs, Register rhs, Label* label)
DEFINED_ON(arm, arm64, mips_shared, x86, x64);
// Given a pointer to a GC Cell, retrieve the StoreBuffer pointer from its
// chunk trailer, or nullptr if it is in the tenured heap.
void loadStoreBuffer(Register ptr, Register buffer) PER_ARCH;
template <typename T>
inline CodeOffsetJump branchPtrWithPatch(Condition cond, Register lhs, T rhs, RepatchLabel* label) PER_SHARED_ARCH;
template <typename T>
@ -1100,8 +1104,9 @@ class MacroAssembler : public MacroAssemblerSpecific
DEFINED_ON(arm, arm64, mips_shared, x86, x64);
void branchPtrInNurseryChunk(Condition cond, const Address& address, Register temp, Label* label)
DEFINED_ON(x86);
void branchValueIsNurseryObject(Condition cond, const Address& address, Register temp, Label* label) PER_ARCH;
void branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label) PER_ARCH;
void branchValueIsNurseryCell(Condition cond, const Address& address, Register temp, Label* label) PER_ARCH;
void branchValueIsNurseryCell(Condition cond, ValueOperand value, Register temp, Label* label) PER_ARCH;
// This function compares a Value (lhs) which is having a private pointer
// boxed inside a js::Value, with a raw pointer (rhs).
@ -1320,7 +1325,7 @@ class MacroAssembler : public MacroAssemblerSpecific
void branchPtrInNurseryChunkImpl(Condition cond, Register ptr, Label* label)
DEFINED_ON(x86);
template <typename T>
void branchValueIsNurseryObjectImpl(Condition cond, const T& value, Register temp, Label* label)
void branchValueIsNurseryCellImpl(Condition cond, const T& value, Register temp, Label* label)
DEFINED_ON(arm64, mips64, x64);
template <typename T>
@ -1989,6 +1994,7 @@ class MacroAssembler : public MacroAssemblerSpecific
void loadDependentStringBase(Register str, Register dest);
void storeDependentStringBase(Register base, Register str);
void leaNewDependentStringBase(Register str, Register dest);
void loadStringIndexValue(Register str, Register dest, Label* fail);
@ -2226,11 +2232,15 @@ class MacroAssembler : public MacroAssemblerSpecific
private:
void checkAllocatorState(Label* fail);
bool shouldNurseryAllocate(gc::AllocKind allocKind, gc::InitialHeap initialHeap);
void nurseryAllocate(Register result, Register temp, gc::AllocKind allocKind,
size_t nDynamicSlots, gc::InitialHeap initialHeap, Label* fail);
void nurseryAllocateObject(Register result, Register temp, gc::AllocKind allocKind,
size_t nDynamicSlots, Label* fail);
void freeListAllocate(Register result, Register temp, gc::AllocKind allocKind, Label* fail);
void allocateObject(Register result, Register temp, gc::AllocKind allocKind,
uint32_t nDynamicSlots, gc::InitialHeap initialHeap, Label* fail);
void nurseryAllocateString(Register result, Register temp, gc::AllocKind allocKind,
Label* fail);
void allocateString(Register result, Register temp, gc::AllocKind allocKind,
gc::InitialHeap initialHeap, Label* fail);
void allocateNonObject(Register result, Register temp, gc::AllocKind allocKind, Label* fail);
void copySlotsFromTemplate(Register obj, const NativeObject* templateObj,
uint32_t start, uint32_t end);
@ -2256,8 +2266,8 @@ class MacroAssembler : public MacroAssemblerSpecific
void initUnboxedObjectContents(Register object, UnboxedPlainObject* templateObject);
void newGCString(Register result, Register temp, Label* fail);
void newGCFatInlineString(Register result, Register temp, Label* fail);
void newGCString(Register result, Register temp, Label* fail, bool attemptNursery);
void newGCFatInlineString(Register result, Register temp, Label* fail, bool attemptNursery);
// Compares two strings for equality based on the JSOP.
// This checks for identical pointers, atoms and length and fails for everything else.

2
js/src/jit/VMFunctions.cpp
Просмотреть файл

@ -1335,6 +1335,7 @@ AssertValidObjectPtr(JSContext* cx, JSObject* obj)
// Check what we can, so that we'll hopefully assert/crash if we get a
// bogus object (pointer).
MOZ_ASSERT(obj->compartment() == cx->compartment());
MOZ_ASSERT(obj->zoneFromAnyThread() == cx->zone());
MOZ_ASSERT(obj->runtimeFromActiveCooperatingThread() == cx->runtime());
MOZ_ASSERT_IF(!obj->hasLazyGroup() && obj->maybeShape(),
@ -1344,7 +1345,6 @@ AssertValidObjectPtr(JSContext* cx, JSObject* obj)
MOZ_ASSERT(obj->isAligned());
gc::AllocKind kind = obj->asTenured().getAllocKind();
MOZ_ASSERT(gc::IsObjectAllocKind(kind));
MOZ_ASSERT(obj->asTenured().zone() == cx->zone());
}
#endif
}

74
js/src/jit/arm/MacroAssembler-arm.cpp
Просмотреть файл

@ -4791,32 +4791,65 @@ MacroAssembler::moveValue(const Value& src, const ValueOperand& dest)
// Branch functions
void
MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
Label* label)
MacroAssembler::loadStoreBuffer(Register ptr, Register buffer)
{
SecondScratchRegisterScope scratch2(*this);
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
MOZ_ASSERT(ptr != temp);
MOZ_ASSERT(ptr != scratch2);
ma_lsr(Imm32(gc::ChunkShift), ptr, scratch2);
ma_lsl(Imm32(gc::ChunkShift), scratch2, scratch2);
load32(Address(scratch2, gc::ChunkLocationOffset), scratch2);
branch32(cond, scratch2, Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
ma_lsr(Imm32(gc::ChunkShift), ptr, buffer);
ma_lsl(Imm32(gc::ChunkShift), buffer, buffer);
load32(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address,
Register temp, Label* label)
MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
Label* label)
{
Maybe<SecondScratchRegisterScope> scratch2;
if (temp == Register::Invalid()) {
scratch2.emplace(*this);
temp = scratch2.ref();
}
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
MOZ_ASSERT(ptr != temp);
ma_lsr(Imm32(gc::ChunkShift), ptr, temp);
ma_lsl(Imm32(gc::ChunkShift), temp, temp);
load32(Address(temp, gc::ChunkLocationOffset), temp);
branch32(cond, temp, Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, const Address& address,
Register temp, Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done, checkAddress;
Label done;
branchTestObject(Assembler::NotEqual, address, cond == Assembler::Equal ? &done : label);
Register tag = temp;
extractTag(address, tag);
branchTestObject(Assembler::Equal, tag, &checkAddress);
branchTestString(Assembler::NotEqual, tag, cond == Assembler::Equal ? &done : label);
loadPtr(address, temp);
branchPtrInNurseryChunk(cond, temp, InvalidReg, label);
bind(&checkAddress);
loadPtr(ToPayload(address), temp);
SecondScratchRegisterScope scratch2(*this);
branchPtrInNurseryChunk(cond, temp, scratch2, label);
bind(&done);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value,
Register temp, Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done, checkAddress;
branchTestObject(Assembler::Equal, value.typeReg(), &checkAddress);
branchTestString(Assembler::NotEqual, value.typeReg(),
cond == Assembler::Equal ? &done : label);
bind(&checkAddress);
branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
bind(&done);
}
@ -4826,11 +4859,10 @@ MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value,
Register temp, Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done;
branchTestObject(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label);
branchPtrInNurseryChunk(cond, value.payloadReg(), InvalidReg, label);
branchTestObject(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label);
branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
bind(&done);
}

57
js/src/jit/arm64/MacroAssembler-arm64.cpp
Просмотреть файл

@ -760,6 +760,15 @@ MacroAssembler::moveValue(const Value& src, const ValueOperand& dest)
// ===============================================================
// Branch functions
void
MacroAssembler::loadStoreBuffer(Register ptr, Register buffer)
{
if (ptr != buffer)
movePtr(ptr, buffer);
orPtr(Imm32(gc::ChunkMask), buffer);
loadPtr(Address(buffer, gc::ChunkStoreBufferOffsetFromLastByte), buffer);
}
void
MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
Label* label)
@ -776,23 +785,49 @@ MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register t
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp,
Label* label)
MacroAssembler::branchValueIsNurseryCell(Condition cond, const Address& address, Register temp,
Label* label)
{
branchValueIsNurseryObjectImpl(cond, address, temp, label);
branchValueIsNurseryCellImpl(cond, address, temp, label);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value, Register temp,
Label* label)
{
branchValueIsNurseryCellImpl(cond, value, temp, label);
}
template <typename T>
void
MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, const T& value, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
MOZ_ASSERT(temp != ScratchReg && temp != ScratchReg2); // Both may be used internally.
Label done, checkAddress, checkObjectAddress;
bool testNursery = (cond == Assembler::Equal);
branchTestObject(Assembler::Equal, value, &checkObjectAddress);
branchTestString(Assembler::NotEqual, value, testNursery ? &done : label);
unboxString(value, temp);
jump(&checkAddress);
bind(&checkObjectAddress);
unboxObject(value, temp);
bind(&checkAddress);
orPtr(Imm32(gc::ChunkMask), temp);
branch32(cond, Address(temp, gc::ChunkLocationOffsetFromLastByte),
Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
bind(&done);
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp,
Label* label)
{
branchValueIsNurseryObjectImpl(cond, value, temp, label);
}
template <typename T>
void
MacroAssembler::branchValueIsNurseryObjectImpl(Condition cond, const T& value, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
MOZ_ASSERT(temp != ScratchReg && temp != ScratchReg2); // Both may be used internally.

20
js/src/jit/mips32/MacroAssembler-mips32.cpp
Просмотреть файл

@ -2266,14 +2266,16 @@ MacroAssembler::moveValue(const Value& src, const ValueOperand& dest)
// Branch functions
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address,
Register temp, Label* label)
MacroAssembler::branchValueIsNurseryCell(Condition cond, const Address& address,
Register temp, Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done, checkAddress;
Label done;
branchTestObject(Assembler::Equal, address, &checkAddress);
branchTestString(Assembler::NotEqual, address, cond == Assembler::Equal ? &done : label);
branchTestObject(Assembler::NotEqual, address, cond == Assembler::Equal ? &done : label);
bind(&checkAddress);
loadPtr(address, temp);
branchPtrInNurseryChunk(cond, temp, InvalidReg, label);
@ -2281,14 +2283,16 @@ MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& addres
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value,
Register temp, Label* label)
MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value,
Register temp, Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done;
Label done, checkAddress;
branchTestObject(Assembler::Equal, value, &checkAddress);
branchTestString(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label);
branchTestObject(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label);
bind(&checkAddress);
branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
bind(&done);

49
js/src/jit/mips64/MacroAssembler-mips64.cpp
Просмотреть файл

@ -2334,24 +2334,9 @@ MacroAssembler::moveValue(const Value& src, const ValueOperand& dest)
// ===============================================================
// Branch functions
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp,
Label* label)
{
branchValueIsNurseryObjectImpl(cond, address, temp, label);
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value,
Register temp, Label* label)
{
branchValueIsNurseryObjectImpl(cond, value, temp, label);
}
template <typename T>
void
MacroAssembler::branchValueIsNurseryObjectImpl(Condition cond, const T& value, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
@ -2366,6 +2351,40 @@ MacroAssembler::branchValueIsNurseryObjectImpl(Condition cond, const T& value, R
bind(&done);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, const Address& address, Register temp,
Label* label)
{
branchValueIsNurseryCellImpl(cond, address, temp, label);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value,
Register temp, Label* label)
{
branchValueIsNurseryCellImpl(cond, value, temp, label);
}
template <typename T>
void
MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, const T& value, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done, checkAddress;
branchTestObject(Assembler::Equal, value, &checkAddress);
branchTestString(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label);
bind(&checkAddress);
extractCell(value, SecondScratchReg);
orPtr(Imm32(gc::ChunkMask), SecondScratchReg);
branch32(cond, Address(SecondScratchReg, gc::ChunkLocationOffsetFromLastByte),
Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
bind(&done);
}
void
MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
const Value& rhs, Label* label)

7
js/src/jit/mips64/MacroAssembler-mips64.h
Просмотреть файл

@ -448,6 +448,13 @@ class MacroAssemblerMIPS64Compat : public MacroAssemblerMIPS64
unboxSymbol(value, scratch);
return scratch;
}
Register extractCell(const Address& address, Register scratch) {
return extractObject(address, scratch);
}
Register extractCell(const ValueOperand& value, Register scratch) {
unboxNonDouble(value, scratch);
return scratch;
}
Register extractInt32(const ValueOperand& value, Register scratch) {
unboxInt32(value, scratch);
return scratch;

4
js/src/jit/shared/CodeGenerator-shared.h
Просмотреть файл

@ -147,6 +147,10 @@ class CodeGeneratorShared : public LElementVisitor
return gen->isProfilerInstrumentationEnabled();
}
bool stringsCanBeInNursery() const {
return gen->stringsCanBeInNursery();
}
js::Vector<NativeToTrackedOptimizations, 0, SystemAllocPolicy> trackedOptimizations_;
uint8_t* trackedOptimizationsMap_;
uint32_t trackedOptimizationsMapSize_;

63
js/src/jit/shared/LIR-shared.h
Просмотреть файл

@ -7871,6 +7871,33 @@ class LPostWriteBarrierO : public LInstructionHelper<0, 2, 1>
}
};
// Generational write barrier used when writing a string to an object.
class LPostWriteBarrierS : public LInstructionHelper<0, 2, 1>
{
public:
LIR_HEADER(PostWriteBarrierS)
LPostWriteBarrierS(const LAllocation& obj, const LAllocation& value,
const LDefinition& temp) {
setOperand(0, obj);
setOperand(1, value);
setTemp(0, temp);
}
const MPostWriteBarrier* mir() const {
return mir_->toPostWriteBarrier();
}
const LAllocation* object() {
return getOperand(0);
}
const LAllocation* value() {
return getOperand(1);
}
const LDefinition* temp() {
return getTemp(0);
}
};
// Generational write barrier used when writing a value to another object.
class LPostWriteBarrierV : public LInstructionHelper<0, 1 + BOX_PIECES, 1>
{
@ -7933,6 +7960,42 @@ class LPostWriteElementBarrierO : public LInstructionHelper<0, 3, 1>
}
};
// Generational write barrier used when writing a string to an object's
// elements.
class LPostWriteElementBarrierS : public LInstructionHelper<0, 3, 1>
{
public:
LIR_HEADER(PostWriteElementBarrierS)
LPostWriteElementBarrierS(const LAllocation& obj, const LAllocation& value,
const LAllocation& index, const LDefinition& temp) {
setOperand(0, obj);
setOperand(1, value);
setOperand(2, index);
setTemp(0, temp);
}
const MPostWriteElementBarrier* mir() const {
return mir_->toPostWriteElementBarrier();
}
const LAllocation* object() {
return getOperand(0);
}
const LAllocation* value() {
return getOperand(1);
}
const LAllocation* index() {
return getOperand(2);
}
const LDefinition* temp() {
return getTemp(0);
}
};
// Generational write barrier used when writing a value to another object's
// elements.
class LPostWriteElementBarrierV : public LInstructionHelper<0, 2 + BOX_PIECES, 1>

2
js/src/jit/shared/LOpcodes-shared.h
Просмотреть файл

@ -272,8 +272,10 @@
_(TypeBarrierO) \
_(MonitorTypes) \
_(PostWriteBarrierO) \
_(PostWriteBarrierS) \
_(PostWriteBarrierV) \
_(PostWriteElementBarrierO) \
_(PostWriteElementBarrierS) \
_(PostWriteElementBarrierV) \
_(InitializedLength) \
_(SetInitializedLength) \

66
js/src/jit/x64/MacroAssembler-x64.cpp
Просмотреть файл

@ -450,6 +450,15 @@ MacroAssembler::moveValue(const Value& src, const ValueOperand& dest)
// ===============================================================
// Branch functions
void
MacroAssembler::loadStoreBuffer(Register ptr, Register buffer)
{
if (ptr != buffer)
movePtr(ptr, buffer);
orPtr(Imm32(gc::ChunkMask), buffer);
loadPtr(Address(buffer, gc::ChunkStoreBufferOffsetFromLastByte), buffer);
}
void
MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp, Label* label)
{
@ -465,24 +474,9 @@ MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register t
Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp,
Label* label)
{
branchValueIsNurseryObjectImpl(cond, address, temp, label);
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp,
Label* label)
{
branchValueIsNurseryObjectImpl(cond, value, temp, label);
}
template <typename T>
void
MacroAssembler::branchValueIsNurseryObjectImpl(Condition cond, const T& value, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
MOZ_ASSERT(temp != InvalidReg);
@ -498,6 +492,48 @@ MacroAssembler::branchValueIsNurseryObjectImpl(Condition cond, const T& value, R
bind(&done);
}
template <typename T>
void
MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, const T& value, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
MOZ_ASSERT(temp != InvalidReg);
Label done, checkAddress, checkObjectAddress;
Register tag = temp;
splitTag(value, tag);
branchTestObject(Assembler::Equal, tag, &checkObjectAddress);
branchTestString(Assembler::NotEqual, tag, cond == Assembler::Equal ? &done : label);
unboxString(value, temp);
jump(&checkAddress);
bind(&checkObjectAddress);
unboxObject(value, temp);
bind(&checkAddress);
orPtr(Imm32(gc::ChunkMask), temp);
branch32(cond, Address(temp, gc::ChunkLocationOffsetFromLastByte),
Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
bind(&done);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, const Address& address, Register temp,
Label* label)
{
branchValueIsNurseryCellImpl(cond, address, temp, label);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value, Register temp,
Label* label)
{
branchValueIsNurseryCellImpl(cond, value, temp, label);
}
void
MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
const Value& rhs, Label* label)

57
js/src/jit/x86/MacroAssembler-x86.cpp
Просмотреть файл

@ -452,6 +452,15 @@ MacroAssembler::moveValue(const Value& src, const ValueOperand& dest)
// ===============================================================
// Branch functions
void
MacroAssembler::loadStoreBuffer(Register ptr, Register buffer)
{
if (ptr != buffer)
movePtr(ptr, buffer);
orPtr(Imm32(gc::ChunkMask), buffer);
loadPtr(Address(buffer, gc::ChunkStoreBufferOffsetFromLastByte), buffer);
}
void
MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
Label* label)
@ -481,20 +490,6 @@ MacroAssembler::branchPtrInNurseryChunkImpl(Condition cond, Register ptr, Label*
Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done;
branchTestObject(Assembler::NotEqual, address, cond == Assembler::Equal ? &done : label);
branchPtrInNurseryChunk(cond, address, temp, label);
bind(&done);
}
void
MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp,
Label* label)
@ -509,6 +504,40 @@ MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, R
bind(&done);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, const Address& address, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done, checkAddress;
Register tag = extractTag(address, temp);
MOZ_ASSERT(tag == temp);
branchTestObject(Assembler::Equal, tag, &checkAddress);
branchTestString(Assembler::NotEqual, tag, cond == Assembler::Equal ? &done : label);
bind(&checkAddress);
branchPtrInNurseryChunk(cond, ToPayload(address), temp, label);
bind(&done);
}
void
MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value, Register temp,
Label* label)
{
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done, checkAddress;
branchTestObject(Assembler::Equal, value, &checkAddress);
branchTestString(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label);
bind(&checkAddress);
branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
bind(&done);
}
void
MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
const Value& rhs, Label* label)

3
js/src/tests/lib/tests.py
Просмотреть файл

@ -41,6 +41,9 @@ JITFLAGS = {
'--ion-check-range-analysis', '--ion-extra-checks', '--no-sse3', '--no-threads'],
['--no-asmjs', '--no-wasm', '--no-baseline', '--no-ion'],
],
'baseline': [
['--no-ion'],
],
# Interpreter-only, for tools that cannot handle binary code generation.
'interp': [
['--no-baseline', '--no-asmjs', '--no-wasm', '--no-native-regexp']

10
js/src/vm/String.h
Просмотреть файл

@ -629,6 +629,16 @@ class JSString : public js::gc::Cell
js::gc::TenuredCell::writeBarrierPre(&thing->asTenured());
}
static void addCellAddressToStoreBuffer(js::gc::StoreBuffer* buffer, js::gc::Cell** cellp)
{
buffer->putCell(cellp);
}
static void removeCellAddressFromStoreBuffer(js::gc::StoreBuffer* buffer, js::gc::Cell** cellp)
{
buffer->unputCell(cellp);
}
static void writeBarrierPost(void* cellp, JSString* prev, JSString* next) {
// See JSObject::writeBarrierPost for a description of the logic here.
MOZ_ASSERT(cellp);

12
js/src/vm/UnboxedObject.cpp
Просмотреть файл

@ -136,13 +136,23 @@ UnboxedLayout::makeConstructorCode(JSContext* cx, HandleObjectGroup group)
Label postBarrier;
for (size_t i = 0; i < layout.properties().length(); i++) {
const UnboxedLayout::Property& property = layout.properties()[i];
if (!UnboxedTypeNeedsPostBarrier(property.type))
continue;
Address valueAddress(propertiesReg, i * sizeof(IdValuePair) + offsetof(IdValuePair, value));
if (property.type == JSVAL_TYPE_OBJECT) {
Address valueAddress(propertiesReg, i * sizeof(IdValuePair) + offsetof(IdValuePair, value));
Label notObject;
masm.branchTestObject(Assembler::NotEqual, valueAddress, &notObject);
Register valueObject = masm.extractObject(valueAddress, scratch1);
masm.branchPtrInNurseryChunk(Assembler::Equal, valueObject, scratch2, &postBarrier);
masm.bind(&notObject);
} else {
MOZ_ASSERT(property.type == JSVAL_TYPE_STRING);
Label notString;
masm.branchTestString(Assembler::NotEqual, valueAddress, &notString);
masm.unboxString(valueAddress, scratch1);
masm.branchPtrInNurseryChunk(Assembler::Equal, scratch1, scratch2, &postBarrier);
masm.bind(&notString);
}
}

2
js/src/vm/UnboxedObject.h
Просмотреть файл

@ -39,7 +39,7 @@ UnboxedTypeNeedsPreBarrier(JSValueType type)
static inline bool
UnboxedTypeNeedsPostBarrier(JSValueType type)
{
return type == JSVAL_TYPE_OBJECT;
return type == JSVAL_TYPE_STRING || type == JSVAL_TYPE_OBJECT;
}
// Class tracking information specific to unboxed objects.