diff --git a/js/src/methodjit/Compiler.cpp b/js/src/methodjit/Compiler.cpp index 6cf35137f2b6..19b60fe6b69b 100644 --- a/js/src/methodjit/Compiler.cpp +++ b/js/src/methodjit/Compiler.cpp @@ -602,7 +602,7 @@ mjit::Compiler::generateMethod() OpcodeStatus &opinfo = analysis[PC]; frame.setInTryBlock(opinfo.inTryBlock); if (opinfo.nincoming || opinfo.trap) { - frame.forgetEverything(opinfo.stackDepth); + frame.syncAndForgetEverything(opinfo.stackDepth); opinfo.safePoint = true; } jumpMap[uint32(PC - script->code)] = masm.label(); @@ -683,7 +683,7 @@ mjit::Compiler::generateMethod() BEGIN_CASE(JSOP_GOTO) { /* :XXX: this isn't really necessary if we follow the branch. */ - frame.forgetEverything(); + frame.syncAndForgetEverything(); Jump j = masm.jump(); jumpAndTrace(j, PC + GET_JUMP_OFFSET(PC)); } @@ -786,7 +786,7 @@ mjit::Compiler::generateMethod() /* Branch is never taken, don't bother doing anything. */ if (result) { - frame.forgetEverything(); + frame.syncAndForgetEverything(); Jump j = masm.jump(); jumpAndTrace(j, target); } @@ -1105,10 +1105,10 @@ mjit::Compiler::generateMethod() END_CASE(JSOP_AND) BEGIN_CASE(JSOP_TABLESWITCH) - frame.forgetEverything(); + frame.syncAndForgetEverything(); masm.move(ImmPtr(PC), Registers::ArgReg1); - /* prepareStubCall() is not needed due to forgetEverything() */ + /* prepareStubCall() is not needed due to syncAndForgetEverything() */ stubCall(stubs::TableSwitch); frame.pop(); @@ -1118,10 +1118,10 @@ mjit::Compiler::generateMethod() END_CASE(JSOP_TABLESWITCH) BEGIN_CASE(JSOP_LOOKUPSWITCH) - frame.forgetEverything(); + frame.syncAndForgetEverything(); masm.move(ImmPtr(PC), Registers::ArgReg1); - /* prepareStubCall() is not needed due to forgetEverything() */ + /* prepareStubCall() is not needed due to syncAndForgetEverything() */ stubCall(stubs::LookupSwitch); frame.pop(); @@ -1213,12 +1213,23 @@ mjit::Compiler::generateMethod() END_CASE(JSOP_GETLOCAL) BEGIN_CASE(JSOP_SETLOCAL) - BEGIN_CASE(JSOP_SETLOCALPOP) - frame.storeLocal(GET_SLOTNO(PC)); - if (op == JSOP_SETLOCALPOP) + { + jsbytecode *next = &PC[JSOP_SETLOCAL_LENGTH]; + bool pop = JSOp(*next) == JSOP_POP && !analysis[next].nincoming; + frame.storeLocal(GET_SLOTNO(PC), pop); + if (pop) { frame.pop(); + PC += JSOP_SETLOCAL_LENGTH + JSOP_POP_LENGTH; + break; + } + } END_CASE(JSOP_SETLOCAL) + BEGIN_CASE(JSOP_SETLOCALPOP) + frame.storeLocal(GET_SLOTNO(PC), true); + frame.pop(); + END_CASE(JSOP_SETLOCALPOP) + BEGIN_CASE(JSOP_UINT16) frame.push(Value(Int32Value((int32_t) GET_UINT16(PC)))); END_CASE(JSOP_UINT16) @@ -1361,7 +1372,7 @@ mjit::Compiler::generateMethod() if (fun) { JSLocalKind localKind = fun->lookupLocal(cx, inner->atom, NULL); if (localKind != JSLOCAL_NONE) - frame.forgetEverything(); + frame.syncAndForgetEverything(); } prepareStubCall(Uses(0)); @@ -1428,6 +1439,7 @@ mjit::Compiler::generateMethod() END_CASE(JSOP_LAMBDA) BEGIN_CASE(JSOP_TRY) + frame.syncAndForgetEverything(); END_CASE(JSOP_TRY) BEGIN_CASE(JSOP_GETFCSLOT) @@ -1552,7 +1564,7 @@ mjit::Compiler::generateMethod() /* For now, don't bother doing anything for this opcode. */ JSObject *obj = script->getObject(fullAtomIndex(PC)); - frame.forgetEverything(); + frame.syncAndForgetEverything(); masm.move(ImmPtr(obj), Registers::ArgReg1); uint32 n = js_GetEnterBlockStackDefs(cx, script, PC); stubCall(stubs::EnterBlock); @@ -1821,14 +1833,14 @@ mjit::Compiler::emitReturn() /* There will always be a call object. */ prepareStubCall(Uses(0)); stubCall(stubs::PutCallObject); - frame.throwaway(); + frame.discardFrame(); } else { /* if (hasCallObj() || hasArgsObj()) stubs::PutActivationObjects() */ Jump putObjs = masm.branchTest32(Assembler::NonZero, Address(JSFrameReg, JSStackFrame::offsetOfFlags()), Imm32(JSFRAME_HAS_CALL_OBJ | JSFRAME_HAS_ARGS_OBJ)); stubcc.linkExit(putObjs, Uses(frame.frameDepth())); - frame.throwaway(); + frame.discardFrame(); stubcc.leave(); stubcc.call(stubs::PutActivationObjects); @@ -1914,6 +1926,8 @@ mjit::Compiler::interruptCheckHelper() Jump noInterrupt = stubcc.masm.branchTest32(Assembler::Zero, flag); #endif + frame.freeReg(reg); + frame.sync(stubcc.masm, Uses(0)); stubcc.masm.move(ImmPtr(PC), Registers::ArgReg1); stubcc.call(stubs::Interrupt); @@ -1923,8 +1937,6 @@ mjit::Compiler::interruptCheckHelper() #ifdef JS_THREADSAFE stubcc.linkRejoin(noInterrupt); #endif - - frame.freeReg(reg); } void @@ -2016,7 +2028,9 @@ mjit::Compiler::inlineCallHelper(uint32 argc, bool callingNew) * registers we've preserved. */ frame.syncAndKill(Registers(Registers::AvailRegs), Uses(argc + 2)); - frame.resetRegState(); + frame.unpinKilledReg(dataReg); + if (typeReg.isSet()) + frame.unpinKilledReg(typeReg.reg()); Registers tempRegs; @@ -2280,7 +2294,7 @@ mjit::Compiler::emitStubCmpOp(BoolStub stub, jsbytecode *target, JSOp fused) } else { JS_ASSERT(fused == JSOP_IFEQ || fused == JSOP_IFNE); - frame.forgetEverything(); + frame.syncAndForgetEverything(); Assembler::Condition cond = (fused == JSOP_IFEQ) ? Assembler::Zero : Assembler::NonZero; @@ -3680,7 +3694,7 @@ mjit::Compiler::iterMore() /* Get props_cursor, test */ RegisterID T2 = frame.allocReg(); - frame.forgetEverything(); + frame.syncAndForgetEverything(); masm.loadPtr(Address(T1, offsetof(NativeIterator, props_cursor)), T2); masm.loadPtr(Address(T1, offsetof(NativeIterator, props_end)), T1); Jump jFast = masm.branchPtr(Assembler::LessThan, T2, T1); @@ -3917,8 +3931,7 @@ mjit::Compiler::jsop_setgname(uint32 index) mic.shape); masm.move(ImmPtr(obj), objReg); } else { - objReg = frame.tempRegForData(objFe); - frame.pinReg(objReg); + objReg = frame.copyDataIntoReg(objFe); RegisterID reg = frame.allocReg(); masm.loadShape(objReg, reg); @@ -4012,8 +4025,7 @@ mjit::Compiler::jsop_setgname(uint32 index) JS_ASSERT(mic.patchValueOffset == masm.differenceBetween(mic.load, masm.label())); #endif - if (objFe->isConstant()) - frame.freeReg(objReg); + frame.freeReg(objReg); frame.popn(2); if (mic.u.name.dataConst) { frame.push(v); diff --git a/js/src/methodjit/FastArithmetic.cpp b/js/src/methodjit/FastArithmetic.cpp index e319e818e9ed..e9406f8ee7e0 100644 --- a/js/src/methodjit/FastArithmetic.cpp +++ b/js/src/methodjit/FastArithmetic.cpp @@ -1000,7 +1000,7 @@ mjit::Compiler::jsop_equality_int_string(JSOp op, BoolStub stub, jsbytecode *tar frame.pop(); frame.pop(); - frame.throwaway(); + frame.discardFrame(); /* Start of the slow path for equality stub call. */ Label stubCall = stubcc.masm.label(); @@ -1286,7 +1286,7 @@ mjit::Compiler::jsop_relational_double(JSOp op, BoolStub stub, jsbytecode *targe stubcc.call(stub); frame.popn(2); - frame.forgetEverything(); + frame.syncAndForgetEverything(); Jump j = masm.branchDouble(dblCond, fpLeft, fpRight); @@ -1453,7 +1453,12 @@ mjit::Compiler::jsop_relational_full(JSOp op, BoolStub stub, jsbytecode *target, frame.pinReg(reg.reg()); frame.popn(2); - frame.forgetEverything(); + + frame.syncAndKillEverything(); + frame.unpinKilledReg(cmpReg); + if (reg.isSet()) + frame.unpinKilledReg(reg.reg()); + frame.syncAndForgetEverything(); /* Operands could have been reordered, so use cmpOp. */ Assembler::Condition i32Cond; diff --git a/js/src/methodjit/FastOps.cpp b/js/src/methodjit/FastOps.cpp index 7f3616390413..d324515201b6 100644 --- a/js/src/methodjit/FastOps.cpp +++ b/js/src/methodjit/FastOps.cpp @@ -660,7 +660,7 @@ mjit::Compiler::jsop_equality(JSOp op, BoolStub stub, jsbytecode *target, JSOp f */ if (target) { - frame.forgetEverything(); + frame.syncAndForgetEverything(); if ((op == JSOP_EQ && fused == JSOP_IFNE) || (op == JSOP_NE && fused == JSOP_IFEQ)) { @@ -907,8 +907,7 @@ mjit::Compiler::booleanJumpScript(JSOp op, jsbytecode *target) type.setReg(frame.copyTypeIntoReg(fe)); data.setReg(frame.copyDataIntoReg(fe)); - /* :FIXME: Can something more lightweight be used? */ - frame.forgetEverything(); + frame.syncAndForgetEverything(); Assembler::Condition cond = (op == JSOP_IFNE || op == JSOP_OR) ? Assembler::NonZero @@ -995,7 +994,7 @@ mjit::Compiler::jsop_ifneq(JSOp op, jsbytecode *target) if (op == JSOP_IFEQ) b = !b; if (b) { - frame.forgetEverything(); + frame.syncAndForgetEverything(); jumpAndTrace(masm.jump(), target); } return; @@ -1015,7 +1014,7 @@ mjit::Compiler::jsop_andor(JSOp op, jsbytecode *target) /* Short-circuit. */ if ((op == JSOP_OR && b == JS_TRUE) || (op == JSOP_AND && b == JS_FALSE)) { - frame.forgetEverything(); + frame.syncAndForgetEverything(); jumpAndTrace(masm.jump(), target); } diff --git a/js/src/methodjit/FrameEntry.h b/js/src/methodjit/FrameEntry.h index 662bfab82389..cb411488f38a 100644 --- a/js/src/methodjit/FrameEntry.h +++ b/js/src/methodjit/FrameEntry.h @@ -128,6 +128,7 @@ class FrameEntry void track(uint32 index) { clear(); index_ = index; + tracked = true; } void clear() { @@ -210,6 +211,14 @@ class FrameEntry copy = fe; } + inline bool isTracked() const { + return tracked; + } + + inline void untrack() { + tracked = false; + } + private: JSValueType knownType; jsval_layout v_; @@ -219,7 +228,8 @@ class FrameEntry FrameEntry *copy; bool copied; bool isNumber; - char padding[2]; + bool tracked; + char padding[1]; }; } /* namespace mjit */ diff --git a/js/src/methodjit/FrameState-inl.h b/js/src/methodjit/FrameState-inl.h index 09dbfd468f85..234463399161 100644 --- a/js/src/methodjit/FrameState-inl.h +++ b/js/src/methodjit/FrameState-inl.h @@ -43,16 +43,13 @@ namespace js { namespace mjit { -inline FrameEntry * -FrameState::addToTracker(uint32 index) +inline void +FrameState::addToTracker(FrameEntry *fe) { - JS_ASSERT(!base[index]); - FrameEntry *fe = &entries[index]; - base[index] = fe; + JS_ASSERT(!fe->isTracked()); fe->track(tracker.nentries); tracker.add(fe); JS_ASSERT(tracker.nentries <= script->nslots); - return fe; } inline FrameEntry * @@ -60,9 +57,9 @@ FrameState::peek(int32 depth) { JS_ASSERT(depth < 0); JS_ASSERT(sp + depth >= spBase); - FrameEntry *fe = sp[depth]; - if (!fe) { - fe = addToTracker(indexOf(depth)); + FrameEntry *fe = &sp[depth]; + if (!fe->isTracked()) { + addToTracker(fe); fe->resetSynced(); } return fe; @@ -89,11 +86,13 @@ inline JSC::MacroAssembler::RegisterID FrameState::allocReg() { RegisterID reg; - if (!freeRegs.empty()) + if (!freeRegs.empty()) { reg = freeRegs.takeAnyReg(); - else + } else { reg = evictSomeReg(); - regstate[reg].fe = NULL; + regstate[reg].forget(); + } + return reg; } @@ -101,11 +100,13 @@ inline JSC::MacroAssembler::RegisterID FrameState::allocReg(uint32 mask) { RegisterID reg; - if (freeRegs.hasRegInMask(mask)) + if (freeRegs.hasRegInMask(mask)) { reg = freeRegs.takeRegInMask(mask); - else + } else { reg = evictSomeReg(mask); - regstate[reg].fe = NULL; + regstate[reg].forget(); + } + return reg; } @@ -113,11 +114,15 @@ inline JSC::MacroAssembler::RegisterID FrameState::allocReg(FrameEntry *fe, RematInfo::RematType type) { RegisterID reg; - if (!freeRegs.empty()) + if (!freeRegs.empty()) { reg = freeRegs.takeAnyReg(); - else + } else { reg = evictSomeReg(); - regstate[reg] = RegisterState(fe, type); + regstate[reg].forget(); + } + + regstate[reg].associate(fe, type); + return reg; } @@ -162,8 +167,8 @@ FrameState::pop() { JS_ASSERT(sp > spBase); - FrameEntry *fe = *--sp; - if (!fe) + FrameEntry *fe = --sp; + if (!fe->isTracked()) return; forgetAllRegs(fe); @@ -172,7 +177,8 @@ FrameState::pop() inline void FrameState::freeReg(RegisterID reg) { - JS_ASSERT(regstate[reg].fe == NULL); + JS_ASSERT(!regstate[reg].usedBy()); + freeRegs.putReg(reg); } @@ -183,28 +189,30 @@ FrameState::forgetReg(RegisterID reg) * Important: Do not touch the fe here. We can peephole optimize away * loads and stores by re-using the contents of old FEs. */ - JS_ASSERT_IF(regstate[reg].fe, !regstate[reg].fe->isCopy()); - freeRegs.putReg(reg); + JS_ASSERT_IF(regstate[reg].fe(), !regstate[reg].fe()->isCopy()); + + if (!regstate[reg].isPinned()) { + regstate[reg].forget(); + freeRegs.putReg(reg); + } } inline void -FrameState::forgetEverything(uint32 newStackDepth) +FrameState::syncAndForgetEverything(uint32 newStackDepth) { - forgetEverything(); + syncAndForgetEverything(); sp = spBase + newStackDepth; } inline FrameEntry * FrameState::rawPush() { - JS_ASSERT(unsigned(sp - base) < nargs + script->nslots); + JS_ASSERT(unsigned(sp - entries) < nargs + script->nslots); - sp++; + if (!sp->isTracked()) + addToTracker(sp); - if (FrameEntry *fe = sp[-1]) - return fe; - - return addToTracker(&sp[-1] - base); + return sp++; } inline void @@ -217,10 +225,9 @@ FrameState::push(const Value &v) inline void FrameState::pushSynced() { + if (sp->isTracked()) + sp->resetSynced(); sp++; - - if (FrameEntry *fe = sp[-1]) - fe->resetSynced(); } inline void @@ -242,7 +249,7 @@ FrameState::pushSynced(JSValueType type, RegisterID reg) fe->data.sync(); fe->setType(type); fe->data.setRegister(reg); - regstate[reg] = RegisterState(fe, RematInfo::DATA); + regstate[reg].associate(fe, RematInfo::DATA); } inline void @@ -281,8 +288,8 @@ FrameState::pushRegs(RegisterID type, RegisterID data) fe->resetUnsynced(); fe->type.setRegister(type); fe->data.setRegister(data); - regstate[type] = RegisterState(fe, RematInfo::TYPE); - regstate[data] = RegisterState(fe, RematInfo::DATA); + regstate[type].associate(fe, RematInfo::TYPE); + regstate[data].associate(fe, RematInfo::DATA); } inline void @@ -295,7 +302,7 @@ FrameState::pushTypedPayload(JSValueType type, RegisterID payload) fe->resetUnsynced(); fe->setType(type); fe->data.setRegister(payload); - regstate[payload] = RegisterState(fe, RematInfo::DATA); + regstate[payload].associate(fe, RematInfo::DATA); } inline void @@ -320,7 +327,7 @@ FrameState::pushNumber(MaybeRegisterID payload, bool asInt32) if (payload.isSet()) { fe->data.unsync(); fe->data.setRegister(payload.reg()); - regstate[payload.reg()] = RegisterState(fe, RematInfo::DATA); + regstate[payload.reg()].associate(fe, RematInfo::DATA); } else { fe->data.setMemory(); } @@ -339,7 +346,7 @@ FrameState::pushInt32(RegisterID payload) fe->isNumber = true; fe->data.unsync(); fe->data.setRegister(payload); - regstate[payload] = RegisterState(fe, RematInfo::DATA); + regstate[payload].associate(fe, RematInfo::DATA); } inline void @@ -362,13 +369,13 @@ FrameState::pushUntypedPayload(JSValueType type, RegisterID payload) fe->setNotCopied(); fe->setCopyOf(NULL); fe->data.setRegister(payload); - regstate[payload] = RegisterState(fe, RematInfo::DATA); + regstate[payload].associate(fe, RematInfo::DATA); } inline JSC::MacroAssembler::RegisterID FrameState::tempRegForType(FrameEntry *fe, RegisterID fallback) { - JS_ASSERT(regstate[fallback].fe == NULL); + JS_ASSERT(!regstate[fallback].fe()); if (fe->isCopy()) fe = fe->copyOf(); @@ -435,7 +442,7 @@ FrameState::tempRegInMaskForData(FrameEntry *fe, uint32 mask) return old; /* Keep the old register pinned. */ - regstate[old].fe = NULL; + regstate[old].forget(); reg = allocReg(mask); masm.move(old, reg); freeReg(old); @@ -443,7 +450,7 @@ FrameState::tempRegInMaskForData(FrameEntry *fe, uint32 mask) reg = allocReg(mask); masm.loadPayload(addressOf(fe), reg); } - regstate[reg] = RegisterState(fe, RematInfo::DATA); + regstate[reg].associate(fe, RematInfo::DATA); fe->data.setRegister(reg); return reg; } @@ -618,28 +625,31 @@ inline FrameEntry * FrameState::getLocal(uint32 slot) { uint32 index = nargs + slot; - if (FrameEntry *fe = base[index]) - return fe; - FrameEntry *fe = addToTracker(index); - fe->resetSynced(); + FrameEntry *fe = &entries[index]; + if (!fe->isTracked()) { + addToTracker(fe); + fe->resetSynced(); + } return fe; } inline void FrameState::pinReg(RegisterID reg) { - JS_ASSERT(!freeRegs.hasReg(reg)); - JS_ASSERT(regstate[reg].fe); - regstate[reg].save = regstate[reg].fe; - regstate[reg].fe = NULL; + regstate[reg].pin(); } inline void FrameState::unpinReg(RegisterID reg) { - JS_ASSERT(!freeRegs.hasReg(reg)); - JS_ASSERT(!regstate[reg].fe); - regstate[reg].fe = regstate[reg].save; + regstate[reg].unpin(); +} + +inline void +FrameState::unpinKilledReg(RegisterID reg) +{ + regstate[reg].unpinUnsafe(); + freeRegs.putReg(reg); } inline void @@ -651,12 +661,6 @@ FrameState::forgetAllRegs(FrameEntry *fe) forgetReg(fe->data.reg()); } -inline FrameEntry * -FrameState::tosFe() const -{ - return &entries[uint32(sp - base)]; -} - inline void FrameState::swapInTracker(FrameEntry *lhs, FrameEntry *rhs) { @@ -711,8 +715,8 @@ FrameState::pushLocal(uint32 n) * SETLOCAL equivocation of stack slots, and let expressions, just * weakly assert on the fixed local vars. */ - FrameEntry *fe = base[localIndex(n)]; - if (fe && n < script->nfixed) { + FrameEntry *fe = &locals[n]; + if (fe->isTracked() && n < script->nfixed) { JS_ASSERT(fe->type.inMemory()); JS_ASSERT(fe->data.inMemory()); } diff --git a/js/src/methodjit/FrameState.cpp b/js/src/methodjit/FrameState.cpp index acc2f09ab4da..a6210be0501c 100644 --- a/js/src/methodjit/FrameState.cpp +++ b/js/src/methodjit/FrameState.cpp @@ -64,19 +64,17 @@ FrameState::init(uint32 nargs) uint32 nslots = script->nslots + nargs; if (!nslots) { - sp = spBase = locals = args = base = NULL; + sp = spBase = locals = args = NULL; return true; } - uint32 nlocals = script->nslots; - if ((eval = script->usesEval || cx->compartment->debugMode)) - nlocals = 0; + eval = script->usesEval || cx->compartment->debugMode; - uint8 *cursor = (uint8 *)cx->malloc(sizeof(FrameEntry) * nslots + // entries[] - sizeof(FrameEntry *) * nslots + // base[] - sizeof(FrameEntry *) * nslots + // tracker.entries[] - sizeof(uint32) * nlocals // escaping[] - ); + size_t totalBytes = sizeof(FrameEntry) * nslots + // entries[] + sizeof(FrameEntry *) * nslots + // tracker.entries + (eval ? 0 : sizeof(uint32) * nslots); // closedVars[] + + uint8 *cursor = (uint8 *)cx->calloc(totalBytes); if (!cursor) return false; @@ -86,22 +84,21 @@ FrameState::init(uint32 nargs) entries = (FrameEntry *)cursor; cursor += sizeof(FrameEntry) * nslots; - base = (FrameEntry **)cursor; - args = base; - locals = base + nargs; + args = entries; + locals = args + nargs; spBase = locals + script->nfixed; sp = spBase; - memset(base, 0, sizeof(FrameEntry *) * nslots); - cursor += sizeof(FrameEntry *) * nslots; tracker.entries = (FrameEntry **)cursor; cursor += sizeof(FrameEntry *) * nslots; - if (nlocals) { + if (!eval && nslots) { escaping = (uint32 *)cursor; - memset(escaping, 0, sizeof(uint32) * nlocals); + cursor += sizeof(uint32) * nslots; } + JS_ASSERT(reinterpret_cast(entries) + totalBytes == cursor); + return true; } @@ -110,19 +107,20 @@ FrameState::takeReg(RegisterID reg) { if (freeRegs.hasReg(reg)) { freeRegs.takeReg(reg); + JS_ASSERT(!regstate[reg].usedBy()); } else { - JS_ASSERT(regstate[reg].fe); + JS_ASSERT(regstate[reg].fe()); evictReg(reg); + regstate[reg].forget(); } - regstate[reg].fe = NULL; } void FrameState::evictReg(RegisterID reg) { - FrameEntry *fe = regstate[reg].fe; + FrameEntry *fe = regstate[reg].fe(); - if (regstate[reg].type == RematInfo::TYPE) { + if (regstate[reg].type() == RematInfo::TYPE) { if (!fe->type.synced()) { syncType(fe, addressOf(fe), masm); fe->type.sync(); @@ -153,7 +151,7 @@ FrameState::evictSomeReg(uint32 mask) continue; /* Register is not owned by the FrameState. */ - FrameEntry *fe = regstate[i].fe; + FrameEntry *fe = regstate[i].fe(); if (!fe) continue; @@ -163,11 +161,11 @@ FrameState::evictSomeReg(uint32 mask) #endif fallback = reg; - if (regstate[i].type == RematInfo::TYPE && fe->type.synced()) { + if (regstate[i].type() == RematInfo::TYPE && fe->type.synced()) { fe->type.setMemory(); return fallback; } - if (regstate[i].type == RematInfo::DATA && fe->data.synced()) { + if (regstate[i].type() == RematInfo::DATA && fe->data.synced()) { fe->data.setMemory(); return fallback; } @@ -181,24 +179,42 @@ FrameState::evictSomeReg(uint32 mask) void -FrameState::forgetEverything() +FrameState::syncAndForgetEverything() { syncAndKill(Registers(Registers::AvailRegs), Uses(frameDepth())); - - throwaway(); + forgetEverything(); } - void -FrameState::throwaway() +FrameState::resetInternalState() { for (uint32 i = 0; i < tracker.nentries; i++) - base[indexOfFe(tracker[i])] = NULL; + tracker[i]->untrack(); tracker.reset(); freeRegs.reset(); } +void +FrameState::discardFrame() +{ + resetInternalState(); + + memset(regstate, 0, sizeof(regstate)); +} + +void +FrameState::forgetEverything() +{ + resetInternalState(); + +#ifdef DEBUG + for (uint32 i = 0; i < JSC::MacroAssembler::TotalRegisters; i++) { + JS_ASSERT(!regstate[i].usedBy()); + } +#endif +} + void FrameState::storeTo(FrameEntry *fe, Address address, bool popped) { @@ -253,46 +269,49 @@ FrameState::assertValidRegisterState() const { Registers checkedFreeRegs; - FrameEntry *tos = tosFe(); for (uint32 i = 0; i < tracker.nentries; i++) { FrameEntry *fe = tracker[i]; - if (fe >= tos) + if (fe >= sp) continue; JS_ASSERT(i == fe->trackerIndex()); JS_ASSERT_IF(fe->isCopy(), fe->trackerIndex() > fe->copyOf()->trackerIndex()); + JS_ASSERT_IF(fe->isCopy(), fe > fe->copyOf()); JS_ASSERT_IF(fe->isCopy(), !fe->type.inRegister() && !fe->data.inRegister()); - JS_ASSERT_IF(fe->isCopy(), fe->copyOf() < tos); + JS_ASSERT_IF(fe->isCopy(), fe->copyOf() < sp); JS_ASSERT_IF(fe->isCopy(), fe->copyOf()->isCopied()); if (fe->isCopy()) continue; if (fe->type.inRegister()) { checkedFreeRegs.takeReg(fe->type.reg()); - JS_ASSERT(regstate[fe->type.reg()].fe == fe); + JS_ASSERT(regstate[fe->type.reg()].fe() == fe); } if (fe->data.inRegister()) { checkedFreeRegs.takeReg(fe->data.reg()); - JS_ASSERT(regstate[fe->data.reg()].fe == fe); + JS_ASSERT(regstate[fe->data.reg()].fe() == fe); } } JS_ASSERT(checkedFreeRegs == freeRegs); + + for (uint32 i = 0; i < JSC::MacroAssembler::TotalRegisters; i++) { + JS_ASSERT(!regstate[i].isPinned()); + JS_ASSERT_IF(regstate[i].fe(), !freeRegs.hasReg(RegisterID(i))); + JS_ASSERT_IF(regstate[i].fe(), regstate[i].fe()->isTracked()); + } } #endif void -FrameState::syncFancy(Assembler &masm, Registers avail, uint32 resumeAt, +FrameState::syncFancy(Assembler &masm, Registers avail, FrameEntry *resumeAt, FrameEntry *bottom) const { - /* :TODO: can be resumeAt? */ - reifier.reset(&masm, avail, tracker.nentries, bottom); + reifier.reset(&masm, avail, resumeAt, bottom); - FrameEntry *tos = tosFe(); - for (uint32 i = resumeAt; i < tracker.nentries; i--) { - FrameEntry *fe = tracker[i]; - if (fe >= tos) + for (FrameEntry *fe = resumeAt; fe >= bottom; fe--) { + if (!fe->isTracked()) continue; reifier.sync(fe); @@ -302,6 +321,29 @@ FrameState::syncFancy(Assembler &masm, Registers avail, uint32 resumeAt, void FrameState::sync(Assembler &masm, Uses uses) const { + if (!entries) + return; + + /* Sync all registers up-front. */ + for (uint32 i = 0; i < JSC::MacroAssembler::TotalRegisters; i++) { + RegisterID reg = RegisterID(i); + FrameEntry *fe = regstate[reg].usedBy(); + if (!fe) + continue; + + JS_ASSERT(fe->isTracked()); + + if (regstate[reg].type() == RematInfo::DATA) { + JS_ASSERT(fe->data.reg() == reg); + if (!fe->data.synced()) + syncData(fe, addressOf(fe), masm); + } else { + JS_ASSERT(fe->type.reg() == reg); + if (!fe->type.synced()) + syncType(fe, addressOf(fe), masm); + } + } + /* * Keep track of free registers using a bitmask. If we have to drop into * syncFancy(), then this mask will help avoid eviction. @@ -309,35 +351,33 @@ FrameState::sync(Assembler &masm, Uses uses) const Registers avail(freeRegs); Registers temp(Registers::TempRegs); - FrameEntry *tos = tosFe(); - FrameEntry *bottom = tos - uses.nuses; + FrameEntry *bottom = sp - uses.nuses; - if (inTryBlock) - bottom = NULL; - - for (uint32 i = tracker.nentries - 1; i < tracker.nentries; i--) { - FrameEntry *fe = tracker[i]; - if (fe >= tos) + for (FrameEntry *fe = sp - 1; fe >= bottom; fe--) { + if (!fe->isTracked()) continue; Address address = addressOf(fe); if (!fe->isCopy()) { - /* Keep track of registers that can be clobbered. */ - if (fe->data.inRegister()) + /* + * If this |fe| has registers, track them as available. They've + * already been synced. Otherwise, see if a constant needs to be + * synced. + */ + if (fe->data.inRegister()) { avail.putReg(fe->data.reg()); - if (fe->type.inRegister()) - avail.putReg(fe->type.reg()); - - /* Sync. */ - if (!fe->data.synced() && (fe->data.inRegister() || fe >= bottom)) { + } else if (!fe->data.synced()) { syncData(fe, address, masm); if (fe->isConstant()) continue; } - if (!fe->type.synced() && (fe->type.inRegister() || fe >= bottom)) - syncType(fe, addressOf(fe), masm); - } else if (fe >= bottom) { + + if (fe->type.inRegister()) + avail.putReg(fe->type.reg()); + else if (!fe->type.synced()) + syncType(fe, address, masm); + } else { FrameEntry *backing = fe->copyOf(); JS_ASSERT(backing != fe); JS_ASSERT(!backing->isConstant() && !fe->isConstant()); @@ -348,7 +388,7 @@ FrameState::sync(Assembler &masm, Uses uses) const */ if ((!fe->type.synced() && !backing->type.inRegister()) || (!fe->data.synced() && !backing->data.inRegister())) { - syncFancy(masm, avail, i, bottom); + syncFancy(masm, avail, fe, bottom); return; } @@ -371,93 +411,130 @@ FrameState::sync(Assembler &masm, Uses uses) const void FrameState::syncAndKill(Registers kill, Uses uses, Uses ignore) { - /* Backwards, so we can allocate registers to backing slots better. */ - FrameEntry *tos = tosFe(); - FrameEntry *bottom = tos - uses.nuses; + FrameEntry *spStop = sp - ignore.nuses; - tos -= ignore.nuses; + /* Sync all kill-registers up-front. */ + Registers search(kill.freeMask & ~freeRegs.freeMask); + while (!search.empty()) { + RegisterID reg = search.takeAnyReg(); + FrameEntry *fe = regstate[reg].usedBy(); + if (!fe || fe >= spStop) + continue; - if (inTryBlock) - bottom = NULL; + JS_ASSERT(fe->isTracked()); - for (uint32 i = tracker.nentries - 1; i < tracker.nentries; i--) { - FrameEntry *fe = tracker[i]; - if (fe >= tos) + if (regstate[reg].type() == RematInfo::DATA) { + JS_ASSERT(fe->data.reg() == reg); + if (!fe->data.synced()) { + syncData(fe, addressOf(fe), masm); + fe->data.sync(); + } + } else { + JS_ASSERT(fe->type.reg() == reg); + if (!fe->type.synced()) { + syncType(fe, addressOf(fe), masm); + fe->type.sync(); + } + } + } + + uint32 maxvisits = tracker.nentries; + FrameEntry *bottom = sp - uses.nuses; + + for (FrameEntry *fe = sp - 1; fe >= bottom && maxvisits; fe--) { + if (!fe->isTracked()) + continue; + + maxvisits--; + + if (fe >= spStop) continue; Address address = addressOf(fe); FrameEntry *backing = fe; - if (fe->isCopy()) { - if (!inTryBlock && fe < bottom) - continue; + + if (fe->isCopy()) backing = fe->copyOf(); - } - JS_ASSERT_IF(i == 0, !fe->isCopy()); - - bool killData = fe->data.inRegister() && kill.hasReg(fe->data.reg()); - if (!fe->data.synced() && (killData || fe >= bottom)) { + if (!fe->data.synced()) { if (backing != fe && backing->data.inMemory()) tempRegForData(backing); syncData(backing, address, masm); fe->data.sync(); - if (fe->isConstant() && !fe->type.synced()) + if (fe->isConstant() && !fe->type.synced()) { fe->type.sync(); - } - if (killData) { - JS_ASSERT(backing == fe); - JS_ASSERT(fe->data.synced()); - if (regstate[fe->data.reg()].fe) + } else if (fe->data.inRegister() && kill.hasReg(fe->data.reg())) { forgetReg(fe->data.reg()); - fe->data.setMemory(); + fe->data.setMemory(); + } } - bool killType = fe->type.inRegister() && kill.hasReg(fe->type.reg()); - if (!fe->type.synced() && (killType || fe >= bottom)) { + if (!fe->type.synced()) { if (backing != fe && backing->type.inMemory()) tempRegForType(backing); syncType(backing, address, masm); fe->type.sync(); - } - if (killType) { - JS_ASSERT(backing == fe); - JS_ASSERT(fe->type.synced()); - if (regstate[fe->type.reg()].fe) + if (fe->type.inRegister() && kill.hasReg(fe->type.reg())) { forgetReg(fe->type.reg()); - fe->type.setMemory(); + fe->type.setMemory(); + } } } -} -void -FrameState::resetRegState() -{ - freeRegs = Registers(); + /* + * Anything still alive at this point is guaranteed to be synced. However, + * it is necessary to evict temporary registers. + */ + search = Registers(kill.freeMask & ~freeRegs.freeMask); + while (!search.empty()) { + RegisterID reg = search.takeAnyReg(); + FrameEntry *fe = regstate[reg].usedBy(); + if (!fe || fe >= spStop) + continue; + + JS_ASSERT(fe->isTracked()); + + if (regstate[reg].type() == RematInfo::DATA) { + JS_ASSERT(fe->data.reg() == reg); + JS_ASSERT(fe->data.synced()); + fe->data.setMemory(); + } else { + JS_ASSERT(fe->type.reg() == reg); + JS_ASSERT(fe->type.synced()); + fe->type.setMemory(); + } + + forgetReg(reg); + } } void FrameState::merge(Assembler &masm, Changes changes) const { - FrameEntry *tos = tosFe(); - Registers temp(Registers::TempRegs); + Registers search(Registers::AvailRegs & ~freeRegs.freeMask); - for (uint32 i = 0; i < tracker.nentries; i++) { - FrameEntry *fe = tracker[i]; - if (fe >= tos) - continue; + while (!search.empty()) { + RegisterID reg = search.peekReg(); + FrameEntry *fe = regstate[reg].usedBy(); - /* Copies do not have registers. */ - if (fe->isCopy()) { - JS_ASSERT(!fe->data.inRegister()); - JS_ASSERT(!fe->type.inRegister()); + if (!fe) { + search.takeReg(reg); continue; } - if (fe->data.inRegister() && fe->type.inRegister()) + if (fe->data.inRegister() && fe->type.inRegister()) { + search.takeReg(fe->data.reg()); + search.takeReg(fe->type.reg()); masm.loadValueAsComponents(addressOf(fe), fe->type.reg(), fe->data.reg()); - else if (fe->data.inRegister()) - masm.loadPayload(addressOf(fe), fe->data.reg()); - else if (fe->type.inRegister()) - masm.loadTypeTag(addressOf(fe), fe->type.reg()); + } else { + if (fe->data.inRegister()) { + search.takeReg(fe->data.reg()); + masm.loadPayload(addressOf(fe), fe->data.reg()); + } + if (fe->type.inRegister()) { + search.takeReg(fe->type.reg()); + masm.loadTypeTag(addressOf(fe), fe->type.reg()); + } + } } } @@ -488,9 +565,9 @@ FrameState::copyDataIntoReg(FrameEntry *fe, RegisterID hint) reg = allocReg(); masm.move(hint, reg); fe->data.setRegister(reg); - regstate[reg] = regstate[hint]; + regstate[reg].associate(regstate[hint].fe(), RematInfo::DATA); } - regstate[hint].fe = NULL; + regstate[hint].forget(); } else { pinReg(reg); takeReg(hint); @@ -513,7 +590,7 @@ FrameState::copyDataIntoReg(Assembler &masm, FrameEntry *fe) if (!fe->data.synced()) syncData(fe, addressOf(fe), masm); fe->data.setMemory(); - regstate[reg].fe = NULL; + regstate[reg].forget(); } else { RegisterID newReg = allocReg(); masm.move(reg, newReg); @@ -546,7 +623,7 @@ FrameState::copyTypeIntoReg(FrameEntry *fe) if (!fe->type.synced()) syncType(fe, addressOf(fe), masm); fe->type.setMemory(); - regstate[reg].fe = NULL; + regstate[reg].forget(); } else { RegisterID newReg = allocReg(); masm.move(reg, newReg); @@ -633,7 +710,7 @@ FrameState::ownRegForType(FrameEntry *fe) syncType(backing, addressOf(backing), masm); reg = backing->type.reg(); backing->type.setMemory(); - moveOwnership(reg, NULL); + regstate[reg].forget(); } else { reg = allocReg(); masm.move(backing->type.reg(), reg); @@ -643,10 +720,11 @@ FrameState::ownRegForType(FrameEntry *fe) if (fe->type.inRegister()) { reg = fe->type.reg(); + /* Remove ownership of this register. */ - JS_ASSERT(regstate[reg].fe == fe); - JS_ASSERT(regstate[reg].type == RematInfo::TYPE); - regstate[reg].fe = NULL; + JS_ASSERT(regstate[reg].fe() == fe); + JS_ASSERT(regstate[reg].type() == RematInfo::TYPE); + regstate[reg].forget(); fe->type.invalidate(); } else { JS_ASSERT(fe->type.inMemory()); @@ -676,7 +754,7 @@ FrameState::ownRegForData(FrameEntry *fe) syncData(backing, addressOf(backing), masm); reg = backing->data.reg(); backing->data.setMemory(); - moveOwnership(reg, NULL); + regstate[reg].forget(); } else { reg = allocReg(); masm.move(backing->data.reg(), reg); @@ -696,9 +774,9 @@ FrameState::ownRegForData(FrameEntry *fe) if (fe->data.inRegister()) { reg = fe->data.reg(); /* Remove ownership of this register. */ - JS_ASSERT(regstate[reg].fe == fe); - JS_ASSERT(regstate[reg].type == RematInfo::DATA); - regstate[reg].fe = NULL; + JS_ASSERT(regstate[reg].fe() == fe); + JS_ASSERT(regstate[reg].type() == RematInfo::DATA); + regstate[reg].forget(); fe->data.invalidate(); } else { JS_ASSERT(fe->data.inMemory()); @@ -739,34 +817,14 @@ FrameState::pushCopyOf(uint32 index) } FrameEntry * -FrameState::uncopy(FrameEntry *original) +FrameState::walkTrackerForUncopy(FrameEntry *original) { - JS_ASSERT(original->isCopied()); - - /* - * Copies have two critical invariants: - * 1) The backing store precedes all copies in the tracker. - * 2) The backing store of a copy cannot be popped from the stack - * while the copy is still live. - * - * Maintaining this invariant iteratively is kind of hard, so we choose - * the "lowest" copy in the frame up-front. - * - * For example, if the stack is: - * [A, B, C, D] - * And the tracker has: - * [A, D, C, B] - * - * If B, C, and D are copies of A - we will walk the tracker to the end - * and select D, not B (see bug 583684). - */ uint32 firstCopy = InvalidIndex; - FrameEntry *tos = tosFe(); FrameEntry *bestFe = NULL; uint32 ncopies = 0; - for (uint32 i = 0; i < tracker.nentries; i++) { + for (uint32 i = original->trackerIndex() + 1; i < tracker.nentries; i++) { FrameEntry *fe = tracker[i]; - if (fe >= tos) + if (fe >= sp) continue; if (fe->isCopy() && fe->copyOf() == original) { if (firstCopy == InvalidIndex) { @@ -782,12 +840,12 @@ FrameState::uncopy(FrameEntry *original) if (!ncopies) { JS_ASSERT(firstCopy == InvalidIndex); JS_ASSERT(!bestFe); - original->copied = false; return NULL; } JS_ASSERT(firstCopy != InvalidIndex); JS_ASSERT(bestFe); + JS_ASSERT(bestFe > original); /* Mark all extra copies as copies of the new backing index. */ bestFe->setCopyOf(NULL); @@ -795,7 +853,7 @@ FrameState::uncopy(FrameEntry *original) bestFe->setCopied(); for (uint32 i = firstCopy; i < tracker.nentries; i++) { FrameEntry *other = tracker[i]; - if (other >= tos || other == bestFe) + if (other >= sp || other == bestFe) continue; /* The original must be tracked before copies. */ @@ -820,7 +878,80 @@ FrameState::uncopy(FrameEntry *original) bestFe->setNotCopied(); } - FrameEntry *fe = bestFe; + return bestFe; +} + +FrameEntry * +FrameState::walkFrameForUncopy(FrameEntry *original) +{ + FrameEntry *bestFe = NULL; + uint32 ncopies = 0; + + /* It's only necessary to visit as many FEs are being tracked. */ + uint32 maxvisits = tracker.nentries; + + for (FrameEntry *fe = original + 1; fe < sp && maxvisits; fe++) { + if (!fe->isTracked()) + continue; + + maxvisits--; + + if (fe->isCopy() && fe->copyOf() == original) { + if (!bestFe) { + bestFe = fe; + bestFe->setCopyOf(NULL); + } else { + fe->setCopyOf(bestFe); + if (fe->trackerIndex() < bestFe->trackerIndex()) + swapInTracker(bestFe, fe); + } + ncopies++; + } + } + + if (ncopies) + bestFe->setCopied(); + + return bestFe; +} + +FrameEntry * +FrameState::uncopy(FrameEntry *original) +{ + JS_ASSERT(original->isCopied()); + + /* + * Copies have three critical invariants: + * 1) The backing store precedes all copies in the tracker. + * 2) The backing store precedes all copies in the FrameState. + * 3) The backing store of a copy cannot be popped from the stack + * while the copy is still live. + * + * Maintaining this invariant iteratively is kind of hard, so we choose + * the "lowest" copy in the frame up-front. + * + * For example, if the stack is: + * [A, B, C, D] + * And the tracker has: + * [A, D, C, B] + * + * If B, C, and D are copies of A - we will walk the tracker to the end + * and select B, not D (see bug 583684). + * + * Note: |tracker.nentries <= (nslots + nargs)|. However, this walk is + * sub-optimal if |tracker.nentries - original->trackerIndex() > sp - original|. + * With large scripts this may be a problem worth investigating. Note that + * the tracker is walked twice, so we multiply by 2 for pessimism. + */ + FrameEntry *fe; + if ((tracker.nentries - original->trackerIndex()) * 2 > uint32(sp - original)) + fe = walkFrameForUncopy(original); + else + fe = walkTrackerForUncopy(original); + if (!fe) { + original->setNotCopied(); + return NULL; + } /* * Switch the new backing store to the old backing store. During @@ -837,7 +968,7 @@ FrameState::uncopy(FrameEntry *original) tempRegForType(original); fe->type.inherit(original->type); if (fe->type.inRegister()) - moveOwnership(fe->type.reg(), fe); + regstate[fe->type.reg()].reassociate(fe); } else { JS_ASSERT(fe->isTypeKnown()); JS_ASSERT(fe->getKnownType() == original->getKnownType()); @@ -846,7 +977,7 @@ FrameState::uncopy(FrameEntry *original) tempRegForData(original); fe->data.inherit(original->data); if (fe->data.inRegister()) - moveOwnership(fe->data.reg(), fe); + regstate[fe->data.reg()].reassociate(fe); return fe; } @@ -854,52 +985,86 @@ FrameState::uncopy(FrameEntry *original) void FrameState::storeLocal(uint32 n, bool popGuaranteed, bool typeChange) { - FrameEntry *localFe = getLocal(n); - bool cacheable = !eval && !escaping[n]; + FrameEntry *local = getLocal(n); - if (!popGuaranteed && !cacheable) { - JS_ASSERT_IF(base[localIndex(n)] && (!eval || n < script->nfixed), - entries[localIndex(n)].type.inMemory() && - entries[localIndex(n)].data.inMemory()); - Address local(JSFrameReg, sizeof(JSStackFrame) + n * sizeof(Value)); - storeTo(peek(-1), local, false); - forgetAllRegs(getLocal(n)); - localFe->resetSynced(); - return; + storeTop(local, popGuaranteed, typeChange); + + bool closed = eval || escaping[n]; + if (closed || inTryBlock) { + /* Ensure that the local variable remains synced. */ + if (local->isCopy()) { + FrameEntry *backing = local->copyOf(); + if (!local->data.synced()) { + if (backing->data.inMemory()) + tempRegForData(backing); + syncData(backing, addressOf(local), masm); + } + if (!local->type.synced()) { + if (backing->type.inMemory()) + tempRegForType(backing); + syncType(backing, addressOf(local), masm); + } + } else if (local->isConstant()) { + if (!local->data.synced()) + syncData(local, addressOf(local), masm); + } else { + if (!local->data.synced()) { + syncData(local, addressOf(local), masm); + local->data.sync(); + } + if (!local->type.synced()) { + syncType(local, addressOf(local), masm); + local->type.sync(); + } + if (closed) + forgetEntry(local); + } + + if (closed) + local->resetSynced(); } +} - bool wasSynced = localFe->type.synced(); +void +FrameState::forgetEntry(FrameEntry *fe) +{ + if (fe->isCopied()) { + uncopy(fe); + if (!fe->isCopied()) + forgetAllRegs(fe); + } else { + forgetAllRegs(fe); + } +} +void +FrameState::storeTop(FrameEntry *target, bool popGuaranteed, bool typeChange) +{ + bool wasSynced = target->type.synced(); /* Detect something like (x = x) which is a no-op. */ FrameEntry *top = peek(-1); - if (top->isCopy() && top->copyOf() == localFe) { - JS_ASSERT(localFe->isCopied()); + if (top->isCopy() && top->copyOf() == target) { + JS_ASSERT(target->isCopied()); return; } /* Completely invalidate the local variable. */ - if (localFe->isCopied()) { - uncopy(localFe); - if (!localFe->isCopied()) - forgetAllRegs(localFe); - } else { - forgetAllRegs(localFe); - } - - localFe->resetUnsynced(); + forgetEntry(target); + target->resetUnsynced(); /* Constants are easy to propagate. */ if (top->isConstant()) { - localFe->setCopyOf(NULL); - localFe->setNotCopied(); - localFe->setConstant(Jsvalify(top->getValue())); + target->setCopyOf(NULL); + target->setNotCopied(); + target->setConstant(Jsvalify(top->getValue())); return; } /* - * When dealing with copies, there are two important invariants: + * When dealing with copies, there are three important invariants: * * 1) The backing store precedes all copies in the tracker. + * 2) The backing store precedes all copies in the FrameState. * 2) The backing store of a local is never a stack slot, UNLESS the local * variable itself is a stack slot (blocks) that precedes the stack * slot. @@ -909,24 +1074,23 @@ FrameState::storeLocal(uint32 n, bool popGuaranteed, bool typeChange) * condition does not hold, force it to hold by swapping in-place. */ FrameEntry *backing = top; + bool copied = false; if (top->isCopy()) { backing = top->copyOf(); JS_ASSERT(backing->trackerIndex() < top->trackerIndex()); - uint32 backingIndex = indexOfFe(backing); - uint32 tol = uint32(spBase - base); - if (backingIndex < tol || backingIndex < localIndex(n)) { + if (backing < target) { /* local.idx < backing.idx means local cannot be a copy yet */ - if (localFe->trackerIndex() < backing->trackerIndex()) - swapInTracker(backing, localFe); - localFe->setNotCopied(); - localFe->setCopyOf(backing); + if (target->trackerIndex() < backing->trackerIndex()) + swapInTracker(backing, target); + target->setNotCopied(); + target->setCopyOf(backing); if (backing->isTypeKnown()) - localFe->setType(backing->getKnownType()); + target->setType(backing->getKnownType()); else - localFe->type.invalidate(); - localFe->data.invalidate(); - localFe->isNumber = backing->isNumber; + target->type.invalidate(); + target->data.invalidate(); + target->isNumber = backing->isNumber; return; } @@ -948,13 +1112,14 @@ FrameState::storeLocal(uint32 n, bool popGuaranteed, bool typeChange) * but even so there's a quick workaround. We take all copies of the * backing fe, and redirect them to be copies of the destination. */ - FrameEntry *tos = tosFe(); for (uint32 i = backing->trackerIndex() + 1; i < tracker.nentries; i++) { FrameEntry *fe = tracker[i]; - if (fe >= tos) + if (fe >= sp) continue; - if (fe->isCopy() && fe->copyOf() == backing) - fe->setCopyOf(localFe); + if (fe->isCopy() && fe->copyOf() == backing) { + fe->setCopyOf(target); + copied = true; + } } } backing->setNotCopied(); @@ -964,50 +1129,50 @@ FrameState::storeLocal(uint32 n, bool popGuaranteed, bool typeChange) * consistent ordering - all copies of |backing| are tracked after * |backing|. Transitively, only one swap is needed. */ - if (backing->trackerIndex() < localFe->trackerIndex()) - swapInTracker(backing, localFe); + if (backing->trackerIndex() < target->trackerIndex()) + swapInTracker(backing, target); /* * Move the backing store down - we spill registers here, but we could be * smarter and re-use the type reg. */ RegisterID reg = tempRegForData(backing); - localFe->data.setRegister(reg); - moveOwnership(reg, localFe); + target->data.setRegister(reg); + regstate[reg].reassociate(target); if (typeChange) { if (backing->isTypeKnown()) { - localFe->setType(backing->getKnownType()); + target->setType(backing->getKnownType()); } else { RegisterID reg = tempRegForType(backing); - localFe->type.setRegister(reg); - moveOwnership(reg, localFe); + target->type.setRegister(reg); + regstate[reg].reassociate(target); } } else { if (!wasSynced) - masm.storeTypeTag(ImmType(backing->getKnownType()), addressOf(localFe)); - localFe->type.setMemory(); + masm.storeTypeTag(ImmType(backing->getKnownType()), addressOf(target)); + target->type.setMemory(); } if (!backing->isTypeKnown()) backing->type.invalidate(); backing->data.invalidate(); - backing->setCopyOf(localFe); - backing->isNumber = localFe->isNumber; - localFe->setCopied(); + backing->setCopyOf(target); + backing->isNumber = target->isNumber; - if (!cacheable) { - /* TODO: x64 optimization */ - if (!localFe->type.synced()) - syncType(localFe, addressOf(localFe), masm); - if (!localFe->data.synced()) - syncData(localFe, addressOf(localFe), masm); - forgetAllRegs(localFe); - localFe->type.setMemory(); - localFe->data.setMemory(); - } + JS_ASSERT(top->copyOf() == target); - JS_ASSERT(top->copyOf() == localFe); + /* + * Right now, |backing| is a copy of |target| (note the reversal), but + * |target| is not marked as copied. This is an optimization so uncopy() + * may avoid frame traversal. + * + * There are two cases where we must set the copy bit, however: + * - The fixup phase redirected more copies to |target|. + * - An immediate pop is not guaranteed. + */ + if (copied || !popGuaranteed) + target->setCopied(); } void @@ -1015,7 +1180,7 @@ FrameState::shimmy(uint32 n) { JS_ASSERT(sp - n >= spBase); int32 depth = 0 - int32(n); - storeLocal(uint32(&sp[depth - 1] - locals), true); + storeTop(&sp[depth - 1], true); popn(n); } @@ -1024,7 +1189,7 @@ FrameState::shift(int32 n) { JS_ASSERT(n < 0); JS_ASSERT(sp + n - 1 >= spBase); - storeLocal(uint32(&sp[n - 1] - locals), true); + storeTop(&sp[n - 1], true); pop(); } diff --git a/js/src/methodjit/FrameState.h b/js/src/methodjit/FrameState.h index 7ed851b4f580..2bb1ee154fff 100644 --- a/js/src/methodjit/FrameState.h +++ b/js/src/methodjit/FrameState.h @@ -175,22 +175,103 @@ class FrameState uint32 nentries; }; + /* + * Some RegisterState invariants. + * + * If |fe| is non-NULL, |save| is NULL. + * If |save| is non-NULL, |fe| is NULL. + * That is, both |fe| and |save| cannot be non-NULL. + * + * If either |fe| or |save| is non-NULL, the register is not in freeRegs. + * If both |fe| and |save| are NULL, the register is either in freeRegs, + * or owned by the compiler. + */ struct RegisterState { - RegisterState() + RegisterState() : fe_(NULL), save_(NULL) { } RegisterState(FrameEntry *fe, RematInfo::RematType type) - : fe(fe), type(type) - { } + : fe_(fe), save_(NULL), type_(type) + { + JS_ASSERT(!save_); + } + bool isPinned() const { + assertConsistency(); + return !!save_; + } + + void assertConsistency() const { + JS_ASSERT_IF(fe_, !save_); + JS_ASSERT_IF(save_, !fe_); + } + + FrameEntry *fe() const { + assertConsistency(); + return fe_; + } + + RematInfo::RematType type() const { + assertConsistency(); + return type_; + } + + FrameEntry *usedBy() const { + if (fe_) + return fe_; + return save_; + } + + void associate(FrameEntry *fe, RematInfo::RematType type) { + JS_ASSERT(!fe_); + JS_ASSERT(!save_); + + fe_ = fe; + type_ = type; + JS_ASSERT(!save_); + } + + /* Change ownership. */ + void reassociate(FrameEntry *fe) { + assertConsistency(); + JS_ASSERT(fe); + + fe_ = fe; + } + + /* Unassociate this register from the FE. */ + void forget() { + JS_ASSERT(fe_); + fe_ = NULL; + JS_ASSERT(!save_); + } + + void pin() { + assertConsistency(); + save_ = fe_; + fe_ = NULL; + } + + void unpin() { + assertConsistency(); + fe_ = save_; + save_ = NULL; + } + + void unpinUnsafe() { + assertConsistency(); + save_ = NULL; + } + + private: /* FrameEntry owning this register, or NULL if not owned by a frame. */ - FrameEntry *fe; + FrameEntry *fe_; /* Hack - simplifies register allocation for pairs. */ - FrameEntry *save; + FrameEntry *save_; /* Part of the FrameEntry that owns the FE. */ - RematInfo::RematType type; + RematInfo::RematType type_; }; public: @@ -505,9 +586,10 @@ class FrameState void storeTo(FrameEntry *fe, Address address, bool popHint); /* - * Stores the top stack slot back to a local variable. + * Stores the top stack slot back to a slot. */ void storeLocal(uint32 n, bool popGuaranteed = false, bool typeChange = true); + void storeTop(FrameEntry *target, bool popGuaranteed = false, bool typeChange = true); /* * Restores state from a slow path. @@ -526,28 +608,34 @@ class FrameState void syncAndKill(Registers kill, Uses uses, Uses ignored); void syncAndKill(Registers kill, Uses uses) { syncAndKill(kill, uses, Uses(0)); } - /* - * Reset the register state. - */ - void resetRegState(); + /* Syncs and kills everything. */ + void syncAndKillEverything() { + syncAndKill(Registers(Registers::AvailRegs), Uses(frameDepth())); + } /* * Clear all tracker entries, syncing all outstanding stores in the process. * The stack depth is in case some merge points' edges did not immediately * precede the current instruction. */ - inline void forgetEverything(uint32 newStackDepth); + inline void syncAndForgetEverything(uint32 newStackDepth); /* * Same as above, except the stack depth is not changed. This is used for * branching opcodes. */ - void forgetEverything(); + void syncAndForgetEverything(); /* * Throw away the entire frame state, without syncing anything. + * This can only be called after a syncAndKill() against all registers. */ - void throwaway(); + void forgetEverything(); + + /* + * Discard the entire framestate forcefully. + */ + void discardFrame(); /* * Mark an existing slot with a type. @@ -603,8 +691,9 @@ class FrameState /* * Marks a register such that it cannot be spilled by the register - * allocator. Any pinned registers must be unpinned at the end of the op. - * Note: This function should only be used on registers tied to FEs. + * allocator. Any pinned registers must be unpinned at the end of the op, + * no matter what. In addition, pinReg() can only be used on registers + * which are associated with FrameEntries. */ inline void pinReg(RegisterID reg); @@ -613,6 +702,11 @@ class FrameState */ inline void unpinReg(RegisterID reg); + /* + * Same as unpinReg(), but does not restore the FrameEntry. + */ + inline void unpinKilledReg(RegisterID reg); + /* * Dups the top item on the stack. */ @@ -639,7 +733,6 @@ class FrameState */ uint32 stackDepth() const { return sp - spBase; } uint32 frameDepth() const { return stackDepth() + script->nfixed; } - inline FrameEntry *tosFe() const; #ifdef DEBUG void assertValidRegisterState() const; @@ -687,7 +780,7 @@ class FrameState RegisterID evictSomeReg(uint32 mask); void evictReg(RegisterID reg); inline FrameEntry *rawPush(); - inline FrameEntry *addToTracker(uint32 index); + inline void addToTracker(FrameEntry *fe); inline void syncType(const FrameEntry *fe, Address to, Assembler &masm) const; inline void syncData(const FrameEntry *fe, Address to, Assembler &masm) const; inline FrameEntry *getLocal(uint32 slot); @@ -695,9 +788,10 @@ class FrameState inline void swapInTracker(FrameEntry *lhs, FrameEntry *rhs); inline uint32 localIndex(uint32 n); void pushCopyOf(uint32 index); - void syncFancy(Assembler &masm, Registers avail, uint32 resumeAt, + void syncFancy(Assembler &masm, Registers avail, FrameEntry *resumeAt, FrameEntry *bottom) const; inline bool tryFastDoubleLoad(FrameEntry *fe, FPRegisterID fpReg, Assembler &masm) const; + void resetInternalState(); /* * "Uncopies" the backing store of a FrameEntry that has been copied. The @@ -708,25 +802,29 @@ class FrameState * Later addition: uncopy() returns the first copy found. */ FrameEntry *uncopy(FrameEntry *original); + FrameEntry *walkTrackerForUncopy(FrameEntry *original); + FrameEntry *walkFrameForUncopy(FrameEntry *original); + + /* + * All registers in the FE are forgotten. If it is copied, it is uncopied + * beforehand. + */ + void forgetEntry(FrameEntry *fe); FrameEntry *entryFor(uint32 index) const { - JS_ASSERT(base[index]); + JS_ASSERT(entries[index].isTracked()); return &entries[index]; } - void moveOwnership(RegisterID reg, FrameEntry *newFe) { - regstate[reg].fe = newFe; - } - RegisterID evictSomeReg() { return evictSomeReg(Registers::AvailRegs); } uint32 indexOf(int32 depth) { - return uint32((sp + depth) - base); + return uint32((sp + depth) - entries); } - uint32 indexOfFe(FrameEntry *fe) { + uint32 indexOfFe(FrameEntry *fe) const { return uint32(fe - entries); } @@ -742,20 +840,17 @@ class FrameState /* Cache of FrameEntry objects. */ FrameEntry *entries; - /* Base pointer of the FrameEntry vector. */ - FrameEntry **base; - /* Base pointer for arguments. */ - FrameEntry **args; + FrameEntry *args; /* Base pointer for local variables. */ - FrameEntry **locals; + FrameEntry *locals; /* Base pointer for the stack. */ - FrameEntry **spBase; + FrameEntry *spBase; /* Dynamic stack pointer. */ - FrameEntry **sp; + FrameEntry *sp; /* Vector of tracked slot indexes. */ Tracker tracker; diff --git a/js/src/methodjit/ImmutableSync.cpp b/js/src/methodjit/ImmutableSync.cpp index 2fb01bc9a5e1..46260aef3029 100644 --- a/js/src/methodjit/ImmutableSync.cpp +++ b/js/src/methodjit/ImmutableSync.cpp @@ -45,7 +45,7 @@ using namespace js; using namespace js::mjit; ImmutableSync::ImmutableSync(JSContext *cx, const FrameState &frame) - : cx(cx), entries(NULL), frame(frame) + : cx(cx), entries(NULL), frame(frame), generation(0) { } @@ -57,19 +57,18 @@ ImmutableSync::~ImmutableSync() bool ImmutableSync::init(uint32 nentries) { - entries = (SyncEntry *)cx->malloc(sizeof(SyncEntry) * nentries); + entries = (SyncEntry *)cx->calloc(sizeof(SyncEntry) * nentries); return !!entries; } void -ImmutableSync::reset(Assembler *masm, Registers avail, uint32 n, - FrameEntry *bottom) +ImmutableSync::reset(Assembler *masm, Registers avail, FrameEntry *top, FrameEntry *bottom) { this->avail = avail; - this->nentries = n; this->masm = masm; + this->top = top; this->bottom = bottom; - memset(entries, 0, sizeof(SyncEntry) * nentries); + this->generation++; memset(regs, 0, sizeof(regs)); } @@ -92,17 +91,10 @@ ImmutableSync::allocReg() if (!regs[i]) { /* If the frame does not own this register, take it! */ - FrameEntry *fe = frame.regstate[i].fe; + FrameEntry *fe = frame.regstate[i].fe(); if (!fe) return reg; - /* - * The Reifier does not own this register, but the frame does. - * This must mean that we've not yet processed this entry, and - * that it's data has not been clobbered. - */ - JS_ASSERT(fe->trackerIndex() < nentries); - evictFromFrame = i; /* @@ -115,18 +107,14 @@ ImmutableSync::allocReg() } if (evictFromFrame != FrameState::InvalidIndex) { - FrameEntry *fe = frame.regstate[evictFromFrame].fe; + FrameEntry *fe = frame.regstate[evictFromFrame].fe(); SyncEntry &e = entryFor(fe); - if (frame.regstate[evictFromFrame].type == RematInfo::TYPE) { + if (frame.regstate[evictFromFrame].type() == RematInfo::TYPE) { JS_ASSERT(!e.typeClobbered); - e.typeSynced = true; e.typeClobbered = true; - masm->storeTypeTag(fe->type.reg(), frame.addressOf(fe)); } else { JS_ASSERT(!e.dataClobbered); - e.dataSynced = true; e.dataClobbered = true; - masm->storePayload(fe->data.reg(), frame.addressOf(fe)); } return RegisterID(evictFromFrame); } @@ -150,39 +138,38 @@ ImmutableSync::allocReg() inline ImmutableSync::SyncEntry & ImmutableSync::entryFor(FrameEntry *fe) { - JS_ASSERT(fe->trackerIndex() < nentries); - return entries[fe->trackerIndex()]; + JS_ASSERT(fe <= top); + SyncEntry &e = entries[frame.indexOfFe(fe)]; + if (e.generation != generation) + e.reset(generation); + return e; } void ImmutableSync::sync(FrameEntry *fe) { - JS_ASSERT(nentries); +#ifdef DEBUG + top = fe; +#endif + if (fe->isCopy()) syncCopy(fe); else syncNormal(fe); - nentries--; } bool ImmutableSync::shouldSyncType(FrameEntry *fe, SyncEntry &e) { - if (fe->type.inRegister() && !e.typeClobbered) - return true; - if (e.hasTypeReg) - return true; - return frame.inTryBlock || fe >= bottom; + /* Registers are synced up-front. */ + return !fe->type.synced() && !fe->type.inRegister(); } bool ImmutableSync::shouldSyncData(FrameEntry *fe, SyncEntry &e) { - if (fe->data.inRegister() && !e.dataClobbered) - return true; - if (e.hasDataReg) - return true; - return frame.inTryBlock || fe >= bottom; + /* Registers are synced up-front. */ + return !fe->data.synced() && !fe->data.inRegister(); } JSC::MacroAssembler::RegisterID @@ -216,8 +203,7 @@ ImmutableSync::ensureDataReg(FrameEntry *fe, SyncEntry &e) void ImmutableSync::syncCopy(FrameEntry *fe) { - if (!frame.inTryBlock && fe < bottom) - return; + JS_ASSERT(fe >= bottom); FrameEntry *backing = fe->copyOf(); SyncEntry &e = entryFor(backing); @@ -254,7 +240,7 @@ ImmutableSync::syncNormal(FrameEntry *fe) e.type = fe->getKnownType(); } - if (!fe->data.synced() && !e.dataSynced && shouldSyncData(fe, e)) { + if (shouldSyncData(fe, e)) { if (fe->isConstant()) { masm->storeValue(fe->getValue(), addr); return; @@ -262,7 +248,7 @@ ImmutableSync::syncNormal(FrameEntry *fe) masm->storePayload(ensureDataReg(fe, e), addr); } - if (!fe->type.synced() && !e.typeSynced && shouldSyncType(fe, e)) { + if (shouldSyncType(fe, e)) { if (e.learnedType) masm->storeTypeTag(ImmType(e.type), addr); else @@ -272,14 +258,14 @@ ImmutableSync::syncNormal(FrameEntry *fe) if (e.hasDataReg) { avail.putReg(e.dataReg); regs[e.dataReg] = NULL; - } else if (!e.dataClobbered && fe->data.inRegister() && frame.regstate[fe->data.reg()].fe) { + } else if (!e.dataClobbered && fe->data.inRegister() && frame.regstate[fe->data.reg()].fe()) { avail.putReg(fe->data.reg()); } if (e.hasTypeReg) { avail.putReg(e.typeReg); regs[e.typeReg] = NULL; - } else if (!e.typeClobbered && fe->type.inRegister() && frame.regstate[fe->type.reg()].fe) { + } else if (!e.typeClobbered && fe->type.inRegister() && frame.regstate[fe->type.reg()].fe()) { avail.putReg(fe->type.reg()); } } diff --git a/js/src/methodjit/ImmutableSync.h b/js/src/methodjit/ImmutableSync.h index 26b884520026..28234af2f6db 100644 --- a/js/src/methodjit/ImmutableSync.h +++ b/js/src/methodjit/ImmutableSync.h @@ -70,16 +70,24 @@ class ImmutableSync * * They are separated for readability. */ - bool dataSynced; - bool typeSynced; + uint32 generation; bool dataClobbered; bool typeClobbered; - RegisterID dataReg; - RegisterID typeReg; bool hasDataReg; bool hasTypeReg; bool learnedType; + RegisterID dataReg; + RegisterID typeReg; JSValueType type; + + void reset(uint32 gen) { + dataClobbered = false; + typeClobbered = false; + hasDataReg = false; + hasTypeReg = false; + learnedType = false; + generation = gen; + } }; public: @@ -87,8 +95,7 @@ class ImmutableSync ~ImmutableSync(); bool init(uint32 nentries); - void reset(Assembler *masm, Registers avail, uint32 n, - FrameEntry *bottom); + void reset(Assembler *masm, Registers avail, FrameEntry *top, FrameEntry *bottom); void sync(FrameEntry *fe); private: @@ -111,7 +118,9 @@ class ImmutableSync Registers avail; Assembler *masm; SyncEntry *regs[Assembler::TotalRegisters]; + FrameEntry *top; FrameEntry *bottom; + uint32 generation; }; } /* namespace mjit */ diff --git a/js/src/methodjit/MachineRegs.h b/js/src/methodjit/MachineRegs.h index a8d88d3da87a..2fe1b6dfa5d8 100644 --- a/js/src/methodjit/MachineRegs.h +++ b/js/src/methodjit/MachineRegs.h @@ -190,11 +190,16 @@ struct Registers { return !(freeMask & mask); } - RegisterID takeAnyReg() { + RegisterID peekReg() { JS_ASSERT(!empty()); int ireg; JS_FLOOR_LOG2(ireg, freeMask); RegisterID reg = (RegisterID)ireg; + return reg; + } + + RegisterID takeAnyReg() { + RegisterID reg = peekReg(); takeReg(reg); return reg; }