diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h index 73db96d347a8..5e1501ceef9b 100644 --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -823,6 +823,18 @@ class MacroAssembler : public MacroAssemblerSpecific // On ARM, the chip must have hardware division instructions. inline void remainder32(Register rhs, Register srcDest, bool isUnsigned) PER_SHARED_ARCH; + // Perform an integer division, returning the integer part rounded toward zero. + // rhs must not be zero, and the division must not overflow. The remainder + // is stored into the third argument register here. + // + // This variant preserves registers, and doesn't require hardware division + // instructions on ARM (will call out to a runtime routine). + // + // rhs is preserved, srdDest and remOutput are clobbered. + void flexibleDivMod32(Register rhs, Register srcDest, Register remOutput, + bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) + DEFINED_ON(mips_shared, arm, arm64, x86_shared); + inline void divFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; inline void divDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; diff --git a/js/src/jit/arm/Assembler-arm.h b/js/src/jit/arm/Assembler-arm.h index 8031687c08e5..03eaed15fd88 100644 --- a/js/src/jit/arm/Assembler-arm.h +++ b/js/src/jit/arm/Assembler-arm.h @@ -159,6 +159,15 @@ static constexpr Register StackPointer = sp; static constexpr Register FramePointer = r11; static constexpr Register ReturnReg = r0; static constexpr Register64 ReturnReg64(r1, r0); + +// The attribute '__value_in_regs' alters the calling convention of a function so that +// a structure of up to four elements can be returned via the argument registers rather +// than being written to memory. +static constexpr Register ReturnRegVal0 = IntArgReg0; +static constexpr Register ReturnRegVal1 = IntArgReg1; +static constexpr Register ReturnRegVal2 = IntArgReg2; +static constexpr Register ReturnRegVal3 = IntArgReg3; + static constexpr FloatRegister ReturnFloat32Reg = { FloatRegisters::d0, VFPRegister::Single }; static constexpr FloatRegister ReturnDoubleReg = { FloatRegisters::d0, VFPRegister::Double}; static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg; diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp index 83469549fe52..3a215a2397c1 100644 --- a/js/src/jit/arm/MacroAssembler-arm.cpp +++ b/js/src/jit/arm/MacroAssembler-arm.cpp @@ -5925,6 +5925,48 @@ MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, Regist addDouble(scratchDouble, dest); } +extern "C" { + extern MOZ_EXPORT int64_t __aeabi_idivmod(int,int); + extern MOZ_EXPORT int64_t __aeabi_uidivmod(int,int); +} + +void +MacroAssembler::flexibleDivMod32(Register rhs, Register lhsOutput, Register remOutput, + bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) +{ + // Currently this helper can't handle this situation. + MOZ_ASSERT(lhsOutput != rhs); + + if (HasIDIV()) { + mov(lhsOutput, remOutput); + remainder32(rhs, remOutput, isUnsigned); + quotient32(rhs, lhsOutput, isUnsigned); + } else { + // Ensure that the output registers are saved and restored properly, + MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal0)); + MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal1)); + PushRegsInMask(volatileLiveRegs); + + { + ScratchRegisterScope scratch(*this); + setupUnalignedABICall(scratch); + } + passABIArg(lhsOutput); + passABIArg(rhs); + callWithABI(isUnsigned ? JS_FUNC_TO_DATA_PTR(void*, __aeabi_uidivmod) : + JS_FUNC_TO_DATA_PTR(void*, __aeabi_idivmod), + MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + mov(ReturnRegVal1, remOutput); + mov(ReturnRegVal0, lhsOutput); + + LiveRegisterSet ignore; + ignore.add(remOutput); + ignore.add(lhsOutput); + PopRegsInMaskIgnore(volatileLiveRegs, ignore); + } +} + // ======================================================================== // Spectre Mitigations. diff --git a/js/src/jit/arm64/MacroAssembler-arm64.cpp b/js/src/jit/arm64/MacroAssembler-arm64.cpp index 59d8dd8b2c68..993ab91c8ca9 100644 --- a/js/src/jit/arm64/MacroAssembler-arm64.cpp +++ b/js/src/jit/arm64/MacroAssembler-arm64.cpp @@ -1860,6 +1860,26 @@ MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& atomicEffectOp(arrayType, sync, op, value, mem, temp); } +void +MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest, Register remOutput, + bool isUnsigned, const LiveRegisterSet&) +{ + vixl::UseScratchRegisterScope temps(this); + ARMRegister scratch = temps.AcquireW(); + ARMRegister src = temps.AcquireW(); + + // Preserve src for remainder computation + Mov(src, ARMRegister(srcDest, 32)); + + if (isUnsigned) + Udiv(ARMRegister(srcDest, 32), src, ARMRegister(rhs, 32)); + else + Sdiv(ARMRegister(srcDest, 32), src, ARMRegister(rhs, 32)); + //Compute remainder + Mul(scratch, ARMRegister(srcDest, 32), ARMRegister(rhs, 32)); + Sub(ARMRegister(remOutput, 32), src, scratch); +} + // ======================================================================== // Spectre Mitigations. diff --git a/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp b/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp index c29965119034..4735d7752046 100644 --- a/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp +++ b/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp @@ -2784,6 +2784,18 @@ MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& atomicEffectOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp, maskTemp); } +void +MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest, Register remOutput, + bool isUnsigned, const LiveRegisterSet&) +{ + if (isUnsigned) + as_divu(srcDest, rhs); + else + as_div(srcDest, rhs); + as_mfhi(remOutput); + as_mflo(srcDest); +} + // ======================================================================== // Spectre Mitigations. diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp index 95c1dfc7fa23..c72fa619e594 100644 --- a/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp +++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp @@ -8,6 +8,7 @@ #include "jit/JitFrames.h" #include "jit/MacroAssembler.h" +#include "jit/MoveEmitter.h" #include "jit/MacroAssembler-inl.h" @@ -281,6 +282,108 @@ MacroAssembler::comment(const char* msg) masm.comment(msg); } +class MOZ_RAII ScopedMoveResolution +{ + MacroAssembler& masm_; + MoveResolver& resolver_; + + public: + explicit ScopedMoveResolution(MacroAssembler& masm) + : masm_(masm), + resolver_(masm.moveResolver()) + { + + } + + void addMove(Register src, Register dest) { + if (src != dest) + masm_.propagateOOM(resolver_.addMove(MoveOperand(src), MoveOperand(dest), MoveOp::GENERAL)); + } + + ~ScopedMoveResolution() { + masm_.propagateOOM(resolver_.resolve()); + if (masm_.oom()) + return; + + resolver_.sortMemoryToMemoryMoves(); + + MoveEmitter emitter(masm_); + emitter.emit(resolver_); + emitter.finish(); + } + +}; + +// This operation really consists of five phases, in order to enforce the restriction that +// on x86_shared, srcDest must be eax and edx will be clobbered. +// +// Input: { rhs, lhsOutput } +// +// [PUSH] Preserve registers +// [MOVE] Generate moves to specific registers +// +// [DIV] Input: { regForRhs, EAX } +// [DIV] extend EAX into EDX +// [DIV] x86 Division operator +// [DIV] Ouptut: { EAX, EDX } +// +// [MOVE] Move specific registers to outputs +// [POP] Restore registers +// +// Output: { lhsOutput, remainderOutput } +void +MacroAssembler::flexibleDivMod32(Register rhs, Register lhsOutput, Register remOutput, + bool isUnsigned, const LiveRegisterSet&) +{ + // Currently this helper can't handle this situation. + MOZ_ASSERT(lhsOutput != rhs); + MOZ_ASSERT(lhsOutput != remOutput); + + // Choose a register that is not edx, or eax to hold the rhs; + // ebx is chosen arbitrarily, and will be preserved if necessary. + Register regForRhs = (rhs == eax || rhs == edx) ? ebx : rhs; + + // Add registers we will be clobbering as live, but + // also remove the set we do not restore. + LiveRegisterSet preserve; + preserve.add(edx); + preserve.add(eax); + preserve.add(regForRhs); + + preserve.takeUnchecked(lhsOutput); + preserve.takeUnchecked(remOutput); + + PushRegsInMask(preserve); + + // Marshal Registers For operation + { + ScopedMoveResolution resolution(*this); + resolution.addMove(rhs, regForRhs); + resolution.addMove(lhsOutput, eax); + } + if (oom()) + return; + + // Sign extend eax into edx to make (edx:eax): idiv/udiv are 64-bit. + if (isUnsigned) { + mov(ImmWord(0), edx); + udiv(regForRhs); + } else { + cdq(); + idiv(regForRhs); + } + + { + ScopedMoveResolution resolution(*this); + resolution.addMove(eax, lhsOutput); + resolution.addMove(edx, remOutput); + } + if (oom()) + return; + + PopRegsInMask(preserve); +} + // =============================================================== // Stack manipulation functions. diff --git a/js/src/jsapi-tests/testJitMacroAssembler.cpp b/js/src/jsapi-tests/testJitMacroAssembler.cpp index 0fd4daee9566..cce1385bfc17 100644 --- a/js/src/jsapi-tests/testJitMacroAssembler.cpp +++ b/js/src/jsapi-tests/testJitMacroAssembler.cpp @@ -29,7 +29,7 @@ typedef void (*EnterTest)(); static bool Prepare(MacroAssembler& masm) { - AllocatableRegisterSet regs(RegisterSet::Volatile()); + AllocatableRegisterSet regs(RegisterSet::All()); LiveRegisterSet save(regs.asLiveSet()); masm.PushRegsInMask(save); return true; @@ -37,7 +37,7 @@ static bool Prepare(MacroAssembler& masm) static bool Execute(JSContext* cx, MacroAssembler& masm) { - AllocatableRegisterSet regs(RegisterSet::Volatile()); + AllocatableRegisterSet regs(RegisterSet::All()); LiveRegisterSet save(regs.asLiveSet()); masm.PopRegsInMask(save); masm.ret(); // Add return statement to be sure. @@ -58,6 +58,59 @@ static bool Execute(JSContext* cx, MacroAssembler& masm) return true; } +BEGIN_TEST(testJitMacroAssembler_flexibleDivMod) +{ + StackMacroAssembler masm(cx); + + if (!Prepare(masm)) + return false; + + // Test case divides 9/2; + const uintptr_t quotient_result = 4; + const uintptr_t remainder_result = 1; + const uintptr_t dividend = 9; + const uintptr_t divisor = 2; + + AllocatableGeneralRegisterSet leftOutputHandSides(GeneralRegisterSet::All()); + + while (!leftOutputHandSides.empty()) { + Register lhsOutput = leftOutputHandSides.takeAny(); + + AllocatableGeneralRegisterSet rightHandSides(GeneralRegisterSet::All()); + while (!rightHandSides.empty()) { + Register rhs = rightHandSides.takeAny(); + + AllocatableGeneralRegisterSet remainders(GeneralRegisterSet::All()); + while (!remainders.empty()) { + Register remainderOutput = remainders.takeAny(); + if (lhsOutput == rhs || lhsOutput == remainderOutput || rhs == remainderOutput) + continue; + + AllocatableRegisterSet regs(RegisterSet::Volatile()); + LiveRegisterSet save(regs.asLiveSet()); + + Label next, fail; + masm.mov(ImmWord(dividend), lhsOutput); + masm.mov(ImmWord(divisor), rhs); + masm.flexibleDivMod32(rhs, lhsOutput, remainderOutput, false, save); + masm.branch32(Assembler::NotEqual, AbsoluteAddress("ient_result), lhsOutput, &fail); + masm.branch32(Assembler::NotEqual, AbsoluteAddress(&remainder_result), remainderOutput, &fail); + // Ensure RHS was not clobbered + masm.branch32(Assembler::NotEqual, AbsoluteAddress(&divisor), rhs, &fail); + masm.jump(&next); + masm.bind(&fail); + masm.printf("Failed"); + masm.breakpoint(); + + masm.bind(&next); + } + } + } + + return Execute(cx, masm); +} +END_TEST(testJitMacroAssembler_flexibleDivMod) + BEGIN_TEST(testJitMacroAssembler_truncateDoubleToInt64) { StackMacroAssembler masm(cx);