diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index 86865ebaa39b..4dece28fad91 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -15323,7 +15323,7 @@ void CodeGenerator::emitIonToWasmCallBase(LIonToWasmCallBase* lir) { const wasm::FuncExport& funcExport = lir->mir()->funcExport(); const wasm::FuncType& sig = funcExport.funcType(); - ABIArgGenerator abi; + WasmABIArgGenerator abi; for (size_t i = 0; i < lir->numOperands(); i++) { MIRType argMir; switch (sig.args()[i].kind()) { diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp index 4427f25c22fd..d1c9ec718707 100644 --- a/js/src/jit/MacroAssembler.cpp +++ b/js/src/jit/MacroAssembler.cpp @@ -3049,8 +3049,8 @@ void MacroAssembler::freeStack(Register amount) { addToStackPtr(amount); } // =============================================================== // ABI function calls. - -void MacroAssembler::setupABICall() { +template +void MacroAssembler::setupABICallHelper() { #ifdef DEBUG MOZ_ASSERT(!inCall_); inCall_ = true; @@ -3061,7 +3061,7 @@ void MacroAssembler::setupABICall() { #endif // Reinitialize the ABIArg generator. - abiArgs_ = ABIArgGenerator(); + abiArgs_ = ABIArgGeneratorT(); #if defined(JS_CODEGEN_ARM) // On ARM, we need to know what ABI we are using, either in the @@ -3084,9 +3084,13 @@ void MacroAssembler::setupABICall() { #endif } +void MacroAssembler::setupNativeABICall() { + setupABICallHelper(); +} + void MacroAssembler::setupWasmABICall() { MOZ_ASSERT(IsCompilingWasm(), "non-wasm should use setupAlignedABICall"); - setupABICall(); + setupABICallHelper(); #if defined(JS_CODEGEN_ARM) // The builtin thunk does the FP -> GPR moving on soft-FP, so @@ -3098,7 +3102,7 @@ void MacroAssembler::setupWasmABICall() { void MacroAssembler::setupAlignedABICall() { MOZ_ASSERT(!IsCompilingWasm(), "wasm should use setupWasmABICall"); - setupABICall(); + setupNativeABICall(); dynamicAlignment_ = false; #if defined(JS_CODEGEN_ARM64) diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h index cfe1009d3600..cebfd9891fa9 100644 --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -667,7 +667,12 @@ class MacroAssembler : public MacroAssemblerSpecific { private: // Reinitialize the variables which have to be cleared before making a call // with callWithABI. - void setupABICall(); + template + void setupABICallHelper(); + + // Reinitialize the variables which have to be cleared before making a call + // with native abi. + void setupNativeABICall(); // Reserve the stack and resolve the arguments move. void callWithABIPre(uint32_t* stackAdjust, @@ -4335,9 +4340,9 @@ static inline MIRType ToMIRType(ABIArgType argType) { // Helper for generatePreBarrier. inline DynFn JitMarkFunction(MIRType type); -template -class ABIArgIter { - ABIArgGenerator gen_; +template +class ABIArgIterBase { + ABIArgGeneratorT gen_; const VecT& types_; unsigned i_; @@ -4346,7 +4351,9 @@ class ABIArgIter { } public: - explicit ABIArgIter(const VecT& types) : types_(types), i_(0) { settle(); } + explicit ABIArgIterBase(const VecT& types) : types_(types), i_(0) { + settle(); + } void operator++(int) { MOZ_ASSERT(!done()); i_++; @@ -4376,6 +4383,29 @@ class ABIArgIter { } }; +// This is not an alias because we want to allow class template argument +// deduction. +template +class ABIArgIter : public ABIArgIterBase { + public: + explicit ABIArgIter(const VecT& types) + : ABIArgIterBase(types) {} +}; + +class WasmABIArgGenerator : public ABIArgGenerator { + public: + WasmABIArgGenerator() { + increaseStackOffset(wasm::FrameWithTls::sizeWithoutFrame()); + } +}; + +template +class WasmABIArgIter : public ABIArgIterBase { + public: + explicit WasmABIArgIter(const VecT& types) + : ABIArgIterBase(types) {} +}; + } // namespace jit } // namespace js diff --git a/js/src/jit/arm/Assembler-arm.h b/js/src/jit/arm/Assembler-arm.h index d7e3cb067234..f47c92a7bf00 100644 --- a/js/src/jit/arm/Assembler-arm.h +++ b/js/src/jit/arm/Assembler-arm.h @@ -147,6 +147,7 @@ class ABIArgGenerator { ABIArg next(MIRType argType); ABIArg& current() { return current_; } uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; } }; bool IsUnaligned(const wasm::MemoryAccessDesc& access); diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp index eb740bcd2872..a58e5acd984f 100644 --- a/js/src/jit/arm/MacroAssembler-arm.cpp +++ b/js/src/jit/arm/MacroAssembler-arm.cpp @@ -4359,7 +4359,7 @@ void MacroAssembler::popReturnAddress() { pop(lr); } // ABI function calls. void MacroAssembler::setupUnalignedABICall(Register scratch) { - setupABICall(); + setupNativeABICall(); dynamicAlignment_ = true; ma_mov(sp, scratch); diff --git a/js/src/jit/arm64/Assembler-arm64.h b/js/src/jit/arm64/Assembler-arm64.h index 0d449502df37..aa0218b995da 100644 --- a/js/src/jit/arm64/Assembler-arm64.h +++ b/js/src/jit/arm64/Assembler-arm64.h @@ -421,6 +421,7 @@ class ABIArgGenerator { ABIArg next(MIRType argType); ABIArg& current() { return current_; } uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; } protected: unsigned intRegIndex_; diff --git a/js/src/jit/arm64/MacroAssembler-arm64.cpp b/js/src/jit/arm64/MacroAssembler-arm64.cpp index c2cf302b34c7..9d376935932f 100644 --- a/js/src/jit/arm64/MacroAssembler-arm64.cpp +++ b/js/src/jit/arm64/MacroAssembler-arm64.cpp @@ -1099,7 +1099,7 @@ void MacroAssembler::popReturnAddress() { // ABI function calls. void MacroAssembler::setupUnalignedABICall(Register scratch) { - setupABICall(); + setupNativeABICall(); dynamicAlignment_ = true; int64_t alignment = ~(int64_t(ABIStackAlignment) - 1); diff --git a/js/src/jit/mips32/Assembler-mips32.h b/js/src/jit/mips32/Assembler-mips32.h index 1dad9d309624..ad5c091798aa 100644 --- a/js/src/jit/mips32/Assembler-mips32.h +++ b/js/src/jit/mips32/Assembler-mips32.h @@ -43,6 +43,8 @@ class ABIArgGenerator { return usedArgSlots_ * sizeof(intptr_t); } + + void increaseStackOffset(uint32_t bytes) { MOZ_CRASH("NYI"); } }; // These registers may be volatile or nonvolatile. diff --git a/js/src/jit/mips32/MacroAssembler-mips32.cpp b/js/src/jit/mips32/MacroAssembler-mips32.cpp index bb329b076a23..49811a30292e 100644 --- a/js/src/jit/mips32/MacroAssembler-mips32.cpp +++ b/js/src/jit/mips32/MacroAssembler-mips32.cpp @@ -2033,7 +2033,7 @@ void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest, void MacroAssembler::setupUnalignedABICall(Register scratch) { MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls"); - setupABICall(); + setupNativeABICall(); dynamicAlignment_ = true; ma_move(scratch, StackPointer); diff --git a/js/src/jit/mips64/Assembler-mips64.h b/js/src/jit/mips64/Assembler-mips64.h index 4ad0db8fc143..6f756b4ab200 100644 --- a/js/src/jit/mips64/Assembler-mips64.h +++ b/js/src/jit/mips64/Assembler-mips64.h @@ -38,6 +38,7 @@ class ABIArgGenerator { return (usedArgSlots_ - 8) * sizeof(int64_t); } + void increaseStackOffset(uint32_t bytes) { MOZ_CRASH("NYI"); } }; // These registers may be volatile or nonvolatile. diff --git a/js/src/jit/mips64/MacroAssembler-mips64.cpp b/js/src/jit/mips64/MacroAssembler-mips64.cpp index 5962f8485753..d86b06abc890 100644 --- a/js/src/jit/mips64/MacroAssembler-mips64.cpp +++ b/js/src/jit/mips64/MacroAssembler-mips64.cpp @@ -1875,7 +1875,7 @@ void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest, void MacroAssembler::setupUnalignedABICall(Register scratch) { MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls"); - setupABICall(); + setupNativeABICall(); dynamicAlignment_ = true; ma_move(scratch, StackPointer); diff --git a/js/src/jit/none/MacroAssembler-none.h b/js/src/jit/none/MacroAssembler-none.h index 62c086a9db2c..b127b87a80cf 100644 --- a/js/src/jit/none/MacroAssembler-none.h +++ b/js/src/jit/none/MacroAssembler-none.h @@ -619,6 +619,7 @@ class ABIArgGenerator { ABIArg next(MIRType) { MOZ_CRASH(); } ABIArg& current() { MOZ_CRASH(); } uint32_t stackBytesConsumedSoFar() const { MOZ_CRASH(); } + void increaseStackOffset(uint32_t) { MOZ_CRASH(); } }; static inline bool GetTempRegForIntArg(uint32_t, uint32_t, Register*) { diff --git a/js/src/jit/x64/Assembler-x64.h b/js/src/jit/x64/Assembler-x64.h index bfeed8af266c..2be31b5387f5 100644 --- a/js/src/jit/x64/Assembler-x64.h +++ b/js/src/jit/x64/Assembler-x64.h @@ -191,6 +191,7 @@ class ABIArgGenerator { ABIArg next(MIRType argType); ABIArg& current() { return current_; } uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; } }; // These registers may be volatile or nonvolatile. diff --git a/js/src/jit/x64/MacroAssembler-x64.cpp b/js/src/jit/x64/MacroAssembler-x64.cpp index b54e22c37e7e..ea2ae72ef124 100644 --- a/js/src/jit/x64/MacroAssembler-x64.cpp +++ b/js/src/jit/x64/MacroAssembler-x64.cpp @@ -335,7 +335,7 @@ void MacroAssembler::subFromStackPtr(Imm32 imm32) { // ABI function calls. void MacroAssembler::setupUnalignedABICall(Register scratch) { - setupABICall(); + setupNativeABICall(); dynamicAlignment_ = true; movq(rsp, scratch); diff --git a/js/src/jit/x86/Assembler-x86.h b/js/src/jit/x86/Assembler-x86.h index ea50e2f3c898..305029d9bdfb 100644 --- a/js/src/jit/x86/Assembler-x86.h +++ b/js/src/jit/x86/Assembler-x86.h @@ -87,6 +87,7 @@ class ABIArgGenerator { ABIArg next(MIRType argType); ABIArg& current() { return current_; } uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } + void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; } }; // These registers may be volatile or nonvolatile. diff --git a/js/src/jit/x86/MacroAssembler-x86.cpp b/js/src/jit/x86/MacroAssembler-x86.cpp index 4235ecccff90..1ef83bc59eeb 100644 --- a/js/src/jit/x86/MacroAssembler-x86.cpp +++ b/js/src/jit/x86/MacroAssembler-x86.cpp @@ -334,7 +334,7 @@ void MacroAssembler::subFromStackPtr(Imm32 imm32) { // ABI function calls. void MacroAssembler::setupUnalignedABICall(Register scratch) { - setupABICall(); + setupNativeABICall(); dynamicAlignment_ = true; movl(esp, scratch); diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp index ac961bb2e5a6..b5099b8d529e 100644 --- a/js/src/wasm/WasmBaselineCompile.cpp +++ b/js/src/wasm/WasmBaselineCompile.cpp @@ -5268,7 +5268,7 @@ class BaseCompiler final : public BaseCompilerInterface { } // Identify GC-managed pointers passed on the stack. - for (ABIArgIter i(args); !i.done(); i++) { + for (WasmABIArgIter i(args); !i.done(); i++) { ABIArg argLoc = *i; if (argLoc.kind() == ABIArg::Stack && args[i.index()] == MIRType::RefOrNull) { @@ -5354,7 +5354,7 @@ class BaseCompiler final : public BaseCompilerInterface { } // Copy arguments from registers to stack. - for (ABIArgIter i(args); !i.done(); i++) { + for (WasmABIArgIter i(args); !i.done(); i++) { if (args.isSyntheticStackResultPointerArg(i.index())) { // If there are stack results and the pointer to stack results // was passed in a register, store it to the stack. @@ -5619,7 +5619,7 @@ class BaseCompiler final : public BaseCompilerInterface { } uint32_t lineOrBytecode; - ABIArgGenerator abi; + WasmABIArgGenerator abi; bool isInterModule; bool usesSystemAbi; #ifdef JS_CODEGEN_ARM diff --git a/js/src/wasm/WasmBaselineCompile.h b/js/src/wasm/WasmBaselineCompile.h index fec2fa675459..70dc873ec66f 100644 --- a/js/src/wasm/WasmBaselineCompile.h +++ b/js/src/wasm/WasmBaselineCompile.h @@ -41,7 +41,7 @@ class BaseLocalIter { const ValTypeVector& locals_; const ArgTypeVector& args_; - jit::ABIArgIter argsIter_; + jit::WasmABIArgIter argsIter_; size_t index_; int32_t frameSize_; int32_t nextFrameSize_; diff --git a/js/src/wasm/WasmGC.cpp b/js/src/wasm/WasmGC.cpp index ed0068524157..48bfd256ddb8 100644 --- a/js/src/wasm/WasmGC.cpp +++ b/js/src/wasm/WasmGC.cpp @@ -128,7 +128,7 @@ bool CreateStackMapForFunctionEntryTrap(const wasm::ArgTypeVector& argTypes, return false; } - for (ABIArgIter i(argTypes); !i.done(); i++) { + for (WasmABIArgIter i(argTypes); !i.done(); i++) { ABIArg argLoc = *i; if (argLoc.kind() == ABIArg::Stack && argTypes[i.index()] == MIRType::RefOrNull) { @@ -183,7 +183,7 @@ bool GenerateStackmapEntriesForTrapExit(const ArgTypeVector& args, return false; } - for (ABIArgIter i(args); !i.done(); i++) { + for (WasmABIArgIter i(args); !i.done(); i++) { if (!i->argInRegister() || i.mirType() != MIRType::RefOrNull) { continue; } diff --git a/js/src/wasm/WasmGC.h b/js/src/wasm/WasmGC.h index 361c58542597..ff52ffe89be9 100644 --- a/js/src/wasm/WasmGC.h +++ b/js/src/wasm/WasmGC.h @@ -261,7 +261,7 @@ class StackMaps { // the complete native-ABI-level call signature. template static inline size_t StackArgAreaSizeUnaligned(const T& argTypes) { - ABIArgIter i(argTypes); + WasmABIArgIter i(argTypes); while (!i.done()) { i++; } @@ -270,7 +270,7 @@ static inline size_t StackArgAreaSizeUnaligned(const T& argTypes) { static inline size_t StackArgAreaSizeUnaligned( const SymbolicAddressSignature& saSig) { - // ABIArgIter::ABIArgIter wants the items to be iterated over to be + // WasmABIArgIter::ABIArgIter wants the items to be iterated over to be // presented in some type that has methods length() and operator[]. So we // have to wrap up |saSig|'s array of types in this API-matching class. class MOZ_STACK_CLASS ItemsAndLength { diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp index b4786169100b..bdd8c7edb006 100644 --- a/js/src/wasm/WasmIonCompile.cpp +++ b/js/src/wasm/WasmIonCompile.cpp @@ -67,7 +67,7 @@ class FunctionCompiler; class CallCompileState { // A generator object that is passed each argument as it is compiled. - ABIArgGenerator abi_; + WasmABIArgGenerator abi_; // Accumulates the register arguments while compiling arguments. MWasmCall::Args regArgs_; @@ -167,7 +167,7 @@ class FunctionCompiler { return false; } - for (ABIArgIter i(args); !i.done(); i++) { + for (WasmABIArgIter i(args); !i.done(); i++) { MWasmParameter* ins = MWasmParameter::New(alloc(), *i, i.mirType()); curBlock_->add(ins); if (args.isSyntheticStackResultPointerArg(i.index())) { diff --git a/js/src/wasm/WasmStubs.cpp b/js/src/wasm/WasmStubs.cpp index ac433549e680..2ad1154b0ad2 100644 --- a/js/src/wasm/WasmStubs.cpp +++ b/js/src/wasm/WasmStubs.cpp @@ -336,18 +336,28 @@ static void AssertStackAlignment(MacroAssembler& masm, uint32_t alignment, masm.assertStackAlignment(alignment, addBeforeAssert); } -template -static unsigned StackArgBytes(const VectorT& args) { - ABIArgIter iter(args); +template class ABIArgIterT> +static unsigned StackArgBytesHelper(const VectorT& args) { + ABIArgIterT iter(args); while (!iter.done()) { iter++; } return iter.stackBytesConsumedSoFar(); } -static unsigned StackArgBytes(const FuncType& funcType) { +template +static unsigned StackArgBytesForNativeABI(const VectorT& args) { + return StackArgBytesHelper(args); +} + +template +static unsigned StackArgBytesForWasmABI(const VectorT& args) { + return StackArgBytesHelper(args); +} + +static unsigned StackArgBytesForWasmABI(const FuncType& funcType) { ArgTypeVector args(funcType); - return StackArgBytes(args); + return StackArgBytesForWasmABI(args); } static void Move64(MacroAssembler& masm, const Address& src, @@ -367,12 +377,12 @@ static void Move64(MacroAssembler& masm, const Address& src, static void SetupABIArguments(MacroAssembler& masm, const FuncExport& fe, Register argv, Register scratch) { // Copy parameters out of argv and into the registers/stack-slots specified by - // the system ABI. + // the wasm ABI. // // SetupABIArguments are only used for C++ -> wasm calls through callExport(), // and V128 and Ref types (other than externref) are not currently allowed. ArgTypeVector args(fe.funcType()); - for (ABIArgIter iter(args); !iter.done(); iter++) { + for (WasmABIArgIter iter(args); !iter.done(); iter++) { unsigned argOffset = iter.index() * sizeof(ExportArg); Address src(argv, argOffset); MIRType type = iter.mirType(); @@ -808,9 +818,10 @@ static bool GenerateInterpEntry(MacroAssembler& masm, const FuncExport& fe, masm.Push(scratch); #endif - // Reserve stack space for the call. - unsigned argDecrement = StackDecrementForCall( - WasmStackAlignment, masm.framePushed(), StackArgBytes(fe.funcType())); + // Reserve stack space for the wasm call. + unsigned argDecrement = + StackDecrementForCall(WasmStackAlignment, masm.framePushed(), + StackArgBytesForWasmABI(fe.funcType())); masm.reserveStack(argDecrement); // Copy parameters out of argv and into the wasm ABI registers/stack-slots. @@ -1003,13 +1014,13 @@ static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex, // left): // <-- retAddr | descriptor | callee | argc | this | arg1..N - unsigned normalBytesNeeded = StackArgBytes(fe.funcType()); + unsigned normalBytesNeeded = StackArgBytesForWasmABI(fe.funcType()); MIRTypeVector coerceArgTypes; MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Int32)); MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer)); MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer)); - unsigned oolBytesNeeded = StackArgBytes(coerceArgTypes); + unsigned oolBytesNeeded = StackArgBytesForWasmABI(coerceArgTypes); unsigned bytesNeeded = std::max(normalBytesNeeded, oolBytesNeeded); @@ -1201,7 +1212,7 @@ static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex, // Convert all the expected values to unboxed values on the stack. ArgTypeVector args(fe.funcType()); - for (ABIArgIter iter(args); !iter.done(); iter++) { + for (WasmABIArgIter iter(args); !iter.done(); iter++) { unsigned jitArgOffset = frameSize + JitFrameLayout::offsetOfActualArg(iter.index()); Address argv(sp, jitArgOffset); @@ -1382,7 +1393,9 @@ static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex, masm.bind(&oolCall); masm.setFramePushed(frameSize); - ABIArgMIRTypeIter argsIter(coerceArgTypes); + // Baseline and Ion call C++ runtime via BuiltinThunk with wasm abi, so to + // unify the BuiltinThunk's interface we call it here with wasm abi. + jit::WasmABIArgIter argsIter(coerceArgTypes); // argument 0: function export index. if (argsIter->kind() == ABIArg::GPR) { @@ -1471,7 +1484,7 @@ void wasm::GenerateDirectCallFromJit(MacroAssembler& masm, const FuncExport& fe, masm.orPtr(Imm32(ExitOrJitEntryFPTag), FramePointer); // Move stack arguments to their final locations. - unsigned bytesNeeded = StackArgBytes(fe.funcType()); + unsigned bytesNeeded = StackArgBytesForWasmABI(fe.funcType()); bytesNeeded = StackDecrementForCall(WasmStackAlignment, masm.framePushed(), bytesNeeded); if (bytesNeeded) { @@ -1482,7 +1495,7 @@ void wasm::GenerateDirectCallFromJit(MacroAssembler& masm, const FuncExport& fe, fe.funcIndex()); ArgTypeVector args(fe.funcType()); - for (ABIArgIter iter(args); !iter.done(); iter++) { + for (WasmABIArgIter iter(args); !iter.done(); iter++) { MOZ_ASSERT_IF(iter->kind() == ABIArg::GPR, iter->gpr() != scratch); MOZ_ASSERT_IF(iter->kind() == ABIArg::GPR, iter->gpr() != FramePointer); if (iter->kind() != ABIArg::Stack) { @@ -1868,9 +1881,8 @@ static void FillArgumentArrayForExit( #endif masm.load64(src, scratch64); GenPrintI64(DebugChannel::Import, masm, scratch64); - GenerateBigIntInitialization(masm, offsetFromFPToCallerStackArgs, - scratch64, scratch, nullptr, - throwLabel); + GenerateBigIntInitialization(masm, sizeof(Frame), scratch64, + scratch, nullptr, throwLabel); masm.storeValue(JSVAL_TYPE_BIGINT, scratch, dst); } else if (type == MIRType::RefOrNull) { // This works also for FuncRef because it is distinguishable from a @@ -1939,10 +1951,10 @@ static bool GenerateImportFunction(jit::MacroAssembler& masm, MOZ_ASSERT(masm.framePushed() == 0); const unsigned sizeOfTlsSlot = sizeof(void*); - unsigned framePushed = - StackDecrementForCall(WasmStackAlignment, - sizeof(Frame), // pushed by prologue - StackArgBytes(fi.funcType()) + sizeOfTlsSlot); + unsigned framePushed = StackDecrementForCall( + WasmStackAlignment, + sizeof(Frame), // pushed by prologue + StackArgBytesForWasmABI(fi.funcType()) + sizeOfTlsSlot); masm.wasmReserveStackChecked(framePushed, BytecodeOffset(0)); MOZ_ASSERT(masm.framePushed() == framePushed); @@ -1956,7 +1968,7 @@ static bool GenerateImportFunction(jit::MacroAssembler& masm, // Copy our frame's stack arguments to the callee frame's stack argument. unsigned offsetFromFPToCallerStackArgs = sizeof(Frame); ArgTypeVector args(fi.funcType()); - for (ABIArgIter i(args); !i.done(); i++) { + for (WasmABIArgIter i(args); !i.done(); i++) { if (i->kind() != ABIArg::Stack) { continue; } @@ -2041,7 +2053,7 @@ static bool GenerateImportInterpExit(MacroAssembler& masm, const FuncImport& fi, // The padding between stack args and argv ensures that argv is aligned. The // padding between argv and retaddr ensures that sp is aligned. unsigned argOffset = - AlignBytes(StackArgBytes(invokeArgTypes), sizeof(double)); + AlignBytes(StackArgBytesForNativeABI(invokeArgTypes), sizeof(double)); // The abiArgCount includes a stack result pointer argument if needed. unsigned abiArgCount = ArgTypeVector(fi.funcType()).lengthWithStackResults(); unsigned argBytes = std::max(1, abiArgCount) * sizeof(Value); @@ -2054,7 +2066,7 @@ static bool GenerateImportInterpExit(MacroAssembler& masm, const FuncImport& fi, offsets); // Fill the argument array. - unsigned offsetFromFPToCallerStackArgs = sizeof(Frame); + unsigned offsetFromFPToCallerStackArgs = sizeof(FrameWithTls); Register scratch = ABINonArgReturnReg0; Register scratch2 = ABINonArgReturnReg1; // The scratch3 reg does not need to be non-volatile, but has to be @@ -2262,7 +2274,7 @@ static bool GenerateImportJitExit(MacroAssembler& masm, const FuncImport& fi, argOffset += sizeof(Value); // 5. Fill the arguments. - const uint32_t offsetFromFPToCallerStackArgs = sizeof(Frame); + const uint32_t offsetFromFPToCallerStackArgs = sizeof(FrameWithTls); Register scratch = ABINonArgReturnReg1; // Repeatedly clobbered Register scratch2 = ABINonArgReturnReg0; // Reused as callee below // The scratch3 reg does not need to be non-volatile, but has to be @@ -2441,7 +2453,7 @@ static bool GenerateImportJitExit(MacroAssembler& masm, const FuncImport& fi, MIRTypeVector coerceArgTypes; MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer)); unsigned offsetToCoerceArgv = - AlignBytes(StackArgBytes(coerceArgTypes), sizeof(Value)); + AlignBytes(StackArgBytesForWasmABI(coerceArgTypes), sizeof(Value)); MOZ_ASSERT(nativeFramePushed >= offsetToCoerceArgv + sizeof(Value)); AssertStackAlignment(masm, ABIStackAlignment); @@ -2573,14 +2585,14 @@ bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIFunctionType abiType, uint32_t framePushed = StackDecrementForCall(ABIStackAlignment, sizeof(Frame), // pushed by prologue - StackArgBytes(args)); + StackArgBytesForNativeABI(args)); GenerateExitPrologue(masm, framePushed, exitReason, offsets); // Copy out and convert caller arguments, if needed. - unsigned offsetFromFPToCallerStackArgs = sizeof(Frame); + unsigned offsetFromFPToCallerStackArgs = sizeof(FrameWithTls); Register scratch = ABINonArgReturnReg0; - for (ABIArgIter i(args); !i.done(); i++) { + for (ABIArgIter i(args); !i.done(); i++) { if (i->argInRegister()) { #ifdef JS_CODEGEN_ARM // Non hard-fp passes the args values in GPRs. diff --git a/js/src/wasm/WasmTypes.h b/js/src/wasm/WasmTypes.h index 2be7a44f1d53..1bccb7dd7495 100644 --- a/js/src/wasm/WasmTypes.h +++ b/js/src/wasm/WasmTypes.h @@ -50,8 +50,8 @@ namespace js { namespace jit { class JitScript; enum class RoundingMode; -template -class ABIArgIter; +template +class ABIArgIterBase; } // namespace jit // This is a widespread header, so lets keep out the core wasm impl types. @@ -1311,13 +1311,13 @@ class ArgTypeVector { const ValTypeVector& args_; bool hasStackResults_; - // To allow ABIArgIter, we define a private length() - // method. To prevent accidental errors, other users need to be + // To allow ABIArgIterBase, we define a private + // length() method. To prevent accidental errors, other users need to be // explicit and call lengthWithStackResults() or // lengthWithoutStackResults(). size_t length() const { return args_.length() + size_t(hasStackResults_); } - friend jit::ABIArgIter; - friend jit::ABIArgIter; + template + friend class jit::ABIArgIterBase; public: ArgTypeVector(const ValTypeVector& args, StackResults stackResults) @@ -3303,6 +3303,19 @@ class Frame { static_assert(!std::is_polymorphic_v, "Frame doesn't need a vtable."); +class FrameWithTls : public Frame { + public: + TlsData* calleeTls_; + TlsData* callerTls_; + + constexpr static uint32_t sizeWithoutFrame() { + return sizeof(wasm::FrameWithTls) - sizeof(wasm::Frame); + } +}; + +static_assert(FrameWithTls::sizeWithoutFrame() == 2 * sizeof(void*), + "There are only two additional slots"); + #if defined(JS_CODEGEN_ARM64) static_assert(sizeof(Frame) % 16 == 0, "frame is aligned"); #endif