зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1639153 - Introduce indirect stubs to optimize call_indirect. r=lth
This patch introduces indirect stubs. An indirect stub is a stub that takes care of any switching activities needed for call_indirect. Before this patch, we have to conservatively assume that any call_indirect's target can be from a foreign instance, but now we don't need to do this because now Table[i].code is replaced with the address of the corresponding indirect stub. There is one optimization - if we 100% sure that call_indirect's target belongs to our instance then we can omit indirect stub and use raw function call with signature check of course. We 100% sure that call_indirect's target is the same instance iff Table is private and we don't export it. However, even if a table is exported but not used we can still use the optimization but this logic for some next patch. Differential Revision: https://phabricator.services.mozilla.com/D117123
This commit is contained in:
Dmitry Bezhetskov
Родитель
ee274fe350
Коммит
8f6dd5d182
|
|
@ -7920,37 +7920,34 @@ void CodeGenerator::visitWasmCall(LWasmCall* lir) {
|
|||
#endif
|
||||
|
||||
// LWasmCallBase::isCallPreserved() assumes that all MWasmCalls preserve the
|
||||
// TLS and pinned regs. The only case where where we don't have to reload
|
||||
// the TLS and pinned regs is when the callee preserves them.
|
||||
bool reloadRegs = true;
|
||||
bool switchRealm = true;
|
||||
// TLS and pinned regs. The only case where where we have to reload
|
||||
// the TLS and pinned regs is when we emit import or builtin instance calls.
|
||||
bool reloadRegs = false;
|
||||
bool switchRealm = false;
|
||||
|
||||
const wasm::CallSiteDesc& desc = mir->desc();
|
||||
const wasm::CalleeDesc& callee = mir->callee();
|
||||
switch (callee.which()) {
|
||||
case wasm::CalleeDesc::Func:
|
||||
masm.call(desc, callee.funcIndex());
|
||||
reloadRegs = false;
|
||||
switchRealm = false;
|
||||
break;
|
||||
case wasm::CalleeDesc::Import:
|
||||
masm.wasmCallImport(desc, callee);
|
||||
reloadRegs = true;
|
||||
switchRealm = true;
|
||||
break;
|
||||
case wasm::CalleeDesc::AsmJSTable:
|
||||
case wasm::CalleeDesc::WasmTable:
|
||||
masm.wasmCallIndirect(desc, callee, needsBoundsCheck);
|
||||
reloadRegs = switchRealm = callee.which() == wasm::CalleeDesc::WasmTable;
|
||||
break;
|
||||
case wasm::CalleeDesc::Builtin:
|
||||
masm.call(desc, callee.builtin());
|
||||
reloadRegs = false;
|
||||
switchRealm = false;
|
||||
break;
|
||||
case wasm::CalleeDesc::BuiltinInstanceMethod:
|
||||
masm.wasmCallBuiltinInstanceMethod(desc, mir->instanceArg(),
|
||||
callee.builtin(),
|
||||
mir->builtinMethodFailureMode());
|
||||
switchRealm = false;
|
||||
reloadRegs = true;
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3941,22 +3941,13 @@ CodeOffset MacroAssembler::wasmCallIndirect(const wasm::CallSiteDesc& desc,
|
|||
addPtr(index, scratch);
|
||||
}
|
||||
|
||||
storePtr(WasmTlsReg,
|
||||
Address(getStackPointer(), WasmCallerTLSOffsetBeforeCall));
|
||||
loadPtr(Address(scratch, offsetof(wasm::FunctionTableElem, tls)), WasmTlsReg);
|
||||
storePtr(WasmTlsReg,
|
||||
Address(getStackPointer(), WasmCalleeTLSOffsetBeforeCall));
|
||||
loadPtr(Address(scratch, offsetof(wasm::FunctionTableElem, code)), scratch);
|
||||
|
||||
Label nonNull;
|
||||
branchTest32(Assembler::NonZero, WasmTlsReg, WasmTlsReg, &nonNull);
|
||||
branchTest32(Assembler::NonZero, scratch, scratch, &nonNull);
|
||||
wasmTrap(wasm::Trap::IndirectCallToNull, trapOffset);
|
||||
bind(&nonNull);
|
||||
|
||||
loadWasmPinnedRegsFromTls();
|
||||
switchToWasmTlsRealm(index, WasmTableCallScratchReg1);
|
||||
|
||||
loadPtr(Address(scratch, offsetof(wasm::FunctionTableElem, code)), scratch);
|
||||
|
||||
return call(desc, scratch);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -270,6 +270,11 @@ constexpr uint32_t WasmCallerTLSOffsetBeforeCall =
|
|||
constexpr uint32_t WasmCalleeTLSOffsetBeforeCall =
|
||||
wasm::FrameWithTls::calleeTLSOffset() + ShadowStackSpace;
|
||||
|
||||
constexpr uint32_t WasmCallerTLSOffsetAfterCall =
|
||||
WasmCallerTLSOffsetBeforeCall + SizeOfReturnAddressAfterCall;
|
||||
constexpr uint32_t WasmCalleeTLSOffsetAfterCall =
|
||||
WasmCalleeTLSOffsetBeforeCall + SizeOfReturnAddressAfterCall;
|
||||
|
||||
// Allocation sites may be passed to GC thing allocation methods either via a
|
||||
// register (for baseline compilation) or an enum indicating one of the
|
||||
// catch-all allocation sites (for optimized compilation).
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ static const int32_t NUNBOX32_TYPE_OFFSET = 4;
|
|||
static const int32_t NUNBOX32_PAYLOAD_OFFSET = 0;
|
||||
|
||||
static const uint32_t ShadowStackSpace = 0;
|
||||
static const uint32_t SizeOfReturnAddressAfterCall = 0u;
|
||||
|
||||
// How far forward/back can a jump go? Provide a generous buffer for thunks.
|
||||
static const uint32_t JumpImmediateRange = 20 * 1024 * 1024;
|
||||
|
|
|
|||
|
|
@ -548,6 +548,7 @@ class FloatRegisters {
|
|||
};
|
||||
|
||||
static const uint32_t ShadowStackSpace = 0;
|
||||
static const uint32_t SizeOfReturnAddressAfterCall = 0u;
|
||||
|
||||
// When our only strategy for far jumps is to encode the offset directly, and
|
||||
// not insert any jump islands during assembly for even further jumps, then the
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ namespace js {
|
|||
namespace jit {
|
||||
|
||||
static const uint32_t ShadowStackSpace = 4 * sizeof(uintptr_t);
|
||||
static const uint32_t SizeOfReturnAddressAfterCall = 0;
|
||||
|
||||
// These offsets are specific to nunboxing, and capture offsets into the
|
||||
// components of a js::Value.
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ namespace jit {
|
|||
|
||||
// Shadow stack space is not required on MIPS64.
|
||||
static const uint32_t ShadowStackSpace = 0;
|
||||
static const uint32_t SizeOfReturnAddressAfterCall = 0;
|
||||
|
||||
// MIPS64 have 64 bit floating-point coprocessor. There are 32 double
|
||||
// precision register which can also be used as single precision registers.
|
||||
|
|
|
|||
|
|
@ -158,6 +158,7 @@ inline bool hasUnaliasedDouble() { MOZ_CRASH(); }
|
|||
inline bool hasMultiAlias() { MOZ_CRASH(); }
|
||||
|
||||
static const uint32_t ShadowStackSpace = 0;
|
||||
static const uint32_t SizeOfReturnAddressAfterCall = 0u;
|
||||
static const uint32_t JumpImmediateRange = INT32_MAX;
|
||||
|
||||
#ifdef JS_NUNBOX32
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ static const uint32_t ShadowStackSpace = 32;
|
|||
static const uint32_t ShadowStackSpace = 0;
|
||||
#endif
|
||||
|
||||
static const uint32_t SizeOfReturnAddressAfterCall = sizeof(void*);
|
||||
|
||||
static const uint32_t JumpImmediateRange = INT32_MAX;
|
||||
|
||||
class Registers {
|
||||
|
|
|
|||
|
|
@ -1444,7 +1444,7 @@ CodeOffset BaseCompiler::callIndirect(uint32_t funcTypeIndex,
|
|||
|
||||
loadI32(indexVal, RegI32(WasmTableCallIndexReg));
|
||||
|
||||
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Dynamic);
|
||||
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Indirect);
|
||||
CalleeDesc callee = CalleeDesc::wasmTable(table, funcTypeId);
|
||||
return masm.wasmCallIndirect(desc, callee, NeedsBoundsCheck(true));
|
||||
}
|
||||
|
|
@ -1453,7 +1453,7 @@ CodeOffset BaseCompiler::callIndirect(uint32_t funcTypeIndex,
|
|||
|
||||
CodeOffset BaseCompiler::callImport(unsigned globalDataOffset,
|
||||
const FunctionCall& call) {
|
||||
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Dynamic);
|
||||
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Import);
|
||||
CalleeDesc callee = CalleeDesc::import(globalDataOffset);
|
||||
return masm.wasmCallImport(desc, callee);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ using namespace js;
|
|||
using namespace js::jit;
|
||||
using namespace js::wasm;
|
||||
using mozilla::BinarySearch;
|
||||
using mozilla::BinarySearchIf;
|
||||
using mozilla::MakeEnumeratedRange;
|
||||
using mozilla::PodAssign;
|
||||
|
||||
|
|
@ -111,6 +112,17 @@ CodeSegment::~CodeSegment() {
|
|||
}
|
||||
}
|
||||
|
||||
template <class T, class Container, class Comparator>
|
||||
void InsertIntoSortedContainer(Container& container, T&& value,
|
||||
Comparator&& comparator) {
|
||||
size_t indexInSortedVector = 0;
|
||||
MOZ_ALWAYS_FALSE(BinarySearchIf(container, 0, container.length(),
|
||||
std::forward<Comparator>(comparator),
|
||||
&indexInSortedVector));
|
||||
MOZ_ALWAYS_TRUE(container.insert(container.begin() + indexInSortedVector,
|
||||
std::forward<T>(value)));
|
||||
}
|
||||
|
||||
static uint32_t RoundupCodeLength(uint32_t codeLength) {
|
||||
// AllocateExecutableMemory() requires a multiple of ExecutableCodePageSize.
|
||||
return RoundUp(codeLength, ExecutableCodePageSize);
|
||||
|
|
@ -649,6 +661,34 @@ bool LazyStubSegment::addStubs(size_t codeLength,
|
|||
return true;
|
||||
}
|
||||
|
||||
bool LazyStubSegment::addIndirectStubs(
|
||||
size_t codeLength, const VectorOfIndirectStubTarget& targets,
|
||||
const CodeRangeVector& codeRanges, uint8_t** codePtr,
|
||||
size_t* indexFirstInsertedCodeRange) {
|
||||
MOZ_ASSERT(hasSpace(codeLength));
|
||||
|
||||
if (!codeRanges_.reserve(codeRanges_.length() + codeRanges.length())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t offsetInSegment = usedBytes_;
|
||||
*codePtr = base() + usedBytes_;
|
||||
usedBytes_ += codeLength;
|
||||
*indexFirstInsertedCodeRange = codeRanges_.length();
|
||||
|
||||
for (size_t i = 0; i < targets.length(); ++i) {
|
||||
DebugOnly<uint32_t> funcIndex = targets[i].functionIdx;
|
||||
const CodeRange& indirectStubRange = codeRanges[i];
|
||||
MOZ_ASSERT(indirectStubRange.isIndirectStub());
|
||||
MOZ_ASSERT(indirectStubRange.funcIndex() == funcIndex);
|
||||
|
||||
codeRanges_.infallibleAppend(indirectStubRange);
|
||||
codeRanges_.back().offsetBy(offsetInSegment);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
const CodeRange* LazyStubSegment::lookupRange(const void* pc) const {
|
||||
return LookupInSorted(codeRanges_,
|
||||
CodeRange::OffsetInCode((uint8_t*)pc - base()));
|
||||
|
|
@ -661,21 +701,12 @@ void LazyStubSegment::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code,
|
|||
*data += mallocSizeOf(this);
|
||||
}
|
||||
|
||||
struct ProjectLazyFuncIndex {
|
||||
const LazyFuncExportVector& funcExports;
|
||||
explicit ProjectLazyFuncIndex(const LazyFuncExportVector& funcExports)
|
||||
: funcExports(funcExports) {}
|
||||
uint32_t operator[](size_t index) const {
|
||||
return funcExports[index].funcIndex;
|
||||
}
|
||||
};
|
||||
|
||||
static constexpr unsigned LAZY_STUB_LIFO_DEFAULT_CHUNK_SIZE = 8 * 1024;
|
||||
|
||||
bool LazyStubTier::createMany(const Uint32Vector& funcExportIndices,
|
||||
const CodeTier& codeTier,
|
||||
bool flushAllThreadsIcaches,
|
||||
size_t* stubSegmentIndex) {
|
||||
bool LazyStubTier::createManyEntryStubs(const Uint32Vector& funcExportIndices,
|
||||
const CodeTier& codeTier,
|
||||
bool flushAllThreadsIcaches,
|
||||
size_t* stubSegmentIndex) {
|
||||
MOZ_ASSERT(funcExportIndices.length());
|
||||
|
||||
LifoAlloc lifo(LAZY_STUB_LIFO_DEFAULT_CHUNK_SIZE);
|
||||
|
|
@ -778,9 +809,13 @@ bool LazyStubTier::createMany(const Uint32Vector& funcExportIndices,
|
|||
interpRangeIndex);
|
||||
|
||||
size_t exportIndex;
|
||||
MOZ_ALWAYS_FALSE(BinarySearch(ProjectLazyFuncIndex(exports_), 0,
|
||||
exports_.length(), fe.funcIndex(),
|
||||
&exportIndex));
|
||||
const uint32_t targetFunctionIndex = fe.funcIndex();
|
||||
MOZ_ALWAYS_FALSE(BinarySearchIf(
|
||||
exports_, 0, exports_.length(),
|
||||
[targetFunctionIndex](const LazyFuncExport& funcExport) {
|
||||
return targetFunctionIndex - funcExport.funcIndex;
|
||||
},
|
||||
&exportIndex));
|
||||
MOZ_ALWAYS_TRUE(
|
||||
exports_.insert(exports_.begin() + exportIndex, std::move(lazyExport)));
|
||||
|
||||
|
|
@ -791,8 +826,8 @@ bool LazyStubTier::createMany(const Uint32Vector& funcExportIndices,
|
|||
return true;
|
||||
}
|
||||
|
||||
bool LazyStubTier::createOne(uint32_t funcExportIndex,
|
||||
const CodeTier& codeTier) {
|
||||
bool LazyStubTier::createOneEntryStub(uint32_t funcExportIndex,
|
||||
const CodeTier& codeTier) {
|
||||
Uint32Vector funcExportIndexes;
|
||||
if (!funcExportIndexes.append(funcExportIndex)) {
|
||||
return false;
|
||||
|
|
@ -804,8 +839,8 @@ bool LazyStubTier::createOne(uint32_t funcExportIndex,
|
|||
bool flushAllThreadIcaches = false;
|
||||
|
||||
size_t stubSegmentIndex;
|
||||
if (!createMany(funcExportIndexes, codeTier, flushAllThreadIcaches,
|
||||
&stubSegmentIndex)) {
|
||||
if (!createManyEntryStubs(funcExportIndexes, codeTier, flushAllThreadIcaches,
|
||||
&stubSegmentIndex)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -829,6 +864,127 @@ bool LazyStubTier::createOne(uint32_t funcExportIndex,
|
|||
return true;
|
||||
}
|
||||
|
||||
auto IndirectStubComparator = [](uint32_t funcIndex, void* tlsData,
|
||||
const IndirectStub& stub) -> int {
|
||||
if (funcIndex != stub.funcIndex) {
|
||||
return static_cast<int>(funcIndex - stub.funcIndex);
|
||||
}
|
||||
|
||||
// If function indices are equal then compare by tls.
|
||||
const auto tlsDataAsIntegral = reinterpret_cast<uintptr_t>(tlsData);
|
||||
const auto stubTlsAsIntegral = reinterpret_cast<uintptr_t>(stub.tls);
|
||||
return static_cast<int>(tlsDataAsIntegral - stubTlsAsIntegral);
|
||||
};
|
||||
|
||||
bool LazyStubTier::createManyIndirectStubs(
|
||||
const VectorOfIndirectStubTarget& targets, const CodeTier& codeTier) {
|
||||
MOZ_ASSERT(targets.length());
|
||||
|
||||
LifoAlloc lifo(LAZY_STUB_LIFO_DEFAULT_CHUNK_SIZE);
|
||||
TempAllocator alloc(&lifo);
|
||||
JitContext jitContext(&alloc);
|
||||
WasmMacroAssembler masm(alloc);
|
||||
AutoCreatedBy acb(masm, "LazyStubTier::createManyIndirectStubs");
|
||||
|
||||
CodeRangeVector codeRanges;
|
||||
for (const auto& target : targets) {
|
||||
MOZ_ASSERT(!lookupIndirectStub(target.functionIdx, target.tls));
|
||||
|
||||
Offsets offsets;
|
||||
if (!GenerateIndirectStub(
|
||||
masm, static_cast<uint8_t*>(target.checkedCallEntryAddress),
|
||||
target.tls, &offsets)) {
|
||||
return false;
|
||||
}
|
||||
if (!codeRanges.emplaceBack(CodeRange::IndirectStub, target.functionIdx,
|
||||
offsets)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
masm.finish();
|
||||
|
||||
MOZ_ASSERT(masm.callSites().empty());
|
||||
MOZ_ASSERT(masm.callSiteTargets().empty());
|
||||
MOZ_ASSERT(masm.trapSites().empty());
|
||||
|
||||
if (masm.oom()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t codeLength = LazyStubSegment::AlignBytesNeeded(masm.bytesNeeded());
|
||||
|
||||
if (!stubSegments_.length() ||
|
||||
!stubSegments_[lastStubSegmentIndex_]->hasSpace(codeLength)) {
|
||||
size_t newSegmentSize = std::max(codeLength, ExecutableCodePageSize);
|
||||
UniqueLazyStubSegment newSegment =
|
||||
LazyStubSegment::create(codeTier, newSegmentSize);
|
||||
if (!newSegment) {
|
||||
return false;
|
||||
}
|
||||
lastStubSegmentIndex_ = stubSegments_.length();
|
||||
if (!stubSegments_.emplaceBack(std::move(newSegment))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
LazyStubSegment* segment = stubSegments_[lastStubSegmentIndex_].get();
|
||||
|
||||
size_t indirectStubRangeIndex;
|
||||
uint8_t* codePtr = nullptr;
|
||||
if (!segment->addIndirectStubs(codeLength, targets, codeRanges, &codePtr,
|
||||
&indirectStubRangeIndex)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
masm.executableCopy(codePtr);
|
||||
PatchDebugSymbolicAccesses(codePtr, masm);
|
||||
memset(codePtr + masm.bytesNeeded(), 0, codeLength - masm.bytesNeeded());
|
||||
|
||||
for (const CodeLabel& label : masm.codeLabels()) {
|
||||
Assembler::Bind(codePtr, label);
|
||||
}
|
||||
|
||||
if (!ExecutableAllocator::makeExecutableAndFlushICache(
|
||||
FlushICacheSpec::LocalThreadOnly, codePtr, codeLength)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Record the runtime info about generated indirect stubs.
|
||||
if (!indirectStubVector_.reserve(indirectStubVector_.length() +
|
||||
targets.length())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto& target : targets) {
|
||||
auto stub = IndirectStub{target.functionIdx, lastStubSegmentIndex_,
|
||||
indirectStubRangeIndex, target.tls};
|
||||
|
||||
size_t indirectStubIndex;
|
||||
MOZ_ALWAYS_FALSE(BinarySearchIf(
|
||||
indirectStubVector_, 0, indirectStubVector_.length(),
|
||||
[&stub](const IndirectStub& otherStub) {
|
||||
return IndirectStubComparator(stub.funcIndex, stub.tls, otherStub);
|
||||
},
|
||||
&indirectStubIndex));
|
||||
MOZ_ALWAYS_TRUE(indirectStubVector_.insert(
|
||||
indirectStubVector_.begin() + indirectStubIndex, std::move(stub)));
|
||||
|
||||
++indirectStubRangeIndex;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const CodeRange* LazyStubTier::lookupRange(const void* pc) const {
|
||||
for (const UniqueLazyStubSegment& stubSegment : stubSegments_) {
|
||||
const CodeRange* result = stubSegment->lookupRange(pc);
|
||||
if (result) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool LazyStubTier::createTier2(const Uint32Vector& funcExportIndices,
|
||||
const CodeTier& codeTier,
|
||||
Maybe<size_t>* outStubSegmentIndex) {
|
||||
|
|
@ -841,8 +997,8 @@ bool LazyStubTier::createTier2(const Uint32Vector& funcExportIndices,
|
|||
bool flushAllThreadIcaches = true;
|
||||
|
||||
size_t stubSegmentIndex;
|
||||
if (!createMany(funcExportIndices, codeTier, flushAllThreadIcaches,
|
||||
&stubSegmentIndex)) {
|
||||
if (!createManyEntryStubs(funcExportIndices, codeTier, flushAllThreadIcaches,
|
||||
&stubSegmentIndex)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -864,16 +1020,24 @@ void LazyStubTier::setJitEntries(const Maybe<size_t>& stubSegmentIndex,
|
|||
}
|
||||
}
|
||||
|
||||
bool LazyStubTier::hasStub(uint32_t funcIndex) const {
|
||||
bool LazyStubTier::hasEntryStub(uint32_t funcIndex) const {
|
||||
size_t match;
|
||||
return BinarySearch(ProjectLazyFuncIndex(exports_), 0, exports_.length(),
|
||||
funcIndex, &match);
|
||||
return BinarySearchIf(
|
||||
exports_, 0, exports_.length(),
|
||||
[funcIndex](const LazyFuncExport& funcExport) {
|
||||
return funcIndex - funcExport.funcIndex;
|
||||
},
|
||||
&match);
|
||||
}
|
||||
|
||||
void* LazyStubTier::lookupInterpEntry(uint32_t funcIndex) const {
|
||||
size_t match;
|
||||
if (!BinarySearch(ProjectLazyFuncIndex(exports_), 0, exports_.length(),
|
||||
funcIndex, &match)) {
|
||||
if (!BinarySearchIf(
|
||||
exports_, 0, exports_.length(),
|
||||
[funcIndex](const LazyFuncExport& funcExport) {
|
||||
return funcIndex - funcExport.funcIndex;
|
||||
},
|
||||
&match)) {
|
||||
return nullptr;
|
||||
}
|
||||
const LazyFuncExport& fe = exports_[match];
|
||||
|
|
@ -881,6 +1045,24 @@ void* LazyStubTier::lookupInterpEntry(uint32_t funcIndex) const {
|
|||
return stub.base() + stub.codeRanges()[fe.funcCodeRangeIndex].begin();
|
||||
}
|
||||
|
||||
void* LazyStubTier::lookupIndirectStub(uint32_t funcIndex, void* tls) const {
|
||||
size_t match;
|
||||
if (!BinarySearchIf(
|
||||
indirectStubVector_, 0, indirectStubVector_.length(),
|
||||
[funcIndex, tls](const IndirectStub& stub) {
|
||||
return IndirectStubComparator(funcIndex, tls, stub);
|
||||
},
|
||||
&match)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const IndirectStub& indirectStub = indirectStubVector_[match];
|
||||
|
||||
const LazyStubSegment& segment = *stubSegments_[indirectStub.segmentIndex];
|
||||
return segment.base() +
|
||||
segment.codeRanges()[indirectStub.codeRangeIndex].begin();
|
||||
}
|
||||
|
||||
void LazyStubTier::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code,
|
||||
size_t* data) const {
|
||||
*data += sizeof(*this);
|
||||
|
|
@ -1070,7 +1252,7 @@ bool CodeTier::initialize(IsTier2 isTier2, const Code& code,
|
|||
MOZ_ASSERT(!initialized());
|
||||
code_ = &code;
|
||||
|
||||
MOZ_ASSERT(lazyStubs_.lock()->empty());
|
||||
MOZ_ASSERT(lazyStubs_.lock()->entryStubsEmpty());
|
||||
|
||||
// See comments in CodeSegment::initialize() for why this must be last.
|
||||
if (!segment_->initialize(isTier2, *this, linkData, metadata, *metadata_)) {
|
||||
|
|
@ -1327,6 +1509,22 @@ const CodeRange* Code::lookupFuncRange(void* pc) const {
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
const CodeRange* Code::lookupIndirectStubRange(void* pc) const {
|
||||
// TODO / OPTIMIZE:
|
||||
// There is only one client of this function - Table::getFuncRef.
|
||||
// Table::getFuncRef can return only exported function,
|
||||
// so if we pregenerate indirect stubs for all exported functions
|
||||
// we can eliminate the lock below.
|
||||
for (Tier tier : tiers()) {
|
||||
auto stubs = codeTier(tier).lazyStubs().lock();
|
||||
const CodeRange* result = stubs->lookupRange(pc);
|
||||
if (result && result->isIndirectStub()) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const StackMap* Code::lookupStackMap(uint8_t* nextPC) const {
|
||||
for (Tier t : tiers()) {
|
||||
const StackMap* result = metadata(t).stackMaps.findMap(nextPC);
|
||||
|
|
|
|||
|
|
@ -72,6 +72,21 @@ class MacroAssembler;
|
|||
|
||||
namespace wasm {
|
||||
|
||||
struct IndirectStubTarget {
|
||||
uint32_t functionIdx;
|
||||
void* checkedCallEntryAddress;
|
||||
TlsData* tls;
|
||||
|
||||
bool operator==(const IndirectStubTarget& other) const {
|
||||
return functionIdx == other.functionIdx &&
|
||||
checkedCallEntryAddress == other.checkedCallEntryAddress &&
|
||||
tls == other.tls;
|
||||
}
|
||||
};
|
||||
|
||||
using VectorOfIndirectStubTarget =
|
||||
Vector<IndirectStubTarget, 8, SystemAllocPolicy>;
|
||||
|
||||
struct MetadataTier;
|
||||
struct Metadata;
|
||||
|
||||
|
|
@ -489,7 +504,7 @@ struct MetadataTier {
|
|||
using UniqueMetadataTier = UniquePtr<MetadataTier>;
|
||||
|
||||
// LazyStubSegment is a code segment lazily generated for function entry stubs
|
||||
// (both interpreter and jit ones).
|
||||
// (both interpreter and jit ones) and for indirect stubs.
|
||||
//
|
||||
// Because a stub is usually small (a few KiB) and an executable code segment
|
||||
// isn't (64KiB), a given stub segment can contain entry stubs of many
|
||||
|
|
@ -521,6 +536,11 @@ class LazyStubSegment : public CodeSegment {
|
|||
const CodeRangeVector& codeRanges, uint8_t** codePtr,
|
||||
size_t* indexFirstInsertedCodeRange);
|
||||
|
||||
bool addIndirectStubs(size_t codeLength,
|
||||
const VectorOfIndirectStubTarget& targets,
|
||||
const CodeRangeVector& codeRanges, uint8_t** codePtr,
|
||||
size_t* indexFirstInsertedCodeRange);
|
||||
|
||||
const CodeRangeVector& codeRanges() const { return codeRanges_; }
|
||||
const CodeRange* lookupRange(const void* pc) const;
|
||||
|
||||
|
|
@ -545,8 +565,27 @@ struct LazyFuncExport {
|
|||
|
||||
using LazyFuncExportVector = Vector<LazyFuncExport, 0, SystemAllocPolicy>;
|
||||
|
||||
// IndirectStub provides a mapping between function indices and
|
||||
// indirect stubs code ranges.
|
||||
|
||||
struct IndirectStub {
|
||||
size_t funcIndex;
|
||||
size_t segmentIndex;
|
||||
size_t codeRangeIndex;
|
||||
void* tls;
|
||||
IndirectStub(size_t funcIndex, size_t segmentIndex, size_t codeRangeIndex,
|
||||
TlsData* tls)
|
||||
: funcIndex(funcIndex),
|
||||
segmentIndex(segmentIndex),
|
||||
codeRangeIndex(codeRangeIndex),
|
||||
tls(tls) {}
|
||||
};
|
||||
|
||||
using IndirectStubVector = Vector<IndirectStub, 0, SystemAllocPolicy>;
|
||||
|
||||
// LazyStubTier contains all the necessary information for lazy function entry
|
||||
// stubs that are generated at runtime. None of its data is ever serialized.
|
||||
// stubs and indirect stubs that are generated at runtime.
|
||||
// None of its data are ever serialized.
|
||||
//
|
||||
// It must be protected by a lock, because the main thread can both read and
|
||||
// write lazy stubs at any time while a background thread can regenerate lazy
|
||||
|
|
@ -555,25 +594,36 @@ using LazyFuncExportVector = Vector<LazyFuncExport, 0, SystemAllocPolicy>;
|
|||
class LazyStubTier {
|
||||
LazyStubSegmentVector stubSegments_;
|
||||
LazyFuncExportVector exports_;
|
||||
IndirectStubVector indirectStubVector_;
|
||||
size_t lastStubSegmentIndex_;
|
||||
|
||||
bool createMany(const Uint32Vector& funcExportIndices,
|
||||
const CodeTier& codeTier, bool flushAllThreadsIcaches,
|
||||
size_t* stubSegmentIndex);
|
||||
bool createManyEntryStubs(const Uint32Vector& funcExportIndices,
|
||||
const CodeTier& codeTier,
|
||||
bool flushAllThreadsIcaches,
|
||||
size_t* stubSegmentIndex);
|
||||
|
||||
public:
|
||||
LazyStubTier() : lastStubSegmentIndex_(0) {}
|
||||
|
||||
bool empty() const { return stubSegments_.empty(); }
|
||||
bool hasStub(uint32_t funcIndex) const;
|
||||
|
||||
// Returns a pointer to the raw interpreter entry of a given function which
|
||||
// stubs have been lazily generated.
|
||||
void* lookupInterpEntry(uint32_t funcIndex) const;
|
||||
|
||||
// Creates one lazy stub for the exported function, for which the jit entry
|
||||
// will be set to the lazily-generated one.
|
||||
bool createOne(uint32_t funcExportIndex, const CodeTier& codeTier);
|
||||
bool createOneEntryStub(uint32_t funcExportIndex, const CodeTier& codeTier);
|
||||
|
||||
bool entryStubsEmpty() const { return stubSegments_.empty(); }
|
||||
bool hasEntryStub(uint32_t funcIndex) const;
|
||||
|
||||
// Returns a pointer to the raw interpreter entry of a given function for
|
||||
// which stubs have been lazily generated.
|
||||
void* lookupInterpEntry(uint32_t funcIndex) const;
|
||||
|
||||
// Creates many indirect stubs.
|
||||
bool createManyIndirectStubs(const VectorOfIndirectStubTarget& targets,
|
||||
const CodeTier& codeTier);
|
||||
|
||||
// Returns a pointer to the indirect stub of a given function.
|
||||
void* lookupIndirectStub(uint32_t funcIndex, void* tls) const;
|
||||
|
||||
const CodeRange* lookupRange(const void* pc) const;
|
||||
|
||||
// Create one lazy stub for all the functions in funcExportIndices, putting
|
||||
// them in a single stub. Jit entries won't be used until
|
||||
|
|
@ -795,6 +845,7 @@ class Code : public ShareableBase<Code> {
|
|||
|
||||
const CallSite* lookupCallSite(void* returnAddress) const;
|
||||
const CodeRange* lookupFuncRange(void* pc) const;
|
||||
const CodeRange* lookupIndirectStubRange(void* pc) const;
|
||||
const StackMap* lookupStackMap(uint8_t* nextPC) const;
|
||||
#ifdef ENABLE_WASM_EXCEPTIONS
|
||||
const WasmTryNote* lookupWasmTryNote(void* pc, Tier* tier) const;
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@ CodeRange::CodeRange(Kind kind, uint32_t funcIndex, Offsets offsets)
|
|||
u.func.lineOrBytecode_ = 0;
|
||||
u.func.beginToUncheckedCallEntry_ = 0;
|
||||
u.func.beginToTierEntry_ = 0;
|
||||
MOZ_ASSERT(isEntry());
|
||||
MOZ_ASSERT(isEntry() || isIndirectStub());
|
||||
MOZ_ASSERT(begin_ <= end_);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -245,7 +245,9 @@ class CodeRange {
|
|||
TrapExit, // calls C++ to report and jumps to throw stub
|
||||
DebugTrap, // calls C++ to handle debug event
|
||||
FarJumpIsland, // inserted to connect otherwise out-of-range insns
|
||||
Throw // special stack-unwinding stub jumped to by other stubs
|
||||
Throw, // special stack-unwinding stub jumped to by other stubs
|
||||
IndirectStub, // a stub that take care of switching instance specific state
|
||||
// at cross-instance boundaries
|
||||
};
|
||||
|
||||
private:
|
||||
|
|
@ -327,8 +329,9 @@ class CodeRange {
|
|||
bool isJitEntry() const { return kind() == JitEntry; }
|
||||
bool isInterpEntry() const { return kind() == InterpEntry; }
|
||||
bool isEntry() const { return isInterpEntry() || isJitEntry(); }
|
||||
bool isIndirectStub() const { return kind() == IndirectStub; }
|
||||
bool hasFuncIndex() const {
|
||||
return isFunction() || isImportExit() || isEntry();
|
||||
return isFunction() || isImportExit() || isEntry() || isIndirectStub();
|
||||
}
|
||||
uint32_t funcIndex() const {
|
||||
MOZ_ASSERT(hasFuncIndex());
|
||||
|
|
@ -410,7 +413,8 @@ class CallSiteDesc {
|
|||
|
||||
enum Kind {
|
||||
Func, // pc-relative call to a specific function
|
||||
Dynamic, // dynamic callee called via register
|
||||
Import, // wasm import call
|
||||
Indirect, // wasm indirect call
|
||||
Symbolic, // call to a single symbolic callee
|
||||
EnterFrame, // call to a enter frame handler
|
||||
LeaveFrame, // call to a leave frame handler
|
||||
|
|
@ -427,7 +431,8 @@ class CallSiteDesc {
|
|||
}
|
||||
uint32_t lineOrBytecode() const { return lineOrBytecode_; }
|
||||
Kind kind() const { return Kind(kind_); }
|
||||
bool mightBeCrossInstance() const { return kind() == CallSiteDesc::Dynamic; }
|
||||
bool isImportCall() const { return kind() == CallSiteDesc::Import; }
|
||||
bool isIndirectCall() const { return kind() == CallSiteDesc::Indirect; }
|
||||
};
|
||||
|
||||
class CallSite : public CallSiteDesc {
|
||||
|
|
|
|||
|
|
@ -34,8 +34,12 @@ using namespace js::wasm;
|
|||
DebugFrame* DebugFrame::from(Frame* fp) {
|
||||
MOZ_ASSERT(
|
||||
GetNearestEffectiveTls(fp)->instance->code().metadata().debugEnabled);
|
||||
auto* df =
|
||||
reinterpret_cast<DebugFrame*>((uint8_t*)fp - DebugFrame::offsetOfFrame());
|
||||
size_t offset = DebugFrame::offsetOfFrame();
|
||||
if (fp->callerIsTrampolineFP()) {
|
||||
offset +=
|
||||
FrameWithTls::sizeWithoutFrame() + IndirectStubAdditionalAlignment;
|
||||
}
|
||||
auto* df = reinterpret_cast<DebugFrame*>((uint8_t*)fp - offset);
|
||||
MOZ_ASSERT(GetNearestEffectiveTls(fp)->instance == df->instance());
|
||||
return df;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,25 @@ struct TlsData;
|
|||
|
||||
constexpr uintptr_t ExitOrJitEntryFPTag = 0x1;
|
||||
|
||||
// Tag for determining whether trampoline frame is on the stack or not.
|
||||
constexpr uintptr_t TrampolineFpTag = 0x2;
|
||||
|
||||
// In a normal call without an indirect stub SP is 16-byte aligned before
|
||||
// the call and, after the call, SP is (16 - 4) bytes aligned because
|
||||
// the call instruction pushes the return address. Callee expects SP to be
|
||||
// (16 - 4) bytes aligned for the checkedCallEntry and to be (16 - 8) byte
|
||||
// aligned for the tailCheckedEntry. When we call indirect stub instead of
|
||||
// callee, SP is still 16 bytes aligned, then we push two Frames here - one for
|
||||
// the stub and one on callee's behalf. Since each frame is (8) bytes aligned,
|
||||
// resulted SP will be also aligned on (16) when we jump into callee, so we
|
||||
// will step into the alignment assert. To prevent this we allocate additional
|
||||
// (8) bytes to satisfy caller's expectations about SP.
|
||||
#if defined(JS_CODEGEN_X86)
|
||||
constexpr uint32_t IndirectStubAdditionalAlignment = 8u;
|
||||
#else
|
||||
constexpr uint32_t IndirectStubAdditionalAlignment = 0u;
|
||||
#endif
|
||||
|
||||
// wasm::Frame represents the bytes pushed by the call instruction and the
|
||||
// fixed prologue generated by wasm::GenerateCallablePrologue.
|
||||
//
|
||||
|
|
@ -89,10 +108,25 @@ class Frame {
|
|||
return reinterpret_cast<Frame*>(callerFP_);
|
||||
}
|
||||
|
||||
Frame* trampolineCaller() const {
|
||||
MOZ_ASSERT(callerIsTrampolineFP());
|
||||
return reinterpret_cast<Frame*>(reinterpret_cast<uintptr_t>(callerFP_) &
|
||||
~TrampolineFpTag);
|
||||
}
|
||||
|
||||
bool callerIsExitOrJitEntryFP() const {
|
||||
return isExitOrJitEntryFP(callerFP_);
|
||||
}
|
||||
|
||||
bool callerIsTrampolineFP() const { return isTrampolineFP(callerFP_); }
|
||||
|
||||
size_t trampolineSlots() const {
|
||||
return callerIsTrampolineFP()
|
||||
? ((sizeof(Frame) + IndirectStubAdditionalAlignment) /
|
||||
sizeof(void*))
|
||||
: 0;
|
||||
}
|
||||
|
||||
uint8_t* jitEntryCaller() const { return toJitEntryCaller(callerFP_); }
|
||||
|
||||
static const Frame* fromUntaggedWasmExitFP(const void* savedFP) {
|
||||
|
|
@ -110,6 +144,10 @@ class Frame {
|
|||
~ExitOrJitEntryFPTag);
|
||||
}
|
||||
|
||||
static bool isTrampolineFP(const void* fp) {
|
||||
return reinterpret_cast<uintptr_t>(fp) & TrampolineFpTag;
|
||||
}
|
||||
|
||||
static uint8_t* addExitOrJitEntryFPTag(const Frame* fp) {
|
||||
MOZ_ASSERT(!isExitOrJitEntryFP(fp));
|
||||
return reinterpret_cast<uint8_t*>(reinterpret_cast<uintptr_t>(fp) |
|
||||
|
|
|
|||
|
|
@ -150,7 +150,11 @@ void WasmFrameIter::popFrame() {
|
|||
}
|
||||
|
||||
Frame* prevFP = fp_;
|
||||
fp_ = fp_->wasmCaller();
|
||||
if (fp_->callerIsTrampolineFP()) {
|
||||
fp_ = fp_->trampolineCaller();
|
||||
} else {
|
||||
fp_ = fp_->wasmCaller();
|
||||
}
|
||||
resumePCinCurrentFrame_ = prevFP->returnAddress();
|
||||
|
||||
if (!fp_) {
|
||||
|
|
@ -206,7 +210,9 @@ void WasmFrameIter::popFrame() {
|
|||
const CallSite* callsite = code_->lookupCallSite(returnAddress);
|
||||
MOZ_ASSERT(callsite);
|
||||
|
||||
if (callsite->mightBeCrossInstance()) {
|
||||
if (callsite->isImportCall()) {
|
||||
tls_ = ExtractCallerTlsFromFrameWithTls(prevFP);
|
||||
} else if (callsite->isIndirectCall() && prevFP->callerIsTrampolineFP()) {
|
||||
tls_ = ExtractCallerTlsFromFrameWithTls(prevFP);
|
||||
}
|
||||
|
||||
|
|
@ -921,6 +927,7 @@ void ProfilingFrameIterator::initFromExitFP(const Frame* fp) {
|
|||
callerFP_ = fp->rawCaller();
|
||||
AssertMatchesCallSite(callerPC_, callerFP_);
|
||||
break;
|
||||
case CodeRange::IndirectStub:
|
||||
case CodeRange::ImportJitExit:
|
||||
case CodeRange::ImportInterpExit:
|
||||
case CodeRange::BuiltinThunk:
|
||||
|
|
@ -952,6 +959,10 @@ static bool isSignatureCheckFail(uint32_t offsetInCode,
|
|||
|
||||
const TlsData* js::wasm::GetNearestEffectiveTls(const Frame* fp) {
|
||||
while (true) {
|
||||
if (fp->callerIsTrampolineFP()) {
|
||||
return ExtractCalleeTlsFromFrameWithTls(fp);
|
||||
}
|
||||
|
||||
if (fp->callerIsExitOrJitEntryFP()) {
|
||||
// It is a direct call from JIT.
|
||||
MOZ_ASSERT(!LookupCode(fp->returnAddress()));
|
||||
|
|
@ -967,10 +978,14 @@ const TlsData* js::wasm::GetNearestEffectiveTls(const Frame* fp) {
|
|||
return ExtractCalleeTlsFromFrameWithTls(fp);
|
||||
}
|
||||
|
||||
if (codeRange->isIndirectStub()) {
|
||||
return ExtractCalleeTlsFromFrameWithTls(fp->wasmCaller());
|
||||
}
|
||||
|
||||
MOZ_ASSERT(codeRange->kind() == CodeRange::Function);
|
||||
MOZ_ASSERT(code);
|
||||
const CallSite* callsite = code->lookupCallSite(returnAddress);
|
||||
if (callsite->mightBeCrossInstance()) {
|
||||
if (callsite->isImportCall()) {
|
||||
return ExtractCalleeTlsFromFrameWithTls(fp);
|
||||
}
|
||||
|
||||
|
|
@ -1157,7 +1172,9 @@ bool js::wasm::StartUnwinding(const RegisterState& registers,
|
|||
if (isSignatureCheckFail(offsetInCode, codeRange)) {
|
||||
// Frame have been pushed and FP has been set.
|
||||
const auto* frame = Frame::fromUntaggedWasmExitFP(fp);
|
||||
fixedFP = frame->rawCaller();
|
||||
fixedFP = frame->callerIsTrampolineFP()
|
||||
? reinterpret_cast<uint8_t*>(frame->trampolineCaller())
|
||||
: reinterpret_cast<uint8_t*>(frame->wasmCaller());
|
||||
fixedPC = frame->returnAddress();
|
||||
AssertMatchesCallSite(fixedPC, fixedFP);
|
||||
break;
|
||||
|
|
@ -1187,6 +1204,16 @@ bool js::wasm::StartUnwinding(const RegisterState& registers,
|
|||
// entry trampoline also doesn't GeneratePrologue/Epilogue so we can't
|
||||
// use the general unwinding logic above.
|
||||
break;
|
||||
case CodeRange::IndirectStub: {
|
||||
// IndirectStub is used now as a trivial proxy into the function
|
||||
// so we aren't in the prologue/epilogue.
|
||||
fixedPC = pc;
|
||||
fixedFP = fp;
|
||||
*unwoundCaller = false;
|
||||
AssertMatchesCallSite(Frame::fromUntaggedWasmExitFP(fp)->returnAddress(),
|
||||
Frame::fromUntaggedWasmExitFP(fp)->rawCaller());
|
||||
break;
|
||||
}
|
||||
case CodeRange::JitEntry:
|
||||
// There's a jit frame above the current one; we don't care about pc
|
||||
// since the Jit entry frame is a jit frame which can be considered as
|
||||
|
|
@ -1367,6 +1394,8 @@ void ProfilingFrameIterator::operator++() {
|
|||
}
|
||||
case CodeRange::InterpEntry:
|
||||
MOZ_CRASH("should have had null caller fp");
|
||||
case CodeRange::IndirectStub:
|
||||
MOZ_CRASH("we can't profile indirect stub");
|
||||
case CodeRange::JitEntry:
|
||||
MOZ_CRASH("should have been guarded above");
|
||||
case CodeRange::Throw:
|
||||
|
|
@ -1588,6 +1617,8 @@ const char* ProfilingFrameIterator::label() const {
|
|||
return code_->profilingLabel(codeRange_->funcIndex());
|
||||
case CodeRange::InterpEntry:
|
||||
MOZ_CRASH("should be an ExitReason");
|
||||
case CodeRange::IndirectStub:
|
||||
return "indirect stub";
|
||||
case CodeRange::JitEntry:
|
||||
return "fast entry trampoline (in wasm)";
|
||||
case CodeRange::ImportJitExit:
|
||||
|
|
|
|||
|
|
@ -72,6 +72,21 @@ struct StackMap final {
|
|||
// is also noted. This is used in Instance::traceFrame to check that the
|
||||
// TrapExitDummyValue is in the expected place in the frame.
|
||||
|
||||
// StackMap and indirect stubs.
|
||||
// StackMaps track every word in wasm frame except indirect stubs words
|
||||
// because these stubs can be dynamically added or not at runtime.
|
||||
// For example, when we do `call_indirect foo` for some cross-instance call
|
||||
// clang-format off
|
||||
// | some stack arg | <- covered by StackMap.
|
||||
// | caller TLS slot | <- covered by StackMap.
|
||||
// | callee TLS slot | <- covered by StackMap.
|
||||
// | return pc: caller | <- At runtime we read the tag below and skip this Frame in Instance::traceFrame.
|
||||
// | callerFP (tag) |
|
||||
// | some stubs data |
|
||||
// | return pc: to stub | <- callee's wasm::Frame, covered by StackMap.
|
||||
// | caller FP |
|
||||
// clang-format on
|
||||
|
||||
// The total number of stack words covered by the map ..
|
||||
uint32_t numMappedWords : 30;
|
||||
|
||||
|
|
|
|||
|
|
@ -474,7 +474,8 @@ bool ModuleGenerator::linkCallSites() {
|
|||
const CallSiteTarget& target = callSiteTargets_[lastPatchedCallSite_];
|
||||
uint32_t callerOffset = callSite.returnAddressOffset();
|
||||
switch (callSite.kind()) {
|
||||
case CallSiteDesc::Dynamic:
|
||||
case CallSiteDesc::Import:
|
||||
case CallSiteDesc::Indirect:
|
||||
case CallSiteDesc::Symbolic:
|
||||
break;
|
||||
case CallSiteDesc::Func: {
|
||||
|
|
@ -578,6 +579,9 @@ void ModuleGenerator::noteCodeRange(uint32_t codeRangeIndex,
|
|||
case CodeRange::Throw:
|
||||
// Jumped to by other stubs, so nothing to do.
|
||||
break;
|
||||
case CodeRange::IndirectStub:
|
||||
MOZ_CRASH("Indirect stub generates later - at runtime.");
|
||||
break;
|
||||
case CodeRange::FarJumpIsland:
|
||||
case CodeRange::BuiltinThunk:
|
||||
MOZ_CRASH("Unexpected CodeRange kind");
|
||||
|
|
|
|||
|
|
@ -117,6 +117,35 @@ TableTls& Instance::tableTls(const TableDesc& td) const {
|
|||
return *(TableTls*)(globalData() + td.globalDataOffset);
|
||||
}
|
||||
|
||||
void* Instance::checkedCallEntry(const uint32_t functionIndex,
|
||||
const Tier tier) const {
|
||||
uint8_t* codeBaseTier = codeBase(tier);
|
||||
const MetadataTier& metadataTier = metadata(tier);
|
||||
const CodeRangeVector& codeRanges = metadataTier.codeRanges;
|
||||
const Uint32Vector& funcToCodeRange = metadataTier.funcToCodeRange;
|
||||
return codeBaseTier +
|
||||
codeRanges[funcToCodeRange[functionIndex]].funcCheckedCallEntry();
|
||||
}
|
||||
|
||||
void* Instance::checkedCallEntryForWasmImportedFunction(JSFunction* fun,
|
||||
const Tier tier) const {
|
||||
MOZ_ASSERT(IsWasmExportedFunction(fun));
|
||||
return checkedCallEntry(ExportedFunctionToFuncIndex(fun), tier);
|
||||
}
|
||||
|
||||
static bool IsImportedFunction(const uint32_t functionIndex,
|
||||
const MetadataTier& metadataTier) {
|
||||
return functionIndex < metadataTier.funcImports.length();
|
||||
}
|
||||
|
||||
static bool IsJSExportedFunction(JSFunction* fun) {
|
||||
return !IsWasmExportedFunction(fun);
|
||||
}
|
||||
|
||||
static bool IsNullFunction(const uint32_t functionIndex) {
|
||||
return functionIndex == NullFuncIndex;
|
||||
}
|
||||
|
||||
// TODO(1626251): Consolidate definitions into Iterable.h
|
||||
static bool IterableToArray(JSContext* cx, HandleValue iterable,
|
||||
MutableHandle<ArrayObject*> array) {
|
||||
|
|
@ -679,62 +708,195 @@ inline int32_t WasmMemoryFill32(T memBase, size_t memLen, uint32_t byteOffset,
|
|||
return 0;
|
||||
}
|
||||
|
||||
void* Instance::createIndirectStub(uint32_t funcIndex, TlsData* targetTlsData) {
|
||||
const Tier tier = code().bestTier();
|
||||
const CodeTier& codeTier = code(tier);
|
||||
auto stubs = codeTier.lazyStubs().lock();
|
||||
|
||||
void* stub_entry = stubs->lookupIndirectStub(funcIndex, targetTlsData);
|
||||
if (stub_entry) {
|
||||
return stub_entry;
|
||||
}
|
||||
|
||||
VectorOfIndirectStubTarget targets;
|
||||
const Tier calleeTier = targetTlsData->instance->code().bestTier();
|
||||
void* checkedCallEntry =
|
||||
targetTlsData->instance->checkedCallEntry(funcIndex, calleeTier);
|
||||
if (!targets.append(
|
||||
IndirectStubTarget{funcIndex, checkedCallEntry, targetTlsData}) ||
|
||||
!stubs->createManyIndirectStubs(targets, codeTier)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return stubs->lookupIndirectStub(funcIndex, targetTlsData);
|
||||
}
|
||||
|
||||
bool Instance::createManyIndirectStubs(
|
||||
const VectorOfIndirectStubTarget& targets, const Tier tier) {
|
||||
if (targets.empty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const CodeTier& codeTier = code(tier);
|
||||
auto stubs = codeTier.lazyStubs().lock();
|
||||
return stubs->createManyIndirectStubs(targets, codeTier);
|
||||
}
|
||||
|
||||
void* Instance::getIndirectStub(uint32_t funcIndex, TlsData* targetTlsData,
|
||||
const Tier tier) const {
|
||||
MOZ_ASSERT(funcIndex != NullFuncIndex);
|
||||
|
||||
if (!code().hasTier(tier)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto stubs = code(tier).lazyStubs().lock();
|
||||
return stubs->lookupIndirectStub(funcIndex, targetTlsData);
|
||||
}
|
||||
|
||||
static void RemoveDuplicates(VectorOfIndirectStubTarget* vector) {
|
||||
auto comparator = [](const IndirectStubTarget& lhs,
|
||||
const IndirectStubTarget& rhs) {
|
||||
if (lhs.functionIdx == rhs.functionIdx) {
|
||||
const auto lshTls = reinterpret_cast<uintptr_t>(lhs.tls);
|
||||
const auto rshTls = reinterpret_cast<uintptr_t>(rhs.tls);
|
||||
return lshTls < rshTls;
|
||||
}
|
||||
return lhs.functionIdx < rhs.functionIdx;
|
||||
};
|
||||
std::sort(vector->begin(), vector->end(), comparator);
|
||||
auto* newEnd = std::unique(vector->begin(), vector->end());
|
||||
vector->erase(newEnd, vector->end());
|
||||
}
|
||||
|
||||
bool Instance::ensureIndirectStubs(const Uint32Vector& elemFuncIndices,
|
||||
uint32_t srcOffset, uint32_t len,
|
||||
const Tier tier,
|
||||
const bool tableIsImportedOrExported) {
|
||||
const MetadataTier& metadataTier = metadata(tier);
|
||||
VectorOfIndirectStubTarget targets;
|
||||
|
||||
for (uint32_t i = 0; i < len; i++) {
|
||||
const uint32_t funcIndex = elemFuncIndices[srcOffset + i];
|
||||
if (IsNullFunction(funcIndex)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsImportedFunction(funcIndex, metadataTier)) {
|
||||
FuncImportTls& import =
|
||||
funcImportTls(metadataTier.funcImports[funcIndex]);
|
||||
JSFunction* fun = import.fun;
|
||||
TlsData* calleeTls = import.tls->instance->tlsData();
|
||||
if (IsJSExportedFunction(fun) ||
|
||||
getIndirectStub(funcIndex, calleeTls, tier)) {
|
||||
continue;
|
||||
}
|
||||
void* calleeEntry =
|
||||
import.tls->instance->checkedCallEntryForWasmImportedFunction(fun,
|
||||
tier);
|
||||
if (!targets.append(
|
||||
IndirectStubTarget{funcIndex, calleeEntry, calleeTls})) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!tableIsImportedOrExported ||
|
||||
getIndirectStub(funcIndex, tlsData(), tier)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!targets.append(IndirectStubTarget{
|
||||
funcIndex, checkedCallEntry(funcIndex, tier), tlsData()})) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
RemoveDuplicates(&targets);
|
||||
return createManyIndirectStubs(targets, tier);
|
||||
}
|
||||
|
||||
bool Instance::ensureIndirectStub(FuncRef* ref, const Tier tier,
|
||||
const bool tableIsImportedOrExported) {
|
||||
if (ref->isNull()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t functionIndex = ExportedFunctionToFuncIndex(ref->asJSFunction());
|
||||
Uint32Vector functionIndices;
|
||||
if (!functionIndices.append(functionIndex)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return ensureIndirectStubs(functionIndices, 0, 1u, tier,
|
||||
tableIsImportedOrExported);
|
||||
}
|
||||
|
||||
bool Instance::initElems(uint32_t tableIndex, const ElemSegment& seg,
|
||||
uint32_t dstOffset, uint32_t srcOffset, uint32_t len) {
|
||||
Table& table = *tables_[tableIndex];
|
||||
MOZ_ASSERT(dstOffset <= table.length());
|
||||
MOZ_ASSERT(len <= table.length() - dstOffset);
|
||||
|
||||
Tier tier = code().bestTier();
|
||||
const Tier tier = code().bestTier();
|
||||
const MetadataTier& metadataTier = metadata(tier);
|
||||
const FuncImportVector& funcImports = metadataTier.funcImports;
|
||||
const CodeRangeVector& codeRanges = metadataTier.codeRanges;
|
||||
const Uint32Vector& funcToCodeRange = metadataTier.funcToCodeRange;
|
||||
const Uint32Vector& elemFuncIndices = seg.elemFuncIndices;
|
||||
MOZ_ASSERT(srcOffset <= elemFuncIndices.length());
|
||||
MOZ_ASSERT(len <= elemFuncIndices.length() - srcOffset);
|
||||
|
||||
uint8_t* codeBaseTier = codeBase(tier);
|
||||
if (table.isFunction()) {
|
||||
if (!ensureIndirectStubs(elemFuncIndices, srcOffset, len, tier,
|
||||
table.isImportedOrExported())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < len; i++) {
|
||||
uint32_t funcIndex = elemFuncIndices[srcOffset + i];
|
||||
if (funcIndex == NullFuncIndex) {
|
||||
if (IsNullFunction(funcIndex)) {
|
||||
table.setNull(dstOffset + i);
|
||||
} else if (!table.isFunction()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!table.isFunction()) {
|
||||
// Note, fnref must be rooted if we do anything more than just store it.
|
||||
void* fnref = Instance::refFunc(this, funcIndex);
|
||||
if (fnref == AnyRef::invalid().forCompiledCode()) {
|
||||
return false; // OOM, which has already been reported.
|
||||
}
|
||||
table.fillAnyRef(dstOffset + i, 1, AnyRef::fromCompiledCode(fnref));
|
||||
} else {
|
||||
if (funcIndex < funcImports.length()) {
|
||||
FuncImportTls& import = funcImportTls(funcImports[funcIndex]);
|
||||
JSFunction* fun = import.fun;
|
||||
if (IsWasmExportedFunction(fun)) {
|
||||
// This element is a wasm function imported from another
|
||||
// instance. To preserve the === function identity required by
|
||||
// the JS embedding spec, we must set the element to the
|
||||
// imported function's underlying CodeRange.funcCheckedCallEntry and
|
||||
// Instance so that future Table.get()s produce the same
|
||||
// function object as was imported.
|
||||
WasmInstanceObject* calleeInstanceObj =
|
||||
ExportedFunctionToInstanceObject(fun);
|
||||
Instance& calleeInstance = calleeInstanceObj->instance();
|
||||
Tier calleeTier = calleeInstance.code().bestTier();
|
||||
const CodeRange& calleeCodeRange =
|
||||
calleeInstanceObj->getExportedFunctionCodeRange(fun, calleeTier);
|
||||
void* code = calleeInstance.codeBase(calleeTier) +
|
||||
calleeCodeRange.funcCheckedCallEntry();
|
||||
table.setFuncRef(dstOffset + i, code, &calleeInstance);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
void* code =
|
||||
codeBaseTier +
|
||||
codeRanges[funcToCodeRange[funcIndex]].funcCheckedCallEntry();
|
||||
table.setFuncRef(dstOffset + i, code, this);
|
||||
continue;
|
||||
}
|
||||
|
||||
void* code = nullptr;
|
||||
if (IsImportedFunction(funcIndex, metadataTier)) {
|
||||
FuncImportTls& import = funcImportTls(funcImports[funcIndex]);
|
||||
JSFunction* fun = import.fun;
|
||||
if (IsJSExportedFunction(fun)) {
|
||||
code = checkedCallEntry(funcIndex, tier);
|
||||
} else {
|
||||
code = getIndirectStub(funcIndex, import.tls, tier);
|
||||
MOZ_ASSERT(code);
|
||||
}
|
||||
} else {
|
||||
// The function is an internal wasm function that belongs to the current
|
||||
// instance. If table is isImportedOrExported then some other module can
|
||||
// import this table and call its functions so we have to use indirect
|
||||
// stub, otherwise we can use checked call entry because we don't cross
|
||||
// instance's borders.
|
||||
if (table.isImportedOrExported()) {
|
||||
code = getIndirectStub(funcIndex, tlsData(), tier);
|
||||
MOZ_ASSERT(code);
|
||||
} else {
|
||||
code = checkedCallEntry(funcIndex, tier);
|
||||
}
|
||||
}
|
||||
|
||||
MOZ_ASSERT(code);
|
||||
table.setFuncRef(dstOffset + i, code, this);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -810,7 +972,11 @@ bool Instance::initElems(uint32_t tableIndex, const ElemSegment& seg,
|
|||
break;
|
||||
case TableRepr::Func:
|
||||
MOZ_RELEASE_ASSERT(!table.isAsmJS());
|
||||
table.fillFuncRef(start, len, FuncRef::fromCompiledCode(value), cx);
|
||||
if (!table.fillFuncRef(start, len, FuncRef::fromCompiledCode(value),
|
||||
cx)) {
|
||||
ReportOutOfMemory(cx);
|
||||
return -1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -850,19 +1016,28 @@ bool Instance::initElems(uint32_t tableIndex, const ElemSegment& seg,
|
|||
RootedAnyRef ref(TlsContext.get(), AnyRef::fromCompiledCode(initValue));
|
||||
Table& table = *instance->tables()[tableIndex];
|
||||
|
||||
if (table.isFunction()) {
|
||||
RootedFuncRef functionForFill(TlsContext.get(),
|
||||
FuncRef::fromAnyRefUnchecked(ref));
|
||||
if (!instance->ensureIndirectStub(functionForFill.address(),
|
||||
instance->code().bestTier(),
|
||||
table.isImportedOrExported())) {
|
||||
return -1;
|
||||
}
|
||||
uint32_t oldSize = table.grow(delta);
|
||||
if (oldSize != uint32_t(-1) && initValue != nullptr) {
|
||||
MOZ_RELEASE_ASSERT(!table.isAsmJS());
|
||||
MOZ_RELEASE_ASSERT(
|
||||
table.fillFuncRef(oldSize, delta, functionForFill, TlsContext.get()));
|
||||
}
|
||||
return oldSize;
|
||||
}
|
||||
|
||||
MOZ_ASSERT(!table.isFunction());
|
||||
uint32_t oldSize = table.grow(delta);
|
||||
|
||||
if (oldSize != uint32_t(-1) && initValue != nullptr) {
|
||||
switch (table.repr()) {
|
||||
case TableRepr::Ref:
|
||||
table.fillAnyRef(oldSize, delta, ref);
|
||||
break;
|
||||
case TableRepr::Func:
|
||||
MOZ_RELEASE_ASSERT(!table.isAsmJS());
|
||||
table.fillFuncRef(oldSize, delta, FuncRef::fromAnyRefUnchecked(ref),
|
||||
TlsContext.get());
|
||||
break;
|
||||
}
|
||||
table.fillAnyRef(oldSize, delta, ref);
|
||||
}
|
||||
|
||||
return oldSize;
|
||||
|
|
@ -885,8 +1060,11 @@ bool Instance::initElems(uint32_t tableIndex, const ElemSegment& seg,
|
|||
break;
|
||||
case TableRepr::Func:
|
||||
MOZ_RELEASE_ASSERT(!table.isAsmJS());
|
||||
table.fillFuncRef(index, 1, FuncRef::fromCompiledCode(value),
|
||||
TlsContext.get());
|
||||
if (!table.fillFuncRef(index, 1, FuncRef::fromCompiledCode(value),
|
||||
TlsContext.get())) {
|
||||
ReportOutOfMemory(TlsContext.get());
|
||||
return -1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -1547,7 +1725,8 @@ uintptr_t Instance::traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi,
|
|||
// We have to calculate |scanStart|, the lowest address that is described by
|
||||
// |map|, by consulting |map->frameOffsetFromTop|.
|
||||
|
||||
const size_t numMappedBytes = map->numMappedWords * sizeof(void*);
|
||||
const size_t numMappedBytes =
|
||||
(map->numMappedWords + frame->trampolineSlots()) * sizeof(void*);
|
||||
const uintptr_t scanStart = uintptr_t(frame) +
|
||||
(map->frameOffsetFromTop * sizeof(void*)) -
|
||||
numMappedBytes;
|
||||
|
|
@ -1575,6 +1754,9 @@ uintptr_t Instance::traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi,
|
|||
#endif
|
||||
|
||||
uintptr_t* stackWords = (uintptr_t*)scanStart;
|
||||
const uint32_t trampolineBeginIdx =
|
||||
map->numMappedWords - static_cast<uint32_t>(map->frameOffsetFromTop) +
|
||||
frame->trampolineSlots();
|
||||
|
||||
// If we have some exit stub words, this means the map also covers an area
|
||||
// created by a exit stub, and so the highest word of that should be a
|
||||
|
|
@ -1585,8 +1767,15 @@ uintptr_t Instance::traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi,
|
|||
TrapExitDummyValue);
|
||||
|
||||
// And actually hand them off to the GC.
|
||||
for (uint32_t i = 0; i < map->numMappedWords; i++) {
|
||||
if (map->getBit(i) == 0) {
|
||||
uint32_t stackIdx = 0;
|
||||
for (uint32_t mapIdx = 0; mapIdx < map->numMappedWords;
|
||||
++mapIdx, ++stackIdx) {
|
||||
if (frame->callerIsTrampolineFP() && mapIdx == trampolineBeginIdx) {
|
||||
// Skip trampoline frame.
|
||||
stackIdx += frame->trampolineSlots();
|
||||
}
|
||||
|
||||
if (map->getBit(mapIdx) == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -1596,10 +1785,11 @@ uintptr_t Instance::traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi,
|
|||
|
||||
// This assertion seems at least moderately effective in detecting
|
||||
// discrepancies or misalignments between the map and reality.
|
||||
MOZ_ASSERT(js::gc::IsCellPointerValidOrNull((const void*)stackWords[i]));
|
||||
MOZ_ASSERT(
|
||||
js::gc::IsCellPointerValidOrNull((const void*)stackWords[stackIdx]));
|
||||
|
||||
if (stackWords[i]) {
|
||||
TraceRoot(trc, (JSObject**)&stackWords[i],
|
||||
if (stackWords[stackIdx]) {
|
||||
TraceRoot(trc, (JSObject**)&stackWords[stackIdx],
|
||||
"Instance::traceWasmFrame: normal word");
|
||||
}
|
||||
}
|
||||
|
|
@ -1634,6 +1824,26 @@ uintptr_t Instance::traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi,
|
|||
return scanStart + numMappedBytes - 1;
|
||||
}
|
||||
|
||||
Instance* Instance::getOriginalInstanceAndFunction(uint32_t funcIdx,
|
||||
JSFunction** fun) {
|
||||
Tier tier = code().bestTier();
|
||||
const MetadataTier& metadataTier = metadata(tier);
|
||||
const FuncImportVector& funcImports = metadataTier.funcImports;
|
||||
|
||||
if (IsImportedFunction(funcIdx, metadataTier)) {
|
||||
FuncImportTls& import = funcImportTls(funcImports[funcIdx]);
|
||||
*fun = import.fun;
|
||||
if (IsJSExportedFunction(*fun)) {
|
||||
return this;
|
||||
}
|
||||
WasmInstanceObject* calleeInstanceObj =
|
||||
ExportedFunctionToInstanceObject(*fun);
|
||||
return &calleeInstanceObj->instance();
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
WasmMemoryObject* Instance::memory() const { return memory_; }
|
||||
|
||||
SharedMem<uint8_t*> Instance::memoryBase() const {
|
||||
|
|
@ -1700,7 +1910,7 @@ static bool EnsureEntryStubs(const Instance& instance, uint32_t funcIndex,
|
|||
tier = instance.code().bestTier();
|
||||
const CodeTier& codeTier = instance.code(tier);
|
||||
if (tier == prevTier) {
|
||||
if (!stubs->createOne(funcExportIndex, codeTier)) {
|
||||
if (!stubs->createOneEntryStub(funcExportIndex, codeTier)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -1714,9 +1924,9 @@ static bool EnsureEntryStubs(const Instance& instance, uint32_t funcIndex,
|
|||
|
||||
// If it didn't have a stub in the first tier, background compilation
|
||||
// shouldn't have made one in the second tier.
|
||||
MOZ_ASSERT(!stubs2->hasStub(fe.funcIndex()));
|
||||
MOZ_ASSERT(!stubs2->hasEntryStub(fe.funcIndex()));
|
||||
|
||||
if (!stubs2->createOne(funcExportIndex, codeTier)) {
|
||||
if (!stubs2->createOneEntryStub(funcExportIndex, codeTier)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,9 @@ class Instance {
|
|||
const void** addressOfTypeId(const TypeIdDesc& typeId) const;
|
||||
FuncImportTls& funcImportTls(const FuncImport& fi);
|
||||
TableTls& tableTls(const TableDesc& td) const;
|
||||
void* checkedCallEntry(const uint32_t functionIndex, const Tier tier) const;
|
||||
void* checkedCallEntryForWasmImportedFunction(JSFunction* fun,
|
||||
const Tier tier) const;
|
||||
|
||||
// Only WasmInstanceObject can call the private trace function.
|
||||
friend class js::WasmInstanceObject;
|
||||
|
|
@ -98,6 +101,8 @@ class Instance {
|
|||
uint8_t* nextPC,
|
||||
uintptr_t highestByteVisitedInPrevFrame);
|
||||
|
||||
Instance* getOriginalInstanceAndFunction(uint32_t funcIdx, JSFunction** fun);
|
||||
|
||||
JS::Realm* realm() const { return realm_; }
|
||||
const Code& code() const { return *code_; }
|
||||
const CodeTier& code(Tier t) const { return code_->codeTier(t); }
|
||||
|
|
@ -173,6 +178,17 @@ class Instance {
|
|||
uint32_t dstOffset, uint32_t srcOffset,
|
||||
uint32_t len);
|
||||
|
||||
void* getIndirectStub(uint32_t funcIndex, TlsData* targetTlsData,
|
||||
const Tier tier) const;
|
||||
void* createIndirectStub(uint32_t funcIndex, TlsData* targetTlsData);
|
||||
bool createManyIndirectStubs(const VectorOfIndirectStubTarget& targets,
|
||||
const Tier tier);
|
||||
bool ensureIndirectStubs(const Uint32Vector& elemFuncIndices,
|
||||
uint32_t srcOffset, uint32_t len, const Tier tier,
|
||||
const bool tableIsImportedOrExported);
|
||||
bool ensureIndirectStub(FuncRef* ref, const Tier tier,
|
||||
const bool tableIsImportedOrExported);
|
||||
|
||||
// Debugger support:
|
||||
|
||||
JSString* createDisplayURL(JSContext* cx);
|
||||
|
|
|
|||
|
|
@ -1630,7 +1630,7 @@ class FunctionCompiler {
|
|||
callee = CalleeDesc::wasmTable(table, funcTypeId);
|
||||
}
|
||||
|
||||
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Dynamic);
|
||||
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Indirect);
|
||||
ArgTypeVector args(funcType);
|
||||
ResultType resultType = ResultType::Vector(funcType.results());
|
||||
auto* ins = MWasmCall::New(alloc(), desc, callee, call.regArgs_,
|
||||
|
|
@ -1651,7 +1651,7 @@ class FunctionCompiler {
|
|||
return true;
|
||||
}
|
||||
|
||||
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Dynamic);
|
||||
CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Import);
|
||||
auto callee = CalleeDesc::import(globalDataOffset);
|
||||
ArgTypeVector args(funcType);
|
||||
ResultType resultType = ResultType::Vector(funcType.results());
|
||||
|
|
|
|||
|
|
@ -3276,7 +3276,10 @@ bool WasmTableObject::fillRange(JSContext* cx, uint32_t index, uint32_t length,
|
|||
switch (tab.repr()) {
|
||||
case TableRepr::Func:
|
||||
MOZ_RELEASE_ASSERT(!tab.isAsmJS());
|
||||
tab.fillFuncRef(index, length, FuncRef::fromJSFunction(fun), cx);
|
||||
if (!tab.fillFuncRef(index, length, FuncRef::fromJSFunction(fun), cx)) {
|
||||
ReportOutOfMemory(cx);
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
case TableRepr::Ref:
|
||||
tab.fillAnyRef(index, length, any);
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ bool Module::finishTier2(const LinkData& linkData2,
|
|||
auto stubs1 = code().codeTier(Tier::Baseline).lazyStubs().lock();
|
||||
auto stubs2 = code().codeTier(Tier::Optimized).lazyStubs().lock();
|
||||
|
||||
MOZ_ASSERT(stubs2->empty());
|
||||
MOZ_ASSERT(stubs2->entryStubsEmpty());
|
||||
|
||||
Uint32Vector funcExportIndices;
|
||||
for (size_t i = 0; i < metadataTier1.funcExports.length(); i++) {
|
||||
|
|
@ -145,7 +145,7 @@ bool Module::finishTier2(const LinkData& linkData2,
|
|||
if (fe.hasEagerStubs()) {
|
||||
continue;
|
||||
}
|
||||
if (!stubs1->hasStub(fe.funcIndex())) {
|
||||
if (!stubs1->hasEntryStub(fe.funcIndex())) {
|
||||
continue;
|
||||
}
|
||||
if (!funcExportIndices.emplaceBack(i)) {
|
||||
|
|
|
|||
|
|
@ -1979,7 +1979,7 @@ static bool GenerateImportFunction(jit::MacroAssembler& masm,
|
|||
}
|
||||
|
||||
// Call the import exit stub.
|
||||
CallSiteDesc desc(CallSiteDesc::Dynamic);
|
||||
CallSiteDesc desc(CallSiteDesc::Import);
|
||||
MoveSPForJitABI(masm);
|
||||
masm.wasmCallImport(desc, CalleeDesc::import(fi.tlsDataOffset()));
|
||||
|
||||
|
|
@ -2568,6 +2568,167 @@ struct ABIFunctionArgs {
|
|||
}
|
||||
};
|
||||
|
||||
static void PushFrame(MacroAssembler& masm) {
|
||||
#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
|
||||
masm.subFromStackPtr(Imm32(sizeof(Frame)));
|
||||
masm.storePtr(ra, Address(StackPointer, Frame::returnAddressOffset()));
|
||||
masm.storePtr(FramePointer, Address(StackPointer, Frame::callerFPOffset()));
|
||||
#elif defined(JS_CODEGEN_ARM64)
|
||||
{
|
||||
AutoForbidPoolsAndNops afp(&masm,
|
||||
/* number of instructions in scope = */ 3);
|
||||
masm.Sub(sp, sp, sizeof(Frame));
|
||||
masm.Str(ARMRegister(lr, 64), MemOperand(sp, Frame::returnAddressOffset()));
|
||||
masm.Str(ARMRegister(FramePointer, 64),
|
||||
MemOperand(sp, Frame::callerFPOffset()));
|
||||
}
|
||||
#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_X64) || \
|
||||
defined(JS_CODEGEN_X86)
|
||||
{
|
||||
# if defined(JS_CODEGEN_ARM)
|
||||
AutoForbidPoolsAndNops afp(&masm,
|
||||
/* number of instructions in scope = */ 3);
|
||||
masm.push(lr);
|
||||
# endif
|
||||
masm.push(FramePointer);
|
||||
}
|
||||
#else
|
||||
MOZ_CRASH("Unknown architecture");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void PopFrame(MacroAssembler& masm) {
|
||||
#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
|
||||
masm.loadPtr(Address(StackPointer, Frame::callerFPOffset()), FramePointer);
|
||||
masm.loadPtr(Address(StackPointer, Frame::returnAddressOffset()), ra);
|
||||
masm.addToStackPtr(Imm32(sizeof(Frame)));
|
||||
#elif defined(JS_CODEGEN_ARM64)
|
||||
{
|
||||
AutoForbidPoolsAndNops afp(&masm,
|
||||
/* number of instructions in scope = */ 3);
|
||||
|
||||
masm.Ldr(ARMRegister(FramePointer, 64),
|
||||
MemOperand(sp, Frame::callerFPOffset()));
|
||||
masm.Ldr(ARMRegister(lr, 64), MemOperand(sp, Frame::returnAddressOffset()));
|
||||
masm.Add(sp, sp, sizeof(Frame));
|
||||
}
|
||||
#elif defined(JS_CODEGEN_ARM)
|
||||
{
|
||||
AutoForbidPoolsAndNops afp(&masm,
|
||||
/* number of instructions in scope = */ 3);
|
||||
masm.pop(FramePointer);
|
||||
masm.pop(lr);
|
||||
}
|
||||
#elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
|
||||
masm.pop(FramePointer);
|
||||
#else
|
||||
MOZ_CRASH("Unknown architecture");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void AllocateStackBytes(MacroAssembler& masm, const uint32_t bytes) {
|
||||
if (!bytes) {
|
||||
return;
|
||||
}
|
||||
|
||||
#if defined(JS_CODEGEN_ARM64)
|
||||
masm.Sub(sp, sp, bytes);
|
||||
#else
|
||||
masm.subPtr(Imm32(static_cast<int32_t>(bytes)), masm.getStackPointer());
|
||||
#endif
|
||||
}
|
||||
|
||||
static void DeallocateStackBytes(MacroAssembler& masm, const uint32_t bytes) {
|
||||
if (!bytes) {
|
||||
return;
|
||||
}
|
||||
|
||||
#if defined(JS_CODEGEN_ARM64)
|
||||
masm.Add(sp, sp, bytes);
|
||||
#else
|
||||
masm.addPtr(Imm32(static_cast<int32_t>(bytes)), masm.getStackPointer());
|
||||
#endif
|
||||
}
|
||||
|
||||
bool wasm::GenerateIndirectStub(MacroAssembler& masm,
|
||||
uint8_t* calleeCheckedEntry, TlsData* tlsPtr,
|
||||
Offsets* offsets) {
|
||||
#if defined(JS_CODEGEN_ARM64)
|
||||
// See comment in |GenerateCallablePrologue|.
|
||||
// Revisit after resolution of the
|
||||
// https://bugzilla.mozilla.org/show_bug.cgi?id=1709853.
|
||||
const vixl::Register stashedSPreg = masm.GetStackPointer64();
|
||||
masm.SetStackPointer64(vixl::sp);
|
||||
#endif
|
||||
|
||||
ImmWord tlsPtrImm(reinterpret_cast<uintptr_t>(tlsPtr));
|
||||
Label sameInstanceCase;
|
||||
|
||||
masm.haltingAlign(CodeAlignment);
|
||||
offsets->begin = masm.currentOffset();
|
||||
masm.setFramePushed(0);
|
||||
|
||||
masm.movePtr(tlsPtrImm, WasmTableCallScratchReg0);
|
||||
masm.branchPtr(Assembler::Condition::Equal, WasmTlsReg,
|
||||
WasmTableCallScratchReg0, &sameInstanceCase);
|
||||
|
||||
// Preserve caller's TLS and callee's TLS.
|
||||
masm.storePtr(WasmTlsReg,
|
||||
Address(masm.getStackPointer(), WasmCallerTLSOffsetAfterCall));
|
||||
masm.movePtr(WasmTableCallScratchReg0, WasmTlsReg);
|
||||
masm.storePtr(WasmTableCallScratchReg0,
|
||||
Address(masm.getStackPointer(), WasmCalleeTLSOffsetAfterCall));
|
||||
masm.loadWasmPinnedRegsFromTls();
|
||||
masm.switchToWasmTlsRealm(WasmTableCallIndexReg, WasmTableCallScratchReg1);
|
||||
|
||||
// Setup the frame for this stub and tag caller FP so runtime can recongize
|
||||
// it during stack walking.
|
||||
PushFrame(masm);
|
||||
masm.moveStackPtrTo(FramePointer);
|
||||
masm.addPtr(Imm32(wasm::TrampolineFpTag), Address(FramePointer, 0));
|
||||
|
||||
Label prepareFrameOnCalleeBehalfAndJumpCallee;
|
||||
|
||||
AllocateStackBytes(masm, IndirectStubAdditionalAlignment);
|
||||
masm.call(&prepareFrameOnCalleeBehalfAndJumpCallee);
|
||||
DeallocateStackBytes(masm, IndirectStubAdditionalAlignment);
|
||||
|
||||
// Restore the caller state and return.
|
||||
PopFrame(masm);
|
||||
masm.subPtr(Imm32(wasm::TrampolineFpTag), FramePointer);
|
||||
masm.loadPtr(Address(masm.getStackPointer(), WasmCallerTLSOffsetAfterCall),
|
||||
WasmTlsReg);
|
||||
masm.loadWasmPinnedRegsFromTls();
|
||||
masm.switchToWasmTlsRealm(WasmTableCallIndexReg, WasmTableCallScratchReg1);
|
||||
|
||||
#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
|
||||
masm.as_jr(ra);
|
||||
#elif defined(JS_CODEGEN_ARM64)
|
||||
// See comment in |GenerateCallablePrologue|.
|
||||
masm.Mov(PseudoStackPointer64, vixl::sp);
|
||||
masm.Ret(ARMRegister(lr, 64));
|
||||
masm.SetStackPointer64(stashedSPreg);
|
||||
#elif defined(JS_CODEGEN_ARM)
|
||||
masm.branch(lr);
|
||||
#elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
|
||||
masm.ret();
|
||||
#else
|
||||
MOZ_CRASH("Unknown architecture");
|
||||
#endif
|
||||
|
||||
masm.bind(&prepareFrameOnCalleeBehalfAndJumpCallee);
|
||||
PushFrame(masm);
|
||||
ImmPtr calleeTailEntry(calleeCheckedEntry + WasmCheckedTailEntryOffset,
|
||||
ImmPtr::NoCheckToken());
|
||||
masm.jump(calleeTailEntry);
|
||||
|
||||
masm.bind(&sameInstanceCase);
|
||||
ImmPtr calleeCheckedCallEntry(calleeCheckedEntry, ImmPtr::NoCheckToken());
|
||||
masm.jump(calleeCheckedCallEntry);
|
||||
|
||||
return FinishOffsets(masm, offsets);
|
||||
}
|
||||
|
||||
bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIFunctionType abiType,
|
||||
ExitReason exitReason, void* funcPtr,
|
||||
CallableOffsets* offsets) {
|
||||
|
|
|
|||
|
|
@ -247,6 +247,10 @@ class ABIResultIter {
|
|||
}
|
||||
};
|
||||
|
||||
extern bool GenerateIndirectStub(jit::MacroAssembler& masm,
|
||||
uint8_t* calleeCheckedEntry, TlsData* tlsPtr,
|
||||
Offsets* offsets);
|
||||
|
||||
extern bool GenerateBuiltinThunk(jit::MacroAssembler& masm,
|
||||
jit::ABIFunctionType abiType,
|
||||
ExitReason exitReason, void* funcPtr,
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ Table::Table(JSContext* cx, const TableDesc& desc,
|
|||
functions_(std::move(functions)),
|
||||
elemType_(desc.elemType),
|
||||
isAsmJS_(desc.isAsmJS),
|
||||
importedOrExported(desc.importedOrExported),
|
||||
length_(desc.initialLength),
|
||||
maximum_(desc.maximumLength) {
|
||||
MOZ_ASSERT(repr() == TableRepr::Func);
|
||||
|
|
@ -50,6 +51,7 @@ Table::Table(JSContext* cx, const TableDesc& desc,
|
|||
objects_(std::move(objects)),
|
||||
elemType_(desc.elemType),
|
||||
isAsmJS_(desc.isAsmJS),
|
||||
importedOrExported(desc.importedOrExported),
|
||||
length_(desc.initialLength),
|
||||
maximum_(desc.maximumLength) {
|
||||
MOZ_ASSERT(repr() == TableRepr::Ref);
|
||||
|
|
@ -151,17 +153,38 @@ bool Table::getFuncRef(JSContext* cx, uint32_t index,
|
|||
MOZ_ASSERT(isFunction());
|
||||
|
||||
const FunctionTableElem& elem = getFuncRef(index);
|
||||
if (!elem.code) {
|
||||
if (!elem.tls) {
|
||||
fun.set(nullptr);
|
||||
return true;
|
||||
}
|
||||
|
||||
Instance& instance = *elem.tls->instance;
|
||||
const CodeRange& codeRange = *instance.code().lookupFuncRange(elem.code);
|
||||
const CodeRange* codeRange =
|
||||
instance.code().lookupIndirectStubRange(elem.code);
|
||||
if (!codeRange) {
|
||||
codeRange = instance.code().lookupFuncRange(elem.code);
|
||||
}
|
||||
MOZ_ASSERT(codeRange);
|
||||
|
||||
RootedWasmInstanceObject instanceObj(cx, instance.object());
|
||||
return instanceObj->getExportedFunction(cx, instanceObj,
|
||||
codeRange.funcIndex(), fun);
|
||||
// If the element is a wasm function imported from another
|
||||
// instance then to preserve the === function identity required by
|
||||
// the JS embedding spec, we must set the element to the
|
||||
// imported function's underlying CodeRange.funcCheckedCallEntry and
|
||||
// Instance so that future Table.get()s produce the same
|
||||
// function object as was imported.
|
||||
JSFunction* callee = nullptr;
|
||||
Instance* calleeInstance =
|
||||
instance.getOriginalInstanceAndFunction(codeRange->funcIndex(), &callee);
|
||||
RootedWasmInstanceObject calleeInstanceObj(cx, calleeInstance->object());
|
||||
uint32_t calleeFunctionIndex = codeRange->funcIndex();
|
||||
if (callee && (calleeInstance != &instance)) {
|
||||
const Tier calleeTier = calleeInstance->code().bestTier();
|
||||
calleeFunctionIndex =
|
||||
calleeInstanceObj->getExportedFunctionCodeRange(callee, calleeTier)
|
||||
.funcIndex();
|
||||
}
|
||||
return WasmInstanceObject::getExportedFunction(cx, calleeInstanceObj,
|
||||
calleeFunctionIndex, fun);
|
||||
}
|
||||
|
||||
void Table::setFuncRef(uint32_t index, void* code, const Instance* instance) {
|
||||
|
|
@ -183,7 +206,7 @@ void Table::setFuncRef(uint32_t index, void* code, const Instance* instance) {
|
|||
}
|
||||
}
|
||||
|
||||
void Table::fillFuncRef(uint32_t index, uint32_t fillCount, FuncRef ref,
|
||||
bool Table::fillFuncRef(uint32_t index, uint32_t fillCount, FuncRef ref,
|
||||
JSContext* cx) {
|
||||
MOZ_ASSERT(isFunction());
|
||||
|
||||
|
|
@ -191,7 +214,7 @@ void Table::fillFuncRef(uint32_t index, uint32_t fillCount, FuncRef ref,
|
|||
for (uint32_t i = index, end = index + fillCount; i != end; i++) {
|
||||
setNull(i);
|
||||
}
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
|
||||
RootedFunction fun(cx, ref.asJSFunction());
|
||||
|
|
@ -208,14 +231,16 @@ void Table::fillFuncRef(uint32_t index, uint32_t fillCount, FuncRef ref,
|
|||
#endif
|
||||
|
||||
Instance& instance = instanceObj->instance();
|
||||
Tier tier = instance.code().bestTier();
|
||||
const MetadataTier& metadata = instance.metadata(tier);
|
||||
const CodeRange& codeRange =
|
||||
metadata.codeRange(metadata.lookupFuncExport(funcIndex));
|
||||
void* code = instance.codeBase(tier) + codeRange.funcCheckedCallEntry();
|
||||
void* code = instance.createIndirectStub(funcIndex, instance.tlsData());
|
||||
if (!code) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (uint32_t i = index, end = index + fillCount; i != end; i++) {
|
||||
setFuncRef(i, code, &instance);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
AnyRef Table::getAnyRef(uint32_t index) const {
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ class Table : public ShareableBase<Table> {
|
|||
TableAnyRefVector objects_; // or objects_, but not both
|
||||
const RefType elemType_;
|
||||
const bool isAsmJS_;
|
||||
const bool importedOrExported;
|
||||
uint32_t length_;
|
||||
const Maybe<uint32_t> maximum_;
|
||||
|
||||
|
|
@ -78,6 +79,12 @@ class Table : public ShareableBase<Table> {
|
|||
MOZ_ASSERT(elemType_.isFunc());
|
||||
return isAsmJS_;
|
||||
}
|
||||
|
||||
bool isImportedOrExported() const {
|
||||
MOZ_ASSERT(elemType_.isFunc());
|
||||
return importedOrExported;
|
||||
}
|
||||
|
||||
bool isFunction() const { return elemType().isFunc(); }
|
||||
uint32_t length() const { return length_; }
|
||||
Maybe<uint32_t> maximum() const { return maximum_; }
|
||||
|
|
@ -93,7 +100,7 @@ class Table : public ShareableBase<Table> {
|
|||
bool getFuncRef(JSContext* cx, uint32_t index,
|
||||
MutableHandleFunction fun) const;
|
||||
void setFuncRef(uint32_t index, void* code, const Instance* instance);
|
||||
void fillFuncRef(uint32_t index, uint32_t fillCount, FuncRef ref,
|
||||
bool fillFuncRef(uint32_t index, uint32_t fillCount, FuncRef ref,
|
||||
JSContext* cx);
|
||||
|
||||
AnyRef getAnyRef(uint32_t index) const;
|
||||
|
|
|
|||
|
|
@ -239,7 +239,7 @@ class FuncRef {
|
|||
|
||||
JSFunction* asJSFunction() { return value_; }
|
||||
|
||||
bool isNull() { return value_ == nullptr; }
|
||||
bool isNull() const { return value_ == nullptr; }
|
||||
|
||||
void trace(JSTracer* trc) const;
|
||||
};
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче