mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1692065 - wasm: Add tagging for JSString* in AnyRef. r=yury 

Before this commit we would need to box a JSString* in WasmValueBox
when we received one from JS and converted to an AnyRef.

This commit adds a tag 0x2 to AnyRef to distinguish JSString* from
JSObject*. We do not use tag 0x1 as that will be used for i31ref
values. This relies on alignment of all GC cell allocations.

This commit refactors the code generation routines around AnyRef
so that they all exist in MacroAssembler. This makes it possible
to update one place when adding support for a new tag.

Our existing tests about passing values to/from externref should
be sufficient to cover this new addition. There is no user visible
change to this.

Differential Revision: https://phabricator.services.mozilla.com/D177813
This commit is contained in:
Ryan Hunt 2023-08-04 14:06:50 +00:00
Родитель 1b7268cfe7
Коммит b0b3bb75a6
24 изменённых файлов: 509 добавлений и 293 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
js/src/gc/Marking-inl.h
Просмотреть файл

@ -71,6 +71,9 @@ struct TaggedPtr<wasm::AnyRef> {
static wasm::AnyRef wrap(JSObject* obj) {
return wasm::AnyRef::fromJSObjectOrNull(obj);
}
static wasm::AnyRef wrap(JSString* str) {
return wasm::AnyRef::fromJSString(str);
}
static wasm::AnyRef empty() { return wasm::AnyRef(); }
};

6
js/src/gc/Tenuring.cpp
Просмотреть файл

@ -183,6 +183,12 @@ void TenuringTracer::traverse(wasm::AnyRef* thingp) {
post = wasm::AnyRef::fromJSObject(*obj);
break;
}
case wasm::AnyRefKind::String: {
JSString* str = value.toJSString();
onStringEdge(&str, "string");
post = wasm::AnyRef::fromJSString(str);
break;
}
case wasm::AnyRefKind::Null: {
// This function must only be called for GC things.
MOZ_CRASH();

54
js/src/jit/CodeGenerator.cpp
Просмотреть файл

@ -9246,14 +9246,9 @@ void CodeGenerator::visitWasmPostWriteBarrier(LWasmPostWriteBarrier* lir) {
lir, valueBase, temp, lir->valueOffset());
addOutOfLineCode(ool, lir->mir());
// If the pointer being stored is null, no barrier.
masm.branchTestPtr(Assembler::Zero, value, value, ool->rejoin());
// If there is a containing object and it is in the nursery, no barrier.
masm.branchPtrInNurseryChunk(Assembler::Equal, object, temp, ool->rejoin());
// If the pointer being stored is to a tenured object, no barrier.
masm.branchPtrInNurseryChunk(Assembler::Equal, value, temp, ool->entry());
wasm::EmitWasmPostBarrierGuard(masm, mozilla::Some(object), temp, value,
ool->rejoin());
masm.jump(ool->entry());
masm.bind(ool->rejoin());
}
@ -17330,9 +17325,9 @@ void CodeGenerator::visitWasmTrapIfNull(LWasmTrapIfNull* lir) {
MOZ_ASSERT(gen->compilingWasm());
const MWasmTrapIfNull* mir = lir->mir();
Label nonNull;
Register input = ToRegister(lir->object());
Register ref = ToRegister(lir->ref());
masm.branchTestPtr(Assembler::NonZero, input, input, &nonNull);
masm.branchWasmAnyRefIsNull(false, ref, &nonNull);
masm.wasmTrap(mir->trap(), mir->bytecodeOffset());
masm.bind(&nonNull);
}
@ -18807,44 +18802,27 @@ void CodeGenerator::visitWasmFence(LWasmFence* lir) {
masm.memoryBarrier(MembarFull);
}
void CodeGenerator::visitWasmBoxValue(LWasmBoxValue* lir) {
ValueOperand input = ToValue(lir, LWasmBoxValue::InputIndex);
void CodeGenerator::visitWasmAnyRefFromJSValue(LWasmAnyRefFromJSValue* lir) {
ValueOperand input = ToValue(lir, LWasmAnyRefFromJSValue::InputIndex);
Register output = ToRegister(lir->output());
Label nullValue, objectValue, done;
{
ScratchTagScope tag(masm, input);
masm.splitTagForTest(input, tag);
masm.branchTestObject(Assembler::Equal, tag, &objectValue);
masm.branchTestNull(Assembler::Equal, tag, &nullValue);
}
using Fn = JSObject* (*)(JSContext*, HandleValue);
using Fn = JSObject* (*)(JSContext * cx, HandleValue value);
OutOfLineCode* oolBoxValue = oolCallVM<Fn, wasm::AnyRef::boxValue>(
lir, ArgList(input), StoreRegisterTo(output));
masm.jump(oolBoxValue->entry());
masm.bind(&nullValue);
// See the definition of AnyRef for a discussion of pointer representation.
masm.xorPtr(output, output);
masm.jump(&done);
masm.bind(&objectValue);
// See the definition of AnyRef for a discussion of pointer representation.
masm.unboxObject(input, output);
masm.bind(&done);
masm.convertValueToWasmAnyRef(input, output, oolBoxValue->entry());
masm.bind(oolBoxValue->rejoin());
}
void CodeGenerator::visitWasmAnyRefFromJSObject(LWasmAnyRefFromJSObject* lir) {
Register input = ToRegister(lir->input());
Register output = ToRegister(lir->output());
// See the definition of AnyRef for a discussion of pointer representation.
if (input != output) {
masm.movePtr(input, output);
}
masm.convertObjectToWasmAnyRef(input, output);
}
void CodeGenerator::visitWasmAnyRefFromJSString(LWasmAnyRefFromJSString* lir) {
Register input = ToRegister(lir->input());
Register output = ToRegister(lir->output());
masm.convertStringToWasmAnyRef(input, output);
}
#ifdef FUZZING_JS_FUZZILLI

9
js/src/jit/LIROps.yaml
Просмотреть файл

@ -299,7 +299,7 @@
- name: WasmTrapIfNull
operands:
object: WordSized
ref: WordSized
mir_op: true
- name: WasmRefIsSubtypeOfConcrete
@ -3575,7 +3575,7 @@
- name: IonToWasmCallI64
gen_boilerplate: false
- name: WasmBoxValue
- name: WasmAnyRefFromJSValue
result_type: WordSized
operands:
input: BoxedValue
@ -3585,6 +3585,11 @@
operands:
input: WordSized
- name: WasmAnyRefFromJSString
result_type: WordSized
operands:
input: WordSized
# Constant Simd128
- name: Simd128
result_type: WordSized

13
js/src/jit/Lowering.cpp
Просмотреть файл

@ -3069,8 +3069,9 @@ void LIRGenerator::visitWasmBuiltinTruncateToInt32(
lowerWasmBuiltinTruncateToInt32(truncate);
}
void LIRGenerator::visitWasmBoxValue(MWasmBoxValue* ins) {
LWasmBoxValue* lir = new (alloc()) LWasmBoxValue(useBox(ins->input()));
void LIRGenerator::visitWasmAnyRefFromJSValue(MWasmAnyRefFromJSValue* ins) {
LWasmAnyRefFromJSValue* lir =
new (alloc()) LWasmAnyRefFromJSValue(useBox(ins->input()));
define(lir, ins);
assignSafepoint(lir, ins);
}
@ -3081,6 +3082,12 @@ void LIRGenerator::visitWasmAnyRefFromJSObject(MWasmAnyRefFromJSObject* ins) {
define(lir, ins);
}
void LIRGenerator::visitWasmAnyRefFromJSString(MWasmAnyRefFromJSString* ins) {
LWasmAnyRefFromJSString* lir =
new (alloc()) LWasmAnyRefFromJSString(useRegisterAtStart(ins->input()));
define(lir, ins);
}
void LIRGenerator::visitWrapInt64ToInt32(MWrapInt64ToInt32* ins) {
define(new (alloc()) LWrapInt64ToInt32(useInt64AtStart(ins->input())), ins);
}
@ -3422,7 +3429,7 @@ void LIRGenerator::visitWasmTrap(MWasmTrap* ins) {
}
void LIRGenerator::visitWasmTrapIfNull(MWasmTrapIfNull* ins) {
auto* lir = new (alloc()) LWasmTrapIfNull(useRegister(ins->value()));
auto* lir = new (alloc()) LWasmTrapIfNull(useRegister(ins->ref()));
add(lir, ins);
}

16
js/src/jit/MIROps.yaml
Просмотреть файл

@ -570,9 +570,7 @@
- name: WasmTruncateToInt32
gen_boilerplate: false
# Store a JS Value that can't be represented as an AnyRef pointer into an
# object that holds the value (opaquely) as such a pointer.
- name: WasmBoxValue
- name: WasmAnyRefFromJSValue
operands:
def: Value
result_type: WasmAnyRef
@ -587,6 +585,14 @@
congruent_to: if_operands_equal
alias_set: none
- name: WasmAnyRefFromJSString
operands:
def: String
type_policy: none
result_type: WasmAnyRef
congruent_to: if_operands_equal
alias_set: none
- name: Int32ToIntPtr
gen_boilerplate: false
@ -1028,10 +1034,10 @@
- name: WasmTrap
gen_boilerplate: false
# Trap if the given value is null
# Trap if the given ref is null
- name: WasmTrapIfNull
operands:
value: RefOrNull
ref: WasmAnyRef
arguments:
trap: wasm::Trap
bytecodeOffset: wasm::BytecodeOffset

34
js/src/jit/MacroAssembler-inl.h
Просмотреть файл

@ -390,6 +390,24 @@ void MacroAssembler::moveValue(const ConstantOrRegister& src,
moveValue(src.reg(), dest);
}
// ===============================================================
// Copy instructions
void MacroAssembler::copy64(const Address& src, const Address& dest,
Register scratch) {
#if JS_BITS_PER_WORD == 32
MOZ_RELEASE_ASSERT(src.base != scratch && dest.base != scratch);
load32(LowWord(src), scratch);
store32(scratch, LowWord(dest));
load32(HighWord(src), scratch);
store32(scratch, HighWord(dest));
#else
Register64 scratch64(scratch);
load64(src, scratch64);
store64(scratch64, dest);
#endif
}
// ===============================================================
// Arithmetic functions
@ -739,22 +757,6 @@ void MacroAssembler::branchTestObjectIsProxy(bool proxy, Register object,
Imm32(ShiftedMask), label);
}
void MacroAssembler::branchTestObjectIsWasmGcObject(bool isGcObject,
Register object,
Register scratch,
Label* label) {
constexpr uint32_t ShiftedMask = (Shape::kindMask() << Shape::kindShift());
constexpr uint32_t ShiftedKind =
(uint32_t(Shape::Kind::WasmGC) << Shape::kindShift());
MOZ_ASSERT(object != scratch);
loadPtr(Address(object, JSObject::offsetOfShape()), scratch);
load32(Address(scratch, Shape::offsetOfImmutableFlags()), scratch);
and32(Imm32(ShiftedMask), scratch);
branch32(isGcObject ? Assembler::Equal : Assembler::NotEqual, scratch,
Imm32(ShiftedKind), label);
}
void MacroAssembler::branchTestProxyHandlerFamily(Condition cond,
Register proxy,
Register scratch,

198
js/src/jit/MacroAssembler.cpp
Просмотреть файл

@ -5520,7 +5520,7 @@ void MacroAssembler::branchWasmRefIsSubtypeAny(
// Check for null.
if (sourceType.isNullable()) {
branchTestPtr(Assembler::Zero, ref, ref, nullLabel);
branchWasmAnyRefIsNull(true, ref, nullLabel);
}
// The only value that can inhabit 'none' is null. So, early out if we got
@ -5542,7 +5542,8 @@ void MacroAssembler::branchWasmRefIsSubtypeAny(
// Test for non-gc objects.
MOZ_ASSERT(scratch1 != Register::Invalid());
if (!wasm::RefType::isSubTypeOf(sourceType, wasm::RefType::eq())) {
branchTestObjectIsWasmGcObject(false, ref, scratch1, failLabel);
branchWasmAnyRefIsObjectOrNull(false, ref, failLabel);
branchObjectIsWasmGcObject(false, ref, scratch1, failLabel);
}
if (destType.isEq()) {
@ -5746,6 +5747,195 @@ void MacroAssembler::branchWasmSuperTypeVectorIsSubtype(
// Fallthrough to the success case
}
void MacroAssembler::branchWasmAnyRefIsNull(bool isNull, Register src,
Label* label) {
branchTestPtr(isNull ? Assembler::Zero : Assembler::NonZero, src, src, label);
}
void MacroAssembler::branchWasmAnyRefIsObjectOrNull(bool isObject, Register src,
Label* label) {
branchTestPtr(isObject ? Assembler::Zero : Assembler::NonZero, src,
Imm32(int32_t(wasm::AnyRef::TagMask)), label);
}
void MacroAssembler::branchWasmAnyRefIsGCThing(bool isGCThing, Register src,
Label* label) {
// The only non-GC thing in anyref currently is 'null'.
branchWasmAnyRefIsNull(!isGCThing, src, label);
}
void MacroAssembler::branchWasmAnyRefIsNurseryCell(Condition cond, Register src,
Register temp,
Label* label) {
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
Label done;
branchWasmAnyRefIsGCThing(false, src,
cond == Assembler::Equal ? &done : label);
getWasmAnyRefGCThingChunk(src, temp);
branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
ImmWord(0), label);
bind(&done);
}
void MacroAssembler::branchValueConvertsToWasmAnyRefInline(
ValueOperand src, Label* label) {
ScratchTagScope tag(*this, src);
splitTagForTest(src, tag);
branchTestObject(Assembler::Equal, tag, label);
branchTestString(Assembler::Equal, tag, label);
branchTestNull(Assembler::Equal, tag, label);
}
void MacroAssembler::convertValueToWasmAnyRef(ValueOperand src, Register dest,
Label* oolConvert) {
Label nullValue, stringValue, objectValue, done;
{
ScratchTagScope tag(*this, src);
splitTagForTest(src, tag);
branchTestObject(Assembler::Equal, tag, &objectValue);
branchTestString(Assembler::Equal, tag, &stringValue);
branchTestNull(Assembler::Equal, tag, &nullValue);
jump(oolConvert);
}
bind(&nullValue);
static_assert(wasm::AnyRef::NullRefValue == 0);
xorPtr(dest, dest);
jump(&done);
bind(&stringValue);
unboxString(src, dest);
orPtr(Imm32((int32_t)wasm::AnyRefTag::String), dest);
jump(&done);
bind(&objectValue);
unboxObject(src, dest);
bind(&done);
}
void MacroAssembler::convertObjectToWasmAnyRef(Register src, Register dest) {
// JS objects are represented without any tagging.
movePtr(src, dest);
}
void MacroAssembler::convertStringToWasmAnyRef(Register src, Register dest) {
// JS strings require a tag.
movePtr(src, dest);
orPtr(Imm32(int32_t(wasm::AnyRefTag::String)), dest);
}
void MacroAssembler::branchObjectIsWasmGcObject(bool isGcObject, Register src,
Register scratch,
Label* label) {
constexpr uint32_t ShiftedMask = (Shape::kindMask() << Shape::kindShift());
constexpr uint32_t ShiftedKind =
(uint32_t(Shape::Kind::WasmGC) << Shape::kindShift());
MOZ_ASSERT(src != scratch);
loadPtr(Address(src, JSObject::offsetOfShape()), scratch);
load32(Address(scratch, Shape::offsetOfImmutableFlags()), scratch);
and32(Imm32(ShiftedMask), scratch);
branch32(isGcObject ? Assembler::Equal : Assembler::NotEqual, scratch,
Imm32(ShiftedKind), label);
}
// Unboxing is branchy and contorted because of Spectre mitigations - we don't
// have enough scratch registers. Were it not for the spectre mitigations in
// branchTestObjClass, the branch nest below would be restructured significantly
// by inverting branches and using fewer registers.
// Unbox an anyref in src (clobbering src in the process) and then re-box it as
// a Value in *dst. See the definition of AnyRef for a discussion of pointer
// representation.
void MacroAssembler::convertWasmAnyRefToValue(Register instance, Register src,
ValueOperand dst,
Register scratch) {
MOZ_ASSERT(src != scratch);
#if JS_BITS_PER_WORD == 32
MOZ_ASSERT(dst.typeReg() != scratch);
MOZ_ASSERT(dst.payloadReg() != scratch);
#else
MOZ_ASSERT(dst.valueReg() != scratch);
#endif
Label isObjectOrNull, isObject, isWasmValueBox, done;
// Check for the object or null tag first
branchTest32(Assembler::Zero, src, Imm32(wasm::AnyRef::TagMask), &isObjectOrNull);
// If we're not object or null, we must be a string
rshiftPtr(Imm32(wasm::AnyRef::TagShift), src);
lshiftPtr(Imm32(wasm::AnyRef::TagShift), src);
moveValue(TypedOrValueRegister(MIRType::String, AnyRegister(src)), dst);
jump(&done);
// Check for the null value
bind(&isObjectOrNull);
branchTestPtr(Assembler::NonZero, src, src, &isObject);
moveValue(NullValue(), dst);
jump(&done);
// Otherwise we must be a non-null object. We next to check if it's storing a
// boxed value
bind(&isObject);
// The type test will clear src if the test fails, so store early.
moveValue(TypedOrValueRegister(MIRType::Object, AnyRegister(src)), dst);
// Spectre mitigations: see comment above about efficiency.
branchTestObjClass(Assembler::Equal, src,
Address(instance, wasm::Instance::offsetOfValueBoxClass()),
scratch, src, &isWasmValueBox);
jump(&done);
// This is a boxed JS value, unbox it.
bind(&isWasmValueBox);
loadValue(Address(src, wasm::AnyRef::valueBoxOffsetOfValue()), dst);
bind(&done);
}
void MacroAssembler::convertWasmAnyRefToValue(Register instance, Register src,
const Address& dst,
Register scratch) {
MOZ_ASSERT(src != scratch);
Label isObjectOrNull, isObject, isWasmValueBox, done;
// Check for the object or null tag first
branchTest32(Assembler::Zero, src, Imm32(wasm::AnyRef::TagMask), &isObjectOrNull);
// If we're not object or null, we must be a string
rshiftPtr(Imm32(wasm::AnyRef::TagShift), src);
lshiftPtr(Imm32(wasm::AnyRef::TagShift), src);
storeValue(JSVAL_TYPE_STRING, src, dst);
jump(&done);
// Check for the null value
bind(&isObjectOrNull);
branchTestPtr(Assembler::NonZero, src, src, &isObject);
storeValue(NullValue(), dst);
jump(&done);
// Otherwise we must be a non-null object. We next to check if it's storing a
// boxed value
bind(&isObject);
// The type test will clear src if the test fails, so store early.
storeValue(JSVAL_TYPE_OBJECT, src, dst);
// Spectre mitigations: see comment above about efficiency.
branchTestObjClass(Assembler::Equal, src,
Address(instance, wasm::Instance::offsetOfValueBoxClass()),
scratch, src, &isWasmValueBox);
jump(&done);
// This is a boxed JS value, unbox it.
bind(&isWasmValueBox);
copy64(Address(src, wasm::AnyRef::valueBoxOffsetOfValue()), dst, scratch);
bind(&done);
}
void MacroAssembler::nopPatchableToCall(const wasm::CallSiteDesc& desc) {
CodeOffset offset = nopPatchableToCall();
append(desc, offset);
@ -5761,9 +5951,11 @@ void MacroAssembler::emitPreBarrierFastPath(JSRuntime* rt, MIRType type,
// Load the GC thing in temp1.
if (type == MIRType::Value) {
unboxGCThingForGCBarrier(Address(PreBarrierReg, 0), temp1);
} else if (type == MIRType::WasmAnyRef) {
unboxWasmAnyRefGCThingForGCBarrier(Address(PreBarrierReg, 0), temp1);
} else {
MOZ_ASSERT(type == MIRType::Object || type == MIRType::String ||
type == MIRType::Shape || type == MIRType::WasmAnyRef);
type == MIRType::Shape);
loadPtr(Address(PreBarrierReg, 0), temp1);
}

42
js/src/jit/MacroAssembler.h
Просмотреть файл

@ -1008,6 +1008,11 @@ class MacroAssembler : public MacroAssemblerSpecific {
inline void loadAbiReturnAddress(Register dest) PER_SHARED_ARCH;
// ===============================================================
// Copy instructions
inline void copy64(const Address& src, const Address& dest, Register scratch);
public:
// ===============================================================
// Logical instructions
@ -1816,9 +1821,6 @@ class MacroAssembler : public MacroAssemblerSpecific {
Register scratch,
const void* handlerp, Label* label);
inline void branchTestObjectIsWasmGcObject(bool isGcObject, Register obj,
Register scratch, Label* label);
inline void branchTestNeedsIncrementalBarrier(Condition cond, Label* label);
inline void branchTestNeedsIncrementalBarrierAnyZone(Condition cond,
Label* label,
@ -3943,6 +3945,40 @@ class MacroAssembler : public MacroAssemblerSpecific {
uint32_t superTypeDepth, Label* label,
bool onSuccess);
// Branch if the wasm anyref `src` is or is not the null value.
void branchWasmAnyRefIsNull(bool isNull, Register src, Label* label);
// Branch if the wasm anyref `src` is or is not a JSObject*.
void branchWasmAnyRefIsObjectOrNull(bool isObject, Register src, Label* label);
// Branch if the wasm anyref `src` is or is not a GC thing.
void branchWasmAnyRefIsGCThing(bool isGCThing, Register src, Label* label);
// Branch if the wasm anyref `src` is or is not pointing to a nursery cell.
void branchWasmAnyRefIsNurseryCell(Condition cond, Register src,
Register scratch, Label* label);
// Branch if the JS value `src` would need to be boxed out of line to be
// converted to a wasm anyref.
void branchValueConvertsToWasmAnyRefInline(ValueOperand src, Label* label);
// Convert a JS value to a wasm anyref. If the value requires boxing, this
// will branch to `oolConvert`.
void convertValueToWasmAnyRef(ValueOperand src, Register dest,
Label* oolConvert);
// Convert a JS object to a wasm anyref. This cannot fail.
void convertObjectToWasmAnyRef(Register src, Register dest);
// Convert a JS string to a wasm anyref. This cannot fail.
void convertStringToWasmAnyRef(Register src, Register dest);
// Convert a wasm anyref to a JS value. This cannot fail.
//
// Due to spectre mitigations, these methods may clobber src.
void convertWasmAnyRefToValue(Register instance, Register src,
ValueOperand dst, Register scratch);
void convertWasmAnyRefToValue(Register instance, Register src,
const Address& dst, Register scratch);
// Branch if the object `src` is or is not a WasmGcObject.
void branchObjectIsWasmGcObject(bool isGcObject, Register src,
Register scratch, Label* label);
// Compute ptr += (indexTemp32 << shift) where shift can be any value < 32.
// May destroy indexTemp32. The value of indexTemp32 must be positive, and it
// is implementation-defined what happens if bits are lost or the value

5
js/src/jit/WarpCacheIRTranspiler.cpp
Просмотреть файл

@ -5396,12 +5396,15 @@ MDefinition* WarpCacheIRTranspiler::convertWasmArg(MDefinition* arg,
case MIRType::Object:
conversion = MWasmAnyRefFromJSObject::New(alloc(), arg);
break;
case MIRType::String:
conversion = MWasmAnyRefFromJSString::New(alloc(), arg);
break;
case MIRType::Null:
arg->setImplicitlyUsedUnchecked();
conversion = MWasmNullConstant::New(alloc());
break;
default:
conversion = MWasmBoxValue::New(alloc(), arg);
conversion = MWasmAnyRefFromJSValue::New(alloc(), arg);
break;
}
break;

13
js/src/jit/arm/MacroAssembler-arm.h
Просмотреть файл

@ -865,6 +865,19 @@ class MacroAssemblerARMCompat : public MacroAssemblerARM {
load32(ToPayload(src), dest);
}
void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
load32(ToPayload(src), dest);
{
ScratchRegisterScope scratch(asMasm());
ma_and(Imm32(wasm::AnyRef::GCThingMask), dest, scratch);
}
}
void getWasmAnyRefGCThingChunk(Register src, Register dest) {
ScratchRegisterScope scratch(asMasm());
ma_and(Imm32(wasm::AnyRef::GCThingChunkMask), src, dest, scratch);
}
void notBoolean(const ValueOperand& val) {
as_eor(val.payloadReg(), val.payloadReg(), Imm8(1));
}

1
js/src/jit/arm64/MacroAssembler-arm64.cpp
Просмотреть файл

@ -1709,7 +1709,6 @@ void MacroAssembler::branchValueIsNurseryCell(Condition cond,
Label* label) {
branchValueIsNurseryCellImpl(cond, value, temp, label);
}
template <typename T>
void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
const T& value, Register temp,

15
js/src/jit/arm64/MacroAssembler-arm64.h
Просмотреть файл

@ -1449,7 +1449,17 @@ class MacroAssemblerCompat : public vixl::MacroAssembler {
Operand(JS::detail::ValueGCThingPayloadMask));
}
void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
loadPtr(src, dest);
And(ARMRegister(dest, 64), ARMRegister(dest, 64),
Operand(wasm::AnyRef::GCThingMask));
}
// Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
void getGCThingValueChunk(Register src, Register dest) {
And(ARMRegister(src, 64), ARMRegister(dest, 64),
Operand(JS::detail::ValueGCThingPayloadChunkMask));
}
void getGCThingValueChunk(const Address& src, Register dest) {
loadPtr(src, dest);
And(ARMRegister(dest, 64), ARMRegister(dest, 64),
@ -1460,6 +1470,11 @@ class MacroAssemblerCompat : public vixl::MacroAssembler {
Operand(JS::detail::ValueGCThingPayloadChunkMask));
}
void getWasmAnyRefGCThingChunk(Register src, Register dest) {
And(ARMRegister(dest, 64), ARMRegister(src, 64),
Operand(wasm::AnyRef::GCThingChunkMask));
}
inline void unboxValue(const ValueOperand& src, AnyRegister dest,
JSValueType type);

10
js/src/jit/none/MacroAssembler-none.h
Просмотреть файл

@ -359,6 +359,16 @@ class MacroAssemblerNone : public Assembler {
void unboxGCThingForGCBarrier(const T&, Register) {
MOZ_CRASH();
}
template <typename T>
void unboxWasmAnyRefGCThingForGCBarrier(const T&, Register) {
MOZ_CRASH();
}
void getWasmAnyRefGCThingChunk(Register, Register) {
MOZ_CRASH();
}
template <typename T>
void unboxObjectOrNull(const T& src, Register dest) {
MOZ_CRASH();

9
js/src/jit/wasm32/MacroAssembler-wasm32.h
Просмотреть файл

@ -435,6 +435,15 @@ class MacroAssemblerWasm32 : public Assembler {
MOZ_CRASH();
}
template <typename T>
void unboxWasmAnyRefGCThingForGCBarrier(const T&, Register) {
MOZ_CRASH();
}
void getWasmAnyRefGCThingChunk(Register, Register) {
MOZ_CRASH();
}
template <typename T>
void unboxObjectOrNull(const T& src, Register dest) {
MOZ_CRASH();

11
js/src/jit/x64/MacroAssembler-x64.h
Просмотреть файл

@ -907,6 +907,11 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared {
andq(src.valueReg(), dest);
}
void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
movq(ImmWord(wasm::AnyRef::GCThingMask), dest);
andq(Operand(src), dest);
}
// Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
void getGCThingValueChunk(const Address& src, Register dest) {
movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
@ -918,6 +923,12 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared {
andq(src.valueReg(), dest);
}
void getWasmAnyRefGCThingChunk(Register src, Register dest) {
MOZ_ASSERT(src != dest);
movq(ImmWord(wasm::AnyRef::GCThingChunkMask), dest);
andq(src, dest);
}
inline void fallibleUnboxPtrImpl(const Operand& src, Register dest,
JSValueType type, Label* fail);

11
js/src/jit/x86/MacroAssembler-x86.h
Просмотреть файл

@ -889,6 +889,17 @@ class MacroAssemblerX86 : public MacroAssemblerX86Shared {
movl(payloadOf(src), dest);
}
void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest) {
movl(ImmWord(wasm::AnyRef::GCThingMask), dest);
andl(Operand(src), dest);
}
void getWasmAnyRefGCThingChunk(Register src, Register dest) {
MOZ_ASSERT(src != dest);
movl(ImmWord(wasm::AnyRef::GCThingChunkMask), dest);
andl(src, dest);
}
void notBoolean(const ValueOperand& val) { xorl(Imm32(1), val.payloadReg()); }
template <typename T>

8
js/src/wasm/WasmAnyRef.cpp
Просмотреть файл

@ -62,6 +62,12 @@ bool AnyRef::fromJSValue(JSContext* cx, HandleValue value,
return true;
}
if (value.isString()) {
JSString* string = value.toString();
result.set(AnyRef::fromJSString(string));
return true;
}
if (value.isObject()) {
JSObject& obj = value.toObject();
MOZ_ASSERT(!obj.is<WasmValueBox>());
@ -89,6 +95,8 @@ Value wasm::AnyRef::toJSValue() const {
Value value;
if (isNull()) {
value.setNull();
} else if (isJSString()) {
value.setString(toJSString());
} else {
JSObject& obj = toJSObject();
if (obj.is<WasmValueBox>()) {

104
js/src/wasm/WasmAnyRef.h
Просмотреть файл

@ -44,9 +44,7 @@ namespace wasm {
// host type that the host system allows to flow into and out of wasm
// transparently. It is a pointer-sized datum that has the same representation
// as all its subtypes (funcref, externref, eqref, (ref T), et al) due to the
// non-coercive subtyping of the wasm type system. Its current representation
// is a plain JSObject*, and the private JSObject subtype WasmValueBox is used
// to box non-object non-null JS values.
// non-coercive subtyping of the wasm type system.
//
// The C++/wasm boundary always uses a 'void*' type to express AnyRef values, to
// emphasize the pointer-ness of the value. The C++ code must transform the
@ -62,13 +60,6 @@ namespace wasm {
//
// The lowest bits of the pointer value are used for tagging, to allow for some
// representation optimizations and to distinguish various types.
//
// For version 0, we simply equate AnyRef and JSObject* (this means that there
// are technically no tags at all yet). We use a simple boxing scheme that
// wraps a JS value that is not already JSObject in a distinguishable JSObject
// that holds the value, see WasmTypes.cpp for details. Knowledge of this
// mapping is embedded in CodeGenerator.cpp (in WasmBoxValue and
// WasmAnyRefFromJSObject) and in WasmStubs.cpp (in functions Box* and Unbox*).
// The kind of value stored in an AnyRef. This is not 1:1 with the pointer tag
// of AnyRef as this separates the 'Null' and 'Object' cases which are
@ -76,36 +67,16 @@ namespace wasm {
enum class AnyRefKind : uint8_t {
Null,
Object,
String,
};
// The pointer tag of an AnyRef.
enum class AnyRefTag : uint8_t {
// This value is either a JSObject& or a null pointer.
ObjectOrNull = 0x0,
// This value is a JSString*.
String = 0x2,
};
static constexpr uintptr_t TAG_MASK = 0x3;
static constexpr uintptr_t TAG_SHIFT = 2;
static_assert(TAG_SHIFT <= gc::CellAlignShift, "not enough free bits");
static constexpr uintptr_t AddAnyRefTag(uintptr_t value, AnyRefTag tag) {
MOZ_ASSERT(!(value & TAG_MASK));
return value | uintptr_t(tag);
}
static constexpr AnyRefTag GetAnyRefTag(uintptr_t value) {
return (AnyRefTag)(value & TAG_MASK);
}
static constexpr uintptr_t RemoveAnyRefTag(uintptr_t value) {
return value & ~TAG_MASK;
}
// The representation of a null reference value throughout the compiler for
// when we need an integer constant. This is asserted to be equivalent to
// nullptr in wasm::Init.
static constexpr uintptr_t NULLREF_VALUE = 0;
static_assert(GetAnyRefTag(NULLREF_VALUE) == AnyRefTag::ObjectOrNull);
static constexpr uintptr_t INVALIDREF_VALUE = UINTPTR_MAX & ~TAG_MASK;
static_assert(GetAnyRefTag(INVALIDREF_VALUE) == AnyRefTag::ObjectOrNull);
// A reference to any wasm reference type or host (JS) value. AnyRef is
// optimized for efficient access to wasm GC objects and possibly a tagged
@ -116,34 +87,66 @@ class AnyRef {
uintptr_t value_;
// Get the pointer tag stored in value_.
AnyRefTag pointerTag() const { return GetAnyRefTag(value_); }
AnyRefTag pointerTag() const { return GetUintptrTag(value_); }
explicit AnyRef(uintptr_t value) : value_(value) {}
static constexpr uintptr_t TagUintptr(uintptr_t value, AnyRefTag tag) {
MOZ_ASSERT(!(value & TagMask));
return value | uintptr_t(tag);
}
static constexpr uintptr_t UntagUintptr(uintptr_t value) {
return value & ~TagMask;
}
static constexpr AnyRefTag GetUintptrTag(uintptr_t value) {
return (AnyRefTag)(value & TagMask);
}
public:
explicit AnyRef() : value_(NULLREF_VALUE) {}
MOZ_IMPLICIT AnyRef(std::nullptr_t) : value_(NULLREF_VALUE) {}
static constexpr uintptr_t TagMask = 0x3;
static constexpr uintptr_t TagShift = 2;
static_assert(TagShift <= gc::CellAlignShift, "not enough free bits");
// A mask for getting the GC thing an AnyRef represents.
static constexpr uintptr_t GCThingMask = ~TagMask;
// A combined mask for getting the gc::Chunk for an AnyRef that is a GC
// thing.
static constexpr uintptr_t GCThingChunkMask =
GCThingMask & ~js::gc::ChunkMask;
// The representation of a null reference value throughout the compiler for
// when we need an integer constant. This is asserted to be equivalent to
// nullptr in wasm::Init.
static constexpr uintptr_t NullRefValue = 0;
static constexpr uintptr_t InvalidRefValue = UINTPTR_MAX << TagShift;
explicit AnyRef() : value_(NullRefValue) {}
MOZ_IMPLICIT AnyRef(std::nullptr_t) : value_(NullRefValue) {}
// The null AnyRef value.
static AnyRef null() { return AnyRef(NULLREF_VALUE); }
static AnyRef null() { return AnyRef(NullRefValue); }
// An invalid AnyRef cannot arise naturally from wasm and so can be used as
// a sentinel value to indicate failure from an AnyRef-returning function.
static AnyRef invalid() { return AnyRef(INVALIDREF_VALUE); }
static AnyRef invalid() { return AnyRef(InvalidRefValue); }
// Given a JSObject* that comes from JS, turn it into AnyRef.
static AnyRef fromJSObjectOrNull(JSObject* objectOrNull) {
MOZ_ASSERT(GetAnyRefTag((uintptr_t)objectOrNull) ==
MOZ_ASSERT(GetUintptrTag((uintptr_t)objectOrNull) ==
AnyRefTag::ObjectOrNull);
return AnyRef((uintptr_t)objectOrNull);
}
// Given a JSObject& that comes from JS, turn it into AnyRef.
static AnyRef fromJSObject(JSObject& object) {
MOZ_ASSERT(GetAnyRefTag((uintptr_t)&object) == AnyRefTag::ObjectOrNull);
MOZ_ASSERT(GetUintptrTag((uintptr_t)&object) == AnyRefTag::ObjectOrNull);
return AnyRef((uintptr_t)&object);
}
// Given a JSString* that comes from JS, turn it into AnyRef.
static AnyRef fromJSString(JSString* string) {
return AnyRef(TagUintptr((uintptr_t)string, AnyRefTag::String));
}
// Given a void* that comes from compiled wasm code, turn it into AnyRef.
static AnyRef fromCompiledCode(void* pointer) {
return AnyRef((uintptr_t)pointer);
@ -156,7 +159,7 @@ class AnyRef {
// Returns whether a JS value will need to be boxed.
static bool valueNeedsBoxing(JS::HandleValue value) {
return !value.isObjectOrNull();
return !(value.isObjectOrNull() || value.isString());
}
// Box a JS Value that needs boxing.
@ -173,7 +176,7 @@ class AnyRef {
}
AnyRefKind kind() const {
if (value_ == NULLREF_VALUE) {
if (value_ == NullRefValue) {
return AnyRefKind::Null;
}
switch (pointerTag()) {
@ -183,19 +186,23 @@ class AnyRef {
// We ruled out the null case above
return AnyRefKind::Object;
}
case AnyRefTag::String: {
return AnyRefKind::String;
}
default: {
MOZ_CRASH("unknown AnyRef tag");
}
}
}
bool isNull() const { return value_ == NULLREF_VALUE; }
bool isGCThing() const { return kind() == AnyRefKind::Object; }
bool isNull() const { return value_ == NullRefValue; }
bool isGCThing() const { return !isNull(); }
bool isJSObject() const { return kind() == AnyRefKind::Object; }
bool isJSString() const { return kind() == AnyRefKind::String; }
gc::Cell* toGCThing() const {
MOZ_ASSERT(isGCThing());
return (gc::Cell*)RemoveAnyRefTag(value_);
return (gc::Cell*)UntagUintptr(value_);
}
JSObject& toJSObject() const {
MOZ_ASSERT(isJSObject());
@ -203,8 +210,13 @@ class AnyRef {
}
JSObject* toJSObjectOrNull() const {
MOZ_ASSERT(!isInvalid());
MOZ_ASSERT(pointerTag() == AnyRefTag::ObjectOrNull);
return (JSObject*)value_;
}
JSString* toJSString() const {
MOZ_ASSERT(isJSString());
return (JSString*)UntagUintptr(value_);
}
// Convert from AnyRef to a JS Value. This currently does not require any
// allocation. If this changes in the future, this function will become
@ -252,6 +264,8 @@ auto MapGCThingTyped(const wasm::AnyRef& val, F&& f) {
switch (val.kind()) {
case wasm::AnyRefKind::Object:
return mozilla::Some(f(&val.toJSObject()));
case wasm::AnyRefKind::String:
return mozilla::Some(f(val.toJSString()));
case wasm::AnyRefKind::Null: {
using ReturnType = decltype(f(static_cast<JSObject*>(nullptr)));
return mozilla::Maybe<ReturnType>();

52
js/src/wasm/WasmBaselineCompile.cpp
Просмотреть файл

@ -3841,15 +3841,15 @@ bool BaseCompiler::emitBrOnNull() {
if (b.hasBlockResults()) {
needResultRegisters(b.resultType);
}
RegRef rp = popRef();
RegRef ref = popRef();
if (b.hasBlockResults()) {
freeResultRegisters(b.resultType);
}
if (!jumpConditionalWithResults(&b, Assembler::Equal, rp,
ImmWord(NULLREF_VALUE))) {
if (!jumpConditionalWithResults(&b, Assembler::Equal, ref,
ImmWord(AnyRef::NullRefValue))) {
return false;
}
pushRef(rp);
pushRef(ref);
return true;
}
@ -3880,22 +3880,22 @@ bool BaseCompiler::emitBrOnNonNull() {
needIntegerResultRegisters(b.resultType);
// Get the ref from the top of the stack
RegRef condition = popRef();
RegRef refCondition = popRef();
// Create a copy of the ref for passing to the on_non_null label,
// the original ref is used in the condition.
RegRef rp = needRef();
moveRef(condition, rp);
pushRef(rp);
RegRef ref = needRef();
moveRef(refCondition, ref);
pushRef(ref);
freeIntegerResultRegisters(b.resultType);
if (!jumpConditionalWithResults(&b, Assembler::NotEqual, condition,
ImmWord(NULLREF_VALUE))) {
if (!jumpConditionalWithResults(&b, Assembler::NotEqual, refCondition,
ImmWord(AnyRef::NullRefValue))) {
return false;
}
freeRef(condition);
freeRef(refCondition);
// Dropping null reference.
dropValue();
@ -5895,7 +5895,7 @@ bool BaseCompiler::emitRefNull() {
return true;
}
pushRef(NULLREF_VALUE);
pushRef(AnyRef::NullRefValue);
return true;
}
@ -5912,7 +5912,7 @@ bool BaseCompiler::emitRefIsNull() {
RegRef r = popRef();
RegI32 rd = narrowRef(r);
masm.cmpPtrSet(Assembler::Equal, r, ImmWord(NULLREF_VALUE), rd);
masm.cmpPtrSet(Assembler::Equal, r, ImmWord(AnyRef::NullRefValue), rd);
pushI32(rd);
return true;
}
@ -6559,7 +6559,7 @@ void BaseCompiler::emitBarrieredClear(RegPtr valueAddr) {
emitPreBarrier(valueAddr);
// Store null
masm.storePtr(ImmWord(NULLREF_VALUE), Address(valueAddr, 0));
masm.storePtr(ImmWord(AnyRef::NullRefValue), Address(valueAddr, 0));
// No post-barrier is needed, as null does not require a store buffer entry
}
@ -7474,10 +7474,10 @@ bool BaseCompiler::emitArrayCopy() {
void BaseCompiler::emitRefTestCommon(RefType sourceType, RefType destType) {
Label success;
Label join;
RegRef object = popRef();
RegRef ref = popRef();
RegI32 result = needI32();
branchIfRefSubtype(object, sourceType, destType, &success,
branchIfRefSubtype(ref, sourceType, destType, &success,
/*onSuccess=*/true);
masm.xor32(result, result);
masm.jump(&join);
@ -7486,7 +7486,7 @@ void BaseCompiler::emitRefTestCommon(RefType sourceType, RefType destType) {
masm.bind(&join);
pushI32(result);
freeRef(object);
freeRef(ref);
}
void BaseCompiler::emitRefCastCommon(RefType sourceType, RefType destType) {
@ -7651,22 +7651,24 @@ bool BaseCompiler::emitBrOnCastCommon(bool onSuccess,
needIntegerResultRegisters(b.resultType);
}
// Create a copy of the ref for passing to the br_on_cast label,
// the original ref is used for casting in the condition.
RegRef object = popRef();
RegRef objectCondition = needRef();
moveRef(object, objectCondition);
pushRef(object);
// Get the ref from the top of the stack
RegRef refCondition = popRef();
// Create a copy of the ref for passing to the on_cast label,
// the original ref is used in the condition.
RegRef ref = needRef();
moveRef(refCondition, ref);
pushRef(ref);
if (b.hasBlockResults()) {
freeIntegerResultRegisters(b.resultType);
}
if (!jumpConditionalWithResults(&b, objectCondition, sourceType, destType,
if (!jumpConditionalWithResults(&b, refCondition, sourceType, destType,
onSuccess)) {
return false;
}
freeRef(objectCondition);
freeRef(refCondition);
return true;
}

11
js/src/wasm/WasmGC.cpp
Просмотреть файл

@ -219,9 +219,9 @@ void wasm::EmitWasmPreBarrierGuard(MacroAssembler& masm, Register instance,
wasm::TrapSite(masm.currentOffset(), *trapOffset));
}
// If the previous value is null, we don't need the barrier.
// If the previous value is not a GC thing, we don't need the barrier.
masm.loadPtr(Address(valueAddr, valueOffset), scratch);
masm.branchTestPtr(Assembler::Zero, scratch, scratch, skipBarrier);
masm.branchWasmAnyRefIsGCThing(false, scratch, skipBarrier);
}
void wasm::EmitWasmPreBarrierCall(MacroAssembler& masm, Register instance,
@ -258,9 +258,6 @@ void wasm::EmitWasmPostBarrierGuard(MacroAssembler& masm,
const Maybe<Register>& object,
Register otherScratch, Register setValue,
Label* skipBarrier) {
// If the pointer being stored is null, no barrier.
masm.branchTestPtr(Assembler::Zero, setValue, setValue, skipBarrier);
// If there is a containing object and it is in the nursery, no barrier.
if (object) {
masm.branchPtrInNurseryChunk(Assembler::Equal, *object, otherScratch,
@ -268,8 +265,8 @@ void wasm::EmitWasmPostBarrierGuard(MacroAssembler& masm,
}
// If the pointer being stored is to a tenured object, no barrier.
masm.branchPtrInNurseryChunk(Assembler::NotEqual, setValue, otherScratch,
skipBarrier);
masm.branchWasmAnyRefIsNurseryCell(Assembler::NotEqual, setValue,
otherScratch, skipBarrier);
}
#ifdef DEBUG

14
js/src/wasm/WasmIonCompile.cpp
Просмотреть файл

@ -839,21 +839,21 @@ class FunctionCompiler {
curBlock_->setSlot(info().localSlot(slot), def);
}
MDefinition* compareIsNull(MDefinition* value, JSOp compareOp) {
MDefinition* compareIsNull(MDefinition* ref, JSOp compareOp) {
MDefinition* nullVal = constantNullRef();
if (!nullVal) {
return nullptr;
}
return compare(value, nullVal, compareOp, MCompare::Compare_WasmAnyRef);
return compare(ref, nullVal, compareOp, MCompare::Compare_WasmAnyRef);
}
[[nodiscard]] bool refAsNonNull(MDefinition* value) {
[[nodiscard]] bool refAsNonNull(MDefinition* ref) {
if (inDeadCode()) {
return true;
}
auto* ins = MWasmTrapIfNull::New(
alloc(), value, wasm::Trap::NullPointerDereference, bytecodeOffset());
alloc(), ref, wasm::Trap::NullPointerDereference, bytecodeOffset());
curBlock_->add(ins);
return true;
@ -6800,12 +6800,12 @@ static bool EmitStoreLaneSimd128(FunctionCompiler& f, uint32_t laneSize) {
#ifdef ENABLE_WASM_FUNCTION_REFERENCES
static bool EmitRefAsNonNull(FunctionCompiler& f) {
MDefinition* value;
if (!f.iter().readRefAsNonNull(&value)) {
MDefinition* ref;
if (!f.iter().readRefAsNonNull(&ref)) {
return false;
}
return f.refAsNonNull(value);
return f.refAsNonNull(ref);
}
static bool EmitBrOnNull(FunctionCompiler& f) {

2
js/src/wasm/WasmProcess.cpp
Просмотреть файл

@ -404,7 +404,7 @@ bool wasm::Init() {
// at compile time.
uintptr_t pageSize = gc::SystemPageSize();
MOZ_RELEASE_ASSERT(wasm::NullPtrGuardSize <= pageSize);
MOZ_RELEASE_ASSERT(intptr_t(nullptr) == wasm::NULLREF_VALUE);
MOZ_RELEASE_ASSERT(intptr_t(nullptr) == AnyRef::NullRefValue);
ConfigureHugeMemory();

161
js/src/wasm/WasmStubs.cpp
Просмотреть файл

@ -348,21 +348,6 @@ static unsigned StackArgBytesForWasmABI(const FuncType& funcType) {
return StackArgBytesForWasmABI(args);
}
static void Move64(MacroAssembler& masm, const Address& src,
const Address& dest, Register scratch) {
#if JS_BITS_PER_WORD == 32
MOZ_RELEASE_ASSERT(src.base != scratch && dest.base != scratch);
masm.load32(LowWord(src), scratch);
masm.store32(scratch, LowWord(dest));
masm.load32(HighWord(src), scratch);
masm.store32(scratch, HighWord(dest));
#else
Register64 scratch64(scratch);
masm.load64(src, scratch64);
masm.store64(scratch64, dest);
#endif
}
static void SetupABIArguments(MacroAssembler& masm, const FuncExport& fe,
const FuncType& funcType, Register argv,
Register scratch) {
@ -435,7 +420,7 @@ static void SetupABIArguments(MacroAssembler& masm, const FuncExport& fe,
break;
case MIRType::Int64: {
RegisterOrSP sp = masm.getStackPointer();
Move64(masm, src, Address(sp, iter->offsetFromArgBase()), scratch);
masm.copy64(src, Address(sp, iter->offsetFromArgBase()), scratch);
break;
}
case MIRType::WasmAnyRef:
@ -611,107 +596,6 @@ static void CallFuncExport(MacroAssembler& masm, const FuncExport& fe,
}
}
STATIC_ASSERT_ANYREF_IS_JSOBJECT; // Strings are currently boxed
// Unboxing is branchy and contorted because of Spectre mitigations - we don't
// have enough scratch registers. Were it not for the spectre mitigations in
// branchTestObjClass, the branch nest below would be restructured significantly
// by inverting branches and using fewer registers.
// Unbox an anyref in src (clobbering src in the process) and then re-box it as
// a Value in *dst. See the definition of AnyRef for a discussion of pointer
// representation.
static void UnboxAnyrefIntoValue(MacroAssembler& masm, Register instance,
Register src, const Address& dst,
Register scratch) {
MOZ_ASSERT(src != scratch);
// Not actually the value we're passing, but we've no way of
// decoding anything better.
GenPrintPtr(DebugChannel::Import, masm, src);
Label notNull, mustUnbox, done;
masm.branchTestPtr(Assembler::NonZero, src, src, &notNull);
masm.storeValue(NullValue(), dst);
masm.jump(&done);
masm.bind(&notNull);
// The type test will clear src if the test fails, so store early.
masm.storeValue(JSVAL_TYPE_OBJECT, src, dst);
// Spectre mitigations: see comment above about efficiency.
masm.branchTestObjClass(Assembler::Equal, src,
Address(instance, Instance::offsetOfValueBoxClass()),
scratch, src, &mustUnbox);
masm.jump(&done);
masm.bind(&mustUnbox);
Move64(masm, Address(src, AnyRef::valueBoxOffsetOfValue()), dst, scratch);
masm.bind(&done);
}
// Unbox an anyref in src and then re-box it as a Value in dst.
// See the definition of AnyRef for a discussion of pointer representation.
static void UnboxAnyrefIntoValueReg(MacroAssembler& masm, Register instance,
Register src, ValueOperand dst,
Register scratch) {
MOZ_ASSERT(src != scratch);
#if JS_BITS_PER_WORD == 32
MOZ_ASSERT(dst.typeReg() != scratch);
MOZ_ASSERT(dst.payloadReg() != scratch);
#else
MOZ_ASSERT(dst.valueReg() != scratch);
#endif
// Not actually the value we're passing, but we've no way of
// decoding anything better.
GenPrintPtr(DebugChannel::Import, masm, src);
Label notNull, mustUnbox, done;
masm.branchTestPtr(Assembler::NonZero, src, src, &notNull);
masm.moveValue(NullValue(), dst);
masm.jump(&done);
masm.bind(&notNull);
// The type test will clear src if the test fails, so store early.
masm.moveValue(TypedOrValueRegister(MIRType::Object, AnyRegister(src)), dst);
// Spectre mitigations: see comment above about efficiency.
masm.branchTestObjClass(Assembler::Equal, src,
Address(instance, Instance::offsetOfValueBoxClass()),
scratch, src, &mustUnbox);
masm.jump(&done);
masm.bind(&mustUnbox);
masm.loadValue(Address(src, AnyRef::valueBoxOffsetOfValue()), dst);
masm.bind(&done);
}
// Box the Value in src as an anyref in dest. src and dest must not overlap.
// See the definition of AnyRef for a discussion of pointer representation.
static void BoxValueIntoAnyref(MacroAssembler& masm, ValueOperand src,
Register dest, Label* oolConvert) {
Label nullValue, objectValue, done;
{
ScratchTagScope tag(masm, src);
masm.splitTagForTest(src, tag);
masm.branchTestObject(Assembler::Equal, tag, &objectValue);
masm.branchTestNull(Assembler::Equal, tag, &nullValue);
masm.jump(oolConvert);
}
masm.bind(&nullValue);
// See the definition of AnyRef for a discussion of pointer representation.
static_assert(wasm::NULLREF_VALUE == 0);
masm.xorPtr(dest, dest);
masm.jump(&done);
masm.bind(&objectValue);
masm.unboxObject(src, dest);
masm.bind(&done);
}
// Generate a stub that enters wasm from a C++ caller via the native ABI. The
// signature of the entry point is Module::ExportFuncPtr. The exported wasm
// function has an ABI derived from its specific signature, so this function
@ -1191,14 +1075,7 @@ static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex,
case ValType::Ref: {
// Guarded against by temporarilyUnsupportedReftypeForEntry()
MOZ_RELEASE_ASSERT(funcType.args()[i].refType().isExtern());
ScratchTagScope tag(masm, scratchV);
masm.splitTagForTest(scratchV, tag);
// For object inputs, we handle object and null inline, everything
// else requires an actual box and we go out of line to allocate
// that.
masm.branchTestObject(Assembler::Equal, tag, &next);
masm.branchTestNull(Assembler::Equal, tag, &next);
masm.branchValueConvertsToWasmAnyRefInline(scratchV, &next);
masm.jump(&oolCall);
break;
}
@ -1276,8 +1153,18 @@ static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex,
break;
}
case MIRType::WasmAnyRef: {
ValueOperand src = ScratchValIonEntry;
Register target = isStackArg ? ScratchIonEntry : iter->gpr();
masm.unboxObjectOrNull(argv, target);
masm.loadValue(argv, src);
// The loop before should ensure that all values that require boxing
// have been taken care of.
Label join;
Label fail;
masm.convertValueToWasmAnyRef(src, target, &fail);
masm.jump(&join);
masm.bind(&fail);
masm.breakpoint();
masm.bind(&join);
GenPrintPtr(DebugChannel::Function, masm, target);
if (isStackArg) {
masm.storePtr(target, Address(sp, iter->offsetFromArgBase()));
@ -1364,12 +1251,12 @@ static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex,
MOZ_CRASH("unexpected return type when calling from ion to wasm");
}
case ValType::Ref: {
STATIC_ASSERT_ANYREF_IS_JSOBJECT;
// Per comment above, the call may have clobbered the instance
// register, so reload since unboxing will need it.
GenerateJitEntryLoadInstance(masm);
UnboxAnyrefIntoValueReg(masm, InstanceReg, ReturnReg, JSReturnOperand,
WasmJitEntryReturnScratch);
GenPrintPtr(DebugChannel::Import, masm, ReturnReg);
masm.convertWasmAnyRefToValue(InstanceReg, ReturnReg, JSReturnOperand,
WasmJitEntryReturnScratch);
break;
}
}
@ -1647,10 +1534,11 @@ void wasm::GenerateDirectCallFromJit(MacroAssembler& masm, const FuncExport& fe,
GenPrintF64(DebugChannel::Function, masm, ReturnDoubleReg);
break;
case wasm::ValType::Ref:
GenPrintPtr(DebugChannel::Import, masm, ReturnReg);
// The call to wasm above preserves the InstanceReg, we don't
// need to reload it here.
UnboxAnyrefIntoValueReg(masm, InstanceReg, ReturnReg, JSReturnOperand,
WasmJitEntryReturnScratch);
masm.convertWasmAnyRefToValue(InstanceReg, ReturnReg, JSReturnOperand,
WasmJitEntryReturnScratch);
break;
case wasm::ValType::V128:
MOZ_CRASH("unexpected return type when calling from ion to wasm");
@ -1850,7 +1738,8 @@ static void FillArgumentArrayForJitExit(MacroAssembler& masm, Register instance,
// This works also for FuncRef because it is distinguishable from
// a boxed AnyRef.
masm.movePtr(i->gpr(), scratch2);
UnboxAnyrefIntoValue(masm, instance, scratch2, dst, scratch);
GenPrintPtr(DebugChannel::Import, masm, scratch2);
masm.convertWasmAnyRefToValue(instance, scratch2, dst, scratch);
} else if (type == MIRType::StackResults) {
MOZ_CRASH("Multi-result exit to JIT unimplemented");
} else {
@ -1914,7 +1803,8 @@ static void FillArgumentArrayForJitExit(MacroAssembler& masm, Register instance,
// This works also for FuncRef because it is distinguishable from a
// boxed AnyRef.
masm.loadPtr(src, scratch);
UnboxAnyrefIntoValue(masm, instance, scratch, dst, scratch2);
GenPrintPtr(DebugChannel::Import, masm, scratch);
masm.convertWasmAnyRefToValue(instance, scratch, dst, scratch2);
} else if (IsFloatingPointType(type)) {
ScratchDoubleScope dscratch(masm);
FloatRegister fscratch = dscratch.asSingle();
@ -2198,7 +2088,6 @@ static bool GenerateImportInterpExit(MacroAssembler& masm, const FuncImport& fi,
GenPrintF64(DebugChannel::Import, masm, ReturnDoubleReg);
break;
case ValType::Ref:
STATIC_ASSERT_ANYREF_IS_JSOBJECT;
masm.loadPtr(argv, ReturnReg);
GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ",
funcImportIndex);
@ -2399,7 +2288,7 @@ static bool GenerateImportJitExit(MacroAssembler& masm, const FuncImport& fi,
case ValType::Ref:
// Guarded by temporarilyUnsupportedReftypeForExit()
MOZ_RELEASE_ASSERT(results[0].refType().isExtern());
BoxValueIntoAnyref(masm, JSReturnOperand, ReturnReg, &oolConvert);
masm.convertValueToWasmAnyRef(JSReturnOperand, ReturnReg, &oolConvert);
GenPrintPtr(DebugChannel::Import, masm, ReturnReg);
break;
}
@ -2498,7 +2387,7 @@ static bool GenerateImportJitExit(MacroAssembler& masm, const FuncImport& fi,
// Guarded by temporarilyUnsupportedReftypeForExit()
MOZ_RELEASE_ASSERT(results[0].refType().isExtern());
masm.call(SymbolicAddress::BoxValue_Anyref);
masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
masm.branchWasmAnyRefIsNull(true, ReturnReg, throwLabel);
break;
default:
MOZ_CRASH("Unsupported convert type");