From 61dcca4d6b86af2a9523bf2b03f58b8f27058463 Mon Sep 17 00:00:00 2001 From: Lars T Hansen Date: Tue, 12 Dec 2017 12:58:29 -0600 Subject: [PATCH] Bug 1425149 - Revamp the MASM atomics API. r=nbp We do two things that are intertwined: 1. Expose primitive atomic operations for JS and Wasm in MacroAssembler.h, and rewrite the back-ends and the code generators to conform to the new APIs. 2. Parameterize the atomic operations with a synchronization specification, which is an abstraction of the barriers that are needed. Previously most implementations assumed seq_cst but this will change with Wasm. Note that by and large, the MIPS back-ends are not updated and they will need to be adapted separately. --HG-- extra : source : 8dc620adcb86dea1c902069e15665891672e32db extra : histedit_source : 315fef2adaadd183a681977a01008a419db1822b%2Ce51e63492279231517b0590c2ffd8add55fa6136 --- js/src/jit/AtomicOp.h | 31 + js/src/jit/Lowering.cpp | 10 +- js/src/jit/MacroAssembler.cpp | 115 ++ js/src/jit/MacroAssembler.h | 215 +++ js/src/jit/arm/CodeGenerator-arm.cpp | 356 +---- js/src/jit/arm/MacroAssembler-arm.cpp | 1275 ++++++++--------- js/src/jit/arm/MacroAssembler-arm.h | 302 ---- js/src/jit/arm64/CodeGenerator-arm64.cpp | 8 +- js/src/jit/arm64/MacroAssembler-arm64.cpp | 182 +-- js/src/jit/arm64/MacroAssembler-arm64.h | 305 ---- .../mips-shared/CodeGenerator-mips-shared.cpp | 14 +- js/src/jit/mips32/CodeGenerator-mips32.cpp | 20 +- js/src/jit/mips64/CodeGenerator-mips64.cpp | 25 +- js/src/jit/mips64/MacroAssembler-mips64.h | 3 + js/src/jit/none/MacroAssembler-none.h | 59 - js/src/jit/shared/Assembler-shared.h | 14 +- js/src/jit/x64/CodeGenerator-x64.cpp | 61 +- js/src/jit/x64/MacroAssembler-x64.cpp | 110 +- js/src/jit/x64/MacroAssembler-x64.h | 82 -- .../x86-shared/CodeGenerator-x86-shared.cpp | 298 +--- .../jit/x86-shared/CodeGenerator-x86-shared.h | 9 - .../x86-shared/MacroAssembler-x86-shared.cpp | 460 ++++-- .../x86-shared/MacroAssembler-x86-shared.h | 442 ------ js/src/jit/x86/CodeGenerator-x86.cpp | 52 +- js/src/jit/x86/MacroAssembler-x86.cpp | 239 +-- js/src/jit/x86/MacroAssembler-x86.h | 71 +- js/src/wasm/WasmBaselineCompile.cpp | 89 +- js/src/wasm/WasmIonCompile.cpp | 20 +- 28 files changed, 1882 insertions(+), 2985 deletions(-) diff --git a/js/src/jit/AtomicOp.h b/js/src/jit/AtomicOp.h index 9a686cdd7b58..9e1fc3ba5185 100644 --- a/js/src/jit/AtomicOp.h +++ b/js/src/jit/AtomicOp.h @@ -67,6 +67,37 @@ static constexpr MemoryBarrierBits MembarAfterLoad = MembarLoadLoad|MembarLoadSt static constexpr MemoryBarrierBits MembarBeforeStore = MembarStoreStore; static constexpr MemoryBarrierBits MembarAfterStore = MembarStoreLoad; +struct Synchronization +{ + const MemoryBarrierBits barrierBefore; + const MemoryBarrierBits barrierAfter; + + constexpr Synchronization(MemoryBarrierBits before, MemoryBarrierBits after) + : barrierBefore(before), + barrierAfter(after) + {} + + static Synchronization None() { + return Synchronization(MemoryBarrierBits(MembarNobits), MemoryBarrierBits(MembarNobits)); + } + + static Synchronization Full() { + return Synchronization(MembarFull, MembarFull); + } + + static Synchronization Load() { + return Synchronization(MembarBeforeLoad, MembarAfterLoad); + } + + static Synchronization Store() { + return Synchronization(MembarBeforeStore, MembarAfterStore); + } + + bool isNone() const { + return (barrierBefore | barrierAfter) == MembarNobits; + } +}; + } // namespace jit } // namespace js diff --git a/js/src/jit/Lowering.cpp b/js/src/jit/Lowering.cpp index bc220768db3f..89ddbfe3e487 100644 --- a/js/src/jit/Lowering.cpp +++ b/js/src/jit/Lowering.cpp @@ -3565,8 +3565,9 @@ LIRGenerator::visitLoadUnboxedScalar(MLoadUnboxedScalar* ins) if (ins->readType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) tempDef = temp(); + Synchronization sync = Synchronization::Load(); if (ins->requiresMemoryBarrier()) { - LMemoryBarrier* fence = new(alloc()) LMemoryBarrier(MembarBeforeLoad); + LMemoryBarrier* fence = new(alloc()) LMemoryBarrier(sync.barrierBefore); add(fence, ins); } LLoadUnboxedScalar* lir = new(alloc()) LLoadUnboxedScalar(elements, index, tempDef); @@ -3574,7 +3575,7 @@ LIRGenerator::visitLoadUnboxedScalar(MLoadUnboxedScalar* ins) assignSnapshot(lir, Bailout_Overflow); define(lir, ins); if (ins->requiresMemoryBarrier()) { - LMemoryBarrier* fence = new(alloc()) LMemoryBarrier(MembarAfterLoad); + LMemoryBarrier* fence = new(alloc()) LMemoryBarrier(sync.barrierAfter); add(fence, ins); } } @@ -3675,13 +3676,14 @@ LIRGenerator::visitStoreUnboxedScalar(MStoreUnboxedScalar* ins) // is a store instruction that incorporates the necessary // barriers, and we could use that instead of separate barrier and // store instructions. See bug #1077027. + Synchronization sync = Synchronization::Store(); if (ins->requiresMemoryBarrier()) { - LMemoryBarrier* fence = new(alloc()) LMemoryBarrier(MembarBeforeStore); + LMemoryBarrier* fence = new(alloc()) LMemoryBarrier(sync.barrierBefore); add(fence, ins); } add(new(alloc()) LStoreUnboxedScalar(elements, index, value), ins); if (ins->requiresMemoryBarrier()) { - LMemoryBarrier* fence = new(alloc()) LMemoryBarrier(MembarAfterStore); + LMemoryBarrier* fence = new(alloc()) LMemoryBarrier(sync.barrierAfter); add(fence, ins); } } diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp index daf69ae0f25a..f8042ba300fc 100644 --- a/js/src/jit/MacroAssembler.cpp +++ b/js/src/jit/MacroAssembler.cpp @@ -3243,8 +3243,123 @@ MacroAssembler::emitPreBarrierFastPath(JSRuntime* rt, MIRType type, Register tem branchTestPtr(Assembler::NonZero, temp2, temp1, noBarrier); } +// ======================================================================== +// JS atomic operations. + +template +static void +CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, const Synchronization& sync, + const T& mem, Register oldval, Register newval, Register temp, AnyRegister output) +{ + if (arrayType == Scalar::Uint32) { + masm.compareExchange(arrayType, sync, mem, oldval, newval, temp); + masm.convertUInt32ToDouble(temp, output.fpu()); + } else { + masm.compareExchange(arrayType, sync, mem, oldval, newval, output.gpr()); + } +} + +void +MacroAssembler::compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync, + const Address& mem, Register oldval, Register newval, + Register temp, AnyRegister output) +{ + CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output); +} + +void +MacroAssembler::compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync, + const BaseIndex& mem, Register oldval, Register newval, + Register temp, AnyRegister output) +{ + CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output); +} + +template +static void +AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, const Synchronization& sync, + const T& mem, Register value, Register temp, AnyRegister output) +{ + if (arrayType == Scalar::Uint32) { + masm.atomicExchange(arrayType, sync, mem, value, temp); + masm.convertUInt32ToDouble(temp, output.fpu()); + } else { + masm.atomicExchange(arrayType, sync, mem, value, output.gpr()); + } +} + +void +MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync, + const Address& mem, Register value, Register temp, + AnyRegister output) +{ + AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output); +} + +void +MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync, + const BaseIndex& mem, Register value, Register temp, + AnyRegister output) +{ + AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output); +} + +template +static void +AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType, const Synchronization& sync, + AtomicOp op, Register value, const T& mem, Register temp1, Register temp2, + AnyRegister output) +{ + if (arrayType == Scalar::Uint32) { + masm.atomicFetchOp(arrayType, sync, op, value, mem, temp2, temp1); + masm.convertUInt32ToDouble(temp1, output.fpu()); + } else { + masm.atomicFetchOp(arrayType, sync, op, value, mem, temp1, output.gpr()); + } +} + +void +MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register temp1, Register temp2, + AnyRegister output) +{ + AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output); +} + +void +MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register temp1, Register temp2, + AnyRegister output) +{ + AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output); +} + +void +MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register temp) +{ + atomicEffectOp(arrayType, sync, op, value, mem, temp); +} + +void +MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register temp) +{ + atomicEffectOp(arrayType, sync, op, value, mem, temp); +} + //}}} check_macroassembler_style +void +MacroAssembler::memoryBarrierBefore(const Synchronization& sync) { + memoryBarrier(sync.barrierBefore); +} + +void +MacroAssembler::memoryBarrierAfter(const Synchronization& sync) { + memoryBarrier(sync.barrierAfter); +} + void MacroAssembler::loadWasmTlsRegFromFrame(Register dest) { diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h index dbf517681ba5..79b12cc133f1 100644 --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -1520,6 +1520,218 @@ class MacroAssembler : public MacroAssemblerSpecific inline void clampIntToUint8(Register reg) PER_SHARED_ARCH; + public: + // ======================================================================== + // Primitive atomic operations. + // + // If the access is from JS and the eventual destination of the result is a + // js::Value, it's probably best to use the JS-specific versions of these, + // see further below. + // + // Temp registers must be defined unless otherwise noted in the per-function + // constraints. + + // 8-bit, 16-bit, and 32-bit wide operations. + // + // The 8-bit and 16-bit operations zero-extend or sign-extend the result to + // 32 bits, according to `type`. On 64-bit systems, the upper 32 bits of + // the result will be zero. + + // CompareExchange with memory. Return the value that was in memory, + // whether we wrote or not. + // + // x86-shared: `output` must be eax. + + void compareExchange(Scalar::Type type, const Synchronization& sync, const Address& mem, + Register expected, Register replacement, Register output) + DEFINED_ON(arm, arm64, x86_shared); + + void compareExchange(Scalar::Type type, const Synchronization& sync, const BaseIndex& mem, + Register expected, Register replacement, Register output) + DEFINED_ON(arm, arm64, x86_shared); + + // Exchange with memory. Return the value initially in memory. + + void atomicExchange(Scalar::Type type, const Synchronization& sync, const Address& mem, + Register value, Register output) + DEFINED_ON(arm, arm64, x86_shared); + + void atomicExchange(Scalar::Type type, const Synchronization& sync, const BaseIndex& mem, + Register value, Register output) + DEFINED_ON(arm, arm64, x86_shared); + + // Read-modify-write with memory. Return the value in memory before the + // operation. + // + // x86-shared: + // For 8-bit operations, `value` and `output` must have a byte subregister. + // For Add and Sub, `temp` must be invalid. + // For And, Or, and Xor, `output` must be eax and `temp` must have a byte subregister. + // + // ARM: Registers `value` and `output` must differ. + + void atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register temp, Register output) + DEFINED_ON(arm, arm64, x86_shared); + + void atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Imm32 value, const Address& mem, Register temp, Register output) + DEFINED_ON(x86_shared); + + void atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register temp, Register output) + DEFINED_ON(arm, arm64, x86_shared); + + void atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Imm32 value, const BaseIndex& mem, Register temp, Register output) + DEFINED_ON(x86_shared); + + // Read-modify-write with memory. Return no value. + + void atomicEffectOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, Register value, + const Address& mem, Register temp) + DEFINED_ON(arm, arm64, x86_shared); + + void atomicEffectOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, Imm32 value, + const Address& mem, Register temp) + DEFINED_ON(x86_shared); + + void atomicEffectOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, Register value, + const BaseIndex& mem, Register temp) + DEFINED_ON(arm, arm64, x86_shared); + + void atomicEffectOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, Imm32 value, + const BaseIndex& mem, Register temp) + DEFINED_ON(x86_shared); + + // 64-bit wide operations. + + // 64-bit atomic load. On 64-bit systems, use regular wasm load with + // Synchronization::Load, not this method. + // + // x86: `temp` must be ecx:ebx; `output` must be edx:eax. + // ARM: `temp` should be invalid; `output` must be (even,odd) pair. + + void atomicLoad64(const Synchronization& sync, const Address& mem, Register64 temp, + Register64 output) + DEFINED_ON(arm, x86); + + void atomicLoad64(const Synchronization& sync, const BaseIndex& mem, Register64 temp, + Register64 output) + DEFINED_ON(arm, x86); + + // x86: `expected` must be the same as `output`, and must be edx:eax + // x86: `replacement` must be ecx:ebx + // x64: `output` must be rax. + // ARM: Registers must be distinct; `replacement` and `output` must be (even,odd) pairs. + + void compareExchange64(const Synchronization& sync, const Address& mem, Register64 expected, + Register64 replacement, Register64 output) + DEFINED_ON(arm, arm64, x64, x86); + + void compareExchange64(const Synchronization& sync, const BaseIndex& mem, Register64 expected, + Register64 replacement, Register64 output) + DEFINED_ON(arm, arm64, x64, x86); + + // x86: `value` must be ecx:ebx; `output` must be edx:eax. + // ARM: Registers must be distinct; `value` and `output` must be (even,odd) pairs. + + void atomicExchange64(const Synchronization& sync, const Address& mem, Register64 value, + Register64 output) + DEFINED_ON(arm, arm64, x64, x86); + + void atomicExchange64(const Synchronization& sync, const BaseIndex& mem, Register64 value, + Register64 output) + DEFINED_ON(arm, arm64, x64, x86); + + // x86: `output` must be edx:eax, `temp` must be ecx:ebx. + // x64: For And, Or, and Xor `output` must be rax. + // ARM: Registers must be distinct; `temp` and `output` must be (even,odd) pairs. + + void atomicFetchOp64(const Synchronization& sync, AtomicOp op, Register64 value, + const Address& mem, Register64 temp, Register64 output) + DEFINED_ON(arm, arm64, x64); + + void atomicFetchOp64(const Synchronization& sync, AtomicOp op, Register64 value, + const BaseIndex& mem, Register64 temp, Register64 output) + DEFINED_ON(arm, arm64, x64); + + void atomicFetchOp64(const Synchronization& sync, AtomicOp op, const Address& value, + const Address& mem, Register64 temp, Register64 output) + DEFINED_ON(x86); + + void atomicFetchOp64(const Synchronization& sync, AtomicOp op, const Address& value, + const BaseIndex& mem, Register64 temp, Register64 output) + DEFINED_ON(x86); + + void atomicEffectOp64(const Synchronization& sync, AtomicOp op, Register64 value, + const BaseIndex& mem) + DEFINED_ON(x64); + + // ======================================================================== + // JS atomic operations. + // + // Here the arrayType must be a type that is valid for JS. As of 2017 that + // is an 8-bit, 16-bit, or 32-bit integer type. + // + // If arrayType is Scalar::Uint32 then: + // + // - `output` must be a float register (this is bug 1077305) + // - if the operation takes one temp register then `temp` must be defined + // - if the operation takes two temp registers then `temp2` must be defined. + // + // Otherwise `output` must be a GPR and `temp`/`temp2` should be InvalidReg. + // (`temp1` must always be valid.) + // + // For additional register constraints, see the primitive 32-bit operations + // above. + + void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync, const Address& mem, + Register expected, Register replacement, Register temp, + AnyRegister output); + + void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync, + const BaseIndex& mem, Register expected, Register replacement, + Register temp, AnyRegister output); + + void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync, const Address& mem, + Register value, Register temp, AnyRegister output); + + void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync, const BaseIndex& mem, + Register value, Register temp, AnyRegister output); + + void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register temp1, Register temp2, + AnyRegister output); + + void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register temp1, Register temp2, + AnyRegister output); + + void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Imm32 value, const Address& mem, Register temp1, Register temp2, + AnyRegister output) + DEFINED_ON(x86_shared); + + void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Imm32 value, const BaseIndex& mem, Register temp1, Register temp2, + AnyRegister output) + DEFINED_ON(x86_shared); + + void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register temp); + + void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register temp); + + void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Imm32 value, const Address& mem, Register temp) + DEFINED_ON(x86_shared); + + void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Imm32 value, const BaseIndex& mem, Register temp) + DEFINED_ON(x86_shared); + //}}} check_macroassembler_style public: @@ -1736,6 +1948,9 @@ class MacroAssembler : public MacroAssemblerSpecific void storeToTypedFloatArray(Scalar::Type arrayType, FloatRegister value, const Address& dest, unsigned numElems = 0); + void memoryBarrierBefore(const Synchronization& sync); + void memoryBarrierAfter(const Synchronization& sync); + // Load a property from an UnboxedPlainObject or UnboxedArrayObject. template void loadUnboxedProperty(T address, JSValueType type, TypedOrValueRegister output); diff --git a/js/src/jit/arm/CodeGenerator-arm.cpp b/js/src/jit/arm/CodeGenerator-arm.cpp index 51ed3802a567..c7be4884715b 100644 --- a/js/src/jit/arm/CodeGenerator-arm.cpp +++ b/js/src/jit/arm/CodeGenerator-arm.cpp @@ -1834,10 +1834,10 @@ CodeGeneratorARM::visitCompareExchangeTypedArrayElement(LCompareExchangeTypedArr if (lir->index()->isConstant()) { Address dest(elements, ToInt32(lir->index()) * width); - masm.compareExchangeToTypedIntArray(arrayType, dest, oldval, newval, temp, output); + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, newval, temp, output); } else { BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - masm.compareExchangeToTypedIntArray(arrayType, dest, oldval, newval, temp, output); + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, newval, temp, output); } } @@ -1855,289 +1855,13 @@ CodeGeneratorARM::visitAtomicExchangeTypedArrayElement(LAtomicExchangeTypedArray if (lir->index()->isConstant()) { Address dest(elements, ToInt32(lir->index()) * width); - masm.atomicExchangeToTypedIntArray(arrayType, dest, value, temp, output); + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp, output); } else { BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - masm.atomicExchangeToTypedIntArray(arrayType, dest, value, temp, output); + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp, output); } } -template -void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const S& value, const T& mem, Register flagTemp, - Register outTemp, AnyRegister output) -{ - MOZ_ASSERT(flagTemp != InvalidReg); - MOZ_ASSERT_IF(arrayType == Scalar::Uint32, outTemp != InvalidReg); - - switch (arrayType) { - case Scalar::Int8: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd8SignExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub8SignExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd8SignExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr8SignExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor8SignExtend(value, mem, flagTemp, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Uint8: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd8ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub8ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd8ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr8ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor8ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Int16: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd16SignExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub16SignExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd16SignExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr16SignExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor16SignExtend(value, mem, flagTemp, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Uint16: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd16ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub16ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd16ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr16ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor16ZeroExtend(value, mem, flagTemp, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Int32: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd32(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub32(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd32(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr32(value, mem, flagTemp, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor32(value, mem, flagTemp, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Uint32: - // At the moment, the code in MCallOptimize.cpp requires the output - // type to be double for uint32 arrays. See bug 1077305. - MOZ_ASSERT(output.isFloat()); - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd32(value, mem, flagTemp, outTemp); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub32(value, mem, flagTemp, outTemp); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd32(value, mem, flagTemp, outTemp); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr32(value, mem, flagTemp, outTemp); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor32(value, mem, flagTemp, outTemp); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - masm.convertUInt32ToDouble(outTemp, output.fpu()); - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Imm32& value, const Address& mem, - Register flagTemp, Register outTemp, - AnyRegister output); -template void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Imm32& value, const BaseIndex& mem, - Register flagTemp, Register outTemp, - AnyRegister output); -template void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Register& value, const Address& mem, - Register flagTemp, Register outTemp, - AnyRegister output); -template void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Register& value, const BaseIndex& mem, - Register flagTemp, Register outTemp, - AnyRegister output); - -// Binary operation for effect, result discarded. -template -void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value, - const T& mem, Register flagTemp) -{ - MOZ_ASSERT(flagTemp != InvalidReg); - - switch (arrayType) { - case Scalar::Int8: - case Scalar::Uint8: - switch (op) { - case AtomicFetchAddOp: - masm.atomicAdd8(value, mem, flagTemp); - break; - case AtomicFetchSubOp: - masm.atomicSub8(value, mem, flagTemp); - break; - case AtomicFetchAndOp: - masm.atomicAnd8(value, mem, flagTemp); - break; - case AtomicFetchOrOp: - masm.atomicOr8(value, mem, flagTemp); - break; - case AtomicFetchXorOp: - masm.atomicXor8(value, mem, flagTemp); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Int16: - case Scalar::Uint16: - switch (op) { - case AtomicFetchAddOp: - masm.atomicAdd16(value, mem, flagTemp); - break; - case AtomicFetchSubOp: - masm.atomicSub16(value, mem, flagTemp); - break; - case AtomicFetchAndOp: - masm.atomicAnd16(value, mem, flagTemp); - break; - case AtomicFetchOrOp: - masm.atomicOr16(value, mem, flagTemp); - break; - case AtomicFetchXorOp: - masm.atomicXor16(value, mem, flagTemp); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Int32: - case Scalar::Uint32: - switch (op) { - case AtomicFetchAddOp: - masm.atomicAdd32(value, mem, flagTemp); - break; - case AtomicFetchSubOp: - masm.atomicSub32(value, mem, flagTemp); - break; - case AtomicFetchAndOp: - masm.atomicAnd32(value, mem, flagTemp); - break; - case AtomicFetchOrOp: - masm.atomicOr32(value, mem, flagTemp); - break; - case AtomicFetchXorOp: - masm.atomicXor32(value, mem, flagTemp); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Imm32& value, const Address& mem, - Register flagTemp); -template void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Imm32& value, const BaseIndex& mem, - Register flagTemp); -template void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Register& value, const Address& mem, - Register flagTemp); -template void -CodeGeneratorARM::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Register& value, const BaseIndex& mem, - Register flagTemp); - - -template -static inline void -AtomicBinopToTypedArray(CodeGeneratorARM* cg, AtomicOp op, - Scalar::Type arrayType, const LAllocation* value, const T& mem, - Register flagTemp, Register outTemp, AnyRegister output) -{ - if (value->isConstant()) - cg->atomicBinopToTypedIntArray(op, arrayType, Imm32(ToInt32(value)), mem, flagTemp, outTemp, output); - else - cg->atomicBinopToTypedIntArray(op, arrayType, ToRegister(value), mem, flagTemp, outTemp, output); -} - void CodeGeneratorARM::visitAtomicTypedArrayElementBinop(LAtomicTypedArrayElementBinop* lir) { @@ -2147,31 +1871,22 @@ CodeGeneratorARM::visitAtomicTypedArrayElementBinop(LAtomicTypedArrayElementBino Register elements = ToRegister(lir->elements()); Register flagTemp = ToRegister(lir->temp1()); Register outTemp = lir->temp2()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp2()); - const LAllocation* value = lir->value(); + Register value = ToRegister(lir->value()); Scalar::Type arrayType = lir->mir()->arrayType(); int width = Scalar::byteSize(arrayType); if (lir->index()->isConstant()) { Address mem(elements, ToInt32(lir->index()) * width); - AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem, flagTemp, outTemp, output); + masm.atomicFetchOpJS(arrayType, Synchronization::Full(), lir->mir()->operation(), value, + mem, flagTemp, outTemp, output); } else { BaseIndex mem(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem, flagTemp, outTemp, output); + masm.atomicFetchOpJS(arrayType, Synchronization::Full(), lir->mir()->operation(), value, + mem, flagTemp, outTemp, output); } } -template -static inline void -AtomicBinopToTypedArray(CodeGeneratorARM* cg, AtomicOp op, Scalar::Type arrayType, - const LAllocation* value, const T& mem, Register flagTemp) -{ - if (value->isConstant()) - cg->atomicBinopToTypedIntArray(op, arrayType, Imm32(ToInt32(value)), mem, flagTemp); - else - cg->atomicBinopToTypedIntArray(op, arrayType, ToRegister(value), mem, flagTemp); -} - void CodeGeneratorARM::visitAtomicTypedArrayElementBinopForEffect(LAtomicTypedArrayElementBinopForEffect* lir) { @@ -2179,16 +1894,18 @@ CodeGeneratorARM::visitAtomicTypedArrayElementBinopForEffect(LAtomicTypedArrayEl Register elements = ToRegister(lir->elements()); Register flagTemp = ToRegister(lir->flagTemp()); - const LAllocation* value = lir->value(); + Register value = ToRegister(lir->value()); Scalar::Type arrayType = lir->mir()->arrayType(); int width = Scalar::byteSize(arrayType); if (lir->index()->isConstant()) { Address mem(elements, ToInt32(lir->index()) * width); - AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem, flagTemp); + masm.atomicEffectOpJS(arrayType, Synchronization::Full(), lir->mir()->operation(), value, + mem, flagTemp); } else { BaseIndex mem(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem, flagTemp); + masm.atomicEffectOpJS(arrayType, Synchronization::Full(), lir->mir()->operation(), value, + mem, flagTemp); } } @@ -2554,10 +2271,9 @@ CodeGeneratorARM::visitWasmCompareExchangeHeap(LWasmCompareExchangeHeap* ins) Register oldval = ToRegister(ins->oldValue()); Register newval = ToRegister(ins->newValue()); + Register out = ToRegister(ins->output()); - masm.compareExchangeToTypedIntArray(vt == Scalar::Uint32 ? Scalar::Int32 : vt, - srcAddr, oldval, newval, InvalidReg, - ToAnyRegister(ins->output())); + masm.compareExchange(vt, Synchronization::Full(), srcAddr, oldval, newval, out); } void @@ -2568,11 +2284,11 @@ CodeGeneratorARM::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) Scalar::Type vt = mir->access().type(); Register ptrReg = ToRegister(ins->ptr()); Register value = ToRegister(ins->value()); + Register output = ToRegister(ins->output()); BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset()); MOZ_ASSERT(ins->addrTemp()->isBogusTemp()); - masm.atomicExchangeToTypedIntArray(vt == Scalar::Uint32 ? Scalar::Int32 : vt, - srcAddr, value, InvalidReg, ToAnyRegister(ins->output())); + masm.atomicExchange(vt, Synchronization::Full(), srcAddr, value, output); } void @@ -2584,21 +2300,14 @@ CodeGeneratorARM::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) Scalar::Type vt = mir->access().type(); Register ptrReg = ToRegister(ins->ptr()); Register flagTemp = ToRegister(ins->flagTemp()); + Register output = ToRegister(ins->output()); const LAllocation* value = ins->value(); AtomicOp op = mir->operation(); MOZ_ASSERT(ins->addrTemp()->isBogusTemp()); BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset()); - - if (value->isConstant()) { - atomicBinopToTypedIntArray(op, vt == Scalar::Uint32 ? Scalar::Int32 : vt, - Imm32(ToInt32(value)), srcAddr, flagTemp, InvalidReg, - ToAnyRegister(ins->output())); - } else { - atomicBinopToTypedIntArray(op, vt == Scalar::Uint32 ? Scalar::Int32 : vt, - ToRegister(value), srcAddr, flagTemp, InvalidReg, - ToAnyRegister(ins->output())); - } + masm.atomicFetchOp(vt, Synchronization::Full(), op, ToRegister(value), srcAddr, flagTemp, + output); } void @@ -2615,11 +2324,7 @@ CodeGeneratorARM::visitWasmAtomicBinopHeapForEffect(LWasmAtomicBinopHeapForEffec MOZ_ASSERT(ins->addrTemp()->isBogusTemp()); BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset()); - - if (value->isConstant()) - atomicBinopToTypedIntArray(op, vt, Imm32(ToInt32(value)), srcAddr, flagTemp); - else - atomicBinopToTypedIntArray(op, vt, ToRegister(value), srcAddr, flagTemp); + masm.atomicEffectOp(vt, Synchronization::Full(), op, ToRegister(value), srcAddr, flagTemp); } void @@ -3396,7 +3101,7 @@ CodeGeneratorARM::visitWasmAtomicLoadI64(LWasmAtomicLoadI64* lir) Register64 tmp(InvalidReg, InvalidReg); BaseIndex addr(HeapReg, ptr, TimesOne, lir->mir()->access().offset()); - masm.atomicLoad64(addr, tmp, output); + masm.atomicLoad64(Synchronization::Full(), addr, tmp, output); } void @@ -3407,7 +3112,7 @@ CodeGeneratorARM::visitWasmAtomicStoreI64(LWasmAtomicStoreI64* lir) Register64 tmp(ToRegister(lir->tmpHigh()), ToRegister(lir->tmpLow())); BaseIndex addr(HeapReg, ptr, TimesOne, lir->mir()->access().offset()); - masm.atomicExchange64(addr, value, tmp); + masm.atomicExchange64(Synchronization::Full(), addr, value, tmp); } void @@ -3419,7 +3124,7 @@ CodeGeneratorARM::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* lir) Register64 out = ToOutRegister64(lir); BaseIndex addr(HeapReg, ptr, TimesOne, lir->mir()->access().offset()); - masm.compareExchange64(addr, expected, replacement, out); + masm.compareExchange64(Synchronization::Full(), addr, expected, replacement, out); } void @@ -3431,14 +3136,7 @@ CodeGeneratorARM::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* lir) BaseIndex addr(HeapReg, ptr, TimesOne, lir->access().offset()); Register64 tmp(ToRegister(lir->tmpHigh()), ToRegister(lir->tmpLow())); - switch (lir->operation()) { - case AtomicFetchAddOp: masm.atomicFetchAdd64(value, addr, tmp, out); break; - case AtomicFetchSubOp: masm.atomicFetchSub64(value, addr, tmp, out); break; - case AtomicFetchAndOp: masm.atomicFetchAnd64(value, addr, tmp, out); break; - case AtomicFetchOrOp: masm.atomicFetchOr64(value, addr, tmp, out); break; - case AtomicFetchXorOp: masm.atomicFetchXor64(value, addr, tmp, out); break; - default: MOZ_CRASH(); - } + masm.atomicFetchOp64(Synchronization::Full(), lir->operation(), value, addr, tmp, out); } void @@ -3449,5 +3147,5 @@ CodeGeneratorARM::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* lir) Register64 out = ToOutRegister64(lir); BaseIndex addr(HeapReg, ptr, TimesOne, lir->access().offset()); - masm.atomicExchange64(addr, value, out); + masm.atomicExchange64(Synchronization::Full(), addr, value, out); } diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp index 242fa59a457c..11e32f59f97f 100644 --- a/js/src/jit/arm/MacroAssembler-arm.cpp +++ b/js/src/jit/arm/MacroAssembler-arm.cpp @@ -12,6 +12,7 @@ #include "mozilla/MathAlgorithms.h" #include "jit/arm/Simulator-arm.h" +#include "jit/AtomicOp.h" #include "jit/AtomicOperations.h" #include "jit/Bailouts.h" #include "jit/BaselineFrame.h" @@ -4158,664 +4159,6 @@ MacroAssemblerARMCompat::jumpWithPatch(RepatchLabel* label, Condition cond, Labe return ret; } -namespace js { -namespace jit { - -template<> -Register -MacroAssemblerARMCompat::computePointer(const BaseIndex& src, Register r) -{ - Register base = src.base; - Register index = src.index; - uint32_t scale = Imm32::ShiftOf(src.scale).value; - int32_t offset = src.offset; - - ScratchRegisterScope scratch(asMasm()); - - as_add(r, base, lsl(index, scale)); - if (offset != 0) - ma_add(r, Imm32(offset), r, scratch); - return r; -} - -template<> -Register -MacroAssemblerARMCompat::computePointer
(const Address& src, Register r) -{ - ScratchRegisterScope scratch(asMasm()); - if (src.offset == 0) - return src.base; - ma_add(src.base, Imm32(src.offset), r, scratch); - return r; -} - -} // namespace jit -} // namespace js - -// General algorithm: -// -// ... ptr, ; compute address of item -// dmb -// L0 ldrex* output, [ptr] -// sxt* output, output, 0 ; sign-extend if applicable -// *xt* tmp, oldval, 0 ; sign-extend or zero-extend if applicable -// cmp output, tmp -// bne L1 ; failed - values are different -// strex* tmp, newval, [ptr] -// cmp tmp, 1 -// beq L0 ; failed - location is dirty, retry -// L1 dmb -// -// Discussion here: http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html. -// However note that that discussion uses 'isb' as the trailing fence. -// I've not quite figured out why, and I've gone with dmb here which -// is safe. Also see the LLVM source, which uses 'dmb ish' generally. -// (Apple's Swift CPU apparently handles ish in a non-default, faster -// way.) - -template -void -MacroAssemblerARMCompat::compareExchange(int nbytes, bool signExtend, const T& mem, - Register oldval, Register newval, Register output) -{ - Label again; - Label done; - - asMasm().memoryBarrier(MembarFull); - - SecondScratchRegisterScope scratch2(asMasm()); - Register ptr = computePointer(mem, scratch2); - - ScratchRegisterScope scratch(asMasm()); - - bind(&again); - switch (nbytes) { - case 1: - as_ldrexb(output, ptr); - if (signExtend) { - as_sxtb(output, output, 0); - as_sxtb(scratch, oldval, 0); - } else { - as_uxtb(scratch, oldval, 0); - } - break; - case 2: - as_ldrexh(output, ptr); - if (signExtend) { - as_sxth(output, output, 0); - as_sxth(scratch, oldval, 0); - } else { - as_uxth(scratch, oldval, 0); - } - break; - case 4: - MOZ_ASSERT(!signExtend); - as_ldrex(output, ptr); - break; - } - if (nbytes < 4) - as_cmp(output, O2Reg(scratch)); - else - as_cmp(output, O2Reg(oldval)); - as_b(&done, NotEqual); - switch (nbytes) { - case 1: - as_strexb(scratch, newval, ptr); - break; - case 2: - as_strexh(scratch, newval, ptr); - break; - case 4: - as_strex(scratch, newval, ptr); - break; - } - as_cmp(scratch, Imm8(1)); - as_b(&again, Equal); - bind(&done); - - asMasm().memoryBarrier(MembarFull); -} - -template void -js::jit::MacroAssemblerARMCompat::compareExchange(int nbytes, bool signExtend, - const Address& address, Register oldval, - Register newval, Register output); -template void -js::jit::MacroAssemblerARMCompat::compareExchange(int nbytes, bool signExtend, - const BaseIndex& address, Register oldval, - Register newval, Register output); - -template -void -MacroAssemblerARMCompat::atomicExchange(int nbytes, bool signExtend, const T& mem, - Register value, Register output) -{ - // Bug 1077321: We may further optimize for ARMv8 (AArch32) here. - Label again; - Label done; - - asMasm().memoryBarrier(MembarFull); - - SecondScratchRegisterScope scratch2(asMasm()); - Register ptr = computePointer(mem, scratch2); - - ScratchRegisterScope scratch(asMasm()); - - bind(&again); - switch (nbytes) { - case 1: - as_ldrexb(output, ptr); - if (signExtend) - as_sxtb(output, output, 0); - as_strexb(scratch, value, ptr); - break; - case 2: - as_ldrexh(output, ptr); - if (signExtend) - as_sxth(output, output, 0); - as_strexh(scratch, value, ptr); - break; - case 4: - MOZ_ASSERT(!signExtend); - as_ldrex(output, ptr); - as_strex(scratch, value, ptr); - break; - default: - MOZ_CRASH(); - } - as_cmp(scratch, Imm8(1)); - as_b(&again, Equal); - bind(&done); - - asMasm().memoryBarrier(MembarFull); -} - -template void -js::jit::MacroAssemblerARMCompat::atomicExchange(int nbytes, bool signExtend, - const Address& address, Register value, - Register output); -template void -js::jit::MacroAssemblerARMCompat::atomicExchange(int nbytes, bool signExtend, - const BaseIndex& address, Register value, - Register output); - -template -void -MacroAssemblerARMCompat::atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, const Imm32& value, - const T& mem, Register flagTemp, Register output) -{ - // The Imm32 case is not needed yet because lowering always forces - // the value into a register at present (bug 1077317). - // - // This would be useful for immediates small enough to fit into - // add/sub/and/or/xor. - MOZ_CRASH("Feature NYI"); -} - -// General algorithm: -// -// ... ptr, ; compute address of item -// dmb -// L0 ldrex* output, [ptr] -// sxt* output, output, 0 ; sign-extend if applicable -// OP tmp, output, value ; compute value to store -// strex* tmp2, tmp, [ptr] ; tmp2 required by strex -// cmp tmp2, 1 -// beq L0 ; failed - location is dirty, retry -// dmb ; ordering barrier required -// -// Also see notes above at compareExchange re the barrier strategy. -// -// Observe that the value being operated into the memory element need -// not be sign-extended because no OP will make use of bits to the -// left of the bits indicated by the width of the element, and neither -// output nor the bits stored are affected by OP. - -template -void -MacroAssemblerARMCompat::atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, - const Register& value, const T& mem, Register flagTemp, - Register output) -{ - MOZ_ASSERT(flagTemp != InvalidReg); - MOZ_ASSERT(output != value); - - Label again; - - SecondScratchRegisterScope scratch2(asMasm()); - Register ptr = computePointer(mem, scratch2); - - asMasm().memoryBarrier(MembarFull); - - ScratchRegisterScope scratch(asMasm()); - - bind(&again); - switch (nbytes) { - case 1: - as_ldrexb(output, ptr); - if (signExtend) - as_sxtb(output, output, 0); - break; - case 2: - as_ldrexh(output, ptr); - if (signExtend) - as_sxth(output, output, 0); - break; - case 4: - MOZ_ASSERT(!signExtend); - as_ldrex(output, ptr); - break; - } - switch (op) { - case AtomicFetchAddOp: - as_add(scratch, output, O2Reg(value)); - break; - case AtomicFetchSubOp: - as_sub(scratch, output, O2Reg(value)); - break; - case AtomicFetchAndOp: - as_and(scratch, output, O2Reg(value)); - break; - case AtomicFetchOrOp: - as_orr(scratch, output, O2Reg(value)); - break; - case AtomicFetchXorOp: - as_eor(scratch, output, O2Reg(value)); - break; - } - // Rd must differ from the two other arguments to strex. - switch (nbytes) { - case 1: - as_strexb(flagTemp, scratch, ptr); - break; - case 2: - as_strexh(flagTemp, scratch, ptr); - break; - case 4: - as_strex(flagTemp, scratch, ptr); - break; - } - as_cmp(flagTemp, Imm8(1)); - as_b(&again, Equal); - - asMasm().memoryBarrier(MembarFull); -} - -// Uses both scratch registers, one for the address and one for a temp, -// but needs two temps for strex: -// -// ... ptr, ; compute address of item -// dmb -// L0 ldrex* temp, [ptr] -// OP temp, temp, value ; compute value to store -// strex* temp2, temp, [ptr] -// cmp temp2, 1 -// beq L0 ; failed - location is dirty, retry -// dmb ; ordering barrier required - -template -void -MacroAssemblerARMCompat::atomicEffectOp(int nbytes, AtomicOp op, const Register& value, - const T& mem, Register flagTemp) -{ - MOZ_ASSERT(flagTemp != InvalidReg); - - Label again; - - SecondScratchRegisterScope scratch2(asMasm()); - Register ptr = computePointer(mem, scratch2); - - asMasm().memoryBarrier(MembarFull); - - ScratchRegisterScope scratch(asMasm()); - - bind(&again); - switch (nbytes) { - case 1: - as_ldrexb(scratch, ptr); - break; - case 2: - as_ldrexh(scratch, ptr); - break; - case 4: - as_ldrex(scratch, ptr); - break; - } - switch (op) { - case AtomicFetchAddOp: - as_add(scratch, scratch, O2Reg(value)); - break; - case AtomicFetchSubOp: - as_sub(scratch, scratch, O2Reg(value)); - break; - case AtomicFetchAndOp: - as_and(scratch, scratch, O2Reg(value)); - break; - case AtomicFetchOrOp: - as_orr(scratch, scratch, O2Reg(value)); - break; - case AtomicFetchXorOp: - as_eor(scratch, scratch, O2Reg(value)); - break; - } - // Rd must differ from the two other arguments to strex. - switch (nbytes) { - case 1: - as_strexb(flagTemp, scratch, ptr); - break; - case 2: - as_strexh(flagTemp, scratch, ptr); - break; - case 4: - as_strex(flagTemp, scratch, ptr); - break; - } - as_cmp(flagTemp, Imm8(1)); - as_b(&again, Equal); - - asMasm().memoryBarrier(MembarFull); -} - -template -void -MacroAssemblerARMCompat::atomicEffectOp(int nbytes, AtomicOp op, const Imm32& value, - const T& mem, Register flagTemp) -{ - // The Imm32 case is not needed yet because lowering always forces - // the value into a register at present (bug 1077317). - // - // This would be useful for immediates small enough to fit into - // add/sub/and/or/xor. - MOZ_CRASH("NYI"); -} - -template void -js::jit::MacroAssemblerARMCompat::atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, - const Imm32& value, const Address& mem, - Register flagTemp, Register output); -template void -js::jit::MacroAssemblerARMCompat::atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, - const Imm32& value, const BaseIndex& mem, - Register flagTemp, Register output); -template void -js::jit::MacroAssemblerARMCompat::atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, - const Register& value, const Address& mem, - Register flagTemp, Register output); -template void -js::jit::MacroAssemblerARMCompat::atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, - const Register& value, const BaseIndex& mem, - Register flagTemp, Register output); - -template void -js::jit::MacroAssemblerARMCompat::atomicEffectOp(int nbytes, AtomicOp op, const Imm32& value, - const Address& mem, Register flagTemp); -template void -js::jit::MacroAssemblerARMCompat::atomicEffectOp(int nbytes, AtomicOp op, const Imm32& value, - const BaseIndex& mem, Register flagTemp); -template void -js::jit::MacroAssemblerARMCompat::atomicEffectOp(int nbytes, AtomicOp op, const Register& value, - const Address& mem, Register flagTemp); -template void -js::jit::MacroAssemblerARMCompat::atomicEffectOp(int nbytes, AtomicOp op, const Register& value, - const BaseIndex& mem, Register flagTemp); - -template -void -MacroAssemblerARMCompat::compareExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, - Register oldval, Register newval, - Register temp, AnyRegister output) -{ - switch (arrayType) { - case Scalar::Int8: - compareExchange8SignExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint8: - compareExchange8ZeroExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Int16: - compareExchange16SignExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint16: - compareExchange16ZeroExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Int32: - compareExchange32(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint32: - // At the moment, the code in MCallOptimize.cpp requires the output - // type to be double for uint32 arrays. See bug 1077305. - MOZ_ASSERT(output.isFloat()); - compareExchange32(mem, oldval, newval, temp); - convertUInt32ToDouble(temp, output.fpu()); - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -MacroAssemblerARMCompat::compareExchangeToTypedIntArray(Scalar::Type arrayType, const Address& mem, - Register oldval, Register newval, Register temp, - AnyRegister output); -template void -MacroAssemblerARMCompat::compareExchangeToTypedIntArray(Scalar::Type arrayType, const BaseIndex& mem, - Register oldval, Register newval, Register temp, - AnyRegister output); - -template -void -MacroAssemblerARMCompat::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, - Register value, Register temp, AnyRegister output) -{ - switch (arrayType) { - case Scalar::Int8: - atomicExchange8SignExtend(mem, value, output.gpr()); - break; - case Scalar::Uint8: - atomicExchange8ZeroExtend(mem, value, output.gpr()); - break; - case Scalar::Int16: - atomicExchange16SignExtend(mem, value, output.gpr()); - break; - case Scalar::Uint16: - atomicExchange16ZeroExtend(mem, value, output.gpr()); - break; - case Scalar::Int32: - atomicExchange32(mem, value, output.gpr()); - break; - case Scalar::Uint32: - // At the moment, the code in MCallOptimize.cpp requires the output - // type to be double for uint32 arrays. See bug 1077305. - MOZ_ASSERT(output.isFloat()); - atomicExchange32(mem, value, temp); - convertUInt32ToDouble(temp, output.fpu()); - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -MacroAssemblerARMCompat::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const Address& mem, - Register value, Register temp, AnyRegister output); -template void -MacroAssemblerARMCompat::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const BaseIndex& mem, - Register value, Register temp, AnyRegister output); - -template -void -MacroAssemblerARMCompat::atomicLoad64(const T& mem, Register64 temp, Register64 output) -{ - MOZ_ASSERT(temp.low == InvalidReg && temp.high == InvalidReg); - MOZ_ASSERT((output.low.code() & 1) == 0); - MOZ_ASSERT(output.low.code() + 1 == output.high.code()); - - asMasm().memoryBarrier(MembarFull); - - SecondScratchRegisterScope scratch2(asMasm()); - Register ptr = computePointer(mem, scratch2); - - as_ldrexd(output.low, output.high, ptr); - as_clrex(); - - asMasm().memoryBarrier(MembarFull); -} - -template void -MacroAssemblerARMCompat::atomicLoad64(const Address& mem, Register64 temp, Register64 output); -template void -MacroAssemblerARMCompat::atomicLoad64(const BaseIndex& mem, Register64 temp, Register64 output); - -template -void -MacroAssemblerARMCompat::atomicFetchOp64(AtomicOp op, Register64 value, const T& mem, - Register64 temp, Register64 output) -{ - MOZ_ASSERT(temp.low != InvalidReg && temp.high != InvalidReg); - MOZ_ASSERT(output != value); - - MOZ_ASSERT((temp.low.code() & 1) == 0); - MOZ_ASSERT(temp.low.code() + 1 == temp.high.code()); - - // We could avoid this pair requirement but in that case we would end up - // with two moves in the loop to preserve the loaded value in output. The - // prize would be less register spilling around this op since the pair - // requirement will tend to force more spilling. - - MOZ_ASSERT((output.low.code() & 1) == 0); - MOZ_ASSERT(output.low.code() + 1 == output.high.code()); - - Label again; - - SecondScratchRegisterScope scratch2(asMasm()); - Register ptr = computePointer(mem, scratch2); - - asMasm().memoryBarrier(MembarFull); - - bind(&again); - as_ldrexd(output.low, output.high, ptr); - switch (op) { - case AtomicFetchAddOp: - as_add(temp.low, output.low, O2Reg(value.low), SetCC); - as_adc(temp.high, output.high, O2Reg(value.high)); - break; - case AtomicFetchSubOp: - as_sub(temp.low, output.low, O2Reg(value.low), SetCC); - as_sbc(temp.high, output.high, O2Reg(value.high)); - break; - case AtomicFetchAndOp: - as_and(temp.low, output.low, O2Reg(value.low)); - as_and(temp.high, output.high, O2Reg(value.high)); - break; - case AtomicFetchOrOp: - as_orr(temp.low, output.low, O2Reg(value.low)); - as_orr(temp.high, output.high, O2Reg(value.high)); - break; - case AtomicFetchXorOp: - as_eor(temp.low, output.low, O2Reg(value.low)); - as_eor(temp.high, output.high, O2Reg(value.high)); - break; - } - - ScratchRegisterScope scratch(asMasm()); - - // Rd (temp) must differ from the two other arguments to strex. - as_strexd(scratch, temp.low, temp.high, ptr); - as_cmp(scratch, Imm8(1)); - as_b(&again, Equal); - - asMasm().memoryBarrier(MembarFull); -} - -template void -MacroAssemblerARMCompat::atomicFetchOp64(AtomicOp op, Register64 value, const Address& mem, - Register64 temp, Register64 output); -template void -MacroAssemblerARMCompat::atomicFetchOp64(AtomicOp op, Register64 value, const BaseIndex& mem, - Register64 temp, Register64 output); - -template -void -MacroAssemblerARMCompat::atomicExchange64(const T& mem, Register64 value, Register64 output) -{ - MOZ_ASSERT(output != value); - - MOZ_ASSERT((value.low.code() & 1) == 0); - MOZ_ASSERT(value.low.code() + 1 == value.high.code()); - - MOZ_ASSERT((output.low.code() & 1) == 0); - MOZ_ASSERT(output.low.code() + 1 == output.high.code()); - - Label again; - - SecondScratchRegisterScope scratch2(asMasm()); - Register ptr = computePointer(mem, scratch2); - - asMasm().memoryBarrier(MembarFull); - - bind(&again); - as_ldrexd(output.low, output.high, ptr); - - ScratchRegisterScope scratch(asMasm()); - - as_strexd(scratch, value.low, value.high, ptr); - as_cmp(scratch, Imm8(1)); - as_b(&again, Equal); - - asMasm().memoryBarrier(MembarFull); -} - -template void -MacroAssemblerARMCompat::atomicExchange64(const Address& mem, Register64 value, Register64 output); -template void -MacroAssemblerARMCompat::atomicExchange64(const BaseIndex& mem, Register64 value, Register64 output); - -template -void -MacroAssemblerARMCompat::compareExchange64(const T& mem, Register64 expect, - Register64 replace, Register64 output) -{ - MOZ_ASSERT(expect != replace && replace != output && output != expect); - - MOZ_ASSERT((replace.low.code() & 1) == 0); - MOZ_ASSERT(replace.low.code() + 1 == replace.high.code()); - - MOZ_ASSERT((output.low.code() & 1) == 0); - MOZ_ASSERT(output.low.code() + 1 == output.high.code()); - - Label again; - Label done; - - SecondScratchRegisterScope scratch2(asMasm()); - Register ptr = computePointer(mem, scratch2); - - asMasm().memoryBarrier(MembarFull); - - bind(&again); - as_ldrexd(output.low, output.high, ptr); - - as_cmp(output.low, O2Reg(expect.low)); - as_cmp(output.high, O2Reg(expect.high), Equal); - as_b(&done, NotEqual); - - ScratchRegisterScope scratch(asMasm()); - - // Rd (temp) must differ from the two other arguments to strex. - as_strexd(scratch, replace.low, replace.high, ptr); - as_cmp(scratch, Imm8(1)); - as_b(&again, Equal); - bind(&done); - - asMasm().memoryBarrier(MembarFull); -} - -template void -MacroAssemblerARMCompat::compareExchange64(const Address& mem, Register64 expect, - Register64 replace, Register64 output); -template void -MacroAssemblerARMCompat::compareExchange64(const BaseIndex& mem, Register64 expect, - Register64 replace, Register64 output); - void MacroAssemblerARMCompat::profilerEnterFrame(Register framePtr, Register scratch) { @@ -5672,6 +5015,606 @@ MacroAssembler::wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access, Regi wasmUnalignedStoreImpl(access, FloatRegister(), val64, memoryBase, ptr, ptrScratch, tmp); } +// ======================================================================== +// Primitive atomic operations. + +static Register +ComputePointerForAtomic(MacroAssembler& masm, const BaseIndex& src, Register r) +{ + Register base = src.base; + Register index = src.index; + uint32_t scale = Imm32::ShiftOf(src.scale).value; + int32_t offset = src.offset; + + ScratchRegisterScope scratch(masm); + + masm.as_add(r, base, lsl(index, scale)); + if (offset != 0) + masm.ma_add(r, Imm32(offset), r, scratch); + return r; +} + +static Register +ComputePointerForAtomic(MacroAssembler& masm, const Address& src, Register r) +{ + ScratchRegisterScope scratch(masm); + if (src.offset == 0) + return src.base; + masm.ma_add(src.base, Imm32(src.offset), r, scratch); + return r; +} + +// General algorithm: +// +// ... ptr, ; compute address of item +// dmb +// L0 ldrex* output, [ptr] +// sxt* output, output, 0 ; sign-extend if applicable +// *xt* tmp, oldval, 0 ; sign-extend or zero-extend if applicable +// cmp output, tmp +// bne L1 ; failed - values are different +// strex* tmp, newval, [ptr] +// cmp tmp, 1 +// beq L0 ; failed - location is dirty, retry +// L1 dmb +// +// Discussion here: http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html. +// However note that that discussion uses 'isb' as the trailing fence. +// I've not quite figured out why, and I've gone with dmb here which +// is safe. Also see the LLVM source, which uses 'dmb ish' generally. +// (Apple's Swift CPU apparently handles ish in a non-default, faster +// way.) + +template +static void +CompareExchange(MacroAssembler& masm, Scalar::Type type, const Synchronization& sync, + const T& mem, Register oldval, Register newval, Register output) +{ + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + MOZ_ASSERT(nbytes <= 4); + + Label again; + Label done; + + SecondScratchRegisterScope scratch2(masm); + Register ptr = ComputePointerForAtomic(masm, mem, scratch2); + + ScratchRegisterScope scratch(masm); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + switch (nbytes) { + case 1: + masm.as_ldrexb(output, ptr); + if (signExtend) { + masm.as_sxtb(output, output, 0); + masm.as_sxtb(scratch, oldval, 0); + } else { + masm.as_uxtb(scratch, oldval, 0); + } + break; + case 2: + masm.as_ldrexh(output, ptr); + if (signExtend) { + masm.as_sxth(output, output, 0); + masm.as_sxth(scratch, oldval, 0); + } else { + masm.as_uxth(scratch, oldval, 0); + } + break; + case 4: + masm.as_ldrex(output, ptr); + break; + } + if (nbytes < 4) + masm.as_cmp(output, O2Reg(scratch)); + else + masm.as_cmp(output, O2Reg(oldval)); + masm.as_b(&done, MacroAssembler::NotEqual); + switch (nbytes) { + case 1: + masm.as_strexb(scratch, newval, ptr); + break; + case 2: + masm.as_strexh(scratch, newval, ptr); + break; + case 4: + masm.as_strex(scratch, newval, ptr); + break; + } + masm.as_cmp(scratch, Imm8(1)); + masm.as_b(&again, MacroAssembler::Equal); + masm.bind(&done); + + masm.memoryBarrierAfter(sync); +} + +void +MacroAssembler::compareExchange(Scalar::Type type, const Synchronization& sync, + const Address& address, Register oldval, + Register newval, Register output) +{ + CompareExchange(*this, type, sync, address, oldval, newval, output); +} + +void +MacroAssembler::compareExchange(Scalar::Type type, const Synchronization& sync, + const BaseIndex& address, Register oldval, + Register newval, Register output) +{ + CompareExchange(*this, type, sync, address, oldval, newval, output); +} + +template +static void +AtomicExchange(MacroAssembler& masm, Scalar::Type type, const Synchronization& sync, + const T& mem, Register value, Register output) +{ + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + MOZ_ASSERT(nbytes <= 4); + + // Bug 1077321: We may further optimize for ARMv8 (AArch32) here. + Label again; + Label done; + + SecondScratchRegisterScope scratch2(masm); + Register ptr = ComputePointerForAtomic(masm, mem, scratch2); + + ScratchRegisterScope scratch(masm); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + switch (nbytes) { + case 1: + masm.as_ldrexb(output, ptr); + if (signExtend) + masm.as_sxtb(output, output, 0); + masm.as_strexb(scratch, value, ptr); + break; + case 2: + masm.as_ldrexh(output, ptr); + if (signExtend) + masm.as_sxth(output, output, 0); + masm.as_strexh(scratch, value, ptr); + break; + case 4: + masm.as_ldrex(output, ptr); + masm.as_strex(scratch, value, ptr); + break; + } + masm.as_cmp(scratch, Imm8(1)); + masm.as_b(&again, MacroAssembler::Equal); + masm.bind(&done); + + masm.memoryBarrierAfter(sync); +} + +void +MacroAssembler::atomicExchange(Scalar::Type type, const Synchronization& sync, + const Address& address, Register value, Register output) +{ + AtomicExchange(*this, type, sync, address, value, output); +} + +void +MacroAssembler::atomicExchange(Scalar::Type type, const Synchronization& sync, + const BaseIndex& address, Register value, Register output) +{ + AtomicExchange(*this, type, sync, address, value, output); +} + +// General algorithm: +// +// ... ptr, ; compute address of item +// dmb +// L0 ldrex* output, [ptr] +// sxt* output, output, 0 ; sign-extend if applicable +// OP tmp, output, value ; compute value to store +// strex* tmp2, tmp, [ptr] ; tmp2 required by strex +// cmp tmp2, 1 +// beq L0 ; failed - location is dirty, retry +// dmb ; ordering barrier required +// +// Also see notes above at compareExchange re the barrier strategy. +// +// Observe that the value being operated into the memory element need +// not be sign-extended because no OP will make use of bits to the +// left of the bits indicated by the width of the element, and neither +// output nor the bits stored are affected by OP. + +template +static void +AtomicFetchOp(MacroAssembler& masm, Scalar::Type type, const Synchronization& sync, AtomicOp op, + const Register& value, const T& mem, Register flagTemp, Register output) +{ + bool signExtend = Scalar::isSignedIntType(type); + unsigned nbytes = Scalar::byteSize(type); + + MOZ_ASSERT(nbytes <= 4); + MOZ_ASSERT(flagTemp != InvalidReg); + MOZ_ASSERT(output != value); + + Label again; + + SecondScratchRegisterScope scratch2(masm); + Register ptr = ComputePointerForAtomic(masm, mem, scratch2); + + masm.memoryBarrierBefore(sync); + + ScratchRegisterScope scratch(masm); + + masm.bind(&again); + switch (nbytes) { + case 1: + masm.as_ldrexb(output, ptr); + if (signExtend) + masm.as_sxtb(output, output, 0); + break; + case 2: + masm.as_ldrexh(output, ptr); + if (signExtend) + masm.as_sxth(output, output, 0); + break; + case 4: + masm.as_ldrex(output, ptr); + break; + } + switch (op) { + case AtomicFetchAddOp: + masm.as_add(scratch, output, O2Reg(value)); + break; + case AtomicFetchSubOp: + masm.as_sub(scratch, output, O2Reg(value)); + break; + case AtomicFetchAndOp: + masm.as_and(scratch, output, O2Reg(value)); + break; + case AtomicFetchOrOp: + masm.as_orr(scratch, output, O2Reg(value)); + break; + case AtomicFetchXorOp: + masm.as_eor(scratch, output, O2Reg(value)); + break; + } + // Rd must differ from the two other arguments to strex. + switch (nbytes) { + case 1: + masm.as_strexb(flagTemp, scratch, ptr); + break; + case 2: + masm.as_strexh(flagTemp, scratch, ptr); + break; + case 4: + masm.as_strex(flagTemp, scratch, ptr); + break; + } + masm.as_cmp(flagTemp, Imm8(1)); + masm.as_b(&again, MacroAssembler::Equal); + + masm.memoryBarrierAfter(sync); +} + +void +MacroAssembler::atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register temp, Register output) +{ + AtomicFetchOp(*this, type, sync, op, value, mem, temp, output); +} + +void +MacroAssembler::atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register temp, Register output) +{ + AtomicFetchOp(*this, type, sync, op, value, mem, temp, output); +} + +// Uses both scratch registers, one for the address and one for a temp, +// but needs two temps for strex: +// +// ... ptr, ; compute address of item +// dmb +// L0 ldrex* temp, [ptr] +// OP temp, temp, value ; compute value to store +// strex* temp2, temp, [ptr] +// cmp temp2, 1 +// beq L0 ; failed - location is dirty, retry +// dmb ; ordering barrier required + +template +static void +AtomicEffectOp(MacroAssembler& masm, Scalar::Type type, const Synchronization& sync, AtomicOp op, + const Register& value, const T& mem, Register flagTemp) +{ + unsigned nbytes = Scalar::byteSize(type); + + MOZ_ASSERT(nbytes <= 4); + MOZ_ASSERT(flagTemp != InvalidReg); + + Label again; + + SecondScratchRegisterScope scratch2(masm); + Register ptr = ComputePointerForAtomic(masm, mem, scratch2); + + masm.memoryBarrierBefore(sync); + + ScratchRegisterScope scratch(masm); + + masm.bind(&again); + switch (nbytes) { + case 1: + masm.as_ldrexb(scratch, ptr); + break; + case 2: + masm.as_ldrexh(scratch, ptr); + break; + case 4: + masm.as_ldrex(scratch, ptr); + break; + } + switch (op) { + case AtomicFetchAddOp: + masm.as_add(scratch, scratch, O2Reg(value)); + break; + case AtomicFetchSubOp: + masm.as_sub(scratch, scratch, O2Reg(value)); + break; + case AtomicFetchAndOp: + masm.as_and(scratch, scratch, O2Reg(value)); + break; + case AtomicFetchOrOp: + masm.as_orr(scratch, scratch, O2Reg(value)); + break; + case AtomicFetchXorOp: + masm.as_eor(scratch, scratch, O2Reg(value)); + break; + } + // Rd must differ from the two other arguments to strex. + switch (nbytes) { + case 1: + masm.as_strexb(flagTemp, scratch, ptr); + break; + case 2: + masm.as_strexh(flagTemp, scratch, ptr); + break; + case 4: + masm.as_strex(flagTemp, scratch, ptr); + break; + } + masm.as_cmp(flagTemp, Imm8(1)); + masm.as_b(&again, MacroAssembler::Equal); + + masm.memoryBarrierAfter(sync); +} + +void +MacroAssembler::atomicEffectOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register flagTemp) +{ + AtomicEffectOp(*this, type, sync, op, value, mem, flagTemp); +} + +void +MacroAssembler::atomicEffectOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register flagTemp) +{ + AtomicEffectOp(*this, type, sync, op, value, mem, flagTemp); +} + +template +static void +AtomicLoad64(MacroAssembler& masm, const Synchronization& sync, const T& mem, Register64 temp, + Register64 output) +{ + MOZ_ASSERT(temp.low == InvalidReg && temp.high == InvalidReg); + MOZ_ASSERT((output.low.code() & 1) == 0); + MOZ_ASSERT(output.low.code() + 1 == output.high.code()); + + masm.memoryBarrierBefore(sync); + + SecondScratchRegisterScope scratch2(masm); + Register ptr = ComputePointerForAtomic(masm, mem, scratch2); + + masm.as_ldrexd(output.low, output.high, ptr); + masm.as_clrex(); + + masm.memoryBarrierAfter(sync); +} + +void +MacroAssembler::atomicLoad64(const Synchronization& sync, const Address& mem, Register64 temp, + Register64 output) +{ + AtomicLoad64(*this, sync, mem, temp, output); +} + +void +MacroAssembler::atomicLoad64(const Synchronization& sync, const BaseIndex& mem, Register64 temp, + Register64 output) +{ + AtomicLoad64(*this, sync, mem, temp, output); +} + +template +static void +CompareExchange64(MacroAssembler& masm, const Synchronization& sync, const T& mem, + Register64 expect, Register64 replace, Register64 output) +{ + MOZ_ASSERT(expect != replace && replace != output && output != expect); + + MOZ_ASSERT((replace.low.code() & 1) == 0); + MOZ_ASSERT(replace.low.code() + 1 == replace.high.code()); + + MOZ_ASSERT((output.low.code() & 1) == 0); + MOZ_ASSERT(output.low.code() + 1 == output.high.code()); + + Label again; + Label done; + + SecondScratchRegisterScope scratch2(masm); + Register ptr = ComputePointerForAtomic(masm, mem, scratch2); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + masm.as_ldrexd(output.low, output.high, ptr); + + masm.as_cmp(output.low, O2Reg(expect.low)); + masm.as_cmp(output.high, O2Reg(expect.high), MacroAssembler::Equal); + masm.as_b(&done, MacroAssembler::NotEqual); + + ScratchRegisterScope scratch(masm); + + // Rd (temp) must differ from the two other arguments to strex. + masm.as_strexd(scratch, replace.low, replace.high, ptr); + masm.as_cmp(scratch, Imm8(1)); + masm.as_b(&again, MacroAssembler::Equal); + masm.bind(&done); + + masm.memoryBarrierAfter(sync); +} + +void +MacroAssembler::compareExchange64(const Synchronization& sync, const Address& mem, + Register64 expect, Register64 replace, Register64 output) +{ + CompareExchange64(*this, sync, mem, expect, replace, output); +} + +void +MacroAssembler::compareExchange64(const Synchronization& sync, const BaseIndex& mem, + Register64 expect, Register64 replace, Register64 output) +{ + CompareExchange64(*this, sync, mem, expect, replace, output); +} + +template +static void +AtomicExchange64(MacroAssembler& masm, const Synchronization& sync, const T& mem, Register64 value, + Register64 output) +{ + MOZ_ASSERT(output != value); + + MOZ_ASSERT((value.low.code() & 1) == 0); + MOZ_ASSERT(value.low.code() + 1 == value.high.code()); + + MOZ_ASSERT((output.low.code() & 1) == 0); + MOZ_ASSERT(output.low.code() + 1 == output.high.code()); + + Label again; + + SecondScratchRegisterScope scratch2(masm); + Register ptr = ComputePointerForAtomic(masm, mem, scratch2); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + masm.as_ldrexd(output.low, output.high, ptr); + + ScratchRegisterScope scratch(masm); + + masm.as_strexd(scratch, value.low, value.high, ptr); + masm.as_cmp(scratch, Imm8(1)); + masm.as_b(&again, MacroAssembler::Equal); + + masm.memoryBarrierAfter(sync); +} + +void +MacroAssembler::atomicExchange64(const Synchronization& sync, const Address& mem, Register64 value, + Register64 output) +{ + AtomicExchange64(*this, sync, mem, value, output); +} + +void +MacroAssembler::atomicExchange64(const Synchronization& sync, const BaseIndex& mem, Register64 value, + Register64 output) +{ + AtomicExchange64(*this, sync, mem, value, output); +} + +template +static void +AtomicFetchOp64(MacroAssembler& masm, const Synchronization& sync, AtomicOp op, Register64 value, + const T& mem, Register64 temp, Register64 output) +{ + MOZ_ASSERT(temp.low != InvalidReg && temp.high != InvalidReg); + MOZ_ASSERT(output != value); + + MOZ_ASSERT((temp.low.code() & 1) == 0); + MOZ_ASSERT(temp.low.code() + 1 == temp.high.code()); + + // We could avoid this pair requirement but in that case we would end up + // with two moves in the loop to preserve the loaded value in output. The + // prize would be less register spilling around this op since the pair + // requirement will tend to force more spilling. + + MOZ_ASSERT((output.low.code() & 1) == 0); + MOZ_ASSERT(output.low.code() + 1 == output.high.code()); + + Label again; + + SecondScratchRegisterScope scratch2(masm); + Register ptr = ComputePointerForAtomic(masm, mem, scratch2); + + masm.memoryBarrierBefore(sync); + + masm.bind(&again); + masm.as_ldrexd(output.low, output.high, ptr); + switch (op) { + case AtomicFetchAddOp: + masm.as_add(temp.low, output.low, O2Reg(value.low), SetCC); + masm.as_adc(temp.high, output.high, O2Reg(value.high)); + break; + case AtomicFetchSubOp: + masm.as_sub(temp.low, output.low, O2Reg(value.low), SetCC); + masm.as_sbc(temp.high, output.high, O2Reg(value.high)); + break; + case AtomicFetchAndOp: + masm.as_and(temp.low, output.low, O2Reg(value.low)); + masm.as_and(temp.high, output.high, O2Reg(value.high)); + break; + case AtomicFetchOrOp: + masm.as_orr(temp.low, output.low, O2Reg(value.low)); + masm.as_orr(temp.high, output.high, O2Reg(value.high)); + break; + case AtomicFetchXorOp: + masm.as_eor(temp.low, output.low, O2Reg(value.low)); + masm.as_eor(temp.high, output.high, O2Reg(value.high)); + break; + } + + ScratchRegisterScope scratch(masm); + + // Rd (temp) must differ from the two other arguments to strex. + masm.as_strexd(scratch, temp.low, temp.high, ptr); + masm.as_cmp(scratch, Imm8(1)); + masm.as_b(&again, MacroAssembler::Equal); + + masm.memoryBarrierAfter(sync); +} + +void +MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, Register64 value, + const Address& mem, Register64 temp, Register64 output) +{ + AtomicFetchOp64(*this, sync, op, value, mem, temp, output); +} + +void +MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, Register64 value, + const BaseIndex& mem, Register64 temp, Register64 output) +{ + AtomicFetchOp64(*this, sync, op, value, mem, temp, output); +} + + //}}} check_macroassembler_style void @@ -5839,7 +5782,7 @@ MacroAssemblerARM::wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register m type == Scalar::Int64; unsigned byteSize = access.byteSize(); - asMasm().memoryBarrier(access.barrierBefore()); + asMasm().memoryBarrierBefore(access.sync()); uint32_t framePushed = asMasm().framePushed(); BufferOffset load; @@ -5878,7 +5821,7 @@ MacroAssemblerARM::wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register m } } - asMasm().memoryBarrier(access.barrierAfter()); + asMasm().memoryBarrierAfter(access.sync()); } void @@ -5901,7 +5844,7 @@ MacroAssemblerARM::wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegist ma_add(Imm32(offset), ptr, scratch); } - asMasm().memoryBarrier(access.barrierBefore()); + asMasm().memoryBarrierAfter(access.sync()); uint32_t framePushed = asMasm().framePushed(); @@ -5937,7 +5880,7 @@ MacroAssemblerARM::wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegist } } - asMasm().memoryBarrier(access.barrierAfter()); + asMasm().memoryBarrierAfter(access.sync()); } void @@ -5981,7 +5924,7 @@ MacroAssemblerARM::wasmUnalignedLoadImpl(const wasm::MemoryAccessDesc& access, R MOZ_ASSERT(low != tmp); MOZ_ASSERT(low != ptr); - asMasm().memoryBarrier(access.barrierBefore()); + asMasm().memoryBarrierBefore(access.sync()); emitUnalignedLoad(isSigned, Min(byteSize, 4u), ptr, tmp, low); @@ -6011,7 +5954,7 @@ MacroAssemblerARM::wasmUnalignedLoadImpl(const wasm::MemoryAccessDesc& access, R } } - asMasm().memoryBarrier(access.barrierAfter()); + asMasm().memoryBarrierAfter(access.sync()); } void @@ -6037,7 +5980,7 @@ MacroAssemblerARM::wasmUnalignedStoreImpl(const wasm::MemoryAccessDesc& access, // Add memoryBase to ptr, so we can use base+index addressing in the byte loads. ma_add(memoryBase, ptr); - asMasm().memoryBarrier(access.barrierBefore()); + asMasm().memoryBarrierAfter(access.sync()); if (val64 != Register64::Invalid()) { if (val64.low != tmp) @@ -6062,7 +6005,7 @@ MacroAssemblerARM::wasmUnalignedStoreImpl(const wasm::MemoryAccessDesc& access, emitUnalignedStore(4, ptr, tmp, /* offset */ 4); } - asMasm().memoryBarrier(access.barrierAfter()); + asMasm().memoryBarrierAfter(access.sync()); } void diff --git a/js/src/jit/arm/MacroAssembler-arm.h b/js/src/jit/arm/MacroAssembler-arm.h index fd9c92d620a5..f75d26aec1fe 100644 --- a/js/src/jit/arm/MacroAssembler-arm.h +++ b/js/src/jit/arm/MacroAssembler-arm.h @@ -12,7 +12,6 @@ #include "jsopcode.h" #include "jit/arm/Assembler-arm.h" -#include "jit/AtomicOp.h" #include "jit/JitFrames.h" #include "jit/MoveResolver.h" @@ -1107,307 +1106,6 @@ class MacroAssemblerARMCompat : public MacroAssemblerARM ma_vmov(src, dest, cc); } - private: - template - Register computePointer(const T& src, Register r); - - template - void compareExchange(int nbytes, bool signExtend, const T& address, Register oldval, - Register newval, Register output); - - template - void atomicExchange(int nbytes, bool signExtend, const T& address, Register value, - Register output); - - template - void atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, const Imm32& value, - const T& address, Register flagTemp, Register output); - - template - void atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, const Register& value, - const T& address, Register flagTemp, Register output); - - template - void atomicEffectOp(int nbytes, AtomicOp op, const Imm32& value, const T& address, - Register flagTemp); - - template - void atomicEffectOp(int nbytes, AtomicOp op, const Register& value, const T& address, - Register flagTemp); - - template - void atomicFetchOp64(AtomicOp op, Register64 value, const T& mem, Register64 temp, - Register64 output); - - public: - // T in {Address,BaseIndex} - // S in {Imm32,Register} - - template - void compareExchange8SignExtend(const T& mem, Register oldval, Register newval, Register output) - { - compareExchange(1, true, mem, oldval, newval, output); - } - template - void compareExchange8ZeroExtend(const T& mem, Register oldval, Register newval, Register output) - { - compareExchange(1, false, mem, oldval, newval, output); - } - template - void compareExchange16SignExtend(const T& mem, Register oldval, Register newval, Register output) - { - compareExchange(2, true, mem, oldval, newval, output); - } - template - void compareExchange16ZeroExtend(const T& mem, Register oldval, Register newval, Register output) - { - compareExchange(2, false, mem, oldval, newval, output); - } - template - void compareExchange32(const T& mem, Register oldval, Register newval, Register output) { - compareExchange(4, false, mem, oldval, newval, output); - } - - template - void atomicExchange8SignExtend(const T& mem, Register value, Register output) - { - atomicExchange(1, true, mem, value, output); - } - template - void atomicExchange8ZeroExtend(const T& mem, Register value, Register output) - { - atomicExchange(1, false, mem, value, output); - } - template - void atomicExchange16SignExtend(const T& mem, Register value, Register output) - { - atomicExchange(2, true, mem, value, output); - } - template - void atomicExchange16ZeroExtend(const T& mem, Register value, Register output) - { - atomicExchange(2, false, mem, value, output); - } - template - void atomicExchange32(const T& mem, Register value, Register output) { - atomicExchange(4, false, mem, value, output); - } - - template - void atomicFetchAdd8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicFetchAdd8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicFetchAdd16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicFetchAdd16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicFetchAdd32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicAdd8(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(1, AtomicFetchAddOp, value, mem, flagTemp); - } - template - void atomicAdd16(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(2, AtomicFetchAddOp, value, mem, flagTemp); - } - template - void atomicAdd32(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(4, AtomicFetchAddOp, value, mem, flagTemp); - } - - template - void atomicFetchSub8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicFetchSub8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicFetchSub16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicFetchSub16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicFetchSub32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicSub8(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(1, AtomicFetchSubOp, value, mem, flagTemp); - } - template - void atomicSub16(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(2, AtomicFetchSubOp, value, mem, flagTemp); - } - template - void atomicSub32(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(4, AtomicFetchSubOp, value, mem, flagTemp); - } - - template - void atomicFetchAnd8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicFetchAnd8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicFetchAnd16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicFetchAnd16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicFetchAnd32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicAnd8(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(1, AtomicFetchAndOp, value, mem, flagTemp); - } - template - void atomicAnd16(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(2, AtomicFetchAndOp, value, mem, flagTemp); - } - template - void atomicAnd32(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(4, AtomicFetchAndOp, value, mem, flagTemp); - } - - template - void atomicFetchOr8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicFetchOr8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicFetchOr16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicFetchOr16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicFetchOr32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicOr8(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(1, AtomicFetchOrOp, value, mem, flagTemp); - } - template - void atomicOr16(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(2, AtomicFetchOrOp, value, mem, flagTemp); - } - template - void atomicOr32(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(4, AtomicFetchOrOp, value, mem, flagTemp); - } - - template - void atomicFetchXor8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicFetchXor8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicFetchXor16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicFetchXor16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicFetchXor32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicXor8(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(1, AtomicFetchXorOp, value, mem, flagTemp); - } - template - void atomicXor16(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(2, AtomicFetchXorOp, value, mem, flagTemp); - } - template - void atomicXor32(const S& value, const T& mem, Register flagTemp) { - atomicEffectOp(4, AtomicFetchXorOp, value, mem, flagTemp); - } - - // Temp should be invalid; output must be (even,odd) pair. - template - void atomicLoad64(const T& mem, Register64 temp, Register64 output); - - // Registers must be distinct; temp and output must be (even,odd) pairs. - template - void atomicFetchAdd64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchAddOp, value, mem, temp, output); - } - - // Registers must be distinct; temp and output must be (even,odd) pairs. - template - void atomicFetchSub64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchSubOp, value, mem, temp, output); - } - - // Registers must be distinct; temp and output must be (even,odd) pairs. - template - void atomicFetchAnd64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchAndOp, value, mem, temp, output); - } - - // Registers must be distinct; temp and output must be (even,odd) pairs. - template - void atomicFetchOr64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchOrOp, value, mem, temp, output); - } - - // Registers must be distinct; temp and output must be (even,odd) pairs. - template - void atomicFetchXor64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchXorOp, value, mem, temp, output); - } - - // Registers must be distinct; value and output must be (even,odd) pairs. - template - void atomicExchange64(const T& mem, Register64 value, Register64 output); - - // Registers must be distinct; replace and output must be (even,odd) pairs. - template - void compareExchange64(const T& mem, Register64 expect, Register64 replace, Register64 output); - - template - void compareExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, Register oldval, Register newval, - Register temp, AnyRegister output); - - template - void atomicExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, Register value, - Register temp, AnyRegister output); - inline void incrementInt32Value(const Address& addr); void cmp32(Register lhs, Imm32 rhs); diff --git a/js/src/jit/arm64/CodeGenerator-arm64.cpp b/js/src/jit/arm64/CodeGenerator-arm64.cpp index f0f83631d921..2ba87080cf9b 100644 --- a/js/src/jit/arm64/CodeGenerator-arm64.cpp +++ b/js/src/jit/arm64/CodeGenerator-arm64.cpp @@ -728,10 +728,10 @@ CodeGeneratorARM64::visitCompareExchangeTypedArrayElement(LCompareExchangeTypedA if (lir->index()->isConstant()) { Address dest(elements, ToInt32(lir->index()) * width); - masm.compareExchangeToTypedIntArray(arrayType, dest, oldval, newval, temp, output); + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, newval, temp, output); } else { BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - masm.compareExchangeToTypedIntArray(arrayType, dest, oldval, newval, temp, output); + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, newval, temp, output); } } @@ -749,10 +749,10 @@ CodeGeneratorARM64::visitAtomicExchangeTypedArrayElement(LAtomicExchangeTypedArr if (lir->index()->isConstant()) { Address dest(elements, ToInt32(lir->index()) * width); - masm.atomicExchangeToTypedIntArray(arrayType, dest, value, temp, output); + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp, output); } else { BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - masm.atomicExchangeToTypedIntArray(arrayType, dest, value, temp, output); + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp, output); } } diff --git a/js/src/jit/arm64/MacroAssembler-arm64.cpp b/js/src/jit/arm64/MacroAssembler-arm64.cpp index 38e35456a77a..5c8da0e90ee7 100644 --- a/js/src/jit/arm64/MacroAssembler-arm64.cpp +++ b/js/src/jit/arm64/MacroAssembler-arm64.cpp @@ -233,89 +233,6 @@ MacroAssemblerCompat::breakpoint() Brk((code++) & 0xffff); } -template -void -MacroAssemblerCompat::compareExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, - Register oldval, Register newval, - Register temp, AnyRegister output) -{ - switch (arrayType) { - case Scalar::Int8: - compareExchange8SignExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint8: - compareExchange8ZeroExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Int16: - compareExchange16SignExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint16: - compareExchange16ZeroExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Int32: - compareExchange32(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint32: - // At the moment, the code in MCallOptimize.cpp requires the output - // type to be double for uint32 arrays. See bug 1077305. - MOZ_ASSERT(output.isFloat()); - compareExchange32(mem, oldval, newval, temp); - convertUInt32ToDouble(temp, output.fpu()); - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -MacroAssemblerCompat::compareExchangeToTypedIntArray(Scalar::Type arrayType, const Address& mem, - Register oldval, Register newval, Register temp, - AnyRegister output); -template void -MacroAssemblerCompat::compareExchangeToTypedIntArray(Scalar::Type arrayType, const BaseIndex& mem, - Register oldval, Register newval, Register temp, - AnyRegister output); - -template -void -MacroAssemblerCompat::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, - Register value, Register temp, AnyRegister output) -{ - switch (arrayType) { - case Scalar::Int8: - atomicExchange8SignExtend(mem, value, output.gpr()); - break; - case Scalar::Uint8: - atomicExchange8ZeroExtend(mem, value, output.gpr()); - break; - case Scalar::Int16: - atomicExchange16SignExtend(mem, value, output.gpr()); - break; - case Scalar::Uint16: - atomicExchange16ZeroExtend(mem, value, output.gpr()); - break; - case Scalar::Int32: - atomicExchange32(mem, value, output.gpr()); - break; - case Scalar::Uint32: - // At the moment, the code in MCallOptimize.cpp requires the output - // type to be double for uint32 arrays. See bug 1077305. - MOZ_ASSERT(output.isFloat()); - atomicExchange32(mem, value, temp); - convertUInt32ToDouble(temp, output.fpu()); - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -MacroAssemblerCompat::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const Address& mem, - Register value, Register temp, AnyRegister output); -template void -MacroAssemblerCompat::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const BaseIndex& mem, - Register value, Register temp, AnyRegister output); - void MacroAssembler::reserveStack(uint32_t amount) { @@ -959,6 +876,105 @@ MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, Register output, MOZ_CRASH("NYI"); } +// ======================================================================== +// Primitive atomic operations. + +void +MacroAssembler::compareExchange(Scalar::Type type, const Synchronization& sync, const Address& mem, + Register oldval, Register newval, Register output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::compareExchange(Scalar::Type type, const Synchronization& sync, const BaseIndex& mem, + Register oldval, Register newval, Register output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicExchange(Scalar::Type type, const Synchronization& sync, const Address& mem, + Register value, Register output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicExchange(Scalar::Type type, const Synchronization& sync, const BaseIndex& mem, + Register value, Register output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register temp, Register output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicFetchOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register temp, Register output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicEffectOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const Address& mem, Register temp) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicEffectOp(Scalar::Type type, const Synchronization& sync, AtomicOp op, + Register value, const BaseIndex& mem, Register temp) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::compareExchange64(const Synchronization& sync, const Address& mem, Register64 expect, + Register64 replace, Register64 output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::compareExchange64(const Synchronization& sync, const BaseIndex& mem, Register64 expect, + Register64 replace, Register64 output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicExchange64(const Synchronization& sync, const Address& mem, Register64 value, Register64 output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicExchange64(const Synchronization& sync, const BaseIndex& mem, Register64 value, Register64 output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, Register64 value, const Address& mem, + Register64 temp, Register64 output) +{ + MOZ_CRASH("NYI"); +} + +void +MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op, Register64 value, const BaseIndex& mem, + Register64 temp, Register64 output) +{ + MOZ_CRASH("NYI"); +} + //}}} check_macroassembler_style } // namespace jit diff --git a/js/src/jit/arm64/MacroAssembler-arm64.h b/js/src/jit/arm64/MacroAssembler-arm64.h index f0bf7ef2315f..ce04e2f975b6 100644 --- a/js/src/jit/arm64/MacroAssembler-arm64.h +++ b/js/src/jit/arm64/MacroAssembler-arm64.h @@ -1874,311 +1874,6 @@ class MacroAssemblerCompat : public vixl::MacroAssembler return value; } - private: - template - void compareExchange(int nbytes, bool signExtend, const T& address, Register oldval, - Register newval, Register output) - { - MOZ_CRASH("compareExchange"); - } - - template - void atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, const Imm32& value, - const T& address, Register temp, Register output) - { - MOZ_CRASH("atomicFetchOp"); - } - - template - void atomicFetchOp(int nbytes, bool signExtend, AtomicOp op, const Register& value, - const T& address, Register temp, Register output) - { - MOZ_CRASH("atomicFetchOp"); - } - - template - void atomicEffectOp(int nbytes, AtomicOp op, const Register& value, const T& mem) { - MOZ_CRASH("atomicEffectOp"); - } - - template - void atomicEffectOp(int nbytes, AtomicOp op, const Imm32& value, const T& mem) { - MOZ_CRASH("atomicEffectOp"); - } - - template - void atomicFetchOp64(AtomicOp op, Register64 value, const T& mem, Register64 temp, - Register64 output) - { - MOZ_CRASH("AtomicFetchOp64"); - } - - public: - // T in {Address,BaseIndex} - // S in {Imm32,Register} - - template - void compareExchange8SignExtend(const T& mem, Register oldval, Register newval, Register output) - { - compareExchange(1, true, mem, oldval, newval, output); - } - template - void compareExchange8ZeroExtend(const T& mem, Register oldval, Register newval, Register output) - { - compareExchange(1, false, mem, oldval, newval, output); - } - template - void compareExchange16SignExtend(const T& mem, Register oldval, Register newval, Register output) - { - compareExchange(2, true, mem, oldval, newval, output); - } - template - void compareExchange16ZeroExtend(const T& mem, Register oldval, Register newval, Register output) - { - compareExchange(2, false, mem, oldval, newval, output); - } - template - void compareExchange32(const T& mem, Register oldval, Register newval, Register output) { - compareExchange(4, false, mem, oldval, newval, output); - } - template - void atomicExchange32(const T& mem, Register value, Register output) { - MOZ_CRASH("atomicExchang32"); - } - - template - void atomicExchange8ZeroExtend(const T& mem, Register value, Register output) { - MOZ_CRASH("atomicExchange8ZeroExtend"); - } - template - void atomicExchange8SignExtend(const T& mem, Register value, Register output) { - MOZ_CRASH("atomicExchange8SignExtend"); - } - - template - void atomicFetchAdd8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicFetchAdd8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicFetchAdd16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicFetchAdd16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchAddOp, value, mem, temp, output); - } - template - void atomicFetchAdd32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchAddOp, value, mem, temp, output); - } - - template - void atomicAdd8(const S& value, const T& mem) { - atomicEffectOp(1, AtomicFetchAddOp, value, mem); - } - template - void atomicAdd16(const S& value, const T& mem) { - atomicEffectOp(2, AtomicFetchAddOp, value, mem); - } - template - void atomicAdd32(const S& value, const T& mem) { - atomicEffectOp(4, AtomicFetchAddOp, value, mem); - } - - template - void atomicExchange16ZeroExtend(const T& mem, Register value, Register output) { - MOZ_CRASH("atomicExchange16ZeroExtend"); - } - template - void atomicExchange16SignExtend(const T& mem, Register value, Register output) { - MOZ_CRASH("atomicExchange16SignExtend"); - } - - template - void atomicFetchSub8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicFetchSub8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicFetchSub16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicFetchSub16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchSubOp, value, mem, temp, output); - } - template - void atomicFetchSub32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchSubOp, value, mem, temp, output); - } - - template - void atomicSub8(const S& value, const T& mem) { - atomicEffectOp(1, AtomicFetchSubOp, value, mem); - } - template - void atomicSub16(const S& value, const T& mem) { - atomicEffectOp(2, AtomicFetchSubOp, value, mem); - } - template - void atomicSub32(const S& value, const T& mem) { - atomicEffectOp(4, AtomicFetchSubOp, value, mem); - } - - template - void atomicFetchAnd8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicFetchAnd8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicFetchAnd16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicFetchAnd16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchAndOp, value, mem, temp, output); - } - template - void atomicFetchAnd32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchAndOp, value, mem, temp, output); - } - - template - void atomicAnd8(const S& value, const T& mem) { - atomicEffectOp(1, AtomicFetchAndOp, value, mem); - } - template - void atomicAnd16(const S& value, const T& mem) { - atomicEffectOp(2, AtomicFetchAndOp, value, mem); - } - template - void atomicAnd32(const S& value, const T& mem) { - atomicEffectOp(4, AtomicFetchAndOp, value, mem); - } - - template - void atomicFetchOr8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicFetchOr8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicFetchOr16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicFetchOr16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchOrOp, value, mem, temp, output); - } - template - void atomicFetchOr32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchOrOp, value, mem, temp, output); - } - - template - void atomicOr8(const S& value, const T& mem) { - atomicEffectOp(1, AtomicFetchOrOp, value, mem); - } - template - void atomicOr16(const S& value, const T& mem) { - atomicEffectOp(2, AtomicFetchOrOp, value, mem); - } - template - void atomicOr32(const S& value, const T& mem) { - atomicEffectOp(4, AtomicFetchOrOp, value, mem); - } - - template - void atomicFetchXor8SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, true, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicFetchXor8ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(1, false, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicFetchXor16SignExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, true, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicFetchXor16ZeroExtend(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(2, false, AtomicFetchXorOp, value, mem, temp, output); - } - template - void atomicFetchXor32(const S& value, const T& mem, Register temp, Register output) { - atomicFetchOp(4, false, AtomicFetchXorOp, value, mem, temp, output); - } - - template - void atomicXor8(const S& value, const T& mem) { - atomicEffectOp(1, AtomicFetchXorOp, value, mem); - } - template - void atomicXor16(const S& value, const T& mem) { - atomicEffectOp(2, AtomicFetchXorOp, value, mem); - } - template - void atomicXor32(const S& value, const T& mem) { - atomicEffectOp(4, AtomicFetchXorOp, value, mem); - } - - template - void atomicFetchAdd64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchAddOp, value, mem, temp, output); - } - - template - void atomicFetchSub64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchSubOp, value, mem, temp, output); - } - - template - void atomicFetchAnd64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchAndOp, value, mem, temp, output); - } - - template - void atomicFetchOr64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchOrOp, value, mem, temp, output); - } - - template - void atomicFetchXor64(Register64 value, const T& mem, Register64 temp, Register64 output) { - atomicFetchOp64(AtomicFetchXorOp, value, mem, temp, output); - } - - template - void atomicExchange64(const T& mem, Register64 src, Register64 output) { - MOZ_CRASH("atomicExchange64"); - } - - template - void compareExchange64(const T& mem, Register64 expected, Register64 replacement, - Register64 output) - { - MOZ_CRASH("compareExchange64"); - } - - template - void compareExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, Register oldval, Register newval, - Register temp, AnyRegister output); - - template - void atomicExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, Register value, - Register temp, AnyRegister output); - // Emit a BLR or NOP instruction. ToggleCall can be used to patch // this instruction. CodeOffset toggledCall(JitCode* target, bool enabled) { diff --git a/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp b/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp index bd23638d714e..58384c680bcb 100644 --- a/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp +++ b/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp @@ -1899,7 +1899,7 @@ CodeGeneratorMIPSShared::emitWasmLoad(T* lir) default: MOZ_CRASH("unexpected array type"); } - masm.memoryBarrier(mir->access().barrierBefore()); + masm.memoryBarrierBefore(mir->access().sync()); BaseIndex address(HeapReg, ptr, TimesOne); @@ -1919,7 +1919,7 @@ CodeGeneratorMIPSShared::emitWasmLoad(T* lir) isSigned ? SignExtend : ZeroExtend); } - masm.memoryBarrier(mir->access().barrierAfter()); + masm.memoryBarrierAfter(mir->access().sync()); return; } @@ -1938,7 +1938,8 @@ CodeGeneratorMIPSShared::emitWasmLoad(T* lir) isSigned ? SignExtend : ZeroExtend); } masm.append(mir->access(), masm.size() - 4, masm.framePushed()); - masm.memoryBarrier(mir->access().barrierAfter()); + + masm.memoryBarrierAfter(mir->access().sync()); } void @@ -1990,7 +1991,7 @@ CodeGeneratorMIPSShared::emitWasmStore(T* lir) default: MOZ_CRASH("unexpected array type"); } - masm.memoryBarrier(mir->access().barrierBefore()); + masm.memoryBarrierBefore(mir->access().sync()); BaseIndex address(HeapReg, ptr, TimesOne); @@ -2010,7 +2011,7 @@ CodeGeneratorMIPSShared::emitWasmStore(T* lir) isSigned ? SignExtend : ZeroExtend); } - masm.memoryBarrier(mir->access().barrierAfter()); + masm.memoryBarrierAfter(mir->access().sync()); return; } @@ -2032,7 +2033,8 @@ CodeGeneratorMIPSShared::emitWasmStore(T* lir) } // Only the last emitted instruction is a memory access. masm.append(mir->access(), masm.size() - 4, masm.framePushed()); - masm.memoryBarrier(mir->access().barrierAfter()); + + masm.memoryBarrierAfter(mir->access().sync()); } void diff --git a/js/src/jit/mips32/CodeGenerator-mips32.cpp b/js/src/jit/mips32/CodeGenerator-mips32.cpp index 1b08334a3fc6..54bec1046207 100644 --- a/js/src/jit/mips32/CodeGenerator-mips32.cpp +++ b/js/src/jit/mips32/CodeGenerator-mips32.cpp @@ -481,7 +481,7 @@ CodeGeneratorMIPS::emitWasmLoadI64(T* lir) default: MOZ_CRASH("unexpected array type"); } - masm.memoryBarrier(mir->access().barrierBefore()); + masm.memoryBarrierBefore(mir->access().sync()); MOZ_ASSERT(INT64LOW_OFFSET == 0); if (IsUnaligned(mir->access())) { @@ -503,10 +503,7 @@ CodeGeneratorMIPS::emitWasmLoadI64(T* lir) BaseIndex(HeapReg, ptr, TimesOne, INT64HIGH_OFFSET), temp, SizeWord, SignExtend); } - return; - } - - if (byteSize <= 4) { + } else if (byteSize <= 4) { masm.ma_load(output.low, BaseIndex(HeapReg, ptr, TimesOne), static_cast(8 * byteSize), isSigned ? SignExtend : ZeroExtend); masm.append(mir->access(), masm.size() - 4 , masm.framePushed()); @@ -522,7 +519,7 @@ CodeGeneratorMIPS::emitWasmLoadI64(T* lir) masm.append(mir->access(), masm.size() - 4 , masm.framePushed()); } - masm.memoryBarrier(mir->access().barrierAfter()); + masm.memoryBarrierAfter(mir->access().sync()); } void @@ -570,7 +567,7 @@ CodeGeneratorMIPS::emitWasmStoreI64(T* lir) default: MOZ_CRASH("unexpected array type"); } - masm.memoryBarrier(mir->access().barrierBefore()); + masm.memoryBarrierBefore(mir->access().sync()); MOZ_ASSERT(INT64LOW_OFFSET == 0); if (IsUnaligned(mir->access())) { @@ -587,10 +584,7 @@ CodeGeneratorMIPS::emitWasmStoreI64(T* lir) masm.ma_store_unaligned(mir->access(), value.low, BaseIndex(HeapReg, ptr, TimesOne), temp, SizeWord, ZeroExtend); } - return; - } - - if (byteSize <= 4) { + } else if (byteSize <= 4) { masm.ma_store(value.low, BaseIndex(HeapReg, ptr, TimesOne), static_cast(8 * byteSize)); masm.append(mir->access(), masm.size() - 4, masm.framePushed()); @@ -601,7 +595,7 @@ CodeGeneratorMIPS::emitWasmStoreI64(T* lir) masm.ma_store(value.low, BaseIndex(HeapReg, ptr, TimesOne), SizeWord); } - masm.memoryBarrier(mir->access().barrierAfter()); + masm.memoryBarrierAfter(mir->access().sync()); } void @@ -827,4 +821,4 @@ CodeGeneratorMIPS::setReturnDoubleRegs(LiveRegisterSet* regs) MOZ_ASSERT(ReturnFloat32Reg.code_ == ReturnDoubleReg.code_); regs->add(ReturnFloat32Reg); regs->add(ReturnDoubleReg); -} \ No newline at end of file +} diff --git a/js/src/jit/mips64/CodeGenerator-mips64.cpp b/js/src/jit/mips64/CodeGenerator-mips64.cpp index 4187496917ba..b58e4dd7f357 100644 --- a/js/src/jit/mips64/CodeGenerator-mips64.cpp +++ b/js/src/jit/mips64/CodeGenerator-mips64.cpp @@ -447,7 +447,7 @@ CodeGeneratorMIPS64::emitWasmLoadI64(T* lir) default: MOZ_CRASH("unexpected array type"); } - masm.memoryBarrier(mir->access().barrierBefore()); + masm.memoryBarrierBefore(mir->access().sync()); if (IsUnaligned(mir->access())) { Register temp = ToRegister(lir->getTemp(1)); @@ -455,14 +455,13 @@ CodeGeneratorMIPS64::emitWasmLoadI64(T* lir) masm.ma_load_unaligned(mir->access(), ToOutRegister64(lir).reg, BaseIndex(HeapReg, ptr, TimesOne), temp, static_cast(8 * byteSize), isSigned ? SignExtend : ZeroExtend); - return; + } else { + masm.ma_load(ToOutRegister64(lir).reg, BaseIndex(HeapReg, ptr, TimesOne), + static_cast(8 * byteSize), isSigned ? SignExtend : ZeroExtend); + masm.append(mir->access(), masm.size() - 4, masm.framePushed()); } - masm.ma_load(ToOutRegister64(lir).reg, BaseIndex(HeapReg, ptr, TimesOne), - static_cast(8 * byteSize), isSigned ? SignExtend : ZeroExtend); - masm.append(mir->access(), masm.size() - 4, masm.framePushed()); - - masm.memoryBarrier(mir->access().barrierAfter()); + masm.memoryBarrierAfter(mir->access().sync()); } void @@ -513,7 +512,7 @@ CodeGeneratorMIPS64::emitWasmStoreI64(T* lir) default: MOZ_CRASH("unexpected array type"); } - masm.memoryBarrier(mir->access().barrierBefore()); + masm.memoryBarrierBefore(mir->access().sync()); if (IsUnaligned(mir->access())) { Register temp = ToRegister(lir->getTemp(1)); @@ -521,13 +520,13 @@ CodeGeneratorMIPS64::emitWasmStoreI64(T* lir) masm.ma_store_unaligned(mir->access(), ToRegister64(lir->value()).reg, BaseIndex(HeapReg, ptr, TimesOne), temp, static_cast(8 * byteSize), isSigned ? SignExtend : ZeroExtend); - return; + } else { + masm.ma_store(ToRegister64(lir->value()).reg, BaseIndex(HeapReg, ptr, TimesOne), + static_cast(8 * byteSize), isSigned ? SignExtend : ZeroExtend); + masm.append(mir->access(), masm.size() - 4, masm.framePushed()); } - masm.ma_store(ToRegister64(lir->value()).reg, BaseIndex(HeapReg, ptr, TimesOne), - static_cast(8 * byteSize), isSigned ? SignExtend : ZeroExtend); - masm.append(mir->access(), masm.size() - 4, masm.framePushed()); - masm.memoryBarrier(mir->access().barrierAfter()); + masm.memoryBarrierAfter(mir->access().sync()); } void diff --git a/js/src/jit/mips64/MacroAssembler-mips64.h b/js/src/jit/mips64/MacroAssembler-mips64.h index 991d6f8cdcd8..6947455a331a 100644 --- a/js/src/jit/mips64/MacroAssembler-mips64.h +++ b/js/src/jit/mips64/MacroAssembler-mips64.h @@ -527,6 +527,8 @@ class MacroAssemblerMIPS64Compat : public MacroAssemblerMIPS64 public: // The following functions are exposed for use in platform-shared code. + // TODO: These are no longer used in platform code. + private: template void compareExchange8SignExtend(const T& mem, Register oldval, Register newval, Register valueTemp, Register offsetTemp, Register maskTemp, Register output) @@ -834,6 +836,7 @@ class MacroAssemblerMIPS64Compat : public MacroAssemblerMIPS64 atomicEffectOp(4, AtomicFetchXorOp, value, mem, flagTemp, valueTemp, offsetTemp, maskTemp); } + public: template void compareExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, Register oldval, Register newval, Register temp, Register valueTemp, Register offsetTemp, Register maskTemp, diff --git a/js/src/jit/none/MacroAssembler-none.h b/js/src/jit/none/MacroAssembler-none.h index abd0c810350a..aa833f92fd86 100644 --- a/js/src/jit/none/MacroAssembler-none.h +++ b/js/src/jit/none/MacroAssembler-none.h @@ -310,65 +310,6 @@ class MacroAssemblerNone : public Assembler template void computeEffectiveAddress(T, Register) { MOZ_CRASH(); } - template void compareExchange8SignExtend(const T& mem, Register oldval, Register newval, Register output) { MOZ_CRASH(); } - template void compareExchange8ZeroExtend(const T& mem, Register oldval, Register newval, Register output) { MOZ_CRASH(); } - template void compareExchange16SignExtend(const T& mem, Register oldval, Register newval, Register output) { MOZ_CRASH(); } - template void compareExchange16ZeroExtend(const T& mem, Register oldval, Register newval, Register output) { MOZ_CRASH(); } - template void compareExchange32(const T& mem, Register oldval, Register newval, Register output) { MOZ_CRASH(); } - template void atomicExchange8SignExtend(const T& mem, Register value, Register output) { MOZ_CRASH(); } - template void atomicExchange8ZeroExtend(const T& mem, Register value, Register output) { MOZ_CRASH(); } - template void atomicExchange16SignExtend(const T& mem, Register value, Register output) { MOZ_CRASH(); } - template void atomicExchange16ZeroExtend(const T& mem, Register value, Register output) { MOZ_CRASH(); } - template void atomicExchange32(const T& mem, Register value, Register output) { MOZ_CRASH(); } - template void atomicFetchAdd8SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchAdd8ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchAdd16SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchAdd16ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchAdd32(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicAdd8(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicAdd16(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicAdd32(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicFetchSub8SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchSub8ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchSub16SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchSub16ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchSub32(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicSub8(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicSub16(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicSub32(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicFetchAnd8SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchAnd8ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchAnd16SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchAnd16ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchAnd32(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicAnd8(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicAnd16(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicAnd32(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicFetchOr8SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchOr8ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchOr16SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchOr16ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchOr32(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicOr8(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicOr16(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicOr32(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicFetchXor8SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchXor8ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchXor16SignExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchXor16ZeroExtend(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicFetchXor32(const T& value, const S& mem, Register temp, Register output) { MOZ_CRASH(); } - template void atomicXor8(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicXor16(const T& value, const S& mem) { MOZ_CRASH(); } - template void atomicXor32(const T& value, const S& mem) { MOZ_CRASH(); } - - template void atomicFetchAdd64(Register64 value, const T& mem, Register64 temp, Register64 output) { MOZ_CRASH(); } - template void atomicFetchSub64(Register64 value, const T& mem, Register64 temp, Register64 output) { MOZ_CRASH(); } - template void atomicFetchAnd64(Register64 value, const T& mem, Register64 temp, Register64 output) { MOZ_CRASH(); } - template void atomicFetchOr64(Register64 value, const T& mem, Register64 temp, Register64 output) { MOZ_CRASH(); } - template void atomicFetchXor64(Register64 value, const T& mem, Register64 temp, Register64 output) { MOZ_CRASH(); } - template void atomicExchange64(const T& mem, Register64 src, Register64 output) { MOZ_CRASH(); } - template void compareExchange64(const T& mem, Register64 expect, Register64 replace, Register64 output) { MOZ_CRASH(); } - Register splitTagForTest(ValueOperand) { MOZ_CRASH(); } void boxDouble(FloatRegister, ValueOperand, FloatRegister) { MOZ_CRASH(); } diff --git a/js/src/jit/shared/Assembler-shared.h b/js/src/jit/shared/Assembler-shared.h index 61a3b1e3c207..65a0d9b1956c 100644 --- a/js/src/jit/shared/Assembler-shared.h +++ b/js/src/jit/shared/Assembler-shared.h @@ -735,22 +735,19 @@ class MemoryAccessDesc uint32_t align_; Scalar::Type type_; unsigned numSimdElems_; - jit::MemoryBarrierBits barrierBefore_; - jit::MemoryBarrierBits barrierAfter_; + jit::Synchronization sync_; mozilla::Maybe trapOffset_; public: explicit MemoryAccessDesc(Scalar::Type type, uint32_t align, uint32_t offset, const mozilla::Maybe& trapOffset, unsigned numSimdElems = 0, - jit::MemoryBarrierBits barrierBefore = jit::MembarNobits, - jit::MemoryBarrierBits barrierAfter = jit::MembarNobits) + const jit::Synchronization& sync = jit::Synchronization::None()) : offset_(offset), align_(align), type_(type), numSimdElems_(numSimdElems), - barrierBefore_(barrierBefore), - barrierAfter_(barrierAfter), + sync_(sync), trapOffset_(trapOffset) { MOZ_ASSERT(Scalar::isSimdType(type) == (numSimdElems > 0)); @@ -769,11 +766,10 @@ class MemoryAccessDesc : Scalar::byteSize(type()); } unsigned numSimdElems() const { MOZ_ASSERT(isSimd()); return numSimdElems_; } - jit::MemoryBarrierBits barrierBefore() const { return barrierBefore_; } - jit::MemoryBarrierBits barrierAfter() const { return barrierAfter_; } + const jit::Synchronization& sync() const { return sync_; } bool hasTrap() const { return !!trapOffset_; } BytecodeOffset trapOffset() const { return *trapOffset_; } - bool isAtomic() const { return (barrierBefore_ | barrierAfter_) != jit::MembarNobits; } + bool isAtomic() const { return !sync_.isNone(); } bool isSimd() const { return Scalar::isSimdType(type_); } bool isPlainAsmJS() const { return !hasTrap(); } diff --git a/js/src/jit/x64/CodeGenerator-x64.cpp b/js/src/jit/x64/CodeGenerator-x64.cpp index f27bdbc9da95..9da854d8e67f 100644 --- a/js/src/jit/x64/CodeGenerator-x64.cpp +++ b/js/src/jit/x64/CodeGenerator-x64.cpp @@ -400,7 +400,7 @@ CodeGeneratorX64::wasmStore(const wasm::MemoryAccessDesc& access, const LAllocat if (value->isConstant()) { MOZ_ASSERT(!access.isSimd()); - masm.memoryBarrier(access.barrierBefore()); + masm.memoryBarrierBefore(access.sync()); const MConstant* mir = value->toConstant(); Imm32 cst = Imm32(mir->type() == MIRType::Int32 ? mir->toInt32() : mir->toInt64()); @@ -432,7 +432,7 @@ CodeGeneratorX64::wasmStore(const wasm::MemoryAccessDesc& access, const LAllocat } masm.append(access, storeOffset, masm.framePushed()); - masm.memoryBarrier(access.barrierAfter()); + masm.memoryBarrierAfter(access.sync()); } else { masm.wasmStore(access, ToAnyRegister(value), dstAddr); } @@ -562,15 +562,11 @@ CodeGeneratorX64::visitWasmCompareExchangeHeap(LWasmCompareExchangeHeap* ins) if (accessType == Scalar::Int64) { MOZ_ASSERT(!mir->access().isPlainAsmJS()); - masm.compareExchange64(srcAddr, Register64(oldval), Register64(newval), - ToOutRegister64(ins)); + masm.compareExchange64(Synchronization::Full(), srcAddr, Register64(oldval), + Register64(newval), ToOutRegister64(ins)); } else { - masm.compareExchangeToTypedIntArray(accessType == Scalar::Uint32 ? Scalar::Int32 : accessType, - srcAddr, - oldval, - newval, - InvalidReg, - ToAnyRegister(ins->output())); + masm.compareExchange(accessType, Synchronization::Full(), srcAddr, oldval, newval, + ToRegister(ins->output())); } } @@ -589,13 +585,11 @@ CodeGeneratorX64::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) if (accessType == Scalar::Int64) { MOZ_ASSERT(!mir->access().isPlainAsmJS()); - masm.atomicExchange64(srcAddr, Register64(value), ToOutRegister64(ins)); + masm.atomicExchange64(Synchronization::Full(), srcAddr, Register64(value), + ToOutRegister64(ins)); } else { - masm.atomicExchangeToTypedIntArray(accessType == Scalar::Uint32 ? Scalar::Int32 : accessType, - srcAddr, - value, - InvalidReg, - ToAnyRegister(ins->output())); + masm.atomicExchange(accessType, Synchronization::Full(), srcAddr, value, + ToRegister(ins->output())); } } @@ -608,7 +602,7 @@ CodeGeneratorX64::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) Register ptr = ToRegister(ins->ptr()); const LAllocation* value = ins->value(); Register temp = ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp()); - AnyRegister output = ToAnyRegister(ins->output()); + Register output = ToRegister(ins->output()); MOZ_ASSERT(ins->addrTemp()->isBogusTemp()); Scalar::Type accessType = mir->access().type(); @@ -620,22 +614,15 @@ CodeGeneratorX64::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) if (accessType == Scalar::Int64) { Register64 val = Register64(ToRegister(value)); - Register64 out = Register64(output.gpr()); + Register64 out = Register64(output); Register64 tmp = Register64(temp); - switch (op) { - case AtomicFetchAddOp: masm.atomicFetchAdd64(val, srcAddr, tmp, out); break; - case AtomicFetchSubOp: masm.atomicFetchSub64(val, srcAddr, tmp, out); break; - case AtomicFetchAndOp: masm.atomicFetchAnd64(val, srcAddr, tmp, out); break; - case AtomicFetchOrOp: masm.atomicFetchOr64(val, srcAddr, tmp, out); break; - case AtomicFetchXorOp: masm.atomicFetchXor64(val, srcAddr, tmp, out); break; - default: MOZ_CRASH("Invalid typed array atomic operation"); - } + masm.atomicFetchOp64(Synchronization::Full(), op, val, srcAddr, tmp, out); } else if (value->isConstant()) { - atomicBinopToTypedIntArray(op, accessType, Imm32(ToInt32(value)), srcAddr, temp, InvalidReg, - output); + masm.atomicFetchOp(accessType, Synchronization::Full(), op, Imm32(ToInt32(value)), + srcAddr, temp, output); } else { - atomicBinopToTypedIntArray(op, accessType, ToRegister(value), srcAddr, temp, InvalidReg, - output); + masm.atomicFetchOp(accessType, Synchronization::Full(), op, ToRegister(value), + srcAddr, temp, output); } } @@ -656,23 +643,17 @@ CodeGeneratorX64::visitWasmAtomicBinopHeapForEffect(LWasmAtomicBinopHeapForEffec if (accessType == Scalar::Int64) { Register64 val = Register64(ToRegister(value)); - switch (op) { - case AtomicFetchAddOp: masm.atomicAdd64(srcAddr, val); break; - case AtomicFetchSubOp: masm.atomicSub64(srcAddr, val); break; - case AtomicFetchAndOp: masm.atomicAnd64(srcAddr, val); break; - case AtomicFetchOrOp: masm.atomicOr64(srcAddr, val); break; - case AtomicFetchXorOp: masm.atomicXor64(srcAddr, val); break; - default: MOZ_CRASH("Invalid typed array atomic operation"); - } + masm.atomicEffectOp64(Synchronization::Full(), op, val, srcAddr); } else if (value->isConstant()) { Imm32 c(0); if (value->toConstant()->type() == MIRType::Int64) c = Imm32(ToInt64(value)); else c = Imm32(ToInt32(value)); - atomicBinopToTypedIntArray(op, accessType, c, srcAddr); + masm.atomicEffectOp(accessType, Synchronization::Full(), op, c, srcAddr, InvalidReg); } else { - atomicBinopToTypedIntArray(op, accessType, ToRegister(value), srcAddr); + masm.atomicEffectOp(accessType, Synchronization::Full(), op, ToRegister(value), srcAddr, + InvalidReg); } } diff --git a/js/src/jit/x64/MacroAssembler-x64.cpp b/js/src/jit/x64/MacroAssembler-x64.cpp index 58d510713006..b72a52b6c3ee 100644 --- a/js/src/jit/x64/MacroAssembler-x64.cpp +++ b/js/src/jit/x64/MacroAssembler-x64.cpp @@ -722,7 +722,7 @@ MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, MIRType value void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr, AnyRegister out) { - memoryBarrier(access.barrierBefore()); + memoryBarrierBefore(access.sync()); size_t loadOffset = size(); switch (access.type()) { @@ -784,13 +784,13 @@ MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr, } append(access, loadOffset, framePushed()); - memoryBarrier(access.barrierAfter()); + memoryBarrierAfter(access.sync()); } void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Operand srcAddr, Register64 out) { - memoryBarrier(access.barrierBefore()); + memoryBarrierBefore(access.sync()); MOZ_ASSERT(!access.isSimd()); @@ -831,13 +831,13 @@ MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Operand srcAdd } append(access, loadOffset, framePushed()); - memoryBarrier(access.barrierAfter()); + memoryBarrierAfter(access.sync()); } void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Operand dstAddr) { - memoryBarrier(access.barrierBefore()); + memoryBarrierBefore(access.sync()); size_t storeOffset = size(); switch (access.type()) { @@ -896,7 +896,7 @@ MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister valu } append(access, storeOffset, framePushed()); - memoryBarrier(access.barrierAfter()); + memoryBarrierAfter(access.sync()); } void @@ -923,4 +923,102 @@ MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, Register output j(Assembler::Above, oolEntry); } +// ======================================================================== +// Primitive atomic operations. + +void +MacroAssembler::compareExchange64(const Synchronization&, const Address& mem, Register64 expected, + Register64 replacement, Register64 output) +{ + MOZ_ASSERT(output.reg == rax); + if (expected != output) + movq(expected.reg, output.reg); + lock_cmpxchgq(replacement.reg, Operand(mem)); +} + +void +MacroAssembler::compareExchange64(const Synchronization&, const BaseIndex& mem, Register64 expected, + Register64 replacement, Register64 output) +{ + MOZ_ASSERT(output.reg == rax); + if (expected != output) + movq(expected.reg, output.reg); + lock_cmpxchgq(replacement.reg, Operand(mem)); +} + +void +MacroAssembler::atomicExchange64(const Synchronization& sync, const Address& mem, Register64 value, Register64 output) +{ + if (value != output) + movq(value.reg, output.reg); + xchgq(output.reg, Operand(mem)); +} + +void +MacroAssembler::atomicExchange64(const Synchronization& sync, const BaseIndex& mem, Register64 value, Register64 output) +{ + if (value != output) + movq(value.reg, output.reg); + xchgq(output.reg, Operand(mem)); +} + +template +static void +AtomicFetchOp64(MacroAssembler& masm, AtomicOp op, Register value, const T& mem, Register temp, + Register output) +{ + if (op == AtomicFetchAddOp) { + if (value != output) + masm.movq(value, output); + masm.lock_xaddq(output, Operand(mem)); + } else if (op == AtomicFetchSubOp) { + if (value != output) + masm.movq(value, output); + masm.negq(output); + masm.lock_xaddq(output, Operand(mem)); + } else { + Label again; + MOZ_ASSERT(output == rax); + masm.movq(Operand(mem), rax); + masm.bind(&again); + masm.movq(rax, temp); + switch (op) { + case AtomicFetchAndOp: masm.andq(value, temp); break; + case AtomicFetchOrOp: masm.orq(value, temp); break; + case AtomicFetchXorOp: masm.xorq(value, temp); break; + default: MOZ_CRASH(); + } + masm.lock_cmpxchgq(temp, Operand(mem)); + masm.j(MacroAssembler::NonZero, &again); + } +} + +void +MacroAssembler::atomicFetchOp64(const Synchronization&, AtomicOp op, Register64 value, + const Address& mem, Register64 temp, Register64 output) +{ + AtomicFetchOp64(*this, op, value.reg, mem, temp.reg, output.reg); +} + +void +MacroAssembler::atomicFetchOp64(const Synchronization&, AtomicOp op, Register64 value, + const BaseIndex& mem, Register64 temp, Register64 output) +{ + AtomicFetchOp64(*this, op, value.reg, mem, temp.reg, output.reg); +} + +void +MacroAssembler::atomicEffectOp64(const Synchronization&, AtomicOp op, Register64 value, + const BaseIndex& mem) +{ + switch (op) { + case AtomicFetchAddOp: lock_addq(value.reg, Operand(mem)); break; + case AtomicFetchSubOp: lock_subq(value.reg, Operand(mem)); break; + case AtomicFetchAndOp: lock_andq(value.reg, Operand(mem)); break; + case AtomicFetchOrOp: lock_orq(value.reg, Operand(mem)); break; + case AtomicFetchXorOp: lock_xorq(value.reg, Operand(mem)); break; + default: MOZ_CRASH(); + } +} + //}}} check_macroassembler_style diff --git a/js/src/jit/x64/MacroAssembler-x64.h b/js/src/jit/x64/MacroAssembler-x64.h index eef79ce581b5..c24439da513c 100644 --- a/js/src/jit/x64/MacroAssembler-x64.h +++ b/js/src/jit/x64/MacroAssembler-x64.h @@ -643,88 +643,6 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared storePtr(ImmWord(imm.value), address); } - template - void atomicFetchAdd64(Register64 src, const T& mem, Register64 temp, Register64 output) { - MOZ_ASSERT(temp.reg == InvalidReg); - if (src != output) - movq(src.reg, output.reg); - lock_xaddq(output.reg, Operand(mem)); - } - template - void atomicAdd64(const T& mem, Register64 value) { - lock_addq(value.reg, Operand(mem)); - } - - template - void atomicFetchSub64(Register64 src, const T& mem, Register64 temp, Register64 output) { - MOZ_ASSERT(temp.reg == InvalidReg); - if (src != output) - movq(src.reg, output.reg); - negq(output.reg); - lock_xaddq(output.reg, Operand(mem)); - } - template - void atomicSub64(const T& mem, Register64 value) { - lock_subq(value.reg, Operand(mem)); - } - - // requires output == rax -#define ATOMIC_BITOP_BODY(OP) \ - MOZ_ASSERT(output.reg == rax); \ - movq(Operand(mem), rax); \ - Label again; \ - bind(&again); \ - movq(rax, temp.reg); \ - OP(src.reg, temp.reg); \ - lock_cmpxchgq(temp.reg, Operand(mem)); \ - j(NonZero, &again); - - template - void atomicFetchAnd64(const S& src, const T& mem, Register64 temp, Register64 output) { - ATOMIC_BITOP_BODY(andq) - } - template - void atomicAnd64(const T& mem, Register64 value) { - lock_andq(value.reg, Operand(mem)); - } - - template - void atomicFetchOr64(const S& src, const T& mem, Register64 temp, Register64 output) { - ATOMIC_BITOP_BODY(orq) - } - template - void atomicOr64(const T& mem, Register64 value) { - lock_orq(value.reg, Operand(mem)); - } - - template - void atomicFetchXor64(const S& src, const T& mem, Register64 temp, Register64 output) { - ATOMIC_BITOP_BODY(xorq) - } - template - void atomicXor64(const T& mem, Register64 value) { - lock_xorq(value.reg, Operand(mem)); - } - -#undef ATOMIC_BITOP_BODY - - template - void atomicExchange64(const T& mem, Register64 src, Register64 output) { - if (src != output) - movq(src.reg, output.reg); - xchgq(output.reg, Operand(mem)); - } - - template - void compareExchange64(const T& mem, Register64 expected, Register64 replacement, - Register64 output) - { - MOZ_ASSERT(output.reg == rax); - if (expected != output) - movq(expected.reg, output.reg); - lock_cmpxchgq(replacement.reg, Operand(mem)); - } - void splitTag(Register src, Register dest) { if (src != dest) movq(src, dest); diff --git a/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp b/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp index 7b9fa278ac25..d93f9eaddd67 100644 --- a/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp +++ b/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp @@ -4229,10 +4229,10 @@ CodeGeneratorX86Shared::visitCompareExchangeTypedArrayElement(LCompareExchangeTy if (lir->index()->isConstant()) { Address dest(elements, ToInt32(lir->index()) * width); - masm.compareExchangeToTypedIntArray(arrayType, dest, oldval, newval, temp, output); + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, newval, temp, output); } else { BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - masm.compareExchangeToTypedIntArray(arrayType, dest, oldval, newval, temp, output); + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, newval, temp, output); } } @@ -4250,273 +4250,26 @@ CodeGeneratorX86Shared::visitAtomicExchangeTypedArrayElement(LAtomicExchangeType if (lir->index()->isConstant()) { Address dest(elements, ToInt32(lir->index()) * width); - masm.atomicExchangeToTypedIntArray(arrayType, dest, value, temp, output); + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp, output); } else { BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - masm.atomicExchangeToTypedIntArray(arrayType, dest, value, temp, output); + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp, output); } } -template -void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value, - const T& mem, Register temp1, Register temp2, AnyRegister output) -{ - switch (arrayType) { - case Scalar::Int8: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd8SignExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub8SignExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd8SignExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr8SignExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor8SignExtend(value, mem, temp1, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Uint8: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd8ZeroExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub8ZeroExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd8ZeroExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr8ZeroExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor8ZeroExtend(value, mem, temp1, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Int16: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd16SignExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub16SignExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd16SignExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr16SignExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor16SignExtend(value, mem, temp1, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Uint16: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd16ZeroExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub16ZeroExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd16ZeroExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr16ZeroExtend(value, mem, temp1, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor16ZeroExtend(value, mem, temp1, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Int32: - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd32(value, mem, temp1, output.gpr()); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub32(value, mem, temp1, output.gpr()); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd32(value, mem, temp1, output.gpr()); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr32(value, mem, temp1, output.gpr()); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor32(value, mem, temp1, output.gpr()); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Uint32: - // At the moment, the code in MCallOptimize.cpp requires the output - // type to be double for uint32 arrays. See bug 1077305. - MOZ_ASSERT(output.isFloat()); - switch (op) { - case AtomicFetchAddOp: - masm.atomicFetchAdd32(value, mem, InvalidReg, temp1); - break; - case AtomicFetchSubOp: - masm.atomicFetchSub32(value, mem, InvalidReg, temp1); - break; - case AtomicFetchAndOp: - masm.atomicFetchAnd32(value, mem, temp2, temp1); - break; - case AtomicFetchOrOp: - masm.atomicFetchOr32(value, mem, temp2, temp1); - break; - case AtomicFetchXorOp: - masm.atomicFetchXor32(value, mem, temp2, temp1); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - masm.convertUInt32ToDouble(temp1, output.fpu()); - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Imm32& value, const Address& mem, - Register temp1, Register temp2, AnyRegister output); -template void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Imm32& value, const BaseIndex& mem, - Register temp1, Register temp2, AnyRegister output); -template void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Register& value, const Address& mem, - Register temp1, Register temp2, AnyRegister output); -template void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Register& value, const BaseIndex& mem, - Register temp1, Register temp2, AnyRegister output); - -// Binary operation for effect, result discarded. -template -void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value, - const T& mem) -{ - switch (arrayType) { - case Scalar::Int8: - case Scalar::Uint8: - switch (op) { - case AtomicFetchAddOp: - masm.atomicAdd8(value, mem); - break; - case AtomicFetchSubOp: - masm.atomicSub8(value, mem); - break; - case AtomicFetchAndOp: - masm.atomicAnd8(value, mem); - break; - case AtomicFetchOrOp: - masm.atomicOr8(value, mem); - break; - case AtomicFetchXorOp: - masm.atomicXor8(value, mem); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Int16: - case Scalar::Uint16: - switch (op) { - case AtomicFetchAddOp: - masm.atomicAdd16(value, mem); - break; - case AtomicFetchSubOp: - masm.atomicSub16(value, mem); - break; - case AtomicFetchAndOp: - masm.atomicAnd16(value, mem); - break; - case AtomicFetchOrOp: - masm.atomicOr16(value, mem); - break; - case AtomicFetchXorOp: - masm.atomicXor16(value, mem); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - case Scalar::Int32: - case Scalar::Uint32: - switch (op) { - case AtomicFetchAddOp: - masm.atomicAdd32(value, mem); - break; - case AtomicFetchSubOp: - masm.atomicSub32(value, mem); - break; - case AtomicFetchAndOp: - masm.atomicAnd32(value, mem); - break; - case AtomicFetchOrOp: - masm.atomicOr32(value, mem); - break; - case AtomicFetchXorOp: - masm.atomicXor32(value, mem); - break; - default: - MOZ_CRASH("Invalid typed array atomic operation"); - } - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Imm32& value, const Address& mem); -template void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Imm32& value, const BaseIndex& mem); -template void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Register& value, const Address& mem); -template void -CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, - const Register& value, const BaseIndex& mem); - - template static inline void -AtomicBinopToTypedArray(CodeGeneratorX86Shared* cg, AtomicOp op, - Scalar::Type arrayType, const LAllocation* value, const T& mem, - Register temp1, Register temp2, AnyRegister output) +AtomicBinopToTypedArray(MacroAssembler& masm, AtomicOp op, Scalar::Type arrayType, + const LAllocation* value, const T& mem, Register temp1, + Register temp2, AnyRegister output) { - if (value->isConstant()) - cg->atomicBinopToTypedIntArray(op, arrayType, Imm32(ToInt32(value)), mem, temp1, temp2, output); - else - cg->atomicBinopToTypedIntArray(op, arrayType, ToRegister(value), mem, temp1, temp2, output); + if (value->isConstant()) { + masm.atomicFetchOpJS(arrayType, Synchronization::Full(), op, Imm32(ToInt32(value)), mem, + temp1, temp2, output); + } else { + masm.atomicFetchOpJS(arrayType, Synchronization::Full(), op, ToRegister(value), mem, temp1, + temp2, output); + } } void @@ -4535,22 +4288,25 @@ CodeGeneratorX86Shared::visitAtomicTypedArrayElementBinop(LAtomicTypedArrayEleme if (lir->index()->isConstant()) { Address mem(elements, ToInt32(lir->index()) * width); - AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem, temp1, temp2, output); + AtomicBinopToTypedArray(masm, lir->mir()->operation(), arrayType, value, mem, temp1, temp2, output); } else { BaseIndex mem(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem, temp1, temp2, output); + AtomicBinopToTypedArray(masm, lir->mir()->operation(), arrayType, value, mem, temp1, temp2, output); } } template static inline void -AtomicBinopToTypedArray(CodeGeneratorX86Shared* cg, AtomicOp op, - Scalar::Type arrayType, const LAllocation* value, const T& mem) +AtomicBinopToTypedArray(MacroAssembler& masm, Scalar::Type arrayType, AtomicOp op, + const LAllocation* value, const T& mem) { - if (value->isConstant()) - cg->atomicBinopToTypedIntArray(op, arrayType, Imm32(ToInt32(value)), mem); - else - cg->atomicBinopToTypedIntArray(op, arrayType, ToRegister(value), mem); + if (value->isConstant()) { + masm.atomicEffectOpJS(arrayType, Synchronization::Full(), op, Imm32(ToInt32(value)), mem, + InvalidReg); + } else { + masm.atomicEffectOpJS(arrayType, Synchronization::Full(), op, ToRegister(value), mem, + InvalidReg); + } } void @@ -4565,10 +4321,10 @@ CodeGeneratorX86Shared::visitAtomicTypedArrayElementBinopForEffect(LAtomicTypedA if (lir->index()->isConstant()) { Address mem(elements, ToInt32(lir->index()) * width); - AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem); + AtomicBinopToTypedArray(masm, arrayType, lir->mir()->operation(), value, mem); } else { BaseIndex mem(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); - AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem); + AtomicBinopToTypedArray(masm, arrayType, lir->mir()->operation(), value, mem); } } diff --git a/js/src/jit/x86-shared/CodeGenerator-x86-shared.h b/js/src/jit/x86-shared/CodeGenerator-x86-shared.h index fac6ec8584ef..daca6fe61d5a 100644 --- a/js/src/jit/x86-shared/CodeGenerator-x86-shared.h +++ b/js/src/jit/x86-shared/CodeGenerator-x86-shared.h @@ -322,15 +322,6 @@ class CodeGeneratorX86Shared : public CodeGeneratorShared void visitOutOfLineSimdFloatToIntCheck(OutOfLineSimdFloatToIntCheck* ool); void generateInvalidateEpilogue(); - // Generating a result. - template - void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value, - const T& mem, Register temp1, Register temp2, AnyRegister output); - - // Generating no result. - template - void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value, const T& mem); - void setReturnDoubleRegs(LiveRegisterSet* regs); void canonicalizeIfDeterministic(Scalar::Type type, const LAllocation* value); diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp index 2f321070f994..3be5d040e304 100644 --- a/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp +++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp @@ -144,89 +144,6 @@ MacroAssemblerX86Shared::asMasm() const return *static_cast(this); } -template -void -MacroAssemblerX86Shared::compareExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, - Register oldval, Register newval, - Register temp, AnyRegister output) -{ - switch (arrayType) { - case Scalar::Int8: - compareExchange8SignExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint8: - compareExchange8ZeroExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Int16: - compareExchange16SignExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint16: - compareExchange16ZeroExtend(mem, oldval, newval, output.gpr()); - break; - case Scalar::Int32: - compareExchange32(mem, oldval, newval, output.gpr()); - break; - case Scalar::Uint32: - // At the moment, the code in MCallOptimize.cpp requires the output - // type to be double for uint32 arrays. See bug 1077305. - MOZ_ASSERT(output.isFloat()); - compareExchange32(mem, oldval, newval, temp); - asMasm().convertUInt32ToDouble(temp, output.fpu()); - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -MacroAssemblerX86Shared::compareExchangeToTypedIntArray(Scalar::Type arrayType, const Address& mem, - Register oldval, Register newval, Register temp, - AnyRegister output); -template void -MacroAssemblerX86Shared::compareExchangeToTypedIntArray(Scalar::Type arrayType, const BaseIndex& mem, - Register oldval, Register newval, Register temp, - AnyRegister output); - -template -void -MacroAssemblerX86Shared::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, - Register value, Register temp, AnyRegister output) -{ - switch (arrayType) { - case Scalar::Int8: - atomicExchange8SignExtend(mem, value, output.gpr()); - break; - case Scalar::Uint8: - atomicExchange8ZeroExtend(mem, value, output.gpr()); - break; - case Scalar::Int16: - atomicExchange16SignExtend(mem, value, output.gpr()); - break; - case Scalar::Uint16: - atomicExchange16ZeroExtend(mem, value, output.gpr()); - break; - case Scalar::Int32: - atomicExchange32(mem, value, output.gpr()); - break; - case Scalar::Uint32: - // At the moment, the code in MCallOptimize.cpp requires the output - // type to be double for uint32 arrays. See bug 1077305. - MOZ_ASSERT(output.isFloat()); - atomicExchange32(mem, value, temp); - asMasm().convertUInt32ToDouble(temp, output.fpu()); - break; - default: - MOZ_CRASH("Invalid typed array type"); - } -} - -template void -MacroAssemblerX86Shared::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const Address& mem, - Register value, Register temp, AnyRegister output); -template void -MacroAssemblerX86Shared::atomicExchangeToTypedIntArray(Scalar::Type arrayType, const BaseIndex& mem, - Register value, Register temp, AnyRegister output); - template T* MacroAssemblerX86Shared::getConstant(const typename T::Pod& value, Map& map, @@ -888,4 +805,381 @@ MacroAssembler::outOfLineWasmTruncateFloat32ToInt64(FloatRegister input, bool is jump(rejoin); } +// ======================================================================== +// Primitive atomic operations. + +static void +ExtendTo32(MacroAssembler& masm, Scalar::Type type, Register r) +{ + switch (Scalar::byteSize(type)) { + case 1: + if (Scalar::isSignedIntType(type)) + masm.movsbl(r, r); + else + masm.movzbl(r, r); + break; + case 2: + if (Scalar::isSignedIntType(type)) + masm.movswl(r, r); + else + masm.movzwl(r, r); + break; + default: + break; + } +} + +static inline void +CheckBytereg(Register r) { +#ifdef DEBUG + AllocatableGeneralRegisterSet byteRegs(Registers::SingleByteRegs); + MOZ_ASSERT(byteRegs.has(r)); +#endif +} + +static inline void +CheckBytereg(Imm32 r) { + // Nothing +} + +template +static void +CompareExchange(MacroAssembler& masm, Scalar::Type type, const T& mem, Register oldval, + Register newval, Register output) +{ + MOZ_ASSERT(output == eax); + + if (oldval != output) + masm.movl(oldval, output); + + switch (Scalar::byteSize(type)) { + case 1: + CheckBytereg(newval); + masm.lock_cmpxchgb(newval, Operand(mem)); + break; + case 2: + masm.lock_cmpxchgw(newval, Operand(mem)); + break; + case 4: + masm.lock_cmpxchgl(newval, Operand(mem)); + break; + } + + ExtendTo32(masm, type, output); +} + +void +MacroAssembler::compareExchange(Scalar::Type type, const Synchronization&, const Address& mem, + Register oldval, Register newval, Register output) +{ + CompareExchange(*this, type, mem, oldval, newval, output); +} + +void +MacroAssembler::compareExchange(Scalar::Type type, const Synchronization&, const BaseIndex& mem, + Register oldval, Register newval, Register output) +{ + CompareExchange(*this, type, mem, oldval, newval, output); +} + +template +static void +AtomicExchange(MacroAssembler& masm, Scalar::Type type, const T& mem, Register value, + Register output) + +{ + if (value != output) + masm.movl(value, output); + + switch (Scalar::byteSize(type)) { + case 1: + CheckBytereg(output); + masm.xchgb(output, Operand(mem)); + break; + case 2: + masm.xchgw(output, Operand(mem)); + break; + case 4: + masm.xchgl(output, Operand(mem)); + break; + default: + MOZ_CRASH("Invalid"); + } + ExtendTo32(masm, type, output); +} + +void +MacroAssembler::atomicExchange(Scalar::Type type, const Synchronization&, const Address& mem, + Register value, Register output) +{ + AtomicExchange(*this, type, mem, value, output); +} + +void +MacroAssembler::atomicExchange(Scalar::Type type, const Synchronization&, const BaseIndex& mem, + Register value, Register output) +{ + AtomicExchange(*this, type, mem, value, output); +} + +static void +SetupValue(MacroAssembler& masm, AtomicOp op, Imm32 src, Register output) { + if (op == AtomicFetchSubOp) + masm.movl(Imm32(-src.value), output); + else + masm.movl(src, output); +} + +static void +SetupValue(MacroAssembler& masm, AtomicOp op, Register src, Register output) { + if (src != output) + masm.movl(src, output); + if (op == AtomicFetchSubOp) + masm.negl(output); +} + +template +static void +AtomicFetchOp(MacroAssembler& masm, Scalar::Type arrayType, AtomicOp op, V value, + const T& mem, Register temp, Register output) +{ +// Note value can be an Imm or a Register. + +#define ATOMIC_BITOP_BODY(LOAD, OP, LOCK_CMPXCHG) \ + do { \ + MOZ_ASSERT(output != temp); \ + MOZ_ASSERT(output == eax); \ + masm.LOAD(Operand(mem), eax); \ + Label again; \ + masm.bind(&again); \ + masm.movl(eax, temp); \ + masm.OP(value, temp); \ + masm.LOCK_CMPXCHG(temp, Operand(mem)); \ + masm.j(MacroAssembler::NonZero, &again); \ + } while (0) + + MOZ_ASSERT_IF(op == AtomicFetchAddOp || op == AtomicFetchSubOp, temp == InvalidReg); + + switch (Scalar::byteSize(arrayType)) { + case 1: + CheckBytereg(value); + CheckBytereg(output); + switch (op) { + case AtomicFetchAddOp: + case AtomicFetchSubOp: + SetupValue(masm, op, value, output); + masm.lock_xaddb(output, Operand(mem)); + break; + case AtomicFetchAndOp: + CheckBytereg(temp); + ATOMIC_BITOP_BODY(movb, andl, lock_cmpxchgb); + break; + case AtomicFetchOrOp: + CheckBytereg(temp); + ATOMIC_BITOP_BODY(movb, orl, lock_cmpxchgb); + break; + case AtomicFetchXorOp: + CheckBytereg(temp); + ATOMIC_BITOP_BODY(movb, xorl, lock_cmpxchgb); + break; + default: + MOZ_CRASH(); + } + break; + case 2: + switch (op) { + case AtomicFetchAddOp: + case AtomicFetchSubOp: + SetupValue(masm, op, value, output); + masm.lock_xaddw(output, Operand(mem)); + break; + case AtomicFetchAndOp: + ATOMIC_BITOP_BODY(movw, andl, lock_cmpxchgw); + break; + case AtomicFetchOrOp: + ATOMIC_BITOP_BODY(movw, orl, lock_cmpxchgw); + break; + case AtomicFetchXorOp: + ATOMIC_BITOP_BODY(movw, xorl, lock_cmpxchgw); + break; + default: + MOZ_CRASH(); + } + break; + case 4: + switch (op) { + case AtomicFetchAddOp: + case AtomicFetchSubOp: + SetupValue(masm, op, value, output); + masm.lock_xaddl(output, Operand(mem)); + break; + case AtomicFetchAndOp: + ATOMIC_BITOP_BODY(movl, andl, lock_cmpxchgl); + break; + case AtomicFetchOrOp: + ATOMIC_BITOP_BODY(movl, orl, lock_cmpxchgl); + break; + case AtomicFetchXorOp: + ATOMIC_BITOP_BODY(movl, xorl, lock_cmpxchgl); + break; + default: + MOZ_CRASH(); + } + break; + } + ExtendTo32(masm, arrayType, output); + +#undef ATOMIC_BITOP_BODY +} + +void +MacroAssembler::atomicFetchOp(Scalar::Type arrayType, const Synchronization&, AtomicOp op, + Register value, const BaseIndex& mem, Register temp, Register output) +{ + AtomicFetchOp(*this, arrayType, op, value, mem, temp, output); +} + +void +MacroAssembler::atomicFetchOp(Scalar::Type arrayType, const Synchronization&, AtomicOp op, + Register value, const Address& mem, Register temp, Register output) +{ + AtomicFetchOp(*this, arrayType, op, value, mem, temp, output); +} + +void +MacroAssembler::atomicFetchOp(Scalar::Type arrayType, const Synchronization&, AtomicOp op, + Imm32 value, const BaseIndex& mem, Register temp, Register output) +{ + AtomicFetchOp(*this, arrayType, op, value, mem, temp, output); +} + +void +MacroAssembler::atomicFetchOp(Scalar::Type arrayType, const Synchronization&, AtomicOp op, + Imm32 value, const Address& mem, Register temp, Register output) +{ + AtomicFetchOp(*this, arrayType, op, value, mem, temp, output); +} + +template +static void +AtomicEffectOp(MacroAssembler& masm, Scalar::Type arrayType, AtomicOp op, V value, const T& mem) +{ + switch (Scalar::byteSize(arrayType)) { + case 1: + switch (op) { + case AtomicFetchAddOp: masm.lock_addb(value, Operand(mem)); break; + case AtomicFetchSubOp: masm.lock_subb(value, Operand(mem)); break; + case AtomicFetchAndOp: masm.lock_andb(value, Operand(mem)); break; + case AtomicFetchOrOp: masm.lock_orb(value, Operand(mem)); break; + case AtomicFetchXorOp: masm.lock_xorb(value, Operand(mem)); break; + default: + MOZ_CRASH(); + } + break; + case 2: + switch (op) { + case AtomicFetchAddOp: masm.lock_addw(value, Operand(mem)); break; + case AtomicFetchSubOp: masm.lock_subw(value, Operand(mem)); break; + case AtomicFetchAndOp: masm.lock_andw(value, Operand(mem)); break; + case AtomicFetchOrOp: masm.lock_orw(value, Operand(mem)); break; + case AtomicFetchXorOp: masm.lock_xorw(value, Operand(mem)); break; + default: + MOZ_CRASH(); + } + break; + case 4: + switch (op) { + case AtomicFetchAddOp: masm.lock_addl(value, Operand(mem)); break; + case AtomicFetchSubOp: masm.lock_subl(value, Operand(mem)); break; + case AtomicFetchAndOp: masm.lock_andl(value, Operand(mem)); break; + case AtomicFetchOrOp: masm.lock_orl(value, Operand(mem)); break; + case AtomicFetchXorOp: masm.lock_xorl(value, Operand(mem)); break; + default: + MOZ_CRASH(); + } + break; + default: + MOZ_CRASH(); + } +} + +void +MacroAssembler::atomicEffectOp(Scalar::Type arrayType, const Synchronization&, AtomicOp op, + Register value, const BaseIndex& mem, Register temp) +{ + MOZ_ASSERT(temp == InvalidReg); + AtomicEffectOp(*this, arrayType, op, value, mem); +} + +void +MacroAssembler::atomicEffectOp(Scalar::Type arrayType, const Synchronization&, AtomicOp op, + Register value, const Address& mem, Register temp) +{ + MOZ_ASSERT(temp == InvalidReg); + AtomicEffectOp(*this, arrayType, op, value, mem); +} + +void +MacroAssembler::atomicEffectOp(Scalar::Type arrayType, const Synchronization&, AtomicOp op, + Imm32 value, const BaseIndex& mem, Register temp) +{ + MOZ_ASSERT(temp == InvalidReg); + AtomicEffectOp(*this, arrayType, op, value, mem); +} + +void +MacroAssembler::atomicEffectOp(Scalar::Type arrayType, const Synchronization&, AtomicOp op, + Imm32 value, const Address& mem, Register temp) +{ + MOZ_ASSERT(temp == InvalidReg); + AtomicEffectOp(*this, arrayType, op, value, mem); +} + +// ======================================================================== +// JS atomic operations. + +template +static void +AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType, const Synchronization& sync, + AtomicOp op, Imm32 value, const T& mem, Register temp1, Register temp2, + AnyRegister output) +{ + if (arrayType == Scalar::Uint32) { + masm.atomicFetchOp(arrayType, sync, op, value, mem, temp2, temp1); + masm.convertUInt32ToDouble(temp1, output.fpu()); + } else { + masm.atomicFetchOp(arrayType, sync, op, value, mem, temp1, output.gpr()); + } +} + +void +MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Imm32 value, const Address& mem, Register temp1, Register temp2, + AnyRegister output) +{ + AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output); +} + +void +MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Imm32 value, const BaseIndex& mem, Register temp1, Register temp2, + AnyRegister output) +{ + AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output); +} + +void +MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Imm32 value, const Address& mem, Register temp) +{ + atomicEffectOp(arrayType, sync, op, value, mem, temp); +} + +void +MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync, AtomicOp op, + Imm32 value, const BaseIndex& mem, Register temp) +{ + atomicEffectOp(arrayType, sync, op, value, mem, temp); +} + //}}} check_macroassembler_style diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared.h b/js/src/jit/x86-shared/MacroAssembler-x86-shared.h index ba170527131c..723479882b00 100644 --- a/js/src/jit/x86-shared/MacroAssembler-x86-shared.h +++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared.h @@ -15,23 +15,6 @@ # include "jit/x64/Assembler-x64.h" #endif -#ifdef DEBUG - #define CHECK_BYTEREG(reg) \ - JS_BEGIN_MACRO \ - AllocatableGeneralRegisterSet byteRegs(Registers::SingleByteRegs); \ - MOZ_ASSERT(byteRegs.has(reg)); \ - JS_END_MACRO - #define CHECK_BYTEREGS(r1, r2) \ - JS_BEGIN_MACRO \ - AllocatableGeneralRegisterSet byteRegs(Registers::SingleByteRegs); \ - MOZ_ASSERT(byteRegs.has(r1)); \ - MOZ_ASSERT(byteRegs.has(r2)); \ - JS_END_MACRO -#else - #define CHECK_BYTEREG(reg) (void)0 - #define CHECK_BYTEREGS(r1, r2) (void)0 -#endif - namespace js { namespace jit { @@ -184,343 +167,6 @@ class MacroAssemblerX86Shared : public Assembler lock_decl(addr); } - template - void atomicFetchAdd8SignExtend(Register src, const T& mem, Register temp, Register output) { - CHECK_BYTEREGS(src, output); - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - lock_xaddb(output, Operand(mem)); - movsbl(output, output); - } - - template - void atomicFetchAdd8ZeroExtend(Register src, const T& mem, Register temp, Register output) { - CHECK_BYTEREGS(src, output); - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - lock_xaddb(output, Operand(mem)); - movzbl(output, output); - } - - template - void atomicFetchAdd8SignExtend(Imm32 src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(output); - MOZ_ASSERT(temp == InvalidReg); - movb(src, output); - lock_xaddb(output, Operand(mem)); - movsbl(output, output); - } - - template - void atomicFetchAdd8ZeroExtend(Imm32 src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(output); - MOZ_ASSERT(temp == InvalidReg); - movb(src, output); - lock_xaddb(output, Operand(mem)); - movzbl(output, output); - } - - template - void atomicFetchAdd16SignExtend(Register src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - lock_xaddw(output, Operand(mem)); - movswl(output, output); - } - - template - void atomicFetchAdd16ZeroExtend(Register src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - lock_xaddw(output, Operand(mem)); - movzwl(output, output); - } - - template - void atomicFetchAdd16SignExtend(Imm32 src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - movl(src, output); - lock_xaddw(output, Operand(mem)); - movswl(output, output); - } - - template - void atomicFetchAdd16ZeroExtend(Imm32 src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - movl(src, output); - lock_xaddw(output, Operand(mem)); - movzwl(output, output); - } - - template - void atomicFetchAdd32(Register src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - lock_xaddl(output, Operand(mem)); - } - - template - void atomicFetchAdd32(Imm32 src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - movl(src, output); - lock_xaddl(output, Operand(mem)); - } - - template - void atomicFetchSub8SignExtend(Register src, const T& mem, Register temp, Register output) { - CHECK_BYTEREGS(src, output); - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - negl(output); - lock_xaddb(output, Operand(mem)); - movsbl(output, output); - } - - template - void atomicFetchSub8ZeroExtend(Register src, const T& mem, Register temp, Register output) { - CHECK_BYTEREGS(src, output); - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - negl(output); - lock_xaddb(output, Operand(mem)); - movzbl(output, output); - } - - template - void atomicFetchSub8SignExtend(Imm32 src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(output); - MOZ_ASSERT(temp == InvalidReg); - movb(Imm32(-src.value), output); - lock_xaddb(output, Operand(mem)); - movsbl(output, output); - } - - template - void atomicFetchSub8ZeroExtend(Imm32 src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(output); - MOZ_ASSERT(temp == InvalidReg); - movb(Imm32(-src.value), output); - lock_xaddb(output, Operand(mem)); - movzbl(output, output); - } - - template - void atomicFetchSub16SignExtend(Register src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - negl(output); - lock_xaddw(output, Operand(mem)); - movswl(output, output); - } - - template - void atomicFetchSub16ZeroExtend(Register src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - negl(output); - lock_xaddw(output, Operand(mem)); - movzwl(output, output); - } - - template - void atomicFetchSub16SignExtend(Imm32 src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - movl(Imm32(-src.value), output); - lock_xaddw(output, Operand(mem)); - movswl(output, output); - } - - template - void atomicFetchSub16ZeroExtend(Imm32 src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - movl(Imm32(-src.value), output); - lock_xaddw(output, Operand(mem)); - movzwl(output, output); - } - - template - void atomicFetchSub32(Register src, const T& mem, Register temp, Register output) { - MOZ_ASSERT(temp == InvalidReg); - if (src != output) - movl(src, output); - negl(output); - lock_xaddl(output, Operand(mem)); - } - - template - void atomicFetchSub32(Imm32 src, const T& mem, Register temp, Register output) { - movl(Imm32(-src.value), output); - lock_xaddl(output, Operand(mem)); - } - - // Requires output == eax. Note src can be an Imm and we can't directly - // assert that it's different from output or tmp. -#define ATOMIC_BITOP_BODY(LOAD, OP, LOCK_CMPXCHG) \ - MOZ_ASSERT(output != temp); \ - MOZ_ASSERT(output == eax); \ - LOAD(Operand(mem), eax); \ - Label again; \ - bind(&again); \ - movl(eax, temp); \ - OP(src, temp); \ - LOCK_CMPXCHG(temp, Operand(mem)); \ - j(NonZero, &again); - - template - void atomicFetchAnd8SignExtend(const S& src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(temp); - ATOMIC_BITOP_BODY(movb, andl, lock_cmpxchgb) - movsbl(eax, eax); - } - template - void atomicFetchAnd8ZeroExtend(const S& src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(temp); - ATOMIC_BITOP_BODY(movb, andl, lock_cmpxchgb) - movzbl(eax, eax); - } - template - void atomicFetchAnd16SignExtend(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movw, andl, lock_cmpxchgw) - movswl(eax, eax); - } - template - void atomicFetchAnd16ZeroExtend(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movw, andl, lock_cmpxchgw) - movzwl(eax, eax); - } - template - void atomicFetchAnd32(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movl, andl, lock_cmpxchgl) - } - - template - void atomicFetchOr8SignExtend(const S& src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(temp); - ATOMIC_BITOP_BODY(movb, orl, lock_cmpxchgb) - movsbl(eax, eax); - } - template - void atomicFetchOr8ZeroExtend(const S& src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(temp); - ATOMIC_BITOP_BODY(movb, orl, lock_cmpxchgb) - movzbl(eax, eax); - } - template - void atomicFetchOr16SignExtend(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movw, orl, lock_cmpxchgw) - movswl(eax, eax); - } - template - void atomicFetchOr16ZeroExtend(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movw, orl, lock_cmpxchgw) - movzwl(eax, eax); - } - template - void atomicFetchOr32(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movl, orl, lock_cmpxchgl) - } - - template - void atomicFetchXor8SignExtend(const S& src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(temp); - ATOMIC_BITOP_BODY(movb, xorl, lock_cmpxchgb) - movsbl(eax, eax); - } - template - void atomicFetchXor8ZeroExtend(const S& src, const T& mem, Register temp, Register output) { - CHECK_BYTEREG(temp); - ATOMIC_BITOP_BODY(movb, xorl, lock_cmpxchgb) - movzbl(eax, eax); - } - template - void atomicFetchXor16SignExtend(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movw, xorl, lock_cmpxchgw) - movswl(eax, eax); - } - template - void atomicFetchXor16ZeroExtend(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movw, xorl, lock_cmpxchgw) - movzwl(eax, eax); - } - template - void atomicFetchXor32(const S& src, const T& mem, Register temp, Register output) { - ATOMIC_BITOP_BODY(movl, xorl, lock_cmpxchgl) - } - -#undef ATOMIC_BITOP_BODY - - // S is Register or Imm32; T is Address or BaseIndex. - - template - void atomicAdd8(const S& src, const T& mem) { - lock_addb(src, Operand(mem)); - } - template - void atomicAdd16(const S& src, const T& mem) { - lock_addw(src, Operand(mem)); - } - template - void atomicAdd32(const S& src, const T& mem) { - lock_addl(src, Operand(mem)); - } - template - void atomicSub8(const S& src, const T& mem) { - lock_subb(src, Operand(mem)); - } - template - void atomicSub16(const S& src, const T& mem) { - lock_subw(src, Operand(mem)); - } - template - void atomicSub32(const S& src, const T& mem) { - lock_subl(src, Operand(mem)); - } - template - void atomicAnd8(const S& src, const T& mem) { - lock_andb(src, Operand(mem)); - } - template - void atomicAnd16(const S& src, const T& mem) { - lock_andw(src, Operand(mem)); - } - template - void atomicAnd32(const S& src, const T& mem) { - lock_andl(src, Operand(mem)); - } - template - void atomicOr8(const S& src, const T& mem) { - lock_orb(src, Operand(mem)); - } - template - void atomicOr16(const S& src, const T& mem) { - lock_orw(src, Operand(mem)); - } - template - void atomicOr32(const S& src, const T& mem) { - lock_orl(src, Operand(mem)); - } - template - void atomicXor8(const S& src, const T& mem) { - lock_xorb(src, Operand(mem)); - } - template - void atomicXor16(const S& src, const T& mem) { - lock_xorw(src, Operand(mem)); - } - template - void atomicXor32(const S& src, const T& mem) { - lock_xorl(src, Operand(mem)); - } - void storeLoadFence() { // This implementation follows Linux. if (HasSSE2()) @@ -669,40 +315,6 @@ class MacroAssemblerX86Shared : public Assembler AutoEnsureByteRegister ensure(this, dest, src); movb(ensure.reg(), Operand(dest)); } - template - void compareExchange8ZeroExtend(const T& mem, Register oldval, Register newval, Register output) { - MOZ_ASSERT(output == eax); - CHECK_BYTEREG(newval); - if (oldval != output) - movl(oldval, output); - lock_cmpxchgb(newval, Operand(mem)); - movzbl(output, output); - } - template - void compareExchange8SignExtend(const T& mem, Register oldval, Register newval, Register output) { - MOZ_ASSERT(output == eax); - CHECK_BYTEREG(newval); - if (oldval != output) - movl(oldval, output); - lock_cmpxchgb(newval, Operand(mem)); - movsbl(output, output); - } - template - void atomicExchange8ZeroExtend(const T& mem, Register value, Register output) { - CHECK_BYTEREG(output); - if (value != output) - movl(value, output); - xchgb(output, Operand(mem)); - movzbl(output, output); - } - template - void atomicExchange8SignExtend(const T& mem, Register value, Register output) { - CHECK_BYTEREG(output); - if (value != output) - movl(value, output); - xchgb(output, Operand(mem)); - movsbl(output, output); - } void load16ZeroExtend(const Operand& src, Register dest) { movzwl(src, dest); } @@ -716,36 +328,6 @@ class MacroAssemblerX86Shared : public Assembler void store16(const S& src, const T& dest) { movw(src, Operand(dest)); } - template - void compareExchange16ZeroExtend(const T& mem, Register oldval, Register newval, Register output) { - MOZ_ASSERT(output == eax); - if (oldval != output) - movl(oldval, output); - lock_cmpxchgw(newval, Operand(mem)); - movzwl(output, output); - } - template - void compareExchange16SignExtend(const T& mem, Register oldval, Register newval, Register output) { - MOZ_ASSERT(output == eax); - if (oldval != output) - movl(oldval, output); - lock_cmpxchgw(newval, Operand(mem)); - movswl(output, output); - } - template - void atomicExchange16ZeroExtend(const T& mem, Register value, Register output) { - if (value != output) - movl(value, output); - xchgw(output, Operand(mem)); - movzwl(output, output); - } - template - void atomicExchange16SignExtend(const T& mem, Register value, Register output) { - if (value != output) - movl(value, output); - xchgw(output, Operand(mem)); - movswl(output, output); - } void load16SignExtend(const Operand& src, Register dest) { movswl(src, dest); } @@ -768,19 +350,6 @@ class MacroAssemblerX86Shared : public Assembler void store32(const S& src, const T& dest) { movl(src, Operand(dest)); } - template - void compareExchange32(const T& mem, Register oldval, Register newval, Register output) { - MOZ_ASSERT(output == eax); - if (oldval != output) - movl(oldval, output); - lock_cmpxchgl(newval, Operand(mem)); - } - template - void atomicExchange32(const T& mem, Register value, Register output) { - if (value != output) - movl(value, output); - xchgl(output, Operand(mem)); - } template void store32_NoSecondScratch(const S& src, const T& dest) { store32(src, dest); @@ -1349,14 +918,6 @@ class MacroAssemblerX86Shared : public Assembler ret(); } - template - void compareExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, Register oldval, Register newval, - Register temp, AnyRegister output); - - template - void atomicExchangeToTypedIntArray(Scalar::Type arrayType, const T& mem, Register value, - Register temp, AnyRegister output); - protected: bool buildOOLFakeExitFrame(void* fakeReturnAddr); }; @@ -1428,7 +989,4 @@ MacroAssemblerX86Shared::storeScalar(FloatRegister src, const Address& de } // namespace jit } // namespace js -#undef CHECK_BYTEREG -#undef CHECK_BYTEREGS - #endif /* jit_x86_shared_MacroAssembler_x86_shared_h */ diff --git a/js/src/jit/x86/CodeGenerator-x86.cpp b/js/src/jit/x86/CodeGenerator-x86.cpp index c90df70ab8e9..d2466494cc4c 100644 --- a/js/src/jit/x86/CodeGenerator-x86.cpp +++ b/js/src/jit/x86/CodeGenerator-x86.cpp @@ -510,16 +510,12 @@ CodeGeneratorX86::visitWasmCompareExchangeHeap(LWasmCompareExchangeHeap* ins) Register newval = ToRegister(ins->newValue()); Register addrTemp = ToRegister(ins->addrTemp()); Register memoryBase = ToRegister(ins->memoryBase()); + Register output = ToRegister(ins->output()); masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()), addrTemp); Address memAddr(addrTemp, 0); - masm.compareExchangeToTypedIntArray(accessType == Scalar::Uint32 ? Scalar::Int32 : accessType, - memAddr, - oldval, - newval, - InvalidReg, - ToAnyRegister(ins->output())); + masm.compareExchange(accessType, Synchronization::Full(), memAddr, oldval, newval, output); } void @@ -532,15 +528,12 @@ CodeGeneratorX86::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) Register value = ToRegister(ins->value()); Register addrTemp = ToRegister(ins->addrTemp()); Register memoryBase = ToRegister(ins->memoryBase()); + Register output = ToRegister(ins->output()); masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()), addrTemp); Address memAddr(addrTemp, 0); - masm.atomicExchangeToTypedIntArray(accessType == Scalar::Uint32 ? Scalar::Int32 : accessType, - memAddr, - value, - InvalidReg, - ToAnyRegister(ins->output())); + masm.atomicExchange(accessType, Synchronization::Full(), memAddr, value, output); } void @@ -552,6 +545,7 @@ CodeGeneratorX86::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) Register ptrReg = ToRegister(ins->ptr()); Register temp = ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp()); Register addrTemp = ToRegister(ins->addrTemp()); + Register out = ToRegister(ins->output()); const LAllocation* value = ins->value(); AtomicOp op = mir->operation(); Register memoryBase = ToRegister(ins->memoryBase()); @@ -560,19 +554,11 @@ CodeGeneratorX86::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) Address memAddr(addrTemp, 0); if (value->isConstant()) { - atomicBinopToTypedIntArray(op, accessType == Scalar::Uint32 ? Scalar::Int32 : accessType, - Imm32(ToInt32(value)), - memAddr, - temp, - InvalidReg, - ToAnyRegister(ins->output())); + masm.atomicFetchOp(accessType, Synchronization::Full(), op, Imm32(ToInt32(value)), + memAddr, temp, out); } else { - atomicBinopToTypedIntArray(op, accessType == Scalar::Uint32 ? Scalar::Int32 : accessType, - ToRegister(value), - memAddr, - temp, - InvalidReg, - ToAnyRegister(ins->output())); + masm.atomicFetchOp(accessType, Synchronization::Full(), op, ToRegister(value), + memAddr, temp, out); } } @@ -592,10 +578,13 @@ CodeGeneratorX86::visitWasmAtomicBinopHeapForEffect(LWasmAtomicBinopHeapForEffec masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()), addrTemp); Address memAddr(addrTemp, 0); - if (value->isConstant()) - atomicBinopToTypedIntArray(op, accessType, Imm32(ToInt32(value)), memAddr); - else - atomicBinopToTypedIntArray(op, accessType, ToRegister(value), memAddr); + if (value->isConstant()) { + masm.atomicEffectOp(accessType, Synchronization::Full(), op, Imm32(ToInt32(value)), memAddr, + InvalidReg); + } else { + masm.atomicEffectOp(accessType, Synchronization::Full(), op, ToRegister(value), memAddr, + InvalidReg); + } } void @@ -715,14 +704,7 @@ CodeGeneratorX86::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* ins) Address valueAddr(esp, 0); // Here the `value` register acts as a temp, we'll restore it below. - switch (ins->operation()) { - case AtomicFetchAddOp: masm.atomicFetchAdd64(valueAddr, srcAddr, value, output); break; - case AtomicFetchSubOp: masm.atomicFetchSub64(valueAddr, srcAddr, value, output); break; - case AtomicFetchAndOp: masm.atomicFetchAnd64(valueAddr, srcAddr, value, output); break; - case AtomicFetchOrOp: masm.atomicFetchOr64(valueAddr, srcAddr, value, output); break; - case AtomicFetchXorOp: masm.atomicFetchXor64(valueAddr, srcAddr, value, output); break; - default: MOZ_CRASH(); - } + masm.atomicFetchOp64(Synchronization::Full(), ins->operation(), valueAddr, srcAddr, value, output); masm.Pop(ebx); masm.Pop(ecx); diff --git a/js/src/jit/x86/MacroAssembler-x86.cpp b/js/src/jit/x86/MacroAssembler-x86.cpp index f7a13cd7000c..dce1a9456cb4 100644 --- a/js/src/jit/x86/MacroAssembler-x86.cpp +++ b/js/src/jit/x86/MacroAssembler-x86.cpp @@ -9,6 +9,7 @@ #include "mozilla/Alignment.h" #include "mozilla/Casting.h" +#include "jit/AtomicOp.h" #include "jit/Bailouts.h" #include "jit/BaselineFrame.h" #include "jit/JitFrames.h" @@ -660,7 +661,7 @@ MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr, { MOZ_ASSERT(srcAddr.kind() == Operand::MEM_REG_DISP || srcAddr.kind() == Operand::MEM_SCALE); - memoryBarrier(access.barrierBefore()); + memoryBarrierBefore(access.sync()); size_t loadOffset = size(); switch (access.type()) { @@ -721,7 +722,7 @@ MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr, } append(access, loadOffset, framePushed()); - memoryBarrier(access.barrierAfter()); + memoryBarrierAfter(access.sync()); } void @@ -732,7 +733,7 @@ MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Operand srcAdd MOZ_ASSERT(!access.isSimd()); MOZ_ASSERT(srcAddr.kind() == Operand::MEM_REG_DISP || srcAddr.kind() == Operand::MEM_SCALE); - memoryBarrier(access.barrierBefore()); + memoryBarrierBefore(access.sync()); size_t loadOffset = size(); switch (access.type()) { @@ -804,7 +805,7 @@ MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Operand srcAdd MOZ_CRASH("unexpected array type"); } - memoryBarrier(access.barrierAfter()); + memoryBarrierAfter(access.sync()); } void @@ -812,7 +813,7 @@ MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister valu { MOZ_ASSERT(dstAddr.kind() == Operand::MEM_REG_DISP || dstAddr.kind() == Operand::MEM_SCALE); - memoryBarrier(access.barrierBefore()); + memoryBarrierBefore(access.sync()); size_t storeOffset = size(); switch (access.type()) { @@ -870,7 +871,7 @@ MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister valu } append(access, storeOffset, framePushed()); - memoryBarrier(access.barrierAfter()); + memoryBarrierAfter(access.sync()); } void @@ -890,94 +891,158 @@ MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 va append(access, storeOffset, framePushed()); } +template +static void +AtomicLoad64(MacroAssembler& masm, const T& address, Register64 temp, Register64 output) +{ + MOZ_ASSERT(temp.low == ebx); + MOZ_ASSERT(temp.high == ecx); + MOZ_ASSERT(output.high == edx); + MOZ_ASSERT(output.low == eax); + + // In the event edx:eax matches what's in memory, ecx:ebx will be + // stored. The two pairs must therefore have the same values. + + masm.movl(edx, ecx); + masm.movl(eax, ebx); + + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(address)); +} + +void +MacroAssembler::atomicLoad64(const Synchronization&, const Address& mem, Register64 temp, + Register64 output) +{ + AtomicLoad64(*this, mem, temp, output); +} + +void +MacroAssembler::atomicLoad64(const Synchronization&, const BaseIndex& mem, Register64 temp, + Register64 output) +{ + AtomicLoad64(*this, mem, temp, output); +} + +template +static void +CompareExchange64(MacroAssembler& masm, const T& mem, Register64 expected, + Register64 replacement, Register64 output) +{ + MOZ_ASSERT(expected == output); + MOZ_ASSERT(expected.high == edx); + MOZ_ASSERT(expected.low == eax); + MOZ_ASSERT(replacement.high == ecx); + MOZ_ASSERT(replacement.low == ebx); + + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem)); +} + +void +MacroAssembler::compareExchange64(const Synchronization&, const Address& mem, Register64 expected, + Register64 replacement, Register64 output) +{ + CompareExchange64(*this, mem, expected, replacement, output); +} + +void +MacroAssembler::compareExchange64(const Synchronization&, const BaseIndex& mem, Register64 expected, + Register64 replacement, Register64 output) +{ + CompareExchange64(*this, mem, expected, replacement, output); +} + +template +static void +AtomicExchange64(MacroAssembler& masm, const T& mem, Register64 value, Register64 output) +{ + MOZ_ASSERT(value.low == ebx); + MOZ_ASSERT(value.high == ecx); + MOZ_ASSERT(output.high == edx); + MOZ_ASSERT(output.low == eax); + + // edx:eax has garbage initially, and that is the best we can do unless + // we can guess with high probability what's in memory. + + Label again; + masm.bind(&again); + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem)); + masm.j(MacroAssembler::NonZero, &again); +} + +void +MacroAssembler::atomicExchange64(const Synchronization&, const Address& mem, Register64 value, + Register64 output) +{ + AtomicExchange64(*this, mem, value, output); +} + +void +MacroAssembler::atomicExchange64(const Synchronization&, const BaseIndex& mem, Register64 value, + Register64 output) +{ + AtomicExchange64(*this, mem, value, output); +} + +template +static void +AtomicFetchOp64(MacroAssembler& masm, AtomicOp op, const Address& value, const T& mem, + Register64 temp, Register64 output) +{ + // We don't have enough registers for all the operands on x86, so the rhs // operand is in memory. -#define ATOMIC_OP_BODY(OPERATE) \ - MOZ_ASSERT(output.low == eax); \ - MOZ_ASSERT(output.high == edx); \ - MOZ_ASSERT(temp.low == ebx); \ - MOZ_ASSERT(temp.high == ecx); \ - load64(address, output); \ - Label again; \ - bind(&again); \ - asMasm().move64(output, temp); \ - OPERATE(value, temp); \ - lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(address)); \ - j(NonZero, &again); +#define ATOMIC_OP_BODY(OPERATE) \ + do { \ + MOZ_ASSERT(output.low == eax); \ + MOZ_ASSERT(output.high == edx); \ + MOZ_ASSERT(temp.low == ebx); \ + MOZ_ASSERT(temp.high == ecx); \ + masm.load64(mem, output); \ + Label again; \ + masm.bind(&again); \ + masm.move64(output, temp); \ + masm.OPERATE(Operand(value), temp); \ + masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem)); \ + masm.j(MacroAssembler::NonZero, &again); \ + } while(0) -template -void -MacroAssemblerX86::atomicFetchAdd64(const T& value, const U& address, Register64 temp, - Register64 output) -{ - ATOMIC_OP_BODY(add64) -} - -template -void -MacroAssemblerX86::atomicFetchSub64(const T& value, const U& address, Register64 temp, - Register64 output) -{ - ATOMIC_OP_BODY(sub64) -} - -template -void -MacroAssemblerX86::atomicFetchAnd64(const T& value, const U& address, Register64 temp, - Register64 output) -{ - ATOMIC_OP_BODY(and64) -} - -template -void -MacroAssemblerX86::atomicFetchOr64(const T& value, const U& address, Register64 temp, - Register64 output) -{ - ATOMIC_OP_BODY(or64) -} - -template -void -MacroAssemblerX86::atomicFetchXor64(const T& value, const U& address, Register64 temp, - Register64 output) -{ - ATOMIC_OP_BODY(xor64) -} + switch (op) { + case AtomicFetchAddOp: + ATOMIC_OP_BODY(add64FromMemory); + break; + case AtomicFetchSubOp: + ATOMIC_OP_BODY(sub64FromMemory); + break; + case AtomicFetchAndOp: + ATOMIC_OP_BODY(and64FromMemory); + break; + case AtomicFetchOrOp: + ATOMIC_OP_BODY(or64FromMemory); + break; + case AtomicFetchXorOp: + ATOMIC_OP_BODY(xor64FromMemory); + break; + default: + MOZ_CRASH(); + } #undef ATOMIC_OP_BODY +} -template void -js::jit::MacroAssemblerX86::atomicFetchAdd64(const Address& value, const Address& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchAdd64(const Address& value, const BaseIndex& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchSub64(const Address& value, const Address& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchSub64(const Address& value, const BaseIndex& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchAnd64(const Address& value, const Address& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchAnd64(const Address& value, const BaseIndex& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchOr64(const Address& value, const Address& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchOr64(const Address& value, const BaseIndex& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchXor64(const Address& value, const Address& address, - Register64 temp, Register64 output); -template void -js::jit::MacroAssemblerX86::atomicFetchXor64(const Address& value, const BaseIndex& address, - Register64 temp, Register64 output); +void +MacroAssembler::atomicFetchOp64(const Synchronization&, AtomicOp op, const Address& value, + const Address& mem, Register64 temp, Register64 output) +{ + AtomicFetchOp64(*this, op, value, mem, temp, output); +} + +void +MacroAssembler::atomicFetchOp64(const Synchronization&, AtomicOp op, const Address& value, + const BaseIndex& mem, Register64 temp, Register64 output) +{ + AtomicFetchOp64(*this, op, value, mem, temp, output); +} void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, Register output, Label* oolEntry) diff --git a/js/src/jit/x86/MacroAssembler-x86.h b/js/src/jit/x86/MacroAssembler-x86.h index a9699d942d4c..51df2c78444a 100644 --- a/js/src/jit/x86/MacroAssembler-x86.h +++ b/js/src/jit/x86/MacroAssembler-x86.h @@ -94,48 +94,31 @@ class MacroAssemblerX86 : public MacroAssemblerX86Shared } template - void add64(const T& address, Register64 dest) { + void add64FromMemory(const T& address, Register64 dest) { addl(Operand(LowWord(address)), dest.low); adcl(Operand(HighWord(address)), dest.high); } template - void sub64(const T& address, Register64 dest) { + void sub64FromMemory(const T& address, Register64 dest) { subl(Operand(LowWord(address)), dest.low); sbbl(Operand(HighWord(address)), dest.high); } template - void and64(const T& address, Register64 dest) { + void and64FromMemory(const T& address, Register64 dest) { andl(Operand(LowWord(address)), dest.low); andl(Operand(HighWord(address)), dest.high); } template - void or64(const T& address, Register64 dest) { + void or64FromMemory(const T& address, Register64 dest) { orl(Operand(LowWord(address)), dest.low); orl(Operand(HighWord(address)), dest.high); } template - void xor64(const T& address, Register64 dest) { + void xor64FromMemory(const T& address, Register64 dest) { xorl(Operand(LowWord(address)), dest.low); xorl(Operand(HighWord(address)), dest.high); } - // Here, `value` is an address to an Int64 because we don't have enough - // registers for all the operands. It is allowed to be SP-relative. - template - void atomicFetchAdd64(const T& value, const U& address, Register64 temp, Register64 output); - - template - void atomicFetchSub64(const T& value, const U& address, Register64 temp, Register64 output); - - template - void atomicFetchAnd64(const T& value, const U& address, Register64 temp, Register64 output); - - template - void atomicFetchOr64(const T& value, const U& address, Register64 temp, Register64 output); - - template - void atomicFetchXor64(const T& value, const U& address, Register64 temp, Register64 output); - ///////////////////////////////////////////////////////////////// // X86/X64-common interface. ///////////////////////////////////////////////////////////////// @@ -675,50 +658,6 @@ class MacroAssemblerX86 : public MacroAssemblerX86Shared movl(imm.hi(), Operand(HighWord(address))); } - template - void atomicLoad64(const T& address, Register64 temp, Register64 output) { - MOZ_ASSERT(temp.low == ebx); - MOZ_ASSERT(temp.high == ecx); - MOZ_ASSERT(output.high == edx); - MOZ_ASSERT(output.low == eax); - - // In the event edx:eax matches what's in memory, ecx:ebx will be - // stored. The two pairs must therefore have the same values. - movl(edx, ecx); - movl(eax, ebx); - - lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(address)); - } - - template - void atomicExchange64(const T& address, Register64 value, Register64 output) { - MOZ_ASSERT(value.low == ebx); - MOZ_ASSERT(value.high == ecx); - MOZ_ASSERT(output.high == edx); - MOZ_ASSERT(output.low == eax); - - // edx:eax has garbage initially, and that is the best we can do unless - // we can guess with high probability what's in memory. - - Label again; - bind(&again); - lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(address)); - j(Assembler::Condition::NonZero, &again); - } - - template - void compareExchange64(const T& address, Register64 expected, Register64 replacement, - Register64 output) - { - MOZ_ASSERT(expected == output); - MOZ_ASSERT(expected.high == edx); - MOZ_ASSERT(expected.low == eax); - MOZ_ASSERT(replacement.high == ecx); - MOZ_ASSERT(replacement.low == ebx); - - lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(address)); - } - void setStackArg(Register reg, uint32_t arg) { movl(reg, Operand(esp, arg * sizeof(intptr_t))); } diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp index 966cac4a2fad..a09007b65873 100644 --- a/js/src/wasm/WasmBaselineCompile.cpp +++ b/js/src/wasm/WasmBaselineCompile.cpp @@ -4181,6 +4181,7 @@ class BaseCompiler final : public BaseCompilerInterface void atomicRMW32(T srcAddr, Scalar::Type viewType, AtomicOp op, RegI32 rv, RegI32 rd, RegI32 temp) { + Synchronization sync = Synchronization::Full(); switch (viewType) { case Scalar::Uint8: { #ifdef JS_CODEGEN_X86 @@ -4190,39 +4191,14 @@ class BaseCompiler final : public BaseCompilerInterface if (op != AtomicFetchAddOp && op != AtomicFetchSubOp) temp = scratch; #endif - switch (op) { - case AtomicFetchAddOp: masm.atomicFetchAdd8ZeroExtend(rv, srcAddr, temp, rd); break; - case AtomicFetchSubOp: masm.atomicFetchSub8ZeroExtend(rv, srcAddr, temp, rd); break; - case AtomicFetchAndOp: masm.atomicFetchAnd8ZeroExtend(rv, srcAddr, temp, rd); break; - case AtomicFetchOrOp: masm.atomicFetchOr8ZeroExtend(rv, srcAddr, temp, rd); break; - case AtomicFetchXorOp: masm.atomicFetchXor8ZeroExtend(rv, srcAddr, temp, rd); break; - default: MOZ_CRASH("No such op"); - } - break; - } - case Scalar::Uint16: { - switch (op) { - case AtomicFetchAddOp: masm.atomicFetchAdd16ZeroExtend(rv, srcAddr, temp, rd); break; - case AtomicFetchSubOp: masm.atomicFetchSub16ZeroExtend(rv, srcAddr, temp, rd); break; - case AtomicFetchAndOp: masm.atomicFetchAnd16ZeroExtend(rv, srcAddr, temp, rd); break; - case AtomicFetchOrOp: masm.atomicFetchOr16ZeroExtend(rv, srcAddr, temp, rd); break; - case AtomicFetchXorOp: masm.atomicFetchXor16ZeroExtend(rv, srcAddr, temp, rd); break; - default: MOZ_CRASH("No such op"); - } + masm.atomicFetchOp(viewType, sync, op, rv, srcAddr, temp, rd); break; } + case Scalar::Uint16: case Scalar::Int32: - case Scalar::Uint32: { - switch (op) { - case AtomicFetchAddOp: masm.atomicFetchAdd32(rv, srcAddr, temp, rd); break; - case AtomicFetchSubOp: masm.atomicFetchSub32(rv, srcAddr, temp, rd); break; - case AtomicFetchAndOp: masm.atomicFetchAnd32(rv, srcAddr, temp, rd); break; - case AtomicFetchOrOp: masm.atomicFetchOr32(rv, srcAddr, temp, rd); break; - case AtomicFetchXorOp: masm.atomicFetchXor32(rv, srcAddr, temp, rd); break; - default: MOZ_CRASH("No such op"); - } + case Scalar::Uint32: + masm.atomicFetchOp(viewType, sync, op, rv, srcAddr, temp, rd); break; - } default: { MOZ_CRASH("Bad type for atomic operation"); } @@ -4234,20 +4210,14 @@ class BaseCompiler final : public BaseCompilerInterface template void atomicRMW64(const T& srcAddr, AtomicOp op, V value, Register64 temp, Register64 rd) { - switch (op) { - case AtomicFetchAddOp: masm.atomicFetchAdd64(value, srcAddr, temp, rd); break; - case AtomicFetchSubOp: masm.atomicFetchSub64(value, srcAddr, temp, rd); break; - case AtomicFetchAndOp: masm.atomicFetchAnd64(value, srcAddr, temp, rd); break; - case AtomicFetchOrOp: masm.atomicFetchOr64(value, srcAddr, temp, rd); break; - case AtomicFetchXorOp: masm.atomicFetchXor64(value, srcAddr, temp, rd); break; - default: MOZ_CRASH("No such op"); - } + masm.atomicFetchOp64(Synchronization::Full(), op, value, srcAddr, temp, rd); } template void atomicCmpXchg32(T srcAddr, Scalar::Type viewType, RegI32 rexpect, RegI32 rnew, RegI32 rd) { + Synchronization sync = Synchronization::Full(); switch (viewType) { case Scalar::Uint8: { #if defined(JS_CODEGEN_X86) @@ -4259,15 +4229,13 @@ class BaseCompiler final : public BaseCompilerInterface rnew = scratch; } #endif - masm.compareExchange8ZeroExtend(srcAddr, rexpect, rnew, rd); + masm.compareExchange(viewType, sync, srcAddr, rexpect, rnew, rd); break; } case Scalar::Uint16: - masm.compareExchange16ZeroExtend(srcAddr, rexpect, rnew, rd); - break; case Scalar::Int32: case Scalar::Uint32: - masm.compareExchange32(srcAddr, rexpect, rnew, rd); + masm.compareExchange(viewType, sync, srcAddr, rexpect, rnew, rd); break; default: MOZ_CRASH("Bad type for atomic operation"); @@ -4278,28 +4246,26 @@ class BaseCompiler final : public BaseCompilerInterface void atomicXchg32(T srcAddr, Scalar::Type viewType, RegI32 rv, RegI32 rd) { + Synchronization sync = Synchronization::Full(); switch (viewType) { - case Scalar::Uint8: { + case Scalar::Uint8: #if defined(JS_CODEGEN_X86) + { if (!ra.isSingleByteI32(rd)) { ScratchI8 scratch(*this); // The output register must have a byte persona. - masm.atomicExchange8ZeroExtend(srcAddr, rv, scratch); + masm.atomicExchange(viewType, sync, srcAddr, rv, scratch); masm.movl(scratch, rd); } else { - masm.atomicExchange8ZeroExtend(srcAddr, rv, rd); + masm.atomicExchange(viewType, sync, srcAddr, rv, rd); } -#else - masm.atomicExchange8ZeroExtend(srcAddr, rv, rd); -#endif break; } +#endif case Scalar::Uint16: - masm.atomicExchange16ZeroExtend(srcAddr, rv, rd); - break; case Scalar::Int32: case Scalar::Uint32: - masm.atomicExchange32(srcAddr, rv, rd); + masm.atomicExchange(viewType, sync, srcAddr, rv, rd); break; default: MOZ_CRASH("Bad type for atomic operation"); @@ -4557,13 +4523,14 @@ class BaseCompiler final : public BaseCompilerInterface atomicCmpXchg64(T srcAddr, RegI32 ebx) { MOZ_ASSERT(ebx == js::jit::ebx); bc->masm.move32(rnew.low, ebx); - bc->masm.compareExchange64(srcAddr, rexpect, bc->specific.ecx_ebx, getRd()); + bc->masm.compareExchange64(Synchronization::Full(), srcAddr, rexpect, + bc->specific.ecx_ebx, getRd()); } #else template void atomicCmpXchg64(T srcAddr) { - bc->masm.compareExchange64(srcAddr, rexpect, rnew, getRd()); + bc->masm.compareExchange64(Synchronization::Full(), srcAddr, rexpect, rnew, getRd()); } #endif }; @@ -4601,13 +4568,13 @@ class BaseCompiler final : public BaseCompilerInterface void atomicLoad64(T srcAddr, RegI32 ebx) { MOZ_ASSERT(ebx == js::jit::ebx); - bc->masm.atomicLoad64(srcAddr, bc->specific.ecx_ebx, getRd()); + bc->masm.atomicLoad64(Synchronization::Full(), srcAddr, bc->specific.ecx_ebx, getRd()); } # else template void atomicLoad64(T srcAddr) { - bc->masm.atomicLoad64(srcAddr, RegI64::Invalid(), getRd()); + bc->masm.atomicLoad64(Synchronization::Full(), srcAddr, RegI64::Invalid(), getRd()); } # endif }; @@ -4850,12 +4817,12 @@ class BaseCompiler final : public BaseCompilerInterface void atomicXchg64(T srcAddr, RegI32 ebx) const { MOZ_ASSERT(ebx == js::jit::ebx); bc->masm.move32(rv.low, ebx); - bc->masm.atomicExchange64(srcAddr, bc->specific.ecx_ebx, getRd()); + bc->masm.atomicExchange64(Synchronization::Full(), srcAddr, bc->specific.ecx_ebx, getRd()); } #else template void atomicXchg64(T srcAddr) const { - bc->masm.atomicExchange64(srcAddr, rv, getRd()); + bc->masm.atomicExchange64(Synchronization::Full(), srcAddr, rv, getRd()); } #endif }; @@ -8175,7 +8142,7 @@ BaseCompiler::emitAtomicCmpXchg(ValType type, Scalar::Type viewType) return true; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarFull, MembarFull); + /*numSimdExprs=*/ 0, Synchronization::Full()); if (Scalar::byteSize(viewType) <= 4) { PopAtomicCmpXchg32Regs regs(this, type); @@ -8231,7 +8198,7 @@ BaseCompiler::emitAtomicLoad(ValType type, Scalar::Type viewType) return true; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(bytecodeOffset()), - /*numSimdElems=*/ 0, MembarBeforeLoad, MembarAfterLoad); + /*numSimdElems=*/ 0, Synchronization::Load()); if (Scalar::byteSize(viewType) <= sizeof(void*)) return loadCommon(&access, type); @@ -8275,7 +8242,7 @@ BaseCompiler::emitAtomicRMW(ValType type, Scalar::Type viewType, AtomicOp op) return true; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(bytecodeOffset()), - /*numSimdElems=*/ 0, MembarFull, MembarFull); + /*numSimdElems=*/ 0, Synchronization::Full()); if (Scalar::byteSize(viewType) <= 4) { PopAtomicRMW32Regs regs(this, type, viewType, op); @@ -8338,7 +8305,7 @@ BaseCompiler::emitAtomicStore(ValType type, Scalar::Type viewType) return true; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(bytecodeOffset()), - /*numSimdElems=*/ 0, MembarBeforeStore, MembarAfterStore); + /*numSimdElems=*/ 0, Synchronization::Store()); if (Scalar::byteSize(viewType) <= sizeof(void*)) return storeCommon(&access, type); @@ -8366,7 +8333,7 @@ BaseCompiler::emitAtomicXchg(ValType type, Scalar::Type viewType) AccessCheck check; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(bytecodeOffset()), - /*numSimdElems=*/ 0, MembarFull, MembarFull); + /*numSimdElems=*/ 0, Synchronization::Full()); if (Scalar::byteSize(viewType) <= 4) { PopAtomicXchg32Regs regs(this, type); diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp index 4fe5199784a0..26a9c615af22 100644 --- a/js/src/wasm/WasmIonCompile.cpp +++ b/js/src/wasm/WasmIonCompile.cpp @@ -2781,7 +2781,7 @@ EmitOldAtomicsLoad(FunctionCompiler& f) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarBeforeLoad, MembarAfterLoad); + /*numSimdExprs=*/ 0, Synchronization::Load()); auto* ins = f.load(addr.base, &access, ValType::I32); if (!f.inDeadCode() && !ins) @@ -2801,7 +2801,7 @@ EmitOldAtomicsStore(FunctionCompiler& f) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarBeforeStore, MembarAfterStore); + /*numSimdExprs=*/ 0, Synchronization::Store()); f.store(addr.base, &access, value); f.iter().setResult(value); @@ -2819,7 +2819,7 @@ EmitOldAtomicsBinOp(FunctionCompiler& f) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarFull, MembarFull); + /*numSimdExprs=*/ 0, Synchronization::Full()); auto* ins = f.atomicBinopHeap(op, addr.base, &access, ValType::I32, value); if (!f.inDeadCode() && !ins) @@ -2840,7 +2840,7 @@ EmitOldAtomicsCompareExchange(FunctionCompiler& f) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarFull, MembarFull); + /*numSimdExprs=*/ 0, Synchronization::Full()); auto* ins = f.atomicCompareExchangeHeap(addr.base, &access, ValType::I32, oldValue, newValue); if (!f.inDeadCode() && !ins) @@ -2860,7 +2860,7 @@ EmitOldAtomicsExchange(FunctionCompiler& f) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarFull, MembarFull); + /*numSimdExprs=*/ 0, Synchronization::Full()); auto* ins = f.atomicExchangeHeap(addr.base, &access, ValType::I32, value); if (!f.inDeadCode() && !ins) @@ -3416,7 +3416,7 @@ EmitAtomicCmpXchg(FunctionCompiler& f, ValType type, Scalar::Type viewType) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarFull, MembarFull); + /*numSimdExprs=*/ 0, Synchronization::Full()); auto* ins = f.atomicCompareExchangeHeap(addr.base, &access, type, oldValue, newValue); if (!f.inDeadCode() && !ins) return false; @@ -3433,7 +3433,7 @@ EmitAtomicLoad(FunctionCompiler& f, ValType type, Scalar::Type viewType) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarBeforeLoad, MembarAfterLoad); + /*numSimdExprs=*/ 0, Synchronization::Load()); auto* ins = f.load(addr.base, &access, type); if (!f.inDeadCode() && !ins) return false; @@ -3451,7 +3451,7 @@ EmitAtomicRMW(FunctionCompiler& f, ValType type, Scalar::Type viewType, jit::Ato return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarFull, MembarFull); + /*numSimdExprs=*/ 0, Synchronization::Full()); auto* ins = f.atomicBinopHeap(op, addr.base, &access, type, value); if (!f.inDeadCode() && !ins) return false; @@ -3469,7 +3469,7 @@ EmitAtomicStore(FunctionCompiler& f, ValType type, Scalar::Type viewType) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarBeforeStore, MembarAfterStore); + /*numSimdExprs=*/ 0, Synchronization::Store()); f.store(addr.base, &access, value); return true; } @@ -3573,7 +3573,7 @@ EmitAtomicXchg(FunctionCompiler& f, ValType type, Scalar::Type viewType) return false; MemoryAccessDesc access(viewType, addr.align, addr.offset, Some(f.bytecodeOffset()), - /*numSimdExprs=*/ 0, MembarFull, MembarFull); + /*numSimdExprs=*/ 0, Synchronization::Full()); MDefinition* ins = f.atomicExchangeHeap(addr.base, &access, type, value); if (!f.inDeadCode() && !ins) return false;