diff --git a/js/src/asmjs/Wasm.cpp b/js/src/asmjs/Wasm.cpp index 2231c8ea923e..b028e03f67ed 100644 --- a/js/src/asmjs/Wasm.cpp +++ b/js/src/asmjs/Wasm.cpp @@ -285,7 +285,7 @@ DecodeExpr(FunctionDecoder& f) case Expr::I64Clz: case Expr::I64Ctz: case Expr::I64Popcnt: - return f.iter().notYetImplemented("i64") && + return f.checkI64Support() && f.iter().readUnary(ValType::I64, nullptr); case Expr::F32Abs: case Expr::F32Neg: diff --git a/js/src/asmjs/WasmIonCompile.cpp b/js/src/asmjs/WasmIonCompile.cpp index 88d4e5e8a793..81273cf615e0 100644 --- a/js/src/asmjs/WasmIonCompile.cpp +++ b/js/src/asmjs/WasmIonCompile.cpp @@ -2913,6 +2913,12 @@ EmitExpr(FunctionCompiler& f) return EmitRotate(f, ValType::I64, expr == Expr::I64Rotl); case Expr::I64Eqz: return EmitConversion(f, ValType::I64, ValType::I32); + case Expr::I64Clz: + return EmitUnary(f, ValType::I64); + case Expr::I64Ctz: + return EmitUnary(f, ValType::I64); + case Expr::I64Popcnt: + return EmitUnary(f, ValType::I64); // F32 case Expr::F32Const: { @@ -3194,9 +3200,6 @@ EmitExpr(FunctionCompiler& f) case Expr::I64Store16: case Expr::I64Store32: case Expr::I64Store: - case Expr::I64Clz: - case Expr::I64Ctz: - case Expr::I64Popcnt: case Expr::CurrentMemory: case Expr::GrowMemory: MOZ_CRASH("NYI"); diff --git a/js/src/jit-test/tests/wasm/basic-integer.js b/js/src/jit-test/tests/wasm/basic-integer.js index e27673b35be2..815195ee8992 100644 --- a/js/src/jit-test/tests/wasm/basic-integer.js +++ b/js/src/jit-test/tests/wasm/basic-integer.js @@ -213,9 +213,45 @@ if (hasI64()) { testComparison64('ge_s', 1, "0x8000000000000000", 1); testComparison64('ge_u', 1, "0x8000000000000000", 0); - //testUnary('i64', 'clz', 40, 58); // TODO: NYI - //testUnary('i64', 'ctz', 40, 0); // TODO: NYI - //testUnary('i64', 'popcnt', 40, 0); // TODO: NYI + testUnary('i64', 'clz', 40, 58); + testUnary('i64', 'clz', "0x8000000000000000", 0); + testUnary('i64', 'clz', "0x7fffffffffffffff", 1); + testUnary('i64', 'clz', "0x4000000000000000", 1); + testUnary('i64', 'clz', "0x3000000000000000", 2); + testUnary('i64', 'clz', "0x2000000000000000", 2); + testUnary('i64', 'clz', "0x1000000000000000", 3); + testUnary('i64', 'clz', "0x0030000000000000", 10); + testUnary('i64', 'clz', "0x0000800000000000", 16); + testUnary('i64', 'clz', "0x00000000ffffffff", 32); + testUnary('i64', 'clz', -1, 0); + testUnary('i64', 'clz', 0, 64); + + testUnary('i64', 'ctz', 40, 3); + testUnary('i64', 'ctz', "0x8000000000000000", 63); + testUnary('i64', 'ctz', "0x7fffffffffffffff", 0); + testUnary('i64', 'ctz', "0x4000000000000000", 62); + testUnary('i64', 'ctz', "0x3000000000000000", 60); + testUnary('i64', 'ctz', "0x2000000000000000", 61); + testUnary('i64', 'ctz', "0x1000000000000000", 60); + testUnary('i64', 'ctz', "0x0030000000000000", 52); + testUnary('i64', 'ctz', "0x0000800000000000", 47); + testUnary('i64', 'ctz', "0x00000000ffffffff", 0); + testUnary('i64', 'ctz', -1, 0); + testUnary('i64', 'ctz', 0, 64); + + testUnary('i64', 'popcnt', 40, 2); + testUnary('i64', 'popcnt', 0, 0); + testUnary('i64', 'popcnt', "0x8000000000000000", 1); + testUnary('i64', 'popcnt', "0x7fffffffffffffff", 63); + testUnary('i64', 'popcnt', "0x4000000000000000", 1); + testUnary('i64', 'popcnt', "0x3000000000000000", 2); + testUnary('i64', 'popcnt', "0x2000000000000000", 1); + testUnary('i64', 'popcnt', "0x1000000000000000", 1); + testUnary('i64', 'popcnt', "0x0030000000000000", 2); + testUnary('i64', 'popcnt', "0x0000800000000000", 1); + testUnary('i64', 'popcnt', "0x00000000ffffffff", 32); + testUnary('i64', 'popcnt', -1, 64); + testUnary('i64', 'popcnt', 0, 0); testI64Eqz(40, 0); testI64Eqz(0, 1); diff --git a/js/src/jit-test/tests/wasm/spec/list.js b/js/src/jit-test/tests/wasm/spec/list.js index 3ba68081f349..2fc62c89c36a 100644 --- a/js/src/jit-test/tests/wasm/spec/list.js +++ b/js/src/jit-test/tests/wasm/spec/list.js @@ -6,7 +6,7 @@ specTests.push("block_comments.wast"); specTests.push("block.wast"); specTests.push("break-drop.wast"); //specTests.push("conversions.wast"); // TODO custom NaN -//specTests.push("endianness.wast"); // TODO i64 ops +//specTests.push("endianness.wast"); // TODO i64 loads/stores //specTests.push("exports.wast"); // TODO real memory exports //specTests.push("f32_cmp.wast"); // TODO custom NaN //specTests.push("f32.wast"); // TODO f32.trunc @@ -21,12 +21,12 @@ specTests.push("forward.wast"); //specTests.push("func_ptrs.wast"); // TODO pass table index in bad indirect error message specTests.push("functions.wast"); specTests.push("i32.wast"); -//specTests.push("i64.wast"); // TODO i64 ops +specTests.push("i64.wast"); specTests.push("imports.wast"); specTests.push("int_exprs.wast"); specTests.push("int_literals.wast"); //specTests.push("labels.wast"); // TODO br_if value/cond -//specTests.push("left-to-right.wast"); // TODO i64 ops +//specTests.push("left-to-right.wast"); // TODO i64 loads/stores specTests.push("memory_redundancy.wast"); //specTests.push("memory_trap.wast"); // TODO current_memory opcode //specTests.push("memory.wast"); // TODO alignment @@ -36,7 +36,7 @@ specTests.push("names.wast"); specTests.push("runaway-recursion.wast"); //specTests.push("select.wast"); // TODO custom NaN //specTests.push("start.wast"); // TODO start opcode -//specTests.push("store_retval.wast"); // TODO i64 ops +//specTests.push("store_retval.wast"); // TODO i64 loads/stores //specTests.push("switch.wast"); // TODO value error! //specTests.push("traps.wast"); // TODO trap on OOB specTests.push("unreachable.wast"); diff --git a/js/src/jit/IonTypes.h b/js/src/jit/IonTypes.h index 908c6d391bb9..1e209c62d03e 100644 --- a/js/src/jit/IonTypes.h +++ b/js/src/jit/IonTypes.h @@ -617,6 +617,13 @@ StringFromMIRType(MIRType type) MOZ_CRASH("Unknown MIRType."); } +static inline bool +IsIntType(MIRType type) +{ + return type == MIRType::Int32 || + type == MIRType::Int64; +} + static inline bool IsNumberType(MIRType type) { diff --git a/js/src/jit/Lowering.cpp b/js/src/jit/Lowering.cpp index 9f041d1f3526..c61e75720a33 100644 --- a/js/src/jit/Lowering.cpp +++ b/js/src/jit/Lowering.cpp @@ -1406,8 +1406,16 @@ LIRGenerator::visitClz(MClz* ins) { MDefinition* num = ins->num(); - LClzI* lir = new(alloc()) LClzI(useRegisterAtStart(num)); - define(lir, ins); + MOZ_ASSERT(IsIntType(ins->type())); + + if (ins->type() == MIRType::Int32) { + LClzI* lir = new(alloc()) LClzI(useRegisterAtStart(num)); + define(lir, ins); + return; + } + + auto* lir = new(alloc()) LClzI64(useInt64RegisterAtStart(num)); + defineInt64(lir, ins); } void @@ -1415,8 +1423,16 @@ LIRGenerator::visitCtz(MCtz* ins) { MDefinition* num = ins->num(); - LCtzI* lir = new(alloc()) LCtzI(useRegisterAtStart(num)); - define(lir, ins); + MOZ_ASSERT(IsIntType(ins->type())); + + if (ins->type() == MIRType::Int32) { + LCtzI* lir = new(alloc()) LCtzI(useRegisterAtStart(num)); + define(lir, ins); + return; + } + + auto* lir = new(alloc()) LCtzI64(useInt64RegisterAtStart(num)); + defineInt64(lir, ins); } void @@ -1424,8 +1440,16 @@ LIRGenerator::visitPopcnt(MPopcnt* ins) { MDefinition* num = ins->num(); - LPopcntI* lir = new(alloc()) LPopcntI(useRegisterAtStart(num), temp()); - define(lir, ins); + MOZ_ASSERT(IsIntType(ins->type())); + + if (ins->type() == MIRType::Int32) { + LPopcntI* lir = new(alloc()) LPopcntI(useRegisterAtStart(num), temp()); + define(lir, ins); + return; + } + + auto* lir = new(alloc()) LPopcntI64(useInt64RegisterAtStart(num), tempInt64()); + defineInt64(lir, ins); } void diff --git a/js/src/jit/MIR.cpp b/js/src/jit/MIR.cpp index 368c416dae71..814e6fecf26e 100644 --- a/js/src/jit/MIR.cpp +++ b/js/src/jit/MIR.cpp @@ -5268,10 +5268,17 @@ MDefinition* MClz::foldsTo(TempAllocator& alloc) { if (num()->isConstant()) { - int32_t n = num()->toConstant()->toInt32(); + MConstant* c = num()->toConstant(); + if (type() == MIRType::Int32) { + int32_t n = c->toInt32(); + if (n == 0) + return MConstant::New(alloc, Int32Value(32)); + return MConstant::New(alloc, Int32Value(mozilla::CountLeadingZeroes32(n))); + } + int64_t n = c->toInt64(); if (n == 0) - return MConstant::New(alloc, Int32Value(32)); - return MConstant::New(alloc, Int32Value(mozilla::CountLeadingZeroes32(n))); + return MConstant::NewInt64(alloc, int64_t(64)); + return MConstant::NewInt64(alloc, int64_t(mozilla::CountLeadingZeroes64(n))); } return this; @@ -5281,10 +5288,17 @@ MDefinition* MCtz::foldsTo(TempAllocator& alloc) { if (num()->isConstant()) { - int32_t n = num()->toConstant()->toInt32(); + MConstant* c = num()->toConstant(); + if (type() == MIRType::Int32) { + int32_t n = num()->toConstant()->toInt32(); + if (n == 0) + return MConstant::New(alloc, Int32Value(32)); + return MConstant::New(alloc, Int32Value(mozilla::CountTrailingZeroes32(n))); + } + int64_t n = c->toInt64(); if (n == 0) - return MConstant::New(alloc, Int32Value(32)); - return MConstant::New(alloc, Int32Value(mozilla::CountTrailingZeroes32(n))); + return MConstant::NewInt64(alloc, int64_t(64)); + return MConstant::NewInt64(alloc, int64_t(mozilla::CountTrailingZeroes64(n))); } return this; @@ -5294,8 +5308,13 @@ MDefinition* MPopcnt::foldsTo(TempAllocator& alloc) { if (num()->isConstant()) { - int32_t n = num()->toConstant()->toInt32(); - return MConstant::New(alloc, Int32Value(mozilla::CountPopulation32(n))); + MConstant* c = num()->toConstant(); + if (type() == MIRType::Int32) { + int32_t n = num()->toConstant()->toInt32(); + return MConstant::New(alloc, Int32Value(mozilla::CountPopulation32(n))); + } + int64_t n = c->toInt64(); + return MConstant::NewInt64(alloc, int64_t(mozilla::CountPopulation64(n))); } return this; diff --git a/js/src/jit/MIR.h b/js/src/jit/MIR.h index 4399f88e4027..a417b69ac7e4 100644 --- a/js/src/jit/MIR.h +++ b/js/src/jit/MIR.h @@ -6212,23 +6212,24 @@ class MClz { bool operandIsNeverZero_; - explicit MClz(MDefinition* num) + explicit MClz(MDefinition* num, MIRType type) : MUnaryInstruction(num), operandIsNeverZero_(false) { + MOZ_ASSERT(IsIntType(type)); MOZ_ASSERT(IsNumberType(num->type())); - specialization_ = MIRType::Int32; - setResultType(MIRType::Int32); + specialization_ = type; + setResultType(type); setMovable(); } public: INSTRUCTION_HEADER(Clz) static MClz* New(TempAllocator& alloc, MDefinition* num) { - return new(alloc) MClz(num); + return new(alloc) MClz(num, MIRType::Int32); } static MClz* NewAsmJS(TempAllocator& alloc, MDefinition* num) { - return new(alloc) MClz(num); + return new(alloc) MClz(num, num->type()); } MDefinition* num() const { return getOperand(0); @@ -6256,23 +6257,24 @@ class MCtz { bool operandIsNeverZero_; - explicit MCtz(MDefinition* num) + explicit MCtz(MDefinition* num, MIRType type) : MUnaryInstruction(num), operandIsNeverZero_(false) { + MOZ_ASSERT(IsIntType(type)); MOZ_ASSERT(IsNumberType(num->type())); - specialization_ = MIRType::Int32; - setResultType(MIRType::Int32); + specialization_ = type; + setResultType(type); setMovable(); } public: INSTRUCTION_HEADER(Ctz) static MCtz* New(TempAllocator& alloc, MDefinition* num) { - return new(alloc) MCtz(num); + return new(alloc) MCtz(num, MIRType::Int32); } static MCtz* NewAsmJS(TempAllocator& alloc, MDefinition* num) { - return new(alloc) MCtz(num); + return new(alloc) MCtz(num, num->type()); } MDefinition* num() const { return getOperand(0); @@ -6298,22 +6300,23 @@ class MPopcnt : public MUnaryInstruction , public BitwisePolicy::Data { - explicit MPopcnt(MDefinition* num) + explicit MPopcnt(MDefinition* num, MIRType type) : MUnaryInstruction(num) { MOZ_ASSERT(IsNumberType(num->type())); - specialization_ = MIRType::Int32; - setResultType(MIRType::Int32); + MOZ_ASSERT(IsIntType(type)); + specialization_ = type; + setResultType(type); setMovable(); } public: INSTRUCTION_HEADER(Popcnt) static MPopcnt* New(TempAllocator& alloc, MDefinition* num) { - return new(alloc) MPopcnt(num); + return new(alloc) MPopcnt(num, MIRType::Int32); } static MPopcnt* NewAsmJS(TempAllocator& alloc, MDefinition* num) { - return new(alloc) MPopcnt(num); + return new(alloc) MPopcnt(num, num->type()); } MDefinition* num() const { return getOperand(0); diff --git a/js/src/jit/shared/LIR-shared.h b/js/src/jit/shared/LIR-shared.h index 7ef2263f87af..f71898c9cff6 100644 --- a/js/src/jit/shared/LIR-shared.h +++ b/js/src/jit/shared/LIR-shared.h @@ -3264,7 +3264,7 @@ class LAbsF : public LInstructionHelper<1, 1, 0> } }; -// Count leading zeroes +// Count leading zeroes on an int32. class LClzI : public LInstructionHelper<1, 1, 0> { public: @@ -3278,7 +3278,21 @@ class LClzI : public LInstructionHelper<1, 1, 0> } }; -// Count trailing zeroes +// Count leading zeroes on an int64. +class LClzI64 : public LInstructionHelper +{ + public: + LIR_HEADER(ClzI64) + explicit LClzI64(const LInt64Allocation& num) { + setInt64Operand(0, num); + } + + MClz* mir() const { + return mir_->toClz(); + } +}; + +// Count trailing zeroes on an int32. class LCtzI : public LInstructionHelper<1, 1, 0> { public: @@ -3292,7 +3306,21 @@ class LCtzI : public LInstructionHelper<1, 1, 0> } }; -// Count population +// Count trailing zeroes on an int64. +class LCtzI64 : public LInstructionHelper +{ + public: + LIR_HEADER(CtzI64) + explicit LCtzI64(const LInt64Allocation& num) { + setInt64Operand(0, num); + } + + MCtz* mir() const { + return mir_->toCtz(); + } +}; + +// Count population on an int32. class LPopcntI : public LInstructionHelper<1, 1, 1> { public: @@ -3311,6 +3339,21 @@ class LPopcntI : public LInstructionHelper<1, 1, 1> } }; +// Count population on an int64. +class LPopcntI64 : public LInstructionHelper +{ + public: + LIR_HEADER(PopcntI64) + explicit LPopcntI64(const LInt64Allocation& num, const LInt64Definition& temp) { + setInt64Operand(0, num); + setInt64Temp(0, temp); + } + + MPopcnt* mir() const { + return mir_->toPopcnt(); + } +}; + // Square root of a double. class LSqrtD : public LInstructionHelper<1, 1, 0> { diff --git a/js/src/jit/shared/LOpcodes-shared.h b/js/src/jit/shared/LOpcodes-shared.h index 55f9eb44a181..bc6359e3f90f 100644 --- a/js/src/jit/shared/LOpcodes-shared.h +++ b/js/src/jit/shared/LOpcodes-shared.h @@ -142,8 +142,11 @@ _(AbsD) \ _(AbsF) \ _(ClzI) \ + _(ClzI64) \ _(CtzI) \ + _(CtzI64) \ _(PopcntI) \ + _(PopcntI64) \ _(SqrtD) \ _(SqrtF) \ _(Atan2D) \ diff --git a/js/src/jit/x64/CodeGenerator-x64.cpp b/js/src/jit/x64/CodeGenerator-x64.cpp index 46a7cf285872..f86e3cff032d 100644 --- a/js/src/jit/x64/CodeGenerator-x64.cpp +++ b/js/src/jit/x64/CodeGenerator-x64.cpp @@ -1316,3 +1316,83 @@ CodeGeneratorX64::visitNotI64(LNotI64* lir) masm.cmpq(Imm32(0), ToRegister(lir->input())); masm.emitSet(Assembler::Equal, ToRegister(lir->output())); } + +void +CodeGeneratorX64::visitClzI64(LClzI64* lir) +{ + Register input = ToRegister(lir->input()); + Register output = ToRegister(lir->output()); + + Label nonzero; + masm.bsrq(input, output); + + masm.j(Assembler::NonZero, &nonzero); + // 0x7f ^ 0x3f == 0x40 == 64 + masm.movq(ImmWord(0x7f), output); + + masm.bind(&nonzero); + masm.xorq(Imm32(0x3f), output); +} + +void +CodeGeneratorX64::visitCtzI64(LCtzI64* lir) +{ + Register input = ToRegister(lir->input()); + Register output = ToRegister(lir->output()); + + Label nonzero; + masm.bsfq(input, output); + + masm.j(Assembler::NonZero, &nonzero); + masm.movq(ImmWord(64), output); + + masm.bind(&nonzero); +} + +void +CodeGeneratorX64::visitPopcntI64(LPopcntI64* lir) +{ + Register input = ToRegister(lir->input()); + Register output = ToRegister(lir->output()); + + if (AssemblerX86Shared::HasPOPCNT()) { + masm.popcntq(input, output); + return; + } + + Register tmp = ToRegister(lir->getTemp(0)); + if (input != output) + masm.movq(input, output); + + MOZ_ASSERT(tmp != output); + + ScratchRegisterScope scratch(masm); + + // Equivalent to mozilla::CountPopulation32, adapted for 64 bits. + // x -= (x >> 1) & m1; + masm.movq(input, tmp); + masm.movq(ImmWord(0x5555555555555555), scratch); + masm.shrq(Imm32(1), tmp); + masm.andq(scratch, tmp); + masm.subq(tmp, output); + + // x = (x & m2) + ((x >> 2) & m2); + masm.movq(output, tmp); + masm.movq(ImmWord(0x3333333333333333), scratch); + masm.andq(scratch, output); + masm.shrq(Imm32(2), tmp); + masm.andq(scratch, tmp); + masm.addq(tmp, output); + + // x = (x + (x >> 4)) & m4; + masm.movq(output, tmp); + masm.movq(ImmWord(0x0f0f0f0f0f0f0f0f), scratch); + masm.shrq(Imm32(4), tmp); + masm.addq(tmp, output); + masm.andq(scratch, output); + + // (x * h01) >> 56 + masm.movq(ImmWord(0x0101010101010101), scratch); + masm.imulq(scratch, output); + masm.shrq(Imm32(56), output); +} diff --git a/js/src/jit/x64/CodeGenerator-x64.h b/js/src/jit/x64/CodeGenerator-x64.h index af639acca179..c6c93cdbf447 100644 --- a/js/src/jit/x64/CodeGenerator-x64.h +++ b/js/src/jit/x64/CodeGenerator-x64.h @@ -53,6 +53,9 @@ class CodeGeneratorX64 : public CodeGeneratorX86Shared void visitDivOrModI64(LDivOrModI64* lir); void visitUDivOrMod64(LUDivOrMod64* lir); void visitNotI64(LNotI64* lir); + void visitClzI64(LClzI64* lir); + void visitCtzI64(LCtzI64* lir); + void visitPopcntI64(LPopcntI64* lir); void visitTruncateDToInt32(LTruncateDToInt32* ins); void visitTruncateFToInt32(LTruncateFToInt32* ins); void visitWrapInt64ToInt32(LWrapInt64ToInt32* lir); diff --git a/mfbt/MathAlgorithms.h b/mfbt/MathAlgorithms.h index 28e41826c458..f2d25d5bb724 100644 --- a/mfbt/MathAlgorithms.h +++ b/mfbt/MathAlgorithms.h @@ -340,7 +340,7 @@ CountPopulation32(uint32_t aValue) return detail::CountPopulation32(aValue); } -/** Analogous to CoutPopulation32, but for 64-bit numbers */ +/** Analogous to CountPopulation32, but for 64-bit numbers */ inline uint_fast8_t CountPopulation64(uint64_t aValue) {