Bug 1271972: wasm: Implement i64.ctz, i64.clz, i64.popcount; r=sunfish

MozReview-Commit-ID: 2XXloMWmbjV

--HG--
extra : rebase_source : a332d6b5c3e0cfe3d5018cdc53e4c20b80b6a52a
extra : histedit_source : d413c95a2d1bc56f8d6f0deacecf37190a612397

js/src/asmjs/Wasm.cpp

@@ -285,7 +285,7 @@ DecodeExpr(FunctionDecoder& f)
       case Expr::I64Clz:
       case Expr::I64Ctz:
       case Expr::I64Popcnt:
-        return f.iter().notYetImplemented("i64") &&
+        return f.checkI64Support() &&
                f.iter().readUnary(ValType::I64, nullptr);
       case Expr::F32Abs:
       case Expr::F32Neg:

js/src/asmjs/WasmIonCompile.cpp

@@ -2913,6 +2913,12 @@ EmitExpr(FunctionCompiler& f)
         return EmitRotate(f, ValType::I64, expr == Expr::I64Rotl);
       case Expr::I64Eqz:
         return EmitConversion<MNot>(f, ValType::I64, ValType::I32);
+      case Expr::I64Clz:
+        return EmitUnary<MClz>(f, ValType::I64);
+      case Expr::I64Ctz:
+        return EmitUnary<MCtz>(f, ValType::I64);
+      case Expr::I64Popcnt:
+        return EmitUnary<MPopcnt>(f, ValType::I64);
 
       // F32
       case Expr::F32Const: {
@@ -3194,9 +3200,6 @@ EmitExpr(FunctionCompiler& f)
       case Expr::I64Store16:
       case Expr::I64Store32:
       case Expr::I64Store:
-      case Expr::I64Clz:
-      case Expr::I64Ctz:
-      case Expr::I64Popcnt:
       case Expr::CurrentMemory:
       case Expr::GrowMemory:
         MOZ_CRASH("NYI");

js/src/jit-test/tests/wasm/basic-integer.js

@@ -213,9 +213,45 @@ if (hasI64()) {
     testComparison64('ge_s', 1, "0x8000000000000000", 1);
     testComparison64('ge_u', 1, "0x8000000000000000", 0);
 
-    //testUnary('i64', 'clz', 40, 58); // TODO: NYI
-    //testUnary('i64', 'ctz', 40, 0); // TODO: NYI
-    //testUnary('i64', 'popcnt', 40, 0); // TODO: NYI
+    testUnary('i64', 'clz', 40, 58);
+    testUnary('i64', 'clz', "0x8000000000000000", 0);
+    testUnary('i64', 'clz', "0x7fffffffffffffff", 1);
+    testUnary('i64', 'clz', "0x4000000000000000", 1);
+    testUnary('i64', 'clz', "0x3000000000000000", 2);
+    testUnary('i64', 'clz', "0x2000000000000000", 2);
+    testUnary('i64', 'clz', "0x1000000000000000", 3);
+    testUnary('i64', 'clz', "0x0030000000000000", 10);
+    testUnary('i64', 'clz', "0x0000800000000000", 16);
+    testUnary('i64', 'clz', "0x00000000ffffffff", 32);
+    testUnary('i64', 'clz', -1, 0);
+    testUnary('i64', 'clz', 0, 64);
+
+    testUnary('i64', 'ctz', 40, 3);
+    testUnary('i64', 'ctz', "0x8000000000000000", 63);
+    testUnary('i64', 'ctz', "0x7fffffffffffffff", 0);
+    testUnary('i64', 'ctz', "0x4000000000000000", 62);
+    testUnary('i64', 'ctz', "0x3000000000000000", 60);
+    testUnary('i64', 'ctz', "0x2000000000000000", 61);
+    testUnary('i64', 'ctz', "0x1000000000000000", 60);
+    testUnary('i64', 'ctz', "0x0030000000000000", 52);
+    testUnary('i64', 'ctz', "0x0000800000000000", 47);
+    testUnary('i64', 'ctz', "0x00000000ffffffff", 0);
+    testUnary('i64', 'ctz', -1, 0);
+    testUnary('i64', 'ctz', 0, 64);
+
+    testUnary('i64', 'popcnt', 40, 2);
+    testUnary('i64', 'popcnt', 0, 0);
+    testUnary('i64', 'popcnt', "0x8000000000000000", 1);
+    testUnary('i64', 'popcnt', "0x7fffffffffffffff", 63);
+    testUnary('i64', 'popcnt', "0x4000000000000000", 1);
+    testUnary('i64', 'popcnt', "0x3000000000000000", 2);
+    testUnary('i64', 'popcnt', "0x2000000000000000", 1);
+    testUnary('i64', 'popcnt', "0x1000000000000000", 1);
+    testUnary('i64', 'popcnt', "0x0030000000000000", 2);
+    testUnary('i64', 'popcnt', "0x0000800000000000", 1);
+    testUnary('i64', 'popcnt', "0x00000000ffffffff", 32);
+    testUnary('i64', 'popcnt', -1, 64);
+    testUnary('i64', 'popcnt', 0, 0);
 
     testI64Eqz(40, 0);
     testI64Eqz(0, 1);

js/src/jit-test/tests/wasm/spec/list.js

@@ -6,7 +6,7 @@ specTests.push("block_comments.wast");
 specTests.push("block.wast");
 specTests.push("break-drop.wast");
 //specTests.push("conversions.wast");       // TODO custom NaN
-//specTests.push("endianness.wast");        // TODO i64 ops
+//specTests.push("endianness.wast");        // TODO i64 loads/stores
 //specTests.push("exports.wast");           // TODO real memory exports
 //specTests.push("f32_cmp.wast");           // TODO custom NaN
 //specTests.push("f32.wast");               // TODO f32.trunc
@@ -21,12 +21,12 @@ specTests.push("forward.wast");
 //specTests.push("func_ptrs.wast");         // TODO pass table index in bad indirect error message
 specTests.push("functions.wast");
 specTests.push("i32.wast");
-//specTests.push("i64.wast");               // TODO i64 ops
+specTests.push("i64.wast");
 specTests.push("imports.wast");
 specTests.push("int_exprs.wast");
 specTests.push("int_literals.wast");
 //specTests.push("labels.wast");            // TODO br_if value/cond
-//specTests.push("left-to-right.wast");     // TODO i64 ops
+//specTests.push("left-to-right.wast");     // TODO i64 loads/stores
 specTests.push("memory_redundancy.wast");
 //specTests.push("memory_trap.wast");       // TODO current_memory opcode
 //specTests.push("memory.wast");            // TODO alignment
@@ -36,7 +36,7 @@ specTests.push("names.wast");
 specTests.push("runaway-recursion.wast");
 //specTests.push("select.wast");            // TODO custom NaN
 //specTests.push("start.wast");             // TODO start opcode
-//specTests.push("store_retval.wast");      // TODO i64 ops
+//specTests.push("store_retval.wast");      // TODO i64 loads/stores
 //specTests.push("switch.wast");            // TODO value error!
 //specTests.push("traps.wast");             // TODO trap on OOB
 specTests.push("unreachable.wast");

js/src/jit/IonTypes.h

@@ -617,6 +617,13 @@ StringFromMIRType(MIRType type)
   MOZ_CRASH("Unknown MIRType.");
 }
 
+static inline bool
+IsIntType(MIRType type)
+{
+    return type == MIRType::Int32 ||
+           type == MIRType::Int64;
+}
+
 static inline bool
 IsNumberType(MIRType type)
 {

js/src/jit/Lowering.cpp

@@ -1406,8 +1406,16 @@ LIRGenerator::visitClz(MClz* ins)
 {
     MDefinition* num = ins->num();
 
+    MOZ_ASSERT(IsIntType(ins->type()));
+
+    if (ins->type() == MIRType::Int32) {
         LClzI* lir = new(alloc()) LClzI(useRegisterAtStart(num));
         define(lir, ins);
+        return;
+    }
+
+    auto* lir = new(alloc()) LClzI64(useInt64RegisterAtStart(num));
+    defineInt64(lir, ins);
 }
 
 void
@@ -1415,8 +1423,16 @@ LIRGenerator::visitCtz(MCtz* ins)
 {
     MDefinition* num = ins->num();
 
+    MOZ_ASSERT(IsIntType(ins->type()));
+
+    if (ins->type() == MIRType::Int32) {
         LCtzI* lir = new(alloc()) LCtzI(useRegisterAtStart(num));
         define(lir, ins);
+        return;
+    }
+
+    auto* lir = new(alloc()) LCtzI64(useInt64RegisterAtStart(num));
+    defineInt64(lir, ins);
 }
 
 void
@@ -1424,8 +1440,16 @@ LIRGenerator::visitPopcnt(MPopcnt* ins)
 {
     MDefinition* num = ins->num();
 
+    MOZ_ASSERT(IsIntType(ins->type()));
+
+    if (ins->type() == MIRType::Int32) {
         LPopcntI* lir = new(alloc()) LPopcntI(useRegisterAtStart(num), temp());
         define(lir, ins);
+        return;
+    }
+
+    auto* lir = new(alloc()) LPopcntI64(useInt64RegisterAtStart(num), tempInt64());
+    defineInt64(lir, ins);
 }
 
 void

js/src/jit/MIR.cpp

@@ -5268,11 +5268,18 @@ MDefinition*
 MClz::foldsTo(TempAllocator& alloc)
 {
     if (num()->isConstant()) {
-        int32_t n = num()->toConstant()->toInt32();
+        MConstant* c = num()->toConstant();
+        if (type() == MIRType::Int32) {
+            int32_t n = c->toInt32();
             if (n == 0)
                 return MConstant::New(alloc, Int32Value(32));
             return MConstant::New(alloc, Int32Value(mozilla::CountLeadingZeroes32(n)));
         }
+        int64_t n = c->toInt64();
+        if (n == 0)
+            return MConstant::NewInt64(alloc, int64_t(64));
+        return MConstant::NewInt64(alloc, int64_t(mozilla::CountLeadingZeroes64(n)));
+    }
 
     return this;
 }
@@ -5281,11 +5288,18 @@ MDefinition*
 MCtz::foldsTo(TempAllocator& alloc)
 {
     if (num()->isConstant()) {
+        MConstant* c = num()->toConstant();
+        if (type() == MIRType::Int32) {
             int32_t n = num()->toConstant()->toInt32();
             if (n == 0)
                 return MConstant::New(alloc, Int32Value(32));
             return MConstant::New(alloc, Int32Value(mozilla::CountTrailingZeroes32(n)));
         }
+        int64_t n = c->toInt64();
+        if (n == 0)
+            return MConstant::NewInt64(alloc, int64_t(64));
+        return MConstant::NewInt64(alloc, int64_t(mozilla::CountTrailingZeroes64(n)));
+    }
 
     return this;
 }
@@ -5294,9 +5308,14 @@ MDefinition*
 MPopcnt::foldsTo(TempAllocator& alloc)
 {
     if (num()->isConstant()) {
+        MConstant* c = num()->toConstant();
+        if (type() == MIRType::Int32) {
             int32_t n = num()->toConstant()->toInt32();
             return MConstant::New(alloc, Int32Value(mozilla::CountPopulation32(n)));
         }
+        int64_t n = c->toInt64();
+        return MConstant::NewInt64(alloc, int64_t(mozilla::CountPopulation64(n)));
+    }
 
     return this;
 }

js/src/jit/MIR.h

@@ -6212,23 +6212,24 @@ class MClz
 {
     bool operandIsNeverZero_;
 
-    explicit MClz(MDefinition* num)
+    explicit MClz(MDefinition* num, MIRType type)
       : MUnaryInstruction(num),
         operandIsNeverZero_(false)
     {
+        MOZ_ASSERT(IsIntType(type));
         MOZ_ASSERT(IsNumberType(num->type()));
-        specialization_ = MIRType::Int32;
-        setResultType(MIRType::Int32);
+        specialization_ = type;
+        setResultType(type);
         setMovable();
     }
 
   public:
     INSTRUCTION_HEADER(Clz)
     static MClz* New(TempAllocator& alloc, MDefinition* num) {
-        return new(alloc) MClz(num);
+        return new(alloc) MClz(num, MIRType::Int32);
     }
     static MClz* NewAsmJS(TempAllocator& alloc, MDefinition* num) {
-        return new(alloc) MClz(num);
+        return new(alloc) MClz(num, num->type());
     }
     MDefinition* num() const {
         return getOperand(0);
@@ -6256,23 +6257,24 @@ class MCtz
 {
     bool operandIsNeverZero_;
 
-    explicit MCtz(MDefinition* num)
+    explicit MCtz(MDefinition* num, MIRType type)
       : MUnaryInstruction(num),
         operandIsNeverZero_(false)
     {
+        MOZ_ASSERT(IsIntType(type));
         MOZ_ASSERT(IsNumberType(num->type()));
-        specialization_ = MIRType::Int32;
-        setResultType(MIRType::Int32);
+        specialization_ = type;
+        setResultType(type);
         setMovable();
     }
 
   public:
     INSTRUCTION_HEADER(Ctz)
     static MCtz* New(TempAllocator& alloc, MDefinition* num) {
-        return new(alloc) MCtz(num);
+        return new(alloc) MCtz(num, MIRType::Int32);
     }
     static MCtz* NewAsmJS(TempAllocator& alloc, MDefinition* num) {
-        return new(alloc) MCtz(num);
+        return new(alloc) MCtz(num, num->type());
     }
     MDefinition* num() const {
         return getOperand(0);
@@ -6298,22 +6300,23 @@ class MPopcnt
   : public MUnaryInstruction
   , public BitwisePolicy::Data
 {
-    explicit MPopcnt(MDefinition* num)
+    explicit MPopcnt(MDefinition* num, MIRType type)
       : MUnaryInstruction(num)
     {
         MOZ_ASSERT(IsNumberType(num->type()));
-        specialization_ = MIRType::Int32;
-        setResultType(MIRType::Int32);
+        MOZ_ASSERT(IsIntType(type));
+        specialization_ = type;
+        setResultType(type);
         setMovable();
     }
 
   public:
     INSTRUCTION_HEADER(Popcnt)
     static MPopcnt* New(TempAllocator& alloc, MDefinition* num) {
-        return new(alloc) MPopcnt(num);
+        return new(alloc) MPopcnt(num, MIRType::Int32);
     }
     static MPopcnt* NewAsmJS(TempAllocator& alloc, MDefinition* num) {
-        return new(alloc) MPopcnt(num);
+        return new(alloc) MPopcnt(num, num->type());
     }
     MDefinition* num() const {
         return getOperand(0);

js/src/jit/shared/LIR-shared.h

@@ -3264,7 +3264,7 @@ class LAbsF : public LInstructionHelper<1, 1, 0>
     }
 };
 
-// Count leading zeroes
+// Count leading zeroes on an int32.
 class LClzI : public LInstructionHelper<1, 1, 0>
 {
   public:
@@ -3278,7 +3278,21 @@ class LClzI : public LInstructionHelper<1, 1, 0>
     }
 };
 
-// Count trailing zeroes
+// Count leading zeroes on an int64.
+class LClzI64 : public LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>
+{
+  public:
+    LIR_HEADER(ClzI64)
+    explicit LClzI64(const LInt64Allocation& num) {
+        setInt64Operand(0, num);
+    }
+
+    MClz* mir() const {
+        return mir_->toClz();
+    }
+};
+
+// Count trailing zeroes on an int32.
 class LCtzI : public LInstructionHelper<1, 1, 0>
 {
   public:
@@ -3292,7 +3306,21 @@ class LCtzI : public LInstructionHelper<1, 1, 0>
     }
 };
 
-// Count population
+// Count trailing zeroes on an int64.
+class LCtzI64 : public LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>
+{
+  public:
+    LIR_HEADER(CtzI64)
+    explicit LCtzI64(const LInt64Allocation& num) {
+        setInt64Operand(0, num);
+    }
+
+    MCtz* mir() const {
+        return mir_->toCtz();
+    }
+};
+
+// Count population on an int32.
 class LPopcntI : public LInstructionHelper<1, 1, 1>
 {
   public:
@@ -3311,6 +3339,21 @@ class LPopcntI : public LInstructionHelper<1, 1, 1>
     }
 };
 
+// Count population on an int64.
+class LPopcntI64 : public LInstructionHelper<INT64_PIECES, INT64_PIECES, 1>
+{
+  public:
+    LIR_HEADER(PopcntI64)
+    explicit LPopcntI64(const LInt64Allocation& num, const LInt64Definition& temp) {
+        setInt64Operand(0, num);
+        setInt64Temp(0, temp);
+    }
+
+    MPopcnt* mir() const {
+        return mir_->toPopcnt();
+    }
+};
+
 // Square root of a double.
 class LSqrtD : public LInstructionHelper<1, 1, 0>
 {

js/src/jit/shared/LOpcodes-shared.h

@@ -142,8 +142,11 @@
     _(AbsD)                         \
     _(AbsF)                         \
     _(ClzI)                         \
+    _(ClzI64)                       \
     _(CtzI)                         \
+    _(CtzI64)                       \
     _(PopcntI)                      \
+    _(PopcntI64)                    \
     _(SqrtD)                        \
     _(SqrtF)                        \
     _(Atan2D)                       \

js/src/jit/x64/CodeGenerator-x64.cpp

@@ -1316,3 +1316,83 @@ CodeGeneratorX64::visitNotI64(LNotI64* lir)
     masm.cmpq(Imm32(0), ToRegister(lir->input()));
     masm.emitSet(Assembler::Equal, ToRegister(lir->output()));
 }
+
+void
+CodeGeneratorX64::visitClzI64(LClzI64* lir)
+{
+    Register input = ToRegister(lir->input());
+    Register output = ToRegister(lir->output());
+
+    Label nonzero;
+    masm.bsrq(input, output);
+
+    masm.j(Assembler::NonZero, &nonzero);
+    // 0x7f ^ 0x3f == 0x40 == 64
+    masm.movq(ImmWord(0x7f), output);
+
+    masm.bind(&nonzero);
+    masm.xorq(Imm32(0x3f), output);
+}
+
+void
+CodeGeneratorX64::visitCtzI64(LCtzI64* lir)
+{
+    Register input = ToRegister(lir->input());
+    Register output = ToRegister(lir->output());
+
+    Label nonzero;
+    masm.bsfq(input, output);
+
+    masm.j(Assembler::NonZero, &nonzero);
+    masm.movq(ImmWord(64), output);
+
+    masm.bind(&nonzero);
+}
+
+void
+CodeGeneratorX64::visitPopcntI64(LPopcntI64* lir)
+{
+    Register input = ToRegister(lir->input());
+    Register output = ToRegister(lir->output());
+
+    if (AssemblerX86Shared::HasPOPCNT()) {
+        masm.popcntq(input, output);
+        return;
+    }
+
+    Register tmp = ToRegister(lir->getTemp(0));
+    if (input != output)
+        masm.movq(input, output);
+
+    MOZ_ASSERT(tmp != output);
+
+    ScratchRegisterScope scratch(masm);
+
+    // Equivalent to mozilla::CountPopulation32, adapted for 64 bits.
+    // x -= (x >> 1) & m1;
+    masm.movq(input, tmp);
+    masm.movq(ImmWord(0x5555555555555555), scratch);
+    masm.shrq(Imm32(1), tmp);
+    masm.andq(scratch, tmp);
+    masm.subq(tmp, output);
+
+    // x = (x & m2) + ((x >> 2) & m2);
+    masm.movq(output, tmp);
+    masm.movq(ImmWord(0x3333333333333333), scratch);
+    masm.andq(scratch, output);
+    masm.shrq(Imm32(2), tmp);
+    masm.andq(scratch, tmp);
+    masm.addq(tmp, output);
+
+    // x = (x + (x >> 4)) & m4;
+    masm.movq(output, tmp);
+    masm.movq(ImmWord(0x0f0f0f0f0f0f0f0f), scratch);
+    masm.shrq(Imm32(4), tmp);
+    masm.addq(tmp, output);
+    masm.andq(scratch, output);
+
+    // (x * h01) >> 56
+    masm.movq(ImmWord(0x0101010101010101), scratch);
+    masm.imulq(scratch, output);
+    masm.shrq(Imm32(56), output);
+}

js/src/jit/x64/CodeGenerator-x64.h

@@ -53,6 +53,9 @@ class CodeGeneratorX64 : public CodeGeneratorX86Shared
     void visitDivOrModI64(LDivOrModI64* lir);
     void visitUDivOrMod64(LUDivOrMod64* lir);
     void visitNotI64(LNotI64* lir);
+    void visitClzI64(LClzI64* lir);
+    void visitCtzI64(LCtzI64* lir);
+    void visitPopcntI64(LPopcntI64* lir);
     void visitTruncateDToInt32(LTruncateDToInt32* ins);
     void visitTruncateFToInt32(LTruncateFToInt32* ins);
     void visitWrapInt64ToInt32(LWrapInt64ToInt32* lir);

mfbt/MathAlgorithms.h

@@ -340,7 +340,7 @@ CountPopulation32(uint32_t aValue)
   return detail::CountPopulation32(aValue);
 }
 
-/** Analogous to CoutPopulation32, but for 64-bit numbers */
+/** Analogous to CountPopulation32, but for 64-bit numbers */
 inline uint_fast8_t
 CountPopulation64(uint64_t aValue)
 {