Bug 1279248 - Part 17: Implement the 64bit variant of PopCnt on x86, r=lth

js/src/asmjs/WasmBaselineCompile.cpp

@@ -2503,7 +2503,7 @@ class BaseCompiler
 
     void popcntI64(RegI64 srcDest, RegI64 tmp) {
 #if defined(JS_CODEGEN_X64)
-        masm.popcnt64(srcDest.reg, srcDest.reg, tmp.reg);
+        masm.popcnt64(srcDest.reg, srcDest.reg, tmp.reg.reg);
 #else
         MOZ_CRASH("BaseCompiler platform hook: popcntI64");
 #endif

js/src/jit/Lowering.cpp

@@ -1459,7 +1459,7 @@ LIRGenerator::visitPopcnt(MPopcnt* ins)
         return;
     }
 
-    auto* lir = new(alloc()) LPopcntI64(useInt64RegisterAtStart(num), tempInt64());
+    auto* lir = new(alloc()) LPopcntI64(useInt64RegisterAtStart(num), temp());
     defineInt64(lir, ins);
 }
 

js/src/jit/MacroAssembler.h

@@ -914,7 +914,7 @@ class MacroAssembler : public MacroAssemblerSpecific
     inline void popcnt32(Register src, Register dest, Register temp) DEFINED_ON(arm, x86_shared);
 
     // temp may be invalid only if the chip has the POPCNT instruction.
-    inline void popcnt64(Register64 src, Register64 dest, Register64 temp) DEFINED_ON(x64);
+    inline void popcnt64(Register64 src, Register64 dest, Register temp) DEFINED_ON(x86, x64);
 
     // ===============================================================
     // Branch functions

js/src/jit/shared/LIR-shared.h

@@ -3561,9 +3561,9 @@ class LPopcntI64 : public LInstructionHelper<INT64_PIECES, INT64_PIECES, 1>
 {
   public:
     LIR_HEADER(PopcntI64)
-    explicit LPopcntI64(const LInt64Allocation& num, const LInt64Definition& temp) {
+    explicit LPopcntI64(const LInt64Allocation& num, const LDefinition& temp) {
         setInt64Operand(0, num);
-        setInt64Temp(0, temp);
+        setTemp(0, temp);
     }
 
     MPopcnt* mir() const {

js/src/jit/x64/CodeGenerator-x64.cpp

@@ -1254,18 +1254,6 @@ CodeGeneratorX64::visitCtzI64(LCtzI64* lir)
     masm.ctz64(input, output);
 }
 
-void
-CodeGeneratorX64::visitPopcntI64(LPopcntI64* lir)
-{
-    Register64 input = ToRegister64(lir->input());
-    Register64 output = ToRegister64(lir->output());
-    Register64 temp = Register64(AssemblerX86Shared::HasPOPCNT()
-                                 ? InvalidReg :
-                                 ToRegister(lir->getTemp(0)));
-
-    masm.popcnt64(input, output, temp);
-}
-
 void
 CodeGeneratorX64::visitTestI64AndBranch(LTestI64AndBranch* lir)
 {

js/src/jit/x64/CodeGenerator-x64.h

@@ -57,7 +57,6 @@ class CodeGeneratorX64 : public CodeGeneratorX86Shared
     void visitNotI64(LNotI64* lir);
     void visitClzI64(LClzI64* lir);
     void visitCtzI64(LCtzI64* lir);
-    void visitPopcntI64(LPopcntI64* lir);
     void visitTruncateDToInt32(LTruncateDToInt32* ins);
     void visitTruncateFToInt32(LTruncateFToInt32* ins);
     void visitWrapInt64ToInt32(LWrapInt64ToInt32* lir);

js/src/jit/x64/MacroAssembler-x64-inl.h

@@ -472,13 +472,13 @@ MacroAssembler::ctz64(Register64 src, Register64 dest)
 }
 
 void
-MacroAssembler::popcnt64(Register64 src64, Register64 dest64, Register64 tmp64)
+MacroAssembler::popcnt64(Register64 src64, Register64 dest64, Register tmp)
 {
     Register src = src64.reg;
     Register dest = dest64.reg;
-    Register tmp = tmp64.reg;
 
     if (AssemblerX86Shared::HasPOPCNT()) {
+        MOZ_ASSERT(tmp == InvalidReg);
         popcntq(src, dest);
         return;
     }

js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp

@@ -4833,5 +4833,17 @@ CodeGeneratorX86Shared::visitRotateI64(LRotateI64* lir)
     }
 }
 
+void
+CodeGeneratorX86Shared::visitPopcntI64(LPopcntI64* lir)
+{
+    Register64 input = ToRegister64(lir->getInt64Operand(0));
+    Register64 output = ToOutRegister64(lir);
+    Register temp = InvalidReg;
+    if (!AssemblerX86Shared::HasPOPCNT())
+        temp = ToRegister(lir->getTemp(0));
+
+    masm.popcnt64(input, output, temp);
+}
+
 } // namespace jit
 } // namespace js

js/src/jit/x86-shared/CodeGenerator-x86-shared.h

@@ -234,6 +234,7 @@ class CodeGeneratorX86Shared : public CodeGeneratorShared
     virtual void visitClzI(LClzI* ins);
     virtual void visitCtzI(LCtzI* ins);
     virtual void visitPopcntI(LPopcntI* ins);
+    virtual void visitPopcntI64(LPopcntI64* lir);
     virtual void visitSqrtD(LSqrtD* ins);
     virtual void visitSqrtF(LSqrtF* ins);
     virtual void visitPowHalfD(LPowHalfD* ins);

js/src/jit/x86/MacroAssembler-x86-inl.h

@@ -544,6 +544,29 @@ MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest, Regi
         xchgl(dest.high, dest.low);
 }
 
+// ===============================================================
+// Bit counting functions
+
+void
+MacroAssembler::popcnt64(Register64 src, Register64 dest, Register tmp)
+{
+    // The tmp register is only needed if there is no native POPCNT.
+
+    MOZ_ASSERT(src.low != tmp && src.high != tmp);
+    MOZ_ASSERT(dest.low != tmp && dest.high != tmp);
+
+    if (dest.low != src.high) {
+        popcnt32(src.low, dest.low, tmp);
+        popcnt32(src.high, dest.high, tmp);
+    } else {
+        MOZ_ASSERT(dest.high != src.high);
+        popcnt32(src.low, dest.high, tmp);
+        popcnt32(src.high, dest.low, tmp);
+    }
+    addl(dest.high, dest.low);
+    xorl(dest.high, dest.high);
+}
+
 // ===============================================================
 // Branch functions