Bug 1524136 - Fix BigInt errors in string conversion and remainder operations r=jwalden

Differential Revision: https://phabricator.services.mozilla.com/D18171

--HG--
extra : moz-landing-system : lando

js/src/tests/non262/BigInt/decimal.js

@@ -0,0 +1,31 @@
+// |reftest| skip-if(!this.hasOwnProperty("BigInt"))
+// Any copyright is dedicated to the Public Domain.
+// https://creativecommons.org/licenses/publicdomain/
+
+// Check base-10 BigInt string conversion
+const decimalTests = [
+    [32n, -1n, 1n, "4294967295"],
+    [32n, -1n, -1n, "-4294967295"],
+    [32n, 0n, 1n, "4294967296"],
+    [32n, 0n, -1n, "-4294967296"],
+    [32n, 1n, 1n, "4294967297"],
+    [32n, 1n, -1n, "-4294967297"],
+    [64n, -1n, 1n, "18446744073709551615"],
+    [64n, -1n, -1n, "-18446744073709551615"],
+    [64n, 0n, 1n, "18446744073709551616"],
+    [64n, 0n, -1n, "-18446744073709551616"],
+    [64n, 1n, 1n, "18446744073709551617"],
+    [64n, 1n, -1n, "-18446744073709551617"],
+    [128n, -1n, 1n, "340282366920938463463374607431768211455"],
+    [128n, -1n, -1n, "-340282366920938463463374607431768211455"],
+    [128n, 0n, 1n, "340282366920938463463374607431768211456"],
+    [128n, 0n, -1n, "-340282366920938463463374607431768211456"],
+    [128n, 1n, 1n, "340282366920938463463374607431768211457"],
+    [128n, 1n, -1n, "-340282366920938463463374607431768211457"],
+];
+for (const [power, offset, sign, result] of decimalTests) {
+    assertEq(((2n**power+offset)*sign).toString(),
+             result);
+}
+
+reportCompare(true, true);

js/src/tests/non262/BigInt/mod.js

@@ -0,0 +1,9 @@
+// |reftest| skip-if(!this.hasOwnProperty("BigInt"))
+// Any copyright is dedicated to the Public Domain.
+// https://creativecommons.org/licenses/publicdomain/
+
+// Check that |x % x| returns zero when |x| contains multiple digits
+assertEq(0x10000000000000000n % 0x10000000000000000n, 0n);
+assertEq(-0x10000000000000000n % -0x10000000000000000n, 0n);
+
+reportCompare(true, true);

js/src/vm/BigIntType.cpp

@@ -1164,26 +1164,29 @@ JSLinearString* BigInt::toStringSingleDigitBaseTen(JSContext* cx, Digit digit, b
 
   MOZ_ASSERT(digit != 0, "zero case should have been handled in toString");
 
-  const size_t charsRequired = CeilDiv(digit, 10) + isNegative;
-  auto resultChars = cx->make_pod_array<char>(charsRequired);
-  if (!resultChars) {
-    return nullptr;
-  }
+  constexpr size_t maxLength = 1 + (std::numeric_limits<Digit>::digits10 + 1);
+  static_assert(maxLength == 11 || maxLength == 21,
+                "unexpected decimal string length");
+
+  char resultChars[maxLength];
+  size_t writePos = maxLength;
 
-  size_t pos = charsRequired;
   while (digit != 0) {
-    MOZ_ASSERT(pos);
-    resultChars[--pos] = radixDigits[digit % 10];
+    MOZ_ASSERT(writePos > 0);
+    resultChars[--writePos] = radixDigits[digit % 10];
     digit /= 10;
   }
+  MOZ_ASSERT(writePos < maxLength);
+  MOZ_ASSERT(resultChars[writePos] != '0');
 
   if (isNegative) {
-    MOZ_ASSERT(pos);
-    resultChars[--pos] = '-';
+    MOZ_ASSERT(writePos > 0);
+    resultChars[--writePos] = '-';
   }
 
-  MOZ_ASSERT(pos == 0);
-  return NewStringCopyN<allowGC>(cx, resultChars.get(), charsRequired);
+  MOZ_ASSERT(writePos < maxLength);
+  return NewStringCopyN<allowGC>(cx, resultChars + writePos,
+                                 maxLength - writePos);
 }
 
 static constexpr BigInt::Digit MaxPowerInDigit(uint8_t radix) {
@@ -1806,7 +1809,7 @@ BigInt* BigInt::mod(JSContext* cx, HandleBigInt x, HandleBigInt y) {
       return nullptr;
     }
     MOZ_ASSERT(remainder);
-    return remainder;
+    return destructivelyTrimHighZeroDigits(cx, remainder);
   }
 }
 
@@ -2213,7 +2216,7 @@ uint64_t BigInt::toUint64(BigInt* x) {
 
   uint64_t digit = x->digit(0);
 
-  if (DigitBits == 32 && x->digitLength() >= 1) {
+  if (DigitBits == 32 && x->digitLength() > 1) {
     digit |= static_cast<uint64_t>(x->digit(1)) << 32;
   }