Bug 1657830 part 6 - Don't use extended jumps for cross-JitCode jumps on x64. r=tcampbell

A lot of entries in the extended jump table were never used because they were for
jumps/calls to other addresses in the executable memory (JitCodes or trampolines).

This patch takes advantage of the contiguous 2 GB executable code buffer: we know
any address in this buffer can always be jumped to without needing an extended jump
table. This also lets us simplify the jump relocation code more.

With Fission coming soon, max 2 GB JIT code per process will hopefully be sufficient.

Depends on D86374

Differential Revision: https://phabricator.services.mozilla.com/D86375

js/src/jit/ProcessExecutableMemory.cpp

@@ -571,6 +571,11 @@ class ProcessExecutableMemory {
                            uintptr_t(base_) + MaxCodeBytesPerProcess);
   }
 
+  bool containsAddress(const void* p) const {
+    return p >= base_ &&
+           uintptr_t(p) < uintptr_t(base_) + MaxCodeBytesPerProcess;
+  }
+
   void* allocate(size_t bytes, ProtectionSetting protection,
                  MemCheckKind checkKind);
   void deallocate(void* addr, size_t bytes, bool decommit);
@@ -725,6 +730,10 @@ bool js::jit::CanLikelyAllocateMoreExecutableMemory() {
   return execMemory.bytesAllocated() + BufferSize <= MaxCodeBytesPerProcess;
 }
 
+bool js::jit::AddressIsInExecutableMemory(const void* p) {
+  return execMemory.containsAddress(p);
+}
+
 bool js::jit::ReprotectRegion(void* start, size_t size,
                               ProtectionSetting protection,
                               MustFlushICache flushICache) {

js/src/jit/ProcessExecutableMemory.h

@@ -100,6 +100,9 @@ extern bool CanLikelyAllocateMoreExecutableMemory();
 // the process allocate executable memory.
 extern size_t LikelyAvailableExecutableMemory();
 
+// Returns whether |p| is stored in the executable code buffer.
+extern bool AddressIsInExecutableMemory(const void* p);
+
 }  // namespace jit
 }  // namespace js
 

js/src/jit/x64/Assembler-x64.cpp

@@ -119,8 +119,20 @@ void Assembler::addPendingJump(JmpSrc src, ImmPtr target,
   if (reloc == RelocationKind::JITCODE) {
     jumpRelocations_.writeUnsigned(src.offset());
   }
-  enoughMemory_ &=
-      jumps_.append(RelativePatch(src.offset(), target.value, reloc));
+
+  static_assert(MaxCodeBytesPerProcess <= uint64_t(2) * 1024 * 1024 * 1024,
+                "Code depends on using int32_t for cross-JitCode jump offsets");
+
+  MOZ_ASSERT_IF(reloc == RelocationKind::JITCODE,
+                AddressIsInExecutableMemory(target.value));
+
+  RelativePatch patch(src.offset(), target.value, reloc);
+  if (reloc == RelocationKind::JITCODE ||
+      AddressIsInExecutableMemory(target.value)) {
+    enoughMemory_ &= codeJumps_.append(patch);
+  } else {
+    enoughMemory_ &= extendedJumps_.append(patch);
+  }
 }
 
 void Assembler::finish() {
@@ -128,7 +140,7 @@ void Assembler::finish() {
     return;
   }
 
-  if (!jumps_.length()) {
+  if (!extendedJumps_.length()) {
     // Since we may be folowed by non-executable data, eagerly insert an
     // undefined instruction byte to prevent processors from decoding
     // gibberish into their pipelines. See Intel performance guides.
@@ -141,7 +153,7 @@ void Assembler::finish() {
   extendedJumpTable_ = masm.size();
 
   // Zero the extended jumps table.
-  for (size_t i = 0; i < jumps_.length(); i++) {
+  for (size_t i = 0; i < extendedJumps_.length(); i++) {
 #ifdef DEBUG
     size_t oldSize = masm.size();
 #endif
@@ -160,10 +172,19 @@ void Assembler::finish() {
 void Assembler::executableCopy(uint8_t* buffer) {
   AssemblerX86Shared::executableCopy(buffer);
 
-  for (size_t i = 0; i < jumps_.length(); i++) {
-    RelativePatch& rp = jumps_[i];
+  for (RelativePatch& rp : codeJumps_) {
     uint8_t* src = buffer + rp.offset;
     MOZ_ASSERT(rp.target);
+
+    MOZ_RELEASE_ASSERT(X86Encoding::CanRelinkJump(src, rp.target));
+    X86Encoding::SetRel32(src, rp.target);
+  }
+
+  for (size_t i = 0; i < extendedJumps_.length(); i++) {
+    RelativePatch& rp = extendedJumps_[i];
+    uint8_t* src = buffer + rp.offset;
+    MOZ_ASSERT(rp.target);
+
     if (X86Encoding::CanRelinkJump(src, rp.target)) {
       X86Encoding::SetRel32(src, rp.target);
     } else {
@@ -204,15 +225,9 @@ class RelocationIterator {
 
 JitCode* Assembler::CodeFromJump(JitCode* code, uint8_t* jump) {
   uint8_t* target = (uint8_t*)X86Encoding::GetRel32Target(jump);
-  if (target >= code->raw() &&
-      target < code->raw() + code->instructionsSize()) {
-    // This jump is within the code buffer, so it has been redirected to
-    // the extended jump table.
-    MOZ_ASSERT(target + SizeOfJumpTableEntry <=
-               code->raw() + code->instructionsSize());
 
-    target = (uint8_t*)X86Encoding::GetPointer(target + SizeOfExtendedJump);
-  }
+  MOZ_ASSERT(!code->containsNativePC(target),
+             "Extended jump table not used for cross-JitCode jumps");
 
   return JitCode::FromExecutable(target);
 }

js/src/jit/x64/Assembler-x64.h

@@ -292,8 +292,6 @@ class Assembler : public AssemblerX86Shared {
   // jump table at the bottom of the instruction stream, and if a jump
   // overflows its range, it will redirect here.
   //
-  // The relocation table stores the offset to the short jump.
-  //
   // Each entry in this table is a jmp [rip], followed by a ud2 to hint to the
   // hardware branch predictor that there is no fallthrough, followed by the
   // eight bytes containing an immediate address. This comes out to 16 bytes.
@@ -306,7 +304,19 @@ class Assembler : public AssemblerX86Shared {
   static const uint32_t SizeOfExtendedJump = 1 + 1 + 4 + 2 + 8;
   static const uint32_t SizeOfJumpTableEntry = 16;
 
-  Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;
+  // Two kinds of jumps on x64:
+  //
+  // * codeJumps_ tracks jumps with target within the executable code region
+  //   for the process. These jumps don't need entries in the extended jump
+  //   table because source and target must be within 2 GB of each other.
+  //
+  // * extendedJumps_ tracks jumps with target outside the executable code
+  //   region. These jumps need entries in the extended jump table described
+  //   above.
+  using PendingJumpVector = Vector<RelativePatch, 8, SystemAllocPolicy>;
+  PendingJumpVector codeJumps_;
+  PendingJumpVector extendedJumps_;
+
   uint32_t extendedJumpTable_;
 
   static JitCode* CodeFromJump(JitCode* code, uint8_t* jump);
@@ -337,7 +347,10 @@ class Assembler : public AssemblerX86Shared {
   void assertNoGCThings() const {
 #ifdef DEBUG
     MOZ_ASSERT(dataRelocations_.length() == 0);
-    for (auto& j : jumps_) {
+    for (auto& j : codeJumps_) {
+      MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+    }
+    for (auto& j : extendedJumps_) {
       MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
     }
 #endif