Bug 1628227 - Add WarpCacheIRTranspiler prototype. r=iain

The transpiler lets us inline CacheIR IC stubs for JSOp::Get{G}Name ops.
It supports just enough instructions to pass jit-tests.

There are still missing pieces (see TODOs, especially in WarpOracle::maybeInlineIC)
and ideally the transpiler class should share more code with CacheIRCompiler
(for example the reader.readFoo() calls).

Differential Revision: https://phabricator.services.mozilla.com/D70355

--HG--
extra : moz-landing-system : lando

js/src/jit/CacheIRCompiler.cpp

@@ -963,10 +963,15 @@ static GCPtr<T>* AsGCPtr(uintptr_t* ptr) {
   return reinterpret_cast<GCPtr<T>*>(ptr);
 }
 
+uintptr_t CacheIRStubInfo::getStubRawWord(const uint8_t* stubData,
+                                          uint32_t offset) const {
+  MOZ_ASSERT(uintptr_t(stubData) % sizeof(uintptr_t) == 0);
+  return *reinterpret_cast<const uintptr_t*>(stubData + offset);
+}
+
 uintptr_t CacheIRStubInfo::getStubRawWord(ICStub* stub, uint32_t offset) const {
   uint8_t* stubData = (uint8_t*)stub + stubDataOffset_;
-  MOZ_ASSERT(uintptr_t(stubData) % sizeof(uintptr_t) == 0);
-  return *(uintptr_t*)(stubData + offset);
+  return getStubRawWord(stubData, offset);
 }
 
 template <class Stub, class T>

js/src/jit/CacheIRCompiler.h

@@ -1261,14 +1261,15 @@ class CacheIRStubInfo {
                               const CacheIRWriter& writer);
 
   template <class Stub, class T>
-  js::GCPtr<T>& getStubField(Stub* stub, uint32_t field) const;
+  js::GCPtr<T>& getStubField(Stub* stub, uint32_t offset) const;
 
   template <class T>
-  js::GCPtr<T>& getStubField(ICStub* stub, uint32_t field) const {
-    return getStubField<ICStub, T>(stub, field);
+  js::GCPtr<T>& getStubField(ICStub* stub, uint32_t offset) const {
+    return getStubField<ICStub, T>(stub, offset);
   }
 
-  uintptr_t getStubRawWord(ICStub* stub, uint32_t field) const;
+  uintptr_t getStubRawWord(const uint8_t* stubData, uint32_t offset) const;
+  uintptr_t getStubRawWord(ICStub* stub, uint32_t offset) const;
 };
 
 template <typename T>

js/src/jit/WarpBuilder.cpp

@@ -9,6 +9,7 @@
 #include "jit/MIR.h"
 #include "jit/MIRGenerator.h"
 #include "jit/MIRGraph.h"
+#include "jit/WarpCacheIRTranspiler.h"
 #include "jit/WarpSnapshot.h"
 #include "vm/Opcodes.h"
 
@@ -1747,6 +1748,10 @@ MConstant* WarpBuilder::globalLexicalEnvConstant() {
 }
 
 bool WarpBuilder::buildGetNameOp(BytecodeLocation loc, MDefinition* env) {
+  if (auto* snapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+    return buildCacheIR(loc, snapshot, env);
+  }
+
   MGetNameCache* ins = MGetNameCache::New(alloc(), env);
   current->add(ins);
   current->push(ins);
@@ -2779,3 +2784,21 @@ bool WarpBuilder::build_ThrowSetConst(BytecodeLocation loc) {
   setTerminatedBlock();
   return true;
 }
+
+bool WarpBuilder::buildCacheIR(BytecodeLocation loc,
+                               const WarpCacheIR* snapshot,
+                               MDefinition* input) {
+  MDefinitionStackVector inputs;
+  MOZ_ALWAYS_TRUE(inputs.append(input));  // Can't fail due to inline capacity.
+
+  TranspilerOutput output;
+  if (!TranspileCacheIRToMIR(mirGen_, current, snapshot, inputs, output)) {
+    return false;
+  }
+
+  if (output.result) {
+    current->push(output.result);
+  }
+
+  return true;
+}

js/src/jit/WarpBuilder.h

@@ -140,6 +140,10 @@ class MOZ_STACK_CLASS WarpBuilder {
   MOZ_MUST_USE bool buildBody();
   MOZ_MUST_USE bool buildEpilogue();
 
+  MOZ_MUST_USE bool buildCacheIR(BytecodeLocation loc,
+                                 const WarpCacheIR* snapshot,
+                                 MDefinition* input);
+
   MOZ_MUST_USE bool buildEnvironmentChain();
   MInstruction* buildNamedLambdaEnv(MDefinition* callee, MDefinition* env,
                                     LexicalEnvironmentObject* templateObj);

js/src/jit/WarpCacheIRTranspiler.cpp

@@ -0,0 +1,207 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/WarpCacheIRTranspiler.h"
+
+#include "jit/CacheIR.h"
+#include "jit/CacheIRCompiler.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "jit/WarpSnapshot.h"
+
+using namespace js;
+using namespace js::jit;
+
+// List of supported ops. Eventually we should use the full CacheIR ops list
+// instead.
+#define WARP_CACHE_IR_OPS(_)          \
+  _(GuardShape)                       \
+  _(LoadEnclosingEnvironment)         \
+  _(LoadDynamicSlotResult)            \
+  _(LoadEnvironmentFixedSlotResult)   \
+  _(LoadEnvironmentDynamicSlotResult) \
+  _(TypeMonitorResult)
+
+// The CacheIR transpiler generates MIR from Baseline CacheIR.
+class MOZ_RAII WarpCacheIRTranspiler {
+  TempAllocator& alloc_;
+  const CacheIRStubInfo* stubInfo_;
+  const uint8_t* stubData_;
+  TranspilerOutput& output_;
+
+  // Vector mapping OperandId to corresponding MDefinition.
+  MDefinitionStackVector operands_;
+
+  CacheIRReader reader;
+  MBasicBlock* current;
+
+  TempAllocator& alloc() { return alloc_; }
+
+  // CacheIR instructions writing to the IC's result register (the *Result
+  // instructions) must call this to pass the corresponding MIR node back to
+  // WarpBuilder.
+  void setResult(MDefinition* result) {
+    MOZ_ASSERT(!output_.result, "Can't have more than one result");
+    output_.result = result;
+  }
+
+  MDefinition* getOperand(OperandId id) const { return operands_[id.id()]; }
+
+  void setOperand(OperandId id, MDefinition* def) { operands_[id.id()] = def; }
+
+  MOZ_MUST_USE bool defineOperand(OperandId id, MDefinition* def) {
+    MOZ_ASSERT(id.id() == operands_.length());
+    return operands_.append(def);
+  }
+
+  uintptr_t readStubWord(uint32_t offset) {
+    return stubInfo_->getStubRawWord(stubData_, offset);
+  }
+
+  Shape* shapeStubField(uint32_t offset) {
+    return reinterpret_cast<Shape*>(readStubWord(offset));
+  }
+  int32_t int32StubField(uint32_t offset) {
+    return static_cast<int32_t>(readStubWord(offset));
+  }
+
+#define DEFINE_OP(op, ...) MOZ_MUST_USE bool transpile_##op();
+  WARP_CACHE_IR_OPS(DEFINE_OP)
+#undef DEFINE_OP
+
+ public:
+  WarpCacheIRTranspiler(MIRGenerator& mirGen, MBasicBlock* current,
+                        const WarpCacheIR* snapshot, TranspilerOutput& output)
+      : alloc_(mirGen.alloc()),
+        stubInfo_(snapshot->stubInfo()),
+        stubData_(snapshot->stubData()),
+        output_(output),
+        reader(stubInfo_),
+        current(current) {}
+
+  MOZ_MUST_USE bool transpile(const MDefinitionStackVector& inputs);
+};
+
+bool WarpCacheIRTranspiler::transpile(const MDefinitionStackVector& inputs) {
+  if (!operands_.appendAll(inputs)) {
+    return false;
+  }
+
+  do {
+    CacheOp op = reader.readOp();
+    switch (op) {
+#define DEFINE_OP(op, ...)   \
+  case CacheOp::op:          \
+    if (!transpile_##op()) { \
+      return false;          \
+    }                        \
+    break;
+      WARP_CACHE_IR_OPS(DEFINE_OP)
+#undef DEFINE_OP
+
+      default:
+        fprintf(stderr, "Unsupported op: %s\n", CacheIrOpNames[size_t(op)]);
+        MOZ_CRASH("Unsupported op");
+    }
+  } while (reader.more());
+
+  return true;
+}
+
+bool WarpCacheIRTranspiler::transpile_GuardShape() {
+  ObjOperandId objId = reader.objOperandId();
+  MDefinition* def = getOperand(objId);
+  Shape* shape = shapeStubField(reader.stubOffset());
+
+  auto* ins = MGuardShape::New(alloc(), def, shape, Bailout_ShapeGuard);
+  current->add(ins);
+
+  setOperand(objId, ins);
+  return true;
+}
+
+bool WarpCacheIRTranspiler::transpile_LoadEnclosingEnvironment() {
+  ObjOperandId inputId = reader.objOperandId();
+  ObjOperandId outputId = reader.objOperandId();
+
+  MDefinition* env = getOperand(inputId);
+  auto* ins = MEnclosingEnvironment::New(alloc(), env);
+  current->add(ins);
+
+  return defineOperand(outputId, ins);
+}
+
+bool WarpCacheIRTranspiler::transpile_LoadDynamicSlotResult() {
+  ObjOperandId objId = reader.objOperandId();
+  int32_t offset = int32StubField(reader.stubOffset());
+
+  MDefinition* obj = getOperand(objId);
+  size_t slotIndex = NativeObject::getDynamicSlotIndexFromOffset(offset);
+
+  auto* slots = MSlots::New(alloc(), obj);
+  current->add(slots);
+
+  auto* load = MLoadSlot::New(alloc(), slots, slotIndex);
+  current->add(load);
+
+  setResult(load);
+  return true;
+}
+
+bool WarpCacheIRTranspiler::transpile_LoadEnvironmentFixedSlotResult() {
+  ObjOperandId objId = reader.objOperandId();
+  int32_t offset = int32StubField(reader.stubOffset());
+
+  MDefinition* obj = getOperand(objId);
+  uint32_t slotIndex = NativeObject::getFixedSlotIndexFromOffset(offset);
+
+  auto* load = MLoadFixedSlot::New(alloc(), obj, slotIndex);
+  current->add(load);
+
+  auto* lexicalCheck = MLexicalCheck::New(alloc(), load);
+  current->add(lexicalCheck);
+
+  setResult(lexicalCheck);
+  return true;
+}
+
+bool WarpCacheIRTranspiler::transpile_LoadEnvironmentDynamicSlotResult() {
+  ObjOperandId objId = reader.objOperandId();
+  int32_t offset = int32StubField(reader.stubOffset());
+
+  MDefinition* obj = getOperand(objId);
+  size_t slotIndex = NativeObject::getDynamicSlotIndexFromOffset(offset);
+
+  auto* slots = MSlots::New(alloc(), obj);
+  current->add(slots);
+
+  auto* load = MLoadSlot::New(alloc(), slots, slotIndex);
+  current->add(load);
+
+  auto* lexicalCheck = MLexicalCheck::New(alloc(), load);
+  current->add(lexicalCheck);
+
+  setResult(lexicalCheck);
+  return true;
+}
+
+bool WarpCacheIRTranspiler::transpile_TypeMonitorResult() {
+  MOZ_ASSERT(output_.result, "Didn't set result MDefinition");
+  return true;
+}
+
+bool jit::TranspileCacheIRToMIR(MIRGenerator& mirGen, MBasicBlock* current,
+                                const WarpCacheIR* snapshot,
+                                const MDefinitionStackVector& inputs,
+                                TranspilerOutput& output) {
+  WarpCacheIRTranspiler transpiler(mirGen, current, snapshot, output);
+  if (!transpiler.transpile(inputs)) {
+    return false;
+  }
+
+  return true;
+}

js/src/jit/WarpCacheIRTranspiler.h

@@ -0,0 +1,46 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_WarpCacheIRTranspiler_h
+#define jit_WarpCacheIRTranspiler_h
+
+#include "js/AllocPolicy.h"
+#include "js/Vector.h"
+
+namespace js {
+namespace jit {
+
+class MBasicBlock;
+class MDefinition;
+class MInstruction;
+class MIRGenerator;
+class WarpCacheIR;
+
+using MDefinitionStackVector = Vector<MDefinition*, 8, SystemAllocPolicy>;
+
+// TranspilerOutput contains information from the transpiler that needs to be
+// passed back to WarpBuilder.
+struct MOZ_STACK_CLASS TranspilerOutput {
+  // For ICs that return a result, this is the corresponding MIR instruction.
+  MDefinition* result = nullptr;
+
+  TranspilerOutput() = default;
+
+  TranspilerOutput(const TranspilerOutput&) = delete;
+  void operator=(const TranspilerOutput&) = delete;
+};
+
+// Generate MIR from a Baseline ICStub's CacheIR.
+MOZ_MUST_USE bool TranspileCacheIRToMIR(MIRGenerator& mirGen,
+                                        MBasicBlock* current,
+                                        const WarpCacheIR* snapshot,
+                                        const MDefinitionStackVector& inputs,
+                                        TranspilerOutput& output);
+
+}  // namespace jit
+}  // namespace js
+
+#endif /* jit_WarpCacheIRTranspiler_h */

js/src/jit/WarpOracle.cpp

@@ -9,6 +9,9 @@
 #include "mozilla/IntegerPrintfMacros.h"
 #include "mozilla/ScopeExit.h"
 
+#include <algorithm>
+
+#include "jit/CacheIRCompiler.h"
 #include "jit/JitScript.h"
 #include "jit/JitSpewer.h"
 #include "jit/MIRGenerator.h"
@@ -380,6 +383,11 @@ AbortReasonOr<WarpScriptSnapshot*> WarpOracle::createScriptSnapshot(
         break;
       }
 
+      case JSOp::GetName:
+      case JSOp::GetGName:
+        MOZ_TRY(maybeInlineIC(opSnapshots, script, loc));
+        break;
+
       case JSOp::Nop:
       case JSOp::NopDestructuring:
       case JSOp::TryDestructuring:
@@ -488,8 +496,6 @@ AbortReasonOr<WarpScriptSnapshot*> WarpOracle::createScriptSnapshot(
       case JSOp::FunApply:
       case JSOp::New:
       case JSOp::SuperCall:
-      case JSOp::GetName:
-      case JSOp::GetGName:
       case JSOp::BindName:
       case JSOp::BindGName:
       case JSOp::GetProp:
@@ -595,3 +601,77 @@ AbortReasonOr<WarpScriptSnapshot*> WarpOracle::createScriptSnapshot(
 
   return scriptSnapshot;
 }
+
+AbortReasonOr<Ok> WarpOracle::maybeInlineIC(WarpOpSnapshotList& snapshots,
+                                            HandleScript script,
+                                            BytecodeLocation loc) {
+  // Add a WarpCacheIR snapshot if the Baseline IC has a single ICStub we can
+  // inline.
+
+  MOZ_ASSERT(loc.opHasIC());
+
+  uint32_t offset = loc.bytecodeToOffset(script);
+
+  // TODO: slow. Should traverse ICEntries as we go, like BaselineCompiler.
+  const ICEntry& entry = script->jitScript()->icEntryFromPCOffset(offset);
+
+  ICStub* stub = entry.firstStub();
+
+  if (stub->isFallback()) {
+    // No optimized stubs.
+    // TODO: add logging for failure/success cases.
+    return Ok();
+  }
+
+  if (!stub->next()->isFallback()) {
+    // More than one optimized stub.
+    return Ok();
+  }
+
+  // TODO: check stub's hit count if we're not doing eager compilation.
+  // TODO: check stub data for nursery pointers.
+  // TODO: don't inline if the IC had unhandled cases => CacheIR is incomplete.
+  // TOOD: have a consistent bailout => invalidate story. Set a flag on the IC?
+
+  const CacheIRStubInfo* stubInfo = nullptr;
+  const uint8_t* stubData = nullptr;
+  switch (stub->kind()) {
+    case ICStub::CacheIR_Regular:
+      stubInfo = stub->toCacheIR_Regular()->stubInfo();
+      stubData = stub->toCacheIR_Regular()->stubDataStart();
+      break;
+    case ICStub::CacheIR_Monitored:
+      stubInfo = stub->toCacheIR_Monitored()->stubInfo();
+      stubData = stub->toCacheIR_Monitored()->stubDataStart();
+      break;
+    case ICStub::CacheIR_Updated:
+      stubInfo = stub->toCacheIR_Updated()->stubInfo();
+      stubData = stub->toCacheIR_Updated()->stubDataStart();
+      break;
+    default:
+      MOZ_CRASH("Unexpected stub");
+  }
+
+  // Copy the ICStub data to protect against the stub being unlinked or mutated.
+  // We don't need to copy the CacheIRStubInfo: because we store and trace the
+  // stub's JitCode*, the baselineCacheIRStubCodes_ map in JitZone will keep it
+  // alive.
+  size_t bytesNeeded = stubInfo->stubDataSize();
+  uint8_t* stubDataCopy = alloc_.allocateArray<uint8_t>(bytesNeeded);
+  if (!stubDataCopy) {
+    return abort(AbortReason::Alloc);
+  }
+
+  // We don't need any GC barriers because the stub data does not contain
+  // nursery pointers (checked above) so we can do a bitwise copy.
+  std::copy_n(stubData, bytesNeeded, stubDataCopy);
+
+  JitCode* jitCode = stub->jitCode();
+
+  if (!AddOpSnapshot<WarpCacheIR>(alloc_, snapshots, offset, jitCode, stubInfo,
+                                  stubDataCopy)) {
+    return abort(AbortReason::Alloc);
+  }
+
+  return Ok();
+}

js/src/jit/WarpOracle.h

@@ -31,6 +31,9 @@ class MOZ_STACK_CLASS WarpOracle {
   AbortReasonOr<WarpEnvironment> createEnvironment(HandleScript script);
   AbortReasonOr<WarpScriptSnapshot*> createScriptSnapshot(HandleScript script);
 
+  AbortReasonOr<Ok> maybeInlineIC(WarpOpSnapshotList& snapshots,
+                                  HandleScript script, BytecodeLocation loc);
+
  public:
   WarpOracle(JSContext* cx, MIRGenerator& mirGen, HandleScript script);
 

js/src/jit/WarpSnapshot.cpp

@@ -141,6 +141,13 @@ void WarpLambda::dumpData(GenericPrinter& out) const {
 void WarpRest::dumpData(GenericPrinter& out) const {
   out.printf("    template: 0x%p\n", templateObject());
 }
+
+void WarpCacheIR::dumpData(GenericPrinter& out) const {
+  out.printf("    stubCode: 0x%p\n", static_cast<JitCode*>(stubCode_));
+  out.printf("    stubInfo: 0x%p\n", stubInfo_);
+  out.printf("    stubData: 0x%p\n", stubData_);
+  // TODO: print CacheIR
+}
 #endif  // JS_JITSPEW
 
 template <typename T>
@@ -227,3 +234,8 @@ void WarpLambda::traceData(JSTracer* trc) {
 void WarpRest::traceData(JSTracer* trc) {
   TraceWarpGCPtr(trc, templateObject_, "warp-rest-template");
 }
+
+void WarpCacheIR::traceData(JSTracer* trc) {
+  TraceWarpGCPtr(trc, stubCode_, "warp-stub-code");
+  // TODO: trace pointers in stub data.
+}

js/src/jit/WarpSnapshot.h

@@ -21,6 +21,8 @@ class ModuleEnvironmentObject;
 
 namespace jit {
 
+class CacheIRStubInfo;
+
 #define WARP_OP_SNAPSHOT_LIST(_) \
   _(WarpArguments)               \
   _(WarpRegExp)                  \
@@ -28,7 +30,8 @@ namespace jit {
   _(WarpGetIntrinsic)            \
   _(WarpGetImport)               \
   _(WarpLambda)                  \
-  _(WarpRest)
+  _(WarpRest)                    \
+  _(WarpCacheIR)
 
 // Wrapper for GC things stored in WarpSnapshot. Asserts the GC pointer is not
 // nursery-allocated. These pointers must be traced using TraceWarpGCPtr.
@@ -229,7 +232,35 @@ class WarpLambda : public WarpOpSnapshot {
 #endif
 };
 
-// Template object for JSOp::Rest.
+// Information from a Baseline IC stub.
+class WarpCacheIR : public WarpOpSnapshot {
+  // Baseline stub code. Stored here to keep the CacheIRStubInfo alive.
+  WarpGCPtr<JitCode*> stubCode_;
+  const CacheIRStubInfo* stubInfo_;
+
+  // Copied Baseline stub data. Allocated in the same LifoAlloc.
+  const uint8_t* stubData_;
+
+ public:
+  static constexpr Kind ThisKind = Kind::WarpCacheIR;
+
+  WarpCacheIR(uint32_t offset, JitCode* stubCode,
+              const CacheIRStubInfo* stubInfo, const uint8_t* stubData)
+      : WarpOpSnapshot(ThisKind, offset),
+        stubCode_(stubCode),
+        stubInfo_(stubInfo),
+        stubData_(stubData) {}
+
+  const CacheIRStubInfo* stubInfo() const { return stubInfo_; }
+  const uint8_t* stubData() const { return stubData_; }
+
+  void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+  void dumpData(GenericPrinter& out) const;
+#endif
+};
+
 class WarpRest : public WarpOpSnapshot {
   WarpGCPtr<ArrayObject*> templateObject_;
 

js/src/jit/moz.build

@@ -89,6 +89,7 @@ UNIFIED_SOURCES += [
     'ValueNumbering.cpp',
     'VMFunctions.cpp',
     'WarpBuilder.cpp',
+    'WarpCacheIRTranspiler.cpp',
     'WarpOracle.cpp',
     'WarpSnapshot.cpp',
     'WasmBCE.cpp',

js/src/vm/BytecodeLocation.h

@@ -178,6 +178,7 @@ class BytecodeLocation {
     return IsBackedgeForLoopHead(rawBytecode_, loopHead.rawBytecode_);
   }
 
+  bool opHasIC() const { return BytecodeOpHasIC(getOp()); }
   bool opHasTypeSet() const { return BytecodeOpHasTypeSet(getOp()); }
 
   bool fallsThrough() const { return BytecodeFallsThrough(getOp()); }

js/src/vm/NativeObject.h

@@ -1478,6 +1478,17 @@ class NativeObject : public JSObject {
   static constexpr size_t getPrivateDataOffset(size_t nfixed) {
     return getFixedSlotOffset(nfixed);
   }
+  static constexpr size_t getFixedSlotIndexFromOffset(size_t offset) {
+    MOZ_ASSERT(offset >= sizeof(NativeObject));
+    offset -= sizeof(NativeObject);
+    MOZ_ASSERT(offset % sizeof(Value) == 0);
+    MOZ_ASSERT(offset / sizeof(Value) < MAX_FIXED_SLOTS);
+    return offset / sizeof(Value);
+  }
+  static constexpr size_t getDynamicSlotIndexFromOffset(size_t offset) {
+    MOZ_ASSERT(offset % sizeof(Value) == 0);
+    return offset / sizeof(Value);
+  }
   static size_t offsetOfSlots() { return offsetof(NativeObject, slots_); }
 };