Bug 1635375 part 4 - Bail out in Warp for cold Baseline ICs. r=iain

Until now we always used an Ion IC if there was no Baseline IC stub to transpile.
This isn't great for Baseline ICs that were never hit because we could get stuck
in Ion ICs instead of using the transpiler.

With this patch we bail out and if that happens more than 10 times we invalidate
the Warp code and then warm up and try a second time.

Depends on D81064

Differential Revision: https://phabricator.services.mozilla.com/D81065

js/src/jit/CodeGenerator.cpp

@@ -6093,6 +6093,14 @@ void CodeGenerator::visitEncodeSnapshot(LEncodeSnapshot* lir) {
   encode(lir->snapshot());
 }
 
+void CodeGenerator::visitUnreachableResultV(LUnreachableResultV* lir) {
+  masm.assumeUnreachable("must be unreachable");
+}
+
+void CodeGenerator::visitUnreachableResultT(LUnreachableResultT* lir) {
+  masm.assumeUnreachable("must be unreachable");
+}
+
 void CodeGenerator::visitGetDynamicName(LGetDynamicName* lir) {
   Register envChain = ToRegister(lir->getEnvironmentChain());
   Register name = ToRegister(lir->getName());

js/src/jit/Lowering.cpp

@@ -544,6 +544,16 @@ void LIRGenerator::visitEncodeSnapshot(MEncodeSnapshot* mir) {
   add(lir, mir);
 }
 
+void LIRGenerator::visitUnreachableResult(MUnreachableResult* mir) {
+  if (mir->type() == MIRType::Value) {
+    auto* lir = new (alloc()) LUnreachableResultV();
+    defineBox(lir, mir);
+  } else {
+    auto* lir = new (alloc()) LUnreachableResultT();
+    define(lir, mir);
+  }
+}
+
 void LIRGenerator::visitAssertFloat32(MAssertFloat32* assertion) {
   MIRType type = assertion->input()->type();
   DebugOnly<bool> checkIsFloat32 = assertion->mustBeFloat32();

js/src/jit/MIR.h

@@ -12747,6 +12747,25 @@ class MUnknownValue : public MNullaryInstruction {
   TRIVIAL_NEW_WRAPPERS
 };
 
+// Used by MIR building to represent the bytecode result of an operation for
+// which an MBail was generated, to balance the basic block's MDefinition stack.
+class MUnreachableResult : public MNullaryInstruction {
+ protected:
+  explicit MUnreachableResult(MIRType type) : MNullaryInstruction(classOpcode) {
+    MOZ_ASSERT(type != MIRType::None);
+    setResultType(type);
+  }
+
+ public:
+  INSTRUCTION_HEADER(UnreachableResult)
+  TRIVIAL_NEW_WRAPPERS
+
+  bool congruentTo(const MDefinition* ins) const override {
+    return congruentIfOperandsEqual(ins);
+  }
+  AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
 class MIonToWasmCall final : public MVariadicInstruction,
                              public NoTypePolicy::Data {
   CompilerGCPointer<WasmInstanceObject*> instanceObj_;

js/src/jit/WarpBuilder.cpp

@@ -39,7 +39,8 @@ BytecodeSite* WarpBuilder::newBytecodeSite(BytecodeLocation loc) {
   return new (alloc()) BytecodeSite(info().inlineScriptTree(), pc);
 }
 
-const WarpOpSnapshot* WarpBuilder::getOpSnapshotImpl(BytecodeLocation loc) {
+const WarpOpSnapshot* WarpBuilder::getOpSnapshotImpl(
+    BytecodeLocation loc, WarpOpSnapshot::Kind kind) {
   uint32_t offset = loc.bytecodeToOffset(script_);
 
   // Skip snapshots until we get to a snapshot with offset >= offset. This is
@@ -48,7 +49,8 @@ const WarpOpSnapshot* WarpBuilder::getOpSnapshotImpl(BytecodeLocation loc) {
     opSnapshotIter_ = opSnapshotIter_->getNext();
   }
 
-  if (!opSnapshotIter_ || opSnapshotIter_->offset() != offset) {
+  if (!opSnapshotIter_ || opSnapshotIter_->offset() != offset ||
+      opSnapshotIter_->kind() != kind) {
     return nullptr;
   }
 
@@ -1681,6 +1683,11 @@ bool WarpBuilder::buildCallOp(BytecodeLocation loc) {
                                  cacheIRSnapshot, callInfo);
   }
 
+  if (getOpSnapshot<WarpBailout>(loc)) {
+    callInfo.setImplicitlyUsedUnchecked();
+    return buildBailoutForColdIC(loc, CacheKind::Call);
+  }
+
   // TODO: consider adding a Call IC like Baseline has.
 
   bool needsThisCheck = false;
@@ -2726,6 +2733,13 @@ bool WarpBuilder::buildIC(BytecodeLocation loc, CacheKind kind,
                                  cacheIRSnapshot, inputs_);
   }
 
+  if (getOpSnapshot<WarpBailout>(loc)) {
+    for (MDefinition* input : inputs) {
+      input->setImplicitlyUsedUnchecked();
+    }
+    return buildBailoutForColdIC(loc, kind);
+  }
+
   // Work around std::initializer_list not defining operator[].
   auto getInput = [&](size_t index) -> MDefinition* {
     MOZ_ASSERT(index < numInputs);
@@ -2886,3 +2900,52 @@ bool WarpBuilder::buildIC(BytecodeLocation loc, CacheKind kind,
 
   return true;
 }
+
+bool WarpBuilder::buildBailoutForColdIC(BytecodeLocation loc, CacheKind kind) {
+  MOZ_ASSERT(loc.opHasIC());
+
+  // TODO: ideally we would terminate the block here and set the implicitly-used
+  // flag for skipped bytecode ops. OSR makes this more tricky though.
+  MBail* bail = MBail::New(alloc(), Bailout_FirstExecution);
+  current->add(bail);
+
+  MIRType resultType;
+  switch (kind) {
+    case CacheKind::UnaryArith:
+    case CacheKind::BinaryArith:
+    case CacheKind::GetName:
+    case CacheKind::GetProp:
+    case CacheKind::GetElem:
+    case CacheKind::GetPropSuper:
+    case CacheKind::GetElemSuper:
+    case CacheKind::GetIntrinsic:
+    case CacheKind::Call:
+    case CacheKind::ToPropertyKey:
+      resultType = MIRType::Value;
+      break;
+    case CacheKind::BindName:
+    case CacheKind::GetIterator:
+    case CacheKind::NewObject:
+      resultType = MIRType::Object;
+      break;
+    case CacheKind::TypeOf:
+      resultType = MIRType::String;
+      break;
+    case CacheKind::ToBool:
+    case CacheKind::Compare:
+    case CacheKind::In:
+    case CacheKind::HasOwn:
+    case CacheKind::InstanceOf:
+      resultType = MIRType::Boolean;
+      break;
+    case CacheKind::SetProp:
+    case CacheKind::SetElem:
+      return true;  // No result.
+  }
+
+  auto* ins = MUnreachableResult::New(alloc(), resultType);
+  current->add(ins);
+  current->push(ins);
+
+  return true;
+}

js/src/jit/WarpBuilder.h

@@ -101,11 +101,12 @@ class MOZ_STACK_CLASS WarpBuilder : public WarpBuilderShared {
 
   BytecodeSite* newBytecodeSite(BytecodeLocation loc);
 
-  const WarpOpSnapshot* getOpSnapshotImpl(BytecodeLocation loc);
+  const WarpOpSnapshot* getOpSnapshotImpl(BytecodeLocation loc,
+                                          WarpOpSnapshot::Kind kind);
 
   template <typename T>
   const T* getOpSnapshot(BytecodeLocation loc) {
-    const WarpOpSnapshot* snapshot = getOpSnapshotImpl(loc);
+    const WarpOpSnapshot* snapshot = getOpSnapshotImpl(loc, T::ThisKind);
     return snapshot ? snapshot->as<T>() : nullptr;
   }
 
@@ -133,6 +134,7 @@ class MOZ_STACK_CLASS WarpBuilder : public WarpBuilderShared {
 
   MOZ_MUST_USE bool buildIC(BytecodeLocation loc, CacheKind kind,
                             std::initializer_list<MDefinition*> inputs);
+  MOZ_MUST_USE bool buildBailoutForColdIC(BytecodeLocation loc, CacheKind kind);
 
   MOZ_MUST_USE bool buildEnvironmentChain();
   MInstruction* buildNamedLambdaEnv(MDefinition* callee, MDefinition* env,

js/src/jit/WarpOracle.cpp

@@ -713,14 +713,23 @@ static void LineNumberAndColumn(HandleScript script, BytecodeLocation loc,
 
 AbortReasonOr<Ok> WarpScriptOracle::maybeInlineIC(WarpOpSnapshotList& snapshots,
                                                   BytecodeLocation loc) {
-  // Add a WarpCacheIR snapshot if the Baseline IC has a single ICStub we can
-  // inline.
+  // Do one of the following:
+  //
+  // * If the Baseline IC has a single ICStub we can inline, add a WarpCacheIR
+  //   snapshot to transpile it to MIR.
+  //
+  // * If the Baseline IC is cold (never executed), add a WarpBailout snapshot
+  //   so that we can collect information in Baseline.
+  //
+  // * Else, don't add a snapshot and rely on WarpBuilder adding an Ion IC.
 
   MOZ_ASSERT(loc.opHasIC());
 
   const ICEntry& entry = getICEntry(loc);
   ICStub* stub = entry.firstStub();
 
+  uint32_t offset = loc.bytecodeToOffset(script_);
+
   if (stub->isFallback()) {
     [[maybe_unused]] unsigned line, column;
     LineNumberAndColumn(script_, loc, &line, &column);
@@ -731,6 +740,16 @@ AbortReasonOr<Ok> WarpScriptOracle::maybeInlineIC(WarpOpSnapshotList& snapshots,
             ") for JSOp::%s @ %s:%u:%u",
             stub->toFallbackStub()->enteredCount(), CodeName(loc.getOp()),
             script_->filename(), line, column);
+
+    // If the fallback stub was used but there's no optimized stub, use an IC.
+    if (stub->toFallbackStub()->enteredCount() != 0) {
+      return Ok();
+    }
+
+    // Cold IC. Bailout to collect information.
+    if (!AddOpSnapshot<WarpBailout>(alloc_, snapshots, offset)) {
+      return abort(AbortReason::Alloc);
+    }
     return Ok();
   }
 
@@ -846,7 +865,6 @@ AbortReasonOr<Ok> WarpScriptOracle::maybeInlineIC(WarpOpSnapshotList& snapshots,
 
   JitCode* jitCode = stub->jitCode();
 
-  uint32_t offset = loc.bytecodeToOffset(script_);
   if (!AddOpSnapshot<WarpCacheIR>(alloc_, snapshots, offset, jitCode, stubInfo,
                                   stubDataCopy)) {
     return abort(AbortReason::Alloc);

js/src/jit/WarpSnapshot.cpp

@@ -155,6 +155,10 @@ void WarpNewObject::dumpData(GenericPrinter& out) const {
   out.printf("    template: 0x%p\n", templateObject());
 }
 
+void WarpBailout::dumpData(GenericPrinter& out) const {
+  // No fields.
+}
+
 void WarpCacheIR::dumpData(GenericPrinter& out) const {
   out.printf("    stubCode: 0x%p\n", static_cast<JitCode*>(stubCode_));
   out.printf("    stubInfo: 0x%p\n", stubInfo_);
@@ -261,6 +265,10 @@ void WarpNewObject::traceData(JSTracer* trc) {
   TraceWarpGCPtr(trc, templateObject_, "warp-newobject-template");
 }
 
+void WarpBailout::traceData(JSTracer* trc) {
+  // No GC pointers.
+}
+
 void WarpCacheIR::traceData(JSTracer* trc) {
   TraceWarpGCPtr(trc, stubCode_, "warp-stub-code");
   // TODO: trace pointers in stub data.

js/src/jit/WarpSnapshot.h

@@ -34,6 +34,7 @@ class CacheIRStubInfo;
   _(WarpRest)                    \
   _(WarpNewArray)                \
   _(WarpNewObject)               \
+  _(WarpBailout)                 \
   _(WarpCacheIR)
 
 // Wrapper for GC things stored in WarpSnapshot. Asserts the GC pointer is not
@@ -82,6 +83,7 @@ class WarpOpSnapshot : public TempObject,
 
  public:
   uint32_t offset() const { return offset_; }
+  Kind kind() const { return kind_; }
 
   template <typename T>
   const T* as() const {
@@ -235,6 +237,20 @@ class WarpLambda : public WarpOpSnapshot {
 #endif
 };
 
+// Informs WarpBuilder that an IC site is cold and execution should bail out.
+class WarpBailout : public WarpOpSnapshot {
+ public:
+  static constexpr Kind ThisKind = Kind::WarpBailout;
+
+  explicit WarpBailout(uint32_t offset) : WarpOpSnapshot(ThisKind, offset) {}
+
+  void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+  void dumpData(GenericPrinter& out) const;
+#endif
+};
+
 // Information from a Baseline IC stub.
 class WarpCacheIR : public WarpOpSnapshot {
   // Baseline stub code. Stored here to keep the CacheIRStubInfo alive.

js/src/jit/shared/LIR-shared.h

@@ -1107,6 +1107,20 @@ class LEncodeSnapshot : public LInstructionHelper<0, 0, 0> {
   LEncodeSnapshot() : LInstructionHelper(classOpcode) {}
 };
 
+class LUnreachableResultV : public LInstructionHelper<BOX_PIECES, 0, 0> {
+ public:
+  LIR_HEADER(UnreachableResultV)
+
+  LUnreachableResultV() : LInstructionHelper(classOpcode) {}
+};
+
+class LUnreachableResultT : public LInstructionHelper<1, 0, 0> {
+ public:
+  LIR_HEADER(UnreachableResultT)
+
+  LUnreachableResultT() : LInstructionHelper(classOpcode) {}
+};
+
 template <size_t defs, size_t ops>
 class LDOMPropertyInstructionHelper
     : public LCallInstructionHelper<defs, 1 + ops, 3> {