Bug 1448136 - Ensure Debug OSR transition is respected in InstanceOf Fallback stub. r=jandem

--HG--
extra : rebase_source : 38eb775cc8846f00a3f0ef90e1f6b66bf0af7ab7

js/src/jit-test/tests/cacheir/bug1448136.js

@@ -0,0 +1,23 @@
+print = function(s) { return s.toString(); }
+assertEq = function(a,b) {
+  try { print(a); print(b); } catch(exc) {}
+}
+g = newGlobal();
+g.parent = this;
+g.eval("(" + function() {
+  Debugger(parent).onExceptionUnwind = function(frame) {
+    frame.older
+  }
+} + ")()")
+function a() {};
+function b() {};
+for (let _ of Array(100))
+  assertEq(b instanceof a, true);
+function c(){};
+function d(){};
+function e(){};
+Object.defineProperty(a, Symbol.hasInstance, {value: assertEq });
+let funcs = [a, b, c, d];
+for (let f of funcs)
+  assertEq(e instanceof f, true);
+

js/src/jit/BaselineIC.cpp

@@ -4146,9 +4146,12 @@ TryAttachInstanceOfStub(JSContext* cx, BaselineFrame* frame, ICInstanceOf_Fallba
 }
 
 static bool
-DoInstanceOfFallback(JSContext* cx, BaselineFrame* frame, ICInstanceOf_Fallback* stub,
+DoInstanceOfFallback(JSContext* cx, BaselineFrame* frame, ICInstanceOf_Fallback* stub_,
                      HandleValue lhs, HandleValue rhs, MutableHandleValue res)
 {
+    // This fallback stub may trigger debug mode toggling.
+    DebugModeOSRVolatileStub<ICInstanceOf_Fallback*> stub(ICStubEngine::Baseline, frame, stub_);
+
     FallbackICSpew(cx, stub, "InstanceOf");
 
     if (!rhs.isObject()) {
@@ -4163,6 +4166,10 @@ DoInstanceOfFallback(JSContext* cx, BaselineFrame* frame, ICInstanceOf_Fallback*
 
     res.setBoolean(cond);
 
+    // Check if debug mode toggling made the stub invalid.
+    if (stub.invalid())
+        return true;
+
     if (!obj->is<JSFunction>()) {
         stub->noteUnoptimizableAccess();
         return true;