Bug 1582727: wasm: tweak stack-limit comparison so it also works for interrupts; r=luke

Differential Revision: https://phabricator.services.mozilla.com/D46594

--HG--
extra : moz-landing-system : lando

js/src/jit-test/tests/wasm/timeout/stack-overflow.js

@@ -0,0 +1,97 @@
+// |jit-test| exitstatus: 3; skip-if: !wasmIsSupported()
+
+let { exports } = new WebAssembly.Instance(
+    new WebAssembly.Module(wasmTextToBinary(`
+    (module
+        (func (export "f") (param i32) (param i32) (param i32) (param i32) (result i32)
+            get_local 0
+            i32.popcnt
+            get_local 1
+            i32.popcnt
+            i32.add
+
+            get_local 2
+            i32.popcnt
+            get_local 3
+            i32.popcnt
+            i32.add
+
+            get_local 0
+            get_local 1
+            i32.sub
+
+            get_local 2
+            get_local 3
+            i32.sub
+
+            get_local 0
+            get_local 1
+            i32.mul
+
+            get_local 2
+            get_local 3
+            i32.mul
+
+            get_local 0
+            get_local 2
+            i32.sub
+
+            get_local 1
+            get_local 3
+            i32.sub
+
+            get_local 0
+            get_local 1
+            i32.add
+
+            get_local 2
+            get_local 3
+            i32.add
+
+            get_local 0
+            get_local 2
+            i32.add
+
+            get_local 1
+            get_local 3
+            i32.add
+
+            get_local 0
+            i32.ctz
+            get_local 1
+            i32.ctz
+            i32.add
+
+            get_local 2
+            i32.ctz
+            get_local 3
+            i32.ctz
+
+            get_local 0
+            get_local 1
+            get_local 2
+            get_local 3
+            call 0
+
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+            i32.add
+        )
+    )
+    `))
+);
+
+timeout(1, function() {});
+exports.f()

js/src/jit/MacroAssembler.cpp

@@ -3047,10 +3047,16 @@ std::pair<CodeOffset, uint32_t> MacroAssembler::wasmReserveStackChecked(
     Label ok;
     Register scratch = ABINonArgReg0;
     moveStackPtrTo(scratch);
-    subPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, stackLimit)), scratch);
-    branchPtr(Assembler::GreaterThan, scratch, Imm32(amount), &ok);
+
+    Label trap;
+    branchPtr(Assembler::Below, scratch, Imm32(amount), &trap);
+    subPtr(Imm32(amount), scratch);
+    branchPtr(Assembler::Below, Address(WasmTlsReg, offsetof(wasm::TlsData, stackLimit)), scratch, &ok);
+
+    bind(&trap);
     wasmTrap(wasm::Trap::StackOverflow, trapOffset);
     CodeOffset trapInsnOffset = CodeOffset(currentOffset());
+
     bind(&ok);
     reserveStack(amount);
     return std::pair<CodeOffset, uint32_t>(trapInsnOffset, 0);