Bug 1751699 - wasm: Handle OOM in allocating try note correctly. r=jseward

The method for allocating a wasm::TryNote in MacroAssembler would
mark that an OOM had occurred, but still return a bogus index
into the try notes. This commit propagates the OOM to the caller
so that it knows that there is no valid try note index to use.

Differential Revision: https://phabricator.services.mozilla.com/D136755

js/src/jit-test/tests/wasm/exceptions/bug-1751699.js

@@ -0,0 +1,6 @@
+oomTest(() => {
+	wasmEvalText(`
+      (import "" "" (func $d))
+      (func try call $d end)
+    `);
+});

js/src/jit/CodeGenerator.cpp

@@ -8017,11 +8017,13 @@ void CodeGenerator::visitWasmCall(LWasmCall* lir) {
 #ifdef ENABLE_WASM_EXCEPTIONS
   // If this call is in Wasm try code block, initialise a WasmTryNote for this
   // call.
-  bool inTry_ = mir->inTry();
+  bool inTry = mir->inTry();
   size_t tryNoteIndex = 0;
-
-  if (inTry_) {
-    tryNoteIndex = masm.wasmStartTry();
+  if (inTry && !masm.wasmStartTry(&tryNoteIndex)) {
+    // Handle an OOM in allocating a try note by forcing inTry to false, this
+    // will skip the logic below that uses the try note. This is okay as
+    // compilation will be aborted after this.
+    inTry = false;
   }
 #endif
 
@@ -8096,7 +8098,7 @@ void CodeGenerator::visitWasmCall(LWasmCall* lir) {
   }
 
 #ifdef ENABLE_WASM_EXCEPTIONS
-  if (inTry_) {
+  if (inTry) {
     // A call that threw will not return here normally, but will jump to this
     // WasmCall's WasmTryNote entryPoint below. To make exceptional control flow
     // easier to track, we set the entry point in this very call. The exception

js/src/jit/MacroAssembler.cpp

@@ -3751,9 +3751,9 @@ void MacroAssembler::wasmInterruptCheck(Register tls,
 }
 
 #ifdef ENABLE_WASM_EXCEPTIONS
-size_t MacroAssembler::wasmStartTry() {
+[[nodiscard]] bool MacroAssembler::wasmStartTry(size_t* tryNoteIndex) {
   wasm::WasmTryNote tryNote = wasm::WasmTryNote(currentOffset(), 0, 0);
-  return append(tryNote);
+  return append(tryNote, tryNoteIndex);
 }
 #endif
 

js/src/jit/MacroAssembler.h

@@ -3656,7 +3656,7 @@ class MacroAssembler : public MacroAssemblerSpecific {
   void wasmTrap(wasm::Trap trap, wasm::BytecodeOffset bytecodeOffset);
   void wasmInterruptCheck(Register tls, wasm::BytecodeOffset bytecodeOffset);
 #ifdef ENABLE_WASM_EXCEPTIONS
-  size_t wasmStartTry();
+  [[nodiscard]] bool wasmStartTry(size_t* tryNoteIndex);
 #endif
 
   // Returns a pair: the offset of the undefined (trapping) instruction, and

js/src/jit/shared/Assembler-shared.h

@@ -660,9 +660,13 @@ class AssemblerShared {
   // This one returns an index as the try note so that it can be looked up
   // later to add the end point and stack position of the try block.
 #ifdef ENABLE_WASM_EXCEPTIONS
-  size_t append(wasm::WasmTryNote tryNote) {
-    enoughMemory_ &= tryNotes_.append(tryNote);
-    return tryNotes_.length() - 1;
+  [[nodiscard]] bool append(wasm::WasmTryNote tryNote, size_t* tryNoteIndex) {
+    if (!tryNotes_.append(tryNote)) {
+      enoughMemory_ = false;
+      return false;
+    }
+    *tryNoteIndex = tryNotes_.length() - 1;
+    return true;
   }
 #endif
 

js/src/wasm/WasmBaselineCompile.cpp

@@ -3523,7 +3523,9 @@ bool BaseCompiler::emitTry() {
     // Be conservative for BCE due to complex control flow in try blocks.
     controlItem().bceSafeOnExit = 0;
     // Mark the beginning of the try block, the rest is filled in by catch.
-    controlItem().tryNoteIndex = masm.wasmStartTry();
+    if (!masm.wasmStartTry(&controlItem().tryNoteIndex)) {
+      return false;
+    }
   }
 
   return true;