Bug 1721718 - wasm: Add OOM checks on getOrCreatePrototype for wasm objects. r=jseward

Differential Revision: https://phabricator.services.mozilla.com/D120533

js/src/jit-test/tests/wasm/oom/directives.txt

@@ -0,0 +1 @@
+|jit-test| test-also=--wasm-compiler=optimizing; test-also=--wasm-compiler=baseline; test-also=--disable-wasm-huge-memory; skip-variant-if: --disable-wasm-huge-memory, !wasmHugeMemorySupported(); include:wasm.js

js/src/jit-test/tests/wasm/oom/jsapi-prototype.js

@@ -0,0 +1,29 @@
+// |jit-test| skip-if: !('oomTest' in this)
+
+oomTest(() => {
+  let memory = new WebAssembly.Memory({initial: 0});
+  assertEq(Object.getPrototypeOf(memory), WebAssembly.Memory.prototype, "prototype");
+});
+
+oomTest(() => {
+  let global = new WebAssembly.Global({value: 'i32'});
+  assertEq(Object.getPrototypeOf(global), WebAssembly.Global.prototype, "prototype");
+});
+
+oomTest(() => {
+  let table = new WebAssembly.Table({element: 'anyfunc', initial: 0});
+  assertEq(Object.getPrototypeOf(table), WebAssembly.Table.prototype, "prototype");
+});
+
+oomTest(() => {
+  let bytecode = wasmTextToBinary('(module)');
+  let module = new WebAssembly.Module(bytecode);
+  assertEq(Object.getPrototypeOf(module), WebAssembly.Module.prototype, "prototype");
+});
+
+oomTest(() => {
+  let bytecode = wasmTextToBinary('(module)');
+  let module = new WebAssembly.Module(bytecode);
+  let instance = new WebAssembly.Instance(module, {});
+  assertEq(Object.getPrototypeOf(instance), WebAssembly.Instance.prototype, "prototype");
+});

js/src/wasm/WasmJS.cpp

@@ -1041,6 +1041,19 @@ static JSObject* CreateWasmConstructor(JSContext* cx, JSProtoKey key) {
   return NewNativeConstructor(cx, Class::construct, 1, className);
 }
 
+static JSObject* GetWasmConstructorPrototype(JSContext* cx,
+                                             const CallArgs& callArgs,
+                                             JSProtoKey key) {
+  RootedObject proto(cx);
+  if (!GetPrototypeFromBuiltinConstructor(cx, callArgs, key, &proto)) {
+    return nullptr;
+  }
+  if (!proto) {
+    proto = GlobalObject::getOrCreatePrototype(cx, key);
+  }
+  return proto;
+}
+
 // ============================================================================
 // WebAssembly.Module class and methods
 
@@ -1581,13 +1594,11 @@ bool WasmModuleObject::construct(JSContext* cx, unsigned argc, Value* vp) {
     return false;
   }
 
-  RootedObject proto(cx);
-  if (!GetPrototypeFromBuiltinConstructor(cx, callArgs, JSProto_WasmModule,
-                                          &proto)) {
-    return false;
-  }
+  RootedObject proto(
+      cx, GetWasmConstructorPrototype(cx, callArgs, JSProto_WasmModule));
   if (!proto) {
-    proto = GlobalObject::getOrCreatePrototype(cx, JSProto_WasmModule);
+    ReportOutOfMemory(cx);
+    return false;
   }
 
   RootedObject moduleObj(cx, WasmModuleObject::create(cx, *module, proto));
@@ -1861,15 +1872,12 @@ bool WasmInstanceObject::construct(JSContext* cx, unsigned argc, Value* vp) {
     return false;
   }
 
-  RootedObject instanceProto(cx);
-  if (!GetPrototypeFromBuiltinConstructor(cx, args, JSProto_WasmInstance,
-                                          &instanceProto)) {
+  RootedObject proto(
+      cx, GetWasmConstructorPrototype(cx, args, JSProto_WasmInstance));
+  if (!proto) {
+    ReportOutOfMemory(cx);
     return false;
   }
-  if (!instanceProto) {
-    instanceProto =
-        GlobalObject::getOrCreatePrototype(cx, JSProto_WasmInstance);
-  }
 
   Rooted<ImportValues> imports(cx);
   if (!GetImports(cx, *module, importObj, imports.address())) {
@@ -1877,7 +1885,7 @@ bool WasmInstanceObject::construct(JSContext* cx, unsigned argc, Value* vp) {
   }
 
   RootedWasmInstanceObject instanceObj(cx);
-  if (!module->instantiate(cx, imports.get(), instanceProto, &instanceObj)) {
+  if (!module->instantiate(cx, imports.get(), proto, &instanceObj)) {
     return false;
   }
 
@@ -2418,13 +2426,11 @@ bool WasmMemoryObject::construct(JSContext* cx, unsigned argc, Value* vp) {
     return false;
   }
 
-  RootedObject proto(cx);
-  if (!GetPrototypeFromBuiltinConstructor(cx, args, JSProto_WasmMemory,
-                                          &proto)) {
-    return false;
-  }
+  RootedObject proto(cx,
+                     GetWasmConstructorPrototype(cx, args, JSProto_WasmMemory));
   if (!proto) {
-    proto = GlobalObject::getOrCreatePrototype(cx, JSProto_WasmMemory);
+    ReportOutOfMemory(cx);
+    return false;
   }
 
   RootedWasmMemoryObject memoryObj(cx,
@@ -2955,13 +2961,11 @@ bool WasmTableObject::construct(JSContext* cx, unsigned argc, Value* vp) {
     return false;
   }
 
-  RootedObject proto(cx);
-  if (!GetPrototypeFromBuiltinConstructor(cx, args, JSProto_WasmTable,
-                                          &proto)) {
-    return false;
-  }
+  RootedObject proto(cx,
+                     GetWasmConstructorPrototype(cx, args, JSProto_WasmTable));
   if (!proto) {
-    proto = GlobalObject::getOrCreatePrototype(cx, JSProto_WasmTable);
+    ReportOutOfMemory(cx);
+    return false;
   }
 
   // The rest of the runtime expects table limits to be within a 32-bit range.
@@ -3406,13 +3410,11 @@ bool WasmGlobalObject::construct(JSContext* cx, unsigned argc, Value* vp) {
     }
   }
 
-  RootedObject proto(cx);
-  if (!GetPrototypeFromBuiltinConstructor(cx, args, JSProto_WasmGlobal,
-                                          &proto)) {
-    return false;
-  }
+  RootedObject proto(cx,
+                     GetWasmConstructorPrototype(cx, args, JSProto_WasmGlobal));
   if (!proto) {
-    proto = GlobalObject::getOrCreatePrototype(cx, JSProto_WasmGlobal);
+    ReportOutOfMemory(cx);
+    return false;
   }
 
   WasmGlobalObject* global =

js/src/wasm/WasmModule.cpp

@@ -1141,6 +1141,7 @@ static bool CreateExportObject(
     propertyAttr |= JSPROP_READONLY | JSPROP_PERMANENT;
   }
   if (!exportObj) {
+    ReportOutOfMemory(cx);
     return false;
   }
 
@@ -1261,6 +1262,7 @@ bool Module::instantiate(JSContext* cx, ImportValues& imports,
 
     maybeDebug = cx->make_unique<DebugState>(*code, *this);
     if (!maybeDebug) {
+      ReportOutOfMemory(cx);
       return false;
     }
   } else {
@@ -1288,6 +1290,7 @@ bool Module::instantiate(JSContext* cx, ImportValues& imports,
   // final pre-requisite for a live instance.
 
   if (!cx->realm()->wasm.registerInstance(cx, instance)) {
+    ReportOutOfMemory(cx);
     return false;
   }