Bug 1589493 - Extending BrowserTestUtils.crashFrame to allow crashing with an OOM;r=mconley

BrowserTestUtils.crashFrame now accepts additional `options`, with an argument `crashType` that may
take "CRASH_OOM" or "CRASH_INVALID_POINTER_DEREF"|null to specify the nature of the crash. The names
are taken from CrashTestUtils.jsm but this module cannot be imported as such as it has non-trivial
binary dependencies.

Depends on D54130

Differential Revision: https://phabricator.services.mozilla.com/D54700

--HG--
extra : moz-landing-system : lando

testing/mochitest/BrowserTestUtils/BrowserTestUtils.jsm

@@ -1628,6 +1628,10 @@ var BrowserTestUtils = {
    * @param (BrowsingContext) browsingContext
    *        The context where the frame leaves. Default to
    *        top level context if not supplied.
+   * @param (object?) options
+   *        An object with any of the following fields:
+   *          crashType: "CRASH_INVALID_POINTER_DEREF" | "CRASH_OOM"
+   *            The type of crash. If unspecified, default to "CRASH_INVALID_POINTER_DEREF"
    *
    * @returns (Promise)
    * @resolves An Object with key-value pairs representing the data from the
@@ -1637,7 +1641,8 @@ var BrowserTestUtils = {
     browser,
     shouldShowTabCrashPage = true,
     shouldClearMinidumps = true,
-    browsingContext
+    browsingContext,
+    options = {}
   ) {
     let extra = {};
 
@@ -1772,7 +1777,9 @@ var BrowserTestUtils = {
     this.sendAsyncMessage(
       browsingContext || browser.browsingContext,
       "BrowserTestUtils:CrashFrame",
-      {}
+      {
+        crashType: options.crashType || "",
+      }
     );
 
     await Promise.all(expectedPromises);

testing/mochitest/BrowserTestUtils/BrowserTestUtilsChild.jsm

@@ -210,8 +210,8 @@ class BrowserTestUtilsChild extends JSWindowActorChild {
 
       case "BrowserTestUtils:CrashFrame": {
         // This is to intentionally crash the frame.
-        // We crash by using js-ctypes and dereferencing
-        // a bad pointer. The crash should happen immediately
+        // We crash by using js-ctypes. The crash
+        // should happen immediately
         // upon loading this frame script.
 
         const { ctypes } = ChromeUtils.import(
@@ -220,9 +220,38 @@ class BrowserTestUtilsChild extends JSWindowActorChild {
 
         let dies = function() {
           ChromeUtils.privateNoteIntentionalCrash();
-          let zero = new ctypes.intptr_t(8);
-          let badptr = ctypes.cast(zero, ctypes.PointerType(ctypes.int32_t));
-          badptr.contents;
+
+          switch (aMessage.data.crashType) {
+            case "CRASH_OOM": {
+              // Allocate waaaaaay too much memory to encourage the system
+              // to crash with an OOM.
+              const OS = ChromeUtils.import(
+                "resource://gre/modules/osfile/osfile_shared_allthreads.jsm"
+              );
+              let libxul = ctypes.open(OS.Constants.Path.libxul);
+              let moz_xmalloc = libxul.declare(
+                "moz_xmalloc",
+                ctypes.default_abi,
+                /* return type */ ctypes.voidptr_t,
+                /* size */ ctypes.size_t
+              );
+              let max_value = ctypes.cast(ctypes.ssize_t(-1), ctypes.size_t);
+              moz_xmalloc(max_value);
+              moz_xmalloc(max_value);
+              moz_xmalloc(max_value);
+              break;
+            }
+            case "CRASH_INVALID_POINTER_DEREF": // Fallthrough
+            default: {
+              // Dereference a bad pointer.
+              let zero = new ctypes.intptr_t(8);
+              let badptr = ctypes.cast(
+                zero,
+                ctypes.PointerType(ctypes.int32_t)
+              );
+              badptr.contents;
+            }
+          }
         };
 
         dump("\nEt tu, Brute?\n");