Bug 1873068 - remove broken SubDialog code to ensure alerts() from freshly opened about:blank documents do not hang, r=pbz,hsivonen

The early closing checks were added when SubDialog was only used in Firefox's
settings UI, and instances were reused. It doesn't look like we still reuse
subdialog instances anyway, so I don't think the code needs to cater for this
at all. Conveniently, this also addresses the hang.

It does not cause the alert to actually appear (hence the todo_is checking for
the dialog which isn't there right now), but fixing that appears to
be a problem to do with about:blank creation that will be significantly
trickier to resolve.

Differential Revision: https://phabricator.services.mozilla.com/D200858

dom/tests/browser/browser.toml

@@ -24,6 +24,8 @@ support-files = [
 
 ["browser_ConsoleStoragePBTest_perwindowpb.js"]
 
+["browser_alert_from_about_blank.js"]
+
 ["browser_autofocus_background.js"]
 
 ["browser_autofocus_preference.js"]

dom/tests/browser/browser_alert_from_about_blank.js

@@ -0,0 +1,42 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function test_check_alert_from_blank() {
+  await BrowserTestUtils.withNewTab(
+    "https://example.com/blank",
+    async browser => {
+      await SpecialPowers.spawn(browser, [], () => {
+        let button = content.document.createElement("button");
+        button.addEventListener("click", () => {
+          let newWin = content.open(
+            "about:blank",
+            "",
+            "popup,width=600,height=600"
+          );
+          newWin.alert("Alert from the popup.");
+        });
+        content.document.body.append(button);
+      });
+      let newWinPromise = BrowserTestUtils.waitForNewWindow();
+      await BrowserTestUtils.synthesizeMouseAtCenter("button", {}, browser);
+      let otherWin = await newWinPromise;
+      Assert.equal(
+        otherWin.gBrowser.currentURI?.spec,
+        "about:blank",
+        "Should have opened about:blank"
+      );
+      Assert.equal(
+        otherWin.menubar.visible,
+        false,
+        "Should be a popup window."
+      );
+      let contentDialogManager = gBrowser
+        .getTabDialogBox(gBrowser.selectedBrowser)
+        .getContentDialogManager();
+      todo_is(contentDialogManager._dialogs.length, 1, "Should have 1 dialog.");
+      await BrowserTestUtils.closeWindow(otherWin);
+    }
+  );
+});

toolkit/modules/SubDialog.sys.mjs

@@ -173,24 +173,8 @@ SubDialog.prototype = {
     // Wait until frame is ready to prevent browser crash in tests
     await this._frameCreated;
 
-    if (!this._frame.contentWindow) {
-      // Given the binding constructor execution is asynchronous, and "load"
-      // event can be dispatched before the browser element is shown, the
-      // browser binding might not be constructed at this point.  Forcibly
-      // construct the frame and construct the binding.
-      // FIXME: Remove this (bug 1437247)
-      this._frame.getBoundingClientRect();
-    }
-
-    // If we're open on some (other) URL or we're closing, open when closing has finished.
-    if (this._openedURL || this._isClosing) {
-      if (!this._isClosing) {
-        this.close();
-      }
-      let args = Array.from(arguments);
-      this._closingPromise.then(() => {
-        this.open.apply(this, args);
-      });
+    // If we're closing now that we've waited for the dialog to load, abort.
+    if (this._isClosing) {
       return;
     }
     this._addDialogEventListeners();