Bug 1696793 - fix window modal dialog handling of html:dialog closing separately, r=mtigley

The html:dialog event can respond to [esc] keypresses itself. In that case,
we need to close the embedded dialog ourselves, or the UI ends up broken.

Differential Revision: https://phabricator.services.mozilla.com/D107463

browser/base/content/browser.js

@@ -9415,12 +9415,15 @@ var gDialogBox = {
       Cu.reportError(ex);
     } finally {
       let dialog = document.getElementById("window-modal-dialog");
-      dialog.close();
+      if (dialog.open) {
+        dialog.close();
+      }
       dialog.style.visibility = "hidden";
       dialog.style.height = "0";
       dialog.style.width = "0";
       document.documentElement.removeAttribute("window-modal-open");
       dialog.removeEventListener("dialogopen", this);
+      dialog.removeEventListener("close", this);
       this._updateMenuAndCommandState(true /* to enable */);
       this._dialog = null;
     }
@@ -9428,8 +9431,13 @@ var gDialogBox = {
   },
 
   handleEvent(event) {
-    if (event.type == "dialogopen") {
-      this._dialog.focus(true);
+    switch (event.type) {
+      case "dialogopen":
+        this._dialog.focus(true);
+        break;
+      case "close":
+        this._dialog.close();
+        break;
     }
   },
 
@@ -9457,6 +9465,7 @@ var gDialogBox = {
     let template = document.getElementById("window-modal-dialog-template")
       .content.firstElementChild;
     parentElement.addEventListener("dialogopen", this);
+    parentElement.addEventListener("close", this);
     this._dialog = new SubDialog({
       template,
       parentElement,

browser/base/content/test/tabPrompts/browser_windowPrompt.js

@@ -68,3 +68,52 @@ add_task(async function test_check_window_modal_prompt_service() {
     ok(!menu.disabled, `Menu ${menu.id} should not be disabled anymore.`);
   }
 });
+
+/**
+ * Check that the dialog's own closing methods being invoked don't break things.
+ */
+add_task(async function test_check_window_modal_prompt_service() {
+  await SpecialPowers.pushPrefEnv({
+    set: [["prompts.windowPromptSubDialog", true]],
+  });
+  let dialogPromise = BrowserTestUtils.promiseAlertDialogOpen(null);
+  // Avoid blocking the test on the (sync) alert by sticking it in a timeout:
+  setTimeout(
+    () => Services.prompt.alert(window, "Some title", "some message"),
+    0
+  );
+  let dialogWin = await dialogPromise;
+
+  // Check circumstances of opening.
+  ok(
+    dialogWin?.docShell?.chromeEventHandler,
+    "Should have embedded the dialog."
+  );
+  is(
+    window.getComputedStyle(document.body).getPropertyValue("-moz-user-input"),
+    "none",
+    "Browser window should be inert."
+  );
+
+  let container = dialogWin.docShell.chromeEventHandler.closest("dialog");
+  let closedPromise = BrowserTestUtils.waitForMutationCondition(
+    container,
+    { childList: true, attributes: true },
+    () => !container.hasChildNodes() && !container.open
+  );
+
+  // This can also be invoked by the user if the escape key is handled
+  // outside of our embedded dialog.
+  container.close();
+  await closedPromise;
+
+  // Check we cleaned up:
+  is(
+    window.getComputedStyle(document.body).getPropertyValue("-moz-user-input"),
+    "auto",
+    "Browser window should no longer be inert."
+  );
+  for (let menu of document.querySelectorAll("menubar > menu")) {
+    ok(!menu.disabled, `Menu ${menu.id} should not be disabled anymore.`);
+  }
+});