From 86da8f87a36a88eb4d6a41c405852dc09ac4693e Mon Sep 17 00:00:00 2001
From: Gijs Kruitbosch <gijskruitbosch@gmail.com>
Date: Thu, 11 Feb 2021 23:57:57 +0000
Subject: [PATCH] Bug 1685313 - allow window modal dialogs to display inside
 windows, r=jaws,mtigley

Differential Revision: https://phabricator.services.mozilla.com/D103388
---
 browser/app/profile/firefox.js              |  9 ++
 browser/base/content/browser.css            | 19 +++-
 browser/base/content/browser.js             | 87 ++++++++++++++++++
 browser/base/content/browser.xhtml          | 11 +++
 netwerk/base/nsIPrompt.idl                  |  3 +
 toolkit/components/prompts/src/Prompter.jsm | 97 +++++++++++++++++----
 toolkit/content/widgets/dialog.js           | 14 ++-
 7 files changed, 218 insertions(+), 22 deletions(-)

diff --git a/browser/app/profile/firefox.js b/browser/app/profile/firefox.js
index 178c66a550f3..73e0f279e04b 100644
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1370,6 +1370,15 @@ pref("prompts.tabChromePromptSubDialog", true);
   pref("prompts.contentPromptSubDialog", false);
 #endif
 
+// Whether to show window-modal dialogs opened for browser windows
+// in a SubDialog inside their parent, instead of an OS level window.
+#ifdef NIGHTLY_BUILD
+  pref("prompts.windowPromptSubDialog", true);
+#else
+  pref("prompts.windowPromptSubDialog", false);
+#endif
+
+
 // Activates preloading of the new tab url.
 pref("browser.newtab.preload", true);
 
diff --git a/browser/base/content/browser.css b/browser/base/content/browser.css
index f9e4040aa143..14ad32e01528 100644
--- a/browser/base/content/browser.css
+++ b/browser/base/content/browser.css
@@ -1479,7 +1479,7 @@ toolbar[keyNav=true]:not([collapsed=true], [customizing=true]) toolbartabstop {
 }
 
 /**
- * Tab Dialogs
+ * Dialogs
  */
 
 .dialogStack {
@@ -1572,6 +1572,21 @@ toolbar[keyNav=true]:not([collapsed=true], [customizing=true]) toolbartabstop {
   place-content: center;
 }
 
+/* Override default <html:dialog> styles */
+#window-modal-dialog {
+  border-width: 0;
+  background-color: transparent;
+}
+
+#window-modal-dialog::backdrop {
+  background-color: rgba(0, 0, 0, 0.5);
+}
+
+/* Hide tab-modal dialogs when a window-modal one is up. */
+:root[window-modal-open] .browserStack > .dialogStack {
+  visibility: hidden;
+}
+
 /**
- * End Tab Dialogs
+ * End Dialogs
  */
diff --git a/browser/base/content/browser.js b/browser/base/content/browser.js
index d04ec827af63..a34df7cfc63d 100644
--- a/browser/base/content/browser.js
+++ b/browser/base/content/browser.js
@@ -71,6 +71,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
   SimpleServiceDiscovery: "resource://gre/modules/SimpleServiceDiscovery.jsm",
   SiteDataManager: "resource:///modules/SiteDataManager.jsm",
   SitePermissions: "resource:///modules/SitePermissions.jsm",
+  SubDialog: "resource://gre/modules/SubDialog.jsm",
   SubDialogManager: "resource://gre/modules/SubDialog.jsm",
   TabModalPrompt: "chrome://global/content/tabprompts.jsm",
   TabCrashHandler: "resource:///modules/ContentCrashHandlers.jsm",
@@ -9301,6 +9302,92 @@ TabModalPromptBox.prototype = {
   },
 };
 
+// Handle window-modal prompts that we want to display with the same style as
+// tab-modal prompts.
+var gDialogBox = {
+  _dialog: null,
+
+  get isOpen() {
+    return !!this._dialog;
+  },
+
+  async open(uri, args) {
+    try {
+      await this._open(uri, args);
+    } catch (ex) {
+      Cu.reportError(ex);
+    } finally {
+      let dialog = document.getElementById("window-modal-dialog");
+      dialog.close();
+      dialog.style.visibility = "hidden";
+      dialog.style.height = "0";
+      dialog.style.width = "0";
+      document.documentElement.removeAttribute("window-modal-open");
+      dialog.removeEventListener("dialogopen", this);
+      this._dialog = null;
+    }
+    return args;
+  },
+
+  handleEvent(event) {
+    if (event.type == "dialogopen") {
+      this._dialog.focus(true);
+    }
+  },
+
+  _open(uri, args) {
+    // Get this offset before we touch style below, as touching style seems
+    // to reset the cached layout bounds.
+    let offset = window.windowUtils.getBoundsWithoutFlushing(
+      gBrowser.selectedBrowser
+    ).top;
+    let parentElement = document.getElementById("window-modal-dialog");
+    // The dialog has 1em padding; compensate for that:
+    parentElement.style.marginTop = `calc(${offset}px - 1em)`;
+    parentElement.style.removeProperty("visibility");
+    parentElement.style.removeProperty("width");
+    parentElement.style.removeProperty("height");
+    document.documentElement.setAttribute("window-modal-open", true);
+    // Call this first so the contents show up and get layout, which is
+    // required for SubDialog to work.
+    parentElement.showModal();
+
+    // Now actually set up the dialog contents:
+    let template = document.getElementById("window-modal-dialog-template")
+      .content.firstElementChild;
+    parentElement.addEventListener("dialogopen", this);
+    this._dialog = new SubDialog({
+      template,
+      parentElement,
+      id: "window-modal-dialog-subdialog",
+      options: {
+        consumeOutsideClicks: false,
+      },
+    });
+    let closedPromise = new Promise(resolve => {
+      this._closedCallback = resolve;
+    });
+    this._dialog.open(
+      uri,
+      {
+        features: "resizable=no",
+        modalType: Ci.nsIPrompt.MODAL_TYPE_INTERNAL_WINDOW,
+        closedCallback: () => {
+          this._closedCallback();
+        },
+      },
+      args
+    );
+    return closedPromise;
+  },
+};
+
+// browser.js loads in the library window, too, but we can only show prompts
+// in the main browser window:
+if (window.location.href != AppConstants.BROWSER_CHROME_URL) {
+  gDialogBox = null;
+}
+
 var ConfirmationHint = {
   _timerID: null,
 
diff --git a/browser/base/content/browser.xhtml b/browser/base/content/browser.xhtml
index fb69c72d02f5..d750b8104f26 100644
--- a/browser/base/content/browser.xhtml
+++ b/browser/base/content/browser.xhtml
@@ -1618,6 +1618,17 @@
     </panelview>
   </html:template>
 
+  <html:dialog id="window-modal-dialog" style="visibility: hidden; height: 0; width: 0"/>
+  <html:template id="window-modal-dialog-template">
+    <vbox class="dialogTemplate dialogOverlay" align="center" topmost="true">
+      <hbox class="dialogBox">
+        <browser class="dialogFrame"
+                 autoscroll="false"
+                 disablehistory="true"/>
+      </hbox>
+    </vbox>
+  </html:template>
+
   <!-- Temporary wrapper until we move away from XUL flex to allow a negative
        margin-top to slide the toolbox off screen in fullscreen layout.-->
   <box>
diff --git a/netwerk/base/nsIPrompt.idl b/netwerk/base/nsIPrompt.idl
index a1166d151dd0..6a6bc0deac64 100644
--- a/netwerk/base/nsIPrompt.idl
+++ b/netwerk/base/nsIPrompt.idl
@@ -66,6 +66,9 @@ interface nsIPrompt : nsISupports
     const unsigned long MODAL_TYPE_CONTENT = 1;
     const unsigned long MODAL_TYPE_TAB = 2;
     const unsigned long MODAL_TYPE_WINDOW = 3;
+    // Like MODAL_TYPE_WINDOW, but shown inside a parent window (with similar
+    // styles as _TAB and _CONTENT types) rather than as a new window:
+    const unsigned long MODAL_TYPE_INTERNAL_WINDOW = 4;
 
     int32_t confirmEx(in wstring dialogTitle,
                    in wstring text,
diff --git a/toolkit/components/prompts/src/Prompter.jsm b/toolkit/components/prompts/src/Prompter.jsm
index 5ecc818eb8c8..963599d12200 100644
--- a/toolkit/components/prompts/src/Prompter.jsm
+++ b/toolkit/components/prompts/src/Prompter.jsm
@@ -13,6 +13,16 @@ var { PromptUtils } = ChromeUtils.import(
   "resource://gre/modules/SharedPromptUtils.jsm"
 );
 
+const {
+  // MODAL_TYPE_TAB, // currently not read in this file.
+  MODAL_TYPE_CONTENT,
+  MODAL_TYPE_WINDOW,
+  MODAL_TYPE_INTERNAL_WINDOW,
+} = Ci.nsIPrompt;
+
+const COMMON_DIALOG = "chrome://global/content/commonDialog.xhtml";
+const SELECT_DIALOG = "chrome://global/content/selectDialog.xhtml";
+
 function Prompter() {
   // Note that EmbedPrompter clones this implementation.
 }
@@ -967,6 +977,14 @@ class ModalPrompter {
       this.browsingContext = browsingContext;
     }
 
+    if (
+      domWin &&
+      (!modalType || modalType == MODAL_TYPE_WINDOW) &&
+      !this.browsingContext?.isContent
+    ) {
+      modalType = MODAL_TYPE_INTERNAL_WINDOW;
+    }
+
     // Use given modal type or fallback to default
     this.modalType = modalType || ModalPrompter.defaultModalType;
 
@@ -982,19 +1000,42 @@ class ModalPrompter {
 
   set modalType(modalType) {
     // Setting modal type window is always allowed
-    if (modalType == Ci.nsIPrompt.MODAL_TYPE_WINDOW) {
+    if (modalType == MODAL_TYPE_WINDOW) {
       this._modalType = modalType;
       return;
     }
 
-    // We can't use content / tab prompts if we don't have a suitable parent.
-    if (!this.browsingContext?.isContent) {
-      modalType = Ci.nsIPrompt.MODAL_TYPE_WINDOW;
-
-      Cu.reportError(
-        "Prompter: Browser not available. Falling back to window prompt."
-      );
+    // For content prompts for non-content windows, use window prompts:
+    if (modalType == MODAL_TYPE_CONTENT && !this.browsingContext?.isContent) {
+      this._modalType = MODAL_TYPE_WINDOW;
+      return;
     }
+
+    // We can't use content / tab prompts if we don't have a suitable parent.
+    if (
+      !this.browsingContext?.isContent &&
+      modalType != MODAL_TYPE_INTERNAL_WINDOW
+    ) {
+      // Only show this error if we're not about to fall back again and show a different one.
+      if (this.browsingContext?.associatedWindow?.gDialogBox) {
+        Cu.reportError(
+          "Prompter: Browser not available. Falling back to internal window prompt."
+        );
+      }
+      modalType = MODAL_TYPE_INTERNAL_WINDOW;
+    }
+
+    if (
+      modalType == MODAL_TYPE_INTERNAL_WINDOW &&
+      (this.browsingContext?.isContent ||
+        !this.browsingContext?.associatedWindow?.gDialogBox)
+    ) {
+      Cu.reportError(
+        "Prompter: internal dialogs not available in this context. Falling back to window prompt."
+      );
+      modalType = MODAL_TYPE_WINDOW;
+    }
+
     this._modalType = modalType;
   }
 
@@ -1031,17 +1072,22 @@ class ModalPrompter {
       return args;
     }
 
+    if (this._modalType == MODAL_TYPE_INTERNAL_WINDOW) {
+      await this.openInternalWindowPrompt(
+        this.browsingContext.associatedWindow,
+        args
+      );
+      return args;
+    }
+
     // Select prompts are not part of CommonDialog
     // and thus not supported as tab or content prompts yet. See Bug 1622817.
     // Once they are integrated this override should be removed.
-    if (
-      args.promptType == "select" &&
-      this.modalType !== Ci.nsIPrompt.MODAL_TYPE_WINDOW
-    ) {
+    if (args.promptType == "select" && this.modalType !== MODAL_TYPE_WINDOW) {
       Cu.reportError(
         "Prompter: 'select' prompts do not support tab/content prompting. Falling back to window prompt."
       );
-      args.modalType = Ci.nsIPrompt.MODAL_TYPE_WINDOW;
+      args.modalType = MODAL_TYPE_WINDOW;
     } else {
       args.modalType = this.modalType;
     }
@@ -1082,7 +1128,7 @@ class ModalPrompter {
       args.inPermitUnload = inPermitUnload;
       let eventDetail = Cu.cloneInto(
         {
-          tabPrompt: this.modalType != Ci.nsIPrompt.MODAL_TYPE_WINDOW,
+          tabPrompt: this.modalType != MODAL_TYPE_WINDOW,
           inPermitUnload,
         },
         this.browsingContext.window
@@ -1164,9 +1210,6 @@ class ModalPrompter {
    * @param {Object} args - Prompt options and return values.
    */
   openWindowPrompt(parentWindow = null, args) {
-    const COMMON_DIALOG = "chrome://global/content/commonDialog.xhtml";
-    const SELECT_DIALOG = "chrome://global/content/selectDialog.xhtml";
-
     let uri = args.promptType == "select" ? SELECT_DIALOG : COMMON_DIALOG;
     let propBag = PromptUtils.objectToPropBag(args);
     Services.ww.openWindow(
@@ -1179,6 +1222,17 @@ class ModalPrompter {
     PromptUtils.propBagToObject(propBag, args);
   }
 
+  async openInternalWindowPrompt(parentWindow, args) {
+    if (!parentWindow?.gDialogBox || !ModalPrompter.windowPromptSubDialog) {
+      this.openWindowPrompt(parentWindow, args);
+      return;
+    }
+    let propBag = PromptUtils.objectToPropBag(args);
+    let uri = args.promptType == "select" ? SELECT_DIALOG : COMMON_DIALOG;
+    await parentWindow.gDialogBox.open(uri, propBag);
+    PromptUtils.propBagToObject(propBag, args);
+  }
+
   /**
    * Calls async prompt method and optionally runs promise chained task on
    * result data. Converts result data to nsIPropertyBag.
@@ -1671,7 +1725,14 @@ XPCOMUtils.defineLazyPreferenceGetter(
   ModalPrompter,
   "defaultModalType",
   "prompts.defaultModalType",
-  Ci.nsIPrompt.MODAL_TYPE_WINDOW
+  MODAL_TYPE_WINDOW
+);
+
+XPCOMUtils.defineLazyPreferenceGetter(
+  ModalPrompter,
+  "windowPromptSubDialog",
+  "prompts.windowPromptSubDialog",
+  false
 );
 
 function AuthPromptAdapterFactory() {}
diff --git a/toolkit/content/widgets/dialog.js b/toolkit/content/widgets/dialog.js
index d80c42af7761..c6b5d8ff2f56 100644
--- a/toolkit/content/widgets/dialog.js
+++ b/toolkit/content/widgets/dialog.js
@@ -264,7 +264,12 @@
         // give focus to the first focusable element in the dialog
         let focusedElt = document.commandDispatcher.focusedElement;
         if (!focusedElt) {
-          document.commandDispatcher.advanceFocusIntoSubtree(this);
+          Services.focus.moveFocus(
+            window,
+            null,
+            Services.focus.MOVEFOCUS_FORWARD,
+            Services.focus.FLAG_NOPARENTFRAME
+          );
 
           focusedElt = document.commandDispatcher.focusedElement;
           if (focusedElt) {
@@ -273,7 +278,12 @@
               focusedElt.localName == "tab" ||
               focusedElt.getAttribute("noinitialfocus") == "true"
             ) {
-              document.commandDispatcher.advanceFocusIntoSubtree(focusedElt);
+              Services.focus.moveFocus(
+                window,
+                focusedElt,
+                Services.focus.MOVEFOCUS_FORWARD,
+                Services.focus.FLAG_NOPARENTFRAME
+              );
               focusedElt = document.commandDispatcher.focusedElement;
               if (focusedElt) {
                 if (focusedElt == initialFocusedElt) {