Bug 1905750 - Display shortcuts on text selection r=tarek

Detect selection to show shortcut icon near the cursor. Open panel with prompts when hovering the icon. Hide these on various events that might move content or no selection.

Differential Revision: https://phabricator.services.mozilla.com/D217545

browser/app/profile/firefox.js

@@ -1957,6 +1957,7 @@ pref("browser.ml.chat.prompts.1", '{"label":"Simplify Language","value":"Please
 pref("browser.ml.chat.prompts.2", '{"label":"Quiz Me","value":"Please quiz me on this selection. Ask me a variety of types of questions, for example multiple choice, true or false, and short answer. Wait for my response before moving on to the next question.","id":"quiz","targeting":"!provider|regExpMatch(\'gemini\')"}');
 pref("browser.ml.chat.provider", "");
 pref("browser.ml.chat.shortcuts", false);
+pref("browser.ml.chat.shortcuts.custom", false);
 pref("browser.ml.chat.sidebar", true);
 
 pref("security.protectionspopup.recordEventTelemetry", true);

browser/components/genai/GenAI.sys.mjs

@@ -9,6 +9,8 @@ import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
 const lazy = {};
 ChromeUtils.defineESModuleGetters(lazy, {
   ASRouterTargeting: "resource:///modules/asrouter/ASRouterTargeting.sys.mjs",
+  clearTimeout: "resource://gre/modules/Timer.sys.mjs",
+  setTimeout: "resource://gre/modules/Timer.sys.mjs",
 });
 XPCOMUtils.defineLazyPreferenceGetter(
   lazy,
@@ -38,6 +40,22 @@ XPCOMUtils.defineLazyPreferenceGetter(
   null,
   (_pref, _old, val) => onChatProviderChange(val)
 );
+XPCOMUtils.defineLazyPreferenceGetter(
+  lazy,
+  "chatShortcuts",
+  "browser.ml.chat.shortcuts"
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+  lazy,
+  "chatShortcutsCustom",
+  "browser.ml.chat.shortcuts.custom"
+);
+XPCOMUtils.defineLazyPreferenceGetter(
+  lazy,
+  "chatShortcutsDebounce",
+  "browser.ml.chat.shortcutsDebounce",
+  200
+);
 XPCOMUtils.defineLazyPreferenceGetter(
   lazy,
   "chatSidebar",
@@ -167,7 +185,147 @@ export const GenAI = {
   },
 
   /**
-   * Build prompts menu to ask chat for context menu or popup.
+   * Add chat items to menu or popup.
+   *
+   * @param {MozBrowser} browser providing context
+   * @param {string} selection text
+   * @param {Function} itemAdder creates and returns the item
+   * @param {Function} cleanup optional on item activation
+   * @returns {object} context used for selecting prompts
+   */
+  async addAskChatItems(browser, selection, itemAdder, cleanup) {
+    // Prepare context used for both targeting and handling prompts
+    const window = browser.ownerGlobal;
+    const tab = window.gBrowser.getTabForBrowser(browser);
+    const uri = browser.currentURI;
+    const context = {
+      provider: lazy.chatProvider,
+      selection,
+      tabTitle: (tab._labelIsContentTitle && tab.label) || "",
+      url: uri.asciiHost + uri.filePath,
+      window,
+    };
+
+    // Add items that pass along context for handling
+    (await this.getContextualPrompts(context)).forEach(promptObj =>
+      itemAdder(promptObj).addEventListener("command", () => {
+        this.handleAskChat(promptObj, context);
+        cleanup?.();
+      })
+    );
+    return context;
+  },
+
+  /**
+   * Handle messages from content to show or hide shortcuts.
+   *
+   * @param {string} name of message
+   * @param {object} data for the message
+   * @param {MozBrowser} browser that provided the message
+   */
+  handleShortcutsMessage(name, data, browser) {
+    if (!lazy.chatEnabled || !lazy.chatShortcuts || lazy.chatProvider == "") {
+      return;
+    }
+    const stack = browser.closest(".browserStack");
+    if (!stack) {
+      return;
+    }
+
+    let shortcuts = stack.querySelector(".content-shortcuts");
+    const window = browser.ownerGlobal;
+    const { document } = window;
+    const popup = document.getElementById("ask-chat-shortcuts");
+    const hide = () => {
+      if (shortcuts) {
+        shortcuts.removeAttribute("shown");
+      }
+      popup.hidePopup();
+    };
+
+    switch (name) {
+      case "GenAI:HideShortcuts":
+        hide();
+        break;
+      case "GenAI:SelectionChange":
+        // Add shortcuts to the current tab's brower stack if it doesn't exist
+        if (!shortcuts) {
+          shortcuts = stack.appendChild(document.createElement("div"));
+          shortcuts.className = "content-shortcuts";
+
+          // Detect hover to build and open the popup
+          shortcuts.addEventListener("mouseover", async () => {
+            if (!shortcuts.hasAttribute("active")) {
+              shortcuts.toggleAttribute("active");
+              const vbox = popup.querySelector("vbox");
+              vbox.innerHTML = "";
+              const context = await this.addAskChatItems(
+                browser,
+                shortcuts.selection,
+                promptObj => {
+                  const button = vbox.appendChild(
+                    document.createXULElement("toolbarbutton")
+                  );
+                  button.className = "subviewbutton";
+                  button.textContent = promptObj.label;
+                  return button;
+                },
+                hide
+              );
+
+              // Add custom input box if configured
+              if (lazy.chatShortcutsCustom) {
+                vbox.appendChild(document.createXULElement("toolbarseparator"));
+                const input = vbox.appendChild(document.createElement("input"));
+                input.placeholder = `Ask ${
+                  this.chatProviders.get(lazy.chatProvider)?.name ??
+                  "AI chatbot"
+                }…`;
+                input.style.margin = "var(--arrowpanel-menuitem-margin)";
+                input.addEventListener("mouseover", () => input.focus());
+                input.addEventListener("change", () => {
+                  this.handleAskChat({ value: input.value }, context);
+                  hide();
+                });
+              }
+
+              popup.openPopup(shortcuts);
+              popup.addEventListener(
+                "popuphidden",
+                () => shortcuts.removeAttribute("active"),
+                { once: true }
+              );
+            }
+          });
+        }
+
+        // Immediately hide shortcuts and debounce multiple selection changes
+        hide();
+        if (shortcuts.timeout) {
+          lazy.clearTimeout(shortcuts.timeout);
+        }
+        shortcuts.timeout = lazy.setTimeout(() => {
+          // Save the latest selection so it can be used by the popup
+          shortcuts.selection = data.selection;
+          shortcuts.toggleAttribute("shown");
+
+          // Position the shortcuts relative to the browser's top-left corner
+          const rect = browser.getBoundingClientRect();
+          shortcuts.style.setProperty(
+            "--shortcuts-x",
+            data.x - window.screenX - rect.x + "px"
+          );
+          shortcuts.style.setProperty(
+            "--shortcuts-y",
+            data.y - window.screenY - rect.y + "px"
+          );
+        }, lazy.chatShortcutsDebounce);
+        break;
+    }
+  },
+
+  /**
+   * Build prompts menu to ask chat for context menu.
    *
    * @param {MozMenu} menu element to update
    * @param {nsContextMenu} nsContextMenu helpers for context menu
@@ -181,24 +339,10 @@ export const GenAI = {
       this.chatProviders.get(lazy.chatProvider)?.name ?? "AI Chatbot"
     }`;
     menu.menupopup?.remove();
-
+    await this.addAskChatItems(
-    // Prepare context used for both targeting and handling prompts
+      nsContextMenu.browser,
-    const window = menu.ownerGlobal;
+      nsContextMenu.selectionInfo.fullText ?? "",
-    const tab = window.gBrowser.getTabForBrowser(nsContextMenu.browser);
+      promptObj => menu.appendItem(promptObj.label)
-    const context = {
-      provider: lazy.chatProvider,
-      selection: nsContextMenu.selectionInfo.fullText ?? "",
-      tabTitle: (tab._labelIsContentTitle && tab.label) || "",
-      window,
-    };
-
-    // Add menu items that pass along context for handling
-    (await this.getContextualPrompts(context)).forEach(promptObj =>
-      menu
-        .appendItem(promptObj.label, promptObj.value)
-        .addEventListener("command", () =>
-          this.handleAskChat(promptObj, context)
-        )
     );
     nsContextMenu.showItem(menu, menu.itemCount > 0);
   },

browser/components/genai/GenAIChild.sys.mjs

@@ -15,11 +15,31 @@ export class GenAIChild extends JSWindowActorChild {
   }
 
   handleEvent(event) {
+    const sendHide = () => this.sendQuery("GenAI:HideShortcuts", event.type);
     switch (event.type) {
       case "mousemove":
+        // Track the pointer's screen position to avoid container positioning
+        this.lastX = event.screenX;
+        this.lastY = event.screenY;
+        break;
       case "resize":
       case "scroll":
-      case "selectionchange":
+        // Hide if selection might have shifted away from shortcuts
+        sendHide();
+        break;
+      case "selectionchange": {
+        const selection = this.contentWindow.getSelection().toString().trim();
+        if (!selection) {
+          sendHide();
+          break;
+        }
+        this.sendQuery("GenAI:SelectionChange", {
+          x: this.lastX ?? 0,
+          y: this.lastY ?? 0,
+          selection,
+        });
+        break;
+      }
     }
   }
 }

browser/components/genai/GenAIParent.sys.mjs

@@ -2,9 +2,20 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+const lazy = {};
+ChromeUtils.defineESModuleGetters(lazy, {
+  GenAI: "resource:///modules/GenAI.sys.mjs",
+});
+
 /**
  * JSWindowActor to pass data between GenAI singleton and content pages.
  */
 export class GenAIParent extends JSWindowActorParent {
-  receiveMessage() {}
+  receiveMessage({ data, name }) {
+    lazy.GenAI.handleShortcutsMessage(
+      name,
+      data,
+      this.browsingContext.topFrameElement
+    );
+  }
 }

browser/components/genai/tests/browser/browser.toml

@@ -7,6 +7,7 @@ prefs = [
 
 ["browser_chat_contextmenu.js"]
 ["browser_chat_request.js"]
+["browser_chat_shortcuts.js"]
 ["browser_chat_sidebar.js"]
 ["browser_chat_telemetry.js"]
 ["browser_genai_actors.js"]

browser/components/genai/tests/browser/browser_chat_shortcuts.js

@@ -0,0 +1,52 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Check that shortcuts aren't shown by default
+ */
+add_task(async function test_no_shortcuts() {
+  await SpecialPowers.pushPrefEnv({
+    set: [["browser.ml.chat.provider", "http://localhost:8080"]],
+  });
+  await BrowserTestUtils.withNewTab("data:text/plain,hi", async browser => {
+    browser.focus();
+    goDoCommand("cmd_selectAll");
+    Assert.ok(
+      !document.querySelector(".content-shortcuts"),
+      "No shortcuts found"
+    );
+  });
+});
+
+/**
+ * Check that shortcuts get shown on selection and open popup and sidebar
+ */
+add_task(async function test_show_shortcuts() {
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["browser.ml.chat.shortcuts", true],
+      ["browser.ml.chat.shortcutsDebounce", 0],
+    ],
+  });
+  await BrowserTestUtils.withNewTab("data:text/plain,hi", async browser => {
+    await SimpleTest.promiseFocus(browser);
+    goDoCommand("cmd_selectAll");
+    const shortcuts = await TestUtils.waitForCondition(() =>
+      document.querySelector(".content-shortcuts")
+    );
+    Assert.ok(shortcuts, "Shortcuts added on select");
+
+    const popup = document.getElementById("ask-chat-shortcuts");
+    Assert.equal(popup.state, "closed", "Popup is closed");
+
+    EventUtils.sendMouseEvent({ type: "mouseover" }, shortcuts);
+    await BrowserTestUtils.waitForEvent(popup, "popupshown");
+    Assert.equal(popup.state, "open", "Popup is open");
+
+    Assert.ok(!SidebarController.isOpen, "Sidebar is closed");
+    popup.querySelector("toolbarbutton").click();
+    Assert.ok(SidebarController.isOpen, "Chat opened sidebar");
+
+    SidebarController.hide();
+  });
+});