Bug 1844495: Implement base of MDN Suggestions r=adw,dao

Differential Revision: https://phabricator.services.mozilla.com/D184075

browser/app/profile/firefox.js

@@ -614,6 +614,10 @@ pref("browser.urlbar.addons.featureGate", false);
 // addons suggestions are turned on.
 pref("browser.urlbar.suggest.addons", true);
 
+// If `browser.urlbar.mdn.featureGate` is true, this controls whether
+// mdn suggestions are turned on.
+pref("browser.urlbar.suggest.mdn", true);
+
 // The minimum prefix length of addons keyword the user must type to trigger
 // the suggestion. 0 means the min length should be taken from Nimbus.
 pref("browser.urlbar.addons.minKeywordLength", 0);

browser/components/urlbar/QuickSuggest.sys.mjs

@@ -20,6 +20,7 @@ const FEATURES = {
   BlockedSuggestions:
     "resource:///modules/urlbar/private/BlockedSuggestions.sys.mjs",
   ImpressionCaps: "resource:///modules/urlbar/private/ImpressionCaps.sys.mjs",
+  MDNSuggestions: "resource:///modules/urlbar/private/MDNSuggestions.sys.mjs",
   PocketSuggestions:
     "resource:///modules/urlbar/private/PocketSuggestions.sys.mjs",
   Weather: "resource:///modules/urlbar/private/Weather.sys.mjs",

browser/components/urlbar/UrlbarPrefs.sys.mjs

@@ -167,6 +167,9 @@ const PREF_URLBAR_DEFAULTS = new Map([
   // The maximum number of results in the urlbar popup.
   ["maxRichResults", 10],
 
+  // Feature gate pref for mdn suggestions in the urlbar.
+  ["mdn.featureGate", false],
+
   // Comma-separated list of client variants to send to Merino
   ["merino.clientVariants", ""],
 
@@ -284,6 +287,10 @@ const PREF_URLBAR_DEFAULTS = new Map([
   // addon suggestions are turned on.
   ["suggest.addons", true],
 
+  // If `browser.urlbar.mdn.featureGate` is true, this controls whether
+  // mdn suggestions are turned on.
+  ["suggest.mdn", true],
+
   // Whether results will include search suggestions.
   ["suggest.searches", false],
 

browser/components/urlbar/moz.build

@@ -61,6 +61,7 @@ EXTRA_JS_MODULES["urlbar/private"] += [
     "private/BaseFeature.sys.mjs",
     "private/BlockedSuggestions.sys.mjs",
     "private/ImpressionCaps.sys.mjs",
+    "private/MDNSuggestions.sys.mjs",
     "private/PocketSuggestions.sys.mjs",
     "private/QuickSuggestRemoteSettings.sys.mjs",
     "private/Weather.sys.mjs",

browser/components/urlbar/private/MDNSuggestions.sys.mjs

@@ -0,0 +1,114 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { BaseFeature } from "resource:///modules/urlbar/private/BaseFeature.sys.mjs";
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+  QuickSuggestRemoteSettings:
+    "resource:///modules/urlbar/private/QuickSuggestRemoteSettings.sys.mjs",
+  SuggestionsMap:
+    "resource:///modules/urlbar/private/QuickSuggestRemoteSettings.sys.mjs",
+  UrlbarPrefs: "resource:///modules/UrlbarPrefs.sys.mjs",
+  UrlbarResult: "resource:///modules/UrlbarResult.sys.mjs",
+  UrlbarUtils: "resource:///modules/UrlbarUtils.sys.mjs",
+});
+
+/**
+ * A feature that supports MDN suggestions.
+ */
+export class MDNSuggestions extends BaseFeature {
+  get shouldEnable() {
+    return (
+      lazy.UrlbarPrefs.get("mdn.featureGate") &&
+      lazy.UrlbarPrefs.get("suggest.mdn") &&
+      lazy.UrlbarPrefs.get("suggest.quicksuggest.nonsponsored")
+    );
+  }
+
+  get enablingPreferences() {
+    return [
+      "mdn.featureGate",
+      "suggest.mdn",
+      "suggest.quicksuggest.nonsponsored",
+    ];
+  }
+
+  get merinoProvider() {
+    return "mdn";
+  }
+
+  enable(enabled) {
+    if (enabled) {
+      lazy.QuickSuggestRemoteSettings.register(this);
+    } else {
+      lazy.QuickSuggestRemoteSettings.unregister(this);
+    }
+  }
+
+  queryRemoteSettings(searchString) {
+    const suggestions = this.#suggestionsMap?.get(searchString);
+    return suggestions
+      ? suggestions.map(suggestion => ({ ...suggestion }))
+      : [];
+  }
+
+  async onRemoteSettingsSync(rs) {
+    const records = await rs.get({ filters: { type: "mdn-suggestions" } });
+    if (!this.isEnabled) {
+      return;
+    }
+
+    const suggestionsMap = new lazy.SuggestionsMap();
+
+    for (const record of records) {
+      const { buffer } = await rs.attachments.download(record);
+      if (!this.isEnabled) {
+        return;
+      }
+
+      const results = JSON.parse(new TextDecoder("utf-8").decode(buffer));
+      await suggestionsMap.add(results, {
+        mapKeyword:
+          lazy.SuggestionsMap.MAP_KEYWORD_PREFIXES_STARTING_AT_FIRST_WORD,
+      });
+      if (!this.isEnabled) {
+        return;
+      }
+    }
+
+    this.#suggestionsMap = suggestionsMap;
+  }
+
+  async makeResult(queryContext, suggestion, searchString) {
+    if (!this.isEnabled) {
+      // The feature is disabled on the client, but Merino may still return
+      // mdn suggestions anyway, and we filter them out here.
+      return null;
+    }
+
+    const payload = {
+      icon: "chrome://devtools/skin/images/mdn.svg",
+      url: suggestion.url,
+      title: [suggestion.title, lazy.UrlbarUtils.HIGHLIGHT.TYPED],
+      description: suggestion.description,
+      shouldShowUrl: true,
+    };
+
+    return Object.assign(
+      new lazy.UrlbarResult(
+        lazy.UrlbarUtils.RESULT_TYPE.URL,
+        lazy.UrlbarUtils.RESULT_SOURCE.OTHER_NETWORK,
+        ...lazy.UrlbarResult.payloadAndSimpleHighlights(
+          queryContext.tokens,
+          payload
+        )
+      ),
+      { showFeedbackMenu: true }
+    );
+  }
+
+  #suggestionsMap = null;
+}

browser/components/urlbar/tests/quicksuggest/browser/browser.ini

@@ -16,6 +16,7 @@ support-files =
 [browser_quicksuggest_indexes.js]
 skip-if =
   os == "win" && os_version == "6.1" # Skip on Azure - frequent failure
+[browser_quicksuggest_mdn.js]
 [browser_quicksuggest_merinoSessions.js]
 [browser_quicksuggest_onboardingDialog.js]
 skip-if =

browser/components/urlbar/tests/quicksuggest/browser/browser_quicksuggest_mdn.js

@@ -0,0 +1,90 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test for mdn suggestions.
+
+const REMOTE_SETTINGS_DATA = [
+  {
+    type: "mdn-suggestions",
+    attachment: [
+      {
+        url: "https://example.com/array-filter",
+        title: "Array.prototype.filter()",
+        description:
+          "The filter() method creates a shallow copy of a portion of a given array, filtered down to just the elements from the given array that pass the test implemented by the provided function.",
+        keywords: ["array"],
+      },
+    ],
+  },
+];
+
+add_setup(async function () {
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["browser.urlbar.quicksuggest.enabled", true],
+      ["browser.urlbar.quicksuggest.nonsponsored", true],
+      ["browser.urlbar.quicksuggest.remoteSettings.enabled", true],
+      ["browser.urlbar.bestMatch.enabled", true],
+      ["browser.urlbar.suggest.mdn", true],
+    ],
+  });
+
+  await QuickSuggestTestUtils.ensureQuickSuggestInit({
+    remoteSettingsResults: REMOTE_SETTINGS_DATA,
+  });
+});
+
+add_task(async function basic() {
+  await SpecialPowers.pushPrefEnv({
+    set: [["browser.urlbar.mdn.featureGate", true]],
+  });
+
+  const suggestion = REMOTE_SETTINGS_DATA[0].attachment[0];
+  await UrlbarTestUtils.promiseAutocompleteResultPopup({
+    window,
+    value: suggestion.keywords[0],
+  });
+  Assert.equal(UrlbarTestUtils.getResultCount(window), 2);
+
+  const { element, result } = await UrlbarTestUtils.getDetailsOfResultAt(
+    window,
+    1
+  );
+  Assert.equal(
+    result.providerName,
+    UrlbarProviderQuickSuggest.name,
+    "The result should be from the expected provider"
+  );
+  Assert.equal(result.payload.provider, "MDNSuggestions");
+
+  const onLoad = BrowserTestUtils.browserLoaded(
+    gBrowser.selectedBrowser,
+    false,
+    suggestion.url
+  );
+  EventUtils.synthesizeMouseAtCenter(element.row, {});
+  await onLoad;
+  Assert.ok(true, "Expected page is loaded");
+
+  await PlacesUtils.history.clear();
+  await SpecialPowers.popPrefEnv();
+});
+
+add_task(async function disable() {
+  await SpecialPowers.pushPrefEnv({
+    set: [["browser.urlbar.mdn.featureGate", false]],
+  });
+
+  await UrlbarTestUtils.promiseAutocompleteResultPopup({
+    window,
+    value: "array",
+  });
+  Assert.equal(UrlbarTestUtils.getResultCount(window), 1);
+
+  const { result } = await UrlbarTestUtils.getDetailsOfResultAt(window, 0);
+  Assert.equal(result.providerName, "HeuristicFallback");
+
+  await SpecialPowers.popPrefEnv();
+});

browser/components/urlbar/tests/quicksuggest/unit/test_quicksuggest_mdn.js

@@ -0,0 +1,224 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Tests Pocket quick suggest results.
+
+"use strict";
+
+const REMOTE_SETTINGS_DATA = [
+  {
+    type: "mdn-suggestions",
+    attachment: [
+      {
+        url: "https://example.com/array-filter",
+        title: "Array.prototype.filter()",
+        description:
+          "The filter() method creates a shallow copy of a portion of a given array, filtered down to just the elements from the given array that pass the test implemented by the provided function.",
+        is_top_pick: true,
+        keywords: ["array filter"],
+      },
+      {
+        url: "https://example.com/input",
+        title: "<input>: The Input (Form Input) element",
+        description:
+          "The <input> HTML element is used to create interactive controls for web-based forms in order to accept data from the user; a wide variety of types of input data and control widgets are available, depending on the device and user agent. The <input> element is one of the most powerful and complex in all of HTML due to the sheer number of combinations of input types and attributes.",
+        is_top_pick: false,
+        keywords: ["input"],
+      },
+      {
+        url: "https://example.com/grid",
+        title: "CSS Grid Layout",
+        description:
+          "CSS Grid Layout excels at dividing a page into major regions or defining the relationship in terms of size, position, and layer, between parts of a control built from HTML primitives.",
+        keywords: ["grid"],
+      },
+    ],
+  },
+];
+
+add_setup(async function init() {
+  UrlbarPrefs.set("quicksuggest.enabled", true);
+  UrlbarPrefs.set("bestMatch.enabled", true);
+  UrlbarPrefs.set("suggest.quicksuggest.nonsponsored", true);
+  UrlbarPrefs.set("suggest.quicksuggest.sponsored", false);
+  UrlbarPrefs.set("suggest.mdn", true);
+  UrlbarPrefs.set("mdn.featureGate", true);
+
+  // Disable search suggestions so we don't hit the network.
+  Services.prefs.setBoolPref("browser.search.suggest.enabled", false);
+
+  await QuickSuggestTestUtils.ensureQuickSuggestInit({
+    remoteSettingsResults: REMOTE_SETTINGS_DATA,
+  });
+  await waitForSuggestions();
+});
+
+add_task(async function basic() {
+  for (const suggestion of REMOTE_SETTINGS_DATA[0].attachment) {
+    const fullKeyword = suggestion.keywords[0];
+    const firstWord = fullKeyword.split(" ")[0];
+    for (let i = 1; i < fullKeyword.length; i++) {
+      const keyword = fullKeyword.substring(0, i);
+      const shouldMatch = i >= firstWord.length;
+      const matches = shouldMatch
+        ? [makeExpectedResult({ searchString: keyword, suggestion })]
+        : [];
+      await check_results({
+        context: createContext(keyword, {
+          providers: [UrlbarProviderQuickSuggest.name],
+          isPrivate: false,
+        }),
+        matches,
+      });
+    }
+
+    await check_results({
+      context: createContext(fullKeyword + " ", {
+        providers: [UrlbarProviderQuickSuggest.name],
+        isPrivate: false,
+      }),
+      matches: [],
+    });
+  }
+});
+
+// Check wheather the MDN suggestions will be hidden by the pref.
+add_task(async function disableByLocalPref() {
+  const suggestion = REMOTE_SETTINGS_DATA[0].attachment[0];
+  const keyword = suggestion.keywords[0];
+
+  const prefs = [
+    "suggest.mdn",
+    "quicksuggest.enabled",
+    "suggest.quicksuggest.nonsponsored",
+  ];
+
+  for (const pref of prefs) {
+    // First make sure the suggestion is added.
+    await check_results({
+      context: createContext(keyword, {
+        providers: [UrlbarProviderQuickSuggest.name],
+        isPrivate: false,
+      }),
+      matches: [
+        makeExpectedResult({
+          searchString: keyword,
+          suggestion,
+        }),
+      ],
+    });
+
+    // Now disable them.
+    UrlbarPrefs.set(pref, false);
+    await check_results({
+      context: createContext(keyword, {
+        providers: [UrlbarProviderQuickSuggest.name],
+        isPrivate: false,
+      }),
+      matches: [],
+    });
+
+    // Revert.
+    UrlbarPrefs.set(pref, true);
+    await waitForSuggestions();
+  }
+});
+
+// Check wheather the MDN suggestions will be shown by the setup of Nimbus
+// variable.
+add_task(async function nimbus() {
+  // Nimbus variable mdn.featureGate changes the pref in default branch
+  // (by setPref in FeatureManifest). So, as it will not override the user branch
+  // pref, should use default branch if the test needs Nimbus and needs to change
+  // mdn.featureGate in local.
+  UrlbarPrefs.clear("mdn.featureGate");
+  const defaultPrefs = Services.prefs.getDefaultBranch("browser.urlbar.");
+
+  const suggestion = REMOTE_SETTINGS_DATA[0].attachment[0];
+  const keyword = suggestion.keywords[0];
+
+  // Disable the fature gate.
+  defaultPrefs.setBoolPref("mdn.featureGate", false);
+  await check_results({
+    context: createContext(keyword, {
+      providers: [UrlbarProviderQuickSuggest.name],
+      isPrivate: false,
+    }),
+    matches: [],
+  });
+
+  // Enable by Nimbus.
+  const cleanUpNimbusEnable = await UrlbarTestUtils.initNimbusFeature(
+    { mdnFeatureGate: true },
+    "urlbar",
+    "config"
+  );
+  await waitForSuggestions();
+  await check_results({
+    context: createContext(keyword, {
+      providers: [UrlbarProviderQuickSuggest.name],
+      isPrivate: false,
+    }),
+    matches: [makeExpectedResult({ searchString: keyword, suggestion })],
+  });
+  await cleanUpNimbusEnable();
+
+  // Enable locally.
+  defaultPrefs.setBoolPref("mdn.featureGate", true);
+  await waitForSuggestions();
+
+  // Disable by Nimbus.
+  const cleanUpNimbusDisable = await UrlbarTestUtils.initNimbusFeature(
+    { mdnFeatureGate: false },
+    "urlbar",
+    "config"
+  );
+  await check_results({
+    context: createContext(keyword, {
+      providers: [UrlbarProviderQuickSuggest.name],
+      isPrivate: false,
+    }),
+    matches: [],
+  });
+  await cleanUpNimbusDisable();
+
+  // Revert.
+  defaultPrefs.setBoolPref("mdn.featureGate", true);
+  await waitForSuggestions();
+});
+
+function makeExpectedResult({
+  searchString,
+  suggestion,
+  source = "remote-settings",
+} = {}) {
+  const isTopPick = !!suggestion.is_top_pick;
+  return {
+    isBestMatch: isTopPick,
+    suggestedIndex: isTopPick ? 1 : -1,
+    type: UrlbarUtils.RESULT_TYPE.URL,
+    source: UrlbarUtils.RESULT_SOURCE.OTHER_NETWORK,
+    heuristic: false,
+    payload: {
+      source,
+      provider: source == "remote-settings" ? "MDNSuggestions" : "mdn",
+      telemetryType: "mdn",
+      title: suggestion.title,
+      url: suggestion.url,
+      displayUrl: suggestion.url.replace(/^https:\/\//, ""),
+      description: isTopPick ? suggestion.description : "",
+      icon: "chrome://devtools/skin/images/mdn.svg",
+      shouldShowUrl: true,
+    },
+  };
+}
+
+async function waitForSuggestions() {
+  let keyword = REMOTE_SETTINGS_DATA[0].attachment[0].keywords[0];
+  let feature = QuickSuggest.getFeature("MDNSuggestions");
+  await TestUtils.waitForCondition(async () => {
+    let suggestions = await feature.queryRemoteSettings(keyword);
+    return !!suggestions.length;
+  }, "Waiting for MDNSuggestions to serve remote settings suggestions");
+}

browser/components/urlbar/tests/quicksuggest/unit/xpcshell.ini

@@ -10,6 +10,7 @@ firefox-appdir = browser
 [test_quicksuggest_bestMatch.js]
 [test_quicksuggest_dynamicWikipedia.js]
 [test_quicksuggest_impressionCaps.js]
+[test_quicksuggest_mdn.js]
 [test_quicksuggest_merino.js]
 [test_quicksuggest_merinoSessions.js]
 [test_quicksuggest_migrate_v1.js]

toolkit/components/nimbus/FeatureManifest.yaml

@@ -199,6 +199,12 @@ urlbar:
       type: boolean
       description: >-
         Whether the experiment (or rollout) is related to best match. If true, then the Nimbus exposure event will be recorded when the user first triggers a best match (or would have triggered a best match, for users in the control group). Deprecated, please use `experimentType: "best-match"` instead.
+    mdnFeatureGate:
+      type: boolean
+      setPref: browser.urlbar.mdn.featureGate
+      description: >-
+        Feature gate that controls whether all aspects of the mdn suggestion
+        feature are exposed to the user.
     merinoClientVariants:
       type: string
       fallbackPref: browser.urlbar.merino.clientVariants