Bug 1727668 - Integrate Merino with Firefox Suggest/quick suggest. r=mak,nanj

This integrates a fetch to Merino in UrlbarProviderQuickSuggest. We continue to
do the remote settings fetch too. Per the Jira ticket, we should prefer the
Merino suggestion when both sources return one.

Each fetch is controlled by a new pref and Nimbus variable, so we can enable
them independently.

At first I started making a UrlbarProviderMerino class, but it's better to
modify UrlbarProviderQuickSuggest because everything besides the fetch source is
the same: We want to collect the same telemetry, have the same results and
payloads, etc.

Depends on D123852

Differential Revision: https://phabricator.services.mozilla.com/D123707

browser/app/profile/firefox.js

@@ -380,6 +380,9 @@ pref("browser.urlbar.quicksuggest.showOnboardingDialogAfterNRestarts", 0);
 pref("browser.urlbar.quicksuggest.sponsoredIndex", -1);
 pref("browser.urlbar.quicksuggest.nonSponsoredIndex", -1);
 
+// Whether Remote Settings is enabled as a quick suggest source.
+pref("browser.urlbar.quicksuggest.remoteSettings.enabled", true);
+
 // Whether unit conversion is enabled.
 #ifdef NIGHTLY_BUILD
 pref("browser.urlbar.unitConversion.enabled", true);
@@ -449,6 +452,12 @@ pref("browser.urlbar.keepPanelOpenDuringImeComposition", false);
 // Whether Firefox Suggest group labels are shown in the urlbar view.
 pref("browser.urlbar.groupLabels.enabled", true);
 
+// Whether Merino is enabled as a quick suggest source in the urlbar.
+pref("browser.urlbar.merino.enabled", false);
+
+// The Merino endpoint URL, not including parameters.
+pref("browser.urlbar.merino.endpointURL", "https://merino.services.mozilla.com/api/v1/suggest");
+
 pref("browser.altClickSave", false);
 
 // Enable logging downloads operations to the Console.

browser/components/urlbar/UrlbarInput.jsm

@@ -1334,11 +1334,10 @@ class UrlbarInput {
       ),
       currentPage: this.window.gBrowser.currentURI.spec,
       formHistoryName: this.formHistoryName,
-      allowSearchSuggestions:
-        !event ||
-        !UrlbarUtils.isPasteEvent(event) ||
-        !event.data ||
-        event.data.length <= UrlbarPrefs.get("maxCharsForSearchSuggestions"),
+      prohibitRemoteResults:
+        event &&
+        UrlbarUtils.isPasteEvent(event) &&
+        UrlbarPrefs.get("maxCharsForSearchSuggestions") < event.data?.length,
     };
 
     if (this.searchMode) {

browser/components/urlbar/UrlbarPrefs.jsm

@@ -112,9 +112,10 @@ const PREF_URLBAR_DEFAULTS = new Map([
   // Whether the results panel should be kept open during IME composition.
   ["keepPanelOpenDuringImeComposition", false],
 
-  // For search suggestion results, we truncate the user's search string to this
-  // number of characters before fetching results.
-  ["maxCharsForSearchSuggestions", 20],
+  // As a user privacy measure, don't fetch results from remote services for
+  // searches that start by pasting a string longer than this. The pref name
+  // indicates search suggestions, but this is used for all remote results.
+  ["maxCharsForSearchSuggestions", 100],
 
   // The maximum number of form history results to include.
   ["maxHistoricalSearchSuggestions", 0],
@@ -122,6 +123,12 @@ const PREF_URLBAR_DEFAULTS = new Map([
   // The maximum number of results in the urlbar popup.
   ["maxRichResults", 10],
 
+  // Whether Merino is enabled as a quick suggest source.
+  ["merino.enabled", false],
+
+  // The Merino endpoint URL, not including parameters.
+  ["merino.endpointURL", "https://merino.services.mozilla.com/api/v1/suggest"],
+
   // Whether addresses and search results typed into the address bar
   // should be opened in new tabs by default.
   ["openintab", false],
@@ -184,15 +191,25 @@ const PREF_URLBAR_DEFAULTS = new Map([
   // Whether results will include QuickSuggest suggestions.
   ["suggest.quicksuggest", false],
 
+  // Whether the quick suggest feature is enabled, i.e., sponsored and
+  // recommended results related to the user's search string.
+  ["quicksuggest.enabled", false],
+
+  // Whether to show QuickSuggest related logs.
+  ["quicksuggest.log", false],
+
+  // Whether Remote Settings is enabled as a quick suggest source.
+  ["quicksuggest.remoteSettings.enabled", true],
+
+  // Whether to show the quick suggest onboarding dialog.
+  ["quicksuggest.shouldShowOnboardingDialog", true],
+
   // Whether the user has seen the onboarding dialog.
   ["quicksuggest.showedOnboardingDialog", false],
 
   // Count the restarts before showing the onboarding dialog.
   ["quicksuggest.seenRestarts", 0],
 
-  // Whether to show QuickSuggest related logs.
-  ["quicksuggest.log", false],
-
   // When using switch to tabs, if set to true this will move the tab into the
   // active window.
   ["switchTabs.adoptIntoActiveWindow", false],

browser/components/urlbar/UrlbarProviderQuickSuggest.jsm

@@ -23,8 +23,9 @@ XPCOMUtils.defineLazyModuleGetters(this, {
   UrlbarUtils: "resource:///modules/UrlbarUtils.jsm",
 });
 
-// These prefs are relative to the `browser.urlbar` branch.
-const SUGGEST_PREF = "suggest.quicksuggest";
+XPCOMUtils.defineLazyGlobalGetters(this, ["AbortController", "fetch"]);
+
+const MERINO_ENDPOINT_PARAM_QUERY = "q";
 
 const TELEMETRY_SCALAR_IMPRESSION =
   "contextual.services.quicksuggest.impression";
@@ -95,7 +96,7 @@ class ProviderQuickSuggest extends UrlbarProvider {
       !queryContext.searchMode &&
       !queryContext.isPrivate &&
       UrlbarPrefs.get("quickSuggestEnabled") &&
-      UrlbarPrefs.get(SUGGEST_PREF) &&
+      UrlbarPrefs.get("suggest.quicksuggest") &&
       UrlbarPrefs.get("suggest.searches") &&
       UrlbarPrefs.get("browser.search.suggest.enabled")
     );
@@ -111,15 +112,33 @@ class ProviderQuickSuggest extends UrlbarProvider {
    */
   async startQuery(queryContext, addCallback) {
     let instance = this.queryInstance;
-    let suggestion = await UrlbarQuickSuggest.query(
-      queryContext.searchString.trimStart()
-    );
-    if (!suggestion || instance != this.queryInstance) {
+
+    // Trim only the start of the search string because a trailing space can
+    // affect the suggestions.
+    let searchString = queryContext.searchString.trimStart();
+
+    // We currently have two sources for quick suggest: remote settings
+    // (from `UrlbarQuickSuggest`) and Merino.
+    let promises = [];
+    if (UrlbarPrefs.get("quickSuggestRemoteSettingsEnabled")) {
+      promises.push(UrlbarQuickSuggest.query(searchString));
+    }
+    if (UrlbarPrefs.get("merinoEnabled") && queryContext.allowRemoteResults()) {
+      promises.push(this._fetchMerinoSuggestion(searchString));
+    }
+    let [rsSuggestion, merinoSuggestion] = await Promise.all(promises);
+    if (instance != this.queryInstance) {
+      return;
+    }
+
+    // We prefer the Merino suggestion.
+    let suggestion = merinoSuggestion || rsSuggestion;
+    if (!suggestion) {
       return;
     }
 
     let payload = {
-      qsSuggestion: [suggestion.fullKeyword, UrlbarUtils.HIGHLIGHT.SUGGESTED],
+      qsSuggestion: [suggestion.full_keyword, UrlbarUtils.HIGHLIGHT.SUGGESTED],
       title: suggestion.title,
       url: suggestion.url,
       icon: suggestion.icon,
@@ -132,7 +151,7 @@ class ProviderQuickSuggest extends UrlbarProvider {
       helpL10nId: "firefox-suggest-urlbar-learn-more",
     };
 
-    if (!suggestion.isSponsored) {
+    if (!suggestion.is_sponsored) {
       payload.sponsoredL10nId = "firefox-suggest-urlbar-nonsponsored-action";
     }
 
@@ -143,7 +162,7 @@ class ProviderQuickSuggest extends UrlbarProvider {
     );
     result.isSuggestedIndexRelativeToGroup = true;
     result.suggestedIndex = UrlbarPrefs.get(
-      suggestion.isSponsored
+      suggestion.is_sponsored
         ? "quickSuggestSponsoredIndex"
         : "quickSuggestNonSponsoredIndex"
     );
@@ -282,16 +301,93 @@ class ProviderQuickSuggest extends UrlbarProvider {
    */
   onPrefChanged(pref) {
     switch (pref) {
-      case SUGGEST_PREF:
+      case "suggest.quicksuggest":
         Services.telemetry.recordEvent(
           TELEMETRY_EVENT_CATEGORY,
           "enable_toggled",
-          UrlbarPrefs.get(SUGGEST_PREF) ? "enabled" : "disabled"
+          UrlbarPrefs.get("suggest.quicksuggest") ? "enabled" : "disabled"
         );
         break;
     }
   }
 
+  /**
+   * Cancels the current query.
+   *
+   * @param {UrlbarQueryContext} queryContext
+   */
+  cancelQuery(queryContext) {
+    try {
+      this._merinoFetchController?.abort();
+    } catch (error) {
+      Cu.reportError(error);
+    }
+    this._merinoFetchController = null;
+  }
+
+  /**
+   * Fetches a Merino suggestion.
+   *
+   * @param {string} searchString
+   * @returns {object}
+   *   The Merino suggestion object, or null if there isn't one.
+   */
+  async _fetchMerinoSuggestion(searchString) {
+    let instance = this.queryInstance;
+
+    // Fetch a response from the endpoint.
+    let response;
+    let controller;
+    try {
+      let url = new URL(UrlbarPrefs.get("merino.endpointURL"));
+      url.searchParams.set(MERINO_ENDPOINT_PARAM_QUERY, searchString);
+
+      controller = this._merinoFetchController = new AbortController();
+      response = await fetch(url, {
+        signal: controller.signal,
+      });
+      if (instance != this.queryInstance) {
+        return null;
+      }
+    } catch (error) {
+      if (error.name != "AbortError") {
+        Cu.reportError(error);
+      }
+    } finally {
+      if (controller == this._merinoFetchController) {
+        this._merinoFetchController = null;
+      }
+    }
+
+    if (!response) {
+      return null;
+    }
+
+    // Get the response body as an object.
+    let body;
+    try {
+      body = await response.json();
+      if (instance != this.queryInstance) {
+        return null;
+      }
+    } catch (error) {
+      Cu.reportError(error);
+    }
+
+    if (!body?.suggestions?.length) {
+      return null;
+    }
+
+    let { suggestions } = body;
+    if (!Array.isArray(suggestions)) {
+      Cu.reportError("Unexpected Merino response: " + JSON.stringify(body));
+      return null;
+    }
+
+    // Return the first suggestion.
+    return suggestions[0];
+  }
+
   /**
    * Updates state based on the `browser.urlbar.quicksuggest.enabled` pref.
    * Enable/disable event telemetry and ensure QuickSuggest module is loaded

browser/components/urlbar/UrlbarProviderSearchSuggestions.jsm

@@ -139,7 +139,6 @@ class ProviderSearchSuggestions extends UrlbarProvider {
    */
   _allowSuggestions(queryContext) {
     if (
-      !queryContext.allowSearchSuggestions ||
       // If the user typed a restriction token or token alias, we ignore the
       // pref to disable suggestions in the Urlbar.
       (!UrlbarPrefs.get("suggest.searches") &&
@@ -165,6 +164,13 @@ class ProviderSearchSuggestions extends UrlbarProvider {
     queryContext,
     searchString = queryContext.searchString
   ) {
+    // This is checked by `queryContext.allowRemoteResults` below, but we can
+    // short-circuit that call with the `_isTokenOrRestrictionPresent` block
+    // before that. Make sure we don't allow remote suggestions if this is set.
+    if (queryContext.prohibitRemoteResults) {
+      return false;
+    }
+
     // TODO (Bug 1626964): Support zero prefix suggestions.
     if (!searchString.trim()) {
       return false;
@@ -188,30 +194,7 @@ class ProviderSearchSuggestions extends UrlbarProvider {
       return false;
     }
 
-    // We're unlikely to get useful remote suggestions for a single character.
-    if (searchString.length < 2) {
-      return false;
-    }
-
-    // Disallow remote suggestions if only an origin is typed to avoid
-    // disclosing information about sites the user visits. This also catches
-    // partially-typed origins, like mozilla.o, because the URIFixup check
-    // below can't validate those.
-    if (
-      queryContext.tokens.length == 1 &&
-      queryContext.tokens[0].type == UrlbarTokenizer.TYPE.POSSIBLE_ORIGIN
-    ) {
-      return false;
-    }
-
-    // Disallow remote suggestions for strings containing tokens that look like
-    // URIs, to avoid disclosing information about networks or passwords.
-    if (queryContext.fixupInfo?.href && !queryContext.fixupInfo?.isSearch) {
-      return false;
-    }
-
-    // Allow remote suggestions.
-    return true;
+    return queryContext.allowRemoteResults(searchString);
   }
 
   /**

browser/components/urlbar/UrlbarQuickSuggest.jsm

@@ -90,7 +90,7 @@ class Suggestions {
       `${pad(d.getDate())}${pad(d.getHours())}`;
     let icon = await this.fetchIcon(result.icon);
     return {
-      fullKeyword: this.getFullKeyword(phrase, result.keywords),
+      full_keyword: this.getFullKeyword(phrase, result.keywords),
       title: result.title,
       url: result.url.replace("%YYYYMMDDHH%", date),
       click_url: result.click_url.replace("%YYYYMMDDHH%", date),
@@ -98,7 +98,7 @@ class Suggestions {
       impression_url: result.impression_url,
       block_id: result.id,
       advertiser: result.advertiser.toLocaleLowerCase(),
-      isSponsored: !NONSPONSORED_IAB_CATEGORIES.has(result.iab_category),
+      is_sponsored: !NONSPONSORED_IAB_CATEGORIES.has(result.iab_category),
       icon,
     };
   }

browser/components/urlbar/UrlbarUtils.jsm

@@ -980,7 +980,7 @@ var UrlbarUtils = {
       userContextId: window.gBrowser.selectedBrowser.getAttribute(
         "usercontextid"
       ),
-      allowSearchSuggestions: false,
+      prohibitRemoteResults: true,
       providers: ["AliasEngines", "BookmarkKeywords", "HeuristicFallback"],
     };
     if (window.gURLBar.searchMode) {
@@ -1532,11 +1532,11 @@ class UrlbarQueryContext {
    * @param {object} [options.searchMode]
    *   The input's current search mode.  See UrlbarInput.setSearchMode for a
    *   description.
-   * @param {boolean} [options.allowSearchSuggestions]
-   *   Whether to allow search suggestions.  This is a veto, meaning that when
-   *   false, suggestions will not be fetched, but when true, some other
-   *   condition may still prohibit suggestions, like private browsing mode.
-   *   Defaults to true.
+   * @param {boolean} [options.prohibitRemoteResults]
+   *   This provides a short-circuit override for `context.allowRemoteResults`.
+   *   If it's false, then `allowRemoteResults` will do its usual checks to
+   *   determine whether remote results are allowed. If it's true, then
+   *   `allowRemoteResults` will immediately return false. Defaults to false.
    * @param {string} [options.formHistoryName]
    *   The name under which the local form history is registered.
    */
@@ -1556,9 +1556,9 @@ class UrlbarQueryContext {
 
     // Manage optional properties of options.
     for (let [prop, checkFn, defaultValue] of [
-      ["allowSearchSuggestions", v => true, true],
       ["currentPage", v => typeof v == "string" && !!v.length],
       ["formHistoryName", v => typeof v == "string" && !!v.length],
+      ["prohibitRemoteResults", v => true, false],
       ["providers", v => Array.isArray(v) && v.length],
       ["searchMode", v => v && typeof v == "object"],
       ["sources", v => Array.isArray(v) && v.length],
@@ -1646,6 +1646,47 @@ class UrlbarQueryContext {
 
     return null;
   }
+
+  /**
+   * Returns whether results from remote services are generally allowed for the
+   * context. Callers can impose further restrictions as appropriate, but
+   * typically they should not fetch remote results if this returns false.
+   *
+   * @param {string} [searchString]
+   *   Usually this is just the context's search string, but if you need to
+   *   fetch remote results based on a modified version, you can pass it here.
+   * @returns {boolean}
+   *   Whether remote results are allowed.
+   */
+  allowRemoteResults(searchString = this.searchString) {
+    if (this.prohibitRemoteResults) {
+      return false;
+    }
+
+    // We're unlikely to get useful remote results for a single character.
+    if (searchString.length < 2) {
+      return false;
+    }
+
+    // Disallow remote results if only an origin is typed to avoid disclosing
+    // sites the user visits. This also catches partially typed origins, like
+    // mozilla.o, because the fixup check below can't validate them.
+    if (
+      this.tokens.length == 1 &&
+      this.tokens[0].type == UrlbarTokenizer.TYPE.POSSIBLE_ORIGIN
+    ) {
+      return false;
+    }
+
+    // Disallow remote results for strings containing tokens that look like URIs
+    // to avoid disclosing information about networks and passwords.
+    if (this.fixupInfo?.href && !this.fixupInfo?.isSearch) {
+      return false;
+    }
+
+    // Allow remote results.
+    return true;
+  }
 }
 
 /**

browser/components/urlbar/docs/overview.rst

@@ -52,12 +52,12 @@ It is augmented as it progresses through the system, with various information:
                 // with the search service.
     currentPage: // {string} url of the page that was loaded when the search
                  // began.
-    allowSearchSuggestions: // {boolean} Whether to allow search suggestions.
-                            // This is a veto, meaning that when false,
-                            // suggestions will not be fetched, but when true,
-                            // some other condition may still prohibit
-                            // suggestions, like private browsing mode. Defaults
-                            // to true.
+    prohibitRemoteResults:
+      // {boolean} This provides a short-circuit override for
+      // context.allowRemoteResults(). If it's false, then allowRemoteResults()
+      // will do its usual checks to determine whether remote results are
+      // allowed. If it's true, then allowRemoteResults() will immediately
+      // return false. Defaults to false.
 
     // Properties added by the Model.
     results; // {array} list of UrlbarResult objects.

browser/components/urlbar/docs/preferences.rst

@@ -64,27 +64,9 @@ Hidden
 These preferences are normally hidden, and should not be used unless you really
 know what you are doing.
 
-browser.urlbar.speculativeConnect.enabled (boolean, default: true)
-  Speculative connections allow to resolve domains pre-emptively when the user
-  is likely to pick a result from the Address Bar. This allows for faster
-  navigation.
-
 browser.urlbar.accessibility.tabToSearch.announceResults (boolean: default: true)
   Whether we announce to screen readers when tab-to-search results are inserted.
 
-browser.urlbar.quicksuggest.log (boolean, default: false)
-  Whether to show QuickSuggest related logs, by default only logs Warnings.
-
-browser.urlbar.switchTabs.adoptIntoActiveWindow (boolean, default: false)
-  When using switch to tabs, if set to true this will move the tab into the
-  active window, instead of just switching to it.
-
-browser.urlbar.trimURLs (boolean, default: true)
-  Clean-up URLs when showing them in the Address Bar.
-
-browser.urlbar.sponsoredTopSites (boolean, default: false)
-  Whether top sites may include sponsored ones.
-
 browser.urlbar.autoFill (boolean, default: true)
   Autofill is the the feature that automatically completes domains and URLs that
   the user has visited as the user is typing them in the urlbar textbox.
@@ -132,8 +114,9 @@ browser.urlbar.formatting.enabled (boolean, default: true)
   field. This should usually be enabled for security reasons.
 
 browser.urlbar.maxCharsForSearchSuggestions (integer, default: 100)
-  When fetching search suggestion results, we truncate the user's search string
-  to this number of characters.
+  As a user privacy measure, we don't fetch results from remote services for
+  searches that start by pasting a string longer than this. The pref name
+  indicates search suggestions, but this is used for all remote results.
 
 browser.urlbar.maxHistoricalSearchSuggestions (integer, default: 2)
   The maximum number of form history results to include as search history.
@@ -141,9 +124,25 @@ browser.urlbar.maxHistoricalSearchSuggestions (integer, default: 2)
 browser.urlbar.maxRichResults (integer, default: 10)
   The maximum number of results in the urlbar popup.
 
+browser.urlbar.merino.enabled (boolean, default: false)
+  Whether Merino is enabled as a quick suggest source.
+
 browser.urlbar.openintab (boolean, default: false)
   Whether address bar results should be opened in new tabs by default.
 
+browser.urlbar.quicksuggest.enabled (boolean, default: false)
+  Whether the quick suggest feature is enabled, i.e., sponsored and recommended
+  results related to the user's search string.
+
+browser.urlbar.quicksuggest.log (boolean, default: false)
+  Whether to show QuickSuggest related logs, by default only logs Warnings.
+
+browser.urlbar.quicksuggest.remoteSettings.enabled (boolean, default: true)
+  Whether remote settings is enabled as a quick suggest source.
+
+browser.urlbar.quicksuggest.shouldShowOnboardingDialog (boolean, default: true)
+  Whether to show the quick suggest onboarding dialog.
+
 browser.urlbar.richSuggestions.tail (boolean, default: true)
   If true, we show tail search suggestions when available.
 
@@ -152,6 +151,21 @@ browser.urlbar.searchTips.test.ignoreShowLimits (boolean, default: false)
   time the newtab page or the default search engine homepage is opened.
   This is useful for testing purposes.
 
+browser.urlbar.speculativeConnect.enabled (boolean, default: true)
+  Speculative connections allow to resolve domains pre-emptively when the user
+  is likely to pick a result from the Address Bar. This allows for faster
+  navigation.
+
+browser.urlbar.sponsoredTopSites (boolean, default: false)
+  Whether top sites may include sponsored ones.
+
+browser.urlbar.switchTabs.adoptIntoActiveWindow (boolean, default: false)
+  When using switch to tabs, if set to true this will move the tab into the
+  active window, instead of just switching to it.
+
+browser.urlbar.trimURLs (boolean, default: true)
+  Clean-up URLs when showing them in the Address Bar.
+
 keyword.enabled (boolean, default: true)
   By default, when the search string is not recognized as a potential url,
   search for it with the default search engine. If set to false any string will

browser/components/urlbar/tests/UrlbarTestUtils.jsm

@@ -40,6 +40,7 @@ XPCOMUtils.defineLazyServiceGetter(
 // throw an "unknown pref" error if a test enrolls in a mock experiment and hits
 // a code path that accesses a Nimbus feature variable not defined here.
 const DEFAULT_EXPERIMENT_FEATURE_VARIABLES = {
+  merinoEnabled: false,
   quickSuggestEnabled: false,
   quickSuggestNonSponsoredIndex: -1,
   quickSuggestShouldShowOnboardingDialog: true,

browser/components/urlbar/tests/browser/browser_quicksuggest_configuration.js

@@ -132,3 +132,26 @@ add_task(async function test_indexes() {
     },
   });
 });
+
+add_task(async function test_merino() {
+  await UrlbarTestUtils.withExperiment({
+    valueOverrides: {
+      merinoEnabled: true,
+      merinoEndpointURL: "http://example.com/test_merino_config",
+      merinoEndpointParamQuery: "test_merino_config_param",
+    },
+    callback: () => {
+      Assert.equal(UrlbarPrefs.get("merinoEnabled"), true, "merinoEnabled");
+      Assert.equal(
+        UrlbarPrefs.get("merinoEndpointURL"),
+        "http://example.com/test_merino_config",
+        "merinoEndpointURL"
+      );
+      Assert.equal(
+        UrlbarPrefs.get("merinoEndpointParamQuery"),
+        "test_merino_config_param",
+        "merinoEndpointParamQuery"
+      );
+    },
+  });
+});

browser/components/urlbar/tests/browser/browser_urlbar_telemetry_quicksuggest.js

@@ -337,6 +337,8 @@ add_task(async function nimbusExposure() {
   UrlbarProviderQuickSuggest._recordedExposureEvent = false;
   let doExperimentCleanup = await UrlbarTestUtils.enrollExperiment({
     valueOverrides: {
+      // Make sure Merino is disabled so we don't hit the network.
+      merinoEnabled: false,
       quickSuggestEnabled: true,
       quickSuggestShouldShowOnboardingDialog: false,
     },

browser/components/urlbar/tests/unit/test_quicksuggest_merino.js

@@ -0,0 +1,432 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Tests Merino integration with the quick suggest feature/provider.
+
+"use strict";
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  UrlbarProviderQuickSuggest:
+    "resource:///modules/UrlbarProviderQuickSuggest.jsm",
+  UrlbarQuickSuggest: "resource:///modules/UrlbarQuickSuggest.jsm",
+});
+
+// relative to `browser.urlbar`
+const PREF_MERINO_ENABLED = "merino.enabled";
+const PREF_REMOTE_SETTINGS_ENABLED = "quicksuggest.remoteSettings.enabled";
+const PREF_MERINO_ENDPOINT_URL = "merino.endpointURL";
+
+const REMOTE_SETTINGS_DATA = [
+  {
+    id: 1,
+    url: "http://test.com/q=frabbits",
+    title: "frabbits",
+    keywords: ["fra", "frab"],
+    click_url: "http://click.reporting.test.com/",
+    impression_url: "http://impression.reporting.test.com/",
+    advertiser: "TestAdvertiser",
+  },
+];
+
+let gMerinoResponse;
+
+add_task(async function init() {
+  UrlbarPrefs.set("quicksuggest.enabled", true);
+  UrlbarPrefs.set("suggest.quicksuggest", true);
+  UrlbarPrefs.set("quicksuggest.shouldShowOnboardingDialog", false);
+
+  // Set up the Merino server.
+  let path = "/merino";
+  let server = makeMerinoServer(path);
+  let url = new URL("http://localhost/");
+  url.pathname = path;
+  url.port = server.identity.primaryPort;
+  UrlbarPrefs.set(PREF_MERINO_ENDPOINT_URL, url.toString());
+
+  // Set up the remote settings client with the test data.
+  await UrlbarQuickSuggest.init();
+  await UrlbarQuickSuggest._processSuggestionsJSON(REMOTE_SETTINGS_DATA);
+  UrlbarQuickSuggest.onEnabledUpdate = () => {};
+});
+
+// Tests with Merino enabled and remote settings disabled.
+add_task(async function merinoOnly() {
+  UrlbarPrefs.set(PREF_MERINO_ENABLED, true);
+  UrlbarPrefs.set(PREF_REMOTE_SETTINGS_ENABLED, false);
+
+  setMerinoResponse({
+    body: {
+      suggestions: [
+        {
+          full_keyword: "merinoOnly full_keyword",
+          title: "merinoOnly title",
+          url: "merinoOnly url",
+          icon: "merinoOnly icon",
+          impression_url: "merinoOnly impression_url",
+          click_url: "merinoOnly click_url",
+          block_id: 111,
+          advertiser: "merinoOnly advertiser",
+          is_sponsored: true,
+        },
+      ],
+    },
+  });
+
+  let context = createContext("frab", {
+    providers: [UrlbarProviderQuickSuggest.name],
+    isPrivate: false,
+  });
+  await check_results({
+    context,
+    matches: [
+      {
+        type: UrlbarUtils.RESULT_TYPE.URL,
+        source: UrlbarUtils.RESULT_SOURCE.SEARCH,
+        heuristic: false,
+        payload: {
+          qsSuggestion: "merinoOnly full_keyword",
+          title: "merinoOnly title",
+          url: "merinoOnly url",
+          icon: "merinoOnly icon",
+          sponsoredImpressionUrl: "merinoOnly impression_url",
+          sponsoredClickUrl: "merinoOnly click_url",
+          sponsoredBlockId: 111,
+          sponsoredAdvertiser: "merinoOnly advertiser",
+          isSponsored: true,
+          helpUrl: UrlbarProviderQuickSuggest.helpUrl,
+          helpL10nId: "firefox-suggest-urlbar-learn-more",
+          displayUrl: "merinoOnly url",
+        },
+      },
+    ],
+  });
+});
+
+// Tests with Merino disabled and remote settings enabled.
+add_task(async function remoteSettingsOnly() {
+  UrlbarPrefs.set(PREF_MERINO_ENABLED, false);
+  UrlbarPrefs.set(PREF_REMOTE_SETTINGS_ENABLED, true);
+
+  // Make sure the server is prepared to return a response so we can make sure
+  // we don't fetch it.
+  setMerinoResponse({
+    body: {
+      suggestions: [
+        {
+          full_keyword: "remoteSettingsOnly full_keyword",
+          title: "remoteSettingsOnly title",
+          url: "remoteSettingsOnly url",
+          icon: "remoteSettingsOnly icon",
+          impression_url: "remoteSettingsOnly impression_url",
+          click_url: "remoteSettingsOnly click_url",
+          block_id: 111,
+          advertiser: "remoteSettingsOnly advertiser",
+          is_sponsored: true,
+        },
+      ],
+    },
+  });
+
+  let context = createContext("frab", {
+    providers: [UrlbarProviderQuickSuggest.name],
+    isPrivate: false,
+  });
+  await check_results({
+    context,
+    matches: [
+      {
+        type: UrlbarUtils.RESULT_TYPE.URL,
+        source: UrlbarUtils.RESULT_SOURCE.SEARCH,
+        heuristic: false,
+        payload: {
+          qsSuggestion: "frab",
+          title: "frabbits",
+          url: "http://test.com/q=frabbits",
+          icon: null,
+          sponsoredImpressionUrl: "http://impression.reporting.test.com/",
+          sponsoredClickUrl: "http://click.reporting.test.com/",
+          sponsoredBlockId: 1,
+          sponsoredAdvertiser: "testadvertiser",
+          isSponsored: true,
+          helpUrl: UrlbarProviderQuickSuggest.helpUrl,
+          helpL10nId: "firefox-suggest-urlbar-learn-more",
+          displayUrl: "http://test.com/q=frabbits",
+        },
+      },
+    ],
+  });
+});
+
+// Tests with both Merino and remote settings enabled. The Merino suggestion
+// should be preferred over the remote settings one.
+add_task(async function bothEnabled() {
+  UrlbarPrefs.set(PREF_MERINO_ENABLED, true);
+  UrlbarPrefs.set(PREF_REMOTE_SETTINGS_ENABLED, true);
+
+  setMerinoResponse({
+    body: {
+      suggestions: [
+        {
+          full_keyword: "bothEnabled full_keyword",
+          title: "bothEnabled title",
+          url: "bothEnabled url",
+          icon: "bothEnabled icon",
+          impression_url: "bothEnabled impression_url",
+          click_url: "bothEnabled click_url",
+          block_id: 111,
+          advertiser: "bothEnabled advertiser",
+          is_sponsored: true,
+        },
+      ],
+    },
+  });
+
+  let context = createContext("frab", {
+    providers: [UrlbarProviderQuickSuggest.name],
+    isPrivate: false,
+  });
+  await check_results({
+    context,
+    matches: [
+      {
+        type: UrlbarUtils.RESULT_TYPE.URL,
+        source: UrlbarUtils.RESULT_SOURCE.SEARCH,
+        heuristic: false,
+        payload: {
+          qsSuggestion: "bothEnabled full_keyword",
+          title: "bothEnabled title",
+          url: "bothEnabled url",
+          icon: "bothEnabled icon",
+          sponsoredImpressionUrl: "bothEnabled impression_url",
+          sponsoredClickUrl: "bothEnabled click_url",
+          sponsoredBlockId: 111,
+          sponsoredAdvertiser: "bothEnabled advertiser",
+          isSponsored: true,
+          helpUrl: UrlbarProviderQuickSuggest.helpUrl,
+          helpL10nId: "firefox-suggest-urlbar-learn-more",
+          displayUrl: "bothEnabled url",
+        },
+      },
+    ],
+  });
+});
+
+// Tests with both Merino and remote settings disabled.
+add_task(async function bothDisabled() {
+  UrlbarPrefs.set(PREF_MERINO_ENABLED, false);
+  UrlbarPrefs.set(PREF_REMOTE_SETTINGS_ENABLED, false);
+
+  // Make sure the server is prepared to return a response so we can make sure
+  // we don't fetch it.
+  setMerinoResponse({
+    body: {
+      suggestions: [
+        {
+          full_keyword: "bothDisabled full_keyword",
+          title: "bothDisabled title",
+          url: "bothDisabled url",
+          icon: "bothDisabled icon",
+          impression_url: "bothDisabled impression_url",
+          click_url: "bothDisabled click_url",
+          block_id: 111,
+          advertiser: "bothDisabled advertiser",
+          is_sponsored: true,
+        },
+      ],
+    },
+  });
+
+  let context = createContext("frab", {
+    providers: [UrlbarProviderQuickSuggest.name],
+    isPrivate: false,
+  });
+  await check_results({ context, matches: [] });
+});
+
+// Checks that we use the first Merino suggestion if the response includes
+// multiple suggestions.
+add_task(async function multipleMerinoSuggestions() {
+  UrlbarPrefs.set(PREF_MERINO_ENABLED, true);
+  UrlbarPrefs.set(PREF_REMOTE_SETTINGS_ENABLED, false);
+
+  setMerinoResponse({
+    body: {
+      suggestions: [
+        {
+          full_keyword: "multipleMerinoSuggestions 0 full_keyword",
+          title: "multipleMerinoSuggestions 0 title",
+          url: "multipleMerinoSuggestions 0 url",
+          icon: "multipleMerinoSuggestions 0 icon",
+          impression_url: "multipleMerinoSuggestions 0 impression_url",
+          click_url: "multipleMerinoSuggestions 0 click_url",
+          block_id: 0,
+          advertiser: "multipleMerinoSuggestions 0 advertiser",
+          is_sponsored: true,
+        },
+        {
+          full_keyword: "multipleMerinoSuggestions 1 full_keyword",
+          title: "multipleMerinoSuggestions 1 title",
+          url: "multipleMerinoSuggestions 1 url",
+          icon: "multipleMerinoSuggestions 1 icon",
+          impression_url: "multipleMerinoSuggestions 1 impression_url",
+          click_url: "multipleMerinoSuggestions 1 click_url",
+          block_id: 1,
+          advertiser: "multipleMerinoSuggestions 1 advertiser",
+          is_sponsored: false,
+        },
+      ],
+    },
+  });
+
+  let context = createContext("test", {
+    providers: [UrlbarProviderQuickSuggest.name],
+    isPrivate: false,
+  });
+  await check_results({
+    context,
+    matches: [
+      {
+        type: UrlbarUtils.RESULT_TYPE.URL,
+        source: UrlbarUtils.RESULT_SOURCE.SEARCH,
+        heuristic: false,
+        payload: {
+          qsSuggestion: "multipleMerinoSuggestions 0 full_keyword",
+          title: "multipleMerinoSuggestions 0 title",
+          url: "multipleMerinoSuggestions 0 url",
+          icon: "multipleMerinoSuggestions 0 icon",
+          sponsoredImpressionUrl: "multipleMerinoSuggestions 0 impression_url",
+          sponsoredClickUrl: "multipleMerinoSuggestions 0 click_url",
+          sponsoredBlockId: 0,
+          sponsoredAdvertiser: "multipleMerinoSuggestions 0 advertiser",
+          isSponsored: true,
+          helpUrl: UrlbarProviderQuickSuggest.helpUrl,
+          helpL10nId: "firefox-suggest-urlbar-learn-more",
+          displayUrl: "multipleMerinoSuggestions 0 url",
+        },
+      },
+    ],
+  });
+});
+
+// Checks a response that's valid but also has some unexpected properties.
+add_task(async function unexpectedResponseProperties() {
+  UrlbarPrefs.set(PREF_MERINO_ENABLED, true);
+  UrlbarPrefs.set(PREF_REMOTE_SETTINGS_ENABLED, false);
+
+  setMerinoResponse({
+    body: {
+      unexpectedString: "some value",
+      unexpectedArray: ["a", "b", "c"],
+      unexpectedObject: { foo: "bar" },
+      suggestions: [
+        {
+          full_keyword: "unexpected full_keyword",
+          title: "unexpected title",
+          url: "unexpected url",
+          icon: "unexpected icon",
+          impression_url: "unexpected impression_url",
+          click_url: "unexpected click_url",
+          block_id: 1234,
+          advertiser: "unexpected advertiser",
+          is_sponsored: true,
+        },
+      ],
+    },
+  });
+
+  let context = createContext("test", {
+    providers: [UrlbarProviderQuickSuggest.name],
+    isPrivate: false,
+  });
+  await check_results({
+    context,
+    matches: [
+      {
+        type: UrlbarUtils.RESULT_TYPE.URL,
+        source: UrlbarUtils.RESULT_SOURCE.SEARCH,
+        heuristic: false,
+        payload: {
+          qsSuggestion: "unexpected full_keyword",
+          title: "unexpected title",
+          url: "unexpected url",
+          icon: "unexpected icon",
+          sponsoredImpressionUrl: "unexpected impression_url",
+          sponsoredClickUrl: "unexpected click_url",
+          sponsoredBlockId: 1234,
+          sponsoredAdvertiser: "unexpected advertiser",
+          isSponsored: true,
+          helpUrl: UrlbarProviderQuickSuggest.helpUrl,
+          helpL10nId: "firefox-suggest-urlbar-learn-more",
+          displayUrl: "unexpected url",
+        },
+      },
+    ],
+  });
+});
+
+// Checks some bad/unexpected responses.
+add_task(async function badResponses() {
+  UrlbarPrefs.set(PREF_MERINO_ENABLED, true);
+  UrlbarPrefs.set(PREF_REMOTE_SETTINGS_ENABLED, false);
+
+  let context;
+  let contextArgs = [
+    "test",
+    { providers: [UrlbarProviderQuickSuggest.name], isPrivate: false },
+  ];
+
+  setMerinoResponse({
+    body: {},
+  });
+  context = createContext(...contextArgs);
+  await check_results({ context, matches: [] });
+
+  setMerinoResponse({
+    body: { bogus: [] },
+  });
+  context = createContext(...contextArgs);
+  await check_results({ context, matches: [] });
+
+  setMerinoResponse({
+    body: { suggestions: {} },
+  });
+  context = createContext(...contextArgs);
+  await check_results({ context, matches: [] });
+
+  setMerinoResponse({
+    body: { suggestions: [] },
+  });
+  context = createContext(...contextArgs);
+  await check_results({ context, matches: [] });
+
+  setMerinoResponse({
+    body: "",
+  });
+  context = createContext(...contextArgs);
+  await check_results({ context, matches: [] });
+
+  setMerinoResponse({
+    contentType: "text/html",
+    body: "bogus",
+  });
+  context = createContext(...contextArgs);
+  await check_results({ context, matches: [] });
+});
+
+function makeMerinoServer(endpointPath) {
+  let server = makeTestServer();
+  server.registerPathHandler(endpointPath, (req, resp) => {
+    resp.setHeader("Content-Type", gMerinoResponse.contentType, false);
+    if (typeof gMerinoResponse.body == "string") {
+      resp.write(gMerinoResponse.body);
+    } else if (gMerinoResponse.body) {
+      resp.write(JSON.stringify(gMerinoResponse.body));
+    }
+  });
+  return server;
+}
+
+function setMerinoResponse({ body, contentType = "application/json" }) {
+  gMerinoResponse = { body, contentType };
+}

browser/components/urlbar/tests/unit/xpcshell.ini

@@ -70,6 +70,7 @@ skip-if = !sync
 [test_tags_matchBookmarkTitles.js]
 [test_tags_returnedInSearches.js]
 [test_quicksuggest_keywordtree.js]
+[test_quicksuggest_merino.js]
 [test_tokenizer.js]
 [test_trimming.js]
 [test_unitConversion.js]

toolkit/components/nimbus/FeatureManifest.js

@@ -14,6 +14,11 @@ const FeatureManifest = {
   urlbar: {
     description: "The Address Bar",
     variables: {
+      merinoEnabled: {
+        type: "boolean",
+        fallbackPref: "browser.urlbar.merino.enabled",
+        description: "Whether Merino is enabled as a quick suggest source",
+      },
       quickSuggestEnabled: {
         type: "boolean",
         fallbackPref: "browser.urlbar.quicksuggest.enabled",
@@ -25,6 +30,12 @@ const FeatureManifest = {
         description:
           "The index of non-sponsored QuickSuggest results within the general group. A negative index is relative to the end of the group",
       },
+      quickSuggestRemoteSettingsEnabled: {
+        type: "boolean",
+        fallbackPref: "browser.urlbar.quicksuggest.remoteSettings.enabled",
+        description:
+          "Whether Remote Settings is enabled as a quick suggest source",
+      },
       quickSuggestShouldShowOnboardingDialog: {
         type: "boolean",
         fallbackPref: "browser.urlbar.quicksuggest.shouldShowOnboardingDialog",