Bug 1645521 - Part 1 - Add ProviderHeuristicFallback. r=adw

Differential Revision: https://phabricator.services.mozilla.com/D80291

browser/components/urlbar/UrlbarProviderHeuristicFallback.jsm

@@ -0,0 +1,290 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * This module exports a provider that provides a heuristic result. The result
+ * either vists a URL or does a search with the current engine. This result is
+ * always the ultimate fallback for any query, so this provider is always active.
+ */
+
+var EXPORTED_SYMBOLS = ["UrlbarProviderHeuristicFallback"];
+
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
+);
+XPCOMUtils.defineLazyModuleGetters(this, {
+  Log: "resource://gre/modules/Log.jsm",
+  Services: "resource://gre/modules/Services.jsm",
+  UrlbarPrefs: "resource:///modules/UrlbarPrefs.jsm",
+  UrlbarProvider: "resource:///modules/UrlbarUtils.jsm",
+  UrlbarResult: "resource:///modules/UrlbarResult.jsm",
+  UrlbarTokenizer: "resource:///modules/UrlbarTokenizer.jsm",
+  UrlbarUtils: "resource:///modules/UrlbarUtils.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(this, "logger", () =>
+  Log.repository.getLogger("Urlbar.Provider.HeuristicFallback")
+);
+
+/**
+ * Class used to create the provider.
+ */
+class ProviderHeuristicFallback extends UrlbarProvider {
+  constructor() {
+    super();
+    // Maps the running queries by queryContext.
+    this.queries = new Map();
+  }
+
+  /**
+   * Returns the name of this provider.
+   * @returns {string} the name of this provider.
+   */
+  get name() {
+    return "HeuristicFallback";
+  }
+
+  /**
+   * Returns the type of this provider.
+   * @returns {integer} one of the types from UrlbarUtils.PROVIDER_TYPE.*
+   */
+  get type() {
+    return UrlbarUtils.PROVIDER_TYPE.HEURISTIC;
+  }
+
+  /**
+   * Whether this provider should be invoked for the given context.
+   * If this method returns false, the providers manager won't start a query
+   * with this provider, to save on resources.
+   * @param {UrlbarQueryContext} queryContext The query context object
+   * @returns {boolean} Whether this provider should be invoked for the search.
+   */
+  isActive(queryContext) {
+    return false; // TODO: Set to true in future part of this patch.
+  }
+
+  /**
+   * Gets the provider's priority.
+   * @param {UrlbarQueryContext} queryContext The query context object
+   * @returns {number} The provider's priority for the given query.
+   */
+  getPriority(queryContext) {
+    return 0;
+  }
+
+  /**
+   * Starts querying.
+   * @param {object} queryContext The query context object
+   * @param {function} addCallback Callback invoked by the provider to add a new
+   *        result.
+   * @returns {Promise} resolved when the query stops.
+   */
+  async startQuery(queryContext, addCallback) {
+    logger.info(`Starting query for ${queryContext.searchString}`);
+    let instance = {};
+    this.queries.set(queryContext, instance);
+
+    let result = this._matchUnknownUrl(queryContext);
+    if (result) {
+      addCallback(this, result);
+      // Since we can't tell if this is a real URL and whether the user wants
+      // to visit or search for it, we provide an alternative searchengine
+      // match if the string looks like an alphanumeric origin or an e-mail.
+      let str = queryContext.searchString;
+      try {
+        new URL(str);
+      } catch (ex) {
+        if (
+          UrlbarPrefs.get("keyword.enabled") &&
+          (UrlbarTokenizer.looksLikeOrigin(str, {
+            noIp: true,
+            noPort: true,
+          }) ||
+            UrlbarTokenizer.REGEXP_COMMON_EMAIL.test(str))
+        ) {
+          let searchResult = await this._defaultEngineSearchResult(
+            queryContext
+          );
+          if (!this.queries.has(queryContext)) {
+            return;
+          }
+          addCallback(this, searchResult);
+        }
+      }
+    } else {
+      result = await this._defaultEngineSearchResult(queryContext);
+      if (!result || !this.queries.has(queryContext)) {
+        return;
+      }
+      result.heuristic = true;
+      addCallback(this, result);
+    }
+
+    this.queries.delete(queryContext);
+  }
+
+  /**
+   * Cancels a running query.
+   * @param {object} queryContext The query context object
+   */
+  cancelQuery(queryContext) {
+    logger.info(`Canceling query for ${queryContext.searchString}`);
+    this.queries.delete(queryContext);
+  }
+
+  // TODO (bug 1054814): Use visited URLs to inform which scheme to use, if the
+  // scheme isn't specificed.
+  _matchUnknownUrl(queryContext) {
+    let unescapedSearchString = Services.textToSubURI.unEscapeURIForUI(
+      queryContext.searchString
+    );
+    let [prefix, suffix] = UrlbarUtils.stripURLPrefix(unescapedSearchString);
+    if (!suffix && prefix) {
+      // The user just typed a stripped protocol, don't build a non-sense url
+      // like http://http/ for it.
+      return null;
+    }
+    // The user may have typed something like "word?" to run a search, we should
+    // not convert that to a url.
+    if (
+      queryContext.restrictSource &&
+      queryContext.restrictSource == UrlbarUtils.RESULT_SOURCE.SEARCH
+    ) {
+      return null;
+    }
+
+    let searchUrl = queryContext.searchString.trim();
+
+    if (queryContext.fixupError) {
+      if (
+        queryContext.fixupError == Cr.NS_ERROR_MALFORMED_URI &&
+        !UrlbarPrefs.get("keyword.enabled")
+      ) {
+        let result = new UrlbarResult(
+          UrlbarUtils.RESULT_TYPE.URL,
+          UrlbarUtils.RESULT_SOURCE.OTHER_LOCAL,
+          ...UrlbarResult.payloadAndSimpleHighlights(queryContext.tokens, {
+            title: [searchUrl, UrlbarUtils.HIGHLIGHT.TYPED],
+            url: [searchUrl, UrlbarUtils.HIGHLIGHT.TYPED],
+            icon: "",
+          })
+        );
+        result.heuristic = true;
+        return result;
+      }
+
+      return null;
+    }
+
+    // If the URI cannot be fixed or the preferred URI would do a keyword search,
+    // that basically means this isn't useful to us. Note that
+    // fixupInfo.keywordAsSent will never be true if the keyword.enabled pref
+    // is false or there are no engines, so in that case we will always return
+    // a "visit".
+    if (!queryContext.fixupInfo?.href || queryContext.fixupInfo?.isSearch) {
+      return null;
+    }
+
+    let uri = new URL(queryContext.fixupInfo.href);
+    // Check the host, as "http:///" is a valid nsIURI, but not useful to us.
+    // But, some schemes are expected to have no host. So we check just against
+    // schemes we know should have a host. This allows new schemes to be
+    // implemented without us accidentally blocking access to them.
+    let hostExpected = ["http:", "https:", "ftp:", "chrome:"].includes(
+      uri.protocol
+    );
+    if (hostExpected && !uri.host) {
+      return null;
+    }
+
+    // getFixupURIInfo() escaped the URI, so it may not be pretty.  Embed the
+    // escaped URL in the result since that URL should be "canonical".  But
+    // pass the pretty, unescaped URL as the result's title, since it is
+    // displayed to the user.
+    let escapedURL = uri.toString();
+    let displayURL = decodeURI(uri);
+
+    // We don't know if this url is in Places or not, and checking that would
+    // be expensive. Thus we also don't know if we may have an icon.
+    // If we'd just try to fetch the icon for the typed string, we'd cause icon
+    // flicker, since the url keeps changing while the user types.
+    // By default we won't provide an icon, but for the subset of urls with a
+    // host we'll check for a typed slash and set favicon for the host part.
+    let iconUri = "";
+    if (hostExpected && (searchUrl.endsWith("/") || uri.pathname.length > 1)) {
+      // Look for an icon with the entire URL except for the pathname, including
+      // scheme, usernames, passwords, hostname, and port.
+      let pathIndex = uri.toString().lastIndexOf(uri.pathname);
+      let prePath = uri.toString().slice(0, pathIndex);
+      iconUri = `page-icon:${prePath}/`;
+    }
+
+    let result = new UrlbarResult(
+      UrlbarUtils.RESULT_TYPE.URL,
+      UrlbarUtils.RESULT_SOURCE.OTHER_LOCAL,
+      ...UrlbarResult.payloadAndSimpleHighlights(queryContext.tokens, {
+        title: [displayURL, UrlbarUtils.HIGHLIGHT.TYPED],
+        url: [escapedURL, UrlbarUtils.HIGHLIGHT.TYPED],
+        icon: iconUri,
+      })
+    );
+    result.heuristic = true;
+    return result;
+  }
+
+  async _defaultEngineSearchResult(queryContext) {
+    let engine;
+    if (queryContext.engineName) {
+      engine = Services.search.getEngineByName(queryContext.engineName);
+    } else if (queryContext.isPrivate) {
+      engine = Services.search.defaultPrivateEngine;
+    } else {
+      engine = Services.search.defaultEngine;
+    }
+
+    if (!engine) {
+      return null;
+    }
+
+    // Strip a leading search restriction char, because we prepend it to text
+    // when the search shortcut is used and it's not user typed. Don't strip
+    // other restriction chars, so that it's possible to search for things
+    // including one of those (e.g. "c#").
+    let query = queryContext.searchString;
+    if (
+      queryContext.tokens[0] &&
+      queryContext.tokens[0].value === UrlbarTokenizer.RESTRICT.SEARCH
+    ) {
+      query = UrlbarUtils.substringAfter(
+        query,
+        queryContext.tokens[0].value
+      ).trim();
+    }
+
+    let result = new UrlbarResult(
+      UrlbarUtils.RESULT_TYPE.SEARCH,
+      UrlbarUtils.RESULT_SOURCE.SEARCH,
+      ...UrlbarResult.payloadAndSimpleHighlights(queryContext.tokens, {
+        engine: [engine.name, UrlbarUtils.HIGHLIGHT.TYPED],
+        icon: [engine.iconURI?.spec || ""],
+        query: [query, UrlbarUtils.HIGHLIGHT.NONE],
+        // We're confident that there is no alias, since UnifiedComplete
+        // handles heuristic searches with aliases.
+        keyword: undefined,
+        keywordOffer: UrlbarUtils.KEYWORD_OFFER.NONE,
+        // For test interoperabilty with UrlbarProviderSearchSuggestions.
+        suggestion: undefined,
+        tailPrefix: undefined,
+        tail: undefined,
+        tailOffsetIndex: -1,
+        isSearchHistory: false,
+      })
+    );
+    return result;
+  }
+}
+
+var UrlbarProviderHeuristicFallback = new ProviderHeuristicFallback();

browser/components/urlbar/UrlbarProvidersManager.jsm

@@ -35,6 +35,8 @@ XPCOMUtils.defineLazyGetter(this, "logger", () =>
 var localProviderModules = {
   UrlbarProviderUnifiedComplete:
     "resource:///modules/UrlbarProviderUnifiedComplete.jsm",
+  UrlbarProviderHeuristicFallback:
+    "resource:///modules/UrlbarProviderHeuristicFallback.jsm",
   UrlbarProviderInterventions:
     "resource:///modules/UrlbarProviderInterventions.jsm",
   UrlbarProviderOmnibox: "resource:///modules/UrlbarProviderOmnibox.jsm",

browser/components/urlbar/UrlbarUtils.jsm

@@ -607,6 +607,28 @@ var UrlbarUtils = {
     return index < 0 ? "" : sourceStr.substr(index + targetStr.length);
   },
 
+  /**
+   * Strips the prefix from a URL and returns the prefix and the remainder of the
+   * URL.  "Prefix" is defined to be the scheme and colon, plus, if present, two
+   * slashes.  If the given string is not actually a URL, then an empty prefix and
+   * the string itself is returned.
+   *
+   * @param {string} str The possible URL to strip.
+   * @returns {array} If `str` is a URL, then [prefix, remainder].  Otherwise, ["", str].
+   */
+  stripURLPrefix(str) {
+    const REGEXP_STRIP_PREFIX = /^[a-z]+:(?:\/){0,2}/i;
+    let match = REGEXP_STRIP_PREFIX.exec(str);
+    if (!match) {
+      return ["", str];
+    }
+    let prefix = match[0];
+    if (prefix.length < str.length && str[prefix.length] == " ") {
+      return ["", str];
+    }
+    return [prefix, str.substr(prefix.length)];
+  },
+
   /**
    * Runs a search for the given string, and returns the heuristic result.
    * @param {string} searchString The string to search for.
@@ -1008,7 +1030,7 @@ class UrlbarQueryContext {
    * serializable so they can be sent to extensions.
    */
   get fixupInfo() {
-    if (this.searchString && !this._fixupInfo) {
+    if (this.searchString.trim() && !this._fixupInfo) {
       let flags =
         Ci.nsIURIFixup.FIXUP_FLAG_FIX_SCHEME_TYPOS |
         Ci.nsIURIFixup.FIXUP_FLAG_ALLOW_KEYWORD_LOOKUP;
@@ -1016,18 +1038,35 @@ class UrlbarQueryContext {
         flags |= Ci.nsIURIFixup.FIXUP_FLAG_PRIVATE_CONTEXT;
       }
 
-      let info = Services.uriFixup.getFixupURIInfo(
-        this.searchString.trim(),
-        flags
-      );
-      this._fixupInfo = {
-        href: info.fixedURI.spec,
-        isSearch: !!info.keywordAsSent,
-      };
+      try {
+        let info = Services.uriFixup.getFixupURIInfo(
+          this.searchString.trim(),
+          flags
+        );
+        this._fixupInfo = {
+          href: info.fixedURI.spec,
+          isSearch: !!info.keywordAsSent,
+        };
+      } catch (ex) {
+        this._fixupError = ex.result;
+      }
     }
 
     return this._fixupInfo || null;
   }
+
+  /**
+   * Returns the error that was thrown when fixupInfo was fetched, if any. If
+   * fixupInfo has not yet been fetched for this queryContext, it is fetched
+   * here.
+   */
+  get fixupError() {
+    if (!this.fixupInfo) {
+      return this._fixupError;
+    }
+
+    return null;
+  }
 }
 
 /**

browser/components/urlbar/moz.build

@@ -12,6 +12,7 @@ EXTRA_JS_MODULES += [
     'UrlbarMuxerUnifiedComplete.jsm',
     'UrlbarPrefs.jsm',
     'UrlbarProviderExtension.jsm',
+    'UrlbarProviderHeuristicFallback.jsm',
     'UrlbarProviderInterventions.jsm',
     'UrlbarProviderOmnibox.jsm',
     'UrlbarProviderOpenTabs.jsm',

toolkit/components/places/UnifiedComplete.jsm

@@ -1338,50 +1338,7 @@ Search.prototype = {
       }
     }
 
-    if (this.pending && this._searchTokens.length && this._enableActions) {
-      // If we don't have a result that matches what we know about, then
-      // we use a fallback for things we don't know about.
-
-      // We may not have auto-filled, but this may still look like a URL.
-      // However, even if the input is a valid URL, we may not want to use
-      // it as such. This can happen if the host isn't using a known TLD
-      // and hasn't been added to a user-controlled list of known domains,
-      // and/or if it looks like an email address.
-      let matched = await this._matchUnknownUrl();
-      if (matched) {
-        // Since we can't tell if this is a real URL and whether the user wants
-        // to visit or search for it, we provide an alternative searchengine
-        // match if the string looks like an alphanumeric origin or an e-mail.
-        let str = this._originalSearchString;
-        try {
-          new URL(str);
-        } catch (ex) {
-          if (
-            UrlbarPrefs.get("keyword.enabled") &&
-            (UrlbarTokenizer.looksLikeOrigin(str, {
-              noIp: true,
-              noPort: true,
-            }) ||
-              UrlbarTokenizer.REGEXP_COMMON_EMAIL.test(str))
-          ) {
-            this._addingHeuristicResult = false;
-            await this._matchCurrentSearchEngine();
-            this._addingHeuristicResult = true;
-          }
-        }
-        return true;
-      }
-    }
-
-    if (this.pending && this._enableActions && this._originalSearchString) {
-      // When all else fails, and the search string is non-empty, we search
-      // using the current search engine.
-      let matched = await this._matchCurrentSearchEngine();
-      if (matched) {
-        return true;
-      }
-    }
-
+    // Fall back to UrlbarProviderHeuristicFallback.
     return false;
   },
 
@@ -1564,32 +1521,6 @@ Search.prototype = {
     return true;
   },
 
-  async _matchCurrentSearchEngine() {
-    let engine;
-    if (this._engineName) {
-      engine = Services.search.getEngineByName(this._engineName);
-    } else if (this._inPrivateWindow) {
-      engine = Services.search.defaultPrivateEngine;
-    } else {
-      engine = Services.search.defaultEngine;
-    }
-
-    if (!engine || !this.pending) {
-      return false;
-    }
-
-    // Strip a leading search restriction char, because we prepend it to text
-    // when the search shortcut is used and it's not user typed. Don't strip
-    // other restriction chars, so that it's possible to search for things
-    // including one of those (e.g. "c#").
-    let query = this._trimmedOriginalSearchString;
-    if (this._leadingRestrictionToken === UrlbarTokenizer.RESTRICT.SEARCH) {
-      query = substringAfter(query, this._leadingRestrictionToken).trim();
-    }
-    this._addSearchEngineMatch({ engine, query });
-    return true;
-  },
-
   _addExtensionMatch(content, comment) {
     this._addMatch({
       value: makeActionUrl("extension", {
@@ -1703,106 +1634,6 @@ Search.prototype = {
     }
   },
 
-  // TODO (bug 1054814): Use visited URLs to inform which scheme to use, if the
-  // scheme isn't specificed.
-  _matchUnknownUrl() {
-    if (!this._searchString && this._strippedPrefix) {
-      // The user just typed a stripped protocol, don't build a non-sense url
-      // like http://http/ for it.
-      return false;
-    }
-    // The user may have typed something like "word?" to run a search, we should
-    // not convert that to a url.
-    if (this.hasBehavior("search") && this.hasBehavior("restrict")) {
-      return false;
-    }
-    let flags =
-      Ci.nsIURIFixup.FIXUP_FLAG_FIX_SCHEME_TYPOS |
-      Ci.nsIURIFixup.FIXUP_FLAG_ALLOW_KEYWORD_LOOKUP;
-    if (this._inPrivateWindow) {
-      flags |= Ci.nsIURIFixup.FIXUP_FLAG_PRIVATE_CONTEXT;
-    }
-    let fixupInfo = null;
-    let searchUrl = this._trimmedOriginalSearchString;
-    try {
-      fixupInfo = Services.uriFixup.getFixupURIInfo(searchUrl, flags);
-    } catch (e) {
-      if (
-        e.result == Cr.NS_ERROR_MALFORMED_URI &&
-        !UrlbarPrefs.get("keyword.enabled")
-      ) {
-        let value = makeActionUrl("visiturl", {
-          url: searchUrl,
-          input: searchUrl,
-        });
-        this._addMatch({
-          value,
-          comment: searchUrl,
-          style: "action visiturl",
-          frecency: Infinity,
-        });
-
-        return true;
-      }
-      return false;
-    }
-
-    // If the URI cannot be fixed or the preferred URI would do a keyword search,
-    // that basically means this isn't useful to us. Note that
-    // fixupInfo.keywordAsSent will never be true if the keyword.enabled pref
-    // is false or there are no engines, so in that case we will always return
-    // a "visit".
-    if (!fixupInfo.fixedURI || fixupInfo.keywordAsSent) {
-      return false;
-    }
-
-    let uri = fixupInfo.fixedURI;
-    // Check the host, as "http:///" is a valid nsIURI, but not useful to us.
-    // But, some schemes are expected to have no host. So we check just against
-    // schemes we know should have a host. This allows new schemes to be
-    // implemented without us accidentally blocking access to them.
-    let hostExpected = ["http", "https", "ftp", "chrome"].includes(uri.scheme);
-    if (hostExpected && !uri.host) {
-      return false;
-    }
-
-    // getFixupURIInfo() escaped the URI, so it may not be pretty.  Embed the
-    // escaped URL in the action URI since that URL should be "canonical".  But
-    // pass the pretty, unescaped URL as the match comment, since it's likely
-    // to be displayed to the user, and in any case the front-end should not
-    // rely on it being canonical.
-    let escapedURL = uri.displaySpec;
-    let displayURL = Services.textToSubURI.unEscapeURIForUI(escapedURL);
-
-    let value = makeActionUrl("visiturl", {
-      url: escapedURL,
-      input: searchUrl,
-    });
-
-    let match = {
-      value,
-      comment: displayURL,
-      style: "action visiturl",
-      frecency: Infinity,
-    };
-
-    // We don't know if this url is in Places or not, and checking that would
-    // be expensive. Thus we also don't know if we may have an icon.
-    // If we'd just try to fetch the icon for the typed string, we'd cause icon
-    // flicker, since the url keeps changing while the user types.
-    // By default we won't provide an icon, but for the subset of urls with a
-    // host we'll check for a typed slash and set favicon for the host part.
-    if (
-      hostExpected &&
-      (searchUrl.endsWith("/") || uri.pathQueryRef.length > 1)
-    ) {
-      match.icon = `page-icon:${uri.prePath}/`;
-    }
-
-    this._addMatch(match);
-    return true;
-  },
-
   _onResultRow(row, cancel) {
     let queryType = row.getResultByIndex(QUERYINDEX_QUERYTYPE);
     switch (queryType) {