Bug 612453 - Provide search suggestions on Firefox Start Page (about:home) (part 2, ContentSearch). r=MattN,felipe

browser/base/content/content.js

@@ -246,6 +246,7 @@ addMessageListener("WebChannelMessageToContent", function (e) {
 let ContentSearchMediator = {
 
   whitelist: new Set([
+    "about:home",
     "about:newtab",
   ]),
 
@@ -274,7 +275,7 @@ let ContentSearchMediator = {
   },
 
   get _contentWhitelisted() {
-    return this.whitelist.has(content.document.documentURI.toLowerCase());
+    return this.whitelist.has(content.document.documentURI);
   },
 
   _sendMsg: function (type, data=null) {
@@ -285,12 +286,14 @@ let ContentSearchMediator = {
   },
 
   _fireEvent: function (type, data=null) {
-    content.dispatchEvent(new content.CustomEvent("ContentSearchService", {
+    let event = Cu.cloneInto({
       detail: {
         type: type,
         data: data,
       },
-    }));
+    }, content);
+    content.dispatchEvent(new content.CustomEvent("ContentSearchService",
+                                                  event));
   },
 };
 ContentSearchMediator.init(this);

browser/modules/ContentSearch.jsm

@@ -13,6 +13,14 @@ const { classes: Cc, interfaces: Ci, utils: Cu } = Components;
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/Promise.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "FormHistory",
+  "resource://gre/modules/FormHistory.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "PrivateBrowsingUtils",
+  "resource://gre/modules/PrivateBrowsingUtils.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "SearchSuggestionController",
+  "resource://gre/modules/SearchSuggestionController.jsm");
 
 const INBOUND_MESSAGE = "ContentSearch";
 const OUTBOUND_MESSAGE = INBOUND_MESSAGE;
@@ -26,30 +34,45 @@ const OUTBOUND_MESSAGE = INBOUND_MESSAGE;
  *
  * Inbound messages have the following types:
  *
+ *   AddFormHistoryEntry
+ *     Adds an entry to the search form history.
+ *     data: the entry, a string
+ *   GetSuggestions
+ *     Retrieves an array of search suggestions given a search string.
+ *     data: { engineName, searchString, [remoteTimeout] }
  *   GetState
- *      Retrieves the current search engine state.
- *      data: null
+ *     Retrieves the current search engine state.
+ *     data: null
  *   ManageEngines
- *      Opens the search engine management window.
- *      data: null
+ *     Opens the search engine management window.
+ *     data: null
+ *   RemoveFormHistoryEntry
+ *     Removes an entry from the search form history.
+ *     data: the entry, a string
  *   Search
- *      Performs a search.
- *      data: an object { engineName, searchString, whence }
+ *     Performs a search.
+ *     data: { engineName, searchString, whence }
  *   SetCurrentEngine
- *      Sets the current engine.
- *      data: the name of the engine
+ *     Sets the current engine.
+ *     data: the name of the engine
+ *   SpeculativeConnect
+ *     Speculatively connects to an engine.
+ *     data: the name of the engine
  *
  * Outbound messages have the following types:
  *
  *   CurrentEngine
- *     Sent when the current engine changes.
+ *     Broadcast when the current engine changes.
  *     data: see _currentEngineObj
  *   CurrentState
- *     Sent when the current search state changes.
+ *     Broadcast when the current search state changes.
  *     data: see _currentStateObj
  *   State
  *     Sent in reply to GetState.
  *     data: see _currentStateObj
+ *   Suggestions
+ *     Sent in reply to GetSuggestions.
+ *     data: see _onMessageGetSuggestions
  */
 
 this.ContentSearch = {
@@ -60,6 +83,10 @@ this.ContentSearch = {
   _eventQueue: [],
   _currentEvent: null,
 
+  // This is used to handle search suggestions.  It maps xul:browsers to objects
+  // { controller, previousFormHistoryResult }.  See _onMessageGetSuggestions.
+  _suggestionMap: new WeakMap(),
+
   init: function () {
     Cc["@mozilla.org/globalmessagemanager;1"].
       getService(Ci.nsIMessageListenerManager).
@@ -72,10 +99,15 @@ this.ContentSearch = {
     // the event queue.  If the message's source docshell changes browsers in
     // the meantime, then we need to update msg.target.  event.detail will be
     // the docshell's new parent <xul:browser> element.
-    msg.handleEvent = function (event) {
-      this.target.removeEventListener("SwapDocShells", this, true);
-      this.target = event.detail;
-      this.target.addEventListener("SwapDocShells", this, true);
+    msg.handleEvent = event => {
+      let browserData = this._suggestionMap.get(msg.target);
+      if (browserData) {
+        this._suggestionMap.delete(msg.target);
+        this._suggestionMap.set(event.detail, browserData);
+      }
+      msg.target.removeEventListener("SwapDocShells", msg, true);
+      msg.target = event.detail;
+      msg.target.addEventListener("SwapDocShells", msg, true);
     };
     msg.target.addEventListener("SwapDocShells", msg, true);
 
@@ -106,6 +138,9 @@ this.ContentSearch = {
     try {
       yield this["_on" + this._currentEvent.type](this._currentEvent.data);
     }
+    catch (err) {
+      Cu.reportError(err);
+    }
     finally {
       this._currentEvent = null;
       this._processEventQueue();
@@ -128,17 +163,11 @@ this.ContentSearch = {
   },
 
   _onMessageSearch: function (msg, data) {
-    let expectedDataProps = [
+    this._ensureDataHasProperties(data, [
       "engineName",
       "searchString",
       "whence",
-    ];
-    for (let prop of expectedDataProps) {
-      if (!(prop in data)) {
-        Cu.reportError("Message data missing required property: " + prop);
-        return Promise.resolve();
-      }
-    }
+    ]);
     let browserWin = msg.target.ownerDocument.defaultView;
     let engine = Services.search.getEngineByName(data.engineName);
     browserWin.BrowserSearch.recordSearchInHealthReport(engine, data.whence);
@@ -158,6 +187,92 @@ this.ContentSearch = {
     return Promise.resolve();
   },
 
+  _onMessageGetSuggestions: Task.async(function* (msg, data) {
+    this._ensureDataHasProperties(data, [
+      "engineName",
+      "searchString",
+    ]);
+
+    let engine = Services.search.getEngineByName(data.engineName);
+    if (!engine) {
+      throw new Error("Unknown engine name: " + data.engineName);
+    }
+
+    let browserData = this._suggestionDataForBrowser(msg.target, true);
+    let { controller } = browserData;
+    let ok = SearchSuggestionController.engineOffersSuggestions(engine);
+    controller.maxLocalResults = ok ? 2 : 6;
+    controller.maxRemoteResults = ok ? 6 : 0;
+    controller.remoteTimeout = data.remoteTimeout || undefined;
+    let priv = PrivateBrowsingUtils.isWindowPrivate(msg.target.contentWindow);
+    // fetch() rejects its promise if there's a pending request, but since we
+    // process our event queue serially, there's never a pending request.
+    let suggestions = yield controller.fetch(data.searchString, priv, engine);
+
+    // Keep the form history result so RemoveFormHistoryEntry can remove entries
+    // from it.  Keeping only one result isn't foolproof because the client may
+    // try to remove an entry from one set of suggestions after it has requested
+    // more but before it's received them.  In that case, the entry may not
+    // appear in the new suggestions.  But that should happen rarely.
+    browserData.previousFormHistoryResult = suggestions.formHistoryResult;
+
+    this._reply(msg, "Suggestions", {
+      engineName: data.engineName,
+      searchString: suggestions.term,
+      formHistory: suggestions.local,
+      remote: suggestions.remote,
+    });
+  }),
+
+  _onMessageAddFormHistoryEntry: function (msg, entry) {
+    // There are some tests that use about:home and newtab that trigger a search
+    // and then immediately close the tab.  In those cases, the browser may have
+    // been destroyed by the time we receive this message, and as a result
+    // contentWindow is undefined.
+    if (!msg.target.contentWindow ||
+        PrivateBrowsingUtils.isWindowPrivate(msg.target.contentWindow)) {
+      return Promise.resolve();
+    }
+    let browserData = this._suggestionDataForBrowser(msg.target, true);
+    FormHistory.update({
+      op: "bump",
+      fieldname: browserData.controller.formHistoryParam,
+      value: entry,
+    }, {
+      handleCompletion: () => {},
+      handleError: err => {
+        Cu.reportError("Error adding form history entry: " + err);
+      },
+    });
+    return Promise.resolve();
+  },
+
+  _onMessageRemoveFormHistoryEntry: function (msg, entry) {
+    let browserData = this._suggestionDataForBrowser(msg.target);
+    if (browserData && browserData.previousFormHistoryResult) {
+      let { previousFormHistoryResult } = browserData;
+      for (let i = 0; i < previousFormHistoryResult.matchCount; i++) {
+        if (previousFormHistoryResult.getValueAt(i) == entry) {
+          previousFormHistoryResult.removeValueAt(i, true);
+          break;
+        }
+      }
+    }
+    return Promise.resolve();
+  },
+
+  _onMessageSpeculativeConnect: function (msg, engineName) {
+    let engine = Services.search.getEngineByName(engineName);
+    if (!engine) {
+      throw new Error("Unknown engine name: " + engineName);
+    }
+    if (msg.target.contentWindow) {
+      engine.speculativeConnect({
+        window: msg.target.contentWindow,
+      });
+    }
+  },
+
   _onObserve: Task.async(function* (data) {
     if (data == "engine-current") {
       let engine = yield this._currentEngineObj();
@@ -171,6 +286,20 @@ this.ContentSearch = {
     }
   }),
 
+  _suggestionDataForBrowser: function (browser, create=false) {
+    let data = this._suggestionMap.get(browser);
+    if (!data && create) {
+      // Since one SearchSuggestionController instance is meant to be used per
+      // autocomplete widget, this means that we assume each xul:browser has at
+      // most one such widget.
+      data = {
+        controller: new SearchSuggestionController(),
+      };
+      this._suggestionMap.set(browser, data);
+    }
+    return data;
+  },
+
   _reply: function (msg, type, data) {
     // We reply asyncly to messages, and by the time we reply the browser we're
     // responding to may have been destroyed.  messageManager is null then.
@@ -241,6 +370,14 @@ this.ContentSearch = {
     return deferred.promise;
   },
 
+  _ensureDataHasProperties: function (data, requiredProperties) {
+    for (let prop of requiredProperties) {
+      if (!(prop in data)) {
+        throw new Error("Message data missing required property: " + prop);
+      }
+    }
+  },
+
   _initService: function () {
     if (!this._initServicePromise) {
       let deferred = Promise.defer();

browser/modules/test/browser.ini

@@ -9,6 +9,8 @@ support-files =
 support-files =
   contentSearch.js
   contentSearchBadImage.xml
+  contentSearchSuggestions.sjs
+  contentSearchSuggestions.xml
 [browser_NetworkPrioritizer.js]
 skip-if = e10s # Bug 666804 - Support NetworkPrioritizer in e10s
 [browser_SignInToWebsite.js]

browser/modules/test/browser_ContentSearch.js

@@ -171,6 +171,92 @@ add_task(function* badImage() {
   yield waitForTestMsg("CurrentState");
 });
 
+add_task(function* GetSuggestions_AddFormHistoryEntry_RemoveFormHistoryEntry() {
+  yield addTab();
+
+  // Add the test engine that provides suggestions.
+  let vals = yield waitForNewEngine("contentSearchSuggestions.xml", 0);
+  let engine = vals[0];
+
+  let searchStr = "browser_ContentSearch.js-suggestions-";
+
+  // Add a form history suggestion and wait for Satchel to notify about it.
+  gMsgMan.sendAsyncMessage(TEST_MSG, {
+    type: "AddFormHistoryEntry",
+    data: searchStr + "form",
+  });
+  let deferred = Promise.defer();
+  Services.obs.addObserver(function onAdd(subj, topic, data) {
+    if (data == "formhistory-add") {
+      executeSoon(() => deferred.resolve());
+    }
+  }, "satchel-storage-changed", false);
+  yield deferred.promise;
+
+  // Send GetSuggestions using the test engine.  Its suggestions should appear
+  // in the remote suggestions in the Suggestions response below.
+  gMsgMan.sendAsyncMessage(TEST_MSG, {
+    type: "GetSuggestions",
+    data: {
+      engineName: engine.name,
+      searchString: searchStr,
+      remoteTimeout: 5000,
+    },
+  });
+
+  // Check the Suggestions response.
+  let msg = yield waitForTestMsg("Suggestions");
+  checkMsg(msg, {
+    type: "Suggestions",
+    data: {
+      engineName: engine.name,
+      searchString: searchStr,
+      formHistory: [searchStr + "form"],
+      remote: [searchStr + "foo", searchStr + "bar"],
+    },
+  });
+
+  // Delete the form history suggestion and wait for Satchel to notify about it.
+  gMsgMan.sendAsyncMessage(TEST_MSG, {
+    type: "RemoveFormHistoryEntry",
+    data: searchStr + "form",
+  });
+  deferred = Promise.defer();
+  Services.obs.addObserver(function onRemove(subj, topic, data) {
+    if (data == "formhistory-remove") {
+      executeSoon(() => deferred.resolve());
+    }
+  }, "satchel-storage-changed", false);
+  yield deferred.promise;
+
+  // Send GetSuggestions again.
+  gMsgMan.sendAsyncMessage(TEST_MSG, {
+    type: "GetSuggestions",
+    data: {
+      engineName: engine.name,
+      searchString: searchStr,
+      remoteTimeout: 5000,
+    },
+  });
+
+  // The formHistory suggestions in the Suggestions response should be empty.
+  msg = yield waitForTestMsg("Suggestions");
+  checkMsg(msg, {
+    type: "Suggestions",
+    data: {
+      engineName: engine.name,
+      searchString: searchStr,
+      formHistory: [],
+      remote: [searchStr + "foo", searchStr + "bar"],
+    },
+  });
+
+  // Finally, clean up by removing the test engine.
+  Services.search.removeEngine(engine);
+  yield waitForTestMsg("CurrentState");
+});
+
+
 function checkMsg(actualMsg, expectedMsgData) {
   SimpleTest.isDeeply(actualMsg.data, expectedMsgData, "Checking message");
 }
@@ -226,7 +312,7 @@ function addTab() {
   let tab = gBrowser.addTab();
   gBrowser.selectedTab = tab;
   tab.linkedBrowser.addEventListener("load", function load() {
-    tab.removeEventListener("load", load, true);
+    tab.linkedBrowser.removeEventListener("load", load, true);
     let url = getRootDirectory(gTestPath) + TEST_CONTENT_SCRIPT_BASENAME;
     gMsgMan = tab.linkedBrowser.messageManager;
     gMsgMan.sendAsyncMessage(CONTENT_SEARCH_MSG, {

browser/modules/test/contentSearchSuggestions.sjs

@@ -0,0 +1,9 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function handleRequest(req, resp) {
+  let suffixes = ["foo", "bar"];
+  let data = [req.queryString, suffixes.map(s => req.queryString + s)];
+  resp.setHeader("Content-Type", "application/json", false);
+  resp.write(JSON.stringify(data));
+}

browser/modules/test/contentSearchSuggestions.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/">
+<ShortName>browser_ContentSearch contentSearchSuggestions.xml</ShortName>
+<Url type="application/x-suggestions+json" method="GET" template="http://mochi.test:8888/browser/browser/modules/test/contentSearchSuggestions.sjs?{searchTerms}"/>
+<Url type="text/html" method="GET" template="http://browser-ContentSearch.com/contentSearchSuggestions" rel="searchform"/>
+</SearchPlugin>