Bug 1547669 - Improve the TopSites WebExtension API with further options. r=mixedpuppy,Mardak,adw

Adds includePinned and includeSearchShortcuts options to the topSites API, so
that it's possible to get a list of results like it's shown on the newtab page.
Both options default to false, so that the existing behavior is preserved.
The API gets disabled on Android; this API always depended on APIs not well
supported there, and the test is disabled on Android, so it was untested.

Differential Revision: https://phabricator.services.mozilla.com/D36467

--HG--
extra : moz-landing-system : lando

browser/components/extensions/ext-browser.json

@@ -198,6 +198,14 @@
       ["tabs"]
     ]
   },
+  "topSites": {
+    "url": "chrome://extensions/content/parent/ext-topSites.js",
+    "schema": "chrome://extensions/content/schemas/top_sites.json",
+    "scopes": ["addon_parent"],
+    "paths": [
+      ["topSites"]
+    ]
+  },
   "urlbar": {
     "url": "chrome://browser/content/parent/ext-urlbar.js",
     "schema": "chrome://browser/content/schemas/urlbar.json",

toolkit/components/extensions/ext-toolkit.json

@@ -181,14 +181,6 @@
       ["theme"]
     ]
   },
-  "topSites": {
-    "url": "chrome://extensions/content/parent/ext-topSites.js",
-    "schema": "chrome://extensions/content/schemas/top_sites.json",
-    "scopes": ["addon_parent"],
-    "paths": [
-      ["topSites"]
-    ]
-  },
   "userScripts": {
     "url": "chrome://extensions/content/parent/ext-userScripts.js",
     "schema": "chrome://extensions/content/schemas/user_scripts.json",

toolkit/components/extensions/jar.mn

@@ -40,7 +40,9 @@ toolkit.jar:
     content/extensions/parent/ext-telemetry.js (parent/ext-telemetry.js)
     content/extensions/parent/ext-theme.js (parent/ext-theme.js)
     content/extensions/parent/ext-toolkit.js (parent/ext-toolkit.js)
+#ifndef ANDROID
     content/extensions/parent/ext-topSites.js (parent/ext-topSites.js)
+#endif
     content/extensions/parent/ext-userScripts.js (parent/ext-userScripts.js)
     content/extensions/parent/ext-webRequest.js (parent/ext-webRequest.js)
     content/extensions/parent/ext-webNavigation.js (parent/ext-webNavigation.js)

toolkit/components/extensions/parent/ext-topSites.js

@@ -2,8 +2,14 @@
 /* vim: set sts=2 sw=2 et tw=80: */
 "use strict";
 
-ChromeUtils.defineModuleGetter(this, "NewTabUtils",
-                               "resource://gre/modules/NewTabUtils.jsm");
+XPCOMUtils.defineLazyModuleGetters(this, {
+  NewTabUtils: "resource://gre/modules/NewTabUtils.jsm",
+  shortURL: "resource://activity-stream/lib/ShortURL.jsm",
+  getSearchProvider: "resource://activity-stream/lib/SearchShortcuts.jsm",
+});
+
+const SHORTCUTS_PREF =
+  "browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts";
 
 this.topSites = class extends ExtensionAPI {
   getAPI(context) {
@@ -16,13 +22,52 @@ this.topSites = class extends ExtensionAPI {
             numItems: options.limit,
             includeFavicon: options.includeFavicon,
           });
-          return links.map(link => {
-            return {
-              url: link.url,
-              title: link.title,
-              favicon: link.favicon,
-            };
-          });
+
+          if (options.includePinned) {
+            let pinnedLinks = NewTabUtils.pinnedLinks.links;
+            if (options.includeFavicon) {
+              pinnedLinks = NewTabUtils.activityStreamProvider._faviconBytesToDataURI(
+                await NewTabUtils.activityStreamProvider._addFavicons(pinnedLinks));
+            }
+            pinnedLinks.forEach((pinnedLink, index) => {
+              if (pinnedLink &&
+                  (!pinnedLink.searchTopSite || options.includeSearchShortcuts)) {
+                // Remove any dupes from history.
+                links = links.filter(link => link.url != pinnedLink.url &&
+                  (!options.onePerDomain ||
+                   NewTabUtils.extractSite(link.url) != pinnedLink.baseDomain));
+                links.splice(index, 0, pinnedLink);
+              }
+            });
+            // Because we may have addded links, we must crop again.
+            if (options.limit) {
+              links = links.slice(0, options.limit);
+            }
+          }
+
+          // Convert links to search shortcuts, if necessary.
+          if (options.includeSearchShortcuts &&
+            Services.prefs.getBoolPref(SHORTCUTS_PREF, false)) {
+            // Pinned shortcuts are already returned as searchTopSite links,
+            // with a proper label and url. But certain non-pinned links may
+            // also be promoted to search shortcuts; here we convert them.
+            links = links.map(link => {
+              let searchProvider = getSearchProvider(shortURL(link));
+              if (searchProvider) {
+                link.searchTopSite = true;
+                link.label = searchProvider.keyword;
+                link.url = searchProvider.url;
+              }
+              return link;
+            });
+          }
+
+          return links.map(link => ({
+            url: link.url,
+            title: link.searchTopSite ? link.label : link.title,
+            favicon: link.favicon,
+            type: link.searchTopSite ? "search" : "url",
+          }));
         },
       },
     };

toolkit/components/extensions/schemas/jar.mn

@@ -40,7 +40,9 @@ toolkit.jar:
     content/extensions/schemas/telemetry.json
     content/extensions/schemas/test.json
     content/extensions/schemas/theme.json
+#ifndef ANDROID
     content/extensions/schemas/top_sites.json
+#endif
     content/extensions/schemas/types.json
     content/extensions/schemas/user_scripts.json
     content/extensions/schemas/user_scripts_content.json

toolkit/components/extensions/schemas/top_sites.json

@@ -40,6 +40,16 @@
             "type": "string",
             "optional": true,
             "description": "Data URL for the favicon, if available."
+          },
+          "type": {
+            "type": "string",
+            "enum": [
+              "url",
+              "search"
+            ],
+            "optional": true,
+            "default": "url",
+            "description": "The entry type, either <code>url</code> for a normal page link, or <code>search</code> for a search shortcut."
           }
         }
       }
@@ -87,6 +97,18 @@
                 "default": false,
                 "optional": true,
                 "description": "Include sites favicon if available."
+              },
+              "includePinned": {
+                "type": "boolean",
+                "default": false,
+                "optional": true,
+                "description": "Include sites that the user has pinned on the Firefox new tab."
+              },
+              "includeSearchShortcuts": {
+                "type": "boolean",
+                "default": false,
+                "optional": true,
+                "description": "Include search shortcuts appearing on the Firefox new tab."
               }
             },
             "default": {},

toolkit/components/extensions/test/xpcshell/test_ext_topSites.js

@@ -4,13 +4,13 @@ const {PlacesUtils} = ChromeUtils.import("resource://gre/modules/PlacesUtils.jsm
 const {NewTabUtils} = ChromeUtils.import("resource://gre/modules/NewTabUtils.jsm");
 const {PlacesTestUtils} = ChromeUtils.import("resource://testing-common/PlacesTestUtils.jsm");
 
-// Disable top site search shortcuts for this test
-Services.prefs.setBoolPref("browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts", false);
+const SEARCH_SHORTCUTS_EXPERIMENT_PREF = "browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts";
 
 // A small 1x1 test png
 const IMAGE_1x1 = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVQI12NgAAAAAgAB4iG8MwAAAABJRU5ErkJggg==";
 
 add_task(async function test_topSites() {
+  Services.prefs.setBoolPref(SEARCH_SHORTCUTS_EXPERIMENT_PREF, false);
   let visits = [];
   const numVisits = 15; // To make sure we get frecency.
   let visitDate = new Date(1999, 9, 9, 9, 9).getTime();
@@ -49,7 +49,7 @@ add_task(async function test_topSites() {
   equal(links.length, visits.length, "Top sites has been successfully initialized");
 
   // Drop the visits.visits for later testing.
-  visits = visits.map(v => { return {url: v.url, title: v.title, favicon: undefined}; });
+  visits = visits.map(v => { return {url: v.url, title: v.title, favicon: undefined, type: "url"}; });
 
   // Test that results from all providers are returned by default.
   let extension = ExtensionTestUtils.loadExtension({
@@ -104,4 +104,149 @@ add_task(async function test_topSites() {
   NewTabUtils.uninit();
   await extension.unload();
   await PlacesUtils.history.clear();
+  Services.prefs.clearUserPref(SEARCH_SHORTCUTS_EXPERIMENT_PREF);
+});
+
+// Test pinned likns and search shortcuts.
+add_task(async function test_topSites_complete() {
+  Services.prefs.setBoolPref(SEARCH_SHORTCUTS_EXPERIMENT_PREF, true);
+  NewTabUtils.init();
+  let time = new Date();
+  let pinnedIndex = 0;
+  let entries = [
+    {
+      url: `http://pinned1.com/`,
+      title: "pinned1",
+      type: "url",
+      pinned: pinnedIndex++,
+      visitDate: time,
+    },
+    {
+      url: `http://search1.com/`,
+      title: "@search1",
+      type: "search",
+      pinned: pinnedIndex++,
+      visitDate: new Date(--time),
+    },
+    {
+      url: `https://amazon.com`,
+      title: "@amazon",
+      type: "search",
+      visitDate: new Date(--time),
+    },
+    {
+      url: `http://history1.com/`,
+      title: "history1",
+      type: "url",
+      visitDate: new Date(--time),
+    },
+    {
+      url: `http://history2.com/`,
+      title: "history2",
+      type: "url",
+      visitDate: new Date(--time),
+    },
+    {
+      url: `https://blocked1.com/`,
+      title: "blocked1",
+      type: "blocked",
+      visitDate: new Date(--time),
+    },
+  ];
+
+  for (let entry of entries) {
+    // Build up frecency.
+    await PlacesUtils.history.insert({
+      url: entry.url,
+      title: entry.title,
+      visits: new Array(15).fill({
+        date: entry.visitDate,
+        transition: PlacesUtils.history.TRANSITIONS.LINK,
+      }),
+    });
+    // Insert a favicon to show that favicons are not returned by default.
+    await PlacesTestUtils.addFavicons(new Map([[entry.url, IMAGE_1x1]]));
+    if (entry.pinned !== undefined) {
+      let info = entry.type == "search" ?
+        {url: entry.url, label: entry.title, searchTopSite: true} :
+        {url: entry.url, title: entry.title};
+      NewTabUtils.pinnedLinks.pin(info, entry.pinned);
+    }
+    if (entry.type == "blocked") {
+      NewTabUtils.activityStreamLinks.blockURL({url: entry.url});
+    }
+  }
+
+  // Some transformation is necessary to match output data.
+  let expectedResults =
+    entries.filter(e => e.type != "blocked").map(e => {
+      e.favicon = undefined;
+      delete e.visitDate;
+      delete e.pinned;
+      return e;
+    });
+
+  let extension = ExtensionTestUtils.loadExtension({
+    manifest: {
+      "permissions": [
+        "topSites",
+      ],
+    },
+    background() {
+      browser.test.onMessage.addListener(async options => {
+        let sites;
+        if (typeof options !== undefined) {
+          sites = await browser.topSites.get(options);
+        } else {
+          sites = await browser.topSites.get();
+        }
+        browser.test.sendMessage("sites", sites);
+      });
+    },
+  });
+
+  await extension.startup();
+
+  // Test that results are returned by the API.
+  function getSites(options) {
+    extension.sendMessage(options);
+    return extension.awaitMessage("sites");
+  }
+
+  Assert.deepEqual(expectedResults,
+                   await getSites({includePinned: true,
+                                   includeSearchShortcuts: true}),
+                   "got topSites all links");
+
+  // Test no shortcuts.
+  dump(JSON.stringify(await getSites({includePinned: true})) + "\n");
+  Assert.ok(!(await getSites({includePinned: true})).some(link => link.type == "search"),
+            "should get no shortcuts");
+
+  // Test favicons.
+  let topSites = await getSites({includePinned: true,
+                                 includeSearchShortcuts: true,
+                                 includeFavicon: true});
+  Assert.ok(topSites.every(f => f.favicon == IMAGE_1x1), "favicon is correct");
+
+  // Test options.limit.
+  Assert.equal(1, (await getSites({includePinned: true,
+                                   includeSearchShortcuts: true,
+                                   limit: 1})).length,
+               "limit to 1 topSite");
+
+  // Clear history for a pinned entry, then check results.
+  await PlacesUtils.history.remove("http://pinned1.com/");
+  let links = await getSites({includePinned: true});
+  Assert.ok(links.find(link => link.url == "http://pinned1.com/"),
+            "Check unvisited pinned links are returned.");
+  links = await getSites();
+  Assert.ok(!links.find(link => link.url == "http://pinned1.com/"),
+            "Check unvisited pinned links are not returned.");
+
+  await extension.unload();
+  NewTabUtils.uninit();
+  await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
+  Services.prefs.clearUserPref(SEARCH_SHORTCUTS_EXPERIMENT_PREF);
 });

toolkit/components/places/Bookmarks.jsm

@@ -1973,7 +1973,7 @@ async function fetchBookmark(info, options = {}) {
        WHERE b.guid = :guid
       `, { guid: info.guid });
 
-    return rows.length ? rowsToItemsArray(rows, options.ignoreInvalidURLs)[0] : null;
+    return rows.length ? rowsToItemsArray(rows, !!options.ignoreInvalidURLs)[0] : null;
   };
   if (options.concurrent) {
     let db = await PlacesUtils.promiseDBConnection();