gecko-dev/browser/components/newtab/lib/HighlightsFeed.jsm

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";

const {Services} = ChromeUtils.import("resource://gre/modules/Services.jsm");

const {actionTypes: at} = ChromeUtils.import("resource://activity-stream/common/Actions.jsm");

const {shortURL} = ChromeUtils.import("resource://activity-stream/lib/ShortURL.jsm");
const {SectionsManager} = ChromeUtils.import("resource://activity-stream/lib/SectionsManager.jsm");
const {TOP_SITES_DEFAULT_ROWS, TOP_SITES_MAX_SITES_PER_ROW} = ChromeUtils.import("resource://activity-stream/common/Reducers.jsm");
const {Dedupe} = ChromeUtils.import("resource://activity-stream/common/Dedupe.jsm");

ChromeUtils.defineModuleGetter(this, "filterAdult",
  "resource://activity-stream/lib/FilterAdult.jsm");
ChromeUtils.defineModuleGetter(this, "LinksCache",
  "resource://activity-stream/lib/LinksCache.jsm");
ChromeUtils.defineModuleGetter(this, "NewTabUtils",
  "resource://gre/modules/NewTabUtils.jsm");
ChromeUtils.defineModuleGetter(this, "Screenshots",
  "resource://activity-stream/lib/Screenshots.jsm");
ChromeUtils.defineModuleGetter(this, "PageThumbs",
  "resource://gre/modules/PageThumbs.jsm");
ChromeUtils.defineModuleGetter(this, "DownloadsManager",
  "resource://activity-stream/lib/DownloadsManager.jsm");

const HIGHLIGHTS_MAX_LENGTH = 16;
const MANY_EXTRA_LENGTH = HIGHLIGHTS_MAX_LENGTH * 5 + TOP_SITES_DEFAULT_ROWS * TOP_SITES_MAX_SITES_PER_ROW;
const SECTION_ID = "highlights";
const SYNC_BOOKMARKS_FINISHED_EVENT = "weave:engine:sync:applied";
const BOOKMARKS_RESTORE_SUCCESS_EVENT = "bookmarks-restore-success";
const BOOKMARKS_RESTORE_FAILED_EVENT = "bookmarks-restore-failed";
const RECENT_DOWNLOAD_THRESHOLD = 36 * 60 * 60 * 1000;

this.HighlightsFeed = class HighlightsFeed {
  constructor() {
    this.dedupe = new Dedupe(this._dedupeKey);
    this.linksCache = new LinksCache(NewTabUtils.activityStreamLinks,
      "getHighlights", ["image"]);
    PageThumbs.addExpirationFilter(this);
    this.downloadsManager = new DownloadsManager();
  }

  _dedupeKey(site) {
    // Treat bookmarks, pocket, and downloaded items as un-dedupable, otherwise show one of a url
    return site && ((site.pocket_id || site.type === "bookmark" || site.type === "download") ? {} : site.url);
  }

  init() {
    Services.obs.addObserver(this, SYNC_BOOKMARKS_FINISHED_EVENT);
    Services.obs.addObserver(this, BOOKMARKS_RESTORE_SUCCESS_EVENT);
    Services.obs.addObserver(this, BOOKMARKS_RESTORE_FAILED_EVENT);
    SectionsManager.onceInitialized(this.postInit.bind(this));
  }

  postInit() {
    SectionsManager.enableSection(SECTION_ID);
    this.fetchHighlights({broadcast: true});
    this.downloadsManager.init(this.store);
  }

  uninit() {
    SectionsManager.disableSection(SECTION_ID);
    PageThumbs.removeExpirationFilter(this);
    Services.obs.removeObserver(this, SYNC_BOOKMARKS_FINISHED_EVENT);
    Services.obs.removeObserver(this, BOOKMARKS_RESTORE_SUCCESS_EVENT);
    Services.obs.removeObserver(this, BOOKMARKS_RESTORE_FAILED_EVENT);
  }

  observe(subject, topic, data) {
    // When we receive a notification that a sync has happened for bookmarks,
    // or Places finished importing or restoring bookmarks, refresh highlights
    const manyBookmarksChanged =
      (topic === SYNC_BOOKMARKS_FINISHED_EVENT && data === "bookmarks") ||
      topic === BOOKMARKS_RESTORE_SUCCESS_EVENT ||
      topic === BOOKMARKS_RESTORE_FAILED_EVENT;
    if (manyBookmarksChanged) {
      this.fetchHighlights({broadcast: true});
    }
  }

  filterForThumbnailExpiration(callback) {
    const state = this.store.getState().Sections.find(section => section.id === SECTION_ID);

    callback(state && state.initialized ? state.rows.reduce((acc, site) => {
      // Screenshots call in `fetchImage` will search for preview_image_url or
      // fallback to URL, so we prevent both from being expired.
      acc.push(site.url);
      if (site.preview_image_url) {
        acc.push(site.preview_image_url);
      }
      return acc;
    }, []) : []);
  }

  /**
   * Chronologically sort highlights of all types except 'visited'. Then just append
   * the rest at the end of highlights.
   * @param {Array} pages The full list of links to order.
   * @return {Array} A sorted array of highlights
   */
  _orderHighlights(pages) {
    const splitHighlights = {chronologicalCandidates: [], visited: []};
    for (let page of pages) {
      // If we have a page that is both a history item and a bookmark, treat it
      // as a bookmark
      if (page.type === "history" && !page.bookmarkGuid) {
        splitHighlights.visited.push(page);
      } else {
        splitHighlights.chronologicalCandidates.push(page);
      }
    }

    return splitHighlights.chronologicalCandidates
            .sort((a, b) => a.date_added < b.date_added)
            .concat(splitHighlights.visited);
  }

  /**
   * Refresh the highlights data for content.
   * @param {bool} options.broadcast Should the update be broadcasted.
   */
  async fetchHighlights(options = {}) {
    // If TopSites are enabled we need them for deduping, so wait for
    // TOP_SITES_UPDATED. We also need the section to be registered to update
    // state, so wait for postInit triggered by SectionsManager initializing.
    if ((!this.store.getState().TopSites.initialized && this.store.getState().Prefs.values["feeds.topsites"]) ||
        !this.store.getState().Sections.length) {
      return;
    }

    // We broadcast when we want to force an update, so get fresh links
    if (options.broadcast) {
      this.linksCache.expire();
    }

    // Request more than the expected length to allow for items being removed by
    // deduping against Top Sites or multiple history from the same domain, etc.
    const manyPages = await this.linksCache.request({
      numItems: MANY_EXTRA_LENGTH,
      excludeBookmarks: !this.store.getState().Prefs.values["section.highlights.includeBookmarks"],
      excludeHistory: !this.store.getState().Prefs.values["section.highlights.includeVisited"],
      excludePocket: !this.store.getState().Prefs.values["section.highlights.includePocket"],
    });

    if (this.store.getState().Prefs.values["section.highlights.includeDownloads"]) {
      // We only want 1 download that is less than 36 hours old, and the file currently exists
      let results = await this.downloadsManager.getDownloads(RECENT_DOWNLOAD_THRESHOLD, {numItems: 1, onlySucceeded: true, onlyExists: true});
      if (results.length) {
        // We only want 1 download, the most recent one
        manyPages.push({
          ...results[0],
          type: "download",
        });
      }
    }

    const orderedPages = this._orderHighlights(manyPages);

    // Remove adult highlights if we need to
    const checkedAdult = this.store.getState().Prefs.values.filterAdult ?
      filterAdult(orderedPages) : orderedPages;

    // Remove any Highlights that are in Top Sites already
    const [, deduped] = this.dedupe.group(this.store.getState().TopSites.rows, checkedAdult);

    // Keep all "bookmark"s and at most one (most recent) "history" per host
    const highlights = [];
    const hosts = new Set();
    for (const page of deduped) {
      const hostname = shortURL(page);
      // Skip this history page if we already something from the same host
      if (page.type === "history" && hosts.has(hostname)) {
        continue;
      }

      // If we already have the image for the card, use that immediately. Else
      // asynchronously fetch the image. NEVER fetch a screenshot for downloads
      if (!page.image && page.type !== "download") {
        this.fetchImage(page);
      }

      // Adjust the type for 'history' items that are also 'bookmarked' when we
      // want to include bookmarks
      if (page.type === "history" && page.bookmarkGuid &&
          this.store.getState().Prefs.values["section.highlights.includeBookmarks"]) {
        page.type = "bookmark";
      }

      // We want the page, so update various fields for UI
      Object.assign(page, {
        hasImage: page.type !== "download", // Downloads do not have an image - all else types fall back to a screenshot
        hostname,
        type: page.type,
        pocket_id: page.pocket_id,
      });

      // Add the "bookmark", "pocket", or not-skipped "history"
      highlights.push(page);
      hosts.add(hostname);

      // Remove internal properties that might be updated after dispatch
      delete page.__sharedCache;

      // Skip the rest if we have enough items
      if (highlights.length === HIGHLIGHTS_MAX_LENGTH) {
        break;
      }
    }

    const {initialized} = this.store.getState().Sections.find(section => section.id === SECTION_ID);
    // Broadcast when required or if it is the first update.
    const shouldBroadcast = options.broadcast || !initialized;

    SectionsManager.updateSection(SECTION_ID, {rows: highlights}, shouldBroadcast);
  }

  /**
   * Fetch an image for a given highlight and update the card with it. If no
   * image is available then fallback to fetching a screenshot.
   */
  fetchImage(page) {
    // Request a screenshot if we don't already have one pending
    const {preview_image_url: imageUrl, url} = page;
    return Screenshots.maybeCacheScreenshot(page, imageUrl || url, "image", image => {
      SectionsManager.updateSectionCard(SECTION_ID, url, {image}, true);
    });
  }

  onAction(action) {
    // Relay the downloads actions to DownloadsManager - it is a child of HighlightsFeed
    this.downloadsManager.onAction(action);
    switch (action.type) {
      case at.INIT:
        this.init();
        break;
      case at.SYSTEM_TICK:
      case at.TOP_SITES_UPDATED:
        this.fetchHighlights({broadcast: false});
        break;
      case at.PREF_CHANGED:
        // Update existing pages when the user changes what should be shown
        if (action.data.name.startsWith("section.highlights.include")) {
          this.fetchHighlights({broadcast: true});
        }
        break;
      case at.PLACES_HISTORY_CLEARED:
      case at.PLACES_LINK_BLOCKED:
      case at.DOWNLOAD_CHANGED:
      case at.POCKET_LINK_DELETED_OR_ARCHIVED:
        this.fetchHighlights({broadcast: true});
        break;
      case at.PLACES_LINKS_CHANGED:
      case at.PLACES_SAVED_TO_POCKET:
        this.linksCache.expire();
        this.fetchHighlights({broadcast: false});
        break;
      case at.UNINIT:
        this.uninit();
        break;
    }
  }
};

const EXPORTED_SYMBOLS = ["HighlightsFeed", "SECTION_ID", "MANY_EXTRA_LENGTH", "SYNC_BOOKMARKS_FINISHED_EVENT", "BOOKMARKS_RESTORE_SUCCESS_EVENT", "BOOKMARKS_RESTORE_FAILED_EVENT"];