Bug 1719697 - Add an API for accessing Snapshots based on metadata. r=mak

Differential Revision: https://phabricator.services.mozilla.com/D119227

browser/components/places/Interactions.jsm

@@ -104,6 +104,8 @@ class TypingInteraction {
  *   Time in milliseconds that the page has been actively viewed for.
  * @property {string} url
  *   The url of the page that was interacted with.
+ * @property {Interactions.DOCUMENT_TYPE} documentType
+ *   The type of the document.
  * @property {number} typingTime
  *   Time in milliseconds that the user typed on the page
  * @property {number} keypresses
@@ -112,6 +114,10 @@ class TypingInteraction {
  *   Time in milliseconds that the user spent scrolling the page
  * @property {number} scrollingDistance
  *   The distance, in pixels, that the user scrolled the page
+ * @property {number} created_at
+ *   Creation time as the number of milliseconds since the epoch.
+ * @property {number} updated_at
+ *   Last updated time as the number of milliseconds since the epoch.
  */
 
 /**
@@ -119,6 +125,13 @@ class TypingInteraction {
  * obtaining interaction information and passing it to the InteractionsManager.
  */
 class _Interactions {
+  DOCUMENT_TYPE = {
+    // Used when the document type is unknown.
+    GENERIC: 0,
+    // Used for pages serving media, e.g. videos.
+    MEDIA: 1,
+  };
+
   /**
    * This is used to store potential interactions. It maps the browser
    * to the current interaction information.
@@ -611,6 +624,8 @@ class InteractionsStore {
         params[`url${i}`] = interaction.url;
         params[`created_at${i}`] = interaction.created_at;
         params[`updated_at${i}`] = interaction.updated_at;
+        params[`document_type${i}`] =
+          interaction.documentType ?? Interactions.DOCUMENT_TYPE.GENERIC;
         params[`total_view_time${i}`] =
           Math.round(interaction.totalViewTime) || 0;
         params[`typing_time${i}`] = Math.round(interaction.typingTime) || 0;
@@ -626,6 +641,7 @@ class InteractionsStore {
           (SELECT id FROM moz_places WHERE url_hash = hash(:url${i}) AND url = :url${i}),
           :created_at${i},
           :updated_at${i},
+          :document_type${i},
           :total_view_time${i},
           :typing_time${i},
           :key_presses${i},
@@ -642,11 +658,11 @@ class InteractionsStore {
       async db => {
         await db.executeCached(
           `
-          WITH inserts (id, place_id, created_at, updated_at, total_view_time, typing_time, key_presses, scrolling_time, scrolling_distance) AS (
+          WITH inserts (id, place_id, created_at, updated_at, document_type, total_view_time, typing_time, key_presses, scrolling_time, scrolling_distance) AS (
             VALUES ${SQLInsertFragments.join(", ")}
           )
           INSERT OR REPLACE INTO moz_places_metadata (
-            id, place_id, created_at, updated_at, total_view_time, typing_time, key_presses, scrolling_time, scrolling_distance
+            id, place_id, created_at, updated_at, document_type, total_view_time, typing_time, key_presses, scrolling_time, scrolling_distance
           ) SELECT * FROM inserts WHERE place_id NOT NULL;
         `,
           params

browser/components/places/Snapshots.jsm

@@ -0,0 +1,146 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["Snapshots"];
+
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  PlacesUtils: "resource://gre/modules/PlacesUtils.jsm",
+  Services: "resource://gre/modules/Services.jsm",
+});
+
+/**
+ * @typedef {object} Snapshot
+ *   A snapshot summarises a collection of interactions.
+ * @property {string} url
+ *   The associated URL.
+ * @property {Date} createdAt
+ *   The date/time the snapshot was created.
+ * @property {Date} removedAt
+ *   The date/time the snapshot was deleted.
+ * @property {Date} firstInteractionAt
+ *   The date/time of the first interaction with the snapshot.
+ * @property {Date} lastInteractionAt
+ *   The date/time of the last interaction with the snapshot.
+ * @property {Interactions.DOCUMENT_TYPE} documentType
+ *   The document type of the snapshot.
+ * @property {boolean} userPersisted
+ *   True if the user created or persisted the snapshot in some way.
+ */
+
+/**
+ * Handles storing and retrieving of Snapshots in the Places database.
+ *
+ * Notifications of updates are sent via the observer service:
+ * - places-snapshot-added, data: url
+ *     Sent when a new snapshot is added
+ * - places-snapshot-deleted, data: url
+ *     Sent when a snapshot is removed.
+ */
+const Snapshots = new (class Snapshots {
+  #snapshots = new Map();
+
+  /**
+   * Adds a new snapshot.
+   *
+   * @param {object} details
+   * @param {string} details.url
+   *   The url associated with the snapshot.
+   * @param {boolean} [details.userPersisted]
+   *   True if the user created or persisted the snapshot in some way, defaults to
+   *   false.
+   */
+  async add({ url, userPersisted = false }) {
+    let now = new Date();
+
+    let db = await PlacesUtils.promiseDBConnection();
+    let rows = await db.executeCached(
+      `
+      WITH places(place_id) AS (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url),
+        inserts(created_at, updated_at, document_type) AS (
+          VALUES(
+            (SELECT min(created_at) FROM moz_places_metadata WHERE place_id in places),
+            (SELECT max(updated_at) FROM moz_places_metadata WHERE place_id in places),
+            (SELECT document_type FROM moz_places_metadata WHERE place_id in places ORDER BY updated_at DESC LIMIT 1)
+          )
+        )
+      SELECT * from inserts WHERE created_at is not null
+    `,
+      { url }
+    );
+    if (!rows.length) {
+      throw new Error("Could not find existing interactions");
+    }
+
+    this.#snapshots.set(url, {
+      url,
+      userPersisted,
+      createdAt: now,
+      removedAt: null,
+      documentType: rows[0].getResultByName("document_type"),
+      firstInteractionAt: new Date(rows[0].getResultByName("created_at")),
+      lastInteractionAt: new Date(rows[0].getResultByName("updated_at")),
+    });
+
+    Services.obs.notifyObservers(null, "places-snapshot-added", url);
+  }
+
+  /**
+   * Deletes a snapshot, creating a tombstone.
+   *
+   * @param {string} url
+   *   The url of the snapshot to delete.
+   */
+  async delete(url) {
+    let snapshot = this.#snapshots.get(url);
+
+    if (snapshot) {
+      snapshot.removedAt = new Date();
+      Services.obs.notifyObservers(null, "places-snapshot-deleted", url);
+    }
+  }
+
+  /**
+   * Gets the details for a particular snapshot based on the url.
+   *
+   * @param {string} url
+   *   The url of the snapshot to obtain.
+   * @param {boolean} [includeTombstones]
+   *   Whether to include tombstones in the snapshots to obtain.
+   * @returns {?Snapshot}
+   */
+  async get(url, includeTombstones = false) {
+    let snapshot = this.#snapshots.get(url);
+    if (!snapshot || (snapshot.removedAt && !includeTombstones)) {
+      return null;
+    }
+
+    return snapshot;
+  }
+
+  /**
+   * Queries the current snapshots in the database.
+   *
+   * @param {object} [options]
+   * @param {number} [options.limit]
+   *   A numerical limit to the number of snapshots to retrieve, defaults to 100.
+   * @param {boolean} [options.includeTombstones]
+   *   Whether to include tombstones in the snapshots to obtain.
+   * @returns {Snapshot[]}
+   *   Returns snapshots in order of descending last interaction time.
+   */
+  async query({ limit = 100, includeTombstones = false } = {}) {
+    let snapshots = Array.from(this.#snapshots.values());
+
+    if (!includeTombstones) {
+      return snapshots.filter(s => !s.removedAt).slice(0, limit);
+    }
+    return snapshots.slice(0, limit);
+  }
+})();

browser/components/places/moz.build

@@ -4,7 +4,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-XPCSHELL_TESTS_MANIFESTS += ["tests/unit/xpcshell.ini"]
+XPCSHELL_TESTS_MANIFESTS += [
+    "tests/unit/interactions/xpcshell.ini",
+    "tests/unit/xpcshell.ini",
+]
 MOCHITEST_CHROME_MANIFESTS += ["tests/chrome/chrome.ini"]
 BROWSER_CHROME_MANIFESTS += [
     "tests/browser/browser.ini",
@@ -17,6 +20,7 @@ EXTRA_JS_MODULES += [
     "Interactions.jsm",
     "InteractionsBlocklist.jsm",
     "PlacesUIUtils.jsm",
+    "Snapshots.jsm",
 ]
 
 FINAL_TARGET_FILES.actors += [

browser/components/places/tests/unit/interactions/head_interactions.js

@@ -0,0 +1,127 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  Interactions: "resource:///modules/Interactions.jsm",
+  PlacesTestUtils: "resource://testing-common/PlacesTestUtils.jsm",
+  PlacesUtils: "resource://gre/modules/PlacesUtils.jsm",
+  setTimeout: "resource://gre/modules/Timer.jsm",
+  Snapshots: "resource:///modules/Snapshots.jsm",
+  TestUtils: "resource://testing-common/TestUtils.jsm",
+});
+
+// Initialize profile.
+var gProfD = do_get_profile(true);
+
+/**
+ * Adds a test interaction to the database.
+ *
+ * @param {InteractionInfo[]} interactions
+ */
+async function addInteractions(interactions) {
+  await PlacesTestUtils.addVisits(interactions.map(i => i.url));
+
+  for (let interaction of interactions) {
+    await Interactions.store.add({
+      url: interaction.url,
+      documentType:
+        interaction.documentType ?? Interactions.DOCUMENT_TYPE.GENERIC,
+      totalViewTime: interaction.totalViewTime ?? 0,
+      typingTime: interaction.typingTime ?? 0,
+      keypresses: interaction.keypresses ?? 0,
+      scrollingTime: interaction.scrollingTime ?? 0,
+      scrollingDistance: interaction.scrollingDistance ?? 0,
+      created_at: interaction.created_at || Date.now(),
+      updated_at: interaction.updated_at || Date.now(),
+    });
+  }
+  await Interactions.store.flush();
+}
+
+/**
+ * Asserts that a date looks reasonably valid, i.e. created no earlier than
+ * 24 hours prior to the current date.
+ *
+ * @param {Date} date
+ *   The date to check.
+ */
+function assertRecentDate(date) {
+  Assert.greater(
+    date.getTime(),
+    Date.now() - 1000 * 60 * 60 * 24,
+    "Should have a reasonable value for the date"
+  );
+}
+
+/**
+ * Asserts that an individual snapshot contains the expected values.
+ *
+ * @param {Snapshot} actual
+ *  The snapshot to test.
+ * @param {Snapshot} expected
+ *  The snapshot to test against.
+ */
+function assertSnapshot(actual, expected) {
+  Assert.equal(actual.url, expected.url, "Should have the expected URL");
+  // Avoid falsey-types that we might get from the database.
+  Assert.strictEqual(
+    actual.userPersisted,
+    expected.userPersisted ?? false,
+    "Should have the expected user persisted value"
+  );
+  Assert.strictEqual(
+    actual.documentType,
+    expected.documentType ?? Interactions.DOCUMENT_TYPE.GENERIC,
+    "Should have the expected document type"
+  );
+  assertRecentDate(actual.createdAt);
+  assertRecentDate(actual.firstInteractionAt);
+  assertRecentDate(actual.lastInteractionAt);
+  if (expected.lastUpdated) {
+    Assert.greaterOrEqual(
+      actual.lastInteractionAt,
+      expected.lastUpdated,
+      "Should have a last interaction time greater than or equal to the expected last updated time"
+    );
+  }
+  if (expected.removedAt) {
+    Assert.greaterOrEqual(
+      actual.removedAt.getTime(),
+      expected.removedAt.getTime(),
+      "Should have the removed at time greater than or equal to the expected removed at time"
+    );
+  } else {
+    Assert.strictEqual(
+      actual.removedAt,
+      null,
+      "Should not have a removed at time"
+    );
+  }
+}
+
+/**
+ * Asserts that the snapshots in the database match the expected values.
+ *
+ * @param {Snapshot[]} expected
+ *   The expected snapshots.
+ * @param {object} options
+ *   @see Snapshots.query().
+ */
+async function assertSnapshots(expected, options) {
+  let snapshots = await Snapshots.query(options);
+
+  info(`Found ${snapshots.length} snapshots:\n ${JSON.stringify(snapshots)}`);
+  Assert.equal(
+    snapshots.length,
+    expected.length,
+    "Should have the expected number of snapshots"
+  );
+  for (let i = 0; i < expected.length; i++) {
+    assertSnapshot(snapshots[i], expected[i]);
+  }
+}

browser/components/places/tests/unit/interactions/test_snapshots_basics.js

@@ -0,0 +1,77 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Tests for snapshot creation and removal.
+ */
+
+const TEST_URL1 = "https://example.com/";
+const TEST_URL2 = "https://example.com/12345";
+const TOPIC_ADDED = "places-snapshot-added";
+const TOPIC_DELETED = "places-snapshot-deleted";
+
+add_task(async function test_add_simple_snapshot() {
+  await addInteractions([
+    { url: TEST_URL1, documentType: Interactions.DOCUMENT_TYPE.MEDIA },
+    { url: TEST_URL2 },
+  ]);
+
+  let promise = TestUtils.topicObserved(
+    TOPIC_ADDED,
+    (subject, data) => !subject && data == TEST_URL1
+  );
+  await Snapshots.add({ url: TEST_URL1 });
+  await promise;
+
+  promise = TestUtils.topicObserved(
+    TOPIC_ADDED,
+    (subject, data) => !subject && data == TEST_URL2
+  );
+  await Snapshots.add({ url: TEST_URL2, userPersisted: true });
+  await promise;
+
+  await assertSnapshots([
+    { url: TEST_URL1, documentType: Interactions.DOCUMENT_TYPE.MEDIA },
+    { url: TEST_URL2, userPersisted: true },
+  ]);
+
+  let snapshot = await Snapshots.get(TEST_URL2);
+  assertSnapshot(snapshot, { url: TEST_URL2, userPersisted: true });
+});
+
+add_task(async function test_add_snapshot_without_interaction() {
+  await Assert.rejects(
+    Snapshots.add({ url: "https://invalid.com/" }),
+    /Could not find existing interactions/,
+    "Should reject if an interaction is missing"
+  );
+});
+
+add_task(async function test_get_snapshot_not_found() {
+  let snapshot = await Snapshots.get("https://invalid.com/");
+  Assert.ok(
+    !snapshot,
+    "Should not have found a snapshot for a URL not in the database"
+  );
+});
+
+add_task(async function test_remove_snapshot() {
+  let removedAt = new Date();
+
+  let promise = TestUtils.topicObserved(
+    TOPIC_DELETED,
+    (subject, data) => !subject && data == TEST_URL1
+  );
+  await Snapshots.delete(TEST_URL1);
+  await promise;
+
+  let snapshot = await Snapshots.get(TEST_URL1);
+  Assert.ok(!snapshot, "Tombstone snapshots should not be returned by default");
+
+  snapshot = await Snapshots.get(TEST_URL1, true);
+  assertSnapshot(snapshot, {
+    url: TEST_URL1,
+    documentType: Interactions.DOCUMENT_TYPE.MEDIA,
+    removedAt,
+  });
+});

browser/components/places/tests/unit/interactions/test_snapshots_queries.js

@@ -0,0 +1,78 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/**
+ * Tests for queries on snapshots.
+ */
+
+add_task(async function setup() {
+  for (let i = 0; i < 10; i++) {
+    let url = `https://example.com/${i}/`;
+    await addInteractions([{ url }]);
+
+    await Snapshots.add({ url: `https://example.com/${i}/` });
+
+    // Give a little time between updates to help the ordering.
+    /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */
+    await new Promise(r => setTimeout(r, 20));
+  }
+});
+
+add_task(async function test_query_returns_correct_order() {
+  await assertSnapshots([
+    { url: "https://example.com/0/" },
+    { url: "https://example.com/1/" },
+    { url: "https://example.com/2/" },
+    { url: "https://example.com/3/" },
+    { url: "https://example.com/4/" },
+    { url: "https://example.com/5/" },
+    { url: "https://example.com/6/" },
+    { url: "https://example.com/7/" },
+    { url: "https://example.com/8/" },
+    { url: "https://example.com/9/" },
+  ]);
+
+  // TODO: Update an interaction, and check the new order is returned corectly.
+});
+
+add_task(async function test_query_limit() {
+  await assertSnapshots(
+    [{ url: `https://example.com/0/` }, { url: `https://example.com/1/` }],
+    { limit: 2 }
+  );
+});
+
+add_task(async function test_query_handles_tombstones() {
+  let removedAt = new Date();
+  await Snapshots.delete("https://example.com/3/");
+  await Snapshots.delete("https://example.com/6/");
+
+  info("Deleted snapshots should not appear in the query");
+  await assertSnapshots([
+    { url: "https://example.com/0/" },
+    { url: "https://example.com/1/" },
+    { url: "https://example.com/2/" },
+    { url: "https://example.com/4/" },
+    { url: "https://example.com/5/" },
+    { url: "https://example.com/7/" },
+    { url: "https://example.com/8/" },
+    { url: "https://example.com/9/" },
+  ]);
+
+  info("Deleted snapshots should appear if tombstones are requested");
+  await assertSnapshots(
+    [
+      { url: "https://example.com/0/" },
+      { url: "https://example.com/1/" },
+      { url: "https://example.com/2/" },
+      { url: "https://example.com/3/", removedAt },
+      { url: "https://example.com/4/" },
+      { url: "https://example.com/5/" },
+      { url: "https://example.com/6/", removedAt },
+      { url: "https://example.com/7/" },
+      { url: "https://example.com/8/" },
+      { url: "https://example.com/9/" },
+    ],
+    { includeTombstones: true }
+  );
+});

browser/components/places/tests/unit/interactions/xpcshell.ini

@@ -0,0 +1,7 @@
+[DEFAULT]
+head = head_interactions.js
+firefox-appdir = browser
+skip-if = toolkit == 'android'
+
+[test_snapshots_basics.js]
+[test_snapshots_queries.js]