diff --git a/browser/components/places/Interactions.jsm b/browser/components/places/Interactions.jsm index 3285800214e1..c95c7a0b3300 100644 --- a/browser/components/places/Interactions.jsm +++ b/browser/components/places/Interactions.jsm @@ -104,6 +104,8 @@ class TypingInteraction { * Time in milliseconds that the page has been actively viewed for. * @property {string} url * The url of the page that was interacted with. + * @property {Interactions.DOCUMENT_TYPE} documentType + * The type of the document. * @property {number} typingTime * Time in milliseconds that the user typed on the page * @property {number} keypresses @@ -112,6 +114,10 @@ class TypingInteraction { * Time in milliseconds that the user spent scrolling the page * @property {number} scrollingDistance * The distance, in pixels, that the user scrolled the page + * @property {number} created_at + * Creation time as the number of milliseconds since the epoch. + * @property {number} updated_at + * Last updated time as the number of milliseconds since the epoch. */ /** @@ -119,6 +125,13 @@ class TypingInteraction { * obtaining interaction information and passing it to the InteractionsManager. */ class _Interactions { + DOCUMENT_TYPE = { + // Used when the document type is unknown. + GENERIC: 0, + // Used for pages serving media, e.g. videos. + MEDIA: 1, + }; + /** * This is used to store potential interactions. It maps the browser * to the current interaction information. @@ -611,6 +624,8 @@ class InteractionsStore { params[`url${i}`] = interaction.url; params[`created_at${i}`] = interaction.created_at; params[`updated_at${i}`] = interaction.updated_at; + params[`document_type${i}`] = + interaction.documentType ?? Interactions.DOCUMENT_TYPE.GENERIC; params[`total_view_time${i}`] = Math.round(interaction.totalViewTime) || 0; params[`typing_time${i}`] = Math.round(interaction.typingTime) || 0; @@ -626,6 +641,7 @@ class InteractionsStore { (SELECT id FROM moz_places WHERE url_hash = hash(:url${i}) AND url = :url${i}), :created_at${i}, :updated_at${i}, + :document_type${i}, :total_view_time${i}, :typing_time${i}, :key_presses${i}, @@ -642,11 +658,11 @@ class InteractionsStore { async db => { await db.executeCached( ` - WITH inserts (id, place_id, created_at, updated_at, total_view_time, typing_time, key_presses, scrolling_time, scrolling_distance) AS ( + WITH inserts (id, place_id, created_at, updated_at, document_type, total_view_time, typing_time, key_presses, scrolling_time, scrolling_distance) AS ( VALUES ${SQLInsertFragments.join(", ")} ) INSERT OR REPLACE INTO moz_places_metadata ( - id, place_id, created_at, updated_at, total_view_time, typing_time, key_presses, scrolling_time, scrolling_distance + id, place_id, created_at, updated_at, document_type, total_view_time, typing_time, key_presses, scrolling_time, scrolling_distance ) SELECT * FROM inserts WHERE place_id NOT NULL; `, params diff --git a/browser/components/places/Snapshots.jsm b/browser/components/places/Snapshots.jsm new file mode 100644 index 000000000000..e1c906bce3db --- /dev/null +++ b/browser/components/places/Snapshots.jsm @@ -0,0 +1,146 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +var EXPORTED_SYMBOLS = ["Snapshots"]; + +const { XPCOMUtils } = ChromeUtils.import( + "resource://gre/modules/XPCOMUtils.jsm" +); + +XPCOMUtils.defineLazyModuleGetters(this, { + PlacesUtils: "resource://gre/modules/PlacesUtils.jsm", + Services: "resource://gre/modules/Services.jsm", +}); + +/** + * @typedef {object} Snapshot + * A snapshot summarises a collection of interactions. + * @property {string} url + * The associated URL. + * @property {Date} createdAt + * The date/time the snapshot was created. + * @property {Date} removedAt + * The date/time the snapshot was deleted. + * @property {Date} firstInteractionAt + * The date/time of the first interaction with the snapshot. + * @property {Date} lastInteractionAt + * The date/time of the last interaction with the snapshot. + * @property {Interactions.DOCUMENT_TYPE} documentType + * The document type of the snapshot. + * @property {boolean} userPersisted + * True if the user created or persisted the snapshot in some way. + */ + +/** + * Handles storing and retrieving of Snapshots in the Places database. + * + * Notifications of updates are sent via the observer service: + * - places-snapshot-added, data: url + * Sent when a new snapshot is added + * - places-snapshot-deleted, data: url + * Sent when a snapshot is removed. + */ +const Snapshots = new (class Snapshots { + #snapshots = new Map(); + + /** + * Adds a new snapshot. + * + * @param {object} details + * @param {string} details.url + * The url associated with the snapshot. + * @param {boolean} [details.userPersisted] + * True if the user created or persisted the snapshot in some way, defaults to + * false. + */ + async add({ url, userPersisted = false }) { + let now = new Date(); + + let db = await PlacesUtils.promiseDBConnection(); + let rows = await db.executeCached( + ` + WITH places(place_id) AS (SELECT id FROM moz_places WHERE url_hash = hash(:url) AND url = :url), + inserts(created_at, updated_at, document_type) AS ( + VALUES( + (SELECT min(created_at) FROM moz_places_metadata WHERE place_id in places), + (SELECT max(updated_at) FROM moz_places_metadata WHERE place_id in places), + (SELECT document_type FROM moz_places_metadata WHERE place_id in places ORDER BY updated_at DESC LIMIT 1) + ) + ) + SELECT * from inserts WHERE created_at is not null + `, + { url } + ); + if (!rows.length) { + throw new Error("Could not find existing interactions"); + } + + this.#snapshots.set(url, { + url, + userPersisted, + createdAt: now, + removedAt: null, + documentType: rows[0].getResultByName("document_type"), + firstInteractionAt: new Date(rows[0].getResultByName("created_at")), + lastInteractionAt: new Date(rows[0].getResultByName("updated_at")), + }); + + Services.obs.notifyObservers(null, "places-snapshot-added", url); + } + + /** + * Deletes a snapshot, creating a tombstone. + * + * @param {string} url + * The url of the snapshot to delete. + */ + async delete(url) { + let snapshot = this.#snapshots.get(url); + + if (snapshot) { + snapshot.removedAt = new Date(); + Services.obs.notifyObservers(null, "places-snapshot-deleted", url); + } + } + + /** + * Gets the details for a particular snapshot based on the url. + * + * @param {string} url + * The url of the snapshot to obtain. + * @param {boolean} [includeTombstones] + * Whether to include tombstones in the snapshots to obtain. + * @returns {?Snapshot} + */ + async get(url, includeTombstones = false) { + let snapshot = this.#snapshots.get(url); + if (!snapshot || (snapshot.removedAt && !includeTombstones)) { + return null; + } + + return snapshot; + } + + /** + * Queries the current snapshots in the database. + * + * @param {object} [options] + * @param {number} [options.limit] + * A numerical limit to the number of snapshots to retrieve, defaults to 100. + * @param {boolean} [options.includeTombstones] + * Whether to include tombstones in the snapshots to obtain. + * @returns {Snapshot[]} + * Returns snapshots in order of descending last interaction time. + */ + async query({ limit = 100, includeTombstones = false } = {}) { + let snapshots = Array.from(this.#snapshots.values()); + + if (!includeTombstones) { + return snapshots.filter(s => !s.removedAt).slice(0, limit); + } + return snapshots.slice(0, limit); + } +})(); diff --git a/browser/components/places/moz.build b/browser/components/places/moz.build index 794894e68c2c..872077e099e4 100644 --- a/browser/components/places/moz.build +++ b/browser/components/places/moz.build @@ -4,7 +4,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -XPCSHELL_TESTS_MANIFESTS += ["tests/unit/xpcshell.ini"] +XPCSHELL_TESTS_MANIFESTS += [ + "tests/unit/interactions/xpcshell.ini", + "tests/unit/xpcshell.ini", +] MOCHITEST_CHROME_MANIFESTS += ["tests/chrome/chrome.ini"] BROWSER_CHROME_MANIFESTS += [ "tests/browser/browser.ini", @@ -17,6 +20,7 @@ EXTRA_JS_MODULES += [ "Interactions.jsm", "InteractionsBlocklist.jsm", "PlacesUIUtils.jsm", + "Snapshots.jsm", ] FINAL_TARGET_FILES.actors += [ diff --git a/browser/components/places/tests/unit/interactions/head_interactions.js b/browser/components/places/tests/unit/interactions/head_interactions.js new file mode 100644 index 000000000000..3bb3d35f02c5 --- /dev/null +++ b/browser/components/places/tests/unit/interactions/head_interactions.js @@ -0,0 +1,127 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { XPCOMUtils } = ChromeUtils.import( + "resource://gre/modules/XPCOMUtils.jsm" +); + +XPCOMUtils.defineLazyModuleGetters(this, { + Interactions: "resource:///modules/Interactions.jsm", + PlacesTestUtils: "resource://testing-common/PlacesTestUtils.jsm", + PlacesUtils: "resource://gre/modules/PlacesUtils.jsm", + setTimeout: "resource://gre/modules/Timer.jsm", + Snapshots: "resource:///modules/Snapshots.jsm", + TestUtils: "resource://testing-common/TestUtils.jsm", +}); + +// Initialize profile. +var gProfD = do_get_profile(true); + +/** + * Adds a test interaction to the database. + * + * @param {InteractionInfo[]} interactions + */ +async function addInteractions(interactions) { + await PlacesTestUtils.addVisits(interactions.map(i => i.url)); + + for (let interaction of interactions) { + await Interactions.store.add({ + url: interaction.url, + documentType: + interaction.documentType ?? Interactions.DOCUMENT_TYPE.GENERIC, + totalViewTime: interaction.totalViewTime ?? 0, + typingTime: interaction.typingTime ?? 0, + keypresses: interaction.keypresses ?? 0, + scrollingTime: interaction.scrollingTime ?? 0, + scrollingDistance: interaction.scrollingDistance ?? 0, + created_at: interaction.created_at || Date.now(), + updated_at: interaction.updated_at || Date.now(), + }); + } + await Interactions.store.flush(); +} + +/** + * Asserts that a date looks reasonably valid, i.e. created no earlier than + * 24 hours prior to the current date. + * + * @param {Date} date + * The date to check. + */ +function assertRecentDate(date) { + Assert.greater( + date.getTime(), + Date.now() - 1000 * 60 * 60 * 24, + "Should have a reasonable value for the date" + ); +} + +/** + * Asserts that an individual snapshot contains the expected values. + * + * @param {Snapshot} actual + * The snapshot to test. + * @param {Snapshot} expected + * The snapshot to test against. + */ +function assertSnapshot(actual, expected) { + Assert.equal(actual.url, expected.url, "Should have the expected URL"); + // Avoid falsey-types that we might get from the database. + Assert.strictEqual( + actual.userPersisted, + expected.userPersisted ?? false, + "Should have the expected user persisted value" + ); + Assert.strictEqual( + actual.documentType, + expected.documentType ?? Interactions.DOCUMENT_TYPE.GENERIC, + "Should have the expected document type" + ); + assertRecentDate(actual.createdAt); + assertRecentDate(actual.firstInteractionAt); + assertRecentDate(actual.lastInteractionAt); + if (expected.lastUpdated) { + Assert.greaterOrEqual( + actual.lastInteractionAt, + expected.lastUpdated, + "Should have a last interaction time greater than or equal to the expected last updated time" + ); + } + if (expected.removedAt) { + Assert.greaterOrEqual( + actual.removedAt.getTime(), + expected.removedAt.getTime(), + "Should have the removed at time greater than or equal to the expected removed at time" + ); + } else { + Assert.strictEqual( + actual.removedAt, + null, + "Should not have a removed at time" + ); + } +} + +/** + * Asserts that the snapshots in the database match the expected values. + * + * @param {Snapshot[]} expected + * The expected snapshots. + * @param {object} options + * @see Snapshots.query(). + */ +async function assertSnapshots(expected, options) { + let snapshots = await Snapshots.query(options); + + info(`Found ${snapshots.length} snapshots:\n ${JSON.stringify(snapshots)}`); + Assert.equal( + snapshots.length, + expected.length, + "Should have the expected number of snapshots" + ); + for (let i = 0; i < expected.length; i++) { + assertSnapshot(snapshots[i], expected[i]); + } +} diff --git a/browser/components/places/tests/unit/interactions/test_snapshots_basics.js b/browser/components/places/tests/unit/interactions/test_snapshots_basics.js new file mode 100644 index 000000000000..68123056f083 --- /dev/null +++ b/browser/components/places/tests/unit/interactions/test_snapshots_basics.js @@ -0,0 +1,77 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests for snapshot creation and removal. + */ + +const TEST_URL1 = "https://example.com/"; +const TEST_URL2 = "https://example.com/12345"; +const TOPIC_ADDED = "places-snapshot-added"; +const TOPIC_DELETED = "places-snapshot-deleted"; + +add_task(async function test_add_simple_snapshot() { + await addInteractions([ + { url: TEST_URL1, documentType: Interactions.DOCUMENT_TYPE.MEDIA }, + { url: TEST_URL2 }, + ]); + + let promise = TestUtils.topicObserved( + TOPIC_ADDED, + (subject, data) => !subject && data == TEST_URL1 + ); + await Snapshots.add({ url: TEST_URL1 }); + await promise; + + promise = TestUtils.topicObserved( + TOPIC_ADDED, + (subject, data) => !subject && data == TEST_URL2 + ); + await Snapshots.add({ url: TEST_URL2, userPersisted: true }); + await promise; + + await assertSnapshots([ + { url: TEST_URL1, documentType: Interactions.DOCUMENT_TYPE.MEDIA }, + { url: TEST_URL2, userPersisted: true }, + ]); + + let snapshot = await Snapshots.get(TEST_URL2); + assertSnapshot(snapshot, { url: TEST_URL2, userPersisted: true }); +}); + +add_task(async function test_add_snapshot_without_interaction() { + await Assert.rejects( + Snapshots.add({ url: "https://invalid.com/" }), + /Could not find existing interactions/, + "Should reject if an interaction is missing" + ); +}); + +add_task(async function test_get_snapshot_not_found() { + let snapshot = await Snapshots.get("https://invalid.com/"); + Assert.ok( + !snapshot, + "Should not have found a snapshot for a URL not in the database" + ); +}); + +add_task(async function test_remove_snapshot() { + let removedAt = new Date(); + + let promise = TestUtils.topicObserved( + TOPIC_DELETED, + (subject, data) => !subject && data == TEST_URL1 + ); + await Snapshots.delete(TEST_URL1); + await promise; + + let snapshot = await Snapshots.get(TEST_URL1); + Assert.ok(!snapshot, "Tombstone snapshots should not be returned by default"); + + snapshot = await Snapshots.get(TEST_URL1, true); + assertSnapshot(snapshot, { + url: TEST_URL1, + documentType: Interactions.DOCUMENT_TYPE.MEDIA, + removedAt, + }); +}); diff --git a/browser/components/places/tests/unit/interactions/test_snapshots_queries.js b/browser/components/places/tests/unit/interactions/test_snapshots_queries.js new file mode 100644 index 000000000000..76c8ae86cafd --- /dev/null +++ b/browser/components/places/tests/unit/interactions/test_snapshots_queries.js @@ -0,0 +1,78 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * Tests for queries on snapshots. + */ + +add_task(async function setup() { + for (let i = 0; i < 10; i++) { + let url = `https://example.com/${i}/`; + await addInteractions([{ url }]); + + await Snapshots.add({ url: `https://example.com/${i}/` }); + + // Give a little time between updates to help the ordering. + /* eslint-disable-next-line mozilla/no-arbitrary-setTimeout */ + await new Promise(r => setTimeout(r, 20)); + } +}); + +add_task(async function test_query_returns_correct_order() { + await assertSnapshots([ + { url: "https://example.com/0/" }, + { url: "https://example.com/1/" }, + { url: "https://example.com/2/" }, + { url: "https://example.com/3/" }, + { url: "https://example.com/4/" }, + { url: "https://example.com/5/" }, + { url: "https://example.com/6/" }, + { url: "https://example.com/7/" }, + { url: "https://example.com/8/" }, + { url: "https://example.com/9/" }, + ]); + + // TODO: Update an interaction, and check the new order is returned corectly. +}); + +add_task(async function test_query_limit() { + await assertSnapshots( + [{ url: `https://example.com/0/` }, { url: `https://example.com/1/` }], + { limit: 2 } + ); +}); + +add_task(async function test_query_handles_tombstones() { + let removedAt = new Date(); + await Snapshots.delete("https://example.com/3/"); + await Snapshots.delete("https://example.com/6/"); + + info("Deleted snapshots should not appear in the query"); + await assertSnapshots([ + { url: "https://example.com/0/" }, + { url: "https://example.com/1/" }, + { url: "https://example.com/2/" }, + { url: "https://example.com/4/" }, + { url: "https://example.com/5/" }, + { url: "https://example.com/7/" }, + { url: "https://example.com/8/" }, + { url: "https://example.com/9/" }, + ]); + + info("Deleted snapshots should appear if tombstones are requested"); + await assertSnapshots( + [ + { url: "https://example.com/0/" }, + { url: "https://example.com/1/" }, + { url: "https://example.com/2/" }, + { url: "https://example.com/3/", removedAt }, + { url: "https://example.com/4/" }, + { url: "https://example.com/5/" }, + { url: "https://example.com/6/", removedAt }, + { url: "https://example.com/7/" }, + { url: "https://example.com/8/" }, + { url: "https://example.com/9/" }, + ], + { includeTombstones: true } + ); +}); diff --git a/browser/components/places/tests/unit/interactions/xpcshell.ini b/browser/components/places/tests/unit/interactions/xpcshell.ini new file mode 100644 index 000000000000..ccc87d54ca90 --- /dev/null +++ b/browser/components/places/tests/unit/interactions/xpcshell.ini @@ -0,0 +1,7 @@ +[DEFAULT] +head = head_interactions.js +firefox-appdir = browser +skip-if = toolkit == 'android' + +[test_snapshots_basics.js] +[test_snapshots_queries.js]