Bug 1713385 - Add typing metrics to history metadata r=Standard8

Collect typing metrics (time spent typing, number of keys pressed) to improve history metadata.

Differential Revision: https://phabricator.services.mozilla.com/D116719
This commit is contained in:
Andrew Creskey 2021-06-14 19:56:06 +00:00
Родитель e72a7b286d
Коммит de66360ca3
4 изменённых файлов: 716 добавлений и 22 удалений

Просмотреть файл

@ -16,6 +16,18 @@ XPCOMUtils.defineLazyModuleGetters(this, {
Services: "resource://gre/modules/Services.jsm",
});
XPCOMUtils.defineLazyGetter(this, "logConsole", function() {
return console.createInstance({
prefix: "InteractionsManager",
maxLogLevel: Services.prefs.getBoolPref(
"browser.places.interactions.log",
false
)
? "Debug"
: "Warn",
});
});
XPCOMUtils.defineLazyServiceGetters(this, {
idleService: ["@mozilla.org/widget/useridleservice;1", "nsIUserIdleService"],
});
@ -29,6 +41,200 @@ XPCOMUtils.defineLazyPreferenceGetter(
const DOMWINDOW_OPENED_TOPIC = "domwindowopened";
/**
* The TypingInteraction object measures time spent typing on the current interaction.
* This is consists of the current typing metrics as well as accumulated typing metrics.
*/
class TypingInteraction {
/**
* The time, in milliseconds, at which the user started typing in the current typing sequence.
*
* @type {number}
*/
#typingStartTime = null;
/**
* The time, in milliseconds, at which the last keypress was made in the current typing sequence.
*
* @type {number}
*/
#typingEndTime = null;
/**
* The number of keypresses made in the current typing sequence.
*
* @type {number}
*/
#keypresses = 0;
/**
* Time, in milliseconds, after the last keypress after which we consider the current typing sequence to have ended.
*
* @type {number}
*/
static _TYPING_TIMEOUT = 3000;
/**
* The number of keypresses accumulated in this interaction.
* Each typing sequence will contribute to accumulated keypresses.
*
* @type {number}
*/
#accumulatedKeypresses = 0;
/**
* Typing time, in milliseconds, accumulated in this interaction.
* Each typing sequence will contribute to accumulated typing time.
*
* @type {number}
*/
#accumulatedTypingTime = 0;
/**
* Adds a system event listener to the given window.
*
* @param {DOMWindow} win
* The window to register in.
*/
registerWindow(win) {
Services.els.addSystemEventListener(
win.document,
"keyup",
this,
/* useCapture */ false
);
}
/**
* Removes system event listener from the given window.
*
* @param {DOMWindow} win
* The window to removed listeners from
*/
unregisterWindow(win) {
Services.els.removeSystemEventListener(
win.document,
"keyup",
this,
/* useCapture */ false
);
}
/**
* Determines if the given key stroke is considered typing.
*
* @param {string} code
* @returns {boolean} whether the key code is considered typing
*
*/
#isTypingKey(code) {
if (["Space", "Comma", "Period", "Quote"].includes(code)) {
return true;
}
return (
code.startsWith("Key") ||
code.startsWith("Digit") ||
code.startsWith("Numpad")
);
}
/**
* Reset current typing metrics.
*/
#resetCurrentTypingMetrics() {
this.#keypresses = 0;
this.#typingStartTime = null;
this.#typingEndTime = null;
}
/**
* Reset all typing interaction metrics, included those accumulated.
*/
resetTypingInteraction() {
this.#resetCurrentTypingMetrics();
this.#accumulatedKeypresses = 0;
this.#accumulatedTypingTime = 0;
}
/**
* Returns an object with all current and accumulated typing metrics.
*
* @returns {object} with properties typingTime, keypresses
*/
getTypingInteraction() {
let typingInteraction = this.#getCurrentTypingMetrics();
typingInteraction.typingTime += this.#accumulatedTypingTime;
typingInteraction.keypresses += this.#accumulatedKeypresses;
return typingInteraction;
}
/**
* Returns an object with the current typing metrics.
*
* @returns {object} with properties typingTime, keypresses
*/
#getCurrentTypingMetrics() {
let typingInteraction = { typingTime: 0, keypresses: 0 };
// We don't consider a single keystroke to be typing, not least because it would have 0 typing
// time which would equate to infinite keystrokes per minute.
if (this.#keypresses > 1) {
let typingTime = this.#typingEndTime - this.#typingStartTime;
typingInteraction.typingTime = typingTime;
typingInteraction.keypresses = this.#keypresses;
}
return typingInteraction;
}
/**
* The user has stopped typing, accumulate the current metrics and reset the counters.
*
*/
#onTypingEnded() {
let typingInteraction = this.#getCurrentTypingMetrics();
this.#accumulatedTypingTime += typingInteraction.typingTime;
this.#accumulatedKeypresses += typingInteraction.keypresses;
this.#resetCurrentTypingMetrics();
}
/**
* Handles received window events to detect keypresses.
*
* @param {object} event The event details.
*/
handleEvent(event) {
switch (event.type) {
case "keyup":
if (
event.target.ownerGlobal.gBrowser?.selectedBrowser == event.target &&
this.#isTypingKey(event.code)
) {
const now = Cu.now();
// Detect typing end from previous keystroke
const lastKeyDelay = now - this.#typingEndTime;
if (
this.#keypresses > 0 &&
lastKeyDelay > TypingInteraction._TYPING_TIMEOUT
) {
this.#onTypingEnded();
}
this.#keypresses++;
if (!this.#typingStartTime) {
this.#typingStartTime = now;
}
this.#typingEndTime = now;
}
break;
}
}
}
/**
* @typedef {object} DocumentInfo
* DocumentInfo is used to pass document information from the child process
@ -46,6 +252,10 @@ const DOMWINDOW_OPENED_TOPIC = "domwindowopened";
* Time in milliseconds that the page has been actively viewed for.
* @property {string} url
* The url of the page that was interacted with.
* @property {number} typingTime
* Time in milliseconds that the user typed on the page
* @property {number} keypresses
* The number of keypresses made on the page
*/
/**
@ -64,6 +274,13 @@ class _Interactions {
*/
#interactions = new WeakMap();
/**
* This tracks and reports the typing interactions
*
* @type {TypingInteraction}
*/
#typingInteraction = new TypingInteraction();
/**
* Tracks the currently active window so that we can avoid recording
* interactions in non-active windows.
@ -98,17 +315,6 @@ class _Interactions {
return;
}
this.logConsole = console.createInstance({
prefix: "InteractionsManager",
maxLogLevel: Services.prefs.getBoolPref(
"browser.places.interactions.log",
false
)
? "Debug"
: "Warn",
});
this.logConsole.debug("init");
ChromeUtils.registerWindowActor("Interactions", {
parent: {
moduleURI: "resource:///actors/InteractionsParent.jsm",
@ -141,6 +347,18 @@ class _Interactions {
idleService.removeIdleObserver(this, pageViewIdleTime);
}
/**
* Resets any stored user or interaction state.
* Used by tests.
*/
reset() {
logConsole.debug("Reset");
this.#interactions = new WeakMap();
this.#userIsIdle = false;
this._pageViewStartTime = Cu.now();
this.#typingInteraction?.resetTypingInteraction();
}
/**
* Registers the start of a new interaction.
*
@ -155,10 +373,12 @@ class _Interactions {
this.registerEndOfInteraction(browser);
}
this.logConsole.debug("New interaction", docInfo);
logConsole.debug("New interaction", docInfo);
interaction = {
url: docInfo.url,
totalViewTime: 0,
typingTime: 0,
keypresses: 0,
};
this.#interactions.set(browser, interaction);
@ -186,7 +406,7 @@ class _Interactions {
if (!browser) {
return;
}
this.logConsole.debug("End of interaction");
logConsole.debug("End of interaction");
this.#updateInteraction(browser);
this.#interactions.delete(browser);
@ -205,7 +425,7 @@ class _Interactions {
!this.#activeWindow ||
(browser && browser.ownerGlobal != this.#activeWindow)
) {
this.logConsole.debug("No update due to no active window");
logConsole.debug("No update due to no active window");
return;
}
@ -214,7 +434,7 @@ class _Interactions {
// Sometimes an interaction may be signalled before idle is cleared, however
// worst case we'd only loose approx 2 seconds of interaction detail.
if (this.#userIsIdle) {
this.logConsole.debug("No update due to user is idle");
logConsole.debug("No update due to user is idle");
return;
}
@ -224,12 +444,18 @@ class _Interactions {
let interaction = this.#interactions.get(browser);
if (!interaction) {
this.logConsole.debug("No interaction to update");
logConsole.debug("No interaction to update");
return;
}
interaction.totalViewTime += Cu.now() - this._pageViewStartTime;
this._pageViewStartTime = Cu.now();
const typingInteraction = this.#typingInteraction.getTypingInteraction();
interaction.typingTime += typingInteraction.typingTime;
interaction.keypresses += typingInteraction.keypresses;
this.#typingInteraction.resetTypingInteraction();
this._updateDatabase(interaction);
}
@ -239,7 +465,7 @@ class _Interactions {
* @param {DOMWindow} win
*/
#onActivateWindow(win) {
this.logConsole.debug("Activate window");
logConsole.debug("Activate window");
if (PrivateBrowsingUtils.isWindowPrivate(win)) {
return;
@ -255,12 +481,32 @@ class _Interactions {
* @param {DOMWindow} win
*/
#onDeactivateWindow(win) {
this.logConsole.debug("Deactivate window");
logConsole.debug("Deactivate window");
this.#updateInteraction();
this.#activeWindow = undefined;
}
/**
* Get the timeout duration used for end-of-typing detection.
*
* @returns {number} timeout, in in milliseconds
*/
_getTypingTimeout() {
return TypingInteraction._TYPING_TIMEOUT;
}
/**
* Set the timeout duration used for end-of-typing detection.
* Provided to for test usage.
*
* @param {number} timeout
* Delay in milliseconds after a keypress after which end of typing is determined
*/
_setTypingTimeout(timeout) {
TypingInteraction._TYPING_TIMEOUT = timeout;
}
/**
* Handles the TabSelect notification. Updates the current interaction and
* then switches it to the interaction for the new tab. The new interaction
@ -270,7 +516,7 @@ class _Interactions {
* The instance of the browser that the user switched away from.
*/
#onTabSelect(previousBrowser) {
this.logConsole.debug("Tab switch notified");
logConsole.debug("Tab switch notified");
this.#updateInteraction(previousBrowser);
this._pageViewStartTime = Cu.now();
@ -311,14 +557,14 @@ class _Interactions {
this.#onWindowOpen(subject);
break;
case "idle":
this.logConsole.debug("idle");
logConsole.debug("idle");
// We save the state of the current interaction when we are notified
// that the user is idle.
this.#updateInteraction();
this.#userIsIdle = true;
break;
case "active":
this.logConsole.debug("active");
logConsole.debug("active");
this.#userIsIdle = false;
this._pageViewStartTime = Cu.now();
break;
@ -339,6 +585,7 @@ class _Interactions {
win.addEventListener("TabSelect", this, true);
win.addEventListener("deactivate", this, true);
win.addEventListener("activate", this, true);
this.#typingInteraction.registerWindow(win);
}
/**
@ -351,6 +598,7 @@ class _Interactions {
win.removeEventListener("TabSelect", this, true);
win.removeEventListener("deactivate", this, true);
win.removeEventListener("activate", this, true);
this.#typingInteraction.unregisterWindow(win);
}
/**
@ -389,7 +637,7 @@ class _Interactions {
* The document information to write.
*/
async _updateDatabase(interactionInfo) {
this.logConsole.debug("Would update database: ", interactionInfo);
logConsole.debug("Would update database: ", interactionInfo);
}
}

Просмотреть файл

@ -8,5 +8,7 @@ prefs =
browser.places.interactions.log=true
support-files =
head.js
../keyword_form.html
[browser_interactions_view_time.js]
[browser_interactions_typing.js]

Просмотреть файл

@ -0,0 +1,443 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
* Tests reporting of typing interactions.
*/
"use strict";
const TEST_URL =
"https://example.com/browser/browser/components/places/tests/browser/keyword_form.html";
const TEST_URL2 = "https://example.com/browser";
const TEST_URL3 =
"https://example.com/browser/browser/base/content/test/contextMenu/subtst_contextmenu_input.html";
const sentence = "The quick brown fox jumps over the lazy dog.";
const sentenceFragments = [
"The quick",
" brown fox",
" jumps over the lazy dog.",
];
const longSentence =
"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ut purus a libero cursus scelerisque. In hac habitasse platea dictumst. Quisque posuere ante sed consequat volutpat.";
// Reduce the end-of-typing threshold to improve test speed
const TYPING_INTERACTION_TIMEOUT = 50;
// A delay that's long enough to trigger the end of typing timeout
const POST_TYPING_DELAY = TYPING_INTERACTION_TIMEOUT + 50;
const defaultTypingTimeout = Interactions._getTypingTimeout();
function reduceTypingTimeoutForTests(timeout) {
Interactions._setTypingTimeout(timeout);
}
function resetTypingTimeout() {
Interactions._setTypingTimeout(defaultTypingTimeout);
}
add_task(async function setup() {
sinon.spy(Interactions, "_updateDatabase");
Interactions.reset();
disableIdleService();
reduceTypingTimeoutForTests(TYPING_INTERACTION_TIMEOUT);
registerCleanupFunction(() => {
sinon.restore();
resetTypingTimeout();
});
});
async function assertDatabaseValues(expected) {
await BrowserTestUtils.waitForCondition(
() => Interactions._updateDatabase.callCount == expected.length,
`Should have saved to the database Interactions._updateDatabase.callCount: ${Interactions._updateDatabase.callCount},
expected.length: ${expected.length}`
);
let args = Interactions._updateDatabase.args;
for (let i = 0; i < expected.length; i++) {
let actual = args[i][0];
Assert.equal(
actual.url,
expected[i].url,
"Should have saved the interaction into the database"
);
if (expected[i].keypresses) {
Assert.equal(
actual.keypresses,
expected[i].keypresses,
"Should have saved the keypresses into the database"
);
}
if (expected[i].exactTypingTime) {
Assert.equal(
actual.typingTime,
expected[i].exactTypingTime,
"Should have stored the exact typing time."
);
} else if (expected[i].typingTimeIsGreaterThan) {
Assert.greater(
actual.typingTime,
expected[i].typingTimeIsGreaterThan,
"Should have stored at least this amount of typing time."
);
} else if (expected[i].typingTimeIsLessThan) {
Assert.less(
actual.typingTime,
expected[i].typingTimeIsLessThan,
"Should have stored less than this amount of typing time."
);
}
}
}
async function sendTextToInput(browser, text) {
await SpecialPowers.spawn(browser, [], function() {
const input = content.document.querySelector(
"#form1 > input[name='search']"
);
input.focus();
});
await EventUtils.sendString(text);
}
add_task(async function test_load_and_navigate_away_no_keypresses() {
sinon.reset();
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {
BrowserTestUtils.loadURI(browser, TEST_URL2);
await BrowserTestUtils.browserLoaded(browser, false, TEST_URL2);
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: 0,
exactTypingTime: 0,
},
]);
BrowserTestUtils.loadURI(browser, "about:blank");
await BrowserTestUtils.browserLoaded(browser, false, "about:blank");
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: 0,
exactTypingTime: 0,
},
{
url: TEST_URL2,
keypresses: 0,
exactTypingTime: 0,
},
]);
});
});
add_task(async function test_load_type_and_navigate_away() {
sinon.reset();
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {
await sendTextToInput(browser, sentence);
BrowserTestUtils.loadURI(browser, TEST_URL2);
await BrowserTestUtils.browserLoaded(browser, false, TEST_URL2);
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentence.length,
typingTimeIsGreaterThan: 0,
},
]);
BrowserTestUtils.loadURI(browser, "about:blank");
await BrowserTestUtils.browserLoaded(browser, false, "about:blank");
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentence.length,
typingTimeIsGreaterThan: 0,
},
{
url: TEST_URL2,
keypresses: 0,
exactTypingTime: 0,
},
]);
});
});
add_task(async function test_no_typing_close_tab() {
sinon.reset();
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {});
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: 0,
exactTypingTime: 0,
},
]);
});
add_task(async function test_typing_close_tab() {
sinon.reset();
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {
await sendTextToInput(browser, sentence);
});
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentence.length,
typingTimeIsGreaterThan: 0,
},
]);
});
add_task(async function test_single_key_typing_and_delay() {
sinon.reset();
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {
// Single keystrokes with a delay between each, are not considered typing
const text = ["T", "h", "e"];
for (let i = 0; i < text.length; i++) {
await sendTextToInput(browser, text[i]);
// We do need to wait here because typing is defined as a series of keystrokes followed by a delay.
// eslint-disable-next-line mozilla/no-arbitrary-setTimeout
await new Promise(r => setTimeout(r, POST_TYPING_DELAY));
}
});
// Since we typed single keys with delays between each, there should be no typing added to the database
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: 0,
exactTypingTime: 0,
},
]);
});
add_task(async function test_double_key_typing_and_delay() {
sinon.reset();
const text = ["Ab", "cd", "ef"];
const testStartTime = Cu.now();
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {
for (let i = 0; i < text.length; i++) {
await sendTextToInput(browser, text[i]);
// We do need to wait here because typing is defined as a series of keystrokes followed by a delay.
// eslint-disable-next-line mozilla/no-arbitrary-setTimeout
await new Promise(r => setTimeout(r, POST_TYPING_DELAY));
}
});
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: text.reduce(
(accumulator, current) => accumulator + current.length,
0
),
typingTimeIsGreaterThan: 0,
typingTimeIsLessThan: Cu.now() - testStartTime,
},
]);
});
add_task(async function test_typing_and_delay() {
sinon.reset();
const testStartTime = Cu.now();
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {
for (let i = 0; i < sentenceFragments.length; i++) {
await sendTextToInput(browser, sentenceFragments[i]);
// We do need to wait here because typing is defined as a series of keystrokes followed by a delay.
// eslint-disable-next-line mozilla/no-arbitrary-setTimeout
await new Promise(r => setTimeout(r, POST_TYPING_DELAY));
}
});
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentenceFragments.reduce(
(accumulator, current) => accumulator + current.length,
0
),
typingTimeIsGreaterThan: 0,
typingTimeIsLessThan: Cu.now() - testStartTime,
},
]);
});
add_task(async function test_typing_and_reload() {
sinon.reset();
const testStartTime = Cu.now();
await BrowserTestUtils.withNewTab(TEST_URL, async browser => {
await sendTextToInput(browser, sentenceFragments[0]);
info("reload");
browser.reload();
await BrowserTestUtils.browserLoaded(browser);
// First typing should have been recorded
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentenceFragments[0].length,
typingTimeIsGreaterThan: 0,
},
]);
await sendTextToInput(browser, sentenceFragments[1]);
info("reload");
browser.reload();
await BrowserTestUtils.browserLoaded(browser);
// Second typing should have been recorded
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentenceFragments[0].length,
typingTimeIsGreaterThan: 0,
typingTimeIsLessThan: Cu.now() - testStartTime,
},
{
url: TEST_URL,
keypresses: sentenceFragments[1].length,
typingTimeIsGreaterThan: 0,
typingTimeIsLessThan: Cu.now() - testStartTime,
},
]);
});
});
add_task(async function test_switch_tabs_no_typing() {
sinon.reset();
let tab1 = await BrowserTestUtils.openNewForegroundTab({
gBrowser,
url: TEST_URL,
});
let tab2 = BrowserTestUtils.addTab(gBrowser, TEST_URL2);
await BrowserTestUtils.browserLoaded(tab2.linkedBrowser, false, TEST_URL2);
info("Switch to second tab");
gBrowser.selectedTab = tab2;
// Only the interaction of the first tab should be recorded so far, and with no typing
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: 0,
exactTypingTime: 0,
},
]);
BrowserTestUtils.removeTab(tab1);
BrowserTestUtils.removeTab(tab2);
});
add_task(async function test_typing_switch_tabs() {
sinon.reset();
let tab1 = await BrowserTestUtils.openNewForegroundTab({
gBrowser,
url: TEST_URL,
});
await sendTextToInput(tab1.linkedBrowser, sentence);
let tab2 = BrowserTestUtils.addTab(gBrowser, TEST_URL3);
await BrowserTestUtils.browserLoaded(tab2.linkedBrowser, false, TEST_URL3);
info("Switch to second tab");
await BrowserTestUtils.switchTab(gBrowser, tab2);
// Only the interaction of the first tab should be recorded so far
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentence.length,
typingTimeIsGreaterThan: 0,
},
]);
const tab1TyingTime = Interactions._updateDatabase.args[0][0].typingTime;
info("Switch back to first tab");
await BrowserTestUtils.switchTab(gBrowser, tab1);
// The interaction of the second tab should now be recorded (no typing)
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentence.length,
exactTypingTime: tab1TyingTime,
},
{
url: TEST_URL3,
keypresses: 0,
typingTimeIsGreaterThan: 0,
},
]);
info("Switch back to the second tab");
await BrowserTestUtils.switchTab(gBrowser, tab2);
// Typing into the second tab
await SpecialPowers.spawn(tab2.linkedBrowser, [], function() {
const input = content.document.getElementById("input_text");
input.focus();
});
await EventUtils.sendString(longSentence);
info("Switch back to first tab");
await BrowserTestUtils.switchTab(gBrowser, tab1);
// The interaction of the second tab should now also be recorded (with typing)
await assertDatabaseValues([
{
url: TEST_URL,
keypresses: sentence.length,
exactTypingTime: tab1TyingTime,
},
{
url: TEST_URL3,
keypresses: 0,
typingTimeIsGreaterThan: 0,
},
{
url: TEST_URL,
keypresses: 0,
typingTimeIsGreaterThan: 0,
},
{
url: TEST_URL3,
keypresses: longSentence.length,
typingTimeIsGreaterThan: 0,
},
]);
BrowserTestUtils.removeTab(tab1);
BrowserTestUtils.removeTab(tab2);
});

Просмотреть файл

@ -13,6 +13,7 @@ const TEST_URL4 = "https://example.com/browser/browser/components";
add_task(async function setup() {
sinon.spy(Interactions, "_updateDatabase");
Interactions.reset();
disableIdleService();
registerCleanupFunction(() => {