Bug 1505411 - Add basic monitoring for partner search pages with ads and clicks. Depends on D11188 r=adw,Felipe,chutten

Differential Revision: https://phabricator.services.mozilla.com/D11656

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Mark Banner 2018-12-05 20:33:01 +00:00
Родитель 3075d95605
Коммит 47d25a826d
12 изменённых файлов: 818 добавлений и 47 удалений

Просмотреть файл

@ -0,0 +1,165 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var EXPORTED_SYMBOLS = ["SearchTelemetryChild"];
ChromeUtils.import("resource://gre/modules/ActorChild.jsm");
ChromeUtils.import("resource://gre/modules/Services.jsm");
const SHARED_DATA_KEY = "SearchTelemetry:ProviderInfo";
/**
* SearchProviders looks after keeping track of the search provider information
* received from the main process.
*
* It is separate to SearchTelemetryChild so that it is not constructed for each
* tab, but once per process.
*/
class SearchProviders {
constructor() {
this._searchProviderInfo = null;
Services.cpmm.sharedData.addEventListener("change", this);
}
/**
* Gets the search provider information for any provider with advert information.
* If there is nothing in the cache, it will obtain it from shared data.
*
* @returns {object} Returns the search provider information. @see SearchTelemetry.jsm
*/
get info() {
if (this._searchProviderInfo) {
return this._searchProviderInfo;
}
this._searchProviderInfo = Services.cpmm.sharedData.get(SHARED_DATA_KEY);
if (!this._searchProviderInfo) {
return null;
}
// Filter-out non-ad providers so that we're not trying to match against
// those unnecessarily.
for (let [providerName, info] of Object.entries(this._searchProviderInfo)) {
if (!("extraAdServersRegexps" in info)) {
delete this._searchProviderInfo[providerName];
}
}
return this._searchProviderInfo;
}
/**
* Handles events received from sharedData notifications.
*
* @param {object} event The event details.
*/
handleEvent(event) {
switch (event.type) {
case "change": {
if (event.changedKeys.includes(SHARED_DATA_KEY)) {
// Just null out the provider information for now, we'll fetch it next
// time we need it.
this._searchProviderInfo = null;
}
break;
}
}
}
}
const searchProviders = new SearchProviders();
/**
* SearchTelemetryChild monitors for pages that are partner searches, and
* looks through them to find links which looks like adverts and sends back
* a notification to SearchTelemetry for possible telemetry reporting.
*
* Only the partner details and the fact that at least one ad was found on the
* page are returned to SearchTelemetry. If no ads are found, no notification is
* given.
*/
class SearchTelemetryChild extends ActorChild {
/**
* Determines if there is a provider that matches the supplied URL and returns
* the information associated with that provider.
*
* @param {string} url The url to check
* @returns {array|null} Returns null if there's no match, otherwise an array
* of provider name and the provider information.
*/
_getProviderInfoForUrl(url) {
return Object.entries(searchProviders.info || []).find(
([_, info]) => info.regexp.test(url)
);
}
/**
* Checks to see if the page is a partner and has an ad link within it. If so,
* it will notify SearchTelemetry.
*
* @param {object} doc The document object to check.
*/
_checkForAdLink(doc) {
let providerInfo = this._getProviderInfoForUrl(doc.documentURI);
if (!providerInfo) {
return;
}
let regexps = providerInfo[1].extraAdServersRegexps;
let anchors = doc.getElementsByTagName("a");
let hasAds = false;
for (let anchor of anchors) {
if (!anchor.href) {
continue;
}
for (let regexp of regexps) {
if (regexp.test(anchor.href)) {
hasAds = true;
break;
}
}
if (hasAds) {
break;
}
}
if (hasAds) {
this.sendAsyncMessage("SearchTelemetry:PageInfo", {
hasAds: true,
url: doc.documentURI,
});
}
}
/**
* Handles events received from the actor child notifications.
*
* @param {object} event The event details.
*/
handleEvent(event) {
// We are only interested in the top-level frame.
if (event.target.ownerGlobal != this.content) {
return;
}
switch (event.type) {
case "pageshow": {
// If a page is loaded from the bfcache, we won't get a "DOMContentLoaded"
// event, so we need to rely on "pageshow" in this case. Note: we do this
// so that we remain consistent with the *.in-content:sap* count for the
// SEARCH_COUNTS histogram.
if (event.persisted) {
this._checkForAdLink(this.content.document);
}
break;
}
case "DOMContentLoaded": {
this._checkForAdLink(this.content.document);
break;
}
}
}
}

Просмотреть файл

@ -43,6 +43,7 @@ FINAL_TARGET_FILES.actors += [
'PageMetadataChild.jsm',
'PageStyleChild.jsm',
'PluginChild.jsm',
'SearchTelemetryChild.jsm',
'URIFixupChild.jsm',
'WebRTCChild.jsm',
]

Просмотреть файл

@ -53,6 +53,7 @@ const whitelist = {
"resource:///modules/ContentMetaHandler.jsm",
"resource:///actors/LinkHandlerChild.jsm",
"resource:///actors/PageStyleChild.jsm",
"resource:///actors/SearchTelemetryChild.jsm",
"resource://gre/modules/ActorChild.jsm",
"resource://gre/modules/ActorManagerChild.jsm",
"resource://gre/modules/E10SUtils.jsm",

Просмотреть файл

@ -252,6 +252,16 @@ let ACTORS = {
},
},
SearchTelemetry: {
child: {
module: "resource:///actors/SearchTelemetryChild.jsm",
events: {
DOMContentLoaded: {},
"pageshow": {mozSystemGroup: true},
},
},
},
ShieldFrame: {
child: {
module: "resource://normandy-content/ShieldFrameChild.jsm",
@ -426,6 +436,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
SafeBrowsing: "resource://gre/modules/SafeBrowsing.jsm",
Sanitizer: "resource:///modules/Sanitizer.jsm",
SaveToPocket: "chrome://pocket/content/SaveToPocket.jsm",
SearchTelemetry: "resource:///modules/SearchTelemetry.jsm",
SessionStartup: "resource:///modules/sessionstore/SessionStartup.jsm",
SessionStore: "resource:///modules/sessionstore/SessionStore.jsm",
ShellService: "resource:///modules/ShellService.jsm",
@ -1444,6 +1455,7 @@ BrowserGlue.prototype = {
}
BrowserUsageTelemetry.uninit();
SearchTelemetry.uninit();
// Only uninit PingCentre if the getter has initialized it
if (Object.prototype.hasOwnProperty.call(this, "pingCentre")) {
this.pingCentre.uninit();
@ -1511,6 +1523,7 @@ BrowserGlue.prototype = {
}
BrowserUsageTelemetry.init();
SearchTelemetry.init();
// Show update notification, if needed.
if (Services.prefs.prefHasUserValue("app.update.postupdate"))

Просмотреть файл

@ -12,9 +12,31 @@ XPCOMUtils.defineLazyModuleGetters(this, {
Services: "resource://gre/modules/Services.jsm",
});
// The various histograms and scalars that we report to.
const SEARCH_COUNTS_HISTOGRAM_KEY = "SEARCH_COUNTS";
const SEARCH_WITH_ADS_SCALAR = "browser.search.with_ads";
const SEARCH_AD_CLICKS_SCALAR = "browser.search.ad_clicks";
// Used to identify various parameters (query, code, etc.) in search URLS.
/**
* Used to identify various parameters used with partner search providers. This
* consists of the following structure:
* - {<string>} name
* Details for a particular provider with the string name.
* - {regexp} <string>.regexp
* The regular expression used to match the url for the search providers main page.
* - {string} <string>.queryParam
* The query parameter name that indicates a search has been made.
* - {string} [<string>.codeParam]
* The query parameter name that indicates a search provider's code.
* - {array} [<string>.codePrefixes]
* An array of the possible string prefixes for a codeParam, indicating a
* partner search.
* - {array} [<string>.followOnParams]
* An array of parameters name that indicates this is a follow-on search.
* - {array} [<string>.extraAdServersRegexps]
* An array of regular expressions used to determine if a link on a search
* page mightbe an advert.
*/
const SEARCH_PROVIDER_INFO = {
"google": {
"regexp": /^https:\/\/www\.google\.(?:.+)\/search/,
@ -22,6 +44,7 @@ const SEARCH_PROVIDER_INFO = {
"codeParam": "client",
"codePrefixes": ["firefox"],
"followonParams": ["oq", "ved", "ei"],
"extraAdServersRegexps": [/^https:\/\/www\.googleadservices\.com\/(?:pagead\/)?aclk/],
},
"duckduckgo": {
"regexp": /^https:\/\/duckduckgo\.com\//,
@ -52,11 +75,81 @@ const BROWSER_SEARCH_PREF = "browser.search.";
XPCOMUtils.defineLazyPreferenceGetter(this, "loggingEnabled", BROWSER_SEARCH_PREF + "log", false);
/**
* TelemetryHandler is the main class handling search telemetry. It primarily
* deals with tracking of what pages are loaded into tabs.
*
* It handles the *in-content:sap* keys of the SEARCH_COUNTS histogram.
*/
class TelemetryHandler {
constructor() {
this._browserInfoByUrl = new Map();
this._initialized = false;
this.__searchProviderInfo = null;
this._contentHandler = new ContentHandler({
browserInfoByUrl: this._browserInfoByUrl,
getProviderInfoForUrl: this._getProviderInfoForUrl.bind(this),
});
}
/**
* Initializes the TelemetryHandler and its ContentHandler. It will add
* appropriate listeners to the window so that window opening and closing
* can be tracked.
*/
init() {
if (this._initialized) {
return;
}
this._contentHandler.init();
for (let win of Services.wm.getEnumerator("navigator:browser")) {
this._registerWindow(win);
}
Services.wm.addListener(this);
this._initialized = true;
}
/**
* Uninitializes the TelemetryHandler and its ContentHandler.
*/
uninit() {
if (!this._initialized) {
return;
}
this._contentHandler.uninit();
for (let win of Services.wm.getEnumerator("navigator:browser")) {
this._unregisterWindow(win);
}
Services.wm.removeListener(this);
this._initialized = false;
}
/**
* Handles the TabClose event received from the listeners.
*
* @param {object} event
*/
handleEvent(event) {
if (event.type != "TabClose") {
Cu.reportError(`Received unexpected event type ${event.type}`);
return;
}
this.stopTrackingBrowser(event.target.linkedBrowser);
}
/**
* Test-only function, used to override the provider information, so that
* unit tests can set it to easy to test values.
*
* @param {object} infoByProvider @see SEARCH_PROVIDER_INFO for type information.
*/
overrideSearchTelemetryForTests(infoByProvider) {
if (infoByProvider) {
for (let info of Object.values(infoByProvider)) {
@ -66,19 +159,162 @@ class TelemetryHandler {
} else {
this.__searchProviderInfo = SEARCH_PROVIDER_INFO;
}
this._contentHandler.overrideSearchTelemetryForTests(this.__searchProviderInfo);
}
recordSearchURLTelemetry(url) {
let entry = Object.entries(this._searchProviderInfo).find(
([_, info]) => info.regexp.test(url)
);
if (!entry) {
/**
* This may start tracking a tab based on the URL. If the URL matches a search
* partner, and it has a code, then we'll start tracking it. This will aid
* determining if it is a page we should be tracking for adverts.
*
* @param {object} browser The browser associated with the page.
* @param {string} url The url that was loaded in the browser.
*/
updateTrackingStatus(browser, url) {
let info = this._checkURLForSerpMatch(url);
if (!info) {
this.stopTrackingBrowser(browser);
return;
}
let [provider, searchProviderInfo] = entry;
this._reportSerpPage(info, url);
// If we have a code, then we also track this for potential ad clicks.
if (info.code) {
let item = this._browserInfoByUrl.get(url);
if (item) {
item.browsers.add(browser);
} else {
this._browserInfoByUrl.set(url, {
browser: new WeakSet([browser]),
info,
});
}
}
}
/**
* Stops tracking of a tab, for example the tab has loaded a different URL.
*
* @param {object} browser The browser associated with the tab to stop being
* tracked.
*/
stopTrackingBrowser(browser) {
for (let [url, item] of this._browserInfoByUrl) {
item.browser.delete(browser);
if (!item.browser.length) {
this._browserInfoByUrl.delete(url);
}
}
}
// nsIWindowMediatorListener
/**
* This is called when a new window is opened, and handles registration of
* that window if it is a browser window.
*
* @param {nsIXULWindow} xulWin The xul window that was opened.
*/
onOpenWindow(xulWin) {
let win = xulWin.docShell.domWindow;
win.addEventListener("load", () => {
if (win.document.documentElement.getAttribute("windowtype") != "navigator:browser") {
return;
}
this._registerWindow(win);
}, {once: true});
}
/**
* Listener that is called when a window is closed, and handles deregistration of
* that window if it is a browser window.
*
* @param {nsIXULWindow} xulWin The xul window that was closed.
*/
onCloseWindow(xulWin) {
let win = xulWin.docShell.domWindow;
if (win.document.documentElement.getAttribute("windowtype") != "navigator:browser") {
return;
}
this._unregisterWindow(win);
}
/**
* Adds event listeners for the window and registers it with the content handler.
*
* @param {object} win The window to register.
*/
_registerWindow(win) {
this._contentHandler.registerWindow(win);
win.gBrowser.tabContainer.addEventListener("TabClose", this);
}
/**
* Removes event listeners for the window and unregisters it with the content
* handler.
*
* @param {object} win The window to unregister.
*/
_unregisterWindow(win) {
for (let tab of win.gBrowser.tabs) {
this.stopTrackingBrowser(tab);
}
this._contentHandler.unregisterWindow(win);
win.gBrowser.tabContainer.removeEventListener("TabClose", this);
}
/**
* Searches for provider information for a given url.
*
* @param {string} url The url to match for a provider.
* @param {boolean} useOnlyExtraAdServers If true, this will use the extra
* ad server regexp to match instead of the main regexp.
* @returns {array|null} Returns an array of provider name and the provider information.
*/
_getProviderInfoForUrl(url, useOnlyExtraAdServers = false) {
if (useOnlyExtraAdServers) {
return Object.entries(this._searchProviderInfo).find(
([_, info]) => {
if (info.extraAdServersRegexps) {
for (let regexp of info.extraAdServersRegexps) {
if (regexp.test(url)) {
return true;
}
}
}
return false;
}
);
}
return Object.entries(this._searchProviderInfo).find(
([_, info]) => info.regexp.test(url)
);
}
/**
* Checks to see if a url is a search partner location, and determines the
* provider and codes used.
*
* @param {string} url The url to match.
* @returns {null|object} Returns null if there is no match found. Otherwise,
* returns an object of strings for provider, code and type.
*/
_checkURLForSerpMatch(url) {
let info = this._getProviderInfoForUrl(url);
if (!info) {
return null;
}
let [provider, searchProviderInfo] = info;
let queries = new URLSearchParams(url.split("#")[0].split("?")[1]);
if (!queries.get(searchProviderInfo.queryParam)) {
return;
return null;
}
// Default to organic to simplify things.
// We override type in the sap cases.
@ -114,13 +350,29 @@ class TelemetryHandler {
}
}
}
let payload = `${provider}.in-content:${type}:${code || "none"}`;
let histogram = Services.telemetry.getKeyedHistogramById(SEARCH_COUNTS_HISTOGRAM_KEY);
histogram.add(payload);
LOG("recordSearchURLTelemetry: " + payload);
return {provider, type, code};
}
/**
* Logs telemetry for a search provider visit.
*
* @param {object} info
* @param {string} info.provider The name of the provider.
* @param {string} info.type The type of search.
* @param {string} [info.code] The code for the provider.
* @param {string} url The url that was matched (for debug logging only).
*/
_reportSerpPage(info, url) {
let payload = `${info.provider}.in-content:${info.type}:${info.code || "none"}`;
let histogram = Services.telemetry.getKeyedHistogramById(SEARCH_COUNTS_HISTOGRAM_KEY);
histogram.add(payload);
LOG(`SearchTelemetry::recordSearchURLTelemetry: ${payload} for ${url}`);
}
/**
* Returns the current search provider information in use.
* @see SEARCH_PROVIDER_INFO
*/
get _searchProviderInfo() {
if (!this.__searchProviderInfo) {
this.__searchProviderInfo = SEARCH_PROVIDER_INFO;
@ -130,12 +382,168 @@ class TelemetryHandler {
}
/**
* Outputs aText to the JavaScript console as well as to stdout.
* ContentHandler deals with handling telemetry of the content within a tab -
* when ads detected and when they are selected.
*
* It handles the "browser.search.with_ads" and "browser.search.ad_clicks"
* scalars.
*/
function LOG(aText) {
class ContentHandler {
/**
* Constructor.
*
* @param {object} options
* @param {Map} options.browserInfoByUrl The map of urls from TelemetryHandler.
* @param {function} options.getProviderInfoForUrl A function that obtains
* the provider information for a url.
*/
constructor(options) {
this._browserInfoByUrl = options.browserInfoByUrl;
this._getProviderInfoForUrl = options.getProviderInfoForUrl;
}
/**
* Initializes the content handler. This will also set up the shared data that is
* shared with the SearchTelemetryChild actor.
*/
init() {
Services.ppmm.sharedData.set("SearchTelemetry:ProviderInfo", SEARCH_PROVIDER_INFO);
Cc["@mozilla.org/network/http-activity-distributor;1"]
.getService(Ci.nsIHttpActivityDistributor)
.addObserver(this);
}
/**
* Uninitializes the content handler.
*/
uninit() {
Cc["@mozilla.org/network/http-activity-distributor;1"]
.getService(Ci.nsIHttpActivityDistributor)
.removeObserver(this);
}
/**
* Receives a message from the SearchTelemetryChild actor.
*
* @param {object} msg
*/
receiveMessage(msg) {
if (msg.name != "SearchTelemetry:PageInfo") {
LOG(`"Received unexpected message: ${msg.name}`);
return;
}
this._reportPageWithAds(msg.data);
}
/**
* Test-only function to override the search provider information for use
* with tests. Passes it to the SearchTelemetryChild actor.
*
* @param {object} providerInfo @see SEARCH_PROVIDER_INFO for type information.
*/
overrideSearchTelemetryForTests(providerInfo) {
Services.ppmm.sharedData.set("SearchTelemetry:ProviderInfo", providerInfo);
}
/**
* Listener that observes network activity, so that we can determine if a link
* from a search provider page was followed, and if then if that link was an
* ad click or not.
*
* @param {nsISupports} httpChannel The channel that generated the activity.
* @param {number} activityType The type of activity.
* @param {number} activitySubtype The subtype for the activity.
* @param {PRTime} timestamp The time of the activity.
* @param {number} [extraSizeData] Any size data available for the activity.
* @param {string} [extraStringData] Any extra string data available for the
* activity.
*/
observeActivity(httpChannel, activityType, activitySubtype, timestamp, extraSizeData, extraStringData) {
if (!this._browserInfoByUrl.size ||
activityType != Ci.nsIHttpActivityObserver.ACTIVITY_TYPE_HTTP_TRANSACTION ||
activitySubtype != Ci.nsIHttpActivityObserver.ACTIVITY_SUBTYPE_TRANSACTION_CLOSE) {
return;
}
let channel = httpChannel.QueryInterface(Ci.nsIHttpChannel);
let loadInfo;
try {
loadInfo = channel.loadInfo;
} catch (e) {
// Channels without a loadInfo are not pertinent.
return;
}
try {
let uri = channel.URI;
let triggerURI = loadInfo.triggeringPrincipal.URI;
if (!triggerURI || !this._browserInfoByUrl.has(triggerURI.spec)) {
return;
}
let info = this._getProviderInfoForUrl(uri.spec, true);
if (!info) {
return;
}
Services.telemetry.keyedScalarAdd(SEARCH_AD_CLICKS_SCALAR, info[0], 1);
LOG(`SearchTelemetry::recordSearchURLTelemetry: Counting ad click in page for ${info[0]} ${triggerURI.spec}`);
} catch (e) {
Cu.reportError(e);
}
}
/**
* Adds a message listener for the window being registered to receive messages
* from SearchTelemetryChild.
*
* @param {object} win The window to register.
*/
registerWindow(win) {
win.messageManager.addMessageListener("SearchTelemetry:PageInfo", this);
}
/**
* Removes the message listener for the window.
*
* @param {object} win The window to unregister.
*/
unregisterWindow(win) {
win.messageManager.removeMessageListener("SearchTelemetry:PageInfo", this);
}
/**
* Logs telemetry for a page with adverts, if it is one of the partner search
* provider pages that we're tracking.
*
* @param {object} info
* @param {boolean} info.hasAds Whether or not the page has adverts.
* @param {string} info.url The url of the page.
*/
_reportPageWithAds(info) {
let item = this._browserInfoByUrl.get(info.url);
if (!item) {
LOG(`Expected to report URI with ads but couldn't find the information`);
return;
}
Services.telemetry.keyedScalarAdd(SEARCH_WITH_ADS_SCALAR, item.info.provider, 1);
LOG(`SearchTelemetry::recordSearchURLTelemetry: Counting ads in page for ${item.info.provider} ${info.url}`);
}
}
/**
* Outputs the message to the JavaScript console as well as to stdout.
*
* @param {string} msg The message to output.
*/
function LOG(msg) {
if (loggingEnabled) {
dump(`*** SearchTelemetry: ${aText}\n"`);
Services.console.logStringMessage(aText);
dump(`*** SearchTelemetry: ${msg}\n"`);
Services.console.logStringMessage(msg);
}
}

Просмотреть файл

@ -49,5 +49,6 @@ skip-if = artifact # bug 1315953
[browser_searchTelemetry.js]
support-files =
searchTelemetry.html
searchTelemetryAd.html
[browser_webapi.js]
[browser_tooManyEnginesOffered.js]

Просмотреть файл

@ -11,27 +11,39 @@ const {SearchTelemetry} = ChromeUtils.import("resource:///modules/SearchTelemetr
const TEST_PROVIDER_INFO = {
"example": {
"regexp": /^http:\/\/mochi.test:.+\/browser\/browser\/components\/search\/test\/browser\/searchTelemetry.html/,
"regexp": /^http:\/\/mochi.test:.+\/browser\/browser\/components\/search\/test\/browser\/searchTelemetry(?:Ad)?.html/,
"queryParam": "s",
"codeParam": "abc",
"codePrefixes": ["ff"],
"followonParams": ["a"],
"extraAdServersRegexp": /^https:\/\/example\.com/,
"adPrefixes": ["ad", "ad2"],
"extraAdServersRegexps": [/^https:\/\/example\.com\/ad2?/],
},
};
const MAIN_TEST_PAGE =
"http://mochi.test:8888/browser/browser/components/search/test/browser/searchTelemetry.html";
const TEST_PROVIDER_SERP_URL =
MAIN_TEST_PAGE + "?s=test&abc=ff";
const TEST_PROVIDER_SERP_FOLLOWON_URL =
MAIN_TEST_PAGE + "?s=test&abc=ff&a=foo";
const SEARCH_AD_CLICK_SCALARS = [
"browser.search.with_ads",
"browser.search.ad_clicks",
];
function getPageUrl(useExample = false, useAdPage = false) {
let server = useExample ? "example.com" : "mochi.test:8888";
let page = useAdPage ? "searchTelemetryAd.html" : "searchTelemetry.html";
return `http://${server}/browser/browser/components/search/test/browser/${page}`;
}
function getSERPUrl(page) {
return page + "?s=test&abc=ff";
}
function getSERPFollowOnUrl(page) {
return page + "?s=test&abc=ff&a=foo";
}
const searchCounts = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS");
async function assertTelemetry(expectedHistograms) {
let histSnapshot;
async function assertTelemetry(expectedHistograms, expectedScalars) {
let histSnapshot = {};
let scalars = {};
await TestUtils.waitForCondition(() => {
histSnapshot = searchCounts.snapshot();
@ -39,6 +51,15 @@ async function assertTelemetry(expectedHistograms) {
Object.getOwnPropertyNames(expectedHistograms).length;
});
if (Object.entries(expectedScalars).length > 0) {
await TestUtils.waitForCondition(() => {
scalars = Services.telemetry.getSnapshotForKeyedScalars(
"main", false).parent || {};
return Object.getOwnPropertyNames(expectedScalars)[0] in
scalars;
});
}
Assert.equal(Object.getOwnPropertyNames(histSnapshot).length,
Object.getOwnPropertyNames(expectedHistograms).length,
"Should only have one key");
@ -49,6 +70,18 @@ async function assertTelemetry(expectedHistograms) {
Assert.equal(histSnapshot[key].sum, value,
`Should have counted the correct number of visits for ${key}`);
}
for (let [name, value] of Object.entries(expectedScalars)) {
Assert.ok(name in scalars,
`Scalar ${name} should have been added.`);
Assert.deepEqual(scalars[name], value,
`Should have counted the correct number of visits for ${name}`);
}
for (let name of SEARCH_AD_CLICK_SCALARS) {
Assert.equal(name in scalars, name in expectedScalars,
`Should have matched ${name} in scalars and expectedScalars`);
}
}
add_task(async function setup() {
@ -70,20 +103,119 @@ add_task(async function test_simple_search_page_visit() {
await BrowserTestUtils.withNewTab({
gBrowser,
url: TEST_PROVIDER_SERP_URL,
url: getSERPUrl(getPageUrl()),
}, async () => {
await assertTelemetry({"example.in-content:sap:ff": 1});
await assertTelemetry({"example.in-content:sap:ff": 1}, {});
});
});
add_task(async function test_follow_on_visit() {
await BrowserTestUtils.withNewTab({
gBrowser,
url: TEST_PROVIDER_SERP_FOLLOWON_URL,
url: getSERPFollowOnUrl(getPageUrl()),
}, async () => {
await assertTelemetry({
"example.in-content:sap:ff": 1,
"example.in-content:sap-follow-on:ff": 1,
});
}, {});
});
});
add_task(async function test_track_ad() {
searchCounts.clear();
let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser,
getSERPUrl(getPageUrl(false, true)));
await assertTelemetry({"example.in-content:sap:ff": 1}, {
"browser.search.with_ads": {"example": 1},
});
BrowserTestUtils.removeTab(tab);
});
add_task(async function test_track_ad_new_window() {
searchCounts.clear();
Services.telemetry.clearScalars();
let win = await BrowserTestUtils.openNewBrowserWindow();
let url = getSERPUrl(getPageUrl(false, true));
await BrowserTestUtils.loadURI(win.gBrowser.selectedBrowser, url);
await BrowserTestUtils.browserLoaded(win.gBrowser.selectedBrowser, false, url);
await assertTelemetry({"example.in-content:sap:ff": 1}, {
"browser.search.with_ads": {"example": 1},
});
await BrowserTestUtils.closeWindow(win);
});
add_task(async function test_track_ad_pages_without_ads() {
// Note: the above tests have already checked a page with no ad-urls.
searchCounts.clear();
Services.telemetry.clearScalars();
let tabs = [];
tabs.push(await BrowserTestUtils.openNewForegroundTab(gBrowser,
getSERPUrl(getPageUrl(false, false))));
tabs.push(await BrowserTestUtils.openNewForegroundTab(gBrowser,
getSERPUrl(getPageUrl(false, true))));
await assertTelemetry({"example.in-content:sap:ff": 2}, {
"browser.search.with_ads": {"example": 1},
});
for (let tab of tabs) {
BrowserTestUtils.removeTab(tab);
}
});
add_task(async function test_track_ad_click() {
// Note: the above tests have already checked a page with no ad-urls.
searchCounts.clear();
Services.telemetry.clearScalars();
let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser,
getSERPUrl(getPageUrl(false, true)));
await assertTelemetry({"example.in-content:sap:ff": 1}, {
"browser.search.with_ads": {"example": 1},
});
let pageLoadPromise = BrowserTestUtils.waitForLocationChange(gBrowser);
await ContentTask.spawn(tab.linkedBrowser, {}, () => {
content.document.getElementById("ad1").click();
});
await pageLoadPromise;
await assertTelemetry({"example.in-content:sap:ff": 1}, {
"browser.search.with_ads": {"example": 1},
"browser.search.ad_clicks": {"example": 1},
});
// Now go back, and click again.
pageLoadPromise = BrowserTestUtils.waitForLocationChange(gBrowser);
gBrowser.goBack();
await pageLoadPromise;
// We've gone back, so we register an extra display & if it is with ads or not.
await assertTelemetry({"example.in-content:sap:ff": 2}, {
"browser.search.with_ads": {"example": 2},
"browser.search.ad_clicks": {"example": 1},
});
pageLoadPromise = BrowserTestUtils.waitForLocationChange(gBrowser);
await ContentTask.spawn(tab.linkedBrowser, {}, () => {
content.document.getElementById("ad1").click();
});
await pageLoadPromise;
await assertTelemetry({"example.in-content:sap:ff": 2}, {
"browser.search.with_ads": {"example": 2},
"browser.search.ad_clicks": {"example": 2},
});
BrowserTestUtils.removeTab(tab);
});

Просмотреть файл

@ -2,5 +2,9 @@
<html>
<head>
</head>
<body></body>
<body>
<a href="https://example.com/otherpage">Non ad link</a>
<a href="https://example1.com/ad">Matching path prefix, different server</a>
<a href="https://mochi.test:8888/otherpage">Non ad link</a>
</body>
</html>

Просмотреть файл

@ -0,0 +1,9 @@
<!DOCTYPE html>
<html>
<head>
</head>
<body>
<a id="ad1" href="https://example.com/ad">Ad link</a>
<a id="ad2" href="https://example.com/ad2">Second Ad link</a>
</body>
</html>

Просмотреть файл

@ -7,79 +7,79 @@ ChromeUtils.import("resource:///modules/SearchTelemetry.jsm");
add_task(async function test_parsing_search_urls() {
let hs;
// Google search access point.
SearchTelemetry.recordSearchURLTelemetry("https://www.google.com/search?q=test&ie=utf-8&oe=utf-8&client=firefox-b-1-ab");
SearchTelemetry.updateTrackingStatus({}, "https://www.google.com/search?q=test&ie=utf-8&oe=utf-8&client=firefox-b-1-ab");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("google.in-content:sap:firefox-b-1-ab" in hs, "The histogram must contain the correct key");
// Google search access point follow-on.
SearchTelemetry.recordSearchURLTelemetry("https://www.google.com/search?client=firefox-b-1-ab&ei=EI_VALUE&q=test2&oq=test2&gs_l=GS_L_VALUE");
SearchTelemetry.updateTrackingStatus({}, "https://www.google.com/search?client=firefox-b-1-ab&ei=EI_VALUE&q=test2&oq=test2&gs_l=GS_L_VALUE");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("google.in-content:sap-follow-on:firefox-b-1-ab" in hs, "The histogram must contain the correct key");
// Google organic.
SearchTelemetry.recordSearchURLTelemetry("https://www.google.com/search?source=hp&ei=EI_VALUE&q=test&oq=test&gs_l=GS_L_VALUE");
SearchTelemetry.updateTrackingStatus({}, "https://www.google.com/search?source=hp&ei=EI_VALUE&q=test&oq=test&gs_l=GS_L_VALUE");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("google.in-content:organic:none" in hs, "The histogram must contain the correct key");
// Google organic UK.
SearchTelemetry.recordSearchURLTelemetry("https://www.google.co.uk/search?source=hp&ei=EI_VALUE&q=test&oq=test&gs_l=GS_L_VALUE");
SearchTelemetry.updateTrackingStatus({}, "https://www.google.co.uk/search?source=hp&ei=EI_VALUE&q=test&oq=test&gs_l=GS_L_VALUE");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("google.in-content:organic:none" in hs, "The histogram must contain the correct key");
// Yahoo organic.
SearchTelemetry.recordSearchURLTelemetry("https://search.yahoo.com/search?p=test&fr=yfp-t&fp=1&toggle=1&cop=mss&ei=UTF-8");
SearchTelemetry.updateTrackingStatus({}, "https://search.yahoo.com/search?p=test&fr=yfp-t&fp=1&toggle=1&cop=mss&ei=UTF-8");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("yahoo.in-content:organic:none" in hs, "The histogram must contain the correct key");
// Yahoo organic UK.
SearchTelemetry.recordSearchURLTelemetry("https://uk.search.yahoo.com/search?p=test&fr=yfp-t&fp=1&toggle=1&cop=mss&ei=UTF-8");
SearchTelemetry.updateTrackingStatus({}, "https://uk.search.yahoo.com/search?p=test&fr=yfp-t&fp=1&toggle=1&cop=mss&ei=UTF-8");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("yahoo.in-content:organic:none" in hs, "The histogram must contain the correct key");
// Bing search access point.
SearchTelemetry.recordSearchURLTelemetry("https://www.bing.com/search?q=test&pc=MOZI&form=MOZLBR");
SearchTelemetry.updateTrackingStatus({}, "https://www.bing.com/search?q=test&pc=MOZI&form=MOZLBR");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("bing.in-content:sap:MOZI" in hs, "The histogram must contain the correct key");
// Bing organic.
SearchTelemetry.recordSearchURLTelemetry("https://www.bing.com/search?q=test&qs=n&form=QBLH&sp=-1&pq=&sc=0-0&sk=&cvid=CVID_VALUE");
SearchTelemetry.updateTrackingStatus({}, "https://www.bing.com/search?q=test&qs=n&form=QBLH&sp=-1&pq=&sc=0-0&sk=&cvid=CVID_VALUE");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("bing.in-content:organic:none" in hs, "The histogram must contain the correct key");
// DuckDuckGo search access point.
SearchTelemetry.recordSearchURLTelemetry("https://duckduckgo.com/?q=test&t=ffab");
SearchTelemetry.updateTrackingStatus({}, "https://duckduckgo.com/?q=test&t=ffab");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("duckduckgo.in-content:sap:ffab" in hs, "The histogram must contain the correct key");
// DuckDuckGo organic.
SearchTelemetry.recordSearchURLTelemetry("https://duckduckgo.com/?q=test&t=hi&ia=news");
SearchTelemetry.updateTrackingStatus({}, "https://duckduckgo.com/?q=test&t=hi&ia=news");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("duckduckgo.in-content:organic:hi" in hs, "The histogram must contain the correct key");
// Baidu search access point.
SearchTelemetry.recordSearchURLTelemetry("https://www.baidu.com/baidu?wd=test&tn=monline_dg&ie=utf-8");
SearchTelemetry.updateTrackingStatus({}, "https://www.baidu.com/baidu?wd=test&tn=monline_dg&ie=utf-8");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("baidu.in-content:sap:monline_dg" in hs, "The histogram must contain the correct key");
// Baidu search access point follow-on.
SearchTelemetry.recordSearchURLTelemetry("https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=monline_dg&wd=test2&oq=test&rsv_pq=RSV_PQ_VALUE&rsv_t=RSV_T_VALUE&rqlang=cn&rsv_enter=1&rsv_sug3=2&rsv_sug2=0&inputT=227&rsv_sug4=397");
SearchTelemetry.updateTrackingStatus({}, "https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=monline_dg&wd=test2&oq=test&rsv_pq=RSV_PQ_VALUE&rsv_t=RSV_T_VALUE&rqlang=cn&rsv_enter=1&rsv_sug3=2&rsv_sug2=0&inputT=227&rsv_sug4=397");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("baidu.in-content:sap-follow-on:monline_dg" in hs, "The histogram must contain the correct key");
// Baidu organic.
SearchTelemetry.recordSearchURLTelemetry("https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=1&ch=&tn=baidu&bar=&wd=test&rn=&oq=&rsv_pq=RSV_PQ_VALUE&rsv_t=RSV_T_VALUE&rqlang=cn");
SearchTelemetry.updateTrackingStatus({}, "https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=1&ch=&tn=baidu&bar=&wd=test&rn=&oq=&rsv_pq=RSV_PQ_VALUE&rsv_t=RSV_T_VALUE&rqlang=cn");
hs = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS").snapshot();
Assert.ok(hs);
Assert.ok("baidu.in-content:organic:baidu" in hs, "The histogram must contain the correct key");

Просмотреть файл

@ -154,6 +154,9 @@ let URICountListener = {
},
onLocationChange(browser, webProgress, request, uri, flags) {
// By default, assume we no longer need to track this tab.
SearchTelemetry.stopTrackingBrowser(browser);
// Don't count this URI if it's an error page.
if (flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_ERROR_PAGE) {
return;
@ -219,7 +222,7 @@ let URICountListener = {
if (shouldRecordSearchCount(browser.getTabBrowser()) &&
!(flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT)) {
SearchTelemetry.recordSearchURLTelemetry(uriSpec);
SearchTelemetry.updateTrackingStatus(browser, uriSpec);
}
if (!shouldCountURI) {

Просмотреть файл

@ -2288,6 +2288,40 @@ update:
record_in_processes:
- main
# The following section contains search counters.
browser.search:
with_ads:
bug_numbers:
- 1495548
- 1505411
description: >
Records counts of SERP pages with adverts displayed. The key format is <engine-name>.
expires: never
keyed: true
kind: uint
notification_emails:
- fx-search@mozilla.com
- adw@mozilla.com
release_channel_collection: opt-out
record_in_processes:
- main
ad_clicks:
bug_numbers:
- 1495548
- 1505411
description: >
Records clicks of adverts on SERP pages. The key format is <engine-name>.
expires: never
keyed: true
kind: uint
notification_emails:
- fx-search@mozilla.com
- adw@mozilla.com
release_channel_collection: opt-out
record_in_processes:
- main
# The following section is for probes testing the Telemetry system. They will not be
# submitted in pings and are only used for testing.
telemetry.test: