mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1150134 - Part 6: Refactor archiving to the new Telemetry module design. r=vladan

This commit is contained in:
Georg Fritzsche 2015-04-23 19:22:28 +02:00
Родитель 9fa566d188
Коммит 562a8df802
7 изменённых файлов: 581 добавлений и 287 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

180
toolkit/components/telemetry/TelemetryArchive.jsm Normal file
Просмотреть файл

@ -0,0 +1,180 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
this.EXPORTED_SYMBOLS = [
"TelemetryArchive"
];
const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm", this);
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
Cu.import("resource://gre/modules/Preferences.jsm", this);
Cu.import("resource://gre/modules/Task.jsm", this);
Cu.import("resource://gre/modules/osfile.jsm", this);
const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetryArchive::";
const PREF_BRANCH = "toolkit.telemetry.";
const PREF_ARCHIVE_ENABLED = PREF_BRANCH + "archive.enabled";
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryStorage",
"resource://gre/modules/TelemetryStorage.jsm");
this.TelemetryArchive = {
/**
* Get a list of the archived pings, sorted by the creation date.
* Note that scanning the archived pings on disk is delayed on startup,
* use promizeInitialized() to access this after scanning.
*
* @return {Promise<sequence<Object>>}
* A list of the archived ping info in the form:
* { id: <string>,
* timestampCreated: <number>,
* type: <string> }
*/
promiseArchivedPingList: function() {
return TelemetryArchiveImpl.promiseArchivedPingList();
},
/**
* Load an archived ping from disk by id, asynchronously.
*
* @param id {String} The pings UUID.
* @return {Promise<PingData>} A promise resolved with the pings data on success.
*/
promiseArchivedPingById: function(id) {
return TelemetryArchiveImpl.promiseArchivedPingById(id);
},
/**
* Archive a ping and persist it to disk.
*
* @param {object} ping The ping data to archive.
* @return {promise} Promise that is resolved when the ping is successfully archived.
*/
promiseArchivePing: function(ping) {
return TelemetryArchiveImpl.promiseArchivePing(ping);
},
/**
* Used in tests only to fake a restart of the module.
*/
_testReset: function() {
TelemetryArchiveImpl._testReset();
},
};
/**
* Checks if pings can be archived. Some products (e.g. Thunderbird) might not want
* to do that.
* @return {Boolean} True if pings should be archived, false otherwise.
*/
function shouldArchivePings() {
return Preferences.get(PREF_ARCHIVE_ENABLED, false);
}
let TelemetryArchiveImpl = {
_logger: null,
// Tracks the archived pings in a Map of (id -> {timestampCreated, type}).
// We use this to cache info on archived pings to avoid scanning the disk more than once.
_archivedPings: new Map(),
// Whether we already scanned the archived pings on disk.
_scannedArchiveDirectory: false,
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
}
return this._logger;
},
_testReset: function() {
this._archivedPings = new Map();
this._scannedArchiveDirectory = false;
},
promiseArchivePing: function(ping) {
if (!shouldArchivePings()) {
this._log.trace("promiseArchivePing - archiving is disabled");
return Promise.resolve();
}
for (let field of ["creationDate", "id", "type"]) {
if (!(field in ping)) {
this._log.warn("promiseArchivePing - missing field " + field)
return Promise.reject(new Error("missing field " + field));
}
}
const creationDate = new Date(ping.creationDate);
if (this._archivedPings.has(ping.id)) {
const data = this._archivedPings.get(ping.id);
if (data.timestampCreated > creationDate.getTime()) {
this._log.error("promiseArchivePing - trying to overwrite newer ping with the same id");
return Promise.reject(new Error("trying to overwrite newer ping with the same id"));
} else {
this._log.warn("promiseArchivePing - overwriting older ping with the same id");
}
}
this._archivedPings.set(ping.id, {
timestampCreated: creationDate.getTime(),
type: ping.type,
});
return TelemetryStorage.saveArchivedPing(ping);
},
_buildArchivedPingList: function() {
let list = [for (p of this._archivedPings) {
id: p[0],
timestampCreated: p[1].timestampCreated,
type: p[1].type,
}];
list.sort((a, b) => a.timestampCreated - b.timestampCreated);
return list;
},
promiseArchivedPingList: function() {
this._log.trace("promiseArchivedPingList");
if (this._scannedArchiveDirectory) {
return Promise.resolve(this._buildArchivedPingList())
}
return TelemetryStorage.loadArchivedPingList().then((loadedInfo) => {
// Add the ping info from scanning to the existing info.
// We might have pings added before lazily loading this list.
for (let [id, info] of loadedInfo) {
this._log.trace("promiseArchivedPingList - id: " + id + ", info: " + info);
this._archivedPings.set(id, {
timestampCreated: info.timestampCreated,
type: info.type,
});
}
this._scannedArchiveDirectory = true;
return this._buildArchivedPingList();
});
},
promiseArchivedPingById: function(id) {
this._log.trace("promiseArchivedPingById - id: " + id);
const data = this._archivedPings.get(id);
if (!data) {
this._log.trace("promiseArchivedPingById - no ping with id: " + id);
return Promise.reject(new Error("TelemetryArchive.promiseArchivedPingById - no ping with id " + id));
}
return TelemetryStorage.loadArchivedPing(id, data.timestampCreated, data.type);
},
};

251
toolkit/components/telemetry/TelemetryPing.jsm
Просмотреть файл

@ -29,7 +29,6 @@ const PREF_BRANCH = "toolkit.telemetry.";
const PREF_BRANCH_LOG = PREF_BRANCH + "log.";
const PREF_SERVER = PREF_BRANCH + "server";
const PREF_ENABLED = PREF_BRANCH + "enabled";
const PREF_ARCHIVE_ENABLED = PREF_BRANCH + "archive.enabled";
const PREF_LOG_LEVEL = PREF_BRANCH_LOG + "level";
const PREF_LOG_DUMP = PREF_BRANCH_LOG + "dump";
const PREF_CACHED_CLIENTID = PREF_BRANCH + "cachedClientID";
@ -57,8 +56,6 @@ XPCOMUtils.defineLazyModuleGetter(this, "AsyncShutdown",
"resource://gre/modules/AsyncShutdown.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryStorage",
"resource://gre/modules/TelemetryStorage.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryLog",
"resource://gre/modules/TelemetryLog.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "ThirdPartyCookieProbe",
"resource://gre/modules/ThirdPartyCookieProbe.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryEnvironment",
@ -67,13 +64,8 @@ XPCOMUtils.defineLazyModuleGetter(this, "SessionRecorder",
"resource://gre/modules/SessionRecorder.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "UpdateChannel",
"resource://gre/modules/UpdateChannel.jsm");
// Compute the path of the pings archive on the first use.
const DATAREPORTING_DIR = "datareporting";
const PINGS_ARCHIVE_DIR = "archived";
XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() {
return OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR, PINGS_ARCHIVE_DIR);
});
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryArchive",
"resource://gre/modules/TelemetryArchive.jsm");
/**
* Setup Telemetry logging. This function also gets called when loggin related
@ -122,25 +114,6 @@ function isNewPingFormat(aPing) {
("version" in aPing) && (aPing.version >= 2);
}
/**
* Build the path to the archived ping.
* @param {String} aPingId The ping id.
* @param {Object} aDate The ping creation date.
* @param {String} aType The ping type.
* @return {String} The full path to the archived ping.
*/
function getArchivedPingPath(aPingId, aDate, aType) {
// Helper to pad the month to 2 digits, if needed (e.g. "1" -> "01").
let addLeftPadding = value => (value < 10) ? ("0" + value) : value;
// Get the ping creation date and generate the archive directory to hold it. Note
// that getMonth returns a 0-based month, so we need to add an offset.
let archivedPingDir = OS.Path.join(gPingsArchivePath,
aDate.getFullYear() + '-' + addLeftPadding(aDate.getMonth() + 1));
// Generate the archived ping file path as YYYY-MM/<TIMESTAMP>.UUID.type.json
let fileName = [aDate.getTime(), aPingId, aType, "json"].join(".");
return OS.Path.join(archivedPingDir, fileName);
};
/**
* This is a policy object used to override behavior for testing.
*/
@ -353,31 +326,6 @@ this.TelemetryPing = Object.freeze({
promiseInitialized: function() {
return Impl.promiseInitialized();
},
/**
* Get a list of the archived pings, sorted by the creation date.
* Note that scanning the archived pings on disk is delayed on startup,
* use promizeInitialized() to access this after scanning.
*
* @return {Promise<sequence<Object>>}
* A list of the archived ping info in the form:
* { id: <string>,
* timestampCreated: <number>,
* type: <string> }
*/
promiseArchivedPingList: function() {
return Impl.promiseArchivedPingList();
},
/**
* Load an archived ping from disk by id, asynchronously.
*
* @param id {String} The pings UUID.
* @return {Promise<PingData>} A promise resolved with the pings data on success.
*/
promiseArchivedPingById: function(id) {
return Impl.promiseArchivedPingById(id);
},
});
let Impl = {
@ -407,10 +355,6 @@ let Impl = {
// This tracks all pending ping requests to the server.
_pendingPingRequests: new Map(),
// This tracks the archived pings in a Map of (id -> {timestampCreated, type}).
// We use this to cache info on archived pings to avoid scanning the disk more than once.
_archivedPings: null,
/**
* Get the data for the "application" section of the ping.
*/
@ -556,7 +500,7 @@ let Impl = {
let pingData = this.assemblePing(aType, aPayload, aOptions);
// Always persist the pings if we are allowed to.
let archivePromise = this._archivePing(pingData)
let archivePromise = TelemetryArchive.promiseArchivePing(pingData)
.catch(e => this._log.error("send - Failed to archive ping " + pingData.id, e));
// Once ping is assembled, send it along with the persisted pings in the backlog.
@ -644,7 +588,7 @@ let Impl = {
let pingData = this.assemblePing(aType, aPayload, aOptions);
let savePromise = TelemetryStorage.savePing(pingData, aOptions.overwrite);
let archivePromise = this._archivePing(pingData).catch(e => {
let archivePromise = TelemetryArchive.promiseArchivePing(pingData).catch(e => {
this._log.error("addPendingPing - Failed to archive ping " + pingData.id, e);
});
@ -936,9 +880,6 @@ let Impl = {
return Promise.resolve();
}
// Initialize some members that may need resetting for restart tests.
this._archivedPings = new Map();
// For very short session durations, we may never load the client
// id from disk.
// We try to cache it in prefs to avoid this, even though this may
@ -969,20 +910,6 @@ let Impl = {
this._clientID = yield ClientID.getClientID();
Preferences.set(PREF_CACHED_CLIENTID, this._clientID);
// If pings should be archived, make sure the archive directory exists.
if (this._shouldArchivePings()) {
const DATAREPORTING_PATH = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR);
let reportError =
e => this._log.error("setupTelemetry - Unable to create the directory", e);
// Don't bail out if we can't create the archive folder, we might still be
// able to send telemetry pings to the servers.
yield OS.File.makeDir(DATAREPORTING_PATH, {ignoreExisting: true}).catch(reportError);
yield OS.File.makeDir(gPingsArchivePath, {ignoreExisting: true}).catch(reportError);
yield this._scanArchivedPingDirectory()
.catch((e) => this._log.error("setupTelemetry - failure scanning archived ping directory", e));
}
Telemetry.asyncFetchTelemetryData(function () {});
this._delayedInitTaskDeferred.resolve();
} catch (e) {
@ -1101,37 +1028,6 @@ let Impl = {
Preferences.get(PREF_FHR_UPLOAD_ENABLED, false);
},
/**
* Checks if pings can be archived. Some products (e.g. Thunderbird) might not want
* to do that.
* @return {Boolean} True if pings should be archived, false otherwise.
*/
_shouldArchivePings: function() {
return Preferences.get(PREF_ARCHIVE_ENABLED, true);
},
/**
* Save a ping to the pings archive. Note that any error should be handled by the caller.
* @param {Object} aPingData The content of the ping.
* @return {Promise} A promise resolved when the ping is saved to the archive.
*/
_archivePing: Task.async(function*(aPingData) {
if (!this._shouldArchivePings()) {
return;
}
const creationDate = new Date(aPingData.creationDate);
const filePath = getArchivedPingPath(aPingData.id, creationDate, aPingData.type);
yield OS.File.makeDir(OS.Path.dirname(filePath), { ignoreExisting: true,
from: OS.Constants.Path.profileDir });
yield TelemetryStorage.savePingToFile(aPingData, filePath, true);
this._archivedPings.set(aPingData.id, {
timestampCreated: creationDate.getTime(),
type: aPingData.type,
});
}),
/**
* Get an object describing the current state of this module for AsyncShutdown diagnostics.
*/
@ -1145,44 +1041,6 @@ let Impl = {
};
},
/**
* Get a list of the archived pings, sorted by the creation date.
* @return sequence<Object>>
* A list of the archived ping info in the form:
* { id: <string>,
* timestampCreated: <number>,
* type: <string> }
*/
promiseArchivedPingList: function() {
this._log.trace("getArchivedPingList");
let list = [for (p of this._archivedPings) {
id: p[0],
timestampCreated: p[1].timestampCreated,
type: p[1].type,
}];
list.sort((a, b) => a.timestampCreated - b.timestampCreated);
return list;
},
/**
* Load an archived ping from disk by id, asynchronously.
* @return {Promise<Object>} Promise that is resolved with the ping data.
*/
promiseArchivedPingById: function(id) {
this._log.trace("getArchivedPingById - id: " + id);
const data = this._archivedPings.get(id);
if (!data) {
this._log.trace("getArchivedPingById - no ping with id: " + id);
return Promise.reject(new Error("TelemetryPing.getArchivedPingById - no ping with id " + id));
}
const path = getArchivedPingPath(id, new Date(data.timestampCreated), data.type);
this._log.trace("getArchivedPingById - loading ping from: " + path);
return TelemetryStorage.loadPingFile(path);
},
/**
* Allows waiting for TelemetryPings delayed initialization to complete.
* This will complete before TelemetryPing is shutting down.
@ -1191,105 +1049,4 @@ let Impl = {
promiseInitialized: function() {
return this._delayedInitTaskDeferred.promise;
},
/**
* Archived pings are saved with file names of the form:
* "<timestamp>.<uuid>.<type>.json"
* This helper extracts that data from a given filename.
*
* @param fileName {String} The filename.
* @return {Object} Null if the filename didn't match the expected form.
* Otherwise an object with the extracted data in the form:
* { timestamp: <number>,
* id: <string>,
* type: <string> }
*/
_getArchivedPingDataFromFileName: function(fileName) {
// Extract the parts.
let parts = fileName.split(".");
if (parts.length != 4) {
this._log.trace("_getArchivedPingDataFromFileName - should have 4 parts");
return null;
}
let [timestamp, uuid, type, extension] = parts;
if (extension != "json") {
this._log.trace("_getArchivedPingDataFromFileName - should have a 'json' extension");
return null;
}
// Check for a valid timestamp.
timestamp = parseInt(timestamp);
if (Number.isNaN(timestamp)) {
this._log.trace("_getArchivedPingDataFromFileName - should have a valid timestamp");
return null;
}
// Check for a valid UUID.
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
if (!uuidRegex.test(uuid)) {
this._log.trace("_getArchivedPingDataFromFileName - should have a valid id");
return null;
}
// Check for a valid type string.
const typeRegex = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i;
if (!typeRegex.test(type)) {
this._log.trace("_getArchivedPingDataFromFileName - should have a valid type");
return null;
}
return {
timestamp: timestamp,
id: uuid,
type: type,
};
},
/**
* Scan the archived pings directory and cache the info on the pings found.
*/
_scanArchivedPingDirectory: Task.async(function*() {
this._log.trace("_scanArchivedPingDirectory");
let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);
let subdirs = (yield dirIterator.nextBatch()).filter(e => e.isDir);
// Walk through the monthly subdirs of the form <YYYY-MM>/
for (let dir of subdirs) {
const dirRegEx = /^[0-9]{4}-[0-9]{2}$/;
if (!dirRegEx.test(dir.name)) {
this._log.warn("_scanArchivedPingDirectory - skipping invalidly named subdirectory " + dir.path);
continue;
}
this._log.trace("_scanArchivedPingDirectory - checking in subdir: " + dir.path);
let pingIterator = new OS.File.DirectoryIterator(dir.path);
let pings = (yield pingIterator.nextBatch()).filter(e => !e.isDir);
// Now process any ping files of the form "<timestamp>.<uuid>.<type>.json"
for (let p of pings) {
// data may be null if the filename doesn't match the above format.
let data = this._getArchivedPingDataFromFileName(p.name);
if (!data) {
continue;
}
// In case of conflicts, overwrite only with newer pings.
if (this._archivedPings.has(data.id)) {
const overwrite = data.timestamp > this._archivedPings.get(data.id).timestampCreated;
this._log.warn("_scanArchivedPingDirectory - have seen this id before: " + data.id +
", overwrite: " + overwrite);
if (!overwrite) {
continue;
}
}
this._archivedPings.set(data.id, {
timestampCreated: data.timestamp,
type: data.type,
});
}
}
}),
};

382
toolkit/components/telemetry/TelemetryStorage.jsm
Просмотреть файл

@ -12,6 +12,7 @@ const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://gre/modules/Services.jsm", this);
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
Cu.import("resource://gre/modules/osfile.jsm", this);
@ -21,8 +22,18 @@ Cu.import("resource://gre/modules/Promise.jsm", this);
XPCOMUtils.defineLazyModuleGetter(this, 'Deprecated',
'resource://gre/modules/Deprecated.jsm');
const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetryStorage::";
const Telemetry = Services.telemetry;
// Compute the path of the pings archive on the first use.
const DATAREPORTING_DIR = "datareporting";
const PINGS_ARCHIVE_DIR = "archived";
XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() {
return OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR, PINGS_ARCHIVE_DIR);
});
// Files that have been lying around for longer than MAX_PING_FILE_AGE are
// deleted without being loaded.
const MAX_PING_FILE_AGE = 14 * 24 * 60 * 60 * 1000; // 2 weeks
@ -52,7 +63,6 @@ let pendingPings = [];
let isPingDirectoryCreated = false;
this.TelemetryStorage = {
get MAX_PING_FILE_AGE() {
return MAX_PING_FILE_AGE;
},
@ -69,6 +79,39 @@ this.TelemetryStorage = {
return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings");
},
/**
* Save an archived ping to disk.
*
* @param {object} ping The ping data to archive.
* @return {promise} Promise that is resolved when the ping is successfully archived.
*/
saveArchivedPing: function(ping) {
return TelemetryStorageImpl.saveArchivedPing(ping);
},
/**
* Load an archived ping from disk.
*
* @param {string} id The pings id.
* @param {number} timestampCreated The pings creation timestamp.
* @param {string} type The pings type.
* @return {promise<object>} Promise that is resolved with the ping data.
*/
loadArchivedPing: function(id, timestampCreated, type) {
return TelemetryStorageImpl.loadArchivedPing(id, timestampCreated, type);
},
/**
* Get a list of info on the archived pings.
* This will scan the archive directory and grab basic data about the existing
* pings out of their filename.
*
* @return {promise<sequence<object>>}
*/
loadArchivedPingList: function() {
return TelemetryStorageImpl.loadArchivedPingList();
},
/**
* Save a single ping to a file.
*
@ -80,10 +123,259 @@ this.TelemetryStorage = {
* @returns {promise}
*/
savePingToFile: function(ping, file, overwrite) {
return TelemetryStorageImpl.savePingToFile(ping, file, overwrite);
},
/**
* Save a ping to its file.
*
* @param {object} ping The content of the ping to save.
* @param {bool} overwrite If |true|, the file will be overwritten
* if it exists.
* @returns {promise}
*/
savePing: function(ping, overwrite) {
return TelemetryStorageImpl.savePing(ping, overwrite);
},
/**
* Save all pending pings.
*
* @param {object} sessionPing The additional session ping.
* @returns {promise}
*/
savePendingPings: function(sessionPing) {
return TelemetryStorageImpl.savePendingPings(sessionPing);
},
/**
* Add a ping to the saved pings directory so that it gets along with other pings. Note
* that the original ping file will not be modified.
*
* @param {String} aFilePath The path to the ping file that needs to be added to the
* saved pings directory.
* @return {Promise} A promise resolved when the ping is saved to the pings directory.
*/
addPendingPing: function(aPingPath) {
return TelemetryStorageImpl.addPendingPing(aPingPath);
},
/**
* Remove the file for a ping
*
* @param {object} ping The ping.
* @returns {promise}
*/
cleanupPingFile: function(ping) {
return TelemetryStorageImpl.cleanupPingFile(ping);
},
/**
* Load all saved pings.
*
* Once loaded, the saved pings can be accessed (destructively only)
* through |popPendingPings|.
*
* @returns {promise}
*/
loadSavedPings: function() {
return TelemetryStorageImpl.loadSavedPings();
},
/**
* Load the histograms from a file.
*
* Once loaded, the saved pings can be accessed (destructively only)
* through |popPendingPings|.
*
* @param {string} file The file to load.
* @returns {promise}
*/
loadHistograms: function loadHistograms(file) {
return TelemetryStorageImpl.loadHistograms(file);
},
/**
* The number of pings loaded since the beginning of time.
*/
get pingsLoaded() {
return TelemetryStorageImpl.pingsLoaded;
},
/**
* The number of pings loaded that are older than OVERDUE_PING_FILE_AGE
* but younger than MAX_PING_FILE_AGE.
*/
get pingsOverdue() {
return TelemetryStorageImpl.pingsOverdue;
},
/**
* The number of pings that we just tossed out for being older than
* MAX_PING_FILE_AGE.
*/
get pingsDiscarded() {
return TelemetryStorageImpl.pingsDiscarded;
},
/**
* Iterate destructively through the pending pings.
*
* @return {iterator}
*/
popPendingPings: function*() {
while (pendingPings.length > 0) {
let data = pendingPings.pop();
yield data;
}
},
testLoadHistograms: function(file) {
return TelemetryStorageImpl.testLoadHistograms(file);
},
/**
* Loads a ping file.
* @param {String} aFilePath The path of the ping file.
* @return {Promise<Object>} A promise resolved with the ping content or rejected if the
* ping contains invalid data.
*/
loadPingFile: Task.async(function* (aFilePath) {
return TelemetryStorageImpl.loadPingFile(aFilePath);
}),
};
let TelemetryStorageImpl = {
_logger: null,
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
}
return this._logger;
},
/**
* Save an archived ping to disk.
*
* @param {object} ping The ping data to archive.
* @return {promise} Promise that is resolved when the ping is successfully archived.
*/
saveArchivedPing: Task.async(function*(ping) {
const creationDate = new Date(ping.creationDate);
const filePath = getArchivedPingPath(ping.id, creationDate, ping.type);
yield OS.File.makeDir(OS.Path.dirname(filePath), { ignoreExisting: true,
from: OS.Constants.Path.profileDir });
yield TelemetryStorage.savePingToFile(ping, filePath, true);
}),
/**
* Load an archived ping from disk.
*
* @param {string} id The pings id.
* @param {number} timestampCreated The pings creation timestamp.
* @param {string} type The pings type.
* @return {promise<object>} Promise that is resolved with the ping data.
*/
loadArchivedPing: function(id, timestampCreated, type) {
this._log.trace("loadArchivedPing - id: " + id + ", timestampCreated: " + timestampCreated + ", type: " + type);
const path = getArchivedPingPath(id, new Date(timestampCreated), type);
this._log.trace("loadArchivedPing - loading ping from: " + path);
return this.loadPingFile(path);
},
/**
* Remove an archived ping from disk.
*
* @param {string} id The pings id.
* @param {number} timestampCreated The pings creation timestamp.
* @param {string} type The pings type.
* @return {promise<object>} Promise that is resolved when the pings is removed.
*/
_removeArchivedPing: function(id, timestampCreated, type) {
this._log.trace("_removeArchivedPing - id: " + id + ", timestampCreated: " + timestampCreated + ", type: " + type);
const path = getArchivedPingPath(id, new Date(timestampCreated), type);
this._log.trace("_removeArchivedPing - removing ping from: " + path);
return OS.File.remove(path);
},
/**
* Get a list of info on the archived pings.
* This will scan the archive directory and grab basic data about the existing
* pings out of their filename.
*
* @return {promise<sequence<object>>}
*/
loadArchivedPingList: Task.async(function*() {
this._log.trace("loadArchivedPingList");
if (!(yield OS.File.exists(gPingsArchivePath))) {
return new Map();
}
let archivedPings = new Map();
let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);
let subdirs = (yield dirIterator.nextBatch()).filter(e => e.isDir);
// Walk through the monthly subdirs of the form <YYYY-MM>/
for (let dir of subdirs) {
const dirRegEx = /^[0-9]{4}-[0-9]{2}$/;
if (!dirRegEx.test(dir.name)) {
this._log.warn("loadArchivedPingList - skipping invalidly named subdirectory " + dir.path);
continue;
}
this._log.trace("loadArchivedPingList - checking in subdir: " + dir.path);
let pingIterator = new OS.File.DirectoryIterator(dir.path);
let pings = (yield pingIterator.nextBatch()).filter(e => !e.isDir);
// Now process any ping files of the form "<timestamp>.<uuid>.<type>.json"
for (let p of pings) {
// data may be null if the filename doesn't match the above format.
let data = this._getArchivedPingDataFromFileName(p.name);
if (!data) {
continue;
}
// In case of conflicts, overwrite only with newer pings.
if (archivedPings.has(data.id)) {
const overwrite = data.timestamp > archivedPings.get(data.id).timestampCreated;
this._log.warn("loadArchivedPingList - have seen this id before: " + data.id +
", overwrite: " + overwrite);
if (!overwrite) {
continue;
}
yield this._removeArchivedPing(data.id, data.timestampCreated, data.type)
.catch((e) => this._log.warn("loadArchivedPingList - failed to remove ping", e));
}
archivedPings.set(data.id, {
timestampCreated: data.timestamp,
type: data.type,
});
}
}
return archivedPings;
}),
/**
* Save a single ping to a file.
*
* @param {object} ping The content of the ping to save.
* @param {string} file The destination file.
* @param {bool} overwrite If |true|, the file will be overwritten if it exists,
* if |false| the file will not be overwritten and no error will be reported if
* the file exists.
* @returns {promise}
*/
savePingToFile: function(ping, filePath, overwrite) {
return Task.spawn(function*() {
try {
let pingString = JSON.stringify(ping);
yield OS.File.writeAtomic(file, pingString, {tmpPath: file + ".tmp",
yield OS.File.writeAtomic(filePath, pingString, {tmpPath: filePath + ".tmp",
noOverwrite: !overwrite});
} catch(e if e.becauseExists) {
}
@ -261,18 +553,6 @@ this.TelemetryStorage = {
return pingsDiscarded;
},
/**
* Iterate destructively through the pending pings.
*
* @return {iterator}
*/
popPendingPings: function*() {
while (pendingPings.length > 0) {
let data = pendingPings.pop();
yield data;
}
},
testLoadHistograms: function(file) {
pingsLoaded = 0;
return this.loadHistograms(file.path);
@ -296,9 +576,64 @@ this.TelemetryStorage = {
}
return ping;
}),
/**
* Archived pings are saved with file names of the form:
* "<timestamp>.<uuid>.<type>.json"
* This helper extracts that data from a given filename.
*
* @param fileName {String} The filename.
* @return {Object} Null if the filename didn't match the expected form.
* Otherwise an object with the extracted data in the form:
* { timestamp: <number>,
* id: <string>,
* type: <string> }
*/
_getArchivedPingDataFromFileName: function(fileName) {
// Extract the parts.
let parts = fileName.split(".");
if (parts.length != 4) {
this._log.trace("_getArchivedPingDataFromFileName - should have 4 parts");
return null;
}
let [timestamp, uuid, type, extension] = parts;
if (extension != "json") {
this._log.trace("_getArchivedPingDataFromFileName - should have a 'json' extension");
return null;
}
// Check for a valid timestamp.
timestamp = parseInt(timestamp);
if (Number.isNaN(timestamp)) {
this._log.trace("_getArchivedPingDataFromFileName - should have a valid timestamp");
return null;
}
// Check for a valid UUID.
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
if (!uuidRegex.test(uuid)) {
this._log.trace("_getArchivedPingDataFromFileName - should have a valid id");
return null;
}
// Check for a valid type string.
const typeRegex = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i;
if (!typeRegex.test(type)) {
this._log.trace("_getArchivedPingDataFromFileName - should have a valid type");
return null;
}
return {
timestamp: timestamp,
id: uuid,
type: type,
};
},
};
///// Utility functions
function pingFilePath(ping) {
// Support legacy ping formats, who don't have an "id" field, but a "slug" field.
let pingIdentifier = (ping.slug) ? ping.slug : ping.id;
@ -333,3 +668,22 @@ function addToPendingPings(file) {
return OS.File.remove(file);
});
}
/**
* Build the path to the archived ping.
* @param {String} aPingId The ping id.
* @param {Object} aDate The ping creation date.
* @param {String} aType The ping type.
* @return {String} The full path to the archived ping.
*/
function getArchivedPingPath(aPingId, aDate, aType) {
// Helper to pad the month to 2 digits, if needed (e.g. "1" -> "01").
let addLeftPadding = value => (value < 10) ? ("0" + value) : value;
// Get the ping creation date and generate the archive directory to hold it. Note
// that getMonth returns a 0-based month, so we need to add an offset.
let archivedPingDir = OS.Path.join(gPingsArchivePath,
aDate.getFullYear() + '-' + addLeftPadding(aDate.getMonth() + 1));
// Generate the archived ping file path as YYYY-MM/<TIMESTAMP>.UUID.type.json
let fileName = [aDate.getTime(), aPingId, aType, "json"].join(".");
return OS.Path.join(archivedPingDir, fileName);
}

1
toolkit/components/telemetry/moz.build
Просмотреть файл

@ -30,6 +30,7 @@ EXTRA_COMPONENTS += [
]
EXTRA_JS_MODULES += [
'TelemetryArchive.jsm',
'TelemetryLog.jsm',
'TelemetryStopwatch.jsm',
'TelemetryStorage.jsm',

10
toolkit/components/telemetry/tests/unit/head.js
Просмотреть файл

@ -140,8 +140,8 @@ function fakeSchedulerTimer(set, clear) {
*
* @return Date The new faked date.
*/
function fakeNow(...arguments) {
const date = new Date(...arguments);
function fakeNow(...args) {
const date = new Date(...args);
let ping = Cu.import("resource://gre/modules/TelemetryPing.jsm");
ping.Policy.now = () => date;
@ -162,6 +162,12 @@ function truncateToDays(aMsec) {
return Math.floor(aMsec / MILLISECONDS_PER_DAY);
}
// Returns a promise that resolves to true when the passed promise rejects,
// false otherwise.
function promiseRejects(promise) {
return promise.then(() => false, () => true);
}
// Set logging preferences for all the tests.
Services.prefs.setCharPref("toolkit.telemetry.log.level", "Trace");
TelemetryPing.initLogging();

22
toolkit/components/telemetry/tests/unit/test_PingAPI.js
Просмотреть файл

@ -7,10 +7,11 @@
"use strict";
Cu.import("resource://gre/modules/TelemetryPing.jsm", this);
Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);
Cu.import("resource://gre/modules/TelemetryArchive.jsm", this);
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
Cu.import("resource://gre/modules/osfile.jsm", this);
Cu.import("resource://gre/modules/Task.jsm", this);
Cu.import("resource://gre/modules/Services.jsm", this);
XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() {
return OS.Path.join(OS.Constants.Path.profileDir, "datareporting", "archived");
@ -44,7 +45,7 @@ add_task(function* test_archivedPings() {
for (let data of PINGS) {
fakeNow(data.dateCreated);
data.id = yield TelemetryPing.send(data.type, data.payload);
let list = yield TelemetryPing.promiseArchivedPingList();
let list = yield TelemetryArchive.promiseArchivedPingList();
expectedPingList.push({
id: data.id,
@ -58,7 +59,7 @@ add_task(function* test_archivedPings() {
let ids = [for (p of PINGS) p.id];
let checkLoadingPings = Task.async(function*() {
for (let data of PINGS) {
let ping = yield TelemetryPing.promiseArchivedPingById(data.id);
let ping = yield TelemetryArchive.promiseArchivedPingById(data.id);
Assert.equal(ping.id, data.id, "Archived ping should have matching id");
Assert.equal(ping.type, data.type, "Archived ping should have matching type");
Assert.equal(ping.creationDate, data.dateCreated.toISOString(),
@ -71,7 +72,7 @@ add_task(function* test_archivedPings() {
// Check that we find the archived pings again by scanning after a restart.
yield TelemetryPing.setup();
let pingList = yield TelemetryPing.promiseArchivedPingList();
let pingList = yield TelemetryArchive.promiseArchivedPingList();
Assert.deepEqual(expectedPingList, pingList,
"Should have submitted pings in archive list after restart");
yield checkLoadingPings();
@ -111,17 +112,18 @@ add_task(function* test_archivedPings() {
});
expectedPingList.sort((a, b) => a.timestampCreated - b.timestampCreated);
// Trigger scanning the ping dir.
yield TelemetryPing.setup();
// Reset the TelemetryArchive so we scan the archived dir again.
yield TelemetryArchive._testReset();
// Check that we are still picking up the valid archived pings on disk,
// plus the valid ones above.
pingList = yield TelemetryPing.promiseArchivedPingList();
pingList = yield TelemetryArchive.promiseArchivedPingList();
Assert.deepEqual(expectedPingList, pingList, "Should have picked up valid archived pings");
yield checkLoadingPings();
// Now check that we fail to load the two invalid pings from above.
let rejects = (promise) => promise.then(() => false, () => true);
Assert.ok((yield rejects(TelemetryPing.promiseArchivedPingById(FAKE_ID1))), "Should not have scanned invalid ping");
Assert.ok((yield rejects(TelemetryPing.promiseArchivedPingById(FAKE_ID2))), "Should not have scanned invalid ping");
Assert.ok((yield promiseRejects(TelemetryArchive.promiseArchivedPingById(FAKE_ID1))),
"Should have rejected invalid ping");
Assert.ok((yield promiseRejects(TelemetryArchive.promiseArchivedPingById(FAKE_ID2))),
"Should have rejected invalid ping");
});

22
toolkit/components/telemetry/tests/unit/test_TelemetryPing.js
Просмотреть файл

@ -14,6 +14,7 @@ Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
Cu.import("resource://gre/modules/TelemetryPing.jsm", this);
Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);
Cu.import("resource://gre/modules/TelemetryArchive.jsm", this);
Cu.import("resource://gre/modules/Task.jsm", this);
Cu.import("resource://gre/modules/Promise.jsm", this);
Cu.import("resource://gre/modules/Preferences.jsm");
@ -39,11 +40,6 @@ let gServerStarted = false;
let gRequestIterator = null;
let gClientID = null;
function getArchiveFilename(uuid, date, type) {
let ping = Cu.import("resource://gre/modules/TelemetryPing.jsm");
return ping.getArchivedPingPath(uuid, date, type);
}
function sendPing(aSendClientId, aSendEnvironment) {
if (gServerStarted) {
TelemetryPing.setServer("http://localhost:" + gHttpServer.identity.primaryPort);
@ -240,18 +236,17 @@ add_task(function* test_archivePings() {
registerPingHandler(() => Assert.ok(false, "Telemetry must not send pings if not allowed to."));
let pingId = yield sendPing(true, true);
// Check that the ping was persisted to the pings archive, even with upload disabled.
let pingPath = getArchiveFilename(pingId, now, TEST_PING_TYPE);
Assert.ok((yield OS.File.exists(pingPath)),
"TelemetryPing must archive pings if FHR is enabled.");
// Check that the ping was archived, even with upload disabled.
let ping = yield TelemetryArchive.promiseArchivedPingById(pingId);
Assert.equal(ping.id, pingId, "TelemetryPing must archive pings if FHR is enabled.");
// Check that pings don't get archived if not allowed to.
now = new Date(2010, 10, 18, 12, 0, 0);
fakeNow(now);
Preferences.set(PREF_ARCHIVE_ENABLED, false);
pingId = yield sendPing(true, true);
pingPath = getArchiveFilename(pingId, now, TEST_PING_TYPE);
Assert.ok(!(yield OS.File.exists(pingPath)),
let promise = TelemetryArchive.promiseArchivedPingById(pingId);
Assert.ok((yield promiseRejects(promise)),
"TelemetryPing must not archive pings if the archive pref is disabled.");
// Enable archiving and the upload so that pings get sent and archived again.
@ -266,9 +261,8 @@ add_task(function* test_archivePings() {
// Check that we archive pings when successfully sending them.
yield gRequestIterator.next();
pingPath = getArchiveFilename(pingId, now, TEST_PING_TYPE);
Assert.ok((yield OS.File.exists(pingPath)),
"TelemetryPing must archive pings if FHR is enabled.");
ping = yield TelemetryArchive.promiseArchivedPingById(pingId);
Assert.equal(ping.id, pingId, "TelemetryPing must archive pings if FHR is enabled.");
});
add_task(function* stopServer(){