From a81aa4ad16ca035dc25f8ab686823642c2120918 Mon Sep 17 00:00:00 2001 From: Ed Morley Date: Tue, 31 Jan 2012 18:20:33 +0000 Subject: [PATCH] Backout d74a924a149b (in effect relanding bug 707320) due to M-oth orange --- toolkit/components/telemetry/Telemetry.cpp | 345 +++++++++++++++++- toolkit/components/telemetry/TelemetryPing.js | 109 +++++- toolkit/components/telemetry/nsITelemetry.idl | 61 +++- .../telemetry/tests/unit/test_nsITelemetry.js | 41 +++ 4 files changed, 531 insertions(+), 25 deletions(-) diff --git a/toolkit/components/telemetry/Telemetry.cpp b/toolkit/components/telemetry/Telemetry.cpp index 0467ffb61d55..1300a935a264 100644 --- a/toolkit/components/telemetry/Telemetry.cpp +++ b/toolkit/components/telemetry/Telemetry.cpp @@ -38,6 +38,7 @@ * ***** END LICENSE BLOCK ***** */ #include "base/histogram.h" +#include "base/pickle.h" #include "nsIComponentManager.h" #include "nsIServiceManager.h" #include "nsCOMPtr.h" @@ -47,6 +48,8 @@ #include "jsapi.h" #include "nsStringGlue.h" #include "nsITelemetry.h" +#include "nsIFile.h" +#include "nsILocalFile.h" #include "Telemetry.h" #include "nsTHashtable.h" #include "nsHashKeys.h" @@ -54,6 +57,7 @@ #include "nsXULAppAPI.h" #include "nsThreadUtils.h" #include "mozilla/Mutex.h" +#include "mozilla/FileUtils.h" namespace { @@ -227,11 +231,9 @@ enum reflectStatus { }; enum reflectStatus -ReflectHistogramSnapshot(JSContext *cx, JSObject *obj, Histogram *h) +ReflectHistogramAndSamples(JSContext *cx, JSObject *obj, Histogram *h, + const Histogram::SampleSet &ss) { - Histogram::SampleSet ss; - h->SnapshotSample(&ss); - // We don't want to reflect corrupt histograms. if (h->FindCorruption(ss) != Histogram::NO_INCONSISTENCIES) { return REFLECT_CORRUPT; @@ -260,6 +262,14 @@ ReflectHistogramSnapshot(JSContext *cx, JSObject *obj, Histogram *h) return REFLECT_OK; } +enum reflectStatus +ReflectHistogramSnapshot(JSContext *cx, JSObject *obj, Histogram *h) +{ + Histogram::SampleSet ss; + h->SnapshotSample(&ss); + return ReflectHistogramAndSamples(cx, obj, h, ss); +} + JSBool JSHistogram_Add(JSContext *cx, uintN argc, jsval *vp) { @@ -677,6 +687,333 @@ TelemetryImpl::GetHistogramById(const nsACString &name, JSContext *cx, jsval *re return WrapAndReturnHistogram(h, cx, ret); } +class TelemetrySessionData : public nsITelemetrySessionData +{ + NS_DECL_ISUPPORTS + NS_DECL_NSITELEMETRYSESSIONDATA + +public: + static nsresult LoadFromDisk(nsIFile *, TelemetrySessionData **ptr); + static nsresult SaveToDisk(nsIFile *, const nsACString &uuid); + + TelemetrySessionData(const char *uuid); + ~TelemetrySessionData(); + +private: + struct EnumeratorArgs { + JSContext *cx; + JSObject *snapshots; + }; + typedef nsBaseHashtableET EntryType; + typedef nsTHashtable SessionMapType; + static PLDHashOperator ReflectSamples(EntryType *entry, void *arg); + SessionMapType mSampleSetMap; + nsCString mUUID; + + bool DeserializeHistogramData(Pickle &pickle, void **iter); + static bool SerializeHistogramData(Pickle &pickle); + + // The file format version. Should be incremented whenever we change + // how individual SampleSets are stored in the file. + static const unsigned int sVersion = 1; +}; + +NS_IMPL_THREADSAFE_ISUPPORTS1(TelemetrySessionData, nsITelemetrySessionData) + +TelemetrySessionData::TelemetrySessionData(const char *uuid) + : mUUID(uuid) +{ + mSampleSetMap.Init(); +} + +TelemetrySessionData::~TelemetrySessionData() +{ + mSampleSetMap.Clear(); +} + +NS_IMETHODIMP +TelemetrySessionData::GetUuid(nsACString &uuid) +{ + uuid = mUUID; + return NS_OK; +} + +PLDHashOperator +TelemetrySessionData::ReflectSamples(EntryType *entry, void *arg) +{ + struct EnumeratorArgs *args = static_cast(arg); + // This has the undesirable effect of creating a histogram for the + // current session with the given ID. But there's no good way to + // compute the ranges and buckets from scratch. + Histogram *h = nsnull; + nsresult rv = GetHistogramByEnumId(Telemetry::ID(entry->GetKey()), &h); + if (NS_FAILED(rv)) { + return PL_DHASH_STOP; + } + + // Don't reflect histograms with no data associated with them. + if (entry->mData.sum() == 0) { + return PL_DHASH_NEXT; + } + + JSObject *snapshot = JS_NewObject(args->cx, NULL, NULL, NULL); + if (!(snapshot + && ReflectHistogramAndSamples(args->cx, snapshot, h, entry->mData) + && JS_DefineProperty(args->cx, args->snapshots, + h->histogram_name().c_str(), + OBJECT_TO_JSVAL(snapshot), NULL, NULL, + JSPROP_ENUMERATE))) { + return PL_DHASH_STOP; + } + + return PL_DHASH_NEXT; +} + +NS_IMETHODIMP +TelemetrySessionData::GetSnapshots(JSContext *cx, jsval *ret) +{ + JSObject *snapshots = JS_NewObject(cx, NULL, NULL, NULL); + if (!snapshots) { + return NS_ERROR_FAILURE; + } + + struct EnumeratorArgs args = { cx, snapshots }; + PRUint32 count = mSampleSetMap.EnumerateEntries(ReflectSamples, + static_cast(&args)); + if (count != mSampleSetMap.Count()) { + return NS_ERROR_FAILURE; + } + + *ret = OBJECT_TO_JSVAL(snapshots); + return NS_OK; +} + +bool +TelemetrySessionData::DeserializeHistogramData(Pickle &pickle, void **iter) +{ + PRUint32 count = 0; + if (!pickle.ReadUInt32(iter, &count)) { + return false; + } + + for (size_t i = 0; i < count; ++i) { + int stored_length; + const char *name; + if (!pickle.ReadData(iter, &name, &stored_length)) { + return false; + } + + Telemetry::ID id; + nsresult rv = TelemetryImpl::GetHistogramEnumId(name, &id); + if (NS_FAILED(rv)) { + // We serialized a non-static histogram. Just drop its data on + // the floor. If we can't deserialize the data, though, we're in + // trouble. + Histogram::SampleSet ss; + if (!ss.Deserialize(iter, pickle)) { + return false; + } + } + + EntryType *entry = mSampleSetMap.GetEntry(id); + if (!entry) { + entry = mSampleSetMap.PutEntry(id); + if (NS_UNLIKELY(!entry)) { + return false; + } + if (!entry->mData.Deserialize(iter, pickle)) { + return false; + } + } + } + + return true; +} + +nsresult +TelemetrySessionData::LoadFromDisk(nsIFile *file, TelemetrySessionData **ptr) +{ + *ptr = nsnull; + nsresult rv; + nsCOMPtr f(do_QueryInterface(file, &rv)); + if (NS_FAILED(rv)) { + return rv; + } + + AutoFDClose fd; + rv = f->OpenNSPRFileDesc(PR_RDONLY, 0, &fd); + if (NS_FAILED(rv)) { + return NS_ERROR_FAILURE; + } + + PRInt32 size = PR_Available(fd); + if (size == -1) { + return NS_ERROR_FAILURE; + } + + nsAutoArrayPtr data(new char[size]); + PRInt32 amount = PR_Read(fd, data, size); + if (amount != size) { + return NS_ERROR_FAILURE; + } + + Pickle pickle(data, size); + void *iter = NULL; + + unsigned int storedVersion; + if (!(pickle.ReadUInt32(&iter, &storedVersion) + && storedVersion == sVersion)) { + return NS_ERROR_FAILURE; + } + + const char *uuid; + int uuidLength; + if (!pickle.ReadData(&iter, &uuid, &uuidLength)) { + return NS_ERROR_FAILURE; + } + + nsAutoPtr sessionData(new TelemetrySessionData(uuid)); + if (!sessionData->DeserializeHistogramData(pickle, &iter)) { + return NS_ERROR_FAILURE; + } + + *ptr = sessionData.forget(); + return NS_OK; +} + +bool +TelemetrySessionData::SerializeHistogramData(Pickle &pickle) +{ + StatisticsRecorder::Histograms hs; + StatisticsRecorder::GetHistograms(&hs); + + if (!pickle.WriteUInt32(hs.size())) { + return false; + } + + for (StatisticsRecorder::Histograms::const_iterator it = hs.begin(); + it != hs.end(); + ++it) { + const Histogram *h = *it; + const char *name = h->histogram_name().c_str(); + + Histogram::SampleSet ss; + h->SnapshotSample(&ss); + + if (!(pickle.WriteData(name, strlen(name)+1) + && ss.Serialize(&pickle))) { + return false; + } + } + + return true; +} + +nsresult +TelemetrySessionData::SaveToDisk(nsIFile *file, const nsACString &uuid) +{ + nsresult rv; + nsCOMPtr f(do_QueryInterface(file, &rv)); + if (NS_FAILED(rv)) { + return rv; + } + + AutoFDClose fd; + rv = f->OpenNSPRFileDesc(PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE, 0600, &fd); + if (NS_FAILED(rv)) { + return rv; + } + + Pickle pickle; + if (!pickle.WriteUInt32(sVersion)) { + return NS_ERROR_FAILURE; + } + + // Include the trailing NULL for the UUID to make reading easier. + const char *data; + size_t length = uuid.GetData(&data); + if (!(pickle.WriteData(data, length+1) + && SerializeHistogramData(pickle))) { + return NS_ERROR_FAILURE; + } + + PRInt32 amount = PR_Write(fd, static_cast(pickle.data()), + pickle.size()); + if (amount != pickle.size()) { + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +class SaveHistogramEvent : public nsRunnable +{ +public: + SaveHistogramEvent(nsIFile *file, const nsACString &uuid, + nsITelemetrySaveSessionDataCallback *callback) + : mFile(file), mUUID(uuid), mCallback(callback) + {} + + NS_IMETHOD Run() + { + nsresult rv = TelemetrySessionData::SaveToDisk(mFile, mUUID); + mCallback->Handle(!!NS_SUCCEEDED(rv)); + return rv; + } + +private: + nsCOMPtr mFile; + nsCString mUUID; + nsCOMPtr mCallback; +}; + +NS_IMETHODIMP +TelemetryImpl::SaveHistograms(nsIFile *file, const nsACString &uuid, + nsITelemetrySaveSessionDataCallback *callback, + bool isSynchronous) +{ + nsCOMPtr event = new SaveHistogramEvent(file, uuid, callback); + if (isSynchronous) { + return event ? event->Run() : NS_ERROR_FAILURE; + } else { + return NS_DispatchToCurrentThread(event); + } +} + +class LoadHistogramEvent : public nsRunnable +{ +public: + LoadHistogramEvent(nsIFile *file, + nsITelemetryLoadSessionDataCallback *callback) + : mFile(file), mCallback(callback) + {} + + NS_IMETHOD Run() + { + TelemetrySessionData *sessionData = nsnull; + nsresult rv = TelemetrySessionData::LoadFromDisk(mFile, &sessionData); + if (NS_FAILED(rv)) { + mCallback->Handle(nsnull); + } else { + nsCOMPtr data(sessionData); + mCallback->Handle(data); + } + return rv; + } + +private: + nsCOMPtr mFile; + nsCOMPtr mCallback; +}; + +NS_IMETHODIMP +TelemetryImpl::LoadHistograms(nsIFile *file, + nsITelemetryLoadSessionDataCallback *callback) +{ + nsCOMPtr event = new LoadHistogramEvent(file, callback); + return NS_DispatchToCurrentThread(event); +} + NS_IMETHODIMP TelemetryImpl::GetCanRecord(bool *ret) { *ret = mCanRecord; diff --git a/toolkit/components/telemetry/TelemetryPing.js b/toolkit/components/telemetry/TelemetryPing.js index 5aa510fb82e9..7a6c2a85f905 100644 --- a/toolkit/components/telemetry/TelemetryPing.js +++ b/toolkit/components/telemetry/TelemetryPing.js @@ -42,6 +42,7 @@ const Cu = Components.utils; Cu.import("resource://gre/modules/Services.jsm"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/LightweightThemeManager.jsm"); +Cu.import("resource://gre/modules/ctypes.jsm"); // When modifying the payload in incompatible ways, please bump this version number const PAYLOAD_VERSION = 1; @@ -179,6 +180,10 @@ TelemetryPing.prototype = { _histograms: {}, _initialized: false, _prevValues: {}, + // Generate a unique id once per session so the server can cope with + // duplicate submissions. + _uuid: generateUUID(), + _prevSession: null, /** * Returns a set of histograms that can be converted into JSON @@ -187,8 +192,7 @@ TelemetryPing.prototype = { * histogram_type: <0 for exponential, 1 for linear>, bucketX:countX, ....} ...} * where bucket[XY], count[XY] are positive integers. */ - getHistograms: function getHistograms() { - let hls = Telemetry.histogramSnapshots; + getHistograms: function getHistograms(hls) { let info = Telemetry.registeredHistograms; let ret = {}; @@ -397,24 +401,48 @@ TelemetryPing.prototype = { send: function send(reason, server) { // populate histograms one last time this.gatherMemory(); + let data = this.getSessionPayloadAndSlug(reason); + + // Don't record a successful ping for previous session data. + this.doPing(server, data.slug, data.payload, !data.previous); + this._prevSession = null; + + // We were sending off data from before; now send the actual data + // we've collected this session. + if (data.previous) { + data = this.getSessionPayloadAndSlug(reason); + this.doPing(server, data.slug, data.payload, true); + } + }, + + getSessionPayloadAndSlug: function getSessionPayloadAndSlug(reason) { + // Use a deterministic url for testing. + let isTestPing = (reason == "test-ping"); + let havePreviousSession = !!this._prevSession; + let slug = (isTestPing + ? reason + : (havePreviousSession + ? this._prevSession.uuid + : this._uuid)); let payloadObj = { ver: PAYLOAD_VERSION, - info: this.getMetadata(reason), - simpleMeasurements: getSimpleMeasurements(), - histograms: this.getHistograms(), - slowSQL: Telemetry.slowSQL + // Send a different reason string for previous session data. + info: this.getMetadata(havePreviousSession ? "saved-session" : reason), }; + if (havePreviousSession) { + payloadObj.histograms = this.getHistograms(this._prevSession.snapshots); + } + else { + payloadObj.simpleMeasurements = getSimpleMeasurements(); + payloadObj.histograms = this.getHistograms(Telemetry.histogramSnapshots); + payloadObj.slowSQL = Telemetry.slowSQL; + } + return { previous: !!havePreviousSession, slug: slug, payload: JSON.stringify(payloadObj) }; + }, - let isTestPing = (reason == "test-ping"); - // Generate a unique id once per session so the server can cope with duplicate submissions. - // Use a deterministic url for testing. - if (!this._path) - this._path = "/submit/telemetry/" + (isTestPing ? reason : generateUUID()); - - let hping = Telemetry.getHistogramById("TELEMETRY_PING"); - let hsuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS"); - - let url = server + this._path; + doPing: function doPing(server, slug, payload, recordSuccess) { + let submitPath = "/submit/telemetry/" + slug; + let url = server + submitPath; let request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"] .createInstance(Ci.nsIXMLHttpRequest); request.mozBackgroundRequest = true; @@ -423,6 +451,7 @@ TelemetryPing.prototype = { request.setRequestHeader("Content-Type", "application/json"); let startTime = new Date(); + let file = this.savedHistogramsFile(); function finishRequest(channel) { let success = false; @@ -430,15 +459,23 @@ TelemetryPing.prototype = { success = channel.QueryInterface(Ci.nsIHttpChannel).requestSucceeded; } catch(e) { } - hsuccess.add(success); - hping.add(new Date() - startTime); - if (isTestPing) + if (recordSuccess) { + let hping = Telemetry.getHistogramById("TELEMETRY_PING"); + let hsuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS"); + + hsuccess.add(success); + hping.add(new Date() - startTime); + } + if (success && file.exists()) { + file.remove(true); + } + if (slug == "test-ping") Services.obs.notifyObservers(null, "telemetry-test-xhr-complete", null); } request.addEventListener("error", function(aEvent) finishRequest(request.channel), false); request.addEventListener("load", function(aEvent) finishRequest(request.channel), false); - request.send(JSON.stringify(payloadObj)); + request.send(payload); }, attachObservers: function attachObservers() { @@ -459,6 +496,25 @@ TelemetryPing.prototype = { } }, + savedHistogramsFile: function savedHistogramsFile() { + let profileDirectory = Services.dirsvc.get("ProfD", Ci.nsILocalFile); + let profileFile = profileDirectory.clone(); + + // There's a bunch of binary data in the file, so we need to be + // sensitive to multiple machine types. Use ctypes to get some + // discriminating information. + let size = ctypes.voidptr_t.size; + // Hack to figure out endianness. + let uint32_array_t = ctypes.uint32_t.array(1); + let array = uint32_array_t([0xdeadbeef]); + let uint8_array_t = ctypes.uint8_t.array(4); + let array_as_bytes = ctypes.cast(array, uint8_array_t); + let endian = (array_as_bytes[0] === 0xde) ? "big" : "little" + let name = "sessionHistograms.dat." + size + endian; + profileFile.append(name); + return profileFile; + }, + /** * Initializes telemetry within a timer. If there is no PREF_SERVER set, don't turn on telemetry. */ @@ -479,6 +535,7 @@ TelemetryPing.prototype = { Services.obs.addObserver(this, "private-browsing", false); Services.obs.addObserver(this, "profile-before-change", false); Services.obs.addObserver(this, "sessionstore-windows-restored", false); + Services.obs.addObserver(this, "quit-application-granted", false); // Delay full telemetry initialization to give the browser time to // run various late initializers. Otherwise our gathered memory @@ -492,6 +549,12 @@ TelemetryPing.prototype = { delete self._timer } this._timer.initWithCallback(timerCallback, TELEMETRY_DELAY, Ci.nsITimer.TYPE_ONE_SHOT); + + // Load data from the previous session. + let loadCallback = function(data) { + self._prevSession = data; + } + Telemetry.loadHistograms(this.savedHistogramsFile(), loadCallback); }, /** @@ -502,6 +565,7 @@ TelemetryPing.prototype = { Services.obs.removeObserver(this, "sessionstore-windows-restored"); Services.obs.removeObserver(this, "profile-before-change"); Services.obs.removeObserver(this, "private-browsing"); + Services.obs.removeObserver(this, "quit-application-granted"); }, /** @@ -567,6 +631,11 @@ TelemetryPing.prototype = { } this.send(aTopic == "idle" ? "idle-daily" : aTopic, server); break; + case "quit-application-granted": + Telemetry.saveHistograms(this.savedHistogramsFile(), + this._uuid, function (success) success, + /*isSynchronous=*/true); + break; } }, diff --git a/toolkit/components/telemetry/nsITelemetry.idl b/toolkit/components/telemetry/nsITelemetry.idl index 5e82bf126f0a..b001b48ed8d9 100644 --- a/toolkit/components/telemetry/nsITelemetry.idl +++ b/toolkit/components/telemetry/nsITelemetry.idl @@ -38,8 +38,39 @@ * ***** END LICENSE BLOCK ***** */ #include "nsISupports.idl" +#include "nsIFile.idl" -[scriptable, uuid(db854295-478d-4de9-8211-d73ed7d81cd0)] +[scriptable, uuid(c177b6b0-5ef1-44f5-bc67-6bcf7d2518e5)] +interface nsITelemetrySessionData : nsISupports +{ + /** + * The UUID of our previous session. + */ + readonly attribute ACString uuid; + + /** + * An object containing a snapshot from all registered histograms that had + * data recorded in the previous session. + * { name1: data1, name2: data2, .... } + * where the individual dataN are as nsITelemetry.histogramSnapshots. + */ + [implicit_jscontext] + readonly attribute jsval snapshots; +}; + +[scriptable, function, uuid(aff36c9d-7e4c-41ab-a9b6-53773bbca0cd)] +interface nsITelemetryLoadSessionDataCallback : nsISupports +{ + void handle(in nsITelemetrySessionData data); +}; + +[scriptable, function, uuid(40065f26-afd2-4417-93de-c1de9adb1548)] +interface nsITelemetrySaveSessionDataCallback : nsISupports +{ + void handle(in bool success); +}; + +[scriptable, uuid(22fc825e-288f-457e-80d5-5bb35f06d37e)] interface nsITelemetry : nsISupports { /** @@ -127,6 +158,34 @@ interface nsITelemetry : nsISupports [implicit_jscontext] jsval getHistogramById(in ACString id); + /** + * Save persistent histograms to the given file. + * + * @param file - filename for saving + * @param uuid - UUID of this session + * @param callback - function to be caled when file writing is complete + * @param isSynchronous - whether the save is done synchronously. Defaults + * to asynchronous saving. + */ + void saveHistograms(in nsIFile file, in ACString uuid, + in nsITelemetrySaveSessionDataCallback callback, + [optional] in boolean isSynchronous); + + /* Reconstruct an nsITelemetryDataSession object containing histogram + * information from the given file; the file must have been produced + * via saveHistograms. The file is read asynchronously. + * + * This method does not modify the histogram information being + * collected in the current session. + * + * The reconstructed object is then passed to the given callback. + * + * @param file - the file to load histogram information from + * @param callback - function to process histogram information + */ + void loadHistograms(in nsIFile file, + in nsITelemetryLoadSessionDataCallback callback); + /** * Set this to false to disable gathering of telemetry statistics. */ diff --git a/toolkit/components/telemetry/tests/unit/test_nsITelemetry.js b/toolkit/components/telemetry/tests/unit/test_nsITelemetry.js index e0711d7cedda..56dc92afbd81 100644 --- a/toolkit/components/telemetry/tests/unit/test_nsITelemetry.js +++ b/toolkit/components/telemetry/tests/unit/test_nsITelemetry.js @@ -105,6 +105,45 @@ function test_privateMode() { do_check_neq(uneval(orig), uneval(h.snapshot())); } +function generateUUID() { + let str = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator).generateUUID().toString(); + // strip {} + return str.substring(1, str.length - 1); +} + +// Check that we do sane things when saving to disk. +function test_loadSave() +{ + let dirService = Cc["@mozilla.org/file/directory_service;1"] + .getService(Ci.nsIProperties); + let tmpDir = dirService.get("TmpD", Ci.nsILocalFile); + let tmpFile = tmpDir.clone(); + tmpFile.append("saved-histograms.dat"); + if (tmpFile.exists()) { + tmpFile.remove(true); + } + + let saveFinished = false; + let loadFinished = false; + let uuid = generateUUID(); + let loadCallback = function(data) { + do_check_true(data != null); + do_check_eq(uuid, data.uuid); + loadFinished = true; + do_test_finished(); + }; + let saveCallback = function(success) { + do_check_true(success); + Telemetry.loadHistograms(tmpFile, loadCallback); + saveFinished = true; + }; + do_test_pending(); + Telemetry.saveHistograms(tmpFile, uuid, saveCallback); + do_register_cleanup(function () do_check_true(saveFinished)); + do_register_cleanup(function () do_check_true(loadFinished)); + do_register_cleanup(function () tmpFile.exists() && tmpFile.remove(true)); +} + function run_test() { let kinds = [Telemetry.HISTOGRAM_EXPONENTIAL, Telemetry.HISTOGRAM_LINEAR] @@ -121,4 +160,6 @@ function run_test() test_getHistogramById(); test_getSlowSQL(); test_privateMode(); + test_loadSave(); + }