bug 585196: Telemetry clientside r=Mossop

modules/libpref/src/init/all.js

@@ -272,6 +272,11 @@ pref("toolkit.scrollbox.smoothScroll", true);
 pref("toolkit.scrollbox.scrollIncrement", 20);
 pref("toolkit.scrollbox.clickToScroll.scrollDelay", 150);
 
+// Telemetry
+pref("toolkit.telemetry.enabled", false);
+// Telemetry test server to be used until the official one is public
+pref("toolkit.telemetry.server", "http://telemetry.allizom.org");
+
 // view source
 pref("view_source.syntax_highlight", true);
 pref("view_source.wrap_long_lines", false);

toolkit/components/Makefile.in

@@ -72,6 +72,7 @@ PARALLEL_DIRS += \
   viewconfig \
   viewsource \
   webapps \
+  telemetry \
   $(NULL)
 
 ifdef BUILD_CTYPES

toolkit/components/telemetry/Makefile.in

@@ -0,0 +1,57 @@
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# The Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#  Taras Glek <tglek@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+DEPTH = ../../..
+topsrcdir = @top_srcdir@
+srcdir = @srcdir@
+VPATH = @srcdir@
+
+include $(DEPTH)/config/autoconf.mk
+
+MODULE = telemetry
+
+EXTRA_COMPONENTS = \
+  TelemetryPing.manifest \
+  TelemetryPing.js \
+  $(NULL)
+
+ifdef ENABLE_TESTS
+DIRS += tests
+endif
+
+include $(topsrcdir)/config/rules.mk

toolkit/components/telemetry/TelemetryPing.js

@@ -0,0 +1,270 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is the nsTryToClose component.
+ *
+ * The Initial Developer of the Original Code is
+ * the Mozilla Foundation
+ * Portions created by the Initial Developer are Copyright (C) 2011
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Taras Glek <tglek@mozilla.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+const PREF_SERVER = "toolkit.telemetry.server";
+const PREF_ENABLED = "toolkit.telemetry.enabled";
+// Do not gather data more than once a minute
+const TELEMETRY_INTERVAL = 60;
+// Delay before intializing telemetry (ms)
+const TELEMETRY_DELAY = 60000;
+// about:memory values to turn into histograms
+const MEM_HISTOGRAMS = {
+  "heap-used/js/gc-heap": [1024, 1024 * 500, 10],
+  "mapped/heap/used": [1024, 2 * 1024 * 1024, 10],
+  "heap-used/layout/all": [1024, 50 * 1025, 10]
+};
+
+XPCOMUtils.defineLazyGetter(this, "Telemetry", function () {
+  return Cc["@mozilla.org/base/telemetry;1"].getService(Ci.nsITelemetry);
+});
+
+/**
+ * Returns a set of histograms that can be converted into JSON
+ * @return a snapshot of the histograms of form:
+ *  { histogram_name: {range:[minvalue,maxvalue], bucket_count:<number of buckets>,
+ *    histogram_type: <0 for exponential, 1 for linear>, bucketX:countX, ....} ...}
+ * where bucket[XY], count[XY] are positive integers.
+ */
+function getHistograms() {
+  let hls = Telemetry.histogramSnapshots;
+  let ret = {};
+
+  for (let key in hls) {
+    let hgram = hls[key];
+    let r = hgram.ranges;
+    let c = hgram.counts;
+    let retgram = {
+      range: [r[1], r[r.length - 1]],
+      bucket_count: r.length,
+      histogram_type: hgram.histogram_type,
+      values: {}
+    };
+    let first = true;
+    let last = 0;
+
+    for (let i = 0; i < c.length; i++) {
+      let value = c[i];
+      if (!value)
+        continue;
+
+      // add a lower bound
+      if (i && first) {
+        first = false;
+        retgram.values[r[i - 1]] = 0;
+      }
+      last = i + 1;
+      retgram.values[r[i]] = value;
+    }
+
+    // add an upper bound
+    if (last && last < c.length)
+      retgram.values[r[last]] = 0;
+    ret[key] = retgram;
+  }
+  return ret;
+}
+
+function generateUUID() {
+  return Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator)
+         .generateUUID().toString();
+}
+
+/**
+ * @return request metadata
+ */
+function getMetadata(reason) {
+  let si = Cc["@mozilla.org/toolkit/app-startup;1"].
+           getService(Ci.nsIAppStartup).getStartupInfo();
+  let ai = Services.appinfo;
+  let ret = {
+    uptime: (new Date() - si.process),
+    reason: reason,
+    OS: ai.OS,
+    XPCOMABI: ai.XPCOMABI,
+    ID: ai.ID,
+    vesion: ai.version,
+    name: ai.name,
+    appBuildID: ai.appBuildID,
+    platformBuildID: ai.platformBuildID,
+  }
+  return ret;
+}
+
+function TelemetryPing() {}
+
+TelemetryPing.prototype = {
+  _histograms: {},
+
+  /**
+   * Pull values from about:memory into corresponding histograms
+   */
+  gatherMemory: function gatherMemory() {
+    let mgr;
+    try {
+      mgr = Cc["@mozilla.org/memory-reporter-manager;1"].
+            getService(Ci.nsIMemoryReporterManager);
+    } catch (e) {
+      // OK to skip memory reporters in xpcshell
+      return
+    }
+
+    let e = mgr.enumerateReporters();
+    let memReporters = {};
+    while (e.hasMoreElements()) {
+      let mr = e.getNext().QueryInterface(Ci.nsIMemoryReporter);
+      //  memReporters[mr.path] = mr.memoryUsed;
+      let specs = MEM_HISTOGRAMS[mr.path];
+      if (!specs) {
+        continue;
+      }
+
+      let name = "Memory:" + mr.path + " (KB)";
+      let h = this._histograms[name];
+      if (!h) {
+        h = Telemetry.newExponentialHistogram(name, specs[0], specs[1], specs[2]);
+        this._histograms[name] = h;
+      }
+      let v = Math.floor(mr.memoryUsed / 1024);
+      h.add(v);
+    }
+    return memReporters;
+  },
+  
+  /**
+   * Send data to the server. Record success/send-time in histograms
+   */
+  send: function send(reason, server) {
+    // populate histograms one last time
+    this.gatherMemory();
+    let nativeJSON = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
+    let payload = {
+      info: getMetadata(reason),
+      histograms: getHistograms()
+    };
+    let isTestPing = (reason == "test-ping");
+    // Generate a unique id once per session so the server can cope with duplicate submissions.
+    // Use a deterministic url for testing.
+    if (!this._path)
+      this._path = "/submit/telemetry/" + (isTestPing ? reason : generateUUID());
+    
+    const TELEMETRY_PING = "telemetry.ping (ms)";
+    const TELEMETRY_SUCCESS = "telemetry.success (No, Yes)";
+
+    let hping = Telemetry.newExponentialHistogram(TELEMETRY_PING, 1, 3000, 10);
+    let hsuccess = Telemetry.newLinearHistogram(TELEMETRY_SUCCESS, 1, 2, 3);
+
+    let url = server + this._path;
+    let request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+                  .createInstance(Ci.nsIXMLHttpRequest);
+    request.mozBackgroundRequest = true;
+    request.open("POST", url, true);
+    request.overrideMimeType("text/plain");
+
+    let startTime = new Date()
+
+    function finishRequest(success_metric) {
+      hsuccess.add(success_metric);
+      hping.add(new Date() - startTime);
+      if (isTestPing)
+        Services.obs.notifyObservers(null, "telemetry-test-xhr-complete", null);
+    }
+    request.onerror = function(aEvent) finishRequest(1);
+    request.onload = function(aEvent) finishRequest(2);
+    request.send(nativeJSON.encode(payload));
+  },
+  
+  /**
+   * Initializes telemetry within a timer. If there is no PREF_SERVER set, don't turn on telemetry.
+   */
+  setup: function setup() {
+    let enabled = false; 
+    try {
+      enabled = Services.prefs.getBoolPref(PREF_ENABLED);
+      this._server = Services.prefs.getCharPref(PREF_SERVER);
+    } catch (e) {
+      // Prerequesite prefs aren't set
+    }
+    if (!enabled) 
+      return;
+  
+    let self = this;
+    this._timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+    let timerCallback = function() {
+      let idleService = Cc["@mozilla.org/widget/idleservice;1"].
+                        getService(Ci.nsIIdleService);
+      idleService.addIdleObserver(self, TELEMETRY_INTERVAL); 
+      self.gatherMemory();
+    }
+    this._timer.initWithCallback(timerCallback, TELEMETRY_DELAY, Ci.nsITimer.TYPE_ONE_SHOT);
+  },
+
+  /**
+   * This observer drives telemetry.
+   */
+  observe: function (aSubject, aTopic, aData) {
+    // Allows to change the server for testing
+    var server = this._server;
+
+    switch (aTopic) {
+    case "profile-after-change":
+      this.setup();
+      break;
+    case "idle":
+      this.gatherMemory();
+      break;
+    case "test-ping":
+      server = aData;
+      // fall through
+    case "idle-daily":
+      this.send(aTopic, server);
+      break;
+    }
+  },
+
+  classID: Components.ID("{55d6a5fa-130e-4ee6-a158-0133af3b86ba}"),
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
+};
+
+let NSGetFactory = XPCOMUtils.generateNSGetFactory([TelemetryPing]);

toolkit/components/telemetry/TelemetryPing.manifest

@@ -0,0 +1,3 @@
+component {55d6a5fa-130e-4ee6-a158-0133af3b86ba} TelemetryPing.js
+contract @mozilla.org/base/telemetry-ping;1 {55d6a5fa-130e-4ee6-a158-0133af3b86ba}
+category profile-after-change TelemetryPing @mozilla.org/base/telemetry-ping;1

toolkit/components/telemetry/tests/Makefile.in

@@ -0,0 +1,53 @@
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Mozilla.org.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Taras Glek <tglek@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either of the GNU General Public License Version 2 or later (the "GPL"),
+# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+DEPTH		= ../../../..
+topsrcdir	= @top_srcdir@
+srcdir		= @srcdir@
+VPATH		= @srcdir@
+relativesrcdir = toolkit/components/telemetry/test
+
+include $(DEPTH)/config/autoconf.mk
+
+#MODULE		= test_harness_telemetry
+
+XPCSHELL_TESTS = \
+                 unit \
+                 $(NULL)
+
+include $(topsrcdir)/config/rules.mk

toolkit/components/telemetry/tests/unit/test_TelemetryPing.js

@@ -0,0 +1,140 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ 
+*/
+/* This testcase triggers two telemetry pings.
+ * 
+ * Telemetry code keeps histograms of past telemetry pings. The first
+ * ping populates these histograms. One of those histograms is then
+ * checked in the second request.
+ */
+
+do_load_httpd_js();
+Cu.import("resource://gre/modules/Services.jsm");
+
+const PATH = "/submit/telemetry/test-ping"
+const BinaryInputStream = Components.Constructor(
+  "@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream",
+  "setInputStream");
+
+var httpserver = new nsHttpServer();
+
+function telemetry_ping () {
+  let tp = Cc["@mozilla.org/base/telemetry-ping;1"].getService(Ci.nsIObserver);
+  tp.observe(tp, "test-ping", "http://localhost:4444");
+}
+
+function telemetryObserver(aSubject, aTopic, aData) {
+  Services.obs.removeObserver(telemetryObserver, aTopic);
+  httpserver.registerPathHandler(PATH, checkHistograms);
+  telemetry_ping();
+}
+
+function run_test() {
+  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  httpserver.start(4444);
+
+  Services.obs.addObserver(telemetryObserver, "telemetry-test-xhr-complete", false);
+
+  telemetry_ping();
+  // spin the event loop
+  do_test_pending();
+}
+
+function readBytesFromInputStream(inputStream, count) {
+  if (!count) {
+    count = inputStream.available();
+  }
+  return new BinaryInputStream(inputStream).readBytes(count);
+}
+
+function checkHistograms(request, response) {
+  // do not need the http server anymore
+  httpserver.stop(do_test_finished);
+  let s = request.bodyInputStream
+  let payload = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON)
+                                             .decode(readBytesFromInputStream(s))
+
+  do_check_true(payload.info.uptime >= 0)
+
+  // get rid of the non-deterministic field
+  payload.info.uptime = 0;
+  const expected_info = {
+    uptime: 0,
+    reason: "test-ping",
+    OS: "XPCShell", 
+    XPCOMABI: "noarch-spidermonkey", 
+    ID: "xpcshell@tests.mozilla.org", 
+    vesion: "1", 
+    name: "XPCShell", 
+    appBuildID: "2007010101",
+    platformBuildID: "2007010101"
+  };
+
+  do_check_eq(uneval(payload.info), 
+              uneval(expected_info));
+
+  const TELEMETRY_PING = "telemetry.ping (ms)";
+  const TELEMETRY_SUCCESS = "telemetry.success (No, Yes)";
+  do_check_true(TELEMETRY_PING in payload.histograms)
+
+  // There should be one successful report from the previos telemetry ping
+  const expected_tc = {
+    range: [1, 2],
+    bucket_count: 3,
+    histogram_type: 1,
+    values: {1:0, 2:1}
+  }
+  let tc = payload.histograms[TELEMETRY_SUCCESS]
+  do_check_eq(uneval(tc), 
+              uneval(expected_tc));
+}
+
+// copied from toolkit/mozapps/extensions/test/xpcshell/head_addons.js
+const XULAPPINFO_CONTRACTID = "@mozilla.org/xre/app-info;1";
+const XULAPPINFO_CID = Components.ID("{c763b610-9d49-455a-bbd2-ede71682a1ac}");
+
+function createAppInfo(id, name, version, platformVersion) {
+  gAppInfo = {
+    // nsIXULAppInfo
+    vendor: "Mozilla",
+    name: name,
+    ID: id,
+    version: version,
+    appBuildID: "2007010101",
+    platformVersion: platformVersion,
+    platformBuildID: "2007010101",
+
+    // nsIXULRuntime
+    inSafeMode: false,
+    logConsoleErrors: true,
+    OS: "XPCShell",
+    XPCOMABI: "noarch-spidermonkey",
+    invalidateCachesOnRestart: function invalidateCachesOnRestart() {
+      // Do nothing
+    },
+
+    // nsICrashReporter
+    annotations: {},
+
+    annotateCrashReport: function(key, data) {
+      this.annotations[key] = data;
+    },
+
+    QueryInterface: XPCOMUtils.generateQI([Ci.nsIXULAppInfo,
+                                           Ci.nsIXULRuntime,
+                                           Ci.nsICrashReporter,
+                                           Ci.nsISupports])
+  };
+
+  var XULAppInfoFactory = {
+    createInstance: function (outer, iid) {
+      if (outer != null)
+        throw Components.results.NS_ERROR_NO_AGGREGATION;
+      return gAppInfo.QueryInterface(iid);
+    }
+  };
+  var registrar = Components.manager.QueryInterface(Ci.nsIComponentRegistrar);
+  registrar.registerFactory(XULAPPINFO_CID, "XULAppInfo",
+                            XULAPPINFO_CONTRACTID, XULAppInfoFactory);
+}