Merge pull request #574 from mozilla/data

New data format (fix #515)

data/lightbeam.js

@@ -141,21 +141,6 @@ function resetAdditionalUI() {
 }
 
 
-/****************************************
- *   Upload data
- */
-
-
-global.startSharing = function startSharing(askForConfirmation, callback) {
-  if (askForConfirmation) {
-    askForDataSharingConfirmationDialog(function (confirmed) {
-      callback(confirmed);
-    });
-  } else {
-    callback(true);
-  }
-};
-
 /****************************************
  *   Format date string
  */

data/ui.js

@@ -87,9 +87,10 @@ document.querySelector(".toggle-btn.share-btn").addEventListener("click",
     }
   });
 
-global.confirmStartSharing = function confirmStartSharing(askForConfirmation, elmClicked) {
-  global.startSharing(askForConfirmation, function (confirmed) {
+function confirmStartSharing(askForConfirmation, elmClicked) {
+  let callback = function (confirmed) {
     if (confirmed) {
+      console.log("Sharing confirmed!");
       toggleBtnOnEffect(document.querySelector(".share-btn"));
       global.self.port.emit("prefChanged", {
         "contributeData": true
@@ -97,7 +98,13 @@ global.confirmStartSharing = function confirmStartSharing(askForConfirmation, el
     } else {
       elmClicked.checked = false;
     }
-  });
+  };
+  if (askForConfirmation) {
+    askForDataSharingConfirmationDialog(callback);
+  } else {
+    callback(true);
+  }
+
 };
 
 global.confirmStopSharing = function confirmStopSharing(elmClicked) {

doc/data_format.v1.2.md

@@ -0,0 +1,59 @@
+# File Formats in Lightbeam for Firefox
+
+As we have gained experience with Lightbeam for Firefox and built more visualization, talked with folks in the security arena, etc., we've identified more information that would be useful to track than what was included in Format 1.0. This documents the new format, paying special attention to the extensions.
+
+Version 1.2 is an extension of 1.1.
+New in Version 1.2: userId, userAgentData.
+
+## Format 1.2
+
+This format has the following structure:
+
+A root object whose keys are `format`, `version`, `connections`, `userId`, and `userAgentData`.
+
+The `format` value is the string "Lightbeam Save File" and is for documentation and identification of JSON files which are Lightbeam for Firefox-specific. 
+
+The `version` value is the string "1.2" and identifies the specific format documented here. A missing `version` key is the same as a `version` value of "0" and should be parsed and interpreted as specified for Format 0 above. 
+
+The `userId` value is a string with the format <identifier>:<timestamp>. User
+Ids will never be shared publicly. User IDs are rotated every three months and
+necessary to compute metrics such as the number of users who contribute data,
+and any metrics or heuristics that rely on knowing the sample size for
+statistical significance or confidence intervals. For example, calculating the
+mean number of 3rd party cookies received by any Lightbeam user within a given
+confidence interval requires knowing the sample size. For more information, see
+https://docs.google.com/a/mozilla.com/document/d/1ai2Fhl_DjqdXcIpmmeu9qJmGcaUmKngvFJ66T8yEe4s/edit?pli=1#heading=h.343bsagct5ij.
+
+The `userAgentData` value is an object containing information about the version
+of Firefox in use, as well as addons and preferences that may affect the
+behavior of Lightbeam.
+
+The `connections` value makes up the bulk of the file. It is an array of connection array objects, where each connection is represented as an array of values in the following order: `[source, target, timestamp, contentType, cookie, sourceVisited, secure, sourcePathDepth, sourceQueryDepth, sourceSub, targetSub, method, status, cacheable]`. Because we will be storing a lot more information over time than the older format, we do not store keys repetitively with each connection, but effectively a 14-tuple corresponding closely to a database row.
+
+The `source` value is a URL containing domain and subdomain information for the requested site, but stripped of protocol, path, query, and fragment. See notes for "target" for changes from 0.0 and 1.0 formats.
+
+The `target` value is a URL containing domain and subdomain information for a resource loaded from a third-party site, and like the source the target is stripped of protocol, path, query, and fragment. Connections which differ only by subdomain are not considered third-party content, which means that if you visit example.com and it loads content from ads.example.com, those connections will not be tracked by Lightbeam for Firefox. This is changed from both 0.0 and 1.0. In 0.0 we did not track subdomain information at all, and in 1.0 we kept subdomains in the "source" and "target" attributes. Now we have reverted to the 0.0 form of stripping subdomains from "source" and "target", but we store the subdomain information in the "sourceSub" and "targetSub" attributes (which will be empty if there is no subdomain).
+
+The `timestamp` is an integer number of milliseconds since the Unix epoch (January 1, 1970) as normally used for Javascript Date objects. The granularity of the timestamp is intentionally reduced when this data is shared, by rounding timestamps down to the last 5 minutes to prevent trackers from comparing our data with theirs to re-associate our data with individual users. Note, this is a change from the earlier (version 0) data format which only stored timestamps relative to the time Lightbeam for Firefox was started, and couldn't be used to restore actual session dates or times.
+
+The `contentType` value is the string reported by the target in the `Content-Type` header, although we MAY also compare with the actual type returned to improve the accuracy of this value. If there is no Content-Type header we WILL attempt to determine the type of the content. If all attempts fail, a default content type of "text/plain" (the standard default content type) will be used, but the value WILL NOT be null.
+
+The `cookie` value is either `true` or `false` representing the existence of one or more `Set-Cookie` headers returned by the target.
+
+The `sourceVisited` value is either `true` or `false`, indicating whether or not the source was loaded by the user in a page or tab. While it is expected that this value will generally be true, it may be false for sources in iframes.
+
+The `secure` value will be `true` for content loaded via the `HTTPS` protocol, `false` for content loaded via `HTTP` protocol. No other protocols are currently tracked by Lightbeam for Firefox as connections.
+
+The `sourcePathDepth` is a metric of how many path elements there were in the source URL before it was sanitized. The URL http://example.com/ has a depth of 0, while the URL http://example.com/blog/post/2012/12/21 has a depth of 5.
+
+The `sourceQueryDepth` is a metric of how many items there were in the query string. This is not a test of unique keys, just simple breaking after the "?" and splitting on "&" and ";". Again, http://example.com/ has a depth of 0, as does http://example.com/?, while http://example.com/?captain=kirk&ship=enterprise, http://example.com/?captain=kirk&captain=picard, and http://example.com/?captain=kirk;ship=enterprise all have a depth of 2.
+
+The `sourceSub` is the remainder of the domain after stripping off the top-level domain (so for "ec2.amazon.com", the source would be "amazon.com" and the source-sub would be "ec2").
+
+The `targetSub` is the same as "sourceSub", but for targets.
+
+The `method` attribute is whether this connection was loaded via GET, POST, PUT, etc.
+
+The `status` attribute is the numeric status of the response (200, 404, 500, etc.) as an integer.
+
+The `cacheable` attribute will be false if the server responded with any of a variety of mechanisms (as headers) for preventing cacheing such as "Cache-control: no-cache", "Pragma: no-cache", "Expires: 0", or "Expires" with a date value in the past relative to the value of the "Date" header.

lib/persist.js

@@ -22,13 +22,49 @@ const STORAGE_KEYS = [
 function serializeConnections(connections) {
   let exportSet = {
     format: 'Lightbeam Save File',
-    version: '1.1',
+    version: '1.2',
+    userId: storage.userId,
+    userAgentData: getUserAgentData(),
     uploadTime: Date.now(),
     connections: connections
   };
+  console.debug(JSON.stringify(exportSet));
   return JSON.stringify(exportSet);
 }
 
+function getUserAgentData() {
+  let retval = {};
+  let app = require("sdk/system/xul-app");
+  retval.appname = app.name;
+  retval.fxVersion = app.version;
+  let prefService = require("sdk/preferences/service");
+  let prefs = [ "app.update.channel",
+                "network.cookie.behavior",
+                "privacy.donottrackheader.enabled",
+                "privacy.donottrack.header.value" ];
+  prefs.forEach(function(p) { retval[p] = prefService.get(p); });
+  retval.addons = getAddons();
+  console.debug(JSON.stringify(retval));
+  return retval;
+}
+
+function getAddons() {
+  const { Cu } = require('chrome');
+  let { AddonManager } = Cu.import("resource://gre/modules/AddonManager.jsm");
+  let addons = [];
+  AddonManager.getAllAddons(function(addonList) {
+    addonList.forEach(function(addon) {
+      let o = {};
+      let states = ['id', 'name', 'appDisabled', 'isActive', 'type',
+                    'userDisabled'];
+      states.forEach(function(state) { o[state] = addon[state]; });
+      addons.push(o);
+    });
+  });
+  console.debug(JSON.stringify(addons));
+  return addons;
+}
+
 function upload(connections) {
   console.debug("received upload event in addon");
   let uploadServer = 'https://data.mozilla.com/submit/lightbeam';
@@ -38,7 +74,7 @@ function upload(connections) {
     onComplete: function (response) {
       let status = Number(response.status);
       if (status >= 200 && status < 300) {
-        console.debug("successful upload: ", response.text);
+        console.log("successful upload: ", response.text);
       } else {
         // Ignore errors for now. We could save last upload time and try
         // uploading again the ones that failed previously.
@@ -74,11 +110,31 @@ exports.storeConnections = function storeConnections(connections) {
 exports.reset = function reset() {
   storage.connections.length = 0;
   storage.blockmap = {};
+  storage.userId = generateUserId();
   prefs.contributeData = false;
   prefs.defaultVisualization = "graph";
   prefs.defaultFilter = "daily";
 };
 
+// Generate a new user id.
+function generateUserId() {
+  // Short hex string.
+  let userId = Math.floor(0xFFFFFFFF * Math.random()).toString(16);
+  storage.userId = userId + ":" + Date.now();
+  return storage.userId;
+}
+
+// Possibly rotate the user id.
+function maybeRotateUserId(forceChange) {
+  let parts = storage.userId.split(":");
+  // 90 days in ms
+  let MAX_LIFETIME_MS = 90 * 24 * 60 * 60 * 1000;
+  let timeToChange = Date(parts[1] + MAX_LIFETIME_MS);
+  if (forceChange || Date.now() >= timeToChange) {
+    generateUserId();
+  }
+}
+
 // Initialize all of our storage
 if (!storage.connections) {
   storage.connections = [];
@@ -88,4 +144,11 @@ if (!storage.blockmap) {
   storage.blockmap = {};
 }
 
+if (!storage.userId) {
+  generateUserId();
+}
+
+// Rotate user id if necessary
+maybeRotateUserId();
+
 console.log('Current quota usage:', Math.round(ss.quotaUsage * 100));

lib/ui.js

@@ -157,6 +157,7 @@ function attachToLightbeamPage(worker) {
   }
 
   function onPrefChanged(event) {
+    console.debug("Received updated prefs", JSON.stringify(event));
     if ("contributeData" in event) {
       prefs.contributeData = event.contributeData;
     }