Bug 808126 - Crash report collection for Firefox Health Report; r=rnewman

services/healthreport/HealthReportComponents.manifest

@@ -11,6 +11,7 @@ category app-startup HealthReportService service,@mozilla.org/healthreport/servi
 
 category healthreport-js-provider AddonsProvider resource://gre/modules/services/healthreport/providers.jsm
 category healthreport-js-provider AppInfoProvider resource://gre/modules/services/healthreport/providers.jsm
+category healthreport-js-provider CrashesProvider resource://gre/modules/services/healthreport/providers.jsm
 category healthreport-js-provider SysInfoProvider resource://gre/modules/services/healthreport/providers.jsm
 category healthreport-js-provider ProfileMetadataProvider resource://gre/modules/services/healthreport/profile.jsm
 category healthreport-js-provider SessionsProvider resource://gre/modules/services/healthreport/providers.jsm

services/healthreport/modules-testing/utils.jsm

@@ -7,12 +7,19 @@
 this.EXPORTED_SYMBOLS = [
   "getAppInfo",
   "updateAppInfo",
+  "makeFakeAppDir",
+  "createFakeCrash",
 ];
 
 
-const {interfaces: Ci, results: Cr, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
 
+Cu.import("resource://gre/modules/commonjs/promise/core.js");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+Cu.import("resource://gre/modules/osfile.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/services-common/utils.js");
+
 
 let APP_INFO = {
   vendor: "Mozilla",
@@ -71,3 +78,133 @@ this.updateAppInfo = function (obj) {
 
   registrar.registerFactory(id, "XULAppInfo", cid, factory);
 };
+
+// Reference needed in order for fake app dir provider to be active.
+let gFakeAppDirectoryProvider;
+
+/**
+ * Installs a fake UAppData directory.
+ *
+ * This is needed by tests because a UAppData directory typically isn't
+ * present in the test environment.
+ *
+ * This function is suitable for use in different components. If we ever
+ * establish a central location for convenient test helpers, this should
+ * go there.
+ *
+ * We create the new UAppData directory under the profile's directory
+ * because the profile directory is automatically cleaned as part of
+ * test shutdown.
+ *
+ * This returns a promise that will be resolved once the new directory
+ * is created and installed.
+ */
+this.makeFakeAppDir = function () {
+  let dirMode = OS.Constants.libc.S_IRWXU;
+  let dirService = Cc["@mozilla.org/file/directory_service;1"]
+                     .getService(Ci.nsIProperties);
+  let baseFile = dirService.get("ProfD", Ci.nsIFile);
+  let appD = baseFile.clone();
+  appD.append("UAppData");
+
+  if (gFakeAppDirectoryProvider) {
+    return Promise.resolve(appD.path);
+  }
+
+  function makeDir(f) {
+    if (f.exists()) {
+      return;
+    }
+
+    dump("Creating directory: " + f.path + "\n");
+    f.create(Ci.nsIFile.DIRECTORY_TYPE, dirMode);
+  }
+
+  makeDir(appD);
+
+  let reportsD = appD.clone();
+  reportsD.append("Crash Reports");
+
+  let pendingD = reportsD.clone();
+  pendingD.append("pending");
+  let submittedD = reportsD.clone();
+  submittedD.append("submitted");
+
+  makeDir(reportsD);
+  makeDir(pendingD);
+  makeDir(submittedD);
+
+  let provider = {
+    getFile: function (prop, persistent) {
+      persistent.value = true;
+      if (prop == "UAppData") {
+        return appD.clone();
+      }
+
+      throw Cr.NS_ERROR_FAILURE;
+    },
+
+    QueryInterace: function (iid) {
+      if (iid.equals(Ci.nsIDirectoryServiceProvider) ||
+          iid.equals(Ci.nsISupports)) {
+        return this;
+      }
+
+      throw Cr.NS_ERROR_NO_INTERFACE;
+    },
+  };
+
+  // Register the new provider.
+  dirService.QueryInterface(Ci.nsIDirectoryService)
+            .registerProvider(provider);
+
+  // And undefine the old one.
+  try {
+    dirService.undefine("UAppData");
+  } catch (ex) {};
+
+  gFakeAppDirectoryProvider = provider;
+
+  dump("Successfully installed fake UAppDir\n");
+  return Promise.resolve(appD.path);
+};
+
+
+/**
+ * Creates a fake crash in the Crash Reports directory.
+ *
+ * Currently, we just create a dummy file. A more robust implementation would
+ * create something that actually resembles a crash report file.
+ *
+ * This is very similar to code in crashreporter/tests/browser/head.js.
+ *
+ * FUTURE consolidate code in a shared JSM.
+ */
+this.createFakeCrash = function (submitted=false, date=new Date()) {
+  let id = CommonUtils.generateUUID();
+  let filename;
+
+  let paths = ["Crash Reports"];
+  let mode;
+
+  if (submitted) {
+    paths.push("submitted");
+    filename = "bp-" + id + ".txt";
+    mode = OS.Constants.libc.S_IRUSR | OS.Constants.libc.S_IWUSR |
+           OS.Constants.libc.S_IRGRP | OS.Constants.libc.S_IROTH;
+  } else {
+    paths.push("pending");
+    filename = id + ".dmp";
+    mode = OS.Constants.libc.S_IRUSR | OS.Constants.libc.S_IWUSR;
+  }
+
+  paths.push(filename);
+
+  let file = FileUtils.getFile("UAppData", paths, true);
+  file.create(file.NORMAL_FILE_TYPE, mode);
+  file.lastModifiedTime = date.getTime();
+  dump("Created fake crash: " + id + "\n");
+
+  return id;
+};
+

services/healthreport/providers.jsm

@@ -17,6 +17,8 @@
 this.EXPORTED_SYMBOLS = [
   "AddonsProvider",
   "AppInfoProvider",
+  "CrashDirectoryService",
+  "CrashesProvider",
   "SessionsProvider",
   "SysInfoProvider",
 ];
@@ -24,6 +26,7 @@ this.EXPORTED_SYMBOLS = [
 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 
 Cu.import("resource://gre/modules/commonjs/promise/core.js");
+Cu.import("resource://gre/modules/osfile.jsm");
 Cu.import("resource://gre/modules/Metrics.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/Task.jsm");
@@ -811,3 +814,156 @@ AddonsProvider.prototype = Object.freeze({
   },
 });
 
+
+function DailyCrashesMeasurement() {
+  Metrics.Measurement.call(this);
+}
+
+DailyCrashesMeasurement.prototype = Object.freeze({
+  __proto__: Metrics.Measurement.prototype,
+
+  name: "crashes",
+  version: 1,
+
+  configureStorage: function () {
+    this.registerStorageField("pending", this.storage.FIELD_DAILY_COUNTER);
+    this.registerStorageField("submitted", this.storage.FIELD_DAILY_COUNTER);
+  },
+});
+
+this.CrashesProvider = function () {
+  Metrics.Provider.call(this);
+};
+
+CrashesProvider.prototype = Object.freeze({
+  __proto__: Metrics.Provider.prototype,
+
+  name: "org.mozilla.crashes",
+
+  measurementTypes: [DailyCrashesMeasurement],
+
+  collectConstantData: function () {
+    return Task.spawn(this._populateCrashCounts.bind(this));
+  },
+
+  _populateCrashCounts: function () {
+    let now = new Date();
+    let service = new CrashDirectoryService();
+
+    let pending = yield service.getPendingFiles();
+    let submitted = yield service.getSubmittedFiles();
+
+    let lastCheck = yield this.getState("lastCheck");
+    if (!lastCheck) {
+      lastCheck = 0;
+    } else {
+      lastCheck = parseInt(lastCheck, 10);
+      if (Number.isNaN(lastCheck)) {
+        lastCheck = 0;
+      }
+    }
+
+    let m = this.getMeasurement("crashes", 1);
+
+    // FUTURE detect mtimes in the future and react more intelligently.
+    for (let filename in pending) {
+      let modified = pending[filename].modified;
+
+      if (modified.getTime() < lastCheck) {
+        continue;
+      }
+
+      yield m.incrementDailyCounter("pending", modified);
+    }
+
+    for (let filename in submitted) {
+      let modified = submitted[filename].modified;
+
+      if (modified.getTime() < lastCheck) {
+        continue;
+      }
+
+      yield m.incrementDailyCounter("submitted", modified);
+    }
+
+    yield this.setState("lastCheck", "" + now.getTime());
+  },
+});
+
+
+/**
+ * Helper for interacting with the crashes directory.
+ *
+ * FUTURE Extract to JSM alongside crashreporter. Use in about:crashes.
+ */
+this.CrashDirectoryService = function () {
+  let base = Cc["@mozilla.org/file/directory_service;1"]
+               .getService(Ci.nsIProperties)
+               .get("UAppData", Ci.nsIFile);
+
+  let cr = base.clone();
+  cr.append("Crash Reports");
+
+  let submitted = cr.clone();
+  submitted.append("submitted");
+
+  let pending = cr.clone();
+  pending.append("pending");
+
+  this._baseDir = base.path;
+  this._submittedDir = submitted.path;
+  this._pendingDir = pending.path;
+};
+
+CrashDirectoryService.prototype = Object.freeze({
+  RE_SUBMITTED_FILENAME: /^bp-.+\.txt$/,
+  RE_PENDING_FILENAME: /^.+\.dmp$/,
+
+  getPendingFiles: function () {
+    return this._getDirectoryEntries(this._pendingDir,
+                                     this.RE_PENDING_FILENAME);
+  },
+
+  getSubmittedFiles: function () {
+    return this._getDirectoryEntries(this._submittedDir,
+                                     this.RE_SUBMITTED_FILENAME);
+  },
+
+  _getDirectoryEntries: function (path, re) {
+    let files = {};
+
+    return Task.spawn(function iterateDirectory() {
+      if (!(yield OS.File.exists(path))) {
+        throw new Task.Result(files);
+      }
+
+      let iterator = new OS.File.DirectoryIterator(path);
+      try {
+        while (true) {
+          let entry;
+          try {
+            entry = yield iterator.next();
+          } catch (ex if ex == StopIteration) {
+            break;
+          }
+
+          if (!entry.name.match(re)) {
+            continue;
+          }
+
+          let info = yield OS.File.stat(entry.path);
+          files[entry.name] = {
+            created: info.creationDate,
+            modified: info.lastModificationDate,
+            size: info.size,
+          };
+        }
+
+        throw new Task.Result(files);
+      } finally {
+        iterator.close();
+      }
+    });
+  },
+});
+

services/healthreport/tests/xpcshell/test_provider_crashes.js

@@ -0,0 +1,146 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const {utils: Cu} = Components;
+
+
+Cu.import("resource://gre/modules/Metrics.jsm");
+Cu.import("resource://gre/modules/services/healthreport/providers.jsm");
+Cu.import("resource://testing-common/services/healthreport/utils.jsm");
+
+
+const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
+
+
+function run_test() {
+  makeFakeAppDir().then(run_next_test, do_throw);
+}
+
+let gPending = {};
+let gSubmitted = {};
+
+add_task(function test_directory_service() {
+  let d = new CrashDirectoryService();
+
+  let entries = yield d.getPendingFiles();
+  do_check_eq(typeof(entries), "object");
+  do_check_eq(Object.keys(entries).length, 0);
+
+  entries = yield d.getSubmittedFiles();
+  do_check_eq(typeof(entries), "object");
+  do_check_eq(Object.keys(entries).length, 0);
+
+  let now = new Date();
+
+  // We lose granularity when writing to filesystem.
+  now.setUTCMilliseconds(0);
+  let dates = [];
+  for (let i = 0; i < 10; i++) {
+    dates.push(new Date(now.getTime() - i * MILLISECONDS_PER_DAY));
+  }
+
+  let pending = {};
+  let submitted = {};
+  for (let date of dates) {
+    pending[createFakeCrash(false, date)] = date;
+    submitted[createFakeCrash(true, date)] = date;
+  }
+
+  entries = yield d.getPendingFiles();
+  do_check_eq(Object.keys(entries).length, Object.keys(pending).length);
+  for (let id in pending) {
+    let filename = id + ".dmp";
+    do_check_true(filename in entries);
+    do_check_eq(entries[filename].modified.getTime(), pending[id].getTime());
+  }
+
+  entries = yield d.getSubmittedFiles();
+  do_check_eq(Object.keys(entries).length, Object.keys(submitted).length);
+  for (let id in submitted) {
+    let filename = "bp-" + id + ".txt";
+    do_check_true(filename in entries);
+    do_check_eq(entries[filename].modified.getTime(), submitted[id].getTime());
+  }
+
+  gPending = pending;
+  gSubmitted = submitted;
+});
+
+add_test(function test_constructor() {
+  let provider = new CrashesProvider();
+
+  run_next_test();
+});
+
+add_task(function test_init() {
+  let storage = yield Metrics.Storage("init");
+  let provider = new CrashesProvider();
+  yield provider.init(storage);
+  yield provider.shutdown();
+
+  yield storage.close();
+});
+
+add_task(function test_collect() {
+  let storage = yield Metrics.Storage("collect");
+  let provider = new CrashesProvider();
+  yield provider.init(storage);
+
+  // FUTURE Don't rely on state from previous test.
+  yield provider.collectConstantData();
+
+  let m = provider.getMeasurement("crashes", 1);
+  let values = yield m.getValues();
+  do_check_eq(values.days.size, Object.keys(gPending).length);
+  for each (let date in gPending) {
+    do_check_true(values.days.hasDay(date));
+
+    let value = values.days.getDay(date);
+    do_check_true(value.has("pending"));
+    do_check_true(value.has("submitted"));
+    do_check_eq(value.get("pending"), 1);
+    do_check_eq(value.get("submitted"), 1);
+  }
+
+  let currentState = yield provider.getState("lastCheck");
+  do_check_eq(typeof(currentState), "string");
+  do_check_true(currentState.length > 0);
+  let lastState = currentState;
+
+  // If we collect again, we should get no new data.
+  yield provider.collectConstantData();
+  values = yield m.getValues();
+  for each (let date in gPending) {
+    let day = values.days.getDay(date);
+    do_check_eq(day.get("pending"), 1);
+    do_check_eq(day.get("submitted"), 1);
+  }
+
+  currentState = yield provider.getState("lastCheck");
+  do_check_neq(currentState, lastState);
+  do_check_true(currentState > lastState);
+
+  let now = new Date();
+  let tomorrow = new Date(now.getTime() + MILLISECONDS_PER_DAY);
+  let yesterday = new Date(now.getTime() - MILLISECONDS_PER_DAY);
+
+  let yesterdayID = createFakeCrash(false, yesterday);
+  let tomorrowID = createFakeCrash(false, tomorrow);
+
+  yield provider.collectConstantData();
+  values = yield m.getValues();
+  do_check_eq(values.days.size, 11);
+  do_check_eq(values.days.getDay(tomorrow).get("pending"), 1);
+
+  for each (let date in gPending) {
+    let day = values.days.getDay(date);
+    do_check_eq(day.get("pending"), 1);
+    do_check_eq(day.get("submitted"), 1);
+  }
+
+  yield provider.shutdown();
+  yield storage.close();
+});
+

services/healthreport/tests/xpcshell/xpcshell.ini

@@ -8,6 +8,7 @@ tail =
 [test_healthreporter.js]
 [test_provider_addons.js]
 [test_provider_appinfo.js]
+[test_provider_crashes.js]
 [test_provider_sysinfo.js]
 [test_provider_sessions.js]