Bug 1649700 - Fallback to dump data if fails to read from IndexedDB r=Standard8

Differential Revision: https://phabricator.services.mozilla.com/D81826

services/settings/Database.jsm

@@ -54,7 +54,7 @@ class Database {
               const cursor = event.target.result;
               if (cursor) {
                 const { value } = cursor;
-                if (filterObject(objFilters, value)) {
+                if (Utils.filterObject(objFilters, value)) {
                   results.push(value);
                 }
                 cursor.continue();
@@ -73,7 +73,7 @@ class Database {
     for (const result of results) {
       delete result._cid;
     }
-    return sort ? sortObjects(sort, results) : results;
+    return sort ? Utils.sortObjects(sort, results) : results;
   }
 
   async importChanges(metadata, timestamp, records = [], options = {}) {
@@ -389,10 +389,6 @@ async function executeIDB(storeNames, callback, options = {}) {
   return promise.finally(finishedFn);
 }
 
-function _isUndefined(value) {
-  return typeof value === "undefined";
-}
-
 function makeNestedObjectFromArr(arr, val, nestedFiltersObj) {
   const last = arr.length - 1;
   return arr.reduce((acc, cv, i) => {
@@ -414,52 +410,6 @@ function transformSubObjectFilters(filtersObj) {
   return transformedFilters;
 }
 
-/**
- * Test if a single object matches all given filters.
- *
- * @param  {Object} filters  The filters object.
- * @param  {Object} entry    The object to filter.
- * @return {Boolean}
- */
-function filterObject(filters, entry) {
-  return Object.entries(filters).every(([filter, value]) => {
-    if (Array.isArray(value)) {
-      return value.some(candidate => candidate === entry[filter]);
-    } else if (typeof value === "object") {
-      return filterObject(value, entry[filter]);
-    } else if (!Object.prototype.hasOwnProperty.call(entry, filter)) {
-      console.error(`The property ${filter} does not exist`);
-      return false;
-    }
-    return entry[filter] === value;
-  });
-}
-
-/**
- * Sorts records in a list according to a given ordering.
- *
- * @param  {String} order The ordering, eg. `-last_modified`.
- * @param  {Array}  list  The collection to order.
- * @return {Array}
- */
-function sortObjects(order, list) {
-  const hasDash = order[0] === "-";
-  const field = hasDash ? order.slice(1) : order;
-  const direction = hasDash ? -1 : 1;
-  return list.slice().sort((a, b) => {
-    if (a[field] && _isUndefined(b[field])) {
-      return direction;
-    }
-    if (b[field] && _isUndefined(a[field])) {
-      return -direction;
-    }
-    if (_isUndefined(a[field]) && _isUndefined(b[field])) {
-      return 0;
-    }
-    return a[field] > b[field] ? direction : -direction;
-  });
-}
-
 // We need to expose this wrapper function so we can test
 // shutdown handling.
 Database._executeIDB = executeIDB;

services/settings/RemoteSettingsClient.jsm

@@ -21,6 +21,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
   ObjectUtils: "resource://gre/modules/ObjectUtils.jsm",
   PerformanceCounters: "resource://gre/modules/PerformanceCounters.jsm",
   RemoteSettingsWorker: "resource://services-settings/RemoteSettingsWorker.jsm",
+  SharedUtils: "resource://services-settings/SharedUtils.jsm",
   UptakeTelemetry: "resource://services-common/uptake-telemetry.js",
   Utils: "resource://services-settings/Utils.jsm",
 });
@@ -333,6 +334,7 @@ class RemoteSettingsClient extends EventEmitter {
    * @param  {Object} options                  The options object.
    * @param  {Object} options.filters          Filter the results (default: `{}`).
    * @param  {String} options.order            The order to apply (eg. `"-last_modified"`).
+   * @param  {boolean} options.dumpFallback    Fallback to dump data if read of local DB fails (default: `true`).
    * @param  {boolean} options.syncIfEmpty     Synchronize from server if local data is empty (default: `true`).
    * @param  {boolean} options.verifySignature Verify the signature of the local data (default: `false`).
    * @return {Promise}
@@ -341,11 +343,44 @@ class RemoteSettingsClient extends EventEmitter {
     const {
       filters = {},
       order = "", // not sorted by default.
+      dumpFallback = true,
       syncIfEmpty = true,
     } = options;
     let { verifySignature = false } = options;
 
-    if (syncIfEmpty && !(await Utils.hasLocalData(this))) {
+    let hasLocalData;
+    try {
+      hasLocalData = await Utils.hasLocalData(this);
+    } catch (e) {
+      // If the local DB cannot be read (for unknown reasons, Bug 1649393)
+      // We fallback to the packaged data, and filter/sort in memory.
+      if (!dumpFallback) {
+        throw e;
+      }
+      Cu.reportError(e);
+      let { data } = await SharedUtils.loadJSONDump(
+        this.bucketName,
+        this.collectionName
+      );
+      if (data !== null) {
+        console.info(`${this.identifier} falling back to JSON dump`);
+      } else {
+        console.info(`${this.identifier} no dump fallback, return empty list`);
+        data = [];
+      }
+      if (!ObjectUtils.isEmpty(filters)) {
+        data = data.filter(r => Utils.filterObject(filters, r));
+      }
+      if (order) {
+        data = Utils.sortObjects(order, data);
+      }
+      // No need to verify signature on JSON dumps.
+      // If local DB cannot be read, then we don't even try to do anything,
+      // we return results early.
+      return this._filterEntries(data);
+    }
+
+    if (syncIfEmpty && !hasLocalData) {
       // .get() was called before we had the chance to synchronize the local database.
       // We'll try to avoid returning an empty list.
       if (!this._importingPromise) {

services/settings/RemoteSettingsWorker.js

@@ -67,11 +67,10 @@ const Agent = {
    * @returns {int} Number of records loaded from dump or -1 if no dump found.
    */
   async importJSONDump(bucket, collection) {
-    // When using the preview bucket, we still want to load the main dump.
-    // But we store it locally in the preview bucket.
-    const jsonBucket = bucket.replace("-preview", "");
-
-    const { data: records } = await loadJSONDump(jsonBucket, collection);
+    const { data: records } = await SharedUtils.loadJSONDump(
+      bucket,
+      collection
+    );
     if (records === null) {
       // Return -1 if file is missing.
       return -1;
@@ -139,27 +138,6 @@ self.onmessage = event => {
     });
 };
 
-/**
- * Load (from disk) the JSON file distributed with the release for this collection.
- * @param {String}  bucket
- * @param {String}  collection
- */
-async function loadJSONDump(bucket, collection) {
-  const fileURI = `resource://app/defaults/settings/${bucket}/${collection}.json`;
-  let response;
-  try {
-    response = await fetch(fileURI);
-  } catch (e) {
-    // Return null if file is missing.
-    return { data: null };
-  }
-  if (gShutdown) {
-    throw new Error("Can't import when we've started shutting down.");
-  }
-  // Will throw if JSON is invalid.
-  return response.json();
-}
-
 let gPendingTransactions = new Set();
 
 /**
@@ -185,7 +163,7 @@ async function importDumpIDB(bucket, collection, records) {
     // Each entry of the dump will be stored in the records store.
     // They are indexed by `_cid`.
     const cid = bucket + "/" + collection;
-    // We can just modify the items in-place, as we got them from loadJSONDump.
+    // We can just modify the items in-place, as we got them from SharedUtils.loadJSONDump().
     records.forEach(item => {
       item._cid = cid;
     });

services/settings/SharedUtils.jsm

@@ -10,7 +10,7 @@ var EXPORTED_SYMBOLS = ["SharedUtils"];
 
 // Import globals that are available by default in workers but not in JSMs.
 if (typeof crypto == "undefined") {
-  Cu.importGlobalProperties(["crypto"]);
+  Cu.importGlobalProperties(["fetch", "crypto"]);
 }
 
 var SharedUtils = {
@@ -34,4 +34,25 @@ var SharedUtils = {
     const hashStr = Array.from(hashBytes, toHex).join("");
     return hashStr == hash;
   },
+
+  /**
+   * Load (from disk) the JSON file distributed with the release for this collection.
+   * @param {String}  bucket
+   * @param {String}  collection
+   */
+  async loadJSONDump(bucket, collection) {
+    // When using the preview bucket, we still want to load the main dump.
+    // But we store it locally in the preview bucket.
+    const jsonBucket = bucket.replace("-preview", "");
+    const fileURI = `resource://app/defaults/settings/${jsonBucket}/${collection}.json`;
+    let response;
+    try {
+      response = await fetch(fileURI);
+    } catch (e) {
+      // Return null if file is missing.
+      return { data: null };
+    }
+    // Will throw if JSON is invalid.
+    return response.json();
+  },
 };

services/settings/Utils.jsm

@@ -49,6 +49,10 @@ XPCOMUtils.defineLazyPreferenceGetter(
   "services.settings.server"
 );
 
+function _isUndefined(value) {
+  return typeof value === "undefined";
+}
+
 var Utils = {
   get SERVER_URL() {
     const env = Cc["@mozilla.org/process/environment;1"].getService(
@@ -235,4 +239,50 @@ var Utils = {
       ageSeconds,
     };
   },
+
+  /**
+   * Test if a single object matches all given filters.
+   *
+   * @param  {Object} filters  The filters object.
+   * @param  {Object} entry    The object to filter.
+   * @return {Boolean}
+   */
+  filterObject(filters, entry) {
+    return Object.entries(filters).every(([filter, value]) => {
+      if (Array.isArray(value)) {
+        return value.some(candidate => candidate === entry[filter]);
+      } else if (typeof value === "object") {
+        return Utils.filterObject(value, entry[filter]);
+      } else if (!Object.prototype.hasOwnProperty.call(entry, filter)) {
+        console.error(`The property ${filter} does not exist`);
+        return false;
+      }
+      return entry[filter] === value;
+    });
+  },
+
+  /**
+   * Sorts records in a list according to a given ordering.
+   *
+   * @param  {String} order The ordering, eg. `-last_modified`.
+   * @param  {Array}  list  The collection to order.
+   * @return {Array}
+   */
+  sortObjects(order, list) {
+    const hasDash = order[0] === "-";
+    const field = hasDash ? order.slice(1) : order;
+    const direction = hasDash ? -1 : 1;
+    return list.slice().sort((a, b) => {
+      if (a[field] && _isUndefined(b[field])) {
+        return direction;
+      }
+      if (b[field] && _isUndefined(a[field])) {
+        return -direction;
+      }
+      if (_isUndefined(a[field]) && _isUndefined(b[field])) {
+        return 0;
+      }
+      return a[field] > b[field] ? direction : -direction;
+    });
+  },
 };

services/settings/test/unit/test_remote_settings.js

@@ -276,6 +276,32 @@ add_task(async function test_get_loads_dump_only_once_if_called_in_parallel() {
 });
 add_task(clear_state);
 
+add_task(async function test_get_falls_back_to_dump_if_db_fails() {
+  if (IS_ANDROID) {
+    // Skip test: we don't ship remote settings dumps on Android (see package-manifest).
+    return;
+  }
+  const backup = clientWithDump.db.getLastModified;
+  clientWithDump.db.getLastModified = () => {
+    throw new Error("Unknown error");
+  };
+
+  const records = await clientWithDump.get();
+  ok(records.length > 0, "dump content is returned");
+
+  // If fallback is disabled, error is thrown.
+  let error;
+  try {
+    await clientWithDump.get({ dumpFallback: false });
+  } catch (e) {
+    error = e;
+  }
+  equal(error.message, "Unknown error");
+
+  clientWithDump.db.getLastModified = backup;
+});
+add_task(clear_state);
+
 add_task(async function test_get_does_not_sync_if_empty_dump_is_provided() {
   if (IS_ANDROID) {
     // Skip test: we don't ship remote settings dumps on Android (see package-manifest).