Bug 1658597 - Destroy local DB when synchronization is broken r=barret

Differential Revision: https://phabricator.services.mozilla.com/D147755

services/settings/Database.jsm

@@ -26,6 +26,10 @@ var EXPORTED_SYMBOLS = ["Database"];
  * (with the objective of getting rid of kinto-offline-client)
  */
 class Database {
+  static destroy() {
+    return destroyIDB();
+  }
+
   constructor(identifier) {
     ensureShutdownBlocker();
     this.identifier = identifier;
@@ -405,6 +409,35 @@ async function executeIDB(storeNames, callback, options = {}) {
   return promise.finally(finishedFn);
 }
 
+async function destroyIDB() {
+  if (gDB) {
+    if (gShutdownStarted || Services.startup.shuttingDown) {
+      throw new lazy.IDBHelpers.ShutdownError(
+        "The application is shutting down",
+        "destroyIDB()"
+      );
+    }
+
+    // This will return immediately; the actual close will happen once
+    // there are no more running transactions.
+    gDB.close();
+    const allTransactions = new Set([
+      ...gPendingWriteOperations,
+      ...gPendingReadOnlyTransactions,
+    ]);
+    for (let transaction of Array.from(allTransactions)) {
+      try {
+        transaction.abort();
+      } catch (ex) {
+        // Ignore errors to abort transactions, we'll destroy everything.
+      }
+    }
+  }
+  gDB = null;
+  gDBPromise = null;
+  return lazy.IDBHelpers.destroyIDB();
+}
+
 function makeNestedObjectFromArr(arr, val, nestedFiltersObj) {
   const last = arr.length - 1;
   return arr.reduce((acc, cv, i) => {
@@ -449,7 +482,7 @@ Database._shutdownHandler = () => {
       // Ensure we don't throw/break, because either way we're in shutdown.
 
       // In particular, `transaction.abort` can throw if the transaction
-      // is complete, ie if we manage to get called inbetween the
+      // is complete, ie if we manage to get called in between the
       // transaction completing, and our completion handler being called
       // to remove the item from the set. We don't care about that.
       if (ex.result != NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR) {

services/settings/IDBHelpers.jsm

@@ -202,10 +202,19 @@ async function openIDB(allowUpgrades = true) {
   });
 }
 
+function destroyIDB() {
+  const request = indexedDB.deleteDatabase(DB_NAME);
+  return new Promise((resolve, reject) => {
+    request.onerror = event => reject(new IndexedDBError(event.target.error));
+    request.onsuccess = () => resolve();
+  });
+}
+
 var IDBHelpers = {
   bulkOperationHelper,
   executeIDB,
   openIDB,
+  destroyIDB,
   IndexedDBError,
   ShutdownError,
 };

services/settings/SyncHistory.jsm

@@ -93,6 +93,14 @@ class SyncHistory {
     return entries;
   }
 
+  /**
+   * Return the most recent entry.
+   */
+  async last() {
+    // List is sorted from newer to older.
+    return (await this.list())[0];
+  }
+
   /**
    * Wipe out the **whole** store.
    */

services/settings/remote-settings.js

@@ -27,6 +27,7 @@ XPCOMUtils.defineLazyModuleGetters(lazy, {
   pushBroadcastService: "resource://gre/modules/PushBroadcastService.jsm",
   RemoteSettingsClient: "resource://services-settings/RemoteSettingsClient.jsm",
   SyncHistory: "resource://services-settings/SyncHistory.jsm",
+  Database: "resource://services-settings/Database.jsm",
   Utils: "resource://services-settings/Utils.jsm",
   FilterExpressions:
     "resource://gre/modules/components-utils/FilterExpressions.jsm",
@@ -73,6 +74,13 @@ XPCOMUtils.defineLazyPreferenceGetter(
   10
 );
 
+XPCOMUtils.defineLazyPreferenceGetter(
+  lazy,
+  "gPrefDestroyBrokenEnabled",
+  PREF_SETTINGS_BRANCH + "destroy_broken_db_enabled",
+  true
+);
+
 /**
  * Default entry filtering function, in charge of excluding remote settings entries
  * where the JEXL expression evaluates into a falsy value.
@@ -251,6 +259,38 @@ function remoteSettingsFunction() {
       }
     }
 
+    // When triggered from the daily timer, we try to recover a broken
+    // sync state by destroying the local DB completely and retrying from scratch.
+    if (
+      lazy.gPrefDestroyBrokenEnabled &&
+      trigger == "timer" &&
+      (await isSynchronizationBroken())
+    ) {
+      // We don't want to destroy the local DB if the failures are related to
+      // network or server errors though.
+      const lastStatus = await lazy.gSyncHistory.last();
+      const lastErrorClass =
+        lazy.RemoteSettingsClient[lastStatus?.infos?.errorName] || Error;
+      const isLocalError = !(
+        lastErrorClass.prototype instanceof lazy.RemoteSettingsClient.APIError
+      );
+      if (isLocalError) {
+        console.warn(
+          "Synchronization has failed consistently. Destroy database."
+        );
+        // Clear the last ETag to refetch everything.
+        lazy.gPrefs.clearUserPref(PREF_SETTINGS_LAST_ETAG);
+        // Clear the history, to avoid re-destroying several times in a row.
+        await lazy.gSyncHistory.clear().catch(error => Cu.reportError(error));
+        // Delete the whole IndexedDB database.
+        await lazy.Database.destroy().catch(error => Cu.reportError(error));
+      } else {
+        console.warn(
+          `Synchronization is broken, but last error is ${lastStatus}`
+        );
+      }
+    }
+
     lazy.console.info("Start polling for changes");
     Services.obs.notifyObservers(
       null,

services/settings/test/unit/test_remote_settings_recover_broken.js

@@ -0,0 +1,156 @@
+/* import-globals-from ../../../common/tests/unit/head_helpers.js */
+
+const { AppConstants } = ChromeUtils.import(
+  "resource://gre/modules/AppConstants.jsm"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+const { SyncHistory } = ChromeUtils.import(
+  "resource://services-settings/SyncHistory.jsm"
+);
+const { RemoteSettingsClient } = ChromeUtils.import(
+  "resource://services-settings/RemoteSettingsClient.jsm"
+);
+const { RemoteSettings } = ChromeUtils.import(
+  "resource://services-settings/remote-settings.js"
+);
+const { Utils } = ChromeUtils.import("resource://services-settings/Utils.jsm");
+
+const PREF_SETTINGS_SERVER = "services.settings.server";
+const CHANGES_PATH = "/v1" + Utils.CHANGES_PATH;
+const BROKEN_SYNC_THRESHOLD = 10; // See default pref value
+
+let server;
+let client;
+let maybeSyncBackup;
+
+async function clear_state() {
+  // Disable logging output.
+  Services.prefs.setCharPref("services.settings.loglevel", "critical");
+  // Pull data from the test server.
+  Services.prefs.setCharPref(
+    PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`
+  );
+
+  // Clear sync history.
+  await new SyncHistory("").clear();
+
+  // Simulate a response whose ETag gets incremented on each call
+  // (in order to generate several history entries, indexed by timestamp).
+  let timestamp = 1337;
+  server.registerPathHandler(CHANGES_PATH, (request, response) => {
+    response.setStatusLine(null, 200, "OK");
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Date", new Date(1000000).toUTCString());
+    response.setHeader("ETag", `"${timestamp}"`);
+    response.write(
+      JSON.stringify({
+        timestamp,
+        changes: [
+          {
+            last_modified: ++timestamp,
+            bucket: "main",
+            collection: "desktop-manager",
+          },
+        ],
+      })
+    );
+  });
+
+  // Restore original maybeSync() method between each test.
+  client.maybeSync = maybeSyncBackup;
+}
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  client = RemoteSettings("desktop-manager");
+  maybeSyncBackup = client.maybeSync;
+
+  run_next_test();
+
+  registerCleanupFunction(() => {
+    server.stop(() => {});
+    // Restore original maybeSync() method when test suite is done.
+    client.maybeSync = maybeSyncBackup;
+  });
+}
+
+add_task(clear_state);
+
+add_task(async function test_db_is_destroyed_when_sync_is_broken() {
+  // Simulate a successful sync.
+  client.maybeSync = async () => {
+    // Store some data in local DB.
+    await client.db.importChanges({}, 1515, []);
+  };
+  await RemoteSettings.pollChanges({ trigger: "timer" });
+
+  // Register a client with a failing sync method.
+  client.maybeSync = () => {
+    throw new RemoteSettingsClient.InvalidSignatureError(
+      "main/desktop-manager"
+    );
+  };
+
+  // Now obtain several failures in a row.
+  for (var i = 0; i < BROKEN_SYNC_THRESHOLD; i++) {
+    try {
+      await RemoteSettings.pollChanges({ trigger: "timer" });
+    } catch (e) {}
+  }
+
+  // Synchronization is in broken state.
+  Assert.equal(
+    await client.db.getLastModified(),
+    1515,
+    "Local DB was not destroyed yet"
+  );
+
+  // Synchronize again. Broken state will be detected.
+  try {
+    await RemoteSettings.pollChanges({ trigger: "timer" });
+  } catch (e) {}
+
+  // DB was destroyed.
+  Assert.equal(
+    await client.db.getLastModified(),
+    null,
+    "Local DB was destroyed"
+  );
+});
+
+add_task(clear_state);
+
+add_task(async function test_db_is_not_destroyed_when_state_is_server_error() {
+  // Since we don't mock the server endpoints to obtain the changeset of this
+  // collection, the call to `maybeSync()` will fail with network errors.
+
+  // Store some data in local DB.
+  await client.db.importChanges({}, 1515, []);
+
+  // Now obtain several failures in a row.
+  let lastError;
+  for (var i = 0; i < BROKEN_SYNC_THRESHOLD + 1; i++) {
+    try {
+      await RemoteSettings.pollChanges({ trigger: "timer" });
+    } catch (e) {
+      lastError = e;
+    }
+  }
+  Assert.ok(
+    /Cannot parse server content/.test(lastError.message),
+    "Error is about server"
+  );
+  // DB was not destroyed.
+  Assert.equal(
+    await client.db.getLastModified(),
+    1515,
+    "Local DB was not destroyed"
+  );
+});
+
+add_task(clear_state);

services/settings/test/unit/xpcshell.ini

@@ -12,6 +12,7 @@ support-files = test_attachments_downloader/**
 [test_remote_settings_dump_lastmodified.js]
 [test_remote_settings_offline.js]
 [test_remote_settings_poll.js]
+[test_remote_settings_recover_broken.js]
 [test_remote_settings_worker.js]
 [test_remote_settings_jexl_filters.js]
 [test_remote_settings_release_prefs.js]