Bug 1502146 - Reduce impact of RemoteSettings synchronization on main thread r=Gijs

Move JSON dump loading and CanonicalJSON serialization to a worker to reduce impact on main thread

Differential Revision: https://phabricator.services.mozilla.com/D10064

--HG--
extra : moz-landing-system : lando

services/common/kinto-offline-client.js

@@ -33,7 +33,7 @@ const global = this;
 var EXPORTED_SYMBOLS = ["Kinto"];
 
 /*
- * Version 12.2.0 - 266e100
+ * Version 12.2.4 - 8fb687a
  */
 
 (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Kinto = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(require,module,exports){
@@ -71,16 +71,15 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 ChromeUtils.import("resource://gre/modules/Timer.jsm");
 ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 XPCOMUtils.defineLazyGlobalGetters(global, ["fetch", "indexedDB"]);
-const {
-  EventEmitter
-} = ChromeUtils.import("resource://gre/modules/EventEmitter.jsm", {});
-const {
-  generateUUID
-} = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator); // Use standalone kinto-http module landed in FFx.
+ChromeUtils.defineModuleGetter(global, "EventEmitter", "resource://gre/modules/EventEmitter.jsm"); // Use standalone kinto-http module landed in FFx.
 
-const {
-  KintoHttpClient
-} = ChromeUtils.import("resource://services-common/kinto-http-client.js");
+ChromeUtils.defineModuleGetter(global, "KintoHttpClient", "resource://services-common/kinto-http-client.js");
+XPCOMUtils.defineLazyGetter(global, "generateUUID", () => {
+  const {
+    generateUUID
+  } = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator);
+  return generateUUID;
+});
 
 class Kinto extends _KintoBase.default {
   static get adapters() {
@@ -490,10 +489,13 @@ function createListRequest(cid, store, filters, done) {
 
   if (filterFields.length == 0) {
     const request = store.index("cid").getAll(IDBKeyRange.only(cid));
+
     request.onsuccess = event => done(event.target.result);
+
     return request;
-  }
-  // Introspect filters and check if they leverage an indexed field.
+  } // Introspect filters and check if they leverage an indexed field.
+
+
   const indexField = filterFields.find(field => {
     return INDEXED_FIELDS.includes(field);
   });
@@ -506,13 +508,12 @@ function createListRequest(cid, store, filters, done) {
   } // If `indexField` was used already, don't filter again.
 
 
-  const remainingFilters = (0, _utils.omitKeys)(filters, indexField); // value specified in the filter (eg. `filters: { _status: ["created", "updated"] }`)
+  const remainingFilters = (0, _utils.omitKeys)(filters, [indexField]); // value specified in the filter (eg. `filters: { _status: ["created", "updated"] }`)
 
-  const value = filters[indexField];
-  // For the "id" field, use the primary key.
-  const indexStore = indexField == "id" ? store : store.index(indexField);
+  const value = filters[indexField]; // For the "id" field, use the primary key.
+
+  const indexStore = indexField == "id" ? store : store.index(indexField); // WHERE IN equivalent clause
 
-  // WHERE IN equivalent clause
   if (Array.isArray(value)) {
     if (value.length === 0) {
       return done([]);
@@ -523,12 +524,14 @@ function createListRequest(cid, store, filters, done) {
     const request = indexStore.openCursor(range);
     request.onsuccess = cursorHandlers.in(values, remainingFilters, done);
     return request;
-  }
+  } // If no filters on custom attribute, get all results in one bulk.
+
 
-  // If no filters on custom attribute, get all results in one bulk.
   if (remainingFilters.length == 0) {
     const request = indexStore.getAll(IDBKeyRange.only([cid, value]));
+
     request.onsuccess = event => done(event.target.result);
+
     return request;
   } // WHERE field = value clause
 
@@ -767,10 +770,13 @@ class IDB extends _base.default {
       };
       createListRequest(this.cid, store, filters, records => {
         // Store obtained records by id.
-        const preloaded = records.reduce((acc, record) => {
-          acc[record.id] = (0, _utils.omitKeys)(record, ["_cid"]);
-          return acc;
-        }, {});
+        const preloaded = {};
+
+        for (const record of records) {
+          delete record["_cid"];
+          preloaded[record.id] = record;
+        }
+
         runCallback(preloaded);
       });
     }, {
@@ -820,7 +826,11 @@ class IDB extends _base.default {
         createListRequest(this.cid, store, filters, _results => {
           // we have received all requested records that match the filters,
           // we now park them within current scope and hide the `_cid` attribute.
-          results = _results.map(r => (0, _utils.omitKeys)(r, ["_cid"]));
+          for (const result of _results) {
+            delete result["_cid"];
+          }
+
+          results = _results;
         });
       }); // The resulting list of records is sorted.
       // XXX: with some efforts, this could be fully implemented using IDB API.
@@ -885,7 +895,21 @@ class IDB extends _base.default {
   async loadDump(records) {
     try {
       await this.execute(transaction => {
-        records.forEach(record => transaction.update(record));
+        // Since the put operations are asynchronous, we chain
+        // them together. The last one will be waited for the
+        // `transaction.oncomplete` callback. (see #execute())
+        let i = 0;
+        putNext();
+
+        function putNext() {
+          if (i == records.length) {
+            return;
+          } // On error, `transaction.onerror` is called.
+
+
+          transaction.update(records[i]).onsuccess = putNext;
+          ++i;
+        }
       });
       const previousLastModified = await this.getLastModified();
       const lastModified = Math.max(...records.map(record => record.last_modified));
@@ -924,7 +948,7 @@ function transactionProxy(adapter, store, preloaded = []) {
     },
 
     update(record) {
-      store.put({ ...record,
+      return store.put({ ...record,
         _cid
       });
     },
@@ -3116,13 +3140,14 @@ function deepEqual(a, b) {
 
 
 function omitKeys(obj, keys = []) {
-  return Object.keys(obj).reduce((acc, key) => {
-    if (!keys.includes(key)) {
-      acc[key] = obj[key];
-    }
+  const result = { ...obj
+  };
 
-    return acc;
-  }, {});
+  for (const key of keys) {
+    delete result[key];
+  }
+
+  return result;
 }
 
 function arrayEqual(a, b) {

services/settings/RemoteSettingsWorker.js

@@ -0,0 +1,188 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env mozilla/chrome-worker */
+
+"use strict";
+
+/**
+ * A worker dedicated to Remote Settings.
+ */
+
+importScripts("resource://gre/modules/workers/require.js",
+              "resource://gre/modules/CanonicalJSON.jsm",
+              "resource://gre/modules/third_party/jsesc/jsesc.js");
+
+const IDB_NAME = "remote-settings";
+const IDB_VERSION = 1;
+const IDB_RECORDS_STORE = "records";
+const IDB_TIMESTAMPS_STORE = "timestamps";
+
+const Agent = {
+  /**
+   * Return the canonical JSON serialization of the changes
+   * applied to the local records.
+   * It has to match what is done on the server (See Kinto/kinto-signer).
+   *
+   * @param {Array<Object>} localRecords
+   * @param {Array<Object>} remoteRecords
+   * @param {String} timestamp
+   * @returns {String}
+   */
+  async canonicalStringify(localRecords, remoteRecords, timestamp) {
+    // Sort list by record id.
+    let allRecords = localRecords.concat(remoteRecords).sort((a, b) => {
+      if (a.id < b.id) {
+        return -1;
+      }
+      return a.id > b.id ? 1 : 0;
+    });
+    // All existing records are replaced by the version from the server
+    // and deleted records are removed.
+    for (let i = 0; i < allRecords.length; /* no increment! */) {
+      const rec = allRecords[i];
+      const next = allRecords[i + 1];
+      if ((next && rec.id == next.id) || rec.deleted) {
+        allRecords.splice(i, 1); // remove local record
+      } else {
+        i++;
+      }
+    }
+    const toSerialize = {
+      last_modified: "" + timestamp,
+      data: allRecords,
+    };
+    return CanonicalJSON.stringify(toSerialize, jsesc);
+  },
+
+  /**
+   * If present, import the JSON file into the Remote Settings IndexedDB
+   * for the specified bucket and collection.
+   * (eg. blocklists/certificates, main/onboarding)
+   * @param {String} bucket
+   * @param {String} collection
+   */
+  async importJSONDump(bucket, collection) {
+    const { data: records } = await loadJSONDump(bucket, collection);
+    if (records.length > 0) {
+      await importDumpIDB(bucket, collection, records);
+    }
+    return records.length;
+  },
+};
+
+/**
+ * Wrap worker invocations in order to return the `callbackId` along
+ * the result. This will allow to transform the worker invocations
+ * into promises in `RemoteSettingsWorker.jsm`.
+ */
+self.onmessage = (event) => {
+  const { callbackId, method, args = [] } = event.data;
+  Agent[method](...args)
+    .then((result) => {
+      self.postMessage({ callbackId, result });
+    })
+    .catch(error => {
+      console.log(`RemoteSettingsWorker error: ${error}`);
+      self.postMessage({ callbackId, error: "" + error });
+    });
+};
+
+/**
+ * Load (from disk) the JSON file distributed with the release for this collection.
+ * @param {String}  bucket
+ * @param {String}  collection
+ */
+async function loadJSONDump(bucket, collection) {
+  const fileURI = `resource://app/defaults/settings/${bucket}/${collection}.json`;
+  let response;
+  try {
+    response = await fetch(fileURI);
+  } catch (e) {
+    // Return empty dataset if file is missing.
+    return { data: [] };
+  }
+  // Will throw if JSON is invalid.
+  return response.json();
+}
+
+/**
+ * Import the records into the Remote Settings Chrome IndexedDB.
+ *
+ * Note: This duplicates some logics from `kinto-offline-client.js`.
+ *
+ * @param {String} bucket
+ * @param {String} collection
+ * @param {Array<Object>} records
+ */
+async function importDumpIDB(bucket, collection, records) {
+  // Open the DB. It will exist since if we are running this, it means
+  // we already tried to read the timestamp in `remote-settings.js`
+  const db = await openIDB(IDB_NAME, IDB_VERSION);
+
+  // Each entry of the dump will be stored in the records store.
+  // They are indexed by `_cid`, and their status is `synced`.
+  const cid = bucket + "/" + collection;
+  await executeIDB(db, IDB_RECORDS_STORE, store => {
+    // Chain the put operations together, the last one will be waited by
+    // the `transaction.oncomplete` callback.
+    let i = 0;
+    putNext();
+
+    function putNext() {
+      if (i == records.length) {
+        return;
+      }
+      const entry = { ...records[i], _status: "synced", _cid: cid };
+      store.put(entry).onsuccess = putNext; // On error, `transaction.onerror` is called.
+      ++i;
+    }
+  });
+
+  // Store the highest timestamp as the collection timestamp.
+  const timestamp = Math.max(...records.map(record => record.last_modified));
+  await executeIDB(db, IDB_TIMESTAMPS_STORE, store => store.put({ cid, value: timestamp }));
+}
+
+/**
+ * Helper to wrap indexedDB.open() into a promise.
+ */
+async function openIDB(dbname, version) {
+  return new Promise((resolve, reject) => {
+    const request = indexedDB.open(dbname, version);
+    request.onupgradeneeded = () => {
+      // We should never have to initialize the DB here.
+      reject(new Error(`Error accessing ${dbname} Chrome IDB at version ${version}`));
+    };
+    request.onerror = event => reject(event.target.error);
+    request.onsuccess = event => {
+      const db = event.target.result;
+      resolve(db);
+    };
+  });
+}
+
+/**
+ * Helper to wrap some IDBObjectStore operations into a promise.
+ *
+ * @param {IDBDatabase} db
+ * @param {String} storeName
+ * @param {function} callback
+ */
+async function executeIDB(db, storeName, callback) {
+  const mode = "readwrite";
+  return new Promise((resolve, reject) => {
+    const transaction = db.transaction([storeName], mode);
+    const store = transaction.objectStore(storeName);
+    let result;
+    try {
+      result = callback(store);
+    } catch (e) {
+      transaction.abort();
+      reject(e);
+    }
+    transaction.onerror = event => reject(event.target.error);
+    transaction.oncomplete = event => resolve(result);
+  });
+}

services/settings/RemoteSettingsWorker.jsm

@@ -0,0 +1,53 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/**
+ * Interface to a dedicated thread handling for Remote Settings heavy operations.
+ */
+
+// ChromeUtils.import("resource://gre/modules/PromiseWorker.jsm", this);
+
+var EXPORTED_SYMBOLS = ["RemoteSettingsWorker"];
+
+class Worker {
+
+  constructor(source) {
+    this.worker = new ChromeWorker(source);
+    this.worker.onmessage = this._onWorkerMessage.bind(this);
+
+    this.callbacks = new Map();
+    this.lastCallbackId = 0;
+  }
+
+  async _execute(method, args = []) {
+    return new Promise(async (resolve, reject) => {
+      const callbackId = ++this.lastCallbackId;
+      this.callbacks.set(callbackId, [resolve, reject]);
+      this.worker.postMessage({ callbackId, method, args });
+    });
+  }
+
+  _onWorkerMessage(event) {
+    const { callbackId, result, error } = event.data;
+    const [resolve, reject] = this.callbacks.get(callbackId);
+    if (error) {
+      reject(new Error(error));
+    } else {
+      resolve(result);
+    }
+    this.callbacks.delete(callbackId);
+  }
+
+  async canonicalStringify(localRecords, remoteRecords, timestamp) {
+    return this._execute("canonicalStringify", [localRecords, remoteRecords, timestamp]);
+  }
+
+  async importJSONDump(bucket, collection) {
+    return this._execute("importJSONDump", [bucket, collection]);
+  }
+}
+
+var RemoteSettingsWorker = new Worker("resource://services-settings/RemoteSettingsWorker.js");

services/settings/moz.build

@@ -16,6 +16,8 @@ EXTRA_COMPONENTS += [
 
 EXTRA_JS_MODULES['services-settings'] += [
     'remote-settings.js',
+    'RemoteSettingsWorker.js',
+    'RemoteSettingsWorker.jsm',
 ]
 
 XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']

services/settings/remote-settings.js

@@ -31,6 +31,8 @@ ChromeUtils.defineModuleGetter(this, "FilterExpressions",
                                "resource://gre/modules/components-utils/FilterExpressions.jsm");
 ChromeUtils.defineModuleGetter(this, "pushBroadcastService",
                                "resource://gre/modules/PushBroadcastService.jsm");
+ChromeUtils.defineModuleGetter(this, "RemoteSettingsWorker",
+                               "resource://services-settings/RemoteSettingsWorker.jsm");
 
 const PREF_SETTINGS_DEFAULT_BUCKET     = "services.settings.default_bucket";
 const PREF_SETTINGS_BRANCH             = "services.settings.";
@@ -47,7 +49,7 @@ const PREF_SETTINGS_LOAD_DUMP          = "load_dump";
 // Telemetry update source identifier.
 const TELEMETRY_HISTOGRAM_KEY = "settings-changes-monitoring";
 
-const INVALID_SIGNATURE = "Invalid content/signature";
+const INVALID_SIGNATURE = "Invalid content signature";
 const MISSING_SIGNATURE = "Missing signature";
 
 XPCOMUtils.defineLazyGetter(this, "gPrefs", () => {
@@ -106,32 +108,8 @@ async function jexlFilterFunc(entry, environment) {
   return result ? entry : null;
 }
 
-
-function mergeChanges(collection, localRecords, changes) {
-  const records = {};
-  // Local records by id.
-  localRecords.forEach((record) => records[record.id] = collection.cleanLocalFields(record));
-  // All existing records are replaced by the version from the server.
-  changes.forEach((record) => records[record.id] = record);
-
-  return Object.values(records)
-    // Filter out deleted records.
-    .filter((record) => !record.deleted)
-    // Sort list by record id.
-    .sort((a, b) => {
-      if (a.id < b.id) {
-        return -1;
-      }
-      return a.id > b.id ? 1 : 0;
-    });
-}
-
-
 async function fetchCollectionMetadata(remote, collection, expectedTimestamp) {
   const client = new KintoHttpClient(remote);
-  //
-  // XXX: https://github.com/Kinto/kinto-http.js/issues/307
-  //
   const { signature } = await client.bucket(collection.bucket)
                                     .collection(collection.name)
                                     .getData({ query: { _expected: expectedTimestamp }});
@@ -223,33 +201,6 @@ async function fetchLatestChanges(url, lastEtag, expectedTimestamp) {
   return {changes, currentEtag, serverTimeMillis, backoffSeconds};
 }
 
-/**
- * Load the the JSON file distributed with the release for this collection.
- * @param {String}  bucket
- * @param {String}  collection
- * @param {Object}  options
- * @param {boolean} options.ignoreMissing Do not throw an error if the file is missing.
- */
-async function loadDumpFile(bucket, collection, { ignoreMissing = true } = {}) {
-  const fileURI = `resource://app/defaults/settings/${bucket}/${collection}.json`;
-  let response;
-  try {
-    // Will throw NetworkError is folder/file is missing.
-    response = await fetch(fileURI);
-    if (!response.ok) {
-      throw new Error(`Could not read from '${fileURI}'`);
-    }
-    // Will throw if JSON is invalid.
-    return response.json();
-  } catch (e) {
-    // A missing file is reported as "NetworError" (see Bug 1493709)
-    if (!ignoreMissing || !/NetworkError/.test(e.message)) {
-      throw e;
-    }
-  }
-  return { data: [] };
-}
-
 
 class RemoteSettingsClient {
 
@@ -352,7 +303,7 @@ class RemoteSettingsClient {
   async get(options = {}) {
     // In Bug 1451031, we will do some jexl filtering to limit the list items
     // whose target is matched.
-    const { filters = {}, order } = options;
+    const { filters = {}, order = "" } = options; // not sorted by default.
     const c = await this.openCollection();
 
     const timestamp = await c.db.getLastModified();
@@ -360,8 +311,7 @@ class RemoteSettingsClient {
     // a packaged JSON dump.
     if (timestamp == null) {
       try {
-        const { data } = await loadDumpFile(this.bucketName, this.collectionName);
-        await c.loadDump(data);
+        await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
       } catch (e) {
         // Report but return an empty list since there will be no data anyway.
         Cu.reportError(e);
@@ -398,8 +348,7 @@ class RemoteSettingsClient {
       // cold start.
       if (!collectionLastModified && loadDump) {
         try {
-          const initialData = await loadDumpFile(this.bucketName, this.collectionName);
-          await collection.loadDump(initialData.data);
+          await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
           collectionLastModified = await collection.db.getLastModified();
         } catch (e) {
           // Report but go-on.
@@ -418,8 +367,13 @@ class RemoteSettingsClient {
       // If there is a `signerName` and collection signing is enforced, add a
       // hook for incoming changes that validates the signature.
       if (this.signerName && gVerifySignature) {
-        collection.hooks["incoming-changes"] = [(payload, collection) => {
-          return this._validateCollectionSignature(payload, collection, { expectedTimestamp });
+        collection.hooks["incoming-changes"] = [async (payload, collection) => {
+          await this._validateCollectionSignature(payload.changes,
+                                                  payload.lastModified,
+                                                  collection,
+                                                  { expectedTimestamp });
+          // In case the signature is valid, apply the changes locally.
+          return payload;
         }];
       }
 
@@ -439,7 +393,7 @@ class RemoteSettingsClient {
           throw new Error("Sync failed");
         }
       } catch (e) {
-        if (e.message == INVALID_SIGNATURE) {
+        if (e.message.includes(INVALID_SIGNATURE)) {
           // Signature verification failed during synchronzation.
           reportStatus = UptakeTelemetry.STATUS.SIGNATURE_ERROR;
           // if sync fails with a signature error, it's likely that our
@@ -448,7 +402,10 @@ class RemoteSettingsClient {
           // remote collection.
           const payload = await fetchRemoteCollection(collection, expectedTimestamp);
           try {
-            await this._validateCollectionSignature(payload, collection, { expectedTimestamp, ignoreLocal: true });
+            await this._validateCollectionSignature(payload.data,
+                                                    payload.last_modified,
+                                                    collection,
+                                                    { expectedTimestamp, ignoreLocal: true });
           } catch (e) {
             reportStatus = UptakeTelemetry.STATUS.SIGNATURE_RETRY_ERROR;
             throw e;
@@ -456,7 +413,7 @@ class RemoteSettingsClient {
 
           // The signature is good (we haven't thrown).
           // Now we will Inspect what we had locally.
-          const { data: oldData } = await collection.list();
+          const { data: oldData } = await collection.list({ order: "" }); // no need to sort.
 
           // We build a sync result as if a diff-based sync was performed.
           syncResult = { created: [], updated: [], deleted: [] };
@@ -515,7 +472,7 @@ class RemoteSettingsClient {
       // If every changed entry is filtered, we don't even fire the event.
       if (created.length || updated.length || deleted.length) {
         // Read local collection of records (also filtered).
-        const { data: allData } = await collection.list();
+        const { data: allData } = await collection.list({ order: "" }); // no need to sort.
         const current = await this._filterEntries(allData);
         const payload = { data: { current, created, updated, deleted } };
         try {
@@ -545,8 +502,8 @@ class RemoteSettingsClient {
     }
   }
 
-  async _validateCollectionSignature(payload, collection, options = {}) {
-    const { expectedTimestamp, ignoreLocal } = options;
+  async _validateCollectionSignature(remoteRecords, timestamp, collection, options = {}) {
+    const { expectedTimestamp, ignoreLocal = false } = options;
     // this is a content-signature field from an autograph response.
     const signaturePayload = await fetchCollectionMetadata(gServerURL, collection, expectedTimestamp);
     if (!signaturePayload) {
@@ -559,30 +516,22 @@ class RemoteSettingsClient {
     const verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
                        .createInstance(Ci.nsIContentSignatureVerifier);
 
-    let toSerialize;
-    if (ignoreLocal) {
-      toSerialize = {
-        last_modified: `${payload.last_modified}`,
-        data: payload.data,
-      };
-    } else {
-      const {data: localRecords} = await collection.list();
-      const records = mergeChanges(collection, localRecords, payload.changes);
-      toSerialize = {
-        last_modified: `${payload.lastModified}`,
-        data: records,
-      };
+    let localRecords = [];
+    if (!ignoreLocal) {
+      const { data } = await collection.list({ order: "" }); // no need to sort.
+      // Local fields are stripped to compute the collection signature (server does not have them).
+      localRecords = data.map(r => collection.cleanLocalFields(r));
     }
 
-    const serialized = CanonicalJSON.stringify(toSerialize);
-
-    if (verifier.verifyContentSignature(serialized, "p384ecdsa=" + signature,
-                                        certChain,
-                                        this.signerName)) {
-      // In case the hash is valid, apply the changes locally.
-      return payload;
+    const serialized = await RemoteSettingsWorker.canonicalStringify(localRecords,
+                                                                     remoteRecords,
+                                                                     timestamp);
+    if (!verifier.verifyContentSignature(serialized,
+                                         "p384ecdsa=" + signature,
+                                         certChain,
+                                         this.signerName)) {
+      throw new Error(INVALID_SIGNATURE + ` (${collection.bucket}/${collection.name})`);
     }
-    throw new Error(INVALID_SIGNATURE);
   }
 
   /**
@@ -628,7 +577,7 @@ async function hasLocalData(client) {
  */
 async function hasLocalDump(bucket, collection) {
   try {
-    await loadDumpFile(bucket, collection, {ignoreMissing: false});
+    await fetch(`resource://app/defaults/settings/${bucket}/${collection}.json`);
     return true;
   } catch (e) {
     return false;

services/settings/test/unit/test_remote_settings_worker.js

@@ -0,0 +1,50 @@
+/* import-globals-from ../../../common/tests/unit/head_helpers.js */
+
+ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
+ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+const { RemoteSettingsWorker } = ChromeUtils.import("resource://services-settings/RemoteSettingsWorker.jsm", {});
+const { Kinto } = ChromeUtils.import("resource://services-common/kinto-offline-client.js", {});
+
+const IS_ANDROID = AppConstants.platform == "android";
+
+add_task(async function test_canonicaljson_merges_remote_into_local() {
+  const localRecords = [{ id: "1", title: "title 1" }, { id: "2", title: "title 2" }, { id: "3", title: "title 3" }];
+  const remoteRecords = [{ id: "2", title: "title b" }, { id: "3", deleted: true }];
+  const timestamp = 42;
+
+  const serialized = await RemoteSettingsWorker.canonicalStringify(localRecords, remoteRecords, timestamp);
+
+  Assert.equal('{"data":[{"id":"1","title":"title 1"},{"id":"2","title":"title b"}],"last_modified":"42"}', serialized);
+});
+
+
+add_task(async function test_import_json_dump_into_idb() {
+  if (IS_ANDROID) {
+    // Skip test: we don't ship remote settings dumps on Android (see package-manifest).
+    return;
+  }
+  const kintoCollection = new Kinto({
+    bucket: "main",
+    adapter: Kinto.adapters.IDB,
+    adapterOptions: { dbName: "remote-settings" },
+  }).collection("language-dictionaries");
+  const { data: before } = await kintoCollection.list();
+  Assert.equal(before.length, 0);
+
+  await RemoteSettingsWorker.importJSONDump("main", "language-dictionaries");
+
+  const { data: after } = await kintoCollection.list();
+  Assert.ok(after.length > 0);
+});
+
+
+add_task(async function test_throws_error_if_worker_fails() {
+  let error;
+  try {
+    await RemoteSettingsWorker.canonicalStringify(null, [], 42);
+  } catch (e) {
+    error = e;
+  }
+  Assert.equal("TypeError: localRecords is null; can't access its \"concat\" property", error.message);
+});

services/settings/test/unit/xpcshell.ini

@@ -5,4 +5,5 @@ tags = remote-settings
 
 [test_remote_settings.js]
 [test_remote_settings_poll.js]
+[test_remote_settings_worker.js]
 [test_remote_settings_jexl_filters.js]

toolkit/modules/CanonicalJSON.jsm

@@ -4,9 +4,6 @@
 
 var EXPORTED_SYMBOLS = ["CanonicalJSON"];
 
-ChromeUtils.defineModuleGetter(this, "jsesc",
-                               "resource://gre/modules/third_party/jsesc/jsesc.js");
-
 var CanonicalJSON = {
   /**
    * Return the canonical JSON form of the passed source, sorting all the object
@@ -22,10 +19,14 @@ var CanonicalJSON = {
    * @usage
    *        CanonicalJSON.stringify(listOfRecords);
    **/
-  stringify: function stringify(source) {
+  stringify: function stringify(source, jsescFn) {
+    if (typeof jsescFn != "function") {
+      const { jsesc } = ChromeUtils.import("resource://gre/modules/third_party/jsesc/jsesc.js", {});
+      jsescFn = jsesc;
+    }
     if (Array.isArray(source)) {
       const jsonArray = source.map(x => typeof x === "undefined" ? null : x);
-      return `[${jsonArray.map(stringify).join(",")}]`;
+      return "[" + jsonArray.map(item => stringify(item, jsescFn)).join(",") + "]";
     }
 
     if (typeof source === "number") {
@@ -35,7 +36,7 @@ var CanonicalJSON = {
     }
 
     // Leverage jsesc library, mainly for unicode escaping.
-    const toJSON = (input) => jsesc(input, {lowercaseHex: true, json: true});
+    const toJSON = (input) => jsescFn(input, {lowercaseHex: true, json: true});
 
     if (typeof source !== "object" || source === null) {
       return toJSON(source);
@@ -53,7 +54,7 @@ var CanonicalJSON = {
       const jsonValue = value && value.toJSON ? value.toJSON() : value;
       const suffix = index !== lastIndex ? "," : "";
       const escapedKey = toJSON(key);
-      return serial + `${escapedKey}:${stringify(jsonValue)}${suffix}`;
+      return serial + escapedKey + ":" + stringify(jsonValue, jsescFn) + suffix;
     }, "{") + "}";
   },
 };

tools/lint/eslint/modules.json

@@ -23,6 +23,7 @@
   "BootstrapMonitor.jsm": ["monitor"],
   "browser-loader.js": ["BrowserLoader"],
   "browserid_identity.js": ["BrowserIDManager", "AuthenticationError"],
+  "CanonicalJSON.jsm": ["CanonicalJSON"],
   "CertUtils.jsm": ["CertUtils"],
   "clients.js": ["ClientEngine", "ClientsRec"],
   "collection_repair.js": ["getRepairRequestor", "getAllRepairRequestors", "CollectionRepairRequestor", "getRepairResponder", "CollectionRepairResponder"],
@@ -100,6 +101,7 @@
   "InlineSpellChecker.jsm": ["InlineSpellChecker", "SpellCheckHelper"],
   "JSDOMParser.js": ["JSDOMParser"],
   "jsdebugger.jsm": ["addDebuggerToGlobal"],
+  "jsesc.js": ["jsesc"],
   "json2.js": ["JSON"],
   "keys.js": ["BulkKeyBundle", "SyncKeyBundle"],
   "KeyValueParser.jsm": ["parseKeyValuePairsFromLines", "parseKeyValuePairs", "parseKeyValuePairsFromFile", "parseKeyValuePairsFromFileAsync"],