Bug 1451050 - Generalize loading of packaged remote settings dumps r=Gijs

MozReview-Commit-ID: FwD92fataAy

--HG--
rename : services/blocklists/addons.json => services/settings/dumps/blocklists/addons.json
rename : services/blocklists/certificates.json => services/settings/dumps/blocklists/certificates.json
rename : services/blocklists/gfx.json => services/settings/dumps/blocklists/gfx.json
rename : services/blocklists/plugins.json => services/settings/dumps/blocklists/plugins.json
rename : services/blocklists/moz.build => services/settings/dumps/moz.build
rename : services/blocklists/pins.json => services/settings/dumps/pinning/pins.json
rename : services/blocklists/readme.md => services/settings/dumps/readme.md
extra : rebase_source : f659d946227db7b4266202283f44c1b3eadf53f1

browser/base/content/test/static/browser_all_files_referenced.js

@@ -13,8 +13,9 @@ var isDevtools = SimpleTest.harnessParameters.subsuite == "devtools";
 var gExceptionPaths = [
   "chrome://browser/content/defaultthemes/",
   "chrome://browser/locale/searchplugins/",
-  "resource://app/defaults/blocklists/",
-  "resource://app/defaults/pinning/",
+  "resource://app/defaults/settings/blocklists/",
+  "resource://app/defaults/settings/main/",
+  "resource://app/defaults/settings/pinning/",
   "resource://app/defaults/preferences/",
   "resource://gre/modules/commonjs/",
   "resource://gre/defaults/pref/",

browser/installer/allowed-dupes.mn

@@ -150,3 +150,6 @@ browser/features/formautofill@mozilla.org/chrome/content/editCreditCard.xhtml
 browser/chrome/browser/res/payments/formautofill/editCreditCard.xhtml
 browser/features/formautofill@mozilla.org/chrome/content/autofillEditForms.js
 browser/chrome/browser/res/payments/formautofill/autofillEditForms.js
+# Bug 1451050 - Remote settings empty dumps (will be populated with data eventually)
+browser/defaults/settings/pinning/pins.json
+browser/defaults/settings/main/tippytop.json

browser/installer/package-manifest.in

@@ -449,8 +449,9 @@
 @RESPATH@/greprefs.js
 @RESPATH@/defaults/autoconfig/prefcalls.js
 @RESPATH@/browser/defaults/permissions
-@RESPATH@/browser/defaults/blocklists
-@RESPATH@/browser/defaults/pinning
+@RESPATH@/browser/defaults/settings/blocklists
+@RESPATH@/browser/defaults/settings/pinning
+@RESPATH@/browser/defaults/settings/main
 
 ; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325)
 ; Technically this is an app pref file, but we are keeping it in the original

mobile/android/installer/package-manifest.in

@@ -87,6 +87,8 @@
 @BINPATH@/application.ini
 @BINPATH@/platform.ini
 @BINPATH@/blocklist.xml
+@BINPATH@/defaults/settings/blocklists/addons.json
+@BINPATH@/defaults/settings/blocklists/certificates.json
 
 ; [Components]
 @BINPATH@/components/components.manifest

services/common/docs/RemoteSettings.rst

@@ -90,6 +90,18 @@ When an entry has a file attached to it, it has an ``attachment`` attribute, whi
           }
         });
 
+Initial data
+------------
+
+For newly created user profiles, the list of entries returned by the ``.get()`` method will be empty until the first synchronization happens.
+
+It is possible to package a dump of the server records that will be loaded into the local database when no synchronization has happened yet. It will thus serve as the default dataset and also reduce the amount of data to be downloaded on the first synchronization.
+
+#. Place the JSON dump of the server records in the ``services/settings/dumps/main/`` folder
+#. Add the filename to the ``FINAL_TARGET_FILES`` list in ``services/settings/dumps/main/moz.build``
+
+Now, when ``RemoteSettings("some-key").get()`` is called from an empty profile, the ``some-key.json`` file is going to be loaded before the results are returned.
+
 
 Uptake Telemetry
 ================

services/common/remote-settings.js

@@ -213,6 +213,21 @@ class RemoteSettingsClient {
     // whose target is matched.
     const { filters = {}, order } = options;
     const c = await this.openCollection();
+
+    const timestamp = await c.db.getLastModified();
+    // If the local database was never synchronized, then we attempt to load
+    // a packaged JSON dump.
+    if (timestamp == null) {
+      try {
+        const { data } = await this._loadDumpFile();
+        await c.loadDump(data);
+      } catch (e) {
+        // Report but return an empty list since there will be no data anyway.
+        Cu.reportError(e);
+        return [];
+      }
+    }
+
     const { data } = await c.list({ filters, order });
     return this._filterEntries(data);
   }
@@ -402,7 +417,7 @@ class RemoteSettingsClient {
   async _loadDumpFile() {
     // Replace OS specific path separator by / for URI.
     const { components: folderFile } = OS.Path.split(this.filename);
-    const fileURI = `resource://app/defaults/${folderFile.join("/")}`;
+    const fileURI = `resource://app/defaults/settings/${folderFile.join("/")}`;
     const response = await fetch(fileURI);
     if (!response.ok) {
       throw new Error(`Could not read from '${fileURI}'`);

services/common/tests/unit/test_blocklist_clients.js

@@ -1,5 +1,6 @@
 const { Constructor: CC } = Components;
 
+ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://testing-common/httpd.js");
 const { FileUtils } = ChromeUtils.import("resource://gre/modules/FileUtils.jsm", {});
@@ -10,6 +11,8 @@ const BlocklistClients = ChromeUtils.import("resource://services-common/blocklis
 const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
   "nsIBinaryInputStream", "setInputStream");
 
+const IS_ANDROID = AppConstants.platform == "android";
+
 
 let gBlocklistClients;
 let server;
@@ -99,7 +102,14 @@ function run_test() {
 }
 
 add_task(async function test_initial_dump_is_loaded_as_synced_when_collection_is_empty() {
+  const november2016 = 1480000000000;
+
   for (let {client} of gBlocklistClients) {
+    if (IS_ANDROID && client.collectionName != BlocklistClients.AddonBlocklistClient.collectionName) {
+      // On Android we don't ship the dumps of plugins and gfx.
+      continue;
+    }
+
     // Test an empty db populates, but don't reach server (specified timestamp <= dump).
     await client.maybeSync(1, Date.now());
 
@@ -107,6 +117,28 @@ add_task(async function test_initial_dump_is_loaded_as_synced_when_collection_is
     const collection = await client.openCollection();
     const { data: list } = await collection.list();
     equal(list[0]._status, "synced");
+
+    // Verify that the internal timestamp was updated.
+    const timestamp = await collection.db.getLastModified();
+    ok(timestamp > november2016, `Loaded dump of ${client.collectionName} has timestamp ${timestamp}`);
+  }
+});
+add_task(clear_state);
+
+add_task(async function test_initial_dump_is_loaded_when_using_get_on_empty_collection() {
+  for (let {client} of gBlocklistClients) {
+    if (IS_ANDROID && client.collectionName != BlocklistClients.AddonBlocklistClient.collectionName) {
+      // On Android we don't ship the dumps of plugins and gfx.
+      continue;
+    }
+    // Internal database is empty.
+    const collection = await client.openCollection();
+    const { data: list } = await collection.list();
+    equal(list.length, 0);
+
+    // Calling .get() will load the dump.
+    const afterLoaded = await client.get();
+    ok(afterLoaded.length > 0, `Loaded dump of ${client.collectionName} has ${afterLoaded.length} records`);
   }
 });
 add_task(clear_state);

services/common/tests/unit/test_blocklist_targetapp_filter.js

@@ -16,6 +16,7 @@ async function createRecords(records) {
   for (const record of records) {
     await collection.create(record);
   }
+  collection.db.saveLastModified(42); // Simulate sync (and prevent load dump).
 }
 
 

services/common/tests/unit/test_remote_settings.js

@@ -103,6 +103,17 @@ add_task(async function test_records_changes_are_overwritten_by_server_changes()
 });
 add_task(clear_state);
 
+add_task(async function test_default_records_come_from_a_local_dump_when_database_is_empty() {
+  // When collection is unknown, no dump is loaded, and there is no error.
+  let data = await RemoteSettings("some-unknown-key").get();
+  equal(data.length, 0);
+
+  // When collection has a dump in services/settings/dumps/{bucket}/{collection}.json
+  data = await RemoteSettings("certificates", { bucketName: "blocklists" }).get();
+  notEqual(data.length, 0);
+});
+add_task(clear_state);
+
 add_task(async function test_sync_event_provides_information_about_records() {
   const serverTime = Date.now();
 

services/common/tests/unit/xpcshell.ini

@@ -9,8 +9,8 @@ support-files =
 [test_load_modules.js]
 
 [test_blocklist_certificates.js]
-# Initial JSON data for blocklists are not shipped on Android.
-skip-if = (os == "android" || appname == "thunderbird")
+# Skip signature tests for Thunderbird (Bug 1341983).
+skip-if = appname == "thunderbird"
 tags = blocklist
 [test_blocklist_clients.js]
 tags = blocklist

services/moz.build

@@ -10,12 +10,12 @@ with Files('moz.build'):
 DIRS += [
     'common',
     'crypto',
+    'settings',
 ]
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android':
     DIRS += [
         'fxaccounts',
-        'blocklists',
     ]
 
 if CONFIG['MOZ_SERVICES_SYNC']:

services/settings/dumps/blocklists/moz.build

@@ -7,12 +7,10 @@
 with Files('**'):
     BUG_COMPONENT = ('Toolkit', 'Blocklisting')
 
-FINAL_TARGET_FILES.defaults.blocklists += ['addons.json',
-                                           'certificates.json',
-                                           'gfx.json',
-                                           'plugins.json']
-
-FINAL_TARGET_FILES.defaults.pinning += ['pins.json']
+FINAL_TARGET_FILES.defaults.settings.blocklists += ['addons.json',
+                                                    'certificates.json',
+                                                    'gfx.json',
+                                                    'plugins.json']
 
 if CONFIG['MOZ_BUILD_APP'] == 'browser':
     DIST_SUBDIR = 'browser'

services/settings/dumps/main/moz.build

@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FINAL_TARGET_FILES.defaults.settings.main += [
+    'tippytop.json',
+]
+
+if CONFIG['MOZ_BUILD_APP'] == 'browser':
+    DIST_SUBDIR = 'browser'

services/settings/dumps/moz.build

@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    'blocklists',
+    'main',
+    'pinning',
+]

services/settings/dumps/pinning/moz.build

@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FINAL_TARGET_FILES.defaults.settings.pinning += ['pins.json']
+
+if CONFIG['MOZ_BUILD_APP'] == 'browser':
+    DIST_SUBDIR = 'browser'

services/settings/dumps/pinning/pins.json

@@ -0,0 +1 @@
+{"data":[]}

services/settings/moz.build

@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files('**'):
+    BUG_COMPONENT = ('Firefox', 'Remote Settings Client')
+
+DIRS += [
+    'dumps',
+]