Bug 1620621 - Add initial dump of addons blocklist r=Gijs

The MLBF (addons-mlbf.bin) itself is 64 KB.
Together with the metadata, this is 65 KB.
In contrast, the current JSON-based dump (addons.json) is 913 KB.

Differential Revision: https://phabricator.services.mozilla.com/D73159

services/settings/dumps/blocklists/addons-bloomfilters.json

@@ -0,0 +1 @@
+{"data":[{"schema":1588016498560,"attachment":{"hash":"164992bd106fd2c4cb039f8c1b2581f1d5f2c8ecc1635a2aa69efd10fd2dd7fd","size":65411,"filename":"filter.bin","location":"staging/addons-bloomfilters/1db2b4c3-3608-4657-a66c-18a26e16d2d4.bin","mimetype":"application/octet-stream"},"key_format":"{guid}:{version}","attachment_type":"bloomfilter-base","generation_time":1588098908496,"id":"7b10f7bb-aa73-4733-933b-f03f3cabd5f6","last_modified":1588099019245}]}

services/settings/dumps/blocklists/addons-bloomfilters/addons-mlbf.bin.meta.json

@@ -0,0 +1 @@
+{"schema":1588016498560,"attachment":{"hash":"164992bd106fd2c4cb039f8c1b2581f1d5f2c8ecc1635a2aa69efd10fd2dd7fd","size":65411,"filename":"filter.bin","location":"staging/addons-bloomfilters/1db2b4c3-3608-4657-a66c-18a26e16d2d4.bin","mimetype":"application/octet-stream"},"key_format":"{guid}:{version}","attachment_type":"bloomfilter-base","generation_time":1588098908496,"id":"7b10f7bb-aa73-4733-933b-f03f3cabd5f6","last_modified":1588099019245}

services/settings/dumps/blocklists/moz.build

@@ -7,9 +7,15 @@
 with Files('**'):
     BUG_COMPONENT = ('Toolkit', 'Blocklist Implementation')
 
-FINAL_TARGET_FILES.defaults.settings.blocklists += ['addons.json',
+FINAL_TARGET_FILES.defaults.settings.blocklists += ['addons-bloomfilters.json',
+                                                    'addons.json',
                                                     'gfx.json',
                                                     'plugins.json']
 
+FINAL_TARGET_FILES.defaults.settings.blocklists['addons-bloomfilters'] += [
+  'addons-bloomfilters/addons-mlbf.bin',
+  'addons-bloomfilters/addons-mlbf.bin.meta.json'
+]
+
 if CONFIG['MOZ_BUILD_APP'] == 'browser':
     DIST_SUBDIR = 'browser'

toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js

@@ -0,0 +1,96 @@
+/* Any copyright is dedicated to the Public Domain.
+ * https://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * @fileOverview Verifies that the MLBF dump of the addons blocklist is
+ * correctly registered.
+ */
+
+Services.prefs.setBoolPref("extensions.blocklist.useMLBF", true);
+
+const { ExtensionBlocklist: ExtensionBlocklistMLBF } = Blocklist;
+
+// A known blocked version from bug 1626602.
+const blockedAddon = {
+  id: "{6f62927a-e380-401a-8c9e-c485b7d87f0d}",
+  version: "9.2.0",
+  signedState: 2, // = AddonManager.SIGNEDSTATE_SIGNED.
+  // The following date is the date of the first checked in MLBF. Any MLBF
+  // generated in the future should be generated after this date, to be useful.
+  signedDate: 1588098908496, // 2020-04-28 (dummy date)
+};
+
+// A known add-on that is not blocked, as of writing. It is likely not going
+// to be blocked because it does not have any executable code.
+const nonBlockedAddon = {
+  id: "disable-ctrl-q-and-cmd-q@robwu.nl",
+  version: "1",
+  signedState: 2, // = AddonManager.SIGNEDSTATE_SIGNED.
+  signedDate: 1482430349000, // 2016-12-22 (actual signing time).
+};
+
+async function sha256(arrayBuffer) {
+  Cu.importGlobalProperties(["crypto"]);
+  let hash = await crypto.subtle.digest("SHA-256", arrayBuffer);
+  const toHex = b => b.toString(16).padStart(2, "0");
+  return Array.from(new Uint8Array(hash), toHex).join("");
+}
+
+add_task(async function verify_dump_first_run() {
+  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
+
+  // Tapping into the internals of ExtensionBlocklistMLBF._fetchMLBF to observe
+  // MLBF request details.
+  const observed = [];
+
+  ExtensionBlocklistMLBF.ensureInitialized();
+  // Despite being called "download", this does not actually access the network
+  // when there is a valid dump.
+  const originalImpl = ExtensionBlocklistMLBF._client.attachments.download;
+  ExtensionBlocklistMLBF._client.attachments.download = function(record) {
+    let downloadPromise = originalImpl.apply(this, arguments);
+    observed.push({ inputRecord: record, downloadPromise });
+    return downloadPromise;
+  };
+
+  Assert.equal(
+    await Blocklist.getAddonBlocklistState(blockedAddon),
+    Ci.nsIBlocklistService.STATE_BLOCKED,
+    "A add-on that is known to be on the blocklist should be blocked"
+  );
+  Assert.equal(
+    await Blocklist.getAddonBlocklistState(nonBlockedAddon),
+    Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
+    "A known non-blocked add-on should not be blocked"
+  );
+
+  Assert.equal(observed.length, 1, "expected number of MLBF download requests");
+
+  const { inputRecord, downloadPromise } = observed[0];
+
+  Assert.ok(inputRecord, "addons-bloomfilters collection dump exists");
+
+  const downloadResult = await downloadPromise;
+
+  // Verify that the "download" result really originates from the local dump.
+  // "dump_match" means that the record exists in the collection and that an
+  // attachment was found.
+  //
+  // If this fails:
+  // - "dump_fallback" means that the MLBF attachment is out of sync with the
+  //   collection data.
+  // - undefined could mean that the implementation of Attachments.jsm changed.
+  Assert.equal(
+    downloadResult._source,
+    "dump_match",
+    "MLBF attachment should match the RemoteSettings collection"
+  );
+
+  Assert.equal(
+    await sha256(downloadResult.buffer),
+    inputRecord.attachment.hash,
+    "The content of the attachment should actually matches the record"
+  );
+});

toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_fetch.js

@@ -27,11 +27,10 @@ add_task(async function fetch_invalid_mlbf_record() {
     generation_time: 1,
   };
 
-  let resultPromise = ExtensionBlocklistMLBF._fetchMLBF(invalidRecord);
-
-  // TODO bug ...: When the MLBF is packaged with the application, this
-  // assertion should be updated to pass.
-  await Assert.rejects(resultPromise, /NetworkError/, "record not found");
+  // _fetchMLBF(invalidRecord) may succeed if there is a MLBF dump packaged with
+  // the application. This test intentionally hides the actual path to get
+  // deterministic results. To check whether the dump is correctly registered,
+  // run test_blocklist_mlbf_dump.js
 
   // Forget about the packaged attachment.
   Downloader._RESOURCE_BASE_URL = "invalid://bogus";

toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/xpcshell.ini

@@ -17,6 +17,8 @@ tags = remote-settings
 # Bug 676992: test consistently hangs on Android
 skip-if = os == "android"
 [test_blocklist_mlbf.js]
+[test_blocklist_mlbf_dump.js]
+skip-if = os == "android" # addons-bloomfilters not yet listed in mobile/android/installer/package-manifest.in
 [test_blocklist_mlbf_fetch.js]
 [test_blocklist_mlbf_stashes.js]
 [test_blocklist_mlbf_update.js]