Bug 1224528 - Load initial JSON files for blocklist r=mgoodwin

MozReview-Commit-ID: D53xoTa0PZu

--HG--
extra : rebase_source : a4aa143c627df0f70c15fd3589cde7a49e30d80d
This commit is contained in:
Mathieu Leplatre 2016-12-11 14:37:22 -10:00
Родитель 728d8ad262
Коммит 8e621b659c
15 изменённых файлов: 186 добавлений и 57 удалений

Просмотреть файл

@ -636,6 +636,7 @@
@RESPATH@/greprefs.js
@RESPATH@/defaults/autoconfig/prefcalls.js
@RESPATH@/browser/defaults/permissions
@RESPATH@/browser/defaults/blocklists
; Warning: changing the path to channel-prefs.js can cause bugs (Bug 756325)
; Technically this is an app pref file, but we are keeping it in the original

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Просмотреть файл

@ -0,0 +1,15 @@
# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
FINAL_TARGET_FILES.defaults.blocklists += ['addons.json',
'certificates.json',
'gfx.json',
'plugins.json']
FINAL_TARGET_FILES.defaults.pinning += ['pins.json']
if CONFIG['MOZ_BUILD_APP'] == 'browser':
DIST_SUBDIR = 'browser'

Просмотреть файл

@ -0,0 +1 @@
{"data":[]}

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Просмотреть файл

@ -0,0 +1,29 @@
# Blocklist
The blocklist entries are synchronized locally from the Firefox Settings service.
https://firefox.settings.services.mozilla.com
In order to reduce the amount of data to be downloaded on first synchronization,
a JSON dump from the records present on the remote server is shipped with the
release.
## How to update the JSON files ?
Even though it is not a problem if the dumps are not up-to-date when shipped, here
are the commands to update them:
```
SERVICE_URL="https://firefox.settings.services.mozilla.com/v1"
curl "$SERVICE_URL/buckets/blocklists/collections/certificates/records?" > services/blocklists/certificates.json
curl "$SERVICE_URL/buckets/blocklists/collections/gfx/records?" > services/blocklists/gfx.json
curl "$SERVICE_URL/buckets/blocklists/collections/plugins/records?" > services/blocklists/plugins.json
curl "$SERVICE_URL/buckets/blocklists/collections/addons/records?" > services/blocklists/addons.json
curl "$SERVICE_URL/buckets/pinning/collections/pins/records?" > services/blocklists/pins.json
```
## TODO
- Setup a bot to update it regularly.

Просмотреть файл

@ -7,15 +7,13 @@
this.EXPORTED_SYMBOLS = ["AddonBlocklistClient",
"GfxBlocklistClient",
"OneCRLBlocklistClient",
"PluginBlocklistClient",
"PinningBlocklistClient",
"FILENAME_ADDONS_JSON",
"FILENAME_GFX_JSON",
"FILENAME_PLUGINS_JSON"];
"PluginBlocklistClient"];
const { classes: Cc, interfaces: Ci, utils: Cu } = Components;
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
const { Task } = Cu.import("resource://gre/modules/Task.jsm", {});
const { OS } = Cu.import("resource://gre/modules/osfile.jsm", {});
Cu.importGlobalProperties(["fetch"]);
@ -25,6 +23,9 @@ const { KintoHttpClient } = Cu.import("resource://services-common/kinto-http-cli
const { FirefoxAdapter } = Cu.import("resource://services-common/kinto-storage-adapter.js", {});
const { CanonicalJSON } = Components.utils.import("resource://gre/modules/CanonicalJSON.jsm", {});
XPCOMUtils.defineLazyModuleGetter(this, "FileUtils", "resource://gre/modules/FileUtils.jsm");
const KEY_APPDIR = "XCurProcD";
const PREF_SETTINGS_SERVER = "services.settings.server";
const PREF_BLOCKLIST_BUCKET = "services.blocklist.bucket";
const PREF_BLOCKLIST_ONECRL_COLLECTION = "services.blocklist.onecrl.collection";
@ -48,9 +49,6 @@ const INVALID_SIGNATURE = "Invalid content/signature";
// filename, even though it isn't descriptive of who is using it.
this.KINTO_STORAGE_PATH = "kinto.sqlite";
this.FILENAME_ADDONS_JSON = "blocklist-addons.json";
this.FILENAME_GFX_JSON = "blocklist-gfx.json";
this.FILENAME_PLUGINS_JSON = "blocklist-plugins.json";
function mergeChanges(collection, localRecords, changes) {
@ -104,6 +102,29 @@ class BlocklistClient {
});
}
get filename() {
return `${this.bucketName}/${this.collectionName}.json`;
}
/**
* Load the the JSON file distributed with the release for this blocklist.
*
* For Bug 1257565 this method will have to try to load the file from the profile,
* in order to leverage the updateJSONBlocklist() below, which writes a new
* dump each time the collection changes.
*/
loadDumpFile() {
const fileURI = `resource://app/defaults/${this.filename}`;
return Task.spawn(function* loadFile() {
const response = yield fetch(fileURI);
if (!response.ok) {
throw new Error(`Could not read from '${fileURI}'`);
}
// Will be rejected if JSON is invalid.
return yield response.json();
});
}
validateCollectionSignature(remote, payload, collection, options = {}) {
const {ignoreLocal} = options;
@ -145,14 +166,17 @@ class BlocklistClient {
/**
* Synchronize from Kinto server, if necessary.
*
* @param {int} lastModified the lastModified date (on the server) for
the remote collection.
* @param {Date} serverTime the current date return by the server.
* @return {Promise} which rejects on sync or process failure.
* @param {int} lastModified the lastModified date (on the server) for
the remote collection.
* @param {Date} serverTime the current date return by the server.
* @param {Object} options additional advanced options.
* @param {bool} options.loadDump load initial dump from disk on first sync (default: true)
* @return {Promise} which rejects on sync or process failure.
*/
maybeSync(lastModified, serverTime) {
maybeSync(lastModified, serverTime, options = {loadDump: true}) {
const {loadDump} = options;
const remote = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
let enforceCollectionSigning =
const enforceCollectionSigning =
Services.prefs.getBoolPref(PREF_BLOCKLIST_ENFORCE_SIGNING);
// if there is a signerName and collection signing is enforced, add a
@ -177,13 +201,30 @@ class BlocklistClient {
};
const collection = this._kinto.collection(this.collectionName, options);
const collectionLastModified = yield collection.db.getLastModified();
let collectionLastModified = yield collection.db.getLastModified();
// If there is no data currently in the collection, attempt to import
// initial data from the application defaults.
// This allows to avoid synchronizing the whole collection content on
// cold start.
if (!collectionLastModified && loadDump) {
try {
const initialData = yield this.loadDumpFile();
yield collection.db.loadDump(initialData.data);
collectionLastModified = yield collection.db.getLastModified();
} catch (e) {
// Report but go-on.
Cu.reportError(e);
}
}
// If the data is up to date, there's no need to sync. We still need
// to record the fact that a check happened.
if (lastModified <= collectionLastModified) {
this.updateLastCheck(serverTime);
return;
}
// Fetch changes from server.
try {
const {ok} = yield collection.sync({remote});
@ -315,10 +356,13 @@ function* updatePinningList(records) {
function* updateJSONBlocklist(filename, records) {
// Write JSON dump for synchronous load at startup.
const path = OS.Path.join(OS.Constants.Path.profileDir, filename);
const blocklistFolder = OS.Path.dirname(path);
yield OS.File.makeDir(blocklistFolder, {from: OS.Constants.Path.profileDir});
const serialized = JSON.stringify({data: records}, null, 2);
try {
yield OS.File.writeAtomic(path, serialized, {tmpPath: path + ".tmp"});
// Notify change to `nsBlocklistService`
const eventData = {filename};
Services.cpmm.sendAsyncMessage("Blocklist:reload-from-disk", eventData);
@ -338,21 +382,21 @@ this.OneCRLBlocklistClient = new BlocklistClient(
this.AddonBlocklistClient = new BlocklistClient(
Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS_COLLECTION),
PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS,
updateJSONBlocklist.bind(undefined, FILENAME_ADDONS_JSON),
(records) => updateJSONBlocklist(this.AddonBlocklistClient.filename, records),
Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET)
);
this.GfxBlocklistClient = new BlocklistClient(
Services.prefs.getCharPref(PREF_BLOCKLIST_GFX_COLLECTION),
PREF_BLOCKLIST_GFX_CHECKED_SECONDS,
updateJSONBlocklist.bind(undefined, FILENAME_GFX_JSON),
(records) => updateJSONBlocklist(this.GfxBlocklistClient.filename, records),
Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET)
);
this.PluginBlocklistClient = new BlocklistClient(
Services.prefs.getCharPref(PREF_BLOCKLIST_PLUGINS_COLLECTION),
PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS,
updateJSONBlocklist.bind(undefined, FILENAME_PLUGINS_JSON),
(records) => updateJSONBlocklist(this.PluginBlocklistClient.filename, records),
Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET)
);

Просмотреть файл

@ -12,9 +12,10 @@ const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
let server;
// set up what we need to make storage adapters
const kintoFilename = "kinto.sqlite";
let sqliteHandle;
const KINTO_FILENAME = "kinto.sqlite";
function do_get_kinto_collection(collectionName, sqliteHandle) {
function do_get_kinto_collection(collectionName) {
let config = {
// Set the remote to be some server that will cause test failure when
// hit since we should never hit the server directly, only via maybeSync()
@ -35,8 +36,8 @@ add_task(function* test_something() {
const configPath = "/v1/";
const recordsPath = "/v1/buckets/blocklists/collections/certificates/records";
Services.prefs.setCharPref("services.settings.server",
`http://localhost:${server.identity.primaryPort}/v1`);
const dummyServerURL = `http://localhost:${server.identity.primaryPort}/v1`;
Services.prefs.setCharPref("services.settings.server", dummyServerURL);
// register a handler
function handleResponse(request, response) {
@ -66,24 +67,45 @@ add_task(function* test_something() {
// Test an empty db populates
yield OneCRLBlocklistClient.maybeSync(2000, Date.now());
sqliteHandle = yield FirefoxAdapter.openConnection({path: KINTO_FILENAME});
const collection = do_get_kinto_collection("certificates");
// Open the collection, verify it's been populated:
// Our test data has a single record; it should be in the local collection
let sqliteHandle = yield FirefoxAdapter.openConnection({path: kintoFilename});
let collection = do_get_kinto_collection("certificates", sqliteHandle);
let list = yield collection.list();
// We know there will be initial values from the JSON dump.
// (at least as many as in the dump shipped when this test was written).
do_check_true(list.data.length >= 363);
// No sync will be intented if maybeSync() is up-to-date.
Services.prefs.clearUserPref("services.settings.server");
Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0);
// Use any last_modified older than highest shipped in JSON dump.
yield OneCRLBlocklistClient.maybeSync(123456, Date.now());
// Last check value was updated.
do_check_neq(0, Services.prefs.getIntPref("services.blocklist.onecrl.checked"));
// Restore server pref.
Services.prefs.setCharPref("services.settings.server", dummyServerURL);
// clear the collection, save a non-zero lastModified so we don't do
// import of initial data when we sync again.
yield collection.clear();
// a lastModified value of 1000 means we get a remote collection with a
// single record
yield collection.db.saveLastModified(1000);
yield OneCRLBlocklistClient.maybeSync(2000, Date.now());
// Open the collection, verify it's been updated:
// Our test data now has two records; both should be in the local collection
list = yield collection.list();
do_check_eq(list.data.length, 1);
yield sqliteHandle.close();
// Test the db is updated when we call again with a later lastModified value
yield OneCRLBlocklistClient.maybeSync(4000, Date.now());
// Open the collection, verify it's been updated:
// Our test data now has two records; both should be in the local collection
sqliteHandle = yield FirefoxAdapter.openConnection({path: kintoFilename});
collection = do_get_kinto_collection("certificates", sqliteHandle);
list = yield collection.list();
do_check_eq(list.data.length, 3);
yield sqliteHandle.close();
// Try to maybeSync with the current lastModified value - no connection
// should be attempted.
@ -104,8 +126,7 @@ add_task(function* test_something() {
// Check that a sync completes even when there's bad data in the
// collection. This will throw on fail, so just calling maybeSync is an
// acceptible test.
Services.prefs.setCharPref("services.settings.server",
`http://localhost:${server.identity.primaryPort}/v1`);
Services.prefs.setCharPref("services.settings.server", dummyServerURL);
yield OneCRLBlocklistClient.maybeSync(5000, Date.now());
});
@ -122,6 +143,7 @@ function run_test() {
do_register_cleanup(function() {
server.stop(() => { });
return sqliteHandle.close();
});
}
@ -150,6 +172,17 @@ function getSampleResponse(req, port) {
"responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
},
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{}]})
},
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=1000": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",

Просмотреть файл

@ -17,9 +17,9 @@ const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
const kintoFilename = "kinto.sqlite";
const gBlocklistClients = [
{client: BlocklistClients.AddonBlocklistClient, filename: BlocklistClients.FILENAME_ADDONS_JSON, testData: ["i808", "i720", "i539"]},
{client: BlocklistClients.PluginBlocklistClient, filename: BlocklistClients.FILENAME_PLUGINS_JSON, testData: ["p1044", "p32", "p28"]},
{client: BlocklistClients.GfxBlocklistClient, filename: BlocklistClients.FILENAME_GFX_JSON, testData: ["g204", "g200", "g36"]},
{client: BlocklistClients.AddonBlocklistClient, testData: ["i808", "i720", "i539"]},
{client: BlocklistClients.PluginBlocklistClient, testData: ["p1044", "p32", "p28"]},
{client: BlocklistClients.GfxBlocklistClient, testData: ["g204", "g200", "g36"]},
];
@ -57,14 +57,10 @@ function* clear_state() {
} finally {
yield sqliteHandle.close();
}
}
// Remove profile data.
for (let {filename} of gBlocklistClients) {
const blocklist = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
if (blocklist.exists()) {
blocklist.remove(true);
}
// Remove profile data.
const path = OS.Path.join(OS.Constants.Path.profileDir, client.filename);
yield OS.File.remove(path, { ignoreAbsent: true });
}
}
@ -121,7 +117,7 @@ function run_test() {
add_task(function* test_records_obtained_from_server_are_stored_in_db() {
for (let {client} of gBlocklistClients) {
// Test an empty db populates
yield client.maybeSync(2000, Date.now());
yield client.maybeSync(2000, Date.now(), {loadDump: false});
// Open the collection, verify it's been populated:
// Our test data has a single record; it should be in the local collection
@ -135,11 +131,11 @@ add_task(function* test_records_obtained_from_server_are_stored_in_db() {
add_task(clear_state);
add_task(function* test_list_is_written_to_file_in_profile() {
for (let {client, filename, testData} of gBlocklistClients) {
const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
for (let {client, testData} of gBlocklistClients) {
const profFile = FileUtils.getFile(KEY_PROFILEDIR, client.filename.split("/"));
strictEqual(profFile.exists(), false);
yield client.maybeSync(2000, Date.now());
yield client.maybeSync(2000, Date.now(), {loadDump: false});
strictEqual(profFile.exists(), true);
const content = yield readJSON(profFile.path);
@ -159,9 +155,9 @@ add_task(function* test_current_server_time_is_saved_in_pref() {
add_task(clear_state);
add_task(function* test_update_json_file_when_addons_has_changes() {
for (let {client, filename, testData} of gBlocklistClients) {
yield client.maybeSync(2000, Date.now() - 1000);
const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
for (let {client, testData} of gBlocklistClients) {
yield client.maybeSync(2000, Date.now() - 1000, {loadDump: false});
const profFile = FileUtils.getFile(KEY_PROFILEDIR, client.filename.split("/"));
const fileLastModified = profFile.lastModifiedTime = profFile.lastModifiedTime - 1000;
const serverTime = Date.now();
@ -179,24 +175,24 @@ add_task(function* test_update_json_file_when_addons_has_changes() {
add_task(clear_state);
add_task(function* test_sends_reload_message_when_blocklist_has_changes() {
for (let {client, filename} of gBlocklistClients) {
for (let {client} of gBlocklistClients) {
let received = yield new Promise((resolve, reject) => {
Services.ppmm.addMessageListener("Blocklist:reload-from-disk", {
receiveMessage(aMsg) { resolve(aMsg) }
});
client.maybeSync(2000, Date.now() - 1000);
client.maybeSync(2000, Date.now() - 1000, {loadDump: false});
});
equal(received.data.filename, filename);
equal(received.data.filename, client.filename);
}
});
add_task(clear_state);
add_task(function* test_do_nothing_when_blocklist_is_up_to_date() {
for (let {client, filename} of gBlocklistClients) {
yield client.maybeSync(2000, Date.now() - 1000);
const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
for (let {client} of gBlocklistClients) {
yield client.maybeSync(2000, Date.now() - 1000, {loadDump: false});
const profFile = FileUtils.getFile(KEY_PROFILEDIR, client.filename.split("/"));
const fileLastModified = profFile.lastModifiedTime = profFile.lastModifiedTime - 1000;
const serverTime = Date.now();

Просмотреть файл

@ -294,7 +294,8 @@ add_task(function* test_check_signatures() {
// With all of this set up, we attempt a sync. This will resolve if all is
// well and throw if something goes wrong.
yield OneCRLBlocklistClient.maybeSync(1000, startTime);
// We don't want to load initial json dumps in this test suite.
yield OneCRLBlocklistClient.maybeSync(1000, startTime, {loadDump: false});
// Check that some additions (2 records) to the collection have a valid
// signature.

Просмотреть файл

@ -9,6 +9,8 @@ support-files =
[test_load_modules.js]
[test_blocklist_certificates.js]
# Initial JSON data for blocklists are not shipped on Android.
skip-if = os == "android"
[test_blocklist_clients.js]
[test_blocklist_updater.js]
[test_blocklist_pinning.js]

Просмотреть файл

@ -10,7 +10,10 @@ DIRS += [
]
if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android':
DIRS += ['fxaccounts']
DIRS += [
'fxaccounts',
'blocklists',
]
if CONFIG['MOZ_SERVICES_SYNC']:
DIRS += ['sync']

Просмотреть файл

@ -17,7 +17,7 @@
"AsyncSpellCheckTestHelper.jsm": ["onSpellCheck"],
"AutoMigrate.jsm": ["AutoMigrate"],
"Battery.jsm": ["GetBattery", "Battery"],
"blocklist-clients.js": ["AddonBlocklistClient", "GfxBlocklistClient", "OneCRLBlocklistClient", "PluginBlocklistClient", "FILENAME_ADDONS_JSON", "FILENAME_GFX_JSON", "FILENAME_PLUGINS_JSON"],
"blocklist-clients.js": ["AddonBlocklistClient", "GfxBlocklistClient", "OneCRLBlocklistClient", "PluginBlocklistClient"],
"blocklist-updater.js": ["checkVersions", "addTestBlocklistClient"],
"bogus_element_type.jsm": [],
"bookmark_validator.js": ["BookmarkValidator", "BookmarkProblemData"],