diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index f1cf471fc324..3eb512673e60 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -2211,6 +2211,10 @@ pref("services.blocklist.addons.collection", "addons"); pref("services.blocklist.addons.checked", 0); pref("services.blocklist.plugins.collection", "plugins"); pref("services.blocklist.plugins.checked", 0); +pref("services.blocklist.pinning.enabled", true); +pref("services.blocklist.pinning.bucket", "pinning"); +pref("services.blocklist.pinning.collection", "pins"); +pref("services.blocklist.pinning.checked", 0); pref("services.blocklist.gfx.collection", "gfx"); pref("services.blocklist.gfx.checked", 0); diff --git a/services/common/blocklist-clients.js b/services/common/blocklist-clients.js index b4df5c6a3824..5db18ca5b289 100644 --- a/services/common/blocklist-clients.js +++ b/services/common/blocklist-clients.js @@ -8,6 +8,7 @@ this.EXPORTED_SYMBOLS = ["AddonBlocklistClient", "GfxBlocklistClient", "OneCRLBlocklistClient", "PluginBlocklistClient", + "PinningBlocklistClient", "FILENAME_ADDONS_JSON", "FILENAME_GFX_JSON", "FILENAME_PLUGINS_JSON"]; @@ -32,6 +33,10 @@ const PREF_BLOCKLIST_ADDONS_COLLECTION = "services.blocklist.addons.collec const PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS = "services.blocklist.addons.checked"; const PREF_BLOCKLIST_PLUGINS_COLLECTION = "services.blocklist.plugins.collection"; const PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS = "services.blocklist.plugins.checked"; +const PREF_BLOCKLIST_PINNING_ENABLED = "services.blocklist.pinning.enabled"; +const PREF_BLOCKLIST_PINNING_BUCKET = "services.blocklist.pinning.bucket"; +const PREF_BLOCKLIST_PINNING_COLLECTION = "services.blocklist.pinning.collection"; +const PREF_BLOCKLIST_PINNING_CHECKED_SECONDS = "services.blocklist.pinning.checked"; const PREF_BLOCKLIST_GFX_COLLECTION = "services.blocklist.gfx.collection"; const PREF_BLOCKLIST_GFX_CHECKED_SECONDS = "services.blocklist.gfx.checked"; const PREF_BLOCKLIST_ENFORCE_SIGNING = "services.blocklist.signing.enforced"; @@ -82,9 +87,8 @@ function fetchRemoteCollection(collection) { * URL and bucket name. It uses the `FirefoxAdapter` which relies on SQLite to * persist the local DB. */ -function kintoClient(connection) { +function kintoClient(connection, bucket) { let base = Services.prefs.getCharPref(PREF_SETTINGS_SERVER); - let bucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET); let config = { remote: base, @@ -99,10 +103,11 @@ function kintoClient(connection) { class BlocklistClient { - constructor(collectionName, lastCheckTimePref, processCallback, signerName) { + constructor(collectionName, lastCheckTimePref, processCallback, bucketName, signerName) { this.collectionName = collectionName; this.lastCheckTimePref = lastCheckTimePref; this.processCallback = processCallback; + this.bucketName = bucketName; this.signerName = signerName; } @@ -168,7 +173,7 @@ class BlocklistClient { let connection; try { connection = yield FirefoxAdapter.openConnection({path: KINTO_STORAGE_PATH}); - let db = kintoClient(connection); + let db = kintoClient(connection, this.bucketName); let collection = db.collection(this.collectionName, opts); let collectionLastModified = yield collection.db.getLastModified(); @@ -247,14 +252,54 @@ function* updateCertBlocklist(records) { } } catch (e) { // prevent errors relating to individual blocklist entries from - // causing sync to fail. At some point in the future, we may want to - // accumulate telemetry on these failures. + // causing sync to fail. We will accumulate telemetry on these failures in + // bug 1254099. Cu.reportError(e); } } certList.saveEntries(); } +/** + * Modify the appropriate security pins based on records from the remote + * collection. + * + * @param {Object} records current records in the local db. + */ +function* updatePinningList(records) { + if (Services.prefs.getBoolPref(PREF_BLOCKLIST_PINNING_ENABLED)) { + const appInfo = Cc["@mozilla.org/xre/app-info;1"] + .getService(Ci.nsIXULAppInfo); + + const siteSecurityService = Cc["@mozilla.org/ssservice;1"] + .getService(Ci.nsISiteSecurityService); + + // clear the current preload list + siteSecurityService.clearPreloads(); + + // write each KeyPin entry to the preload list + for (let item of records) { + try { + const {pinType, pins=[], versions} = item; + if (pinType == "KeyPin" && pins.length && + versions.indexOf(appInfo.version) != -1) { + siteSecurityService.setKeyPins(item.hostName, + item.includeSubdomains, + item.expires, + pins.length, + pins, true); + } + } catch (e) { + // prevent errors relating to individual preload entries from causing + // sync to fail. We will accumulate telemetry for such failures in bug + // 1254099. + } + } + } else { + return; + } +} + /** * Write list of records into JSON file, and notify nsBlocklistService. * @@ -276,28 +321,39 @@ function* updateJSONBlocklist(filename, records) { } } - this.OneCRLBlocklistClient = new BlocklistClient( Services.prefs.getCharPref(PREF_BLOCKLIST_ONECRL_COLLECTION), PREF_BLOCKLIST_ONECRL_CHECKED_SECONDS, updateCertBlocklist, + Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET), "onecrl.content-signature.mozilla.org" ); this.AddonBlocklistClient = new BlocklistClient( Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS_COLLECTION), PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS, - updateJSONBlocklist.bind(undefined, FILENAME_ADDONS_JSON) + updateJSONBlocklist.bind(undefined, FILENAME_ADDONS_JSON), + Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET) ); this.GfxBlocklistClient = new BlocklistClient( Services.prefs.getCharPref(PREF_BLOCKLIST_GFX_COLLECTION), PREF_BLOCKLIST_GFX_CHECKED_SECONDS, - updateJSONBlocklist.bind(undefined, FILENAME_GFX_JSON) + updateJSONBlocklist.bind(undefined, FILENAME_GFX_JSON), + Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET) ); this.PluginBlocklistClient = new BlocklistClient( Services.prefs.getCharPref(PREF_BLOCKLIST_PLUGINS_COLLECTION), PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS, - updateJSONBlocklist.bind(undefined, FILENAME_PLUGINS_JSON) + updateJSONBlocklist.bind(undefined, FILENAME_PLUGINS_JSON), + Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET) +); + +this.PinningPreloadClient = new BlocklistClient( + Services.prefs.getCharPref(PREF_BLOCKLIST_PINNING_COLLECTION), + PREF_BLOCKLIST_PINNING_CHECKED_SECONDS, + updatePinningList, + Services.prefs.getCharPref(PREF_BLOCKLIST_PINNING_BUCKET), + "pinning-preload.content-signature.mozilla.org" ); diff --git a/services/common/blocklist-updater.js b/services/common/blocklist-updater.js index 3b39b955245f..05f3aa87dc9f 100644 --- a/services/common/blocklist-updater.js +++ b/services/common/blocklist-updater.js @@ -13,7 +13,6 @@ const BlocklistClients = Cu.import("resource://services-common/blocklist-clients const PREF_SETTINGS_SERVER = "services.settings.server"; const PREF_BLOCKLIST_CHANGES_PATH = "services.blocklist.changes.path"; -const PREF_BLOCKLIST_BUCKET = "services.blocklist.bucket"; const PREF_BLOCKLIST_LAST_UPDATE = "services.blocklist.last_update_seconds"; const PREF_BLOCKLIST_LAST_ETAG = "services.blocklist.last_etag"; const PREF_BLOCKLIST_CLOCK_SKEW_SECONDS = "services.blocklist.clock_skew_seconds"; @@ -23,7 +22,8 @@ const gBlocklistClients = { [BlocklistClients.OneCRLBlocklistClient.collectionName]: BlocklistClients.OneCRLBlocklistClient, [BlocklistClients.AddonBlocklistClient.collectionName]: BlocklistClients.AddonBlocklistClient, [BlocklistClients.GfxBlocklistClient.collectionName]: BlocklistClients.GfxBlocklistClient, - [BlocklistClients.PluginBlocklistClient.collectionName]: BlocklistClients.PluginBlocklistClient + [BlocklistClients.PluginBlocklistClient.collectionName]: BlocklistClients.PluginBlocklistClient, + [BlocklistClients.PinningPreloadClient.collectionName]: BlocklistClients.PinningPreloadClient }; // Add a blocklist client for testing purposes. Do not use for any other purpose @@ -44,7 +44,6 @@ this.checkVersions = function() { // Right now, we only use the collection name and the last modified info let kintoBase = Services.prefs.getCharPref(PREF_SETTINGS_SERVER); let changesEndpoint = kintoBase + Services.prefs.getCharPref(PREF_BLOCKLIST_CHANGES_PATH); - let blocklistsBucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET); // Use ETag to obtain a `304 Not modified` when no change occurred. const headers = {}; @@ -82,14 +81,11 @@ this.checkVersions = function() { let firstError; for (let collectionInfo of versionInfo.data) { - // Skip changes that don't concern configured blocklist bucket. - if (collectionInfo.bucket != blocklistsBucket) { - continue; - } - let collection = collectionInfo.collection; let client = gBlocklistClients[collection]; - if (client && client.maybeSync) { + if (client && + client.bucketName == collectionInfo.bucket && + client.maybeSync) { let lastModified = 0; if (collectionInfo.last_modified) { lastModified = collectionInfo.last_modified; diff --git a/services/common/tests/unit/test_blocklist_pinning.js b/services/common/tests/unit/test_blocklist_pinning.js new file mode 100644 index 000000000000..b1e2bb4efc56 --- /dev/null +++ b/services/common/tests/unit/test_blocklist_pinning.js @@ -0,0 +1,298 @@ +"use strict" + +const { Constructor: CC } = Components; + +Cu.import("resource://testing-common/httpd.js"); + +const { Kinto } = Cu.import("resource://services-common/kinto-offline-client.js"); +const { FirefoxAdapter } = Cu.import("resource://services-common/kinto-storage-adapter.js"); + +const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", "setInputStream"); + +const PREF_BLOCKLIST_PINNING_COLLECTION = "services.blocklist.pinning.collection"; +const COLLECTION_NAME = "pins"; +const KINTO_STORAGE_PATH = "kinto.sqlite"; + +// First, we need to setup appInfo or we can't do version checks +var id = "xpcshell@tests.mozilla.org"; +var appName = "XPCShell"; +var version = "1"; +var platformVersion = "1.9.2"; +Cu.import("resource://testing-common/AppInfo.jsm", this); + +updateAppInfo({ + name: appName, + ID: id, + version: version, + platformVersion: platformVersion ? platformVersion : "1.0", + crashReporter: true, +}); + +let server; + + +function do_get_kinto_collection(connection, collectionName) { + let config = { + // Set the remote to be some server that will cause test failure when + // hit since we should never hit the server directly (any non-local + // request causes failure in tests), only via maybeSync() + remote: "https://firefox.settings.services.mozilla.com/v1/", + // Set up the adapter and bucket as normal + adapter: FirefoxAdapter, + adapterOptions: {sqliteHandle: connection}, + bucket: "pinning" + }; + let kintoClient = new Kinto(config); + return kintoClient.collection(collectionName); +} + +// Some simple tests to demonstrate that the core preload sync operations work +// correctly and that simple kinto operations are working as expected. +add_task(function* test_something(){ + // set the collection name explicitly - since there will be version + // specific collection names in prefs + Services.prefs.setCharPref(PREF_BLOCKLIST_PINNING_COLLECTION, + COLLECTION_NAME); + + const { PinningPreloadClient } = Cu.import("resource://services-common/blocklist-clients.js"); + + const configPath = "/v1/"; + const recordsPath = "/v1/buckets/pinning/collections/pins/records"; + + Services.prefs.setCharPref("services.settings.server", + `http://localhost:${server.identity.primaryPort}/v1`); + + // register a handler + function handleResponse (request, response) { + try { + const sample = getSampleResponse(request, server.identity.primaryPort); + if (!sample) { + do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`); + } + + response.setStatusLine(null, sample.status.status, + sample.status.statusText); + // send the headers + for (let headerLine of sample.sampleHeaders) { + let headerElements = headerLine.split(':'); + response.setHeader(headerElements[0], headerElements[1].trimLeft()); + } + response.setHeader("Date", (new Date()).toUTCString()); + + response.write(sample.responseBody); + } catch (e) { + do_print(e); + } + } + server.registerPathHandler(configPath, handleResponse); + server.registerPathHandler(recordsPath, handleResponse); + + let sss = Cc["@mozilla.org/ssservice;1"] + .getService(Ci.nsISiteSecurityService); + + // ensure our pins are all missing before we start + ok(!sss.isSecureHost(sss.HEADER_HPKP, "one.example.com", 0)); + ok(!sss.isSecureHost(sss.HEADER_HPKP, "two.example.com", 0)); + ok(!sss.isSecureHost(sss.HEADER_HPKP, "three.example.com", 0)); + + // Test an empty db populates + let result = yield PinningPreloadClient.maybeSync(2000, Date.now()); + + let connection = yield FirefoxAdapter.openConnection({path: KINTO_STORAGE_PATH}); + + // Open the collection, verify it's been populated: + // Our test data has a single record; it should be in the local collection + let collection = do_get_kinto_collection(connection, COLLECTION_NAME); + let list = yield collection.list(); + do_check_eq(list.data.length, 1); + + // check that a pin exists for one.example.com + ok(sss.isSecureHost(sss.HEADER_HPKP, "one.example.com", 0)); + + // Test the db is updated when we call again with a later lastModified value + result = yield PinningPreloadClient.maybeSync(4000, Date.now()); + + // Open the collection, verify it's been updated: + // Our data now has three new records; all should be in the local collection + collection = do_get_kinto_collection(connection, COLLECTION_NAME); + list = yield collection.list(); + do_check_eq(list.data.length, 4); + yield connection.close(); + + // check that a pin exists for two.example.com and three.example.com + ok(sss.isSecureHost(sss.HEADER_HPKP, "two.example.com", 0)); + ok(sss.isSecureHost(sss.HEADER_HPKP, "three.example.com", 0)); + + // check that a pin does not exist for four.example.com - it's in the + // collection but the version should not match + ok(!sss.isSecureHost(sss.HEADER_HPKP, "four.example.com", 0)); + + // Try to maybeSync with the current lastModified value - no connection + // should be attempted. + // Clear the kinto base pref so any connections will cause a test failure + Services.prefs.clearUserPref("services.settings.server"); + yield PinningPreloadClient.maybeSync(4000, Date.now()); + + // Try again with a lastModified value at some point in the past + yield PinningPreloadClient.maybeSync(3000, Date.now()); + + // Check the pinning check time pref is modified, even if the collection + // hasn't changed + Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0); + yield PinningPreloadClient.maybeSync(3000, Date.now()); + let newValue = Services.prefs.getIntPref("services.blocklist.pinning.checked"); + do_check_neq(newValue, 0); + + // Check that a sync completes even when there's bad data in the + // collection. This will throw on fail, so just calling maybeSync is an + // acceptible test (the data below with last_modified of 300 is nonsense). + Services.prefs.setCharPref("services.settings.server", + `http://localhost:${server.identity.primaryPort}/v1`); + yield PinningPreloadClient.maybeSync(5000, Date.now()); +}); + +function run_test() { + // Ensure that signature verification is disabled to prevent interference + // with basic certificate sync tests + Services.prefs.setBoolPref("services.blocklist.signing.enforced", false); + + // Set up an HTTP Server + server = new HttpServer(); + server.start(-1); + + run_next_test(); + + do_register_cleanup(function() { + server.stop(() => { }); + }); +} + +// get a response for a given request from sample data +function getSampleResponse(req, port) { + const appInfo = Cc["@mozilla.org/xre/app-info;1"] + .getService(Ci.nsIXULAppInfo); + + const responses = { + "OPTIONS": { + "sampleHeaders": [ + "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page", + "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS", + "Access-Control-Allow-Origin: *", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress" + ], + "status": {status: 200, statusText: "OK"}, + "responseBody": "null" + }, + "GET:/v1/?": { + "sampleHeaders": [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress" + ], + "status": {status: 200, statusText: "OK"}, + "responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"}) + }, + "GET:/v1/buckets/pinning/collections/pins/records?_sort=-last_modified": { + "sampleHeaders": [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + "Etag: \"3000\"" + ], + "status": {status: 200, statusText: "OK"}, + "responseBody": JSON.stringify({"data":[{ + "pinType": "KeyPin", + "hostName": "one.example.com", + "includeSubdomains": false, + "expires": new Date().getTime() + 1000000, + "pins" : ["cUPcTAZWKaASuYWhhneDttWpY3oBAkE3h2+soZS7sWs=", + "M8HztCzM3elUxkcjR2S5P4hhyBNf6lHkmjAHKhpGPWE="], + "versions" : [appInfo.version], + "id":"78cf8900-fdea-4ce5-f8fb-b78710617718", + "last_modified":3000 + }]}) + }, + "GET:/v1/buckets/pinning/collections/pins/records?_sort=-last_modified&_since=3000": { + "sampleHeaders": [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + "Etag: \"4000\"" + ], + "status": {status: 200, statusText: "OK"}, + "responseBody": JSON.stringify({"data":[{ + "pinType": "KeyPin", + "hostName": "two.example.com", + "includeSubdomains": false, + "expires": new Date().getTime() + 1000000, + "pins" : ["cUPcTAZWKaASuYWhhneDttWpY3oBAkE3h2+soZS7sWs=", + "M8HztCzM3elUxkcjR2S5P4hhyBNf6lHkmjAHKhpGPWE="], + "versions" : [appInfo.version], + "id":"dabafde9-df4a-ddba-2548-748da04cc02c", + "last_modified":4000 + },{ + "pinType": "KeyPin", + "hostName": "three.example.com", + "includeSubdomains": false, + "expires": new Date().getTime() + 1000000, + "pins" : ["cUPcTAZWKaASuYWhhneDttWpY3oBAkE3h2+soZS7sWs=", + "M8HztCzM3elUxkcjR2S5P4hhyBNf6lHkmjAHKhpGPWE="], + "versions" : [appInfo.version, "some other version that won't match"], + "id":"dabafde9-df4a-ddba-2548-748da04cc02d", + "last_modified":4000 + },{ + "pinType": "KeyPin", + "hostName": "four.example.com", + "includeSubdomains": false, + "expires": new Date().getTime() + 1000000, + "pins" : ["cUPcTAZWKaASuYWhhneDttWpY3oBAkE3h2+soZS7sWs=", + "M8HztCzM3elUxkcjR2S5P4hhyBNf6lHkmjAHKhpGPWE="], + "versions" : ["some version that won't match"], + "id":"dabafde9-df4a-ddba-2548-748da04cc02e", + "last_modified":4000 + }]}) + }, + "GET:/v1/buckets/pinning/collections/pins/records?_sort=-last_modified&_since=4000": { + "sampleHeaders": [ + "Access-Control-Allow-Origin: *", + "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff", + "Content-Type: application/json; charset=UTF-8", + "Server: waitress", + "Etag: \"5000\"" + ], + "status": {status: 200, statusText: "OK"}, + "responseBody": JSON.stringify({"data":[{ + "irrelevant":"this entry looks nothing whatsoever like a pin preload", + "pinType": "KeyPin", + "id":"dabafde9-df4a-ddba-2548-748da04cc02f", + "last_modified":5000 + },{ + "irrelevant":"this entry has data of the wrong type", + "pinType": "KeyPin", + "hostName": 3, + "includeSubdomains": "nonsense", + "expires": "more nonsense", + "pins" : [1,2,3,4], + "id":"dabafde9-df4a-ddba-2548-748da04cc030", + "last_modified":5000 + },{ + "irrelevant":"this entry is missing the actual pins", + "pinType": "KeyPin", + "hostName": "missingpins.example.com", + "includeSubdomains": false, + "expires": new Date().getTime() + 1000000, + "versions" : [appInfo.version], + "id":"dabafde9-df4a-ddba-2548-748da04cc031", + "last_modified":5000 + }]}) + } + }; + return responses[`${req.method}:${req.path}?${req.queryString}`] || + responses[req.method]; + +} diff --git a/services/common/tests/unit/test_blocklist_updater.js b/services/common/tests/unit/test_blocklist_updater.js index 1b71c194a040..e91461f96900 100644 --- a/services/common/tests/unit/test_blocklist_updater.js +++ b/services/common/tests/unit/test_blocklist_updater.js @@ -57,6 +57,7 @@ add_task(function* test_check_maybeSync(){ // add a test kinto client that will respond to lastModified information // for a collection called 'test-collection' updater.addTestBlocklistClient("test-collection", { + bucketName: "blocklists", maybeSync(lastModified, serverTime) { do_check_eq(lastModified, 1000); do_check_eq(serverTime, 2000); diff --git a/services/common/tests/unit/xpcshell.ini b/services/common/tests/unit/xpcshell.ini index dbec09519a7b..aa10b112d7ac 100644 --- a/services/common/tests/unit/xpcshell.ini +++ b/services/common/tests/unit/xpcshell.ini @@ -12,6 +12,7 @@ support-files = [test_blocklist_certificates.js] [test_blocklist_clients.js] [test_blocklist_updater.js] +[test_blocklist_pinning.js] [test_kinto.js] [test_blocklist_signatures.js]