зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1509066 - Sync on RemoteSettings.get() when local db is empty r=glasserc
Differential Revision: https://phabricator.services.mozilla.com/D13099 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Mathieu Leplatre
Родитель
bfddcfb70e
Коммит
57a0b8ae80
|
|
@ -31,6 +31,11 @@ The ``get()`` method returns the list of entries for a specific key. Each entry
|
|||
// await InternalAPI.load(entry.id, entry.label, entry.weight);
|
||||
});
|
||||
|
||||
.. note::
|
||||
The data updates are managed internally, and ``.get()`` only returns the local data.
|
||||
The data is pulled from the server only if this collection has no local data yet and no JSON dump
|
||||
could be found (see :ref:`services/initial-data` below).
|
||||
|
||||
.. note::
|
||||
The ``id`` and ``last_modified`` (timestamp) attributes are assigned by the server.
|
||||
|
||||
|
|
@ -89,18 +94,23 @@ When an entry has a file attached to it, it has an ``attachment`` attribute, whi
|
|||
}
|
||||
});
|
||||
|
||||
.. _services/initial-data:
|
||||
|
||||
Initial data
|
||||
------------
|
||||
|
||||
For newly created user profiles, the list of entries returned by the ``.get()`` method will be empty until the first synchronization happens.
|
||||
It is possible to package a dump of the server records that will be loaded into the local database when no synchronization has happened yet.
|
||||
|
||||
It is possible to package a dump of the server records that will be loaded into the local database when no synchronization has happened yet. It will thus serve as the default dataset and also reduce the amount of data to be downloaded on the first synchronization.
|
||||
The JSON dump will serve as the default dataset for ``.get()``, instead of doing a round-trip to pull the latest data. It will also reduce the amount of data to be downloaded on the first synchronization.
|
||||
|
||||
#. Place the JSON dump of the server records in the ``services/settings/dumps/main/`` folder
|
||||
#. Add the filename to the ``FINAL_TARGET_FILES`` list in ``services/settings/dumps/main/moz.build``
|
||||
|
||||
Now, when ``RemoteSettings("some-key").get()`` is called from an empty profile, the ``some-key.json`` file is going to be loaded before the results are returned.
|
||||
|
||||
.. note::
|
||||
|
||||
JSON dumps are not shipped on Android to minimize the installer size.
|
||||
|
||||
Targets and A/B testing
|
||||
=======================
|
||||
|
|
@ -170,14 +180,15 @@ The synchronization of every known remote settings clients can be triggered manu
|
|||
|
||||
await RemoteSettings.pollChanges()
|
||||
|
||||
The synchronization of a single client can be forced with ``maybeSync()``:
|
||||
The synchronization of a single client can be forced with the ``.sync()`` method:
|
||||
|
||||
.. code-block:: js
|
||||
|
||||
const fakeTimestamp = Infinity;
|
||||
const fakeServerTime = Date.now();
|
||||
await RemoteSettings("a-key").sync();
|
||||
|
||||
await RemoteSettings("a-key").maybeSync(fakeTimestamp, fakeServerTime)
|
||||
.. important::
|
||||
|
||||
The above methods are only relevant during development or debugging and should never be called in production code.
|
||||
|
||||
|
||||
Manipulate local data
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ add_task(async function test_something() {
|
|||
server.registerPathHandler(recordsPath, handleResponse);
|
||||
|
||||
// Test an empty db populates
|
||||
await OneCRLBlocklistClient.maybeSync(2000, Date.now());
|
||||
await OneCRLBlocklistClient.maybeSync(2000);
|
||||
|
||||
// Open the collection, verify it's been populated:
|
||||
const list = await OneCRLBlocklistClient.get();
|
||||
|
|
@ -63,9 +63,7 @@ add_task(async function test_something() {
|
|||
Services.prefs.clearUserPref("services.settings.server");
|
||||
Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0);
|
||||
// Use any last_modified older than highest shipped in JSON dump.
|
||||
await OneCRLBlocklistClient.maybeSync(123456, Date.now());
|
||||
// Last check value was updated.
|
||||
Assert.notEqual(0, Services.prefs.getIntPref("services.blocklist.onecrl.checked"));
|
||||
await OneCRLBlocklistClient.maybeSync(123456);
|
||||
|
||||
// Restore server pref.
|
||||
Services.prefs.setCharPref("services.settings.server", dummyServerURL);
|
||||
|
|
@ -78,7 +76,7 @@ add_task(async function test_something() {
|
|||
// single record
|
||||
await collection.db.saveLastModified(1000);
|
||||
|
||||
await OneCRLBlocklistClient.maybeSync(2000, Date.now());
|
||||
await OneCRLBlocklistClient.maybeSync(2000);
|
||||
|
||||
// Open the collection, verify it's been updated:
|
||||
// Our test data now has two records; both should be in the local collection
|
||||
|
|
@ -86,7 +84,7 @@ add_task(async function test_something() {
|
|||
Assert.equal(before.length, 1);
|
||||
|
||||
// Test the db is updated when we call again with a later lastModified value
|
||||
await OneCRLBlocklistClient.maybeSync(4000, Date.now());
|
||||
await OneCRLBlocklistClient.maybeSync(4000);
|
||||
|
||||
// Open the collection, verify it's been updated:
|
||||
// Our test data now has two records; both should be in the local collection
|
||||
|
|
@ -97,23 +95,16 @@ add_task(async function test_something() {
|
|||
// should be attempted.
|
||||
// Clear the kinto base pref so any connections will cause a test failure
|
||||
Services.prefs.clearUserPref("services.settings.server");
|
||||
await OneCRLBlocklistClient.maybeSync(4000, Date.now());
|
||||
await OneCRLBlocklistClient.maybeSync(4000);
|
||||
|
||||
// Try again with a lastModified value at some point in the past
|
||||
await OneCRLBlocklistClient.maybeSync(3000, Date.now());
|
||||
|
||||
// Check the OneCRL check time pref is modified, even if the collection
|
||||
// hasn't changed
|
||||
Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0);
|
||||
await OneCRLBlocklistClient.maybeSync(3000, Date.now());
|
||||
let newValue = Services.prefs.getIntPref("services.blocklist.onecrl.checked");
|
||||
Assert.notEqual(newValue, 0);
|
||||
await OneCRLBlocklistClient.maybeSync(3000);
|
||||
|
||||
// Check that a sync completes even when there's bad data in the
|
||||
// collection. This will throw on fail, so just calling maybeSync is an
|
||||
// acceptible test.
|
||||
Services.prefs.setCharPref("services.settings.server", dummyServerURL);
|
||||
await OneCRLBlocklistClient.maybeSync(5000, Date.now());
|
||||
await OneCRLBlocklistClient.maybeSync(5000);
|
||||
});
|
||||
|
||||
function run_test() {
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ add_task(async function test_initial_dump_is_loaded_as_synced_when_collection_is
|
|||
}
|
||||
|
||||
// Test an empty db populates, but don't reach server (specified timestamp <= dump).
|
||||
await client.maybeSync(1, Date.now());
|
||||
await client.maybeSync(1);
|
||||
|
||||
// Verify the loaded data has status to synced:
|
||||
const collection = await client.openCollection();
|
||||
|
|
@ -134,19 +134,6 @@ add_task(async function test_initial_dump_is_loaded_when_using_get_on_empty_coll
|
|||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_current_server_time_is_saved_in_pref() {
|
||||
for (let {client} of gBlocklistClients) {
|
||||
// The lastCheckTimePref was customized:
|
||||
ok(/services\.blocklist\.(\w+)\.checked/.test(client.lastCheckTimePref), client.lastCheckTimePref);
|
||||
|
||||
const serverTime = Date.now();
|
||||
await client.maybeSync(3000, serverTime);
|
||||
const after = Services.prefs.getIntPref(client.lastCheckTimePref);
|
||||
equal(after, Math.round(serverTime / 1000));
|
||||
}
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_sync_event_data_is_filtered_for_target() {
|
||||
// Here we will synchronize 4 times, the first two to initialize the local DB and
|
||||
// the last two about event filtered data.
|
||||
|
|
@ -154,25 +141,23 @@ add_task(async function test_sync_event_data_is_filtered_for_target() {
|
|||
const timestamp2 = 3001;
|
||||
const timestamp3 = 4001;
|
||||
const timestamp4 = 5001;
|
||||
// Fake a date value obtained from server (used to store a pref, useless here).
|
||||
const fakeServerTime = Date.now();
|
||||
|
||||
for (let {client} of gBlocklistClients) {
|
||||
// Initialize the collection with some data (local is empty, thus no ?_since)
|
||||
await client.maybeSync(timestamp1, fakeServerTime - 30, {loadDump: false});
|
||||
await client.maybeSync(timestamp1, {loadDump: false});
|
||||
// This will pick the data with ?_since=3000.
|
||||
await client.maybeSync(timestamp2, fakeServerTime - 20);
|
||||
await client.maybeSync(timestamp2);
|
||||
|
||||
// In ?_since=4000 entries, no target matches. The sync event is not called.
|
||||
let called = false;
|
||||
client.on("sync", e => called = true);
|
||||
await client.maybeSync(timestamp3, fakeServerTime - 10);
|
||||
await client.maybeSync(timestamp3);
|
||||
equal(called, false, `shouldn't have sync event for ${client.collectionName}`);
|
||||
|
||||
// In ?_since=5000 entries, only one entry matches.
|
||||
let syncEventData;
|
||||
client.on("sync", e => syncEventData = e.data);
|
||||
await client.maybeSync(timestamp4, fakeServerTime);
|
||||
await client.maybeSync(timestamp4);
|
||||
const { current, created, updated, deleted } = syncEventData;
|
||||
equal(created.length + updated.length + deleted.length, 1, `event filtered data for ${client.collectionName}`);
|
||||
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ add_task(async function test_something() {
|
|||
Services.io.newURI("https://five.example.com"), 0));
|
||||
|
||||
// Test an empty db populates
|
||||
await PinningPreloadClient.maybeSync(2000, Date.now());
|
||||
await PinningPreloadClient.maybeSync(2000);
|
||||
|
||||
// Open the collection, verify it's been populated:
|
||||
// Our test data has a single record; it should be in the local collection
|
||||
|
|
@ -92,7 +92,7 @@ add_task(async function test_something() {
|
|||
Services.io.newURI("https://one.example.com"), 0));
|
||||
|
||||
// Test the db is updated when we call again with a later lastModified value
|
||||
await PinningPreloadClient.maybeSync(4000, Date.now());
|
||||
await PinningPreloadClient.maybeSync(4000);
|
||||
|
||||
// Open the collection, verify it's been updated:
|
||||
// Our data now has four new records; all should be in the local collection
|
||||
|
|
@ -114,17 +114,10 @@ add_task(async function test_something() {
|
|||
// should be attempted.
|
||||
// Clear the kinto base pref so any connections will cause a test failure
|
||||
Services.prefs.clearUserPref("services.settings.server");
|
||||
await PinningPreloadClient.maybeSync(4000, Date.now());
|
||||
await PinningPreloadClient.maybeSync(4000);
|
||||
|
||||
// Try again with a lastModified value at some point in the past
|
||||
await PinningPreloadClient.maybeSync(3000, Date.now());
|
||||
|
||||
// Check the pinning check time pref is modified, even if the collection
|
||||
// hasn't changed
|
||||
Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0);
|
||||
await PinningPreloadClient.maybeSync(3000, Date.now());
|
||||
let newValue = Services.prefs.getIntPref("services.blocklist.pinning.checked");
|
||||
Assert.notEqual(newValue, 0);
|
||||
await PinningPreloadClient.maybeSync(3000);
|
||||
|
||||
// Check that the HSTS preload added to the collection works...
|
||||
ok(sss.isSecureURI(sss.HEADER_HSTS,
|
||||
|
|
@ -139,7 +132,7 @@ add_task(async function test_something() {
|
|||
// acceptible test (the data below with last_modified of 300 is nonsense).
|
||||
Services.prefs.setCharPref("services.settings.server",
|
||||
`http://localhost:${server.identity.primaryPort}/v1`);
|
||||
await PinningPreloadClient.maybeSync(5000, Date.now());
|
||||
await PinningPreloadClient.maybeSync(5000);
|
||||
|
||||
// The STS entry for five.example.com now has includeSubdomains set;
|
||||
// ensure that the new includeSubdomains value is honored.
|
||||
|
|
|
|||
|
|
@ -152,9 +152,6 @@ add_task(async function test_check_signatures() {
|
|||
Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
|
||||
`http://localhost:${server.identity.primaryPort}/v1`);
|
||||
|
||||
// Set up some data we need for our test
|
||||
let startTime = Date.now();
|
||||
|
||||
// These are records we'll use in the test collections
|
||||
const RECORD1 = {
|
||||
details: {
|
||||
|
|
@ -298,7 +295,7 @@ add_task(async function test_check_signatures() {
|
|||
// With all of this set up, we attempt a sync. This will resolve if all is
|
||||
// well and throw if something goes wrong.
|
||||
// We don't want to load initial json dumps in this test suite.
|
||||
await OneCRLBlocklistClient.maybeSync(1000, startTime, {loadDump: false});
|
||||
await OneCRLBlocklistClient.maybeSync(1000, { loadDump: false });
|
||||
|
||||
let endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
|
||||
|
||||
|
|
@ -336,7 +333,7 @@ add_task(async function test_check_signatures() {
|
|||
[RESPONSE_META_TWO_ITEMS_SIG],
|
||||
};
|
||||
registerHandlers(twoItemsResponses);
|
||||
await OneCRLBlocklistClient.maybeSync(3000, startTime);
|
||||
await OneCRLBlocklistClient.maybeSync(3000);
|
||||
|
||||
|
||||
// Check the collection with one addition and one removal has a valid
|
||||
|
|
@ -368,7 +365,7 @@ add_task(async function test_check_signatures() {
|
|||
[RESPONSE_META_THREE_ITEMS_SIG],
|
||||
};
|
||||
registerHandlers(oneAddedOneRemovedResponses);
|
||||
await OneCRLBlocklistClient.maybeSync(4000, startTime);
|
||||
await OneCRLBlocklistClient.maybeSync(4000);
|
||||
|
||||
// Check the signature is still valid with no operation (no changes)
|
||||
|
||||
|
|
@ -390,7 +387,7 @@ add_task(async function test_check_signatures() {
|
|||
[RESPONSE_META_THREE_ITEMS_SIG],
|
||||
};
|
||||
registerHandlers(noOpResponses);
|
||||
await OneCRLBlocklistClient.maybeSync(4100, startTime);
|
||||
await OneCRLBlocklistClient.maybeSync(4100);
|
||||
|
||||
|
||||
// Check the collection is reset when the signature is invalid
|
||||
|
|
@ -451,7 +448,7 @@ add_task(async function test_check_signatures() {
|
|||
let syncEventSent = false;
|
||||
OneCRLBlocklistClient.on("sync", ({ data }) => { syncEventSent = true; });
|
||||
|
||||
await OneCRLBlocklistClient.maybeSync(5000, startTime);
|
||||
await OneCRLBlocklistClient.maybeSync(5000);
|
||||
|
||||
endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
|
||||
|
||||
|
|
@ -492,7 +489,7 @@ add_task(async function test_check_signatures() {
|
|||
syncEventSent = false;
|
||||
OneCRLBlocklistClient.on("sync", ({ data }) => { syncEventSent = true; });
|
||||
|
||||
await OneCRLBlocklistClient.maybeSync(5000, startTime);
|
||||
await OneCRLBlocklistClient.maybeSync(5000);
|
||||
|
||||
// Local data was unchanged, since it was never than the one returned by the server,
|
||||
// thus the sync event is not sent.
|
||||
|
|
@ -528,7 +525,7 @@ add_task(async function test_check_signatures() {
|
|||
let syncData;
|
||||
OneCRLBlocklistClient.on("sync", ({ data }) => { syncData = data; });
|
||||
|
||||
await OneCRLBlocklistClient.maybeSync(5000, startTime, { loadDump: false });
|
||||
await OneCRLBlocklistClient.maybeSync(5000, { loadDump: false });
|
||||
|
||||
// Local data was unchanged, since it was never than the one returned by the server.
|
||||
equal(syncData.current.length, 2);
|
||||
|
|
@ -559,7 +556,7 @@ add_task(async function test_check_signatures() {
|
|||
startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
|
||||
registerHandlers(allBadSigResponses);
|
||||
try {
|
||||
await OneCRLBlocklistClient.maybeSync(6000, startTime);
|
||||
await OneCRLBlocklistClient.maybeSync(6000);
|
||||
do_throw("Sync should fail (the signature is intentionally bad)");
|
||||
} catch (e) {
|
||||
await checkRecordCount(OneCRLBlocklistClient, 2);
|
||||
|
|
@ -581,7 +578,7 @@ add_task(async function test_check_signatures() {
|
|||
startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
|
||||
registerHandlers(missingSigResponses);
|
||||
try {
|
||||
await OneCRLBlocklistClient.maybeSync(6000, startTime);
|
||||
await OneCRLBlocklistClient.maybeSync(6000);
|
||||
do_throw("Sync should fail (the signature is missing)");
|
||||
} catch (e) {
|
||||
await checkRecordCount(OneCRLBlocklistClient, 2);
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ ChromeUtils.defineModuleGetter(this, "ClientEnvironmentBase",
|
|||
"resource://gre/modules/components-utils/ClientEnvironment.jsm");
|
||||
ChromeUtils.defineModuleGetter(this, "RemoteSettingsWorker",
|
||||
"resource://services-settings/RemoteSettingsWorker.jsm");
|
||||
ChromeUtils.defineModuleGetter(this, "Utils",
|
||||
"resource://services-settings/Utils.jsm");
|
||||
|
||||
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
|
||||
|
||||
|
|
@ -32,6 +34,8 @@ const MISSING_SIGNATURE = "Missing signature";
|
|||
|
||||
XPCOMUtils.defineLazyPreferenceGetter(this, "gServerURL",
|
||||
"services.settings.server");
|
||||
XPCOMUtils.defineLazyPreferenceGetter(this, "gChangesPath",
|
||||
"services.settings.changes.path");
|
||||
XPCOMUtils.defineLazyPreferenceGetter(this, "gVerifySignature",
|
||||
"services.settings.verify_signature", true);
|
||||
|
||||
|
|
@ -125,7 +129,7 @@ class EventEmitter {
|
|||
* @param {Object} payload the event payload to call the listeners with
|
||||
*/
|
||||
async emit(event, payload) {
|
||||
const callbacks = this._listeners.get("sync");
|
||||
const callbacks = this._listeners.get(event);
|
||||
let lastError;
|
||||
for (const cb of callbacks) {
|
||||
try {
|
||||
|
|
@ -164,7 +168,7 @@ class EventEmitter {
|
|||
class RemoteSettingsClient extends EventEmitter {
|
||||
|
||||
constructor(collectionName, { bucketNamePref, signerName, filterFunc, localFields = [], lastCheckTimePref }) {
|
||||
super(["sync"]);
|
||||
super(["sync"]); // emitted events
|
||||
|
||||
this.collectionName = collectionName;
|
||||
this.signerName = signerName;
|
||||
|
|
@ -212,20 +216,19 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
* @return {Promise}
|
||||
*/
|
||||
async get(options = {}) {
|
||||
const {
|
||||
filters = {},
|
||||
order = "", // not sorted by default.
|
||||
} = options;
|
||||
const { filters = {}, order = "" } = options; // not sorted by default.
|
||||
|
||||
const c = await this.openCollection();
|
||||
|
||||
const timestamp = await c.db.getLastModified();
|
||||
if (timestamp == null) {
|
||||
// The local database for this collection was never synchronized.
|
||||
// Before returning an empty list, we attempt to load a packaged JSON dump.
|
||||
if (!(await Utils.hasLocalData(this))) {
|
||||
try {
|
||||
// Load JSON dump if there is one.
|
||||
await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
|
||||
// .get() was called before we had the chance to synchronize the local database.
|
||||
// We'll try to avoid returning an empty list.
|
||||
if (await Utils.hasLocalDump(this.bucketName, this.collectionName)) {
|
||||
// Since there is a JSON dump, load it as default data.
|
||||
await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
|
||||
} else {
|
||||
// There is no JSON dump, force a synchronization from the server.
|
||||
await this.sync({ loadDump: false });
|
||||
}
|
||||
} catch (e) {
|
||||
// Report but return an empty list since there will be no data anyway.
|
||||
Cu.reportError(e);
|
||||
|
|
@ -234,24 +237,46 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
}
|
||||
|
||||
// Read from the local DB.
|
||||
const { data } = await c.list({ filters, order });
|
||||
const kintoCol = await this.openCollection();
|
||||
const { data } = await kintoCol.list({ filters, order });
|
||||
// Filter the records based on `this.filterFunc` results.
|
||||
return this._filterEntries(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronize from Kinto server, if necessary.
|
||||
* Synchronize the local database with the remote server.
|
||||
*
|
||||
* @param {Object} options See #maybeSync() options.
|
||||
*/
|
||||
async sync(options) {
|
||||
// We want to know which timestamp we are expected to obtain in order to leverage
|
||||
// cache busting. We don't provide ETag because we don't want a 304.
|
||||
const { changes } = await Utils.fetchLatestChanges(gServerURL + gChangesPath, {
|
||||
filters: {
|
||||
collection: this.collectionName,
|
||||
bucket: this.bucketName,
|
||||
},
|
||||
});
|
||||
if (changes.length === 0) {
|
||||
throw new Error(`Unknown collection "${this.identifier}"`);
|
||||
}
|
||||
// According to API, there will be one only (fail if not).
|
||||
const [{ last_modified: expectedTimestamp }] = changes;
|
||||
|
||||
return this.maybeSync(expectedTimestamp, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronize the local database with the remote server, **only if necessary**.
|
||||
*
|
||||
* @param {int} expectedTimestamp the lastModified date (on the server) for the remote collection.
|
||||
* This will be compared to the local timestamp, and will be used for
|
||||
* cache busting if local data is out of date.
|
||||
* @param {int} serverTimeMillis the current date return by the server.
|
||||
* This is only used to track the last check or synchronization.
|
||||
* @param {Object} options additional advanced options.
|
||||
* @param {bool} options.loadDump load initial dump from disk on first sync (default: true)
|
||||
* @return {Promise} which rejects on sync or process failure.
|
||||
*/
|
||||
async maybeSync(expectedTimestamp, serverTimeMillis, options = { loadDump: true }) {
|
||||
async maybeSync(expectedTimestamp, options = { loadDump: true }) {
|
||||
const { loadDump } = options;
|
||||
|
||||
let reportStatus = null;
|
||||
|
|
@ -277,7 +302,6 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
// If the data is up to date, there's no need to sync. We still need
|
||||
// to record the fact that a check happened.
|
||||
if (expectedTimestamp <= collectionLastModified) {
|
||||
this._updateLastCheck(serverTimeMillis);
|
||||
reportStatus = UptakeTelemetry.STATUS.UP_TO_DATE;
|
||||
return;
|
||||
}
|
||||
|
|
@ -398,9 +422,6 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
}
|
||||
}
|
||||
|
||||
// Track last update.
|
||||
this._updateLastCheck(serverTimeMillis);
|
||||
|
||||
} catch (e) {
|
||||
// No specific error was tracked, mark it as unknown.
|
||||
if (reportStatus === null) {
|
||||
|
|
@ -418,11 +439,17 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
}
|
||||
|
||||
/**
|
||||
* Fetch the signature info from the collection metadata and verifies that the
|
||||
* local set of records has the same.
|
||||
*
|
||||
* @param {Array<Object>} remoteRecords
|
||||
* @param {int} timestamp
|
||||
* @param {Collection} collection
|
||||
* @param {Array<Object>} remoteRecords The list of changes to apply to the local database.
|
||||
* @param {int} timestamp The timestamp associated with the list of remote records.
|
||||
* @param {Collection} collection Kinto.js Collection instance.
|
||||
* @param {Object} options
|
||||
* @param {int} options.expectedTimestamp Cache busting of collection metadata
|
||||
* @param {Boolean} options.ignoreLocal When the signature verification is retried, since we refetch
|
||||
* the whole collection, we don't take into account the local
|
||||
* data (default: `false`)
|
||||
* @returns {Promise}
|
||||
*/
|
||||
async _validateCollectionSignature(remoteRecords, timestamp, kintoCollection, options = {}) {
|
||||
|
|
@ -452,21 +479,12 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
}
|
||||
|
||||
/**
|
||||
* Save last time server was checked in users prefs.
|
||||
*
|
||||
* @param {int} serverTimeMillis the current date return by server.
|
||||
*/
|
||||
_updateLastCheck(serverTimeMillis) {
|
||||
const checkedServerTimeInSeconds = Math.round(serverTimeMillis / 1000);
|
||||
Services.prefs.setIntPref(this.lastCheckTimePref, checkedServerTimeInSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter entries for which calls to `this.filterFunc` returns null.
|
||||
*
|
||||
* @param {Array<Objet>} data
|
||||
* @returns {Array<Object>}
|
||||
*/
|
||||
async _filterEntries(data) {
|
||||
// Filter entries for which calls to `this.filterFunc` returns null.
|
||||
if (!this.filterFunc) {
|
||||
return data;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
var EXPORTED_SYMBOLS = [
|
||||
"Utils",
|
||||
];
|
||||
|
||||
ChromeUtils.import("resource://gre/modules/Services.jsm");
|
||||
ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
|
||||
|
||||
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
|
||||
|
||||
var Utils = {
|
||||
|
||||
/**
|
||||
* Check if local data exist for the specified client.
|
||||
*
|
||||
* @param {RemoteSettingsClient} client
|
||||
* @return {bool} Whether it exists or not.
|
||||
*/
|
||||
async hasLocalData(client) {
|
||||
const kintoCol = await client.openCollection();
|
||||
const timestamp = await kintoCol.db.getLastModified();
|
||||
return timestamp !== null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Check if we ship a JSON dump for the specified bucket and collection.
|
||||
*
|
||||
* @param {String} bucket
|
||||
* @param {String} collection
|
||||
* @return {bool} Whether it is present or not.
|
||||
*/
|
||||
async hasLocalDump(bucket, collection) {
|
||||
try {
|
||||
await fetch(`resource://app/defaults/settings/${bucket}/${collection}.json`);
|
||||
return true;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Fetch the list of remote collections and their timestamp.
|
||||
* @param {String} url The poll URL (eg. `http://${server}{pollingEndpoint}`)
|
||||
* @param {int} expectedTimestamp The timestamp that the server is supposed to return.
|
||||
* We obtained it from the Megaphone notification payload,
|
||||
* and we use it only for cache busting (Bug 1497159).
|
||||
* @param {String} lastEtag (optional) The Etag of the latest poll to be matched
|
||||
* by the server (eg. `"123456789"`).
|
||||
* @param {Object} filters
|
||||
*/
|
||||
async fetchLatestChanges(url, options = {}) {
|
||||
const { expectedTimestamp, lastEtag = "", filters = {} } = options;
|
||||
//
|
||||
// Fetch the list of changes objects from the server that looks like:
|
||||
// {"data":[
|
||||
// {
|
||||
// "host":"kinto-ota.dev.mozaws.net",
|
||||
// "last_modified":1450717104423,
|
||||
// "bucket":"blocklists",
|
||||
// "collection":"certificates"
|
||||
// }]}
|
||||
|
||||
// Use ETag to obtain a `304 Not modified` when no change occurred,
|
||||
// and `?_since` parameter to only keep entries that weren't processed yet.
|
||||
const headers = {};
|
||||
const params = { ...filters };
|
||||
if (lastEtag != "") {
|
||||
headers["If-None-Match"] = lastEtag;
|
||||
params._since = lastEtag;
|
||||
}
|
||||
if (expectedTimestamp) {
|
||||
params._expected = expectedTimestamp;
|
||||
}
|
||||
if (params) {
|
||||
url += "?" + Object.entries(params).map(([k, v]) => `${k}=${encodeURIComponent(v)}`).join("&");
|
||||
}
|
||||
const response = await fetch(url, { headers });
|
||||
|
||||
let changes = [];
|
||||
// If no changes since last time, go on with empty list of changes.
|
||||
if (response.status != 304) {
|
||||
let payload;
|
||||
try {
|
||||
payload = await response.json();
|
||||
} catch (e) {
|
||||
payload = e.message;
|
||||
}
|
||||
|
||||
if (!payload.hasOwnProperty("data")) {
|
||||
// If the server is failing, the JSON response might not contain the
|
||||
// expected data. For example, real server errors (Bug 1259145)
|
||||
// or dummy local server for tests (Bug 1481348)
|
||||
const is404FromCustomServer = response.status == 404 && Services.prefs.prefHasUserValue("services.settings.server");
|
||||
if (!is404FromCustomServer) {
|
||||
throw new Error(`Server error ${response.status} ${response.statusText}: ${JSON.stringify(payload)}`);
|
||||
}
|
||||
} else {
|
||||
changes = payload.data;
|
||||
}
|
||||
}
|
||||
// The server should always return ETag. But we've had situations where the CDN
|
||||
// was interfering.
|
||||
const currentEtag = response.headers.has("ETag") ? response.headers.get("ETag") : undefined;
|
||||
let serverTimeMillis = Date.parse(response.headers.get("Date"));
|
||||
// Since the response is served via a CDN, the Date header value could have been cached.
|
||||
const ageSeconds = response.headers.has("Age") ? parseInt(response.headers.get("Age"), 10) : 0;
|
||||
serverTimeMillis += ageSeconds * 1000;
|
||||
|
||||
// Check if the server asked the clients to back off.
|
||||
let backoffSeconds;
|
||||
if (response.headers.has("Backoff")) {
|
||||
const value = parseInt(response.headers.get("Backoff"), 10);
|
||||
if (!isNaN(value)) {
|
||||
backoffSeconds = value;
|
||||
}
|
||||
}
|
||||
|
||||
return { changes, currentEtag, serverTimeMillis, backoffSeconds };
|
||||
},
|
||||
};
|
||||
|
|
@ -19,6 +19,7 @@ EXTRA_JS_MODULES['services-settings'] += [
|
|||
'RemoteSettingsClient.jsm',
|
||||
'RemoteSettingsWorker.js',
|
||||
'RemoteSettingsWorker.jsm',
|
||||
'Utils.jsm',
|
||||
]
|
||||
|
||||
XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ ChromeUtils.defineModuleGetter(this, "pushBroadcastService",
|
|||
"resource://gre/modules/PushBroadcastService.jsm");
|
||||
ChromeUtils.defineModuleGetter(this, "RemoteSettingsClient",
|
||||
"resource://services-settings/RemoteSettingsClient.jsm");
|
||||
ChromeUtils.defineModuleGetter(this, "Utils",
|
||||
"resource://services-settings/Utils.jsm");
|
||||
ChromeUtils.defineModuleGetter(this, "FilterExpressions",
|
||||
"resource://gre/modules/components-utils/FilterExpressions.jsm");
|
||||
|
||||
|
|
@ -73,112 +75,6 @@ async function jexlFilterFunc(entry, environment) {
|
|||
return result ? entry : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the list of remote collections and their timestamp.
|
||||
* @param {String} url The poll URL (eg. `http://${server}{pollingEndpoint}`)
|
||||
* @param {String} lastEtag (optional) The Etag of the latest poll to be matched
|
||||
* by the server (eg. `"123456789"`).
|
||||
* @param {int} expectedTimestamp The timestamp that the server is supposed to return.
|
||||
* We obtained it from the Megaphone notification payload,
|
||||
* and we use it only for cache busting (Bug 1497159).
|
||||
*/
|
||||
async function fetchLatestChanges(url, lastEtag, expectedTimestamp) {
|
||||
//
|
||||
// Fetch the list of changes objects from the server that looks like:
|
||||
// {"data":[
|
||||
// {
|
||||
// "host":"kinto-ota.dev.mozaws.net",
|
||||
// "last_modified":1450717104423,
|
||||
// "bucket":"blocklists",
|
||||
// "collection":"certificates"
|
||||
// }]}
|
||||
|
||||
// Use ETag to obtain a `304 Not modified` when no change occurred,
|
||||
// and `?_since` parameter to only keep entries that weren't processed yet.
|
||||
const headers = {};
|
||||
const params = {};
|
||||
if (lastEtag) {
|
||||
headers["If-None-Match"] = lastEtag;
|
||||
params._since = lastEtag;
|
||||
}
|
||||
if (expectedTimestamp) {
|
||||
params._expected = expectedTimestamp;
|
||||
}
|
||||
if (params) {
|
||||
url += "?" + Object.entries(params).map(([k, v]) => `${k}=${encodeURIComponent(v)}`).join("&");
|
||||
}
|
||||
const response = await fetch(url, {headers});
|
||||
|
||||
let changes = [];
|
||||
// If no changes since last time, go on with empty list of changes.
|
||||
if (response.status != 304) {
|
||||
let payload;
|
||||
try {
|
||||
payload = await response.json();
|
||||
} catch (e) {
|
||||
payload = e.message;
|
||||
}
|
||||
|
||||
if (!payload.hasOwnProperty("data")) {
|
||||
// If the server is failing, the JSON response might not contain the
|
||||
// expected data. For example, real server errors (Bug 1259145)
|
||||
// or dummy local server for tests (Bug 1481348)
|
||||
const is404FromCustomServer = response.status == 404 && gPrefs.prefHasUserValue(PREF_SETTINGS_SERVER);
|
||||
if (!is404FromCustomServer) {
|
||||
throw new Error(`Server error ${response.status} ${response.statusText}: ${JSON.stringify(payload)}`);
|
||||
}
|
||||
} else {
|
||||
changes = payload.data;
|
||||
}
|
||||
}
|
||||
// The server should always return ETag. But we've had situations where the CDN
|
||||
// was interfering.
|
||||
const currentEtag = response.headers.has("ETag") ? response.headers.get("ETag") : undefined;
|
||||
let serverTimeMillis = Date.parse(response.headers.get("Date"));
|
||||
// Since the response is served via a CDN, the Date header value could have been cached.
|
||||
const ageSeconds = response.headers.has("Age") ? parseInt(response.headers.get("Age"), 10) : 0;
|
||||
serverTimeMillis += ageSeconds * 1000;
|
||||
|
||||
// Check if the server asked the clients to back off.
|
||||
let backoffSeconds;
|
||||
if (response.headers.has("Backoff")) {
|
||||
const value = parseInt(response.headers.get("Backoff"), 10);
|
||||
if (!isNaN(value)) {
|
||||
backoffSeconds = value;
|
||||
}
|
||||
}
|
||||
|
||||
return { changes, currentEtag, serverTimeMillis, backoffSeconds };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if local data exist for the specified client.
|
||||
*
|
||||
* @param {RemoteSettingsClient} client
|
||||
* @return {bool} Whether it exists or not.
|
||||
*/
|
||||
async function hasLocalData(client) {
|
||||
const kintoCol = await client.openCollection();
|
||||
const timestamp = await kintoCol.db.getLastModified();
|
||||
return timestamp !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we ship a JSON dump for the specified bucket and collection.
|
||||
*
|
||||
* @param {String} bucket
|
||||
* @param {String} collection
|
||||
* @return {bool} Whether it is present or not.
|
||||
*/
|
||||
async function hasLocalDump(bucket, collection) {
|
||||
try {
|
||||
await fetch(`resource://app/defaults/settings/${bucket}/${collection}.json`);
|
||||
return true;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function remoteSettingsFunction() {
|
||||
const _clients = new Map();
|
||||
|
|
@ -236,8 +132,8 @@ function remoteSettingsFunction() {
|
|||
if (bucketName == Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_BUCKET)) {
|
||||
const c = new RemoteSettingsClient(collectionName, defaultOptions);
|
||||
const [dbExists, localDump] = await Promise.all([
|
||||
hasLocalData(c),
|
||||
hasLocalDump(bucketName, collectionName),
|
||||
Utils.hasLocalData(c),
|
||||
Utils.hasLocalDump(bucketName, collectionName),
|
||||
]);
|
||||
if (dbExists || localDump) {
|
||||
return c;
|
||||
|
|
@ -272,14 +168,11 @@ function remoteSettingsFunction() {
|
|||
}
|
||||
}
|
||||
|
||||
let lastEtag;
|
||||
if (gPrefs.prefHasUserValue(PREF_SETTINGS_LAST_ETAG)) {
|
||||
lastEtag = gPrefs.getCharPref(PREF_SETTINGS_LAST_ETAG);
|
||||
}
|
||||
const lastEtag = gPrefs.getCharPref(PREF_SETTINGS_LAST_ETAG, "");
|
||||
|
||||
let pollResult;
|
||||
try {
|
||||
pollResult = await fetchLatestChanges(remoteSettings.pollingEndpoint, lastEtag, expectedTimestamp);
|
||||
pollResult = await Utils.fetchLatestChanges(remoteSettings.pollingEndpoint, { expectedTimestamp, lastEtag });
|
||||
} catch (e) {
|
||||
// Report polling error to Uptake Telemetry.
|
||||
let report;
|
||||
|
|
@ -313,7 +206,9 @@ function remoteSettingsFunction() {
|
|||
// by the absolute of that value in seconds (positive means it's ahead)
|
||||
const clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
|
||||
gPrefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
|
||||
gPrefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, serverTimeMillis / 1000);
|
||||
const checkedServerTimeInSeconds = Math.round(serverTimeMillis / 1000);
|
||||
gPrefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, checkedServerTimeInSeconds);
|
||||
|
||||
|
||||
const loadDump = gPrefs.getBoolPref(PREF_SETTINGS_LOAD_DUMP, true);
|
||||
|
||||
|
|
@ -330,7 +225,9 @@ function remoteSettingsFunction() {
|
|||
// Start synchronization! It will be a no-op if the specified `lastModified` equals
|
||||
// the one in the local database.
|
||||
try {
|
||||
await client.maybeSync(last_modified, serverTimeMillis, {loadDump});
|
||||
await client.maybeSync(last_modified, { loadDump });
|
||||
// Save last time this client was successfully synced.
|
||||
Services.prefs.setIntPref(client.lastCheckTimePref, checkedServerTimeInSeconds);
|
||||
} catch (e) {
|
||||
if (!firstError) {
|
||||
firstError = e;
|
||||
|
|
@ -356,7 +253,7 @@ function remoteSettingsFunction() {
|
|||
* known remote settings collections.
|
||||
*/
|
||||
remoteSettings.inspect = async () => {
|
||||
const { changes, currentEtag: serverTimestamp } = await fetchLatestChanges(remoteSettings.pollingEndpoint);
|
||||
const { changes, currentEtag: serverTimestamp } = await Utils.fetchLatestChanges(remoteSettings.pollingEndpoint);
|
||||
|
||||
const collections = await Promise.all(changes.map(async (change) => {
|
||||
const { bucket, collection, last_modified: serverTimestamp } = change;
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ const { Constructor: CC } = Components;
|
|||
|
||||
ChromeUtils.import("resource://gre/modules/Services.jsm");
|
||||
ChromeUtils.import("resource://testing-common/httpd.js");
|
||||
ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
|
||||
|
||||
const IS_ANDROID = AppConstants.platform == "android";
|
||||
|
||||
const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js", {});
|
||||
const { UptakeTelemetry } = ChromeUtils.import("resource://services-common/uptake-telemetry.js", {});
|
||||
|
|
@ -13,6 +16,7 @@ const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
|
|||
|
||||
let server;
|
||||
let client;
|
||||
let clientWithDump;
|
||||
|
||||
async function clear_state() {
|
||||
// Clear local DB.
|
||||
|
|
@ -20,6 +24,10 @@ async function clear_state() {
|
|||
await collection.clear();
|
||||
// Reset event listeners.
|
||||
client._listeners.set("sync", []);
|
||||
|
||||
const collectionWithDump = await clientWithDump.openCollection();
|
||||
await collectionWithDump.clear();
|
||||
|
||||
Services.prefs.clearUserPref("services.settings.default_bucket");
|
||||
}
|
||||
|
||||
|
|
@ -37,6 +45,7 @@ function run_test() {
|
|||
Services.prefs.setBoolPref("services.settings.verify_signature", false);
|
||||
|
||||
client = RemoteSettings("password-fields");
|
||||
clientWithDump = RemoteSettings("language-dictionaries");
|
||||
|
||||
// Setup server fake responses.
|
||||
function handleResponse(request, response) {
|
||||
|
|
@ -77,7 +86,7 @@ function run_test() {
|
|||
|
||||
add_task(async function test_records_obtained_from_server_are_stored_in_db() {
|
||||
// Test an empty db populates
|
||||
await client.maybeSync(2000, Date.now());
|
||||
await client.maybeSync(2000);
|
||||
|
||||
// Open the collection, verify it's been populated:
|
||||
// Our test data has a single record; it should be in the local collection
|
||||
|
|
@ -88,21 +97,12 @@ add_task(clear_state);
|
|||
|
||||
add_task(async function test_records_can_have_local_fields() {
|
||||
const c = RemoteSettings("password-fields", { localFields: ["accepted"] });
|
||||
await c.maybeSync(2000, Date.now());
|
||||
await c.maybeSync(2000);
|
||||
|
||||
const col = await c.openCollection();
|
||||
await col.update({ id: "9d500963-d80e-3a91-6e74-66f3811b99cc", accepted: true });
|
||||
|
||||
await c.maybeSync(2000, Date.now()); // Does not fail.
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_current_server_time_is_saved_in_pref() {
|
||||
const serverTime = Date.now();
|
||||
await client.maybeSync(2000, serverTime);
|
||||
equal(client.lastCheckTimePref, "services.settings.main.password-fields.last_check");
|
||||
const after = Services.prefs.getIntPref(client.lastCheckTimePref);
|
||||
equal(after, Math.round(serverTime / 1000));
|
||||
await c.maybeSync(2000); // Does not fail.
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
|
|
@ -114,34 +114,55 @@ add_task(async function test_records_changes_are_overwritten_by_server_changes()
|
|||
"id": "9d500963-d80e-3a91-6e74-66f3811b99cc",
|
||||
}, { useRecordId: true });
|
||||
|
||||
await client.maybeSync(2000, Date.now());
|
||||
await client.maybeSync(2000);
|
||||
|
||||
const data = await client.get();
|
||||
equal(data[0].website, "https://some-website.com");
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_default_records_come_from_a_local_dump_when_database_is_empty() {
|
||||
// When collection is unknown, no dump is loaded, and there is no error.
|
||||
let data = await RemoteSettings("some-unknown-key").get();
|
||||
equal(data.length, 0);
|
||||
add_task(async function test_get_loads_default_records_from_a_local_dump_when_database_is_empty() {
|
||||
if (IS_ANDROID) {
|
||||
// Skip test: we don't ship remote settings dumps on Android (see package-manifest).
|
||||
return;
|
||||
}
|
||||
|
||||
// When collection has a dump in services/settings/dumps/{bucket}/{collection}.json
|
||||
data = await RemoteSettings("certificates", { bucketNamePref: "services.blocklist.bucket" }).get();
|
||||
const data = await clientWithDump.get();
|
||||
notEqual(data.length, 0);
|
||||
// No synchronization happened (responses are not mocked).
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_get_triggers_synchronization_when_database_is_empty() {
|
||||
// The "password-fields" collection has no local dump, and no local data.
|
||||
// Therefore a synchronization will happen.
|
||||
const data = await client.get();
|
||||
|
||||
// Data comes from mocked HTTP response (see below).
|
||||
equal(data.length, 1);
|
||||
equal(data[0].selector, "#webpage[field-pwd]");
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_get_ignores_synchronization_errors() {
|
||||
// The monitor endpoint won't contain any information about this collection.
|
||||
let data = await RemoteSettings("some-unknown-key").get();
|
||||
equal(data.length, 0);
|
||||
// The sync endpoints are not mocked, this fails internally.
|
||||
data = await RemoteSettings("no-mocked-responses").get();
|
||||
equal(data.length, 0);
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_sync_event_provides_information_about_records() {
|
||||
const serverTime = Date.now();
|
||||
|
||||
let eventData;
|
||||
client.on("sync", ({ data }) => eventData = data);
|
||||
|
||||
await client.maybeSync(2000, serverTime - 1000);
|
||||
await client.maybeSync(2000);
|
||||
equal(eventData.current.length, 1);
|
||||
|
||||
await client.maybeSync(3001, serverTime);
|
||||
await client.maybeSync(3001);
|
||||
equal(eventData.current.length, 2);
|
||||
equal(eventData.created.length, 1);
|
||||
equal(eventData.created[0].website, "https://www.other.org/signin");
|
||||
|
|
@ -150,7 +171,7 @@ add_task(async function test_sync_event_provides_information_about_records() {
|
|||
equal(eventData.updated[0].new.website, "https://some-website.com/login");
|
||||
equal(eventData.deleted.length, 0);
|
||||
|
||||
await client.maybeSync(4001, serverTime);
|
||||
await client.maybeSync(4001);
|
||||
equal(eventData.current.length, 1);
|
||||
equal(eventData.created.length, 0);
|
||||
equal(eventData.updated.length, 0);
|
||||
|
|
@ -160,31 +181,39 @@ add_task(async function test_sync_event_provides_information_about_records() {
|
|||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_inspect_method() {
|
||||
const serverTime = Date.now();
|
||||
|
||||
// Synchronize the `password-fields` collection.
|
||||
await client.maybeSync(2000, serverTime);
|
||||
// Synchronize the `password-fields` collection in order to have
|
||||
// some local data when .inspect() is called.
|
||||
await client.maybeSync(2000);
|
||||
|
||||
const inspected = await RemoteSettings.inspect();
|
||||
|
||||
const { mainBucket, serverURL, defaultSigner, collections } = inspected;
|
||||
// Assertion for global attributes.
|
||||
const { mainBucket, serverURL, defaultSigner, collections, serverTimestamp } = inspected;
|
||||
const rsSigner = "remote-settings.content-signature.mozilla.org";
|
||||
equal(mainBucket, "main");
|
||||
equal(serverURL, `http://localhost:${server.identity.primaryPort}/v1`);
|
||||
equal(defaultSigner, rsSigner);
|
||||
equal(serverTimestamp, '"5000"');
|
||||
|
||||
equal(inspected.serverTimestamp, '"4000"');
|
||||
equal(collections.length, 1);
|
||||
// password-fields was synchronized and has local data.
|
||||
equal(collections[0].collection, "password-fields");
|
||||
equal(collections[0].serverTimestamp, 3000);
|
||||
equal(collections[0].localTimestamp, 3000);
|
||||
// A collection is listed in .inspect() if it has local data or if there
|
||||
// is a JSON dump for it.
|
||||
// "password-fields" has no dump but was synchronized above and thus has local data.
|
||||
let col = collections.pop();
|
||||
equal(col.collection, "password-fields");
|
||||
equal(col.serverTimestamp, 3000);
|
||||
equal(col.localTimestamp, 3000);
|
||||
|
||||
if (!IS_ANDROID) {
|
||||
// "language-dictionaries" has a local dump (not on Android)
|
||||
col = collections.pop();
|
||||
equal(col.collection, "language-dictionaries");
|
||||
equal(col.serverTimestamp, 4000);
|
||||
ok(!col.localTimestamp); // not synchronized.
|
||||
}
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_listeners_are_not_deduplicated() {
|
||||
const serverTime = Date.now();
|
||||
|
||||
let count = 0;
|
||||
const plus1 = () => { count += 1; };
|
||||
|
||||
|
|
@ -192,30 +221,26 @@ add_task(async function test_listeners_are_not_deduplicated() {
|
|||
client.on("sync", plus1);
|
||||
client.on("sync", plus1);
|
||||
|
||||
await client.maybeSync(2000, serverTime);
|
||||
await client.maybeSync(2000);
|
||||
|
||||
equal(count, 3);
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_listeners_can_be_removed() {
|
||||
const serverTime = Date.now();
|
||||
|
||||
let count = 0;
|
||||
const onSync = () => { count += 1; };
|
||||
|
||||
client.on("sync", onSync);
|
||||
client.off("sync", onSync);
|
||||
|
||||
await client.maybeSync(2000, serverTime);
|
||||
await client.maybeSync(2000);
|
||||
|
||||
equal(count, 0);
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_all_listeners_are_executed_if_one_fails() {
|
||||
const serverTime = Date.now();
|
||||
|
||||
let count = 0;
|
||||
client.on("sync", () => { count += 1; });
|
||||
client.on("sync", () => { throw new Error("boom"); });
|
||||
|
|
@ -223,7 +248,7 @@ add_task(async function test_all_listeners_are_executed_if_one_fails() {
|
|||
|
||||
let error;
|
||||
try {
|
||||
await client.maybeSync(2000, serverTime);
|
||||
await client.maybeSync(2000);
|
||||
} catch (e) {
|
||||
error = e;
|
||||
}
|
||||
|
|
@ -234,11 +259,10 @@ add_task(async function test_all_listeners_are_executed_if_one_fails() {
|
|||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_telemetry_reports_up_to_date() {
|
||||
await client.maybeSync(2000, Date.now() - 1000);
|
||||
const serverTime = Date.now();
|
||||
await client.maybeSync(2000);
|
||||
const startHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
|
||||
await client.maybeSync(3000, serverTime);
|
||||
await client.maybeSync(3000);
|
||||
|
||||
// No Telemetry was sent.
|
||||
const endHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
|
|
@ -249,10 +273,9 @@ add_task(clear_state);
|
|||
|
||||
add_task(async function test_telemetry_if_sync_succeeds() {
|
||||
// We test each client because Telemetry requires preleminary declarations.
|
||||
const serverTime = Date.now();
|
||||
const startHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
|
||||
await client.maybeSync(2000, serverTime);
|
||||
await client.maybeSync(2000);
|
||||
|
||||
const endHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
const expectedIncrements = {[UptakeTelemetry.STATUS.SUCCESS]: 1};
|
||||
|
|
@ -261,12 +284,11 @@ add_task(async function test_telemetry_if_sync_succeeds() {
|
|||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_telemetry_reports_if_application_fails() {
|
||||
const serverTime = Date.now();
|
||||
const startHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
client.on("sync", () => { throw new Error("boom"); });
|
||||
|
||||
try {
|
||||
await client.maybeSync(2000, serverTime);
|
||||
await client.maybeSync(2000);
|
||||
} catch (e) {}
|
||||
|
||||
const endHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
|
|
@ -276,15 +298,13 @@ add_task(async function test_telemetry_reports_if_application_fails() {
|
|||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_telemetry_reports_if_sync_fails() {
|
||||
const serverTime = Date.now();
|
||||
|
||||
const collection = await client.openCollection();
|
||||
await collection.db.saveLastModified(9999);
|
||||
|
||||
const startHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
|
||||
try {
|
||||
await client.maybeSync(10000, serverTime);
|
||||
await client.maybeSync(10000);
|
||||
} catch (e) {}
|
||||
|
||||
const endHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
|
|
@ -294,13 +314,12 @@ add_task(async function test_telemetry_reports_if_sync_fails() {
|
|||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_telemetry_reports_unknown_errors() {
|
||||
const serverTime = Date.now();
|
||||
const backup = client.openCollection;
|
||||
client.openCollection = () => { throw new Error("Internal"); };
|
||||
const startHistogram = getUptakeTelemetrySnapshot(client.identifier);
|
||||
|
||||
try {
|
||||
await client.maybeSync(2000, serverTime);
|
||||
await client.maybeSync(2000);
|
||||
} catch (e) {}
|
||||
|
||||
client.openCollection = backup;
|
||||
|
|
@ -320,15 +339,23 @@ add_task(async function test_bucketname_changes_when_bucket_pref_changes() {
|
|||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_inspect_changes_the_list_when_bucket_pref_is_changed() {
|
||||
if (IS_ANDROID) {
|
||||
// Skip test: we don't ship remote settings dumps on Android (see package-manifest).
|
||||
return;
|
||||
}
|
||||
// Register a client only listed in -preview...
|
||||
RemoteSettings("crash-rate");
|
||||
|
||||
const { collections: before } = await RemoteSettings.inspect();
|
||||
deepEqual(before.map(c => c.collection).sort(), ["password-fields"]);
|
||||
|
||||
// These two collections are listed in the main bucket in monitor/changes (one with dump, one registered).
|
||||
deepEqual(before.map(c => c.collection).sort(), ["language-dictionaries", "password-fields"]);
|
||||
|
||||
// Switch to main-preview bucket.
|
||||
Services.prefs.setCharPref("services.settings.default_bucket", "main-preview");
|
||||
|
||||
const { collections: after, mainBucket } = await RemoteSettings.inspect();
|
||||
|
||||
// These two collections are listed in the main bucket in monitor/changes (both are registered).
|
||||
deepEqual(after.map(c => c.collection).sort(), ["crash-rate", "password-fields"]);
|
||||
equal(mainBucket, "main-preview");
|
||||
});
|
||||
|
|
@ -374,14 +401,19 @@ function getSampleResponse(req, port) {
|
|||
"Content-Type: application/json; charset=UTF-8",
|
||||
"Server: waitress",
|
||||
`Date: ${new Date().toUTCString()}`,
|
||||
"Etag: \"4000\"",
|
||||
"Etag: \"5000\"",
|
||||
],
|
||||
"status": { status: 200, statusText: "OK" },
|
||||
"responseBody": {
|
||||
"data": [{
|
||||
"id": "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
|
||||
"bucket": "main",
|
||||
"collection": "unknown",
|
||||
"collection": "unknown-locally",
|
||||
"last_modified": 5000,
|
||||
}, {
|
||||
"id": "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
|
||||
"bucket": "main",
|
||||
"collection": "language-dictionaries",
|
||||
"last_modified": 4000,
|
||||
}, {
|
||||
"id": "0af8da0b-3e03-48fb-8d0d-2d8e4cb7514d",
|
||||
|
|
@ -472,6 +504,62 @@ function getSampleResponse(req, port) {
|
|||
error: "Service Unavailable",
|
||||
},
|
||||
},
|
||||
"GET:/v1/buckets/monitor/collections/changes/records?collection=password-fields&bucket=main": {
|
||||
"sampleHeaders": [
|
||||
"Access-Control-Allow-Origin: *",
|
||||
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
|
||||
"Content-Type: application/json; charset=UTF-8",
|
||||
"Server: waitress",
|
||||
`Date: ${new Date().toUTCString()}`,
|
||||
"Etag: \"1338\"",
|
||||
],
|
||||
"status": { status: 200, statusText: "OK" },
|
||||
"responseBody": {
|
||||
"data": [{
|
||||
"id": "fe5758d0-c67a-42d0-bb4f-8f2d75106b65",
|
||||
"bucket": "main",
|
||||
"collection": "password-fields",
|
||||
"last_modified": 1337,
|
||||
}],
|
||||
},
|
||||
},
|
||||
"GET:/v1/buckets/main/collections/password-fields/records?_expected=1337&_sort=-last_modified": {
|
||||
"sampleHeaders": [
|
||||
"Access-Control-Allow-Origin: *",
|
||||
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
|
||||
"Content-Type: application/json; charset=UTF-8",
|
||||
"Server: waitress",
|
||||
"Etag: \"3000\"",
|
||||
],
|
||||
"status": { status: 200, statusText: "OK" },
|
||||
"responseBody": {
|
||||
"data": [{
|
||||
"id": "312cc78d-9c1f-4291-a4fa-a1be56f6cc69",
|
||||
"last_modified": 3000,
|
||||
"website": "https://some-website.com",
|
||||
"selector": "#webpage[field-pwd]",
|
||||
}],
|
||||
},
|
||||
},
|
||||
"GET:/v1/buckets/monitor/collections/changes/records?collection=no-mocked-responses&bucket=main": {
|
||||
"sampleHeaders": [
|
||||
"Access-Control-Allow-Origin: *",
|
||||
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
|
||||
"Content-Type: application/json; charset=UTF-8",
|
||||
"Server: waitress",
|
||||
`Date: ${new Date().toUTCString()}`,
|
||||
"Etag: \"713705\"",
|
||||
],
|
||||
"status": { status: 200, statusText: "OK" },
|
||||
"responseBody": {
|
||||
"data": [{
|
||||
"id": "07a98d1b-7c62-4344-ab18-76856b3facd8",
|
||||
"bucket": "main",
|
||||
"collection": "no-mocked-responses",
|
||||
"last_modified": 713705,
|
||||
}],
|
||||
},
|
||||
},
|
||||
};
|
||||
return responses[`${req.method}:${req.path}?${req.queryString}`] ||
|
||||
responses[`${req.method}:${req.path}`] ||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ const PREF_LAST_UPDATE = "services.settings.last_update_seconds";
|
|||
const PREF_LAST_ETAG = "services.settings.last_etag";
|
||||
const PREF_CLOCK_SKEW_SECONDS = "services.settings.clock_skew_seconds";
|
||||
|
||||
const DB_NAME = "remote-settings";
|
||||
// Telemetry report result.
|
||||
const TELEMETRY_HISTOGRAM_KEY = "settings-changes-monitoring";
|
||||
const CHANGES_PATH = "/v1/buckets/monitor/collections/changes/records";
|
||||
|
|
@ -228,6 +229,34 @@ add_task(async function test_expected_timestamp() {
|
|||
add_task(clear_state);
|
||||
|
||||
|
||||
add_task(async function test_client_last_check_is_saved() {
|
||||
server.registerPathHandler(CHANGES_PATH, (request, response) => {
|
||||
response.write(JSON.stringify({
|
||||
data: [{
|
||||
id: "695c2407-de79-4408-91c7-70720dd59d78",
|
||||
last_modified: 1100,
|
||||
host: "localhost",
|
||||
bucket: "main",
|
||||
collection: "models-recipes",
|
||||
}],
|
||||
}));
|
||||
response.setHeader("ETag", '"42"');
|
||||
response.setHeader("Date", (new Date()).toUTCString());
|
||||
response.setStatusLine(null, 200, "OK");
|
||||
});
|
||||
|
||||
const c = RemoteSettings("models-recipes");
|
||||
c.maybeSync = () => {};
|
||||
|
||||
equal(c.lastCheckTimePref, "services.settings.main.models-recipes.last_check");
|
||||
Services.prefs.setIntPref(c.lastCheckTimePref, 0);
|
||||
|
||||
await RemoteSettings.pollChanges({ expectedTimestamp: '"42"' });
|
||||
|
||||
notEqual(Services.prefs.getIntPref(c.lastCheckTimePref), 0);
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_success_with_partial_list() {
|
||||
function partialList(request, response) {
|
||||
const entries = [{
|
||||
|
|
@ -503,9 +532,8 @@ add_task(async function test_syncs_clients_with_local_database() {
|
|||
// This simulates what remote-settings would do when initializing a local database.
|
||||
// We don't want to instantiate a client using the RemoteSettings() API
|
||||
// since we want to test «unknown» clients that have a local database.
|
||||
const dbName = "remote-settings";
|
||||
await (new Kinto.adapters.IDB("blocklists/addons", { dbName })).saveLastModified(42);
|
||||
await (new Kinto.adapters.IDB("main/recipes", { dbName })).saveLastModified(43);
|
||||
await (new Kinto.adapters.IDB("blocklists/addons", { dbName: DB_NAME })).saveLastModified(42);
|
||||
await (new Kinto.adapters.IDB("main/recipes", { dbName: DB_NAME })).saveLastModified(43);
|
||||
|
||||
let error;
|
||||
try {
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче