зеркало из https://github.com/mozilla/gecko-dev.git
Backed out 2 changesets (bug 1717068) for Build bustages. CLOSED TREE
Backed out changeset 1195f23a6393 (bug 1717068) Backed out changeset 7f269f70c52c (bug 1717068)
This commit is contained in:
Dorel Luca
Родитель
593d81e560
Коммит
dfc7a95d69
|
|
@ -341,7 +341,6 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
* @param {Object} options.filters Filter the results (default: `{}`).
|
||||
* @param {String} options.order The order to apply (eg. `"-last_modified"`).
|
||||
* @param {boolean} options.dumpFallback Fallback to dump data if read of local DB fails (default: `true`).
|
||||
* @param {boolean} options.loadDumpIfNewer Use dump data if it is newer than local data (default: `false`).
|
||||
* @param {boolean} options.syncIfEmpty Synchronize from server if local data is empty (default: `true`).
|
||||
* @param {boolean} options.verifySignature Verify the signature of the local data (default: `false`).
|
||||
* @return {Promise}
|
||||
|
|
@ -351,15 +350,13 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
filters = {},
|
||||
order = "", // not sorted by default.
|
||||
dumpFallback = true,
|
||||
loadDumpIfNewer = false, // TODO bug 1718083: should default to true.
|
||||
syncIfEmpty = true,
|
||||
} = options;
|
||||
let { verifySignature = false } = options;
|
||||
|
||||
let data;
|
||||
try {
|
||||
let lastModified = await this.db.getLastModified();
|
||||
let hasLocalData = lastModified !== null;
|
||||
let hasLocalData = await Utils.hasLocalData(this);
|
||||
|
||||
if (syncIfEmpty && !hasLocalData) {
|
||||
// .get() was called before we had the chance to synchronize the local database.
|
||||
|
|
@ -378,35 +375,9 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
await this.sync({ loadDump: false });
|
||||
}
|
||||
})();
|
||||
} else {
|
||||
console.debug(`${this.identifier} Awaiting existing import.`);
|
||||
}
|
||||
} else if (hasLocalData && loadDumpIfNewer) {
|
||||
// Check whether the local data is older than the packaged dump.
|
||||
// If it is, load the packaged dump (which overwrites the local data).
|
||||
let lastModifiedDump = await Utils.getLocalDumpLastModified(
|
||||
this.bucketName,
|
||||
this.collectionName
|
||||
);
|
||||
if (lastModified < lastModifiedDump) {
|
||||
console.debug(
|
||||
`${this.identifier} Local DB is stale (${lastModified}), using dump instead (${lastModifiedDump})`
|
||||
);
|
||||
if (!this._importingPromise) {
|
||||
// As part of importing, any existing data is wiped.
|
||||
this._importingPromise = this._importJSONDump();
|
||||
} else {
|
||||
console.debug(`${this.identifier} Awaiting existing import.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (this._importingPromise) {
|
||||
try {
|
||||
await this._importingPromise;
|
||||
// No need to verify signature, because either we've just load a trusted
|
||||
// dump (here or in a parallel call), or it was verified during sync.
|
||||
verifySignature = false;
|
||||
} catch (e) {
|
||||
// Report error, but continue because there could have been data
|
||||
// loaded from a parrallel call.
|
||||
|
|
@ -415,6 +386,9 @@ class RemoteSettingsClient extends EventEmitter {
|
|||
// then delete this promise again, as now we should have local data:
|
||||
delete this._importingPromise;
|
||||
}
|
||||
// No need to verify signature, because either we've just load a trusted
|
||||
// dump (here or in a parallel call), or it was verified during sync.
|
||||
verifySignature = false;
|
||||
}
|
||||
|
||||
// Read from the local DB.
|
||||
|
|
|
|||
|
|
@ -126,36 +126,6 @@ var Utils = {
|
|||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Look up the last modification time of the JSON dump.
|
||||
*
|
||||
* @param {String} bucket
|
||||
* @param {String} collection
|
||||
* @return {int} The last modification time of the dump. -1 if non-existent.
|
||||
*/
|
||||
async getLocalDumpLastModified(bucket, collection) {
|
||||
if (!this._dumpStats) {
|
||||
this._dumpStats = {};
|
||||
}
|
||||
const identifier = `${bucket}/${collection}`;
|
||||
let lastModified = this._dumpStats[identifier];
|
||||
if (lastModified === undefined) {
|
||||
try {
|
||||
let res = await fetch(
|
||||
`resource://app/defaults/settings/${bucket}/${collection}.json`
|
||||
);
|
||||
let records = (await res.json()).data;
|
||||
// Records in dumps are sorted by last_modified, newest first.
|
||||
// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
|
||||
lastModified = records[0]?.last_modified || 0;
|
||||
} catch (e) {
|
||||
lastModified = -1;
|
||||
}
|
||||
this._dumpStats[identifier] = lastModified;
|
||||
}
|
||||
return lastModified;
|
||||
},
|
||||
|
||||
/**
|
||||
* Fetch the list of remote collections and their timestamp.
|
||||
* ```
|
||||
|
|
|
|||
|
|
@ -1,127 +0,0 @@
|
|||
const { RemoteSettingsClient } = ChromeUtils.import(
|
||||
"resource://services-settings/RemoteSettingsClient.jsm"
|
||||
);
|
||||
const { RemoteSettingsWorker } = ChromeUtils.import(
|
||||
"resource://services-settings/RemoteSettingsWorker.jsm"
|
||||
);
|
||||
const { SharedUtils } = ChromeUtils.import(
|
||||
"resource://services-settings/SharedUtils.jsm"
|
||||
);
|
||||
|
||||
// A collection with a dump that's packaged on all builds where this test runs,
|
||||
// including on Android at mobile/android/installer/package-manifest.in
|
||||
const TEST_BUCKET = "main";
|
||||
const TEST_COLLECTION = "password-recipes";
|
||||
|
||||
let client;
|
||||
let DUMP_RECORDS;
|
||||
let DUMP_LAST_MODIFIED;
|
||||
|
||||
add_task(async function setup() {
|
||||
// "services.settings.server" pref is not set.
|
||||
// Test defaults to an unreachable server,
|
||||
// and will only load from the dump if any.
|
||||
|
||||
client = new RemoteSettingsClient(TEST_COLLECTION, {
|
||||
bucketName: TEST_BUCKET,
|
||||
});
|
||||
|
||||
DUMP_RECORDS = (await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION))
|
||||
.data;
|
||||
DUMP_LAST_MODIFIED = DUMP_RECORDS.reduce(
|
||||
(max, { last_modified }) => Math.max(last_modified, max),
|
||||
-Infinity
|
||||
);
|
||||
// Dumps are fetched via the following, which sorts the records, newest first.
|
||||
// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
|
||||
equal(
|
||||
DUMP_LAST_MODIFIED,
|
||||
DUMP_RECORDS[0].last_modified,
|
||||
"records in dump ought to be sorted by last_modified"
|
||||
);
|
||||
});
|
||||
|
||||
async function importData(records) {
|
||||
await RemoteSettingsWorker._execute("_test_only_import", [
|
||||
TEST_BUCKET,
|
||||
TEST_COLLECTION,
|
||||
records,
|
||||
]);
|
||||
}
|
||||
|
||||
async function clear_state() {
|
||||
await client.db.clear();
|
||||
}
|
||||
|
||||
add_task(async function test_load_from_dump_when_offline() {
|
||||
// Baseline: verify that the collection is empty at first,
|
||||
// but non-empty after loading from the dump.
|
||||
const before = await client.get({ syncIfEmpty: false });
|
||||
equal(before.length, 0, "collection empty when offline");
|
||||
|
||||
// should import from dump since collection was not initialized.
|
||||
const after = await client.get();
|
||||
equal(after.length, DUMP_RECORDS.length, "collection loaded from dump");
|
||||
equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_skip_dump_after_empty_import() {
|
||||
// clear_state should have wiped the database.
|
||||
const before = await client.get({ syncIfEmpty: false });
|
||||
equal(before.length, 0, "collection empty after clearing");
|
||||
|
||||
// Verify that the dump is not imported again by client.get()
|
||||
// when the database is initialized with an empty dump.
|
||||
await importData([]); // <-- Empty set of records.
|
||||
|
||||
const after = await client.get();
|
||||
equal(after.length, 0, "collection still empty due to import");
|
||||
equal(await client.getLastModified(), 0, "Empty dump has no timestamp");
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_skip_dump_after_non_empty_import() {
|
||||
await importData([{ last_modified: 1234, id: "dummy" }]);
|
||||
|
||||
const after = await client.get();
|
||||
equal(after.length, 1, "Imported dummy data");
|
||||
equal(await client.getLastModified(), 1234, "Expected timestamp of import");
|
||||
|
||||
await importData([]);
|
||||
const after2 = await client.get();
|
||||
equal(after2.length, 0, "Previous data wiped on duplicate import");
|
||||
equal(await client.getLastModified(), 0, "Timestamp of empty collection");
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_load_dump_after_empty_import() {
|
||||
await importData([]); // <-- Empty set of records, i.e. last_modified = 0.
|
||||
|
||||
const after = await client.get({ loadDumpIfNewer: true });
|
||||
equal(after.length, DUMP_RECORDS.length, "Imported dump");
|
||||
equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_load_dump_after_non_empty_import() {
|
||||
// Dump is updated regularly, verify that the dump matches our expectations
|
||||
// before running the test.
|
||||
ok(DUMP_LAST_MODIFIED > 1234, "Assuming dump to be newer than dummy 1234");
|
||||
|
||||
await importData([{ last_modified: 1234, id: "dummy" }]);
|
||||
|
||||
const after = await client.get({ loadDumpIfNewer: true });
|
||||
equal(after.length, DUMP_RECORDS.length, "Imported dump");
|
||||
equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
||||
add_task(async function test_skip_dump_if_same_last_modified() {
|
||||
await importData([{ last_modified: DUMP_LAST_MODIFIED, id: "dummy" }]);
|
||||
|
||||
const after = await client.get({ loadDumpIfNewer: true });
|
||||
equal(after.length, 1, "Not importing dump when time matches");
|
||||
equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "Same timestamp");
|
||||
});
|
||||
add_task(clear_state);
|
||||
|
|
@ -9,8 +9,6 @@ skip-if = appname == "thunderbird" # Bug 1662758 - these tests don't pass with a
|
|||
[test_attachments_downloader.js]
|
||||
support-files = test_attachments_downloader/**
|
||||
[test_remote_settings.js]
|
||||
[test_remote_settings_dump_lastmodified.js]
|
||||
[test_remote_settings_offline.js]
|
||||
[test_remote_settings_poll.js]
|
||||
[test_remote_settings_worker.js]
|
||||
[test_remote_settings_jexl_filters.js]
|
||||
|
|
|
|||
|
|
@ -1008,7 +1008,7 @@ this.ExtensionBlocklistMLBF = {
|
|||
this._stashes = null;
|
||||
return;
|
||||
}
|
||||
let records = await this._client.get({ loadDumpIfNewer: true });
|
||||
let records = await this._client.get();
|
||||
if (isUpdateReplaced()) {
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,8 +7,7 @@ const { ComponentUtils } = ChromeUtils.import(
|
|||
|
||||
const MLBF_RECORD = {
|
||||
id: "A blocklist entry that refers to a MLBF file",
|
||||
// Higher than any last_modified in addons-bloomfilters.json:
|
||||
last_modified: Date.now(),
|
||||
last_modified: 1,
|
||||
attachment: {
|
||||
size: 32,
|
||||
hash: "6af648a5d6ce6dbee99b0aab1780d24d204977a6606ad670d5372ef22fac1052",
|
||||
|
|
|
|||
|
|
@ -38,18 +38,14 @@ async function sha256(arrayBuffer) {
|
|||
return Array.from(new Uint8Array(hash), toHex).join("");
|
||||
}
|
||||
|
||||
// A list of { inputRecord, downloadPromise }:
|
||||
// - inputRecord is the record that was used for looking up the MLBF.
|
||||
// - downloadPromise is the result of trying to download it.
|
||||
const observed = [];
|
||||
|
||||
add_task(async function setup() {
|
||||
add_task(async function verify_dump_first_run() {
|
||||
createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
|
||||
ExtensionBlocklistMLBF.ensureInitialized();
|
||||
|
||||
// Tapping into the internals of ExtensionBlocklistMLBF._fetchMLBF to observe
|
||||
// MLBF request details.
|
||||
const observed = [];
|
||||
|
||||
ExtensionBlocklistMLBF.ensureInitialized();
|
||||
// Despite being called "download", this does not actually access the network
|
||||
// when there is a valid dump.
|
||||
const originalImpl = ExtensionBlocklistMLBF._client.attachments.download;
|
||||
|
|
@ -59,10 +55,6 @@ add_task(async function setup() {
|
|||
return downloadPromise;
|
||||
};
|
||||
|
||||
await promiseStartupManager();
|
||||
});
|
||||
|
||||
async function verifyBlocklistWorksWithDump() {
|
||||
Assert.equal(
|
||||
await Blocklist.getAddonBlocklistState(blockedAddon),
|
||||
Ci.nsIBlocklistService.STATE_BLOCKED,
|
||||
|
|
@ -73,13 +65,10 @@ async function verifyBlocklistWorksWithDump() {
|
|||
Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
|
||||
"A known non-blocked add-on should not be blocked"
|
||||
);
|
||||
}
|
||||
|
||||
add_task(async function verify_dump_first_run() {
|
||||
await verifyBlocklistWorksWithDump();
|
||||
Assert.equal(observed.length, 1, "expected number of MLBF download requests");
|
||||
|
||||
const { inputRecord, downloadPromise } = observed.pop();
|
||||
const { inputRecord, downloadPromise } = observed[0];
|
||||
|
||||
Assert.ok(inputRecord, "addons-bloomfilters collection dump exists");
|
||||
|
||||
|
|
@ -105,52 +94,3 @@ add_task(async function verify_dump_first_run() {
|
|||
"The content of the attachment should actually matches the record"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(async function use_dump_fallback_when_collection_is_out_of_sync() {
|
||||
await AddonTestUtils.loadBlocklistRawData({
|
||||
// last_modified higher than any value in addons-bloomfilters.json.
|
||||
extensionsMLBF: [{ last_modified: Date.now() }],
|
||||
});
|
||||
Assert.equal(observed.length, 1, "Expected new download on update");
|
||||
|
||||
const { inputRecord, downloadPromise } = observed.pop();
|
||||
Assert.equal(inputRecord, null, "No MLBF record found");
|
||||
|
||||
const downloadResult = await downloadPromise;
|
||||
Assert.equal(
|
||||
downloadResult._source,
|
||||
"dump_fallback",
|
||||
"should have used fallback despite the absence of a MLBF record"
|
||||
);
|
||||
|
||||
await verifyBlocklistWorksWithDump();
|
||||
Assert.equal(observed.length, 0, "Blocklist uses cached result");
|
||||
});
|
||||
|
||||
// Verifies that the dump would supersede local data. This can happen after an
|
||||
// application upgrade, where the local database contains outdated records from
|
||||
// a previous application version.
|
||||
add_task(async function verify_dump_supersedes_old_dump() {
|
||||
// Delete in-memory value; otherwise the cached record from the previous test
|
||||
// task would be re-used and nothing would be downloaded.
|
||||
delete ExtensionBlocklistMLBF._mlbfData;
|
||||
|
||||
await AddonTestUtils.loadBlocklistRawData({
|
||||
// last_modified lower than any value in addons-bloomfilters.json.
|
||||
extensionsMLBF: [{ last_modified: 1 }],
|
||||
});
|
||||
Assert.equal(observed.length, 1, "Expected new download on update");
|
||||
|
||||
const { inputRecord, downloadPromise } = observed.pop();
|
||||
Assert.ok(inputRecord, "should have read from addons-bloomfilters dump");
|
||||
|
||||
const downloadResult = await downloadPromise;
|
||||
Assert.equal(
|
||||
downloadResult._source,
|
||||
"dump_match",
|
||||
"Should have replaced outdated collection records with dump"
|
||||
);
|
||||
|
||||
await verifyBlocklistWorksWithDump();
|
||||
Assert.equal(observed.length, 0, "Blocklist uses cached result");
|
||||
});
|
||||
|
|
|
|||
|
|
@ -115,13 +115,7 @@ add_task(async function test_without_stashes() {
|
|||
// Test what happens when the collection was inadvertently emptied,
|
||||
// but still with a cached mlbf from before.
|
||||
add_task(async function test_without_collection_but_cache() {
|
||||
await AddonTestUtils.loadBlocklistRawData({
|
||||
// Insert a dummy record with a value of last_modified which is higher than
|
||||
// any value of last_modified in addons-bloomfilters.json, to prevent the
|
||||
// blocklist implementation from automatically falling back to the packaged
|
||||
// JSON dump.
|
||||
extensionsMLBF: [{ last_modified: Date.now() }],
|
||||
});
|
||||
await AddonTestUtils.loadBlocklistRawData({ extensionsMLBF: [] });
|
||||
assertTelemetryScalars({
|
||||
"blocklist.mlbf_enabled": true,
|
||||
"blocklist.mlbf_source": "cache_fallback",
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче