зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1620621 - Add bloomfilter-based blocklist for addons r=Gijs,aswan
NOTE: This commit does not yet include a dump of the RemoteSettings collection and attachment. This will be added in the near future. Differential Revision: https://phabricator.services.mozilla.com/D72418
This commit is contained in:
Rob Wu
Родитель
4c8480260e
Коммит
5047bb6dea
|
|
@ -2332,12 +2332,14 @@ pref("extensions.abuseReport.amoDetailsURL", "https://services.addons.mozilla.or
|
|||
|
||||
// Blocklist preferences
|
||||
pref("extensions.blocklist.enabled", true);
|
||||
pref("extensions.blocklist.useMLBF", false);
|
||||
// Required blocklist freshness for OneCRL OCSP bypass (default is 30 hours)
|
||||
// Note that this needs to exceed the interval at which we update OneCRL data,
|
||||
// configured in services.settings.poll_interval .
|
||||
pref("security.onecrl.maximum_staleness_in_seconds", 108000);
|
||||
pref("extensions.blocklist.detailsURL", "https://blocked.cdn.mozilla.net/");
|
||||
pref("extensions.blocklist.itemURL", "https://blocked.cdn.mozilla.net/%blockID%.html");
|
||||
pref("extensions.blocklist.addonItemURL", "https://addons.mozilla.org/%LOCALE%/%APP%/blocked-addon/%addonID%/%addonVersion%/");
|
||||
// Controls what level the blocklist switches from warning about items to forcibly
|
||||
// blocking them.
|
||||
pref("extensions.blocklist.level", 2);
|
||||
|
|
@ -2346,6 +2348,9 @@ pref("services.blocklist.bucket", "blocklists");
|
|||
pref("services.blocklist.addons.collection", "addons");
|
||||
pref("services.blocklist.addons.checked", 0);
|
||||
pref("services.blocklist.addons.signer", "remote-settings.content-signature.mozilla.org");
|
||||
pref("services.blocklist.addons-mlbf.collection", "addons-bloomfilters");
|
||||
pref("services.blocklist.addons-mlbf.checked", 0);
|
||||
pref("services.blocklist.addons-mlbf.signer", "remote-settings.content-signature.mozilla.org");
|
||||
pref("services.blocklist.plugins.collection", "plugins");
|
||||
pref("services.blocklist.plugins.checked", 0);
|
||||
pref("services.blocklist.plugins.signer", "remote-settings.content-signature.mozilla.org");
|
||||
|
|
|
|||
|
|
@ -38,6 +38,12 @@ ChromeUtils.defineModuleGetter(
|
|||
"resource://services-settings/remote-settings.js"
|
||||
);
|
||||
|
||||
const CascadeFilter = Components.Constructor(
|
||||
"@mozilla.org/cascade-filter;1",
|
||||
"nsICascadeFilter",
|
||||
"setFilterData"
|
||||
);
|
||||
|
||||
// The whole ID should be surrounded by literal ().
|
||||
// IDs may contain alphanumerics, _, -, {}, @ and a literal '.'
|
||||
// They may also contain backslashes (needed to escape the {} and dot)
|
||||
|
|
@ -127,9 +133,11 @@ function doesAddonEntryMatch(matches, addonProps) {
|
|||
|
||||
const TOOLKIT_ID = "toolkit@mozilla.org";
|
||||
const PREF_BLOCKLIST_ITEM_URL = "extensions.blocklist.itemURL";
|
||||
const PREF_BLOCKLIST_ADDONITEM_URL = "extensions.blocklist.addonItemURL";
|
||||
const PREF_BLOCKLIST_ENABLED = "extensions.blocklist.enabled";
|
||||
const PREF_BLOCKLIST_LEVEL = "extensions.blocklist.level";
|
||||
const PREF_BLOCKLIST_SUPPRESSUI = "extensions.blocklist.suppressUI";
|
||||
const PREF_BLOCKLIST_USE_MLBF = "extensions.blocklist.useMLBF";
|
||||
const PREF_EM_LOGGING_ENABLED = "extensions.logging.enabled";
|
||||
const URI_BLOCKLIST_DIALOG =
|
||||
"chrome://mozapps/content/extensions/blocklist.xhtml";
|
||||
|
|
@ -151,10 +159,17 @@ const PREF_BLOCKLIST_PLUGINS_COLLECTION =
|
|||
const PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS =
|
||||
"services.blocklist.plugins.checked";
|
||||
const PREF_BLOCKLIST_PLUGINS_SIGNER = "services.blocklist.plugins.signer";
|
||||
// Blocklist v2 - legacy JSON format.
|
||||
const PREF_BLOCKLIST_ADDONS_COLLECTION = "services.blocklist.addons.collection";
|
||||
const PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS =
|
||||
"services.blocklist.addons.checked";
|
||||
const PREF_BLOCKLIST_ADDONS_SIGNER = "services.blocklist.addons.signer";
|
||||
// Blocklist v3 - MLBF format.
|
||||
const PREF_BLOCKLIST_ADDONS3_COLLECTION =
|
||||
"services.blocklist.addons-mlbf.collection";
|
||||
const PREF_BLOCKLIST_ADDONS3_CHECKED_SECONDS =
|
||||
"services.blocklist.addons-mlbf.checked";
|
||||
const PREF_BLOCKLIST_ADDONS3_SIGNER = "services.blocklist.addons-mlbf.signer";
|
||||
|
||||
const BlocklistTelemetry = {
|
||||
/**
|
||||
|
|
@ -1060,6 +1075,9 @@ this.PluginBlocklistRS = {
|
|||
* "last_modified": 1480349215672,
|
||||
* }
|
||||
*
|
||||
* This is a legacy format, and implements deprecated operations (bug 1620580).
|
||||
* ExtensionBlocklistMLBF supersedes this implementation.
|
||||
*
|
||||
* Note: we assign to the global to allow tests to reach the object directly.
|
||||
*/
|
||||
this.ExtensionBlocklistRS = {
|
||||
|
|
@ -1141,6 +1159,15 @@ this.ExtensionBlocklistRS = {
|
|||
shutdown() {
|
||||
if (this._client) {
|
||||
this._client.off("sync", this._onUpdate);
|
||||
this._didShutdown = true;
|
||||
}
|
||||
},
|
||||
|
||||
// Called when the blocklist implementation is changed via a pref.
|
||||
undoShutdown() {
|
||||
if (this._didShutdown) {
|
||||
this._client.on("sync", this._onUpdate);
|
||||
this._didShutdown = false;
|
||||
}
|
||||
},
|
||||
|
||||
|
|
@ -1286,6 +1313,233 @@ this.ExtensionBlocklistRS = {
|
|||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* The extensions blocklist implementation, the third version.
|
||||
*
|
||||
* The current blocklist is represented by a multi-level bloom filter (MLBF)
|
||||
* (aka "Cascade Bloom Filter") that works like a set, i.e. supports a has()
|
||||
* operation, except it is probabilistic. The MLBF is 100% accurate for known
|
||||
* entries and unreliable for unknown entries. When the backend generates the
|
||||
* MLBF, all known add-ons are recorded, including their block state. Unknown
|
||||
* add-ons are identified by their signature date being newer than the MLBF's
|
||||
* generation time, and they are considered to not be blocked.
|
||||
*
|
||||
* Legacy blocklists used to distinguish between "soft block" and "hard block",
|
||||
* but the current blocklist only supports one type of block ("hard block").
|
||||
* After checking the blocklist states, any previous "soft blocked" addons will
|
||||
* either be (hard) blocked or unblocked based on the blocklist.
|
||||
*
|
||||
* The MLBF is attached to a RemoteSettings record, as follows:
|
||||
*
|
||||
* {
|
||||
* "generation_time": 1585692000000,
|
||||
* "attachment": { ... RemoteSettings attachment ... }
|
||||
* "attachment_type": "bloomfilter-full",
|
||||
* }
|
||||
*
|
||||
* The collection can have other records, but it should have only one
|
||||
* "bloomfilter-full" entry.
|
||||
*
|
||||
* Note: we assign to the global to allow tests to reach the object directly.
|
||||
*/
|
||||
this.ExtensionBlocklistMLBF = {
|
||||
RS_ATTACHMENT_ID: "addons-mlbf.bin",
|
||||
|
||||
async _fetchMLBF(record) {
|
||||
// |record| may be unset. In that case, the MLBF dump is used instead
|
||||
// (provided that the client has been built with it included).
|
||||
let hash = record?.attachment.hash;
|
||||
if (this._mlbfData && hash && this._mlbfData.cascadeHash === hash) {
|
||||
// Not changed, let's re-use it.
|
||||
return this._mlbfData;
|
||||
}
|
||||
const {
|
||||
buffer,
|
||||
record: actualRecord,
|
||||
} = await this._client.attachments.download(record, {
|
||||
attachmentId: this.RS_ATTACHMENT_ID,
|
||||
useCache: true,
|
||||
fallbackToCache: true,
|
||||
fallbackToDump: true,
|
||||
});
|
||||
return {
|
||||
cascadeHash: actualRecord.attachment.hash,
|
||||
cascadeFilter: new CascadeFilter(new Uint8Array(buffer)),
|
||||
// Note: generation_time is semantically distinct from last_modified.
|
||||
// generation_time is compared with the signing date of the add-on, so it
|
||||
// should be in sync with the signing service's clock.
|
||||
// In contrast, last_modified does not have such strong requirements.
|
||||
generationTime: actualRecord.generation_time,
|
||||
};
|
||||
},
|
||||
|
||||
async _updateMLBF(forceUpdate = false) {
|
||||
// The update process consists of fetching the collection, followed by
|
||||
// potentially multiple network requests. As long as the collection has not
|
||||
// been changed, repeated update requests can be coalesced. But when the
|
||||
// collection has been updated, all pending update requests should await the
|
||||
// new update request instead of the previous one.
|
||||
if (!forceUpdate && this._updatePromise) {
|
||||
return this._updatePromise;
|
||||
}
|
||||
const isUpdateReplaced = () => this._updatePromise != updatePromise;
|
||||
const updatePromise = (async () => {
|
||||
if (!gBlocklistEnabled) {
|
||||
this._mlbfData = null;
|
||||
return;
|
||||
}
|
||||
let records = await this._client.get();
|
||||
if (isUpdateReplaced()) {
|
||||
return;
|
||||
}
|
||||
|
||||
let mlbfRecord = records.find(
|
||||
r => r.attachment_type == "bloomfilter-full" && r.attachment
|
||||
);
|
||||
|
||||
let mlbf = await this._fetchMLBF(mlbfRecord);
|
||||
// When a MLBF dump is packaged with the browser, mlbf will always be
|
||||
// non-null at this point.
|
||||
if (isUpdateReplaced()) {
|
||||
return;
|
||||
}
|
||||
this._mlbfData = mlbf;
|
||||
})()
|
||||
.catch(e => {
|
||||
Cu.reportError(e);
|
||||
})
|
||||
.then(() => {
|
||||
if (!isUpdateReplaced()) {
|
||||
this._updatePromise = null;
|
||||
}
|
||||
return this._updatePromise;
|
||||
});
|
||||
this._updatePromise = updatePromise;
|
||||
return updatePromise;
|
||||
},
|
||||
|
||||
ensureInitialized() {
|
||||
if (!gBlocklistEnabled || this._initialized) {
|
||||
return;
|
||||
}
|
||||
this._initialized = true;
|
||||
this._client = RemoteSettings(
|
||||
Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS3_COLLECTION),
|
||||
{
|
||||
bucketNamePref: PREF_BLOCKLIST_BUCKET,
|
||||
lastCheckTimePref: PREF_BLOCKLIST_ADDONS3_CHECKED_SECONDS,
|
||||
signerName: Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS3_SIGNER),
|
||||
}
|
||||
);
|
||||
this._onUpdate = this._onUpdate.bind(this);
|
||||
this._client.on("sync", this._onUpdate);
|
||||
},
|
||||
|
||||
shutdown() {
|
||||
if (this._client) {
|
||||
this._client.off("sync", this._onUpdate);
|
||||
this._didShutdown = true;
|
||||
}
|
||||
},
|
||||
|
||||
// Called when the blocklist implementation is changed via a pref.
|
||||
undoShutdown() {
|
||||
if (this._didShutdown) {
|
||||
this._client.on("sync", this._onUpdate);
|
||||
this._didShutdown = false;
|
||||
}
|
||||
},
|
||||
|
||||
async _onUpdate() {
|
||||
this.ensureInitialized();
|
||||
await this._updateMLBF(true);
|
||||
|
||||
// Check add-ons from XPIProvider.
|
||||
const types = ["extension", "theme", "locale", "dictionary"];
|
||||
let addons = await AddonManager.getAddonsByTypes(types);
|
||||
for (let addon of addons) {
|
||||
let oldState = addon.blocklistState;
|
||||
await addon.updateBlocklistState(false);
|
||||
let state = addon.blocklistState;
|
||||
|
||||
LOG(
|
||||
"Blocklist state for " +
|
||||
addon.id +
|
||||
" changed from " +
|
||||
oldState +
|
||||
" to " +
|
||||
state
|
||||
);
|
||||
|
||||
// We don't want to re-warn about add-ons
|
||||
if (state == oldState) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ensure that softDisabled is false if the add-on is not soft blocked
|
||||
// (by a previous implementation of the blocklist).
|
||||
if (state != Ci.nsIBlocklistService.STATE_SOFTBLOCKED) {
|
||||
addon.softDisabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
AddonManagerPrivate.updateAddonAppDisabledStates();
|
||||
},
|
||||
|
||||
async getState(addon) {
|
||||
let state = await this.getEntry(addon);
|
||||
return state ? state.state : Ci.nsIBlocklistService.STATE_NOT_BLOCKED;
|
||||
},
|
||||
|
||||
async getEntry(addon) {
|
||||
if (!this._mlbfData) {
|
||||
this.ensureInitialized();
|
||||
await this._updateMLBF(false);
|
||||
}
|
||||
|
||||
let blockKey = addon.id + ":" + addon.version;
|
||||
|
||||
if (!addon.signedState) {
|
||||
// The MLBF does not apply to unsigned add-ons.
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!this._mlbfData) {
|
||||
// This could happen in theory in any of the following cases:
|
||||
// - the blocklist is disabled.
|
||||
// - The RemoteSettings backend served a malformed MLBF.
|
||||
// - The RemoteSettings backend is unreachable, and this client was built
|
||||
// without including a dump of the MLBF.
|
||||
//
|
||||
// ... in other words, this shouldn't happen in practice.
|
||||
return null;
|
||||
}
|
||||
let { cascadeFilter, generationTime } = this._mlbfData;
|
||||
if (!cascadeFilter.has(blockKey)) {
|
||||
// Add-on not blocked or unknown.
|
||||
return null;
|
||||
}
|
||||
// Add-on blocked, or unknown add-on inadvertently labeled as blocked.
|
||||
|
||||
if (addon.signedDate > generationTime) {
|
||||
// The bloom filter only reports 100% accurate results for known add-ons.
|
||||
// Since the add-on was unknown when the bloom filter was generated, the
|
||||
// block decision is incorrect and should be treated as unblocked.
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
state: Ci.nsIBlocklistService.STATE_BLOCKED,
|
||||
url: this.createBlocklistURL(addon.id, addon.version),
|
||||
};
|
||||
},
|
||||
|
||||
createBlocklistURL(id, version) {
|
||||
let url = Services.urlFormatter.formatURLPref(PREF_BLOCKLIST_ADDONITEM_URL);
|
||||
return url.replace(/%addonID%/g, id).replace(/%addonVersion%/g, version);
|
||||
},
|
||||
};
|
||||
|
||||
const EXTENSION_BLOCK_FILTERS = [
|
||||
"id",
|
||||
"name",
|
||||
|
|
@ -1378,6 +1632,7 @@ let Blocklist = {
|
|||
Services.prefs.getIntPref(PREF_BLOCKLIST_LEVEL, DEFAULT_LEVEL),
|
||||
MAX_BLOCK_LEVEL
|
||||
);
|
||||
this._chooseExtensionBlocklistImplementationFromPref();
|
||||
Services.prefs.addObserver("extensions.blocklist.", this);
|
||||
Services.prefs.addObserver(PREF_EM_LOGGING_ENABLED, this);
|
||||
|
||||
|
|
@ -1398,7 +1653,7 @@ let Blocklist = {
|
|||
shutdown() {
|
||||
GfxBlocklistRS.shutdown();
|
||||
PluginBlocklistRS.shutdown();
|
||||
ExtensionBlocklistRS.shutdown();
|
||||
this.ExtensionBlocklist.shutdown();
|
||||
|
||||
Services.obs.removeObserver(this, "xpcom-shutdown");
|
||||
Services.prefs.removeObserver("extensions.blocklist.", this);
|
||||
|
|
@ -1432,6 +1687,15 @@ let Blocklist = {
|
|||
);
|
||||
this._blocklistUpdated();
|
||||
break;
|
||||
case PREF_BLOCKLIST_USE_MLBF:
|
||||
let oldImpl = this.ExtensionBlocklist;
|
||||
this._chooseExtensionBlocklistImplementationFromPref();
|
||||
if (oldImpl._initialized) {
|
||||
oldImpl.shutdown();
|
||||
this.ExtensionBlocklist.undoShutdown();
|
||||
this.ExtensionBlocklist._onUpdate();
|
||||
} // else neither has been initialized yet. Wait for it to happen.
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
@ -1440,7 +1704,7 @@ let Blocklist = {
|
|||
loadBlocklistAsync() {
|
||||
// Need to ensure we notify gfx of new stuff.
|
||||
GfxBlocklistRS.checkForEntries();
|
||||
ExtensionBlocklistRS.ensureInitialized();
|
||||
this.ExtensionBlocklist.ensureInitialized();
|
||||
PluginBlocklistRS.ensureInitialized();
|
||||
},
|
||||
|
||||
|
|
@ -1453,15 +1717,25 @@ let Blocklist = {
|
|||
},
|
||||
|
||||
getAddonBlocklistState(addon, appVersion, toolkitVersion) {
|
||||
return ExtensionBlocklistRS.getState(addon, appVersion, toolkitVersion);
|
||||
// NOTE: appVersion/toolkitVersion are only used by ExtensionBlocklistRS.
|
||||
return this.ExtensionBlocklist.getState(addon, appVersion, toolkitVersion);
|
||||
},
|
||||
|
||||
getAddonBlocklistEntry(addon, appVersion, toolkitVersion) {
|
||||
return ExtensionBlocklistRS.getEntry(addon, appVersion, toolkitVersion);
|
||||
// NOTE: appVersion/toolkitVersion are only used by ExtensionBlocklistRS.
|
||||
return this.ExtensionBlocklist.getEntry(addon, appVersion, toolkitVersion);
|
||||
},
|
||||
|
||||
_chooseExtensionBlocklistImplementationFromPref() {
|
||||
if (Services.prefs.getBoolPref(PREF_BLOCKLIST_USE_MLBF, false)) {
|
||||
this.ExtensionBlocklist = ExtensionBlocklistMLBF;
|
||||
} else {
|
||||
this.ExtensionBlocklist = ExtensionBlocklistRS;
|
||||
}
|
||||
},
|
||||
|
||||
_blocklistUpdated() {
|
||||
ExtensionBlocklistRS._onUpdate();
|
||||
this.ExtensionBlocklist._onUpdate();
|
||||
PluginBlocklistRS._onUpdate();
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ const PROP_JSON_FIELDS = [
|
|||
"targetApplications",
|
||||
"targetPlatforms",
|
||||
"signedState",
|
||||
"signedDate",
|
||||
"seen",
|
||||
"dependencies",
|
||||
"incognito",
|
||||
|
|
@ -1363,7 +1364,7 @@ function defineAddonWrapperProperty(name, getter) {
|
|||
});
|
||||
});
|
||||
|
||||
["installDate", "updateDate"].forEach(function(aProp) {
|
||||
["installDate", "updateDate", "signedDate"].forEach(function(aProp) {
|
||||
defineAddonWrapperProperty(aProp, function() {
|
||||
let addon = addonFor(this);
|
||||
if (addon[aProp]) {
|
||||
|
|
@ -2984,9 +2985,13 @@ this.XPIDatabaseReconcile = {
|
|||
|
||||
let checkSigning =
|
||||
aOldAddon.signedState === undefined && SIGNED_TYPES.has(aOldAddon.type);
|
||||
// signedDate must be set if signedState is set.
|
||||
let signedDateMissing =
|
||||
aOldAddon.signedDate === undefined &&
|
||||
(aOldAddon.signedState || checkSigning);
|
||||
|
||||
let manifest = null;
|
||||
if (checkSigning || aReloadMetadata) {
|
||||
if (checkSigning || aReloadMetadata || signedDateMissing) {
|
||||
try {
|
||||
manifest = XPIInstall.syncLoadManifest(aAddonState, aLocation);
|
||||
} catch (err) {
|
||||
|
|
@ -3003,6 +3008,10 @@ this.XPIDatabaseReconcile = {
|
|||
aOldAddon.signedState = manifest.signedState;
|
||||
}
|
||||
|
||||
if (signedDateMissing) {
|
||||
aOldAddon.signedDate = manifest.signedDate;
|
||||
}
|
||||
|
||||
// May be updating from a version of the app that didn't support all the
|
||||
// properties of the currently-installed add-ons.
|
||||
if (aReloadMetadata) {
|
||||
|
|
|
|||
|
|
@ -678,6 +678,7 @@ var loadManifest = async function(aPackage, aLocation, aOldAddon) {
|
|||
|
||||
let { signedState, cert } = await aPackage.verifySignedState(addon);
|
||||
addon.signedState = signedState;
|
||||
addon.signedDate = cert?.validity?.notBefore / 1000 || null;
|
||||
if (!addon.isPrivileged) {
|
||||
addon.hidden = false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -460,6 +460,7 @@ const JSON_FIELDS = Object.freeze([
|
|||
"rootURI",
|
||||
"runInSafeMode",
|
||||
"signedState",
|
||||
"signedDate",
|
||||
"startupData",
|
||||
"telemetryKey",
|
||||
"type",
|
||||
|
|
@ -551,6 +552,7 @@ class XPIState {
|
|||
rootURI: this.rootURI,
|
||||
runInSafeMode: this.runInSafeMode,
|
||||
signedState: this.signedState,
|
||||
signedDate: this.signedDate,
|
||||
telemetryKey: this.telemetryKey,
|
||||
version: this.version,
|
||||
};
|
||||
|
|
@ -639,6 +641,7 @@ class XPIState {
|
|||
this.dependencies = aDBAddon.dependencies;
|
||||
this.runInSafeMode = canRunInSafeMode(aDBAddon);
|
||||
this.signedState = aDBAddon.signedState;
|
||||
this.signedDate = aDBAddon.signedDate;
|
||||
this.file = aDBAddon._sourceBundle;
|
||||
this.rootURI = aDBAddon.rootURI;
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче