Backed out 2 changesets (bug 1733481, bug 1738345) for causing failures at test_remote_settings_utils_telemetry.js. CLOSED TREE

Backed out changeset 48dc0b288686 (bug 1738345)
Backed out changeset a23df06197e1 (bug 1733481)
This commit is contained in:
Butkovits Atila 2021-12-23 20:29:41 +02:00
Родитель 0f6b43b33a
Коммит 408d492613
20 изменённых файлов: 839 добавлений и 540 удалений

Просмотреть файл

@ -11,6 +11,7 @@ DIRS += [
"webcompat",
"report-site-issue",
"pictureinpicture",
"proxy-failover",
"search-detection",
]

Просмотреть файл

@ -0,0 +1,659 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
/* globals ExtensionAPI, Services, XPCOMUtils, WebExtensionPolicy */
XPCOMUtils.defineLazyGlobalGetters(this, ["ChannelWrapper"]);
XPCOMUtils.defineLazyServiceGetter(
this,
"ProxyService",
"@mozilla.org/network/protocol-proxy-service;1",
"nsIProtocolProxyService"
);
XPCOMUtils.defineLazyServiceGetter(
this,
"NSSErrorsService",
"@mozilla.org/nss_errors_service;1",
"nsINSSErrorsService"
);
XPCOMUtils.defineLazyModuleGetters(this, {
ExtensionParent: "resource://gre/modules/ExtensionParent.jsm",
ExtensionPreferencesManager:
"resource://gre/modules/ExtensionPreferencesManager.jsm",
});
XPCOMUtils.defineLazyGetter(
this,
"Management",
() => ExtensionParent.apiManager
);
const PROXY_DIRECT = "direct";
const DISABLE_HOURS = 48;
const MAX_DISABLED_PI = 2;
const PREF_MONITOR_DATA = "extensions.proxyMonitor.state";
const PREF_MONITOR_LOGGING = "extensions.proxyMonitor.logging.enabled";
const PREF_PROXY_FAILOVER = "network.proxy.failover_direct";
const CHECK_EXTENSION_ONLY =
Services.vc.compare(Services.appinfo.version, "92.0") >= 0;
const PROXY_CONFIG_TYPES = [
"direct",
"manual",
"pac",
"unused", // nsIProtocolProxyService.idl skips index 3.
"wpad",
"system",
];
function hoursSince(dt2, dt1 = Date.now()) {
var diff = (dt2 - dt1) / 1000;
diff /= 60 * 60;
return Math.abs(Math.round(diff));
}
const DEBUG_LOG = Services.prefs.getBoolPref(PREF_MONITOR_LOGGING, true);
function log(msg) {
if (DEBUG_LOG) {
console.log(`proxy-monitor: ${msg}`);
}
}
/**
* ProxyMonitor monitors system and protected requests for failures due to bad
* or unavailable proxy configurations.
*
* In a system with multiple layers of proxy configuration, if there is a
* failing proxy we try to remove just that confuration from the chain.
* However if we get too many failures, we'll make a direct connection the top
* "proxy".
*
* 1. Any proxied system request without a direct failover will have one added.
*
* 2. If a proxied system request fails, the proxy configuration in use will be
* disabled. On later requests, disabled proxies are removed from the proxy
* chain. Disabled proxy configurations remain disabled for 48 hours to allow
* any necessary requests to operate for a period of time. When disabled
* proxies are used as a failover to a direct request (step 3 or 4 below), the
* proxy can be detected as functional and be re-enabled despite not having
* reached the 48 hours.
*
* 3. If too many proxy configurations got disabled, we make a direct config
* first with failover to all other proxy configurations (essentially skipping
* step 2). This state remains for 48 hours before retrying without "direct".
*
* 4. If we've removed all proxies we make a direct config first and failover
* to the other proxy configurations, similar to step 3.
*
* 5. Starting with Fx92, we will only disable proxy configurations provided by
* extensions. Prior to 92, we could not definitively identify extensions from
* the proxyInfo instance.
*
* If we've disabled proxies, we continue to watch the requests for failures in
* "direct" connection mode. If we continue to fail with direct connections,
* we fall back to allowing proxies again.
*/
const ProxyMonitor = {
errors: new Map(),
extensions: new Map(),
disabledTime: 0,
newDirectProxyInfo(failover = null) {
return ProxyService.newProxyInfo(
PROXY_DIRECT,
"",
0,
"",
"",
0,
0,
failover
);
},
async applyFilter(channel, defaultProxyInfo, proxyFilter) {
let proxyInfo = defaultProxyInfo;
// onProxyFilterResult must be called, so we wrap in a try/finally.
try {
if (!proxyInfo) {
// If no proxy is in use, exit early.
return;
}
// If this is not a system request we will allow existing
// proxy behavior.
if (!channel.loadInfo?.loadingPrincipal?.isSystemPrincipal) {
return;
}
// We monitor for successful connections which in some cases may
// re-enable a prior failed proxy configuration.
let wrapper = ChannelWrapper.get(channel);
wrapper.addEventListener("start", this);
if (this.tooManyFailures()) {
log(`too many proxy config failures, prepend direct rid ${wrapper.id}`);
// A lot of failures are happening. Try direct first, but failover to
// any non-extension proxies "just in case".
proxyInfo = this.newDirectProxyInfo(
await this.pruneExtensions(defaultProxyInfo)
);
return;
}
this.dumpProxies(proxyInfo, `starting proxyInfo rid ${wrapper.id}`);
proxyInfo = this.pruneProxyInfo(proxyInfo);
if (!proxyInfo) {
// All current proxies are disabled due to prior failures. Try direct
// first, but failover to any non-extension proxies "just in case".
log(`all proxies disabled, prepend direct`);
proxyInfo = this.newDirectProxyInfo(
await this.pruneExtensions(defaultProxyInfo)
);
return;
}
// If we are not attempting a direct bypass we want to monitor for
// non-connection errors such as invalid proxy servers.
wrapper.addEventListener("error", this);
// A little debug output
this.dumpProxies(proxyInfo, `pruned proxyInfo rid ${wrapper.id}`);
} finally {
// This must be called.
proxyFilter.onProxyFilterResult(proxyInfo);
}
},
relinkProxyInfoChain(proxies) {
if (!proxies.length) {
return null;
}
// Re-link the proxy chain.
// failoverProxy cannot be set to undefined or null, we fixup the last
// failover with a direct failover if necessary.
for (let i = 0; i < proxies.length - 2; i++) {
proxies[i].failoverProxy = proxies[i + 1];
}
let top = proxies[0];
let last = proxies.pop();
// Ensure the last proxy is not linked to something we removed. This
// catches connection failures such as those to non-existant or non-http
// ports. The "error" handler added above catches http connections that
// are not proxy servers.
if (last.failoverProxy || last.type != PROXY_DIRECT) {
last.failoverProxy = this.newDirectProxyInfo();
}
return top;
},
async pruneExtensions(proxyInfo) {
// If an extension controls the settings, we must assume that all PIs
// came from the extension.
let extensionId = await this.getControllingExtension();
if (extensionId) {
return null;
}
let enabledProxies = [];
let pi = proxyInfo;
while (pi) {
if (!pi.sourceId) {
enabledProxies.push(pi);
}
pi = pi.failoverProxy;
}
return this.relinkProxyInfoChain(enabledProxies);
},
// Verify the entire proxy failover chain is clean. There may be multiple
// sources for proxyInfo in the chain, so we remove any disabled entries and
// continue to use configurations that have not yet failed.
pruneProxyInfo(proxyInfo) {
let enabledProxies = [];
let pi = proxyInfo;
while (pi) {
if (!this.proxyDisabled(pi)) {
enabledProxies.push(pi);
}
pi = pi.failoverProxy;
}
return this.relinkProxyInfoChain(enabledProxies);
},
dumpProxies(proxyInfo, msg) {
if (!DEBUG_LOG) {
return;
}
log(msg);
let pi = proxyInfo;
while (pi) {
log(` ${pi.type}:${pi.host}:${pi.port}`);
pi = pi.failoverProxy;
}
},
tooManyFailures() {
// If we have lots of PIs that are failing in a short period of time then
// we back off proxy for a while.
if (this.disabledTime && hoursSince(this.disabledTime) >= DISABLE_HOURS) {
this.recordEvent("timeout", "proxyBypass", "global");
this.reset();
}
return !!this.disabledTime;
},
proxyDisabled(proxyInfo) {
let key = this.getProxyInfoKey(proxyInfo);
if (!key) {
return false;
}
// From 92 forward, if an extension has one disabled PI, we disable all PIs
// from that extension for the DISABLE_HOURS perid.
let extTime = proxyInfo.sourceId && this.extensions.get(proxyInfo.sourceId);
if (extTime && hoursSince(extTime) <= DISABLE_HOURS) {
return true;
}
let err = this.errors.get(key);
if (!err) {
return false;
}
// We keep a proxy config disabled for DISABLE_HOURS to give our daily
// update checks time to complete again.
if (hoursSince(err.time) >= DISABLE_HOURS) {
this.errors.delete(key);
this.logProxySource("timeout", proxyInfo);
return false;
}
// This is harsh, but these requests are too important.
return true;
},
getProxyInfoKey(proxyInfo) {
if (!proxyInfo || proxyInfo.type == PROXY_DIRECT) {
return;
}
let { type, host, port } = proxyInfo;
// eslint-disable-next-line consistent-return
return `${type}:${host}:${port}`;
},
// If proxy.settings is used to change the proxy, an extension will be "in
// control". This returns the id of that extension.
async getControllingExtension() {
// Is this proxied by an extension that set proxy prefs?
let setting = await ExtensionPreferencesManager.getSetting(
"proxy.settings"
);
return setting?.id;
},
async getProxySource(proxyInfo) {
// sourceId is set when using proxy.onRequest
if (proxyInfo.sourceId) {
return {
source: proxyInfo.sourceId,
type: "api",
};
}
let type = PROXY_CONFIG_TYPES[ProxyService.proxyConfigType] || "unknown";
// If we have a policy it will have set the prefs.
if (Services.policies.status === Services.policies.ACTIVE) {
let policies = Services.policies
.getActivePolicies()
?.filter(p => p.Proxy);
if (policies?.length) {
return {
source: "policy",
type,
};
}
}
let source = await this.getControllingExtension();
return {
source: source || "prefs",
type,
};
},
async logProxySource(state, proxyInfo) {
let { source, type } = await this.getProxySource(proxyInfo);
this.recordEvent(state, "proxyInfo", type, { source });
},
recordEvent(method, obj, type = null, source = {}) {
try {
Services.telemetry.recordEvent("proxyMonitor", method, obj, type, source);
log(`event: ${method} ${obj} ${type} ${JSON.stringify(source)}`);
} catch (err) {
// If the telemetry throws just log the error so it doesn't break any
// functionality.
Cu.reportError(err);
}
},
timeoutEntries() {
// remove old entries
for (let [k, err] of this.errors) {
if (hoursSince(err.time) >= DISABLE_HOURS) {
this.errors.delete(k);
this.recordEvent("timeout", "proxyInfo");
}
}
for (let [e, t] of this.extensions) {
if (hoursSince(t) >= DISABLE_HOURS) {
this.extensions.delete(e);
// Not a full bypass, but an extension bypass
this.recordEvent("timeout", "proxyBypass", "extension", { source: e });
}
}
},
async disableProxyInfo(proxyInfo) {
this.dumpProxies(proxyInfo, "disableProxyInfo");
let key = this.getProxyInfoKey(proxyInfo);
if (!key) {
log(`direct request failure`);
return;
}
// From 92 forward, we disable all extension provided proxies if one fails
let extensionId;
if (CHECK_EXTENSION_ONLY) {
extensionId =
proxyInfo.sourceId || (await this.getControllingExtension());
}
this.timeoutEntries();
let err = { time: Date.now(), extensionId };
this.errors.set(key, err);
if (extensionId) {
this.extensions.set(extensionId, err.time);
log(`all proxy configuration from extension ${extensionId} disabled`);
this.recordEvent("start", "proxyBypass", "extension", {
source: extensionId,
});
}
this.logProxySource("disabled", proxyInfo);
// If lots of proxies have failed, we
// disable all proxies for a while to ensure system
// requests have the best oportunity to get
// through.
if (!this.disabledTime && this.errors.size >= MAX_DISABLED_PI) {
this.disabledTime = Date.now();
this.recordEvent("start", "proxyBypass", "global");
}
},
async enableProxyInfo(proxyInfo) {
let key = this.getProxyInfoKey(proxyInfo);
if (!key) {
return;
}
if (this.errors.delete(key)) {
this.logProxySource("enabled", proxyInfo);
}
// From 92 forward, we have tracked extensions. If no keys are disabled,
// remove the extension from the disabled list.
if (!CHECK_EXTENSION_ONLY) {
return;
}
let extensionId =
proxyInfo.sourceId || (await this.getControllingExtension());
if (!extensionId) {
return;
}
// Only delete if no err entries with the id exists.
// eslint-disable-next-line no-unused-vars
for (let [k, err] of this.errors) {
if (err.extensionId == extensionId) {
return;
}
}
this.extensions.delete(extensionId);
},
tlsCheck(channel) {
let securityInfo = channel.securityInfo;
if (!securityInfo) {
return false;
}
securityInfo.QueryInterface(Ci.nsITransportSecurityInfo);
if (NSSErrorsService.isNSSErrorCode(securityInfo.errorCode)) {
return false;
}
const wpl = Ci.nsIWebProgressListener;
const state = securityInfo.securityState;
return !!(state & wpl.STATE_IS_SECURE);
},
handleEvent(event) {
let wrapper = event.currentTarget; // channel wrapper
let { channel } = wrapper;
if (!(channel instanceof Ci.nsIProxiedChannel)) {
log(`got ${event.type} event but not a proxied channel`);
return;
}
// If this is an http request ignore it, it is too easily tampered with.
// Fortunately its use is limited, potentially only captive portal.
if (wrapper.finalURL.startsWith("http:")) {
return;
}
// The tls handshake must succeed to re-enable a request.
let tlsIsSecure = this.tlsCheck(channel);
log(
`request event ${event.type} rid ${wrapper.id} status ${wrapper.statusCode} tls ${tlsIsSecure} for ${channel.URI.spec}`
);
let status = wrapper.statusCode;
switch (event.type) {
case "error":
if (!tlsIsSecure || status == 0) {
this.disableProxyInfo(channel.proxyInfo);
}
break;
case "start":
if (tlsIsSecure && status >= 200 && status < 400) {
this.enableProxyInfo(channel.proxyInfo);
}
break;
default:
break;
}
},
reset() {
this.disabledTime = 0;
this.errors = new Map();
},
store() {
if (!this.disabledTime && !this.errors.size) {
Services.prefs.clearUserPref(PREF_MONITOR_DATA);
return;
}
let data = JSON.stringify({
disabledTime: this.disabledTime,
errors: Array.from(this.errors),
});
Services.prefs.setStringPref(PREF_MONITOR_DATA, data);
},
restore() {
let failovers = Services.prefs.getStringPref(PREF_MONITOR_DATA, null);
if (failovers) {
failovers = JSON.parse(failovers);
this.disabledTime = failovers.disabledTime;
this.errors = new Map(failovers.errors);
this.extensions = new Map(
failovers.errors
.filter(e => e[1].extensionId)
.sort((a, b) => a[1].time - b[1].time)
.map(e => [e[1].extensionId, e[1].time])
);
} else {
this.disabledTime = 0;
this.errors = new Map();
this.extensions = new Map();
}
},
startup() {
// Register filter with a very high position, this will sort to the last
// filter called.
ProxyService.registerChannelFilter(ProxyMonitor, Number.MAX_SAFE_INTEGER);
this.restore();
log("started");
},
shutdown() {
ProxyService.unregisterFilter(ProxyMonitor);
this.store();
log("stopped");
},
};
/**
* Listen for changes in addons and pref to start or stop the ProxyMonitor.
*/
const monitor = {
running: false,
startup() {
if (!this.failoverEnabled) {
return;
}
Management.on("startup", this.handleEvent);
Management.on("shutdown", this.handleEvent);
Management.on("change-permissions", this.handleEvent);
if (this.hasProxyExtension()) {
monitor.startMonitors();
}
},
shutdown() {
Management.off("startup", this.handleEvent);
Management.off("shutdown", this.handleEvent);
Management.off("change-permissions", this.handleEvent);
monitor.stopMonitors();
},
get failoverEnabled() {
return Services.prefs.getBoolPref(PREF_PROXY_FAILOVER, true);
},
observe() {
if (monitor.failoverEnabled) {
monitor.startup();
} else {
monitor.shutdown();
}
},
startMonitors() {
if (!monitor.running) {
ProxyMonitor.startup();
monitor.running = true;
}
},
stopMonitors() {
if (monitor.running) {
ProxyMonitor.shutdown();
monitor.running = false;
}
},
hasProxyExtension(ignore) {
for (let policy of WebExtensionPolicy.getActiveExtensions()) {
if (
policy.id != ignore &&
!policy.extension?.isAppProvided &&
policy.hasPermission("proxy")
) {
return true;
}
}
return false;
},
handleEvent(kind, ...args) {
switch (kind) {
case "startup": {
let [extension] = args;
if (
!monitor.running &&
!extension.isAppProvided &&
extension.hasPermission("proxy")
) {
monitor.startMonitors();
}
break;
}
case "shutdown": {
if (Services.startup.shuttingDown) {
// Let normal shutdown handle things.
break;
}
let [extension] = args;
// WebExtensionPolicy is still active, pass the id to ignore it.
if (
monitor.running &&
!extension.isAppProvided &&
!monitor.hasProxyExtension(extension.id)
) {
monitor.stopMonitors();
}
break;
}
case "change-permissions": {
if (monitor.running) {
break;
}
let { extensionId, added } = args[0];
if (!added?.permissions.includes("proxy")) {
return;
}
let extension = WebExtensionPolicy.getByID(extensionId)?.extension;
if (extension && !extension.isAppProvided) {
monitor.startMonitors();
}
break;
}
}
},
};
this.failover = class extends ExtensionAPI {
onStartup() {
Services.telemetry.registerEvents("proxyMonitor", {
proxyMonitor: {
methods: ["enabled", "disabled", "start", "timeout"],
objects: ["proxyInfo", "proxyBypass"],
extra_keys: ["source"],
record_on_release: true,
},
});
monitor.startup();
Services.prefs.addObserver(PREF_PROXY_FAILOVER, monitor);
}
onShutdown() {
monitor.shutdown();
Services.prefs.removeObserver(PREF_PROXY_FAILOVER, monitor);
}
};

Просмотреть файл

@ -0,0 +1,22 @@
{
"manifest_version": 2,
"name": "Proxy Failover",
"applications": {
"gecko": {
"id": "proxy-failover@mozilla.com",
"strict_min_version": "78.0"
}
},
"version": "1.0.2",
"description": "Direct Failover for system requests.",
"experiment_apis": {
"failover": {
"schema": "schema.json",
"parent": {
"scopes": ["addon_parent"],
"script": "api.js",
"events": ["startup"]
}
}
}
}

Просмотреть файл

@ -0,0 +1,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
DEFINES["MOZ_APP_VERSION"] = CONFIG["MOZ_APP_VERSION"]
DEFINES["MOZ_APP_MAXVERSION"] = CONFIG["MOZ_APP_MAXVERSION"]
FINAL_TARGET_FILES.features["proxy-failover@mozilla.com"] += [
"api.js",
"manifest.json",
"schema.json",
]
BROWSER_CHROME_MANIFESTS += ["tests/browser/browser.ini"]
with Files("**"):
BUG_COMPONENT = ("WebExtensions", "Request Handling")

Просмотреть файл

@ -0,0 +1,2 @@
// no json schema for proxy-failover
[]

Просмотреть файл

@ -0,0 +1,3 @@
[DEFAULT]
[browser_extension_loaded.js]

Просмотреть файл

@ -0,0 +1,16 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const { AddonManager } = ChromeUtils.import(
"resource://gre/modules/AddonManager.jsm"
);
add_task(async function test_proxyfailover_isActive() {
let addon = await AddonManager.getAddonByID("proxy-failover@mozilla.com");
ok(addon, "Add-on exists");
ok(addon.isActive, "Add-on is active");
});

Просмотреть файл

@ -9903,13 +9903,6 @@
mirror: always
#endif
# Whether to allow a bypass flag to be set on httpChannel that will
# prevent proxies from being used for that specific request.
- name: network.proxy.allow_bypass
type: bool
value: true
mirror: always
- name: network.proxy.parse_pac_on_socket_process
type: RelaxedAtomicBool
value: false

Просмотреть файл

@ -27,7 +27,6 @@
#include "mozilla/Components.h"
#include "mozilla/StaticPrefs_browser.h"
#include "mozilla/StaticPrefs_fission.h"
#include "mozilla/StaticPrefs_network.h"
#include "mozilla/StaticPrefs_security.h"
#include "mozilla/Telemetry.h"
#include "mozilla/Tokenizer.h"
@ -3302,26 +3301,17 @@ HttpBaseChannel::SetBeConservative(bool aBeConservative) {
return NS_OK;
}
bool HttpBaseChannel::BypassProxy() {
return StaticPrefs::network_proxy_allow_bypass() && LoadBypassProxy();
}
NS_IMETHODIMP
HttpBaseChannel::GetBypassProxy(bool* aBypassProxy) {
NS_ENSURE_ARG_POINTER(aBypassProxy);
*aBypassProxy = BypassProxy();
*aBypassProxy = LoadBypassProxy();
return NS_OK;
}
NS_IMETHODIMP
HttpBaseChannel::SetBypassProxy(bool aBypassProxy) {
if (StaticPrefs::network_proxy_allow_bypass()) {
StoreBypassProxy(aBypassProxy);
} else {
NS_WARNING("bypassProxy set but network.proxy.allow_bypass is disabled");
return NS_ERROR_FAILURE;
}
StoreBypassProxy(aBypassProxy);
return NS_OK;
}

Просмотреть файл

@ -274,8 +274,6 @@ class HttpBaseChannel : public nsHashPropertyBag,
NS_IMETHOD SetBeConservative(bool aBeConservative) override;
NS_IMETHOD GetBypassProxy(bool* aBypassProxy) override;
NS_IMETHOD SetBypassProxy(bool aBypassProxy) override;
bool BypassProxy();
NS_IMETHOD GetIsTRRServiceChannel(bool* aTRR) override;
NS_IMETHOD SetIsTRRServiceChannel(bool aTRR) override;
NS_IMETHOD GetIsResolvedByTRR(bool* aResolvedByTRR) override;

Просмотреть файл

@ -2150,7 +2150,7 @@ nsresult HttpChannelChild::ContinueAsyncOpen() {
openArgs.allowHttp3() = LoadAllowHttp3();
openArgs.allowAltSvc() = LoadAllowAltSvc();
openArgs.beConservative() = LoadBeConservative();
openArgs.bypassProxy() = BypassProxy();
openArgs.bypassProxy() = LoadBypassProxy();
openArgs.tlsFlags() = mTlsFlags;
openArgs.initialRwin() = mInitialRwin;

Просмотреть файл

@ -5930,7 +5930,7 @@ void nsHttpChannel::MaybeResolveProxyAndBeginConnect() {
// settings if we are never going to make a network connection.
if (!mProxyInfo &&
!(mLoadFlags & (LOAD_ONLY_FROM_CACHE | LOAD_NO_NETWORK_IO)) &&
!BypassProxy() && NS_SUCCEEDED(ResolveProxy())) {
!LoadBypassProxy() && NS_SUCCEEDED(ResolveProxy())) {
return;
}

Просмотреть файл

@ -116,15 +116,14 @@ var Utils = {
if (
// At most one recursive Utils.fetch call (bypassProxy=false to true).
bypassProxy ||
Services.startup.shuttingDown ||
Utils.isOffline ||
!request.isProxied ||
!request.bypassProxyEnabled
) {
reject(err);
return;
}
ServiceRequest.logProxySource(request.channel, "remote-settings");
// TODO: Remove ?. when https://phabricator.services.mozilla.com/D127170 lands
ServiceRequest.logProxySource?.(request.channel, "remote-settings");
resolve(Utils.fetch(input, { ...init, bypassProxy: true }));
}

Просмотреть файл

@ -37,6 +37,36 @@ async function assertTelemetryEvents(expectedEvents) {
add_task(async function setup() {
await TelemetryController.testSetup();
const { ServiceRequest } = ChromeUtils.import(
"resource://gre/modules/ServiceRequest.jsm"
);
if (!ServiceRequest.logProxySource) {
// https://phabricator.services.mozilla.com/D127170 hasn't landed yet.
// Simulate the Events.yaml registration and logProxySource implementation.
Services.telemetry.registerBuiltinEvents("service_request", {
bypass: {
methods: ["bypass"],
objects: ["proxy_info"],
extra_keys: ["source", "type"],
record_on_release: true,
},
});
ServiceRequest.logProxySource = async (channel, service) => {
// D127170 also inspects channel, but for non-extensions the source is
// ultimately just "prefs".
Services.telemetry.setEventRecordingEnabled("service_request", true);
Services.telemetry.recordEvent(
"service_request",
"bypass",
"proxy_info",
service,
{
source: "prefs",
type: "manual",
}
);
};
}
});
add_task(async function test_telemetry() {

Просмотреть файл

@ -3065,24 +3065,3 @@ memory_watcher:
bug_numbers: [1715858]
notification_emails:
- tkikuchi@mozilla.com
service_request:
bypass:
description: >
This event is recorded by a small set of services when a proxy failure
causes a service to re-request with a proxy bypass. It records some
basic information such as the type of proxy configuration, and the source
of the proxy configuration. The value of the event is the name of the
service that triggers the event (e.g. telemetry, remote-settings).
methods: ["bypass"]
objects: ["proxy_info"]
release_channel_collection: opt-out
expiry_version: never
record_in_processes: ["main"]
products: ["firefox"]
bug_numbers: [1732792, 1732793, 1733481, 1733994, 1732388]
notification_emails:
- scaraveo@mozilla.com
extra_keys:
source: the source of the proxy configuration. e.g. policy, prefs or extension_id
type: the type for the proxy configuration source. e.g. api or string version of nsIProtocolProxyService.proxyConfigType

Просмотреть файл

@ -1308,73 +1308,6 @@ var TelemetrySendImpl = {
return "/submit/telemetry/" + slug;
},
_doPingRequest(ping, id, url, options, errorHandler, onloadHandler) {
// Don't send cookies with these requests.
let request = new ServiceRequest({ mozAnon: true });
request.mozBackgroundRequest = true;
request.timeout = Policy.pingSubmissionTimeout();
request.open("POST", url, options);
request.overrideMimeType("text/plain");
request.setRequestHeader("Content-Type", "application/json; charset=UTF-8");
request.setRequestHeader("Date", Policy.now().toUTCString());
request.setRequestHeader("Content-Encoding", "gzip");
request.onerror = errorHandler;
request.ontimeout = errorHandler;
request.onabort = errorHandler;
request.onload = onloadHandler;
this._pendingPingRequests.set(id, request);
let startTime = Utils.monotonicNow();
// If that's a legacy ping format, just send its payload.
let networkPayload = isV4PingFormat(ping) ? ping : ping.payload;
let converter = Cc[
"@mozilla.org/intl/scriptableunicodeconverter"
].createInstance(Ci.nsIScriptableUnicodeConverter);
converter.charset = "UTF-8";
let utf8Payload = converter.ConvertFromUnicode(
JSON.stringify(networkPayload)
);
utf8Payload += converter.Finish();
Services.telemetry
.getHistogramById("TELEMETRY_STRINGIFY")
.add(Utils.monotonicNow() - startTime);
let payloadStream = Cc[
"@mozilla.org/io/string-input-stream;1"
].createInstance(Ci.nsIStringInputStream);
startTime = Utils.monotonicNow();
payloadStream.data = Policy.gzipCompressString(utf8Payload);
// Check the size and drop pings which are too big.
const compressedPingSizeBytes = payloadStream.data.length;
if (compressedPingSizeBytes > TelemetryStorage.MAXIMUM_PING_SIZE) {
this._log.error(
"_doPing - submitted ping exceeds the size limit, size: " +
compressedPingSizeBytes
);
Services.telemetry
.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND")
.add();
Services.telemetry
.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB")
.add(Math.floor(compressedPingSizeBytes / 1024 / 1024));
// We don't need to call |request.abort()| as it was not sent yet.
this._pendingPingRequests.delete(id);
TelemetryHealthPing.recordDiscardedPing(ping.type);
return { promise: TelemetryStorage.removePendingPing(id) };
}
Services.telemetry
.getHistogramById("TELEMETRY_COMPRESS")
.add(Utils.monotonicNow() - startTime);
request.sendInputStream(payloadStream);
return { payloadStream };
},
_doPing(ping, id, isPersisted) {
if (!this.sendingEnabled(ping)) {
// We can't send the pings to the server, so don't try to.
@ -1405,6 +1338,18 @@ var TelemetrySendImpl = {
const url = this._buildSubmissionURL(ping);
// Don't send cookies with these requests.
let request = new ServiceRequest({ mozAnon: true });
request.mozBackgroundRequest = true;
request.timeout = Policy.pingSubmissionTimeout();
request.open("POST", url, true);
request.overrideMimeType("text/plain");
request.setRequestHeader("Content-Type", "application/json; charset=UTF-8");
request.setRequestHeader("Date", Policy.now().toUTCString());
this._pendingPingRequests.set(id, request);
const monotonicStartTime = Utils.monotonicNow();
let deferred = PromiseUtils.defer();
@ -1434,39 +1379,7 @@ var TelemetrySendImpl = {
);
};
let retryRequest = request => {
if (
this._shutdown ||
ServiceRequest.isOffline ||
Services.startup.shuttingDown ||
!request.bypassProxyEnabled ||
this._tooLateToSend ||
request.bypassProxy ||
!request.isProxied
) {
return false;
}
ServiceRequest.logProxySource(request.channel, "telemetry.send");
// If the request failed, and it's using a proxy, automatically
// attempt without proxy.
let { payloadStream } = this._doPingRequest(
ping,
id,
url,
{ bypassProxy: true },
errorHandler,
onloadHandler
);
this.payloadStream = payloadStream;
return true;
};
let errorHandler = event => {
let request = event.target;
if (retryRequest(request)) {
return;
}
let errorhandler = event => {
let failure = event.type;
if (failure === "error") {
failure = XHR_ERROR_TYPE[request.errorCode];
@ -1474,6 +1387,23 @@ var TelemetrySendImpl = {
TelemetryHealthPing.recordSendFailure(failure);
if (this.fallbackHttp) {
// only one attempt
this.fallbackHttp = false;
request.channel.securityInfo
.QueryInterface(Ci.nsITransportSecurityInfo)
.QueryInterface(Ci.nsISerializable);
if (request.channel.securityInfo.errorCodeString.startsWith("SEC_")) {
// re-open the request with the HTTP version of the URL
let fallbackUrl = new URL(url);
fallbackUrl.protocol = "http:";
// TODO encrypt payload
request.open("POST", fallbackUrl, true);
request.sendInputStream(this.payloadStream);
}
}
Services.telemetry
.getHistogramById("TELEMETRY_SEND_FAILURE_TYPE")
.add(failure);
@ -1486,9 +1416,11 @@ var TelemetrySendImpl = {
);
onRequestFinished(false, event);
};
request.onerror = errorhandler;
request.ontimeout = errorhandler;
request.onabort = errorhandler;
let onloadHandler = event => {
let request = event.target;
request.onload = event => {
let status = request.status;
let statusClass = status - (status % 100);
let success = false;
@ -1532,24 +1464,57 @@ var TelemetrySendImpl = {
event.type
);
}
if (!success && retryRequest(request)) {
return;
}
onRequestFinished(success, event);
};
let { payloadStream, promise } = this._doPingRequest(
ping,
id,
url,
{},
errorHandler,
onloadHandler
// If that's a legacy ping format, just send its payload.
let networkPayload = isV4PingFormat(ping) ? ping : ping.payload;
request.setRequestHeader("Content-Encoding", "gzip");
let converter = Cc[
"@mozilla.org/intl/scriptableunicodeconverter"
].createInstance(Ci.nsIScriptableUnicodeConverter);
converter.charset = "UTF-8";
let startTime = Utils.monotonicNow();
let utf8Payload = converter.ConvertFromUnicode(
JSON.stringify(networkPayload)
);
if (promise) {
return promise;
utf8Payload += converter.Finish();
Services.telemetry
.getHistogramById("TELEMETRY_STRINGIFY")
.add(Utils.monotonicNow() - startTime);
let payloadStream = Cc[
"@mozilla.org/io/string-input-stream;1"
].createInstance(Ci.nsIStringInputStream);
startTime = Utils.monotonicNow();
payloadStream.data = Policy.gzipCompressString(utf8Payload);
// Check the size and drop pings which are too big.
const compressedPingSizeBytes = payloadStream.data.length;
if (compressedPingSizeBytes > TelemetryStorage.MAXIMUM_PING_SIZE) {
this._log.error(
"_doPing - submitted ping exceeds the size limit, size: " +
compressedPingSizeBytes
);
Services.telemetry
.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND")
.add();
Services.telemetry
.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB")
.add(Math.floor(compressedPingSizeBytes / 1024 / 1024));
// We don't need to call |request.abort()| as it was not sent yet.
this._pendingPingRequests.delete(id);
TelemetryHealthPing.recordDiscardedPing(ping.type);
return TelemetryStorage.removePendingPing(id);
}
Services.telemetry
.getHistogramById("TELEMETRY_COMPRESS")
.add(Utils.monotonicNow() - startTime);
request.sendInputStream(payloadStream);
this.payloadStream = payloadStream;
return deferred.promise;

Просмотреть файл

@ -52,10 +52,6 @@ const PingServer = {
return this._httpServer.identity.primaryPort;
},
get host() {
return this._httpServer.identity.primaryHost;
},
get started() {
return this._started;
},

Просмотреть файл

@ -1,256 +0,0 @@
/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */
/* vim: set sts=2 sw=2 et tw=80: */
"use strict";
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
const { HttpServer } = ChromeUtils.import("resource://testing-common/httpd.js");
const { TelemetryUtils } = ChromeUtils.import(
"resource://gre/modules/TelemetryUtils.jsm"
);
Services.prefs.setBoolPref("network.dns.native-is-localhost", true);
// Trigger a proper telemetry init.
do_get_profile(true);
AddonTestUtils.init(this);
AddonTestUtils.overrideCertDB();
AddonTestUtils.createAppInfo(
"xpcshell@tests.mozilla.org",
"XPCShell",
"42",
"42"
);
// setup and configure a proxy server that will just deny connections.
const proxy = AddonTestUtils.createHttpServer();
proxy.registerPrefixHandler("/", (request, response) => {
response.setStatusLine(request.httpVersion, 504, "hello proxy user");
response.write("ok!");
});
// Register a proxy to be used by TCPSocket connections later.
let proxy_info;
function getBadProxyPort() {
let server = new HttpServer();
server.start(-1);
const badPort = server.identity.primaryPort;
server.stop();
return badPort;
}
function registerProxy() {
let pps = Cc["@mozilla.org/network/protocol-proxy-service;1"].getService(
Ci.nsIProtocolProxyService
);
const proxyFilter = {
applyFilter(uri, defaultProxyInfo, callback) {
if (
proxy_info &&
uri.host == PingServer.host &&
uri.port == PingServer.port
) {
let proxyInfo = pps.newProxyInfo(
proxy_info.type,
proxy_info.host,
proxy_info.port,
"",
"",
0,
4096,
null
);
proxyInfo.sourceId = proxy_info.sourceId;
callback.onProxyFilterResult(proxyInfo);
} else {
callback.onProxyFilterResult(defaultProxyInfo);
}
},
};
pps.registerFilter(proxyFilter, 0);
registerCleanupFunction(() => {
pps.unregisterFilter(proxyFilter);
});
}
add_task(async function setup() {
fakeIntlReady();
// Make sure we don't generate unexpected pings due to pref changes.
await setEmptyPrefWatchlist();
Services.prefs.setBoolPref(
TelemetryUtils.Preferences.HealthPingEnabled,
false
);
TelemetryStopwatch.setTestModeEnabled(true);
registerProxy();
PingServer.start();
// accept proxy connections for PingServer
proxy.identity.add("http", PingServer.host, PingServer.port);
await TelemetrySend.setup(true);
TelemetrySend.setTestModeEnabled(true);
TelemetrySend.setServer(`http://localhost:${PingServer.port}`);
});
function checkEvent() {
// ServiceRequest should have recorded an event for this.
let expected = [
"service_request",
"bypass",
"proxy_info",
"telemetry.send",
{
source: proxy_info.sourceId,
type: "api",
},
];
let snapshot = Telemetry.snapshotEvents(
Ci.nsITelemetry.DATASET_ALL_CHANNELS,
false
);
let received = snapshot.parent[0];
received.shift();
Assert.deepEqual(
expected,
received,
`retry telemetry data matched ${JSON.stringify(received)}`
);
Telemetry.clearEvents();
}
async function submitPingWithDate(date, expected) {
fakeNow(new Date(date));
let pingId = await TelemetryController.submitExternalPing(
"test-send-date-header",
{}
);
let req = await PingServer.promiseNextRequest();
let ping = decodeRequestPayload(req);
Assert.equal(
req.getHeader("Date"),
expected,
"Telemetry should send the correct Date header with requests."
);
Assert.equal(ping.id, pingId, "Should have received the correct ping id.");
}
// While there is no specific indiction, this test causes the
// telemetrySend doPing onload handler to be invoked.
add_task(async function test_failed_server() {
proxy_info = {
type: "http",
host: proxy.identity.primaryHost,
port: proxy.identity.primaryPort,
sourceId: "failed_server_test",
};
await TelemetrySend.reset();
await submitPingWithDate(
Date.UTC(2011, 1, 1, 11, 0, 0),
"Tue, 01 Feb 2011 11:00:00 GMT"
);
checkEvent();
});
// While there is no specific indiction, this test causes the
// telemetrySend doPing error handler to be invoked.
add_task(async function test_no_server() {
// Make sure the underlying proxy failover is disabled to easily force
// telemetry to retry the request.
Services.prefs.setBoolPref("network.proxy.failover_direct", false);
proxy_info = {
type: "http",
host: "localhost",
port: getBadProxyPort(),
sourceId: "no_server_test",
};
await TelemetrySend.reset();
await submitPingWithDate(
Date.UTC(2012, 1, 1, 11, 0, 0),
"Wed, 01 Feb 2012 11:00:00 GMT"
);
checkEvent();
});
// Mock out the send timer activity.
function waitForTimer() {
return new Promise(resolve => {
fakePingSendTimer(
(callback, timeout) => {
resolve([callback, timeout]);
},
() => {}
);
});
}
add_task(async function test_no_bypass() {
// Make sure the underlying proxy failover is disabled to easily force
// telemetry to retry the request.
Services.prefs.setBoolPref("network.proxy.failover_direct", false);
// Disable the retry and submit again.
Services.prefs.setBoolPref("network.proxy.allow_bypass", false);
proxy_info = {
type: "http",
host: "localhost",
port: getBadProxyPort(),
sourceId: "no_server_test",
};
await TelemetrySend.reset();
fakeNow(new Date(Date.UTC(2013, 1, 1, 11, 0, 0)));
let timerPromise = waitForTimer();
let pingId = await TelemetryController.submitExternalPing(
"test-send-date-header",
{}
);
let [pingSendTimerCallback] = await timerPromise;
Assert.ok(!!pingSendTimerCallback, "Should have a timer callback");
Assert.equal(
TelemetrySend.pendingPingCount,
1,
"Should have correct pending ping count"
);
// Reset the proxy, trigger the next tick - we should receive the ping.
proxy_info = null;
pingSendTimerCallback();
let req = await PingServer.promiseNextRequest();
let ping = decodeRequestPayload(req);
// PingServer finished before telemetry, so make sure it's done.
await TelemetrySend.testWaitOnOutgoingPings();
Assert.equal(
req.getHeader("Date"),
"Fri, 01 Feb 2013 11:00:00 GMT",
"Telemetry should send the correct Date header with requests."
);
Assert.equal(ping.id, pingId, "Should have received the correct ping id.");
// reset to save any pending pings
Assert.equal(
TelemetrySend.pendingPingCount,
0,
"Should not have any pending pings"
);
await TelemetrySend.reset();
PingServer.stop();
});

Просмотреть файл

@ -106,5 +106,3 @@ tags = coverage
[test_bug1555798.js]
[test_UninstallPing.js]
run-if = os == 'win'
[test_failover_retry.js]
skip-if = os == "android" # Android doesn't support telemetry though some tests manage to pass with xpcshell

Просмотреть файл

@ -10,110 +10,17 @@
*/
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
const { Log } = ChromeUtils.import("resource://gre/modules/Log.jsm");
const { XPCOMUtils } = ChromeUtils.import(
"resource://gre/modules/XPCOMUtils.jsm"
);
XPCOMUtils.defineLazyGlobalGetters(this, ["XMLHttpRequest"]);
XPCOMUtils.defineLazyServiceGetter(
this,
"ProxyService",
"@mozilla.org/network/protocol-proxy-service;1",
"nsIProtocolProxyService"
);
XPCOMUtils.defineLazyModuleGetters(this, {
ExtensionPreferencesManager:
"resource://gre/modules/ExtensionPreferencesManager.jsm",
});
XPCOMUtils.defineLazyServiceGetter(
this,
"CaptivePortalService",
"@mozilla.org/network/captive-portal-service;1",
"nsICaptivePortalService"
);
XPCOMUtils.defineLazyServiceGetter(
this,
"gNetworkLinkService",
"@mozilla.org/network/network-link-service;1",
"nsINetworkLinkService"
);
var EXPORTED_SYMBOLS = ["ServiceRequest"];
const PROXY_CONFIG_TYPES = [
"direct",
"manual",
"pac",
"unused", // nsIProtocolProxyService.idl skips index 3.
"wpad",
"system",
];
function recordEvent(service, source = {}) {
try {
Services.telemetry.setEventRecordingEnabled("service_request", true);
Services.telemetry.recordEvent(
"service_request",
"bypass",
"proxy_info",
service,
source
);
} catch (err) {
// If the telemetry throws just log the error so it doesn't break any
// functionality.
Cu.reportError(err);
}
}
// If proxy.settings is used to change the proxy, an extension will
// be "in control". This returns the id of that extension.
async function getControllingExtension() {
if (
!WebExtensionPolicy.getActiveExtensions().some(p =>
p.permissions.includes("proxy")
)
) {
return undefined;
}
// Is this proxied by an extension that set proxy prefs?
let setting = await ExtensionPreferencesManager.getSetting("proxy.settings");
return setting?.id;
}
async function getProxySource(proxyInfo) {
// sourceId is set when using proxy.onRequest
if (proxyInfo.sourceId) {
return {
source: proxyInfo.sourceId,
type: "api",
};
}
let type = PROXY_CONFIG_TYPES[ProxyService.proxyConfigType] || "unknown";
// If we have a policy it will have set the prefs.
if (
Services.policies &&
Services.policies.status === Services.policies.ACTIVE
) {
let policies = Services.policies.getActivePolicies()?.filter(p => p.Proxy);
if (policies?.length) {
return {
source: "policy",
type,
};
}
}
let source = await getControllingExtension();
return {
source: source || "prefs",
type,
};
}
const logger = Log.repository.getLogger("ServiceRequest");
logger.level = Log.Level.Debug;
logger.addAppender(new Log.ConsoleAppender(new Log.BasicFormatter()));
/**
* ServiceRequest is intended to be a drop-in replacement for current users
@ -160,24 +67,4 @@ class ServiceRequest extends XMLHttpRequest {
get bypassProxyEnabled() {
return Services.prefs.getBoolPref("network.proxy.allow_bypass", true);
}
static async logProxySource(channel, service) {
if (channel.proxyInfo) {
let source = await getProxySource(channel.proxyInfo);
recordEvent(service, source);
}
}
static get isOffline() {
try {
return (
Services.io.offline ||
CaptivePortalService.state == CaptivePortalService.LOCKED_PORTAL ||
!gNetworkLinkService.isLinkUp
);
} catch (ex) {
// we cannot get state, assume the best.
}
return false;
}
}