diff --git a/toolkit/components/extensions/ext-webRequest.js b/toolkit/components/extensions/ext-webRequest.js index 793d08a2c897..6796d9c54cef 100644 --- a/toolkit/components/extensions/ext-webRequest.js +++ b/toolkit/components/extensions/ext-webRequest.js @@ -33,6 +33,7 @@ function WebRequestEventManager(context, eventName) { } let data2 = { + requestId: data.requestId, url: data.url, method: data.method, type: data.type, diff --git a/toolkit/components/extensions/test/mochitest/file_WebRequest_page1.html b/toolkit/components/extensions/test/mochitest/file_WebRequest_page1.html index 218ccc5d22e3..6308d424aa1b 100644 --- a/toolkit/components/extensions/test/mochitest/file_WebRequest_page1.html +++ b/toolkit/components/extensions/test/mochitest/file_WebRequest_page1.html @@ -25,5 +25,7 @@ + + diff --git a/toolkit/components/extensions/test/mochitest/test_ext_webrequest.html b/toolkit/components/extensions/test/mochitest/test_ext_webrequest.html index 2c578a5592e9..056cda91f774 100644 --- a/toolkit/components/extensions/test/mochitest/test_ext_webrequest.html +++ b/toolkit/components/extensions/test/mochitest/test_ext_webrequest.html @@ -31,7 +31,9 @@ const expected_requested = [BASE + "/file_WebRequest_page1.html", BASE + "/file_WebRequest_page2.html", BASE + "/nonexistent_script_url.js", BASE + "/redirection.sjs", - BASE + "/xhr_resource"]; + BASE + "/dummy_page.html", + BASE + "/xhr_resource", + "data:text/plain,webRequestTest"]; const expected_beforeSendHeaders = [BASE + "/file_WebRequest_page1.html", BASE + "/file_style_good.css", @@ -53,7 +55,7 @@ const expected_sendHeaders = expected_beforeSendHeaders.filter(u => !/_redirect\ const expected_redirect = expected_beforeSendHeaders.filter(u => /_redirect\./.test(u)) .concat(BASE + "/redirection.sjs"); -const expected_complete = [BASE + "/file_WebRequest_page1.html", +const expected_response = [BASE + "/file_WebRequest_page1.html", BASE + "/file_style_good.css", BASE + "/file_image_good.png", BASE + "/file_script_good.js", @@ -63,6 +65,8 @@ const expected_complete = [BASE + "/file_WebRequest_page1.html", BASE + "/dummy_page.html", BASE + "/xhr_resource"]; +const expected_complete = expected_response.concat("data:text/plain,webRequestTest"); + function removeDupes(list) { let j = 0; for (let i = 1; i < list.length; i++) { @@ -85,11 +89,13 @@ function compareLists(list1, list2, kind) { } function backgroundScript() { - const BASE = "http://mochi.test:8888/tests/toolkit/components/extensions/test/mochitest"; - let checkCompleted = true; let savedTabId = -1; + function shouldRecord(url) { + return url.startsWith(BASE) || /^data:.*\bwebRequestTest\b/.test(url); + } + function checkType(details) { let expected_type = "???"; if (details.url.indexOf("style") != -1) { @@ -100,7 +106,7 @@ function backgroundScript() { expected_type = "script"; } else if (details.url.indexOf("page1") != -1) { expected_type = "main_frame"; - } else if (/page2|redirection|dummy_page/.test(details.url)) { + } else if (/page2|redirection|dummy_page|data:text\/(?:plain|html),/.test(details.url)) { expected_type = "sub_frame"; } else if (details.url.indexOf("xhr") != -1) { expected_type = "xmlhttprequest"; @@ -108,6 +114,11 @@ function backgroundScript() { browser.test.assertEq(details.type, expected_type, "resource type is correct"); } + let requestIDs = new Map(); + function checkRequestId(details) { + browser.test.assertEq(requestIDs.get(details.url), details.requestId, `correct requestId for ${details.url} (${details.requestId})`); + } + let frameIDs = new Map(); let recorded = {requested: [], @@ -123,9 +134,13 @@ function backgroundScript() { } function onBeforeRequest(details) { - browser.test.log(`onBeforeRequest ${details.url}`); + browser.test.log(`onBeforeRequest ${details.requestId} ${details.url}`); + + browser.test.assertTrue(details.requestId > 0, `valid request ID ${details.requestId}`); + requestIDs.set(details.url, details.requestId); + checkResourceType(details.type); - if (details.url.startsWith(BASE)) { + if (shouldRecord(details.url)) { recorded.requested.push(details.url); if (savedTabId == -1) { @@ -155,8 +170,9 @@ function backgroundScript() { function onBeforeSendHeaders(details) { browser.test.log(`onBeforeSendHeaders ${details.url}`); + checkRequestId(details); checkResourceType(details.type); - if (details.url.startsWith(BASE)) { + if (shouldRecord(details.url)) { recorded.beforeSendHeaders.push(details.url); browser.test.assertEq(details.tabId, savedTabId, "correct tab ID"); @@ -173,8 +189,9 @@ function backgroundScript() { function onBeforeRedirect(details) { browser.test.log(`onBeforeRedirect ${details.url} -> ${details.redirectUrl}`); + checkRequestId(details); checkResourceType(details.type); - if (details.url.startsWith(BASE)) { + if (shouldRecord(details.url)) { recorded.beforeRedirect.push(details.url); browser.test.assertEq(details.tabId, savedTabId, "correct tab ID"); @@ -193,7 +210,8 @@ function backgroundScript() { function onRecord(kind, details) { checkResourceType(details.type); - if (details.url.startsWith(BASE)) { + checkRequestId(details); + if (shouldRecord(details.url)) { recorded[kind].push(details.url); } } @@ -209,7 +227,10 @@ function backgroundScript() { // When resources are cached, the ip property is not present, // so only check for the ip property the first time around. if (checkCompleted && !completedUrls[kind].has(details.url)) { - browser.test.assertEq(details.ip, "127.0.0.1", "correct ip"); + // We can only tell IPs for HTTP requests. + if (/^https?:/.test(details.url)) { + browser.test.assertEq(details.ip, "127.0.0.1", "correct ip"); + } completedUrls[kind].add(details.url); } } @@ -243,7 +264,7 @@ function* test_once(skipCompleted) { "webRequestBlocking", ], }, - background: "(" + backgroundScript.toString() + ")()", + background: `const BASE = ${JSON.stringify(BASE)}; (${backgroundScript.toString()})()`, }; let extension = ExtensionTestUtils.loadExtension(extensionData); @@ -297,7 +318,7 @@ function* test_once(skipCompleted) { compareLists(recorded.beforeSendHeaders, expected_beforeSendHeaders, "beforeSendHeaders"); compareLists(recorded.sendHeaders, expected_sendHeaders, "sendHeaders"); compareLists(recorded.beforeRedirect, expected_redirect, "beforeRedirect"); - compareLists(recorded.responseStarted, expected_complete, "responseStarted"); + compareLists(recorded.responseStarted, expected_response, "responseStarted"); compareLists(recorded.completed, expected_complete, "completed"); yield extension.unload(); diff --git a/toolkit/modules/addons/MatchPattern.jsm b/toolkit/modules/addons/MatchPattern.jsm index 7699d5b2e522..52404ad24e5e 100644 --- a/toolkit/modules/addons/MatchPattern.jsm +++ b/toolkit/modules/addons/MatchPattern.jsm @@ -15,7 +15,8 @@ this.EXPORTED_SYMBOLS = ["MatchPattern"]; /* globals MatchPattern */ -const PERMITTED_SCHEMES = ["http", "https", "file", "ftp", "app"]; +const PERMITTED_SCHEMES = ["http", "https", "file", "ftp", "app", "data"]; +const PERMITTED_SCHEMES_REGEXP = PERMITTED_SCHEMES.join("|"); // This function converts a glob pattern (containing * and possibly ? // as wildcards) to a regular expression. @@ -42,7 +43,7 @@ function SingleMatchPattern(pat) { } else if (!pat) { this.schemes = []; } else { - let re = new RegExp("^(http|https|file|ftp|app|\\*)://(\\*|\\*\\.[^*/]+|[^*/]+|)(/.*)$"); + let re = new RegExp(`^(${PERMITTED_SCHEMES_REGEXP}|\\*)://(\\*|\\*\\.[^*/]+|[^*/]+|)(/.*)$`); let match = re.exec(pat); if (!match) { Cu.reportError(`Invalid match pattern: '${pat}'`); diff --git a/toolkit/modules/addons/WebRequest.jsm b/toolkit/modules/addons/WebRequest.jsm index de2803fa86e4..f1c1ea21a08b 100644 --- a/toolkit/modules/addons/WebRequest.jsm +++ b/toolkit/modules/addons/WebRequest.jsm @@ -21,10 +21,44 @@ XPCOMUtils.defineLazyModuleGetter(this, "BrowserUtils", XPCOMUtils.defineLazyModuleGetter(this, "WebRequestCommon", "resource://gre/modules/WebRequestCommon.jsm"); -// TODO -// Figure out how to handle requestId. Gecko seems to have no such thing. (Bug 1163862) -// We also don't know the method for content policy. (Bug 1163862) -// We don't even have a window ID for HTTP observer stuff. (Bug 1163861) +function attachToChannel(channel, key, data) { + if (channel instanceof Ci.nsIWritablePropertyBag2) { + let wrapper = {value: data}; + wrapper.wrappedJSObject = wrapper; + channel.setPropertyAsInterface(key, wrapper); + } +} + +function extractFromChannel(channel, key) { + if (channel instanceof Ci.nsIPropertyBag2 && channel.hasKey(key)) { + let data = channel.get(key); + if (data && data.wrappedJSObject) { + data = data.wrappedJSObject; + } + return "value" in data ? data.value : data; + } + return null; +} + +var RequestId = { + count: 1, + KEY: "mozilla.webRequest.requestId", + create(channel = null) { + let id = this.count++; + if (channel) { + attachToChannel(channel, this.KEY, id); + } + return id; + }, + + get(channel) { + return channel && extractFromChannel(channel, this.KEY) || this.create(channel); + }, +}; + +function runLater(job) { + Services.tm.currentThread.dispatch(job, Ci.nsIEventTarget.DISPATCH_NORMAL); +} function parseFilter(filter) { if (!filter) { @@ -53,6 +87,8 @@ function parseExtra(extra, allowed) { return result; } +var HttpObserverManager; + var ContentPolicyManager = { policyData: new Map(), policies: new Map(), @@ -77,32 +113,50 @@ var ContentPolicyManager = { continue; } let response = null; + let data = { + url: msg.data.url, + windowId: msg.data.windowId, + parentWindowId: msg.data.parentWindowId, + type: msg.data.type, + browser: browser, + requestId: RequestId.create(), + }; try { - response = callback({ - url: msg.data.url, - windowId: msg.data.windowId, - parentWindowId: msg.data.parentWindowId, - type: msg.data.type, - browser: browser, - }); + response = callback(data); + if (response && response.cancel) { + return {cancel: true}; + } + + // FIXME: Need to handle redirection here. (Bug 1163862) } catch (e) { Cu.reportError(e); + } finally { + runLater(() => this.runChannelListener("onStop", data)); } - - if (response && response.cancel) { - return {cancel: true}; - } - - // FIXME: Need to handle redirection here. (Bug 1163862) } return {}; }, + runChannelListener(kind, data) { + let listeners = HttpObserverManager.listeners[kind]; + let uri = BrowserUtils.makeURI(data.url); + let policyType = data.type; + for (let [callback, opts] of listeners.entries()) { + if (!HttpObserverManager.shouldRunListener(policyType, uri, opts.filter)) { + continue; + } + callback(data); + } + }, + addListener(callback, opts) { + // Clone opts, since we're going to modify them for IPC. + opts = Object.assign({}, opts); let id = this.nextId++; opts.id = id; if (opts.filter.urls) { + opts.filter = Object.assign({}, opts.filter); opts.filter.urls = opts.filter.urls.serialize(); } Services.ppmm.broadcastAsyncMessage("WebRequest:AddContentPolicy", opts); @@ -151,8 +205,6 @@ StartStopListener.prototype = { }, }; -var HttpObserverManager; - var ChannelEventSink = { _classDescription: "WebRequest channel event sink", _classID: Components.ID("115062f8-92f1-11e5-8b7f-080027b0f7ec"), @@ -178,7 +230,7 @@ var ChannelEventSink = { // nsIChannelEventSink implementation asyncOnChannelRedirect(oldChannel, newChannel, flags, redirectCallback) { - Services.tm.currentThread.dispatch(() => redirectCallback.onRedirectVerifyCallback(Cr.NS_OK), Ci.nsIEventTarget.DISPATCH_NORMAL); + runLater(() => redirectCallback.onRedirectVerifyCallback(Cr.NS_OK)); try { HttpObserverManager.onChannelReplaced(oldChannel, newChannel); } catch (e) { @@ -203,6 +255,7 @@ HttpObserverManager = { redirectInitialized: false, listeners: { + opening: new Map(), modify: new Map(), afterModify: new Map(), headersReceived: new Map(), @@ -212,7 +265,7 @@ HttpObserverManager = { }, addOrRemove() { - let needModify = this.listeners.modify.size || this.listeners.afterModify.size; + let needModify = this.listeners.opening.size || this.listeners.modify.size || this.listeners.afterModify.size; if (needModify && !this.modifyInitialized) { this.modifyInitialized = true; Services.obs.addObserver(this, "http-on-modify-request", false); @@ -289,13 +342,15 @@ HttpObserverManager = { observe(subject, topic, data) { let channel = subject.QueryInterface(Ci.nsIHttpChannel); - - if (topic == "http-on-modify-request") { - this.modify(channel, topic, data); - } else if (topic == "http-on-examine-response" || - topic == "http-on-examine-cached-response" || - topic == "http-on-examine-merged-response") { - this.examine(channel, topic, data); + switch (topic) { + case "http-on-modify-request": + this.modify(channel, topic, data); + break; + case "http-on-examine-response": + case "http-on-examine-cached-response": + case "http-on-examine-merged-response": + this.examine(channel, topic, data); + break; } }, @@ -305,6 +360,9 @@ HttpObserverManager = { }, runChannelListener(channel, loadContext, kind, extraData = null) { + if (channel.status === Cr.NS_ERROR_ABORT) { + return false; + } let listeners = this.listeners[kind]; let browser = loadContext ? loadContext.topFrameElement : null; let loadInfo = channel.loadInfo; @@ -326,6 +384,7 @@ HttpObserverManager = { } let data = { + requestId: RequestId.get(channel), url: channel.URI.spec, method: channel.requestMethod, browser: browser, @@ -372,7 +431,7 @@ HttpObserverManager = { return true; } if (result.cancel) { - channel.cancel(); + channel.cancel(Cr.NS_ERROR_ABORT); return false; } if (result.redirectUrl) { @@ -407,7 +466,8 @@ HttpObserverManager = { modify(channel, topic, data) { let loadContext = this.getLoadContext(channel); - if (this.runChannelListener(channel, loadContext, "modify")) { + if (this.runChannelListener(channel, loadContext, "opening") && + this.runChannelListener(channel, loadContext, "modify")) { this.runChannelListener(channel, loadContext, "afterModify"); } }, @@ -450,9 +510,11 @@ var onBeforeRequest = { let opts = parseExtra(opt_extraInfoSpec, ["blocking"]); opts.filter = parseFilter(filter); ContentPolicyManager.addListener(callback, opts); + HttpObserverManager.addListener("opening", callback, opts); }, removeListener(callback) { + HttpObserverManager.removeListener("opening", callback); ContentPolicyManager.removeListener(callback); }, }; @@ -482,7 +544,7 @@ var onResponseStarted = new HttpEvent("onStart", ["responseHeaders"]); var onCompleted = new HttpEvent("onStop", ["responseHeaders"]); var WebRequest = { - // Handled via content policy. + // http-on-modify observer for HTTP(S), content policy for the other protocols (notably, data:) onBeforeRequest: onBeforeRequest, // http-on-modify observer. diff --git a/toolkit/modules/addons/WebRequestContent.js b/toolkit/modules/addons/WebRequestContent.js index 21a6e3e7bf2e..fc947be199a6 100644 --- a/toolkit/modules/addons/WebRequestContent.js +++ b/toolkit/modules/addons/WebRequestContent.js @@ -17,6 +17,8 @@ XPCOMUtils.defineLazyModuleGetter(this, "MatchPattern", XPCOMUtils.defineLazyModuleGetter(this, "WebRequestCommon", "resource://gre/modules/WebRequestCommon.jsm"); +const IS_HTTP = /^https?:/; + var ContentPolicy = { _classDescription: "WebRequest content policy", _classID: Components.ID("938e5d24-9ccc-4b55-883e-c252a41f7ce9"), @@ -78,6 +80,12 @@ var ContentPolicy = { shouldLoad(policyType, contentLocation, requestOrigin, node, mimeTypeGuess, extra, requestPrincipal) { + let url = contentLocation.spec; + if (IS_HTTP.test(url)) { + // We'll handle this in our parent process HTTP observer. + return Ci.nsIContentPolicy.ACCEPT; + } + let block = false; let ids = []; for (let [id, {blocking, filter}] of this.contentPolicies.entries()) { @@ -146,7 +154,7 @@ var ContentPolicy = { } let data = {ids, - url: contentLocation.spec, + url, type: WebRequestCommon.typeForPolicyType(policyType), windowId, parentWindowId}; diff --git a/toolkit/modules/tests/browser/browser_WebRequest.js b/toolkit/modules/tests/browser/browser_WebRequest.js index 82804ec645e7..5b1426a802e3 100644 --- a/toolkit/modules/tests/browser/browser_WebRequest.js +++ b/toolkit/modules/tests/browser/browser_WebRequest.js @@ -117,6 +117,7 @@ const expected_requested = [BASE + "/file_WebRequest_page1.html", BASE + "/file_WebRequest_page2.html", BASE + "/nonexistent_script_url.js", BASE + "/WebRequest_redirection.sjs", + BASE + "/dummy_page.html", BASE + "/xhr_resource"]; const expected_sendHeaders = [BASE + "/file_WebRequest_page1.html",