feat: net.fetch() supports custom protocols (#36606)

This commit is contained in:
Jeremy Rose 2023-03-02 15:47:45 -08:00 коммит произвёл GitHub
Родитель 76c825d619
Коммит 6bd9ee6988
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
8 изменённых файлов: 321 добавлений и 36 удалений

Просмотреть файл

@ -101,6 +101,10 @@ Limitations:
* The `.type` and `.url` values of the returned `Response` object are
incorrect.
Requests made with `net.fetch` can be made to [custom protocols](protocol.md)
as well as `file:`, and will trigger [webRequest](web-request.md) handlers if
present.
### `net.isOnline()`
Returns `boolean` - Whether there is currently internet connection.

Просмотреть файл

@ -768,6 +768,10 @@ Limitations:
* The `.type` and `.url` values of the returned `Response` object are
incorrect.
Requests made with `ses.fetch` can be made to [custom protocols](protocol.md)
as well as `file:`, and will trigger [webRequest](web-request.md) handlers if
present.
#### `ses.disableNetworkEmulation()`
Disables any network emulation already active for the `session`. Resets to

Просмотреть файл

@ -10,7 +10,7 @@ const {
} = process._linkedBinding('electron_browser_net');
const { Session } = process._linkedBinding('electron_browser_session');
const kSupportedProtocols = new Set(['http:', 'https:']);
const kHttpProtocols = new Set(['http:', 'https:']);
// set of headers that Node.js discards duplicates for
// see https://nodejs.org/api/http.html#http_message_headers
@ -195,7 +195,20 @@ class ChunkedBodyStream extends Writable {
type RedirectPolicy = 'manual' | 'follow' | 'error';
function parseOptions (optionsIn: ClientRequestConstructorOptions | string): NodeJS.CreateURLLoaderOptions & { redirectPolicy: RedirectPolicy, headers: Record<string, { name: string, value: string | string[] }> } {
const kAllowNonHttpProtocols = Symbol('kAllowNonHttpProtocols');
export function allowAnyProtocol (opts: ClientRequestConstructorOptions): ClientRequestConstructorOptions {
return {
...opts,
[kAllowNonHttpProtocols]: true
} as any;
}
type ExtraURLLoaderOptions = {
redirectPolicy: RedirectPolicy;
headers: Record<string, { name: string, value: string | string[] }>;
allowNonHttpProtocols: boolean;
}
function parseOptions (optionsIn: ClientRequestConstructorOptions | string): NodeJS.CreateURLLoaderOptions & ExtraURLLoaderOptions {
const options: any = typeof optionsIn === 'string' ? url.parse(optionsIn) : { ...optionsIn };
let urlStr: string = options.url;
@ -203,9 +216,6 @@ function parseOptions (optionsIn: ClientRequestConstructorOptions | string): Nod
if (!urlStr) {
const urlObj: url.UrlObject = {};
const protocol = options.protocol || 'http:';
if (!kSupportedProtocols.has(protocol)) {
throw new Error('Protocol "' + protocol + '" not supported');
}
urlObj.protocol = protocol;
if (options.host) {
@ -247,7 +257,7 @@ function parseOptions (optionsIn: ClientRequestConstructorOptions | string): Nod
throw new TypeError('headers must be an object');
}
const urlLoaderOptions: NodeJS.CreateURLLoaderOptions & { redirectPolicy: RedirectPolicy, headers: Record<string, { name: string, value: string | string[] }> } = {
const urlLoaderOptions: NodeJS.CreateURLLoaderOptions & { redirectPolicy: RedirectPolicy, headers: Record<string, { name: string, value: string | string[] }>, allowNonHttpProtocols: boolean } = {
method: (options.method || 'GET').toUpperCase(),
url: urlStr,
redirectPolicy,
@ -257,7 +267,8 @@ function parseOptions (optionsIn: ClientRequestConstructorOptions | string): Nod
credentials: options.credentials,
origin: options.origin,
referrerPolicy: options.referrerPolicy,
cache: options.cache
cache: options.cache,
allowNonHttpProtocols: Object.prototype.hasOwnProperty.call(options, kAllowNonHttpProtocols)
};
const headers: Record<string, string | string[]> = options.headers || {};
for (const [name, value] of Object.entries(headers)) {
@ -308,6 +319,10 @@ export class ClientRequest extends Writable implements Electron.ClientRequest {
}
const { redirectPolicy, ...urlLoaderOptions } = parseOptions(options);
const urlObj = new URL(urlLoaderOptions.url);
if (!urlLoaderOptions.allowNonHttpProtocols && !kHttpProtocols.has(urlObj.protocol)) {
throw new Error('ClientRequest only supports http: and https: protocols');
}
if (urlLoaderOptions.credentials === 'same-origin' && !urlLoaderOptions.origin) { throw new Error('credentials: same-origin requires origin to be set'); }
this._urlLoaderOptions = urlLoaderOptions;
this._redirectPolicy = redirectPolicy;

Просмотреть файл

@ -1,5 +1,6 @@
import { net, IncomingMessage, Session as SessionT } from 'electron/main';
import { Readable, Writable, isReadable } from 'stream';
import { allowAnyProtocol } from '@electron/internal/browser/api/net-client-request';
function createDeferredPromise<T, E extends Error = Error> (): { promise: Promise<T>; resolve: (x: T) => void; reject: (e: E) => void; } {
let res: (x: T) => void;
@ -72,7 +73,7 @@ export function fetchWithSession (input: RequestInfo, init: RequestInit | undefi
// We can't set credentials to same-origin unless there's an origin set.
const credentials = req.credentials === 'same-origin' && !origin ? 'include' : req.credentials;
const r = net.request({
const r = net.request(allowAnyProtocol({
session,
method: req.method,
url: req.url,
@ -81,7 +82,7 @@ export function fetchWithSession (input: RequestInfo, init: RequestInit | undefi
cache: req.cache,
referrerPolicy: req.referrerPolicy,
redirect: req.redirect
});
}));
// cors is the default mode, but we can't set mode=cors without an origin.
if (req.mode && (req.mode !== 'cors' || origin)) {

Просмотреть файл

@ -18,14 +18,19 @@
#include "mojo/public/cpp/system/data_pipe_producer.h"
#include "net/base/load_flags.h"
#include "net/http/http_util.h"
#include "net/url_request/redirect_util.h"
#include "services/network/public/cpp/resource_request.h"
#include "services/network/public/cpp/simple_url_loader.h"
#include "services/network/public/cpp/url_util.h"
#include "services/network/public/cpp/wrapper_shared_url_loader_factory.h"
#include "services/network/public/mojom/chunked_data_pipe_getter.mojom.h"
#include "services/network/public/mojom/http_raw_headers.mojom.h"
#include "services/network/public/mojom/url_loader_factory.mojom.h"
#include "shell/browser/api/electron_api_session.h"
#include "shell/browser/electron_browser_context.h"
#include "shell/browser/javascript_environment.h"
#include "shell/browser/net/asar/asar_url_loader_factory.h"
#include "shell/browser/protocol_registry.h"
#include "shell/common/gin_converters/callback_converter.h"
#include "shell/common/gin_converters/gurl_converter.h"
#include "shell/common/gin_converters/net_converter.h"
@ -336,34 +341,49 @@ gin::WrapperInfo SimpleURLLoaderWrapper::kWrapperInfo = {
gin::kEmbedderNativeGin};
SimpleURLLoaderWrapper::SimpleURLLoaderWrapper(
ElectronBrowserContext* browser_context,
std::unique_ptr<network::ResourceRequest> request,
network::mojom::URLLoaderFactory* url_loader_factory,
int options) {
if (!request->trusted_params)
request->trusted_params = network::ResourceRequest::TrustedParams();
int options)
: browser_context_(browser_context),
request_options_(options),
request_(std::move(request)) {
if (!request_->trusted_params)
request_->trusted_params = network::ResourceRequest::TrustedParams();
mojo::PendingRemote<network::mojom::URLLoaderNetworkServiceObserver>
url_loader_network_observer_remote;
url_loader_network_observer_receivers_.Add(
this,
url_loader_network_observer_remote.InitWithNewPipeAndPassReceiver());
request->trusted_params->url_loader_network_observer =
request_->trusted_params->url_loader_network_observer =
std::move(url_loader_network_observer_remote);
// Chromium filters headers using browser rules, while for net module we have
// every header passed. The following setting will allow us to capture the
// raw headers in the URLLoader.
request->trusted_params->report_raw_headers = true;
// SimpleURLLoader wants to control the request body itself. We have other
// ideas.
auto request_body = std::move(request->request_body);
auto* request_ref = request.get();
request_->trusted_params->report_raw_headers = true;
Start();
}
void SimpleURLLoaderWrapper::Start() {
// Make a copy of the request; we'll need to re-send it if we get redirected.
auto request = std::make_unique<network::ResourceRequest>();
*request = *request_;
// SimpleURLLoader has no way to set a data pipe as the request body, which
// we need to do for streaming upload, so instead we "cheat" and pretend to
// SimpleURLLoader like there is no request_body when we construct it. Later,
// we will sneakily put the request_body back while it isn't looking.
scoped_refptr<network::ResourceRequestBody> request_body =
std::move(request->request_body);
network::ResourceRequest* request_ref = request.get();
loader_ =
network::SimpleURLLoader::Create(std::move(request), kTrafficAnnotation);
if (request_body) {
if (request_body)
request_ref->request_body = std::move(request_body);
}
loader_->SetAllowHttpErrorResults(true);
loader_->SetURLLoaderFactoryOptions(options);
loader_->SetURLLoaderFactoryOptions(request_options_);
loader_->SetOnResponseStartedCallback(base::BindOnce(
&SimpleURLLoaderWrapper::OnResponseStarted, base::Unretained(this)));
loader_->SetOnRedirectCallback(base::BindRepeating(
@ -373,7 +393,8 @@ SimpleURLLoaderWrapper::SimpleURLLoaderWrapper(
loader_->SetOnDownloadProgressCallback(base::BindRepeating(
&SimpleURLLoaderWrapper::OnDownloadProgress, base::Unretained(this)));
loader_->DownloadAsStream(url_loader_factory, this);
url_loader_factory_ = GetURLLoaderFactoryForURL(request_ref->url);
loader_->DownloadAsStream(url_loader_factory_.get(), this);
}
void SimpleURLLoaderWrapper::Pin() {
@ -458,6 +479,42 @@ void SimpleURLLoaderWrapper::Cancel() {
// This ensures that no further callbacks will be called, so there's no need
// for additional guards.
}
scoped_refptr<network::SharedURLLoaderFactory>
SimpleURLLoaderWrapper::GetURLLoaderFactoryForURL(const GURL& url) {
scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory;
auto* protocol_registry =
ProtocolRegistry::FromBrowserContext(browser_context_);
// Explicitly handle intercepted protocols here, even though
// ProxyingURLLoaderFactory would handle them later on, so that we can
// correctly intercept file:// scheme URLs.
if (protocol_registry->IsProtocolIntercepted(url.scheme())) {
auto& protocol_handler =
protocol_registry->intercept_handlers().at(url.scheme());
mojo::PendingRemote<network::mojom::URLLoaderFactory> pending_remote =
ElectronURLLoaderFactory::Create(protocol_handler.first,
protocol_handler.second);
url_loader_factory = network::SharedURLLoaderFactory::Create(
std::make_unique<network::WrapperPendingSharedURLLoaderFactory>(
std::move(pending_remote)));
} else if (protocol_registry->IsProtocolRegistered(url.scheme())) {
auto& protocol_handler = protocol_registry->handlers().at(url.scheme());
mojo::PendingRemote<network::mojom::URLLoaderFactory> pending_remote =
ElectronURLLoaderFactory::Create(protocol_handler.first,
protocol_handler.second);
url_loader_factory = network::SharedURLLoaderFactory::Create(
std::make_unique<network::WrapperPendingSharedURLLoaderFactory>(
std::move(pending_remote)));
} else if (url.SchemeIsFile()) {
mojo::PendingRemote<network::mojom::URLLoaderFactory> pending_remote =
AsarURLLoaderFactory::Create();
url_loader_factory = network::SharedURLLoaderFactory::Create(
std::make_unique<network::WrapperPendingSharedURLLoaderFactory>(
std::move(pending_remote)));
} else {
url_loader_factory = browser_context_->GetURLLoaderFactory();
}
return url_loader_factory;
}
// static
gin::Handle<SimpleURLLoaderWrapper> SimpleURLLoaderWrapper::Create(
@ -634,12 +691,9 @@ gin::Handle<SimpleURLLoaderWrapper> SimpleURLLoaderWrapper::Create(
session = Session::FromPartition(args->isolate(), "");
}
auto url_loader_factory = session->browser_context()->GetURLLoaderFactory();
auto ret = gin::CreateHandle(
args->isolate(),
new SimpleURLLoaderWrapper(std::move(request), url_loader_factory.get(),
options));
args->isolate(), new SimpleURLLoaderWrapper(session->browser_context(),
std::move(request), options));
ret->Pin();
if (!chunk_pipe_getter.IsEmpty()) {
ret->PinBodyGetter(chunk_pipe_getter);
@ -691,6 +745,45 @@ void SimpleURLLoaderWrapper::OnRedirect(
const network::mojom::URLResponseHead& response_head,
std::vector<std::string>* removed_headers) {
Emit("redirect", redirect_info, response_head.headers.get());
if (!loader_)
// The redirect was aborted by JS.
return;
// Optimization: if both the old and new URLs are handled by the network
// service, just FollowRedirect.
if (network::IsURLHandledByNetworkService(redirect_info.new_url) &&
network::IsURLHandledByNetworkService(request_->url))
return;
// Otherwise, restart the request (potentially picking a new
// URLLoaderFactory). See
// https://source.chromium.org/chromium/chromium/src/+/main:content/browser/loader/navigation_url_loader_impl.cc;l=534-550;drc=fbaec92ad5982f83aa4544d5c88d66d08034a9f4
bool should_clear_upload = false;
net::RedirectUtil::UpdateHttpRequest(
request_->url, request_->method, redirect_info, *removed_headers,
/* modified_headers = */ absl::nullopt, &request_->headers,
&should_clear_upload);
if (should_clear_upload) {
// The request body is no longer applicable.
request_->request_body.reset();
}
request_->url = redirect_info.new_url;
request_->method = redirect_info.new_method;
request_->site_for_cookies = redirect_info.new_site_for_cookies;
// See if navigation network isolation key needs to be updated.
request_->trusted_params->isolation_info =
request_->trusted_params->isolation_info.CreateForRedirect(
url::Origin::Create(request_->url));
request_->referrer = GURL(redirect_info.new_referrer);
request_->referrer_policy = redirect_info.new_referrer_policy;
request_->navigation_redirect_chain.push_back(redirect_info.new_url);
Start();
}
void SimpleURLLoaderWrapper::OnUploadProgress(uint64_t position,

Просмотреть файл

@ -31,8 +31,13 @@ class Handle;
namespace network {
class SimpleURLLoader;
struct ResourceRequest;
class SharedURLLoaderFactory;
} // namespace network
namespace electron {
class ElectronBrowserContext;
}
namespace electron::api {
/** Wraps a SimpleURLLoader to make it usable from JavaScript */
@ -54,8 +59,8 @@ class SimpleURLLoaderWrapper
const char* GetTypeName() override;
private:
SimpleURLLoaderWrapper(std::unique_ptr<network::ResourceRequest> request,
network::mojom::URLLoaderFactory* url_loader_factory,
SimpleURLLoaderWrapper(ElectronBrowserContext* browser_context,
std::unique_ptr<network::ResourceRequest> request,
int options);
// SimpleURLLoaderStreamConsumer:
@ -99,6 +104,9 @@ class SimpleURLLoaderWrapper
mojo::PendingReceiver<network::mojom::URLLoaderNetworkServiceObserver>
observer) override;
scoped_refptr<network::SharedURLLoaderFactory> GetURLLoaderFactoryForURL(
const GURL& url);
// SimpleURLLoader callbacks
void OnResponseStarted(const GURL& final_url,
const network::mojom::URLResponseHead& response_head);
@ -112,6 +120,10 @@ class SimpleURLLoaderWrapper
void Pin();
void PinBodyGetter(v8::Local<v8::Value>);
ElectronBrowserContext* browser_context_;
int request_options_;
std::unique_ptr<network::ResourceRequest> request_;
scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;
std::unique_ptr<network::SimpleURLLoader> loader_;
v8::Global<v8::Value> pinned_wrapper_;
v8::Global<v8::Value> pinned_chunk_pipe_getter_;

Просмотреть файл

@ -1,8 +1,9 @@
import { expect } from 'chai';
import * as dns from 'dns';
import { net, session, ClientRequest, BrowserWindow, ClientRequestConstructorOptions } from 'electron/main';
import { net, session, ClientRequest, BrowserWindow, ClientRequestConstructorOptions, protocol } from 'electron/main';
import * as http from 'http';
import * as url from 'url';
import * as path from 'path';
import { Socket } from 'net';
import { defer, listen } from './lib/spec-helpers';
import { once } from 'events';
@ -163,9 +164,9 @@ describe('net module', () => {
it('should post the correct data in a POST request', async () => {
const bodyData = 'Hello World!';
let postedBodyData: string = '';
const serverUrl = await respondOnce.toSingleURL(async (request, response) => {
const postedBodyData = await collectStreamBody(request);
expect(postedBodyData).to.equal(bodyData);
postedBodyData = await collectStreamBody(request);
response.end();
});
const urlRequest = net.request({
@ -175,16 +176,72 @@ describe('net module', () => {
urlRequest.write(bodyData);
const response = await getResponse(urlRequest);
expect(response.statusCode).to.equal(200);
expect(postedBodyData).to.equal(bodyData);
});
it('a 307 redirected POST request preserves the body', async () => {
const bodyData = 'Hello World!';
let postedBodyData: string = '';
let methodAfterRedirect: string | undefined;
const serverUrl = await respondNTimes.toRoutes({
'/redirect': (req, res) => {
res.statusCode = 307;
res.setHeader('location', serverUrl);
return res.end();
},
'/': async (req, res) => {
methodAfterRedirect = req.method;
postedBodyData = await collectStreamBody(req);
res.end();
}
}, 2);
const urlRequest = net.request({
method: 'POST',
url: serverUrl + '/redirect'
});
urlRequest.write(bodyData);
const response = await getResponse(urlRequest);
expect(response.statusCode).to.equal(200);
await collectStreamBody(response);
expect(methodAfterRedirect).to.equal('POST');
expect(postedBodyData).to.equal(bodyData);
});
it('a 302 redirected POST request DOES NOT preserve the body', async () => {
const bodyData = 'Hello World!';
let postedBodyData: string = '';
let methodAfterRedirect: string | undefined;
const serverUrl = await respondNTimes.toRoutes({
'/redirect': (req, res) => {
res.statusCode = 302;
res.setHeader('location', serverUrl);
return res.end();
},
'/': async (req, res) => {
methodAfterRedirect = req.method;
postedBodyData = await collectStreamBody(req);
res.end();
}
}, 2);
const urlRequest = net.request({
method: 'POST',
url: serverUrl + '/redirect'
});
urlRequest.write(bodyData);
const response = await getResponse(urlRequest);
expect(response.statusCode).to.equal(200);
await collectStreamBody(response);
expect(methodAfterRedirect).to.equal('GET');
expect(postedBodyData).to.equal('');
});
it('should support chunked encoding', async () => {
let receivedRequest: http.IncomingMessage = null as any;
const serverUrl = await respondOnce.toSingleURL((request, response) => {
response.statusCode = 200;
response.statusMessage = 'OK';
response.chunkedEncoding = true;
expect(request.method).to.equal('POST');
expect(request.headers['transfer-encoding']).to.equal('chunked');
expect(request.headers['content-length']).to.equal(undefined);
receivedRequest = request;
request.on('data', (chunk: Buffer) => {
response.write(chunk);
});
@ -210,6 +267,9 @@ describe('net module', () => {
}
const response = await getResponse(urlRequest);
expect(receivedRequest.method).to.equal('POST');
expect(receivedRequest.headers['transfer-encoding']).to.equal('chunked');
expect(receivedRequest.headers['content-length']).to.equal(undefined);
expect(response.statusCode).to.equal(200);
const received = await collectStreamBodyBuffer(response);
expect(sent.equals(received)).to.be.true();
@ -1446,6 +1506,9 @@ describe('net module', () => {
urlRequest.end();
urlRequest.on('redirect', () => { urlRequest.abort(); });
urlRequest.on('error', () => {});
urlRequest.on('response', () => {
expect.fail('Unexpected response');
});
await once(urlRequest, 'abort');
});
@ -2078,6 +2141,20 @@ describe('net module', () => {
});
});
describe('non-http schemes', () => {
it('should be rejected by net.request', async () => {
expect(() => {
net.request('file://bar');
}).to.throw('ClientRequest only supports http: and https: protocols');
});
it('should be rejected by net.request when passed in url:', async () => {
expect(() => {
net.request({ url: 'file://bar' });
}).to.throw('ClientRequest only supports http: and https: protocols');
});
});
describe('net.fetch', () => {
// NB. there exist much more comprehensive tests for fetch() in the form of
// the WPT: https://github.com/web-platform-tests/wpt/tree/master/fetch
@ -2167,5 +2244,83 @@ describe('net module', () => {
await expect(r.text()).to.be.rejectedWith(/ERR_INCOMPLETE_CHUNKED_ENCODING/);
});
});
it('can request file:// URLs', async () => {
const resp = await net.fetch(url.pathToFileURL(path.join(__dirname, 'fixtures', 'hello.txt')).toString());
expect(resp.ok).to.be.true();
// trimRight instead of asserting the whole string to avoid line ending shenanigans on WOA
expect((await resp.text()).trimRight()).to.equal('hello world');
});
it('can make requests to custom protocols', async () => {
protocol.registerStringProtocol('electron-test', (req, cb) => { cb('hello ' + req.url); });
defer(() => {
protocol.unregisterProtocol('electron-test');
});
const body = await net.fetch('electron-test://foo').then(r => r.text());
expect(body).to.equal('hello electron-test://foo');
});
it('runs through intercept handlers', async () => {
protocol.interceptStringProtocol('http', (req, cb) => { cb('hello ' + req.url); });
defer(() => {
protocol.uninterceptProtocol('http');
});
const body = await net.fetch('http://foo').then(r => r.text());
expect(body).to.equal('hello http://foo/');
});
it('file: runs through intercept handlers', async () => {
protocol.interceptStringProtocol('file', (req, cb) => { cb('hello ' + req.url); });
defer(() => {
protocol.uninterceptProtocol('file');
});
const body = await net.fetch('file://foo').then(r => r.text());
expect(body).to.equal('hello file://foo/');
});
it('can be redirected', async () => {
protocol.interceptStringProtocol('file', (req, cb) => { cb({ statusCode: 302, headers: { location: 'electron-test://bar' } }); });
defer(() => {
protocol.uninterceptProtocol('file');
});
protocol.registerStringProtocol('electron-test', (req, cb) => { cb('hello ' + req.url); });
defer(() => {
protocol.unregisterProtocol('electron-test');
});
const body = await net.fetch('file://foo').then(r => r.text());
expect(body).to.equal('hello electron-test://bar');
});
it('should not follow redirect when redirect: error', async () => {
protocol.registerStringProtocol('electron-test', (req, cb) => {
if (/redirect/.test(req.url)) return cb({ statusCode: 302, headers: { location: 'electron-test://bar' } });
cb('hello ' + req.url);
});
defer(() => {
protocol.unregisterProtocol('electron-test');
});
await expect(net.fetch('electron-test://redirect', { redirect: 'error' })).to.eventually.be.rejectedWith('Attempted to redirect, but redirect policy was \'error\'');
});
it('a 307 redirected POST request preserves the body', async () => {
const bodyData = 'Hello World!';
let postedBodyData: any;
protocol.registerStringProtocol('electron-test', async (req, cb) => {
if (/redirect/.test(req.url)) return cb({ statusCode: 307, headers: { location: 'electron-test://bar' } });
postedBodyData = req.uploadData![0].bytes.toString();
cb('hello ' + req.url);
});
defer(() => {
protocol.unregisterProtocol('electron-test');
});
const response = await net.fetch('electron-test://redirect', {
method: 'POST',
body: bodyData
});
expect(response.status).to.equal(200);
await response.text();
expect(postedBodyData).to.equal(bodyData);
});
});
});

1
spec/fixtures/hello.txt поставляемый Normal file
Просмотреть файл

@ -0,0 +1 @@
hello world