Add HTTP(S) over HTTP(S) proxy support (#322)

* Add proxy support using tunnel package

# Conflicts:
#	lib/axiosHttpClient.ts
#	lib/policies/proxyPolicy.ts
#	lib/serviceClient.ts
#	package.json

* Fix incorrect merge

* Add tests

* Remove commented code

* Add tunnel to rollup configuration

* Fix test title casing

* Remove only

* Add axios client tests

* Mock buffer

* Remove rewire

* Fix default HTTP client tests

* Add some proxy tests

* Add support for HTTPS proxy

* Address PR comments
This commit is contained in:
Kamil Pajdzik 2019-01-25 09:22:30 -08:00 коммит произвёл GitHub
Родитель 4c2b1c5390
Коммит 1ee5a40d50
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
12 изменённых файлов: 348 добавлений и 63 удалений

Просмотреть файл

@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
import axios, { AxiosError, AxiosRequestConfig, AxiosResponse, AxiosProxyConfig } from "axios";
import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios";
import { Transform, Readable } from "stream";
import FormData from "form-data";
import * as tough from "tough-cookie";
@ -10,9 +10,11 @@ import { HttpHeaders } from "./httpHeaders";
import { HttpOperationResponse } from "./httpOperationResponse";
import { RestError } from "./restError";
import { WebResource, HttpRequestBody } from "./webResource";
import * as tunnel from "tunnel";
import { ProxySettings } from "./serviceClient";
export const axiosClient = axios.create();
import http from "http";
import https from "https";
import { URLBuilder } from "./url";
/**
* A HttpClient implementation that uses axios to send HTTP requests.
@ -130,9 +132,19 @@ export class AxiosHttpClient implements HttpClient {
responseType: httpRequest.streamResponseBody ? "stream" : "text",
cancelToken,
timeout: httpRequest.timeout,
proxy: convertToAxiosProxyConfig(httpRequest.proxySettings)
proxy: false
};
res = await axiosClient(config);
if (httpRequest.proxySettings) {
const agent = createProxyAgent(httpRequest.url, httpRequest.proxySettings, httpRequest.headers);
if (agent.isHttps) {
config.httpsAgent = agent.agent;
} else {
config.httpAgent = agent.agent;
}
}
res = await axios.request(config);
} catch (err) {
if (err instanceof axios.Cancel) {
throw new RestError(err.message, RestError.REQUEST_SEND_ERROR, undefined, httpRequest);
@ -198,25 +210,45 @@ export class AxiosHttpClient implements HttpClient {
}
}
function convertToAxiosProxyConfig(proxySettings: ProxySettings | undefined): AxiosProxyConfig | undefined {
if (!proxySettings) {
return undefined;
}
const axiosAuthConfig = (proxySettings.username && proxySettings.password) ? {
username: proxySettings.username,
password: proxySettings.password
} : undefined;
const axiosProxyConfig: AxiosProxyConfig = {
host: proxySettings.host,
port: proxySettings.port,
auth: axiosAuthConfig
};
return axiosProxyConfig;
}
function isReadableStream(body: any): body is Readable {
return typeof body.pipe === "function";
}
declare type ProxyAgent = { isHttps: boolean; agent: http.Agent | https.Agent };
export function createProxyAgent(requestUrl: string, proxySettings: ProxySettings, headers?: HttpHeaders): ProxyAgent {
const tunnelOptions: tunnel.HttpsOverHttpsOptions = {
proxy: {
host: proxySettings.host,
port: proxySettings.port,
headers: (headers && headers.rawHeaders()) || {}
}
};
if ((proxySettings.username && proxySettings.password)) {
tunnelOptions.proxy!.proxyAuth = `${proxySettings.username}:${proxySettings.password}`;
}
const requestScheme = URLBuilder.parse(requestUrl).getScheme() || "";
const isRequestHttps = requestScheme.toLowerCase() === "https";
const proxyScheme = URLBuilder.parse(proxySettings.host).getScheme() || "";
const isProxyHttps = proxyScheme.toLowerCase() === "https";
const proxyAgent = {
isHttps: isRequestHttps,
agent: createTunnel(isRequestHttps, isProxyHttps, tunnelOptions)
};
return proxyAgent;
}
export function createTunnel(isRequestHttps: boolean, isProxyHttps: boolean, tunnelOptions: tunnel.HttpsOverHttpsOptions): http.Agent | https.Agent {
if (isRequestHttps && isProxyHttps) {
return tunnel.httpsOverHttps(tunnelOptions);
} else if (isRequestHttps && !isProxyHttps) {
return tunnel.httpsOverHttp(tunnelOptions);
} else if (!isRequestHttps && isProxyHttps) {
return tunnel.httpOverHttps(tunnelOptions);
} else {
return tunnel.httpOverHttp(tunnelOptions);
}
}

Просмотреть файл

@ -5,11 +5,47 @@ import { BaseRequestPolicy, RequestPolicy, RequestPolicyFactory, RequestPolicyOp
import { HttpOperationResponse } from "../httpOperationResponse";
import { ProxySettings } from "../serviceClient";
import { WebResource } from "../webResource";
import { Constants } from "../util/constants";
import { URLBuilder } from "../url";
export function proxyPolicy(proxySettings: ProxySettings): RequestPolicyFactory {
function loadEnvironmentProxyValue(): string | undefined {
if (!process) {
return undefined;
}
if (process.env[Constants.HTTPS_PROXY]) {
return process.env[Constants.HTTPS_PROXY];
} else if (process.env[Constants.HTTPS_PROXY.toLowerCase()]) {
return process.env[Constants.HTTPS_PROXY.toLowerCase()];
} else if (process.env[Constants.HTTP_PROXY]) {
return process.env[Constants.HTTP_PROXY];
} else if (process.env[Constants.HTTP_PROXY.toLowerCase()]) {
return process.env[Constants.HTTP_PROXY.toLowerCase()];
}
return undefined;
}
export function getDefaultProxySettings(proxyUrl?: string): ProxySettings | undefined {
if (!proxyUrl) {
proxyUrl = loadEnvironmentProxyValue();
if (!proxyUrl) {
return undefined;
}
}
const parsedUrl = URLBuilder.parse(proxyUrl);
return {
host: parsedUrl.getScheme() + "://" + parsedUrl.getHost(),
port: Number.parseInt(parsedUrl.getPort() || "80")
};
}
export function proxyPolicy(proxySettings?: ProxySettings): RequestPolicyFactory {
return {
create: (nextPolicy: RequestPolicy, options: RequestPolicyOptions) => {
return new ProxyPolicy(nextPolicy, options, proxySettings);
return new ProxyPolicy(nextPolicy, options, proxySettings!);
}
};
}

Просмотреть файл

@ -26,8 +26,8 @@ import { stringifyXML } from "./util/xml";
import { RequestOptionsBase, RequestPrepareOptions, WebResource } from "./webResource";
import { OperationResponse } from "./operationResponse";
import { ServiceCallback } from "./util/utils";
import { proxyPolicy, getDefaultProxySettings } from "./policies/proxyPolicy";
import { throttlingRetryPolicy } from "./policies/throttlingRetryPolicy";
import { proxyPolicy } from "./policies/proxyPolicy";
/**
@ -410,8 +410,9 @@ function createDefaultRequestPolicyFactories(credentials: ServiceClientCredentia
factories.push(deserializationPolicy(options.deserializationContentTypes));
if (options.proxySettings) {
factories.push(proxyPolicy(options.proxySettings));
const proxySettings = options.proxySettings || getDefaultProxySettings();
if (proxySettings) {
factories.push(proxyPolicy(proxySettings));
}
return factories;

Просмотреть файл

@ -52,6 +52,7 @@
"form-data": "^2.3.2",
"tough-cookie": "^2.4.3",
"tslib": "^1.9.2",
"tunnel": "0.0.6",
"uuid": "^3.2.1",
"xml2js": "^0.4.19"
},
@ -67,6 +68,7 @@
"@types/semver": "^5.5.0",
"@types/sinon": "^5.0.6",
"@types/tough-cookie": "^2.3.3",
"@types/tunnel": "0.0.0",
"@types/uuid": "^3.4.4",
"@types/webpack": "^4.4.13",
"@types/webpack-dev-middleware": "^2.0.2",

Просмотреть файл

@ -24,13 +24,14 @@
input: "./es/lib/msRest.js",
external: [
"axios",
"xml2js",
"tough-cookie",
"uuid/v4",
"tslib",
"form-data",
"os",
"stream",
"os"
"tough-cookie",
"tslib",
"tunnel",
"uuid/v4",
"xml2js",
],
output: {
file: "./dist/msRest.node.js",

Просмотреть файл

@ -0,0 +1,111 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
import "chai/register-should";
import { should } from "chai";
import tunnel from "tunnel";
import https from "https";
import { HttpHeaders } from "../lib/msRest";
import { createTunnel, createProxyAgent } from "../lib/axiosHttpClient";
describe("AxiosHttpClient", () => {
describe("createProxyAgent", () => {
type HttpsAgent = https.Agent & {
defaultPort: number | undefined,
options: {
proxy: tunnel.ProxyOptions
},
proxyOptions: tunnel.ProxyOptions
};
[
{ proxy: "http", request: "ftp", port: undefined, isProxyHttps: false },
{ proxy: "http", request: "http", port: undefined, isProxyHttps: false },
{ proxy: "hTtp", request: "https", port: 443, isProxyHttps: true },
{ proxy: "HTTPS", request: "http", port: undefined, isProxyHttps: false },
{ proxy: "https", request: "hTTps", port: 443, isProxyHttps: true }
].forEach(testCase => {
it(`should return ${testCase.isProxyHttps ? "HTTPS" : "HTTP"} proxy for ${testCase.proxy.toUpperCase()} proxy server and ${testCase.request.toUpperCase()} request`, function (done) {
const proxySettings = {
host: `${testCase.proxy}://proxy.microsoft.com`,
port: 8080
};
const requestUrl = `${testCase.request}://example.com`;
const proxyAgent = createProxyAgent(requestUrl, proxySettings);
proxyAgent.isHttps.should.equal(testCase.isProxyHttps);
const agent = proxyAgent.agent as HttpsAgent;
should().equal(agent.defaultPort, testCase.port);
agent.options.proxy.host!.should.equal(proxySettings.host);
agent.options.proxy.port!.should.equal(proxySettings.port);
done();
});
});
it("should copy headers correctly", function (done) {
const proxySettings = {
host: "http://proxy.microsoft.com",
port: 8080
};
const headers = new HttpHeaders({
"User-Agent": "Node.js"
});
const proxyAgent = createProxyAgent("http://example.com", proxySettings, headers);
const agent = proxyAgent.agent as HttpsAgent;
agent.proxyOptions.headers.should.contain({ "user-agent": "Node.js" });
done();
});
});
describe("createTunnel", () => {
const defaultProxySettings = {
host: "http://proxy.microsoft.com",
port: 8080
};
type HttpsAgent = https.Agent & {
defaultPort: number | undefined,
options: {
proxy: tunnel.ProxyOptions
}
};
[true, false].forEach(value => {
it(`returns HTTP agent for HTTP request and HTTP${value ? "S" : ""} proxy`, function () {
const tunnelConfig: tunnel.HttpsOverHttpsOptions = {
proxy: {
host: defaultProxySettings.host,
port: defaultProxySettings.port,
headers: {}
}
};
const tunnel = createTunnel(false, value, tunnelConfig) as HttpsAgent;
tunnel.options.proxy.host!.should.equal(defaultProxySettings.host);
tunnel.options.proxy.port!.should.equal(defaultProxySettings.port);
should().not.exist(tunnel.defaultPort);
});
});
[true, false].forEach(value => {
it(`returns HTTPS agent for HTTPS request and HTTP${value ? "S" : ""} proxy`, function () {
const tunnelConfig: tunnel.HttpsOverHttpsOptions = {
proxy: {
host: defaultProxySettings.host,
port: defaultProxySettings.port,
headers: {}
}
};
const tunnel = createTunnel(true, value, tunnelConfig) as HttpsAgent;
tunnel.options.proxy.host!.should.equal(defaultProxySettings.host);
tunnel.options.proxy.port!.should.equal(defaultProxySettings.port);
tunnel.defaultPort!.should.equal(443);
});
});
});
});

Просмотреть файл

@ -4,15 +4,16 @@
import { assert, AssertionError } from "chai";
import "chai/register-should";
import { createReadStream } from "fs";
import axios from "axios";
import { DefaultHttpClient } from "../lib/defaultHttpClient";
import { RestError } from "../lib/restError";
import { isNode } from "../lib/util/utils";
import { WebResource, HttpRequestBody, TransferProgressEvent } from "../lib/webResource";
import { getHttpMock } from "./mockHttp";
import { getHttpMock, HttpMockFacade } from "./mockHttp";
import { TestFunction } from "mocha";
const nodeIt = isNode ? it : it.skip;
const httpMock = getHttpMock();
const nodeIt = (isNode ? it : it.skip) as TestFunction;
function getAbortController(): AbortController {
let controller: AbortController;
@ -30,8 +31,13 @@ describe("defaultHttpClient", function () {
return new Promise(resolve => setTimeout(resolve, ms));
}
beforeEach(() => httpMock.setup());
let httpMock: HttpMockFacade;
beforeEach(() => {
httpMock = getHttpMock(axios);
httpMock.setup();
});
afterEach(() => httpMock.teardown());
after(() => httpMock.teardown());
it("should return a response instead of throwing for awaited 404", async function () {
const resourceUrl = "/nonexistent/";
@ -138,7 +144,7 @@ describe("defaultHttpClient", function () {
it("for simple bodies", async function () {
httpMock.post("/fileupload", async (_url, _method, body) => {
return { status: 201, body: body, headers: { "Content-Length": "200" } };
return { status: 251, body: body, headers: { "Content-Length": "200" } };
});
const upload: Notified = { notified: false };
@ -152,6 +158,7 @@ describe("defaultHttpClient", function () {
const client = new DefaultHttpClient();
const response = await client.sendRequest(request);
response.should.exist;
response.status.should.equal(251);
upload.notified.should.be.true;
download.notified.should.be.true;
});
@ -167,7 +174,7 @@ describe("defaultHttpClient", function () {
const size = isNode ? payload.toString().length : undefined;
httpMock.post("/fileupload", async (_url, _method, _body) => {
return { status: 201, body: payload, headers: { "Content-Type": "text/javascript", "Content-length": size } };
return { status: 250, body: payload, headers: { "Content-Type": "text/javascript", "Content-length": size } };
});
const upload: Notified = { notified: false };
@ -179,6 +186,7 @@ describe("defaultHttpClient", function () {
const client = new DefaultHttpClient();
const response = await client.sendRequest(request);
response.status.should.equal(250);
if (response.blobBody) {
await response.blobBody;
} else if ((typeof response.readableStreamBody === "function")) {
@ -209,9 +217,9 @@ describe("defaultHttpClient", function () {
});
it("should give a graceful error for nonexistent hosts", async function () {
const requestUrl = "http://foo.notawebsite/";
httpMock.passThrough(requestUrl);
const request = new WebResource(requestUrl);
const requestUrl = "http://fake.domain";
httpMock.passThrough();
const request = new WebResource(requestUrl, "GET");
const client = new DefaultHttpClient();
try {
await client.sendRequest(request);
@ -312,5 +320,6 @@ describe("defaultHttpClient", function () {
assert.strictEqual(
responseBody && responseBody.replace(/\s/g, ""),
expectedResponseBody.replace(/\s/g, ""));
httpMock.teardown();
});
});

Просмотреть файл

@ -4,7 +4,7 @@
import xhrMock, { proxy } from "xhr-mock";
import MockAdapter from "axios-mock-adapter";
import { isNode, HttpMethods } from "../lib/msRest";
import { AxiosRequestConfig } from "axios";
import { AxiosRequestConfig, AxiosInstance } from "axios";
export type UrlFilter = string | RegExp;
@ -18,7 +18,7 @@ export type MockResponseFunction = (url?: string, method?: string, body?: any, h
export type MockResponse = MockResponseData | MockResponseFunction;
interface HttpMockFacade {
export interface HttpMockFacade {
setup(): void;
teardown(): void;
passThrough(url?: UrlFilter): void;
@ -29,22 +29,26 @@ interface HttpMockFacade {
put(url: UrlFilter, response: MockResponse): void;
}
export function getHttpMock(): HttpMockFacade {
return (isNode ? new NodeHttpMock() : new BrowserHttpMock());
export function getHttpMock(axiosInstance?: AxiosInstance): HttpMockFacade {
return (isNode ? new NodeHttpMock(axiosInstance) : new BrowserHttpMock());
}
class NodeHttpMock implements HttpMockFacade {
private _mockAdapter: MockAdapter;
constructor() {
const axiosClient = require("../lib/axiosHttpClient").axiosClient;
this._mockAdapter = new MockAdapter(axiosClient);
constructor(axiosInstance?: AxiosInstance) {
if (!axiosInstance) {
throw new Error("Axios instance cannot be undefined");
}
this._mockAdapter = new MockAdapter(axiosInstance);
}
setup(): void {
this._mockAdapter.reset();
}
teardown(): void {
this._mockAdapter.restore();
}
mockHttpMethod(method: HttpMethods, url: UrlFilter, response: MockResponse): void {

Просмотреть файл

@ -2,15 +2,15 @@
// Licensed under the MIT License. See License.txt in the project root for license information.
import "chai/register-should";
import { should } from "chai";
import { ProxySettings } from "../../lib/serviceClient";
import { RequestPolicyOptions } from "../../lib/policies/requestPolicy";
import { WebResource } from "../../lib/webResource";
import { HttpHeaders } from "../../lib/httpHeaders";
import { proxyPolicy, ProxyPolicy } from "../../lib/policies/proxyPolicy";
import { proxyPolicy, ProxyPolicy, getDefaultProxySettings } from "../../lib/policies/proxyPolicy";
import { Constants } from "../../lib/msRest";
describe("ProxyPolicy", function() {
describe("ProxyPolicy", function () {
const proxySettings: ProxySettings = {
host: "https://example.com",
port: 3030,
@ -36,6 +36,7 @@ describe("ProxyPolicy", function() {
done();
});
it("sets correct proxy settings through constructor", function (done) {
const policy = new ProxyPolicy(emptyRequestPolicy, emptyPolicyOptions, proxySettings);
policy.proxySettings.should.be.deep.equal(proxySettings);
@ -61,4 +62,91 @@ describe("ProxyPolicy", function() {
request.proxySettings!.should.be.deep.equal(requestSpecificProxySettings);
});
});
describe("getDefaultProxySettings", () => {
const proxyUrl = "https://proxy.microsoft.com";
const defaultPort = 80;
it("should return settings with passed address", () => {
const proxySettings: ProxySettings = getDefaultProxySettings(proxyUrl)!;
proxySettings.host.should.equal(proxyUrl);
});
it("should return settings with default port", () => {
const proxySettings: ProxySettings = getDefaultProxySettings(proxyUrl)!;
proxySettings.port.should.equal(defaultPort);
});
it("should return settings with passed port", () => {
const port = 3030;
const proxyUrl = "prot://proxy.microsoft.com";
const proxyUrlWithPort = `${proxyUrl}:${port}`;
const proxySettings: ProxySettings = getDefaultProxySettings(proxyUrlWithPort)!;
proxySettings.host.should.equal(proxyUrl);
proxySettings.port.should.equal(port);
});
describe("with loadEnvironmentProxyValue", () => {
beforeEach(() => {
delete process.env[Constants.HTTP_PROXY];
delete process.env[Constants.HTTPS_PROXY];
delete process.env[Constants.HTTP_PROXY.toLowerCase()];
delete process.env[Constants.HTTPS_PROXY.toLowerCase()];
});
it("should return undefined when no proxy passed and environment variable is not set", () => {
const proxySettings: ProxySettings | undefined = getDefaultProxySettings();
should().not.exist(proxySettings);
});
it("should load settings from environment variables when no proxyUrl passed", () => {
const proxyUrl = "http://proxy.azure.com";
process.env[Constants.HTTP_PROXY] = proxyUrl;
const proxySettings: ProxySettings = getDefaultProxySettings()!;
proxySettings.host.should.equal(proxyUrl);
proxySettings.port.should.equal(defaultPort);
});
describe("should prefer HTTPS proxy over HTTP proxy", () => {
[
{ name: "lower case", func: (envVar: string) => envVar.toLowerCase() },
{ name: "upper case", func: (envVar: string) => envVar.toUpperCase() }
].forEach(testCase => {
it(`with ${testCase.name}`, () => {
const httpProxy = "http://proxy.microsoft.com";
const httpsProxy = "https://proxy.azure.com";
process.env[testCase.func(Constants.HTTP_PROXY)] = httpProxy;
process.env[testCase.func(Constants.HTTPS_PROXY)] = httpsProxy;
const proxySettings: ProxySettings = getDefaultProxySettings()!;
proxySettings.host.should.equal(httpsProxy);
proxySettings.port.should.equal(defaultPort);
});
});
it("should prefer HTTPS proxy over HTTP proxy", () => {
const httpProxy = "http://proxy.microsoft.com";
const httpsProxy = "https://proxy.azure.com";
process.env[Constants.HTTP_PROXY] = httpProxy;
process.env[Constants.HTTPS_PROXY] = httpsProxy;
const proxySettings: ProxySettings = getDefaultProxySettings()!;
proxySettings.host.should.equal(httpsProxy);
proxySettings.port.should.equal(defaultPort);
});
});
["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"].forEach(envVariableName => {
it(`should should load setting from "${envVariableName}" environmental variable`, () => {
process.env[envVariableName] = proxyUrl;
const proxySettings: ProxySettings = getDefaultProxySettings()!;
proxySettings.host.should.equal(proxyUrl);
proxySettings.port.should.equal(defaultPort);
});
});
});
});

Просмотреть файл

@ -12,7 +12,7 @@ const Serializer = new msRest.Serializer({});
const valid_uuid = "ceaafd1e-f936-429f-bbfc-82ee75dddc33";
function stringToByteArray(str: string): Uint8Array {
if (typeof Buffer === "function") {
if (typeof Buffer === "function" && msRest.isNode) {
return Buffer.from(str, "utf-8");
} else {
return new TextEncoder().encode(str);

Просмотреть файл

@ -101,7 +101,7 @@ describe("ServiceClient", function () {
assert.strictEqual(JSON.stringify(response), "{}");
});
it("Should serialize collection:multi query parameters", async function () {
it("should serialize collection:multi query parameters", async function () {
const expected = "?q=1&q=2&q=3";
let request: WebResource;
@ -766,7 +766,7 @@ describe("ServiceClient", function () {
});
function stringToByteArray(str: string): Uint8Array {
if (typeof Buffer === "function") {
if (typeof Buffer === "function" && isNode) {
return Buffer.from(str, "utf-8");
} else {
return new TextEncoder().encode(str);

Просмотреть файл

@ -3,7 +3,7 @@ import * as glob from "glob";
import * as path from "path";
const config: webpack.Configuration = {
entry: glob.sync(path.join(__dirname, "test/**/*.ts")),
entry: glob.sync(path.join(__dirname, "test/**/*[^node\.].ts")),
mode: "development",
devtool: "source-map",
stats: {
@ -46,16 +46,17 @@ const config: webpack.Configuration = {
extensions: [".tsx", ".ts", ".js"]
},
node: {
Buffer: "mock",
dns: false,
fs: "empty",
net: "empty",
path: "empty",
dns: false,
tls: false,
process: "mock",
stream: "empty",
tls: "empty",
tty: false,
tunnel: "empty",
v8: false,
Buffer: false,
process: false,
stream: "empty"
}
};