Improve template performance/reliability (#2007)

This commit is contained in:
Eric Jizba 2020-04-06 13:43:56 -07:00 коммит произвёл GitHub
Родитель 4b57c5db74
Коммит 9f5995d429
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
14 изменённых файлов: 109 добавлений и 88 удалений

Просмотреть файл

@ -976,6 +976,11 @@
"type": "boolean",
"description": "%azureFunctions.showDeployConfirmation%",
"default": true
},
"azureFunctions.requestTimeout": {
"type": "number",
"description": "%azureFunctions.requestTimeout%",
"default": 15
}
}
}

Просмотреть файл

@ -40,8 +40,8 @@
"azureFunctions.openInPortal": "Open in Portal",
"azureFunctions.pickProcess": "Pick Process",
"azureFunctions.pickProcessTimeout": "The timeout (in seconds) to be used when searching for the Azure Functions host process. Since a build is required every time you F5, you may need to adjust this based on how long your build takes.",
"azureFunctions.preDeployTask": "The name of the task to run before zip deployments.",
"azureFunctions.postDeployTask": "The name of the task to run after zip deployments.",
"azureFunctions.preDeployTask": "The name of the task to run before zip deployments.",
"azureFunctions.problemMatchers.funcWatch": "Azure Functions problems (watch mode)",
"azureFunctions.projectLanguage.preview": "(Preview)",
"azureFunctions.projectLanguage": "The default language to use when performing operations like \"Create New Function\".",
@ -54,6 +54,7 @@
"azureFunctions.pythonVenv": "The name of the Python virtual environment used for your project. A virtual environment is required to debug and deploy Python functions.",
"azureFunctions.redeploy": "Redeploy",
"azureFunctions.refresh": "Refresh",
"azureFunctions.requestTimeout": "The timeout (in seconds) to be used when making requests, for example getting the latest templates.",
"azureFunctions.restartFunctionApp": "Restart",
"azureFunctions.scmDoBuildDuringDeployment": "Set to true to build your project on the server. Currently only applicable for Linux Function Apps.",
"azureFunctions.setAzureWebJobsStorage": "Set AzureWebJobsStorage...",

Просмотреть файл

@ -106,7 +106,7 @@ export class PowerShellProjectCreateStep extends ScriptProjectCreateStep {
}
private async getPSGalleryAzModuleInfo(): Promise<string> {
const request: requestUtils.Request = await requestUtils.getDefaultRequest(this.azModuleGalleryUrl, undefined, 'GET');
const request: requestUtils.Request = await requestUtils.getDefaultRequestWithTimeout(this.azModuleGalleryUrl, undefined, 'GET');
return await requestUtils.sendRequest(request);
}

Просмотреть файл

@ -23,7 +23,7 @@ export async function executeFunction(context: IActionContext, node?: FunctionTr
const hostUrl: string = node.parent.parent.hostUrl;
await node.runWithTemporaryDescription(localize('executing', 'Executing...'), async () => {
// https://docs.microsoft.com/azure/azure-functions/functions-manually-run-non-http
const request: requestUtils.Request = await requestUtils.getDefaultRequest(`${hostUrl}/admin/functions/${name}`, undefined, 'POST');
const request: requestUtils.Request = await requestUtils.getDefaultRequestWithTimeout(`${hostUrl}/admin/functions/${name}`, undefined, 'POST');
if (client) {
request.headers['x-functions-key'] = (await client.listHostKeys()).masterKey;
}

Просмотреть файл

@ -18,7 +18,7 @@ interface IPackageMetadata {
}
export async function getNpmDistTag(version: FuncVersion): Promise<INpmDistTag> {
const request: requestUtils.Request = await requestUtils.getDefaultRequest(npmRegistryUri);
const request: requestUtils.Request = await requestUtils.getDefaultRequestWithTimeout(npmRegistryUri);
const packageMetadata: IPackageMetadata = parseJson(await requestUtils.sendRequest(request));
const majorVersion: string = getMajorVersion(version);

Просмотреть файл

@ -107,7 +107,7 @@ async function getNewestFunctionRuntimeVersion(packageManager: PackageManager |
if (packageManager === PackageManager.brew) {
const packageName: string = getBrewPackageName(versionFromSetting);
const brewRegistryUri: string = `https://raw.githubusercontent.com/Azure/homebrew-functions/master/Formula/${packageName}.rb`;
const request: requestUtils.Request = await requestUtils.getDefaultRequest(brewRegistryUri);
const request: requestUtils.Request = await requestUtils.getDefaultRequestWithTimeout(brewRegistryUri);
const brewInfo: string = await requestUtils.sendRequest(request);
const matches: RegExpMatchArray | null = brewInfo.match(/version\s+["']([^"']+)["']/i);
if (matches && matches.length > 1) {

Просмотреть файл

@ -159,7 +159,7 @@ export class CentralTemplateProvider {
if (!this.templateSource || this.templateSource === TemplateSource.Latest || this.templateSource === TemplateSource.Staging) {
context.telemetry.properties.templateSource = 'latest';
const result: ITemplates = await provider.getLatestTemplates(context, latestTemplateVersion);
ext.context.globalState.update(provider.getCacheKey(TemplateProviderBase.templateVersionKey), latestTemplateVersion);
await provider.updateCachedValue(TemplateProviderBase.templateVersionKey, latestTemplateVersion);
await provider.cacheTemplates();
return result;
}
@ -191,7 +191,7 @@ export class CentralTemplateProvider {
const backupTemplateVersion: string = provider.getBackupTemplateVersion();
context.telemetry.properties.backupTemplateVersion = backupTemplateVersion;
const result: ITemplates = await provider.getBackupTemplates(context);
ext.context.globalState.update(provider.getCacheKey(TemplateProviderBase.templateVersionKey), backupTemplateVersion);
await provider.updateCachedValue(TemplateProviderBase.templateVersionKey, backupTemplateVersion);
await provider.cacheTemplates();
return result;
} catch (error) {

Просмотреть файл

@ -34,24 +34,12 @@ export abstract class TemplateProviderBase {
this.projectPath = projectPath;
}
/**
* Adds version, templateType, and language information to a key to ensure there are no collisions in the cache
* For backwards compatability, the original version, templateType, and language will not have this information
*/
public getCacheKey(key: string): string {
if (this.version !== FuncVersion.v1) {
key = `${key}.${this.version}`;
}
public async updateCachedValue(key: string, value: unknown): Promise<void> {
ext.context.globalState.update(await this.getCacheKey(key), value);
}
if (this.templateType !== TemplateType.Script) {
key = `${key}.${this.templateType}`;
}
if (vscode.env.language && !/^en(-us)?$/i.test(vscode.env.language)) {
key = `${key}.${vscode.env.language}`;
}
return key;
public async getCachedValue<T>(key: string): Promise<T | undefined> {
return ext.context.globalState.get<T>(await this.getCacheKey(key));
}
public abstract getLatestTemplateVersion(): Promise<string>;
@ -68,7 +56,7 @@ export abstract class TemplateProviderBase {
}
public async getCachedTemplateVersion(): Promise<string | undefined> {
return ext.context.globalState.get(this.getCacheKey(TemplateProviderBase.templateVersionKey));
return this.getCachedValue(TemplateProviderBase.templateVersionKey);
}
public getBackupTemplateVersion(): string {
@ -83,4 +71,30 @@ export abstract class TemplateProviderBase {
throw new RangeError(localize('invalidVersion', 'Invalid version "{0}".', this.version));
}
}
protected async getCacheKeySuffix(): Promise<string> {
return '';
}
/**
* Adds version, templateType, and language information to a key to ensure there are no collisions in the cache
* For backwards compatability, the original version, templateType, and language will not have this information
*/
private async getCacheKey(key: string): Promise<string> {
key = key + await this.getCacheKeySuffix();
if (this.version !== FuncVersion.v1) {
key = `${key}.${this.version}`;
}
if (this.templateType !== TemplateType.Script) {
key = `${key}.${this.templateType}`;
}
if (vscode.env.language && !/^en(-us)?$/i.test(vscode.env.language)) {
key = `${key}.${vscode.env.language}`;
}
return key;
}
}

Просмотреть файл

@ -8,8 +8,8 @@ import * as path from 'path';
import { IActionContext } from 'vscode-azureextensionui';
import { ext } from '../../extensionVariables';
import { cliFeedUtils } from '../../utils/cliFeedUtils';
import { downloadFile } from '../../utils/fs';
import { parseJson } from '../../utils/parseJson';
import { requestUtils } from '../../utils/requestUtils';
import { ITemplates } from '../ITemplates';
import { TemplateProviderBase, TemplateType } from '../TemplateProviderBase';
import { executeDotnetTemplateCommand, getDotnetItemTemplatePath, getDotnetProjectTemplatePath, getDotnetTemplatesPath, validateDotnetInstalled } from './executeDotnetTemplateCommand';
@ -27,7 +27,7 @@ export class DotnetTemplateProvider extends TemplateProviderBase {
return undefined;
}
const cachedDotnetTemplates: object[] | undefined = ext.context.globalState.get<object[]>(this.getCacheKey(this._dotnetTemplatesKey));
const cachedDotnetTemplates: object[] | undefined = await this.getCachedValue(this._dotnetTemplatesKey);
if (cachedDotnetTemplates) {
return await parseDotnetTemplates(cachedDotnetTemplates, this.version);
} else {
@ -45,10 +45,13 @@ export class DotnetTemplateProvider extends TemplateProviderBase {
const release: cliFeedUtils.IRelease = await cliFeedUtils.getRelease(latestTemplateVersion);
const projectFilePath: string = getDotnetProjectTemplatePath(this.version);
await downloadFile(release.projectTemplates, projectFilePath);
const itemFilePath: string = getDotnetItemTemplatePath(this.version);
await downloadFile(release.itemTemplates, itemFilePath);
await Promise.all([
requestUtils.downloadFile(release.projectTemplates, projectFilePath),
requestUtils.downloadFile(release.itemTemplates, itemFilePath)
]);
return await this.parseTemplates(context);
}
@ -59,7 +62,7 @@ export class DotnetTemplateProvider extends TemplateProviderBase {
}
public async cacheTemplates(): Promise<void> {
ext.context.globalState.update(this.getCacheKey(this._dotnetTemplatesKey), this._rawTemplates);
await this.updateCachedValue(this._dotnetTemplatesKey, this._rawTemplates);
}
private async parseTemplates(context: IActionContext): Promise<ITemplates> {

Просмотреть файл

@ -32,16 +32,17 @@ export class ScriptBundleTemplateProvider extends ScriptTemplateProvider {
const bundleMetadata: IBundleMetadata | undefined = await this.getBundleInfo();
const release: bundleFeedUtils.ITemplatesRelease = await bundleFeedUtils.getRelease(bundleMetadata, latestTemplateVersion);
const bindingsRequest: requestUtils.Request = await requestUtils.getDefaultRequest(release.bindings);
this._rawBindings = parseJson(await requestUtils.sendRequest(bindingsRequest));
const bindingsRequest: requestUtils.Request = await requestUtils.getDefaultRequestWithTimeout(release.bindings);
const language: string = getScriptResourcesLanguage();
const resourcesUrl: string = release.resources.replace('{locale}', language);
const resourcesRequest: requestUtils.Request = await requestUtils.getDefaultRequest(resourcesUrl);
this._rawResources = parseJson(await requestUtils.sendRequest(resourcesRequest));
const resourcesRequest: requestUtils.Request = await requestUtils.getDefaultRequestWithTimeout(resourcesUrl);
const templatesRequest: requestUtils.Request = await requestUtils.getDefaultRequest(release.functions);
this._rawTemplates = parseJson(await requestUtils.sendRequest(templatesRequest));
const templatesRequest: requestUtils.Request = await requestUtils.getDefaultRequestWithTimeout(release.functions);
[this._rawBindings, this._rawResources, this._rawTemplates] = <[object, object, object[]]>(
await Promise.all([bindingsRequest, resourcesRequest, templatesRequest].map(requestUtils.sendRequest))
).map(parseJson);
return parseScriptTemplates(this._rawResources, this._rawTemplates, this._rawBindings);
}

Просмотреть файл

@ -11,7 +11,8 @@ import { ext } from '../../extensionVariables';
import { FuncVersion } from '../../FuncVersion';
import { bundleFeedUtils } from '../../utils/bundleFeedUtils';
import { cliFeedUtils } from '../../utils/cliFeedUtils';
import { downloadFile, getRandomHexString } from '../../utils/fs';
import { getRandomHexString } from '../../utils/fs';
import { requestUtils } from '../../utils/requestUtils';
import { IBindingTemplate } from '../IBindingTemplate';
import { IFunctionTemplate } from '../IFunctionTemplate';
import { ITemplates } from '../ITemplates';
@ -32,9 +33,9 @@ export class ScriptTemplateProvider extends TemplateProviderBase {
private readonly _resourcesKey: string = 'FunctionTemplateResources';
public async getCachedTemplates(): Promise<ITemplates | undefined> {
const cachedResources: object | undefined = ext.context.globalState.get<object>(this.getCacheKey(this._resourcesKey));
const cachedTemplates: object[] | undefined = ext.context.globalState.get<object[]>(this.getCacheKey(this._templatesKey));
const cachedConfig: object | undefined = ext.context.globalState.get<object>(this.getCacheKey(this._bindingsKey));
const cachedResources: object | undefined = await this.getCachedValue(this._resourcesKey);
const cachedTemplates: object[] | undefined = await this.getCachedValue(this._templatesKey);
const cachedConfig: object | undefined = await this.getCachedValue(this._bindingsKey);
if (cachedResources && cachedTemplates && cachedConfig) {
return parseScriptTemplates(cachedResources, cachedTemplates, cachedConfig);
} else {
@ -52,7 +53,7 @@ export class ScriptTemplateProvider extends TemplateProviderBase {
const templatesPath: string = path.join(ext.context.globalStoragePath, 'scriptTemplates');
try {
const filePath: string = path.join(templatesPath, `${getRandomHexString()}.zip`);
await downloadFile(templateRelease.templateApiZip, filePath);
await requestUtils.downloadFile(templateRelease.templateApiZip, filePath);
await new Promise(async (resolve: () => void, reject: (e: Error) => void): Promise<void> => {
// tslint:disable-next-line:no-unsafe-any
@ -79,20 +80,15 @@ export class ScriptTemplateProvider extends TemplateProviderBase {
}
public async cacheTemplates(): Promise<void> {
const suffix: string = await this.getCacheKeySuffix();
ext.context.globalState.update(this.getCacheKey(this._templatesKey + suffix), this._rawTemplates);
ext.context.globalState.update(this.getCacheKey(this._bindingsKey + suffix), this._rawBindings);
ext.context.globalState.update(this.getCacheKey(this._resourcesKey + suffix), this._rawResources);
await this.updateCachedValue(this._templatesKey, this._rawTemplates);
await this.updateCachedValue(this._bindingsKey, this._rawBindings);
await this.updateCachedValue(this._resourcesKey, this._rawResources);
}
public includeTemplate(template: IFunctionTemplate | IBindingTemplate): boolean {
return this.version === FuncVersion.v1 || !bundleFeedUtils.isBundleTemplate(template);
}
protected async getCacheKeySuffix(): Promise<string> {
return '';
}
protected async parseTemplates(templatesPath: string): Promise<ITemplates> {
const language: string = getScriptResourcesLanguage();
// Unlike templates.json and bindings.json, Resources.json has a capital letter

Просмотреть файл

@ -3,8 +3,6 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { parseError } from 'vscode-azureextensionui';
import { localize } from '../localize';
import { parseJson } from './parseJson';
import { requestUtils } from './requestUtils';
@ -24,19 +22,9 @@ export namespace feedUtils {
export async function getJsonFeed<T extends {}>(url: string): Promise<T> {
let cachedFeed: ICachedFeed | undefined = cachedFeeds.get(url);
if (!cachedFeed || Date.now() > cachedFeed.nextRefreshTime) {
const request: requestUtils.Request = await requestUtils.getDefaultRequest(url);
request.timeout = 15 * 1000;
const request: requestUtils.Request = await requestUtils.getDefaultRequestWithTimeout(url);
let response: string;
try {
response = await requestUtils.sendRequest(request);
} catch (error) {
if (parseError(error).errorType === 'ETIMEDOUT') {
throw new Error(localize('timeoutFeed', 'Timed out retrieving feed "{0}".', url));
} else {
throw error;
}
}
const response: string = await requestUtils.sendRequest(request);
cachedFeed = { data: parseJson(response), nextRefreshTime: Date.now() + 10 * 60 * 1000 };
cachedFeeds.set(url, cachedFeed);
}

Просмотреть файл

@ -6,8 +6,6 @@
import * as crypto from "crypto";
import * as fse from 'fs-extra';
import * as path from 'path';
// tslint:disable-next-line:no-require-imports
import request = require('request-promise');
import { MessageItem } from "vscode";
import { DialogResponses } from "vscode-azureextensionui";
import { ext } from "../extensionVariables";
@ -82,24 +80,3 @@ export function isSubpath(expectedParent: string, expectedChild: string, relativ
}
type pathRelativeFunc = (fsPath1: string, fsPath2: string) => string;
export async function downloadFile(url: string, filePath: string): Promise<void> {
return new Promise<void>(async (resolve: () => void, reject: (e: Error) => void): Promise<void> => {
const templateOptions: request.OptionsWithUri = {
method: 'GET',
uri: url
};
await fse.ensureDir(path.dirname(filePath));
request(templateOptions, (err: Error) => {
// tslint:disable-next-line:strict-boolean-expressions
if (err) {
reject(err);
}
}).pipe(fse.createWriteStream(filePath).on('finish', () => {
resolve();
}).on('error', (error: Error) => {
reject(error);
}));
});
}

Просмотреть файл

@ -3,13 +3,29 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as fse from 'fs-extra';
import { HttpMethods, ServiceClientCredentials, WebResource } from "ms-rest";
import * as path from 'path';
import * as requestP from 'request-promise';
import { appendExtensionUserAgent, ISubscriptionContext } from "vscode-azureextensionui";
import { appendExtensionUserAgent, ISubscriptionContext, parseError } from "vscode-azureextensionui";
import { ext } from '../extensionVariables';
import { localize } from '../localize';
import { getWorkspaceSetting } from "../vsCodeConfig/settings";
export namespace requestUtils {
export type Request = WebResource & requestP.RequestPromiseOptions;
const timeoutKey: string = 'requestTimeout';
export async function getDefaultRequestWithTimeout(url: string, credentials?: ServiceClientCredentials, method: HttpMethods = 'GET'): Promise<Request> {
const request: Request = await getDefaultRequest(url, credentials, method);
const timeoutSeconds: number | undefined = getWorkspaceSetting(timeoutKey);
if (timeoutSeconds !== undefined) {
request.timeout = timeoutSeconds * 1000;
}
return request;
}
export async function getDefaultRequest(url: string, credentials?: ServiceClientCredentials, method: HttpMethods = 'GET'): Promise<Request> {
const request: WebResource = new WebResource();
request.url = url;
@ -39,7 +55,15 @@ export namespace requestUtils {
}
export async function sendRequest(request: Request): Promise<string> {
return await <Thenable<string>>requestP(request).promise();
try {
return await <Thenable<string>>requestP(request).promise();
} catch (error) {
if (parseError(error).errorType === 'ETIMEDOUT') {
throw new Error(localize('timeoutFeed', 'Request timed out. Modify setting "{0}.{1}" if you want to extend the timeout.', ext.prefix, timeoutKey));
} else {
throw error;
}
}
}
export async function signRequest(request: Request, cred: ServiceClientCredentials): Promise<void> {
@ -53,4 +77,16 @@ export namespace requestUtils {
});
});
}
export async function downloadFile(url: string, filePath: string): Promise<void> {
const request: Request = await getDefaultRequestWithTimeout(url);
await fse.ensureDir(path.dirname(filePath));
await new Promise(async (resolve, reject): Promise<void> => {
requestP(request, err => {
if (err) {
reject(err);
}
}).pipe(fse.createWriteStream(filePath).on('finish', resolve).on('error', reject));
});
}
}