Generate example (#833)
* generateExample * fix * use translator * scenario * rename * run * test * _swaggerFilePaths * generateExample * fix bug * changelog * 3.0.3 * fix bug
This commit is contained in:
Lei Ni
GitHub
Родитель
c6e95c45fe
Коммит
0e0d49c5b6
|
|
@ -1,5 +1,8 @@
|
|||
# Change Log - oav
|
||||
|
||||
## 07/06/2022 3.0.3
|
||||
- Generate high quality examples from API Scenario tests
|
||||
|
||||
## 06/30/2022 3.0.2
|
||||
|
||||
- traffic-converter
|
||||
|
|
|
|||
|
|
@ -238,6 +238,7 @@ export class ApiScenarioLoader implements Loader<ScenarioDefinition> {
|
|||
prepareSteps: [],
|
||||
scenarios: [],
|
||||
_filePath: this.fileLoader.relativePath(filePath),
|
||||
_swaggerFilePaths: this.opts.swaggerFilePaths!,
|
||||
cleanUpSteps: [],
|
||||
...convertVariables(rawDef.variables),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -252,6 +252,10 @@ export class ApiScenarioRunner {
|
|||
}
|
||||
|
||||
await this.client.sendRestCallRequest(req, step, env);
|
||||
|
||||
if (this.resolveVariables && !step._resolvedParameters) {
|
||||
step._resolvedParameters = env.resolveObjectValues(step.parameters);
|
||||
}
|
||||
}
|
||||
|
||||
private async executeArmTemplateStep(step: StepArmTemplate, env: VariableEnv, scope: Scope) {
|
||||
|
|
|
|||
|
|
@ -136,6 +136,7 @@ export type StepRestCall = StepBase & {
|
|||
parameters: SwaggerExample["parameters"];
|
||||
responses: SwaggerExample["responses"];
|
||||
outputVariables?: OutputVariables;
|
||||
_resolvedParameters?: SwaggerExample["parameters"];
|
||||
};
|
||||
|
||||
//#endregion
|
||||
|
|
@ -306,32 +307,31 @@ export type ScenarioDefinition = TransformRaw<
|
|||
scenarios: Scenario[];
|
||||
cleanUpSteps: Step[];
|
||||
_filePath: string;
|
||||
_swaggerFilePaths: string[];
|
||||
}
|
||||
>;
|
||||
//#endregion
|
||||
|
||||
//#region Runner specific types
|
||||
export interface RawReport {
|
||||
executions: RawExecution[];
|
||||
export interface NewmanReport {
|
||||
executions: NewmanExecution[];
|
||||
timings: any;
|
||||
variables: { [variableName: string]: Variable };
|
||||
testScenarioName?: string;
|
||||
metadata: any;
|
||||
}
|
||||
|
||||
export interface RawExecution {
|
||||
request: RawRequest;
|
||||
response: RawResponse;
|
||||
export interface NewmanExecution {
|
||||
request: NewmanRequest;
|
||||
response: NewmanResponse;
|
||||
annotation?: any;
|
||||
}
|
||||
export interface RawRequest {
|
||||
export interface NewmanRequest {
|
||||
url: string;
|
||||
method: string;
|
||||
headers: { [key: string]: any };
|
||||
body: string;
|
||||
}
|
||||
|
||||
export interface RawResponse {
|
||||
export interface NewmanResponse {
|
||||
statusCode: number;
|
||||
headers: { [key: string]: any };
|
||||
body: string;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { injectable } from "inversify";
|
||||
import { TestScenarioResult } from "./reportGenerator";
|
||||
import { ApiScenarioTestResult } from "./newmanReportValidator";
|
||||
import { generateJUnitCaseReport } from "./markdownReport";
|
||||
|
||||
@injectable()
|
||||
|
|
@ -9,15 +9,15 @@ export class JUnitReporter {
|
|||
this.builder = require("junit-report-builder");
|
||||
}
|
||||
|
||||
public addSuiteToBuild = async (tsr: TestScenarioResult, path: string) => {
|
||||
public addSuiteToBuild = async (tsr: ApiScenarioTestResult, path: string) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const suite = this.builder.testSuite().name(tsr.testScenarioName);
|
||||
const suite = this.builder.testSuite().name(tsr.apiScenarioName);
|
||||
tsr.stepResult.forEach((sr) => {
|
||||
const tc = suite
|
||||
.testCase()
|
||||
.className(tsr.testScenarioName)
|
||||
.name(`${tsr.testScenarioName}.${sr.stepName}`)
|
||||
.className(tsr.apiScenarioName)
|
||||
.name(`${tsr.apiScenarioName}.${sr.stepName}`)
|
||||
.file(sr.exampleFilePath);
|
||||
if (sr.runtimeError && sr.runtimeError.length > 0) {
|
||||
const detail = generateJUnitCaseReport(sr);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,12 @@ import {
|
|||
LiveValidationIssue,
|
||||
RequestResponseLiveValidationResult,
|
||||
} from "../liveValidation/liveValidator";
|
||||
import { ResponseDiffItem, RuntimeError, StepResult, TestScenarioResult } from "./reportGenerator";
|
||||
import {
|
||||
ResponseDiffItem,
|
||||
RuntimeError,
|
||||
StepResult,
|
||||
ApiScenarioTestResult,
|
||||
} from "./newmanReportValidator";
|
||||
|
||||
const spaceReg = /(\n|\t|\r)/gi;
|
||||
|
||||
|
|
@ -178,7 +183,7 @@ const asMarkdownStepResult = (sr: StepResult): TestScenarioMarkdownStepResult =>
|
|||
return r;
|
||||
};
|
||||
|
||||
const asMarkdownResult = (tsr: TestScenarioResult): TestScenarioMarkdownResult => {
|
||||
const asMarkdownResult = (tsr: ApiScenarioTestResult): TestScenarioMarkdownResult => {
|
||||
const fatalCount = tsr.stepResult.filter(
|
||||
(sr) => sr.runtimeError && sr.runtimeError.length > 0
|
||||
).length;
|
||||
|
|
@ -193,7 +198,7 @@ const asMarkdownResult = (tsr: TestScenarioResult): TestScenarioMarkdownResult =
|
|||
}
|
||||
|
||||
const r: TestScenarioMarkdownResult = {
|
||||
testScenarioName: tsr.testScenarioName!,
|
||||
testScenarioName: tsr.apiScenarioName!,
|
||||
result: resultState,
|
||||
swaggerFilePaths: tsr.swaggerFilePaths,
|
||||
startTime: new Date(tsr.startTime!),
|
||||
|
|
@ -209,7 +214,7 @@ const asMarkdownResult = (tsr: TestScenarioResult): TestScenarioMarkdownResult =
|
|||
};
|
||||
|
||||
export const generateMarkdownReportHeader = (): string => "<h3>Azure API Test Report</h3>";
|
||||
export const generateMarkdownReport = (testScenarioResult: TestScenarioResult): string => {
|
||||
export const generateMarkdownReport = (testScenarioResult: ApiScenarioTestResult): string => {
|
||||
const result = asMarkdownResult(testScenarioResult);
|
||||
const body = generateMarkdownReportView(result);
|
||||
return body;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
import {
|
||||
RequestDefinition,
|
||||
ResponseDefinition,
|
||||
ItemDefinition,
|
||||
Request,
|
||||
Response,
|
||||
DescriptionDefinition,
|
||||
} from "postman-collection";
|
||||
import { NewmanExecution, NewmanReport, NewmanRequest, NewmanResponse } from "./apiScenarioTypes";
|
||||
|
||||
export interface RawNewmanReport {
|
||||
run: Run;
|
||||
environment: any;
|
||||
collection: any;
|
||||
}
|
||||
|
||||
interface Run {
|
||||
executions: RawNewmanExecution[];
|
||||
timings: { started: number; completed: number; responseAverage: number };
|
||||
}
|
||||
|
||||
interface RawNewmanExecution {
|
||||
item: ItemDefinition;
|
||||
request: RequestDefinition;
|
||||
response: ResponseDefinition;
|
||||
}
|
||||
|
||||
export function parseNewmanReport(newmanReport: RawNewmanReport): NewmanReport {
|
||||
const ret: NewmanReport = { variables: {}, executions: [], timings: {} };
|
||||
for (const it of newmanReport.run.executions) {
|
||||
ret.executions.push(generateExampleItem(it));
|
||||
}
|
||||
ret.timings = newmanReport.run.timings;
|
||||
ret.variables = parseVariables(newmanReport.environment.values);
|
||||
return ret;
|
||||
}
|
||||
|
||||
function generateExampleItem(it: RawNewmanExecution): NewmanExecution {
|
||||
const resp = new Response(it.response);
|
||||
const req = new Request(it.request);
|
||||
const rawReq = parseRequest(req);
|
||||
const rawResp = parseResponse(resp);
|
||||
const annotation = JSON.parse((it.item.description as DescriptionDefinition)?.content || "{}");
|
||||
return {
|
||||
request: rawReq,
|
||||
response: rawResp,
|
||||
annotation: annotation,
|
||||
};
|
||||
}
|
||||
|
||||
function parseRequest(req: Request): NewmanRequest {
|
||||
const ret: NewmanRequest = {
|
||||
url: "",
|
||||
method: "",
|
||||
headers: [],
|
||||
body: "",
|
||||
};
|
||||
ret.url = req.url.toString();
|
||||
ret.headers = parseHeader(req.headers.toJSON());
|
||||
ret.method = req.method;
|
||||
ret.body = req.body?.toString() || "";
|
||||
return ret;
|
||||
}
|
||||
|
||||
function parseResponse(resp: Response): NewmanResponse {
|
||||
const ret: NewmanResponse = {
|
||||
headers: [],
|
||||
statusCode: resp.code,
|
||||
body: "",
|
||||
};
|
||||
ret.headers = parseHeader(resp.headers.toJSON());
|
||||
|
||||
ret.body = resp.stream?.toString() || "";
|
||||
return ret;
|
||||
}
|
||||
|
||||
function parseHeader(headers: any[]) {
|
||||
const ret: any = {};
|
||||
for (const it of headers) {
|
||||
ret[it.key] = it.value;
|
||||
|
||||
// Currently only mask bearer token header.
|
||||
// For further sensitive data, should add mask module here
|
||||
if (it.key === "Authorization") {
|
||||
ret[it.key] = "<bearer token>";
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
function parseVariables(environment: any[]) {
|
||||
const ret: any = {};
|
||||
for (const it of environment) {
|
||||
if (it.type === "string" || it.type === "any") {
|
||||
ret[it.key] = { type: "string", value: it.value };
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -1,15 +1,7 @@
|
|||
import * as path from "path";
|
||||
import * as uuid from "uuid";
|
||||
import * as _ from "lodash";
|
||||
import { injectable, inject } from "inversify";
|
||||
import { findReadMe } from "@azure/openapi-markdown";
|
||||
import { ExampleQualityValidator } from "../exampleQualityValidator/exampleQualityValidator";
|
||||
import { setDefaultOpts } from "../swagger/loader";
|
||||
import { getApiVersionFromSwaggerPath, getProviderFromFilePath } from "../util/utils";
|
||||
import { SeverityString } from "../util/severity";
|
||||
import { FileLoader } from "../swagger/fileLoader";
|
||||
import { inject, injectable } from "inversify";
|
||||
import { TYPES } from "../inversifyUtils";
|
||||
import { SwaggerExample } from "../swagger/swaggerTypes";
|
||||
import {
|
||||
LiveValidationIssue,
|
||||
LiveValidator,
|
||||
|
|
@ -18,40 +10,35 @@ import {
|
|||
} from "../liveValidation/liveValidator";
|
||||
import { LiveRequest, LiveResponse } from "../liveValidation/operationValidator";
|
||||
import { ReportGenerator as HtmlReportGenerator } from "../report/generateReport";
|
||||
import { FileLoader } from "../swagger/fileLoader";
|
||||
import { setDefaultOpts } from "../swagger/loader";
|
||||
import { SwaggerExample } from "../swagger/swaggerTypes";
|
||||
import {
|
||||
OperationCoverageInfo,
|
||||
RuntimeException,
|
||||
TrafficValidationIssue,
|
||||
TrafficValidationOptions,
|
||||
unCoveredOperationsFormat,
|
||||
} from "../swaggerValidator/trafficValidator";
|
||||
import { RuntimeException } from "../util/validationError";
|
||||
import { SwaggerAnalyzer } from "./swaggerAnalyzer";
|
||||
import { SeverityString } from "../util/severity";
|
||||
import { getApiVersionFromFilePath, getProviderFromFilePath } from "../util/utils";
|
||||
import { ApiScenarioLoaderOption } from "./apiScenarioLoader";
|
||||
import { NewmanExecution, NewmanReport, Scenario, Step, StepRestCall } from "./apiScenarioTypes";
|
||||
import { DataMasker } from "./dataMasker";
|
||||
import { defaultQualityReportFilePath } from "./defaultNaming";
|
||||
import { ApiScenarioLoader, ApiScenarioLoaderOption } from "./apiScenarioLoader";
|
||||
import { NewmanReportParser, NewmanReportParserOption } from "./postmanReportParser";
|
||||
import {
|
||||
RawReport,
|
||||
RawExecution,
|
||||
ScenarioDefinition,
|
||||
Step,
|
||||
StepRestCall,
|
||||
Variable,
|
||||
} from "./apiScenarioTypes";
|
||||
import { VariableEnv } from "./variableEnv";
|
||||
import { getJsonPatchDiff } from "./diffUtils";
|
||||
import { generateMarkdownReport } from "./markdownReport";
|
||||
import { JUnitReporter } from "./junitReport";
|
||||
import { generateMarkdownReport } from "./markdownReport";
|
||||
import { SwaggerAnalyzer } from "./swaggerAnalyzer";
|
||||
import { VariableEnv } from "./variableEnv";
|
||||
|
||||
interface GeneratedExample {
|
||||
exampleFilePath: string;
|
||||
step: string;
|
||||
operationId: string;
|
||||
example: SwaggerExample;
|
||||
}
|
||||
|
||||
export interface TestScenarioResult {
|
||||
testScenarioFilePath: string;
|
||||
export interface ApiScenarioTestResult {
|
||||
apiScenarioFilePath: string;
|
||||
readmeFilePath?: string;
|
||||
swaggerFilePaths: string[];
|
||||
tag?: string;
|
||||
|
|
@ -69,8 +56,8 @@ export interface TestScenarioResult {
|
|||
repository?: string;
|
||||
branch?: string;
|
||||
commitHash?: string;
|
||||
armEndpoint: string;
|
||||
testScenarioName?: string;
|
||||
baseUrl?: string;
|
||||
apiScenarioName?: string;
|
||||
stepResult: StepResult[];
|
||||
}
|
||||
|
||||
|
|
@ -104,99 +91,88 @@ export interface ResponseDiffItem {
|
|||
|
||||
export type ValidationLevel = "validate-request" | "validate-request-response";
|
||||
|
||||
export interface ReportGeneratorOption extends NewmanReportParserOption, ApiScenarioLoaderOption {
|
||||
export interface NewmanReportValidatorOption extends ApiScenarioLoaderOption {
|
||||
apiScenarioFilePath: string;
|
||||
reportOutputFilePath?: string;
|
||||
reportOutputFilePath: string;
|
||||
markdownReportPath?: string;
|
||||
junitReportPath?: string;
|
||||
htmlReportPath?: string;
|
||||
apiScenarioName?: string;
|
||||
baseUrl?: string;
|
||||
runId?: string;
|
||||
validationLevel?: ValidationLevel;
|
||||
savePayload?: boolean;
|
||||
generateExample?: boolean;
|
||||
verbose?: boolean;
|
||||
swaggerFilePaths?: string[];
|
||||
}
|
||||
|
||||
@injectable()
|
||||
export class ReportGenerator {
|
||||
private exampleQualityValidator: ExampleQualityValidator;
|
||||
private swaggerExampleQualityResult: TestScenarioResult;
|
||||
private testDefFile: ScenarioDefinition | undefined;
|
||||
private operationIds: Set<string>;
|
||||
private rawReport: RawReport | undefined;
|
||||
export class NewmanReportValidator {
|
||||
private scenario: Scenario;
|
||||
private testResult: ApiScenarioTestResult;
|
||||
private fileRoot: string;
|
||||
private recording: Map<string, RawExecution>;
|
||||
private liveValidator: LiveValidator;
|
||||
private trafficValidationResult: TrafficValidationIssue[];
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-member-accessibility
|
||||
constructor(
|
||||
@inject(TYPES.opts) private opts: ReportGeneratorOption,
|
||||
private postmanReportParser: NewmanReportParser,
|
||||
private testResourceLoader: ApiScenarioLoader,
|
||||
@inject(TYPES.opts) private opts: NewmanReportValidatorOption,
|
||||
private fileLoader: FileLoader,
|
||||
private dataMasker: DataMasker,
|
||||
private swaggerAnalyzer: SwaggerAnalyzer,
|
||||
private junitReporter: JUnitReporter
|
||||
) {
|
||||
setDefaultOpts(this.opts, {
|
||||
newmanReportFilePath: "",
|
||||
reportOutputFilePath: defaultQualityReportFilePath(this.opts.newmanReportFilePath),
|
||||
apiScenarioFilePath: "",
|
||||
runId: uuid.v4(),
|
||||
validationLevel: "validate-request-response",
|
||||
savePayload: false,
|
||||
generateExample: false,
|
||||
verbose: false,
|
||||
});
|
||||
} as NewmanReportValidatorOption);
|
||||
}
|
||||
|
||||
public async initialize() {
|
||||
const swaggerFileAbsolutePaths = this.opts.swaggerFilePaths!;
|
||||
this.exampleQualityValidator = ExampleQualityValidator.create({
|
||||
swaggerFilePaths: [...swaggerFileAbsolutePaths],
|
||||
});
|
||||
this.recording = new Map<string, RawExecution>();
|
||||
this.operationIds = new Set<string>();
|
||||
this.fileRoot = (await findReadMe(this.opts.apiScenarioFilePath)) || "/";
|
||||
public async initialize(scenario: Scenario) {
|
||||
this.scenario = scenario;
|
||||
|
||||
this.swaggerExampleQualityResult = {
|
||||
testScenarioFilePath: path.relative(this.fileRoot, this.opts.apiScenarioFilePath),
|
||||
this.fileRoot =
|
||||
(await findReadMe(this.opts.apiScenarioFilePath)) ||
|
||||
path.dirname(this.opts.apiScenarioFilePath);
|
||||
|
||||
this.testResult = {
|
||||
apiScenarioFilePath: path.relative(this.fileRoot, this.opts.apiScenarioFilePath),
|
||||
swaggerFilePaths: this.opts.swaggerFilePaths!,
|
||||
providerNamespace: getProviderFromFilePath(this.opts.apiScenarioFilePath),
|
||||
apiVersion: getApiVersionFromSwaggerPath(
|
||||
this.fileLoader.resolvePath(this.opts.swaggerFilePaths![0])
|
||||
),
|
||||
apiVersion: getApiVersionFromFilePath(this.opts.apiScenarioFilePath),
|
||||
runId: this.opts.runId,
|
||||
rootPath: this.fileRoot,
|
||||
repository: process.env.SPEC_REPOSITORY,
|
||||
branch: process.env.SPEC_BRANCH,
|
||||
commitHash: process.env.COMMIT_HASH,
|
||||
environment: process.env.ENVIRONMENT || "test",
|
||||
testScenarioName: this.opts.apiScenarioName,
|
||||
armEndpoint: "https://management.azure.com",
|
||||
apiScenarioName: this.scenario.scenario,
|
||||
baseUrl: this.opts.baseUrl,
|
||||
stepResult: [],
|
||||
};
|
||||
// this.testDefFile = undefined;
|
||||
|
||||
await this.swaggerAnalyzer.initialize();
|
||||
|
||||
this.liveValidator = new LiveValidator({
|
||||
fileRoot: "/",
|
||||
swaggerPaths: this.opts.swaggerFilePaths!,
|
||||
swaggerPaths: [...this.opts.swaggerFilePaths!],
|
||||
});
|
||||
|
||||
this.rawReport = await this.postmanReportParser.generateRawReport(
|
||||
this.opts.newmanReportFilePath
|
||||
);
|
||||
await this.liveValidator.initialize();
|
||||
}
|
||||
|
||||
public async generateTestScenarioResult(rawReport: RawReport) {
|
||||
await this.initialize();
|
||||
await this.liveValidator.initialize();
|
||||
public async generateReport(rawReport: NewmanReport) {
|
||||
await this.generateApiScenarioTestResult(rawReport);
|
||||
|
||||
await this.outputReport();
|
||||
}
|
||||
|
||||
private async generateApiScenarioTestResult(newmanReport: NewmanReport) {
|
||||
this.trafficValidationResult = [];
|
||||
const variables = rawReport.variables;
|
||||
this.swaggerExampleQualityResult.startTime = new Date(rawReport.timings.started).toISOString();
|
||||
this.swaggerExampleQualityResult.endTime = new Date(rawReport.timings.completed).toISOString();
|
||||
this.swaggerExampleQualityResult.subscriptionId = variables.subscriptionId.value as string;
|
||||
for (const it of rawReport.executions) {
|
||||
const variables = newmanReport.variables;
|
||||
this.testResult.startTime = new Date(newmanReport.timings.started).toISOString();
|
||||
this.testResult.endTime = new Date(newmanReport.timings.completed).toISOString();
|
||||
this.testResult.subscriptionId = variables.subscriptionId.value as string;
|
||||
for (const it of newmanReport.executions) {
|
||||
if (it.annotation === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -208,6 +184,7 @@ export class ReportGenerator {
|
|||
runtimeExceptions: [],
|
||||
errors: [],
|
||||
};
|
||||
// Runtime errors
|
||||
if (it.response.statusCode >= 400) {
|
||||
const error = this.getRuntimeError(it);
|
||||
runtimeError.push(error);
|
||||
|
|
@ -219,45 +196,60 @@ export class ReportGenerator {
|
|||
|
||||
trafficValidationIssue.specFilePath = matchedStep.operation._path._spec._filePath;
|
||||
|
||||
let generatedExample = undefined;
|
||||
let roundtripErrors = undefined;
|
||||
const payload = this.convertToLiveValidationPayload(it);
|
||||
|
||||
let responseDiffResult: ResponseDiffItem[] | undefined = undefined;
|
||||
if (it.annotation.exampleName) {
|
||||
generatedExample = this.generateExample(
|
||||
it,
|
||||
variables,
|
||||
rawReport,
|
||||
matchedStep,
|
||||
it.annotation.exampleName
|
||||
);
|
||||
// validate real payload.
|
||||
roundtripErrors = (
|
||||
await this.exampleQualityValidator.validateExternalExamples([
|
||||
{
|
||||
exampleFilePath: generatedExample.exampleFilePath,
|
||||
example: generatedExample.example,
|
||||
operationId: matchedStep.operationId,
|
||||
const statusCode = `${it.response.statusCode}`;
|
||||
const exampleFilePath = `../examples/${matchedStep.operationId}_${statusCode}.json`;
|
||||
if (this.opts.generateExample || it.annotation.exampleName) {
|
||||
const generatedExample: SwaggerExample = {
|
||||
operationId: matchedStep.operationId,
|
||||
title: matchedStep.step,
|
||||
description: matchedStep.description,
|
||||
parameters: matchedStep._resolvedParameters!,
|
||||
responses: {
|
||||
[statusCode]: {
|
||||
headers: payload.liveResponse.headers,
|
||||
body: payload.liveResponse.body,
|
||||
},
|
||||
])
|
||||
).map((it) => _.omit(it, ["exampleName", "exampleFilePath"]));
|
||||
responseDiffResult =
|
||||
this.opts.validationLevel === "validate-request-response"
|
||||
? await this.exampleResponseDiff(generatedExample, matchedStep)
|
||||
: [];
|
||||
},
|
||||
};
|
||||
|
||||
// Example validation
|
||||
if (this.opts.generateExample) {
|
||||
await this.fileLoader.writeFile(
|
||||
path.resolve(path.dirname(this.opts.reportOutputFilePath), exampleFilePath),
|
||||
JSON.stringify(generatedExample, null, 2)
|
||||
);
|
||||
}
|
||||
if (it.annotation.exampleName) {
|
||||
// validate real payload.
|
||||
responseDiffResult =
|
||||
this.opts.validationLevel === "validate-request-response"
|
||||
? await this.exampleResponseDiff(
|
||||
{
|
||||
step: matchedStep.step,
|
||||
operationId: matchedStep.operationId,
|
||||
example: generatedExample,
|
||||
},
|
||||
matchedStep,
|
||||
newmanReport
|
||||
)
|
||||
: [];
|
||||
}
|
||||
}
|
||||
|
||||
// Schema validation
|
||||
const correlationId = it.response.headers["x-ms-correlation-request-id"];
|
||||
const pair = this.convertToLiveValidationPayload(it);
|
||||
if (this.opts.savePayload) {
|
||||
const payloadFilePath = `./payloads/${matchedStep.step}_${correlationId}.json`;
|
||||
await this.fileLoader.writeFile(
|
||||
path.resolve(
|
||||
path.dirname(this.opts.newmanReportFilePath),
|
||||
`payloads/${matchedStep.step}_${correlationId}.json`
|
||||
),
|
||||
JSON.stringify(pair, null, 2)
|
||||
path.resolve(path.dirname(this.opts.reportOutputFilePath), "../", payloadFilePath),
|
||||
JSON.stringify(payload, null, 2)
|
||||
);
|
||||
trafficValidationIssue.payloadFilePath = `payloads/${matchedStep.step}_${correlationId}.json`;
|
||||
trafficValidationIssue.payloadFilePath = payloadFilePath;
|
||||
}
|
||||
const liveValidationResult = await this.liveValidator.validateLiveRequestResponse(pair);
|
||||
const liveValidationResult = await this.liveValidator.validateLiveRequestResponse(payload);
|
||||
|
||||
trafficValidationIssue.operationInfo =
|
||||
liveValidationResult.requestValidationResult.operationInfo;
|
||||
|
|
@ -284,18 +276,16 @@ export class ReportGenerator {
|
|||
|
||||
this.trafficValidationResult.push(trafficValidationIssue);
|
||||
|
||||
this.swaggerExampleQualityResult.stepResult.push({
|
||||
exampleFilePath: generatedExample?.exampleFilePath,
|
||||
this.testResult.stepResult.push({
|
||||
exampleFilePath: exampleFilePath,
|
||||
operationId: it.annotation.operationId,
|
||||
runtimeError,
|
||||
responseDiffResult: responseDiffResult,
|
||||
stepValidationResult: roundtripErrors,
|
||||
correlationId: correlationId,
|
||||
statusCode: it.response.statusCode,
|
||||
stepName: it.annotation.step,
|
||||
liveValidationResult: liveValidationResult,
|
||||
});
|
||||
this.recording.set(correlationId, it);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -320,9 +310,9 @@ export class ReportGenerator {
|
|||
return ret as LiveValidationIssue;
|
||||
}
|
||||
|
||||
private convertToLiveValidationPayload(rawExecution: RawExecution): RequestResponsePair {
|
||||
const request = rawExecution.request;
|
||||
const response = rawExecution.response;
|
||||
private convertToLiveValidationPayload(execution: NewmanExecution): RequestResponsePair {
|
||||
const request = execution.request;
|
||||
const response = execution.response;
|
||||
const liveRequest: LiveRequest = {
|
||||
url: request.url.toString(),
|
||||
method: request.method.toLowerCase(),
|
||||
|
|
@ -340,70 +330,6 @@ export class ReportGenerator {
|
|||
};
|
||||
}
|
||||
|
||||
private async generateExampleQualityReport() {
|
||||
if (this.opts.reportOutputFilePath !== undefined) {
|
||||
console.log(`Write generated report file: ${this.opts.reportOutputFilePath}`);
|
||||
await this.fileLoader.writeFile(
|
||||
this.opts.reportOutputFilePath,
|
||||
JSON.stringify(this.swaggerExampleQualityResult, null, 2)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async generateMarkdownQualityReport() {
|
||||
if (this.opts.markdownReportPath) {
|
||||
await this.fileLoader.appendFile(
|
||||
this.opts.markdownReportPath,
|
||||
generateMarkdownReport(this.swaggerExampleQualityResult)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async generateJUnitReport() {
|
||||
if (this.opts.junitReportPath) {
|
||||
await this.junitReporter.addSuiteToBuild(
|
||||
this.swaggerExampleQualityResult,
|
||||
this.opts.junitReportPath
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private generateExample(
|
||||
it: RawExecution,
|
||||
variables: { [variableName: string]: Variable },
|
||||
rawReport: RawReport,
|
||||
step: StepRestCall,
|
||||
exampleName: string
|
||||
): GeneratedExample {
|
||||
const example: any = {};
|
||||
if (it.annotation.operationId !== undefined) {
|
||||
this.operationIds.add(it.annotation.operationId);
|
||||
}
|
||||
example.parameters = this.generateParametersFromQuery(variables, it, step);
|
||||
try {
|
||||
_.extend(example.parameters, { parameters: JSON.parse(it.request.body) });
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (err) {}
|
||||
const resp: any = this.parseRespBody(it);
|
||||
example.responses = {};
|
||||
_.extend(example.responses, resp);
|
||||
const exampleFilePath = path.relative(
|
||||
this.fileRoot,
|
||||
path.resolve(this.opts.apiScenarioFilePath, exampleName)
|
||||
);
|
||||
const generatedGetExecution = this.findGeneratedGetExecution(it, rawReport);
|
||||
if (generatedGetExecution.length > 0) {
|
||||
const getResp = this.parseRespBody(generatedGetExecution[0]);
|
||||
_.extend(example.responses, getResp);
|
||||
}
|
||||
return {
|
||||
exampleFilePath: exampleFilePath,
|
||||
example,
|
||||
step: it.annotation.step,
|
||||
operationId: it.annotation.operationId,
|
||||
};
|
||||
}
|
||||
|
||||
private convertPostmanFormat<T>(obj: T, convertString: (s: string) => string): T {
|
||||
if (typeof obj === "string") {
|
||||
return convertString(obj) as unknown as T;
|
||||
|
|
@ -427,7 +353,8 @@ export class ReportGenerator {
|
|||
|
||||
private async exampleResponseDiff(
|
||||
example: GeneratedExample,
|
||||
matchedStep: Step
|
||||
matchedStep: Step,
|
||||
rawReport: NewmanReport
|
||||
): Promise<ResponseDiffItem[]> {
|
||||
let res: ResponseDiffItem[] = [];
|
||||
if (matchedStep?.type === "restCall") {
|
||||
|
|
@ -442,7 +369,7 @@ export class ReportGenerator {
|
|||
await this.responseDiff(
|
||||
example.example.responses["200"]?.body || {},
|
||||
matchedStep.responses["200"]?.body || {},
|
||||
this.rawReport!.variables,
|
||||
rawReport.variables,
|
||||
`/200/body`,
|
||||
matchedStep.operation.responses["200"].schema
|
||||
)
|
||||
|
|
@ -539,68 +466,49 @@ export class ReportGenerator {
|
|||
}
|
||||
|
||||
private getMatchedStep(stepName: string): Step | undefined {
|
||||
for (const it of this.testDefFile?.prepareSteps ?? []) {
|
||||
for (const it of this.scenario.steps ?? []) {
|
||||
if (stepName === it.step) {
|
||||
return it;
|
||||
}
|
||||
}
|
||||
for (const testScenario of this.testDefFile?.scenarios ?? []) {
|
||||
for (const step of testScenario.steps) {
|
||||
if (stepName === step.step) {
|
||||
return step;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private findGeneratedGetExecution(it: RawExecution, rawReport: RawReport) {
|
||||
if (it.annotation.type === "LRO") {
|
||||
const finalGet = rawReport.executions.filter(
|
||||
(execution) =>
|
||||
execution.annotation &&
|
||||
execution.annotation.lro_item_name === it.annotation.itemName &&
|
||||
execution.annotation.type === "generated-get"
|
||||
);
|
||||
return finalGet;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
private getRuntimeError(it: RawExecution): RuntimeError {
|
||||
const ret: RuntimeError = {
|
||||
code: "",
|
||||
message: "",
|
||||
private getRuntimeError(it: NewmanExecution): RuntimeError {
|
||||
const responseObj = this.dataMasker.jsonParse(it.response.body);
|
||||
return {
|
||||
code: "RUNTIME_ERROR",
|
||||
message: `statusCode: ${it.response.statusCode}, errorCode: ${responseObj?.error?.code}, errorMessage: ${responseObj?.error?.message}`,
|
||||
severity: "Error",
|
||||
detail: this.dataMasker.jsonStringify(it.response.body),
|
||||
};
|
||||
const responseObj = this.dataMasker.jsonParse(it.response.body);
|
||||
ret.code = `RUNTIME_ERROR`;
|
||||
ret.message = `statusCode: ${it.response.statusCode}, code: ${responseObj?.error?.code}, message: ${responseObj?.error?.message}`;
|
||||
return ret;
|
||||
}
|
||||
|
||||
public async generateReport() {
|
||||
if (this.opts.apiScenarioFilePath !== undefined) {
|
||||
this.testDefFile = await this.testResourceLoader.load(this.opts.apiScenarioFilePath);
|
||||
private async outputReport(): Promise<void> {
|
||||
if (this.opts.reportOutputFilePath !== undefined) {
|
||||
console.log(`Write generated report file: ${this.opts.reportOutputFilePath}`);
|
||||
await this.fileLoader.writeFile(
|
||||
this.opts.reportOutputFilePath,
|
||||
JSON.stringify(this.testResult, null, 2)
|
||||
);
|
||||
}
|
||||
if (this.opts.markdownReportPath) {
|
||||
await this.fileLoader.writeFile(
|
||||
this.opts.markdownReportPath,
|
||||
generateMarkdownReport(this.testResult)
|
||||
);
|
||||
}
|
||||
if (this.opts.junitReportPath) {
|
||||
await this.junitReporter.addSuiteToBuild(this.testResult, this.opts.junitReportPath);
|
||||
}
|
||||
if (this.opts.htmlReportPath) {
|
||||
await this.generateHtmlReport();
|
||||
}
|
||||
await this.swaggerAnalyzer.initialize();
|
||||
await this.initialize();
|
||||
await this.generateTestScenarioResult(this.rawReport!);
|
||||
await this.generateExampleQualityReport();
|
||||
await this.generateMarkdownQualityReport();
|
||||
await this.generateJUnitReport();
|
||||
await this.generateHtmlReport();
|
||||
|
||||
return this.swaggerExampleQualityResult;
|
||||
}
|
||||
|
||||
private async generateHtmlReport() {
|
||||
if (!this.opts.htmlReportPath) {
|
||||
return;
|
||||
}
|
||||
const operationIdCoverageResult = this.swaggerAnalyzer.calculateOperationCoverageBySpec(
|
||||
this.testDefFile!
|
||||
this.scenario._scenarioDef
|
||||
);
|
||||
|
||||
const operationCoverageResult: OperationCoverageInfo[] = [];
|
||||
|
|
@ -613,7 +521,7 @@ export class ReportGenerator {
|
|||
totalOperations: result.totalOperationNumber,
|
||||
spec: specPath,
|
||||
coverageRate: result.coverage,
|
||||
apiVersion: getApiVersionFromSwaggerPath(specPath),
|
||||
apiVersion: getApiVersionFromFilePath(specPath),
|
||||
unCoveredOperations: result.uncoveredOperationIds.length,
|
||||
coveredOperaions: result.totalOperationNumber - result.uncoveredOperationIds.length,
|
||||
validationFailOperations: this.trafficValidationResult.filter(
|
||||
|
|
@ -645,7 +553,7 @@ export class ReportGenerator {
|
|||
const options: TrafficValidationOptions = {
|
||||
reportPath: this.opts.htmlReportPath,
|
||||
overrideLinkInReport: false,
|
||||
sdkPackage: this.swaggerExampleQualityResult.providerNamespace,
|
||||
sdkPackage: this.testResult.providerNamespace,
|
||||
};
|
||||
|
||||
const generator = new HtmlReportGenerator(
|
||||
|
|
@ -656,42 +564,4 @@ export class ReportGenerator {
|
|||
);
|
||||
await generator.generateHtmlReport();
|
||||
}
|
||||
|
||||
private parseRespBody(it: RawExecution) {
|
||||
const resp: any = {};
|
||||
const response = it.response;
|
||||
const statusCode = it.response.statusCode;
|
||||
|
||||
try {
|
||||
resp[statusCode] = { body: JSON.parse(it.response.body) };
|
||||
} catch (err) {
|
||||
resp[statusCode] = { body: it.response.body };
|
||||
}
|
||||
|
||||
if (statusCode === 201 || statusCode === 202) {
|
||||
resp[statusCode].headers = {
|
||||
Location: response.headers.Location,
|
||||
"Azure-AsyncOperation": response.headers["Azure-AsyncOperation"],
|
||||
};
|
||||
}
|
||||
return resp;
|
||||
}
|
||||
|
||||
private generateParametersFromQuery(
|
||||
variables: { [variableName: string]: Variable },
|
||||
execution: RawExecution,
|
||||
step: StepRestCall
|
||||
) {
|
||||
const ret: any = {};
|
||||
for (const k of Object.keys(step.parameters)) {
|
||||
const paramName = Object.keys(variables).includes(k) ? k : `${step.step}_${k}`;
|
||||
const v = variables[paramName];
|
||||
if (v && v.type === "string") {
|
||||
if (execution.request.url.includes(v.value!)) {
|
||||
ret[k] = v.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
|
@ -9,17 +9,22 @@ import { ApiScenarioLoader, ApiScenarioLoaderOption } from "./apiScenarioLoader"
|
|||
import { ApiScenarioRunner } from "./apiScenarioRunner";
|
||||
import { generateMarkdownReportHeader } from "./markdownReport";
|
||||
import { PostmanCollectionRunnerClient } from "./postmanCollectionRunnerClient";
|
||||
import { ValidationLevel } from "./reportGenerator";
|
||||
import {
|
||||
NewmanReportValidator,
|
||||
NewmanReportValidatorOption,
|
||||
ValidationLevel,
|
||||
} from "./newmanReportValidator";
|
||||
import { SwaggerAnalyzer, SwaggerAnalyzerOption } from "./swaggerAnalyzer";
|
||||
import { EnvironmentVariables, VariableEnv } from "./variableEnv";
|
||||
import { NewmanReportAnalyzer, NewmanReportAnalyzerOption } from "./postmanReportAnalyzer";
|
||||
import { NewmanReport } from "./postmanReportParser";
|
||||
import { parseNewmanReport, RawNewmanReport } from "./newmanReportParser";
|
||||
import {
|
||||
defaultCollectionFileName,
|
||||
defaultEnvFileName,
|
||||
defaultNewmanReport,
|
||||
defaultQualityReportFilePath,
|
||||
} from "./defaultNaming";
|
||||
import { DataMasker } from "./dataMasker";
|
||||
import { Scenario } from "./apiScenarioTypes";
|
||||
|
||||
export interface PostmanCollectionGeneratorOption
|
||||
extends ApiScenarioLoaderOption,
|
||||
|
|
@ -38,6 +43,7 @@ export interface PostmanCollectionGeneratorOption
|
|||
testProxy?: string;
|
||||
validationLevel?: ValidationLevel;
|
||||
savePayload?: boolean;
|
||||
generateExample?: boolean;
|
||||
skipCleanUp?: boolean;
|
||||
runId?: string;
|
||||
verbose?: boolean;
|
||||
|
|
@ -96,13 +102,10 @@ export class PostmanCollectionGenerator {
|
|||
const result: Collection[] = [];
|
||||
|
||||
const client = new PostmanCollectionRunnerClient({
|
||||
apiScenarioFileName: this.opt.name,
|
||||
apiScenarioFilePath: this.opt.scenarioDef,
|
||||
runId: this.opt.runId,
|
||||
baseUrl: this.opt.baseUrl,
|
||||
testProxy: this.opt.testProxy,
|
||||
verbose: this.opt.verbose,
|
||||
swaggerFilePaths: this.opt.swaggerFilePaths,
|
||||
skipAuth: this.opt.devMode,
|
||||
skipArmCall: this.opt.devMode,
|
||||
skipLroPoll: this.opt.devMode,
|
||||
|
|
@ -125,11 +128,11 @@ export class PostmanCollectionGenerator {
|
|||
}
|
||||
|
||||
if (this.opt.generateCollection) {
|
||||
await this.writeCollectionToJson(scenario.scenario, collection, runtimeEnv);
|
||||
await this.writeCollectionToJson(scenario, collection, runtimeEnv);
|
||||
}
|
||||
|
||||
if (this.opt.runCollection) {
|
||||
await this.runCollection(scenario.scenario, collection, runtimeEnv);
|
||||
await this.runCollection(scenario, collection, runtimeEnv);
|
||||
}
|
||||
|
||||
result.push(collection);
|
||||
|
|
@ -171,10 +174,11 @@ export class PostmanCollectionGenerator {
|
|||
}
|
||||
|
||||
private async writeCollectionToJson(
|
||||
scenarioName: string,
|
||||
scenario: Scenario,
|
||||
collection: Collection,
|
||||
runtimeEnv: VariableScope
|
||||
) {
|
||||
const scenarioName = scenario.scenario;
|
||||
const collectionPath = resolve(
|
||||
this.opt.outputFolder,
|
||||
`${defaultCollectionFileName(this.opt.name, this.opt.runId!, scenarioName)}`
|
||||
|
|
@ -202,10 +206,11 @@ export class PostmanCollectionGenerator {
|
|||
}
|
||||
|
||||
private async runCollection(
|
||||
scenarioName: string,
|
||||
scenario: Scenario,
|
||||
collection: Collection,
|
||||
runtimeEnv: VariableScope
|
||||
) {
|
||||
const scenarioName = scenario.scenario;
|
||||
const reportExportPath = resolve(
|
||||
this.opt.outputFolder,
|
||||
`${defaultNewmanReport(this.opt.name, this.opt.runId!, scenarioName)}`
|
||||
|
|
@ -231,50 +236,67 @@ export class PostmanCollectionGenerator {
|
|||
}
|
||||
)
|
||||
.on("done", async (_err, _summary) => {
|
||||
const keys = await this.swaggerAnalyzer.getAllSecretKey();
|
||||
const values: string[] = [];
|
||||
for (const [k, v] of Object.entries(runtimeEnv.syncVariablesTo())) {
|
||||
if (this.dataMasker.maybeSecretKey(k)) {
|
||||
values.push(v as string);
|
||||
}
|
||||
}
|
||||
this.dataMasker.addMaskedValues(values);
|
||||
this.dataMasker.addMaskedKeys(keys);
|
||||
// read content and upload. mask newman report.
|
||||
const newmanReport = JSON.parse(
|
||||
await this.fileLoader.load(reportExportPath)
|
||||
) as NewmanReport;
|
||||
|
||||
// add mask environment secret value
|
||||
for (const item of newmanReport.environment.values) {
|
||||
if (this.dataMasker.maybeSecretKey(item.key)) {
|
||||
this.dataMasker.addMaskedValues([item.value]);
|
||||
}
|
||||
}
|
||||
const opts: NewmanReportAnalyzerOption = {
|
||||
newmanReportFilePath: reportExportPath,
|
||||
markdownReportPath: this.opt.markdownReportPath,
|
||||
junitReportPath: this.opt.junitReportPath,
|
||||
htmlReportPath: this.opt.htmlReportPath,
|
||||
runId: this.opt.runId,
|
||||
swaggerFilePaths: this.opt.swaggerFilePaths,
|
||||
validationLevel: this.opt.validationLevel,
|
||||
savePayload: this.opt.savePayload,
|
||||
verbose: this.opt.verbose,
|
||||
};
|
||||
const reportAnalyzer = inversifyGetInstance(NewmanReportAnalyzer, opts);
|
||||
await reportAnalyzer.analyze();
|
||||
if (this.opt.skipCleanUp) {
|
||||
printWarning(
|
||||
`Notice:the resource group '${runtimeEnv.get(
|
||||
"resourceGroupName"
|
||||
)}' was not cleaned up.`
|
||||
);
|
||||
}
|
||||
await this.postRun(scenario, reportExportPath, runtimeEnv);
|
||||
resolve(_summary);
|
||||
});
|
||||
});
|
||||
};
|
||||
await newmanRun();
|
||||
}
|
||||
|
||||
private async postRun(scenario: Scenario, reportExportPath: string, runtimeEnv: VariableScope) {
|
||||
const keys = await this.swaggerAnalyzer.getAllSecretKey();
|
||||
const values: string[] = [];
|
||||
for (const [k, v] of Object.entries(runtimeEnv.syncVariablesTo())) {
|
||||
if (this.dataMasker.maybeSecretKey(k)) {
|
||||
values.push(v as string);
|
||||
}
|
||||
}
|
||||
this.dataMasker.addMaskedValues(values);
|
||||
this.dataMasker.addMaskedKeys(keys);
|
||||
// read content and upload. mask newman report.
|
||||
const rawReport = JSON.parse(await this.fileLoader.load(reportExportPath)) as RawNewmanReport;
|
||||
|
||||
// add mask environment secret value
|
||||
for (const item of rawReport.environment.values) {
|
||||
if (this.dataMasker.maybeSecretKey(item.key)) {
|
||||
this.dataMasker.addMaskedValues([item.value]);
|
||||
}
|
||||
}
|
||||
|
||||
const newmanReport = parseNewmanReport(rawReport);
|
||||
|
||||
const newmanReportValidatorOption: NewmanReportValidatorOption = {
|
||||
apiScenarioFilePath: scenario._scenarioDef._filePath,
|
||||
swaggerFilePaths: scenario._scenarioDef._swaggerFilePaths,
|
||||
reportOutputFilePath: defaultQualityReportFilePath(reportExportPath),
|
||||
checkUnderFileRoot: false,
|
||||
eraseXmsExamples: false,
|
||||
eraseDescription: false,
|
||||
markdownReportPath: this.opt.markdownReportPath,
|
||||
junitReportPath: this.opt.junitReportPath,
|
||||
htmlReportPath: this.opt.htmlReportPath,
|
||||
baseUrl: this.opt.baseUrl,
|
||||
runId: this.opt.runId,
|
||||
validationLevel: this.opt.validationLevel,
|
||||
generateExample: this.opt.generateExample,
|
||||
savePayload: this.opt.savePayload,
|
||||
verbose: this.opt.verbose,
|
||||
};
|
||||
|
||||
const reportValidator = inversifyGetInstance(
|
||||
NewmanReportValidator,
|
||||
newmanReportValidatorOption
|
||||
);
|
||||
|
||||
await reportValidator.initialize(scenario);
|
||||
|
||||
await reportValidator.generateReport(newmanReport);
|
||||
|
||||
if (this.opt.skipCleanUp) {
|
||||
printWarning(
|
||||
`Notice:the resource group '${runtimeEnv.get("resourceGroupName")}' was not cleaned up.`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,11 +22,8 @@ import * as PostmanHelper from "./postmanHelper";
|
|||
import { VariableEnv } from "./variableEnv";
|
||||
|
||||
export interface PostmanCollectionRunnerClientOption {
|
||||
apiScenarioFileName: string;
|
||||
apiScenarioFilePath?: string;
|
||||
apiScenarioName?: string;
|
||||
runId: string;
|
||||
swaggerFilePaths?: string[];
|
||||
baseUrl: string;
|
||||
testProxy?: string;
|
||||
verbose?: boolean;
|
||||
|
|
@ -53,7 +50,6 @@ export class PostmanCollectionRunnerClient implements ApiScenarioRunnerClient {
|
|||
|
||||
public async prepareScenario(scenario: Scenario, env: VariableEnv): Promise<void> {
|
||||
this.opts.apiScenarioName = scenario.scenario;
|
||||
this.opts.apiScenarioFileName = scenario._scenarioDef._filePath;
|
||||
|
||||
this.collection = new Collection({
|
||||
info: {
|
||||
|
|
@ -63,9 +59,9 @@ export class PostmanCollectionRunnerClient implements ApiScenarioRunnerClient {
|
|||
});
|
||||
this.collection.describe(
|
||||
JSON.stringify({
|
||||
apiScenarioFilePath: this.opts.apiScenarioFilePath,
|
||||
apiScenarioName: this.opts.apiScenarioName,
|
||||
swaggerFilePaths: this.opts.swaggerFilePaths,
|
||||
apiScenarioFilePath: scenario._scenarioDef._filePath,
|
||||
apiScenarioName: scenario.scenario,
|
||||
swaggerFilePaths: scenario._scenarioDef._swaggerFilePaths,
|
||||
})
|
||||
);
|
||||
this.collection.auth = new RequestAuth({
|
||||
|
|
@ -342,6 +338,8 @@ pm.test("Stopped TestProxy recording", function() {
|
|||
|
||||
env.resolve();
|
||||
|
||||
step._resolvedParameters = env.resolveObjectValues(step.parameters);
|
||||
|
||||
if (Object.keys(step.variables).length > 0) {
|
||||
PostmanHelper.createEvent(
|
||||
"prerequest",
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
import { dirname } from "path";
|
||||
import { inject, injectable } from "inversify";
|
||||
import uuid from "uuid";
|
||||
import { setDefaultOpts } from "../swagger/loader";
|
||||
import { inversifyGetInstance, TYPES } from "../inversifyUtils";
|
||||
import { defaultQualityReportFilePath } from "./defaultNaming";
|
||||
import { ReportGenerator, ReportGeneratorOption, ValidationLevel } from "./reportGenerator";
|
||||
import { RawReport } from "./apiScenarioTypes";
|
||||
import { NewmanReportParser, NewmanReportParserOption } from "./postmanReportParser";
|
||||
|
||||
export interface NewmanReportAnalyzerOption extends NewmanReportParserOption {
|
||||
htmlReportPath?: string;
|
||||
reportOutputFilePath?: string;
|
||||
markdownReportPath?: string;
|
||||
junitReportPath?: string;
|
||||
runId?: string;
|
||||
swaggerFilePaths?: string[];
|
||||
validationLevel?: ValidationLevel;
|
||||
savePayload?: boolean;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
@injectable()
|
||||
export class NewmanReportAnalyzer {
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-member-accessibility
|
||||
constructor(
|
||||
@inject(TYPES.opts) private opts: NewmanReportAnalyzerOption,
|
||||
private newmanReportParser: NewmanReportParser
|
||||
) {
|
||||
setDefaultOpts(this.opts, {
|
||||
runId: uuid.v4(),
|
||||
newmanReportFilePath: "",
|
||||
reportOutputFilePath: defaultQualityReportFilePath(this.opts.newmanReportFilePath),
|
||||
validationLevel: "validate-request-response",
|
||||
savePayload: false,
|
||||
verbose: false,
|
||||
});
|
||||
}
|
||||
|
||||
public async analyze() {
|
||||
const rawReport: RawReport = await this.newmanReportParser.generateRawReport(
|
||||
this.opts.newmanReportFilePath
|
||||
);
|
||||
const apiScenarioFilePath = rawReport.metadata.apiScenarioFilePath;
|
||||
const reportGeneratorOption: ReportGeneratorOption = {
|
||||
newmanReportFilePath: this.opts.newmanReportFilePath,
|
||||
apiScenarioFilePath,
|
||||
apiScenarioName: rawReport.metadata.apiScenarioName,
|
||||
swaggerFilePaths: rawReport.metadata.swaggerFilePaths,
|
||||
checkUnderFileRoot: false,
|
||||
eraseXmsExamples: false,
|
||||
eraseDescription: false,
|
||||
reportOutputFilePath: this.opts.reportOutputFilePath,
|
||||
markdownReportPath: this.opts.markdownReportPath,
|
||||
junitReportPath: this.opts.junitReportPath,
|
||||
runId: this.opts.runId,
|
||||
validationLevel: this.opts.validationLevel,
|
||||
savePayload: this.opts.savePayload,
|
||||
verbose: this.opts.verbose,
|
||||
fileRoot: dirname(apiScenarioFilePath),
|
||||
htmlReportPath: this.opts.htmlReportPath,
|
||||
};
|
||||
const reportGenerator = inversifyGetInstance(ReportGenerator, reportGeneratorOption);
|
||||
await reportGenerator.generateReport();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
import { inject, injectable } from "inversify";
|
||||
import {
|
||||
RequestDefinition,
|
||||
ResponseDefinition,
|
||||
ItemDefinition,
|
||||
Request,
|
||||
Response,
|
||||
DescriptionDefinition,
|
||||
} from "postman-collection";
|
||||
import { FileLoader, FileLoaderOption } from "../swagger/fileLoader";
|
||||
import { TYPES } from "../inversifyUtils";
|
||||
import { RawExecution, RawReport, RawRequest, RawResponse } from "./apiScenarioTypes";
|
||||
|
||||
export interface NewmanReport {
|
||||
run: Run;
|
||||
environment: any;
|
||||
collection: any;
|
||||
}
|
||||
|
||||
interface Run {
|
||||
executions: NewmanExecution[];
|
||||
timings: { started: number; completed: number; responseAverage: number };
|
||||
}
|
||||
|
||||
interface NewmanExecution {
|
||||
item: ItemDefinition;
|
||||
request: RequestDefinition;
|
||||
response: ResponseDefinition;
|
||||
}
|
||||
|
||||
export interface NewmanReportParserOption extends FileLoaderOption {
|
||||
newmanReportFilePath: string;
|
||||
reportOutputFilePath?: string;
|
||||
}
|
||||
|
||||
@injectable()
|
||||
export class NewmanReportParser {
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-member-accessibility
|
||||
constructor(
|
||||
@inject(TYPES.opts) private opts: NewmanReportParserOption,
|
||||
private fileLoader: FileLoader
|
||||
) {}
|
||||
|
||||
public async generateRawReport(newmanReportFilePath: string) {
|
||||
const ret: RawReport = { variables: {}, executions: [], timings: {}, metadata: {} };
|
||||
const content = await this.fileLoader.load(newmanReportFilePath);
|
||||
const newmanReport = JSON.parse(content) as NewmanReport;
|
||||
ret.metadata = JSON.parse(newmanReport.collection.info.description.content);
|
||||
for (const it of newmanReport.run.executions) {
|
||||
ret.executions.push(this.generateExampleItem(it));
|
||||
}
|
||||
ret.timings = newmanReport.run.timings;
|
||||
ret.variables = this.parseVariables(newmanReport.environment.values);
|
||||
if (this.opts.reportOutputFilePath !== undefined) {
|
||||
await this.fileLoader.writeFile(this.opts.reportOutputFilePath, JSON.stringify(ret, null, 2));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private generateExampleItem(it: NewmanExecution): RawExecution {
|
||||
const resp = new Response(it.response);
|
||||
const req = new Request(it.request);
|
||||
const rawReq = this.parseRequest(req);
|
||||
const rawResp = this.parseResponse(resp);
|
||||
const annotation = JSON.parse((it.item.description as DescriptionDefinition)?.content || "{}");
|
||||
return {
|
||||
request: rawReq,
|
||||
response: rawResp,
|
||||
annotation: annotation,
|
||||
};
|
||||
}
|
||||
|
||||
private parseRequest(req: Request): RawRequest {
|
||||
const ret: RawRequest = {
|
||||
url: "",
|
||||
method: "",
|
||||
headers: [],
|
||||
body: "",
|
||||
};
|
||||
ret.url = req.url.toString();
|
||||
ret.headers = this.parseHeader(req.headers.toJSON());
|
||||
ret.method = req.method;
|
||||
ret.body = req.body?.toString() || "";
|
||||
return ret;
|
||||
}
|
||||
|
||||
private parseResponse(resp: Response): RawResponse {
|
||||
const ret: RawResponse = {
|
||||
headers: [],
|
||||
statusCode: resp.code,
|
||||
body: "",
|
||||
};
|
||||
ret.headers = this.parseHeader(resp.headers.toJSON());
|
||||
|
||||
ret.body = resp.stream?.toString() || "";
|
||||
return ret;
|
||||
}
|
||||
|
||||
private parseHeader(headers: any[]) {
|
||||
const ret: any = {};
|
||||
for (const it of headers) {
|
||||
ret[it.key] = it.value;
|
||||
|
||||
// Currently only mask bearer token header.
|
||||
// For further sensitive data, should add mask module here
|
||||
if (it.key === "Authorization") {
|
||||
ret[it.key] = "<bearer token>";
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private parseVariables(environment: any[]) {
|
||||
const ret: any = {};
|
||||
for (const it of environment) {
|
||||
if (it.type === "string" || it.type === "any") {
|
||||
ret[it.key] = { type: "string", value: it.value };
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
|
@ -99,6 +99,11 @@ export const builder: yargs.CommandBuilder = {
|
|||
boolean: true,
|
||||
default: false,
|
||||
},
|
||||
generateExample: {
|
||||
describe: "Generate examples after API Test",
|
||||
boolean: true,
|
||||
default: false,
|
||||
},
|
||||
testProxy: {
|
||||
describe: "TestProxy endpoint, e.g., http://localhost:5000. If not set, no proxy will be used.",
|
||||
string: true,
|
||||
|
|
@ -188,11 +193,12 @@ export async function handler(argv: yargs.Arguments): Promise<void> {
|
|||
baseUrl: argv.armEndpoint,
|
||||
testProxy: argv.testProxy,
|
||||
validationLevel: argv.level,
|
||||
savePayload: argv.savePayload,
|
||||
generateExample: argv.generateExample,
|
||||
skipCleanUp: argv.skipCleanUp,
|
||||
verbose: argv.verbose,
|
||||
swaggerFilePaths: swaggerFilePaths,
|
||||
devMode: argv.devMode,
|
||||
savePayload: argv.savePayload,
|
||||
};
|
||||
const generator = inversifyGetInstance(PostmanCollectionGenerator, opt);
|
||||
await generator.run();
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import { traverseSwagger } from "../transform/traverseSwagger";
|
|||
import { Operation, Path, LowerHttpMethods } from "../swagger/swaggerTypes";
|
||||
import { LiveValidatorLoader } from "../liveValidation/liveValidatorLoader";
|
||||
import { inversifyGetContainer, inversifyGetInstance } from "../inversifyUtils";
|
||||
import { getApiVersionFromSwaggerPath } from "../util/utils";
|
||||
import { getApiVersionFromFilePath } from "../util/utils";
|
||||
|
||||
export interface TrafficValidationOptions extends Options {
|
||||
sdkPackage?: string;
|
||||
|
|
@ -329,7 +329,7 @@ export class TrafficValidator {
|
|||
|
||||
this.operationCoverageResult.push({
|
||||
spec: key,
|
||||
apiVersion: getApiVersionFromSwaggerPath(key),
|
||||
apiVersion: getApiVersionFromFilePath(key),
|
||||
coveredOperaions: coveredOperaions,
|
||||
coverageRate: coverageRate,
|
||||
unCoveredOperations: value.length - coveredOperaions,
|
||||
|
|
|
|||
|
|
@ -386,11 +386,10 @@ export async function getInputFiles(readMe: string, tag?: string): Promise<strin
|
|||
return result;
|
||||
}
|
||||
|
||||
export function getApiVersionFromSwaggerPath(specPath: string): string {
|
||||
const apiVersionPattern: RegExp = new RegExp(
|
||||
`^.*\/(stable|preview)+\/([0-9]{4}-[0-9]{2}-[0-9]{2}(-preview)?)\/.*\.json$`
|
||||
);
|
||||
const apiVersionMatch = apiVersionPattern.exec(specPath);
|
||||
export function getApiVersionFromFilePath(filePath: string): string {
|
||||
const apiVersionPattern: RegExp =
|
||||
/^.*\/(stable|preview)+\/([0-9]{4}-[0-9]{2}-[0-9]{2}(-preview)?)\/.*\.(json|yaml)$/i;
|
||||
const apiVersionMatch = apiVersionPattern.exec(filePath);
|
||||
return apiVersionMatch === null ? "" : apiVersionMatch[2];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "oav",
|
||||
"version": "3.0.1",
|
||||
"version": "3.0.3",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "oav",
|
||||
"version": "3.0.1",
|
||||
"version": "3.0.3",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@autorest/schemas": "^1.3.4",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "oav",
|
||||
"version": "3.0.2",
|
||||
"version": "3.0.3",
|
||||
"author": {
|
||||
"name": "Microsoft Corporation",
|
||||
"email": "azsdkteam@microsoft.com",
|
||||
|
|
@ -158,4 +158,4 @@
|
|||
"jest-junit": {
|
||||
"output": "test-results.xml"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@
|
|||
exports[`ApiScenarioLoader load valid example based scenario - storage 1`] = `
|
||||
Object {
|
||||
"_filePath": "Microsoft.Storage/stable/2021-08-01/scenarios/storageBasicExample.yaml",
|
||||
"_swaggerFilePaths": Array [
|
||||
"Microsoft.Storage/stable/2021-08-01/storage.json",
|
||||
],
|
||||
"cleanUpSteps": Array [],
|
||||
"prepareSteps": Array [],
|
||||
"requiredVariables": Array [
|
||||
|
|
@ -523,6 +526,9 @@ Object {
|
|||
exports[`ApiScenarioLoader load valid scenario - storage 1`] = `
|
||||
Object {
|
||||
"_filePath": "Microsoft.Storage/stable/2021-08-01/scenarios/storageQuickStart.yaml",
|
||||
"_swaggerFilePaths": Array [
|
||||
"Microsoft.Storage/stable/2021-08-01/storage.json",
|
||||
],
|
||||
"cleanUpSteps": Array [],
|
||||
"prepareSteps": Array [],
|
||||
"requiredVariables": Array [
|
||||
|
|
@ -756,6 +762,9 @@ Object {
|
|||
exports[`ApiScenarioLoader load valid scenario from uri 1`] = `
|
||||
Object {
|
||||
"_filePath": "Microsoft.SignalRService/preview/2021-06-01-preview/scenarios/signalR.yaml",
|
||||
"_swaggerFilePaths": Array [
|
||||
"Microsoft.SignalRService/preview/2021-06-01-preview/signalr.json",
|
||||
],
|
||||
"cleanUpSteps": Array [],
|
||||
"prepareSteps": Array [],
|
||||
"requiredVariables": Array [
|
||||
|
|
|
|||
|
|
@ -1,245 +0,0 @@
|
|||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
exports[`reportGenerator generate report - storage 1`] = `
|
||||
Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"armEndpoint": "https://management.azure.com",
|
||||
"branch": undefined,
|
||||
"commitHash": undefined,
|
||||
"endTime": "2022-07-04T08:17:19.575Z",
|
||||
"environment": "test",
|
||||
"providerNamespace": "Microsoft.Storage",
|
||||
"repository": undefined,
|
||||
"rootPath": "",
|
||||
"runId": "",
|
||||
"startTime": "2022-07-04T08:17:09.368Z",
|
||||
"stepResult": Array [
|
||||
Object {
|
||||
"correlationId": "d6658422-9eff-4e4f-ac2a-b154df45d26a",
|
||||
"exampleFilePath": "../Microsoft.Storage/stable/2021-08-01/scenarios/examples/StorageAccountCheckNameAvailability.json",
|
||||
"liveValidationResult": Object {
|
||||
"requestValidationResult": Object {
|
||||
"errors": Array [],
|
||||
"isSuccessful": true,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_CheckNameAvailability",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
"responseValidationResult": Object {
|
||||
"errors": Array [],
|
||||
"isSuccessful": true,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_CheckNameAvailability",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
},
|
||||
"operationId": "StorageAccounts_CheckNameAvailability",
|
||||
"responseDiffResult": Array [],
|
||||
"runtimeError": Array [],
|
||||
"statusCode": 200,
|
||||
"stepName": "checkName",
|
||||
"stepValidationResult": Array [],
|
||||
},
|
||||
Object {
|
||||
"correlationId": "b69b9ae0-c247-485d-97ed-24940b40191b",
|
||||
"exampleFilePath": "../Microsoft.Storage/stable/2021-08-01/scenarios/examples/StorageAccountCreate.json",
|
||||
"liveValidationResult": Object {
|
||||
"requestValidationResult": Object {
|
||||
"errors": Array [],
|
||||
"isSuccessful": true,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_Create",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
"responseValidationResult": Object {
|
||||
"errors": Array [
|
||||
Object {
|
||||
"code": "INVALID_RESPONSE_CODE",
|
||||
"documentationUrl": "",
|
||||
"jsonPathsInPayload": Array [],
|
||||
"message": "The swagger file does not define '400' response code",
|
||||
"pathsInPayload": Array [],
|
||||
"schemaPath": "",
|
||||
"severity": 0,
|
||||
"source": Object {
|
||||
"position": Object {
|
||||
"column": 22,
|
||||
"line": 180,
|
||||
},
|
||||
"url": "",
|
||||
},
|
||||
},
|
||||
],
|
||||
"isSuccessful": false,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_Create",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
},
|
||||
"operationId": "StorageAccounts_Create",
|
||||
"responseDiffResult": Array [],
|
||||
"runtimeError": Array [
|
||||
Object {
|
||||
"code": "RUNTIME_ERROR",
|
||||
"detail": "{\\"error\\":{\\"code\\":\\"UnsupportedEdgeZone\\",\\"message\\":\\"Edge zone 'losangeles001' not found. The available edge zones in location 'westus' are ''.\\"}}",
|
||||
"message": "statusCode: 400, code: UnsupportedEdgeZone, message: Edge zone 'losangeles001' not found. The available edge zones in location 'westus' are ''.",
|
||||
"severity": "Error",
|
||||
},
|
||||
],
|
||||
"statusCode": 400,
|
||||
"stepName": "createStorageAccount",
|
||||
"stepValidationResult": Array [],
|
||||
},
|
||||
Object {
|
||||
"correlationId": "a1187deb-0f76-4958-a75a-4e74c6895c8a",
|
||||
"exampleFilePath": "../Microsoft.Storage/stable/2021-08-01/scenarios/examples/StorageAccountUpdate.json",
|
||||
"liveValidationResult": Object {
|
||||
"requestValidationResult": Object {
|
||||
"errors": Array [],
|
||||
"isSuccessful": true,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_Update",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
"responseValidationResult": Object {
|
||||
"errors": Array [
|
||||
Object {
|
||||
"code": "INVALID_RESPONSE_CODE",
|
||||
"documentationUrl": "",
|
||||
"jsonPathsInPayload": Array [],
|
||||
"message": "The swagger file does not define '404' response code",
|
||||
"pathsInPayload": Array [],
|
||||
"schemaPath": "",
|
||||
"severity": 0,
|
||||
"source": Object {
|
||||
"position": Object {
|
||||
"column": 22,
|
||||
"line": 334,
|
||||
},
|
||||
"url": "",
|
||||
},
|
||||
},
|
||||
],
|
||||
"isSuccessful": false,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_Update",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
},
|
||||
"operationId": "StorageAccounts_Update",
|
||||
"responseDiffResult": Array [],
|
||||
"runtimeError": Array [
|
||||
Object {
|
||||
"code": "RUNTIME_ERROR",
|
||||
"detail": "{\\"error\\":{\\"code\\":\\"PatchResourceNotFound\\",\\"target\\":\\"staj8625u\\",\\"message\\":\\"The resource '/subscriptions/db5eb68e-73e2-4fa8-b18a-46cd1be4cce5/resourceGroups/apiTest-ep9iua/providers/Microsoft.Storage/storageAccounts/staj8625u' was not found when evaluating policies for a PATCH operation.\\"}}",
|
||||
"message": "statusCode: 404, code: PatchResourceNotFound, message: The resource '/subscriptions/db5eb68e-73e2-4fa8-b18a-46cd1be4cce5/resourceGroups/apiTest-ep9iua/providers/Microsoft.Storage/storageAccounts/staj8625u' was not found when evaluating policies for a PATCH operation.",
|
||||
"severity": "Error",
|
||||
},
|
||||
],
|
||||
"statusCode": 404,
|
||||
"stepName": "updateStorageAccount",
|
||||
"stepValidationResult": Array [],
|
||||
},
|
||||
Object {
|
||||
"correlationId": "71c47075-2193-48c4-8f77-8da2f66de14a",
|
||||
"exampleFilePath": "../Microsoft.Storage/stable/2021-08-01/scenarios/examples/StorageAccountGetProperties.json",
|
||||
"liveValidationResult": Object {
|
||||
"requestValidationResult": Object {
|
||||
"errors": Array [],
|
||||
"isSuccessful": true,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_GetProperties",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
"responseValidationResult": Object {
|
||||
"errors": Array [
|
||||
Object {
|
||||
"code": "INVALID_RESPONSE_CODE",
|
||||
"documentationUrl": "",
|
||||
"jsonPathsInPayload": Array [],
|
||||
"message": "The swagger file does not define '404' response code",
|
||||
"pathsInPayload": Array [],
|
||||
"schemaPath": "",
|
||||
"severity": 0,
|
||||
"source": Object {
|
||||
"position": Object {
|
||||
"column": 22,
|
||||
"line": 270,
|
||||
},
|
||||
"url": "",
|
||||
},
|
||||
},
|
||||
],
|
||||
"isSuccessful": false,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_GetProperties",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
},
|
||||
"operationId": "StorageAccounts_GetProperties",
|
||||
"responseDiffResult": Array [],
|
||||
"runtimeError": Array [
|
||||
Object {
|
||||
"code": "RUNTIME_ERROR",
|
||||
"detail": "{\\"error\\":{\\"code\\":\\"ResourceNotFound\\",\\"message\\":\\"The Resource 'Microsoft.Storage/storageAccounts/staj8625u' under resource group 'apiTest-ep9iua' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix\\"}}",
|
||||
"message": "statusCode: 404, code: ResourceNotFound, message: The Resource 'Microsoft.Storage/storageAccounts/staj8625u' under resource group 'apiTest-ep9iua' was not found. For more details please go to https://aka.ms/ARMResourceNotFoundFix",
|
||||
"severity": "Error",
|
||||
},
|
||||
],
|
||||
"statusCode": 404,
|
||||
"stepName": "getStorageAccount",
|
||||
"stepValidationResult": Array [],
|
||||
},
|
||||
Object {
|
||||
"correlationId": "fd1c31fb-ef89-4671-8865-6b8aeb7cd1c9",
|
||||
"exampleFilePath": "../Microsoft.Storage/stable/2021-08-01/scenarios/examples/StorageAccountDelete.json",
|
||||
"liveValidationResult": Object {
|
||||
"requestValidationResult": Object {
|
||||
"errors": Array [],
|
||||
"isSuccessful": true,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_Delete",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
"responseValidationResult": Object {
|
||||
"errors": Array [],
|
||||
"isSuccessful": true,
|
||||
"operationInfo": Object {
|
||||
"apiVersion": "2021-08-01",
|
||||
"operationId": "StorageAccounts_Delete",
|
||||
},
|
||||
"runtimeException": undefined,
|
||||
},
|
||||
},
|
||||
"operationId": "StorageAccounts_Delete",
|
||||
"responseDiffResult": Array [],
|
||||
"runtimeError": Array [],
|
||||
"statusCode": 204,
|
||||
"stepName": "deleteStorageAccount",
|
||||
"stepValidationResult": Array [],
|
||||
},
|
||||
],
|
||||
"subscriptionId": "aaaaa",
|
||||
"swaggerFilePaths": Array [],
|
||||
"testScenarioFilePath": "../Microsoft.Storage/stable/2021-08-01/scenarios/storageBasicExample.yaml",
|
||||
"testScenarioName": "StorageBasicExample",
|
||||
}
|
||||
`;
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -3,7 +3,7 @@
|
|||
// This software is released under the MIT License.
|
||||
// https://opensource.org/licenses/MIT
|
||||
import { generateJUnitCaseReport } from "../../lib/apiScenario/markdownReport";
|
||||
import { StepResult } from "../../lib/apiScenario/reportGenerator";
|
||||
import { StepResult } from "../../lib/apiScenario/newmanReportValidator";
|
||||
|
||||
describe("junitTestReport", () => {
|
||||
it("Should generate junit case report", () => {
|
||||
|
|
|
|||
|
|
@ -6,19 +6,19 @@ import {
|
|||
generateMarkdownReport,
|
||||
generateMarkdownReportHeader,
|
||||
} from "../../lib/apiScenario/markdownReport";
|
||||
import { TestScenarioResult } from "../../lib/apiScenario/reportGenerator";
|
||||
import { ApiScenarioTestResult } from "../../lib/apiScenario/newmanReportValidator";
|
||||
|
||||
describe("markdownReport", () => {
|
||||
it("Should generate markdown report", () => {
|
||||
const ts = {
|
||||
testScenarioFilePath: "Microsoft.Compute/preview/2020-09-30/test-scenarios/galleries.yaml",
|
||||
apiScenarioFilePath: "Microsoft.Compute/preview/2020-09-30/test-scenarios/galleries.yaml",
|
||||
swaggerFilePaths: ["Microsoft.Compute/preview/2020-09-30/gallery.json"],
|
||||
providerNamespace: "Microsoft.Compute",
|
||||
apiVersion: "2020-09-30",
|
||||
runId: "202106011456-d4udg",
|
||||
rootPath: "/home/zhenglai/repos/azure-rest-api-specs/specification/compute/resource-manager",
|
||||
environment: "test",
|
||||
testScenarioName: "galleries_1",
|
||||
apiScenarioName: "galleries_1",
|
||||
armEndpoint: "https://management.azure.com",
|
||||
stepResult: [
|
||||
{
|
||||
|
|
@ -56,7 +56,7 @@ describe("markdownReport", () => {
|
|||
startTime: "2021-06-01T06:56:46.835Z",
|
||||
endTime: "2021-06-01T06:58:07.066Z",
|
||||
subscriptionId: "db5eb68e-73e2-4fa8-b18a-46cd1be4cce5",
|
||||
} as TestScenarioResult;
|
||||
} as ApiScenarioTestResult;
|
||||
const header = generateMarkdownReportHeader();
|
||||
const body = generateMarkdownReport(ts);
|
||||
expect(header).toMatchSnapshot("header");
|
||||
|
|
|
|||
|
|
@ -1,46 +0,0 @@
|
|||
import { inversifyGetInstance } from "../../lib/inversifyUtils";
|
||||
import { NewmanReportParser } from "../../lib/apiScenario/postmanReportParser";
|
||||
import { RawReport } from "../../lib/apiScenario/apiScenarioTypes";
|
||||
import { ReportGenerator, ReportGeneratorOption } from "../../lib/apiScenario/reportGenerator";
|
||||
|
||||
describe("reportGenerator", () => {
|
||||
it("generate report - storage", async () => {
|
||||
const newmanReportFilePath =
|
||||
"test/apiScenario/fixtures/report/storageBasicExample.yaml/202207041617-tywsob/StorageBasicExample.json";
|
||||
const newmanReportParser = inversifyGetInstance(NewmanReportParser, {
|
||||
newmanReportFilePath,
|
||||
});
|
||||
const rawReport: RawReport = await newmanReportParser.generateRawReport(newmanReportFilePath);
|
||||
const apiScenarioFilePath = rawReport.metadata.apiScenarioFilePath;
|
||||
const reportGeneratorOption: ReportGeneratorOption = {
|
||||
newmanReportFilePath,
|
||||
apiScenarioFilePath,
|
||||
apiScenarioName: rawReport.metadata.apiScenarioName,
|
||||
swaggerFilePaths: rawReport.metadata.swaggerFilePaths,
|
||||
checkUnderFileRoot: false,
|
||||
eraseXmsExamples: false,
|
||||
eraseDescription: false,
|
||||
runId: "",
|
||||
};
|
||||
const reportGenerator = inversifyGetInstance(ReportGenerator, reportGeneratorOption);
|
||||
reportGeneratorOption.reportOutputFilePath = undefined;
|
||||
const report = await reportGenerator.generateReport();
|
||||
report.rootPath = "";
|
||||
report.stepResult.forEach((stepResult) => {
|
||||
stepResult.liveValidationResult?.requestValidationResult.errors.forEach((error) => {
|
||||
error.source = {
|
||||
...error.source,
|
||||
url: "",
|
||||
};
|
||||
});
|
||||
stepResult.liveValidationResult?.responseValidationResult.errors.forEach((error) => {
|
||||
error.source = {
|
||||
...error.source,
|
||||
url: "",
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
expect(report).toMatchSnapshot();
|
||||
});
|
||||
});
|
||||
Загрузка…
Ссылка в новой задаче