diff --git a/ChangeLog.md b/ChangeLog.md index e8920ad8..d2a7ed1c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,10 @@ # Change Log - oav +## 08/09/2021 2.7.2 + +- Add runner architecture drawio +- Oav runner run test scenarios sequentially + ## 08/03/2021 2.7.1 - Disable checkUnderFileRoot in `Validate Traffic` command diff --git a/README.md b/README.md index 1553050d..4a3aa37d 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,6 @@ You can install the latest stable release of node.js from [here](https://nodejs. npm install -g oav@latest ``` -![](./documentation/installOav.gif) - ### Run API test OAV support run API test against Azure and validate request and response. You could define test scenario file which compose with severval swagger example file and then use oav to run it. For more details about API test, please refer to this [API testing doc](https://github.com/Azure/azure-rest-api-specs/tree/test-scenario-main/documentation/test-scenario). diff --git a/documentation/installOav.gif b/documentation/installOav.gif deleted file mode 100644 index 01776ddf..00000000 Binary files a/documentation/installOav.gif and /dev/null differ diff --git a/documentation/postmanRunner.drawio b/documentation/postmanRunner.drawio new file mode 100644 index 00000000..344a7ecd --- /dev/null +++ b/documentation/postmanRunner.drawio @@ -0,0 +1,210 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/documentation/runApiTest.gif b/documentation/runApiTest.gif index 04b7fdae..a35fb91c 100644 Binary files a/documentation/runApiTest.gif and b/documentation/runApiTest.gif differ diff --git a/lib/testScenario/postmanCollectionRunnerClient.ts b/lib/testScenario/postmanCollectionRunnerClient.ts index 99ee70e5..12cc5065 100644 --- a/lib/testScenario/postmanCollectionRunnerClient.ts +++ b/lib/testScenario/postmanCollectionRunnerClient.ts @@ -221,6 +221,9 @@ export class PostmanCollectionRunnerClient implements TestScenarioRunnerClient { url: "", body: { mode: "raw" } as RequestBodyDefinition, }); + if (step.step === "Deployments_CreateOrUpdate") { + console.log(JSON.stringify(stepEnv.env, null, 2)); + } item.description = step.operation.operationId || ""; const queryParams: QueryParamDefinition[] = []; const urlVariables: VariableDefinition[] = []; @@ -539,86 +542,93 @@ export class PostmanCollectionRunnerClient implements TestScenarioRunnerClient { if (this.opts.from || this.opts.to) { runtimeEnvManager.repopulateCollectionItems(this.opts.from, this.opts.to); } - newman - .run( - { - collection: this.collection, - environment: this.collectionEnv, - reporters: ["cli", "json"], - reporter: { json: { export: reportExportPath } }, - }, - function (err, summary) { - if (summary.run.failures.length > 0) { - process.exitCode = 1; - } - if (err) { - console.log(`collection run failed. ${err}`); - } - console.log("collection run complete!"); - } - ) - .on("beforeItem", async function (this: any, _err, _summary) { - if (!_err) { - runtimeEnvManager.save(_summary.item.name, this, "beforeStep"); - } - }) - .on("item", async function (this: any, _err, _summary) { - if (!_err) { - runtimeEnvManager.clean(); - } - }) - .on("done", async (_err, _summary) => { - const keys = await this.swaggerAnalyzer.getAllSecretKey(); - const values: string[] = []; - for (const [k, v] of Object.entries(this.collectionEnv.syncVariablesTo())) { - if (this.dataMasker.maybeSecretKey(k)) { - values.push(v as string); - } - } - this.dataMasker.addMaskedValues(values); - this.dataMasker.addMaskedKeys(keys); - // read content and upload. mask newman report. - const newmanReport = JSON.parse( - await this.fileLoader.load(reportExportPath) - ) as NewmanReport; - // add mask environment secret value - for (const item of newmanReport.environment.values) { - if (this.dataMasker.maybeSecretKey(item.key)) { - this.dataMasker.addMaskedValues([item.value]); - } - } - if (this.opts.enableBlobUploader) { - await this.blobUploader.uploadContent( - "newmanreport", - `${defaultNewmanReport( - this.opts.testScenarioFileName, - this.opts.runId, - this.opts.testScenarioName - )}`, - this.dataMasker.jsonStringify(newmanReport) - ); - } - const opts: NewmanReportAnalyzerOption = { - newmanReportFilePath: reportExportPath, - markdownReportPath: this.opts.markdownReportPath, - junitReportPath: this.opts.junitReportPath, - enableUploadBlob: this.opts.enableBlobUploader, - runId: this.opts.runId, - swaggerFilePaths: this.opts.swaggerFilePaths, - validationLevel: this.opts.validationLevel, - verbose: this.opts.verbose, - }; - const reportAnalyzer = inversifyGetInstance(NewmanReportAnalyzer, opts); - await reportAnalyzer.analyze(); - if (this.opts.skipCleanUp || this.opts.to) { - printWarning( - `Notice:the resource group '${this.collectionEnv.get( - "resourceGroupName" - )}' was not cleaned up.` - ); - } + const newmanRun = async () => { + return new Promise((resolve) => { + newman + .run( + { + collection: this.collection, + environment: this.collectionEnv, + reporters: ["cli", "json"], + reporter: { json: { export: reportExportPath } }, + }, + function (err, summary) { + if (summary.run.failures.length > 0) { + process.exitCode = 1; + } + if (err) { + console.log(`collection run failed. ${err}`); + } + console.log("collection run complete!"); + } + ) + .on("beforeItem", async function (this: any, _err, _summary) { + if (!_err) { + runtimeEnvManager.save(_summary.item.name, this, "beforeStep"); + } + }) + .on("item", async function (this: any, _err, _summary) { + if (!_err) { + runtimeEnvManager.clean(); + } + }) + .on("done", async (_err, _summary) => { + const keys = await this.swaggerAnalyzer.getAllSecretKey(); + const values: string[] = []; + for (const [k, v] of Object.entries(this.collectionEnv.syncVariablesTo())) { + if (this.dataMasker.maybeSecretKey(k)) { + values.push(v as string); + } + } + this.dataMasker.addMaskedValues(values); + this.dataMasker.addMaskedKeys(keys); + // read content and upload. mask newman report. + const newmanReport = JSON.parse( + await this.fileLoader.load(reportExportPath) + ) as NewmanReport; + + // add mask environment secret value + for (const item of newmanReport.environment.values) { + if (this.dataMasker.maybeSecretKey(item.key)) { + this.dataMasker.addMaskedValues([item.value]); + } + } + if (this.opts.enableBlobUploader) { + await this.blobUploader.uploadContent( + "newmanreport", + `${defaultNewmanReport( + this.opts.testScenarioFileName, + this.opts.runId, + this.opts.testScenarioName + )}`, + this.dataMasker.jsonStringify(newmanReport) + ); + } + const opts: NewmanReportAnalyzerOption = { + newmanReportFilePath: reportExportPath, + markdownReportPath: this.opts.markdownReportPath, + junitReportPath: this.opts.junitReportPath, + enableUploadBlob: this.opts.enableBlobUploader, + runId: this.opts.runId, + swaggerFilePaths: this.opts.swaggerFilePaths, + validationLevel: this.opts.validationLevel, + verbose: this.opts.verbose, + }; + const reportAnalyzer = inversifyGetInstance(NewmanReportAnalyzer, opts); + await reportAnalyzer.analyze(); + if (this.opts.skipCleanUp || this.opts.to) { + printWarning( + `Notice:the resource group '${this.collectionEnv.get( + "resourceGroupName" + )}' was not cleaned up.` + ); + } + resolve(_summary); + }); }); + }; + await newmanRun(); } private generatedGetOperationItem( diff --git a/package-lock.json b/package-lock.json index a39a3f4c..0abb226c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "oav", - "version": "2.7.1", + "version": "2.7.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 8d2752a3..67df4d5b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "oav", - "version": "2.7.1", + "version": "2.7.2", "author": { "name": "Microsoft Corporation", "email": "azsdkteam@microsoft.com",