add json report in validate-traffic (#941)

* add json report in validate-traffic

* update changelog

Co-authored-by: Tianxiang Chen <tianxchen@microsoft.com>

ChangeLog.md

@@ -1,5 +1,9 @@
 # Change Log - oav
 
+## 01/15/2023 3.3.1
+
+- Add json report parameter in command `validate-traffic`
+
 ## 12/19/2022 3.3.0
 
 - API Scenario

lib/commands/validate-traffic.ts

@@ -61,6 +61,10 @@ export const builder: yargs.CommandBuilder = {
     string: true,
     default: "https://github.com/scbedd/oav-traffic-converter/blob/main/sample-tables-input/",
   },
+  jsonReport: {
+    describe: "path and file name for json report",
+    string: true,
+  },
 };
 
 export async function handler(argv: yargs.Arguments): Promise<void> {
@@ -79,6 +83,7 @@ export async function handler(argv: yargs.Arguments): Promise<void> {
       outputExceptionInReport: argv.outputExceptionInReport,
       specLinkPrefix: argv.specLinkPrefix,
       payloadLinkPrefix: argv.payloadLinkPrefix,
+      jsonReportPath: argv.jsonReport,
     };
     const errors = await validate.validateTrafficAgainstSpec(specPath, trafficPath, vOptions);
     return errors.length > 0 ? 1 : 0;

lib/swaggerValidator/trafficValidator.ts

@@ -29,16 +29,21 @@ export interface TrafficValidationOptions extends Options {
   specLinkPrefix?: string;
   payloadLinkPrefix?: string;
   markdownPath?: string;
+  jsonReportPath?: string;
 }
 export interface TrafficValidationIssue {
   payloadFilePath?: string;
   payloadFilePathPosition?: FilePosition | undefined;
   specFilePath?: string;
-  errors?: LiveValidationIssue[];
+  errors?: LiveValidationIssueWithSource[];
   runtimeExceptions?: RuntimeException[];
   operationInfo?: OperationContext;
 }
 
+type LiveValidationIssueWithSource = LiveValidationIssue & {
+  issueSource?: "request" | "response";
+};
+
 export interface RuntimeException {
   code: string;
   message: string;
@@ -185,18 +190,28 @@ export class TrafficValidator {
           activityId
         );
 
-        const errorResult: LiveValidationIssue[] = [];
+        const errorResult: LiveValidationIssueWithSource[] = [];
         const runtimeExceptions: RuntimeException[] = [];
         if (validationResult.requestValidationResult.isSuccessful === undefined) {
           runtimeExceptions.push(validationResult.requestValidationResult.runtimeException!);
         } else if (validationResult.requestValidationResult.isSuccessful === false) {
-          errorResult.push(...validationResult.requestValidationResult.errors);
+          errorResult.push(
+            ...validationResult.requestValidationResult.errors.map((e) => {
+              (e as LiveValidationIssueWithSource).issueSource = "request";
+              return e as LiveValidationIssueWithSource;
+            })
+          );
         }
 
         if (validationResult.responseValidationResult.isSuccessful === undefined) {
           runtimeExceptions.push(validationResult.responseValidationResult.runtimeException!);
         } else if (validationResult.responseValidationResult.isSuccessful === false) {
-          errorResult.push(...validationResult.responseValidationResult.errors);
+          errorResult.push(
+            ...validationResult.responseValidationResult.errors.map((e) => {
+              (e as LiveValidationIssueWithSource).issueSource = "response";
+              return e as LiveValidationIssueWithSource;
+            })
+          );
         }
         if (errorResult.length > 0 || runtimeExceptions.length > 0) {
           const trafficSpec = await this.swaggerLoader.load(payloadFilePath);

lib/validate.ts

@@ -25,6 +25,7 @@ import {
   TrafficValidator,
 } from "./swaggerValidator/trafficValidator";
 import { ReportGenerator } from "./report/generateReport";
+import { flatMap } from "@azure-tools/openapi-tools-common";
 
 export interface Options extends XMsExampleExtractor.Options {
   consoleLogLevel?: unknown;
@@ -229,6 +230,42 @@ export async function validateTrafficAgainstSpec(
         ],
       });
     }
+    if (options.jsonReportPath) {
+      const report = {
+        allOperations: validator!.operationCoverageResult
+          .map((item) => item.totalOperations)
+          .reduce((a, b) => a + b, 0),
+        coveredOperations: validator!.operationCoverageResult
+          .map((item) => item.coveredOperaions)
+          .reduce((a, b) => a + b, 0),
+        failedOperations: validator!.operationCoverageResult
+          .map((item) => item.validationFailOperations)
+          .reduce((a, b) => a + b, 0),
+        requestErrors: Array.from(
+          flatMap(
+            trafficValidationResult,
+            (item) =>
+              item.errors
+                ?.filter((it) => it.issueSource === "request")
+                .map((it) => {
+                  return { errorCode: it.code, errorMessage: it.message };
+                }) ?? []
+          )
+        ),
+        responseErrors: Array.from(
+          flatMap(
+            trafficValidationResult,
+            (item) =>
+              item.errors
+                ?.filter((it) => it.issueSource === "response")
+                .map((it) => {
+                  return { errorCode: it.code, errorMessage: it.message };
+                }) ?? []
+          )
+        ),
+      };
+      fs.writeFileSync(options.jsonReportPath, JSON.stringify(report, null, 2));
+    }
     if (options.reportPath) {
       const generator = new ReportGenerator(
         trafficValidationResult,