Isolate RP when running transform in livevalidator

ChangeLog.md

@@ -1,7 +1,8 @@
 # Change Log - oav
 
-## 08/17/2021 2.7.4
+## 08/17/2021 2.8.0
 
+- Isolate RP when running transform on cache model in live validator so that the transform won't fail on global cache level
 - Fix type error in nullableTransformer
 - Throw exception when getting error in transformLoadedSpecs at livevalidator initialization
 - Improve wording of error messages

lib/liveValidation/liveValidator.ts

@@ -24,7 +24,7 @@ import * as utils from "../util/utils";
 import { RuntimeException } from "../util/validationError";
 import { inversifyGetContainer, inversifyGetInstance, TYPES } from "../inversifyUtils";
 import { setDefaultOpts } from "../swagger/loader";
-import { TrafficValidationErrorCode, trafficValidationErrors } from "../util/errorDefinitions";
+import { apiValidationErrors, ApiValidationErrorCode } from "../util/errorDefinitions";
 import { LiveValidatorLoader, LiveValidatorLoaderOption } from "./liveValidatorLoader";
 import { getProviderFromPathTemplate, OperationSearcher } from "./operationSearcher";
 import {
@@ -71,7 +71,7 @@ export interface RequestResponseLiveValidationResult {
 }
 
 export type LiveValidationIssue = {
-  code: TrafficValidationErrorCode;
+  code: ApiValidationErrorCode;
   pathsInPayload: string[];
   documentationUrl?: string;
 } & Omit<SchemaValidateIssue, "code">;
@@ -88,7 +88,7 @@ interface Meta {
  * If `includeErrors` is missing or empty, all error codes will be included.
  */
 export interface ValidateOptions {
-  readonly includeErrors?: TrafficValidationErrorCode[];
+  readonly includeErrors?: ApiValidationErrorCode[];
   readonly includeOperationMatch?: boolean;
 }
 
@@ -198,8 +198,12 @@ export class LiveValidator {
       container,
       fileRoot: this.options.directory,
       ...this.options,
-      loadSuppression: this.options.loadSuppression ?? Object.keys(trafficValidationErrors),
+      loadSuppression: this.options.loadSuppression ?? Object.keys(apiValidationErrors),
     });
+    this.loader.logging = this.logging;
+
+    // re-set the transform context after set the logging function
+    this.loader.setTransformContext();
     const schemaValidator = container.get(TYPES.schemaValidator) as SchemaValidator;
     this.validateRequestResponsePair = await schemaValidator.compileAsync(
       requestResponseDefinition
@@ -233,16 +237,22 @@ export class LiveValidator {
         try {
           const spec = allSpecs.shift()!;
           const loadStart = Date.now();
+          this.logging(
+            `Start building validator for ${spec._filePath}`,
+            LiveValidatorLoggingLevels.debug,
+            LiveValidatorLoggingTypes.trace,
+            "Oav.liveValidator.loader.buildAjvValidator"
+          );
           await this.loader.buildAjvValidator(spec);
           const durationInMs = Date.now() - loadStart;
           this.logging(
-            `Build validator for ${spec._filePath} with DurationInMs:${durationInMs}`,
+            `Complete building validator for ${spec._filePath} with DurationInMs:${durationInMs}`,
             LiveValidatorLoggingLevels.debug,
             LiveValidatorLoggingTypes.trace,
             "Oav.liveValidator.loader.buildAjvValidator"
           );
           this.logging(
-            `Build validator for ${spec._filePath} in initialization time`,
+            `Complete building validator for ${spec._filePath} in initialization time`,
             LiveValidatorLoggingLevels.info,
             LiveValidatorLoggingTypes.perfTrace,
             "Oav.liveValidator.initialize.loader.buildAjvValidator",
@@ -294,16 +304,22 @@ export class LiveValidator {
       try {
         const spec = allSpecs.shift()!;
         const startTime = Date.now();
+        this.logging(
+          `Start building validator for ${spec._filePath} in background`,
+          LiveValidatorLoggingLevels.debug,
+          LiveValidatorLoggingTypes.trace,
+          "Oav.liveValidator.loadAllSpecValidatorInBackground"
+        );
         await this.loader!.buildAjvValidator(spec, { inBackground: true });
         const elapsedTime = Date.now() - startTime;
         this.logging(
-          `Build validator for ${spec._filePath} in background with DurationInMs:${elapsedTime}.`,
+          `Complete building validator for ${spec._filePath} in background with DurationInMs:${elapsedTime}.`,
           LiveValidatorLoggingLevels.debug,
           LiveValidatorLoggingTypes.trace,
           "Oav.liveValidator.loadAllSpecValidatorInBackground"
         );
         this.logging(
-          `Build validator for ${spec._filePath} in background`,
+          `Complete building for ${spec._filePath} in background`,
           LiveValidatorLoggingLevels.info,
           LiveValidatorLoggingTypes.perfTrace,
           "Oav.liveValidator.loadAllSpecValidatorInBackground-1",

lib/liveValidation/liveValidatorLoader.ts

@@ -6,6 +6,7 @@ import { JsonLoader } from "../swagger/jsonLoader";
 import { Loader, setDefaultOpts } from "../swagger/loader";
 import { SwaggerLoader, SwaggerLoaderOption } from "../swagger/swaggerLoader";
 import {
+  LoggingFn,
   Operation,
   Parameter,
   PathParameter,
@@ -41,7 +42,8 @@ export interface LiveValidatorLoaderOption extends SwaggerLoaderOption, SchemaVa
 
 @injectable()
 export class LiveValidatorLoader implements Loader<SwaggerSpec> {
-  public readonly transformContext: TransformContext;
+  private transformContext: TransformContext;
+  public logging: LoggingFn;
 
   public getResponseValidator = getLazyBuilder(
     "_validate",
@@ -98,20 +100,28 @@ export class LiveValidatorLoader implements Loader<SwaggerSpec> {
     setDefaultOpts(opts, {
       transformToNewSchemaFormat: false,
     });
+    this.setTransformContext();
+  }
 
-    this.transformContext = getTransformContext(this.jsonLoader, this.schemaValidator, [
-      xmsPathsTransformer,
-      resolveNestedDefinitionTransformer,
-      this.opts.transformToNewSchemaFormat ? schemaV4ToV7Transformer : undefined,
-      referenceFieldsTransformer,
-      pathRegexTransformer,
+  public setTransformContext() {
+    this.transformContext = getTransformContext(
+      this.jsonLoader,
+      this.schemaValidator,
+      [
+        xmsPathsTransformer,
+        resolveNestedDefinitionTransformer,
+        this.opts.transformToNewSchemaFormat ? schemaV4ToV7Transformer : undefined,
+        referenceFieldsTransformer,
+        pathRegexTransformer,
 
-      discriminatorTransformer,
-      allOfTransformer,
-      noAdditionalPropertiesTransformer,
-      nullableTransformer,
-      pureObjectTransformer,
-    ]);
+        discriminatorTransformer,
+        allOfTransformer,
+        noAdditionalPropertiesTransformer,
+        nullableTransformer,
+        pureObjectTransformer,
+      ],
+      this.logging
+    );
   }
 
   public async load(specFilePath: string): Promise<SwaggerSpec> {

lib/liveValidation/operationValidator.ts

@@ -1,13 +1,23 @@
 import { ParsedUrlQuery } from "querystring";
 import { getInfo, MutableStringMap, StringMap } from "@azure-tools/openapi-tools-common";
-import { LowerHttpMethods, Operation, Response, TransformFn } from "../swagger/swaggerTypes";
+import {
+  LoggingFn,
+  LowerHttpMethods,
+  Operation,
+  Response,
+  TransformFn,
+} from "../swagger/swaggerTypes";
 import { sourceMapInfoToSourceLocation } from "../swaggerValidator/ajvSchemaValidator";
 import { SchemaValidateContext, SchemaValidateIssue } from "../swaggerValidator/schemaValidator";
 import { jsonPathToPointer } from "../util/jsonUtils";
 import { Writable } from "../util/utils";
 import { SourceLocation } from "../util/validationError";
 import { extractPathParamValue } from "../transform/pathRegexTransformer";
-import { getOavErrorMeta, TrafficValidationErrorCode } from "../util/errorDefinitions";
+import {
+  ApiValidationErrorCode,
+  getOavErrorMeta,
+  TrafficValidationErrorCode,
+} from "../util/errorDefinitions";
 import {
   LiveValidationIssue,
   LiveValidatorLoggingLevels,
@@ -53,15 +63,8 @@ export const validateSwaggerLiveRequest = async (
   request: LiveRequest,
   info: OperationContext,
   loader?: LiveValidatorLoader,
-  includeErrors?: TrafficValidationErrorCode[],
-  logging?: (
-    message: string,
-    level?: LiveValidatorLoggingLevels,
-    loggingType?: LiveValidatorLoggingTypes,
-    operationName?: string,
-    durationInMilliseconds?: number,
-    validationRequest?: ValidationRequest
-  ) => void
+  includeErrors?: ApiValidationErrorCode[],
+  logging?: LoggingFn
 ) => {
   const { operation } = info.operationMatch!;
   const { body, query } = request;
@@ -99,8 +102,7 @@ export const validateSwaggerLiveRequest = async (
   transformMapValue(query, operation._queryTransform);
   const headers = transformLiveHeader(request.headers ?? {}, operation);
   validateContentType(operation.consumes!, headers, true, result);
-
-  const ctx = { isResponse: false, includeErrors };
+  const ctx = { isResponse: false, includeErrors: includeErrors as any };
   const errors = validate(ctx, {
     path: pathParam,
     body: transformBodyValue(body, operation),
@@ -116,16 +118,9 @@ export const validateSwaggerLiveResponse = async (
   response: LiveResponse,
   info: OperationContext,
   loader?: LiveValidatorLoader,
-  includeErrors?: TrafficValidationErrorCode[],
+  includeErrors?: ApiValidationErrorCode[],
   isArmCall?: boolean,
-  logging?: (
-    message: string,
-    level?: LiveValidatorLoggingLevels,
-    loggingType?: LiveValidatorLoggingTypes,
-    operationName?: string,
-    durationInMilliseconds?: number,
-    validationRequest?: ValidationRequest
-  ) => void
+  logging?: LoggingFn
 ) => {
   const { operation } = info.operationMatch!;
   const { statusCode, body } = response;
@@ -180,7 +175,7 @@ export const validateSwaggerLiveResponse = async (
 
   const ctx = {
     isResponse: true,
-    includeErrors,
+    includeErrors: includeErrors as any,
     statusCode,
     httpMethod: operation._method,
   };

lib/swagger/swaggerTypes.ts

@@ -1,6 +1,11 @@
 // https://github.com/mohsen1/swagger.d.ts
 
 import { MutableStringMap } from "@azure-tools/openapi-tools-common";
+import {
+  LiveValidatorLoggingLevels,
+  LiveValidatorLoggingTypes,
+} from "../liveValidation/liveValidator";
+import { ValidationRequest } from "../liveValidation/operationValidator";
 import { SchemaValidateFunction } from "../swaggerValidator/schemaValidator";
 import { RegExpWithKeys } from "../transform/pathRegexTransformer";
 import {
@@ -163,6 +168,15 @@ export interface Operation {
 
 export type TransformFn = (val: string) => string | number | boolean;
 
+export type LoggingFn = (
+  message: string,
+  level?: LiveValidatorLoggingLevels,
+  loggingType?: LiveValidatorLoggingTypes,
+  operationName?: string,
+  durationInMilliseconds?: number,
+  validationRequest?: ValidationRequest
+) => void;
+
 // ----------------------------- Response ------------------------------------
 export interface Response {
   description: string;

lib/transform/allOfTransformer.ts

@@ -55,10 +55,20 @@ const transformAllOfSchema = (schema: Schema, baseSchemas: Set<Schema>, jsonLoad
 // Must after transformDiscriminator
 export const allOfTransformer: GlobalTransformer = {
   type: TransformerType.Global,
-  transform({ objSchemas, baseSchemas, jsonLoader }) {
+  transform({ objSchemas, baseSchemas, jsonLoader, logging }) {
     for (const sch of objSchemas) {
-      if (sch.allOf !== undefined) {
-        transformAllOfSchema(sch, baseSchemas, jsonLoader);
+      try {
+        if (sch.allOf !== undefined) {
+          transformAllOfSchema(sch, baseSchemas, jsonLoader);
+        }
+      } catch (e) {
+        if (logging) {
+          logging(`Fail to transform ${sch}. ErrorMessage:${e?.message};ErrorStack:${e?.stack}.`);
+        } else {
+          console.log(
+            `Fail to transform ${sch}. ErrorMessage:${e?.message};ErrorStack:${e?.stack}.`
+          );
+        }
       }
     }
   },

lib/transform/context.ts

@@ -1,5 +1,5 @@
 import { JsonLoader } from "../swagger/jsonLoader";
-import { Parameter, refSelfSymbol, Schema } from "../swagger/swaggerTypes";
+import { LoggingFn, Parameter, refSelfSymbol, Schema } from "../swagger/swaggerTypes";
 import { SchemaValidator } from "../swaggerValidator/schemaValidator";
 import { GlobalTransformer, sortTransformers, SpecTransformer, Transformer } from "./transformer";
 
@@ -15,12 +15,15 @@ export interface TransformContext {
 
   specTransformers: SpecTransformer[];
   globalTransformers: GlobalTransformer[];
+
+  logging?: LoggingFn;
 }
 
 export const getTransformContext = (
   jsonLoader: JsonLoader,
   schemaValidator: SchemaValidator,
-  transformers: Array<Transformer | undefined>
+  transformers: Array<Transformer | undefined>,
+  logging?: LoggingFn
 ): TransformContext => {
   return {
     jsonLoader,
@@ -31,6 +34,7 @@ export const getTransformContext = (
     allParams: [],
     baseSchemas: new Set(),
     ...sortTransformers(transformers.filter(Boolean) as Transformer[]),
+    logging,
   };
 };
 

lib/transform/discriminatorTransformer.ts

@@ -49,25 +49,35 @@ const getDiscriminatorValue = (sch: Schema) => {
 export const discriminatorTransformer: GlobalTransformer = {
   type: TransformerType.Global,
   before: [allOfTransformer],
-  transform({ objSchemas, baseSchemas, jsonLoader }) {
+  transform({ objSchemas, baseSchemas, jsonLoader, logging }) {
     const visited = new Map<Schema, string | null>();
     for (const sch of objSchemas) {
-      const rootRef = getDiscriminatorRoot(sch, visited, baseSchemas, jsonLoader);
-      if (rootRef === null) {
-        continue;
-      }
+      try {
+        const rootRef = getDiscriminatorRoot(sch, visited, baseSchemas, jsonLoader);
+        if (rootRef === null) {
+          continue;
+        }
 
-      const baseSch = jsonLoader.resolveRefObj({ $ref: rootRef } as Schema);
-      const $ref = sch[refSelfSymbol];
-      const discriminatorValue = getDiscriminatorValue(sch);
+        const baseSch = jsonLoader.resolveRefObj({ $ref: rootRef } as Schema);
+        const $ref = sch[refSelfSymbol];
+        const discriminatorValue = getDiscriminatorValue(sch);
 
-      if (baseSch.discriminatorMap === undefined) {
-        baseSch.discriminatorMap = {
-          [getDiscriminatorValue(baseSch)]: null,
-        };
-        copyInfo(baseSch, baseSch.discriminatorMap);
+        if (baseSch.discriminatorMap === undefined) {
+          baseSch.discriminatorMap = {
+            [getDiscriminatorValue(baseSch)]: null,
+          };
+          copyInfo(baseSch, baseSch.discriminatorMap);
+        }
+        baseSch.discriminatorMap[discriminatorValue] = { $ref } as unknown as Schema;
+      } catch (e) {
+        if (logging) {
+          logging(`Fail to transform ${sch}. ErrorMessage:${e?.message};ErrorStack:${e?.stack}.`);
+        } else {
+          console.log(
+            `Fail to transform ${sch}. ErrorMessage:${e?.message};ErrorStack:${e?.stack}.`
+          );
+        }
       }
-      baseSch.discriminatorMap[discriminatorValue] = { $ref } as unknown as Schema;
     }
   },
 };

lib/transform/nullableTransformer.ts

@@ -7,30 +7,50 @@ import { GlobalTransformer, TransformerType } from "./transformer";
 export const nullableTransformer: GlobalTransformer = {
   type: TransformerType.Global,
   after: [allOfTransformer],
-  transform({ objSchemas, allParams, arrSchemas, jsonLoader }) {
+  transform({ objSchemas, allParams, arrSchemas, jsonLoader, logging }) {
     for (const sch of objSchemas) {
-      if (sch.properties !== undefined) {
-        for (const key of Object.keys(sch.properties)) {
-          sch.properties[key] = transformNullable(
-            sch.properties[key],
-            jsonLoader,
-            !sch.required?.includes(key)
+      try {
+        if (sch.properties !== undefined) {
+          for (const key of Object.keys(sch.properties)) {
+            sch.properties[key] = transformNullable(
+              sch.properties[key],
+              jsonLoader,
+              !sch.required?.includes(key)
+            );
+          }
+        }
+
+        const aProperty = sch.additionalProperties;
+        if (typeof aProperty === "object" && aProperty !== null) {
+          sch.additionalProperties = transformNullable(aProperty, jsonLoader);
+        }
+      } catch (e) {
+        if (logging) {
+          logging(`Fail to transform ${sch}. ErrorMessage:${e?.message};ErrorStack:${e?.stack}.`);
+        } else {
+          console.log(
+            `Fail to transform ${sch}. ErrorMessage:${e?.message};ErrorStack:${e?.stack}.`
           );
         }
       }
-
-      const aProperty = sch.additionalProperties;
-      if (typeof aProperty === "object" && aProperty !== null) {
-        sch.additionalProperties = transformNullable(aProperty, jsonLoader);
-      }
     }
 
     for (const sch of arrSchemas) {
-      if (sch.items) {
-        if (Array.isArray(sch.items)) {
-          sch.items = sch.items.map((item) => transformNullable(item, jsonLoader));
+      try {
+        if (sch.items) {
+          if (Array.isArray(sch.items)) {
+            sch.items = sch.items.map((item) => transformNullable(item, jsonLoader));
+          } else {
+            sch.items = transformNullable(sch.items, jsonLoader);
+          }
+        }
+      } catch (e) {
+        if (logging) {
+          logging(`Fail to transform ${sch}. ErrorMessage:${e?.message};ErrorStack:${e?.stack}.`);
         } else {
-          sch.items = transformNullable(sch.items, jsonLoader);
+          console.log(
+            `Fail to transform ${sch}. ErrorMessage:${e?.message};ErrorStack:${e?.stack}.`
+          );
         }
       }
     }

lib/util/errorDefinitions.ts

@@ -7,10 +7,13 @@ export type TrafficValidationErrorCode = keyof typeof trafficValidationErrors;
 
 export type SemanticValidationErrorCode = keyof typeof semanticValidationErrors;
 
+export type ApiValidationErrorCode = keyof typeof apiValidationErrors;
+
 export type OavAllErrorCode =
   | SchemaValidationErrorCode
   | TrafficValidationErrorCode
-  | SemanticValidationErrorCode;
+  | SemanticValidationErrorCode
+  | ApiValidationErrorCode;
 
 let allErrors: {
   [code: string]: { severity: Severity; message: TemplateFunc<string>; id?: string };
@@ -21,6 +24,7 @@ export const getOavErrorMeta = <T extends OavAllErrorCode>(code: T, param: Recor
       ...schemaValidationErrors,
       ...trafficValidationErrors,
       ...semanticValidationErrors,
+      ...apiValidationErrors,
     };
   }
   const errorInfo = allErrors[code];
@@ -62,6 +66,10 @@ export const internalErrors = {
 export const schemaValidationErrors = {
   ...internalErrors,
 
+  UNRESOLVABLE_REFERENCE: {
+    severity: Severity.Critical,
+    message: strTemplate`Reference could not be resolved: ${"ref"}`,
+  },
   DISCRIMINATOR_VALUE_NOT_FOUND: {
     severity: Severity.Critical,
     message: strTemplate`Discriminator value "${"data"}" not found`,
@@ -164,6 +172,7 @@ export const schemaValidationErrors = {
   },
 } as const;
 
+// used in both example validation and api validation
 export const trafficValidationErrors = {
   ...schemaValidationErrors,
 
@@ -211,8 +220,13 @@ export const trafficValidationErrors = {
     severity: Severity.Critical,
     message: strTemplate`Long running operation should return ${"header"} in header but not provided`,
   },
+  INVALID_REQUEST_PARAMETER: {
+    severity: Severity.Critical,
+    message: strTemplate`The type of request parameter ${"param"} is invalid`,
+  },
 } as const;
 
+// used in semantic validation only
 export const semanticValidationErrors = {
   ...schemaValidationErrors,
 
@@ -220,10 +234,6 @@ export const semanticValidationErrors = {
     severity: Severity.Critical,
     message: strTemplate`Json parsing error: ${"details"}`,
   },
-  UNRESOLVABLE_REFERENCE: {
-    severity: Severity.Critical,
-    message: strTemplate`Reference could not be resolved: ${"ref"}`,
-  },
   OBJECT_MISSING_REQUIRED_PROPERTY_DEFINITION: {
     severity: Severity.Critical,
     message: strTemplate`Missing required property definition: ${"property"}`,
@@ -270,3 +280,37 @@ export const semanticValidationErrors = {
     message: strTemplate`Path parameter is declared but is not defined: ${"name"}`,
   },
 } as const;
+
+// used in api validation only
+export const apiValidationRuntimeErrors = {
+  OPERATION_NOT_FOUND_IN_CACHE: {
+    severity: Severity.Critical,
+    message: strTemplate`operation cannot be found in memory cache`,
+  },
+  OPERATION_NOT_FOUND_IN_CACHE_WITH_VERB: {
+    severity: Severity.Critical,
+    message: strTemplate`operation cannot be found in cache.verb array`,
+  },
+  OPERATION_NOT_FOUND_IN_CACHE_WITH_API: {
+    severity: Severity.Critical,
+    message: strTemplate`operation cannot be found in cache.api array`,
+  },
+  OPERATION_NOT_FOUND_IN_CACHE_WITH_PROVIDER: {
+    severity: Severity.Critical,
+    message: strTemplate`operation cannot be found in cache.provider array`,
+  },
+} as const;
+
+export const apiValidationErrors = {
+  ...trafficValidationErrors,
+  ...apiValidationRuntimeErrors,
+
+  MULTIPLE_OPERATIONS_FOUND: {
+    severity: Severity.Critical,
+    message: strTemplate`multiple operations matched from the operations cache`,
+  },
+  PII_MISMATCH: {
+    severity: Severity.Warning,
+    message: strTemplate`The value contains PII data`,
+  },
+};

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "oav",
-  "version": "2.7.4",
+  "version": "2.8.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "oav",
-  "version": "2.7.4",
+  "version": "2.8.0",
   "author": {
     "name": "Microsoft Corporation",
     "email": "azsdkteam@microsoft.com",
@@ -126,6 +126,7 @@
     "dist/lib/**/*.js.map",
     "dist/lib/**/*.d.ts.map",
     "dist/lib/**/*.handlebars",
+    "dist/lib/swaggerValidator/*.json",
     "types/**/*.d.ts",
     "*.ts",
     "lib/**/*.ts"

tsconfig.json

@@ -25,7 +25,7 @@
     // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
     // "strictNullChecks": true,              /* Enable strict null checks. */
     // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
-    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    "strictPropertyInitialization": false,    /* Enable strict checking of property initialization in classes. */
     // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
     // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */