oav/lib/swaggerValidator/ajvSchemaValidator.ts

import * as lodash from "lodash";
import {
  ChildObjectInfo,
  getInfo,
  getRootObjectInfo,
  RootObjectInfo,
} from "@azure-tools/openapi-tools-common";
import { Ajv, default as ajvInit, ErrorObject, ValidateFunction } from "ajv";
import { inject, injectable } from "inversify";
import { TYPES } from "../inversifyUtils";
import { $id, JsonLoader } from "../swagger/jsonLoader";
import { isSuppressed } from "../swagger/suppressionLoader";
import { refSelfSymbol, Schema, SwaggerSpec } from "../swagger/swaggerTypes";
import { getNameFromRef } from "../transform/context";
import {
  xmsAzureResource,
  xmsEnum,
  xmsMutability,
  xmsReadonlyRef,
  xmsSecret,
} from "../util/constants";
import { getOavErrorMeta, TrafficValidationErrorCode } from "../util/errorDefinitions";
import { Severity } from "../util/severity";
import { Writable } from "../util/utils";
import { SourceLocation } from "../util/validationError";
import { ajvEnableAll, ajvEnableArmRule, ajvEnableArmIdFormat } from "./ajv";
import {
  getIncludeErrorsMap,
  SchemaValidateContext,
  SchemaValidateFunction,
  SchemaValidateIssue,
  SchemaValidator,
  SchemaValidatorOption,
} from "./schemaValidator";

@injectable()
export class AjvSchemaValidator implements SchemaValidator {
  private ajv: Ajv;

  public constructor(
    loader: JsonLoader,
    @inject(TYPES.opts) schemaValidatorOption?: SchemaValidatorOption
  ) {
    this.ajv = ajvInit({
      // tslint:disable-next-line: no-submodule-imports
      meta: require("ajv/lib/refs/json-schema-draft-04.json"),
      schemaId: "auto",
      extendRefs: "fail",
      format: "full",
      missingRefs: true,
      addUsedSchema: false,
      removeAdditional: false,
      nullable: true,
      allErrors: true,
      messages: true,
      verbose: true,
      inlineRefs: false,
      passContext: true,
      loopRequired: 2,
      unknownFormats: "ignore",
      loadSchema: async (uri) => {
        const spec: SwaggerSpec = await loader.resolveFile(uri);
        return { [$id]: spec[$id], definitions: spec.definitions, parameters: spec.parameters };
      },
    });
    ajvEnableAll(this.ajv, loader);

    // always enable the armId format validation
    ajvEnableArmIdFormat(this.ajv);

    if (schemaValidatorOption?.isArmCall === true) {
      ajvEnableArmRule(this.ajv);
    }
  }

  public async compileAsync(schema: Schema): Promise<SchemaValidateFunction> {
    const validate = await this.ajv.compileAsync(schema);
    return this.getValidateFunction(validate);
  }

  public compile(schema: Schema): SchemaValidateFunction {
    const validate = this.ajv.compile(schema);
    return this.getValidateFunction(validate);
  }

  private getValidateFunction(validate: ValidateFunction) {
    const ret = function validateSchema(ctx: SchemaValidateContext, data: any) {
      const result: SchemaValidateIssue[] = [];
      const isValid = validateSchema.validate.call(ctx, data);
      if (!isValid) {
        const errors = ReValidateIfNeed(
          validateSchema.validate.errors!,
          ctx,
          data,
          validateSchema.validate
        );
        if (errors.length > 0) {
          ajvErrorListToSchemaValidateIssueList(errors, ctx, result);
        }
        validateSchema.validate.errors = null;
      }
      return result;
    };
    ret.validate = validate;
    return ret;
  }
}

export const ajvErrorListToSchemaValidateIssueList = (
  errors: ErrorObject[],
  ctx: SchemaValidateContext,
  result: SchemaValidateIssue[]
) => {
  const includeErrorsSet = getIncludeErrorsMap(ctx.includeErrors);

  const similarIssues: Map<string, SchemaValidateIssue> = new Map();
  for (const error of errors) {
    const issue = ajvErrorToSchemaValidateIssue(error, ctx);
    if (
      issue === undefined ||
      (includeErrorsSet !== undefined && !includeErrorsSet.has(issue.code))
    ) {
      continue;
    }

    const issueHashKey = [
      issue.code,
      issue.message,
      issue.source.url,
      issue.source.position.column.toString(),
      issue.source.position.line.toString(),
    ].join("|");
    const similarIssue = similarIssues.get(issueHashKey);
    if (similarIssue === undefined) {
      similarIssues.set(issueHashKey, issue);
      result.push(issue);
      continue;
    }

    similarIssue.jsonPathsInPayload.push(issue.jsonPathsInPayload[0]);
  }

  for (const issue of result) {
    if (issue.jsonPathsInPayload.length > 1) {
      (issue as any).jsonPathsInPayload = [...new Set(issue.jsonPathsInPayload)];
    }
  }
};

export const sourceMapInfoToSourceLocation = (
  info?: ChildObjectInfo | RootObjectInfo
): Writable<SourceLocation> => {
  return info === undefined
    ? {
        url: "",
        position: { line: -1, column: -1 },
      }
    : {
        url: getRootObjectInfo(info).url,
        position: { line: info.position.line, column: info.position.column },
      };
};

export const ajvErrorToSchemaValidateIssue = (
  err: ErrorObject,
  ctx: SchemaValidateContext
): SchemaValidateIssue | undefined => {
  const { parentSchema, params } = err;
  let { schema } = err;

  if (shouldSkipError(err, ctx)) {
    return undefined;
  }

  let dataPath = err.dataPath;
  const extraDataPath =
    (params as any).additionalProperty ??
    (params as any).missingProperty ??
    (parentSchema as Schema).discriminator;
  if (extraDataPath !== undefined) {
    dataPath = `${dataPath}.${extraDataPath}`;
    if (schema[extraDataPath] !== undefined) {
      schema = schema[extraDataPath];
    }
  }

  const errInfo = ajvErrorCodeToOavErrorCode(err, ctx);
  if (errInfo === undefined) {
    return undefined;
  }
  if (isSuppressed(err.parentSchema, errInfo.code, errInfo.message)) {
    return undefined;
  }

  let sch: Schema | undefined = parentSchema;
  let info = getInfo(sch);
  if (info === undefined) {
    sch = schema;
    info = getInfo(sch);
  }
  const source = sourceMapInfoToSourceLocation(info);
  if (sch?.[refSelfSymbol] !== undefined) {
    source.jsonRef = sch[refSelfSymbol]!.substr(sch[refSelfSymbol]!.indexOf("#"));
  }

  const result = errInfo as SchemaValidateIssue;
  result.jsonPathsInPayload = [dataPath];
  result.schemaPath = err.schemaPath;
  result.source = source;

  return result;
};

const ReValidateIfNeed = (
  originalErrors: ErrorObject[],
  ctx: SchemaValidateContext,
  data: any,
  validate: ValidateFunction
): ErrorObject[] => {
  const result: ErrorObject[] = [];
  const newData = lodash.cloneDeep(data);

  for (const originalError of originalErrors) {
    validate.errors = null;
    const { schema, parentSchema: parentSch, keyword, data: errorData, dataPath } = originalError;
    const parentSchema = parentSch as Schema;

    // If the value of query parameter is in string format, we can revalidate this error
    if (
      !ctx.isResponse &&
      keyword === "type" &&
      schema === "array" &&
      typeof errorData === "string" &&
      (parentSchema as any)?.["in"] === "query"
    ) {
      const arrayData = errorData.split(",").map((item) => {
        // when item is number
        const numberRegex = /^[+-]?\d+(\.\d+)?([Ee]\+?\d+)?$/g;
        if (numberRegex.test(item)) {
          return parseFloat(item);
        }
        // when item is boolean
        if (item === "true" || item === "false") {
          return item === "true";
        }
        return item;
      });
      const position = dataPath.substr(1);
      lodash.set(newData, position, arrayData);
      const isValid = validate.call(ctx, newData);
      if (!isValid) {
        // if validate.errors have new errors, add them to result
        for (const newError of validate.errors!) {
          let [includedInResult, includedInOriginalErrors] = [false, false];
          for (const resultError of result) {
            if (lodash.isEqual(newError, resultError)) {
              // error is included in result
              includedInResult = true;
              break;
            }
          }
          if (!includedInResult) {
            for (const eachOriginalError of originalErrors) {
              if (lodash.isEqual(newError, eachOriginalError)) {
                // error is included in originalErrors
                includedInOriginalErrors = true;
                break;
              }
            }
            if (!includedInOriginalErrors) {
              result.push(newError);
            }
          }
        }
      }
      continue;
    }
    result.push(originalError);
  }

  return result;
};

const shouldSkipError = (error: ErrorObject, cxt: SchemaValidateContext) => {
  const { parentSchema: parentSch, params, keyword } = error;
  const parentSchema = parentSch as Schema;
  // If schema has allof property, we can get schema in "_realschema", so is data
  const schema = Object.keys(error).includes("_realSchema")
    ? (error as any)._realSchema
    : error.schema;
  const data = Object.keys(error).includes("_realData") ? (error as any)._realData : error.data;

  if (schema?._skipError || parentSchema._skipError) {
    return true;
  }

  // If we're erroring on the added property refWithReadOnly simply ignore the error
  if (
    error.keyword === "additionalProperties" &&
    (params as any).additionalProperty === "refWithReadOnly"
  ) {
    return true;
  }

  // If a response has x-ms-mutability property and its missing the read we can skip this error
  if (
    cxt.isResponse &&
    ((keyword === "required" &&
      (parentSchema.properties?.[(params as any).missingProperty]?.[xmsMutability]?.indexOf(
        "read"
      ) === -1 ||
        // required check is ignored when x-ms-secret is true
        (parentSchema.properties?.[(params as any).missingProperty] as any)?.[xmsSecret] ===
          true)) ||
      (keyword === "type" && data === null && parentSchema[xmsMutability]?.indexOf("read") === -1))
  ) {
    return true;
  }

  // If a request is missing a required property that is readOnly we can skip this error
  if (
    !cxt.isResponse &&
    keyword === "required" &&
    (parentSchema.properties?.[(params as any).missingProperty]?.[xmsReadonlyRef] ||
      parentSchema.properties?.[(params as any).missingProperty]?.readOnly)
  ) {
    return true;
  }

  // If a response has property which x-ms-secret value is "true" in post we can skip this error
  if (
    cxt.isResponse &&
    (cxt as any)?.httpMethod === "post" &&
    // should skip error when x-ms-secret is "true"
    ((keyword === "x-ms-secret" && (parentSchema as any)?.[xmsSecret] === true) ||
      // should skip error when x-ms-secret is "true" and x-ms-mutability is "create" and "update"
      (keyword === "x-ms-mutability" &&
        (parentSchema as any)?.[xmsSecret] === true &&
        parentSchema[xmsMutability]?.indexOf("read") === -1))
  ) {
    return true;
  }

  // If payload has property with date-time parameter and its value is valid except missing "Z" in the end we can skip this error
  if (keyword === "format" && schema === "date-time" && typeof data === "string") {
    const reg = /^\d+-(0\d|1[0-2])-([0-2]\d|3[01])T([01]\d|2[0-3]):[0-5][0-9]:[0-5][0-9]/;
    // intercept time, example: 2008-09-22T14:01:54
    const time = data.slice(0, 19);
    if (reg.test(time)) {
      const dateZ = new Date(data + "Z").toUTCString();
      // validate hour
      const ifHoursAreSame = time.slice(11, 13) === dateZ.slice(17, 19);
      // validate day for leap year, example: 2008-02-29
      const ifDaysAreSame = time.slice(8, 10) === dateZ.slice(5, 7);
      return ifHoursAreSame && ifDaysAreSame;
    }
  }

  // If a response data has multipleOf property, and it divided by multipleOf value is an integer, we can skip this error
  if (keyword === "multipleOf" && typeof schema === "number" && typeof data === "number") {
    let [newSchema, newData] = [schema, data];
    while (newSchema < 1) {
      newSchema *= 10;
      newData *= 10;
    }
    const result = newData / newSchema;
    // should skip error when response data divided by multipleOf value is an integer
    return result === parseInt(String(result));
  }

  return false;
};

const errorKeywordsMapping: { [key: string]: TrafficValidationErrorCode } = {
  additionalProperties: "OBJECT_ADDITIONAL_PROPERTIES",
  required: "OBJECT_MISSING_REQUIRED_PROPERTY",
  format: "INVALID_FORMAT",
  type: "INVALID_TYPE",
  pattern: "PATTERN",
  minimum: "MINIMUM",
  maximum: "MAXIMUM",
  exclusiveMinimum: "MINIMUM_EXCLUSIVE",
  exclusiveMaximum: "MAXIMUM_EXCLUSIVE",
  minLength: "MIN_LENGTH",
  maxLength: "MAX_LENGTH",
  maxItems: "ARRAY_LENGTH_LONG",
  minItems: "ARRAY_LENGTH_SHORT",
  maxProperties: "OBJECT_PROPERTIES_MAXIMUM",
  minProperties: "OBJECT_PROPERTIES_MINIMUM",
  uniqueItems: "ARRAY_UNIQUE",
  additionalItems: "ARRAY_ADDITIONAL_ITEMS",
  anyOf: "ANY_OF_MISSING",
  dependencies: "OBJECT_DEPENDENCY_KEY",
  multiple: "MULTIPLE_OF",
  discriminatorMap: "DISCRIMINATOR_VALUE_NOT_FOUND",
  [xmsAzureResource]: "MISSING_RESOURCE_ID",
};
// Should be type "never" to ensure we've covered all the errors
// export type MissingErrorCode = Exclude<
//   ExtendedErrorCode,
//   | keyof typeof validateErrorMessages
//   | "PII_MISMATCH" // Used in openapi-validate
//   | "INTERNAL_ERROR" // Used in liveValidator
//   | "UNRESOLVABLE_REFERENCE"
//   | "NOT_PASSED" // If keyword mapping not found then we use this error
//   | "OPERATION_NOT_FOUND_IN_CACHE_WITH_PROVIDER" // Covered by liveValidator
//   | "OPERATION_NOT_FOUND_IN_CACHE_WITH_API"
//   | "OPERATION_NOT_FOUND_IN_CACHE_WITH_VERB"
//   | "OPERATION_NOT_FOUND_IN_CACHE"
//   | "MULTIPLE_OPERATIONS_FOUND"
//   | "INVALID_RESPONSE_HEADER"
//   | "INVALID_REQUEST_PARAMETER"
// >;

const transformParamsKeyword = new Set([
  "pattern",
  "additionalProperties",
  "type",
  "format",
  "multipleOf",
  "required",
  xmsAzureResource,
]);

const transformReverseParamsKeyword = new Set([
  "minLength",
  "maxLength",
  "maxItems",
  "minItems",
  "maxProperties",
  "minProperties",
  "minimum",
  "maximum",
  "exclusiveMinimum",
  "exclusiveMaximum",
  "discriminatorMap",
]);

interface MetaErr {
  code: string;
  message: string;
  severity: Severity;
  params?: any;
}
export const ajvErrorCodeToOavErrorCode = (
  error: ErrorObject,
  ctx: SchemaValidateContext
): MetaErr | undefined => {
  const { keyword, parentSchema: parentSch } = error;
  const parentSchema = parentSch as Schema | undefined;
  let { params, data } = error;
  let result: MetaErr | undefined = {
    code: "NOT_PASSED",
    message: error.message!,
    severity: Severity.Verbose,
  };

  // Workaround for incorrect ajv behavior.
  // See https://github.com/ajv-validator/ajv/blob/v6/lib/dot/custom.jst#L74
  if ((error as any)._realData !== undefined) {
    data = (error as any)._realData;
  }

  switch (keyword) {
    case "enum":
      const { allowedValues } = params as any;
      result =
        data === null && parentSchema?.nullable
          ? undefined
          : isEnumCaseMismatch(data, allowedValues)
          ? getOavErrorMeta("ENUM_CASE_MISMATCH", { data })
          : parentSchema?.[xmsEnum]?.modelAsString
          ? undefined
          : getOavErrorMeta("ENUM_MISMATCH", { data });
      params = [data, allowedValues];
      break;

    case "readOnly":
    case xmsMutability:
    case xmsSecret:
      const param = {
        key: getNameFromRef(parentSchema),
        value: Array.isArray(data) ? data.join(",") : JSON.stringify(data),
      };
      params = [param.key, null];
      result =
        keyword === xmsSecret
          ? getOavErrorMeta("SECRET_PROPERTY", param)
          : ctx.isResponse
          ? getOavErrorMeta("WRITEONLY_PROPERTY_NOT_ALLOWED_IN_RESPONSE", param)
          : getOavErrorMeta("READONLY_PROPERTY_NOT_ALLOWED_IN_REQUEST", param);
      break;

    case "oneOf":
      result =
        (params as any).passingSchemas === null
          ? getOavErrorMeta("ONE_OF_MISSING", {})
          : getOavErrorMeta("ONE_OF_MULTIPLE", {});
      params = [];
      break;

    case "type":
      (params as any).type = (params as any).type.replace(",null", "");
      data = schemaType(data);
      break;

    case "discriminatorMap":
      data = (params as any).discriminatorValue;
      break;

    case "maxLength":
    case "minLength":
    case "maxItems":
    case "minItems":
      data = data.length;
      break;

    case "maxProperties":
    case "minProperties":
      data = Object.keys(data).length;
      break;

    case "minimum":
    case "maximum":
    case "exclusiveMinimum":
    case "exclusiveMaximum":
      params = { limit: (params as any).limit };
      break;
  }

  const code = errorKeywordsMapping[keyword];
  if (code !== undefined) {
    result = getOavErrorMeta(code, { ...params, data });
  }

  if (transformParamsKeyword.has(keyword)) {
    params = Object.values(params);
    if (typeof data !== "object") {
      (params as any).push(data);
    }
  } else if (transformReverseParamsKeyword.has(keyword)) {
    params = Object.values(params);
    (params as any[]).unshift(data);
  }

  if (result !== undefined) {
    result.params = params;
  }
  return result;
};

const isEnumCaseMismatch = (data: string, enumList: Array<string | number>) => {
  if (typeof data !== "string") {
    return false;
  }
  data = data.toLowerCase();
  for (const val of enumList) {
    if (typeof val === "string" && val.toLowerCase() === data) {
      return true;
    }
  }
  return false;
};

const schemaType = (what: any): string => {
  const to = typeof what;

  if (to === "object") {
    if (what === null) {
      return "null";
    }
    if (Array.isArray(what)) {
      return "array";
    }
    return "object"; // typeof what === 'object' && what === Object(what) && !Array.isArray(what);
  }

  if (to === "number") {
    if (Number.isFinite(what)) {
      if (what % 1 === 0) {
        return "integer";
      } else {
        return "number";
      }
    }
    if (Number.isNaN(what)) {
      return "not-a-number";
    }
    return "unknown-number";
  }
  return to; // undefined, boolean, string, function
};