Allow to specify specific validation errors to include in results (#406)

* Allow to specify specific validation errors to include in results * Address feedback * remove duplication
2019-03-29 11:12:10 -07:00 · 2019-03-29 11:12:10 -07:00 · 7c49309fdc
--- a/ChangeLog.md
+++ b/ChangeLog.md
@ -1,5 +1,9 @@
 # Changelog

+### 03/28/2019 0.15.1
+
+- Allow for live validation to exclude/include specific errors.
+
 ### 03/27/2019 0.15.0

 - Refactor live validator and new types for validation results.
--- a/index.ts
+++ b/index.ts
@ -1,6 +1,5 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
-
 import * as C from "./lib/util/constants"

 // Easy to use methods from validate.ts
@ -24,11 +23,14 @@ export {
  SemanticValidationError
 } from "./lib/util/getErrorsFromSemanticValidation"
 export {
-  errorConstants,
-  errorCodeToSeverity,
  NodeError,
-  ValidationError,
-  ValidationResult
+  ValidationErrorMetadata,
+  errorCodeToErrorMetadata,
+  ValidationResult,
+  ErrorCode,
+  ExtendedErrorCode,
+  WrapperErrorCode,
+  RuntimeErrorCode
 } from "./lib/util/validationError"

 // Classes
--- a/lib/util/getErrorsFromSemanticValidation.ts
+++ b/lib/util/getErrorsFromSemanticValidation.ts
@ -1,7 +1,7 @@
 import { SpecValidationResult } from "../validators/specValidator"

 import { BaseValidationError } from "./baseValidationError"
-import { errorCodeToSeverity, NodeError, serializeErrors } from "./validationError"
+import { errorCodeToErrorMetadata, NodeError, serializeErrors } from "./validationError"
 import { ValidationResultSource } from "./validationResultSource"

 export interface SemanticValidationError extends BaseValidationError<NodeError<any>> {
@ -32,7 +32,7 @@ export const getErrorsFromSemanticValidation = (

    // process serialized errors
    const semanticErrors: SemanticValidationError[] = serializedErrors.map(serializedError => {
-      const severity = errorCodeToSeverity(serializedError.code)
+      const severity = errorCodeToErrorMetadata(serializedError.code).severity
      const semanticError: SemanticValidationError = {
        source: ValidationResultSource.GLOBAL,
        code: serializedError.code,
--- a/lib/util/toModelErrors.ts
+++ b/lib/util/toModelErrors.ts
@ -1,7 +1,7 @@
 import * as it from "@ts-common/iterator"

 import { ModelValidationError } from "./modelValidationError"
-import { errorCodeToSeverity } from "./validationError"
+import { errorCodeToErrorMetadata, ExtendedErrorCode } from "./validationError"
 import { ValidationResultSource } from "./validationResultSource"

 /**
@ -18,7 +18,7 @@ export function toModelErrors(
    if (value.code === undefined) {
      value.code = "INTERNAL_ERROR"
    }
-    const severity = errorCodeToSeverity(value.code)
+    const severity = errorCodeToErrorMetadata(value.code as ExtendedErrorCode).severity
    const modelError: ModelValidationError = {
      operationId,
      scenario,
--- a/lib/util/utils.ts
+++ b/lib/util/utils.ts
@ -34,32 +34,6 @@ export async function executePromisesSequentially<T>(
  return result
 }

-/*
- * Generates a randomId
- *
- * @param {string} [prefix] A prefix to which the random numbers will be appended.
- *
- * @param {object} [existingIds] An object of existingIds. The function will
- * ensure that the randomId is not one of the existing ones.
- *
- * @return {string} result A random string
- */
-export function generateRandomId(prefix: string, existingIds: {}): string {
-  let randomStr: string
-  while (true) {
-    randomStr = Math.random()
-      .toString(36)
-      .substr(2, 12)
-    if (prefix && typeof prefix.valueOf() === "string") {
-      randomStr = prefix + randomStr
-    }
-    if (!existingIds || !(randomStr in existingIds)) {
-      break
-    }
-  }
-  return randomStr
-}
-
 export interface Reference {
  readonly filePath?: string
  readonly localReference?: LocalReference
@ -147,25 +121,6 @@ export function joinPath(...args: string[]): string {
  return finalPath
 }

-/*
- * Provides a parsed JSON from the given file path or a url. Same as parseJson(). However,
- * this method accepts variable number of path segments as strings and joins them together.
- * After joining the path, it internally calls parseJson().
- *
- * @param variable number of arguments and all the arguments must be of type string.
- *
- * @returns {object} jsonDoc - Parsed document in JSON format.
- */
-/*
-export async function parseJsonWithPathFragments(
-  suppression: Suppression | undefined,
-  ...args: string[],
-): Promise<SwaggerObject> {
-  const specPath = joinPath(...args)
-  return await jsonUtils.parseJson(suppression, specPath)
-}
-*/
-
 /*
 * Merges source object into the target object
 * @param {object} source The object that needs to be merged
--- a/lib/util/validationError.ts
+++ b/lib/util/validationError.ts
@ -1,6 +1,5 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
-
 import { flatMap, fold } from "@ts-common/iterator"
 import * as json from "@ts-common/json"
 import { FilePosition } from "@ts-common/source-map"
@ -15,72 +14,119 @@ import { Severity } from "./severity"
 * @class
 * Error that results from validations.
 */
-export class ValidationError {
-  /**
-   *
-   * @param name Validation Error Name
-   * @param severity The severity of the error
-   */
-  public constructor(public readonly name: string, public readonly severity: Severity) {}
+interface ErrorCodeMetadata {
+  readonly severity: Severity
+  readonly docUrl: string
 }

-const validationErrorEntry = (id: string, severity: Severity): [string, ValidationError] => [
-  id,
-  new ValidationError(id, severity)
-]
+export type ValidationErrorMetadata = ErrorCodeMetadata & { code: ExtendedErrorCode }

-export const errorConstants = new Map<string, ValidationError>([
-  validationErrorEntry("INVALID_TYPE", Severity.Critical),
-  validationErrorEntry("INVALID_FORMAT", Severity.Critical),
-  validationErrorEntry("ENUM_MISMATCH", Severity.Critical),
-  validationErrorEntry("ENUM_CASE_MISMATCH", Severity.Error),
-  validationErrorEntry("PII_MISMATCH", Severity.Warning),
-  validationErrorEntry("ANY_OF_MISSING", Severity.Critical),
-  validationErrorEntry("ONE_OF_MISSING", Severity.Critical),
-  validationErrorEntry("ONE_OF_MULTIPLE", Severity.Critical),
-  validationErrorEntry("NOT_PASSED", Severity.Critical),
-  // arrays
-  validationErrorEntry("ARRAY_LENGTH_SHORT", Severity.Critical),
-  validationErrorEntry("ARRAY_LENGTH_LONG", Severity.Critical),
-  validationErrorEntry("ARRAY_UNIQUE", Severity.Critical),
-  validationErrorEntry("ARRAY_ADDITIONAL_ITEMS", Severity.Critical),
-  // numeric
-  validationErrorEntry("MULTIPLE_OF", Severity.Critical),
-  validationErrorEntry("MINIMUM", Severity.Critical),
-  validationErrorEntry("MINIMUM_EXCLUSIVE", Severity.Critical),
-  validationErrorEntry("MAXIMUM", Severity.Critical),
-  validationErrorEntry("MAXIMUM_EXCLUSIVE", Severity.Critical),
-  // objects
-  validationErrorEntry("OBJECT_PROPERTIES_MINIMUM", Severity.Critical),
-  validationErrorEntry("OBJECT_PROPERTIES_MAXIMUM", Severity.Critical),
-  validationErrorEntry("OBJECT_MISSING_REQUIRED_PROPERTY", Severity.Critical),
-  validationErrorEntry("OBJECT_ADDITIONAL_PROPERTIES", Severity.Critical),
-  validationErrorEntry("OBJECT_DEPENDENCY_KEY", Severity.Warning),
-  // string
-  validationErrorEntry("MIN_LENGTH", Severity.Critical),
-  validationErrorEntry("MAX_LENGTH", Severity.Critical),
-  validationErrorEntry("PATTERN", Severity.Critical),
-  // operation
-  validationErrorEntry("OPERATION_NOT_FOUND_IN_CACHE", Severity.Critical),
-  validationErrorEntry("OPERATION_NOT_FOUND_IN_CACHE_WITH_VERB", Severity.Critical),
-  validationErrorEntry("OPERATION_NOT_FOUND_IN_CACHE_WITH_API", Severity.Critical),
-  validationErrorEntry("OPERATION_NOT_FOUND_IN_CACHE_WITH_PROVIDER", Severity.Critical),
-  validationErrorEntry("MULTIPLE_OPERATIONS_FOUND", Severity.Critical),
-  // others
-  validationErrorEntry("INVALID_RESPONSE_HEADER", Severity.Critical),
-  validationErrorEntry("INVALID_RESPONSE_CODE", Severity.Critical),
-  validationErrorEntry("INVALID_RESPONSE_BODY", Severity.Critical),
-  validationErrorEntry("INVALID_REQUEST_PARAMETER", Severity.Critical),
-  validationErrorEntry("INVALID_CONTENT_TYPE", Severity.Error),
-  validationErrorEntry("INTERNAL_ERROR", Severity.Critical)
-])
+export type ExtendedErrorCode = ErrorCode | WrapperErrorCode | RuntimeErrorCode
+export type ErrorCode = keyof typeof errorConstants
+export type WrapperErrorCode = keyof typeof wrapperErrorConstants
+export type RuntimeErrorCode = keyof typeof runtimeErrorConstants

+const errorConstants = {
+  INVALID_TYPE: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  INVALID_FORMAT: { severity: Severity.Critical, docUrl: "" },
+  ENUM_MISMATCH: { severity: Severity.Critical, docUrl: "" },
+  ENUM_CASE_MISMATCH: { severity: Severity.Error, docUrl: "" },
+  PII_MISMATCH: { severity: Severity.Warning, docUrl: "" },
+  NOT_PASSED: { severity: Severity.Critical, docUrl: "" },
+  ARRAY_LENGTH_SHORT: { severity: Severity.Critical, docUrl: "" },
+  ARRAY_LENGTH_LONG: { severity: Severity.Critical, docUrl: "" },
+  ARRAY_UNIQUE: { severity: Severity.Critical, docUrl: "" },
+  ARRAY_ADDITIONAL_ITEMS: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  MULTIPLE_OF: { severity: Severity.Critical, docUrl: "" },
+  MINIMUM: { severity: Severity.Critical, docUrl: "" },
+  MINIMUM_EXCLUSIVE: { severity: Severity.Critical, docUrl: "" },
+  MAXIMUM: { severity: Severity.Critical, docUrl: "" },
+  MAXIMUM_EXCLUSIVE: { severity: Severity.Critical, docUrl: "" },
+  OBJECT_PROPERTIES_MINIMUM: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  OBJECT_PROPERTIES_MAXIMUM: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  OBJECT_MISSING_REQUIRED_PROPERTY: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  OBJECT_ADDITIONAL_PROPERTIES: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  OBJECT_DEPENDENCY_KEY: { severity: Severity.Warning, docUrl: "" },
+  MIN_LENGTH: { severity: Severity.Critical, docUrl: "" },
+  MAX_LENGTH: { severity: Severity.Critical, docUrl: "" },
+  PATTERN: { severity: Severity.Critical, docUrl: "" },
+  INVALID_RESPONSE_CODE: { severity: Severity.Critical, docUrl: "" },
+  INVALID_CONTENT_TYPE: { severity: Severity.Error, docUrl: "" }
+}
+
+const wrapperErrorConstants = {
+  ANY_OF_MISSING: { severity: Severity.Critical, docUrl: "" },
+  ONE_OF_MISSING: { severity: Severity.Critical, docUrl: "" },
+  ONE_OF_MULTIPLE: { severity: Severity.Critical, docUrl: "" },
+  MULTIPLE_OPERATIONS_FOUND: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  INVALID_RESPONSE_HEADER: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  INVALID_RESPONSE_BODY: { severity: Severity.Critical, docUrl: "" },
+  INVALID_REQUEST_PARAMETER: {
+    severity: Severity.Critical,
+    docUrl: ""
+  }
+}
+
+const runtimeErrorConstants = {
+  OPERATION_NOT_FOUND_IN_CACHE: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  OPERATION_NOT_FOUND_IN_CACHE_WITH_VERB: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  OPERATION_NOT_FOUND_IN_CACHE_WITH_API: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  OPERATION_NOT_FOUND_IN_CACHE_WITH_PROVIDER: {
+    severity: Severity.Critical,
+    docUrl: ""
+  },
+  INTERNAL_ERROR: { severity: Severity.Critical, docUrl: "" }
+}
+
+const allErrorConstants = {
+  ...errorConstants,
+  ...wrapperErrorConstants,
+  ...runtimeErrorConstants
+}
 /**
- * Gets the severity from an error code. If the code is unknown assume critical.
+ * Gets the validation error metadata from an error code. If the code is unknown assume critical.
 */
-export const errorCodeToSeverity = (code: string): Severity => {
-  const errorConstant = errorConstants.get(code)
-  return errorConstant ? errorConstant.severity : Severity.Critical
+export const errorCodeToErrorMetadata = (code: ExtendedErrorCode): ValidationErrorMetadata => {
+  return {
+    ...(allErrorConstants[code] || {
+      severity: Severity.Critical,
+      docUrl: ""
+    }),
+    code
+  }
 }

 export interface LiveValidationIssue {
--- a/lib/validators/liveValidator.ts
+++ b/lib/validators/liveValidator.ts
@ -19,7 +19,8 @@ import { log } from "../util/logging"
 import { Severity } from "../util/severity"
 import * as utils from "../util/utils"
 import {
-  errorCodeToSeverity,
+  ErrorCode,
+  errorCodeToErrorMetadata,
  processValidationErrors,
  RuntimeException,
  SourceLocation
@ -39,11 +40,11 @@ export interface LiveValidatorOptions {
  isPathCaseSensitive: boolean
 }

-export interface ApiVersion {
+interface ApiVersion {
  [method: string]: Operation[]
 }

-export interface Provider {
+interface Provider {
  [apiVersion: string]: ApiVersion
 }

@ -71,15 +72,15 @@ interface OperationInfo {
 export interface RequestValidationResult {
  readonly successfulRequest: boolean
  readonly operationInfo: OperationInfo
-  errors: LiveValidationIssue[]
-  runtimeException?: RuntimeException
+  readonly errors: LiveValidationIssue[]
+  readonly runtimeException?: RuntimeException
 }

 export interface ResponseValidationResult {
  readonly successfulResponse: boolean
  readonly operationInfo: OperationInfo
-  errors: LiveValidationIssue[]
-  runtimeException?: RuntimeException
+  readonly errors: LiveValidationIssue[]
+  readonly runtimeException?: RuntimeException
 }

 export interface ValidationResult {
@ -89,19 +90,28 @@ export interface ValidationResult {
 }

 export interface ApiOperationIdentifier {
-  url: string
-  method: string
+  readonly url: string
+  readonly method: string
 }
+
 export interface LiveValidationIssue {
-  code: string
-  message: string
-  pathInPayload: string
-  severity: Severity
-  similarPaths: string[]
-  source: SourceLocation
-  documentationUrl: string
-  params?: string[]
-  inner?: object[]
+  readonly code: ErrorCode
+  readonly message: string
+  readonly pathInPayload: string
+  readonly severity: Severity
+  readonly similarPaths: string[]
+  readonly source: SourceLocation
+  readonly documentationUrl: string
+  readonly params?: string[]
+  readonly inner?: object[]
+}
+
+/**
+ * Options for a validation operation.
+ * If `includeErrors` is missing or empty, all error codes will be included.
+ */
+export interface ValidateOptions {
+  readonly includeErrors?: ErrorCode[]
 }

 type OperationWithApiVersion = Operation & { apiVersion: string }
@ -345,7 +355,10 @@ export class LiveValidator {
  /**
   *  Validates live request.
   */
-  public validateLiveRequest(liveRequest: LiveRequest): RequestValidationResult {
+  public validateLiveRequest(
+    liveRequest: LiveRequest,
+    options: ValidateOptions = {}
+  ): RequestValidationResult {
    let operation
    try {
      operation = this.findSpecOperation(liveRequest.url, liveRequest.method)
@ -365,7 +378,14 @@ export class LiveValidator {
      const reqResult = operation.validateRequest(liveRequest)
      const processedErrors = processValidationErrors({ errors: [...reqResult.errors] })
      errors = processedErrors
-        ? processedErrors.map(err => this.toLiveValidationIssue(err as any))
+        ? processedErrors
+            .map(err => this.toLiveValidationIssue(err as any))
+            .filter(
+              err =>
+                !options.includeErrors ||
+                options.includeErrors.length === 0 ||
+                options.includeErrors.includes(err.code)
+            )
        : []
    } catch (reqValidationError) {
      const msg =
@ -390,7 +410,7 @@ export class LiveValidator {
      message: err.message,
      pathInPayload: err.path,
      inner: err.inner,
-      severity: errorCodeToSeverity(err.code),
+      severity: errorCodeToErrorMetadata(err.code).severity,
      params: err.params,
      similarPaths: err.similarPaths || [],
      source: {
@ -413,7 +433,8 @@ export class LiveValidator {
   */
  public validateLiveResponse(
    liveResponse: LiveResponse,
-    specOperation: ApiOperationIdentifier
+    specOperation: ApiOperationIdentifier,
+    options: ValidateOptions = {}
  ): ResponseValidationResult {
    let operation: OperationWithApiVersion
    try {
@ -440,7 +461,14 @@ export class LiveValidator {
      const resResult = operation.validateResponse(liveResponse)
      const processedErrors = processValidationErrors({ errors: [...resResult.errors] })
      errors = processedErrors
-        ? processedErrors.map(err => this.toLiveValidationIssue(err as any))
+        ? processedErrors
+            .map(err => this.toLiveValidationIssue(err as any))
+            .filter(
+              err =>
+                !options.includeErrors ||
+                options.includeErrors.length === 0 ||
+                options.includeErrors.includes(err.code)
+            )
        : []
    } catch (resValidationError) {
      const msg =
@ -463,11 +491,11 @@ export class LiveValidator {

  /**
   * Validates live request and response.
-   *
-   * @param requestResponsePair - The wrapper that contains the live request and response
-   * @returns  validationResult - Validation result for given input
   */
-  public validateLiveRequestResponse(requestResponseObj: RequestResponsePair): ValidationResult {
+  public validateLiveRequestResponse(
+    requestResponseObj: RequestResponsePair,
+    options?: ValidateOptions
+  ): ValidationResult {
    const validationResult: ValidationResult = {
      requestValidationResult: {
        successfulRequest: false,
@ -510,11 +538,15 @@ export class LiveValidator {
    const request = requestResponseObj.liveRequest
    const response = requestResponseObj.liveResponse

-    const requestValidationResult = this.validateLiveRequest(request)
-    const responseValidationResult = this.validateLiveResponse(response, {
-      method: request.method,
-      url: request.url
-    })
+    const requestValidationResult = this.validateLiveRequest(request, options)
+    const responseValidationResult = this.validateLiveResponse(
+      response,
+      {
+        method: request.method,
+        url: request.url
+      },
+      options
+    )

    return {
      requestValidationResult,
@ -731,7 +763,6 @@ export class LiveValidator {
 * OAV expects the url that is sent to match exactly with the swagger path. For this we need to keep only the part after
 * where the swagger path starts. Currently those are '/subscriptions' and '/providers'.
 */
-
 export function formatUrlToExpectedFormat(requestUrl: string): string {
  return requestUrl.substring(requestUrl.search("/?(subscriptions|providers)"))
 }
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
  "name": "oav",
-  "version": "0.15.0",
+  "version": "0.15.1",
  "author": {
    "name": "Microsoft Corporation",
    "email": "azsdkteam@microsoft.com",
--- a/test/liveValidation/payloads/multipleErrors_input.json
+++ b/test/liveValidation/payloads/multipleErrors_input.json
@ -0,0 +1,59 @@
+{
+  "liveRequest": {
+    "headers": {
+      "strict-Transport-Security": "max-age=31536000; includeSubDomains",
+      "x-ms-request-id": "97856fec-304a-4317-87f0-f04c328402d3",
+      "date": "Tue, 11 Sep 2018 19:00:21 GMT",
+      "eTag": "\"AAAAAAAAQqUAAAAAAABCpw==\"",
+      "server": "Microsoft-HTTPAPI/2.0",
+      "Content-Type": "application/json"
+    },
+    "method": "PUT",
+    "url": "/subscriptions/randomSub/resourceGroups/randomRG/providers/Microsoft.ApiManagement/service/randomService/users/radomUser?api-version=2018-01-01",
+    "body": {
+      "properties": {
+        "firstName": "aaaaa",
+        "lastName": "aaaaa",
+        "email": "aaaaaaaaaaaaaaaaaaaaaa",
+        "password": "aaaaaaaaaa",
+        "state": "active"
+      }
+    },
+    "query": { "api-version": "2018-01-01" }
+  },
+  "liveResponse": {
+    "statusCode": "201",
+    "headers": {
+      "strict-Transport-Security": "max-age=31536000; includeSubDomains",
+      "x-ms-request-id": "97856fec-304a-4317-87f0-f04c328402d3",
+      "date": "Tue, 11 Sep 2018 19:00:21 GMT",
+      "eTag": "\"AAAAAAAAQqUAAAAAAABCpw==\"",
+      "server": "Microsoft-HTTPAPI/2.0",
+      "Content-Type": "application/json"
+    },
+    "body": {
+      "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "type": "Microsoft.ApiManagement/service/users",
+      "name": "aaaaaaaaaaaa",
+      "properties": {
+        "firstName": "aaaaa",
+        "lastName": 3,
+        "email": "aaaaaaaaaaaaaaaaaaaaaa",
+        "state": "Active",
+        "registrationDate": "2018-09-11",
+        "note": null,
+        "groups": [
+          {
+            "displayName": "aaaaaaaaaaaaaaaaaaa",
+            "id": "aaaaaaaaaaaaaaaaaaaaaaaa",
+            "description": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+            "builtIn": true,
+            "type": "system",
+            "externalId": null
+          }
+        ],
+        "identities": [{ "provider": "aaaaa", "id": "aaaaaaaaaaaaaaaaaaaaaa" }]
+      }
+    }
+  }
+}
--- a/test/liveValidatorTests.ts
+++ b/test/liveValidatorTests.ts
@ -532,4 +532,37 @@ describe("Live validator snapshot validation", () => {
      expect(validationResult.responseValidationResult).toStrictEqual(responseValidationResult)
    })
  })
+  test(`should return all errors for no options`, async () => {
+    const payload = require(`${__dirname}/liveValidation/payloads/multipleErrors_input.json`)
+    const result = validator.validateLiveRequestResponse(payload)
+    expect(result.responseValidationResult.errors.length === 3)
+    expect(result.responseValidationResult.errors.some(err => err.code === "INVALID_TYPE"))
+    expect(result.responseValidationResult.errors.some(err => err.code === "INVALID_FORMAT"))
+    expect(
+      result.responseValidationResult.errors.some(
+        err => err.code === "OBJECT_ADDITIONAL_PROPERTIES"
+      )
+    )
+  })
+  test(`should return all errors for empty includeErrors list`, async () => {
+    const payload = require(`${__dirname}/liveValidation/payloads/multipleErrors_input.json`)
+    const result = validator.validateLiveRequestResponse(payload, { includeErrors: [] })
+    expect(result.responseValidationResult.errors.length === 3)
+    expect(result.responseValidationResult.errors.some(err => err.code === "INVALID_TYPE"))
+    expect(result.responseValidationResult.errors.some(err => err.code === "INVALID_FORMAT"))
+    expect(
+      result.responseValidationResult.errors.some(
+        err => err.code === "OBJECT_ADDITIONAL_PROPERTIES"
+      )
+    )
+  })
+
+  test(`should return only errors specified  in the list`, async () => {
+    const payload = require(`${__dirname}/liveValidation/payloads/multipleErrors_input.json`)
+    const result = validator.validateLiveRequestResponse(payload, {
+      includeErrors: ["INVALID_TYPE"]
+    })
+    expect(result.responseValidationResult.errors.length === 1)
+    expect(result.responseValidationResult.errors[0].code === "INVALID_TYPE")
+  })
 })