oav/test/liveValidatorTests.ts

579 строки
23 KiB
TypeScript
Исходник Обычный вид История

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
import * as assert from "assert"
import * as globby from "globby"
import * as lodash from "lodash"
import * as os from "os"
import * as path from "path"
import { ResponsesObject } from "yasway"
import * as Constants from "../lib/util/constants"
import { LiveValidator } from "../lib/validators/liveValidator"
const numberOfSpecs = 9
jest.setTimeout(150000)
2018-06-04 23:06:00 +03:00
describe("Live Validator", () => {
describe("Initialization", () => {
it("should initialize with defaults", () => {
const options = {
swaggerPaths: [],
git: {
url: "https://github.com/Azure/azure-rest-api-specs.git",
shouldClone: false
},
directory: path.resolve(os.homedir(), "repo"),
isPathCaseSensitive: false
}
2018-06-04 23:06:00 +03:00
const validator = new LiveValidator()
assert.deepStrictEqual(validator.cache, {})
assert.deepStrictEqual(validator.options, options)
2018-05-30 01:38:27 +03:00
})
2018-06-30 01:30:59 +03:00
it("should initialize with cloning", async () => {
const options = {
swaggerPaths: [],
git: {
url: "https://github.com/Azure/oav.git",
shouldClone: true
},
directory: path.resolve(os.homedir(), "repo")
}
2018-06-30 01:30:59 +03:00
const validator = new LiveValidator(options)
await validator.initialize()
assert.deepStrictEqual(validator.cache, {})
assert.deepStrictEqual(validator.options, options)
2018-06-30 01:30:59 +03:00
})
it("should initialize without url", () => {
const options = {
swaggerPaths: [],
git: {
shouldClone: false
},
directory: path.resolve(os.homedir(), "repo")
}
2018-06-30 01:30:59 +03:00
const validator = new LiveValidator(options)
assert.deepStrictEqual(validator.cache, {})
assert.deepStrictEqual(
validator.options.git.url,
"https://github.com/Azure/azure-rest-api-specs.git"
)
2018-06-30 01:30:59 +03:00
})
2018-06-04 23:06:00 +03:00
it("should initialize with user provided swaggerPaths", () => {
const swaggerPaths = ["swaggerPath1", "swaggerPath2"]
const options = {
isPathCaseSensitive: false,
swaggerPaths,
2018-06-04 23:06:00 +03:00
git: {
url: "https://github.com/Azure/azure-rest-api-specs.git",
shouldClone: false
},
2018-06-04 23:06:00 +03:00
directory: path.resolve(os.homedir(), "repo")
2018-05-30 01:38:27 +03:00
}
const validator = new LiveValidator({ swaggerPaths })
assert.deepStrictEqual(validator.cache, {})
assert.deepStrictEqual(validator.options, options)
2018-05-30 01:38:27 +03:00
})
2018-06-04 23:06:00 +03:00
it("should initialize with user provided swaggerPaths & directory", () => {
const swaggerPaths = ["swaggerPath1", "swaggerPath2"]
const directory = "/Users/username/repos/"
const options = {
swaggerPaths,
isPathCaseSensitive: false,
2018-06-04 23:06:00 +03:00
git: {
url: "https://github.com/Azure/azure-rest-api-specs.git",
shouldClone: false
},
directory
2018-05-30 01:38:27 +03:00
}
const validator = new LiveValidator({ swaggerPaths, directory })
assert.deepStrictEqual(validator.cache, {})
assert.deepStrictEqual(validator.options, options)
2018-05-30 01:38:27 +03:00
})
2018-06-04 23:06:00 +03:00
it("should initialize with user provided partial git configuration", () => {
const swaggerPaths = ["swaggerPath1", "swaggerPath2"]
const directory = "/Users/username/repos/"
const git = {
url: "https://github.com/Azure/azure-rest-api-specs.git",
shouldClone: false
}
2018-06-04 23:06:00 +03:00
const options = {
swaggerPaths,
isPathCaseSensitive: false,
2018-06-04 23:06:00 +03:00
git: {
url: git.url,
shouldClone: false
},
directory
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const validator = new LiveValidator({
swaggerPaths,
directory,
git
})
assert.deepStrictEqual(validator.cache, {})
assert.deepStrictEqual(validator.options, options)
2018-05-30 01:38:27 +03:00
})
2018-06-04 23:06:00 +03:00
it("should initialize with user provided full git configuration", () => {
const swaggerPaths = ["swaggerPath1", "swaggerPath2"]
const directory = "/Users/username/repos/"
const git = {
url: "https://github.com/vladbarosan/azure-rest-api-specs.git",
shouldClone: true,
branch: "oav-test-branch"
}
2018-06-04 23:06:00 +03:00
const options = {
swaggerPaths,
git,
directory,
isPathCaseSensitive: false
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const validator = new LiveValidator({
swaggerPaths,
directory,
git
})
assert.deepStrictEqual(validator.cache, {})
assert.deepStrictEqual(validator.options, options)
2018-05-30 01:38:27 +03:00
})
})
2018-06-04 23:06:00 +03:00
describe("Initialize cache", () => {
2018-06-30 01:30:59 +03:00
it("should initialize for arm-mediaservices", async () => {
2018-06-04 23:06:00 +03:00
const expectedProvider = "microsoft.media"
const expectedApiVersion = "2015-10-01"
const options = {
directory: "./test/liveValidation/swaggers/specification",
swaggerPathsPattern: "mediaservices/resource-manager/Microsoft.Media/2015-10-01/media.json"
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const validator = new LiveValidator(options)
2018-06-30 01:30:59 +03:00
try {
await validator.initialize()
assert.strictEqual(true, expectedProvider in validator.cache)
assert.strictEqual(1, Object.keys(validator.cache).length)
const x = validator.cache[expectedProvider]
if (x === undefined) {
throw new Error("x === undefined")
}
assert.strictEqual(true, expectedApiVersion in x)
assert.strictEqual(1, Object.keys(x).length)
assert.strictEqual(2, x[expectedApiVersion].get.length)
assert.strictEqual(1, x[expectedApiVersion].put.length)
assert.strictEqual(1, x[expectedApiVersion].patch.length)
assert.strictEqual(1, x[expectedApiVersion].delete.length)
assert.strictEqual(4, x[expectedApiVersion].post.length)
2018-06-30 01:30:59 +03:00
} catch (err) {
2018-05-30 01:38:27 +03:00
assert.ifError(err)
2018-06-30 01:30:59 +03:00
}
2018-05-30 01:38:27 +03:00
})
2018-06-30 01:30:59 +03:00
it("should initialize for arm-resources", async () => {
2018-06-04 23:06:00 +03:00
const expectedProvider = "microsoft.resources"
const expectedApiVersion = "2016-09-01"
const options = {
directory: "./test/liveValidation/swaggers/"
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const validator = new LiveValidator(options)
2018-06-30 01:30:59 +03:00
await validator.initialize()
assert.strictEqual(true, expectedProvider in validator.cache)
assert.strictEqual(numberOfSpecs, Object.keys(validator.cache).length)
const x = validator.cache[expectedProvider]
if (x === undefined) {
throw new Error("x === undefined")
}
assert.strictEqual(true, expectedApiVersion in x)
assert.strictEqual(1, Object.keys(x).length)
2018-06-30 01:30:59 +03:00
// 'microsoft.resources' -> '2016-09-01'
assert.strictEqual(2, x[expectedApiVersion].get.length)
assert.strictEqual(1, x[expectedApiVersion].delete.length)
assert.strictEqual(3, x[expectedApiVersion].post.length)
assert.strictEqual(1, x[expectedApiVersion].head.length)
assert.strictEqual(1, x[expectedApiVersion].put.length)
const p = validator.cache[Constants.unknownResourceProvider]
if (p === undefined) {
throw new Error("p === undefined")
}
2018-06-30 01:30:59 +03:00
// 'microsoft.unknown' -> 'unknown-api-version'
assert.strictEqual(4, p[Constants.unknownApiVersion].post.length)
assert.strictEqual(11, p[Constants.unknownApiVersion].get.length)
assert.strictEqual(3, p[Constants.unknownApiVersion].head.length)
assert.strictEqual(5, p[Constants.unknownApiVersion].put.length)
assert.strictEqual(5, p[Constants.unknownApiVersion].delete.length)
assert.strictEqual(1, p[Constants.unknownApiVersion].patch.length)
2018-05-30 01:38:27 +03:00
})
it("should initialize for batch", async () => {
const options = {
directory: "./test/liveValidation/swaggers/specification",
swaggerPathsPattern:
"batch/resource-manager/Microsoft.Batch/stable/2017-01-01/BatchManagement.json"
}
const validator = new LiveValidator(options)
await validator.initialize()
assert.notStrictEqual(validator.cache["microsoft.batch"], undefined)
})
2018-06-30 01:30:59 +03:00
it("should initialize for all swaggers", async () => {
2018-06-04 23:06:00 +03:00
const options = {
directory: "./test/liveValidation/swaggers/"
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const validator = new LiveValidator(options)
2018-06-30 01:30:59 +03:00
await validator.initialize()
assert.strictEqual(numberOfSpecs, Object.keys(validator.cache).length)
const microsoftResources = validator.cache["microsoft.resources"]
if (microsoftResources === undefined) {
throw new Error("microsoftResources === undefined")
}
assert.strictEqual(2, microsoftResources["2016-09-01"].get.length)
assert.strictEqual(1, microsoftResources["2016-09-01"].head.length)
const microsoftMedia = validator.cache["microsoft.media"]
if (microsoftMedia === undefined) {
throw new Error("microsoftMedia === undefined")
}
assert.strictEqual(1, microsoftMedia["2015-10-01"].patch.length)
assert.strictEqual(4, microsoftMedia["2015-10-01"].post.length)
const microsoftSearch = validator.cache["microsoft.search"]
if (microsoftSearch === undefined) {
throw new Error("microsoftSearch === undefined")
}
assert.strictEqual(2, microsoftSearch["2015-02-28"].get.length)
assert.strictEqual(3, microsoftSearch["2015-08-19"].get.length)
const microsoftStorage = validator.cache["microsoft.storage"]
if (microsoftStorage === undefined) {
throw new Error("microsoftStorage === undefined")
}
assert.strictEqual(1, microsoftStorage["2015-05-01-preview"].patch.length)
assert.strictEqual(4, microsoftStorage["2015-06-15"].get.length)
assert.strictEqual(3, microsoftStorage["2016-01-01"].post.length)
const microsoftTest = validator.cache["microsoft.test"]
if (microsoftTest === undefined) {
throw new Error("microsoftTest === undefined")
}
assert.strictEqual(4, microsoftTest["2016-01-01"].post.length)
2018-05-30 01:38:27 +03:00
})
})
2018-06-04 23:06:00 +03:00
describe("Initialize cache and search", () => {
2018-06-30 01:30:59 +03:00
it("should return one matched operation for arm-storage", async () => {
2018-06-04 23:06:00 +03:00
const options = {
directory: "./test/liveValidation/swaggers/"
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const listRequestUrl =
"https://management.azure.com/" +
"subscriptions/subscriptionId/providers/Microsoft.Storage/storageAccounts" +
"?api-version=2015-06-15"
const postRequestUrl =
"https://management.azure.com/" +
"subscriptions/subscriptionId/providers/Microsoft.Storage/checkNameAvailability" +
"?api-version=2015-06-15"
const deleteRequestUrl =
"https://management.azure.com/" +
"subscriptions/subscriptionId/resourceGroups/myRG/providers/Microsoft.Storage/" +
"storageAccounts/accname?api-version=2015-06-15"
const validator: any = new LiveValidator(options)
2018-06-30 01:30:59 +03:00
await validator.initialize()
// Operations to match is StorageAccounts_List
let operations = validator.getPotentialOperations(listRequestUrl, "Get").operations
let pathObject = operations[0].pathObject
if (pathObject === undefined) {
throw new Error("pathObject is undefined")
}
assert.strictEqual(1, operations.length)
assert.strictEqual(
2018-06-30 01:30:59 +03:00
"/subscriptions/{subscriptionId}/providers/Microsoft.Storage/storageAccounts",
pathObject.path
)
2018-06-30 01:30:59 +03:00
// Operations to match is StorageAccounts_CheckNameAvailability
operations = validator.getPotentialOperations(postRequestUrl, "PoSt").operations
pathObject = operations[0].pathObject
if (pathObject === undefined) {
throw new Error("pathObject is undefined")
}
assert.strictEqual(1, operations.length)
assert.strictEqual(
2018-06-30 01:30:59 +03:00
"/subscriptions/{subscriptionId}/providers/Microsoft.Storage/checkNameAvailability",
pathObject.path
)
2018-06-30 01:30:59 +03:00
// Operations to match is StorageAccounts_Delete
operations = validator.getPotentialOperations(deleteRequestUrl, "delete").operations
pathObject = operations[0].pathObject
if (pathObject === undefined) {
throw new Error("pathObject is undefined")
}
assert.strictEqual(1, operations.length)
assert.strictEqual(
2018-06-30 01:30:59 +03:00
"/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/" +
"Microsoft.Storage/storageAccounts/{accountName}",
pathObject.path
)
2018-05-30 01:38:27 +03:00
})
2018-06-09 02:35:55 +03:00
it("should return reason for not matched operations", async () => {
2018-06-04 23:06:00 +03:00
const options = {
directory: "./test/liveValidation/swaggers/"
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const nonCachedApiUrl =
"https://management.azure.com/" +
"subscriptions/subscriptionId/providers/Microsoft.Storage/storageAccounts" +
"?api-version=2015-08-15"
const nonCachedProviderUrl =
"https://management.azure.com/" +
"subscriptions/subscriptionId/providers/Hello.World/checkNameAvailability" +
"?api-version=2015-06-15"
const nonCachedVerbUrl =
"https://management.azure.com/" +
"subscriptions/subscriptionId/resourceGroups/myRG/providers/Microsoft.Storage/" +
"storageAccounts/accname?api-version=2015-06-15"
const nonCachedPath =
"https://management.azure.com/" +
"subscriptions/subscriptionId/providers/Microsoft.Storage/storageAccounts/accountName/" +
"properties?api-version=2015-06-15"
const validator: any = new LiveValidator(options)
2018-06-09 02:35:55 +03:00
await validator.initialize()
// Operations to match is StorageAccounts_List with api-version 2015-08-15
// [non cached api version]
let result = validator.getPotentialOperations(nonCachedApiUrl, "Get")
let operations = result.operations
let reason = result.reason
assert.strictEqual(0, operations.length)
2018-06-09 02:35:55 +03:00
if (reason === undefined) {
throw new Error("reason is undefined")
}
assert.strictEqual(Constants.ErrorCodes.OperationNotFoundInCacheWithApi.name, reason.code)
2018-06-09 02:35:55 +03:00
// Operations to match is StorageAccounts_CheckNameAvailability with provider "Hello.World"
// [non cached provider]
result = validator.getPotentialOperations(nonCachedProviderUrl, "PoSt")
operations = result.operations
reason = result.reason
assert.strictEqual(0, operations.length)
2018-06-09 02:35:55 +03:00
if (reason === undefined) {
throw new Error("reason is undefined")
}
assert.strictEqual(
Constants.ErrorCodes.OperationNotFoundInCacheWithProvider.name,
reason.code
)
2018-06-09 02:35:55 +03:00
// Operations to match is StorageAccounts_Delete with verb "head" [non cached http verb]
result = validator.getPotentialOperations(nonCachedVerbUrl, "head")
operations = result.operations
reason = result.reason
assert.strictEqual(0, operations.length)
2018-06-09 02:35:55 +03:00
if (reason === undefined) {
throw new Error("reason is undefined")
}
assert.strictEqual(Constants.ErrorCodes.OperationNotFoundInCacheWithVerb.name, reason.code)
2018-06-09 02:35:55 +03:00
// Operations to match is with path
// "subscriptions/subscriptionId/providers/Microsoft.Storage/" +
// "storageAccounts/storageAccounts/accountName/properties/"
// [non cached path]
result = validator.getPotentialOperations(nonCachedPath, "get")
operations = result.operations
reason = result.reason
assert.strictEqual(0, operations.length)
2018-06-09 02:35:55 +03:00
if (reason === undefined) {
throw new Error("reason is undefined")
}
assert.strictEqual(Constants.ErrorCodes.OperationNotFoundInCache.name, reason.code)
2018-05-30 01:38:27 +03:00
})
it("it shouldn't create an implicit default response", async () => {
2018-06-04 23:06:00 +03:00
const options = {
directory: "./test/liveValidation/swaggers/specification/scenarios",
swaggerPathsPattern: "**/*.json",
shouldModelImplicitDefaultResponse: true
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const validator = new LiveValidator(options)
2018-06-30 01:30:59 +03:00
await validator.initialize()
const microsoftTest = validator.cache["microsoft.test"]
if (microsoftTest === undefined) {
throw new Error("microsoftTest === undefined")
}
2018-06-30 01:30:59 +03:00
// Operations to match is StorageAccounts_List
const operations = microsoftTest["2016-01-01"].post
2018-03-16 00:53:01 +03:00
2018-06-30 01:30:59 +03:00
for (const operation of operations) {
const responses = operation.responses as ResponsesObject
assert.strictEqual(responses.default, undefined)
2018-06-30 01:30:59 +03:00
}
2018-05-30 01:38:27 +03:00
})
})
2018-06-04 23:06:00 +03:00
describe("Initialize cache and validate", () => {
const livePaths = globby.sync(
path.join(__dirname, "test/liveValidation/swaggers/**/live/*.json")
)
livePaths.forEach(livePath => {
2018-06-30 01:30:59 +03:00
it(`should validate request and response for "${livePath}"`, async () => {
2018-06-04 23:06:00 +03:00
const options = {
directory: "./test/liveValidation/swaggers/specification/storage",
swaggerPathsPattern: "**/*.json"
2018-05-30 01:38:27 +03:00
}
2018-06-04 23:06:00 +03:00
const validator = new LiveValidator(options)
2018-06-30 01:30:59 +03:00
await validator.initialize()
const reqRes = require(livePath)
const validationResult = validator.validateLiveRequestResponse(reqRes)
assert.notStrictEqual(validationResult, undefined)
2018-06-30 01:30:59 +03:00
/* tslint:disable-next-line */
// console.dir(validationResult, { depth: null, colors: true })
2018-05-30 01:38:27 +03:00
})
})
it("should initialize for defaultErrorOnly and fail on unknown status code", async () => {
const options = {
directory: "./test/liveValidation/swaggers/specification/defaultIsErrorOnly",
swaggerPathsPattern: "test.json"
}
const validator = new LiveValidator(options)
await validator.initialize()
const result = validator.validateLiveRequestResponse({
liveRequest: {
url: "https://xxx.com/providers/someprovider?api-version=2018-01-01",
method: "get",
headers: {
"content-type": "application/json"
},
query: {
"api-version": "2016-01-01"
}
},
liveResponse: {
statusCode: "300",
headers: {
"content-Type": "application/json"
}
}
})
const errors = result.responseValidationResult.errors
if (errors === undefined) {
throw new Error("errors === undefined")
}
assert.strictEqual((errors[0] as any).code, "INVALID_RESPONSE_CODE")
})
it("should initialize for defaultErrorOnly and pass", async () => {
const options = {
directory: "./test/liveValidation/swaggers/specification/defaultIsErrorOnly",
swaggerPathsPattern: "test.json"
}
const validator = new LiveValidator(options)
await validator.initialize()
const result = validator.validateLiveRequestResponse({
liveRequest: {
url: "https://xxx.com/providers/someprovider?api-version=2018-01-01",
method: "get",
headers: {
"content-type": "application/json"
},
query: {
"api-version": "2016-01-01"
}
},
liveResponse: {
statusCode: "404",
headers: {
"content-Type": "application/json"
}
}
})
const errors = result.responseValidationResult.errors
assert.deepStrictEqual(errors, [])
})
2018-05-30 01:38:27 +03:00
})
})
describe("Live validator snapshot validation", () => {
let validator: LiveValidator
let validatorOneOf: LiveValidator
const errors = [
"OBJECT_MISSING_REQUIRED_PROPERTY",
"OBJECT_ADDITIONAL_PROPERTIES",
"MAX_LENGTH",
"INVALID_FORMAT",
"INVALID_TYPE",
"ENUM_MISMATCH",
"ENUM_CASE_MISMATCH"
]
beforeAll(async () => {
const options = {
directory: `${__dirname}/liveValidation/swaggers/`,
isPathCaseSensitive: false,
useRelativeSourceLocationUrl: true,
swaggerPathsPattern:
"specification\\apimanagement\\resource-manager\\Microsoft.ApiManagement\\preview\\2018-01-01\\*.json",
git: {
shouldClone: false
}
}
validator = new LiveValidator(options)
await validator.initialize()
options.swaggerPathsPattern =
"specification\\mediaservices\\resource-manager\\Microsoft.Media\\2018-07-01\\*.json"
validatorOneOf = new LiveValidator(options)
await validatorOneOf.initialize()
}, 100000)
test(`should return no errors for valid input`, async () => {
const payload = require(`${__dirname}/liveValidation/payloads/valid_input.json`)
const validationResult = validator.validateLiveRequestResponse(payload)
expect(validationResult).toMatchSnapshot()
})
errors.forEach(error => {
test(`should return the expected error requestResponse validation for ${error}`, async () => {
const payload = require(`${__dirname}/liveValidation/payloads/${lodash.camelCase(
error
)}_input.json`)
const validationResult = validator.validateLiveRequestResponse(payload)
expect(validationResult).toMatchSnapshot()
})
test(`should match pair validation with response request validation for ${error}`, async () => {
const payload = require(`${__dirname}/liveValidation/payloads/${lodash.camelCase(
error
)}_input.json`)
const validationResult = validator.validateLiveRequestResponse(payload)
const requestValidationResult = validator.validateLiveRequest(payload.liveRequest)
const responseValidationResult = validator.validateLiveResponse(payload.liveResponse, {
url: payload.liveRequest.url,
method: payload.liveRequest.method
})
expect(validationResult.requestValidationResult).toStrictEqual(requestValidationResult)
expect(validationResult.responseValidationResult).toStrictEqual(responseValidationResult)
})
})
test(`should match for one of missing`, async () => {
const payload = require(`${__dirname}/liveValidation/payloads/oneOfMissing_input.json`)
const result = validatorOneOf.validateLiveRequestResponse(payload)
expect(result).toMatchSnapshot()
})
test(`should return all errors for no options`, async () => {
const payload = require(`${__dirname}/liveValidation/payloads/multipleErrors_input.json`)
const result = validator.validateLiveRequestResponse(payload)
expect(result.responseValidationResult.errors.length === 3)
expect(result.responseValidationResult.errors.some(err => err.code === "INVALID_TYPE"))
expect(result.responseValidationResult.errors.some(err => err.code === "INVALID_FORMAT"))
expect(
result.responseValidationResult.errors.some(
err => err.code === "OBJECT_ADDITIONAL_PROPERTIES"
)
)
})
test(`should return all errors for empty includeErrors list`, async () => {
const payload = require(`${__dirname}/liveValidation/payloads/multipleErrors_input.json`)
const result = validator.validateLiveRequestResponse(payload, { includeErrors: [] })
expect(result.responseValidationResult.errors.length === 3)
expect(result.responseValidationResult.errors.some(err => err.code === "INVALID_TYPE"))
expect(result.responseValidationResult.errors.some(err => err.code === "INVALID_FORMAT"))
expect(
result.responseValidationResult.errors.some(
err => err.code === "OBJECT_ADDITIONAL_PROPERTIES"
)
)
})
test(`should return only errors specified in the list`, async () => {
const payload = require(`${__dirname}/liveValidation/payloads/multipleErrors_input.json`)
const result = validator.validateLiveRequestResponse(payload, {
includeErrors: ["INVALID_TYPE"]
})
expect(result.responseValidationResult.errors.length === 1)
expect(result.responseValidationResult.errors[0].code === "INVALID_TYPE")
})
})