Add live validator init time docs cache (#394)

ChangeLog.md

@@ -1,5 +1,9 @@
 # Changelog
 
+### 03/06/2019 0.14.1
+
+- Add doc cache during `LiveValidator initialization` to reduce memory footprint and execution time.
+
 ### 03/06/2019 0.14.0
 
 - Remove package level global doc cache.

lib/util/jsonUtils.ts

@@ -14,6 +14,7 @@ import { SwaggerObject } from "yasway"
 import { log } from "./logging"
 import { parseContent } from "./makeRequest"
 import { isSubPath, splitPathAndReverse } from "./path"
+import { DocCache } from './documents';
 
 const setSuppression = (
   info: FilePosition | undefined,
@@ -39,9 +40,15 @@ const setSuppression = (
 export async function parseJson(
   suppression: Suppression | undefined,
   specPath: string,
-  reportError: jsonParser.ReportError
+  reportError: jsonParser.ReportError,
+  docsCache?: DocCache
 ): Promise<SwaggerObject> {
 
+  const doc = docsCache && docsCache[specPath]
+  if (doc) {
+    return await doc
+  }
+
   const getSuppressionArray = (
     suppressionItems: ReadonlyArray<SuppressionItem>
   ): ReadonlyArray<SuppressionItem> => {
@@ -110,5 +117,9 @@ export async function parseJson(
 
   const swaggerObjectPromise = createSwaggerObject()
 
+  if (docsCache) {
+    docsCache[specPath] = swaggerObjectPromise
+  }
+
   return swaggerObjectPromise
 }

lib/validators/liveValidator.ts

@@ -18,6 +18,7 @@ import { PotentialOperationsResult } from "../models/potentialOperationsResult"
 import { Operation, Request } from "yasway"
 import { ParsedUrlQuery } from "querystring"
 import { MutableStringMap } from "@ts-common/string-map"
+import { DocCache } from '../util/documents';
 
 export interface Options {
   swaggerPaths: string[]
@@ -201,11 +202,13 @@ export class LiveValidator {
     //   }
     //   ...
     // }
+    const docsCache: DocCache = {}
     const promiseFactories = swaggerPaths.map(swaggerPath => async () =>
-      await this.getSwaggerInitializer(swaggerPath)
+      await this.getSwaggerInitializer(swaggerPath, docsCache)
     )
 
     await utils.executePromisesSequentially(promiseFactories)
+
     log.info("Cache initialization complete.")
   }
 
@@ -397,14 +400,14 @@ export class LiveValidator {
       // We are using this to validate the payload as per the definitions in swagger.
       // We do not need the serialized output from ms-rest.
       const mapper = new models.RequestResponse().mapper()
-      // tslint:disable-next-line:align whitespace
-      ;(msRest as any).models = models
-      // tslint:disable-next-line:align whitespace
-      ;(msRest as any).serialize(
-        mapper,
-        requestResponseObj,
-        "requestResponseObj"
-      )
+        // tslint:disable-next-line:align whitespace
+        ; (msRest as any).models = models
+        // tslint:disable-next-line:align whitespace
+        ; (msRest as any).serialize(
+          mapper,
+          requestResponseObj,
+          "requestResponseObj"
+        )
     } catch (err) {
       const msg =
         `Found errors "${err.message}" in the provided input:\n` +
@@ -617,7 +620,7 @@ export class LiveValidator {
     if (this.options.swaggerPaths.length !== 0) {
       log.debug(
         `Using user provided swagger paths. Total paths: ${
-          this.options.swaggerPaths.length
+        this.options.swaggerPaths.length
         }`
       )
       return this.options.swaggerPaths
@@ -639,18 +642,18 @@ export class LiveValidator {
       const dir = this.options.directory
       log.debug(
         `Using swaggers found from directory "${dir}" and pattern "${jsonsPattern}".` +
-          `Total paths: ${swaggerPaths.length}`
+        `Total paths: ${swaggerPaths.length}`
       )
       return swaggerPaths
     }
   }
 
-  private async getSwaggerInitializer(swaggerPath: string): Promise<void> {
+  private async getSwaggerInitializer(swaggerPath: string, docsCache: DocCache): Promise<void> {
     log.info(`Building cache from: "${swaggerPath}"`)
 
     const validator = new SpecValidator(swaggerPath, null, {
       isPathCaseSensitive: this.options.isPathCaseSensitive
-    })
+    }, docsCache)
 
     try {
       const api = await validator.initialize()
@@ -681,7 +684,7 @@ export class LiveValidator {
           apiVersion = C.unknownApiVersion
           log.debug(
             `Unable to find provider for path : "${pathObject.path}". ` +
-              `Bucketizing into provider: "${provider}"`
+            `Bucketizing into provider: "${provider}"`
           )
         }
         provider = provider.toLowerCase()
@@ -706,7 +709,7 @@ export class LiveValidator {
       )
       log.warn(
         `Unable to initialize "${swaggerPath}" file from SpecValidator. We are ` +
-          `ignoring this swagger file and continuing to build cache for other valid specs.`
+        `ignoring this swagger file and continuing to build cache for other valid specs.`
       )
     }
   }

lib/validators/specResolver.ts

@@ -35,6 +35,7 @@ import { Suppression } from "@azure/openapi-markdown"
 import * as jsonUtils from "../util/jsonUtils"
 import * as jsonParser from "@ts-common/json-parser"
 import * as ps from "@ts-common/property-set"
+import { DocCache } from '../util/documents';
 
 const ErrorCodes = C.ErrorCodes
 
@@ -113,7 +114,8 @@ export class SpecResolver {
     specPath: string,
     specInJson: SwaggerObject,
     options: Options,
-    private readonly reportError: jsonParser.ReportError
+    private readonly reportError: jsonParser.ReportError,
+    private readonly docsCache: DocCache = {}
   ) {
     if (
       specPath === null ||
@@ -229,7 +231,7 @@ export class SpecResolver {
       if (this.options.shouldResolveNullableTypes) {
         this.resolveNullableTypes()
       }
-     } catch (err) {
+    } catch (err) {
       const e = {
         message: "internal error: " + err.message,
         code: ErrorCodes.InternalError.name,
@@ -379,7 +381,7 @@ export class SpecResolver {
       docPath = utils.joinPath(docDir, parsedReference.filePath)
     }
 
-    const result = await jsonUtils.parseJson(suppression, docPath, this.reportError)
+    const result = await jsonUtils.parseJson(suppression, docPath, this.reportError, this.docsCache)
     if (!parsedReference.localReference) {
       // Since there is no local reference we will replace the key in the object with the parsed
       // json (relative) file it is referring to.
@@ -930,7 +932,7 @@ export class SpecResolver {
       if (d) {
         const required = definition.required
         if (!isArray(required)) {
-          definition.required = [ discriminator ]
+          definition.required = [discriminator]
         } else if (required.find(v => v === discriminator) === undefined) {
           definition.required = [...required, discriminator]
         }

lib/validators/specValidator.ts

@@ -20,6 +20,7 @@ import * as jsonParser from "@ts-common/json-parser"
 import * as json from "@ts-common/json"
 import * as processErrors from "../util/processErrors"
 import { getTitle } from './specTransformer';
+import { DocCache } from '../util/documents';
 
 const ErrorCodes = C.ErrorCodes;
 
@@ -90,8 +91,6 @@ export class SpecValidator<T extends CommonValidationResult> {
 
   private readonly specDir: unknown
 
-  private specResolver: SpecResolver | null
-
   private readonly options: Options
 
   /*
@@ -135,7 +134,8 @@ export class SpecValidator<T extends CommonValidationResult> {
   public constructor(
     specPath: string,
     specInJson: SwaggerObject | undefined | null | string,
-    options: Options
+    options: Options,
+    private readonly docsCache: DocCache = {}
   ) {
     if (specPath === null
       || specPath === undefined
@@ -153,7 +153,6 @@ export class SpecValidator<T extends CommonValidationResult> {
     this.specPath = specPath
     this.specDir = path.dirname(this.specPath)
     this.specInJson = specInJson as SwaggerObject
-    this.specResolver = null
     const base: CommonValidationResult = { validityStatus: true, operations: {} }
     this.specValidationResult = base as T
     if (!options) { options = {} }
@@ -176,18 +175,19 @@ export class SpecValidator<T extends CommonValidationResult> {
       let suppression: amd.Suppression | undefined
       if (this.specInJson === undefined || this.specInJson === null) {
         suppression = await getSuppressions(this.specPath)
-        const result = await jsonUtils.parseJson(suppression, this.specPath, reportError)
+        const result = await jsonUtils.parseJson(
+          suppression, this.specPath, reportError, this.docsCache)
         this.specInJson = result
       }
 
-      this.specResolver = new SpecResolver(
+      const resolver = new SpecResolver(
         this.specPath,
         this.specInJson,
         this.options,
         reportError,
+        this.docsCache
       )
-      this.specInJson = (await this.specResolver.resolve(suppression)).specInJson
-
+      this.specInJson = (await resolver.resolve(suppression)).specInJson
       const options = {
         definition: this.specInJson,
         jsonRefs: {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "oav",
-  "version": "0.14.0",
+  "version": "0.14.1",
   "author": {
     "name": "Microsoft Corporation",
     "email": "azsdkteam@microsoft.com",