20200803 checker (#515)

* trace example parameters

* WIP: simple check

* WIP: check replace_value

* format checks

* check list length

* check create/delete status

* check complex array

* dynamic header

* check 200/201/204 body

* nonstrict

* handle name in response

* split json

* replace check path

* remove format in checks

* remove commented code and reformat

* swicher for gen-checks

* option for disable-check

* reorg

src/plugins/azgenerator/CodeModelAz.ts

@@ -29,7 +29,10 @@ export class ExampleParam {
     keys: string[];
     defaultName: string;
     methodParam: MethodParam;
-    public constructor(name: string, value: any, isJson: boolean, isKeyValues: boolean, keys: string[], defaultName: string, methodParam: MethodParam) {
+    ancestors: string[];
+    replacedValue: any;
+    rawValue: any;
+    public constructor(name: string, value: any, isJson: boolean, isKeyValues: boolean, keys: string[], defaultName: string, methodParam: MethodParam, ancestors: string[], rawValue: any) {
         this.name = name;
         this.value = value;
         this.isJson = isJson;
@@ -37,6 +40,8 @@ export class ExampleParam {
         this.keys = keys;
         this.defaultName = defaultName;
         this.methodParam = methodParam;
+        this.ancestors = ancestors;
+        this.rawValue = rawValue;
     }
 }
 export class CommandExample {
@@ -52,6 +57,7 @@ export class CommandExample {
     public MethodResponses: any[];
     public Method_IsLongRun: boolean;
     public MethodParams: MethodParam[];
+    public ExampleObj: any;
 }
 
 export interface CodeModelAz {
@@ -176,7 +182,7 @@ export interface CodeModelAz {
     GenerateTestInit(): void;
     SelectFirstExample(): boolean;
     SelectNextExample(): boolean;
-    FindExampleById(id: string, commandParams: any): string[][];
+    FindExampleById(id: string, commandParams: any, examples: any[]): string[][];
     Example_Body: string[];
     Example_Title: string;
     Example_Params: any;
@@ -186,6 +192,7 @@ export interface CodeModelAz {
     GatherInternalResource();
     FindExampleWaitById(id: string): string[][];
     GetExampleItems(example: CommandExample, isTest: boolean, commandParams: any): string[];
+    GetExampleChecks(example: CommandExample): string[];
     RandomizeNames: boolean;
     
     // readme config

src/plugins/azgenerator/CodeModelAzImpl.ts

@@ -11,7 +11,7 @@ import { isArray, isNullOrUndefined } from "util";
 import { Capitalize, deepCopy, parseResourceId, ToCamelCase, ToJsonString, ToSnakeCase } from '../../utils/helper';
 import { GenerationMode } from "../models";
 import { CodeModelAz, CommandExample, ExampleParam, MethodParam } from "./CodeModelAz";
-import { azOptions, GenerateDefaultTestScenario, GenerateDefaultTestScenarioByDependency, PrintTestScenario, ResourcePool } from './ScenarioTool';
+import { azOptions, GenerateDefaultTestScenario, GenerateDefaultTestScenarioByDependency, PrintTestScenario, ResourcePool, ObjectStatus } from './ScenarioTool';
 
 
 class ActionParam {
@@ -125,15 +125,23 @@ export class CodeModelCliImpl implements CodeModelAz {
     }
 
     public get RandomizeNames(): boolean {
-        if (this.options?.['randomize-names']) return true;
+        let randomizeNames = this.options?.['randomize-names'];
+        if (randomizeNames) return true;
         return false;
     }
 
     public get FormalizeNames(): boolean {
-        if (this.options?.['formalize-names']) return true;
+        let formalizeNames = this.options?.['formalize-names'];
+        if (formalizeNames) return true;
         return false;
     }
 
+    public get GenChecks(): boolean {
+        let disableChecks = this.options?.['disable-checks'];
+        if (disableChecks) return false;
+        return true;
+    }
+
     private calcOptionRequiredByMethod() {
         if (this.SelectFirstCommandGroup()) {
             do {
@@ -1625,7 +1633,7 @@ export class CodeModelCliImpl implements CodeModelAz {
         return (this.Command_GetOriginalOperation && param?.targetProperty?.['isDiscriminator']) ? true : false;
     }
 
-    private AddExampleParameter(methodParam: MethodParam, example_param: ExampleParam[], value: any, polySubParam: MethodParam): boolean {
+    private AddExampleParameter(methodParam: MethodParam, example_param: ExampleParam[], value: any, polySubParam: MethodParam, ancestors: string[]): boolean {
         let isList: boolean = methodParam.isList;
         let isSimpleListOrArray: boolean = methodParam.isSimpleListOrArray;
         let defaultName: string = methodParam.value.language['cli'].cliKey;
@@ -1648,7 +1656,7 @@ export class CodeModelCliImpl implements CodeModelAz {
             if (isSimpleListOrArray) {
                 if (value instanceof Array) {       // spread list
                     for (let e of value) {
-                        this.AddExampleParameter(methodParam, example_param, e, polySubParam);
+                        this.AddExampleParameter(methodParam, example_param, e, polySubParam, ancestors);
                     }
                     handled = true;
                 }
@@ -1692,23 +1700,23 @@ export class CodeModelCliImpl implements CodeModelAz {
                         keys.push(cliName)
                     }
                     if (ret.length > 0) {
-                        example_param.push(new ExampleParam(name, ret, false, true, keys, defaultName, methodParam));
+                        example_param.push(new ExampleParam(name, ret, false, true, keys, defaultName, methodParam, ancestors, value));
                     }
                     handled = true;
                 }
             }
             if (!handled) {
                 if (typeof value == 'string') {
-                    example_param.push(new ExampleParam(name, value, false, false, [], defaultName, methodParam));
+                    example_param.push(new ExampleParam(name, value, false, false, [], defaultName, methodParam, ancestors, value));
                 }
                 else {
                     // JSON form
-                    example_param.push(new ExampleParam(name, JSON.stringify(value).split(/[\r\n]+/).join(""), true, false, [], defaultName, methodParam));
+                    example_param.push(new ExampleParam(name, JSON.stringify(value).split(/[\r\n]+/).join(""), true, false, [], defaultName, methodParam, ancestors, value));
                 }
             }
         }
         else if (typeof value != 'object') {
-            example_param.push(new ExampleParam(name, value, false, false, [], defaultName, methodParam));
+            example_param.push(new ExampleParam(name, value, false, false, [], defaultName, methodParam, ancestors, value));
         }
         else {
             // ignore object values if not isList.
@@ -1807,7 +1815,7 @@ export class CodeModelCliImpl implements CodeModelAz {
                 }
                 if (match) {
                     // example_param.set(name, value);
-                    this.AddExampleParameter(methodParam, example_param, netValue, polySubParam);
+                    this.AddExampleParameter(methodParam, example_param, netValue, polySubParam, ancestors);
                     return;
                 }
             }
@@ -1830,13 +1838,13 @@ export class CodeModelCliImpl implements CodeModelAz {
                 }
                 if (match) {
                     // example_param.set(name, value);
-                    this.AddExampleParameter(methodParam, example_param, netValue, polySubParam);
+                    this.AddExampleParameter(methodParam, example_param, netValue, polySubParam, ancestors);
                     return;
                 }
             }
             else if (ancestors.length == 0) {
                 // example_param.set(name, value);
-                if (this.AddExampleParameter(methodParam, example_param, netValue, polySubParam)) return;
+                if (this.AddExampleParameter(methodParam, example_param, netValue, polySubParam, ancestors)) return;
             }
         }
 
@@ -1861,7 +1869,7 @@ export class CodeModelCliImpl implements CodeModelAz {
                 // }
             }
             param_name = param_name.split("_").join("-");
-            ret.push(new ExampleParam("--" + param_name, param.value, param.isJson, param.isKeyValues, param.keys, param.defaultName, param.methodParam));
+            ret.push(new ExampleParam("--" + param_name, param.value, param.isJson, param.isKeyValues, param.keys, param.defaultName, param.methodParam, param.ancestors, param.rawValue));
         };
         return ret;
     }
@@ -1922,6 +1930,7 @@ export class CodeModelCliImpl implements CodeModelAz {
                 example.Parameters = this.ConvertToCliParameters(params);
                 example.MethodResponses = this.Method.responses || [];
                 example.Method_IsLongRun = this.Method.extensions?.['x-ms-long-running-operation'] ? true : false;
+                example.ExampleObj = example_obj;
                 if (this.Method_GetSplitOriginalOperation) {
                     //filter example by name for generic createorupdate
                     if (this.Command_MethodName.toLowerCase() == "update" && !id.toLowerCase().endsWith("_update"))
@@ -1943,11 +1952,30 @@ export class CodeModelCliImpl implements CodeModelAz {
         return examples;
     }
 
+    public GetExampleChecks(example: CommandExample): string[] {
+        let ret: string[] = [];
+        if (!this.GenChecks)  return ret;
+        let resourceObjectName = undefined;
+        for (let param of example.Parameters) {
+            if (example.ResourceClassName && this.resource_pool.isResource(param.defaultName) == example.ResourceClassName) {
+                resourceObjectName = param.value;
+            }
+        }
+
+        let resourceObject = this.resource_pool.findResource(example.ResourceClassName, resourceObjectName, ObjectStatus.Created);
+        if (resourceObject) {
+            ret.push(...resourceObject.getCheckers(this.resource_pool, example));
+        }
+        ret.push(...this.resource_pool.getListCheckers(example));
+        return ret;
+    }
+
     public GetExampleItems(example: CommandExample, isTest: boolean, commandParams: any): string[] {
         let parameters: string[] = [];
         parameters.push("az " + this.Command_Name);
 
         let hasRG = false;
+        let resourceObjectName = undefined;
         for (let param of example.Parameters) {
             let param_value = param.value;
             if (isTest || this.FormalizeNames) {
@@ -1956,6 +1984,7 @@ export class CodeModelCliImpl implements CodeModelAz {
                     replaced_value = this.resource_pool.addParamResource(param.defaultName, param_value, param.isJson, param.isKeyValues, isTest);
                 }
                 param_value = replaced_value;
+                param.replacedValue = replaced_value;
             }
             let slp = param_value;
             if (!param.isKeyValues) {
@@ -1966,12 +1995,35 @@ export class CodeModelCliImpl implements CodeModelAz {
             if (["--resource-group", "-g"].indexOf(param.name) >= 0) {
                 hasRG = true;
             }
+
+            if (example.ResourceClassName && this.resource_pool.isResource(param.defaultName) == example.ResourceClassName) {
+                resourceObjectName = param.value;
+            }
         }
 
         if (isTest && !hasRG && commandParams && commandParams[this.Command_Name] && commandParams[this.Command_Name].has("resourceGroupName")) {
             parameters.push('-g ""');
         }
 
+        if (isTest) {
+            let resourceObject = this.resource_pool.findResource(example.ResourceClassName, resourceObjectName, undefined);
+            if (resourceObject) {
+                let httpMethod = example.HttpMethod.toLowerCase();
+                if (['put', 'post', 'patch'].indexOf(httpMethod)>=0) {
+                    if (httpMethod == 'post') {
+                        resourceObject.example_params = []; 
+                    }
+                    for (let param of example.Parameters) {
+                        resourceObject.addOrUpdateParam(param);
+                    }
+                    resourceObject.testStatus = ObjectStatus.Created;
+                }
+                if (httpMethod == 'delete') {
+                    resourceObject.testStatus = ObjectStatus.Deleted;
+                }
+            }
+        }
+
         return parameters;
     }
 
@@ -2019,9 +2071,10 @@ export class CodeModelCliImpl implements CodeModelAz {
         }
     }
 
-    public FindExampleById(id: string, commandParams?: any): string[][] {
+    public FindExampleById(id: string, commandParams: any, examples: CommandExample[]): string[][] {
         let ret: string[][] = [];
         this.GetAllExamples(id, (example) => {
+            examples.push(example);
             ret.push(this.GetExampleItems(example, true, commandParams));
         });
         return ret;

src/plugins/azgenerator/ScenarioTool.ts

@@ -1,8 +1,7 @@
 
 import * as path from "path"
 import { CommandExample, ExampleParam } from "./CodeModelAz";
-import { deepCopy, isDict, ToCamelCase, changeCamelToDash } from "../../utils/helper"
-import { Example } from "@azure-tools/codemodel";
+import { deepCopy, isDict, ToCamelCase, ToPythonString, changeCamelToDash } from "../../utils/helper"
 import { EnglishPluralizationService } from "@azure-tools/codegen";
 import { isNullOrUndefined } from "util";
 
@@ -201,16 +200,26 @@ class ResourceClass {
     }
 
 }
+
+export enum ObjectStatus {
+    None,
+    Created,
+    Deleted,
+}
 class ResourceObject {
     object_name: string;
     class_name: string;
     // key: string;
     sub_resources: Map<string, ResourceClass>; //class_name --> resource_class
+    example_params: ExampleParam[];
+    testStatus: ObjectStatus;
 
     public constructor(object_name: string, class_name: string) {
         this.object_name = object_name;
         this.class_name = class_name;
         this.sub_resources = new Map<string, ResourceClass>();
+        this.example_params = [];
+        this.testStatus = ObjectStatus.None;
     }
 
     public get key(): string {
@@ -218,24 +227,142 @@ class ResourceObject {
     }
 
     public placeholder(isTest: boolean): string {
-        if (isTest)    return '{' + this.key + '}';
+        if (isTest) return '{' + this.key + '}';
         return getResourceKey(this.class_name, this.object_name, true);
-        
+    }
+
+    public addOrUpdateParam(example_param: ExampleParam) {
+        // remove all children
+        for (let i = 0; i < this.example_params.length; i++) {
+            let param = this.example_params[i];
+            if (param.ancestors == example_param.ancestors.concat([example_param.name])) {
+                this.example_params.splice(i);
+                i--;
+            }
+        }
+
+        // replace if already there
+        for (let i = 0; i < this.example_params.length; i++) {
+            let param = this.example_params[i];
+            if (JSON.stringify(param.ancestors) == JSON.stringify(example_param.ancestors) && param.name == example_param.name) {
+                this.example_params[i] = example_param;
+                return;
+            }
+        }
+
+        // append to the tail
+        this.example_params.push(example_param);
+    }
+
+    public getCheckers(resource_pool: ResourcePool, example: CommandExample): string[] {
+        function hasComplexArray(obj: any): boolean {
+            if (obj instanceof Array) {
+                if (obj.length > 1) return true;
+                for (let s of obj) {
+                    if (hasComplexArray(s)) return true;
+                }
+            }
+            if (isDict(obj)) {
+                for (let key in obj) {
+                    if (hasComplexArray(obj[key])) return true;
+                }
+            }
+            return false;
+        }
+
+        function addParam(obj: any, param: ExampleParam, checkPath: string, ret: string[]) {
+            if (isDict(obj)) {
+                if (checkPath.length > 0) checkPath += ".";
+                if (param.defaultName in obj && typeof obj[param.defaultName] == typeof param.rawValue && JSON.stringify(obj[param.defaultName]).toLowerCase() == JSON.stringify(param.rawValue).toLowerCase()) {
+                    if (hasComplexArray(param.rawValue)) return;
+                    formatChecker(checkPath + resource_pool.replaceResourceString(param.defaultName, [], [], true), param.rawValue, ret);
+                    return;
+                }
+                else if ('name' in obj && typeof obj['name'] == typeof param.rawValue && JSON.stringify(obj['name']).toLowerCase() == JSON.stringify(param.rawValue).toLowerCase()) {
+                    if (hasComplexArray(param.rawValue)) return;
+                    formatChecker(checkPath + 'name', param.rawValue, ret);
+                    return;
+                }
+
+            }
+            if (obj instanceof Array) {
+                if (obj.length > 1) return;
+                addParam(obj[0], param, checkPath + "[0]", ret);
+            }
+            if (isDict(obj)) {
+                if (checkPath.length > 0) checkPath += ".";
+                for (let key in obj) {
+                    if (checkPath.length==0 && key.toLowerCase()=='properties') {
+                        addParam(obj[key], param, checkPath, ret);
+                    }
+                    else {
+                        addParam(obj[key], param, checkPath + resource_pool.replaceResourceString(key, [], [], true), ret);
+                    }
+                }
+            }
+        }
+
+        function formatChecker(checkPath: string, rawValue: any, ret: string[]) {
+            if (typeof rawValue == 'object') {
+                if (rawValue instanceof Array) {
+                    if (rawValue.length > 1) return;
+                    if (rawValue.length == 0) {
+                        ret.push(`test.check("${checkPath}", []),`);
+                    }
+                    else {
+                        formatChecker(checkPath + "[0]", rawValue[0], ret);
+                    }
+                }
+                else if (isDict(rawValue)) {
+                    if (checkPath.length > 0) checkPath += ".";
+                    if (Object.keys(rawValue).length == 0) {
+                        ret.push(`test.check("${checkPath}", {}),`);
+                    }
+                    for (let key in rawValue) {
+                        formatChecker(checkPath + resource_pool.replaceResourceString(key, [], [], true), rawValue[key], ret);
+                    }
+                }
+            }
+            else {
+                if (typeof rawValue == 'string') {
+                    let replacedValue = resource_pool.replaceResourceString(rawValue, [], [], true);
+                    ret.push(`test.check("${checkPath}", ${ToPythonString(replacedValue, typeof replacedValue)}, case_sensitive=False),`);
+                }
+                else {
+                    ret.push(`test.check("${checkPath}", ${ToPythonString(rawValue, typeof rawValue)}),`);
+                }
+            }
+        }
+
+        let ret: string[] = [];
+        if (['create', 'delete', 'show', 'list', 'update'].indexOf(example.Method) < 0) return ret;
+        let example_resp_body = null;
+        for (let statusCode of [200, 201, 204]) {
+            example_resp_body = example.ExampleObj.responses?.[200]?.body;
+            if (!isNullOrUndefined(example_resp_body)) break;
+        }
+        if (isNullOrUndefined(example_resp_body)) return ret;
+
+        for (let param of this.example_params) {
+            addParam(example_resp_body, param, "", ret);
+        }
+
+        return ret;
     }
 }
 
-function singlizeLast(word:string) {
+function singlizeLast(word: string) {
     let eps = new EnglishPluralizationService();
     let ws = changeCamelToDash(word).split('-');
     let l = ws.length;
-    ws[l-1] = eps.singularize(ws[l-1]);
+    ws[l - 1] = eps.singularize(ws[l - 1]);
     return ws.join('-');
 }
 
 let keyCache = {}  //class_name+objectname->key
 let formalCache = {}
 let keySeq = {}    // class_name ->seq
-export function getResourceKey(class_name: string, object_name: string, formalName=false): string {
+export function getResourceKey(class_name: string, object_name: string, formalName = false): string {
     let longKey = (resourceClassKeys[class_name] || class_name) + '_' + object_name;
     if (formalName && longKey in formalCache) {
         return formalCache[longKey];
@@ -247,12 +374,12 @@ export function getResourceKey(class_name: string, object_name: string, formalNa
     if (keySeq.hasOwnProperty(class_name)) {
         let key = (resourceClassKeys[class_name] || class_name) + '_' + keySeq[class_name];
         keySeq[class_name] += 1;
-        formalCache[longKey] = ToCamelCase(`my-${singlizeLast(class_name)}${keySeq[class_name]-1}`);
+        formalCache[longKey] = ToCamelCase(`my-${singlizeLast(class_name)}${keySeq[class_name] - 1}`);
         if (preparerInfos[class_name]?.name) {  // is external resource
             keyCache[longKey] = key;
         }
         else {
-            keyCache[longKey] = ToCamelCase(`my-${singlizeLast(class_name)}${keySeq[class_name]-1}`);
+            keyCache[longKey] = ToCamelCase(`my-${singlizeLast(class_name)}${keySeq[class_name] - 1}`);
         }
     }
     else {
@@ -268,7 +395,7 @@ export function getResourceKey(class_name: string, object_name: string, formalNa
         }
     }
 
-    return formalName? formalCache[longKey]: keyCache[longKey];
+    return formalName ? formalCache[longKey] : keyCache[longKey];
 }
 
 export class ResourcePool {
@@ -277,11 +404,13 @@ export class ResourcePool {
     map: Map<string, ResourceClass>;
     use_subscription: boolean;
     static KEY_SUBSCRIPTIONID = "subscription_id";
+    replacements: Map<string, string>;
 
     public constructor() {
         this.root = new Map<string, ResourceClass>();
         this.map = new Map<string, ResourceClass>();
         this.use_subscription = false;
+        this.replacements = new Map<string, string>();
     }
 
     private prepareResource(class_name: string, object_name: string, depends: string[][], entitys: PreparerEntity[], preparings: string[][]) {
@@ -393,20 +522,81 @@ export class ResourcePool {
         return resources[class_name].objects[object_name];
     }
 
-    public findTreeResource(class_name: string, object_name: string, root: Map<string, ResourceClass>): ResourceObject {
+    public findResource(class_name: string, object_name: string, testStatus: ObjectStatus): ResourceObject | undefined {
+        if (isNullOrUndefined(class_name) || isNullOrUndefined(object_name)) return null;
+
+        let resource_object = this.findTreeResource(class_name, object_name, this.root, testStatus);
+        if (resource_object) {
+            return resource_object;
+        }
+
+        if (class_name in this.map && object_name in this.map[class_name].objects) {
+            if (isNullOrUndefined(testStatus) || testStatus == this.map[class_name].objects[object_name].testStatus) {
+                return this.map[class_name].objects[object_name];
+            }
+        }
+
+        return undefined;
+    }
+
+    public findAllResource(class_name: string, exampleParams: ExampleParam[] = null, testStatus: ObjectStatus = null): ResourceObject[] {
+        let ret: ResourceObject[] = [];
+        if (isNullOrUndefined(class_name)) return ret;
+
+        this.findAllTreeResource(class_name, this.root, ret);
+
+        if (class_name in this.map) {
+            for (let key in this.map[class_name].objects) {
+                ret.push(this.map[class_name].objects[key]);
+            }
+        }
+        return ret.filter((resourceObject) => {
+            for (let critParam of exampleParams) {
+                let found = false;
+                for (let resourceParam of resourceObject.example_params) {
+                    if (critParam.name == resourceParam.name && critParam.rawValue == resourceParam.rawValue) {
+                        found = true;
+                        break;
+                    }
+                }
+                if (!found) return false;
+            };
+            return isNullOrUndefined(testStatus) || testStatus == resourceObject.testStatus;
+        });
+    }
+
+    public findAllTreeResource(class_name: string, root: Map<string, ResourceClass>, ret: ResourceObject[]) {
+        if (class_name in root) {
+            for (let key in root[class_name].objects) {
+                ret.push(root[class_name].objects[key]);
+            }
+        }
+        for (let c in root) {
+            for (let o in root[c].objects) {
+                this.findAllTreeResource(class_name, root[c].objects[o].sub_resources, ret);
+            }
+        }
+    }
+
+
+    public findTreeResource(class_name: string, object_name: string, root: Map<string, ResourceClass>, testStatus: ObjectStatus = null): ResourceObject {
         if (class_name in root && object_name in root[class_name].objects) {
-            return root[class_name].objects[object_name];
+            if (isNullOrUndefined(testStatus) || testStatus == root[class_name].objects[object_name].testStatus) {
+                return root[class_name].objects[object_name];
+            }
         }
         if (!class_name) {
             for (let c in root) {
                 if (object_name in root[c].objects) {
-                    return root[c].objects[object_name];
+                    if (isNullOrUndefined(testStatus) || testStatus == root[c].objects[object_name].testStatus) {
+                        return root[c].objects[object_name];
+                    }
                 }
             }
         }
         for (let c in root) {
             for (let o in root[c].objects) {
-                let ret = this.findTreeResource(class_name, object_name, root[c].objects[o].sub_resources);
+                let ret = this.findTreeResource(class_name, object_name, root[c].objects[o].sub_resources, testStatus);
                 if (ret) return ret;
             }
         }
@@ -431,7 +621,8 @@ export class ResourcePool {
         return this.map[class_name].objects[object_name];
     }
 
-    public isResource(language): string | null {
+    public isResource(language: string): string | null {
+        if (language.startsWith("--")) language = language.substr(2);
         for (let resource in resourceLanguages) {
             for (let resource_language of resourceLanguages[resource]) {
                 if (resource_language.toLowerCase() == language.toLowerCase()) return resource;
@@ -458,7 +649,7 @@ export class ResourcePool {
         return false;
 
     }
-    public addEndpointResource(endpoint: any, isJson: boolean, isKeyValues: boolean, placeholders: string[], resources: string[], exampleParam: ExampleParam, isTest=true) {
+    public addEndpointResource(endpoint: any, isJson: boolean, isKeyValues: boolean, placeholders: string[], resources: string[], exampleParam: ExampleParam, isTest = true): any {
         if (placeholders == undefined) placeholders = new Array();
         if (isJson) {
             let body = typeof endpoint == 'string' ? JSON.parse(endpoint) : endpoint;
@@ -480,7 +671,7 @@ export class ResourcePool {
 
             if (typeof endpoint == 'string') {
                 let ret = JSON.stringify(body).split(/[\r\n]+/).join("");
-                return isTest ? this.formatable(ret, placeholders): ret;
+                return isTest ? this.formatable(ret, placeholders) : ret;
             }
             else {
                 return body;
@@ -524,7 +715,9 @@ export class ResourcePool {
             }
             return ret;
         }
-
+        return this.replaceResourceString(endpoint, placeholders, resources, isTest);
+    }
+    public replaceResourceString(endpoint: string, placeholders: string[], resources: string[], isTest = true): any {
         let nodes = endpoint.split('/');
         if (nodes.length < 3 || nodes[0].length > 0 || nodes[1].toLowerCase() != SUBSCRIPTIONS) {
             return this.getPlaceholder(endpoint, isTest, placeholders);
@@ -557,14 +750,14 @@ export class ResourcePool {
                 }
             }
             else {
-                nodes[i + 1] = this.getPlaceholder(nodes[i+1], isTest);
+                nodes[i + 1] = this.getPlaceholder(nodes[i + 1], isTest);
             }
             i += 2;
         }
         return nodes.join('/');
     }
 
-    public addParamResource(param_name: string, param_value: string, isJson: boolean, isKeyValues: boolean, isTest=true): string {
+    public addParamResource(param_name: string, param_value: string, isJson: boolean, isKeyValues: boolean, isTest = true): string {
         if (typeof param_value !== 'string' || isJson || isKeyValues) return param_value;
 
         if (param_name.startsWith('--')) {
@@ -587,7 +780,8 @@ export class ResourcePool {
         }
     }
 
-    public getPlaceholder(object_name: string, isTest: boolean, placeholders: string[]=null): string {
+
+    public getPlaceholder(object_name: string, isTest: boolean, placeholders: string[] = null): string {
         // find in MapResource
         for (let class_name in this.map) {
             if (object_name in this.map[class_name].objects) {
@@ -615,9 +809,12 @@ export class ResourcePool {
 
         let regex = /^\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d.\d+Z$/g
         if (azOptions?.['replace-datetime'] && regex.test(object_name)) {
-            let date = new Date();
-            date.setDate(date.getDate() + 10);
-            return date.toISOString();
+            if (!this.replacements.has(object_name)) {
+                let date = new Date();
+                date.setDate(date.getDate() + 10);
+                this.replacements.set(object_name, date.toISOString());
+            }
+            return this.replacements.get(object_name)
         }
         return object_name;
     }
@@ -657,5 +854,15 @@ export class ResourcePool {
         let depends = resourceClassDepends[child];
         return depends && depends.indexOf(parent) >= 0;
     }
+
+    public getListCheckers(example: CommandExample): string[] {
+        let ret = [];
+        if (example.Method != "list") return ret;
+        let len = this.findAllResource(example.ResourceClassName, example.Parameters, ObjectStatus.Created).length;
+        if (len > 0) {
+            ret.push(`test.check('length(@)', ${len}),`);
+        }
+        return ret;
+    }
 }
 

src/plugins/azgenerator/TemplateAzureCliTestScenario.ts

@@ -3,11 +3,10 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { CodeModelAz } from "./CodeModelAz"
+import { CodeModelAz, CommandExample } from "./CodeModelAz"
 import { PreparerEntity, getResourceKey } from "./ScenarioTool"
-import { ToSnakeCase, ToMultiLine, deepCopy } from '../../utils/helper';
+import { ToMultiLine, deepCopy } from '../../utils/helper';
 import { HeaderGenerator } from "./Header";
-import { HttpWithBodyRequest } from "@azure-tools/codemodel";
 
 let usePreparers = false;
 
@@ -30,11 +29,9 @@ export function GenerateAzureCliTestScenario(model: CodeModelAz): string[] {
 
     let header: HeaderGenerator = new HeaderGenerator();
 
-    head.push("");
-    head.push("import os");
+    header.addImport("os");
     head.push("from azure.cli.testsdk import ScenarioTest");
     head.push("from .. import try_manual, raise_if, calc_coverage");
-    //head.push("from .preparers import (VirtualNetworkPreparer, VnetSubnetPreparer)");
     steps.push("");
     steps.push("");
     steps.push("TEST_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))");
@@ -44,18 +41,8 @@ export function GenerateAzureCliTestScenario(model: CodeModelAz): string[] {
     class_info.push("@try_manual");
     class_info.push("class " + model.Extension_NameClass + "ScenarioTest(ScenarioTest):");
     class_info.push("");
-    //initiates.push("    @ResourceGroupPreparer(name_prefix='cli_test_" + model.Extension_NameUnderscored + "')");
-    // initiates.push("    def test_" + model.Extension_NameUnderscored + "(self, resource_group):");
     initiates.push("");
 
-    // go through the examples to recognize resources
-    for (var ci = 0; ci < config.length; ci++) {
-        let exampleId: string = config[ci].name;
-        if (exampleId) {
-            model.FindExampleById(exampleId, commandParams);
-        }
-    }
-
     let subscription_id = model.GetSubscriptionKey();
     if (subscription_id) {
         initiates.push("        self.kwargs.update({");
@@ -67,6 +54,7 @@ export function GenerateAzureCliTestScenario(model: CodeModelAz): string[] {
     let imports: string[] = [];
     let decorators: string[] = [];
     let parameterNames = InitiateDependencies(model, imports, decorators, initiates);
+    let jsonAdded = false;
 
     function parameterLine() {
         let ret = "";
@@ -90,18 +78,34 @@ export function GenerateAzureCliTestScenario(model: CodeModelAz): string[] {
             steps.push(...ToMultiLine(`def ${functionName}(test${parameterLine()}):`));
             // find example by name
             let found = false;
-            for (let exampleCmd of model.FindExampleById(exampleId, commandParams)) {
+            let examples: CommandExample[] = [];
+            let exampleIdx = 0;
+            for (let exampleCmd of model.FindExampleById(exampleId, commandParams, examples)) {
                 if (exampleCmd && exampleCmd.length > 0) {
                     found = true;
-                    if(exampleCmd[0].indexOf(' delete') > -1) {
+                    if (exampleCmd[0].indexOf(' delete') > -1) {
                         exampleCmd[0] += " -y";
-                    } 
+                    }
                     for (let idx = 0; idx < exampleCmd.length; idx++) {
                         let prefix: string = "    " + disabled + ((idx == 0) ? "test.cmd('" : "         '");
                         let postfix: string = (idx < exampleCmd.length - 1) ? " '" : "',";
                         ToMultiLine(prefix + exampleCmd[idx] + postfix, steps);
                     }
-                    steps.push("    " + disabled + "         checks=[])");
+                    let checks = model.GetExampleChecks(examples[exampleIdx++]);
+                    if (checks.length > 0) {
+                        steps.push("    " + disabled + "         checks=[");
+                        for (let check of checks) {
+                            ToMultiLine("    " + disabled + "             " + check, steps);
+                            if (!jsonAdded && !disabled && check.indexOf("json.loads") >= 0) {
+                                header.addImport("json");
+                                jsonAdded = true;
+                            }
+                        }
+                        steps.push("    " + disabled + "         ])");
+                    }
+                    else {
+                        steps.push("    " + disabled + "         checks=[])");
+                    }
                 }
             }
             if (!found) {
@@ -197,11 +201,10 @@ function InitiateDependencies(model: CodeModelAz, imports: string[], decorators:
     if (internalObjects.length > 0) {
         initiates.push("        self.kwargs.update({");
         for (let [class_name, kargs_key, hasCreateExample, object_name] of internalObjects) {
-            if (hasCreateExample && model.RandomizeNames)
-            {
+            if (hasCreateExample && model.RandomizeNames) {
                 const RANDOMIZE_MIN_LEN = 4;
-                let prefixLen = Math.floor(object_name.length/2);
-                if(object_name.length-prefixLen<RANDOMIZE_MIN_LEN)  prefixLen = Math.max(object_name.length-RANDOMIZE_MIN_LEN, 0);
+                let prefixLen = Math.floor(object_name.length / 2);
+                if (object_name.length - prefixLen < RANDOMIZE_MIN_LEN) prefixLen = Math.max(object_name.length - RANDOMIZE_MIN_LEN, 0);
                 ToMultiLine(`            '${kargs_key}': self.create_random_name(prefix='${object_name}'[:${prefixLen}], length=${Math.max(object_name.length, RANDOMIZE_MIN_LEN)}),`, initiates);
             }
             else

src/test/scenarios/datafactory/configuration/readme.az.md

@@ -8,6 +8,7 @@ az:
   package-name: azure-mgmt-datafactory
   namespace: azure.mgmt.datafactory
   formalize-names: true
+  disable-checks: true
 az-output-folder: $(azure-cli-extension-folder)/src/datafactory
 python-sdk-output-folder: "$(az-output-folder)/azext_datafactory/vendored_sdks/datafactory"
 

src/test/scenarios/managed-network/output/src/managed-network/azext_managed_network/tests/latest/test_managed_network_scenario.py

@@ -39,7 +39,10 @@ def step_managednetworksput(test, rg):
              'icrosoft.Network/virtualNetworks/{vn_3}/subnets/default\\"}}]}}" '
              '--name "{myManagedNetwork}" '
              '--resource-group "{rg}"',
-             checks=[])
+             checks=[
+                 test.check("location", "eastus", case_sensitive=False),
+                 test.check("name", "{myManagedNetwork}", case_sensitive=False),
+             ])
 
 
 # EXAMPLE: ManagementNetworkGroupsPut
@@ -56,7 +59,10 @@ def step_managementnetworkgroupsput(test, rg):
              '--group-name "{myManagedNetworkGroup}" '
              '--managed-network-name "{myManagedNetwork}" '
              '--resource-group "{rg}"',
-             checks=[])
+             checks=[
+                 test.check("managementGroups", []),
+                 test.check("name", "{myManagedNetworkGroup}", case_sensitive=False),
+             ])
     test.cmd('az managed-network mn group wait --created '
              '--group-name "{myManagedNetworkGroup}" '
              '--resource-group "{rg}"',
@@ -71,7 +77,11 @@ def step_scopeassignmentsput(test, rg):
              'gedNetwork/managedNetworks/{myManagedNetwork}" '
              '--scope "subscriptions/subscriptionC" '
              '--name "{myScopeAssignment}"',
-             checks=[])
+             checks=[
+                 test.check("assignedManagedNetwork", "/subscriptions/{subscription_id}/resourceGroups/{rg}/providers/M"
+                            "icrosoft.ManagedNetwork/managedNetworks/{myManagedNetwork}", case_sensitive=False),
+                 test.check("name", "{myScopeAssignment}", case_sensitive=False),
+             ])
 
 
 # EXAMPLE: ManagedNetworkPeeringPoliciesPut
@@ -102,7 +112,9 @@ def step_managednetworksget(test, rg):
 def step_managednetworkslistbyresourcegroup(test, rg):
     test.cmd('az managed-network mn list '
              '--resource-group "{rg}"',
-             checks=[])
+             checks=[
+                 test.check('length(@)', 1),
+             ])
 
 
 # EXAMPLE: ManagedNetworksListBySubscription
@@ -110,7 +122,9 @@ def step_managednetworkslistbyresourcegroup(test, rg):
 def step_managednetworkslistbysubscription(test, rg):
     test.cmd('az managed-network mn list '
              '-g ""',
-             checks=[])
+             checks=[
+                 test.check('length(@)', 1),
+             ])
 
 
 # EXAMPLE: ScopeAssignmentsGet
@@ -119,7 +133,11 @@ def step_scopeassignmentsget(test, rg):
     test.cmd('az managed-network mn scope-assignment show '
              '--scope "subscriptions/subscriptionC" '
              '--name "{myScopeAssignment}"',
-             checks=[])
+             checks=[
+                 test.check("assignedManagedNetwork", "/subscriptions/{subscription_id}/resourceGroups/{rg}/providers/M"
+                            "icrosoft.ManagedNetwork/managedNetworks/{myManagedNetwork}", case_sensitive=False),
+                 test.check("name", "{myScopeAssignment}", case_sensitive=False),
+             ])
 
 
 # EXAMPLE: ScopeAssignmentsList
@@ -127,7 +145,9 @@ def step_scopeassignmentsget(test, rg):
 def step_scopeassignmentslist(test, rg):
     test.cmd('az managed-network mn scope-assignment list '
              '--scope "subscriptions/subscriptionC"',
-             checks=[])
+             checks=[
+                 test.check('length(@)', 1),
+             ])
 
 
 # EXAMPLE: ManagementNetworkGroupsGet
@@ -137,7 +157,10 @@ def step_managementnetworkgroupsget(test, rg):
              '--group-name "{myManagedNetworkGroup}" '
              '--managed-network-name "{myManagedNetwork}" '
              '--resource-group "{rg}"',
-             checks=[])
+             checks=[
+                 test.check("managementGroups", []),
+                 test.check("name", "{myManagedNetworkGroup}", case_sensitive=False),
+             ])
 
 
 # EXAMPLE: ManagedNetworksGroupsListByManagedNetwork
@@ -146,7 +169,9 @@ def step_managednetworksgroupslistbymanagednetwork(test, rg):
     test.cmd('az managed-network mn group list '
              '--managed-network-name "{myManagedNetwork}" '
              '--resource-group "{rg}"',
-             checks=[])
+             checks=[
+                 test.check('length(@)', 1),
+             ])
 
 
 # EXAMPLE: ManagedNetworkPeeringPoliciesGet
@@ -156,7 +181,9 @@ def step_managednetworkpeeringpoliciesget(test, rg):
              '--managed-network-name "{myManagedNetwork}" '
              '--policy-name "{myManagedNetworkPeeringPolicy}" '
              '--resource-group "{rg}"',
-             checks=[])
+             checks=[
+                 test.check("name", "{myManagedNetworkPeeringPolicy}", case_sensitive=False),
+             ])
 
 
 # EXAMPLE: ManagedNetworkPeeringPoliciesListByManagedNetwork
@@ -165,7 +192,9 @@ def step_managednetworkpeeringpolicieslistbymanagednetwork(test, rg):
     test.cmd('az managed-network managed-network-peering-policy list '
              '--managed-network-name "{myManagedNetwork}" '
              '--resource-group "{rg}"',
-             checks=[])
+             checks=[
+                 test.check('length(@)', 1),
+             ])
 
 
 # EXAMPLE: ManagedNetworkPeeringPoliciesDelete