From 51a43896255a4c62ea226758652c9381bad8b58f Mon Sep 17 00:00:00 2001
From: Fei Chen <43032123+feich-ms@users.noreply.github.com>
Date: Mon, 13 Apr 2020 09:45:03 +0800
Subject: [PATCH] refactor expander (#2012)

* refactor expander

* adjust code style
---
 libraries/botbuilder-lg/src/expander.ts       | 64 +++++++------
 libraries/botbuilder-lg/src/templates.ts      |  2 +-
 libraries/botbuilder-lg/tests/lg.test.js      | 96 ++++++++++++++++++-
 .../tests/testData/examples/Expand.lg         | 27 +++++-
 4 files changed, 155 insertions(+), 34 deletions(-)

diff --git a/libraries/botbuilder-lg/src/expander.ts b/libraries/botbuilder-lg/src/expander.ts
index eb94fb2d3..8c2743234 100644
--- a/libraries/botbuilder-lg/src/expander.ts
+++ b/libraries/botbuilder-lg/src/expander.ts
@@ -23,19 +23,27 @@ import { TemplateErrors } from './templateErrors';
 /**
  * LG template expander.
  */
-export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFileParserVisitor<string[]> {
+export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFileParserVisitor<any[]> {
     /**
      * Templates.
      */
     public readonly templates: Template[];
 
+    /**
+     * Expander expression parser
+     */
+    private readonly expanderExpressionParser: ExpressionParser;
+
+    /**
+     * Evaluator expression parser
+     */
+    private readonly evaluatorExpressionParser: ExpressionParser;
+
     /**
      * TemplateMap.
      */
     public readonly templateMap: {[name: string]: Template};
     private readonly evaluationTargetStack: EvaluationTarget[] = [];
-    private readonly expanderExpressionParser: ExpressionParser;
-    private readonly evaluatorExpressionParser: ExpressionParser;
     private readonly strictMode: boolean;
 
     public constructor(templates: Template[], expressionParser: ExpressionParser, strictMode: boolean = false) {
@@ -55,7 +63,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
      * @param scope Given scope.
      * @returns All possiable results.
      */
-    public expandTemplate(templateName: string, scope: any): string[] {
+    public expandTemplate(templateName: string, scope: any): any[] {
         if (!(templateName in this.templateMap)) {
             throw new Error(TemplateErrors.templateNotExist(templateName));
         }
@@ -72,13 +80,13 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         
         // Using a stack to track the evalution trace
         this.evaluationTargetStack.push(new EvaluationTarget(templateName, scope));
-        const result: string[] = this.visit(this.templateMap[templateName].parseTree);
+        const result: any[] = this.visit(this.templateMap[templateName].parseTree);
         this.evaluationTargetStack.pop();
 
         return result;
     }
 
-    public visitTemplateDefinition(ctx: lp.TemplateDefinitionContext): string[] {
+    public visitTemplateDefinition(ctx: lp.TemplateDefinitionContext): any[] {
         const templateNameContext: lp.TemplateNameLineContext = ctx.templateNameLine();
         if (templateNameContext.templateName().text === this.currentTarget().templateName) {
             return this.visit(ctx.templateBody());
@@ -87,13 +95,13 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return undefined;
     }
 
-    public visitNormalBody(ctx: lp.NormalBodyContext): string[] {
+    public visitNormalBody(ctx: lp.NormalBodyContext): any[] {
         return this.visit(ctx.normalTemplateBody());
     }
 
-    public visitNormalTemplateBody(ctx: lp.NormalTemplateBodyContext): string[] {
+    public visitNormalTemplateBody(ctx: lp.NormalTemplateBodyContext): any[] {
         const normalTemplateStrs: lp.TemplateStringContext[] = ctx.templateString();
-        let result: string[] = [];
+        let result: any[] = [];
         for (const normalTemplateStr of normalTemplateStrs) {
             result = result.concat(this.visit(normalTemplateStr.normalTemplateString()));
         }
@@ -101,7 +109,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return result;
     }
 
-    public visitIfElseBody(ctx: lp.IfElseBodyContext): string[] {
+    public visitIfElseBody(ctx: lp.IfElseBodyContext): any[] {
         const ifRules: lp.IfConditionRuleContext[] = ctx.ifElseTemplateBody().ifConditionRule();
         for (const ifRule of ifRules) {
             if (this.evalCondition(ifRule.ifCondition())) {
@@ -112,8 +120,8 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return undefined;
     }
 
-    public visitStructuredBody(ctx: lp.StructuredBodyContext): string[] {
-        const templateRefValues: Map<string, string> = new Map<string, string>();
+    public visitStructuredBody(ctx: lp.StructuredBodyContext): any[] {
+        const templateRefValues: Map<string, any> = new Map<string, any>();
         const stb: lp.StructuredTemplateBodyContext = ctx.structuredTemplateBody();
         const result: any = {};
         const typeName: string = stb.structuredBodyNameLine().STRUCTURE_NAME().text.trim();
@@ -141,7 +149,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
                 }
             } else {
                 const propertyObjects: object[] = [];
-                this.evalExpression(body.objectStructureLine().text, body.objectStructureLine()).forEach((x): number => propertyObjects.push(JSON.parse(x)));
+                this.evalExpression(body.objectStructureLine().text, body.objectStructureLine()).forEach((x): number => propertyObjects.push(x));
                 const tempResult = [];
                 for (const res of expandedResult) {
                     for (const propertyObject of propertyObjects) {
@@ -164,14 +172,14 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
             }
         }
 
-        const exps: string[] = expandedResult.map((x: string): string => JSON.stringify(x));
+        const exps: any[] = expandedResult;
 
-        let finalResult: string[] = exps;
+        let finalResult: any[] = exps;
         for (const templateRefValueKey of templateRefValues.keys()) {
-            const tempRes: string[] = [];
+            const tempRes: any[] = [];
             for (const res of finalResult) {
                 for (const refValue of templateRefValues.get(templateRefValueKey)) {
-                    tempRes.push(res.replace(templateRefValueKey, refValue));
+                    tempRes.push(JSON.parse(JSON.stringify(res).replace(templateRefValueKey, refValue.toString().replace(/\"/g, '\\\"'))));
                 }
             }
 
@@ -181,7 +189,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return finalResult;
     }
 
-    private visitStructureValue(ctx: lp.KeyValueStructureLineContext): string[] {
+    private visitStructureValue(ctx: lp.KeyValueStructureLineContext): any[] {
         const values = ctx.keyValueStructureValue();
 
         let result: any[] = [];
@@ -212,7 +220,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return result;
     }
 
-    public visitSwitchCaseBody(ctx: lp.SwitchCaseBodyContext): string[] {
+    public visitSwitchCaseBody(ctx: lp.SwitchCaseBodyContext): any[] {
         const switchcaseNodes: lp.SwitchCaseRuleContext[] = ctx.switchCaseTemplateBody().switchCaseRule();
         const length: number = switchcaseNodes.length;
         const switchNode: lp.SwitchCaseRuleContext = switchcaseNodes[0];
@@ -249,7 +257,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return undefined;
     }
 
-    public visitNormalTemplateString(ctx: lp.NormalTemplateStringContext): string[] {
+    public visitNormalTemplateString(ctx: lp.NormalTemplateStringContext): any[] {
         var prefixErrorMsg = TemplateExtensions.getPrefixErrorMessage(ctx);
         let result: string[] = [''];
         for (const node of ctx.children) {
@@ -335,7 +343,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return true;
     }
 
-    private evalExpression(exp: string, context: ParserRuleContext, errorPrefix: string = ''): string[] {
+    private evalExpression(exp: string, context: ParserRuleContext, errorPrefix: string = ''): any[] {
         exp = TemplateExtensions.trimExpression(exp);
         let result: any;
         let error: string;
@@ -358,7 +366,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
 
         if (Array.isArray(result))
         {
-            return result.map((u): string => u.toString());
+            return result;
         }
 
         return [ result.toString() ];
@@ -426,7 +434,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return undefined;
     }
 
-    private readonly templateEvaluator = (templateName: string): any => (args: readonly any[]): string => {
+    private readonly templateEvaluator = (templateName: string): any => (args: readonly any[]): any => {
         const newScope: any = this.constructScope(templateName, Array.from(args));
 
         const value: string[] = this.expandTemplate(templateName, newScope);
@@ -435,7 +443,7 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         return value[randomNumber];
     }
 
-    private readonly templateExpander = (templateName: string): any => (args: readonly any[]): string[] => {
+    private readonly templateExpander = (templateName: string): any => (args: readonly any[]): any[] => {
         const newScope: any = this.constructScope(templateName, Array.from(args));
 
         return this.expandTemplate(templateName, newScope);
@@ -504,14 +512,12 @@ export class Expander extends AbstractParseTreeVisitor<string[]> implements LGFi
         };
     }
 
-    private readonly templateFunction = (): any => (args: readonly any[]): any => {
+    private readonly templateFunction = (): any => (args: readonly any[]): any[] => {
         const templateName: string = args[0];
         const newScope: any = this.constructScope(templateName, args.slice(1));
+        const value: any[] = this.expandTemplate(templateName, newScope);
 
-        const value: string[] = this.expandTemplate(templateName, newScope);
-        const randomNumber: number = Math.floor(Math.random() * value.length);
-
-        return value[randomNumber];
+        return value;
     }
 
     private readonly validateTemplateFunction = (expression: Expression): void => {
diff --git a/libraries/botbuilder-lg/src/templates.ts b/libraries/botbuilder-lg/src/templates.ts
index c908bac1e..0ec99d1a9 100644
--- a/libraries/botbuilder-lg/src/templates.ts
+++ b/libraries/botbuilder-lg/src/templates.ts
@@ -191,7 +191,7 @@ export class Templates implements Iterable<Template> {
      * @param scope The state visible in the evaluation.
      * @returns Expand result.
      */
-    public expandTemplate(templateName: string, scope?: object): string[] {
+    public expandTemplate(templateName: string, scope?: object): any[] {
         this.checkErrors();
 
         const expander = new Expander(this.allTemplates, this.expressionParser, this.strictMode);
diff --git a/libraries/botbuilder-lg/tests/lg.test.js b/libraries/botbuilder-lg/tests/lg.test.js
index c59abb363..81a7c4d80 100644
--- a/libraries/botbuilder-lg/tests/lg.test.js
+++ b/libraries/botbuilder-lg/tests/lg.test.js
@@ -485,6 +485,90 @@ describe('LG', function() {
         assert(evaled[0] === 'ey' || evaled[0] === 'el');
     });
 
+    it('TestExpandTemplateWithIsTemplateFunction', function() {
+        const templates = Templates.parseFile(GetExampleFilePath('Expand.lg'));
+
+        let evaled = templates.expandTemplate('template2', {templateName: 'Greeting'});
+        assert.strictEqual(evaled.length, 2);
+        assert.strictEqual(evaled[0], 'Hi');
+        assert.strictEqual(evaled[1], 'Hello');
+
+        evaled = templates.expandTemplate('template2', {templateName: 'xxx'});
+        assert.strictEqual(evaled.length, 2);
+        assert.strictEqual(evaled[0], 'Morning');
+        assert.strictEqual(evaled[1], 'Evening');
+    });
+
+    it('TestExpandTemplateWithTemplateFunction', function() {
+        const templates = Templates.parseFile(GetExampleFilePath('Expand.lg'));
+
+        let evaled = templates.expandTemplate('template3', {templateName: 'Greeting'});
+        assert.strictEqual(evaled.length, 2);
+        assert.strictEqual(evaled[0], 'Hi');
+        assert.strictEqual(evaled[1], 'Hello');
+    });
+
+    it('TestExpandTemplateWithDoubleQuotation', function() {
+        const templates = Templates.parseFile(GetExampleFilePath('Expand.lg'));
+
+        let evaled = templates.expandTemplate('ExpanderT1');
+        assert.strictEqual(evaled.length, 2);
+        const expectedResults = [
+            "{\"lgType\":\"MyStruct\",\"text\":\"Hi \\\"quotes\\\" allowed\",\"speak\":\"how old are you?\"}",
+            "{\"lgType\":\"MyStruct\",\"text\":\"Hi \\\"quotes\\\" allowed\",\"speak\":\"what's your age?\"}"
+        ];
+
+        expectedResults.forEach((value, index) => {
+            assert.strictEqual(JSON.stringify(evaled[index]), JSON.stringify(JSON.parse(value)));
+        });
+    });
+
+    it('TestExpandTemplateWithEscapeCharacter', function() {
+        const templates = Templates.parseFile(GetExampleFilePath('EscapeCharacter.lg'));
+        var evaled = templates.expandTemplate('wPhrase');
+        assert.strictEqual(evaled[0], 'Hi \r\n\t\\');
+
+        evaled = templates.expandTemplate('AtEscapeChar');
+        assert.strictEqual(evaled[0], 'Hi{1+1}[wPhrase]{wPhrase()}${wPhrase()}2${1+1}');
+
+        evaled = templates.expandTemplate('otherEscape');
+        assert.strictEqual(evaled[0], 'Hi \\y \\');
+
+        evaled = templates.expandTemplate('escapeInExpression');
+        assert.strictEqual(evaled[0], 'Hi hello\\\\');
+
+        evaled = templates.expandTemplate('escapeInExpression2');
+        assert.strictEqual(evaled[0], "Hi hello'");
+
+        evaled = templates.expandTemplate('escapeInExpression3');
+        assert.strictEqual(evaled[0], 'Hi hello\"');
+
+        evaled = templates.expandTemplate('escapeInExpression4');
+        assert.strictEqual(evaled[0], 'Hi hello\"');
+
+        evaled = templates.expandTemplate('escapeInExpression5');
+        assert.strictEqual(evaled[0], 'Hi hello\n');
+
+        evaled = templates.expandTemplate('escapeInExpression6');
+        assert.strictEqual(evaled[0], 'Hi hello\n');
+
+        var todos = ['A', 'B', 'C'];
+        evaled = templates.expandTemplate('showTodo', {todos});
+        assert.strictEqual(evaled[0].toString().replace(/\r\n/g, '\n'), '\n    Your most recent 3 tasks are\n    * A\n* B\n* C\n    ');
+
+        evaled = templates.expandTemplate('showTodo');
+        assert.strictEqual(evaled[0].toString().replace(/\r\n/g, '\n'), "\n    You don't have any \"t\\\\odo'\".\n    ");
+
+        evaled = templates.expandTemplate('getUserName');
+        assert.strictEqual(evaled[0], "super \"x man\"");
+
+        evaled = templates.expandTemplate('structure1');
+        assert.strictEqual(JSON.stringify(evaled[0]), "{\"lgType\":\"struct\",\"list\":[\"a\",\"b|c\"]}");
+
+        evaled = templates.expandTemplate('dollarsymbol');
+        assert.strictEqual(evaled[0], "$ $ ${'hi'} hi");
+    });
+
     it('TestInlineEvaluate', function() {
         var templates = Templates.parseFile(GetExampleFilePath('2.lg'));
         var evaled = templates.evaluateText('hello');
@@ -676,18 +760,24 @@ describe('LG', function() {
             '{"lgType":"Activity","text":"what\'s your age?","speak":"how old are you?"}',
             '{"lgType":"Activity","text":"what\'s your age?","speak":"what\'s your age?"}'
         ];
-        expectedResults.forEach( u => assert(evaled.includes(u)));
+
+        expectedResults.forEach((value, index) => {
+            assert.strictEqual(JSON.stringify(evaled[index]), JSON.stringify(JSON.parse(value)));
+        });
 
         evaled = templates.expandTemplate('ExpanderT1');
         assert.strictEqual(evaled.length, 4, `Evaled is ${ evaled }`);
     
         expectedResults = [
             '{"lgType":"MyStruct","text":"Hi","speak":"how old are you?"}',
-            '{"lgType":"MyStruct","text":"Hi","speak":"what\'s your age?"}',
             '{"lgType":"MyStruct","text":"Hello","speak":"how old are you?"}',
+            '{"lgType":"MyStruct","text":"Hi","speak":"what\'s your age?"}',
             '{"lgType":"MyStruct","text":"Hello","speak":"what\'s your age?"}'
         ];
-        expectedResults.forEach( u => assert(evaled.includes(u)));
+
+        expectedResults.forEach((value, index) => {
+            assert.strictEqual(JSON.stringify(evaled[index]), JSON.stringify(JSON.parse(value)));
+        });
     });
 
     it('TestExpressionextract', function() {
diff --git a/libraries/botbuilder-lg/tests/testData/examples/Expand.lg b/libraries/botbuilder-lg/tests/testData/examples/Expand.lg
index 38cbd3598..330b6377e 100644
--- a/libraries/botbuilder-lg/tests/testData/examples/Expand.lg
+++ b/libraries/botbuilder-lg/tests/testData/examples/Expand.lg
@@ -75,4 +75,29 @@ They are ${ShowAlarm(alarms[1])}
 - ${count(T1())}
 
 # T4
-- ${substring(T1(), 1, 2)}
\ No newline at end of file
+- ${substring(T1(), 1, 2)}
+
+# template2(templateName)
+- IF: ${isTemplate(templateName)}
+    - ${Greeting()}
+- ELSE:
+    - ${TimeOfDay()}
+
+# template3(templateName)
+  - ${template(templateName)}
+
+# GetAge
+- how old are you?
+- what's your age?
+
+# ExpanderT1
+[MyStruct
+    Text = Hi "quotes" allowed
+    ${ExpanderT2()}
+]
+
+# ExpanderT2
+[MyStruct
+    Speak = ${GetAge()}
+    Text = zoo
+]
\ No newline at end of file