From abc14c6a52d303a4f97e9bc3745f4187f53e0547 Mon Sep 17 00:00:00 2001 From: "Hongyang Du (hond)" Date: Mon, 20 Apr 2020 23:44:32 +0800 Subject: [PATCH] [LG] Robust Template CRUD with Two-phase parsing of LG (#3731) * init * init * init * init * init * init * init * fix error * remove unused code * remove unused file * remove unused code * fix build error * fix error * refine * only allow define option/import at the top of lg file. * refine import recognization * add more comments and remove unused code * remove unused antlr file * remove antlr related file * fix build warning * refine * fix error * fix error * fix error * add context for Template and TemplateImport * fix comments * fix comments * fix comments * rename templatebody to body to avoid the same context name --- CodeCoverage.runsettings | 2 + .../parser/ExpressionAntlrLexer.g4 | 3 + .../parser/ExpressionAntlrParser.g4 | 3 + .../parser/ExpressionParser.cs | 2 - .../Analyzer.cs | 40 +- .../AnalyzerResult.cs | 2 +- .../CustomizedMemory.cs | 12 +- .../ErrorListener.cs | 8 +- .../EvaluationTarget.cs | 8 +- .../Evaluator.cs | 60 +-- .../Expander.cs | 53 +-- .../Extensions.cs | 51 ++- .../LGFileLexer.g4 | 251 +--------- .../LGFileParser.g4 | 145 ++---- .../LGTemplateLexer.g4 | 202 ++++++++ .../LGTemplateParser.g4 | 93 ++++ .../Range.cs | 11 +- .../SourceRange.cs | 50 ++ .../StaticChecker.cs | 114 ++--- .../Template.cs | 137 ++---- .../TemplateErrors.cs | 4 +- .../TemplateImport.cs | 54 +-- .../Templates.cs | 62 +-- .../TemplatesParser.cs | 431 +++++++++++------- .../en-us/sandwichTest-Bread.en-us.lg | 7 +- .../en-us/sandwichTest-BreadEntity.en-us.lg | 5 +- .../en-us/sandwichTest-Cheese.en-us.lg | 7 +- .../en-us/sandwichTest-CheeseEntity.en-us.lg | 5 +- .../en-us/sandwichTest-Length.en-us.lg | 5 +- .../sandwich/en-us/sandwichTest-Meat.en-us.lg | 7 +- .../en-us/sandwichTest-MeatEntity.en-us.lg | 5 +- .../sandwich/en-us/sandwichTest-Name.en-us.lg | 7 +- .../en-us/sandwichTest-Price.en-us.lg | 5 +- .../en-us/sandwichTest-Quantity.en-us.lg | 7 +- .../sandwichTest-library-Assign.en-us.lg | 5 +- .../sandwichTest-library-Choose.en-us.lg | 5 +- .../en-us/sandwichTest-library-Help.en-us.lg | 3 +- .../sandwichTest-library-Missing.en-us.lg | 5 +- .../sandwichTest-library-ReadForm.en-us.lg | 3 +- .../en-us/sandwichTest-library.en-us.lg | 3 +- .../unittests/en-us/unittests-Age.en-us.lg | 5 +- .../en-us/unittests-Datetimev2.en-us.lg | 5 +- .../en-us/unittests-Dimension.en-us.lg | 5 +- .../unittests/en-us/unittests-Email.en-us.lg | 7 +- .../unittests/en-us/unittests-Enum1.en-us.lg | 7 +- .../en-us/unittests-Enum1Entity.en-us.lg | 5 +- .../unittests/en-us/unittests-Enum2.en-us.lg | 7 +- .../en-us/unittests-Enum2Entity.en-us.lg | 5 +- .../en-us/unittests-Geographyv2.en-us.lg | 5 +- .../unittests-Keyphraseproperty.en-us.lg | 7 +- .../unittests/en-us/unittests-Money.en-us.lg | 5 +- .../en-us/unittests-Ordinalv2.en-us.lg | 5 +- .../en-us/unittests-Percentage.en-us.lg | 7 +- .../en-us/unittests-Personname.en-us.lg | 7 +- .../en-us/unittests-Phonenumber.en-us.lg | 7 +- .../en-us/unittests-Quantity.en-us.lg | 7 +- .../en-us/unittests-Temperature.en-us.lg | 5 +- .../unittests/en-us/unittests-Url.en-us.lg | 7 +- .../unittests-Utteranceproperty.en-us.lg | 7 +- .../en-us/unittests-Zipcodepattern.en-us.lg | 7 +- .../en-us/unittests-library-Assign.en-us.lg | 5 +- .../en-us/unittests-library-Choose.en-us.lg | 5 +- .../en-us/unittests-library-Help.en-us.lg | 3 +- .../en-us/unittests-library-Missing.en-us.lg | 5 +- .../en-us/unittests-library-ReadForm.en-us.lg | 3 +- .../en-us/unittests-library.en-us.lg | 3 +- .../Examples/CustomFunction.lg | 5 +- .../Examples/import.lg | 9 +- .../Examples/import2.lg | 8 +- .../ExceptionExamples/ConditionFormatError.lg | 4 +- .../DuplicatedTemplatesInImportFiles.lg | 5 +- .../ExceptionExamples/ErrorLine.lg | 9 + .../InvalidLGFileImportPath.lg | 7 +- .../ExceptionExamples/MultiLineExprError.lg | 15 - ...ot.Builder.LanguageGeneration.Tests.csproj | 4 + .../TemplateDiagnosticTest.cs | 22 +- .../TemplatesTest.cs | 2 +- .../sandwich/en-us/sandwich-Bread.en-us.lg | 7 +- .../en-us/sandwich-BreadEntity.en-us.lg | 5 +- .../sandwich/en-us/sandwich-Cheese.en-us.lg | 7 +- .../en-us/sandwich-CheeseEntity.en-us.lg | 5 +- .../sandwich/en-us/sandwich-Length.en-us.lg | 5 +- .../sandwich/en-us/sandwich-Meat.en-us.lg | 7 +- .../en-us/sandwich-MeatEntity.en-us.lg | 5 +- .../sandwich/en-us/sandwich-Name.en-us.lg | 7 +- .../sandwich/en-us/sandwich-Price.en-us.lg | 5 +- .../sandwich/en-us/sandwich-Quantity.en-us.lg | 7 +- .../en-us/sandwich-library-Assign.en-us.lg | 5 +- .../en-us/sandwich-library-Choose.en-us.lg | 5 +- .../en-us/sandwich-library-Help.en-us.lg | 3 +- .../en-us/sandwich-library-Missing.en-us.lg | 5 +- .../en-us/sandwich-library-ReadForm.en-us.lg | 3 +- .../sandwich/en-us/sandwich-library.en-us.lg | 2 +- 93 files changed, 1080 insertions(+), 1121 deletions(-) create mode 100644 libraries/Microsoft.Bot.Builder.LanguageGeneration/LGTemplateLexer.g4 create mode 100644 libraries/Microsoft.Bot.Builder.LanguageGeneration/LGTemplateParser.g4 create mode 100644 libraries/Microsoft.Bot.Builder.LanguageGeneration/SourceRange.cs create mode 100644 tests/Microsoft.Bot.Builder.LanguageGeneration.Tests/ExceptionExamples/ErrorLine.lg diff --git a/CodeCoverage.runsettings b/CodeCoverage.runsettings index 12529d305..6575840af 100644 --- a/CodeCoverage.runsettings +++ b/CodeCoverage.runsettings @@ -77,6 +77,8 @@ .*\\ExpressionAntlrParser.cs .*\\LGFileLexer.cs .*\\LGFileParser.cs + .*\\LGTemplateLexer.cs + .*\\LGTemplateParser.cs diff --git a/libraries/AdaptiveExpressions/parser/ExpressionAntlrLexer.g4 b/libraries/AdaptiveExpressions/parser/ExpressionAntlrLexer.g4 index 3758d022a..bd1e0f27f 100644 --- a/libraries/AdaptiveExpressions/parser/ExpressionAntlrLexer.g4 +++ b/libraries/AdaptiveExpressions/parser/ExpressionAntlrLexer.g4 @@ -1,5 +1,8 @@ lexer grammar ExpressionAntlrLexer; +@parser::header {#pragma warning disable 3021} // Disable StyleCop warning CS3021 re CLSCompliant attribute in generated files. +@lexer::header {#pragma warning disable 3021} // Disable StyleCop warning CS3021 re CLSCompliant attribute in generated files. + @lexer::members { bool ignoreWS = true; // usually we ignore whitespace, but inside stringInterpolation, whitespace is significant } diff --git a/libraries/AdaptiveExpressions/parser/ExpressionAntlrParser.g4 b/libraries/AdaptiveExpressions/parser/ExpressionAntlrParser.g4 index fe67ef460..7c5c3ef04 100644 --- a/libraries/AdaptiveExpressions/parser/ExpressionAntlrParser.g4 +++ b/libraries/AdaptiveExpressions/parser/ExpressionAntlrParser.g4 @@ -1,5 +1,8 @@ parser grammar ExpressionAntlrParser; +@parser::header {#pragma warning disable 3021} // Disable StyleCop warning CS3021 re CLSCompliant attribute in generated files. +@lexer::header {#pragma warning disable 3021} // Disable StyleCop warning CS3021 re CLSCompliant attribute in generated files. + options { tokenVocab=ExpressionAntlrLexer; } file: expression EOF; diff --git a/libraries/AdaptiveExpressions/parser/ExpressionParser.cs b/libraries/AdaptiveExpressions/parser/ExpressionParser.cs index 4addf0b15..97639f9ee 100644 --- a/libraries/AdaptiveExpressions/parser/ExpressionParser.cs +++ b/libraries/AdaptiveExpressions/parser/ExpressionParser.cs @@ -13,8 +13,6 @@ using Antlr4.Runtime.Misc; using Antlr4.Runtime.Tree; using Newtonsoft.Json.Linq; -[assembly: CLSCompliant(false)] - namespace AdaptiveExpressions { /// diff --git a/libraries/Microsoft.Bot.Builder.LanguageGeneration/Analyzer.cs b/libraries/Microsoft.Bot.Builder.LanguageGeneration/Analyzer.cs index 0e2bddd2a..1afe3b3e0 100644 --- a/libraries/Microsoft.Bot.Builder.LanguageGeneration/Analyzer.cs +++ b/libraries/Microsoft.Bot.Builder.LanguageGeneration/Analyzer.cs @@ -12,7 +12,7 @@ namespace Microsoft.Bot.Builder.LanguageGeneration /// /// LG template analyzer. /// - public class Analyzer : LGFileParserBaseVisitor + public class Analyzer : LGTemplateParserBaseVisitor { private readonly Dictionary templateMap; @@ -23,8 +23,8 @@ namespace Microsoft.Bot.Builder.LanguageGeneration /// /// Initializes a new instance of the class. /// - /// template list. - /// expression parser. + /// Template list. + /// Expression parser. public Analyzer(List