Fxing expression formatting/pretty print bug with reserved keywords (#1509)

Earlier this week, Customer reached out to us about a bug when trying to
format the expression like "Set(Children; true)" in the formula bar.
"Children" is not a reserved keyword and unless the disabled reserved
keyword flag is not set when invoking Lexer, "Children" gets identified
as an ErrorToken. This causes errors in the AST and we don't format when
there are errors and just return the same expression.

To solve this, we need to pass in a Flags.DisabledReservedKeyword flag
to the parser and ultimately lexer when invoking them before the pretty
printing. There was no ability to pass in custom flags so I extended the
"Format" function (which is an entry point for pretty printing) with an
optional argument to allow passing custom flags. These changes would be
later consumed in Canvas Apps Backend (Would be passing in
Flags.DisabledReservedKeyword | Flags.EnableExpressionChaining to Format
in Canvas Apps Backend)

@gregli-msft Could you take a look at this and see if this is the right
way to solve it

src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs

@@ -1766,11 +1766,17 @@ namespace Microsoft.PowerFx.Core.Parser
             }
         }
 
-        public static string Format(string text)
+        /// <summary>
+        /// Formats/Pretty Print the given expression text to more a readable form.
+        /// </summary>
+        /// <param name="text">Expression text to format.</param>
+        /// <param name="flags">Optional flags to customize the behavior of underlying lexer and parser. By default, expression chaining is enabled.</param>
+        /// <returns>Formatted expression text.</returns>
+        public static string Format(string text, Flags flags = Flags.EnableExpressionChaining)
         {
             var result = ParseScript(
                 text,
-                flags: Flags.EnableExpressionChaining);
+                flags: flags);
 
             // Can't pretty print a script with errors.
             if (result.HasError)

src/tests/Microsoft.PowerFx.Core.Tests/FormatterTests.cs

@@ -160,17 +160,80 @@ namespace Microsoft.PowerFx.Tests
         [InlineData("$\"{{{{1+1}}}}\"", "$\"{{{{1+1}}}}\"")]
         [InlineData("Set(str, $\"{{}}\")", "Set(\n    str,\n    $\"{{}}\"\n)")]
         [InlineData("Set(additionText, $\"The sum of 1 and 3 is {{{1 + 3}}})\")", "Set(\n    additionText,\n    $\"The sum of 1 and 3 is {{{1 + 3}}})\"\n)")]
-        [InlineData("$\"This is {{\"Another\"}} interpolated {{string}}\"", "$\"This is {{\"Another\"}} interpolated {{string}}\"")]
+        [InlineData("$\"This is {{\"Another\"}} interpolated {{string}}\"", "$\"This is {{\"Another\"}} interpolated {{string}}\"")]
         public void TestPrettyPrint(string script, string expected)
-        {
+        {
+            // Act & Assert
             var result = Format(script);
             Assert.NotNull(result);
             Assert.Equal(expected, result);
 
-            // Ensure idempotence
+            // Act & Assert: Ensure idempotence
             result = Format(result);
             Assert.NotNull(result);
             Assert.Equal(expected, result);
+        }
+
+        [Theory]
+        [InlineData(TexlLexer.ReservedBlank)]
+        [InlineData(TexlLexer.ReservedChild)]
+        [InlineData(TexlLexer.ReservedChildren)]
+        [InlineData(TexlLexer.ReservedEmpty)]
+        [InlineData(TexlLexer.ReservedIs)]
+        [InlineData(TexlLexer.ReservedNone)]
+        [InlineData(TexlLexer.ReservedNothing)]
+        [InlineData(TexlLexer.ReservedNull)]
+        [InlineData(TexlLexer.ReservedSiblings)]
+        [InlineData(TexlLexer.ReservedThis)]
+        [InlineData(TexlLexer.ReservedUndefined)]
+        public void TestPrettyPrintWithDisabledReservedKeywordsFlag(string keyword)
+        {
+            // Arrange
+            var expression = $"Set({keyword}; true)";
+            var expectedFormattedExpr = $"Set(\n    {keyword};\n    true\n)";
+            var flags = Flags.DisableReservedKeywords | Flags.EnableExpressionChaining;
+
+            // Act
+            var result = Format(expression, flags);
+
+            // Asssert
+            Assert.NotNull(result);
+            Assert.Equal(expectedFormattedExpr, result);
+
+            // Act: Ensure idempotence
+            result = Format(result, flags);
+            
+            // Assert: Ensure idempotence
+            Assert.NotNull(result);
+            Assert.Equal(expectedFormattedExpr, result);
+        }
+
+        [Theory]
+        [InlineData(0)]
+        [InlineData(1)]
+        [InlineData(2)]
+        [InlineData(3)]
+        [InlineData(4)]
+        [InlineData(5)]
+        [InlineData(10)]
+        public void TestPrettyPrintAndRemoveWhitespaceRoundtripWithDisabledReservedKeywordsFlag(int trips)
+        {
+            // Arrange
+            var unformattedExpr = $"Set({TexlLexer.ReservedChildren}; true )";
+            var formatedExpr = $"Set(\n    {TexlLexer.ReservedChildren};\n    true\n)";
+            var expectedOutcome = trips % 2 == 0 ? unformattedExpr : formatedExpr;
+
+            // Act
+            var outcome = unformattedExpr;
+            for (var i = 1; i <= trips; ++i) 
+            {
+                outcome = i % 2 == 0 ?
+                          TexlLexer.InvariantLexer.RemoveWhiteSpace(outcome) :
+                          Format(outcome, Flags.DisableReservedKeywords | Flags.EnableExpressionChaining);
+            }
+
+            // Assert
+            Assert.Equal(expectedOutcome, outcome);
         }
     }
 }