Expose non-source flag on IdentToken to internals (#2199)

For the Record support in PA Client for merge attributes, we're
introducing a function that is only callable from generated expressions,
not user code. This adds a flag to tell if an identifier is from source
directly, or from something we generated.

src/libraries/Microsoft.PowerFx.Core/Lexer/Tokens/IdentToken.cs

@@ -17,6 +17,7 @@ namespace Microsoft.PowerFx.Syntax
         internal readonly bool HasDelimiterStart;
         internal readonly bool HasDelimiterEnd;
         internal readonly bool IsModified;
+        internal readonly bool IsNonSourceIdentToken = false;
 
         // Unescaped, unmodified value.
         internal readonly string _value;
@@ -44,6 +45,7 @@ namespace Microsoft.PowerFx.Syntax
             Contracts.Assert(val.Length == span.Lim - span.Min || isNonSourceIdentToken);
             _value = val;
             Name = DName.MakeValid(val, out IsModified);
+            IsNonSourceIdentToken = isNonSourceIdentToken;
         }
 
         internal IdentToken(string val, Span spanTok, bool fDelimiterStart, bool fDelimiterEnd)
@@ -68,6 +70,7 @@ namespace Microsoft.PowerFx.Syntax
         private IdentToken(IdentToken tok, Span newSpan)
             : this(tok._value, newSpan, tok.HasDelimiterStart, tok.HasDelimiterEnd)
         {
+            IsNonSourceIdentToken = tok.IsNonSourceIdentToken;
         }
 
         internal override Token Clone(Span ts)

src/tests/Microsoft.PowerFx.Core.Tests/AttributeParserTests.cs

@@ -75,6 +75,11 @@ namespace Microsoft.PowerFx.Core.Tests
 
             var genNf = result.NamedFormulas.Last();
             Assert.StartsWith(combinedFunctionName, genNf.Formula.Script);
+
+            // Check that the call in the generated merged expression is tagged as non-source
+            var call = genNf.Formula.ParseTree.AsCall();
+            Assert.NotNull(call);
+            Assert.True(call.Head.Token.IsNonSourceIdentToken);
         }
 
         [Fact]

src/tests/Microsoft.PowerFx.Core.Tests/ParseTests.cs

@@ -500,6 +500,7 @@ namespace Microsoft.PowerFx.Core.Tests
                     Assert.NotNull(node.AsCall().Head);
                     Assert.True(node.AsCall().Head is Identifier);
                     Assert.True((node.AsCall().Head as Identifier).Namespace.IsRoot);
+                    Assert.False(node.AsCall().Head.Token.IsNonSourceIdentToken);
                 });
 
             TestRoundtrip(
@@ -512,6 +513,7 @@ namespace Microsoft.PowerFx.Core.Tests
                     Assert.NotNull(node.AsCall().Head);
                     Assert.True(node.AsCall().Head is Identifier);
                     Assert.False((node.AsCall().Head as Identifier).Namespace.IsRoot);
+                    Assert.False(node.AsCall().Head.Token.IsNonSourceIdentToken);
                     Assert.Equal("Netflix.Services", (node.AsCall().Head as Identifier).Namespace.ToDottedSyntax());
 
                     Assert.NotNull(node.AsCall().HeadNode);