Heavily Refactor LSP (Still Backwards Compatible) and Make it Async and Customizable (#2308)

This pr significantly refactors LSP SDK to truly make it async and bit
customizable. This pr also reorganizes existing tests into a much
modular structure while still keeping around the old tests as a proof
that the changes in this pr are backwards compatible.

The main goal of this PR is to make LSP async in such as a way that
hosts can run expensive operations like nl2fx network calls
asynchronously while making sure the operations that need to run
synchronously acquire correct host specific locks.

This pr accomplishes it goals by 
1) Refactoring a monolith language server class into small handlers
targeting only one specific method. Each handler is async by default and
this allows us to not let LSP block threads inside its hosts
2) LSP in this pr identifies critical sections which are delegated to
hosts for their executions. Host provides an environment using which LSP
can acquire appropriate locks and run those critical sections inside
Host environment
3) LSP, now being async, allows hosts to run Nl2Fx or Fx2Nl network
calls in a non blocking way

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/CodeActions/CodeActionsLanguageServerOperationHandler.cs

@@ -0,0 +1,114 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using static Microsoft.PowerFx.LanguageServerProtocol.LanguageServer;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Handler for code actions.
+    /// </summary>
+    internal sealed class CodeActionsLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public bool IsRequest => true;
+
+        public string LspMethod => TextDocumentNames.CodeAction;
+
+        private readonly OnLogUnhandledExceptionHandler _onLogUnhandledExceptionHandler;
+
+        public CodeActionsLanguageServerOperationHandler(OnLogUnhandledExceptionHandler onLogUnhandledExceptionHandler = null)
+        {
+            _onLogUnhandledExceptionHandler = onLogUnhandledExceptionHandler;
+        }
+         
+        /// <summary>
+        /// Compute quick fixes for the given expression.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="expression">Expression for which quick fixes need to be computed.</param>
+        /// <param name="codeActionParams">Code Action Params.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        /// <returns>Code Action Results.</returns>
+        private Task<CodeActionResult[]> HandleQuickFixes(LanguageServerOperationContext operationContext, string expression, CodeActionParams codeActionParams, CancellationToken cancellationToken)
+        {
+            return operationContext.ExecuteHostTaskAsync(
+            codeActionParams.TextDocument.Uri,
+            (scope) =>
+            {
+                if (scope is EditorContextScope scopeQuickFix)
+                {
+                    return Task.FromResult(scopeQuickFix.SuggestFixes(expression, _onLogUnhandledExceptionHandler));
+                }
+
+                return Task.FromResult(Array.Empty<CodeActionResult>());
+            }, cancellationToken,
+            defaultOutput: Array.Empty<CodeActionResult>());
+        }
+
+        /// <summary>
+        /// Handles the code actions operation.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            operationContext.Logger?.LogInformation($"[PFX] HandleCodeActionRequest: id={operationContext.RequestId ?? "<null>"}, paramsJson={operationContext.RawOperationInput ?? "<null>"}");
+
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out CodeActionParams codeActionParams))
+            {
+                return;
+            }
+
+            var documentUri = codeActionParams.TextDocument.Uri;
+            var uri = new Uri(documentUri);
+            var expression = LanguageServerHelper.ChooseExpression(codeActionParams, HttpUtility.ParseQueryString(uri.Query));
+            if (expression == null)
+            {
+                operationContext.OutputBuilder.AddInvalidParamsError(operationContext.RequestId, "Failed to choose expression for code actions operation");
+                return;
+            }
+
+            var codeActions = new Dictionary<string, CodeAction[]>();
+            foreach (var codeActionKind in codeActionParams.Context.Only)
+            {
+                var results = new CodeActionResult[0];
+                switch (codeActionKind)
+                {
+                    case CodeActionKind.QuickFix:
+                        results = await HandleQuickFixes(operationContext, expression, codeActionParams, cancellationToken).ConfigureAwait(false);
+                        break;
+                    default:
+                        // No action.
+                        return;
+                }
+
+                var items = new List<CodeAction>();
+                foreach (var item in results)
+                {
+                    var range = item.Range ?? codeActionParams.Range;
+                    items.Add(new CodeAction()
+                    {
+                        Title = item.Title,
+                        Kind = codeActionKind,
+                        Edit = new WorkspaceEdit
+                        {
+                            Changes = new Dictionary<string, TextEdit[]> { { documentUri, new[] { new TextEdit { Range = range, NewText = item.Text } } } }
+                        },
+                        ActionResultContext = item.ActionResultContext
+                    });
+                }
+                    
+                codeActions.Add(codeActionKind, items.ToArray());
+            }
+
+            operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, codeActions);
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/CommandsExecuted/CommandExecutedLanguageServerOperationHandler.cs

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    ///  Handler for CommandExecuted operation.
+    /// </summary>
+    internal sealed class CommandExecutedLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public bool IsRequest => true;
+
+        public string LspMethod => CustomProtocolNames.CommandExecuted;
+
+        /// <summary>
+        /// Hook to handle the CodeActionApplied operation.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="commandExecutedParams">Command Executed Params.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        private async Task HandleCodeActionApplied(LanguageServerOperationContext operationContext, CommandExecutedParams commandExecutedParams, CancellationToken cancellationToken)
+        {
+            var codeActionResult = JsonRpcHelper.Deserialize<CodeAction>(commandExecutedParams.Argument);
+            if (codeActionResult.ActionResultContext == null)
+            {
+                operationContext.OutputBuilder.AddProperyValueRequiredError(operationContext.RequestId, $"{nameof(CodeAction.ActionResultContext)} is null or empty.");
+                return;
+            }
+
+            await operationContext.ExecuteHostTaskAsync(
+            commandExecutedParams.TextDocument.Uri,
+            (scope) =>
+            {
+                if (scope is EditorContextScope scopeQuickFix)
+                {
+                    scopeQuickFix.OnCommandExecuted(codeActionResult);
+                }
+            }, cancellationToken).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// Handles the CommandExecuted operation.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            operationContext.Logger?.LogInformation($"[PFX] HandleCommandExecutedRequest: id={operationContext.RequestId ?? "<null>"}, paramsJson={operationContext.RawOperationInput ?? "<null>"}");
+            if (!TryParseAndValidateParams(operationContext, out var commandExecutedParams))
+            {
+                return;
+            }
+
+            switch (commandExecutedParams.Command)
+            {
+                case CommandName.CodeActionApplied:
+                    await HandleCodeActionApplied(operationContext, commandExecutedParams, cancellationToken).ConfigureAwait(false);
+                    return;
+                default:
+                    operationContext.OutputBuilder.AddInvalidRequestError(operationContext.RequestId, $"{commandExecutedParams.Command} is not supported.");
+                    return;
+            }
+        }
+
+        /// <summary>
+        /// A helper method to parse and validate the params for CommandExecuted.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="commandExecutedParams">Reference to hold the commond executed params.</param>
+        /// <returns>True if parsing and validation were successful or false otherwise.</returns>
+        private static bool TryParseAndValidateParams(LanguageServerOperationContext operationContext, out CommandExecutedParams commandExecutedParams)
+        {
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out commandExecutedParams))
+            {
+                return false;
+            }
+
+            if (string.IsNullOrWhiteSpace(commandExecutedParams?.Argument))
+            {
+                operationContext.OutputBuilder.AddProperyValueRequiredError(operationContext.RequestId, $"{nameof(CommandExecutedParams.Argument)} is null or empty.");
+                return false;
+            }
+
+            return true;
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Completions/CompletionsLanguageServerOperationHandler.cs

@@ -0,0 +1,153 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Syntax;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Handler for the completions operation.
+    /// Currently, Language Server SDK delegates full completion logic to the host.
+    /// It only does validation of the input and then transforms the results to the LSP format.
+    /// Those are not needed to be exposed as overridable methods/hooks.
+    /// Therefore, there's only one HandleAsync method.
+    /// </summary>
+    internal sealed class CompletionsLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public bool IsRequest => true;
+
+        public string LspMethod => TextDocumentNames.Completion;
+
+        /// <summary>
+        /// Provides the suggestions for the given expression.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="uri">Document Uri.</param>
+        /// <param name="expression">Expression.</param>
+        /// <param name="cursorPosition">Cursor Position.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        /// <returns>Suggestions and Signatures.</returns>
+        private Task<IIntellisenseResult> SuggestAsync(LanguageServerOperationContext operationContext, string uri, string expression, int cursorPosition, CancellationToken cancellationToken)
+        {
+            return operationContext.ExecuteHostTaskAsync(
+            uri,
+            (scope) => Task.FromResult(scope?.Suggest(expression, cursorPosition)),
+            cancellationToken);
+        }
+
+        /// <summary>
+        /// Handles the completion operation and computes the completions.
+        /// </summary>
+        /// <param name="operationContext">Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            operationContext.Logger?.LogInformation($"[PFX] HandleCompletionRequest: id={operationContext.RequestId ?? "<null>"}, paramsJson={operationContext.RawOperationInput ?? "<null>"}");
+            if (!TryParseAndValidateParams(operationContext, out var completionParams))
+            {
+                operationContext.Logger?.LogError($"[PFX] HandleCompletionRequest: ParseError");
+                return;
+            }
+
+            var documentUri = new Uri(completionParams.TextDocument.Uri);
+            var expression = LanguageServerHelper.ChooseExpression(completionParams, HttpUtility.ParseQueryString(documentUri.Query));
+            if (expression == null)
+            {
+                operationContext.Logger?.LogError($"[PFX] HandleCompletionRequest: InvalidParams, expression is null");
+                operationContext.OutputBuilder.AddInvalidParamsError(operationContext.RequestId, "Failed to choose expression for completions operation");
+                return;
+            }
+
+            var cursorPosition = PositionRangeHelper.GetPosition(expression, completionParams.Position.Line, completionParams.Position.Character);
+
+            operationContext.Logger?.LogInformation($"[PFX] HandleCompletionRequest: calling Suggest...");
+            var results = await SuggestAsync(operationContext, completionParams.TextDocument.Uri, expression, cursorPosition, cancellationToken).ConfigureAwait(false);
+            if (results == null || results == default)
+            {
+                operationContext.OutputBuilder.AddInternalError(operationContext.RequestId, "Failed to get suggestions for completions operation");
+                return;
+            }
+
+            // Note: there are huge number of suggestions in initial requests
+            // Including each of them in the log string is very expensive
+            // Avoid this if possible
+            operationContext.Logger?.LogInformation($"[PFX] HandleCompletionRequest: Suggest results: Count:{results.Suggestions.Count()}, Suggestions:{string.Join(", ", results.Suggestions.Select(s => $@"[{s.Kind}]: '{s.DisplayText.Text}'"))}");
+
+            var precedingCharacter = cursorPosition != 0 ? expression[cursorPosition - 1] : '\0';
+            operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, new
+            {
+                items = results.Suggestions.Select((item, index) => new CompletionItem()
+                {
+                    Label = item.DisplayText.Text,
+                    Detail = item.FunctionParameterDescription,
+                    Documentation = item.Definition,
+                    Kind = GetCompletionItemKind(item.Kind),
+
+                    // The order of the results should be preserved.  To do that, we embed the index
+                    // into the sort text, which clients may sort lexicographically.
+                    SortText = index.ToString("D3", CultureInfo.InvariantCulture),
+
+                    // If the current position is in front of a single quote and the completion result starts with a single quote,
+                    // we don't want to make it harder on the end user by inserting an extra single quote.
+                    InsertText = item.DisplayText.Text is { } label && TexlLexer.IsIdentDelimiter(label[0]) && precedingCharacter == TexlLexer.IdentifierDelimiter ? label.Substring(1) : item.DisplayText.Text
+                }),
+                isIncomplete = false
+            });
+        }
+
+        /// <summary>
+        /// Attempts to parse and validate the completion params.
+        /// </summary>
+        /// <param name="operationContext">Operation Context.</param>
+        /// <param name="completionParams">Reference to capture successfully parsed completion params.</param>
+        /// <returns>True if parsing and validation was successful or false otherewise.</returns>
+        private static bool TryParseAndValidateParams(LanguageServerOperationContext operationContext, out CompletionParams completionParams)
+        {
+            return operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out completionParams);
+        }
+
+        /// <summary>
+        /// Maps the suggestion kind to the completion item kind.
+        /// </summary>
+        /// <param name="kind">Suggestion Kind.</param>
+        /// <returns>Mapped Completion Kind.</returns>
+        private static CompletionItemKind GetCompletionItemKind(SuggestionKind kind)
+        {
+            switch (kind)
+            {
+                case SuggestionKind.Function:
+                    return CompletionItemKind.Method;
+                case SuggestionKind.KeyWord:
+                    return CompletionItemKind.Keyword;
+                case SuggestionKind.Global:
+                    return CompletionItemKind.Variable;
+                case SuggestionKind.Field:
+                    return CompletionItemKind.Field;
+                case SuggestionKind.Alias:
+                    return CompletionItemKind.Variable;
+                case SuggestionKind.Enum:
+                    return CompletionItemKind.Enum;
+                case SuggestionKind.BinaryOperator:
+                    return CompletionItemKind.Operator;
+                case SuggestionKind.Local:
+                    return CompletionItemKind.Variable;
+                case SuggestionKind.ServiceFunctionOption:
+                    return CompletionItemKind.Method;
+                case SuggestionKind.Service:
+                    return CompletionItemKind.Module;
+                case SuggestionKind.ScopeVariable:
+                    return CompletionItemKind.Variable;
+                default:
+                    return CompletionItemKind.Text;
+            }
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Factories/BackwardsCompatibleNLHandlerFactory.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// A factory that always returns the same NLHandler instance.
+    /// Firstly, langauge server sdk took the nl handler to perform nl2fx and fx2nl from the host which was not thread safe.
+    /// So, a second approach using INlHandlerFactory was introduced to create a new instance of NLHandler for each request.
+    /// But with the new handler archritecture, we don't need both of them.
+    /// But we need to backwards compatible with the old code.
+    /// So instead of two backward compatible approaches, this factory unifies first and second so we only have one backwards compatible approach.
+    /// </summary>
+    internal class BackwardsCompatibleNLHandlerFactory : INLHandlerFactory
+    {
+        private readonly NLHandler _nlHandler;
+
+        public BackwardsCompatibleNLHandlerFactory(NLHandler nlHandler)
+        {
+            _nlHandler = nlHandler;
+        }
+
+        public NLHandler GetNLHandler(IPowerFxScope scope, BaseNLParams nlParams)
+        {
+            return _nlHandler;
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Factories/DefaultLanguageServerOperationHandlerFactory.cs

@@ -0,0 +1,58 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// A default and backwards compatible implementation of <see cref="ILanguageServerOperationHandlerFactory"/>.
+    /// </summary>
+    internal class DefaultLanguageServerOperationHandlerFactory : ILanguageServerOperationHandlerFactory
+    {
+        // The following properties are not in the creation context as they exist for backwards compatibility.
+        // if all hosts start using the new design, these properties can be removed.
+        private readonly INLHandlerFactory _nlHandlerFactory;
+        private readonly LanguageServer.NotifyDidChange _notifyDidChange;
+
+        public DefaultLanguageServerOperationHandlerFactory(INLHandlerFactory nlHandlerFactory, LanguageServer.NotifyDidChange notifyDidChange)
+        {
+            _nlHandlerFactory = nlHandlerFactory;
+            _notifyDidChange = notifyDidChange;
+        }
+
+        public ILanguageServerOperationHandler GetHandler(string lspMethod, HandlerCreationContext creationContext)
+        {
+            switch (lspMethod)
+            {
+                case CustomProtocolNames.NL2FX:
+                    return new Nl2FxLanguageServerOperationHandler(_nlHandlerFactory);
+                case CustomProtocolNames.FX2NL:
+                    return new Fx2NlLanguageServerOperationHandler(_nlHandlerFactory);
+                case CustomProtocolNames.GetCapabilities: 
+                    return new GetCustomCapabilitiesLanguageServerOperationHandler(_nlHandlerFactory);
+                case TextDocumentNames.Completion:
+                    return new CompletionsLanguageServerOperationHandler();
+                case TextDocumentNames.SignatureHelp:
+                    return new SignatureHelpLanguageServerOperationHandler();
+                case TextDocumentNames.RangeDocumentSemanticTokens:
+                    return new RangeSemanticTokensLanguageServerOperationHandler();
+                case TextDocumentNames.FullDocumentSemanticTokens:
+                    return new BaseSemanticTokensLanguageServerOperationHandler();
+                case TextDocumentNames.CodeAction:
+                    return new CodeActionsLanguageServerOperationHandler(creationContext.onLogUnhandledExceptionHandler);
+                case TextDocumentNames.DidChange:
+                    return new OnDidChangeLanguageServerNotificationHandler(_notifyDidChange);
+                case TextDocumentNames.DidOpen:
+                    return new OnDidOpenLanguageServerNotificationHandler();
+                case CustomProtocolNames.CommandExecuted:
+                    return new CommandExecutedLanguageServerOperationHandler();
+                case CustomProtocolNames.InitialFixup:
+                    return new InitialFixupLanguageServerOperationHandler();
+                default:
+                    return null;
+            }
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Factories/ILanguageServerOperationHandlerFactory.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Context for handler creation.
+    /// </summary>
+    /// <param name="onLogUnhandledExceptionHandler"> Handler for unhandled exceptions.</param>
+    internal record HandlerCreationContext(LanguageServer.OnLogUnhandledExceptionHandler onLogUnhandledExceptionHandler);
+
+    /// <summary>
+    /// Factory to get the handler for a given method.
+    /// </summary>
+    internal interface ILanguageServerOperationHandlerFactory
+    {
+        /// <summary>
+        /// Get the handler for the given method.
+        /// </summary>
+        /// <param name="lspMethod"> Lsp Method Identifer.</param>
+        /// <param name="creationContext"> Handler Creation Context.</param>
+        /// <returns>Handler for the given method.</returns>
+        ILanguageServerOperationHandler GetHandler(string lspMethod, HandlerCreationContext creationContext);
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Fx2Nl/Fx2NlLanguageServerOperationHandler.cs

@@ -0,0 +1,102 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{ 
+    /// <summary>
+    /// An abstract handler for Fx2Nl operations.
+    /// This is a class and not interface to allow us to add new methods in future without breaking existing implementations.
+    /// </summary>
+    internal sealed class Fx2NlLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public bool IsRequest => true;
+
+        public string LspMethod => CustomProtocolNames.FX2NL;
+
+        private readonly INLHandlerFactory _nLHandlerFactory;
+
+        private CheckResult _preHandleCheckResult;
+
+        private Fx2NLParameters _fx2NlParameters;
+
+        private CustomFx2NLParams _fx2NlRequestParams;
+
+        private NLHandler _nLHandler;
+
+        public Fx2NlLanguageServerOperationHandler(INLHandlerFactory nLHandlerFactory)
+        {
+            _nLHandlerFactory = nLHandlerFactory;
+        }
+
+        /// <summary>
+        /// Performs pre-handle operations for Fx2Nl.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        private Task PreHandleFx2NlAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            _nLHandler = operationContext.GetNLHandler(_fx2NlRequestParams.TextDocument.Uri, _nLHandlerFactory, _fx2NlRequestParams) ?? throw new NullReferenceException("No suitable handler found to handle Fx2Nl");
+            if (!_nLHandler.SupportsFx2NL)
+            {
+                throw new NotSupportedException("Fx2Nl is not supported");
+            }
+
+            return operationContext.ExecuteHostTaskAsync(
+            _fx2NlRequestParams.TextDocument.Uri,
+            (scope) =>
+            {
+                _preHandleCheckResult = scope?.Check(_fx2NlRequestParams.Expression) ?? throw new NullReferenceException("Check result was not found for Fx2NL operation");
+                _fx2NlParameters = GetFx2NlHints(scope, operationContext);
+            }, cancellationToken);
+        }
+
+        /// <summary>
+        /// Gets the Fx2Nl hints.
+        /// </summary>
+        /// <param name="scope"> PowerFx Scope.</param>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <returns>Fx2Nl hints.</returns>
+        private Fx2NLParameters GetFx2NlHints(IPowerFxScope scope,  LanguageServerOperationContext operationContext)
+        {
+            return scope is IPowerFxScopeFx2NL fx2NLScope ? fx2NLScope.GetFx2NLParameters() : new Fx2NLParameters();
+        }
+
+        /// <summary>
+        /// Performs core Fx2Nl operation.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        private async Task Fx2NlAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            var result = await _nLHandler.Fx2NLAsync(_preHandleCheckResult, _fx2NlParameters, cancellationToken).ConfigureAwait(false);
+            operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, result);
+        }
+
+        /// <summary>
+        /// Orchestrates and handles the Fx2Nl operation.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out _fx2NlRequestParams))
+            {
+                return;
+            }
+
+            await PreHandleFx2NlAsync(operationContext, cancellationToken).ConfigureAwait(false);
+            if (_preHandleCheckResult == null)
+            {
+                return;
+            }
+
+            await Fx2NlAsync(operationContext, cancellationToken).ConfigureAwait(false);
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/GetCustomCapibilities/GetCustomCapabilitiesLanguageServerOperationHandler.cs

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Handler to handle the GetCustomCapabilities request.
+    /// This is backwards compatible as it used the NLHandlerFactory to get the NLHandler.
+    /// This should ideally be replaced with the LSP initialization protocol.
+    /// </summary>
+    internal sealed class GetCustomCapabilitiesLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public bool IsRequest => true;
+
+        public string LspMethod => CustomProtocolNames.GetCapabilities;
+
+        private readonly INLHandlerFactory _nlHandlerFactory;
+
+        public GetCustomCapabilitiesLanguageServerOperationHandler(INLHandlerFactory nlHandlerFactory)
+        {
+            _nlHandlerFactory = nlHandlerFactory;
+        }
+
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out CustomGetCapabilitiesParams customGetCapabilitiesParams))
+            {
+                operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, new CustomGetCapabilitiesResult());
+                return;
+            }
+
+            var nlHandler = operationContext.GetNLHandler(customGetCapabilitiesParams.TextDocument.Uri, _nlHandlerFactory, null);
+            if (nlHandler == null)
+            {
+                operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, new CustomGetCapabilitiesResult());
+                return;
+            }
+
+            operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, new CustomGetCapabilitiesResult() { SupportsNL2Fx = nlHandler.SupportsNL2Fx, SupportsFx2NL = nlHandler.SupportsFx2NL });
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/ILanguageServerOperationHandler.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// A interface representing a handler for a language server operation.
+    /// </summary>
+    internal interface ILanguageServerOperationHandler
+    {
+        /// <summary>
+        /// Indicates if the operation is a request.
+        /// </summary>
+        bool IsRequest { get; }
+
+        /// <summary>
+        /// The LSP method that this handler is for.
+        /// </summary>
+        string LspMethod { get; }
+
+        /// <summary>
+        /// Asynchronously handles the operation.
+        /// This function by design doesn't return value. 
+        /// The operation context has a output builder and the handler should use it to write the response.
+        /// </summary>
+        /// <param name="operationContext">Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken);
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/InitialFixup/InitialFixupLanguageServerOperationHandler.cs

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Handles the initial fixup operation.
+    /// </summary>
+    internal sealed class InitialFixupLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public bool IsRequest => true;
+
+        public string LspMethod => CustomProtocolNames.InitialFixup;
+
+        /// <summary>
+        /// Handles the initial fixup operation.
+        /// </summary>
+        /// <param name="operationContext"> Language Server Operation Context. </param>
+        /// <param name="cancellationToken"> Cancellation Token. </param>   
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            operationContext.Logger?.LogInformation($"[PFX] HandleInitialFixupRequest: id={operationContext.RequestId ?? "<null>"}, paramsJson={operationContext.RawOperationInput ?? "<null>"}");
+
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out InitialFixupParams requestParams))
+            {
+                return;
+            }
+
+            var expression = await operationContext.ExecuteHostTaskAsync(requestParams.TextDocument.Uri, (scope) => Task.FromResult(scope?.ConvertToDisplay(requestParams.TextDocument.Text)), cancellationToken, defaultOutput: string.Empty).ConfigureAwait(false);
+
+            operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, new TextDocumentItem()
+            {
+                Uri = requestParams.TextDocument.Uri,
+                Text = expression
+            });
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/LanguageServerOperationContext.cs

@@ -0,0 +1,154 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Core;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// A context for a language server operation with information that might be needed when handling the operation.
+    /// It also has helper methods to simplify the operation handling.
+    /// Lifetime of this context is per request.
+    /// </summary>
+    public class LanguageServerOperationContext
+    {
+        /// <summary>
+        /// A factory to create a scope for the operation.
+        /// </summary>
+        private readonly IPowerFxScopeFactory _scopeFactory;
+
+        public LanguageServerOperationContext(IPowerFxScopeFactory scopeFactory)
+        {
+            _scopeFactory = scopeFactory;
+        }
+
+        /// <summary>
+        /// Language Server Method Identifier for which this context is created.
+        /// </summary>
+        public string LspMethod { get; init; }
+
+        /// <summary>
+        /// Language Server Logger.
+        /// </summary>
+        internal ILanguageServerLogger Logger { get; init; }
+
+        /// <summary>
+        /// ID of the request. This won't be relevant for notification handlers.
+        /// </summary>
+        public string RequestId { get; init; }
+        
+        /// <summary>
+        /// Raw input of the operation. Usually this is passed from the client.
+        /// </summary>
+        public string RawOperationInput { get; init; }
+
+        /// <summary>
+        /// A builder to create the output of the operation.
+        /// No handlers are expected to return a value. They should use this builder to write the response.
+        /// A handler can write multiple responses and the builder will take care of the correct format.
+        /// </summary>
+        internal LanguageServerOutputBuilder OutputBuilder { get; init; }
+
+        /// <summary>
+        /// Host Task Executor to run tasks in host environment.
+        /// </summary>
+        internal IHostTaskExecutor HostTaskExecutor { get; init; }
+
+        private IPowerFxScope _scope = null;
+
+        /// <summary>
+        /// Gets or creates the scope for the operation.
+        /// </summary>
+        /// <param name="uri">Uri to use for scope creation.</param>
+        /// <returns>Scope.</returns>
+        private IPowerFxScope GetScope(string uri)
+        {
+            return _scope ??= _scopeFactory.GetOrCreateInstance(uri);
+        }
+
+        /// <summary>
+        /// Get the NLHandler for the given uri and factory.
+        /// </summary>
+        /// <param name="uri">Uri to use for scope creation.</param>
+        /// <param name="factory">Nl Handler Factory.</param>
+        /// <param name="nLParams">Ml Params.</param>
+        /// <returns>NlHandler Instance.</returns>
+        internal NLHandler GetNLHandler(string uri, INLHandlerFactory factory, BaseNLParams nLParams = null)
+        {
+            return factory.GetNLHandler(GetScope(uri), nLParams);
+        }
+
+        /// <summary>
+        ///  A helper method to execute a task in host environment if host task executor is available.
+        ///  This is critical to trust between LSP and its hosts.
+        ///  LSP should correctly use this and wrap particulat steps that need to run in host using these methods. 
+        /// </summary>
+        /// <typeparam name="TOutput">Output Type.</typeparam>
+        /// <param name="uri">Uri to use for scope creation.</param>
+        /// <param name="task">Task to run inside host.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        /// <param name="defaultOutput"> Default Output if task is canceled by the host.</param>
+        /// <returns>Output.</returns>
+        internal async Task<TOutput> ExecuteHostTaskAsync<TOutput>(string uri, Func<IPowerFxScope, Task<TOutput>> task, CancellationToken cancellationToken, TOutput defaultOutput = default)
+        {
+            if (HostTaskExecutor == null)
+            {
+                return await task(GetScope(uri)).ConfigureAwait(false);
+            }
+
+            Task<TOutput> WrappedTask() => task(GetScope(uri));
+            return await HostTaskExecutor.ExecuteTaskAsync(WrappedTask, this, cancellationToken, defaultOutput).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        ///  A helper method to execute a task in host environment if host task executor is available.
+        ///  This is critical to trust between LSP and its hosts.
+        ///  LSP should correctly use this and wrap particulat steps that need to run in host using these methods. 
+        /// </summary>
+        /// <param name="uri">Uri to use for scope creation.</param>
+        /// <param name="task">Task to run inside host.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        internal async Task ExecuteHostTaskAsync(string uri, Action<IPowerFxScope> task, CancellationToken cancellationToken)
+        {
+            if (HostTaskExecutor == null)
+            {
+                task(GetScope(uri));
+                return;
+            }
+
+            await HostTaskExecutor.ExecuteTaskAsync(
+            () =>
+            {
+                task(GetScope(uri));
+                return Task.FromResult<object>(null);
+            }, this, 
+            cancellationToken).ConfigureAwait(false);
+        }
+    }
+
+    internal static class LanguageServerOperationContextExtensions
+    {
+        /// <summary>
+        /// A helper method to help parse the raw input of the operation and add an error response if parsing fails.
+        /// </summary>
+        /// <typeparam name="T">Deserialized and parsed type.</typeparam>
+        /// <param name="context">Language Server Operation Context.</param>
+        /// <param name="parsedParams">Reference to hold the deserialized/parsed params.</param>
+        /// <returns>True if parsing was successful or false otherwise.</returns>
+        public static bool TryParseParamsAndAddErrorResponseIfNeeded<T>(this LanguageServerOperationContext context, out T parsedParams)
+        {
+            if (!LanguageServerHelper.TryParseParams(context.RawOperationInput, out parsedParams))
+            {
+                context.OutputBuilder.AddParseError(context.RequestId, $"Cannot parse the params for method: ${context.LspMethod}");
+                return false;
+            }
+
+            return true;
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Nl2Fx/Nl2FxLanguageServerOperationHandler.cs

@@ -0,0 +1,136 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// An abstract handler for NL2FX operations.
+    /// This is a class and not interface to allow us to add new methods in future without breaking existing implementations.
+    /// </summary>
+    internal sealed class Nl2FxLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public bool IsRequest => true;
+
+        public string LspMethod => CustomProtocolNames.NL2FX;
+
+        private readonly INLHandlerFactory _nLHandlerFactory;
+
+        private CustomNL2FxParams _nl2FxRequestParams;
+
+        private NLHandler _nlHandler;
+
+        private CustomNL2FxResult _nl2FxResult;
+
+        private NL2FxParameters _nl2FxParameters;
+
+        public Nl2FxLanguageServerOperationHandler(INLHandlerFactory nLHandlerFactory)
+        {
+            _nLHandlerFactory = nLHandlerFactory;
+        }
+
+        /// <summary>
+        /// Performs pre-handle operations for NL2FX.
+        /// </summary>
+        /// <param name="scope"> PowerFx Scope. </param>
+        /// <param name="operationContext"> Language Server Operation Context. </param>
+        private void PreHandleNl2Fx(IPowerFxScope scope, LanguageServerOperationContext operationContext)
+        {
+            _nlHandler = operationContext.GetNLHandler(_nl2FxRequestParams.TextDocument.Uri, _nLHandlerFactory, _nl2FxRequestParams) ?? throw new NullReferenceException("No suitable handler found to handle Nl2Fx");
+            if (!_nlHandler.SupportsNL2Fx)
+            {
+                throw new NotSupportedException("Nl2fx is not supported");
+            }
+
+            var check = scope?.Check(LanguageServer.Nl2FxDummyFormula) ?? throw new NullReferenceException("Check result was not found for NL2Fx operation");
+            var summary = check.ApplyGetContextSummary();
+            _nl2FxParameters = new NL2FxParameters
+            {
+                Sentence = _nl2FxRequestParams.Sentence,
+                SymbolSummary = summary,
+                Engine = check.Engine
+            };
+         }
+
+        /// <summary>
+        /// Performs Core NL2FX operation.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        /// <returns> Nl2Fx Handle Context extended with NL2Fx result. </returns>
+        private async Task Nl2FxAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            _nl2FxResult = await _nlHandler.NL2FxAsync(_nl2FxParameters, cancellationToken).ConfigureAwait(false);
+        }
+
+        /// <summary>
+        /// Performs post-handle operations for NL2FX.
+        /// </summary>
+        /// <param name="scope">PowerFx Scope.</param>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        private void PostHandleNl2FxResults(IPowerFxScope scope, LanguageServerOperationContext operationContext)
+        {
+            var nl2FxResult = _nl2FxResult;
+            if (nl2FxResult?.Expressions != null)
+            {
+                foreach (var item in nl2FxResult.Expressions)
+                {
+                    if (item?.Expression == null)
+                    {
+                        continue;
+                    }
+
+                    var check = scope?.Check(item.Expression);
+                    if (check != null && !check.IsSuccess)
+                    {
+                        item.RawExpression = item.Expression;
+                        item.Expression = null;
+                    }
+
+                    item.AnonymizedExpression = check?.ApplyGetLogging();
+                }
+            }
+
+            operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, nl2FxResult);
+        }
+
+        /// <summary>
+        /// Orchestrates and handles the NL2Fx operation.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out _nl2FxRequestParams))
+            {
+                return;
+            }
+
+            await operationContext.ExecuteHostTaskAsync(
+            _nl2FxRequestParams.TextDocument.Uri,   
+            (scope) => 
+            {
+                PreHandleNl2Fx(scope, operationContext);
+                _nlHandler.PreHandleNl2Fx(_nl2FxRequestParams, _nl2FxParameters, operationContext);
+            }, 
+            cancellationToken).ConfigureAwait(false);
+            if (_nl2FxParameters == null)
+            {
+                return;
+            }
+
+            await Nl2FxAsync(operationContext, cancellationToken).ConfigureAwait(false);
+            if (_nl2FxResult == null)
+            {
+                return;
+            }
+
+            await operationContext.ExecuteHostTaskAsync(_nl2FxRequestParams.TextDocument.Uri, (scope) => PostHandleNl2FxResults(scope, operationContext), cancellationToken).ConfigureAwait(false);
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Notifications/OnDidChangeLanguageServerNotificationHandler.cs

@@ -0,0 +1,64 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Handler to handle the DidChangeTextDocument notification.
+    /// </summary>
+    internal sealed class OnDidChangeLanguageServerNotificationHandler : ILanguageServerOperationHandler
+    {
+        public string LspMethod => TextDocumentNames.DidChange;
+
+        public bool IsRequest => false;
+
+        private readonly LanguageServer.NotifyDidChange _notifyDidChange;
+
+        public OnDidChangeLanguageServerNotificationHandler(LanguageServer.NotifyDidChange notifyDidChange)
+        {
+            _notifyDidChange = notifyDidChange;
+        }
+
+        /// <summary>
+        /// Handles the DidChangeTextDocument notification.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            operationContext.Logger?.LogInformation($"[PFX] HandleDidChangeNotification: paramsJson={operationContext.RawOperationInput ?? "<null>"}");
+
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out DidChangeTextDocumentParams didChangeParams))
+            {
+                return;
+            }
+
+            if (didChangeParams.ContentChanges.Length != 1)
+            {
+                return;
+            }
+
+            var documentUri = didChangeParams.TextDocument.Uri;
+
+            var expression = didChangeParams.ContentChanges[0].Text;
+            await operationContext.ExecuteHostTaskAsync(
+            documentUri,
+            (scope) =>
+            {
+                _notifyDidChange?.Invoke(didChangeParams);
+                var checkResult = scope?.Check(expression);
+
+                operationContext.OutputBuilder.WriteDiagnosticsNotification(didChangeParams.TextDocument.Uri, expression, checkResult.Errors.ToArray());
+
+                operationContext.OutputBuilder.WriteTokensNotification(didChangeParams.TextDocument.Uri, checkResult);
+
+                operationContext.OutputBuilder.WriteExpressionTypeNotification(didChangeParams.TextDocument.Uri, checkResult);
+            }, cancellationToken).ConfigureAwait(false);
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/Notifications/OnDidOpenLanguageServerNotificationHandler.cs

@@ -0,0 +1,52 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Handler to handle the DidOpen notification from the client.
+    /// </summary>
+    internal sealed class OnDidOpenLanguageServerNotificationHandler : ILanguageServerOperationHandler
+    {
+        public string LspMethod => TextDocumentNames.DidOpen;
+
+        public bool IsRequest => false;
+
+        /// <summary>
+        /// Handles the DidOpen notification from the client.
+        /// </summary>
+        /// <param name="operationContext"> Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            operationContext.Logger?.LogInformation($"[PFX] HandleDidOpenNotification: paramsJson={operationContext.RawOperationInput ?? "<null>"}");
+
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out DidOpenTextDocumentParams didOpenParams))
+            {
+                return;
+            }
+
+            await operationContext.ExecuteHostTaskAsync(
+            didOpenParams.TextDocument.Uri,
+            (scope) => 
+            {
+                var checkResult = scope?.Check(didOpenParams.TextDocument.Text);
+                if (checkResult == null)
+                {
+                    return;
+                }
+
+                operationContext.OutputBuilder.WriteDiagnosticsNotification(didOpenParams.TextDocument.Uri, didOpenParams.TextDocument.Text, checkResult.Errors.ToArray());
+
+                operationContext.OutputBuilder.WriteTokensNotification(didOpenParams.TextDocument.Uri, checkResult);
+
+                operationContext.OutputBuilder.WriteExpressionTypeNotification(didOpenParams.TextDocument.Uri, checkResult);
+            }, cancellationToken).ConfigureAwait(false);
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/SemanticTokens/BaseSemanticTokensLanguageServerOperationHandler.cs

@@ -0,0 +1,225 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web;
+using Microsoft.PowerFx.Core.Texl.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Schemas;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Handles all kinds of semantic tokens operations.
+    /// </summary>
+    internal class BaseSemanticTokensLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public virtual string LspMethod => TextDocumentNames.FullDocumentSemanticTokens;
+
+        public bool IsRequest => true;
+
+        private bool IsRangeSemanticTokens => LspMethod == TextDocumentNames.RangeDocumentSemanticTokens;
+
+        public BaseSemanticTokensLanguageServerOperationHandler()
+        {
+        }
+
+        /// <summary>
+        /// Computes the semantic tokens for the given expression.
+        /// Hook to allow consumers to override the token computation.
+        /// </summary>
+        /// <param name="operationContext"> Language Server Operation Context. </param>
+        /// <param name="getTokensContext"> Context that might be needed to compute tokens. </param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        /// <returns>A set of semantic tokens.</returns>
+        private Task<IEnumerable<ITokenTextSpan>> GetTokensAsync(LanguageServerOperationContext operationContext, GetTokensContext getTokensContext, CancellationToken cancellationToken)
+        {
+            return operationContext.ExecuteHostTaskAsync(
+            getTokensContext.documentUri,
+            (scope) =>
+            {
+                var result = scope?.Check(getTokensContext.expression);
+                if (result == null)
+                {
+                    return Task.FromResult(Enumerable.Empty<ITokenTextSpan>());
+                }
+
+                return Task.FromResult(result.GetTokens(getTokensContext.tokenTypesToSkip));
+            }, cancellationToken,
+            defaultOutput: Enumerable.Empty<ITokenTextSpan>());
+        }
+
+        /// <summary>
+        /// Handles publishing a control token notification if any control tokens found.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="controlTokensObj">Collection to add control tokens to.</param>
+        /// <param name="queryParams">Collection of query params.</param>
+        private void PublishControlTokenNotification(LanguageServerOperationContext operationContext, ControlTokens controlTokensObj, NameValueCollection queryParams)
+        {
+            if (controlTokensObj == null || queryParams == null)
+            {
+                return;
+            }
+
+            var version = queryParams.Get("version") ?? string.Empty;
+
+            operationContext.OutputBuilder.AddNotification(CustomProtocolNames.PublishControlTokens, new PublishControlTokensParams()
+            {
+                Version = version,
+                Controls = controlTokensObj
+            });
+        }
+
+        /// <summary>
+        /// Handles the semantic tokens operation.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            var isRangeSemanticTokens = IsRangeSemanticTokens;
+            operationContext.Logger?.LogInformation($"[PFX] {(isRangeSemanticTokens ? "HandleRangeSemanticTokens" : "HandleFullDocumentSemanticTokens")}: id={operationContext.RequestId ?? "<null>"}, paramsJson={operationContext.RawOperationInput ?? "<null>"}");
+
+            if (!TryParseAndValidateSemanticTokenParams(operationContext, out var semanticTokensParams))
+            {
+                return;
+            }
+
+            var uri = new Uri(semanticTokensParams.TextDocument.Uri);
+            var queryParams = HttpUtility.ParseQueryString(uri.Query);
+            var expression = LanguageServerHelper.ChooseExpression(semanticTokensParams, queryParams) ?? string.Empty;
+
+            if (string.IsNullOrWhiteSpace(expression))
+            {
+                // Empty tokens for the empty expression
+                WriteEmptySemanticTokensResponse(operationContext);
+                return;
+            }
+
+            // Monaco-Editor sometimes uses \r\n for the newline character. \n is not always the eol character so allowing clients to pass eol character
+            var eol = queryParams?.Get("eol");
+            eol = !string.IsNullOrEmpty(eol) ? eol : PositionRangeHelper.EOL.ToString();
+
+            var startIndex = -1;
+            var endIndex = -1;
+            if (isRangeSemanticTokens)
+            {
+                (startIndex, endIndex) = (semanticTokensParams as SemanticTokensRangeParams).Range.ConvertRangeToPositions(expression, eol);
+                if (startIndex < 0 || endIndex < 0)
+                {
+                    WriteEmptySemanticTokensResponse(operationContext);
+                    return;
+                }
+            }
+
+            var tokenTypesToSkip = ParseTokenTypesToSkipParam(queryParams?.Get("tokenTypesToSkip"));
+            var tokens = await GetTokensAsync(operationContext, new GetTokensContext(tokenTypesToSkip, semanticTokensParams.TextDocument.Uri, expression), cancellationToken).ConfigureAwait(false);
+
+            if (tokens == null || !tokens.Any())
+            {
+                WriteEmptySemanticTokensResponse(operationContext);
+                return;
+            }
+
+            if (isRangeSemanticTokens)
+            {
+                // Only consider overlapping tokens. end index is exlcusive
+                tokens = tokens.Where(token => !(token.EndIndex <= startIndex || token.StartIndex >= endIndex));
+            }
+
+            var controlTokensObj = !isRangeSemanticTokens ? new ControlTokens() : null;
+
+            var encodedTokens = SemanticTokensEncoder.EncodeTokens(tokens, expression, eol, controlTokensObj);
+            operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, new SemanticTokensResponse() { Data = encodedTokens });
+            PublishControlTokenNotification(operationContext, controlTokensObj, queryParams);
+        }
+
+        /// <summary>
+        /// Attempts to parse and validate the semantic token parameters.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="semanticTokenParams">Reference to hold the parsed semantic token params.</param>
+        /// <returns>True if parsing and validation was successful or false otherwise.</returns>
+        private bool TryParseAndValidateSemanticTokenParams(LanguageServerOperationContext operationContext, out SemanticTokensParams semanticTokenParams)
+        {
+            semanticTokenParams = null;
+            SemanticTokensRangeParams semanticTokensRangeParams = null;
+
+            var parseResult = false;
+            if (IsRangeSemanticTokens)
+            {
+                parseResult = operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out semanticTokensRangeParams);
+                semanticTokenParams = semanticTokensRangeParams;
+            }
+            else
+            {
+                parseResult = operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out semanticTokenParams);
+            }
+
+            if (!parseResult)
+            {
+                return false;
+            }
+
+            if (string.IsNullOrWhiteSpace(semanticTokenParams?.TextDocument.Uri))
+            {
+                operationContext.OutputBuilder.AddParseError(operationContext.RequestId, "Invalid document uri for semantic tokens operation");
+                return false;
+            }
+
+            if (IsRangeSemanticTokens && semanticTokensRangeParams?.Range == null)
+            {
+                WriteEmptySemanticTokensResponse(operationContext);
+                return false;
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Parses the token types to skip parameter.
+        /// These are the token types that should be skipped from the semantic tokens response.
+        /// </summary>
+        /// <param name="rawTokenTypesToSkipParam">Raw token types to skip parameter.</param>
+        /// <returns>Parsed token types to skip parameter.</returns>
+        private static HashSet<TokenType> ParseTokenTypesToSkipParam(string rawTokenTypesToSkipParam)
+        {
+            var tokenTypesToSkip = new HashSet<TokenType>();
+            if (string.IsNullOrWhiteSpace(rawTokenTypesToSkipParam))
+            {
+                return tokenTypesToSkip;
+            }
+
+            if (LanguageServerHelper.TryParseParams(rawTokenTypesToSkipParam, out List<int> tokenTypesToSkipParam))
+            {
+                foreach (var tokenTypeValue in tokenTypesToSkipParam)
+                {
+                    var tokenType = (TokenType)tokenTypeValue;
+                    if (tokenType != TokenType.Lim)
+                    {
+                        tokenType = tokenType == TokenType.Min ? TokenType.Unknown : tokenType;
+                        tokenTypesToSkip.Add(tokenType);
+                    }
+                }
+            }
+
+            return tokenTypesToSkip;
+        }
+
+        /// <summary>
+        /// Adds the empty semantic tokens response to the output builder.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        private static void WriteEmptySemanticTokensResponse(LanguageServerOperationContext operationContext)
+        {
+           operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, new SemanticTokensResponse());
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/SemanticTokens/RangeSemanticTokensLanguageServerOperationHandler.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    internal sealed class RangeSemanticTokensLanguageServerOperationHandler : BaseSemanticTokensLanguageServerOperationHandler
+    {
+        public override string LspMethod => TextDocumentNames.RangeDocumentSemanticTokens;
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/SemanticTokens/SemanticTokensDTOs.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using Microsoft.PowerFx.Core.Texl.Intellisense;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Small DTO to hold the context needed for the GetTokens operation.
+    /// <param name="tokenTypesToSkip">The token types to skip.</param>
+    /// <param name="documentUri">The semantic tokens parameters.</param>
+    /// <param name="expression">The expression to get tokens for.</param>
+    /// </summary>
+    internal record GetTokensContext(HashSet<TokenType> tokenTypesToSkip, string documentUri, string expression);
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Handlers/SignatureHelp/SignatureHelpLanguageServerOperationHandler.cs

@@ -0,0 +1,77 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Web;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol.Handlers
+{
+    /// <summary>
+    /// Handler for the signature help operation.
+    /// Currently, Language Server SDK delegates full signature help logic to the host.
+    /// It only does validation of the input and then transforms the results to the LSP format.
+    /// Those are not needed to be exposed as overridable methods/hooks.
+    /// Therefore, there's only one HandleAsync method.
+    /// </summary>
+    internal sealed class SignatureHelpLanguageServerOperationHandler : ILanguageServerOperationHandler
+    {
+        public string LspMethod => TextDocumentNames.SignatureHelp;
+
+        public bool IsRequest => true;
+
+        /// <summary>
+        /// Provides the signature help for the given expression.
+        /// </summary>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="uri">Document Uri.</param>
+        /// <param name="expression">Expression.</param>
+        /// <param name="cursorPosition">Cursor Position.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        /// <returns>Suggestions and Signatures.</returns>
+        private Task<IIntellisenseResult> SuggestAsync(LanguageServerOperationContext operationContext, string uri, string expression, int cursorPosition, CancellationToken cancellationToken)
+        {
+            return operationContext.ExecuteHostTaskAsync(
+            uri,
+            (scope) => Task.FromResult(scope?.Suggest(expression, cursorPosition)),
+            cancellationToken);
+        }
+
+        /// <summary>
+        /// Handles the signature help operation and computes the signature help.
+        /// </summary>
+        /// <param name="operationContext">Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            operationContext.Logger?.LogInformation($"[PFX] HandleSignatureHelpRequest: id={operationContext.RequestId ?? "<null>"}, paramsJson={operationContext.RawOperationInput ?? "<null>"}");
+
+            if (!operationContext.TryParseParamsAndAddErrorResponseIfNeeded(out SignatureHelpParams signatureHelpParams))
+            {
+                return;
+            }
+
+            var documentUri = new Uri(signatureHelpParams.TextDocument.Uri);
+            var expression = LanguageServerHelper.ChooseExpression(signatureHelpParams, HttpUtility.ParseQueryString(documentUri.Query));
+            if (expression == null)
+            {
+                operationContext.OutputBuilder.AddInvalidParamsError(operationContext.RequestId, "Failed to choose expression for singature help operation");
+                return;
+            }
+
+            var cursorPosition = PositionRangeHelper.GetPosition(expression, signatureHelpParams.Position.Line, signatureHelpParams.Position.Character);
+            var results = await SuggestAsync(operationContext, signatureHelpParams.TextDocument.Uri, expression, cursorPosition, cancellationToken).ConfigureAwait(false);
+            if (results == null || results == default)
+            {
+                operationContext.OutputBuilder.AddInternalError(operationContext.RequestId, "Failed to get suggestions for signature help operation");
+                return;
+            }
+
+            var signatureHelp = new SignatureHelp(results.SignatureHelp);
+            operationContext.OutputBuilder.AddSuccessResponse(operationContext.RequestId, signatureHelp);
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Helpers/DiagnosticsHelper.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using Microsoft.PowerFx.Core.Errors;
+using Microsoft.PowerFx.Core.Utils;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Syntax;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// Contains helper methods for managing diagnostics.
+    /// </summary>
+    internal static class DiagnosticsHelper
+    {
+        /// <summary>
+        /// Writes a diagnostics notification to the builder.
+        /// </summary>
+        /// <param name="builder">Output Builder.</param>
+        /// <param name="uri">Document Uri.</param>
+        /// <param name="expression">Expression.</param>
+        /// <param name="errors">Errors.</param>
+        public static void WriteDiagnosticsNotification(this LanguageServerOutputBuilder builder, string uri, string expression, ExpressionError[] errors)
+        {
+            builder.AddNotification(TextDocumentNames.PublishDiagnostics, CreateDiagnosticsNotification(uri, expression, errors));
+        }
+
+        /// <summary>
+        /// Creates a diagnostics notification from the given information.
+        /// </summary>
+        /// <param name="uri">Document Uri.</param>
+        /// <param name="expression">Expression.</param>
+        /// <param name="errors">Errors.</param>
+        /// <returns>Diagnostics Notification.</returns>
+        public static PublishDiagnosticsParams CreateDiagnosticsNotification(string uri, string expression, ExpressionError[] errors)
+        {
+            Contracts.AssertNonEmpty(uri);
+            Contracts.AssertValue(expression);
+
+            var diagnostics = new List<Diagnostic>();
+            if (errors != null)
+            {
+                foreach (var item in errors)
+                {
+                    var span = item.Span ?? new Span(0, 0);
+                    diagnostics.Add(new Diagnostic()
+                    {
+                        Range = span.ConvertSpanToRange(expression),
+                        Message = item.Message,
+                        Severity = DocumentSeverityToDiagnosticSeverityMap(item.Severity)
+                    });
+                }
+            }
+
+            return new PublishDiagnosticsParams()
+            {
+                Uri = uri,
+                Diagnostics = diagnostics.ToArray()
+            };
+        }
+
+        /// <summary>
+        /// PowerFx classifies diagnostics by <see cref="DocumentErrorSeverity"/>, LSP classifies them by
+        /// <see cref="DiagnosticSeverity"/>. This method maps the former to the latter.
+        /// </summary>
+        /// <param name="severity">
+        /// <see cref="DocumentErrorSeverity"/> which will be mapped to the LSP eequivalent.
+        /// </param>
+        /// <returns>
+        /// <see cref="DiagnosticSeverity"/> equivalent to <see cref="DocumentErrorSeverity"/>.
+        /// </returns>
+        private static DiagnosticSeverity DocumentSeverityToDiagnosticSeverityMap(ErrorSeverity severity) => severity switch
+        {
+            ErrorSeverity.Critical => DiagnosticSeverity.Error,
+            ErrorSeverity.Severe => DiagnosticSeverity.Error,
+            ErrorSeverity.Moderate => DiagnosticSeverity.Error,
+            ErrorSeverity.Warning => DiagnosticSeverity.Warning,
+            ErrorSeverity.Suggestion => DiagnosticSeverity.Hint,
+            ErrorSeverity.Verbose => DiagnosticSeverity.Information,
+            _ => DiagnosticSeverity.Information
+        };
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Helpers/LanguageServerHelper.cs

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Specialized;
+using System.Text.Json;
+using Microsoft.PowerFx.Core.Utils;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// Helper class for Language Server.
+    /// </summary>
+    internal static class LanguageServerHelper
+    {
+        internal static readonly JsonSerializerOptions DefaultJsonSerializerOptions = new ()
+        {
+            PropertyNameCaseInsensitive = true
+        };
+
+        /// <summary>
+        /// Choose expression from multiple options (old or new).
+        /// We used to pass expression in query params, but now we pass it in the request body.
+        /// </summary>
+        /// <param name="baseParams">Base Request params. </param>
+        /// <param name="queryParams">Query Params from document uri.</param>
+        /// <returns>Chosen Expression.</returns>
+        internal static string ChooseExpression(LanguageServerRequestBaseParams baseParams, NameValueCollection queryParams)
+        {
+            return baseParams?.Text ?? queryParams.Get("expression");
+        }
+
+        /// <summary>
+        ///  Attempt to parse the given params into the given type.
+        /// </summary>
+        /// <typeparam name="T">Type of the deserialized params.</typeparam>
+        /// <param name="paramsToParse">Params to be parsed.</param>
+        /// <param name="result">Reference to store the deserialized or parsed params.</param>
+        /// <returns>True if params were parsed successfully or false otherwise.</returns>
+        internal static bool TryParseParams<T>(string paramsToParse, out T result)
+        {
+            Contracts.AssertNonEmpty(paramsToParse);
+
+            try
+            {
+                result = JsonSerializer.Deserialize<T>(paramsToParse, DefaultJsonSerializerOptions);
+                return true;
+            }
+            catch
+            {
+                result = default;
+                return false;
+            }
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Helpers/NotificationHelper.cs

@@ -0,0 +1,98 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Web;
+using Microsoft.PowerFx.Core.Public;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// A helper class to help with creating and writing notifications.
+    /// </summary>
+    internal static class NotificationHelper
+    {
+        /// <summary>
+        /// Create PublishTokens notification and writes it to the builder. 
+        /// Note: This is a legacy notification, and should be replaced by semantic tokens in all hosts.
+        /// </summary>
+        /// <param name="builder">Language Server Output Builder.</param>
+        /// <param name="documentUri">Document Uri.</param>
+        /// <param name="result">Check Result.</param>
+        public static void WriteTokensNotification(this LanguageServerOutputBuilder builder, string documentUri, CheckResult result)
+        {
+            var notification = CreateTokensNotification(documentUri, result);
+            if (notification != null)
+            {
+                builder.AddNotification(CustomProtocolNames.PublishTokens, notification);
+            }
+        }
+
+        /// <summary>
+        /// Create PublishTokens notification. 
+        /// Note: This is a legacy notification, and should be replaced by semantic tokens in all hosts.
+        /// </summary>
+        /// <param name="documentUri">Document Uri.</param>
+        /// <param name="result">Check Result.</param>
+        /// <returns>PublishTokensParams.</returns>
+        public static PublishTokensParams CreateTokensNotification(string documentUri, CheckResult result)
+        {
+            var uri = new Uri(documentUri);
+            var nameValueCollection = HttpUtility.ParseQueryString(uri.Query);
+            if (!uint.TryParse(nameValueCollection.Get("getTokensFlags"), out var flags))
+            {
+                return null;
+            }
+
+            var tokens = result.GetTokens((GetTokensFlags)flags);
+            if (tokens == null || tokens.Count == 0)
+            {
+                return null;
+            }
+
+            return new PublishTokensParams()
+            {
+                Uri = documentUri,
+                Tokens = tokens
+            };
+        }
+
+        /// <summary>
+        /// Create PublishExpressionType notification and writes it to the builder.
+        /// </summary>
+        /// <param name="builder">Language Server Output Builder.</param>
+        /// <param name="documentUri">Document Uri.</param>
+        /// <param name="result">Check Result.</param>
+        public static void WriteExpressionTypeNotification(this LanguageServerOutputBuilder builder, string documentUri, CheckResult result)
+        {
+            var notification = CreateExpressionTypeNotification(documentUri, result);
+            if (notification != null)
+            {
+                builder.AddNotification(CustomProtocolNames.PublishExpressionType, notification);
+            }
+        }
+
+        /// <summary>
+        /// Create PublishExpressionType notification.
+        /// </summary>
+        /// <param name="documentUri">Document Uri.</param>
+        /// <param name="result">Check Result.</param>
+        /// <returns>Expression Type Notification.</returns>
+        public static PublishExpressionTypeParams CreateExpressionTypeNotification(string documentUri, CheckResult result)
+        {
+            var uri = new Uri(documentUri);
+            var nameValueCollection = HttpUtility.ParseQueryString(uri.Query);
+            if (!bool.TryParse(nameValueCollection.Get("getExpressionType"), out var enabled) || !enabled)
+            {
+                return null;
+            }
+
+            return new PublishExpressionTypeParams()
+            {
+                Uri = documentUri,
+                Type = result.ReturnType
+            };
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Helpers/PositionRangeHelper.cs

@@ -1,7 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+using Microsoft.PowerFx.Core.Utils;
 using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Syntax;
 
 namespace Microsoft.PowerFx.LanguageServerProtocol
 {
@@ -10,6 +12,8 @@ namespace Microsoft.PowerFx.LanguageServerProtocol
     /// </summary>
     internal static class PositionRangeHelper
     {
+        public const char EOL = '\n';
+
         /// <summary>
         /// Converts the given range into coressponding start and end indexes.
         /// </summary>
@@ -71,5 +75,106 @@ namespace Microsoft.PowerFx.LanguageServerProtocol
 
             return startIndex < 0 || endIndex < 0 ? (-1, -1) : (startIndex, endIndex);
         }
+
+        /// <summary>
+        /// Get the position offset (starts with 0) in Expression from line/character (starts with 0)
+        /// e.g. "123", line:0, char:1 => 1.
+        /// <param name="expression">Expression.</param>
+        /// <param name="line">Line number in the expression.</param>
+        /// <param name="character">Character number in the expression.</param>
+        /// <param name="eol"> End of line character.</param>
+        /// <returns>Position offset (1D) from line and column/character (2D).</returns>
+        /// </summary>
+        // TODO: This is a buggy implementation. Would revisit this.
+        public static int GetPosition(string expression, int line, int character, char eol = EOL)
+        {
+            Contracts.AssertValue(expression);
+            Contracts.Assert(line >= 0);
+            Contracts.Assert(character >= 0);
+
+            var position = 0;
+            var currentLine = 0;
+            var currentCharacter = 0;
+            while (position < expression.Length)
+            {
+                if (line == currentLine && character == currentCharacter)
+                {
+                    return position;
+                }
+
+                if (expression[position] == EOL)
+                {
+                    currentLine++;
+                    currentCharacter = 0;
+                }
+                else
+                {
+                    currentCharacter++;
+                }
+
+                position++;
+            }
+
+            return position;
+        }
+
+        /// <summary>
+        /// Construct a Range based on a Span for a given expression.
+        /// </summary>
+        /// <param name="expression">The expression.</param>
+        /// <param name="span">The Span.</param>
+        /// <returns>Generated Range.</returns>
+        public static Range ConvertSpanToRange(this Span span, string expression)
+        {
+            var startChar = GetCharPosition(expression, span.Min) - 1;
+            var endChar = GetCharPosition(expression, span.Lim) - 1;
+
+            var startCode = expression.Substring(0, span.Min);
+            var code = expression.Substring(span.Min, span.Lim - span.Min);
+            var startLine = startCode.Split(EOL).Length;
+            var endLine = startLine + code.Split(EOL).Length - 1;
+
+            var range = new Range()
+            {
+                Start = new Position()
+                {
+                    Character = startChar,
+                    Line = startLine
+                },
+                End = new Position()
+                {
+                    Character = endChar,
+                    Line = endLine
+                }
+            };
+
+            Contracts.Assert(range.IsValid());
+
+            return range;
+        }
+
+        /// <summary>
+        /// Get the charactor position (starts with 1) from its line.
+        /// e.g. "123\n1{2}3" ==> 2 ({x} is the input char at position)
+        ///      "12{\n}123" ==> 3 ('\n' belongs to the previous line "12\n", the last char is '2' with index of 3).
+        /// </summary>
+        /// <param name="expression">The expression content.</param>
+        /// <param name="position">The charactor position (starts with 0).</param>
+        /// <returns>The charactor position (starts with 1) from its line.</returns>
+        public static int GetCharPosition(string expression, int position)
+        {
+            Contracts.AssertValue(expression);
+            Contracts.Assert(position >= 0);
+
+            var column = (position < expression.Length && expression[position] == EOL) ? 0 : 1;
+            position--;
+            while (position >= 0 && expression[position] != EOL)
+            {
+                column++;
+                position--;
+            }
+
+            return column;
+        }
     }
 }

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Host/IHostTaskExecutor.cs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// A task executor that can be used to execute LSP tasks in the host environment.
+    /// </summary>
+    public interface IHostTaskExecutor
+    {
+        /// <summary>
+        ///  Executes a task in host environment if host task executor is available.
+        ///  This is critical to trust between LSP and its hosts.
+        ///  LSP should correctly use this and wrap particulat steps that need to run in host using these methods. 
+        /// </summary>
+        /// <typeparam name="TOutput">Output Type.</typeparam>
+        /// <param name="task">Task to run inside host.</param>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <param name="cancellationToken">Cancellation Token.</param>
+        /// <param name="defaultOutput">Default Output to return if the provided task is canceled by host.</param>
+        /// <returns>Output.</returns>
+        Task<TOutput> ExecuteTaskAsync<TOutput>(Func<Task<TOutput>> task, LanguageServerOperationContext operationContext, CancellationToken cancellationToken, TOutput defaultOutput = default);
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/JsonRpc/LanguageServerInput.cs

@@ -0,0 +1,66 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Text.Json;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// Represents the input to the language server.
+    /// </summary>
+    public class LanguageServerInput
+    {
+        /// <summary>
+        /// Id of the request. Applicable only for request messages.
+        /// </summary>
+        public string Id { get; init; }
+
+        /// <summary>
+        /// Method of the request or notification.
+        /// </summary>
+        public string Method { get; init; }
+
+        /// <summary>
+        /// The jsonrpc version.
+        /// </summary>
+        public string Jsonrpc { get; init; }
+
+        /// <summary>
+        /// Parameters of the request or notification.
+        /// </summary>
+        public JsonElement Params { get; init; }
+
+        /// <summary>
+        /// Raw parameters of the request or notification.
+        /// </summary>
+        public string RawParams
+        {
+            get
+            {
+                try
+                {
+                    return Params.GetRawText();
+                }
+                catch
+                {
+                    return string.Empty;
+                }
+            }
+        }
+        
+        /// <summary>
+        /// Parses the input json payload to <see cref="LanguageServerInput"/>.
+        /// </summary>
+        /// <param name="jsonRpcPayload">Raw Payload.</param>
+        /// <returns>Parsed Input.</returns>
+        public static LanguageServerInput Parse(string jsonRpcPayload)
+        {
+            if (LanguageServerHelper.TryParseParams(jsonRpcPayload, out LanguageServerInput input))
+            {
+                return input;
+            }
+
+            return null;
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/JsonRpc/LanguageServerOutput.cs

@@ -0,0 +1,76 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// A representation of the output of a language server.
+    /// </summary>
+    public class LanguageServerOutput
+    {
+        /// <summary>
+        ///  A particular stringified output of the language server.
+        /// </summary>
+        public string Output { get; private set; }
+
+        private LanguageServerOutput()
+        {
+        }
+
+        /// <summary>
+        /// Creates a new instance of <see cref="LanguageServerOutput"/> with the given error information.
+        /// </summary>
+        /// <param name="id"> The id of the request that resulted in the error. </param>
+        /// <param name="code"> The error code. </param>
+        /// <param name="message"> The error message. </param>
+        /// <returns> A new instance of <see cref="LanguageServerOutput"/> with the given error information. </returns>
+        public static LanguageServerOutput CreateErrorResult(string id, JsonRpcHelper.ErrorCode code, string message = null)
+        {
+            return new LanguageServerOutput
+            {
+                Output = JsonRpcHelper.CreateErrorResult(id, code, message),
+            };
+        }
+
+        /// <summary>
+        /// Creates a new instance of <see cref="LanguageServerOutput"/> with the given internal server error.
+        /// </summary>
+        /// <param name="id"> The id of the request that resulted in the error. </param>
+        /// <param name="message"> The error message. </param>
+        /// <returns> A new instance of <see cref="LanguageServerOutput"/> with the given error information. </returns>
+        public static LanguageServerOutput CreateInternalServerErrorOutput(string id, string message = null)
+        {
+            return CreateErrorResult(id, JsonRpcHelper.ErrorCode.InternalError, message);
+        }
+
+        /// <summary>
+        /// Creates a new instance of <see cref="LanguageServerOutput"/> with the given success result.
+        /// </summary>
+        /// <typeparam name="T"> The type of the result. </typeparam>
+        /// <param name="id"> The id of the request that resulted in the success. </param>
+        /// <param name="result"> The result. </param>
+        /// <returns> A new instance of <see cref="LanguageServerOutput"/> with the given success result. </returns>
+        public static LanguageServerOutput CreateSuccessResult<T>(string id, T result)
+        {
+            return new LanguageServerOutput
+            {
+                Output = JsonRpcHelper.CreateSuccessResult(id, result),
+            };
+        }
+
+        /// <summary>
+        /// Creates a new instance of <see cref="LanguageServerOutput"/> with the given notification.
+        /// </summary>
+        /// <typeparam name="T"> The type of the notification parameters. </typeparam>
+        /// <param name="method"> The method of the notification. </param>
+        /// <param name="notificationParams"> The notification parameters. </param>
+        /// <returns> A new instance of <see cref="LanguageServerOutput"/> with the given notification. </returns>
+        public static LanguageServerOutput CreateNotification<T>(string method, T notificationParams)
+        {
+            return new LanguageServerOutput
+            {
+                Output = JsonRpcHelper.CreateNotification(method, notificationParams),
+            };
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/JsonRpc/LanguageServerOutputBuilder.cs

@@ -0,0 +1,162 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// An output builder to build Language Server Output.
+    /// Lifecycle: A new instance of the builder should be created for each request.
+    /// Can hold more than one output and also different types of outputs.
+    /// Request/Notification Hanlders are free to add multiple outputs to the builder.
+    /// </summary>
+    public class LanguageServerOutputBuilder : IEnumerable<LanguageServerOutput>
+    {
+        private static readonly JsonSerializerOptions _jsonSerializerOptions = new ()
+        {
+            PropertyNameCaseInsensitive = true
+        };
+
+        private readonly List<LanguageServerOutput> _outputs = new ();
+
+        /// <summary>
+        /// A serialized output created from all the outputs in the builder.
+        /// </summary>
+        public string Response
+        {
+            get
+            {
+                if (_outputs.Count == 0)
+                {
+                    return string.Empty;
+                }
+
+                if (_outputs.Count == 1)
+                {
+                    return _outputs[0].Output;
+                }
+
+                return JsonSerializer.Serialize(_outputs.Select(output => output.Output), _jsonSerializerOptions);
+            }
+        }
+
+        /// <summary>
+        /// Add a success response with the result.
+        /// </summary>
+        /// <typeparam name="T">Type of the result.</typeparam>
+        /// <param name="id">Id of the request.</param>
+        /// <param name="result">Result.</param>
+        public void AddSuccessResponse<T>(string id, T result)
+        {
+            _outputs.Add(LanguageServerOutput.CreateSuccessResult(id, result));
+        }
+
+        /// <summary>
+        /// Add an error response.
+        /// </summary>
+        /// <param name="id"> Id of the request.</param>
+        /// <param name="code"> Error Code.</param>
+        /// <param name="message"> Error Message.</param>
+        public void AddErrorResponse(string id, JsonRpcHelper.ErrorCode code, string message = null)
+        {
+            _outputs.Add(LanguageServerOutput.CreateErrorResult(id, code, message));
+        }
+
+        /// <summary>
+        /// Add a notification.
+        /// </summary>
+        /// <typeparam name="T"> Type of the notification params.</typeparam>
+        /// <param name="method"> Method of the notification.</param>
+        /// <param name="notificationParams"> Notification Params.</param>
+        public void AddNotification<T>(string method, T notificationParams)
+        {
+            _outputs.Add(LanguageServerOutput.CreateNotification(method, notificationParams));
+        }
+
+        public IEnumerator<LanguageServerOutput> GetEnumerator()
+        {
+            return _outputs.GetEnumerator();
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+    }
+
+    /// <summary>
+    ///  Helper class to build Language Server Output.
+    /// </summary>
+    internal static class LanguageServerOutputBuilderExtensions
+    {
+        /// <summary>
+        /// Add an error response with ParseError error code.
+        /// </summary>
+        /// <param name="outputBuilder">Output Builder.</param>
+        /// <param name="id">Request Id.</param>
+        /// <param name="message">Option Error Message.</param>
+        public static void AddParseError(this LanguageServerOutputBuilder outputBuilder, string id, string message = null)
+        {
+            outputBuilder.AddErrorResponse(id, JsonRpcHelper.ErrorCode.ParseError, message);
+        }
+
+        /// <summary>
+        /// Add an error response with InternalError error code.
+        /// </summary>
+        /// <param name="outputBuilder">Output Builder.</param>
+        /// <param name="id">Request Id.</param>
+        /// <param name="message">Option Error Message.</param>
+        public static void AddInternalError(this LanguageServerOutputBuilder outputBuilder, string id, string message = null)
+        {
+            outputBuilder.AddErrorResponse(id, JsonRpcHelper.ErrorCode.InternalError, message);
+        }
+
+        /// <summary>
+        /// Add an error response with InvalidParams error code.
+        /// </summary>
+        /// <param name="outputBuilder">Output Builder.</param>
+        /// <param name="id">Request Id.</param>
+        /// <param name="message">Option Error Message.</param>
+        public static void AddInvalidParamsError(this LanguageServerOutputBuilder outputBuilder, string id, string message = null)
+        {
+            outputBuilder.AddErrorResponse(id, JsonRpcHelper.ErrorCode.InvalidParams, message);
+        }
+
+        /// <summary>
+        /// Add an error response with PropertyValueRequired error code.
+        /// </summary>
+        /// <param name="outputBuilder">Output Builder.</param>
+        /// <param name="id">Request Id.</param>
+        /// <param name="message">Option Error Message.</param>
+        public static void AddProperyValueRequiredError(this LanguageServerOutputBuilder outputBuilder, string id, string message = null)
+        {
+            outputBuilder.AddErrorResponse(id, JsonRpcHelper.ErrorCode.PropertyValueRequired, message);
+        }
+
+        /// <summary>
+        /// Add an error response with InvalidRequest error code.
+        /// </summary>
+        /// <param name="outputBuilder">Output Builder.</param>
+        /// <param name="id">Request Id.</param>
+        /// <param name="message">Option Error Message.</param>
+        public static void AddInvalidRequestError(this LanguageServerOutputBuilder outputBuilder, string id, string message = null)
+        {
+            outputBuilder.AddErrorResponse(id, JsonRpcHelper.ErrorCode.InvalidRequest, message);
+        }
+
+        /// <summary>
+        /// Add an error response with MethodNotFound error code.
+        /// </summary>
+        /// <param name="outputBuilder">Output Builder.</param>
+        /// <param name="id">Request Id.</param>
+        /// <param name="message">Option Error Message.</param>
+        public static void AddMethodNotFoundError(this LanguageServerOutputBuilder outputBuilder, string id, string message = null)
+        {
+            outputBuilder.AddErrorResponse(id, JsonRpcHelper.ErrorCode.MethodNotFound, message);
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Logging/BackwardsCompatibleLogger.cs

@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// Backwards compatible logger that can be used in place of the ILanguageServerLogger.
+    /// </summary>
+    internal sealed class BackwardsCompatibleLogger : ILanguageServerLogger
+    {
+        private readonly Action<string> _logger;
+
+        public BackwardsCompatibleLogger(Action<string> logger = null)
+        {
+            _logger = logger;
+        }
+
+        private void Log(string message)
+        {
+            _logger?.Invoke(message);
+        }
+
+        public void LogError(string message, object data = null)
+        {
+            Log(message);
+        }
+
+        public void LogException(Exception exception, object data = null)
+        {
+            Log(exception.Message);
+        }
+
+        public void LogInformation(string message, object data = null)
+        {
+            Log(message);
+        }
+
+        public void LogWarning(string message, object data = null)
+        {
+            Log(message);
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Logging/ILanguageServerLogger.cs

@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.PowerFx.LanguageServerProtocol
+{
+    /// <summary>
+    /// Logger interface for language server.
+    /// </summary>
+    public interface ILanguageServerLogger
+    {
+        /// <summary>
+        /// Log information message.
+        /// </summary>
+        /// <param name="message">Message.</param>
+        /// <param name="data">Additional Data to log.</param>
+        public void LogInformation(string message, object data = null);
+
+        /// <summary>
+        /// Log warning message.
+        /// </summary>
+        /// <param name="message">Warning Message.</param>
+       /// <param name="data">Additional Data to log.</param>
+        public void LogWarning(string message, object data = null);
+
+        /// <summary>
+        /// Log error message.
+        /// </summary>
+        /// <param name="message">Message.</param>
+        /// <param name="data">Additional Data to log.</param>
+        public void LogError(string message, object data = null);
+
+        /// <summary>
+        /// Log exception.
+        /// </summary>
+        /// <param name="exception">Exception.</param>
+        /// <param name="data">Additional Data to log.</param>
+        public void LogException(Exception exception, object data = null);
+    }
+}

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Public/NLHandler.cs

@@ -7,6 +7,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.PowerFx.Core.Utils;
 using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
 using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
 using static Microsoft.PowerFx.LanguageServerProtocol.LanguageServer;
 
@@ -38,6 +39,18 @@ namespace Microsoft.PowerFx.LanguageServerProtocol
             return await Fx2NLAsync(check, cancel).ConfigureAwait(false);
 #pragma warning restore CS0618 // Type or member is obsolete
         }
+
+        /// <summary>
+        /// Additional hook to run pre-handle logic for NL2Fx.
+        /// </summary>
+        /// <param name="nl2fxParameters">Nl2fx Parameters computed from defualt pre handle.</param>
+        /// <param name="nl2FxRequestParams">Nl2fx Request Params.</param>
+        /// <param name="operationContext">Language Server Operation Context.</param>
+        /// <exception cref="NotImplementedException">Not implemeted by default.</exception>
+        public virtual void PreHandleNl2Fx(CustomNL2FxParams nl2FxRequestParams, NL2FxParameters nl2fxParameters, LanguageServerOperationContext operationContext)
+        {
+            throw new NotImplementedException();
+        }
     }
 
     /// <summary>

src/libraries/Microsoft.PowerFx.LanguageServerProtocol/Schemas/ControlTokens.cs

@@ -12,7 +12,7 @@ using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
 
 namespace Microsoft.PowerFx.LanguageServerProtocol.Schemas
 {
-    internal class ControlTokens 
+    internal class ControlTokens : IEnumerable<ControlToken>
     {      
         private readonly ICollection<ControlToken> _controlTokens;
         private readonly IDictionary<string, ControlToken> _controlTokenDict;
@@ -44,14 +44,19 @@ namespace Microsoft.PowerFx.LanguageServerProtocol.Schemas
             return null;
         }
 
-        public IEnumerable<ControlToken> GetControlTokens()
+        public IEnumerator<ControlToken> GetEnumerator()
         {
-            return _controlTokens;
+            return _controlTokens.GetEnumerator();
         }
 
         public int Size()
         {
             return _controlTokens.Count;
         }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
     }
 }

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/EditorContextScopeTests.cs

@@ -14,7 +14,7 @@ using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
 using Microsoft.PowerFx.Syntax;
 using Microsoft.PowerFx.Types;
 using Xunit;
-using static Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests.LanguageServerTests;
+using static Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests.LegacyLanguageServerTests;
 
 namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
 {

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/LSPTests.cs

@@ -28,7 +28,7 @@ namespace Microsoft.PowerFx.Interpreter.Tests.LanguageServiceProtocol
         [InlineData("abcdefgh\r\nb", 7, 11, 7, 1, 1, 2)]
         public void TestGetRange(string expression, int min, int lim, char startChar, char endChar, int startLine, int endLine)
         {
-            var range = LanguageServer.GetRange(expression, new Span(min, lim));
+            var range = new Span(min, lim).ConvertSpanToRange(expression);
 
             Assert.Equal(startChar, range.Start.Character);
             Assert.Equal(endChar, range.End.Character);

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/LegacyLanguageServerTests.cs

@@ -26,6 +26,7 @@ using Microsoft.PowerFx.Intellisense;
 using Microsoft.PowerFx.Interpreter.Tests.Helpers;
 using Microsoft.PowerFx.Interpreter.Tests.LanguageServiceProtocol;
 using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
 using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
 using Microsoft.PowerFx.LanguageServerProtocol.Schemas;
 using Microsoft.PowerFx.Types;
@@ -35,7 +36,12 @@ using static Microsoft.PowerFx.Tests.BindingEngineTests;
 
 namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests
 {
-    public class LanguageServerTests : PowerFxTest
+    /// <summary>
+    ///  Do not write tests in this file.
+    ///  Write any new test in RedesignedLanguageServerTest folder.
+    ///  Look there for examples.
+    /// </summary>
+    public class LegacyLanguageServerTests : PowerFxTest
     {
         protected static readonly JsonSerializerOptions _jsonSerializerOptions = new JsonSerializerOptions
         {
@@ -52,7 +58,7 @@ namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests
         private readonly ITestOutputHelper _output;
         private readonly ConcurrentBag<Exception> _exList = new ConcurrentBag<Exception>();
 
-        public LanguageServerTests(ITestOutputHelper output)
+        public LegacyLanguageServerTests(ITestOutputHelper output)
             : base()
         {
             _output = output;
@@ -101,21 +107,13 @@ namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests
         public void TestTopParseError()
         {
             var list = new List<Exception>();
-
-            _testServer.LogUnhandledExceptionHandler += (ex) =>
-            {
-                list.Add(ex);
-            };
             _testServer.OnDataReceived("parse error");
 
-            Assert.Single(list); // ensure handler was invoked. 
-
             Assert.Single(_sendToClientData);
             var errorResponse = JsonSerializer.Deserialize<JsonRpcErrorResponse>(_sendToClientData[0], _jsonSerializerOptions);
             Assert.Equal("2.0", errorResponse.Jsonrpc);
             Assert.Null(errorResponse.Id);
-            Assert.Equal(InternalError, errorResponse.Error.Code);
-            Assert.Equal(list[0].GetDetailedExceptionMessage(), errorResponse.Error.Message);
+            Assert.Equal(ParseError, errorResponse.Error.Code);
         }
 
         // Scope facotry that throws. simulate server crashes.
@@ -940,44 +938,6 @@ namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests
             Assert.Empty(exList);
         }
 
-        [Theory]
-        [InlineData("{1}", 1)]
-        [InlineData("12{3}45", 3)]
-        [InlineData("1234{5}", 5)]
-        [InlineData("123\n1{2}3", 2)]
-        [InlineData("123\n{5}67", 1)]
-        [InlineData("123\n5{6}7", 2)]
-        [InlineData("123\n56{7}", 3)]
-        [InlineData("123\n567{\n}890", 3)]
-        public void TestGetCharPosition(string expression, int expected)
-        {
-            var pattern = @"\{[0-9|\n]\}";
-            var re = new Regex(pattern);
-            var matches = re.Matches(expression);
-            Assert.Single(matches);
-
-            var position = matches[0].Index;
-            expression = expression.Substring(0, position) + expression[position + 1] + expression.Substring(position + 3);
-
-            Assert.Equal(expected, _testServer.TestGetCharPosition(expression, position));
-            Assert.Empty(_exList);
-        }
-
-        [Fact]
-        public void TestGetPosition()
-        {
-            Assert.Equal(0, _testServer.TestGetPosition("123", 0, 0));
-            Assert.Equal(1, _testServer.TestGetPosition("123", 0, 1));
-            Assert.Equal(2, _testServer.TestGetPosition("123", 0, 2));
-            Assert.Equal(4, _testServer.TestGetPosition("123\n123456\n12345", 1, 0));
-            Assert.Equal(5, _testServer.TestGetPosition("123\n123456\n12345", 1, 1));
-            Assert.Equal(11, _testServer.TestGetPosition("123\n123456\n12345", 2, 0));
-            Assert.Equal(13, _testServer.TestGetPosition("123\n123456\n12345", 2, 2));
-            Assert.Equal(3, _testServer.TestGetPosition("123", 0, 999));
-
-            Assert.Empty(_exList);
-        }
-
         // Ensure the disclaimer shows up in a signature.
         [Fact]
         public void TestSignatureDisclaimers()
@@ -1624,6 +1584,11 @@ namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests
                     Explanation = sb.ToString()
                 };
             }
+
+            public override void PreHandleNl2Fx(CustomNL2FxParams nl2FxRequestParams, NL2FxParameters nl2fxParameters, LanguageServerOperationContext operationContext)
+            {
+                // do nothing
+            }
         }
 
         // Test the overload with Fx2NLParameters

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/PositionRangeHelperTests.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+using System.Text.RegularExpressions;
 using Microsoft.PowerFx.LanguageServerProtocol;
 using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
 using Xunit;
@@ -74,5 +75,40 @@ namespace Microsoft.PowerFx.Interpreter.Tests.LanguageServiceProtocol
             Assert.Equal(startIndex, result.startIndex);
             Assert.Equal(endIndex, result.endIndex);
         }
+
+        [Theory]
+        [InlineData("{1}", 1)]
+        [InlineData("12{3}45", 3)]
+        [InlineData("1234{5}", 5)]
+        [InlineData("123\n1{2}3", 2)]
+        [InlineData("123\n{5}67", 1)]
+        [InlineData("123\n5{6}7", 2)]
+        [InlineData("123\n56{7}", 3)]
+        [InlineData("123\n567{\n}890", 3)]
+        public void TestGetCharPosition(string expression, int expected)
+        {
+            var pattern = @"\{[0-9|\n]\}";
+            var re = new Regex(pattern);
+            var matches = re.Matches(expression);
+            Assert.Single(matches);
+
+            var position = matches[0].Index;
+            expression = expression.Substring(0, position) + expression[position + 1] + expression.Substring(position + 3);
+
+            Assert.Equal(expected, PositionRangeHelper.GetCharPosition(expression, position));
+        }
+
+        [Fact]
+        public void TestGetPosition()
+        {
+            Assert.Equal(0, PositionRangeHelper.GetPosition("123", 0, 0));
+            Assert.Equal(1, PositionRangeHelper.GetPosition("123", 0, 1));
+            Assert.Equal(2, PositionRangeHelper.GetPosition("123", 0, 2));
+            Assert.Equal(4, PositionRangeHelper.GetPosition("123\n123456\n12345", 1, 0));
+            Assert.Equal(5, PositionRangeHelper.GetPosition("123\n123456\n12345", 1, 1));
+            Assert.Equal(11, PositionRangeHelper.GetPosition("123\n123456\n12345", 2, 0));
+            Assert.Equal(13, PositionRangeHelper.GetPosition("123\n123456\n12345", 2, 2));
+            Assert.Equal(3, PositionRangeHelper.GetPosition("123", 0, 999));
+        }
     }
 }

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/PublicSurfaceTests.cs

@@ -41,6 +41,12 @@ namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
                 "Microsoft.PowerFx.LanguageServerProtocol.IPowerFxScopeFx2NL",
                 "Microsoft.PowerFx.LanguageServerProtocol.Fx2NLParameters",
                 "Microsoft.PowerFx.LanguageServerProtocol.UsageHints",
+                "Microsoft.PowerFx.LanguageServerProtocol.LanguageServerInput",
+                "Microsoft.PowerFx.LanguageServerProtocol.LanguageServerOutput",
+                "Microsoft.PowerFx.LanguageServerProtocol.LanguageServerOutputBuilder",
+                "Microsoft.PowerFx.LanguageServerProtocol.ILanguageServerLogger",
+                "Microsoft.PowerFx.LanguageServerProtocol.IHostTaskExecutor",
+                "Microsoft.PowerFx.LanguageServerProtocol.Handlers.LanguageServerOperationContext",
 
                 // Internal
                 "Microsoft.PowerFx.LanguageServerProtocol.JsonRpcHelper",

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/CodeActionTests.cs

@@ -0,0 +1,245 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.Interpreter.Tests.LanguageServiceProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public partial class LanguageServerTestBase
+    {
+        private class DummyQuickFixHandler : CodeFixHandler
+        {
+            public override async Task<IEnumerable<CodeFixSuggestion>> SuggestFixesAsync(Engine engine, CheckResult checkResult, CancellationToken cancel)
+            {
+                return new CodeFixSuggestion[]
+                {
+                    new CodeFixSuggestion
+                    {
+                        SuggestedText = "TestText1",
+                        Title = "TestTitle1"
+                    }
+                };
+            }
+        }
+
+        // Failure from one handler shouldn't block others. 
+        public class ExceptionQuickFixHandler : CodeFixHandler
+        {
+            public int _counter = 0;
+
+            public override async Task<IEnumerable<CodeFixSuggestion>> SuggestFixesAsync(Engine engine, CheckResult checkResult, CancellationToken cancel)
+            {
+                _counter++;
+                throw new Exception($"expected failure");
+            }
+        }
+
+        [Fact]
+        public async Task TestCodeAction()
+        {
+            // Arrange
+            var failHandler = new ExceptionQuickFixHandler();
+            var engine = new Engine(new PowerFxConfig());
+            var editor = engine.CreateEditorScope();
+            editor.AddQuickFixHandler(new DummyQuickFixHandler());
+            editor.AddQuickFixHandler(failHandler);
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) => editor);
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var codeActionParams = new CodeActionParams()
+            {
+                TextDocument = GetTextDocument("powerfx://test?expression=IsBlank(&context={\"A\":1,\"B\":[1,2,3]}"),
+                Range = new LanguageServerProtocol.Protocol.Range()
+                {
+                    Start = new Position
+                    {
+                        Line = 0,
+                        Character = 0
+                    },
+                    End = new Position
+                    {
+                        Line = 0,
+                        Character = 10
+                    }
+                },
+                Context = new CodeActionContext() { Only = new[] { CodeActionKind.QuickFix } }
+            };
+            var payload = GetCodeActionPayload(codeActionParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            var response = AssertAndGetResponsePayload<Dictionary<string, CodeAction[]>>(rawResponse, payload.id);
+            Assert.NotEmpty(response);
+            Assert.Contains(CodeActionKind.QuickFix, response.Keys);
+            Assert.True(response[CodeActionKind.QuickFix].Length == 1, "Quick fix didn't return expected suggestion.");
+            Assert.Equal("TestTitle1", response[CodeActionKind.QuickFix][0].Title);
+            Assert.NotEmpty(response[CodeActionKind.QuickFix][0].Edit.Changes);
+            Assert.Contains(codeActionParams.TextDocument.Uri, response[CodeActionKind.QuickFix][0].Edit.Changes.Keys);
+            Assert.Equal("TestText1", response[CodeActionKind.QuickFix][0].Edit.Changes[codeActionParams.TextDocument.Uri][0].NewText);
+
+            // Fail handler was invokde, but didn't block us. 
+            Assert.Equal(1, failHandler._counter); // Invoked
+            Assert.Single(TestServer.UnhandledExceptions);
+        }
+
+        // Test a codefix using a customization, ICodeFixHandler
+        [Fact]
+        public async Task TestCodeActionWithHandlerAndExpressionInUriOrInTextDocument()
+        {
+            // Blank(A) is error, should change to IsBlank(A)
+            var original = "Blank(A)";
+            var updated = "IsBlank(A)";
+            foreach (var addExprToUri in new bool[] { true, false })
+            {
+                var codeActionParams = new CodeActionParams
+                {
+                    TextDocument = addExprToUri ?
+                                   GetTextDocument(GetUri("context={\"A\":1,\"B\":[1,2,3]}&expression=" + original)) :
+                                   GetTextDocument(GetUri("context={\"A\":1,\"B\":[1,2,3]}")),
+                    Text = addExprToUri ? null : original,
+                    Range = SemanticTokensRelatedTestsHelper.CreateRange(0, 0, 0, 10),
+                    Context = GetDefaultCodeActionContext()
+                };
+                await TestCodeActionWithHandlerCore(codeActionParams, updated).ConfigureAwait(false);
+            }
+
+            await TestCodeActionWithHandlerCore(
+                new CodeActionParams
+                {
+                    TextDocument = GetTextDocument(GetUri("context={\"A\":1,\"B\":[1,2,3]}&expression=Max(1,2")),
+                    Text = original,
+                    Range = SemanticTokensRelatedTestsHelper.CreateRange(0, 0, 0, 10),
+                    Context = GetDefaultCodeActionContext()
+                }, updated).ConfigureAwait(false);
+        }
+
+        private async Task TestCodeActionWithHandlerCore(CodeActionParams codeActionParams, string updatedExpr)
+        {
+            var engine = new RecalcEngine();
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) =>
+            {
+                var scope = engine.CreateEditorScope();
+                scope.AddQuickFixHandler(new BlankHandler());
+                return scope;
+            });
+            Init(new InitParams(scopeFactory: scopeFactory));
+
+            var payload = GetCodeActionPayload(codeActionParams);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            var response = AssertAndGetResponsePayload<Dictionary<string, CodeAction[]>>(rawResponse, payload.id);
+            Assert.NotEmpty(response);
+            Assert.Contains(CodeActionKind.QuickFix, response.Keys);
+            Assert.True(response[CodeActionKind.QuickFix].Length == 1, "Quick fix didn't return expected suggestion.");
+            Assert.Equal(BlankHandler.Title, response[CodeActionKind.QuickFix][0].Title);
+            Assert.NotEmpty(response[CodeActionKind.QuickFix][0].Edit.Changes);
+            Assert.Contains(codeActionParams.TextDocument.Uri, response[CodeActionKind.QuickFix][0].Edit.Changes.Keys);
+
+            Assert.Equal(updatedExpr, response[CodeActionKind.QuickFix][0].Edit.Changes[codeActionParams.TextDocument.Uri][0].NewText);
+            Assert.Empty(TestServer.UnhandledExceptions);
+        }
+
+        [Fact]
+        public async Task TestCodeActionCommandExecuted()
+        {
+            var engine = new RecalcEngine();
+
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) =>
+            {
+                var scope = engine.CreateEditorScope();
+                scope.AddQuickFixHandler(new BlankHandler());
+                return scope;
+            });
+            Init(new InitParams(scopeFactory: scopeFactory));
+
+            var documentUri = "powerfx://test?expression=Blank(A)&context={\"A\":1,\"B\":[1,2,3]}";
+            var codeActionsParams1 = new CodeActionParams()
+            {
+                TextDocument = GetTextDocument(documentUri),
+                Range = new LanguageServerProtocol.Protocol.Range()
+                {
+                    Start = new Position
+                    {
+                        Line = 0,
+                        Character = 0
+                    },
+                    End = new Position
+                    {
+                        Line = 0,
+                        Character = 10
+                    }
+                },
+                Context = new CodeActionContext() { Only = new[] { CodeActionKind.QuickFix } }
+            };
+            var payload = GetCodeActionPayload(codeActionsParams1);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<Dictionary<string, CodeAction[]>>(rawResponse, payload.id);
+            Assert.NotEmpty(response);
+
+            var codeActionResult = response[CodeActionKind.QuickFix][0];
+
+            Assert.NotNull(codeActionResult);
+            Assert.NotNull(codeActionResult.ActionResultContext);
+            Assert.Equal(typeof(BlankHandler).FullName, codeActionResult.ActionResultContext.HandlerName);
+            Assert.Equal("Suggestion", codeActionResult.ActionResultContext.ActionIdentifier);
+            var commandExecutedParams = new CommandExecutedParams()
+            {
+                TextDocument = new TextDocumentIdentifier()
+                {
+                    Uri = documentUri
+                },
+                Command = CommandName.CodeActionApplied,
+                Argument = JsonRpcHelper.Serialize(codeActionResult)
+            };
+            var commandExecutedPayload = GetRequestPayload(commandExecutedParams, CustomProtocolNames.CommandExecuted);
+            rawResponse = await TestServer.OnDataReceivedAsync(commandExecutedPayload.payload).ConfigureAwait(false);
+            Assert.True(string.IsNullOrEmpty(rawResponse));
+            
+            commandExecutedParams = new CommandExecutedParams()
+            {
+                TextDocument = new TextDocumentIdentifier()
+                {
+                    Uri = documentUri
+                },
+                Command = CommandName.CodeActionApplied,
+                Argument = string.Empty
+            };
+            commandExecutedPayload = GetRequestPayload(commandExecutedParams, CustomProtocolNames.CommandExecuted);
+            rawResponse = await TestServer.OnDataReceivedAsync(commandExecutedPayload.payload).ConfigureAwait(false);
+            AssertErrorPayload(rawResponse, commandExecutedPayload.id, JsonRpcHelper.ErrorCode.PropertyValueRequired);
+
+            codeActionResult.ActionResultContext = null;
+            commandExecutedParams = new CommandExecutedParams()
+            {
+                TextDocument = new TextDocumentIdentifier()
+                {
+                    Uri = documentUri
+                },
+                Command = CommandName.CodeActionApplied,
+                Argument = JsonRpcHelper.Serialize(codeActionResult)
+            };
+            commandExecutedPayload = GetRequestPayload(commandExecutedParams, CustomProtocolNames.CommandExecuted);
+            rawResponse = await TestServer.OnDataReceivedAsync(commandExecutedPayload.payload).ConfigureAwait(false);
+            AssertErrorPayload(rawResponse, commandExecutedPayload.id, JsonRpcHelper.ErrorCode.PropertyValueRequired);
+        }
+
+        private static CodeActionContext GetDefaultCodeActionContext()
+        {
+            return new CodeActionContext() { Only = new[] { CodeActionKind.QuickFix } };
+        }
+
+        private static (string payload, string id) GetCodeActionPayload(CodeActionParams codeActionParams)
+        {
+            return GetRequestPayload(codeActionParams, TextDocumentNames.CodeAction);
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/CompletionsTests.cs

@@ -0,0 +1,175 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public partial class LanguageServerTestBase
+    {
+        [Theory]
+        [InlineData("Color.AliceBl", 13, false)]
+        [InlineData("Behavior(); Color.AliceBl", 25, true)]
+
+        // $$$ This test generates an internal error as we use an behavior function but we have no way to check its presence
+        [InlineData("Behavior(); Color.AliceBl", 25, false)]
+        public async Task TestCompletionWithExpressionInUriOrNotInUri(string text, int offset, bool withAllowSideEffects)
+        {
+            foreach (var addExprToUri in new bool[] { true, false })
+            {
+                var params1 = new CompletionParams()
+                {
+                    TextDocument = addExprToUri ? GetTextDocument(GetUri("expression=" + text)) : GetTextDocument(),
+                    Text = addExprToUri ? null : text,
+                    Position = GetPosition(offset),
+                    Context = GetCompletionContext()
+                };
+                var params2 = new CompletionParams()
+                {
+                    TextDocument = addExprToUri ?
+                                   GetTextDocument(GetUri("expression=Color.&context={\"A\":\"ABC\",\"B\":{\"Inner\":123}}")) :
+                                   GetTextDocument(GetUri("context={\"A\":\"ABC\",\"B\":{\"Inner\":123}}")),
+                    Text = addExprToUri ? null : "Color.",
+                    Position = GetPosition(7),
+                    Context = GetCompletionContext()
+                };
+                var params3 = new CompletionParams()
+                {
+                    TextDocument = addExprToUri ? GetTextDocument(GetUri("expression={a:{},b:{},c:{}}.")) : GetTextDocument(),
+                    Text = addExprToUri ? null : "{a:{},b:{},c:{}}.",
+                    Position = GetPosition(17),
+                    Context = GetCompletionContext()
+                };
+                await TestCompletionCore(params1, params2, params3, withAllowSideEffects).ConfigureAwait(false);
+            }
+        }
+
+        [Fact]
+        public async Task TestCompletionWithExpressionBothInUriAndTextDocument()
+        {
+            int offset = 25;
+            string text = "Behavior(); Color.AliceBl";
+            string uriText = "Max(1,";
+            bool withAllowSideEffects = false;
+
+            var params1 = new CompletionParams()
+            {
+                TextDocument = GetTextDocument(GetUri("expression=" + uriText)),
+                Text = text,
+                Position = GetPosition(offset),
+                Context = GetCompletionContext()
+            };
+            var params2 = new CompletionParams()
+            {
+                TextDocument = GetTextDocument(GetUri("context={\"A\":\"ABC\",\"B\":{\"Inner\":123}}&expression=1+1")),
+                Text = "Color.",
+                Position = GetPosition(7),
+                Context = GetCompletionContext()
+            };
+            var params3 = new CompletionParams()
+            {
+                TextDocument = GetTextDocument(GetUri("expression=Color.")),
+                Text = "{a:{},b:{},c:{}}.",
+                Position = GetPosition(17),
+                Context = GetCompletionContext()
+            };
+            await TestCompletionCore(params1, params2, params3, withAllowSideEffects).ConfigureAwait(false);
+        }
+
+        private async Task TestCompletionCore(CompletionParams params1, CompletionParams params2, CompletionParams params3, bool withAllowSideEffects)
+        {
+            Init(new InitParams(options: GetParserOptions(withAllowSideEffects)));
+
+            // test good formula
+            var payload = GetCompletionPayload(params1);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<CompletionResult>(rawResponse, payload.id);
+            var foundItems = response.Items.Where(item => item.Label == "AliceBlue");
+            Assert.True(Enumerable.Count(foundItems) == 1, "AliceBlue should be found from suggestion result");
+            Assert.Equal("AliceBlue", foundItems.First().InsertText);
+            Assert.Equal("000", foundItems.First().SortText);
+
+            payload = GetCompletionPayload(params2);
+            rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            response = AssertAndGetResponsePayload<CompletionResult>(rawResponse, payload.id);
+            foundItems = response.Items.Where(item => item.Label == "AliceBlue");
+            Assert.Equal(CompletionItemKind.Variable, foundItems.First().Kind);
+            Assert.True(Enumerable.Count(foundItems) == 1, "AliceBlue should be found from suggestion result");
+            Assert.Equal("AliceBlue", foundItems.First().InsertText);
+            Assert.Equal("000", foundItems.First().SortText);
+
+            payload = GetCompletionPayload(params3);
+            rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            response = AssertAndGetResponsePayload<CompletionResult>(rawResponse, payload.id);
+            foundItems = response.Items.Where(item => item.Label == "a");
+            Assert.True(Enumerable.Count(foundItems) == 1, "'a' should be found from suggestion result");
+            Assert.Equal(CompletionItemKind.Variable, foundItems.First().Kind);
+            Assert.Equal("a", foundItems.First().InsertText);
+            Assert.Equal("000", foundItems.First().SortText);
+
+            foundItems = response.Items.Where(item => item.Label == "b");
+            Assert.True(Enumerable.Count(foundItems) == 1, "'b' should be found from suggestion result");
+            Assert.Equal(CompletionItemKind.Variable, foundItems.First().Kind);
+            Assert.Equal("b", foundItems.First().InsertText);
+            Assert.Equal("001", foundItems.First().SortText);
+
+            foundItems = response.Items.Where(item => item.Label == "c");
+            Assert.True(Enumerable.Count(foundItems) == 1, "'c' should be found from suggestion result");
+            Assert.Equal(CompletionItemKind.Variable, foundItems.First().Kind);
+            Assert.Equal("c", foundItems.First().InsertText);
+            Assert.Equal("002", foundItems.First().SortText);
+
+            // missing 'expression' in documentUri
+            payload = GetCompletionPayload(new CompletionParams()
+            {
+                TextDocument = GetTextDocument("powerfx://test"),
+                Position = GetPosition(1),
+                Context = GetCompletionContext()
+            });
+            var errorResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            AssertErrorPayload(errorResponse, payload.id, JsonRpcHelper.ErrorCode.InvalidParams);
+        }
+
+        [Theory]
+        [InlineData("'A", 1)]
+        [InlineData("'Acc", 1)]
+        public async Task TestCompletionWithIdentifierDelimiter(string text, int offset)
+        {
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) => new MockDataSourceEngine());
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var params1 = new CompletionParams()
+            {
+                TextDocument = GetTextDocument(GetUri("expression=" + text)),
+                Text = text,
+                Position = GetPosition(offset),
+                Context = GetCompletionContext()
+            };
+            var payload = GetCompletionPayload(params1);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<CompletionResult>(rawResponse, payload.id);
+            var foundItems = response.Items.Where(item => item.Label == "'Account'");
+            Assert.True(Enumerable.Count(foundItems) == 1, "'Account' should be found from suggestion result");
+
+            // Test that the Identifier delimiter is ignored in case of insertText,
+            // when preceding character is also the same identifier delimiter
+            Assert.Equal("Account'", foundItems.First().InsertText);
+            Assert.Equal("000", foundItems.First().SortText);
+        }
+
+        private static (string payload, string id) GetCompletionPayload(CompletionParams completionParams)
+        {
+            return GetRequestPayload(completionParams, TextDocumentNames.Completion);
+        }
+
+        private static CompletionContext GetCompletionContext()
+        {
+            return new CompletionContext();
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/Fx2NLTests.cs

@@ -0,0 +1,231 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Types;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    internal class TestFx2NlHandler : NLHandler
+    {
+        public bool _supportsNL2Fx = true;
+        public bool _supportsFx2NL = true;
+
+        public override bool SupportsNL2Fx => _supportsNL2Fx;
+
+        public override bool SupportsFx2NL => _supportsFx2NL;
+
+        public const string ModelIdStr = "Model123";
+
+        // Set the expected expression to return. 
+        public string Expected { get; set; }
+
+        public bool Throw { get; set; }
+
+        public bool Delay { get; set; } = false;
+
+        public bool SupportsParameterHints { get; set; } = false;
+
+        public TestFx2NlHandler()
+        {
+        }
+
+#pragma warning disable CS0672 // Member overrides obsolete member
+        public override async Task<CustomFx2NLResult> Fx2NLAsync(CheckResult check, CancellationToken cancel)
+#pragma warning restore CS0672 // Member overrides obsolete member
+        {
+            if (this.Delay)
+            {
+                await Task.Delay(100, cancel).ConfigureAwait(false);
+            }
+
+            if (this.Throw)
+            {
+                throw new InvalidOperationException($"Simulated error");
+            }
+
+            var sb = new StringBuilder();
+            sb.Append(check.ApplyParse().Text);
+            sb.Append(": ");
+            sb.Append(check.IsSuccess);
+            sb.Append(": ");
+            sb.Append(this.Expected);
+
+            var result = new CustomFx2NLResult
+            {
+                Explanation = sb.ToString()
+            };
+
+            return result;
+        }
+
+        public override async Task<CustomFx2NLResult> Fx2NLAsync(CheckResult check, Fx2NLParameters hints, CancellationToken cancel)
+        {
+            if (!this.SupportsParameterHints)
+            {
+#pragma warning disable CS0618 // Type or member is obsolete
+                return await Fx2NLAsync(check, cancel).ConfigureAwait(false);
+#pragma warning restore CS0618 // Type or member is obsolete
+            }
+
+            {
+                await Task.Delay(100, cancel).ConfigureAwait(false);
+            }
+
+            if (this.Throw)
+            {
+                throw new InvalidOperationException($"Simulated error");
+            }
+
+            var sb = new StringBuilder();
+            sb.Append(hints?.UsageHints?.ControlName);
+            sb.Append("; ");
+            sb.Append(hints?.UsageHints?.ControlKind);
+
+            sb.Append("; ");
+            sb.Append(hints?.UsageHints?.PropertyName);
+
+            return new CustomFx2NLResult
+            {
+                Explanation = sb.ToString()
+            };
+        }
+    }
+
+    public partial class LanguageServerTestBase
+    {
+        [Fact]
+        public async Task TestFx2NL()
+        {
+            // Arrange
+            var documentUri = "powerfx://app?context=1";
+            var expectedExpr = "sentence";
+
+            var engine = new Engine();
+            var symbols = new SymbolTable();
+            symbols.AddVariable("Score", FormulaType.Number);
+            var scope = engine.CreateEditorScope(symbols: symbols);
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) => scope);
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var handler = CreateAndConfigureFx2NlHandler();
+            handler.Delay = true;
+            handler.Expected = expectedExpr;
+            handler.SupportsParameterHints = false;
+
+            // Act
+            var payload = Fx2NlMessageJson(documentUri);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<CustomFx2NLResult>(rawResponse, payload.id);
+
+            // Assert
+            // result has expected concat with symbols. 
+            Assert.Equal("Score > 3: True: sentence", response.Explanation);
+        }
+
+        [Fact]
+        public async Task TestFx2NLUsageHints()
+        {
+            // Arrange
+            var documentUri = "powerfx://app?context=1";
+            var expectedExpr = "sentence";
+
+            var engine = new Engine();
+            var symbols = new SymbolTable();
+            symbols.AddVariable("Score", FormulaType.Number);
+            var scope = new EditorContextScope(engine, null, symbols)
+            {
+                UsageHints = new UsageHints
+                {
+                    ControlKind = "Button",
+                    ControlName = "MyButton",
+                    PropertyName = "Select"
+                }
+            }; 
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) => scope);
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var handler = CreateAndConfigureFx2NlHandler();
+            handler.Delay = true;
+            handler.Expected = expectedExpr;
+            handler.SupportsParameterHints = true;
+
+            // Act
+            var payload = Fx2NlMessageJson(documentUri);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<CustomFx2NLResult>(rawResponse, payload.id);
+
+            // Assert
+            // result has expected concat with symbols. 
+            Assert.Equal("MyButton; Button; Select", response.Explanation);
+        }
+
+        [Fact]
+        public async Task TestFx2NlMissingHandler()
+        {
+            // Arrange
+            HandlerFactory.SetHandler(CustomProtocolNames.FX2NL, null);
+            var documentUri = "powerfx://app?context={\"A\":1,\"B\":[1,2,3]}";
+
+            // Act
+            var payload = Fx2NlMessageJson(documentUri);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            AssertErrorPayload(rawResponse, payload.id, JsonRpcHelper.ErrorCode.MethodNotFound);
+        }
+
+        [Fact]
+        public async Task TestFx2NLThrows()
+        {
+            // Arrange
+            var documentUri = "powerfx://app?context=1";
+            var engine = new Engine();
+            var symbols = new SymbolTable();
+            symbols.AddVariable("Score", FormulaType.Number);
+            var scope = engine.CreateEditorScope(symbols: symbols);
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) => scope);
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var handler = CreateAndConfigureFx2NlHandler();
+            handler.Delay = true;
+            handler.Throw = true;
+            handler.SupportsParameterHints = false;
+
+            // Act
+            var payload = Fx2NlMessageJson(documentUri);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            AssertErrorPayload(rawResponse, payload.id, JsonRpcHelper.ErrorCode.InternalError);
+            Assert.NotEmpty(TestServer.UnhandledExceptions);
+        }
+
+        private static (string payload, string id) Fx2NlMessageJson(string documentUri, string context = null)
+        {
+            var fx2NlParams = new CustomFx2NLParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1
+                },
+                Expression = "Score > 3",
+                Context = context
+            };
+
+            return GetRequestPayload(fx2NlParams, CustomProtocolNames.FX2NL);
+        }
+
+        private TestFx2NlHandler CreateAndConfigureFx2NlHandler()
+        {
+            var fx2NlHandler = new TestFx2NlHandler();
+            HandlerFactory.SetHandler(CustomProtocolNames.FX2NL, new Fx2NlLanguageServerOperationHandler(new BackwardsCompatibleNLHandlerFactory(fx2NlHandler)));
+        
+            return fx2NlHandler;
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/GetCustomCapibilitiesTests.cs

@@ -0,0 +1,64 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    internal class TestGetCustomCapibilitiesHandler : NLHandler
+    {
+        public override bool SupportsFx2NL { get; }
+
+        public override bool SupportsNL2Fx { get; }
+
+        public TestGetCustomCapibilitiesHandler(bool supportsFx2NL, bool supportsNL2Fx)
+        {
+            SupportsFx2NL = supportsFx2NL;
+            SupportsNL2Fx = supportsNL2Fx;
+        }
+    }
+
+    public partial class LanguageServerTestBase
+    {
+        [Theory]
+        [InlineData(true, false)]
+        [InlineData(false, true)]
+        [InlineData(true, true)]
+        [InlineData(false, false)]
+        [InlineData(false, false, true)]
+        public async Task TestGetCapabilities(bool supportNL2Fx, bool supportFx2NL, bool dontRegister = false)
+        {
+            // Arrange
+            var documentUri = "powerfx://app";
+            var testNLHandler = new TestGetCustomCapibilitiesHandler(supportFx2NL, supportNL2Fx);
+            if (dontRegister)
+            {
+                testNLHandler = null;
+            }
+
+            HandlerFactory.SetHandler(CustomProtocolNames.GetCapabilities, new GetCustomCapabilitiesLanguageServerOperationHandler(new BackwardsCompatibleNLHandlerFactory(testNLHandler)));
+            var payload = GetRequestPayload(
+            new CustomGetCapabilitiesParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1
+                }
+            }, CustomProtocolNames.GetCapabilities);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert: result has expected concat with symbols. 
+            var response = AssertAndGetResponsePayload<CustomGetCapabilitiesResult>(rawResponse, payload.id);
+            Assert.Equal(supportNL2Fx, response.SupportsNL2Fx);
+            Assert.Equal(supportFx2NL, response.SupportsFx2NL);
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/InitialFixupTests.cs

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Tests.LanguageServiceProtocol.Tests;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public partial class LanguageServerTestBase
+    {
+        [Fact]
+        public async Task TestInitialFixup()
+        {
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) => new MockSqlEngine());
+
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var documentUri = "powerfx://app?context={\"A\":1,\"B\":[1,2,3]}";
+            var payload = GetRequestPayload(
+            new InitialFixupParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = "new_price * new_quantity"
+                }
+            }, CustomProtocolNames.InitialFixup);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<TextDocumentItem>(rawResponse, payload.id);
+
+            Assert.Equal(documentUri, response.Uri);
+            Assert.Equal("Price * Quantity", response.Text);
+
+            // no change
+            payload = GetRequestPayload(
+            new InitialFixupParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = "Price * Quantity"
+                }
+            }, CustomProtocolNames.InitialFixup);
+            rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            response = AssertAndGetResponsePayload<TextDocumentItem>(rawResponse, payload.id);
+            Assert.Equal(documentUri, response.Uri);
+            Assert.Equal("Price * Quantity", response.Text);
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/InvalidScenarioTests.cs

@@ -0,0 +1,125 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Core.Utils;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    internal class ErrorThrowingHandler : ILanguageServerOperationHandler
+    {
+        public bool IsRequest => false;
+
+        public string LspMethod => TextDocumentNames.DidOpen;
+
+        public async Task HandleAsync(LanguageServerOperationContext operationContext, CancellationToken cancellationToken)
+        {
+            await Task.Delay(200, cancellationToken).ConfigureAwait(false);
+            throw new Exception("Test Exception");
+        }
+    }
+
+    public partial class LanguageServerTestBase
+    {
+        [Fact]
+        public async Task TestTopParseError()
+        {
+            var rawResponse = await TestServer.OnDataReceivedAsync("parse error").ConfigureAwait(false);
+            AssertErrorPayload(rawResponse, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.ParseError);
+        }
+
+        // Exceptions can be thrown oob, test we can register a hook and receive.
+        // Check for exceptions if the scope object we call back to throws
+        [Fact]
+        public async Task TestLogCallbackExceptions()
+        {
+            // Arrange
+            HandlerFactory.SetHandler(TextDocumentNames.DidOpen, new ErrorThrowingHandler());
+
+            // Act
+            var payload = GetDidOpenPayload(
+            new DidOpenTextDocumentParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = "https://none",
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = "123"
+                }
+            });
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+
+            // Assert
+            var error = AssertErrorPayload(rawResponse, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.InternalError);
+            Assert.NotEmpty(TestServer.UnhandledExceptions);
+            Assert.Equal(TestServer.UnhandledExceptions[0].GetDetailedExceptionMessage(), error.message);
+        }
+
+        [Fact]
+        public async Task TestLanguageServerCommunication()
+        {
+            // bad payload
+            var response1 = await TestServer.OnDataReceivedAsync(JsonSerializer.Serialize(new { })).ConfigureAwait(false);
+
+            // bad jsonrpc payload
+            var response2 = await TestServer.OnDataReceivedAsync(JsonSerializer.Serialize(new
+            {
+                jsonrpc = "2.0"
+            })).ConfigureAwait(false);
+
+            // bad notification payload
+            var response3 = await TestServer.OnDataReceivedAsync(JsonSerializer.Serialize(new
+            {
+                jsonrpc = "2.0",
+                method = "unknown",
+                @params = "unkown"
+            })).ConfigureAwait(false);
+
+            // bad request payload
+            var response4 = await TestServer.OnDataReceivedAsync(JsonSerializer.Serialize(new
+            {
+                jsonrpc = "2.0",
+                id = "abc",
+                method = "unknown",
+                @params = "unkown"
+            })).ConfigureAwait(false);
+
+            // verify we have 4 json rpc responeses
+            AssertErrorPayload(response1, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.InvalidRequest);
+            AssertErrorPayload(response2, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.InvalidRequest);
+            AssertErrorPayload(response3, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.MethodNotFound);
+            AssertErrorPayload(response4, "abc", LanguageServerProtocol.JsonRpcHelper.ErrorCode.MethodNotFound);
+        }
+
+        [Fact]
+        public async Task TestHandlerIsMissing()
+        {
+            // Arrange
+            HandlerFactory.SetHandler(TextDocumentNames.DidOpen, null);
+
+            // Act
+            var payload = GetDidOpenPayload(
+            new DidOpenTextDocumentParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = "https://none",
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = "123"
+                }
+            });
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+
+            // Assert
+            AssertErrorPayload(rawResponse, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.MethodNotFound);
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/LanguageServerForTesting.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.PowerFx.Core;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    internal class LanguageServerForTesting : LanguageServer
+    {
+        private readonly List<Exception> _unhandledExceptions = new List<Exception>();
+
+        public List<Exception> UnhandledExceptions => _unhandledExceptions;
+
+        public LanguageServerForTesting(IPowerFxScopeFactory scopeFactory, ILanguageServerOperationHandlerFactory handlerFactory, IHostTaskExecutor hostTaskExecutor, ILanguageServerLogger logger)
+            : base(scopeFactory, hostTaskExecutor, logger)
+        {
+            HandlerFactory = handlerFactory;
+            LogUnhandledExceptionHandler += (e) => _unhandledExceptions.Add(e);
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/LanguageServerTestBase.cs

@@ -0,0 +1,238 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Web;
+using Microsoft.PowerFx.Core;
+using Microsoft.PowerFx.Core.Tests;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Types;
+using Xunit;
+using static Microsoft.PowerFx.Tests.BindingEngineTests;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public record InitParams(Features features = null, ParserOptions options = null, IPowerFxScopeFactory scopeFactory = null);
+
+    public record LSPError(JsonRpcHelper.ErrorCode code, string message = null);
+
+    public partial class LanguageServerTestBase : PowerFxTest
+    {
+        private LanguageServerForTesting TestServer { get; set; }
+
+        private TestHandlerFactory HandlerFactory { get; set; }
+
+        public TestLogger Logger { get; set; }
+
+        public TestHostTaskExecutor HostTaskExecutor { get; set; }
+
+        public LanguageServerTestBase()
+            : base()
+        {
+            Init();
+        }
+
+        internal void Init(InitParams initParams)
+        {
+            var config = new PowerFxConfig(features: initParams?.features ?? Features.None);
+            config.AddFunction(new BehaviorFunction());
+            config.AddFunction(new AISummarizeFunction());
+
+            var engine = new Engine(config);
+            HandlerFactory = new TestHandlerFactory();
+            HostTaskExecutor = new TestHostTaskExecutor();
+            var random = new Random();
+            var useHostTaskExecutor = random.Next(0, 2) == 1;
+
+            var scopeFactory = initParams?.scopeFactory ?? new TestPowerFxScopeFactory(
+                               (string documentUri) => engine.CreateEditorScope(initParams?.options, GetFromUri(documentUri)));
+
+            TestServer = new LanguageServerForTesting(scopeFactory, HandlerFactory, useHostTaskExecutor ? HostTaskExecutor : null, Logger);
+        }
+
+        internal void Init()
+        {
+            Init(new InitParams());
+        }
+
+        // The convention for getting the context from the documentUri is arbitrary and determined by the host. 
+        internal static ReadOnlySymbolTable GetFromUri(string documentUri)
+        {
+            var uriObj = new Uri(documentUri);
+            var json = HttpUtility.ParseQueryString(uriObj.Query).Get("context");
+            json ??= "{}";
+
+            var record = (RecordValue)FormulaValueJSON.FromJson(json);
+            return ReadOnlySymbolTable.NewFromRecord(record.Type);
+        }
+
+        internal static LSPError AssertErrorPayload(string response, string id, JsonRpcHelper.ErrorCode expectedCode)
+        {
+            Assert.NotNull(response);
+            var deserializedResponse = JsonDocument.Parse(response);
+            var root = deserializedResponse.RootElement;
+            Assert.True(root.TryGetProperty("id", out var responseId));
+            Assert.Equal(id, responseId.GetString());
+            Assert.True(root.TryGetProperty("error", out var errElement));
+            Assert.True(errElement.TryGetProperty("code", out var codeElement));
+            var code = (JsonRpcHelper.ErrorCode)codeElement.GetInt32();
+            Assert.Equal(expectedCode, code);
+            Assert.True(root.TryGetProperty("fxVersion", out var fxVersionElement));
+            Assert.Equal(Engine.AssemblyVersion, fxVersionElement.GetString());
+            string message = null;
+            if (errElement.TryGetProperty("message", out var messageElement))
+            {
+                message = messageElement.GetString();
+            }
+
+            return new LSPError(code, message);
+        }
+
+        internal static string GetOutputAtIndexInSerializedResponse(string response, string id = null,  string method = null, int index = -1)
+        {
+            Assert.NotNull(response);
+            try
+            {
+                var possiblyArray = JsonSerializer.Deserialize<List<string>>(response, LanguageServerHelper.DefaultJsonSerializerOptions);
+                if (possiblyArray == null || possiblyArray.Count == 0)
+                {
+                    return response;
+                }
+
+                if (index >= 0 && index < possiblyArray.Count)
+                {
+                    response = possiblyArray[index];
+                }
+                else if (method != null) 
+                {
+                    var match = possiblyArray.Where(item => item.Contains(method)).FirstOrDefault();
+                    if (match != null || match != default)
+                    {
+                        response = match;
+                    }
+                }
+                else
+                {
+                    var match = possiblyArray.Where(item => item.Contains(id)).FirstOrDefault();
+                    if (match != null || match != default)
+                    {
+                        response = match;
+                    }
+                }
+            }
+            catch (JsonException)
+            {
+                // ignore
+            }
+
+            return response;
+        }
+
+        internal static T AssertAndGetResponsePayload<T>(string response, string id, int index = -1)
+        {
+            response = GetOutputAtIndexInSerializedResponse(response, id, null, index);
+            var deserializedResponse = JsonDocument.Parse(response);
+            var root = deserializedResponse.RootElement;
+            root.TryGetProperty("id", out var responseId);
+            Assert.Equal(id, responseId.GetString());
+            root.TryGetProperty("result", out var resultElement);
+            Assert.True(root.TryGetProperty("fxVersion", out var fxVersionElement));
+            Assert.Equal(Engine.AssemblyVersion, fxVersionElement.GetString());
+            var paramsObj = JsonSerializer.Deserialize<T>(resultElement.GetRawText(), LanguageServerHelper.DefaultJsonSerializerOptions);
+            return paramsObj;
+        }
+
+        internal static T AssertAndGetNotificationParams<T>(string response, string method, int index = -1)
+        {
+            Assert.NotNull(response);
+            response = GetOutputAtIndexInSerializedResponse(response, null, method, index);
+            var notification = GetOutputAtIndexInSerializedResponse(response);
+            var deserializedNotification = JsonDocument.Parse(notification);
+            var root = deserializedNotification.RootElement;
+            Assert.True(root.TryGetProperty("method", out var methodElement));
+            Assert.Equal(method, methodElement.GetString());
+            Assert.True(root.TryGetProperty("params", out var paramsElement));
+            T paramsObj;
+
+            if (method == CustomProtocolNames.PublishExpressionType)
+            {
+                paramsObj = JsonRpcHelper.Deserialize<T>(paramsElement.GetRawText());
+            }
+            else
+            {
+                paramsObj = JsonSerializer.Deserialize<T>(paramsElement.GetRawText(), LanguageServerHelper.DefaultJsonSerializerOptions);
+            }
+
+            return paramsObj;
+        }
+
+        internal static string GetUri(string queryParams = null)
+        {
+            var uriBuilder = new UriBuilder("powerfx://app")
+            {
+                Query = queryParams ?? string.Empty
+            };
+            return uriBuilder.Uri.AbsoluteUri;
+        }
+
+        internal static (string payload, string id) GetRequestPayload<T>(T paramsObj, string method, string id = null)
+        {
+            id ??= Guid.NewGuid().ToString();
+            var payload = JsonSerializer.Serialize(
+            new
+            {
+                jsonrpc = "2.0",
+                id,
+                method,
+                @params = paramsObj
+            }, LanguageServerHelper.DefaultJsonSerializerOptions);
+            return (payload, id);
+        }
+
+        internal static string GetNotificationPayload<T>(T paramsObj, string method)
+        {
+            var payload = JsonSerializer.Serialize(
+            new
+            {
+                jsonrpc = "2.0",
+                method,
+                @params = paramsObj
+            }, LanguageServerHelper.DefaultJsonSerializerOptions);
+            return payload;
+        }
+
+        internal static TextDocumentIdentifier GetTextDocument(string uri = null)
+        {
+            return new TextDocumentIdentifier() { Uri = uri ?? GetUri() };
+        }
+
+        internal static string GetExpression(LanguageServerRequestBaseParams requestParams)
+        {
+            if (requestParams?.Text != null)
+            {
+                return requestParams.Text;
+            }
+
+            var uri = new Uri(requestParams.TextDocument.Uri);
+            return HttpUtility.ParseQueryString(uri.Query).Get("expression");
+        }
+
+        internal static Position GetPosition(int offset, int line = 0)
+        {
+            return new Position()
+            {
+                Line = line,
+                Character = offset
+            };
+        }
+
+        internal static ParserOptions GetParserOptions(bool withAllowSideEffects)
+        {
+            return withAllowSideEffects ? new ParserOptions() { AllowsSideEffects = true } : null;
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/Nl2FxTests.cs

@@ -0,0 +1,209 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Types;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    internal class TestNL2FxHandler : NLHandler
+    {
+        public bool _supportsNL2Fx = true;
+        public bool _supportsFx2NL = true;
+
+        public override bool SupportsNL2Fx => _supportsNL2Fx;
+
+        public override bool SupportsFx2NL => _supportsFx2NL;
+
+        public const string ModelIdStr = "Model123";
+
+        // Set on call to NL2Fx 
+        public string _log;
+
+        // Set the expected expression to return. 
+        public string Expected { get; set; }
+
+        public bool Throw { get; set; }
+
+        public bool Delay { get; set; } = false;
+
+        public int PreHandleNl2FxCallCount { get; set; } = 0;
+
+        public TestNL2FxHandler()
+        {
+        }
+
+        public override async Task<CustomNL2FxResult> NL2FxAsync(NL2FxParameters request, CancellationToken cancel)
+        {
+            if (this.Delay)
+            {
+                await Task.Delay(100, cancel).ConfigureAwait(false);
+            }
+
+            if (this.Throw)
+            {
+                throw new InvalidOperationException($"Simulated error");
+            }
+
+            var nl2FxParameters = request;
+
+            Assert.NotNull(nl2FxParameters.Engine);
+
+            var sb = new StringBuilder();
+            sb.Append(nl2FxParameters.Sentence);
+            sb.Append(": ");
+
+            sb.Append(this.Expected);
+            sb.Append(": ");
+
+            foreach (var sym in nl2FxParameters.SymbolSummary.SuggestedSymbols)
+            {
+                sb.Append($"{sym.BestName},{sym.Type}");
+            }
+
+            _log = sb.ToString();
+
+            var nl2FxResult = new CustomNL2FxResult
+            {
+                Expressions = new CustomNL2FxResultItem[]
+                {
+                        new CustomNL2FxResultItem
+                        {
+                            Expression = this.Expected,
+                            ModelId = ModelIdStr
+                        }
+                }
+            };
+
+            return nl2FxResult;
+        }
+
+        public override void PreHandleNl2Fx(CustomNL2FxParams nl2FxRequestParams, NL2FxParameters nl2fxParameters, LanguageServerOperationContext operationContext)
+        {
+            this.PreHandleNl2FxCallCount++;
+        }
+    }
+
+    public partial class LanguageServerTestBase
+    {
+        [Theory]
+        [InlineData("Score < 50", true, "#$PowerFxResolvedObject$# < #$decimal$#")]
+        [InlineData("missing < 50", false, "#$firstname$# < #$decimal$#")] // doesn't compile, should get filtered out by LSP 
+        public async Task TestNL2FX(string expectedExpr, bool success, string anonExpr = null)
+        {
+            // Arrange
+            var documentUri = "powerfx://app?context=1";
+            var engine = new Engine();
+            var symbols = new SymbolTable();
+            symbols.AddVariable("Score", FormulaType.Number);
+            var scope = engine.CreateEditorScope(symbols: symbols);
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) => scope);
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var nl2FxHandler = CreateAndConfigureNl2FxHandler();
+            nl2FxHandler.Delay = true;
+            nl2FxHandler.Expected = expectedExpr;
+
+            // Act
+            var payload = NL2FxMessageJson(documentUri);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<CustomNL2FxResult>(rawResponse, payload.id);
+
+            // Assert
+            // result has expected concat with symbols. 
+            var items = response.Expressions;
+
+            Assert.Single(items);
+            var expression = items[0];
+
+            if (anonExpr != null)
+            {
+                Assert.Equal(anonExpr, items[0].AnonymizedExpression);
+            }
+
+            if (success)
+            {
+                Assert.Equal("my sentence: Score < 50: Score,Number", nl2FxHandler._log);
+
+                Assert.Equal(expectedExpr, expression.Expression);
+                Assert.Null(expression.RawExpression);
+            }
+            else
+            {
+                // Even though model returned an expression, it didn't compile, so it should be filtered out by LSP.
+                Assert.Null(expression.Expression);
+                Assert.Equal(expectedExpr, expression.RawExpression);
+            }
+
+            Assert.Equal(TestNL2FxHandler.ModelIdStr, expression.ModelId);
+            Assert.Equal(1, nl2FxHandler.PreHandleNl2FxCallCount);
+        }
+
+        [Fact]
+        public async Task TestNL2FXMissingHandler()
+        {
+            // Arrange
+            HandlerFactory.SetHandler(CustomProtocolNames.NL2FX, null);
+            var documentUri = "powerfx://app?context={\"A\":1,\"B\":[1,2,3]}";
+
+            // Act
+            var payload = NL2FxMessageJson(documentUri);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            AssertErrorPayload(rawResponse, payload.id, JsonRpcHelper.ErrorCode.MethodNotFound);
+        }
+
+        [Fact]
+        public async Task TestNL2FXHandlerThrows()
+        {
+            // Arrange
+            var documentUri = "powerfx://app?context=1";
+            var engine = new Engine();
+            var symbols = new SymbolTable();
+            symbols.AddVariable("Score", FormulaType.Number);
+            var scope = engine.CreateEditorScope(symbols: symbols);
+            var scopeFactory = new TestPowerFxScopeFactory((string documentUri) => scope);
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var nl2FxHandler = CreateAndConfigureNl2FxHandler();
+            nl2FxHandler.Delay = true;
+            nl2FxHandler.Throw = true;
+
+            // Act
+            var payload = NL2FxMessageJson(documentUri);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            AssertErrorPayload(rawResponse, payload.id, JsonRpcHelper.ErrorCode.InternalError);
+            Assert.NotEmpty(TestServer.UnhandledExceptions);
+            Assert.Equal(1, nl2FxHandler.PreHandleNl2FxCallCount);
+        }
+
+        private static (string payload, string id) NL2FxMessageJson(string documentUri)
+        {
+            var nl2FxParams = new CustomNL2FxParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1
+                },
+                Sentence = "my sentence"
+            };
+
+            return GetRequestPayload(nl2FxParams, CustomProtocolNames.NL2FX);
+        }
+
+        private TestNL2FxHandler CreateAndConfigureNl2FxHandler()
+        {
+            var nl2FxHandler = new TestNL2FxHandler();
+            HandlerFactory.SetHandler(CustomProtocolNames.NL2FX, new Nl2FxLanguageServerOperationHandler(new BackwardsCompatibleNLHandlerFactory(nl2FxHandler)));
+            return nl2FxHandler;
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/OnDidChangeTests.cs

@@ -0,0 +1,121 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Core.Localization;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    internal class TestOnChangeHandler
+    {
+        public int CallCounts = 0;
+
+        public void OnDidChange(DidChangeTextDocumentParams didChangeTextDocumentParams)
+        {
+            CallCounts++;
+        }
+    }
+
+    public partial class LanguageServerTestBase
+    {
+        [Theory]
+        [InlineData("A+CountRows(B)", false)]
+        [InlineData("Behavior(); A+CountRows(B)", true)]
+        public async Task TestDidChange(string text, bool withAllowSideEffects)
+        {
+            Init(new InitParams(options: GetParserOptions(withAllowSideEffects)));
+            var handler = CreateAndConfigureOnChangeHandler();
+
+            // test good formula
+            var payload = GetNotificationPayload(
+            new DidChangeTextDocumentParams()
+            {
+                TextDocument = new VersionedTextDocumentIdentifier()
+                {
+                    Uri = "powerfx://app?context={\"A\":1,\"B\":[1,2,3]}",
+                    Version = 1,
+                },
+                ContentChanges = new TextDocumentContentChangeEvent[]
+                    {
+                        new TextDocumentContentChangeEvent() { Text = text }
+                    }
+            }, TextDocumentNames.DidChange);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            var notification = GetDiagnosticsParams(rawResponse);
+            Assert.Equal("powerfx://app?context={\"A\":1,\"B\":[1,2,3]}", notification.Uri);
+            Assert.Empty(notification.Diagnostics);
+
+            // test bad formula
+            payload = GetNotificationPayload(
+                new DidChangeTextDocumentParams()
+                {
+                    TextDocument = new VersionedTextDocumentIdentifier()
+                    {
+                        Uri = "powerfx://app",
+                        Version = 1,
+                    },
+                    ContentChanges = new TextDocumentContentChangeEvent[]
+                    {
+                        new TextDocumentContentChangeEvent() { Text = "AA" }
+                    }
+                },
+                TextDocumentNames.DidChange);
+            rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            notification = GetDiagnosticsParams(rawResponse);
+            Assert.Equal("powerfx://app", notification.Uri);
+            Assert.Single(notification.Diagnostics);
+            Assert.Equal("Name isn't valid. 'AA' isn't recognized.", notification.Diagnostics[0].Message);
+
+            // some invalid cases
+            rawResponse = await TestServer.OnDataReceivedAsync(JsonSerializer.Serialize(new { })).ConfigureAwait(false);
+            AssertErrorPayload(rawResponse, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.InvalidRequest);
+
+            rawResponse = await TestServer.OnDataReceivedAsync(JsonSerializer.Serialize(new
+            {
+                jsonrpc = "2.0",
+                method = "textDocument/didChange"
+            })).ConfigureAwait(false);
+            AssertErrorPayload(rawResponse, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.InvalidRequest);
+
+            rawResponse = await TestServer.OnDataReceivedAsync(JsonSerializer.Serialize(new
+            {
+                jsonrpc = "2.0",
+                method = "textDocument/didChange",
+                @params = string.Empty
+            })).ConfigureAwait(false);
+            AssertErrorPayload(rawResponse, null, LanguageServerProtocol.JsonRpcHelper.ErrorCode.ParseError);
+
+            Assert.True(handler.CallCounts == 2);
+        }
+
+        private static PublishDiagnosticsParams GetDiagnosticsParams(string response)
+        {
+            return AssertAndGetNotificationParams<PublishDiagnosticsParams>(response, TextDocumentNames.PublishDiagnostics);
+        }
+
+        private TestOnChangeHandler CreateAndConfigureOnChangeHandler()
+        {
+            var handler = new TestOnChangeHandler();
+            HandlerFactory.SetHandler(TextDocumentNames.DidChange, new OnDidChangeLanguageServerNotificationHandler(handler.OnDidChange));
+            return handler;
+        }
+
+        private void CheckBehaviorError(string response, bool expectBehaviorError, out Diagnostic[] diags)
+        {
+            diags = GetDiagnosticsParams(response).Diagnostics;
+
+            if (expectBehaviorError)
+            {
+                Assert.Contains(diags, d => d.Message == StringResources.GetErrorResource(TexlStrings.ErrBehaviorPropertyExpected).GetSingleValue(ErrorResource.ShortMessageTag));
+            }
+            else
+            {
+                Assert.DoesNotContain(diags, d => d.Message == StringResources.GetErrorResource(TexlStrings.ErrBehaviorPropertyExpected).GetSingleValue(ErrorResource.ShortMessageTag));
+            }
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/OnDidOpenTests.cs

@@ -0,0 +1,302 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Intellisense;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.Types;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public partial class LanguageServerTestBase
+    {
+        private async Task TestPublishDiagnostics(string uri, string method, string formula, Diagnostic[] expectedDiagnostics)
+        {
+            var payload = GetDidOpenPayload(new DidOpenTextDocumentParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = uri,
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = formula
+                }
+            });
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            var notification = GetDiagnosticsParams(rawResponse);
+            Assert.Equal(uri, notification.Uri);
+            Assert.Equal(expectedDiagnostics.Length, notification.Diagnostics.Length);
+
+            var diagnosticsSet = new HashSet<Diagnostic>(expectedDiagnostics);
+            for (var i = 0; i < expectedDiagnostics.Length; i++)
+            {
+                var expectedDiagnostic = expectedDiagnostics[i];
+                var actualDiagnostic = notification.Diagnostics[i];
+                Assert.True(diagnosticsSet.Where(x => x.Message == actualDiagnostic.Message).Count() == 1);
+                diagnosticsSet.RemoveWhere(x => x.Message == actualDiagnostic.Message);
+            }
+
+            Assert.True(diagnosticsSet.Count() == 0);
+        }
+
+        [Theory]
+        [InlineData("A+CountRows(B)", "{\"A\":1,\"B\":[1,2,3]}")]
+        public async Task TestDidOpenValidFormula(string formula, string context = null)
+        {
+            var uri = $"powerfx://app{(context != null ? "powerfx://app?context=" + context : string.Empty)}";
+            await TestPublishDiagnostics(uri, "textDocument/didOpen", formula, new Diagnostic[0]).ConfigureAwait(false);
+        }
+
+        [Theory]
+        [InlineData("AA", "Name isn't valid. 'AA' isn't recognized.")]
+        [InlineData("1+CountRowss", "Name isn't valid. 'CountRowss' isn't recognized.")]
+        [InlineData("CountRows(2)", "Invalid argument type (Decimal). Expecting a Table value instead.", "The function 'CountRows' has some invalid arguments.")]
+        public async Task TestDidOpenErroneousFormula(string formula, params string[] expectedErrors)
+        {
+            var expectedDiagnostics = expectedErrors.Select(error => new Diagnostic()
+            {
+                Message = error,
+                Severity = DiagnosticSeverity.Error
+            }).ToArray();
+            await TestPublishDiagnostics("powerfx://app", "textDocument/didOpen", formula, expectedDiagnostics).ConfigureAwait(false);
+        }
+
+        [Fact]
+        public async Task TestDidOpenSeverityFormula()
+        {
+            var formula = "Count([\"test\"])";
+            var expectedDiagnostics = new[]
+            {
+                new Diagnostic()
+                {
+                    Message = "Invalid schema, expected a column of Number values for 'Value'.",
+                    Severity = DiagnosticSeverity.Warning
+                },
+                new Diagnostic()
+                {
+                    Message = "The function 'Count' has some invalid arguments.",
+                    Severity = DiagnosticSeverity.Error
+                },
+            };
+            await TestPublishDiagnostics("powerfx://app", "textDocument/didOpen", formula, expectedDiagnostics).ConfigureAwait(false);
+        }
+
+        [Theory]
+        [InlineData("Concatenate(", 12, false, false)]
+        [InlineData("Behavior(); Concatenate(", 24, true, false)]
+        [InlineData("Behavior(); Concatenate(", 24, false, true)]
+        public async Task TestDidOpenWithErrors(string text, int offset, bool withAllowSideEffects, bool expectBehaviorError)
+        {
+            Init(new InitParams(options: GetParserOptions(withAllowSideEffects)));
+            var payload = GetDidOpenPayload(new DidOpenTextDocumentParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = "powerfx://app",
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = text
+                }
+            }); 
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            CheckBehaviorError(rawResponse, expectBehaviorError, out var diags);
+
+            var diag = diags.First(d => d.Message == "Unexpected characters. The formula contains 'Eof' where 'ParenClose' is expected.");
+
+            Assert.Equal(offset, diag.Range.Start.Character);
+            Assert.Equal(offset, diag.Range.End.Character);
+        }
+
+        [Theory]
+        [InlineData("A+CountRows(B)", 3, false, false)]
+        [InlineData("Behavior(); A+CountRows(B)", 4, true, false)]
+        [InlineData("Behavior(); A+CountRows(B)", 4, false, true)]
+        public async Task TestPublishTokens(string text, int count, bool withAllowSideEffects, bool expectBehaviorError)
+        {
+            Init(new InitParams(options: GetParserOptions(withAllowSideEffects)));
+
+            // getTokensFlags = 0x0 (none), 0x1 (tokens inside expression), 0x2 (all functions)
+            var documentUri = "powerfx://app?context={\"A\":1,\"B\":[1,2,3]}&getTokensFlags=1";
+            var payload = GetDidOpenPayload(new DidOpenTextDocumentParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = text
+                }
+            });
+
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            var response = GetPublishTokensParams(rawResponse);
+            Assert.Equal(documentUri, response.Uri);
+            Assert.Equal(count, response.Tokens.Count);
+            Assert.Equal(TokenResultType.Variable, response.Tokens["A"]);
+            Assert.Equal(TokenResultType.Variable, response.Tokens["B"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["CountRows"]);
+
+            CheckBehaviorError(rawResponse, expectBehaviorError, out _);
+
+            if (count == 4)
+            {
+                Assert.Equal(TokenResultType.Function, response.Tokens["Behavior"]);
+            }
+
+            // getTokensFlags = 0x0 (none), 0x1 (tokens inside expression), 0x2 (all functions)
+            documentUri = "powerfx://app?context={\"A\":1,\"B\":[1,2,3]}&getTokensFlags=2";
+            payload = GetNotificationPayload(
+                new DidChangeTextDocumentParams()
+            {
+                TextDocument = new VersionedTextDocumentIdentifier()
+                {
+                    Uri = documentUri,
+                    Version = 1,
+                },
+                ContentChanges = new TextDocumentContentChangeEvent[]
+                {
+                    new TextDocumentContentChangeEvent() { Text = text }
+                }
+            }, TextDocumentNames.DidChange);
+
+            rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            response = GetPublishTokensParams(rawResponse);
+
+            Assert.Equal(documentUri, response.Uri);
+            Assert.Equal(0, Enumerable.Count(response.Tokens.Where(it => it.Value != TokenResultType.Function)));
+            Assert.Equal(TokenResultType.Function, response.Tokens["Abs"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["Clock.AmPm"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["CountRows"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["VarP"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["Year"]);
+
+            CheckBehaviorError(rawResponse, expectBehaviorError, out _);
+
+            // getTokensFlags = 0x0 (none), 0x1 (tokens inside expression), 0x2 (all functions)
+            documentUri = "powerfx://app?context={\"A\":1,\"B\":[1,2,3]}&getTokensFlags=3";
+            payload = GetNotificationPayload(
+                    new DidChangeTextDocumentParams()
+                    {
+                        TextDocument = new VersionedTextDocumentIdentifier()
+                        {
+                            Uri = documentUri,
+                            Version = 1,
+                        },
+                        ContentChanges = new TextDocumentContentChangeEvent[]
+                        {
+                            new TextDocumentContentChangeEvent() { Text = text }
+                        }
+                    }, TextDocumentNames.DidChange);
+
+            rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            response = GetPublishTokensParams(rawResponse);
+
+            Assert.Equal(documentUri, response.Uri);
+            Assert.Equal(TokenResultType.Variable, response.Tokens["A"]);
+            Assert.Equal(TokenResultType.Variable, response.Tokens["B"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["Abs"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["Clock.AmPm"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["CountRows"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["VarP"]);
+            Assert.Equal(TokenResultType.Function, response.Tokens["Year"]);
+
+            CheckBehaviorError(rawResponse, expectBehaviorError, out _);
+        }
+
+        [Theory]
+        [InlineData("{\"A\": 1 }", "A+2", typeof(DecimalType))]
+        [InlineData("{}", "\"hi\"", typeof(StringType))]
+        [InlineData("{}", "", typeof(BlankType))]
+        [InlineData("{}", "{ A: 1 }", typeof(KnownRecordType))]
+        [InlineData("{}", "[1, 2, 3]", typeof(TableType))]
+        [InlineData("{}", "true", typeof(BooleanType))]
+        public async Task TestPublishExpressionType(string context, string expression, System.Type expectedType)
+        {
+            var documentUri = $"powerfx://app?context={context}&getExpressionType=true";
+            var payload = GetDidOpenPayload(new DidOpenTextDocumentParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = expression
+                }
+            });
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            var response = GetPublishExpressionTypeParams(rawResponse);
+
+            Assert.Equal(documentUri, response.Uri);
+            Assert.IsType(expectedType, response.Type);
+        }
+
+        [Theory]
+        [InlineData("{\"A\": 1 }", "invalid+A")]
+        [InlineData("{}", "B")]
+        [InlineData("{}", "+")]
+        public async Task TestPublishExpressionType_Null(string context, string expression)
+        {
+            var documentUri = $"powerfx://app?context={context}&getExpressionType=true";
+            var payload = GetDidOpenPayload(new DidOpenTextDocumentParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = expression
+                }
+            });
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            var response = GetPublishExpressionTypeParams(rawResponse);
+            Assert.Equal(documentUri, response.Uri);
+            Assert.Null(response.Type);
+        }
+
+        [Theory]
+        [InlineData(false, "{}", "{ A: 1 }", @"{""Type"":""Record"",""Fields"":{""A"":{""Type"":""Decimal""}}}")]
+        [InlineData(false, "{}", "[1, 2]", @"{""Type"":""Table"",""Fields"":{""Value"":{""Type"":""Decimal""}}}")]
+        [InlineData(true, "{}", "[{ A: 1 }, { B: true }]", @"{""Type"":""Table"",""Fields"":{""A"":{""Type"":""Decimal""},""B"":{""Type"":""Boolean""}}}")]
+        [InlineData(false, "{}", "[{ A: 1 }, { B: true }]", @"{""Type"":""Table"",""Fields"":{""Value"":{""Type"":""Record"",""Fields"":{""A"":{""Type"":""Decimal""},""B"":{""Type"":""Boolean""}}}}}")]
+        [InlineData(false, "{}", "{A: 1, B: { C: { D: \"Qwerty\" }, E: true } }", @"{""Type"":""Record"",""Fields"":{""A"":{""Type"":""Decimal""},""B"":{""Type"":""Record"",""Fields"":{""C"":{""Type"":""Record"",""Fields"":{""D"":{""Type"":""String""}}},""E"":{""Type"":""Boolean""}}}}}")]
+        [InlineData(false, "{}", "{ type: 123 }", @"{""Type"":""Record"",""Fields"":{""type"":{""Type"":""Decimal""}}}")]
+        public async Task TestPublishExpressionType_AggregateShapes(bool tableSyntaxDoesntWrapRecords, string context, string expression, string expectedTypeJson)
+        {
+            Init(new InitParams(features: new Features { TableSyntaxDoesntWrapRecords = tableSyntaxDoesntWrapRecords }));
+            var documentUri = $"powerfx://app?context={context}&getExpressionType=true";
+            var payload = GetDidOpenPayload(new DidOpenTextDocumentParams()
+            {
+                TextDocument = new TextDocumentItem()
+                {
+                    Uri = documentUri,
+                    LanguageId = "powerfx",
+                    Version = 1,
+                    Text = expression
+                }
+            });
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload).ConfigureAwait(false);
+            var response = GetPublishExpressionTypeParams(rawResponse);
+            Assert.Equal(documentUri, response.Uri);
+            Assert.Equal(expectedTypeJson, JsonRpcHelper.Serialize(response.Type));
+        }
+
+        private static string GetDidOpenPayload(DidOpenTextDocumentParams openTextDocumentParams)
+        {
+            return GetNotificationPayload(openTextDocumentParams, TextDocumentNames.DidOpen);
+        }
+
+        private static PublishTokensParams GetPublishTokensParams(string response)
+        {
+            return AssertAndGetNotificationParams<PublishTokensParams>(response, CustomProtocolNames.PublishTokens);
+        }
+
+        private static PublishExpressionTypeParams GetPublishExpressionTypeParams(string response)
+        {
+            return AssertAndGetNotificationParams<PublishExpressionTypeParams>(response, CustomProtocolNames.PublishExpressionType);
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/SemanticTokensTests.cs

@@ -0,0 +1,404 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.Core.Texl.Intellisense;
+using Microsoft.PowerFx.Interpreter.Tests.LanguageServiceProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Schemas;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public partial class LanguageServerTestBase
+    {
+        #region Full Document Semantic Tokens Tests
+        [Fact]
+        public async Task TestCorrectFullSemanticTokensAreReturnedWithExpressionInUri()
+        {
+            await TestCorrectFullSemanticTokensAreReturned(new SemanticTokensParams
+            {
+                TextDocument = GetTextDocument(GetUri("expression=Max(1, 2, 3)"))
+            }).ConfigureAwait(false);
+        }
+
+        [Fact]
+        public async Task TestCorrectFullSemanticTokensAreReturnedWithExpressionNotInUri()
+        {
+            await TestCorrectFullSemanticTokensAreReturned(new SemanticTokensParams
+            {
+                TextDocument = GetTextDocument(),
+                Text = "Max(1, 2, 3)"
+            }).ConfigureAwait(false);
+        }
+
+        [Fact]
+        public async Task TestCorrectFullSemanticTokensAreReturnedWithExpressionInBothUriAndTextDocument()
+        {
+            var expression = "Max(1, 2, 3)";
+            var semanticTokenParams = new SemanticTokensParams
+            {
+                TextDocument = GetTextDocument(GetUri("expression=Color.White")),
+                Text = expression
+            };
+            await TestCorrectFullSemanticTokensAreReturned(semanticTokenParams).ConfigureAwait(false);
+        }
+
+        private async Task TestCorrectFullSemanticTokensAreReturned(SemanticTokensParams semanticTokensParams)
+        {
+            // Arrange
+            Init();
+            var expression = GetExpression(semanticTokensParams);
+            Assert.Equal("Max(1, 2, 3)", expression);
+            var payload = GetFullDocumentSemanticTokensRequestPayload(semanticTokensParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var response = AssertAndGetSemanticTokensResponse(rawResponse, payload.id);
+            Assert.NotEmpty(response.Data);
+            var decodedTokens = SemanticTokensRelatedTestsHelper.DecodeEncodedSemanticTokensPartially(response, expression);
+            Assert.Single(decodedTokens.Where(tok => tok.TokenType == TokenType.Function));
+            Assert.Equal(3, decodedTokens.Where(tok => tok.TokenType == TokenType.NumLit || tok.TokenType == TokenType.DecLit).Count());
+        }
+
+        [Theory]
+        [InlineData("Create", TokenType.Function, TokenType.BoolLit)]
+        [InlineData(null)]
+        [InlineData("")]
+        [InlineData("[2, 3")]
+        [InlineData("Create", TokenType.StrInterpStart, TokenType.BinaryOp, TokenType.NumLit, TokenType.DecLit, TokenType.Control)]
+        [InlineData("[]")]
+        [InlineData("1,2]")]
+        [InlineData("[98]")]
+        [InlineData("Create", TokenType.Lim)]
+        [InlineData("Create", TokenType.BoolLit, TokenType.BinaryOp, TokenType.Function, TokenType.Lim)]
+        [InlineData("Create", TokenType.Lim, TokenType.BinaryOp, TokenType.BoolLit)]
+        [InlineData("   ")]
+        [InlineData("NotPresent")]
+        internal async Task TestCorrectFullSemanticTokensAreReturnedWithCertainTokenTypesSkipped(string tokenTypesToSkipParam, params TokenType[] tokenTypesToSkip)
+        {
+            // Arrange
+            var expression = "1+-2;true;\"String Literal\";Sum(1,2);Max(1,2,3);$\"1 + 2 = {3}\";// This is Comment";
+            var expectedTypes = new List<TokenType> { TokenType.DecLit, TokenType.BoolLit, TokenType.Comment, TokenType.Function, TokenType.StrInterpStart, TokenType.IslandEnd, TokenType.IslandStart, TokenType.StrLit, TokenType.StrInterpEnd, TokenType.Delimiter, TokenType.BinaryOp };
+            if (tokenTypesToSkip.Length > 0)
+            {
+                expectedTypes = expectedTypes.Where(expectedType => !tokenTypesToSkip.Contains(expectedType)).ToList();
+            }
+
+            if (tokenTypesToSkipParam == "Create")
+            {
+                tokenTypesToSkipParam = JsonSerializer.Serialize(tokenTypesToSkip.Select(tokType => (int)tokType).ToList());
+            }
+
+            var semanticTokenParams = new SemanticTokensParams
+            {
+                TextDocument = GetTextDocument(GetUri(tokenTypesToSkipParam == "NotPresent" ? string.Empty : "tokenTypesToSkip=" + tokenTypesToSkipParam)),
+                Text = expression
+            };
+            var payload = GetFullDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var response = AssertAndGetSemanticTokensResponse(rawResponse, payload.id);
+            Assert.NotEmpty(response.Data);
+            var decodedTokens = SemanticTokensRelatedTestsHelper.DecodeEncodedSemanticTokensPartially(response, expression);
+            var actualTypes = decodedTokens.Select(tok => tok.TokenType).Distinct().ToList();
+            Assert.Equal(expectedTypes.OrderBy(type => type), actualTypes.OrderBy(type => type));
+        }
+
+        [Fact]
+        public async Task TestErrorResponseReturnedWhenUriIsNullForFullSemanticTokensRequest()
+        {
+            // Arrange
+            var semanticTokenParams = new SemanticTokensParams
+            {
+                TextDocument = new TextDocumentIdentifier() { Uri = null }
+            };
+            var payload = GetFullDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var response = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            AssertErrorPayload(response, payload.id, JsonRpcHelper.ErrorCode.ParseError);
+        }
+
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task TestEmptyFullSemanticTokensResponseReturnedWhenExpressionIsInvalid(bool isNotNull)
+        {
+            // Arrange
+            string expression = string.Empty;
+            var semanticTokenParams = new SemanticTokensParams
+            {
+                TextDocument = GetTextDocument(),
+                Text = isNotNull ? expression : null
+            };
+            var payload = GetFullDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var response = AssertAndGetSemanticTokensResponse(rawResponse, payload.id);
+            Assert.Empty(response.Data);
+        }
+
+        [Fact]
+        public async Task TestFullSemanticTokensResponseReturnedWithDefaultEOL()
+        {
+            // Arrange
+            string expression = "Sum(\n1,1\n)";
+            var semanticTokenParams = new SemanticTokensParams
+            {
+                TextDocument = GetTextDocument(),
+                Text = expression
+            };
+            var payload = GetFullDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var response = AssertAndGetSemanticTokensResponse(rawResponse, payload.id);
+            Assert.NotEmpty(response.Data);
+            Assert.Equal(expression.Where(c => c == '\n').Count(), SemanticTokensRelatedTestsHelper.DetermineNumberOfLinesThatTokensAreSpreadAcross(response));
+        }
+
+        [Theory]
+        [InlineData("9 + 9", 0)]
+        [InlineData("Max(10, 20, 30)", 0)]
+        [InlineData("Color.AliceBlue", 0)]
+        [InlineData("9 + 9\n Max(1, 3, 19); \n Color.AliceBlue", 0)]
+        [InlineData("Label2.Text", 1)]
+        [InlineData("NestedLabel1", 1)]
+        [InlineData("Label2.Text;\nNestedLabel1", 2)]
+        public async Task TestPublishControlTokensNotification(string expression, int expectedNumberOfControlTokens)
+        {
+            // Arrange
+            var checkResult = SemanticTokensRelatedTestsHelper.GetCheckResultWithControlSymbols(expression);
+            var scopeFactory = new TestPowerFxScopeFactory(
+                (string documentUri) => new EditorContextScope(
+                    (expr) => checkResult));
+            Init(new InitParams(scopeFactory: scopeFactory));
+            var payload = GetFullDocumentSemanticTokensRequestPayload(new SemanticTokensParams
+            {
+                TextDocument = GetTextDocument(GetUri("&version=someVersionId")),
+                Text = expression
+            });
+
+            // Act
+            var response = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var notificationParams = AssertAndGetNotificationParams<PublishControlTokensParams>(response, CustomProtocolNames.PublishControlTokens);
+            Assert.Equal("someVersionId", notificationParams.Version);
+            var controlTokenList = notificationParams.Controls;
+            Assert.Equal(expectedNumberOfControlTokens, controlTokenList.Count());
+            foreach (var controlToken in controlTokenList)
+            {
+                Assert.Equal(typeof(ControlToken), controlToken.GetType());
+            }
+        }
+
+        private static (string payload, string id) GetFullDocumentSemanticTokensRequestPayload(SemanticTokensParams semanticTokenParams, string id = null)
+        {
+            return GetRequestPayload(semanticTokenParams, TextDocumentNames.FullDocumentSemanticTokens, id);
+        }
+
+        #endregion
+
+        #region Range Document Semantic Tokens Tests
+        [Theory]
+        [InlineData("Create", TokenType.Function, TokenType.BoolLit)]
+        [InlineData(null)]
+        [InlineData("")]
+        [InlineData("[2, 3")]
+        [InlineData("Create", TokenType.StrInterpStart, TokenType.BinaryOp, TokenType.NumLit, TokenType.DecLit, TokenType.Control)]
+        [InlineData("[]")]
+        [InlineData("1,2]")]
+        [InlineData("[98]")]
+        [InlineData("Create", TokenType.Lim)]
+        [InlineData("Create", TokenType.BoolLit, TokenType.BinaryOp, TokenType.Function, TokenType.Lim)]
+        [InlineData("Create", TokenType.Lim, TokenType.BinaryOp, TokenType.BoolLit)]
+        [InlineData("   ")]
+        [InlineData("NotPresent")]
+        internal async Task TestCorrectRangeSemanticTokensAreReturnedWithCertainTokenTypesSkipped(string tokenTypesToSkipParam, params TokenType[] tokenTypesToSkip)
+        {
+            // Arrange & Assert
+            var expression = "1+1+1+1+1+1+1+1;Sqrt(1);1+-2;true;\n\"String Literal\";Sum(1,2);Max(1,2,3);$\"1 + 2 = {3}\";// This is Comment;//This is comment2;false";
+            var range = SemanticTokensRelatedTestsHelper.CreateRange(1, 2, 3, 35);
+            var expectedTypes = new List<TokenType> { TokenType.DecLit, TokenType.BoolLit, TokenType.Function, TokenType.StrLit, TokenType.Delimiter, TokenType.BinaryOp };
+            if (tokenTypesToSkip.Length > 0)
+            {
+                expectedTypes = expectedTypes.Where(expectedType => !tokenTypesToSkip.Contains(expectedType)).ToList();
+            }
+
+            if (tokenTypesToSkipParam == "Create")
+            {
+                tokenTypesToSkipParam = JsonSerializer.Serialize(tokenTypesToSkip.Select(tokType => (int)tokType).ToList());
+            }
+
+            var semanticTokenParams = new SemanticTokensRangeParams
+            {
+                TextDocument = GetTextDocument(GetUri(tokenTypesToSkipParam == "NotPresent" ? string.Empty : "tokenTypesToSkip=" + tokenTypesToSkipParam)),
+                Text = expression,
+                Range = range
+            };
+            var payload = GetRangeDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var response = AssertAndGetSemanticTokensResponse(rawResponse, payload.id);
+            Assert.NotEmpty(response.Data);
+            var decodedTokens = SemanticTokensRelatedTestsHelper.DecodeEncodedSemanticTokensPartially(response, expression);
+            var actualTypes = decodedTokens.Select(tok => tok.TokenType).Distinct().ToList();
+            Assert.Equal(expectedTypes.OrderBy(type => type), actualTypes.OrderBy(type => type));
+        }
+
+        [Theory]
+        [InlineData(1, 4, 1, 20, false, false)]
+        [InlineData(1, 6, 1, 20, true, false)]
+        [InlineData(1, 1, 1, 12, false, true)]
+        [InlineData(1, 5, 1, 24, true, false)]
+        [InlineData(1, 5, 1, 15, true, true)]
+        [InlineData(1, 9, 1, 14, true, true)]
+        [InlineData(1, 1, 3, 34, false, false)]
+        [InlineData(1, 3, 2, 12, false, true)]
+        [InlineData(2, 11, 3, 3, true, true)]
+        [InlineData(1, 24, 3, 17, false, false)]
+        public async Task TestCorrectRangeSemanticTokensAreReturned(int startLine, int startLineCol, int endLine, int endLineCol, bool tokenDoesNotAlignOnLeft, bool tokenDoesNotAlignOnRight)
+        {
+            // Arrange
+            Init();
+            var expression = "If(Len(Phone_Number) < 10,\nNotify(\"Invalid Phone\nNumber\"),Notify(\"Valid Phone No\"))";
+            var eol = "\n";
+            var semanticTokenParams = new SemanticTokensRangeParams
+            {
+                TextDocument = GetTextDocument(),
+                Text = expression,
+                Range = SemanticTokensRelatedTestsHelper.CreateRange(startLine, endLine, startLineCol, endLineCol)
+            };
+            var payload = GetRangeDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var response = AssertAndGetSemanticTokensResponse(rawResponse, payload.id);
+            var (startIndex, endIndex) = semanticTokenParams.Range.ConvertRangeToPositions(expression, eol);
+            var decodedResponse = SemanticTokensRelatedTestsHelper.DecodeEncodedSemanticTokensPartially(response, expression, eol);
+
+            var leftMostTok = decodedResponse.Min(tok => tok.StartIndex);
+            var rightMostTok = decodedResponse.Max(tok => tok.EndIndex);
+
+            Assert.All(decodedResponse, (tok) => Assert.False(tok.EndIndex <= leftMostTok || tok.StartIndex >= rightMostTok));
+            if (tokenDoesNotAlignOnLeft)
+            {
+                Assert.True(leftMostTok < startIndex);
+            }
+            else
+            {
+                Assert.True(leftMostTok >= startIndex);
+            }
+
+            if (tokenDoesNotAlignOnRight)
+            {
+                Assert.True(rightMostTok > endIndex);
+            }
+            else
+            {
+                Assert.True(rightMostTok <= endIndex);
+            }
+        }
+
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task TestEmptyRangeSemanticTokensResponseReturnedWhenExpressionIsInvalid(bool isNotNull)
+        {
+            // Arrange
+            string expression = string.Empty;
+            var semanticTokenParams = new SemanticTokensRangeParams
+            {
+                TextDocument = GetTextDocument(),
+                Text = isNotNull ? expression : null,
+                Range = SemanticTokensRelatedTestsHelper.CreateRange(1, 1, 1, 4)
+            };
+            var payload = GetRangeDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var response = AssertAndGetSemanticTokensResponse(rawResponse, payload.id);
+            Assert.Empty(response.Data);
+        }
+
+        [Fact]
+        public async Task TestErrorResponseReturnedWhenUriIsNullForRangeSemanticTokensRequest()
+        {
+            // Arrange
+            var semanticTokenParams = new SemanticTokensRangeParams
+            {
+                TextDocument = new TextDocumentIdentifier() { Uri = null },
+                Range = SemanticTokensRelatedTestsHelper.CreateRange(1, 1, 1, 4)
+            };
+            var payload = GetRangeDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var response = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            AssertErrorPayload(response, payload.id, JsonRpcHelper.ErrorCode.ParseError);
+        }
+
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task TestEmptyRangeSemanticTokensResponseReturnedWhenRangeIsNullOrInvalid(bool isNull)
+        {
+            // Arrange
+            var expression = "If(Len(Phone_Number) < 10,\nNotify(\"Invalid Phone\nNumber\"),Notify(\"Valid Phone No\"))";
+            var semanticTokenParams = new SemanticTokensRangeParams
+            {
+                TextDocument = GetTextDocument(),
+                Text = expression,
+                Range = isNull ? null : SemanticTokensRelatedTestsHelper.CreateRange(expression.Length + 2, 2, 1, 2)
+            };
+            var payload = GetRangeDocumentSemanticTokensRequestPayload(semanticTokenParams);
+
+            // Act
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+
+            // Assert
+            var response = AssertAndGetSemanticTokensResponse(rawResponse, payload.id);
+            Assert.Empty(response.Data);
+        }
+
+        private static (string payload, string id) GetRangeDocumentSemanticTokensRequestPayload(SemanticTokensRangeParams semanticTokenRangeParams, string id = null)
+        {
+            return GetRequestPayload(semanticTokenRangeParams, TextDocumentNames.RangeDocumentSemanticTokens, id);
+        }
+        #endregion
+
+        private static SemanticTokensResponse AssertAndGetSemanticTokensResponse(string response, string id)
+        {
+            var tokensResponse = AssertAndGetResponsePayload<SemanticTokensResponse>(response, id);
+            Assert.NotNull(tokensResponse);
+            Assert.NotNull(tokensResponse.Data);
+            return tokensResponse;
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/SignatureHelpTests.cs

@@ -0,0 +1,145 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Linq;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+using Xunit;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public partial class LanguageServerTestBase
+    {
+        // Ensure the disclaimer shows up in a signature.
+        [Fact]
+        public async Task TestSignatureDisclaimers()
+        {
+            // Arrange
+            var text = "AISummarize(";
+            var signatureHelpParams1 = new SignatureHelpParams
+            {
+                TextDocument = GetTextDocument(GetUri("expression=" + text)),
+                Text = text,
+                Position = GetPosition(text.Length),
+                Context = GetSignatureHelpContext("(")
+            };
+
+            Init(new InitParams(options: GetParserOptions(false)));
+
+            // test good formula
+            var payload = GetSignatureHelpPayload(signatureHelpParams1);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<SignatureHelp>(rawResponse, payload.id);
+            var sig = response.Signatures.Single();
+            Assert.Equal("AISummarize()", sig.Label);
+
+            var je = (JsonElement)sig.Documentation;
+            var markdown = JsonSerializer.Deserialize<MarkupContent>(je.ToString(), LanguageServerHelper.DefaultJsonSerializerOptions);
+            Assert.Equal("markdown", markdown.Kind);
+            Assert.StartsWith("Create and set a global variable", markdown.Value); // function's normal description 
+            Assert.Contains("**Disclaimer:** AI-generated content", markdown.Value); // disclaimer appended. 
+        }
+
+        [Theory]
+        [InlineData("Power(", 6, "Power(2,", 8, false)]
+        [InlineData("Behavior(); Power(", 18, "Behavior(); Power(2,", 20, true)]
+
+        // This tests generates an internal error as we use an behavior function but we have no way to check its presence
+        [InlineData("Behavior(); Power(", 18, "Behavior(); Power(2,", 20, false)]
+        public async Task TestSignatureHelpWithExpressionInUriAndNotInUri(string text, int offset, string text2, int offset2, bool withAllowSideEffects)
+        {
+            foreach (var addExprToUri in new bool[] { true, false })
+            {
+                var signatureHelpParams1 = new SignatureHelpParams
+                {
+                    TextDocument = addExprToUri ? GetTextDocument(GetUri("expression=" + text)) : GetTextDocument(),
+                    Text = addExprToUri ? null : text,
+                    Position = GetPosition(offset),
+                    Context = GetSignatureHelpContext("(")
+                };
+                var signatureHelpParams2 = new SignatureHelpParams
+                {
+                    TextDocument = addExprToUri ? GetTextDocument(GetUri("expression=" + text2)) : GetTextDocument(),
+                    Text = addExprToUri ? null : text2,
+                    Position = GetPosition(offset2),
+                    Context = GetSignatureHelpContext(",")
+                };
+                await TestSignatureHelpCore(signatureHelpParams1, signatureHelpParams2, withAllowSideEffects).ConfigureAwait(false);
+            }
+        }
+
+        [Fact]
+        public async Task TestSignatureHelpWithExpressionInBothUriAndTextDocument()
+        {
+            var signatureHelpParams1 = new SignatureHelpParams
+            {
+                TextDocument = GetTextDocument(GetUri("expression=Max(")),
+                Text = "Power(",
+                Position = GetPosition(6),
+                Context = GetSignatureHelpContext("(")
+            };
+            var signatureHelpParams2 = new SignatureHelpParams
+            {
+                TextDocument = GetTextDocument(GetUri("expression=Max(")),
+                Text = "Behavior(); Power(2,",
+                Position = GetPosition(20),
+                Context = GetSignatureHelpContext(",")
+            };
+            await TestSignatureHelpCore(signatureHelpParams1, signatureHelpParams2, true).ConfigureAwait(false);
+        }
+
+        private async Task TestSignatureHelpCore(SignatureHelpParams signatureHelpParams1, SignatureHelpParams signatureHelpParams2, bool withAllowSideEffects)
+        {
+            Init(new InitParams(options: GetParserOptions(withAllowSideEffects)));
+
+            // test good formula
+            var payload = GetSignatureHelpPayload(signatureHelpParams1);
+            var rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            var response = AssertAndGetResponsePayload<SignatureHelp>(rawResponse, payload.id);
+            Assert.Equal(0U, response.ActiveSignature);
+            Assert.Equal(0U, response.ActiveParameter);
+            var foundItems = response.Signatures.Where(item => item.Label.StartsWith("Power"));
+            Assert.True(Enumerable.Count(foundItems) >= 1, "Power should be found from signatures result");
+            Assert.Equal(2, foundItems.First().Parameters.Length);
+            Assert.Equal("base", foundItems.First().Parameters[0].Label);
+            Assert.Equal("exponent", foundItems.First().Parameters[1].Label);
+
+            payload = GetSignatureHelpPayload(signatureHelpParams2);
+            rawResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            response = AssertAndGetResponsePayload<SignatureHelp>(rawResponse, payload.id);
+            Assert.Equal(0U, response.ActiveSignature);
+            Assert.Equal(1U, response.ActiveParameter);
+            foundItems = response.Signatures.Where(item => item.Label.StartsWith("Power"));
+            Assert.True(Enumerable.Count(foundItems) >= 1, "Power should be found from signatures result");
+            Assert.Equal(2, foundItems.First().Parameters.Length);
+            Assert.Equal("base", foundItems.First().Parameters[0].Label);
+            Assert.Equal("exponent", foundItems.First().Parameters[1].Label);
+
+            // missing 'expression' in documentUri
+            payload = GetSignatureHelpPayload(new SignatureHelpParams()
+            {
+                Context = GetSignatureHelpContext("("),
+                TextDocument = GetTextDocument("powerfx://test"),
+                Position = GetPosition(0),
+            });
+            var errorResponse = await TestServer.OnDataReceivedAsync(payload.payload).ConfigureAwait(false);
+            AssertErrorPayload(errorResponse, payload.id, JsonRpcHelper.ErrorCode.InvalidParams);
+        }
+
+        private static (string payload, string id) GetSignatureHelpPayload(SignatureHelpParams signatureHelpParams)
+        {
+            return GetRequestPayload(signatureHelpParams, TextDocumentNames.SignatureHelp);
+        }
+
+        private static SignatureHelpContext GetSignatureHelpContext(string triggerChar)
+        {
+            return new SignatureHelpContext
+            {
+                TriggerKind = SignatureHelpTriggerKind.TriggerCharacter,
+                TriggerCharacter = triggerChar
+            };
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/TestHandlerFactory.cs

@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+using Microsoft.PowerFx.LanguageServerProtocol.Protocol;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    internal class TestHandlerFactory : ILanguageServerOperationHandlerFactory
+    {
+        private readonly Dictionary<string, ILanguageServerOperationHandler> _handlers = new ();
+
+        public TestHandlerFactory()
+        {
+        }
+
+        public TestHandlerFactory SetHandler(string method, ILanguageServerOperationHandler handler)
+        {
+            _handlers[method] = handler;
+            return this;
+        }
+
+        public ILanguageServerOperationHandler GetHandler(string method, HandlerCreationContext creationContext)
+        {
+            if (_handlers.TryGetValue(method, out var handler))
+            {
+                return handler;
+            }
+
+            switch (method)
+            {
+                case CustomProtocolNames.NL2FX:
+                    return new Nl2FxLanguageServerOperationHandler(null);
+                case CustomProtocolNames.FX2NL:
+                    return new Fx2NlLanguageServerOperationHandler(null);
+                case CustomProtocolNames.GetCapabilities:
+                    return new GetCustomCapabilitiesLanguageServerOperationHandler(null);
+                case TextDocumentNames.Completion:
+                    return new CompletionsLanguageServerOperationHandler();
+                case TextDocumentNames.SignatureHelp:
+                    return new SignatureHelpLanguageServerOperationHandler();
+                case TextDocumentNames.RangeDocumentSemanticTokens:
+                    return new RangeSemanticTokensLanguageServerOperationHandler();
+                case TextDocumentNames.FullDocumentSemanticTokens:
+                    return new BaseSemanticTokensLanguageServerOperationHandler();
+                case TextDocumentNames.CodeAction:
+                    return new CodeActionsLanguageServerOperationHandler(creationContext.onLogUnhandledExceptionHandler);
+                case TextDocumentNames.DidChange:
+                    return new OnDidChangeLanguageServerNotificationHandler(null);
+                case TextDocumentNames.DidOpen:
+                    return new OnDidOpenLanguageServerNotificationHandler();
+                case CustomProtocolNames.CommandExecuted:
+                    return new CommandExecutedLanguageServerOperationHandler();
+                case CustomProtocolNames.InitialFixup:
+                    return new InitialFixupLanguageServerOperationHandler();
+                default:
+                    return null;
+            }
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/TestHostTaskExecutor.cs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public class TestHostTaskExecutor : IHostTaskExecutor
+    {
+        public async Task<TOutput> ExecuteTaskAsync<TOutput>(Func<Task<TOutput>> task, LanguageServerOperationContext operationContext, CancellationToken cancellationToken, TOutput defaultOutput = default)
+        {
+            // Simulate a delay in the task execution
+            await Task.Delay(30, cancellationToken).ConfigureAwait(false);
+            return await task().ConfigureAwait(false);
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/RedesignedLanguageServerTests/TestLogger.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.PowerFx.LanguageServerProtocol;
+
+namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
+{
+    public class TestLogger : ILanguageServerLogger
+    {
+        private readonly List<string> _messages = new ();
+
+        public List<string> Messages => _messages;
+
+        public void LogError(string message, object data = null)
+        {
+            _messages.Add(message);
+        }
+
+        public void LogException(Exception exception, object data = null)
+        {
+            _messages.Add(exception.Message);
+        }
+
+        public void LogInformation(string message, object data = null)
+        {
+            _messages.Add(message);
+        }
+
+        public void LogWarning(string message, object data = null)
+        {
+            _messages.Add(message);
+        }
+    }
+}

src/tests/Microsoft.PowerFx.Interpreter.Tests/LanguageServiceProtocol/TestLanguageServer.cs

@@ -1,8 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+using System;
+using System.Collections.Generic;
 using Microsoft.PowerFx.Core;
 using Microsoft.PowerFx.LanguageServerProtocol;
+using Microsoft.PowerFx.LanguageServerProtocol.Handlers;
 using Xunit.Abstractions;
 
 namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
@@ -10,13 +13,28 @@ namespace Microsoft.PowerFx.Tests.LanguageServiceProtocol
     public class TestLanguageServer : LanguageServer
     {
         public TestLanguageServer(ITestOutputHelper output, SendToClient sendToClient, IPowerFxScopeFactory scopeFactory, INLHandlerFactory nlHandlerFactory = null)
+#pragma warning disable CS0618 // Type or member is obsolete
             : base(sendToClient, scopeFactory, (string s) => output.WriteLine(s))
+#pragma warning restore CS0618 // Type or member is obsolete
         {            
             NLHandlerFactory = nlHandlerFactory;
         }
 
-        public int TestGetCharPosition(string expression, int position) => GetCharPosition(expression, position);
+        public int TestGetCharPosition(string expression, int position) => PositionRangeHelper.GetCharPosition(expression, position);
 
-        public int TestGetPosition(string expression, int line, int character) => GetPosition(expression, line, character);
+        public int TestGetPosition(string expression, int line, int character) => PositionRangeHelper.GetPosition(expression, line, character);
+
+        // Language Server class (parent of this) marks OnDataReceived Obsolete
+        // This caused a lot of compiler warnings in LanguageServerTests.cs
+        // since all of them use this OnDataReceived 
+        // To avoid many supress, hide the base OnDataRecieved
+        // With this one below which in turn calls OnDataRecieved from base
+        // And suppresses the call at one place only
+        public new void OnDataReceived(string jsonRpcPayload)
+        {
+#pragma warning disable CS0618 // Type or member is obsolete
+            base.OnDataReceived(jsonRpcPayload);
+#pragma warning restore CS0618 // Type or member is obsolete
+        }
     }
 }