Semantic colorization (#1651)

Semantic colorization

.vscode/launch.json

@@ -25,7 +25,9 @@
       "runtimeExecutable": "${execPath}",
       "args": [
         "${workspaceFolder}/src/Razor/test/testapps/",
-        "--extensionDevelopmentPath=${workspaceFolder}/src/Razor/src/Microsoft.AspNetCore.Razor.VSCode.Extension"
+        "--extensionDevelopmentPath=${workspaceFolder}/src/Razor/src/Microsoft.AspNetCore.Razor.VSCode.Extension",
+        "--enable-proposed-api razor-vscode",
+        "--enable-proposed-api ms-dotnettools.csharp"
       ],
       "outFiles": [
         "${workspaceFolder}/src/Razor/src/Microsoft.AspNetCore.Razor.VSCode.Extension/dist/**/*.js",

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer.Common/LanguageServerConstants.cs

@@ -7,6 +7,10 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer.Common
     {
         public const string ProjectConfigurationFile = "project.razor.json";
 
+        public const string RazorSemanticTokensEndpoint = "razor/semanticTokens";
+
+        public const string RazorSemanticTokenLegendEndpoint = "razor/semanticTokensLegend";
+
         public const string RazorRangeFormattingEndpoint = "razor/rangeFormatting";
 
         public const string RazorUpdateCSharpBufferEndpoint = "razor/updateCSharpBuffer";

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Hover/DefaultRazorHoverInfoService.cs

@@ -57,6 +57,7 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer.Hover
             }
 
             var syntaxTree = codeDocument.GetSyntaxTree();
+
             var change = new SourceChange(location.AbsoluteIndex, length: 0, newText: "");
             var owner = syntaxTree.Root.LocateOwner(change);
 
@@ -94,7 +95,7 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer.Hover
                 {
                     Debug.Assert(binding.Descriptors.Count() > 0);
 
-                    var range = GetRangeFromSyntaxNode(containingTagNameToken, codeDocument);
+                    var range = containingTagNameToken.GetRange(codeDocument.Source);
 
                     var result = ElementInfoToHover(binding.Descriptors, range);
                     return result;
@@ -107,7 +108,6 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer.Hover
                 // Hovering over HTML attribute name
                 var stringifiedAttributes = _tagHelperFactsService.StringifyAttributes(attributes);
 
-
                 var binding = _tagHelperFactsService.GetTagHelperBinding(
                     tagHelperDocumentContext,
                     containingTagNameToken.Content,
@@ -126,7 +126,38 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer.Hover
                     var tagHelperAttributes = _tagHelperFactsService.GetBoundTagHelperAttributes(tagHelperDocumentContext, selectedAttributeName, binding);
 
                     var attribute = attributes.Single(a => a.Span.IntersectsWith(location.AbsoluteIndex));
-                    var range = GetRangeFromSyntaxNode(attribute, codeDocument);
+                    if (attribute is MarkupTagHelperAttributeSyntax thAttributeSyntax)
+                    {
+                        attribute = thAttributeSyntax.Name;
+                    }
+                    else if (attribute is MarkupMinimizedTagHelperAttributeSyntax thMinimizedAttribute)
+                    {
+                        attribute = thMinimizedAttribute.Name;
+                    }
+                    else if (attribute is MarkupTagHelperDirectiveAttributeSyntax directiveAttribute)
+                    {
+                        attribute = directiveAttribute.Name;
+                    }
+                    else if (attribute is MarkupMinimizedTagHelperDirectiveAttributeSyntax miniDirectiveAttribute)
+                    {
+                        attribute = miniDirectiveAttribute;
+                    }
+
+                    var range = attribute.GetRange(codeDocument.Source);
+
+                    // Include the @ in the range
+                    switch (attribute.Parent.Kind)
+                    {
+                        case SyntaxKind.MarkupTagHelperDirectiveAttribute:
+                            var directiveAttribute = attribute.Parent as MarkupTagHelperDirectiveAttributeSyntax;
+                            range.Start.Character -= directiveAttribute.Transition.FullWidth;
+                            break;
+                        case SyntaxKind.MarkupMinimizedTagHelperDirectiveAttribute:
+                            var minimizedAttribute = containingTagNameToken.Parent as MarkupMinimizedTagHelperDirectiveAttributeSyntax;
+                            range.Start.Character -= minimizedAttribute.Transition.FullWidth;
+                            break;
+                    }
+
                     var attributeHoverModel = AttributeInfoToHover(tagHelperAttributes, range);
 
                     return attributeHoverModel;
@@ -136,43 +167,6 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer.Hover
             return null;
         }
 
-        private RangeModel GetRangeFromSyntaxNode(RazorSyntaxNode syntaxNode, RazorCodeDocument codeDocument)
-        {
-            try
-            {
-                int startPosition;
-                int endPosition;
-                if (syntaxNode is MarkupTagHelperAttributeSyntax thAttributeSyntax)
-                {
-                    startPosition = thAttributeSyntax.Name.Position;
-                    endPosition = thAttributeSyntax.Name.EndPosition;
-                }
-                else if (syntaxNode is MarkupMinimizedTagHelperAttributeSyntax thAttrSyntax)
-                {
-                    startPosition = thAttrSyntax.Name.Position;
-                    endPosition = thAttrSyntax.Name.EndPosition;
-                }
-                else
-                {
-                    startPosition = syntaxNode.Position;
-                    endPosition = syntaxNode.EndPosition;
-                }
-                var startLocation = codeDocument.Source.Lines.GetLocation(startPosition);
-                var endLocation = codeDocument.Source.Lines.GetLocation(endPosition);
-
-                return new RangeModel
-                {
-                    Start = new Position(startLocation.LineIndex, startLocation.CharacterIndex),
-                    End = new Position(endLocation.LineIndex, endLocation.CharacterIndex)
-                };
-            }
-            catch (IndexOutOfRangeException)
-            {
-                Debug.Assert(false, "Node position should stay within document length.");
-                return null;
-            }
-        }
-
         private HoverModel AttributeInfoToHover(IEnumerable<BoundAttributeDescriptor> descriptors, RangeModel range)
         {
             var descriptionInfos = descriptors.Select(d => new TagHelperAttributeDescriptionInfo(d.DisplayName, d.GetPropertyName(), d.TypeName, d.Documentation))

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Hover/RazorHoverEndpoint.cs

@@ -90,8 +90,6 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer.Hover
                 return null;
             }
 
-            var tagHelperDocumentContext = codeDocument.GetTagHelperContext();
-
             var sourceText = await document.GetTextAsync();
             var linePosition = new LinePosition((int)request.Position.Line, (int)request.Position.Character);
             var hostDocumentIndex = sourceText.Lines.GetPosition(linePosition);

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/RazorDiagnosticConverter.cs

@@ -81,7 +81,7 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer
             {
                 if (index >= sourceText.Length)
                 {
-                    // Span start index is past the end of the document. Roslyn and VSCode don't support 
+                    // Span start index is past the end of the document. Roslyn and VSCode don't support
                     // virtual positions that don't exist on the document; normalize to the last character.
                     index = sourceText.Length - 1;
                 }

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/RazorLanguageServer.cs

@@ -12,6 +12,7 @@ using Microsoft.AspNetCore.Razor.LanguageServer.Completion;
 using Microsoft.AspNetCore.Razor.LanguageServer.Formatting;
 using Microsoft.AspNetCore.Razor.LanguageServer.Hover;
 using Microsoft.AspNetCore.Razor.LanguageServer.ProjectSystem;
+using Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Completion;
 using Microsoft.CodeAnalysis.Razor.ProjectSystem;
@@ -20,6 +21,7 @@ using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.VisualStudio.Editor.Razor;
 using OmniSharp.Extensions.JsonRpc;
+using OmniSharp.Extensions.JsonRpc.Serialization.Converters;
 using OmniSharp.Extensions.LanguageServer.Protocol.Serialization;
 using OmniSharp.Extensions.LanguageServer.Protocol.Server;
 using OmniSharp.Extensions.LanguageServer.Server;
@@ -65,6 +67,8 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer
                     .WithHandler<RazorLanguageEndpoint>()
                     .WithHandler<RazorConfigurationEndpoint>()
                     .WithHandler<RazorFormattingEndpoint>()
+                    .WithHandler<RazorSemanticTokenEndpoint>()
+                    .WithHandler<RazorSemanticTokenLegendEndpoint>()
                     .WithServices(services =>
                     {
                         services.AddSingleton<FilePathNormalizer>();
@@ -119,6 +123,7 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer
                         services.AddSingleton<RazorFormattingService, DefaultRazorFormattingService>();
 
                         services.AddSingleton<RazorCompletionFactsService, DefaultRazorCompletionFactsService>();
+                        services.AddSingleton<RazorSemanticTokenInfoService, DefaultRazorSemanticTokenInfoService>();
                         services.AddSingleton<RazorHoverInfoService, DefaultRazorHoverInfoService>();
                         services.AddSingleton<HtmlFactsService, DefaultHtmlFactsService>();
                         var documentVersionCache = new DefaultDocumentVersionCache(foregroundDispatcher);

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/DefaultRazorSemanticTokenInfoService.cs

@@ -0,0 +1,166 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using Microsoft.AspNetCore.Razor.Language;
+using Microsoft.AspNetCore.Razor.Language.Syntax;
+using SyntaxNode = Microsoft.AspNetCore.Razor.Language.Syntax.SyntaxNode;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    internal class DefaultRazorSemanticTokenInfoService : RazorSemanticTokenInfoService
+    {
+        public DefaultRazorSemanticTokenInfoService()
+        {
+        }
+
+        public override SemanticTokens GetSemanticTokens(RazorCodeDocument codeDocument, SourceLocation? location = null)
+        {
+            if (codeDocument is null)
+            {
+                throw new ArgumentNullException(nameof(codeDocument));
+            }
+            var syntaxTree = codeDocument.GetSyntaxTree();
+
+            var syntaxNodes = VisitAllNodes(syntaxTree);
+
+            var semanticTokens = ConvertSyntaxTokensToSemanticTokens(syntaxNodes, codeDocument);
+
+            return semanticTokens;
+        }
+
+        private static IEnumerable<SyntaxNode> VisitAllNodes(RazorSyntaxTree syntaxTree)
+        {
+            var visitor = new TagHelperSpanVisitor();
+            visitor.Visit(syntaxTree.Root);
+
+            return visitor.TagHelperNodes;
+        }
+
+        private static SemanticTokens ConvertSyntaxTokensToSemanticTokens(
+            IEnumerable<SyntaxNode> syntaxTokens,
+            RazorCodeDocument razorCodeDocument)
+        {
+            SyntaxNode previousToken = null;
+
+            var data = new List<uint>();
+            foreach (var token in syntaxTokens)
+            {
+                var newData = GetData(token, previousToken, razorCodeDocument);
+                data.AddRange(newData);
+
+                previousToken = token;
+            }
+
+            return new SemanticTokens
+            {
+                Data = data
+            };
+        }
+
+        /**
+         * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following array indices:
+         *  - at index `5*i`   - `deltaLine`: token line number, relative to the previous token
+         *  - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line)
+         *  - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline.
+         *  - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes`
+         *  - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
+        **/
+        private static IEnumerable<uint> GetData(
+            SyntaxNode currentNode,
+            SyntaxNode previousNode,
+            RazorCodeDocument razorCodeDocument)
+        {
+            var previousRange = previousNode?.GetRange(razorCodeDocument.Source);
+            var currentRange = currentNode.GetRange(razorCodeDocument.Source);
+
+            // deltaLine
+            var previousLineIndex = previousNode == null ? 0 : previousRange.Start.Line;
+            yield return (uint)(currentRange.Start.Line - previousLineIndex);
+
+            // deltaStart
+            if (previousRange != null && previousRange?.Start.Line == currentRange.Start.Line)
+            {
+                yield return (uint)(currentRange.Start.Character - previousRange.Start.Character);
+            }
+            else
+            {
+                yield return (uint)(currentRange.Start.Character);
+            }
+
+            // length
+            Debug.Assert(currentNode.Span.Length > 0);
+            yield return (uint)currentNode.Span.Length;
+
+            // tokenType
+            yield return GetTokenTypeData(currentNode);
+
+            // tokenModifiers
+            // We don't currently have any need for tokenModifiers
+            yield return 0;
+        }
+
+        private static uint GetTokenTypeData(SyntaxNode syntaxToken)
+        {
+            switch (syntaxToken.Parent.Kind)
+            {
+                case SyntaxKind.MarkupTagHelperStartTag:
+                case SyntaxKind.MarkupTagHelperEndTag:
+                    return (uint)SemanticTokenLegend.TokenTypesLegend[SemanticTokenLegend.RazorTagHelperElement];
+                case SyntaxKind.MarkupTagHelperAttribute:
+                case SyntaxKind.MarkupMinimizedTagHelperDirectiveAttribute:
+                case SyntaxKind.MarkupTagHelperDirectiveAttribute:
+                case SyntaxKind.MarkupMinimizedTagHelperAttribute:
+                    return (uint)SemanticTokenLegend.TokenTypesLegend[SemanticTokenLegend.RazorTagHelperAttribute];
+                default:
+                    throw new NotImplementedException();
+            }
+        }
+
+        private class TagHelperSpanVisitor : SyntaxWalker
+        {
+            private readonly List<SyntaxNode> _syntaxNodes;
+
+            public TagHelperSpanVisitor()
+            {
+                _syntaxNodes = new List<SyntaxNode>();
+            }
+
+            public IReadOnlyList<SyntaxNode> TagHelperNodes => _syntaxNodes;
+
+            public override void VisitMarkupTagHelperStartTag(MarkupTagHelperStartTagSyntax node)
+            {
+                _syntaxNodes.Add(node.Name);
+                base.VisitMarkupTagHelperStartTag(node);
+            }
+
+            public override void VisitMarkupTagHelperEndTag(MarkupTagHelperEndTagSyntax node)
+            {
+                _syntaxNodes.Add(node.Name);
+                base.VisitMarkupTagHelperEndTag(node);
+            }
+
+            public override void VisitMarkupMinimizedTagHelperAttribute(MarkupMinimizedTagHelperAttributeSyntax node)
+            {
+                if (node.TagHelperAttributeInfo.Bound)
+                {
+                    _syntaxNodes.Add(node.Name);
+                }
+
+                base.VisitMarkupMinimizedTagHelperAttribute(node);
+            }
+
+            public override void VisitMarkupTagHelperAttribute(MarkupTagHelperAttributeSyntax node)
+            {
+                if (node.TagHelperAttributeInfo.Bound)
+                {
+                    _syntaxNodes.Add(node.Name);
+                }
+
+                base.VisitMarkupTagHelperAttribute(node);
+            }
+        }
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/ISemanticTokenHandler.cs

@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using MediatR;
+using Microsoft.AspNetCore.Razor.LanguageServer.Common;
+using OmniSharp.Extensions.JsonRpc;
+using OmniSharp.Extensions.LanguageServer.Protocol;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    [Method(LanguageServerConstants.RazorSemanticTokensEndpoint)]
+    internal interface ISemanticTokenHandler :
+        IJsonRpcRequestHandler<SemanticTokenParams, SemanticTokens>,
+        IRequestHandler<SemanticTokenParams, SemanticTokens>,
+        IJsonRpcHandler,
+        ICapability<SemanticTokenCapability>
+    {
+
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/ISemanticTokenLegendHandler.cs

@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using MediatR;
+using Microsoft.AspNetCore.Razor.LanguageServer.Common;
+using OmniSharp.Extensions.JsonRpc;
+using OmniSharp.Extensions.LanguageServer.Protocol;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    [Method(LanguageServerConstants.RazorSemanticTokenLegendEndpoint)]
+    internal interface ISemanticTokenLegendHandler :
+        IJsonRpcRequestHandler<SemanticTokenLegendParams, SemanticTokenLegendResponse>,
+        IRequestHandler<SemanticTokenLegendParams, SemanticTokenLegendResponse>,
+        IJsonRpcHandler,
+        ICapability<SemanticTokenLegendCapability>
+    {
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/RazorSemanticTokenEndpoint.cs

@@ -0,0 +1,90 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Razor.LanguageServer.Common;
+using Microsoft.AspNetCore.Razor.LanguageServer.ProjectSystem;
+using Microsoft.CodeAnalysis.Razor;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    internal class RazorSemanticTokenEndpoint : ISemanticTokenHandler
+    {
+        private SemanticTokenCapability _capability;
+        private readonly ILogger _logger;
+        private readonly ForegroundDispatcher _foregroundDispatcher;
+        private readonly DocumentResolver _documentResolver;
+        private readonly RazorSemanticTokenInfoService _semanticTokenInfoService;
+
+        public RazorSemanticTokenEndpoint(
+            ForegroundDispatcher foregroundDispatcher,
+            DocumentResolver documentResolver,
+            RazorSemanticTokenInfoService semanticTokenInfoService,
+            ILoggerFactory loggerFactory)
+        {
+            if (foregroundDispatcher is null)
+            {
+                throw new ArgumentNullException(nameof(foregroundDispatcher));
+            }
+
+            if (documentResolver is null)
+            {
+                throw new ArgumentNullException(nameof(documentResolver));
+            }
+
+            if (semanticTokenInfoService is null)
+            {
+                throw new ArgumentNullException(nameof(semanticTokenInfoService));
+            }
+
+            if (loggerFactory is null)
+            {
+                throw new ArgumentNullException(nameof(loggerFactory));
+            }
+
+            _foregroundDispatcher = foregroundDispatcher;
+            _documentResolver = documentResolver;
+            _semanticTokenInfoService = semanticTokenInfoService;
+            _logger = loggerFactory.CreateLogger<RazorSemanticTokenEndpoint>();
+        }
+
+
+        public async Task<SemanticTokens> Handle(SemanticTokenParams request, CancellationToken cancellationToken)
+        {
+            if (request is null)
+            {
+                throw new ArgumentNullException(nameof(request));
+            }
+
+            var document = await Task.Factory.StartNew(() =>
+            {
+                _documentResolver.TryResolveDocument(request.RazorDocumentUri.AbsolutePath, out var documentSnapshot);
+
+                return documentSnapshot;
+            }, cancellationToken, TaskCreationOptions.None, _foregroundDispatcher.ForegroundScheduler);
+
+            if (document is null)
+            {
+                return null;
+            }
+
+            var codeDocument = await document.GetGeneratedOutputAsync();
+            if (codeDocument.IsUnsupported())
+            {
+                return null;
+            }
+
+            var tokens = _semanticTokenInfoService.GetSemanticTokens(codeDocument);
+
+            return tokens;
+        }
+
+        public void SetCapability(SemanticTokenCapability capability)
+        {
+            _capability = capability;
+        }
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/RazorSemanticTokenInfoService.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.AspNetCore.Razor.Language;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    internal abstract class RazorSemanticTokenInfoService
+    {
+        public abstract SemanticTokens GetSemanticTokens(RazorCodeDocument codeDocument, SourceLocation? location = null);
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/RazorSemanticTokenLegendEndpoint.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    public class RazorSemanticTokenLegendEndpoint : ISemanticTokenLegendHandler
+    {
+        private SemanticTokenLegendCapability _capability;
+
+        public Task<SemanticTokenLegendResponse> Handle(SemanticTokenLegendParams request, CancellationToken cancellationToken)
+        {
+            return Task.FromResult(SemanticTokenLegend.GetResponse());
+        }
+
+        public void SetCapability(SemanticTokenLegendCapability capability)
+        {
+            _capability = capability;
+        }
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokenCapability.cs

@@ -0,0 +1,11 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using OmniSharp.Extensions.LanguageServer.Protocol.Client.Capabilities;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    public class SemanticTokenCapability : DynamicCapability
+    {
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokenLegend.cs

@@ -0,0 +1,56 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    public static class SemanticTokenLegend
+    {
+        public const string RazorTagHelperElement = "razorTagHelperElement";
+        public const string RazorTagHelperAttribute = "razorTagHelperAttribute";
+        private static readonly IReadOnlyCollection<string> _tokenTypes = new string[] {
+            RazorTagHelperElement,
+            RazorTagHelperAttribute,
+        };
+
+        private static readonly IReadOnlyCollection<string> _tokenModifiers = new string[] { };
+
+        public static IReadOnlyDictionary<string, int> TokenTypesLegend
+        {
+            get
+            {
+                return GetMap(_tokenTypes);
+            }
+        }
+
+        public static IReadOnlyDictionary<string, int> TokenModifiersLegend
+        {
+            get
+            {
+                return GetMap(_tokenModifiers);
+            }
+        }
+
+        public static SemanticTokenLegendResponse GetResponse()
+        {
+            return new SemanticTokenLegendResponse
+            {
+                TokenModifiers = _tokenModifiers,
+                TokenTypes = _tokenTypes
+            };
+        }
+
+        private static IReadOnlyDictionary<string, int> GetMap(IReadOnlyCollection<string> tokens)
+        {
+            var result = new Dictionary<string, int>();
+            for (var i = 0; i < tokens.Count(); i++)
+            {
+                result.Add(tokens.ElementAt(i), i);
+            }
+
+            return result;
+        }
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokenLegendCapability.cs

@@ -0,0 +1,11 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using OmniSharp.Extensions.LanguageServer.Protocol.Client.Capabilities;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    public class SemanticTokenLegendCapability : DynamicCapability
+    {
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokenLegendParams.cs

@@ -0,0 +1,11 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using MediatR;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    public class SemanticTokenLegendParams : IRequest<SemanticTokenLegendResponse>, IBaseRequest
+    {
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokenLegendResponse.cs

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    public struct SemanticTokenLegendResponse
+    {
+        public IReadOnlyCollection<string> TokenTypes;
+        public IReadOnlyCollection<string> TokenModifiers;
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokenParams.cs

@@ -0,0 +1,14 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using MediatR;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    public class SemanticTokenParams : IRequest<SemanticTokens>
+    {
+        public RazorLanguageKind Kind { get; set; }
+        public Uri RazorDocumentUri { get; set; }
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/Semantic/SemanticTokens.cs

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Semantic
+{
+    public class SemanticTokens
+    {
+        public string ResultId { get; }
+        public IEnumerable<uint> Data { get; set; }
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/SyntaxNodeExtensions.cs

@@ -9,7 +9,7 @@ using Microsoft.CodeAnalysis.Text;
 using OmniSharp.Extensions.LanguageServer.Protocol.Models;
 using Range = OmniSharp.Extensions.LanguageServer.Protocol.Models.Range;
 
-namespace Microsoft.AspNetCore.Razor.LanguageServer.Formatting
+namespace Microsoft.AspNetCore.Razor.LanguageServer
 {
     internal static class SyntaxNodeExtensions
     {

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode.Extension/package.json

@@ -7,6 +7,7 @@
     "defaults": {
         "razor": "0.0.1"
     },
+    "enableProposedApi": true,
     "publisher": "ms-dotnettools",
     "engines": {
         "vscode": "^1.31.0"
@@ -45,6 +46,42 @@
     ],
     "main": "./dist/extension",
     "contributes": {
+        "semanticTokenTypes": [
+          {
+            "id": "razorTagHelperElement",
+            "description": "A Razor TagHelper Element"
+          },
+          {
+            "id": "razorTagHelperAttribute",
+            "description": "A Razor TagHelper Attribute"
+          }
+        ],
+        "semanticTokenStyleDefaults": [
+          {
+            "selector": "razorTagHelperElement",
+            "scope": [ "razor.taghelpers.element" ],
+            "light": {
+              "foreground": "#800080",
+              "fontStyle": "bold"
+            },
+            "dark": {
+              "foreground": "#009696",
+              "fontStyle": "bold"
+            }
+          },
+          {
+            "selector": "razorTagHelperAttribute",
+            "scope": [ "razor.taghelpers.attribute"],
+            "light": {
+              "foreground": "#800080",
+              "fontStyle": "bold"
+            },
+            "dark": {
+              "foreground": "#009696",
+              "fontStyle": "bold"
+            }
+          }
+        ],
         "languages": [
             {
                 "id": "aspnetcorerazor",

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode.Extension/src/SemanticTokensEdit.ts

@@ -0,0 +1,15 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+ export class SemanticTokensEdit {
+    public readonly start: number;
+    public readonly deleteCount: number;
+    public readonly data?: Uint32Array;
+    constructor(start: number, deleteCount: number, data?: Uint32Array) {
+        this.start = start;
+        this.deleteCount = deleteCount;
+        this.data = data;
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode.Extension/src/SemanticTokensEdits.ts

@@ -0,0 +1,15 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import { SemanticTokensEdit } from './SemanticTokensEdit';
+
+export class SemanticTokensEdits {
+    public readonly resultId?: string;
+    public readonly edits: SemanticTokensEdit[];
+    constructor(edits: SemanticTokensEdit[], resultId?: string) {
+        this.resultId = resultId;
+        this.edits = edits;
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode.Extension/src/extension.ts

@@ -18,6 +18,7 @@ export async function activate(context: vscode.ExtensionContext) {
     // Because this extension is only used for local development and tests in CI,
     // we know the Razor Language Server is at a specific path within this repo
     const config = process.env.config ? process.env.config : 'Debug';
+
     const languageServerDir = path.join(
         __dirname, '..', '..', '..', '..', '..', 'artifacts', 'bin', 'rzls', config, 'netcoreapp5.0');
 
@@ -48,9 +49,11 @@ export async function activate(context: vscode.ExtensionContext) {
 
     if (workspaceConfigured) {
         await razorExtensionPackage.activate(
+            vscode,
             context,
             languageServerDir,
-            hostEventStream);
+            hostEventStream,
+            /* enabledProposedApis */true);
     } else {
         console.log('Razor workspace was not configured, extension activation skipped.');
         console.log('To configure your workspace run the following command (ctrl+shift+p) in the experimental instance "Razor: Configure workspace for Razor extension development"');

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode/src/ProposedApisFeature.ts

@@ -0,0 +1,29 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import * as vscode from 'vscode';
+import * as vscodeapi from 'vscode';
+import { RazorDocumentManager } from './RazorDocumentManager';
+import { RazorDocumentSynchronizer } from './RazorDocumentSynchronizer';
+import { RazorLanguage } from './RazorLanguage';
+import { RazorLanguageServiceClient } from './RazorLanguageServiceClient';
+import { RazorLogger } from './RazorLogger';
+import { RazorDocumentSemanticTokensProvider } from './Semantic/RazorDocumentSemanticTokensProvider';
+export class ProposedApisFeature {
+    constructor(
+        private documentSynchronizer: RazorDocumentSynchronizer,
+        private documentManager: RazorDocumentManager,
+        private languageServiceClient: RazorLanguageServiceClient,
+        private logger: RazorLogger,
+    ) {
+    }
+    public async register(vscodeType: typeof vscodeapi, localRegistrations: vscode.Disposable[]) {
+        const legend = await this.languageServiceClient.getSemanticTokenLegend();
+        const semanticTokenProvider = new RazorDocumentSemanticTokensProvider(this.documentSynchronizer, this.documentManager, this.languageServiceClient, this.logger);
+        if (legend) {
+            localRegistrations.push(vscodeType.languages.registerDocumentSemanticTokensProvider(RazorLanguage.id, semanticTokenProvider, legend));
+        }
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode/src/RazorLanguageServiceClient.ts

@@ -11,6 +11,7 @@ import { LanguageQueryResponse } from './RPC/LanguageQueryResponse';
 import { RazorMapToDocumentRangeRequest } from './RPC/RazorMapToDocumentRangeRequest';
 import { RazorMapToDocumentRangeResponse } from './RPC/RazorMapToDocumentRangeResponse';
 import { convertRangeFromSerializable, convertRangeToSerializable } from './RPC/SerializableRange';
+import { SemanticTokensRequest } from './Semantic/SemanticTokensRequest';
 
 export class RazorLanguageServiceClient {
     constructor(private readonly serverClient: RazorLanguageServerClient) {
@@ -38,6 +39,27 @@ export class RazorLanguageServiceClient {
         }
     }
 
+    public async getSemanticTokenLegend(): Promise<vscode.SemanticTokensLegend | undefined> {
+        await this.ensureStarted();
+
+        const response = await this.serverClient.sendRequest<vscode.SemanticTokensLegend>('razor/semanticTokensLegend', /*request param*/null);
+
+        if (response.tokenTypes && response.tokenTypes.length > 0) {
+            return response;
+        }
+    }
+
+    public async mapSemanticTokens(languageKind: LanguageKind, uri: vscode.Uri): Promise<vscode.SemanticTokens | undefined> {
+        await this.ensureStarted();
+
+        const request = new SemanticTokensRequest(languageKind, uri);
+        const response = await this.serverClient.sendRequest<vscode.SemanticTokens>('razor/semanticTokens', request);
+
+        if (response.data && response.data.length > 0) {
+            return response;
+        }
+    }
+
     private async ensureStarted() {
         // If the server is already started this will instantly return.
         await this.serverClient.start();

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode/src/Semantic/RazorDocumentSemanticTokensProvider.ts

@@ -0,0 +1,25 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import * as vscode from 'vscode';
+import { RazorLanguageFeatureBase } from '../RazorLanguageFeatureBase';
+import { LanguageKind } from '../RPC/LanguageKind';
+
+export class RazorDocumentSemanticTokensProvider
+    extends RazorLanguageFeatureBase
+    implements vscode.DocumentSemanticTokensProvider {
+
+    public async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken) {
+        const semanticTokenResponse = await this.serviceClient.mapSemanticTokens(LanguageKind.Razor, document.uri);
+
+        if (semanticTokenResponse) {
+            // However we're serializing into Uint32Array doesn't set byteLength, which is checked by some stuff under the covers.
+            // Solution? Create a new one, blat it over the old one, go home for the weekend.
+            semanticTokenResponse.data = new Uint32Array(semanticTokenResponse.data);
+        }
+
+        return semanticTokenResponse;
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode/src/Semantic/SemanticTokensRequest.ts

@@ -0,0 +1,17 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import * as vscode from 'vscode';
+import { LanguageKind } from '../RPC/LanguageKind';
+
+export class SemanticTokensRequest {
+    public readonly razorDocumentUri: string;
+
+    constructor(
+        public readonly kind: LanguageKind,
+        razorDocumentUri: vscode.Uri) {
+        this.razorDocumentUri = razorDocumentUri.toString();
+    }
+}

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode/src/Semantic/SemanticTokensResponse.ts

@@ -0,0 +1,8 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+export interface SemanticTokens {
+    data: number[];
+}

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode/src/extension.ts

@@ -4,6 +4,7 @@
  * ------------------------------------------------------------------------------------------ */
 
 import * as vscode from 'vscode';
+import * as vscodeapi from 'vscode';
 import { ExtensionContext } from 'vscode';
 import { CompositeCodeActionTranslator } from './CodeActions/CompositeRazorCodeActionTranslator';
 import { RazorCodeActionProvider } from './CodeActions/RazorCodeActionProvider';
@@ -14,6 +15,7 @@ import { reportTelemetryForDocuments } from './DocumentTelemetryListener';
 import { HostEventStream } from './HostEventStream';
 import { RazorHtmlFeature } from './Html/RazorHtmlFeature';
 import { IEventEmitterFactory } from './IEventEmitterFactory';
+import { ProposedApisFeature } from './ProposedApisFeature';
 import { ProvisionalCompletionOrchestrator } from './ProvisionalCompletionOrchestrator';
 import { RazorCodeLensProvider } from './RazorCodeLensProvider';
 import { RazorCompletionItemProvider } from './RazorCompletionItemProvider';
@@ -36,15 +38,19 @@ import { RazorRenameProvider } from './RazorRenameProvider';
 import { RazorSignatureHelpProvider } from './RazorSignatureHelpProvider';
 import { TelemetryReporter } from './TelemetryReporter';
 
-export async function activate(context: ExtensionContext, languageServerDir: string, eventStream: HostEventStream) {
+// We specifically need to take a reference to a particular instance of the vscode namespace,
+// otherwise providers attempt to operate on the null extension.
+export async function activate(vscodeType: typeof vscodeapi, context: ExtensionContext, languageServerDir: string, eventStream: HostEventStream, enableProposedApis = false) {
     const telemetryReporter = new TelemetryReporter(eventStream);
     const eventEmitterFactory: IEventEmitterFactory = {
         create: <T>() => new vscode.EventEmitter<T>(),
     };
-    const languageServerTrace = resolveRazorLanguageServerTrace(vscode);
-    const logger = new RazorLogger(vscode, eventEmitterFactory, languageServerTrace);
+
+    const languageServerTrace = resolveRazorLanguageServerTrace(vscodeType);
+    const logger = new RazorLogger(vscodeType, eventEmitterFactory, languageServerTrace);
+
     try {
-        const languageServerOptions = resolveRazorLanguageServerOptions(vscode, languageServerDir, languageServerTrace, logger);
+        const languageServerOptions = resolveRazorLanguageServerOptions(vscodeType, languageServerDir, languageServerTrace, logger);
         const languageServerClient = new RazorLanguageServerClient(languageServerOptions, telemetryReporter, logger);
         const languageServiceClient = new RazorLanguageServiceClient(languageServerClient);
 
@@ -61,11 +67,11 @@ export async function activate(context: ExtensionContext, languageServerDir: str
         const csharpFeature = new RazorCSharpFeature(documentManager, eventEmitterFactory, logger);
         const htmlFeature = new RazorHtmlFeature(documentManager, languageServiceClient, eventEmitterFactory, logger);
         const localRegistrations: vscode.Disposable[] = [];
-        const reportIssueCommand = new ReportIssueCommand(vscode, documentManager, logger);
+        const reportIssueCommand = new ReportIssueCommand(vscodeType, documentManager, logger);
         const razorFormattingFeature = new RazorFormattingFeature(languageServerClient, documentManager, logger);
 
-        const onStartRegistration = languageServerClient.onStart(() => {
-            vscode.commands.executeCommand<void>('omnisharp.registerLanguageMiddleware', razorLanguageMiddleware);
+        const onStartRegistration = languageServerClient.onStart(async () => {
+            vscodeType.commands.executeCommand<void>('omnisharp.registerLanguageMiddleware', razorLanguageMiddleware);
             const documentSynchronizer = new RazorDocumentSynchronizer(documentManager, logger);
             const provisionalCompletionOrchestrator = new ProvisionalCompletionOrchestrator(
                 documentManager,
@@ -123,33 +129,33 @@ export async function activate(context: ExtensionContext, languageServerDir: str
             localRegistrations.push(
                 languageConfiguration.register(),
                 provisionalCompletionOrchestrator.register(),
-                vscode.languages.registerCodeActionsProvider(
+                vscodeType.languages.registerCodeActionsProvider(
                     RazorLanguage.id,
                     codeActionProvider),
-                vscode.languages.registerCompletionItemProvider(
+                vscodeType.languages.registerCompletionItemProvider(
                     RazorLanguage.id,
                     completionItemProvider,
                     '.', '<', '@'),
-                vscode.languages.registerSignatureHelpProvider(
+                vscodeType.languages.registerSignatureHelpProvider(
                     RazorLanguage.id,
                     signatureHelpProvider,
                     '(', ','),
-                vscode.languages.registerDefinitionProvider(
+                vscodeType.languages.registerDefinitionProvider(
                     RazorLanguage.id,
                     definitionProvider),
-                vscode.languages.registerImplementationProvider(
+                vscodeType.languages.registerImplementationProvider(
                     RazorLanguage.id,
                     implementationProvider),
-                vscode.languages.registerHoverProvider(
+                vscodeType.languages.registerHoverProvider(
                     RazorLanguage.documentSelector,
                     hoverProvider),
-                vscode.languages.registerReferenceProvider(
+                vscodeType.languages.registerReferenceProvider(
                     RazorLanguage.id,
                     referenceProvider),
-                vscode.languages.registerCodeLensProvider(
+                vscodeType.languages.registerCodeLensProvider(
                     RazorLanguage.id,
                     codeLensProvider),
-                vscode.languages.registerRenameProvider(
+                vscodeType.languages.registerRenameProvider(
                     RazorLanguage.id,
                     renameProvider),
                 documentManager.register(),
@@ -157,6 +163,15 @@ export async function activate(context: ExtensionContext, languageServerDir: str
                 htmlFeature.register(),
                 documentSynchronizer.register(),
                 reportIssueCommand.register());
+            if (enableProposedApis) {
+                const proposedApisFeature = new ProposedApisFeature(
+                    documentSynchronizer,
+                    documentManager,
+                    languageServiceClient,
+                    logger);
+
+                await proposedApisFeature.register(vscodeType, localRegistrations);
+            }
 
             razorFormattingFeature.register();
         });
@@ -170,7 +185,7 @@ export async function activate(context: ExtensionContext, languageServerDir: str
             await documentManager.initialize();
         });
 
-        await startLanguageServer(languageServerClient, logger, context);
+        await startLanguageServer(vscodeType, languageServerClient, logger, context);
 
         context.subscriptions.push(languageServerClient, onStartRegistration, onStopRegistration, logger);
     } catch (error) {
@@ -180,23 +195,24 @@ export async function activate(context: ExtensionContext, languageServerDir: str
 }
 
 async function startLanguageServer(
+    vscodeType: typeof vscodeapi,
     languageServerClient: RazorLanguageServerClient,
     logger: RazorLogger,
     context: vscode.ExtensionContext) {
 
-    const razorFiles = await vscode.workspace.findFiles(RazorLanguage.globbingPattern);
+    const razorFiles = await vscodeType.workspace.findFiles(RazorLanguage.globbingPattern);
     if (razorFiles.length === 0) {
         // No Razor files in workspace, language server should stay off until one is added or opened.
         logger.logAlways('No Razor files detected in workspace, delaying language server start.');
 
-        const watcher = vscode.workspace.createFileSystemWatcher(RazorLanguage.globbingPattern);
+        const watcher = vscodeType.workspace.createFileSystemWatcher(RazorLanguage.globbingPattern);
         const delayedLanguageServerStart = async () => {
             razorFileCreatedRegistration.dispose();
             razorFileOpenedRegistration.dispose();
             await languageServerClient.start();
         };
         const razorFileCreatedRegistration = watcher.onDidCreate(() => delayedLanguageServerStart());
-        const razorFileOpenedRegistration = vscode.workspace.onDidOpenTextDocument(async (event) => {
+        const razorFileOpenedRegistration = vscodeType.workspace.onDidOpenTextDocument(async (event) => {
             if (event.languageId === RazorLanguage.id) {
                 await delayedLanguageServerStart();
             }

src/Razor/src/Microsoft.AspNetCore.Razor.VSCode/src/vscode.proposed.d.ts

@@ -0,0 +1,230 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/**
+ * This is the place for API experiments and proposals.
+ * These API are NOT stable and subject to change. They are only available in the Insiders
+ * distribution and CANNOT be used in published extensions.
+ *
+ * To test these API in local environment:
+ * - Use Insiders release of VS Code.
+ * - Add `"enableProposedApi": true` to your package.json.
+ * - Copy this file to your project.
+ */
+
+/* tslint:disable:max-classes-per-file */
+declare module 'vscode' {
+
+    //#region Semantic tokens: https://github.com/microsoft/vscode/issues/86415
+
+    export class SemanticTokensLegend {
+        public readonly tokenTypes: string[];
+        public readonly tokenModifiers: string[];
+
+        constructor(tokenTypes: string[], tokenModifiers: string[]);
+    }
+
+    export class SemanticTokensBuilder {
+        constructor();
+        public push(line: number, char: number, length: number, tokenType: number, tokenModifiers: number): void;
+        public build(): Uint32Array;
+    }
+
+    export class SemanticTokens {
+        /**
+         * The result id of the tokens.
+         *
+         * This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented).
+         */
+        public readonly resultId?: string;
+        public data: Uint32Array;
+
+        constructor(data: Uint32Array, resultId?: string);
+    }
+
+    export class SemanticTokensEdits {
+        /**
+         * The result id of the tokens.
+         *
+         * This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented).
+         */
+        public readonly resultId?: string;
+        public readonly edits: SemanticTokensEdit[];
+
+        constructor(edits: SemanticTokensEdit[], resultId?: string);
+    }
+
+    export class SemanticTokensEdit {
+        public readonly start: number;
+        public readonly deleteCount: number;
+        public readonly data?: Uint32Array;
+
+        constructor(start: number, deleteCount: number, data?: Uint32Array);
+    }
+
+    /**
+     * The document semantic tokens provider interface defines the contract between extensions and
+     * semantic tokens.
+     */
+    export interface DocumentSemanticTokensProvider {
+        /**
+         * A file can contain many tokens, perhaps even hundreds of thousands of tokens. Therefore, to improve
+         * the memory consumption around describing semantic tokens, we have decided to avoid allocating an object
+         * for each token and we represent tokens from a file as an array of integers. Furthermore, the position
+         * of each token is expressed relative to the token before it because most tokens remain stable relative to
+         * each other when edits are made in a file.
+         *
+         * ---
+         * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following array indices:
+         *  - at index `5*i`   - `deltaLine`: token line number, relative to the previous token
+         *  - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line)
+         *  - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline.
+         *  - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes`
+         *  - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
+         *
+         * ---
+         * ### How to encode tokens
+         *
+         * Here is an example for encoding a file with 3 tokens in a uint32 array:
+         * ```
+         *    { line: 2, startChar:  5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] },
+         *    { line: 2, startChar: 10, length: 4, tokenType: "types",      tokenModifiers: [] },
+         *    { line: 5, startChar:  2, length: 7, tokenType: "classes",    tokenModifiers: [] }
+         * ```
+         *
+         * 1. First of all, a legend must be devised. This legend must be provided up-front and capture all possible token types.
+         * For this example, we will choose the following legend which must be passed in when registering the provider:
+         * ```
+         *    tokenTypes: ['properties', 'types', 'classes'],
+         *    tokenModifiers: ['private', 'static']
+         * ```
+         *
+         * 2. The first transformation step is to encode `tokenType` and `tokenModifiers` as integers using the legend. Token types are looked
+         * up by index, so a `tokenType` value of `1` means `tokenTypes[1]`. Multiple token modifiers can be set by using bit flags,
+         * so a `tokenModifier` value of `3` is first viewed as binary `0b00000011`, which means `[tokenModifiers[0], tokenModifiers[1]]` because
+         * bits 0 and 1 are set. Using this legend, the tokens now are:
+         * ```
+         *    { line: 2, startChar:  5, length: 3, tokenType: 0, tokenModifiers: 3 },
+         *    { line: 2, startChar: 10, length: 4, tokenType: 1, tokenModifiers: 0 },
+         *    { line: 5, startChar:  2, length: 7, tokenType: 2, tokenModifiers: 0 }
+         * ```
+         *
+         * 3. The next steps is to encode each token relative to the previous token in the file. In this case, the second token
+         * is on the same line as the first token, so the `startChar` of the second token is made relative to the `startChar`
+         * of the first token, so it will be `10 - 5`. The third token is on a different line than the second token, so the
+         * `startChar` of the third token will not be altered:
+         * ```
+         *    { deltaLine: 2, deltaStartChar: 5, length: 3, tokenType: 0, tokenModifiers: 3 },
+         *    { deltaLine: 0, deltaStartChar: 5, length: 4, tokenType: 1, tokenModifiers: 0 },
+         *    { deltaLine: 3, deltaStartChar: 2, length: 7, tokenType: 2, tokenModifiers: 0 }
+         * ```
+         *
+         * 4. Finally, the last step is to inline each of the 5 fields for a token in a single array, which is a memory friendly representation:
+         * ```
+         *    // 1st token,  2nd token,  3rd token
+         *    [  2,5,3,0,3,  0,5,4,1,0,  3,2,7,2,0 ]
+         * ```
+         */
+        provideDocumentSemanticTokens(document: TextDocument, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
+
+        /**
+         * Instead of always returning all the tokens in a file, it is possible for a `DocumentSemanticTokensProvider` to implement
+         * this method (`updateSemanticTokens`) and then return incremental updates to the previously provided semantic tokens.
+         *
+         * ---
+         * ### How tokens change when the document changes
+         *
+         * Let's look at how tokens might change.
+         *
+         * Continuing with the above example, suppose a new line was inserted at the top of the file.
+         * That would make all the tokens move down by one line (notice how the line has changed for each one):
+         * ```
+         *    { line: 3, startChar:  5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] },
+         *    { line: 3, startChar: 10, length: 4, tokenType: "types",      tokenModifiers: [] },
+         *    { line: 6, startChar:  2, length: 7, tokenType: "classes",    tokenModifiers: [] }
+         * ```
+         * The integer encoding of the tokens does not change substantially because of the delta-encoding of positions:
+         * ```
+         *    // 1st token,  2nd token,  3rd token
+         *    [  3,5,3,0,3,  0,5,4,1,0,  3,2,7,2,0 ]
+         * ```
+         * It is possible to express these new tokens in terms of an edit applied to the previous tokens:
+         * ```
+         *    [  2,5,3,0,3,  0,5,4,1,0,  3,2,7,2,0 ] // old tokens
+         *    [  3,5,3,0,3,  0,5,4,1,0,  3,2,7,2,0 ] // new tokens
+         *
+         *    edit: { start:  0, deleteCount: 1, data: [3] } // replace integer at offset 0 with 3
+         * ```
+         *
+         * Furthermore, let's assume that a new token has appeared on line 4:
+         * ```
+         *    { line: 3, startChar:  5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] },
+         *    { line: 3, startChar: 10, length: 4, tokenType: "types",      tokenModifiers: [] },
+         *    { line: 4, startChar:  3, length: 5, tokenType: "properties", tokenModifiers: ["static"] },
+         *    { line: 6, startChar:  2, length: 7, tokenType: "classes",    tokenModifiers: [] }
+         * ```
+         * The integer encoding of the tokens is:
+         * ```
+         *    // 1st token,  2nd token,  3rd token,  4th token
+         *    [  3,5,3,0,3,  0,5,4,1,0,  1,3,5,0,2,  2,2,7,2,0, ]
+         * ```
+         * Again, it is possible to express these new tokens in terms of an edit applied to the previous tokens:
+         * ```
+         *    [  3,5,3,0,3,  0,5,4,1,0,  3,2,7,2,0 ]               // old tokens
+         *    [  3,5,3,0,3,  0,5,4,1,0,  1,3,5,0,2,  2,2,7,2,0, ]  // new tokens
+         *
+         *    edit: { start: 10, deleteCount: 1, data: [1,3,5,0,2,2] } // replace integer at offset 10 with [1,3,5,0,2,2]
+         * ```
+         *
+         * *NOTE*: When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider.
+         * *NOTE*: If the provider cannot compute `SemanticTokensEdits`, it can "give up" and return all the tokens in the document again.
+         * *NOTE*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state.
+         */
+        provideDocumentSemanticTokensEdits?(document: TextDocument, previousResultId: string, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
+    }
+
+    /**
+     * The document range semantic tokens provider interface defines the contract between extensions and
+     * semantic tokens.
+     */
+    export interface DocumentRangeSemanticTokensProvider {
+        /**
+         * See [provideDocumentSemanticTokens](#DocumentSemanticTokensProvider.provideDocumentSemanticTokens).
+         */
+        provideDocumentRangeSemanticTokens(document: TextDocument, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
+    }
+
+    export namespace languages {
+        /**
+         * Register a semantic tokens provider for a whole document.
+         *
+         * Multiple providers can be registered for a language. In that case providers are sorted
+         * by their [score](#languages.match) and the best-matching provider is used. Failure
+         * of the selected provider will cause a failure of the whole operation.
+         *
+         * @param selector A selector that defines the documents this provider is applicable to.
+         * @param provider A document semantic tokens provider.
+         * @return A [disposable](#Disposable) that unregisters this provider when being disposed.
+         */
+        export function registerDocumentSemanticTokensProvider(selector: DocumentSelector, provider: DocumentSemanticTokensProvider, legend: SemanticTokensLegend): Disposable;
+
+        /**
+         * Register a semantic tokens provider for a document range.
+         *
+         * Multiple providers can be registered for a language. In that case providers are sorted
+         * by their [score](#languages.match) and the best-matching provider is used. Failure
+         * of the selected provider will cause a failure of the whole operation.
+         *
+         * @param selector A selector that defines the documents this provider is applicable to.
+         * @param provider A document range semantic tokens provider.
+         * @return A [disposable](#Disposable) that unregisters this provider when being disposed.
+         */
+        export function registerDocumentRangeSemanticTokensProvider(selector: DocumentSelector, provider: DocumentRangeSemanticTokensProvider, legend: SemanticTokensLegend): Disposable;
+    }
+
+    //#endregion
+
+}
+/* tslint:enable:max-classes-per-file */

src/Razor/test/Microsoft.AspNetCore.Razor.LanguageServer.Test/Hover/DefaultRazorHoverInfoServiceTest.cs

@@ -138,7 +138,7 @@ namespace Microsoft.AspNetCore.Razor.LanguageServer.Test.Hover
             // Assert
             Assert.NotNull(hover);
             Assert.Contains("**Test**", hover.Contents.MarkupContent.Value);
-            var expectedRange = new RangeModel(new Position(1, 4), new Position(1, 22));
+            var expectedRange = new RangeModel(new Position(1, 5), new Position(1, 10));
             Assert.Equal(expectedRange, hover.Range);
         }
 

src/Razor/test/Microsoft.AspNetCore.Razor.LanguageServer.Test/Semantic/DefaultRazorSemanticTokenInfoServiceTest.cs

@@ -0,0 +1,122 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Razor.Language;
+using Microsoft.AspNetCore.Razor.LanguageServer.Completion;
+using Microsoft.AspNetCore.Razor.LanguageServer.Semantic;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Razor.LanguageServer.Test.Semantic
+{
+    public class DefaultRazorSemanticTokenInfoServiceTest : DefaultTagHelperServiceTestBase
+    {
+        #region TagHelpers
+        [Fact]
+        public void GetSemanticTokens_NoAttributes()
+        {
+            var txt = $"@addTagHelper *, TestAssembly{Environment.NewLine}<test1></test1>";
+            var expectedData = new List<uint> {
+                1, 1, 5, 0, 0, //line, character pos, length, tokenType, modifier
+                0, 8, 5, 0, 0
+            };
+
+            AssertSemanticTokens(txt, expectedData);
+        }
+
+        [Fact]
+        public void GetSemanticTokens_WithAttribute()
+        {
+            var txt = $"@addTagHelper *, TestAssembly{Environment.NewLine}<test1 bool-val='true'></test1>";
+            var expectedData = new List<uint> {
+                1, 1, 5, 0, 0, //line, character pos, length, tokenType, modifier
+                0, 6, 8, 1, 0,
+                0, 18, 5, 0, 0
+            };
+
+            AssertSemanticTokens(txt, expectedData);
+        }
+
+        [Fact]
+        public void GetSemanticTokens_MinimizedAttribute()
+        {
+            var txt = $"@addTagHelper *, TestAssembly{Environment.NewLine}<test1 bool-val></test1>";
+            var expectedData = new List<uint> {
+                1, 1, 5, 0, 0, //line, character pos, length, tokenType, modifier
+                0, 6, 8, 1, 0,
+                0, 11, 5, 0, 0
+            };
+
+            AssertSemanticTokens(txt, expectedData);
+        }
+
+        [Fact]
+        public void GetSemanticTokens_IgnoresNonTagHelperAttributes()
+        {
+            var txt = $"@addTagHelper *, TestAssembly{Environment.NewLine}<test1 bool-val='true' class='display:none'></test1>";
+            var expectedData = new List<uint> {
+                1, 1, 5, 0, 0, //line, character pos, length, tokenType, modifier
+                0, 6, 8, 1, 0,
+                0, 39, 5, 0, 0
+            };
+
+            AssertSemanticTokens(txt, expectedData);
+        }
+
+        [Fact]
+        public void GetSemanticTokens_DoesNotApplyOnNonTagHelpers()
+        {
+            var txt = $"@addTagHelper *, TestAssembly{Environment.NewLine}<p bool-val='true'></p>";
+            var expectedData = new List<uint> { };
+
+            AssertSemanticTokens(txt, expectedData);
+        }
+        #endregion TagHelpers
+
+        #region DirectiveAttributes
+        [Fact(Skip = "Haven't implemented directive attributes yet")]
+        public void GetSemanticTokens_DirectiveAttributes()
+        {
+            var txt = $"@addTagHelper *, TestAssembly{Environment.NewLine}<test1 @onclick='Function'></test1>";
+            var expectedData = new List<uint> {
+                1, 1, 5, 1, 0, //line, character pos, length, tokenType, modifier
+                0, 8, 5, 2, 0
+            };
+
+            AssertSemanticTokens(txt, expectedData);
+        }
+
+        [Fact(Skip = "Haven't implemented directive attributes yet")]
+        public void GetSemanticTokens_DirectiveAttributesWithParameters()
+        {
+            var txt = $"@addTagHelper *, TestAssembly{Environment.NewLine}<test1 @onclick:preventDefault='Function'></test1>";
+            var expectedData = new List<uint> {
+                1, 1, 5, 1, 0, //line, character pos, length, tokenType, modifier
+                0, 8, 5, 2, 0
+            };
+
+            AssertSemanticTokens(txt, expectedData);
+        }
+        #endregion DirectiveAttributes
+
+        private void AssertSemanticTokens(string txt, IEnumerable<uint> expectedData)
+        {
+            // Arrange
+            var service = GetDefaultRazorSemanticTokenInfoService();
+            var codeDocument = CreateCodeDocument(txt, DefaultTagHelpers);
+            var location = new SourceLocation(txt.IndexOf("test1"), -1, -1);
+
+            // Act
+            var tokens = service.GetSemanticTokens(codeDocument, location);
+
+            // Assert
+            Assert.Equal(expectedData, tokens.Data);
+        }
+
+        private RazorSemanticTokenInfoService GetDefaultRazorSemanticTokenInfoService()
+        {
+            return new DefaultRazorSemanticTokenInfoService();
+        }
+    }
+}

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tsconfig.json

@@ -1,7 +1,8 @@
 {
 	"extends": "../../tsconfig.json",
 	"compilerOptions": {
-		"outDir": "dist"
+    "outDir": "dist",
+    "sourceMap": true
 	},
 	"include": [
 	  "tests/**/*"

src/Razor/test/VSCode.FunctionalTest/tests/Hover2_2.test.ts

@@ -112,56 +112,62 @@ suite('Hover 2.2', () => {
         }
 
         assert.equal(hoverResult.length, 2, 'Something else may be providing hover results');
+        const envResult = hoverResult.find((hover, index, obj) => {
+            return (hover.contents[0] as vscode.MarkdownString).value.includes('InputTagHelper');
+        });
 
-        let envResult = hoverResult[0];
-        let expectedRange = new vscode.Range(
-            new vscode.Position(0, 1),
-            new vscode.Position(0, 6));
-        assert.deepEqual(envResult.range, expectedRange, 'TagHelper range should be <input>');
-        let mStr = envResult.contents[0] as vscode.MarkdownString;
-        assert.ok(mStr.value.includes('InputTagHelper'), `InputTagHelper not included in '${mStr.value}'`);
+        if (!envResult) {
+            assert.fail('Should have found a TagHelper');
+        } else {
+            let expectedRange = new vscode.Range(
+                new vscode.Position(0, 1),
+                new vscode.Position(0, 6));
+            assert.deepEqual(envResult.range, expectedRange, 'TagHelper range should be <input>');
+            let mStr = envResult.contents[0] as vscode.MarkdownString;
+            assert.ok(mStr.value.includes('InputTagHelper'), `InputTagHelper not included in '${mStr.value}'`);
 
-        hoverResult = await vscode.commands.executeCommand<vscode.Hover[]>(
-            'vscode.executeHoverProvider',
-            cshtmlDoc.uri,
-            new vscode.Position(0, 8));
+            hoverResult = await vscode.commands.executeCommand<vscode.Hover[]>(
+                'vscode.executeHoverProvider',
+                cshtmlDoc.uri,
+                new vscode.Position(0, 8));
 
-        assert.ok(hoverResult, 'Should have a hover result for asp-for');
-        if (!hoverResult) {
-            // This can never happen
-            return;
+            assert.ok(hoverResult, 'Should have a hover result for asp-for');
+            if (!hoverResult) {
+                // This can never happen
+                return;
+            }
+
+            assert.equal(hoverResult.length, 1, 'Something else may be providing hover results');
+
+            const aspForResult = hoverResult[0];
+            expectedRange = new vscode.Range(
+                new vscode.Position(0, 7),
+                new vscode.Position(0, 14));
+            assert.deepEqual(aspForResult.range, expectedRange, 'asp-for should be selected');
+            mStr = aspForResult.contents[0] as vscode.MarkdownString;
+            assert.ok(mStr.value.includes('InputTagHelper.**For**'), `InputTagHelper.For not included in '${mStr.value}'`);
+
+            hoverResult = await vscode.commands.executeCommand<vscode.Hover[]>(
+                'vscode.executeHoverProvider',
+                cshtmlDoc.uri,
+                new vscode.Position(0, 19));
+
+            assert.ok(hoverResult, 'Should have a hover result for class');
+            if (!hoverResult) {
+                // This can never happen
+                return;
+            }
+
+            assert.equal(hoverResult.length, 1, 'Something else may be providing hover results');
+
+            const result = hoverResult[0];
+            expectedRange = new vscode.Range(
+                new vscode.Position(0, 19),
+                new vscode.Position(0, 24));
+            assert.deepEqual(result.range, expectedRange, 'class should be selected');
+            mStr = result.contents[0] as vscode.MarkdownString;
+            assert.ok(mStr.value.includes('class'), `class not included in ${mStr.value}`);
         }
-
-        assert.equal(hoverResult.length, 1, 'Something else may be providing hover results');
-
-        envResult = hoverResult[0];
-        expectedRange = new vscode.Range(
-            new vscode.Position(0, 7),
-            new vscode.Position(0, 14));
-        assert.deepEqual(envResult.range, expectedRange, 'asp-for should be selected');
-        mStr = envResult.contents[0] as vscode.MarkdownString;
-        assert.ok(mStr.value.includes('InputTagHelper.**For**'), `InputTagHelper.For not included in '${mStr.value}'`);
-
-        hoverResult = await vscode.commands.executeCommand<vscode.Hover[]>(
-            'vscode.executeHoverProvider',
-            cshtmlDoc.uri,
-            new vscode.Position(0, 19));
-
-        assert.ok(hoverResult, 'Should have a hover result for class');
-        if (!hoverResult) {
-            // This can never happen
-            return;
-        }
-
-        assert.equal(hoverResult.length, 1, 'Something else may be providing hover results');
-
-        const result = hoverResult[0];
-        expectedRange = new vscode.Range(
-            new vscode.Position(0, 19),
-            new vscode.Position(0, 24));
-        assert.deepEqual(result.range, expectedRange, 'class should be selected');
-        mStr = result.contents[0] as vscode.MarkdownString;
-        assert.ok(mStr.value.includes('class'), `class not included in ${mStr.value}`);
     });
 
     // MvcWithComponents doesn't find TagHelpers because of test setup foibles.

src/Razor/test/VSCode.FunctionalTest/tests/HoverComponents.test.ts

@@ -6,12 +6,21 @@
 import * as assert from 'assert';
 import { beforeEach } from 'mocha';
 import * as path from 'path';
+import { error } from 'util';
 import * as vscode from 'vscode';
 import { componentRoot } from './TestUtil';
 
 let cshtmlDoc: vscode.TextDocument;
 let editor: vscode.TextEditor;
 
+function RangeToStr(range: vscode.Range | undefined): string {
+    if (range) {
+        return `${range.start.line}:${range.start.character} to ${range.end.line}:${range.end.character}`;
+    } else {
+        return 'undefined';
+    }
+}
+
 suite('Hover Components', () => {
     beforeEach(async () => {
         const filePath = path.join(componentRoot, 'Components', 'Shared', 'MainLayout.razor');
@@ -39,11 +48,18 @@ suite('Hover Components', () => {
 
         assert.ok(hoverResult.length > 0, 'Should have atleast one result.');
 
-        const onClickResult = hoverResult[0];
+        const onClickResult = hoverResult.find(hover => (hover.contents[0] as vscode.MarkdownString).value.includes('EventCallback'));
+        if (!onClickResult) {
+            assert.fail('No eventhandler result was found');
+            throw error();
+        }
         const expectedRange = new vscode.Range(
-            new vscode.Position(1, 31),
-            new vscode.Position(1, 58));
-        assert.deepEqual(hoverResult[0].range, expectedRange, 'Directive range should be @onclick');
+            new vscode.Position(1, 32),
+            new vscode.Position(1, 40));
+        const rangeContent = counterDoc.getText(onClickResult.range);
+
+        assert.equal(rangeContent, '@onclick');
+        assert.deepEqual(onClickResult.range, expectedRange, `Directive range should be @onclick: ${RangeToStr(expectedRange)} but was ${rangeContent}: ${RangeToStr(onClickResult.range)}`);
         const mStr = onClickResult.contents[0] as vscode.MarkdownString;
         assert.ok(mStr.value.includes('EventHandlers.**onclick**'), `**onClick** not included in '${mStr.value}'`);
     });

src/Razor/test/VSCode.FunctionalTest/tsconfig.json

@@ -1,9 +1,9 @@
 {
-	"extends": "../../tsconfig.json",
-	"compilerOptions": {
-		"outDir": "dist"
-	},
-	"include": [
-	  "tests/**/*"
-	]
-  }
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "dist"
+  },
+  "include": [
+    "tests/**/*"
+  ]
+}

src/Razor/tslint.json

@@ -1,6 +1,7 @@
 {
     "extends": "tslint:recommended",
     "rules": {
+        "array-type": false,
         "space-within-parens": true,
         "arrow-parens": false,
         "file-header": [true, "------"],