Final lexer changes (#11078)

* Switch the new lexer off-by-default.

* Add a new document for the lexer breaking changes around pragmas.

* Update comment.

* Feedback

Co-authored-by: Jan Jones <jan.jones.cz@gmail.com>

---------

Co-authored-by: Jan Jones <jan.jones.cz@gmail.com>

docs/Compiler Breaking Changes - DotNet 9.md

@@ -0,0 +1,56 @@
+# This document lists known breaking changes in Razor after .NET 9 all the way to .NET 10.
+
+## Preprocessor directive parsing breaks
+
+***Introduced in VS 17.13p1 and .NET 9.0.200***
+
+A new lexing mode was introduced for understanding the C# sections in razor files that brings increased compatibility with how C# is natively lexed. However, this
+also brings some breaking changes to the Razor compiler's understanding of C# preprocessing directives, which previously did not work consistently. Directives are
+now required to start at the beginning of a line in Razor files (only whitespace is allowed before them). Additionally, disabled sections are now properly disabled
+by the Razor compiler when `#if` preprocessor blocks are considered inactive.
+
+### Preprocessor blocks are required to start at the beginning of a line
+
+```razor
+@{ #if DEBUG /* Previously allowed, now triggers RZ1043 */ }
+<div>test</div>
+@{ #endif /* Previously allowed, now triggers RZ1043 */ }
+```
+
+To fix, move the directives to a new line. Only whitespace is allowed before the directive.
+
+```razor
+@{
+#if DEBUG /* This is allowed */
+}
+<div>test</div>
+@{
+    #endif /* This is allowed */
+}
+```
+
+### Disabled blocks are now considered properly in the Razor compiler
+
+Disabled blocks are now considered completely disabled by the Razor compiler, and no attempt to understand the block is made. When combined with the previous break,
+this means that if an `#else`, `#elif`, or `#endif` was not at the start of a line (modulo whitespace), a larger section of the file will be considered disabled than
+in older versions of the Razor compiler. To help diagnose potential breaks here, the Razor compiler will scan disabled text sections for potential misplaced preprocessor
+directives and report a warning if one is encountered.
+
+```razor
+@{
+#if false
+}
+
+This area is now properly considered disabled by the razor compiler, and no attempt to understand it as either C# or HTML is made. This
+can cause changes to how the output is rendered from previous versions of the Razor compiler.
+
+@{ #else
+    In previous versions of the Razor compiler, this directive would have been picked up. It is no longer picked up because it is not at
+    the start of a line. The Razor compiler will report a warning, RZ1044, to help diagnose any potential breaks in this area.
+}
+
+@{
+#endif
+}
+```
+

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/SourceGenerators/RazorSourceGenerator.RazorProviders.cs

@@ -55,9 +55,9 @@ namespace Microsoft.NET.Sdk.Razor.SourceGenerators
 
             var razorConfiguration = new RazorConfiguration(razorLanguageVersion, configurationName ?? "default", Extensions: [], UseConsolidatedMvcViews: true, SuppressAddComponentParameter: !isComponentParameterSupported);
 
-            // We use the new tokenizer by default
-            var useRazorTokenizer = !parseOptions.Features.TryGetValue("use-razor-tokenizer", out var useRazorTokenizerValue)
-                                    || !string.Equals(useRazorTokenizerValue, "false", StringComparison.OrdinalIgnoreCase);
+            // We use the new tokenizer only when requested for now.
+            var useRoslynTokenizer = parseOptions.Features.TryGetValue("use-roslyn-tokenizer", out var useRoslynTokenizerValue)
+                                    && string.Equals(useRoslynTokenizerValue, "true", StringComparison.OrdinalIgnoreCase);
 
             var razorSourceGenerationOptions = new RazorSourceGenerationOptions()
             {
@@ -67,7 +67,7 @@ namespace Microsoft.NET.Sdk.Razor.SourceGenerators
                 SupportLocalizedComponentNames = supportLocalizedComponentNames == "true",
                 CSharpParseOptions = (CSharpParseOptions)parseOptions,
                 TestSuppressUniqueIds = _testSuppressUniqueIds,
-                UseRoslynTokenizer = useRazorTokenizer,
+                UseRoslynTokenizer = useRoslynTokenizer,
             };
 
             return (razorSourceGenerationOptions, diagnostic);

src/Compiler/test/Microsoft.NET.Sdk.Razor.SourceGenerators.Tests/RazorSourceGeneratorTests.cs

@@ -3265,5 +3265,159 @@ namespace MyApp
                e => Assert.Equal("DiscoverTagHelpersFromCompilationStart", e.EventName),
                e => Assert.Equal("DiscoverTagHelpersFromCompilationStop", e.EventName));
         }
+
+        [Theory]
+        [InlineData("true")]
+        [InlineData("True")]
+        [InlineData("TRUE")]
+        [InlineData("tRuE")]
+        public async Task RoslynTokenizerEnabledWithTrue(string value)
+        {
+            var parseOptions = CSharpParseOptions.Default.WithFeatures([new("use-roslyn-tokenizer", value)]);
+
+            var project = CreateTestProject(new()
+            {
+                ["Pages/Index.razor"] = """"
+                    <div>@("""
+                        nested "
+                        """)</div>
+                    """",
+            }, cSharpParseOptions: parseOptions);
+
+            var compilation = await project.GetCompilationAsync();
+            var (driver, additionalTexts) = await GetDriverWithAdditionalTextAsync(project);
+
+            var result = RunGenerator(compilation!, ref driver);
+            result.VerifyPageOutput(
+                """"
+                #pragma checksum "Pages/Index.razor" "{ff1816ec-aa5e-4d10-87f7-6f4963833460}" "c6855f3cabbcb69477e3f5a61f8d77fcfed086c2"
+                // <auto-generated/>
+                #pragma warning disable 1591
+                namespace MyApp.Pages
+                {
+                    #line default
+                    using global::System;
+                    using global::System.Collections.Generic;
+                    using global::System.Linq;
+                    using global::System.Threading.Tasks;
+                    using global::Microsoft.AspNetCore.Components;
+                    #line default
+                    #line hidden
+                    #nullable restore
+                    public partial class Index : global::Microsoft.AspNetCore.Components.ComponentBase
+                    #nullable disable
+                    {
+                        #pragma warning disable 1998
+                        protected override void BuildRenderTree(global::Microsoft.AspNetCore.Components.Rendering.RenderTreeBuilder __builder)
+                        {
+                            __builder.OpenElement(0, "div");
+                            __builder.AddContent(1, 
+                #nullable restore
+                #line (1,8)-(3,8) "Pages/Index.razor"
+                """
+                    nested "
+                    """
+
+                #line default
+                #line hidden
+                #nullable disable
+                            );
+                            __builder.CloseElement();
+                        }
+                        #pragma warning restore 1998
+                    }
+                }
+                #pragma warning restore 1591
+
+                """");
+        }
+
+        [Theory]
+        [InlineData("false")]
+        [InlineData("False")]
+        [InlineData("FALSE")]
+        [InlineData("FaLsE")]
+        [InlineData("")]
+        [InlineData(null)]
+        public async Task RoslynTokenizerDisabledWithFalseOrNothing(string? value)
+        {
+            var parseOptions = CSharpParseOptions.Default;
+
+            if (value != null)
+            {
+                parseOptions = parseOptions.WithFeatures([new("use-roslyn-tokenizer", value)]);
+            }
+
+            var project = CreateTestProject(new()
+            {
+                ["Pages/Index.razor"] = """"
+                    <div>@("""
+                        nested "
+                        """)</div>
+                    """",
+            }, cSharpParseOptions: parseOptions);
+
+            var compilation = await project.GetCompilationAsync();
+            var (driver, additionalTexts) = await GetDriverWithAdditionalTextAsync(project);
+
+            var result = RunGenerator(compilation!, ref driver,
+                // Pages/Index.razor(3,10): error CS1525: Invalid expression term '/'
+                //     """)</div>
+                Diagnostic(ErrorCode.ERR_InvalidExprTerm, "/").WithArguments("/").WithLocation(3, 10),
+                // Pages/Index.razor(3,11): error CS0103: The name 'div' does not exist in the current context
+                //     """)</div>
+                Diagnostic(ErrorCode.ERR_NameNotInContext, "div").WithArguments("div").WithLocation(3, 11),
+                // Pages/Index.razor(3,15): error CS1525: Invalid expression term ')'
+                //     """)</div>
+                Diagnostic(ErrorCode.ERR_InvalidExprTerm, "").WithArguments(")").WithLocation(3, 15),
+                // Pages/Index.razor(3,15): error CS1002: ; expected
+                //     """)</div>
+                Diagnostic(ErrorCode.ERR_SemicolonExpected, "").WithLocation(3, 15),
+                // Pages/Index.razor(3,15): error CS1513: } expected
+                //     """)</div>
+                Diagnostic(ErrorCode.ERR_RbraceExpected, "").WithLocation(3, 15));
+
+            result.VerifyPageOutput(
+                """"
+                #pragma checksum "Pages/Index.razor" "{ff1816ec-aa5e-4d10-87f7-6f4963833460}" "c6855f3cabbcb69477e3f5a61f8d77fcfed086c2"
+                // <auto-generated/>
+                #pragma warning disable 1591
+                namespace MyApp.Pages
+                {
+                    #line default
+                    using global::System;
+                    using global::System.Collections.Generic;
+                    using global::System.Linq;
+                    using global::System.Threading.Tasks;
+                    using global::Microsoft.AspNetCore.Components;
+                    #line default
+                    #line hidden
+                    #nullable restore
+                    public partial class Index : global::Microsoft.AspNetCore.Components.ComponentBase
+                    #nullable disable
+                    {
+                        #pragma warning disable 1998
+                        protected override void BuildRenderTree(global::Microsoft.AspNetCore.Components.Rendering.RenderTreeBuilder __builder)
+                        {
+                            __builder.OpenElement(0, "div");
+                            __builder.AddContent(1, 
+                #nullable restore
+                #line (1,8)-(3,15) "Pages/Index.razor"
+                """
+                    nested "
+                    """)</div>
+                #line default
+                #line hidden
+                #nullable disable
+                            );
+                            __builder.CloseElement();
+                        }
+                        #pragma warning restore 1998
+                    }
+                }
+                #pragma warning restore 1591
+
+                """");
+        }
     }
 }

src/Compiler/test/Microsoft.NET.Sdk.Razor.SourceGenerators.Tests/RazorSourceGeneratorTestsBase.cs

@@ -230,9 +230,10 @@ public abstract class RazorSourceGeneratorTestsBase
 
     protected static Project CreateTestProject(
         OrderedStringDictionary additionalSources,
-        OrderedStringDictionary? sources = null)
+        OrderedStringDictionary? sources = null,
+        CSharpParseOptions? cSharpParseOptions = null)
     {
-        var project = CreateBaseProject();
+        var project = CreateBaseProject(cSharpParseOptions);
 
         if (sources is not null)
         {
@@ -294,7 +295,7 @@ public abstract class RazorSourceGeneratorTestsBase
         }
     }
 
-    private static Project CreateBaseProject()
+    private static Project CreateBaseProject(CSharpParseOptions? cSharpParseOptions)
     {
         var projectId = ProjectId.CreateNewId(debugName: "TestProject");
 
@@ -314,7 +315,7 @@ public abstract class RazorSourceGeneratorTestsBase
                     new("CS8019", ReportDiagnostic.Suppress),
                 }));
 
-        project = project.WithParseOptions(((CSharpParseOptions)project.ParseOptions!).WithLanguageVersion(LanguageVersion.Preview));
+        project = project.WithParseOptions(cSharpParseOptions ?? ((CSharpParseOptions)project.ParseOptions!).WithLanguageVersion(LanguageVersion.Preview));
 
         foreach (var defaultCompileLibrary in DependencyContext.Load(typeof(RazorSourceGeneratorTests).Assembly)!.CompileLibraries)
         {