From d80065912c15963e1c5e5d5e531aec8c6302eee0 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Fri, 10 Feb 2023 15:47:31 -0600 Subject: [PATCH] Move source generator validation to AnalyzerTest --- .../AnalyzerTest`1.cs | 205 +++++++++++- .../PublicAPI.Unshipped.txt | 3 +- .../CSharpSourceGeneratorTest`2.cs | 10 - .../PublicAPI.Unshipped.txt | 1 - .../PublicAPI.Unshipped.txt | 3 - .../SourceGeneratorTest`1.cs | 286 ---------------- .../PublicAPI.Unshipped.txt | 1 - .../VisualBasicSourceGeneratorTest`2.vb | 9 - .../SourceGeneratorTests.cs | 314 +++++++++++++++++- .../SourceGeneratorValidationTests.cs | 39 +-- .../TestGenerators/AddEmptyFile.cs | 0 .../AddEmptyFileWithDiagnostic.cs | 0 .../TestGenerators/AddFileWithCompileError.cs | 0 .../TestGenerators/AddTwoEmptyFiles.cs | 0 14 files changed, 521 insertions(+), 350 deletions(-) rename tests/Microsoft.CodeAnalysis.Testing/{Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests => Microsoft.CodeAnalysis.Testing.Utilities}/TestGenerators/AddEmptyFile.cs (100%) rename tests/Microsoft.CodeAnalysis.Testing/{Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests => Microsoft.CodeAnalysis.Testing.Utilities}/TestGenerators/AddEmptyFileWithDiagnostic.cs (100%) rename tests/Microsoft.CodeAnalysis.Testing/{Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests => Microsoft.CodeAnalysis.Testing.Utilities}/TestGenerators/AddFileWithCompileError.cs (100%) rename tests/Microsoft.CodeAnalysis.Testing/{Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests => Microsoft.CodeAnalysis.Testing.Utilities}/TestGenerators/AddTwoEmptyFiles.cs (100%) diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs index 23836751..67cca652 100644 --- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs +++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs @@ -12,6 +12,10 @@ using System.Reflection; using System.Text; using System.Threading; using System.Threading.Tasks; +using DiffPlex; +using DiffPlex.Chunkers; +using DiffPlex.DiffBuilder; +using DiffPlex.DiffBuilder.Model; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Host.Mef; using Microsoft.CodeAnalysis.Options; @@ -196,7 +200,11 @@ namespace Microsoft.CodeAnalysis.Testing /// A representing the asynchronous operation. protected virtual async Task RunImplAsync(CancellationToken cancellationToken) { - Verify.NotEmpty($"{nameof(TestState)}.{nameof(SolutionState.Sources)}", TestState.Sources); + if (!TestState.GeneratedSources.Any()) + { + // Verify the test state has at least one source, which may or may not be generated + Verify.NotEmpty($"{nameof(TestState)}.{nameof(SolutionState.Sources)}", TestState.Sources); + } var analyzers = GetDiagnosticAnalyzers().ToArray(); var defaultDiagnostic = GetDefaultDiagnostic(analyzers); @@ -204,6 +212,7 @@ namespace Microsoft.CodeAnalysis.Testing var fixableDiagnostics = ImmutableArray.Empty; var testState = TestState.WithInheritedValuesApplied(null, fixableDiagnostics).WithProcessedMarkup(MarkupOptions, defaultDiagnostic, supportedDiagnostics, fixableDiagnostics, DefaultFilePath); + var diagnostics = await VerifySourceGeneratorAsync(testState, Verify, cancellationToken).ConfigureAwait(false); await VerifyDiagnosticsAsync(new EvaluatedProjectState(testState, ReferenceAssemblies), testState.AdditionalProjects.Values.Select(additionalProject => new EvaluatedProjectState(additionalProject, ReferenceAssemblies)).ToImmutableArray(), testState.ExpectedDiagnostics.ToArray(), Verify, cancellationToken).ConfigureAwait(false); } @@ -254,6 +263,187 @@ namespace Microsoft.CodeAnalysis.Testing $" {FormatDiagnostics(analyzers, DefaultFilePath, actual)}{Environment.NewLine}"; } + /// + /// Called to test a C# source generator when applied on the input source as a string. + /// + /// The effective input test state. + /// The verifier to use for test assertions. + /// The that the task will observe. + /// A representing the asynchronous operation. + protected async Task> VerifySourceGeneratorAsync(SolutionState testState, IVerifier verifier, CancellationToken cancellationToken) + { + var sourceGenerators = GetSourceGenerators().ToImmutableArray(); + if (sourceGenerators.IsEmpty) + { + return ImmutableArray.Empty; + } + + return await VerifySourceGeneratorAsync(Language, GetSourceGenerators().ToImmutableArray(), testState, ApplySourceGeneratorAsync, verifier.PushContext("Source generator application"), cancellationToken); + } + + private protected async Task> VerifySourceGeneratorAsync( + string language, + ImmutableArray sourceGenerators, + SolutionState testState, + Func, Project, IVerifier, CancellationToken, Task<(Project project, ImmutableArray diagnostics)>> getFixedProject, + IVerifier verifier, + CancellationToken cancellationToken) + { + var project = await CreateProjectAsync(new EvaluatedProjectState(testState, ReferenceAssemblies), testState.AdditionalProjects.Values.Select(additionalProject => new EvaluatedProjectState(additionalProject, ReferenceAssemblies)).ToImmutableArray(), cancellationToken); + + ImmutableArray diagnostics; + (project, diagnostics) = await getFixedProject(sourceGenerators, project, verifier, cancellationToken).ConfigureAwait(false); + + // After applying the source generator, compare the resulting string to the inputted one + if (!TestBehaviors.HasFlag(TestBehaviors.SkipGeneratedSourcesCheck)) + { + var numOriginalSources = testState.Sources.Count; + var updatedOriginalDocuments = project.Documents.Take(numOriginalSources).ToArray(); + var generatedDocuments = project.Documents.Skip(numOriginalSources).ToArray(); + + // Verify no changes occurred to the original documents + var updatedOriginalDocumentsWithTextBuilder = ImmutableArray.CreateBuilder<(Document document, SourceText content)>(); + foreach (var updatedOriginalDocument in updatedOriginalDocuments) + { + updatedOriginalDocumentsWithTextBuilder.Add((updatedOriginalDocument, await updatedOriginalDocument.GetTextAsync(CancellationToken.None).ConfigureAwait(false))); + } + + VerifyDocuments( + verifier.PushContext("Original files after running source generators"), + updatedOriginalDocumentsWithTextBuilder.ToImmutable(), + testState.Sources.ToImmutableArray(), + allowReordering: false, + DefaultFilePathPrefix, + GetNameAndFoldersFromPath, + MatchDiagnosticsTimeout); + + // Verify the source generated documents match expectations + var generatedDocumentsWithTextBuilder = ImmutableArray.CreateBuilder<(Document document, SourceText content)>(); + foreach (var generatedDocument in generatedDocuments) + { + generatedDocumentsWithTextBuilder.Add((generatedDocument, await generatedDocument.GetTextAsync(CancellationToken.None).ConfigureAwait(false))); + } + + VerifyDocuments( + verifier.PushContext("Verifying source generated files"), + generatedDocumentsWithTextBuilder.ToImmutable(), + testState.GeneratedSources.ToImmutableArray(), + allowReordering: true, + DefaultFilePathPrefix, + static (_, path) => GetNameAndFoldersFromSourceGeneratedFilePath(path), + MatchDiagnosticsTimeout); + } + + return diagnostics; + + static void VerifyDocuments( + IVerifier verifier, + ImmutableArray<(Document document, SourceText content)> actualDocuments, + ImmutableArray<(string filename, SourceText content)> expectedDocuments, + bool allowReordering, + string defaultFilePathPrefix, + Func folders)> getNameAndFolders, + TimeSpan matchTimeout) + { + ImmutableArray> matches; + if (allowReordering) + { + matches = WeightedMatch.Match( + expectedDocuments, + actualDocuments, + ImmutableArray.Create>( + static (expected, actual, exactOnly) => + { + if (actual.content.ToString() == expected.content.ToString()) + { + return 0.0; + } + + if (exactOnly) + { + // Avoid expensive diff calculation when exact match was requested. + return 1.0; + } + + var diffBuilder = new InlineDiffBuilder(new Differ()); + var diff = diffBuilder.BuildDiffModel(expected.content.ToString(), actual.content.ToString(), ignoreWhitespace: true, ignoreCase: false, new LineChunker()); + var changeCount = diff.Lines.Count(static line => line.Type is ChangeType.Inserted or ChangeType.Deleted); + if (changeCount == 0) + { + // We have a failure caused only by line ending or whitespace differences. Make sure + // to use a non-zero value so it can be distinguished from exact matches. + changeCount = 1; + } + + // Apply a multiplier to the content distance to account for its increased importance + // over encoding and checksum algorithm changes. + var priority = 3; + + return priority * changeCount / (double)diff.Lines.Count; + }, + static (expected, actual, exactOnly) => + { + return actual.content.Encoding == expected.content.Encoding ? 0.0 : 1.0; + }, + static (expected, actual, exactOnly) => + { + return actual.content.ChecksumAlgorithm == expected.content.ChecksumAlgorithm ? 0.0 : 1.0; + }, + (expected, actual, exactOnly) => + { + var distance = 0.0; + var (fileName, folders) = getNameAndFolders(defaultFilePathPrefix, expected.filename); + if (fileName != actual.document.Name) + { + distance += 1.0; + } + + if (!folders.SequenceEqual(actual.document.Folders)) + { + distance += 1.0; + } + + return distance; + }), + matchTimeout); + } + else + { + // Matching with an empty set of matching functions always takes the 1:1 alignment without reordering + matches = WeightedMatch.Match( + expectedDocuments, + actualDocuments, + ImmutableArray>.Empty, + matchTimeout); + } + + // Use EqualOrDiff to verify the actual and expected filenames (and total collection length) in a convenient manner + verifier.EqualOrDiff( + string.Join(Environment.NewLine, matches.Select(match => match.TryGetExpected(out var expected) ? expected.filename : string.Empty)), + string.Join(Environment.NewLine, matches.Select(match => match.TryGetActual(out var actual) ? actual.document.FilePath : string.Empty)), + $"Expected source file list to match"); + + // Follow by verifying each property of interest + foreach (var result in matches) + { + if (!result.TryGetExpected(out var expected) + || !result.TryGetActual(out var actual)) + { + throw new InvalidOperationException("Unexpected state: should have failed during the previous assertion."); + } + + verifier.EqualOrDiff(expected.content.ToString(), actual.content.ToString(), $"content of '{expected.filename}' did not match. Diff shown with expected as baseline:"); + verifier.Equal(expected.content.Encoding, actual.content.Encoding, $"encoding of '{expected.filename}' was expected to be '{expected.content.Encoding?.WebName}' but was '{actual.content.Encoding?.WebName}'"); + verifier.Equal(expected.content.ChecksumAlgorithm, actual.content.ChecksumAlgorithm, $"checksum algorithm of '{expected.filename}' was expected to be '{expected.content.ChecksumAlgorithm}' but was '{actual.content.ChecksumAlgorithm}'"); + + // Source-generated sources are implicitly in a subtree, so they have a different folders calculation. + var (fileName, folders) = getNameAndFolders(defaultFilePathPrefix, expected.filename); + verifier.Equal(fileName, actual.document.Name, $"file name was expected to be '{fileName}' but was '{actual.document.Name}'"); + verifier.SequenceEqual(folders, actual.document.Folders, message: $"folders was expected to be '{string.Join("/", folders)}' but was '{string.Join("/", actual.document.Folders)}'"); + } + } + } + /// /// General method that gets a collection of actual s found in the source after the /// analyzer is run, then verifies each of them. @@ -921,10 +1111,11 @@ namespace Microsoft.CodeAnalysis.Testing var diagnostics = ImmutableArray.CreateBuilder<(Project project, Diagnostic diagnostic)>(); foreach (var project in solution.Projects) { - var compilation = await GetProjectCompilationAsync(project, verifier, cancellationToken).ConfigureAwait(false); + var (compilation, generatorDiagnostics) = await GetProjectCompilationAsync(project, verifier, cancellationToken).ConfigureAwait(false); var compilationWithAnalyzers = CreateCompilationWithAnalyzers(compilation, analyzers, GetAnalyzerOptions(project), cancellationToken); var allDiagnostics = await compilationWithAnalyzers.GetAllDiagnosticsAsync().ConfigureAwait(false); + diagnostics.AddRange(generatorDiagnostics.Select(diagnostic => (project, diagnostic))); diagnostics.AddRange(allDiagnostics.Where(diagnostic => !IsCompilerDiagnostic(diagnostic) || IsCompilerDiagnosticIncluded(diagnostic, compilerDiagnostics)).Select(diagnostic => (project, diagnostic))); } @@ -933,13 +1124,13 @@ namespace Microsoft.CodeAnalysis.Testing return results; } - protected virtual async Task GetProjectCompilationAsync(Project project, IVerifier verifier, CancellationToken cancellationToken) + protected virtual async Task<(Compilation compilation, ImmutableArray generatorDiagnostics)> GetProjectCompilationAsync(Project project, IVerifier verifier, CancellationToken cancellationToken) { - var (finalProject, _) = await ApplySourceGeneratorAsync(GetSourceGenerators().ToImmutableArray(), project, verifier, cancellationToken).ConfigureAwait(false); - return (await finalProject.GetCompilationAsync(cancellationToken).ConfigureAwait(false))!; + var (finalProject, generatorDiagnostics) = await ApplySourceGeneratorAsync(GetSourceGenerators().ToImmutableArray(), project, verifier, cancellationToken).ConfigureAwait(false); + return ((await finalProject.GetCompilationAsync(cancellationToken).ConfigureAwait(false))!, generatorDiagnostics); } - private async Task<(Project project, ImmutableArray diagnostics)> ApplySourceGeneratorAsync(ImmutableArray sourceGeneratorTypes, Project project, IVerifier verifier, CancellationToken cancellationToken) + private protected async Task<(Project project, ImmutableArray diagnostics)> ApplySourceGeneratorAsync(ImmutableArray sourceGeneratorTypes, Project project, IVerifier verifier, CancellationToken cancellationToken) { var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false); verifier.True(compilation is { }); @@ -963,7 +1154,7 @@ namespace Microsoft.CodeAnalysis.Testing return (updatedProject, result.Diagnostics); } - private static (string fileName, IEnumerable folders) GetNameAndFoldersFromSourceGeneratedFilePath(string filePath) + private protected static (string fileName, IEnumerable folders) GetNameAndFoldersFromSourceGeneratedFilePath(string filePath) { // Source-generated files are always implicitly subpaths under the project root path. var folders = Path.GetDirectoryName(filePath)!.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt index 6ebf8cc8..126d996b 100644 --- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt +++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt @@ -25,6 +25,7 @@ Microsoft.CodeAnalysis.Testing.AnalyzerTest.TestBehaviors.set -> void Microsoft.CodeAnalysis.Testing.AnalyzerTest.TestCode.set -> void Microsoft.CodeAnalysis.Testing.AnalyzerTest.TestState.get -> Microsoft.CodeAnalysis.Testing.SolutionState Microsoft.CodeAnalysis.Testing.AnalyzerTest.VerifyDiagnosticsAsync(Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState primaryProject, System.Collections.Immutable.ImmutableArray additionalProjects, Microsoft.CodeAnalysis.Testing.DiagnosticResult[] expected, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +Microsoft.CodeAnalysis.Testing.AnalyzerTest.VerifySourceGeneratorAsync(Microsoft.CodeAnalysis.Testing.SolutionState testState, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> Microsoft.CodeAnalysis.Testing.AnalyzerTest.XmlReferences.get -> System.Collections.Generic.Dictionary Microsoft.CodeAnalysis.Testing.AnalyzerVerifier Microsoft.CodeAnalysis.Testing.AnalyzerVerifier.AnalyzerVerifier() -> void @@ -349,7 +350,7 @@ virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.DefaultFilePathPr virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.DefaultTestProjectName.get -> string virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.GetAnalyzerOptions(Microsoft.CodeAnalysis.Project project) -> Microsoft.CodeAnalysis.Diagnostics.AnalyzerOptions virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.GetDefaultDiagnostic(Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer[] analyzers) -> Microsoft.CodeAnalysis.DiagnosticDescriptor -virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.GetProjectCompilationAsync(Microsoft.CodeAnalysis.Project project, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task +virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.GetProjectCompilationAsync(Microsoft.CodeAnalysis.Project project, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<(Microsoft.CodeAnalysis.Compilation compilation, System.Collections.Immutable.ImmutableArray generatorDiagnostics)> virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.GetSourceGenerators() -> System.Collections.Generic.IEnumerable virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.IsCompilerDiagnosticIncluded(Microsoft.CodeAnalysis.Diagnostic diagnostic, Microsoft.CodeAnalysis.Testing.CompilerDiagnostics compilerDiagnostics) -> bool virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest.RunImplAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing/CSharpSourceGeneratorTest`2.cs b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing/CSharpSourceGeneratorTest`2.cs index b620a470..33e03afc 100644 --- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing/CSharpSourceGeneratorTest`2.cs +++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing/CSharpSourceGeneratorTest`2.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; using Microsoft.CodeAnalysis.Testing; namespace Microsoft.CodeAnalysis.CSharp.Testing @@ -23,15 +22,6 @@ namespace Microsoft.CodeAnalysis.CSharp.Testing public override string Language => LanguageNames.CSharp; - protected override GeneratorDriver CreateGeneratorDriver(Project project, ImmutableArray sourceGenerators) - { - return CSharpGeneratorDriver.Create( - sourceGenerators, - project.AnalyzerOptions.AdditionalFiles, - (CSharpParseOptions)project.ParseOptions!, - project.AnalyzerOptions.AnalyzerConfigOptionsProvider); - } - protected override CompilationOptions CreateCompilationOptions() => new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, allowUnsafe: true); diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing/PublicAPI.Unshipped.txt b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing/PublicAPI.Unshipped.txt index 709ccae6..0816d37f 100644 --- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing/PublicAPI.Unshipped.txt +++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing/PublicAPI.Unshipped.txt @@ -3,7 +3,6 @@ Microsoft.CodeAnalysis.CSharp.Testing.CSharpSourceGeneratorTest Microsoft.CodeAnalysis.CSharp.Testing.CSharpSourceGeneratorVerifier.CSharpSourceGeneratorVerifier() -> void override Microsoft.CodeAnalysis.CSharp.Testing.CSharpSourceGeneratorTest.CreateCompilationOptions() -> Microsoft.CodeAnalysis.CompilationOptions -override Microsoft.CodeAnalysis.CSharp.Testing.CSharpSourceGeneratorTest.CreateGeneratorDriver(Microsoft.CodeAnalysis.Project project, System.Collections.Immutable.ImmutableArray sourceGenerators) -> Microsoft.CodeAnalysis.GeneratorDriver override Microsoft.CodeAnalysis.CSharp.Testing.CSharpSourceGeneratorTest.CreateParseOptions() -> Microsoft.CodeAnalysis.ParseOptions override Microsoft.CodeAnalysis.CSharp.Testing.CSharpSourceGeneratorTest.DefaultFileExt.get -> string override Microsoft.CodeAnalysis.CSharp.Testing.CSharpSourceGeneratorTest.GetSourceGenerators() -> System.Collections.Generic.IEnumerable diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/PublicAPI.Unshipped.txt b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/PublicAPI.Unshipped.txt index 6bd854d4..9dcd5037 100644 --- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/PublicAPI.Unshipped.txt +++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/PublicAPI.Unshipped.txt @@ -4,10 +4,7 @@ Microsoft.CodeAnalysis.Testing.EmptySourceGeneratorProvider.Execute(Microsoft.Co Microsoft.CodeAnalysis.Testing.EmptySourceGeneratorProvider.Initialize(Microsoft.CodeAnalysis.GeneratorInitializationContext context) -> void Microsoft.CodeAnalysis.Testing.SourceGeneratorTest Microsoft.CodeAnalysis.Testing.SourceGeneratorTest.SourceGeneratorTest() -> void -Microsoft.CodeAnalysis.Testing.SourceGeneratorTest.VerifySourceGeneratorAsync(Microsoft.CodeAnalysis.Testing.SolutionState testState, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task> Microsoft.CodeAnalysis.Testing.SourceGeneratorVerifier Microsoft.CodeAnalysis.Testing.SourceGeneratorVerifier.SourceGeneratorVerifier() -> void -abstract Microsoft.CodeAnalysis.Testing.SourceGeneratorTest.CreateGeneratorDriver(Microsoft.CodeAnalysis.Project project, System.Collections.Immutable.ImmutableArray sourceGenerators) -> Microsoft.CodeAnalysis.GeneratorDriver override Microsoft.CodeAnalysis.Testing.SourceGeneratorTest.GetDiagnosticAnalyzers() -> System.Collections.Generic.IEnumerable -override Microsoft.CodeAnalysis.Testing.SourceGeneratorTest.RunImplAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task override abstract Microsoft.CodeAnalysis.Testing.SourceGeneratorTest.GetSourceGenerators() -> System.Collections.Generic.IEnumerable diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/SourceGeneratorTest`1.cs b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/SourceGeneratorTest`1.cs index cca09dfa..80e27b56 100644 --- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/SourceGeneratorTest`1.cs +++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/SourceGeneratorTest`1.cs @@ -4,20 +4,8 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO; using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using DiffPlex; -using DiffPlex.Chunkers; -using DiffPlex.DiffBuilder; -using DiffPlex.DiffBuilder.Model; using Microsoft.CodeAnalysis.Diagnostics; -using Microsoft.CodeAnalysis.Formatting; -using Microsoft.CodeAnalysis.Simplification; -using Microsoft.CodeAnalysis.Testing.Model; -using Microsoft.CodeAnalysis.Text; namespace Microsoft.CodeAnalysis.Testing { @@ -32,279 +20,5 @@ namespace Microsoft.CodeAnalysis.Testing /// /// The to be used. protected override abstract IEnumerable GetSourceGenerators(); - - protected abstract GeneratorDriver CreateGeneratorDriver(Project project, ImmutableArray sourceGenerators); - - protected override async Task RunImplAsync(CancellationToken cancellationToken) - { - var analyzers = GetDiagnosticAnalyzers().ToArray(); - var defaultDiagnostic = GetDefaultDiagnostic(analyzers); - var supportedDiagnostics = analyzers.SelectMany(analyzer => analyzer.SupportedDiagnostics).ToImmutableArray(); - var fixableDiagnostics = ImmutableArray.Empty; - var testState = TestState.WithInheritedValuesApplied(null, fixableDiagnostics).WithProcessedMarkup(MarkupOptions, defaultDiagnostic, supportedDiagnostics, fixableDiagnostics, DefaultFilePath); - - var diagnostics = await VerifySourceGeneratorAsync(testState, Verify, cancellationToken).ConfigureAwait(false); - await VerifyDiagnosticsAsync(new EvaluatedProjectState(testState, ReferenceAssemblies).WithAdditionalDiagnostics(diagnostics), testState.AdditionalProjects.Values.Select(additionalProject => new EvaluatedProjectState(additionalProject, ReferenceAssemblies)).ToImmutableArray(), testState.ExpectedDiagnostics.ToArray(), Verify.PushContext("Diagnostics of test state"), cancellationToken).ConfigureAwait(false); - } - - /// - /// Called to test a C# source generator when applied on the input source as a string. - /// - /// The effective input test state. - /// The verifier to use for test assertions. - /// The that the task will observe. - /// A representing the asynchronous operation. - protected async Task> VerifySourceGeneratorAsync(SolutionState testState, IVerifier verifier, CancellationToken cancellationToken) - { - return await VerifySourceGeneratorAsync(Language, InstantiateSourceGenerators(GetSourceGenerators().ToImmutableArray()), testState, ApplySourceGeneratorAsync, verifier.PushContext("Source generator application"), cancellationToken); - } - - private ImmutableArray InstantiateSourceGenerators(ImmutableArray sourceGenerators) - { - return ImmutableArray.CreateRange( - sourceGenerators, - sourceGeneratorType => - { - var instance = Activator.CreateInstance(sourceGeneratorType); - if (instance is ISourceGenerator generator) - { - return generator; - } - - var iincrementalGeneratorType = typeof(ISourceGenerator).Assembly.GetType("Microsoft.CodeAnalysis.IIncrementalGenerator"); - - var asGeneratorMethod = (from method in typeof(ISourceGenerator).Assembly.GetType("Microsoft.CodeAnalysis.GeneratorExtensions")!.GetMethods() - where method is { Name: "AsSourceGenerator", IsStatic: true, IsPublic: true } - let parameterTypes = method.GetParameters().Select(parameter => parameter.ParameterType).ToArray() - where parameterTypes.SequenceEqual(new[] { iincrementalGeneratorType }) - select method).SingleOrDefault(); - return (ISourceGenerator)asGeneratorMethod.Invoke(null, new[] { instance })!; - }); - } - - private async Task> VerifySourceGeneratorAsync( - string language, - ImmutableArray sourceGenerators, - SolutionState testState, - Func, Project, IVerifier, CancellationToken, Task<(Project project, ImmutableArray diagnostics)>> getFixedProject, - IVerifier verifier, - CancellationToken cancellationToken) - { - var project = await CreateProjectAsync(new EvaluatedProjectState(testState, ReferenceAssemblies), testState.AdditionalProjects.Values.Select(additionalProject => new EvaluatedProjectState(additionalProject, ReferenceAssemblies)).ToImmutableArray(), cancellationToken); - _ = await GetCompilerDiagnosticsAsync(project, verifier, cancellationToken).ConfigureAwait(false); - - ImmutableArray diagnostics; - (project, diagnostics) = await getFixedProject(sourceGenerators, project, verifier, cancellationToken).ConfigureAwait(false); - - // After applying the source generator, compare the resulting string to the inputted one - if (!TestBehaviors.HasFlag(TestBehaviors.SkipGeneratedSourcesCheck)) - { - var numOriginalSources = testState.Sources.Count; - var updatedOriginalDocuments = project.Documents.Take(numOriginalSources).ToArray(); - var generatedDocuments = project.Documents.Skip(numOriginalSources).ToArray(); - - // Verify no changes occurred to the original documents - var updatedOriginalDocumentsWithTextBuilder = ImmutableArray.CreateBuilder<(Document document, SourceText content)>(); - foreach (var updatedOriginalDocument in updatedOriginalDocuments) - { - updatedOriginalDocumentsWithTextBuilder.Add((updatedOriginalDocument, await GetSourceTextFromDocumentAsync(updatedOriginalDocument, CancellationToken.None).ConfigureAwait(false))); - } - - VerifyDocuments( - verifier.PushContext("Original files after running source generators"), - updatedOriginalDocumentsWithTextBuilder.ToImmutable(), - testState.Sources.ToImmutableArray(), - allowReordering: false, - DefaultFilePathPrefix, - GetNameAndFoldersFromPath, - MatchDiagnosticsTimeout); - - // Verify the source generated documents match expectations - var generatedDocumentsWithTextBuilder = ImmutableArray.CreateBuilder<(Document document, SourceText content)>(); - foreach (var generatedDocument in generatedDocuments) - { - generatedDocumentsWithTextBuilder.Add((generatedDocument, await GetSourceTextFromDocumentAsync(generatedDocument, CancellationToken.None).ConfigureAwait(false))); - } - - VerifyDocuments( - verifier.PushContext("Verifying source generated files"), - generatedDocumentsWithTextBuilder.ToImmutable(), - testState.GeneratedSources.ToImmutableArray(), - allowReordering: true, - DefaultFilePathPrefix, - static (_, path) => GetNameAndFoldersFromSourceGeneratedFilePath(path), - MatchDiagnosticsTimeout); - } - - return diagnostics; - - static void VerifyDocuments( - IVerifier verifier, - ImmutableArray<(Document document, SourceText content)> actualDocuments, - ImmutableArray<(string filename, SourceText content)> expectedDocuments, - bool allowReordering, - string defaultFilePathPrefix, - Func folders)> getNameAndFolders, - TimeSpan matchTimeout) - { - ImmutableArray> matches; - if (allowReordering) - { - matches = WeightedMatch.Match( - expectedDocuments, - actualDocuments, - ImmutableArray.Create>( - static (expected, actual, exactOnly) => - { - if (actual.content.ToString() == expected.content.ToString()) - { - return 0.0; - } - - if (exactOnly) - { - // Avoid expensive diff calculation when exact match was requested. - return 1.0; - } - - var diffBuilder = new InlineDiffBuilder(new Differ()); - var diff = diffBuilder.BuildDiffModel(expected.content.ToString(), actual.content.ToString(), ignoreWhitespace: true, ignoreCase: false, new LineChunker()); - var changeCount = diff.Lines.Count(static line => line.Type is ChangeType.Inserted or ChangeType.Deleted); - if (changeCount == 0) - { - // We have a failure caused only by line ending or whitespace differences. Make sure - // to use a non-zero value so it can be distinguished from exact matches. - changeCount = 1; - } - - // Apply a multiplier to the content distance to account for its increased importance - // over encoding and checksum algorithm changes. - var priority = 3; - - return priority * changeCount / (double)diff.Lines.Count; - }, - static (expected, actual, exactOnly) => - { - return actual.content.Encoding == expected.content.Encoding ? 0.0 : 1.0; - }, - static (expected, actual, exactOnly) => - { - return actual.content.ChecksumAlgorithm == expected.content.ChecksumAlgorithm ? 0.0 : 1.0; - }, - (expected, actual, exactOnly) => - { - var distance = 0.0; - var (fileName, folders) = getNameAndFolders(defaultFilePathPrefix, expected.filename); - if (fileName != actual.document.Name) - { - distance += 1.0; - } - - if (!folders.SequenceEqual(actual.document.Folders)) - { - distance += 1.0; - } - - return distance; - }), - matchTimeout); - } - else - { - // Matching with an empty set of matching functions always takes the 1:1 alignment without reordering - matches = WeightedMatch.Match( - expectedDocuments, - actualDocuments, - ImmutableArray>.Empty, - matchTimeout); - } - - // Use EqualOrDiff to verify the actual and expected filenames (and total collection length) in a convenient manner - verifier.EqualOrDiff( - string.Join(Environment.NewLine, matches.Select(match => match.TryGetExpected(out var expected) ? expected.filename : string.Empty)), - string.Join(Environment.NewLine, matches.Select(match => match.TryGetActual(out var actual) ? actual.document.FilePath : string.Empty)), - $"Expected source file list to match"); - - // Follow by verifying each property of interest - foreach (var result in matches) - { - if (!result.TryGetExpected(out var expected) - || !result.TryGetActual(out var actual)) - { - throw new InvalidOperationException("Unexpected state: should have failed during the previous assertion."); - } - - verifier.EqualOrDiff(expected.content.ToString(), actual.content.ToString(), $"content of '{expected.filename}' did not match. Diff shown with expected as baseline:"); - verifier.Equal(expected.content.Encoding, actual.content.Encoding, $"encoding of '{expected.filename}' was expected to be '{expected.content.Encoding?.WebName}' but was '{actual.content.Encoding?.WebName}'"); - verifier.Equal(expected.content.ChecksumAlgorithm, actual.content.ChecksumAlgorithm, $"checksum algorithm of '{expected.filename}' was expected to be '{expected.content.ChecksumAlgorithm}' but was '{actual.content.ChecksumAlgorithm}'"); - - // Source-generated sources are implicitly in a subtree, so they have a different folders calculation. - var (fileName, folders) = getNameAndFolders(defaultFilePathPrefix, expected.filename); - verifier.Equal(fileName, actual.document.Name, $"file name was expected to be '{fileName}' but was '{actual.document.Name}'"); - verifier.SequenceEqual(folders, actual.document.Folders, message: $"folders was expected to be '{string.Join("/", folders)}' but was '{string.Join("/", actual.document.Folders)}'"); - } - } - } - - private static (string fileName, IEnumerable folders) GetNameAndFoldersFromSourceGeneratedFilePath(string filePath) - { - // Source-generated files are always implicitly subpaths under the project root path. - var folders = Path.GetDirectoryName(filePath)!.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - var fileName = Path.GetFileName(filePath); - return (fileName, folders); - } - - private async Task<(Project project, ImmutableArray diagnostics)> ApplySourceGeneratorAsync(ImmutableArray sourceGenerators, Project project, IVerifier verifier, CancellationToken cancellationToken) - { - var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false); - verifier.True(compilation is { }); - - var driver = CreateGeneratorDriver(project, sourceGenerators).RunGenerators(compilation, cancellationToken); - var result = driver.GetRunResult(); - - var updatedProject = project; - foreach (var tree in result.GeneratedTrees) - { - var (fileName, folders) = GetNameAndFoldersFromSourceGeneratedFilePath(tree.FilePath); - updatedProject = updatedProject.AddDocument(fileName, await tree.GetTextAsync(cancellationToken).ConfigureAwait(false), folders: folders, filePath: tree.FilePath).Project; - } - - return (updatedProject, result.Diagnostics); - } - - /// - /// Get the existing compiler diagnostics on the input document. - /// - /// The to run the compiler diagnostic analyzers on. - /// The verifier to use for test assertions. - /// The that the task will observe. - /// The compiler diagnostics that were found in the code. - private static async Task> GetCompilerDiagnosticsAsync(Project project, IVerifier verifier, CancellationToken cancellationToken) - { - var allDiagnostics = ImmutableArray.Create(); - - foreach (var document in project.Documents) - { - var semanticModel = await document.GetSemanticModelAsync(cancellationToken).ConfigureAwait(false); - verifier.True(semanticModel is { }); - - allDiagnostics = allDiagnostics.AddRange(semanticModel.GetDiagnostics(cancellationToken: cancellationToken)); - } - - return allDiagnostics; - } - - /// - /// Given a document, turn it into a string based on the syntax root. - /// - /// The to be converted to a string. - /// The that the task will observe. - /// A containing the syntax of the after formatting. - private static async Task GetSourceTextFromDocumentAsync(Document document, CancellationToken cancellationToken) - { - var simplifiedDoc = await Simplifier.ReduceAsync(document, Simplifier.Annotation, cancellationToken: cancellationToken).ConfigureAwait(false); - var formatted = await Formatter.FormatAsync(simplifiedDoc, Formatter.Annotation, cancellationToken: cancellationToken).ConfigureAwait(false); - return await formatted.GetTextAsync(cancellationToken).ConfigureAwait(false); - } } } diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/PublicAPI.Unshipped.txt b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/PublicAPI.Unshipped.txt index 0b95db80..6960e116 100644 --- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/PublicAPI.Unshipped.txt +++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/PublicAPI.Unshipped.txt @@ -3,7 +3,6 @@ Microsoft.CodeAnalysis.VisualBasic.Testing.VisualBasicSourceGeneratorTest(Of TSo Microsoft.CodeAnalysis.VisualBasic.Testing.VisualBasicSourceGeneratorVerifier(Of TSourceGenerator, TVerifier) Microsoft.CodeAnalysis.VisualBasic.Testing.VisualBasicSourceGeneratorVerifier(Of TSourceGenerator, TVerifier).New() -> Void Overrides Microsoft.CodeAnalysis.VisualBasic.Testing.VisualBasicSourceGeneratorTest(Of TSourceGenerator, TVerifier).CreateCompilationOptions() -> Microsoft.CodeAnalysis.CompilationOptions -Overrides Microsoft.CodeAnalysis.VisualBasic.Testing.VisualBasicSourceGeneratorTest(Of TSourceGenerator, TVerifier).CreateGeneratorDriver(project As Microsoft.CodeAnalysis.Project, sourceGenerators As System.Collections.Immutable.ImmutableArray(Of Microsoft.CodeAnalysis.ISourceGenerator)) -> Microsoft.CodeAnalysis.GeneratorDriver Overrides Microsoft.CodeAnalysis.VisualBasic.Testing.VisualBasicSourceGeneratorTest(Of TSourceGenerator, TVerifier).CreateParseOptions() -> Microsoft.CodeAnalysis.ParseOptions Overrides Microsoft.CodeAnalysis.VisualBasic.Testing.VisualBasicSourceGeneratorTest(Of TSourceGenerator, TVerifier).DefaultFileExt() -> String Overrides Microsoft.CodeAnalysis.VisualBasic.Testing.VisualBasicSourceGeneratorTest(Of TSourceGenerator, TVerifier).GetSourceGenerators() -> System.Collections.Generic.IEnumerable(Of System.Type) diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/VisualBasicSourceGeneratorTest`2.vb b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/VisualBasicSourceGeneratorTest`2.vb index 24b54fc1..db2b21eb 100644 --- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/VisualBasicSourceGeneratorTest`2.vb +++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/VisualBasicSourceGeneratorTest`2.vb @@ -1,6 +1,5 @@ ' Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -Imports System.Collections.Immutable Imports Microsoft.CodeAnalysis.Testing Public Class VisualBasicSourceGeneratorTest(Of TSourceGenerator As {ISourceGenerator, New}, TVerifier As {IVerifier, New}) @@ -21,14 +20,6 @@ Public Class VisualBasicSourceGeneratorTest(Of TSourceGenerator As {ISourceGener End Get End Property - Protected Overrides Function CreateGeneratorDriver(project As Project, sourceGenerators As ImmutableArray(Of ISourceGenerator)) As GeneratorDriver - Return VisualBasicGeneratorDriver.Create( - sourceGenerators, - project.AnalyzerOptions.AdditionalFiles, - CType(project.ParseOptions, VisualBasicParseOptions), - project.AnalyzerOptions.AnalyzerConfigOptionsProvider) - End Function - Protected Overrides Function CreateCompilationOptions() As CompilationOptions Return New VisualBasicCompilationOptions(OutputKind.DynamicallyLinkedLibrary) End Function diff --git a/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing.UnitTests/SourceGeneratorTests.cs b/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing.UnitTests/SourceGeneratorTests.cs index a0909ecc..040ec8bf 100644 --- a/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing.UnitTests/SourceGeneratorTests.cs +++ b/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing.UnitTests/SourceGeneratorTests.cs @@ -2,7 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; +using System.Text; using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Testing.TestAnalyzers; +using Microsoft.CodeAnalysis.Testing.TestGenerators; +using Microsoft.CodeAnalysis.Text; using Xunit; using CSharpTest = Microsoft.CodeAnalysis.Testing.TestAnalyzers.CSharpAnalyzerWithSourceGeneratorTest< Microsoft.CodeAnalysis.Testing.EmptyDiagnosticAnalyzer, @@ -23,7 +28,7 @@ namespace Microsoft.CodeAnalysis.Testing TestState = { Sources = { "class MainClass : TestClass { }" }, - GeneratedSources = { (typeof(GenerateSourceFile), "Generated.g.cs", "content not yet validated") }, + GeneratedSources = { (typeof(GenerateSourceFile), "Generated.g.cs", "class TestClass { }") }, }, }.RunAsync(); } @@ -36,7 +41,312 @@ namespace Microsoft.CodeAnalysis.Testing TestState = { Sources = { "Class MainClass : Inherits TestClass : End Class" }, - GeneratedSources = { (typeof(GenerateSourceFile), "Generated.g.vb", "content not yet validated") }, + GeneratedSources = { (typeof(GenerateSourceFile), "Generated.g.vb", "Class TestClass : End Class") }, + }, + }.RunAsync(); + } + + [Fact] + public async Task AddSimpleFile() + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + @"// Comment", + }, + GeneratedSources = + { + ("Microsoft.CodeAnalysis.Testing.Utilities\\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFile\\EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.UTF8)), + }, + }, + }.RunAsync(); + } + + [Fact] + public async Task MultipleFilesAllowEitherOrder() + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + @"// Comment", + }, + GeneratedSources = + { + (typeof(AddTwoEmptyFiles), "EmptyGeneratedFile1.cs", SourceText.From(string.Empty, Encoding.UTF8)), + (typeof(AddTwoEmptyFiles), "EmptyGeneratedFile2.cs", SourceText.From(string.Empty, Encoding.UTF8)), + }, + }, + }.RunAsync(); + + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + @"// Comment", + }, + GeneratedSources = + { + (typeof(AddTwoEmptyFiles), "EmptyGeneratedFile2.cs", SourceText.From(string.Empty, Encoding.UTF8)), + (typeof(AddTwoEmptyFiles), "EmptyGeneratedFile1.cs", SourceText.From(string.Empty, Encoding.UTF8)), + }, + }, + }.RunAsync(); + } + + [Fact] + public async Task AddSimpleFileByGeneratorType() + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + @"// Comment", + }, + GeneratedSources = + { + (typeof(AddEmptyFile), "EmptyGeneratedFile.cs", string.Empty), + }, + }, + }.RunAsync(); + } + + [Fact] + public async Task AddSimpleFileByGeneratorTypeWithEncoding() + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + @"// Comment", + }, + GeneratedSources = + { + (typeof(AddEmptyFile), "EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.UTF8)), + }, + }, + }.RunAsync(); + } + + [Fact] + public async Task AddSimpleFileToEmptyProject() + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + }, + GeneratedSources = + { + (typeof(AddEmptyFile), "EmptyGeneratedFile.cs", string.Empty), + }, + }, + }.RunAsync(); + } + + [Fact] + public async Task AddSimpleFileWithWrongExpectedEncoding() + { + var exception = await Assert.ThrowsAsync(async () => + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + GeneratedSources = + { + (typeof(AddEmptyFile), "EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.Unicode)), + }, + }, + }.RunAsync(); + }); + + var expectedMessage = @"Context: Source generator application +Context: Verifying source generated files +encoding of 'Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFile\EmptyGeneratedFile.cs' was expected to be 'utf-16' but was 'utf-8'"; + new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message); + } + + [Fact] + public async Task AddSimpleFileVerifiesCompilerDiagnostics_CSharp() + { + var exception = await Assert.ThrowsAsync(async () => + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + @"class A {", + }, + GeneratedSources = + { + (typeof(AddFileWithCompileError), "ErrorGeneratedFile.cs", @"class C {"), + }, + }, + }.RunAsync(); + }); + + var expectedMessage = @"Mismatch between number of diagnostics returned, expected ""0"" actual ""2"" + +Diagnostics: +// /0/Test0.cs(1,10): error CS1513: } expected +DiagnosticResult.CompilerError(""CS1513"").WithSpan(1, 10, 1, 10), +// Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs(1,10): error CS1513: } expected +DiagnosticResult.CompilerError(""CS1513"").WithSpan(""Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs"", 1, 10, 1, 10), + +"; + new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message); + } + + [Fact] + public async Task AddSimpleFileVerifiesCompilerDiagnosticsEvenWhenSourceGeneratorOutputsSkipped_CSharp() + { + var exception = await Assert.ThrowsAsync(async () => + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestBehaviors = TestBehaviors.SkipGeneratedSourcesCheck, + TestState = + { + Sources = + { + @"class A {", + }, + }, + }.RunAsync(); + }); + + var expectedMessage = @"Mismatch between number of diagnostics returned, expected ""0"" actual ""2"" + +Diagnostics: +// /0/Test0.cs(1,10): error CS1513: } expected +DiagnosticResult.CompilerError(""CS1513"").WithSpan(1, 10, 1, 10), +// Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs(1,10): error CS1513: } expected +DiagnosticResult.CompilerError(""CS1513"").WithSpan(""Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs"", 1, 10, 1, 10), + +"; + new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message); + } + + [Fact] + public async Task AddSimpleFileVerifiesCompilerDiagnostics_VisualBasic() + { + var exception = await Assert.ThrowsAsync(async () => + { + await new VisualBasicAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + "Class A", + }, + GeneratedSources = + { + (typeof(AddFileWithCompileError), "ErrorGeneratedFile.vb", "Class C"), + }, + }, + }.RunAsync(); + }); + + var expectedMessage = @"Mismatch between number of diagnostics returned, expected ""0"" actual ""2"" + +Diagnostics: +// /0/Test0.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'. +DiagnosticResult.CompilerError(""BC30481"").WithSpan(1, 1, 1, 8), +// Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'. +DiagnosticResult.CompilerError(""BC30481"").WithSpan(""Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb"", 1, 1, 1, 8), + +"; + new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message); + } + + [Fact] + public async Task AddSimpleFileVerifiesCompilerDiagnosticsEvenWhenSourceGeneratorOutputsSkipped_VisualBasic() + { + var exception = await Assert.ThrowsAsync(async () => + { + await new VisualBasicAnalyzerWithSourceGeneratorTest + { + TestBehaviors = TestBehaviors.SkipGeneratedSourcesCheck, + TestState = + { + Sources = + { + "Class A", + }, + }, + }.RunAsync(); + }); + + var expectedMessage = @"Mismatch between number of diagnostics returned, expected ""0"" actual ""2"" + +Diagnostics: +// /0/Test0.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'. +DiagnosticResult.CompilerError(""BC30481"").WithSpan(1, 1, 1, 8), +// Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'. +DiagnosticResult.CompilerError(""BC30481"").WithSpan(""Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb"", 1, 1, 1, 8), + +"; + new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message); + } + + [Fact] + public async Task AddSimpleFileWithDiagnostic() + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestState = + { + Sources = + { + @"{|#0:|}// Comment", + }, + GeneratedSources = + { + ("Microsoft.CodeAnalysis.Testing.Utilities\\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFileWithDiagnostic\\EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.UTF8)), + }, + ExpectedDiagnostics = + { + // /0/Test0.cs(1,1): warning SG0001: Message + new DiagnosticResult(AddEmptyFileWithDiagnostic.Descriptor).WithLocation(0), + }, + }, + }.RunAsync(); + } + + [Fact] + public async Task AddImplicitSimpleFileWithDiagnostic() + { + await new CSharpAnalyzerWithSourceGeneratorTest + { + TestBehaviors = TestBehaviors.SkipGeneratedSourcesCheck, + TestState = + { + Sources = + { + @"{|#0:|}// Comment", + }, + ExpectedDiagnostics = + { + // /0/Test0.cs(1,1): warning SG0001: Message + new DiagnosticResult(AddEmptyFileWithDiagnostic.Descriptor).WithLocation(0), + }, }, }.RunAsync(); } diff --git a/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/SourceGeneratorValidationTests.cs b/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/SourceGeneratorValidationTests.cs index ab50d1e6..d481b41c 100644 --- a/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/SourceGeneratorValidationTests.cs +++ b/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/SourceGeneratorValidationTests.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; using System.Text; using System.Threading.Tasks; using Microsoft.CodeAnalysis.CSharp; @@ -30,7 +29,7 @@ namespace Microsoft.CodeAnalysis.Testing }, GeneratedSources = { - ("Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFile\\EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.UTF8)), + ("Microsoft.CodeAnalysis.Testing.Utilities\\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFile\\EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.UTF8)), }, }, }.RunAsync(); @@ -147,7 +146,7 @@ namespace Microsoft.CodeAnalysis.Testing var expectedMessage = @"Context: Source generator application Context: Verifying source generated files -encoding of 'Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFile\EmptyGeneratedFile.cs' was expected to be 'utf-16' but was 'utf-8'"; +encoding of 'Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFile\EmptyGeneratedFile.cs' was expected to be 'utf-16' but was 'utf-8'"; new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message); } @@ -172,14 +171,13 @@ encoding of 'Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft }.RunAsync(); }); - var expectedMessage = @"Context: Diagnostics of test state -Mismatch between number of diagnostics returned, expected ""0"" actual ""2"" + var expectedMessage = @"Mismatch between number of diagnostics returned, expected ""0"" actual ""2"" Diagnostics: // /0/Test0.cs(1,10): error CS1513: } expected DiagnosticResult.CompilerError(""CS1513"").WithSpan(1, 10, 1, 10), -// Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs(1,10): error CS1513: } expected -DiagnosticResult.CompilerError(""CS1513"").WithSpan(""Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs"", 1, 10, 1, 10), +// Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs(1,10): error CS1513: } expected +DiagnosticResult.CompilerError(""CS1513"").WithSpan(""Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs"", 1, 10, 1, 10), "; new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message); @@ -206,14 +204,13 @@ DiagnosticResult.CompilerError(""CS1513"").WithSpan(""Microsoft.CodeAnalysis.Sou }.RunAsync(); }); - var expectedMessage = @"Context: Diagnostics of test state -Mismatch between number of diagnostics returned, expected ""0"" actual ""2"" + var expectedMessage = @"Mismatch between number of diagnostics returned, expected ""0"" actual ""2"" Diagnostics: // /0/Test0.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'. DiagnosticResult.CompilerError(""BC30481"").WithSpan(1, 1, 1, 8), -// Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'. -DiagnosticResult.CompilerError(""BC30481"").WithSpan(""Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb"", 1, 1, 1, 8), +// Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'. +DiagnosticResult.CompilerError(""BC30481"").WithSpan(""Microsoft.CodeAnalysis.Testing.Utilities\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb"", 1, 1, 1, 8), "; new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message); @@ -232,7 +229,7 @@ DiagnosticResult.CompilerError(""BC30481"").WithSpan(""Microsoft.CodeAnalysis.So }, GeneratedSources = { - ("Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFileWithDiagnostic\\EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.UTF8)), + ("Microsoft.CodeAnalysis.Testing.Utilities\\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFileWithDiagnostic\\EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.UTF8)), }, ExpectedDiagnostics = { @@ -271,15 +268,6 @@ DiagnosticResult.CompilerError(""BC30481"").WithSpan(""Microsoft.CodeAnalysis.So protected override string DefaultFileExt => "cs"; - protected override GeneratorDriver CreateGeneratorDriver(Project project, ImmutableArray sourceGenerators) - { - return CSharpGeneratorDriver.Create( - sourceGenerators, - project.AnalyzerOptions.AdditionalFiles, - (CSharpParseOptions)project.ParseOptions!, - project.AnalyzerOptions.AnalyzerConfigOptionsProvider); - } - protected override CompilationOptions CreateCompilationOptions() { return new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary); @@ -303,15 +291,6 @@ DiagnosticResult.CompilerError(""BC30481"").WithSpan(""Microsoft.CodeAnalysis.So protected override string DefaultFileExt => "vb"; - protected override GeneratorDriver CreateGeneratorDriver(Project project, ImmutableArray sourceGenerators) - { - return VisualBasicGeneratorDriver.Create( - sourceGenerators, - project.AnalyzerOptions.AdditionalFiles, - (VisualBasicParseOptions)project.ParseOptions!, - project.AnalyzerOptions.AnalyzerConfigOptionsProvider); - } - protected override CompilationOptions CreateCompilationOptions() { return new VisualBasicCompilationOptions(OutputKind.DynamicallyLinkedLibrary); diff --git a/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddEmptyFile.cs b/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Testing.Utilities/TestGenerators/AddEmptyFile.cs similarity index 100% rename from tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddEmptyFile.cs rename to tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Testing.Utilities/TestGenerators/AddEmptyFile.cs diff --git a/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddEmptyFileWithDiagnostic.cs b/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Testing.Utilities/TestGenerators/AddEmptyFileWithDiagnostic.cs similarity index 100% rename from tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddEmptyFileWithDiagnostic.cs rename to tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Testing.Utilities/TestGenerators/AddEmptyFileWithDiagnostic.cs diff --git a/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddFileWithCompileError.cs b/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Testing.Utilities/TestGenerators/AddFileWithCompileError.cs similarity index 100% rename from tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddFileWithCompileError.cs rename to tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Testing.Utilities/TestGenerators/AddFileWithCompileError.cs diff --git a/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddTwoEmptyFiles.cs b/tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Testing.Utilities/TestGenerators/AddTwoEmptyFiles.cs similarity index 100% rename from tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddTwoEmptyFiles.cs rename to tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Testing.Utilities/TestGenerators/AddTwoEmptyFiles.cs