Implement MSML_RelaxTestNaming suppressor for VSTHRD200 (#4803)

* Implement MSML_RelaxTestNaming suppressor for VSTHRD200

Allow asynchronous test methods to omit the 'Async' suffix.

test/Microsoft.ML.CodeAnalyzer.Tests/Code/BaseTestClassTest.cs

@@ -14,7 +14,7 @@ namespace Microsoft.ML.CodeAnalyzer.Tests.Code
 {
     public class BaseTestClassTest
     {
-        private static readonly ReferenceAssemblies _referenceAssemblies = ReferenceAssemblies.Default
+        internal static readonly ReferenceAssemblies ReferenceAssemblies = ReferenceAssemblies.Default
             .AddPackages(ImmutableArray.Create(new PackageIdentity("xunit", "2.4.0")));
 
         [Fact]
@@ -31,7 +31,7 @@ public class [|SomeClass|] {
 
             await new VerifyCS.Test
             {
-                ReferenceAssemblies = _referenceAssemblies,
+                ReferenceAssemblies = ReferenceAssemblies,
                 TestState = { Sources = { code } },
             }.RunAsync();
         }
@@ -50,7 +50,7 @@ public class [|SomeClass|] {
 
             await new VerifyCS.Test
             {
-                ReferenceAssemblies = _referenceAssemblies,
+                ReferenceAssemblies = ReferenceAssemblies,
                 TestState = { Sources = { code } },
             }.RunAsync();
         }
@@ -74,7 +74,7 @@ namespace Microsoft.ML.TestFramework {
 
             await new VerifyCS.Test
             {
-                ReferenceAssemblies = _referenceAssemblies,
+                ReferenceAssemblies = ReferenceAssemblies,
                 TestState = { Sources = { code } },
             }.RunAsync();
         }
@@ -100,7 +100,7 @@ namespace Microsoft.ML.TestFramework {
 
             await new VerifyCS.Test
             {
-                ReferenceAssemblies = _referenceAssemblies,
+                ReferenceAssemblies = ReferenceAssemblies,
                 TestState = { Sources = { code } },
             }.RunAsync();
         }

test/Microsoft.ML.CodeAnalyzer.Tests/Code/RelaxTestNamingTest.cs

@@ -0,0 +1,148 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics.CodeAnalysis;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.Testing;
+using Microsoft.ML.InternalCodeAnalyzer;
+using Xunit;
+using VerifyCS = Microsoft.ML.CodeAnalyzer.Tests.Helpers.CSharpCodeFixVerifier<
+    Microsoft.ML.CodeAnalyzer.Tests.Code.RelaxTestNamingTest.WarnForMissingAsyncSuffix,
+    Microsoft.CodeAnalysis.Testing.EmptyCodeFixProvider>;
+
+namespace Microsoft.ML.CodeAnalyzer.Tests.Code
+{
+    public class RelaxTestNamingTest
+    {
+        private static Solution WithoutSuppressedDiagnosticsTransform(Solution solution, ProjectId projectId)
+        {
+            var compilationOptions = solution.GetProject(projectId).CompilationOptions;
+            return solution.WithProjectCompilationOptions(projectId, compilationOptions.WithReportSuppressedDiagnostics(false));
+        }
+
+        [Fact(Skip = "https://github.com/dotnet/roslyn/issues/41584")]
+        public async Task TestClassWithFact()
+        {
+            var code = @"
+using System.Threading.Tasks;
+using Xunit;
+
+public class SomeClass {
+[Fact]
+public async Task [|TestMethod|]() { }
+}
+";
+
+            await new VerifyCS.Test
+            {
+                ReferenceAssemblies = BaseTestClassTest.ReferenceAssemblies,
+                TestState = { Sources = { code } },
+                SolutionTransforms = { WithoutSuppressedDiagnosticsTransform },
+            }.RunAsync();
+
+            await new TestWithSuppressor
+            {
+                ReferenceAssemblies = BaseTestClassTest.ReferenceAssemblies,
+                TestState = { Sources = { code }, MarkupHandling = MarkupMode.Ignore, },
+                SolutionTransforms = { WithoutSuppressedDiagnosticsTransform },
+            }.RunAsync();
+        }
+
+        [Fact(Skip = "https://github.com/dotnet/roslyn/issues/41584")]
+        public async Task TestClassWithTheory()
+        {
+            var code = @"
+using Xunit;
+
+public class [|SomeClass|] {
+[Theory, InlineData(0)]
+public void TestMethod(int arg) { }
+}
+";
+
+            await new VerifyCS.Test
+            {
+                ReferenceAssemblies = BaseTestClassTest.ReferenceAssemblies,
+                TestState = { Sources = { code } },
+                SolutionTransforms = { WithoutSuppressedDiagnosticsTransform },
+            }.RunAsync();
+
+            await new TestWithSuppressor
+            {
+                ReferenceAssemblies = BaseTestClassTest.ReferenceAssemblies,
+                TestState = { Sources = { code }, MarkupHandling = MarkupMode.Ignore, },
+                SolutionTransforms = { WithoutSuppressedDiagnosticsTransform },
+            }.RunAsync();
+        }
+
+        [Fact]
+        public async Task TestAlreadyHasAsyncSuffix()
+        {
+            var code = @"
+using System.Threading.Tasks;
+using Xunit;
+
+public class SomeClass {
+[Fact]
+public async Task TestMethodAsync() { }
+}
+";
+
+            await new VerifyCS.Test
+            {
+                ReferenceAssemblies = BaseTestClassTest.ReferenceAssemblies,
+                TestState = { Sources = { code } },
+            }.RunAsync();
+        }
+
+        [DiagnosticAnalyzer(LanguageNames.CSharp)]
+        public class WarnForMissingAsyncSuffix : DiagnosticAnalyzer
+        {
+            [SuppressMessage("MicrosoftCodeAnalysisDesign", "RS1017:DiagnosticId for analyzers must be a non-null constant.", Justification = "For suppression test only.")]
+            public static readonly DiagnosticDescriptor Rule = new DiagnosticDescriptor(RelaxTestNamingSuppressor.Rule.SuppressedDiagnosticId, "Title", "Message", "Category", DiagnosticSeverity.Warning, isEnabledByDefault: true);
+
+            public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(Rule);
+
+            public override void Initialize(AnalysisContext context)
+            {
+                context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None);
+                context.EnableConcurrentExecution();
+
+                context.RegisterSymbolAction(AnalyzeSymbol, SymbolKind.Method);
+            }
+
+            private void AnalyzeSymbol(SymbolAnalysisContext context)
+            {
+                var method = (IMethodSymbol)context.Symbol;
+                if (method.Name.EndsWith("Async"))
+                {
+                    return;
+                }
+
+                if (method.ReturnType.MetadataName != "Task")
+                {
+                    // Not asynchronous (incomplete checking is sufficient for this test)
+                    return;
+                }
+
+                context.ReportDiagnostic(Diagnostic.Create(Rule, method.Locations[0]));
+            }
+        }
+
+        internal class TestWithSuppressor : VerifyCS.Test
+        {
+            protected override IEnumerable<DiagnosticAnalyzer> GetDiagnosticAnalyzers()
+            {
+                foreach (var analyzer in base.GetDiagnosticAnalyzers())
+                    yield return analyzer;
+
+                yield return new RelaxTestNamingSuppressor();
+            }
+        }
+    }
+}

tools-local/Microsoft.ML.InternalCodeAnalyzer/RelaxTestNamingSuppressor.cs

@@ -0,0 +1,54 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Concurrent;
+using System.Collections.Immutable;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Diagnostics;
+
+namespace Microsoft.ML.InternalCodeAnalyzer
+{
+    [DiagnosticAnalyzer(LanguageNames.CSharp)]
+    public sealed class RelaxTestNamingSuppressor : DiagnosticSuppressor
+    {
+        private const string Id = "MSML_RelaxTestNaming";
+        private const string SuppressedDiagnosticId = "VSTHRD200";
+        private const string Justification = "Asynchronous test methods do not require the 'Async' suffix.";
+
+        internal static readonly SuppressionDescriptor Rule =
+            new SuppressionDescriptor(Id, SuppressedDiagnosticId, Justification);
+
+        public override ImmutableArray<SuppressionDescriptor> SupportedSuppressions { get; } = ImmutableArray.Create(Rule);
+
+        public override void ReportSuppressions(SuppressionAnalysisContext context)
+        {
+            if (!(context.Compilation.GetTypeByMetadataName("Xunit.FactAttribute") is { } factAttribute))
+            {
+                return;
+            }
+
+            var knownTestAttributes = new ConcurrentDictionary<INamedTypeSymbol, bool>();
+
+            foreach (var diagnostic in context.ReportedDiagnostics)
+            {
+                // The diagnostic is reported on the test method
+                if (!(diagnostic.Location.SourceTree is { } tree))
+                {
+                    continue;
+                }
+
+                var root = tree.GetRoot(context.CancellationToken);
+                var node = root.FindNode(diagnostic.Location.SourceSpan, getInnermostNodeForTie: true);
+
+                var semanticModel = context.GetSemanticModel(tree);
+                var declaredSymbol = semanticModel.GetDeclaredSymbol(node, context.CancellationToken);
+                if (declaredSymbol is IMethodSymbol method
+                    && method.IsTestMethod(knownTestAttributes, factAttribute))
+                {
+                    context.ReportSuppression(Suppression.Create(Rule, diagnostic));
+                }
+            }
+        }
+    }
+}