microsoft
/
testfx
зеркало из https://github.com/microsoft/testfx.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Display fixture (Assembly/Class Initialize/Cleanup) methods as "test" entries (#2904) 

Co-authored-by: Amaury Levé <amauryleve@microsoft.com>
This commit is contained in:
fhnaseer 2024-06-18 11:28:05 +02:00 коммит произвёл GitHub
Родитель a1e4196629
Коммит 86489a221a
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
14 изменённых файлов: 564 добавлений и 32 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
TestFx.sln
Просмотреть файл

@ -201,6 +201,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Playground", "samples\Playg
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTest.Acceptance.IntegrationTests", "test\IntegrationTests\MSTest.Acceptance.IntegrationTests\MSTest.Acceptance.IntegrationTests.csproj", "{BCB42780-C559-40B6-8C4A-85EBC464AAA8}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FixturesTestProject", "test\IntegrationTests\TestAssets\FixturesTestProject\FixturesTestProject.csproj", "{A7D0995D-0516-4975-ABBD-EB93E1B79292}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -427,6 +429,10 @@ Global
{BCB42780-C559-40B6-8C4A-85EBC464AAA8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BCB42780-C559-40B6-8C4A-85EBC464AAA8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BCB42780-C559-40B6-8C4A-85EBC464AAA8}.Release|Any CPU.Build.0 = Release|Any CPU
{A7D0995D-0516-4975-ABBD-EB93E1B79292}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A7D0995D-0516-4975-ABBD-EB93E1B79292}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A7D0995D-0516-4975-ABBD-EB93E1B79292}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A7D0995D-0516-4975-ABBD-EB93E1B79292}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -503,6 +509,7 @@ Global
{7E9D98E7-733C-4D6B-A5AC-087D588A40ED} = {CB0CC552-2017-40C0-934A-C8A3B00EF650}
{8A41B37E-0732-4F28-B214-A44233B447FE} = {92F8E9A2-903E-4025-99BC-7DC478D5466D}
{BCB42780-C559-40B6-8C4A-85EBC464AAA8} = {FF69998C-C661-4EF0-804B-845675B3602E}
{A7D0995D-0516-4975-ABBD-EB93E1B79292} = {C9F82701-0E0F-4E61-B05B-AE387E7631F6}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9}

32
src/Adapter/MSTest.TestAdapter/Constants.cs
Просмотреть файл

@ -1,4 +1,4 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
@ -10,6 +10,36 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
/// </summary>
internal static class Constants
{
/// <summary>
/// The 3rd level entry (class) name in the hierarchy array.
/// </summary>
internal const string AssemblyFixturesHierarchyClassName = "[Assembly]";
/// <summary>
/// Discover fixtures or not.
/// </summary>
internal const string FixturesTestTrait = "FixturesTrait";
/// <summary>
/// Assembly initialize.
/// </summary>
internal const string AssemblyInitializeFixtureTrait = "AssemblyInitialize";
/// <summary>
/// Assembly cleanup.
/// </summary>
internal const string AssemblyCleanupFixtureTrait = "AssemblyCleanup";
/// <summary>
/// Class initialize.
/// </summary>
internal const string ClassInitializeFixtureTrait = "ClassInitialize";
/// <summary>
/// Class cleanup.
/// </summary>
internal const string ClassCleanupFixtureTrait = "ClassCleanup";
/// <summary>
/// Uri of the MSTest executor.
/// </summary>

127
src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs
Просмотреть файл

@ -11,6 +11,7 @@ using System.Text;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using FrameworkITestDataSource = Microsoft.VisualStudio.TestTools.UnitTesting.ITestDataSource;
@ -44,12 +45,10 @@ internal class AssemblyEnumerator : MarshalByRefObject
/// </summary>
/// <param name="settings">The settings for the session.</param>
/// <remarks>Use this constructor when creating this object in a new app domain so the settings for this app domain are set.</remarks>
public AssemblyEnumerator(MSTestSettings settings)
{
public AssemblyEnumerator(MSTestSettings settings) =>
// Populate the settings into the domain(Desktop workflow) performing discovery.
// This would just be resetting the settings to itself in non desktop workflows.
MSTestSettings.PopulateSettings(settings);
}
/// <summary>
/// Gets or sets the run settings to use for current discovery session.
@ -77,9 +76,10 @@ internal class AssemblyEnumerator : MarshalByRefObject
internal ICollection<UnitTestElement> EnumerateAssembly(string assemblyFileName, out ICollection<string> warnings)
{
DebugEx.Assert(!StringEx.IsNullOrWhiteSpace(assemblyFileName), "Invalid assembly file name.");
var warningMessages = new List<string>();
var tests = new List<UnitTestElement>();
// Contains list of assembly/class names for which we have already added fixture tests.
var fixturesTests = new HashSet<string>();
Assembly assembly = PlatformServiceProvider.Instance.FileOperations.LoadAssembly(assemblyFileName, isReflectionOnly: false);
@ -109,7 +109,7 @@ internal class AssemblyEnumerator : MarshalByRefObject
}
List<UnitTestElement> testsInType = DiscoverTestsInType(assemblyFileName, RunSettingsXml, type, warningMessages, discoverInternals,
testDataSourceDiscovery, testIdGenerationStrategy);
testDataSourceDiscovery, testIdGenerationStrategy, fixturesTests);
tests.AddRange(testsInType);
}
@ -211,7 +211,8 @@ internal class AssemblyEnumerator : MarshalByRefObject
List<string> warningMessages,
bool discoverInternals,
TestDataSourceDiscoveryOption discoveryOption,
TestIdGenerationStrategy testIdGenerationStrategy)
TestIdGenerationStrategy testIdGenerationStrategy,
HashSet<string> fixturesTests)
{
IDictionary<string, object> tempSourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(assemblyFileName);
tempSourceLevelParameters = RunSettingsUtilities.GetTestRunParameters(runSettingsXml)?.ConcatWithOverwrites(tempSourceLevelParameters)
@ -235,7 +236,15 @@ internal class AssemblyEnumerator : MarshalByRefObject
{
if (discoveryOption == TestDataSourceDiscoveryOption.DuringDiscovery)
{
if (DynamicDataAttached(sourceLevelParameters, test, tests))
Lazy<TestMethodInfo?> testMethodInfo = GetTestMethodInfo(sourceLevelParameters, test);
// Add fixture tests like AssemblyInitialize, AssemblyCleanup, ClassInitialize, ClassCleanup.
if (MSTestSettings.CurrentSettings.ConsiderFixturesAsSpecialTests && testMethodInfo.Value is not null)
{
AddFixtureTests(testMethodInfo.Value, tests, fixturesTests);
}
if (DynamicDataAttached(test, testMethodInfo, tests))
{
continue;
}
@ -257,7 +266,18 @@ internal class AssemblyEnumerator : MarshalByRefObject
return tests;
}
private bool DynamicDataAttached(IDictionary<string, object?> sourceLevelParameters, UnitTestElement test, List<UnitTestElement> tests)
private Lazy<TestMethodInfo?> GetTestMethodInfo(IDictionary<string, object?> sourceLevelParameters, UnitTestElement test) =>
new(() =>
{
// NOTE: From this place we don't have any path that would let the user write a message on the TestContext and we don't do
// anything with what would be printed anyway so we can simply use a simple StringWriter.
using var writer = new StringWriter();
TestMethod testMethod = test.TestMethod;
MSTestAdapter.PlatformServices.Interface.ITestContext testContext = PlatformServiceProvider.Instance.GetTestContext(testMethod, writer, sourceLevelParameters);
return _typeCache.GetTestMethodInfo(testMethod, testContext, MSTestSettings.CurrentSettings.CaptureDebugTraces);
});
private static bool DynamicDataAttached(UnitTestElement test, Lazy<TestMethodInfo?> testMethodInfo, List<UnitTestElement> tests)
{
// It should always be `true`, but if any part of the chain is obsolete; it might not contain those.
// Since we depend on those properties, if they don't exist, we bail out early.
@ -278,13 +298,90 @@ internal class AssemblyEnumerator : MarshalByRefObject
// If you remove this line and acceptance tests still pass you are okay.
test.TestMethod.DataType = DynamicDataType.None;
// NOTE: From this place we don't have any path that would let the user write a message on the TestContext and we don't do
// anything with what would be printed anyway so we can simply use a simple StringWriter.
using var writer = new StringWriter();
TestMethod testMethod = test.TestMethod;
MSTestAdapter.PlatformServices.Interface.ITestContext testContext = PlatformServiceProvider.Instance.GetTestContext(testMethod, writer, sourceLevelParameters);
TestMethodInfo? testMethodInfo = _typeCache.GetTestMethodInfo(testMethod, testContext, MSTestSettings.CurrentSettings.CaptureDebugTraces);
return testMethodInfo != null && TryProcessTestDataSourceTests(test, testMethodInfo, tests);
return testMethodInfo.Value != null && TryProcessTestDataSourceTests(test, testMethodInfo.Value, tests);
}
private static void AddFixtureTests(TestMethodInfo testMethodInfo, List<UnitTestElement> tests, HashSet<string> fixtureTests)
{
string assemblyName = testMethodInfo.Parent.Parent.Assembly.GetName().Name!;
string assemblyLocation = testMethodInfo.Parent.Parent.Assembly.Location;
string className = testMethodInfo.Parent.ClassType.Name;
string classFullName = testMethodInfo.Parent.ClassType.FullName!;
// Check if fixtures for this assembly has already been added.
if (!fixtureTests.Contains(assemblyLocation))
{
_ = fixtureTests.Add(assemblyLocation);
// Add AssemblyInitialize and AssemblyCleanup fixture tests if they exist.
if (testMethodInfo.Parent.Parent.AssemblyInitializeMethod is not null)
{
tests.Add(GetAssemblyFixtureTest(testMethodInfo.Parent.Parent.AssemblyInitializeMethod, assemblyName, className,
classFullName, assemblyLocation, Constants.AssemblyInitializeFixtureTrait));
}
if (testMethodInfo.Parent.Parent.AssemblyCleanupMethod is not null)
{
tests.Add(GetAssemblyFixtureTest(testMethodInfo.Parent.Parent.AssemblyCleanupMethod, assemblyName, className,
classFullName, assemblyLocation, Constants.AssemblyCleanupFixtureTrait));
}
}
// Check if fixtures for this class has already been added.
if (!fixtureTests.Contains(assemblyLocation + classFullName))
{
_ = fixtureTests.Add(assemblyLocation + classFullName);
// Add ClassInitialize and ClassCleanup fixture tests if they exist.
if (testMethodInfo.Parent.ClassInitializeMethod is not null)
{
tests.Add(GetClassFixtureTest(testMethodInfo.Parent.ClassInitializeMethod, assemblyName, className, classFullName,
assemblyLocation, Constants.ClassInitializeFixtureTrait));
}
if (testMethodInfo.Parent.ClassCleanupMethod is not null)
{
tests.Add(GetClassFixtureTest(testMethodInfo.Parent.ClassCleanupMethod, assemblyName, className, classFullName,
assemblyLocation, Constants.ClassCleanupFixtureTrait));
}
}
static UnitTestElement GetAssemblyFixtureTest(MethodInfo methodInfo, string assemblyName, string className, string classFullName,
string assemblyLocation, string fixtureType)
{
string methodName = GetMethodName(methodInfo);
string[] hierarchy = [null!, assemblyName, Constants.AssemblyFixturesHierarchyClassName, methodName];
return GetFixtureTest(classFullName, assemblyLocation, fixtureType, methodName, hierarchy);
}
static UnitTestElement GetClassFixtureTest(MethodInfo methodInfo, string assemblyName, string className, string classFullName,
string assemblyLocation, string fixtureType)
{
string methodName = GetMethodName(methodInfo);
string[] hierarchy = [null!, classFullName, methodName];
return GetFixtureTest(classFullName, assemblyLocation, fixtureType, methodName, hierarchy);
}
static string GetMethodName(MethodInfo methodInfo)
{
ParameterInfo[] args = methodInfo.GetParameters();
return args.Length > 0
? $"{methodInfo.Name}({string.Join(",", args.Select(a => a.ParameterType.FullName))})"
: methodInfo.Name;
}
static UnitTestElement GetFixtureTest(string classFullName, string assemblyLocation, string fixtureType, string methodName, string[] hierarchy)
{
var method = new TestMethod(classFullName, methodName,
hierarchy, methodName, classFullName, assemblyLocation, false,
TestIdGenerationStrategy.FullyQualified);
return new UnitTestElement(method)
{
DisplayName = $"[{fixtureType}] {methodName}",
Ignored = true,
Traits = [new Trait(Constants.FixturesTestTrait, fixtureType)],
};
}
}
private static bool TryProcessTestDataSourceTests(UnitTestElement test, TestMethodInfo testMethodInfo, List<UnitTestElement> tests)

29
src/Adapter/MSTest.TestAdapter/Discovery/UnitTestDiscoverer.cs
Просмотреть файл

@ -1,10 +1,11 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Globalization;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Discovery;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
@ -90,6 +91,8 @@ internal class UnitTestDiscoverer
internal void SendTestCases(string source, IEnumerable<UnitTestElement> testElements, ITestCaseDiscoverySink discoverySink, IDiscoveryContext? discoveryContext, IMessageLogger logger)
{
bool shouldCollectSourceInformation = MSTestSettings.RunConfigurationSettings.CollectSourceInformation;
bool hasAnyRunnableTests = false;
var fixtureTests = new List<TestCase>();
var navigationSessions = new Dictionary<string, object?>();
try
@ -109,13 +112,25 @@ internal class UnitTestDiscoverer
foreach (UnitTestElement testElement in testElements)
{
var testCase = testElement.ToTestCase();
bool hasFixtureTraits = testCase.Traits.Any(t => t.Name == Constants.FixturesTestTrait);
// Filter tests based on test case filters
if (filterExpression != null && !filterExpression.MatchTestCase(testCase, (p) => TestMethodFilter.PropertyValueProvider(testCase, p)))
{
// If test is a fixture test, add it to the list of fixture tests.
if (hasFixtureTraits)
{
fixtureTests.Add(testCase);
}
continue;
}
if (!hasAnyRunnableTests)
{
hasAnyRunnableTests = !hasFixtureTraits;
}
if (!shouldCollectSourceInformation)
{
discoverySink.SendTestCase(testCase);
@ -165,6 +180,18 @@ internal class UnitTestDiscoverer
discoverySink.SendTestCase(testCase);
}
// If there are runnable tests, then add all fixture tests to the discovery sink.
// Scenarios:
// 1. Execute only a fixture test => In this case, we do not need to track any other fixture tests. Selected fixture test will be tracked as will be marked as skipped.
// 2. Execute a runnable test => In this case, case add all fixture tests. We will update status of only those fixtures which are triggered by the selected test.
if (hasAnyRunnableTests)
{
foreach (TestCase testCase in fixtureTests)
{
discoverySink.SendTestCase(testCase);
}
}
}
finally
{

27
src/Adapter/MSTest.TestAdapter/Execution/TestAssemblyInfo.cs
Просмотреть файл

@ -95,6 +95,11 @@ public class TestAssemblyInfo
/// </summary>
public Exception? AssemblyInitializationException { get; internal set; }
/// <summary>
/// Gets the assembly cleanup exception.
/// </summary>
internal Exception? AssemblyCleanupException { get; private set; }
/// <summary>
/// Gets a value indicating whether this assembly has an executable <c>AssemblyCleanup</c> method.
/// </summary>
@ -210,12 +215,11 @@ public class TestAssemblyInfo
return null;
}
Exception? assemblyCleanupException;
lock (_assemblyInfoExecuteSyncObject)
{
try
{
assemblyCleanupException = FixtureMethodRunner.RunWithTimeoutAndCancellation(
AssemblyCleanupException = FixtureMethodRunner.RunWithTimeoutAndCancellation(
() => AssemblyCleanupMethod.InvokeAsSynchronousTask(null),
new CancellationTokenSource(),
AssemblyCleanupMethodTimeoutMilliseconds,
@ -226,17 +230,17 @@ public class TestAssemblyInfo
}
catch (Exception ex)
{
assemblyCleanupException = ex;
AssemblyCleanupException = ex;
}
}
// If assemblyCleanup was successful, then don't do anything
if (assemblyCleanupException is null)
if (AssemblyCleanupException is null)
{
return null;
}
Exception realException = assemblyCleanupException.GetRealException();
Exception realException = AssemblyCleanupException.GetRealException();
// special case AssertFailedException to trim off part of the stack trace
string errorMessage = realException is AssertFailedException or AssertInconclusiveException
@ -267,12 +271,11 @@ public class TestAssemblyInfo
return;
}
Exception? assemblyCleanupException;
lock (_assemblyInfoExecuteSyncObject)
{
try
{
assemblyCleanupException = FixtureMethodRunner.RunWithTimeoutAndCancellation(
AssemblyCleanupException = FixtureMethodRunner.RunWithTimeoutAndCancellation(
() => AssemblyCleanupMethod.InvokeAsSynchronousTask(null),
new CancellationTokenSource(),
AssemblyCleanupMethodTimeoutMilliseconds,
@ -283,23 +286,23 @@ public class TestAssemblyInfo
}
catch (Exception ex)
{
assemblyCleanupException = ex;
AssemblyCleanupException = ex;
}
}
// If assemblyCleanup was successful, then don't do anything
if (assemblyCleanupException is null)
if (AssemblyCleanupException is null)
{
return;
}
// If the exception is already a `TestFailedException` we throw it as-is
if (assemblyCleanupException is TestFailedException)
if (AssemblyCleanupException is TestFailedException)
{
throw assemblyCleanupException;
throw AssemblyCleanupException;
}
Exception realException = assemblyCleanupException.GetRealException();
Exception realException = AssemblyCleanupException.GetRealException();
// special case AssertFailedException to trim off part of the stack trace
string errorMessage = realException is AssertFailedException or AssertInconclusiveException

42
src/Adapter/MSTest.TestAdapter/Execution/TestExecutionManager.cs
Просмотреть файл

@ -196,6 +196,12 @@ public class TestExecutionManager
if (filterExpression != null
&& !filterExpression.MatchTestCase(test, p => testMethodFilter.PropertyValueProvider(test, p)))
{
// If this is a fixture test, return true. Fixture tests are not filtered out and are always available for the status.
if (test.Traits.Any(t => t.Name == Constants.FixturesTestTrait))
{
return true;
}
// Skip test if not fitting filter criteria.
return false;
}
@ -375,6 +381,9 @@ public class TestExecutionManager
IDictionary<string, object> sourceLevelParameters,
UnitTestRunner testRunner)
{
bool hasAnyRunnableTests = false;
var fixtureTests = new List<TestCase>();
foreach (TestCase currentTest in tests)
{
if (_cancellationToken is { Canceled: true })
@ -382,6 +391,15 @@ public class TestExecutionManager
break;
}
// If it is a fixture test, add it to the list of fixture tests and do not execute it.
// It is executed by test itself.
if (currentTest.Traits.Any(t => t.Name == Constants.FixturesTestTrait))
{
fixtureTests.Add(currentTest);
continue;
}
hasAnyRunnableTests = true;
var unitTestElement = currentTest.ToUnitTestElement(source);
testExecutionRecorder.RecordStart(currentTest);
@ -401,6 +419,30 @@ public class TestExecutionManager
SendTestResults(currentTest, unitTestResult, startTime, endTime, testExecutionRecorder);
}
// Once all tests have been executed, update the status of fixture tests.
foreach (TestCase currentTest in fixtureTests)
{
testExecutionRecorder.RecordStart(currentTest);
// If there were only fixture tests, send an inconclusive result.
if (!hasAnyRunnableTests)
{
var result = new UnitTestResult(ObjectModel.UnitTestOutcome.Inconclusive, null);
SendTestResults(currentTest, [result], DateTimeOffset.Now, DateTimeOffset.Now, testExecutionRecorder);
continue;
}
Trait trait = currentTest.Traits.First(t => t.Name == Constants.FixturesTestTrait);
var unitTestElement = currentTest.ToUnitTestElement(source);
FixtureTestResult fixtureTestResult = testRunner.GetFixtureTestResult(unitTestElement.TestMethod, trait.Value);
if (fixtureTestResult.IsExecuted)
{
var result = new UnitTestResult(fixtureTestResult.Outcome, null);
SendTestResults(currentTest, [result], DateTimeOffset.Now, DateTimeOffset.Now, testExecutionRecorder);
}
}
}
/// <summary>

48
src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs
Просмотреть файл

@ -23,6 +23,8 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution;
/// </summary>
internal class UnitTestRunner : MarshalByRefObject
{
private readonly Dictionary<string, TestMethodInfo> _fixtureTests = new();
/// <summary>
/// Type cache.
/// </summary>
@ -97,6 +99,44 @@ internal class UnitTestRunner : MarshalByRefObject
_reflectHelper);
}
internal FixtureTestResult GetFixtureTestResult(TestMethod testMethod, string fixtureType)
{
// For the fixture methods, we need to return the appropriate result.
// Get matching testMethodInfo from the cache and return UnitTestOutcome for the fixture test.
if (_fixtureTests.TryGetValue(testMethod.AssemblyName + testMethod.FullClassName, out TestMethodInfo? testMethodInfo))
{
if (fixtureType == Constants.ClassInitializeFixtureTrait)
{
return testMethodInfo.Parent.IsClassInitializeExecuted
? new(true, GetOutcome(testMethodInfo.Parent.ClassInitializationException), testMethodInfo.Parent.ClassInitializationException?.Message)
: new(true, ObjectModel.UnitTestOutcome.Inconclusive, null);
}
if (fixtureType == Constants.ClassCleanupFixtureTrait)
{
return testMethodInfo.Parent.IsClassInitializeExecuted
? new(testMethodInfo.Parent.IsClassInitializeExecuted, GetOutcome(testMethodInfo.Parent.ClassCleanupException), testMethodInfo.Parent.ClassCleanupException?.Message)
: new(true, ObjectModel.UnitTestOutcome.Inconclusive, null);
}
}
if (_fixtureTests.TryGetValue(testMethod.AssemblyName, out testMethodInfo))
{
if (fixtureType == Constants.AssemblyInitializeFixtureTrait)
{
return new(true, GetOutcome(testMethodInfo.Parent.Parent.AssemblyInitializationException), testMethodInfo.Parent.Parent.AssemblyInitializationException?.Message);
}
else if (fixtureType == Constants.AssemblyCleanupFixtureTrait)
{
return new(true, GetOutcome(testMethodInfo.Parent.Parent.AssemblyCleanupException), testMethodInfo.Parent.Parent.AssemblyInitializationException?.Message);
}
}
return new(false, ObjectModel.UnitTestOutcome.Inconclusive, null);
static ObjectModel.UnitTestOutcome GetOutcome(Exception? exception) => exception == null ? ObjectModel.UnitTestOutcome.Passed : ObjectModel.UnitTestOutcome.Failed;
}
/// <summary>
/// Runs a single test.
/// </summary>
@ -141,6 +181,11 @@ internal class UnitTestRunner : MarshalByRefObject
}
DebugEx.Assert(testMethodInfo is not null, "testMethodInfo should not be null.");
// Keep track of all non-runnable methods so that we can return the appropriate result at the end.
_fixtureTests[testMethod.AssemblyName] = testMethodInfo;
_fixtureTests[testMethod.AssemblyName + testMethod.FullClassName] = testMethodInfo;
var testMethodRunner = new TestMethodRunner(testMethodInfo, testMethod, testContext, MSTestSettings.CurrentSettings.CaptureDebugTraces);
UnitTestResult[] result = testMethodRunner.Execute();
RunRequiredCleanups(testContext, testMethodInfo, testMethod, result);
@ -293,8 +338,9 @@ internal class UnitTestRunner : MarshalByRefObject
ClassCleanupBehavior lifecycleFromAssembly,
ReflectHelper reflectHelper)
{
IEnumerable<UnitTestElement> runnableTests = testsToRun.Where(t => t.Traits is null || !t.Traits.Any(t => t.Name == Constants.FixturesTestTrait));
_remainingTestsByClass =
new(testsToRun.GroupBy(t => t.TestMethod.FullClassName)
new(runnableTests.GroupBy(t => t.TestMethod.FullClassName)
.ToDictionary(
g => g.Key,
g => new HashSet<string>(g.Select(t => t.TestMethod.UniqueName))));

5
src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj
Просмотреть файл

@ -1,4 +1,4 @@
<Project>
<Project>
<!-- SDK top import -->
<Import Project="Sdk.props" Sdk="MSBuild.Sdk.Extras" Condition=" '$(OS)' == 'Windows_NT' " />
@ -97,6 +97,9 @@
<Compile Include="$(Compile);$(IntermediateOutputPath)/MSTestVersion.cs" />
</ItemGroup>
</Target>
<ItemGroup>
<Compile Remove="Discovery\FixturesVisibility.cs" />
</ItemGroup>
<!-- Version templating -->
<ItemGroup>

20
src/Adapter/MSTest.TestAdapter/MSTestSettings.cs
Просмотреть файл

@ -1,4 +1,4 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Diagnostics.CodeAnalysis;
@ -191,6 +191,13 @@ public class MSTestSettings
/// </summary>
public bool TreatClassAndAssemblyCleanupWarningsAsErrors { get; private set; }
/// <summary>
/// Gets a value indicating whether AssemblyInitialize, AssemblyCleanup, ClassInitialize and ClassCleanup methods are
/// reported as special tests (cannot be executed). When this feature is enabled, these methods will be reported as
/// separate entries in the TRX reports, in Test Explorer or in CLI.
/// </summary>
internal bool ConsiderFixturesAsSpecialTests { get; private set; }
/// <summary>
/// Populate settings based on existing settings object.
/// </summary>
@ -222,6 +229,7 @@ public class MSTestSettings
CurrentSettings.ClassCleanupTimeout = settings.ClassCleanupTimeout;
CurrentSettings.TestInitializeTimeout = settings.TestInitializeTimeout;
CurrentSettings.TestCleanupTimeout = settings.TestCleanupTimeout;
CurrentSettings.ConsiderFixturesAsSpecialTests = settings.ConsiderFixturesAsSpecialTests;
}
/// <summary>
@ -553,6 +561,16 @@ public class MSTestSettings
break;
}
case "CONSIDERFIXTURESASSPECIALTESTS":
{
if (bool.TryParse(reader.ReadInnerXml(), out result))
{
settings.ConsiderFixturesAsSpecialTests = result;
}
break;
}
default:
{
PlatformServiceProvider.Instance.SettingsProvider.Load(reader.ReadSubtree());

30
src/Adapter/MSTest.TestAdapter/ObjectModel/FixtureTestResult.cs Normal file
Просмотреть файл

@ -0,0 +1,30 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
[Serializable]
internal sealed class FixtureTestResult
{
internal FixtureTestResult(bool isExecuted, UnitTestOutcome outcome, string? exceptionMessage)
{
IsExecuted = isExecuted;
Outcome = outcome;
ExceptionMessage = exceptionMessage;
}
/// <summary>
/// Gets a value indicating whether the test is executed or not.
/// </summary>
public bool IsExecuted { get; }
/// <summary>
/// Gets the outcome of the test.
/// </summary>
public UnitTestOutcome Outcome { get; }
/// <summary>
/// Gets the exception message if any.
/// </summary>
public string? ExceptionMessage { get; }
}

1
src/Adapter/MSTestAdapter.PlatformServices/Services/MSTestAdapterSettings.cs
Просмотреть файл

@ -58,6 +58,7 @@ public class MSTestAdapterSettings
// <MSTestV2>
// <DeploymentEnabled>true</DeploymentEnabled>
// <DeployTestSourceDependencies>true</DeployTestSourceDependencies>
// <ConsiderFixturesAsSpecialTests>true</ConsiderFixturesAsSpecialTests>
// <DeleteDeploymentDirectoryAfterTestRunIsComplete>true</DeleteDeploymentDirectoryAfterTestRunIsComplete>
// <AssemblyResolution>
// <Directory path= "% HOMEDRIVE %\directory "includeSubDirectories = "true" />

158
test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/FixturesTests.cs Normal file
Просмотреть файл

@ -0,0 +1,158 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using Microsoft.MSTestV2.CLIAutomation;
namespace MSTest.VstestConsoleWrapper.IntegrationTests;
public class FixturesTests : CLITestBase
{
private const string AssetName = "FixturesTestProject";
private static readonly string AssemblyInitialize = "FixturesTestProject1.UnitTest1.AssemblyInitialize(Microsoft.VisualStudio.TestTools.UnitTesting.TestContext)";
private static readonly string AssemblyCleanup = "FixturesTestProject1.UnitTest1.AssemblyCleanup";
private static readonly string ClassInitialize = "FixturesTestProject1.UnitTest1.ClassInitialize(Microsoft.VisualStudio.TestTools.UnitTesting.TestContext)";
private static readonly string ClassCleanup = "FixturesTestProject1.UnitTest1.ClassCleanup";
private static readonly string TestMethod = "FixturesTestProject1.UnitTest1.Test";
private static readonly string PassingTest = "FixturesTestProject1.UnitTest1.PassingTest";
private readonly string[] _tests = new[]
{
AssemblyInitialize,
AssemblyCleanup,
ClassInitialize,
ClassCleanup,
TestMethod,
PassingTest,
};
public void FixturesDisabled_DoesNotReport_FixtureTests()
{
string runSettings = GetRunSettings(false, true, true, true, true, true);
// Discover tests,
InvokeVsTestForDiscovery([AssetName], runSettings);
ValidateDiscoveredTests([TestMethod, PassingTest]);
// Tests,
InvokeVsTestForExecution([AssetName], runSettings);
ValidatePassedTests([TestMethod, PassingTest]);
}
public void FixturesEnabled_DoesReport_FixtureTests()
{
string runSettings = GetRunSettings(true, true, true, true, true, true);
// Discover tests,
InvokeVsTestForDiscovery([AssetName], runSettings);
ValidateDiscoveredTests(_tests);
// Run tests,
InvokeVsTestForExecution([AssetName], runSettings);
ValidatePassedTests(_tests);
}
public void AssemblyInitialize_Fails_TestMethod_Class_Skipped()
{
string runSettings = GetRunSettings(true, false, true, true, true, true);
InvokeVsTestForExecution([AssetName], runSettings);
ValidatePassedTests([AssemblyCleanup]);
ValidateFailedTests(false, [AssemblyInitialize, TestMethod, PassingTest]);
ValidateSkippedTests([ClassInitialize, ClassCleanup]);
}
public void AssemblyCleanup_OnlyFails_AssemblyCleanup()
{
string runSettings = GetRunSettings(true, true, false, true, true, true);
InvokeVsTestForExecution([AssetName], runSettings);
ValidatePassedTests([AssemblyInitialize, ClassInitialize, ClassCleanup, PassingTest]);
// TestMethod fails because AssemblyCleanup is executed after it and hence it fails.
ValidateFailedTests(false, [AssemblyCleanup, TestMethod]);
}
public void ClassInitialize_OnlyFails_ClassInitialize()
{
string runSettings = GetRunSettings(true, true, true, false, true, true);
InvokeVsTestForExecution([AssetName], runSettings);
ValidateFailedTests(false, [ClassInitialize, TestMethod, PassingTest]);
ValidatePassedTests([AssemblyInitialize, AssemblyCleanup, ClassCleanup]);
}
public void ClassCleanup_OnlyFails_ClassCleanup()
{
string runSettings = GetRunSettings(true, true, true, true, false, true);
InvokeVsTestForExecution([AssetName], runSettings);
ValidatePassedTests([AssemblyInitialize, AssemblyCleanup, ClassInitialize, PassingTest]);
// TestMethod fails because ClassCleanup is executed after it and hence it fails.
ValidateFailedTests(false, [ClassCleanup, TestMethod]);
}
public void RunOnlyFixtures_DoesNot_Run_Fixtures()
{
string runSettings = GetRunSettings(true, true, true, true, true, true);
InvokeVsTestForExecution([AssetName], runSettings, testCaseFilter: nameof(AssemblyInitialize));
ValidateSkippedTests(AssemblyInitialize);
InvokeVsTestForExecution([AssetName], runSettings, testCaseFilter: nameof(AssemblyCleanup));
ValidateSkippedTests(AssemblyCleanup);
InvokeVsTestForExecution([AssetName], runSettings, testCaseFilter: nameof(ClassInitialize));
ValidateSkippedTests(ClassInitialize);
InvokeVsTestForExecution([AssetName], runSettings, testCaseFilter: nameof(ClassCleanup));
ValidateSkippedTests(ClassCleanup);
InvokeVsTestForExecution([AssetName], runSettings, testCaseFilter: "ClassCleanup|AssemblyCleanup");
ValidateSkippedTests([AssemblyCleanup, ClassCleanup]);
}
public void RunSingleTest_Runs_Assembly_And_Class_Fixtures()
{
string runSettings = GetRunSettings(true, true, true, true, true, true);
InvokeVsTestForExecution([AssetName], runSettings, testCaseFilter: nameof(PassingTest));
ValidatePassedTests([AssemblyInitialize, AssemblyCleanup, ClassInitialize, ClassCleanup, PassingTest]);
}
public void RunSingleTest_AssemblyInitialize_Failure_Skips_ClassFixtures()
{
string runSettings = GetRunSettings(true, false, true, true, true, true);
InvokeVsTestForExecution([AssetName], runSettings, testCaseFilter: nameof(PassingTest));
ValidateFailedTests(false, [AssemblyInitialize, PassingTest]);
ValidatePassedTests([AssemblyCleanup]);
// Class fixtures are not executed if AssemblyInitialize fails.
ValidateSkippedTests([ClassInitialize, ClassCleanup]);
}
public void RunSingleTest_ClassInitialize_Failure_Runs_AssemblyFixtures()
{
string runSettings = GetRunSettings(true, true, true, false, true, true);
InvokeVsTestForExecution([AssetName], runSettings, testCaseFilter: nameof(PassingTest));
ValidateFailedTests(false, [ClassInitialize, PassingTest]);
ValidatePassedTests([AssemblyInitialize, AssemblyCleanup, ClassCleanup]);
}
private string GetRunSettings(bool fixturesEnabled, bool assemblyInitialize, bool assemblyCleanup, bool classInitialize, bool classCleanup, bool test)
=> $@"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<AssemblyInitialize>{assemblyInitialize}</AssemblyInitialize>
<AssemblyCleanup>{assemblyCleanup}</AssemblyCleanup>
<ClassInitialize>{classInitialize}</ClassInitialize>
<ClassCleanup>{classCleanup}</ClassCleanup>
<Test>{test}</Test>
</EnvironmentVariables>
</RunConfiguration>
<MSTest>
<ConsiderFixturesAsSpecialTests>{fixturesEnabled}</ConsiderFixturesAsSpecialTests>
</MSTest>
</RunSettings>
";
}

12
test/IntegrationTests/TestAssets/FixturesTestProject/FixturesTestProject.csproj Normal file
Просмотреть файл

@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net462</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="$(RepoRoot)src\Adapter\MSTest.TestAdapter\MSTest.TestAdapter.csproj" />
</ItemGroup>
</Project>

58
test/IntegrationTests/TestAssets/FixturesTestProject/UnitTest1.cs Normal file
Просмотреть файл

@ -0,0 +1,58 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using Microsoft.VisualStudio.TestTools.UnitTesting;
namespace FixturesTestProject1;
[TestClass]
public class UnitTest1
{
[AssemblyInitialize]
public static void AssemblyInitialize(TestContext context)
{
bool? condition = GetCondition("AssemblyInitialize");
Assert.IsNotNull(condition);
Assert.IsTrue(condition.Value);
}
[AssemblyCleanup]
public static void AssemblyCleanup()
{
bool? condition = GetCondition("AssemblyCleanup");
Assert.IsNotNull(condition);
Assert.IsTrue(condition.Value);
}
[ClassInitialize]
public static void ClassInitialize(TestContext testContext)
{
bool? condition = GetCondition("ClassInitialize");
Assert.IsNotNull(condition);
Assert.IsTrue(condition.Value);
}
[ClassCleanup]
public static void ClassCleanup()
{
bool? condition = GetCondition("ClassCleanup");
Assert.IsNotNull(condition);
Assert.IsTrue(condition.Value);
}
[TestMethod]
public void PassingTest() => Assert.IsTrue(true);
[TestMethod]
public void Test()
{
bool? condition = GetCondition("Test");
Assert.IsNotNull(condition);
Assert.IsTrue(condition.Value);
}
private static bool? GetCondition(string environmentVariable)
=> bool.TryParse(Environment.GetEnvironmentVariable(environmentVariable), out bool result)
? result
: null;
}