microsoft
/
coyote
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

support for running xunit systematic tests from cli (#481)

This commit is contained in:
Pantazis Deligiannis 2023-06-05 18:04:17 -07:00 коммит произвёл GitHub
Родитель 1f800adcd1
Коммит 58c1d4f8ae
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
25 изменённых файлов: 355 добавлений и 40 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

17
Coyote.sln
Просмотреть файл

@ -47,6 +47,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests.Rewriting", "Tests\Te
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests.Runtime", "Tests\Tests.Runtime\Tests.Runtime.csproj", "{47FD6EEE-7AFB-40B2-BD34-880FA8896199}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests.Tools", "Tests\Tests.Tools\Tests.Tools.csproj", "{161C3CD4-C374-47C2-B306-99D3019D8F47}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -57,14 +59,14 @@ Global
{4B03C121-C1C9-4C08-A673-BFD5FC821983}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4B03C121-C1C9-4C08-A673-BFD5FC821983}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4B03C121-C1C9-4C08-A673-BFD5FC821983}.Release|Any CPU.Build.0 = Release|Any CPU
{8548010B-B99D-44FD-95BD-F716C1D707B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8548010B-B99D-44FD-95BD-F716C1D707B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8548010B-B99D-44FD-95BD-F716C1D707B7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8548010B-B99D-44FD-95BD-F716C1D707B7}.Release|Any CPU.Build.0 = Release|Any CPU
{E75DB9C9-7842-4AE4-A29D-624F6B49F607}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E75DB9C9-7842-4AE4-A29D-624F6B49F607}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E75DB9C9-7842-4AE4-A29D-624F6B49F607}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E75DB9C9-7842-4AE4-A29D-624F6B49F607}.Release|Any CPU.Build.0 = Release|Any CPU
{8548010B-B99D-44FD-95BD-F716C1D707B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8548010B-B99D-44FD-95BD-F716C1D707B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8548010B-B99D-44FD-95BD-F716C1D707B7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8548010B-B99D-44FD-95BD-F716C1D707B7}.Release|Any CPU.Build.0 = Release|Any CPU
{911F1779-3558-4590-836C-C75112D65FD8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{911F1779-3558-4590-836C-C75112D65FD8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{911F1779-3558-4590-836C-C75112D65FD8}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -113,14 +115,18 @@ Global
{47FD6EEE-7AFB-40B2-BD34-880FA8896199}.Debug|Any CPU.Build.0 = Debug|Any CPU
{47FD6EEE-7AFB-40B2-BD34-880FA8896199}.Release|Any CPU.ActiveCfg = Release|Any CPU
{47FD6EEE-7AFB-40B2-BD34-880FA8896199}.Release|Any CPU.Build.0 = Release|Any CPU
{161C3CD4-C374-47C2-B306-99D3019D8F47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{161C3CD4-C374-47C2-B306-99D3019D8F47}.Debug|Any CPU.Build.0 = Debug|Any CPU
{161C3CD4-C374-47C2-B306-99D3019D8F47}.Release|Any CPU.ActiveCfg = Release|Any CPU
{161C3CD4-C374-47C2-B306-99D3019D8F47}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{4B03C121-C1C9-4C08-A673-BFD5FC821983} = {83369B7E-5C21-4D49-A14C-E8A6A4892807}
{8548010B-B99D-44FD-95BD-F716C1D707B7} = {83369B7E-5C21-4D49-A14C-E8A6A4892807}
{E75DB9C9-7842-4AE4-A29D-624F6B49F607} = {83369B7E-5C21-4D49-A14C-E8A6A4892807}
{8548010B-B99D-44FD-95BD-F716C1D707B7} = {83369B7E-5C21-4D49-A14C-E8A6A4892807}
{911F1779-3558-4590-836C-C75112D65FD8} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
{DABC68C1-79D3-4324-A750-7CF72E0A0ACF} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
{61FC86A6-AF87-4007-B184-AF860A57AB9E} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
@ -133,6 +139,7 @@ Global
{130CC5A8-946C-4EB3-982C-F6A57EB65326} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
{D962786B-EBBA-40BA-814C-96C02675E2D1} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
{47FD6EEE-7AFB-40B2-BD34-880FA8896199} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
{161C3CD4-C374-47C2-B306-99D3019D8F47} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {B9407046-CB24-4B07-8031-2749696EC7D8}

3
Scripts/run-tests.ps1 поставляемый
Просмотреть файл

@ -4,7 +4,7 @@
param(
[ValidateSet("net7.0", "net6.0", "netcoreapp3.1", "net462")]
[string]$framework = "net7.0",
[ValidateSet("all", "runtime", "rewriting", "testing", "actors", "actors-testing", "standalone")]
[ValidateSet("all", "runtime", "rewriting", "testing", "actors", "actors-testing", "tools")]
[string]$test = "all",
[string]$filter = "",
[string]$logger = "",
@ -23,6 +23,7 @@ $targets = [ordered]@{
"testing" = "Tests.BugFinding"
"actors" = "Tests.Actors"
"actors-testing" = "Tests.Actors.BugFinding"
"tools" = "Tests.Tools"
}
# Find that paths to the installed .NET runtime.

6
Source/Core/Properties/InternalsVisibleTo.cs
Просмотреть файл

@ -68,6 +68,12 @@ using System.Runtime.CompilerServices;
"d7bc9fd67a08d3fa122120a469158da22a652af4508571ac9b16c6a05d2b3b6d7004ac76be85c3" +
"ca3d55f6ae823cd287a2810243f2bd6be5f4ba7b016c80da954371e591b10c97b0938f721c7149" +
"3bc97f9e")]
[assembly: InternalsVisibleTo("Microsoft.Coyote.Tests.Tools,PublicKey=" +
"0024000004800000940000000602000000240000525341310004000001000100d7971281941569" +
"53fd8af100ac5ecaf1d96fab578562b91133663d6ccbf0b313d037a830a20d7af1ce02a6641d71" +
"d7bc9fd67a08d3fa122120a469158da22a652af4508571ac9b16c6a05d2b3b6d7004ac76be85c3" +
"ca3d55f6ae823cd287a2810243f2bd6be5f4ba7b016c80da954371e591b10c97b0938f721c7149" +
"3bc97f9e")]
[assembly: InternalsVisibleTo("Tests.Performance,PublicKey=" +
"0024000004800000940000000602000000240000525341310004000001000100d7971281941569" +
"53fd8af100ac5ecaf1d96fab578562b91133663d6ccbf0b313d037a830a20d7af1ce02a6641d71" +

6
Source/Test/Properties/InternalsVisibleTo.cs
Просмотреть файл

@ -34,6 +34,12 @@ using System.Runtime.CompilerServices;
"d7bc9fd67a08d3fa122120a469158da22a652af4508571ac9b16c6a05d2b3b6d7004ac76be85c3" +
"ca3d55f6ae823cd287a2810243f2bd6be5f4ba7b016c80da954371e591b10c97b0938f721c7149" +
"3bc97f9e")]
[assembly: InternalsVisibleTo("Microsoft.Coyote.Tests.Tools,PublicKey=" +
"0024000004800000940000000602000000240000525341310004000001000100d7971281941569" +
"53fd8af100ac5ecaf1d96fab578562b91133663d6ccbf0b313d037a830a20d7af1ce02a6641d71" +
"d7bc9fd67a08d3fa122120a469158da22a652af4508571ac9b16c6a05d2b3b6d7004ac76be85c3" +
"ca3d55f6ae823cd287a2810243f2bd6be5f4ba7b016c80da954371e591b10c97b0938f721c7149" +
"3bc97f9e")]
// Tools
[assembly: InternalsVisibleTo("Coyote,PublicKey=" +

4
Source/Test/Rewriting/Attributes.cs
Просмотреть файл

@ -13,7 +13,7 @@ namespace Microsoft.Coyote.Rewriting
/// If this attribute is applied to an assembly manifest, it denotes that the
/// assembly has been rewritten.
/// </remarks>
[AttributeUsage(AttributeTargets.Assembly)]
[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)]
public sealed class RewritingSignatureAttribute : Attribute
{
/// <summary>
@ -39,7 +39,7 @@ namespace Microsoft.Coyote.Rewriting
/// <summary>
/// Attribute for declaring source code targets that must not be rewritten.
/// </summary>
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct)]
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct, AllowMultiple = false)]
public sealed class SkipRewritingAttribute : Attribute
{
/// <summary>

18
Source/Test/SystematicTesting/Frameworks/ITestLog.cs Normal file
Просмотреть файл

@ -0,0 +1,18 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Microsoft.Coyote.Logging;
namespace Microsoft.Coyote.SystematicTesting.Frameworks
{
/// <summary>
/// Logs all test output to the installed <see cref="ILogger"/>.
/// </summary>
internal interface ITestLog
{
/// <summary>
/// Logs messages using the installed <see cref="ILogger"/>.
/// </summary>
LogWriter LogWriter { get; set; }
}
}

31
Source/Test/SystematicTesting/Frameworks/Xunit/TestOutput.cs Normal file
Просмотреть файл

@ -0,0 +1,31 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Microsoft.Coyote.Logging;
using Xunit.Abstractions;
namespace Microsoft.Coyote.SystematicTesting.Frameworks.XUnit
{
/// <summary>
/// Redirects all xUnit test output to the installed <see cref="ILogger"/>.
/// </summary>
internal sealed class TestOutput : ITestOutputHelper, ITestLog
{
/// <summary>
/// Logs messages using the installed <see cref="ILogger"/>.
/// </summary>
public LogWriter LogWriter { get; set; }
/// <inheritdoc/>
public void WriteLine(string message)
{
this.LogWriter.WriteLine(message);
}
/// <inheritdoc/>
public void WriteLine(string format, params object[] args)
{
this.LogWriter.WriteLine(format, args);
}
}
}

5
Tests/Tests.Common/TestOutputLogger.cs → Source/Test/SystematicTesting/Frameworks/Xunit/TestOutputLogger.cs
Просмотреть файл

@ -5,10 +5,10 @@ using System.Text;
using Microsoft.Coyote.Logging;
using Xunit.Abstractions;
namespace Microsoft.Coyote.Tests.Common
namespace Microsoft.Coyote.SystematicTesting.Frameworks.XUnit
{
/// <summary>
/// Logger that writes to the test output.
/// Logger that writes to the xUnit test output.
/// </summary>
public sealed class TestOutputLogger : ILogger
{
@ -63,6 +63,7 @@ namespace Microsoft.Coyote.Tests.Common
public void Write(string format, params object[] args) =>
this.Write(LogSeverity.Info, format, args);
/// <inheritdoc/>
public void Write(LogSeverity severity, string value)
{
lock (this.Lock)

63
Source/Test/SystematicTesting/Reports/TraceReport.cs
Просмотреть файл

@ -63,11 +63,21 @@ namespace Microsoft.Coyote.SystematicTesting
report.Settings.PortfolioMode = configuration.PortfolioMode.ToString().ToLower();
report.Settings.IsLivenessCheckingEnabled = configuration.IsLivenessCheckingEnabled;
report.Settings.LivenessTemperatureThreshold = configuration.LivenessTemperatureThreshold;
report.Settings.IsCollectionAccessRaceCheckingEnabled = configuration.IsCollectionAccessRaceCheckingEnabled;
report.Settings.IsLockAccessRaceCheckingEnabled = configuration.IsLockAccessRaceCheckingEnabled;
report.Settings.IsPartiallyControlledConcurrencyAllowed = configuration.IsPartiallyControlledConcurrencyAllowed;
report.Settings.IsPartiallyControlledDataNondeterminismAllowed = configuration.IsPartiallyControlledDataNondeterminismAllowed;
report.Settings.UncontrolledConcurrencyResolutionAttempts = configuration.UncontrolledConcurrencyResolutionAttempts;
report.Settings.UncontrolledConcurrencyResolutionDelay = configuration.UncontrolledConcurrencyResolutionDelay;
report.Settings.IsAtomicOperationRaceCheckingEnabled = configuration.IsAtomicOperationRaceCheckingEnabled;
report.Settings.IsVolatileOperationRaceCheckingEnabled =
configuration.IsVolatileOperationRaceCheckingEnabled;
report.Settings.IsMemoryAccessRaceCheckingEnabled = configuration.IsMemoryAccessRaceCheckingEnabled;
report.Settings.IsControlFlowRaceCheckingEnabled = configuration.IsControlFlowRaceCheckingEnabled;
report.Settings.IsPartiallyControlledConcurrencyAllowed =
configuration.IsPartiallyControlledConcurrencyAllowed;
report.Settings.IsPartiallyControlledDataNondeterminismAllowed =
configuration.IsPartiallyControlledDataNondeterminismAllowed;
report.Settings.UncontrolledConcurrencyResolutionAttempts =
configuration.UncontrolledConcurrencyResolutionAttempts;
report.Settings.UncontrolledConcurrencyResolutionDelay =
configuration.UncontrolledConcurrencyResolutionDelay;
report.ReportTrace(scheduler.Trace.Clone());
return report.ToJson();
@ -104,11 +114,23 @@ namespace Microsoft.Coyote.SystematicTesting
configuration.PortfolioMode = PortfolioModeExtensions.FromString(report.Settings.PortfolioMode);
configuration.IsLivenessCheckingEnabled = report.Settings.IsLivenessCheckingEnabled;
configuration.LivenessTemperatureThreshold = report.Settings.LivenessTemperatureThreshold;
configuration.IsCollectionAccessRaceCheckingEnabled =
report.Settings.IsCollectionAccessRaceCheckingEnabled;
configuration.IsLockAccessRaceCheckingEnabled = report.Settings.IsLockAccessRaceCheckingEnabled;
configuration.IsPartiallyControlledConcurrencyAllowed = report.Settings.IsPartiallyControlledConcurrencyAllowed;
configuration.IsPartiallyControlledDataNondeterminismAllowed = report.Settings.IsPartiallyControlledDataNondeterminismAllowed;
configuration.UncontrolledConcurrencyResolutionAttempts = report.Settings.UncontrolledConcurrencyResolutionAttempts;
configuration.UncontrolledConcurrencyResolutionDelay = report.Settings.UncontrolledConcurrencyResolutionDelay;
configuration.IsAtomicOperationRaceCheckingEnabled =
report.Settings.IsAtomicOperationRaceCheckingEnabled;
configuration.IsVolatileOperationRaceCheckingEnabled =
report.Settings.IsVolatileOperationRaceCheckingEnabled;
configuration.IsMemoryAccessRaceCheckingEnabled = report.Settings.IsMemoryAccessRaceCheckingEnabled;
configuration.IsControlFlowRaceCheckingEnabled = report.Settings.IsControlFlowRaceCheckingEnabled;
configuration.IsPartiallyControlledConcurrencyAllowed =
report.Settings.IsPartiallyControlledConcurrencyAllowed;
configuration.IsPartiallyControlledDataNondeterminismAllowed =
report.Settings.IsPartiallyControlledDataNondeterminismAllowed;
configuration.UncontrolledConcurrencyResolutionAttempts =
report.Settings.UncontrolledConcurrencyResolutionAttempts;
configuration.UncontrolledConcurrencyResolutionDelay =
report.Settings.UncontrolledConcurrencyResolutionDelay;
for (int idx = 0; idx < report.Steps.Count; idx++)
{
@ -236,11 +258,36 @@ namespace Microsoft.Coyote.SystematicTesting
/// </summary>
public int LivenessTemperatureThreshold { get; set; }
/// <summary>
/// Value specifying if checking races at collection accesses is enabled or not.
/// </summary>
public bool IsCollectionAccessRaceCheckingEnabled { get; set; }
/// <summary>
/// Value specifying if checking races during lock accesses is enabled or not.
/// </summary>
public bool IsLockAccessRaceCheckingEnabled { get; set; }
/// <summary>
/// Value specifying if checking races at atomic operations is enabled or not.
/// </summary>
public bool IsAtomicOperationRaceCheckingEnabled { get; set; }
/// <summary>
/// Value specifying if checking races at volatile operations is enabled or not.
/// </summary>
public bool IsVolatileOperationRaceCheckingEnabled { get; set; }
/// <summary>
/// Value specifying if checking races at memory-access locations is enabled or not.
/// </summary>
public bool IsMemoryAccessRaceCheckingEnabled { get; set; }
/// <summary>
/// Value specifying if checking races at control-flow branching locations is enabled or not.
/// </summary>
public bool IsControlFlowRaceCheckingEnabled { get; set; }
/// <summary>
/// Value specifying if partially controlled concurrency is allowed or not.
/// </summary>

8
Source/Test/SystematicTesting/TestAttributes.cs
Просмотреть файл

@ -8,7 +8,7 @@ namespace Microsoft.Coyote.SystematicTesting
/// <summary>
/// Attribute for declaring the entry point to a Coyote test.
/// </summary>
[AttributeUsage(AttributeTargets.Method)]
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public sealed class TestAttribute : Attribute
{
}
@ -16,7 +16,7 @@ namespace Microsoft.Coyote.SystematicTesting
/// <summary>
/// Attribute for declaring the initialization method to be called before testing starts.
/// </summary>
[AttributeUsage(AttributeTargets.Method)]
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public sealed class TestInitAttribute : Attribute
{
}
@ -24,7 +24,7 @@ namespace Microsoft.Coyote.SystematicTesting
/// <summary>
/// Attribute for declaring a cleanup method to be called when all test iterations terminate.
/// </summary>
[AttributeUsage(AttributeTargets.Method)]
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public sealed class TestDisposeAttribute : Attribute
{
}
@ -32,7 +32,7 @@ namespace Microsoft.Coyote.SystematicTesting
/// <summary>
/// Attribute for declaring a cleanup method to be called when each test iteration terminates.
/// </summary>
[AttributeUsage(AttributeTargets.Method)]
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public sealed class TestIterationDisposeAttribute : Attribute
{
}

89
Source/Test/SystematicTesting/TestMethodInfo.cs
Просмотреть файл

@ -74,6 +74,11 @@ namespace Microsoft.Coyote.SystematicTesting
private readonly ICompilationAssemblyResolver AssemblyResolver;
#endif
/// <summary>
/// Responsible for routing all test framework output to the installed <see cref="ILogger"/>.
/// </summary>
private readonly Frameworks.ITestLog TestFrameworkLog;
/// <summary>
/// Responsible for writing to the installed <see cref="ILogger"/>.
/// </summary>
@ -111,7 +116,8 @@ namespace Microsoft.Coyote.SystematicTesting
this.Assembly = Assembly.LoadFrom(configuration.AssemblyToBeAnalyzed);
#endif
(this.Method, this.Name) = GetTestMethod(this.Assembly, configuration.TestMethodName, logWriter);
(this.Method, this.Name, this.TestFrameworkLog) = GetTestMethod(
this.Assembly, configuration.TestMethodName, logWriter);
this.InitMethod = GetTestSetupMethod(this.Assembly, typeof(TestInitAttribute), logWriter);
this.DisposeMethod = GetTestSetupMethod(this.Assembly, typeof(TestDisposeAttribute), logWriter);
this.IterationDisposeMethod = GetTestSetupMethod(this.Assembly, typeof(TestIterationDisposeAttribute), logWriter);
@ -146,9 +152,11 @@ namespace Microsoft.Coyote.SystematicTesting
/// Returns the test method with the specified name. A test method must
/// be annotated with the <see cref="TestAttribute"/> attribute.
/// </summary>
private static (Delegate testMethod, string testName) GetTestMethod(Assembly assembly, string methodName, LogWriter logWriter)
private static (Delegate testMethod, string testName, Frameworks.ITestLog testLog) GetTestMethod(
Assembly assembly, string methodName, LogWriter logWriter)
{
BindingFlags flags = BindingFlags.Static | BindingFlags.Public | BindingFlags.DeclaredOnly | BindingFlags.InvokeMethod;
BindingFlags flags = BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public |
BindingFlags.DeclaredOnly | BindingFlags.InvokeMethod;
List<MethodInfo> testMethods = FindTestMethodsWithAttribute(typeof(TestAttribute), flags, assembly, logWriter);
if (testMethods.Count > 0)
@ -218,7 +226,7 @@ namespace Microsoft.Coyote.SystematicTesting
if (!((hasVoidReturnType || hasTaskReturnType) &&
(hasNoInputParameters || hasActorInputParameters || hasTaskInputParameters) &&
!testMethod.IsAbstract && !testMethod.IsVirtual && !testMethod.IsConstructor &&
!testMethod.ContainsGenericParameters && testMethod.IsPublic && testMethod.IsStatic))
!testMethod.ContainsGenericParameters && testMethod.IsPublic))
{
throw new InvalidOperationException("Incorrect test method declaration. Please " +
$"make sure your [{typeof(TestAttribute).FullName}] methods have:\n\n" +
@ -234,39 +242,87 @@ namespace Microsoft.Coyote.SystematicTesting
$" async {typeof(Task).FullName}<T>\n");
}
object testInstance = null;
Frameworks.ITestLog testLog = null;
if (!testMethod.IsStatic)
{
flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.DeclaredOnly;
ConstructorInfo[] constructors = testMethod.DeclaringType.GetConstructors(flags);
ConstructorInfo testConstructor = null;
if (constructors.Length > 0)
{
// TODO: add support for more common unit testing frameworks (MSTest, NUnit).
// If there is a constructor that takes an ITestOutputHelper from xUnit, then use that.
testConstructor = constructors.FirstOrDefault(ctor =>
ctor.GetParameters().Length is 1 &&
ctor.GetParameters()[0].ParameterType == typeof(Xunit.Abstractions.ITestOutputHelper));
// Else, try get the parameterless constructor.
testConstructor ??= constructors.FirstOrDefault(ctor => ctor.GetParameters().Length is 0);
if (testConstructor is null)
{
throw new InvalidOperationException($"Cannot detect a public parameterless constructor for " +
$"type '{testMethod.DeclaringType.FullName}' containing test method '{testMethod.Name}'.");
}
}
// Create an instance of the test class.
if (testConstructor.GetParameters().Length is 1 &&
testConstructor.GetParameters()[0].ParameterType == typeof(Xunit.Abstractions.ITestOutputHelper))
{
testLog = new Frameworks.XUnit.TestOutput();
testInstance = testConstructor.Invoke(new[] { testLog });
}
else if (testConstructor.GetParameters().Length is 0)
{
testInstance = Activator.CreateInstance(testMethod.DeclaringType);
}
}
Delegate test;
if (hasTaskReturnType)
{
if (hasActorInputParameters)
{
test = testMethod.CreateDelegate(typeof(Func<IActorRuntime, Task>));
test = testInstance is null ?
testMethod.CreateDelegate(typeof(Func<IActorRuntime, Task>)) :
testMethod.CreateDelegate(typeof(Func<IActorRuntime, Task>), testInstance);
}
else if (hasTaskInputParameters)
{
test = testMethod.CreateDelegate(typeof(Func<ICoyoteRuntime, Task>));
test = testInstance is null ?
testMethod.CreateDelegate(typeof(Func<ICoyoteRuntime, Task>)) :
testMethod.CreateDelegate(typeof(Func<ICoyoteRuntime, Task>), testInstance);
}
else
{
test = testMethod.CreateDelegate(typeof(Func<Task>));
test = testInstance is null ?
testMethod.CreateDelegate(typeof(Func<Task>)) :
testMethod.CreateDelegate(typeof(Func<Task>), testInstance);
}
}
else
{
if (hasActorInputParameters)
{
test = testMethod.CreateDelegate(typeof(Action<IActorRuntime>));
test = testInstance is null ?
testMethod.CreateDelegate(typeof(Action<IActorRuntime>)) :
testMethod.CreateDelegate(typeof(Action<IActorRuntime>), testInstance);
}
else if (hasTaskInputParameters)
{
test = testMethod.CreateDelegate(typeof(Action<ICoyoteRuntime>));
test = testInstance is null ?
testMethod.CreateDelegate(typeof(Action<ICoyoteRuntime>)) :
testMethod.CreateDelegate(typeof(Action<ICoyoteRuntime>), testInstance);
}
else
{
test = testMethod.CreateDelegate(typeof(Action));
test = testInstance is null ?
testMethod.CreateDelegate(typeof(Action)) :
testMethod.CreateDelegate(typeof(Action), testInstance);
}
}
return (test, $"{testMethod.DeclaringType}.{testMethod.Name}");
return (test, $"{testMethod.DeclaringType}.{testMethod.Name}", testLog);
}
/// <summary>
@ -404,6 +460,17 @@ namespace Microsoft.Coyote.SystematicTesting
}
#endif
/// <summary>
/// Sets the test log writer.
/// </summary>
internal void SetTestLogWriter(LogWriter logWriter)
{
if (this.TestFrameworkLog != null)
{
this.TestFrameworkLog.LogWriter = logWriter;
}
}
/// <summary>
/// Releases any held resources.
/// </summary>

7
Source/Test/SystematicTesting/TestingEngine.cs
Просмотреть файл

@ -382,16 +382,19 @@ namespace Microsoft.Coyote.SystematicTesting
runtime = CoyoteRuntime.Create(this.Configuration, this.Scheduler, iterationLogWriter, logManager, actorRuntimeExtension);
actorRuntimeExtension.WithRuntime(runtime);
// Set the test method iteration logger, if one can be set.
methodInfo.SetTestLogWriter(iterationLogWriter);
this.InitializeCustomActorLogging(actorRuntimeExtension);
// Runs the test and waits for it to terminate.
// Run the test and waits for it to terminate.
Task task = runtime.RunTestAsync(methodInfo.Method, methodInfo.Name);
task.Wait();
// Turn off runtime logging for the current iteration.
iterationLogWriter.Close();
// Invokes the user-specified iteration disposal method.
// Invoke the user-specified iteration disposal method.
methodInfo.DisposeCurrentIteration();
// Invoke any registered callbacks at the end of this iteration.

1
Source/Test/Test.csproj
Просмотреть файл

@ -17,6 +17,7 @@
<PackageReference Include="Microsoft.ApplicationInsights" Version="2.20.0" />
<PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
<PackageReference Include="Mono.Cecil" Version="0.11.4" />
<PackageReference Include="xunit.abstractions" Version="2.0.3" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'net7.0'">
<PackageReference Include="Microsoft.Extensions.DependencyModel" Version="7.0.0" />

2
Tests/Tests.Actors.BugFinding/Runtime/EntryPointTests.cs
Просмотреть файл

@ -8,7 +8,7 @@ using System.Threading.Tasks;
using Microsoft.Coyote.Logging;
using Microsoft.Coyote.Runtime;
using Microsoft.Coyote.SystematicTesting;
using Microsoft.Coyote.Tests.Common;
using Microsoft.Coyote.SystematicTesting.Frameworks.XUnit;
using Xunit;
using Xunit.Abstractions;

2
Tests/Tests.Actors/EventQueues/EventQueueStressTests.cs
Просмотреть файл

@ -2,7 +2,7 @@
// Licensed under the MIT License.
using System.Threading.Tasks;
using Microsoft.Coyote.Tests.Common;
using Microsoft.Coyote.SystematicTesting.Frameworks.XUnit;
using Xunit;
using Xunit.Abstractions;

2
Tests/Tests.Actors/EventQueues/EventQueueTests.cs
Просмотреть файл

@ -3,7 +3,7 @@
using System;
using System.Threading.Tasks;
using Microsoft.Coyote.Tests.Common;
using Microsoft.Coyote.SystematicTesting.Frameworks.XUnit;
using Xunit;
using Xunit.Abstractions;

1
Tests/Tests.Actors/Logging/ActorLoggingTests.cs
Просмотреть файл

@ -5,6 +5,7 @@ using System.IO;
using System.Text;
using Microsoft.Coyote.Logging;
using Microsoft.Coyote.Runtime;
using Microsoft.Coyote.SystematicTesting.Frameworks.XUnit;
using Microsoft.Coyote.Tests.Common;
using Xunit;
using Xunit.Abstractions;

1
Tests/Tests.Actors/Logging/RuntimeLogTests.cs
Просмотреть файл

@ -3,6 +3,7 @@
using Microsoft.Coyote.Logging;
using Microsoft.Coyote.Runtime;
using Microsoft.Coyote.SystematicTesting.Frameworks.XUnit;
using Microsoft.Coyote.Tests.Common;
using Xunit;
using Xunit.Abstractions;

1
Tests/Tests.Actors/Logging/RuntimeLogTextFormattingTests.cs
Просмотреть файл

@ -5,6 +5,7 @@ using System.IO;
using System.Text;
using Microsoft.Coyote.Logging;
using Microsoft.Coyote.Runtime;
using Microsoft.Coyote.SystematicTesting.Frameworks.XUnit;
using Microsoft.Coyote.Tests.Common;
using Xunit;
using Xunit.Abstractions;

1
Tests/Tests.Common/BaseTest.cs
Просмотреть файл

@ -12,6 +12,7 @@ using Microsoft.Coyote.Actors.Coverage;
using Microsoft.Coyote.Logging;
using Microsoft.Coyote.Runtime;
using Microsoft.Coyote.SystematicTesting;
using Microsoft.Coyote.SystematicTesting.Frameworks.XUnit;
using Xunit;
using Xunit.Abstractions;
using ActorRuntimeFactory = Microsoft.Coyote.Actors.RuntimeFactory;

18
Tests/Tests.Tools/BaseToolsTest.cs Normal file
Просмотреть файл

@ -0,0 +1,18 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Microsoft.Coyote.Runtime;
using Microsoft.Coyote.Tests.Common;
using Xunit;
using Xunit.Abstractions;
namespace Microsoft.Coyote.Tools.Tests
{
public abstract class BaseToolsTest : BaseTest
{
public BaseToolsTest(ITestOutputHelper output)
: base(output)
{
}
}
}

67
Tests/Tests.Tools/TestLoading/XUnitTestLoadingTests.cs Normal file
Просмотреть файл

@ -0,0 +1,67 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Reflection;
using System.Threading.Tasks;
using Microsoft.Coyote.Logging;
using Microsoft.Coyote.Specifications;
using Microsoft.Coyote.SystematicTesting;
using Microsoft.Coyote.SystematicTesting.Frameworks.XUnit;
using Xunit;
using Xunit.Abstractions;
namespace Microsoft.Coyote.Tools.Tests
{
public class XUnitTestLoadingTests : BaseToolsTest
{
public XUnitTestLoadingTests(ITestOutputHelper output)
: base(output)
{
}
#pragma warning disable CA1822 // Mark members as static
#pragma warning disable xUnit1013 // Public method should be marked as test
[Test]
public void VoidTest() => Assert.True(true);
[Test]
public Task TaskTest()
{
Assert.True(true);
return Task.CompletedTask;
}
#pragma warning restore xUnit1013 // Public method should be marked as test
#pragma warning restore CA1822 // Mark members as static
[Fact(Timeout = 5000)]
public void TestVoidEntryPoint() => this.CheckTestMethod(nameof(this.VoidTest));
[Fact(Timeout = 5000)]
public void TestTaskEntryPoint() => this.CheckTestMethod(nameof(this.TaskTest));
private void CheckTestMethod(string name)
{
Configuration config = this.GetConfiguration();
config.AssemblyToBeAnalyzed = Assembly.GetExecutingAssembly().Location;
config.TestMethodName = name;
var logWriter = new LogWriter(config);
logWriter.SetLogger(new TestOutputLogger(this.TestOutput));
using var testMethodInfo = TestMethodInfo.Create(config, logWriter);
Assert.Equal(Assembly.GetExecutingAssembly(), testMethodInfo.Assembly);
Assert.Equal($"{typeof(XUnitTestLoadingTests).FullName}.{name}", testMethodInfo.Name);
if (testMethodInfo.Method is Action action)
{
action();
}
else if (testMethodInfo.Method is Func<Task> function)
{
function();
}
else
{
Assert.True(false, "Unexpected test method type.");
}
}
}
}

31
Tests/Tests.Tools/Tests.Tools.csproj Normal file
Просмотреть файл

@ -0,0 +1,31 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Description>Tests for the runtime.</Description>
<AssemblyName>Microsoft.Coyote.Tests.Tools</AssemblyName>
<RootNamespace>Microsoft.Coyote.Tools.Tests</RootNamespace>
<OutputPath>.\bin\</OutputPath>
<NetStandard2Supported>false</NetStandard2Supported>
<NetCore31Supported>false</NetCore31Supported>
<Net6Supported>false</Net6Supported>
<IsPackable>false</IsPackable>
<NoWarn>$(NoWarn),1591</NoWarn>
</PropertyGroup>
<Import Project="..\..\Common\build.props" />
<Import Project="..\..\Common\key.props" />
<ItemGroup>
<ProjectReference Include="..\Tests.Common\Tests.Common.csproj" />
<ProjectReference Include="..\..\Tools\Coyote\Coyote.csproj">
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.0" />
<PackageReference Include="xunit" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5" />
</ItemGroup>
<ItemGroup>
<None Update="xunit.runner.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

7
Tests/Tests.Tools/xunit.runner.json поставляемый Normal file
Просмотреть файл

@ -0,0 +1,7 @@
{
"parallelizeAssembly": false,
"parallelizeTestCollections": false,
"diagnosticMessages": true,
"longRunningTestSeconds": 5,
"shadowCopy": false
}

4
Tests/compare-rewriting-diff-logs.ps1 поставляемый
Просмотреть файл

@ -16,8 +16,8 @@ $expected_hashes = [ordered]@{
"rewriting" = "77A92FFF9AF56B818CD5B4D2E8A9CE942DAAC99DB782B51200EE75C59191CB2D"
"rewriting-helpers" = "DAA8156B3911954999DEB6FE30FFC9691F87FDDCEB8E14EC7B986D162F194BD5"
"testing" = "DCF1CDA6C818F68C7FAE8D5B145055FB44AE5FF890DF08806994BFB9492C7C98"
"actors" = "76535EE876332FEF1BCAA3BE4E72FAF76E7E039ABF98EDF1940656A253F59112"
"actors-testing" = "AC7AB8C3FB4C6657C96A4FDCC581070A4483A6F6C8B865074A568389D59C144D"
"actors" = "4CE8FB3275F49CA69B7E6AC5BF0046A97810F57A84A8D99C262ADDF0765D3189"
"actors-testing" = "8D777E634DDF4000BE56F53E2BEA91C4A47D757F72799CA3773FD811D8F2A476"
}
Write-Comment -prefix "." -text "Comparing the test rewriting diff logs" -color "yellow"