microsoft
/
coyote
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 2868: Fix how binary rewriting tests run so they can run in production and systematic test modes 

Fix how binary rewriting tests run so they can run in production and systematic test modes and fix run-tests.ps1 so it tests all frameworks (since that works now!)

Related work items: #4377, #4458, #4705, #4979
This commit is contained in:
Chris Lovett 2020-08-07 17:37:22 +00:00
Родитель 0d6181ae5c
Коммит e7c195d105
20 изменённых файлов: 238 добавлений и 26 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
Coyote.sln
Просмотреть файл

@ -38,6 +38,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Coyote.Performanc
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BinaryRewriting.Tests", "Tests\BinaryRewriting.Tests\BinaryRewriting.Tests.csproj", "{A643CBC2-24DF-4B93-AD7F-84EF329D3AAF}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Standalone.Tests", "Tests\Standalone.Tests\Standalone.Tests.csproj", "{E719FD60-9CDE-4DE9-BD2C-5DB1053AC552}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -84,6 +86,10 @@ Global
{A643CBC2-24DF-4B93-AD7F-84EF329D3AAF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A643CBC2-24DF-4B93-AD7F-84EF329D3AAF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A643CBC2-24DF-4B93-AD7F-84EF329D3AAF}.Release|Any CPU.Build.0 = Release|Any CPU
{E719FD60-9CDE-4DE9-BD2C-5DB1053AC552}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E719FD60-9CDE-4DE9-BD2C-5DB1053AC552}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E719FD60-9CDE-4DE9-BD2C-5DB1053AC552}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E719FD60-9CDE-4DE9-BD2C-5DB1053AC552}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -99,6 +105,7 @@ Global
{4B03C121-C1C9-4C08-A673-BFD5FC821983} = {83369B7E-5C21-4D49-A14C-E8A6A4892807}
{1F4FD737-FEEF-47B0-86DC-3ED72596870B} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
{A643CBC2-24DF-4B93-AD7F-84EF329D3AAF} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
{E719FD60-9CDE-4DE9-BD2C-5DB1053AC552} = {2012300C-6E5D-47A0-9D57-B3F0A73AA1D4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {B9407046-CB24-4B07-8031-2749696EC7D8}

22
Scripts/powershell/common.psm1
Просмотреть файл

@ -21,6 +21,7 @@ function Invoke-DotnetTest([String]$dotnet, [String]$project, [String]$target, [
# Runs the specified tool command.
function Invoke-ToolCommand([String]$tool, [String]$command, [String]$error_msg) {
Write-Host "Invoking $tool $command"
Invoke-Expression "$tool $command"
if (-not ($LASTEXITCODE -eq 0)) {
Write-Error $error_msg
@ -157,3 +158,24 @@ function FindDotNetSdk($dotnet_path)
return $matching_version
}
}
function Rewrite($framework, $target, $keyFile)
{
Write-Host -ForegroundColor Yellow "Rewriting: $target"
if ($framework -eq "netcoreapp3.1") {
$tool = $dotnet
$command = "./bin/$f/coyote.dll rewrite $target"
} else {
$tool = "./bin/$f/coyote.exe"
$command = "rewrite $target"
}
if ($framework -ne "netcoreapp3.1" -and [System.Environment]::OSVersion.Platform -eq "Win32NT")
{
# note: Mono.Cecil cannot sign assemblies on unix platforms.
$command = $command + " --key $key_file"
}
$error_msg = "Failed to rewrite using $target"
Invoke-ToolCommand -tool $tool -command $command -error_msg $error_msg
}

37
Scripts/run-tests.ps1
Просмотреть файл

@ -2,7 +2,7 @@ param(
[string]$dotnet="dotnet",
[ValidateSet("all","netcoreapp3.1","net47","net48")]
[string]$framework="all",
[ValidateSet("all","production","rewriting","testing")]
[ValidateSet("all","production","rewriting","testing","standalone")]
[string]$test="all",
[string]$filter="",
[string]$logger="",
@ -18,6 +18,7 @@ $targets = [ordered]@{
"production" = "Production.Tests"
"rewriting" = "BinaryRewriting.Tests"
"testing" = "SystematicTesting.Tests"
"standalone" = "Standalone.Tests"
}
[System.Environment]::SetEnvironmentVariable('COYOTE_CLI_TELEMETRY_OPTOUT','1')
@ -33,20 +34,30 @@ foreach ($kvp in $targets.GetEnumerator()) {
continue
}
if ($($kvp.Name) -eq "rewriting") {
if ($f -ne "netcoreapp3.1") {
# We only currently support testing .NET Core binary rewriting.
continue
}
$config_file = "$PSScriptRoot/../Tests/$($kvp.Value)/bin/netcoreapp3.1/BinaryRewritingTests.coyote.json"
$command = "./bin/$f/coyote.dll rewrite $config_file"
$error_msg = "Failed to rewrite using 'BinaryRewritingTests.coyote.json'"
Invoke-ToolCommand -tool $dotnet -command $command -error_msg $error_msg
}
$target = "$PSScriptRoot/../Tests/$($kvp.Value)/$($kvp.Value).csproj"
Invoke-DotnetTest -dotnet $dotnet -project $($kvp.Name) -target $target -filter $filter -logger $logger -framework $f -verbosity $v
if ($($kvp.Name) -eq "rewriting") {
$key_file = "$PSScriptRoot/../Common/Key.snk"
$config_file = "$PSScriptRoot/../Tests/$($kvp.Value)/bin/$f/BinaryRewritingTests.coyote.json"
Rewrite -framework $f -target $config_file -keyFile $key_file
# now run the rewritten test.
$target = "$PSScriptRoot/../Tests/$($kvp.Value)/$($kvp.Value).csproj"
Invoke-DotnetTest -dotnet $dotnet -project $($kvp.Name) -target $target -filter $filter -logger $logger -framework $f -verbosity $v
}
if ($($kvp.Name) -eq "standalone") {
$key_file = "$PSScriptRoot/../Common/Key.snk"
$target_file = "$PSScriptRoot/../Tests/bin/$f/Microsoft.Coyote.Standalone.Tests.dll"
Rewrite -framework $f -target $target_file -keyFile $key_file
# now run the rewritten test.
$target = "$PSScriptRoot/../Tests/$($kvp.Value)/$($kvp.Value).csproj"
Invoke-DotnetTest -dotnet $dotnet -project $($kvp.Name) -target $target -filter $filter -logger $logger -framework $f -verbosity $v
}
}
}

14
Source/Core/Configuration.cs
Просмотреть файл

@ -323,6 +323,11 @@ namespace Microsoft.Coyote
/// </summary>
internal string TelemetryServerPath;
/// <summary>
/// Path of strong name key to use for signing new assembly.
/// </summary>
internal string StrongNameKeyFile;
/// <summary>
/// Initializes a new instance of the <see cref="Configuration"/> class.
/// </summary>
@ -590,6 +595,15 @@ namespace Microsoft.Coyote
return this;
}
/// <summary>
/// Enable resigning of assemblies..
/// </summary>
internal Configuration WithStrongNameKeyFile(string keyFile)
{
this.StrongNameKeyFile = keyFile;
return this;
}
/// <summary>
/// Returns the .NET platform version this assembly was compiled for.
/// </summary>

16
Source/Test/Rewriting/AssemblyRewriter.cs
Просмотреть файл

@ -5,6 +5,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using Microsoft.Coyote.IO;
using Microsoft.Coyote.SystematicTesting.Interception;
using Mono.Cecil;
@ -126,11 +127,22 @@ namespace Microsoft.Coyote.Rewriting
string outputPath = Path.Combine(outputDirectory, assemblyName);
Console.WriteLine($"... Writing the modified '{assemblyName}' assembly to " +
$"{(this.Configuration.IsReplacingAssemblies ? assemblyPath : outputPath)}");
assembly.Write(outputPath, new WriterParameters()
var writeParams = new WriterParameters()
{
WriteSymbols = isSymbolFileAvailable,
SymbolWriterProvider = new PortablePdbWriterProvider()
});
};
if (!string.IsNullOrEmpty(this.Configuration.StrongNameKeyFile))
{
using (FileStream fs = File.Open(this.Configuration.StrongNameKeyFile, FileMode.Open))
{
writeParams.StrongNameKeyPair = new StrongNameKeyPair(fs);
}
}
assembly.Write(outputPath, writeParams);
assembly.Dispose();
if (this.Configuration.IsReplacingAssemblies)

5
Source/Test/Rewriting/Configuration.cs
Просмотреть файл

@ -41,6 +41,11 @@ namespace Microsoft.Coyote.Rewriting
/// </summary>
private string DotnetVersion;
/// <summary>
/// Path of strong name key to use for signing new assembly.
/// </summary>
internal string StrongNameKeyFile;
/// <summary>
/// The .NET platform version that Coyote was compiled for.
/// </summary>

17
Source/Test/Rewriting/Transforms/TaskTransform.cs
Просмотреть файл

@ -2,6 +2,7 @@
// Licensed under the MIT License.
using System.Collections.Generic;
using System.Linq;
using Microsoft.Coyote.IO;
using Mono.Cecil;
using Mono.Cecil.Cil;
@ -62,6 +63,7 @@ namespace Microsoft.Coyote.Rewriting
this.TypeDef = type;
this.Method = null;
this.Processor = null;
RewriteTestRewrittenAttribute(type);
}
/// <inheritdoc/>
@ -360,6 +362,21 @@ namespace Microsoft.Coyote.Rewriting
return this.Module.ImportReference(result);
}
/// <summary>
/// Changes the boolean flag in TestRewrittenAttributes to true.
/// </summary>
/// <param name="type">The type that might have a TestRewrittenAttribute.</param>
private static void RewriteTestRewrittenAttribute(TypeDefinition type)
{
CustomAttribute attr = (from a in type.CustomAttributes where a.AttributeType.Name == "TestRewrittenAttribute" select a).FirstOrDefault();
if (attr != null)
{
// found it!
var arg = attr.ConstructorArguments[0];
attr.ConstructorArguments[0] = new CustomAttributeArgument(arg.Type, true);
}
}
/// <summary>
/// Checks if the specified type is the <see cref="SystemTasks.Task"/> type.
/// </summary>

21
Source/Test/SystematicTesting/TestAttributes.cs
Просмотреть файл

@ -51,4 +51,25 @@ namespace Microsoft.Coyote.SystematicTesting
internal sealed class TestRuntimeCreateAttribute : Attribute
{
}
/// <summary>
/// Attribute for monitoring when an assembly is rewritten
/// by coyote.
/// </summary>
[AttributeUsage(AttributeTargets.Class)]
public sealed class TestRewrittenAttribute : Attribute
{
/// <summary>
/// Whether assembly has been rewritten.
/// </summary>
public bool Rewritten;
/// <summary>
/// Initializes a new instance of the <see cref="TestRewrittenAttribute"/> class.
/// </summary>
public TestRewrittenAttribute(bool rewritten)
{
this.Rewritten = rewritten;
}
}
}

1
Tests/BinaryRewriting.Tests/BinaryRewriting.Tests.csproj
Просмотреть файл

@ -29,7 +29,6 @@
<Compile Include="..\Production.Tests\Tasks\Join\TaskWhenAnyTests.cs" Link="Tasks\Join\TaskWhenAnyTests.cs" />
<Compile Include="..\Production.Tests\Tasks\Locks\SynchronizedBlockTests.cs" Link="Tasks\Locks\SynchronizedBlockTests.cs" />
<Compile Include="..\Production.Tests\Tasks\Scheduling\AsyncCallStackTests.cs" Link="Tasks\Scheduling\AsyncCallStackTests.cs" />
<Compile Include="..\SystematicTesting.Tests\BaseSystematicTest.cs" Link="BaseSystematicTest.cs" />
<Compile Include="..\SystematicTesting.Tests\Tasks\Logging\CustomTaskLogTests.cs" Link="Tasks\Logging\CustomTaskLogTests.cs" />
</ItemGroup>
<ItemGroup>

5
Tests/BinaryRewriting.Tests/Configuration/ConfigurationTests.cs
Просмотреть файл

@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
@ -22,7 +23,7 @@ namespace Microsoft.Coyote.BinaryRewriting.Tests.Tasks
{
string configDirectory = this.GetJsonConfigurationDirectory();
string configPath = Path.Combine(configDirectory, "BinaryRewritingTests.coyote.json");
Assert.True(File.Exists(configPath));
Assert.True(File.Exists(configPath), "File not found: " + configPath);
var config = RewritingConfiguration.ParseFromJSON(configPath);
Assert.NotNull(config);
@ -59,7 +60,7 @@ namespace Microsoft.Coyote.BinaryRewriting.Tests.Tasks
{
string binaryDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string configDirectory = subDirectory is null ? binaryDirectory : Path.Combine(binaryDirectory, subDirectory);
Assert.True(Directory.Exists(configDirectory));
Assert.True(Directory.Exists(configDirectory), "Directory not found: " + configDirectory);
return configDirectory;
}

23
Tests/BinaryRewriting.Tests/Threading/ThreadPoolTests.cs
Просмотреть файл

@ -5,12 +5,13 @@ using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Coyote.Specifications;
using Microsoft.Coyote.Tests.Common;
using Xunit;
using Xunit.Abstractions;
namespace Microsoft.Coyote.BinaryRewriting.Tests.Tasks
{
public class ThreadPoolTests : BaseSystematicTest
public class ThreadPoolTests : BaseProductionTest
{
public ThreadPoolTests(ITestOutputHelper output)
: base(output)
@ -70,12 +71,22 @@ namespace Microsoft.Coyote.BinaryRewriting.Tests.Tasks
[Fact(Timeout = 5000)]
public void TestQueueUserWorkItemWithException()
{
this.TestWithError(() =>
if (!this.IsSystematicTest)
{
// production version of this test results in an unhandled exception.
// bugbug: can we rewrite this test so it works both in production and systematic testing modes?
// TestQueueUserWorkItemWithAsyncException makes a lot more sense to me.
return;
}
this.TestWithError(async () =>
{
ThreadPool.QueueUserWorkItem(_ =>
{
ThrowException();
}, null);
await Task.Delay(10);
},
configuration: GetConfiguration().WithTestingIterations(10),
errorChecker: (e) =>
@ -145,6 +156,14 @@ namespace Microsoft.Coyote.BinaryRewriting.Tests.Tasks
[Fact(Timeout = 5000)]
public void TestUnsafeQueueUserWorkItemWithException()
{
if (!this.IsSystematicTest)
{
// production version of this test results in an unhandled exception.
// bugbug: can we rewrite this test so it works both in production and systematic testing modes?
// TestUnsafeQueueUserWorkItemWithAsyncException makes a lot more sense to me.
return;
}
this.TestWithError(() =>
{
ThreadPool.UnsafeQueueUserWorkItem(_ =>

3
Tests/BinaryRewriting.Tests/xunit.runner.json
Просмотреть файл

@ -2,5 +2,6 @@
"parallelizeAssembly": false,
"parallelizeTestCollections": false,
"diagnosticMessages": true,
"longRunningTestSeconds": 5
"longRunningTestSeconds": 5,
"shadowCopy": false
}

14
Tests/Production.Tests/BaseProductionTest.cs
Просмотреть файл

@ -1,11 +1,13 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Reflection;
#if BINARY_REWRITE
using System.Threading.Tasks;
#else
using Microsoft.Coyote.Tasks;
#endif
using Microsoft.Coyote.SystematicTesting;
using Microsoft.Coyote.Tests.Common;
using Xunit.Abstractions;
@ -15,6 +17,7 @@ namespace Microsoft.Coyote.BinaryRewriting.Tests
namespace Microsoft.Coyote.Production.Tests
#endif
{
[TestRewritten(false)]
public abstract class BaseProductionTest : BaseTest
{
public BaseProductionTest(ITestOutputHelper output)
@ -22,9 +25,14 @@ namespace Microsoft.Coyote.Production.Tests
{
}
#if BINARY_REWRITE
public override bool IsSystematicTest => true;
#endif
public override bool IsSystematicTest
{
get
{
var attr = this.GetType().GetCustomAttribute(typeof(TestRewrittenAttribute)) as TestRewrittenAttribute;
return attr.Rewritten;
}
}
public class SharedEntry
{

6
Tests/Production.Tests/Tasks/TaskInterleavingsTests.cs
Просмотреть файл

@ -195,6 +195,12 @@ namespace Microsoft.Coyote.Production.Tests.Tasks
[Fact(Timeout = 5000)]
public void TestExploreAllInterleavings()
{
if (!this.IsSystematicTest)
{
// production version cannot always find all combinations.
return;
}
SortedSet<string> results = new SortedSet<string>();
string success = "Explored interleavings.";

22
Tests/Standalone.Tests/Standalone.Tests.csproj Normal file
Просмотреть файл

@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Description>Tests for the standalone rewriting.</Description>
<AssemblyName>Microsoft.Coyote.Standalone.Tests</AssemblyName>
<RootNamespace>Microsoft.Coyote.Standalone.Tests</RootNamespace>
<OutputPath>..\bin\</OutputPath>
<CoyoteTargetType>Application</CoyoteTargetType>
<NoWarn>$(NoWarn),1591</NoWarn>
</PropertyGroup>
<Import Project="..\..\Common\build.props" />
<Import Project="..\..\Common\key.props" />
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.2" />
</ItemGroup>
<ItemGroup>
<None Update="xunit.runner.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

24
Tests/Standalone.Tests/StandaloneTaskTests.cs Normal file
Просмотреть файл

@ -0,0 +1,24 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Threading.Tasks;
using Xunit;
namespace Standalone.Tests
{
public class StandaloneTaskTests
{
[Fact(Timeout=5000)]
public async Task SimpleTaskTest()
{
int result = 0;
await Task.Run(async () =>
{
await Task.Delay(10);
result = 10;
});
Assert.True(result == 10, "result should be 10");
}
}
}

6
Tests/Standalone.Tests/xunit.runner.json Normal file
Просмотреть файл

@ -0,0 +1,6 @@
{
"parallelizeAssembly": false,
"parallelizeTestCollections": false,
"diagnosticMessages": true,
"longRunningTestSeconds": 5
}

10
Tests/SystematicTesting.Tests/Tasks/Logging/CustomTaskLogTests.cs
Просмотреть файл

@ -18,10 +18,10 @@ using Xunit.Abstractions;
#if BINARY_REWRITE
namespace Microsoft.Coyote.BinaryRewriting.Tests.Tasks
#else
namespace Microsoft.Coyote.SystematicTesting.Tests.Tasks
namespace Microsoft.Coyote.Production.Tests.Tasks
#endif
{
public class CustomTaskLogTests : BaseSystematicTest
public class CustomTaskLogTests : BaseProductionTest
{
public CustomTaskLogTests(ITestOutputHelper output)
: base(output)
@ -82,6 +82,12 @@ Hi mom!
[Fact(Timeout = 5000)]
public void TestCustomTaskRuntimeLog()
{
if (!this.IsSystematicTest)
{
// assembly has not been rewritten, so skip this test.
return;
}
var config = GetConfiguration().WithRandomGeneratorSeed(0);
TestingEngine engine = TestingEngine.Create(config, this.RunAsync);

7
Tools/Coyote/Program.cs
Просмотреть файл

@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.IO;
using Microsoft.Coyote.IO;
using Microsoft.Coyote.Rewriting;
@ -194,6 +195,7 @@ namespace Microsoft.Coyote
Console.WriteLine($". Rewriting {fullPath}");
var config = Rewriting.Configuration.Create(assemblyDir, assemblyDir,
new HashSet<string>(new string[] { fullPath }));
config.StrongNameKeyFile = Configuration.StrongNameKeyFile;
config.PlatformVersion = Configuration.PlatformVersion;
AssemblyRewriter.Rewrite(config);
}
@ -202,6 +204,11 @@ namespace Microsoft.Coyote
Console.WriteLine($". Rewriting the assemblies specified in {Configuration.RewritingConfigurationFile}");
var config = Rewriting.Configuration.ParseFromJSON(Configuration.RewritingConfigurationFile);
config.PlatformVersion = Configuration.PlatformVersion;
if (string.IsNullOrEmpty(config.StrongNameKeyFile))
{
config.StrongNameKeyFile = Configuration.StrongNameKeyFile;
}
AssemblyRewriter.Rewrite(config);
}

4
Tools/Coyote/Utilities/CommandLineOptions.cs
Просмотреть файл

@ -60,6 +60,7 @@ You can provide one or two unsigned integer values", typeof(uint)).IsMultiValue
var rewritingGroup = this.Parser.GetOrCreateGroup("rewritingGroup", "Binary rewriting options");
rewritingGroup.DependsOn = new CommandLineArgumentDependency() { Name = "command", Value = "rewrite" };
rewritingGroup.AddArgument("key", null, "Path to strong name signing key");
var coverageGroup = this.Parser.GetOrCreateGroup("coverageGroup", "Code and activity coverage options");
var coverageArg = coverageGroup.AddArgument("coverage", "c", @"Generate code coverage statistics (via VS instrumentation) with zero or more values equal to:
@ -333,6 +334,9 @@ You can provide one or two unsigned integer values", typeof(uint)).IsMultiValue
case "instrument-list":
configuration.AdditionalCodeCoverageAssemblies[(string)option.Value] = true;
break;
case "key":
configuration.StrongNameKeyFile = (string)option.Value;
break;
case "timeout-delay":
configuration.TimeoutDelay = (uint)option.Value;
break;