зеркало из
1
0
Форкнуть 0

Acceptance tests should be end to end

This commit is contained in:
jnm2 2019-02-16 21:20:55 -05:00
Родитель 1791024852
Коммит b40712b3ad
10 изменённых файлов: 449 добавлений и 55 удалений

2
.gitignore поставляемый
Просмотреть файл

@ -1,4 +1,4 @@
/.acceptanceworkspace
/.acceptance/
## Ignore Visual Studio temporary files, build results, and

Просмотреть файл

@ -64,6 +64,11 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
return workspace;
}
protected static void InconclusiveOnException(Action action)
{
Assume.That(action.Invoke, Throws.Nothing);
}
[TearDown]
public static void TearDown()
{
@ -82,7 +87,7 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
else
{
foreach (var workspace in workspaces)
Directory.Delete(workspace.Directory, recursive: true);
Utils.DeleteDirectoryRobust(workspace.Directory);
}
}
}
@ -100,7 +105,7 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
{
if (File.Exists(Path.Combine(directory, "build.cake")))
{
return Path.Combine(directory, ".acceptanceworkspace");
return Path.Combine(directory, ".acceptance");
}
}

Просмотреть файл

@ -0,0 +1,137 @@
using System.Linq;
using NUnit.Framework;
namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
{
public sealed class BundledDependencyTests : AcceptanceTests
{
[Test]
public static void User_tests_get_the_version_of_Mono_Cecil_referenced_from_the_test_project()
{
var workspace = CreateWorkspace()
.AddProject("Test.csproj", $@"
<Project Sdk='Microsoft.NET.Sdk'>
<PropertyGroup>
<TargetFrameworks>{string.Join(";", TargetFrameworks)}</TargetFrameworks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include='Microsoft.NET.Test.Sdk' Version='*' />
<PackageReference Include='Mono.Cecil' Version='0.10.0-beta5' />
<PackageReference Include='NUnit' Version='*' />
<PackageReference Include='NUnit3TestAdapter' Version='{NuGetPackageVersion}' />
</ItemGroup>
<ItemGroup Condition=""'$(TargetFrameworkIdentifier)' == '.NETCoreApp'"">
<PackageReference Include='System.Diagnostics.FileVersionInfo' Version='*' />
</ItemGroup>
</Project>")
.AddFile("BundledDependencyTests.cs", @"
using System.Diagnostics;
using System.Reflection;
using NUnit.Framework;
public static class BundledDependencyTests
{
[Test]
public static void User_tests_get_the_version_of_Mono_Cecil_referenced_from_the_test_project()
{
var assembly = typeof(Mono.Cecil.ReaderParameters)
#if NETCOREAPP1_0
.GetTypeInfo()
#endif
.Assembly;
var versionBlock = FileVersionInfo.GetVersionInfo(assembly.Location);
Assert.That(versionBlock.ProductVersion, Is.EqualTo(""0.10.0.0-beta5""));
}
}");
workspace.MSBuild(restore: true);
workspace.VSTest(
from targetFramework in TargetFrameworks
select $@"bin\Debug\{targetFramework}\Test.dll");
}
[Test]
public static void Engine_uses_its_bundled_version_of_Mono_Cecil_instead_of_the_version_referenced_by_the_test_project()
{
var workspace = CreateWorkspace()
.AddProject("Test.csproj", $@"
<Project Sdk='Microsoft.NET.Sdk'>
<PropertyGroup>
<TargetFrameworks>{string.Join(";", TargetFrameworks)}</TargetFrameworks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include='Microsoft.NET.Test.Sdk' Version='*' />
<PackageReference Include='Mono.Cecil' Version='0.10.0' />
<PackageReference Include='NUnit' Version='*' />
<PackageReference Include='NUnit3TestAdapter' Version='{NuGetPackageVersion}' />
</ItemGroup>
<ItemGroup Condition=""'$(TargetFrameworkIdentifier)' == '.NETFramework'"">
<PackageReference Include='nunit.engine.api' Version='3.9.0' />
</ItemGroup>
<ItemGroup Condition=""'$(TargetFrameworkIdentifier)' == '.NETCoreApp'"">
<PackageReference Include='System.Diagnostics.FileVersionInfo' Version='*' />
<PackageReference Include='nunit.engine.netstandard' Version='3.8.0' />
</ItemGroup>
<ItemGroup>
<None Update='test.addins' CopyToOutputDirectory='Always' />
</ItemGroup>
</Project>")
.AddFile("BundledDependencyTests.cs", @"
using System.Diagnostics;
using System.Reflection;
using NUnit.Framework;
public class ReferencingMonoCecilTests
{
[Test]
public void Engine_uses_its_bundled_version_of_Mono_Cecil_instead_of_the_version_referenced_by_the_test_project()
{
var assembly = typeof(Mono.Cecil.ReaderParameters)
#if NETCOREAPP1_0
.GetTypeInfo()
#endif
.Assembly;
var versionBlock = FileVersionInfo.GetVersionInfo(assembly.Location);
Assert.That(versionBlock.ProductVersion, Is.EqualTo(""0.10.0.0""));
}
}")
.AddFile("TestNUnitEngineExtension.cs", @"
using NUnit.Engine;
using NUnit.Engine.Extensibility;
// Trigger Mono.Cecil binary break between older versions and 0.10.0
// (test.addins points the engine to search all classes in this file and should result
// in a runtime failure to cast 'Mono.Cecil.InterfaceImplementation' to 'Mono.Cecil.TypeReference'
// if the engine is using the newer version of Mono.Cecil)
[Extension]
public sealed class TestNUnitEngineExtension : ITestEventListener
{
public void OnTestEvent(string report)
{
}
}")
.AddFile("test.addins", @"
Test.dll");
workspace.MSBuild(restore: true);
workspace.VSTest(
from targetFramework in TargetFrameworks
select $@"bin\Debug\{targetFramework}\Test.dll");
}
}
}

Просмотреть файл

@ -1,11 +1,56 @@
using System.Linq;
using NUnit.Framework;
using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools;
namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
{
public sealed class DotNetRestoreTests : AcceptanceTests
public sealed class SinglePassingTestResultTests : AcceptanceTests
{
// TODO: Get packages.config projects to build. Requires downloading nuget.exe and doing `nuget restore`,
// so ToolLocationFacts will need to become a ToolResolver class with tool cache configured via
// constructor. Need to determine how best to get it to IsolatedWorkspace.
// TODO: Verify TRX for all tests.
// TODO: Delete acceptance.cake and make sure this project is picked up in CI
private static void AddTestsCs(IsolatedWorkspace workspace)
{
workspace.AddFile("Tests.cs", @"
using NUnit.Framework;
namespace Test
{
public class Tests
{
[Test]
public void PassingTest()
{
Assert.Pass();
}
}
}");
}
private static void AddTestsVb(IsolatedWorkspace workspace)
{
workspace.AddFile("Tests.vb", @"
Imports NUnit.Framework
Namespace Test
Public Class Tests
<Test>
Public Sub PassingTest()
Assert.Pass()
End Sub
End Class
End Namespace");
}
[TestCaseSource(nameof(TargetFrameworks))]
public static void NuGet_package_can_be_restored_for_single_target_csproj(string targetFramework)
public static void Single_target_csproj(string targetFramework)
{
var workspace = CreateWorkspace()
.AddProject("Test.csproj", $@"
@ -23,11 +68,15 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
</Project>");
workspace.DotNetRestore();
AddTestsCs(workspace);
workspace.MSBuild(restore: true);
workspace.VSTest(new[] { $@"bin\Debug\{targetFramework}\Test.dll" });
}
[TestCaseSource(nameof(TargetFrameworks))]
public static void NuGet_package_can_be_restored_for_single_target_vbproj(string targetFramework)
public static void Single_target_vbproj(string targetFramework)
{
var workspace = CreateWorkspace()
.AddProject("Test.vbproj", $@"
@ -45,11 +94,15 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
</Project>");
workspace.DotNetRestore();
AddTestsVb(workspace);
workspace.MSBuild(restore: true);
workspace.VSTest(new[] { $@"bin\Debug\{targetFramework}\Test.dll" });
}
[Test]
public static void NuGet_package_can_be_restored_for_multi_target_csproj()
public static void Multi_target_csproj()
{
var workspace = CreateWorkspace()
.AddProject("Test.csproj", $@"
@ -67,11 +120,17 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
</Project>");
workspace.DotNetRestore();
AddTestsCs(workspace);
workspace.MSBuild(restore: true);
workspace.VSTest(
from targetFramework in TargetFrameworks
select $@"bin\Debug\{targetFramework}\Test.dll");
}
[Test]
public static void NuGet_package_can_be_restored_for_multi_target_vbproj()
public static void Multi_target_vbproj()
{
var workspace = CreateWorkspace()
.AddProject("Test.vbproj", $@"
@ -89,11 +148,17 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
</Project>");
workspace.DotNetRestore();
AddTestsVb(workspace);
workspace.MSBuild(restore: true);
workspace.VSTest(
from targetFramework in TargetFrameworks
select $@"bin\Debug\{targetFramework}\Test.dll");
}
[Test]
public static void NuGet_package_can_be_restored_for_legacy_csproj_with_PackageReference()
public static void Legacy_csproj_with_PackageReference()
{
var workspace = CreateWorkspace()
.AddProject("Test.csproj", $@"
@ -137,6 +202,9 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
<Reference Include='System.Data' />
<Reference Include='System.Xml' />
</ItemGroup>
<ItemGroup>
<Compile Include='Tests.cs' />
</ItemGroup>
<ItemGroup>
<PackageReference Include='Microsoft.NET.Test.Sdk'>
<Version>15.9.0</Version>
@ -151,11 +219,15 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
<Import Project='$(MSBuildToolsPath)\Microsoft.CSharp.targets' />
</Project>");
workspace.DotNetRestore();
AddTestsCs(workspace);
workspace.MSBuild(restore: true);
workspace.VSTest(new[] { @"bin\Debug\Test.dll" });
}
[Test]
public static void NuGet_package_can_be_restored_for_legacy_vbproj_with_PackageReference()
public static void Legacy_vbproj_with_PackageReference()
{
var workspace = CreateWorkspace()
.AddProject("Test.vbproj", $@"
@ -222,6 +294,9 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
<Import Include='System.Linq' />
<Import Include='System.Xml.Linq' />
</ItemGroup>
<ItemGroup>
<Compile Include='Tests.vb' />
</ItemGroup>
<ItemGroup>
<PackageReference Include='Microsoft.NET.Test.Sdk'>
<Version>15.9.0</Version>
@ -236,11 +311,27 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
<Import Project='$(MSBuildToolsPath)\Microsoft.VisualBasic.targets' />
</Project>");
workspace.DotNetRestore();
AddTestsVb(workspace);
workspace.MSBuild(restore: true);
workspace.VSTest(new[] { @"bin\Debug\Test.dll" });
}
private static void AddPackagesConfig(IsolatedWorkspace workspace)
{
workspace.AddFile("packages.config", $@"
<?xml version='1.0' encoding='utf-8'?>
<packages>
<package id='Microsoft.CodeCoverage' version='15.9.0' targetFramework='net45' />
<package id='Microsoft.NET.Test.Sdk' version='15.9.0' targetFramework='net45' />
<package id='NUnit' version='3.11.0' targetFramework='net45' />
<package id='NUnit3TestAdapter' version='{NuGetPackageVersion}' targetFramework='net45' />
</packages>");
}
[Test]
public static void NuGet_package_can_be_restored_for_legacy_csproj_with_packages_config()
public static void Legacy_csproj_with_packages_config()
{
var workspace = CreateWorkspace()
.AddProject("Test.csproj", $@"
@ -299,6 +390,7 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
<Reference Include='System.Xml' />
</ItemGroup>
<ItemGroup>
<Compile Include='Tests.cs' />
<None Include='packages.config' />
</ItemGroup>
<Import Project='$(MSBuildToolsPath)\Microsoft.CSharp.targets' />
@ -315,21 +407,18 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
</Target>
<Import Project='..\packages\Microsoft.CodeCoverage.15.9.0\build\netstandard1.0\Microsoft.CodeCoverage.targets' Condition=""Exists('..\packages\Microsoft.CodeCoverage.15.9.0\build\netstandard1.0\Microsoft.CodeCoverage.targets')"" />
<Import Project='..\packages\Microsoft.NET.Test.Sdk.15.9.0\build\net45\Microsoft.Net.Test.Sdk.targets' Condition=""Exists('..\packages\Microsoft.NET.Test.Sdk.15.9.0\build\net45\Microsoft.Net.Test.Sdk.targets')"" />
</Project>")
.AddFile("packages.config", $@"
<?xml version='1.0' encoding='utf-8'?>
<packages>
<package id='Microsoft.CodeCoverage' version='15.9.0' targetFramework='net45' />
<package id='Microsoft.NET.Test.Sdk' version='15.9.0' targetFramework='net45' />
<package id='NUnit' version='3.11.0' targetFramework='net45' />
<package id='NUnit3TestAdapter' version='{NuGetPackageVersion}' targetFramework='net45' />
</packages>");
</Project>");
workspace.DotNetRestore();
AddPackagesConfig(workspace);
AddTestsCs(workspace);
workspace.MSBuild(restore: true);
workspace.VSTest(new[] { @"bin\Debug\Test.dll" });
}
[Test]
public static void NuGet_package_can_be_restored_for_legacy_vbproj_with_packages_config()
public static void Legacy_vbproj_with_packages_config()
{
var workspace = CreateWorkspace()
.AddProject("Test.vbproj", $@"
@ -411,6 +500,7 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
<Import Include='System.Threading.Tasks' />
</ItemGroup>
<ItemGroup>
<Compile Include='Tests.vb' />
<None Include='packages.config' />
</ItemGroup>
<Import Project='$(MSBuildToolsPath)\Microsoft.VisualBasic.targets' />
@ -427,17 +517,14 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
</Target>
<Import Project='..\packages\Microsoft.CodeCoverage.15.9.0\build\netstandard1.0\Microsoft.CodeCoverage.targets' Condition=""Exists('..\packages\Microsoft.CodeCoverage.15.9.0\build\netstandard1.0\Microsoft.CodeCoverage.targets')"" />
<Import Project='..\packages\Microsoft.NET.Test.Sdk.15.9.0\build\net45\Microsoft.Net.Test.Sdk.targets' Condition=""Exists('..\packages\Microsoft.NET.Test.Sdk.15.9.0\build\net45\Microsoft.Net.Test.Sdk.targets')"" />
</Project>")
.AddFile("packages.config", $@"
<?xml version='1.0' encoding='utf-8'?>
<packages>
<package id='Microsoft.CodeCoverage' version='15.9.0' targetFramework='net45' />
<package id='Microsoft.NET.Test.Sdk' version='15.9.0' targetFramework='net45' />
<package id='NUnit' version='3.11.0' targetFramework='net45' />
<package id='NUnit3TestAdapter' version='{NuGetPackageVersion}' targetFramework='net45' />
</packages>");
</Project>");
workspace.DotNetRestore();
AddPackagesConfig(workspace);
AddTestsVb(workspace);
workspace.MSBuild(restore: true);
workspace.VSTest(new[] { @"bin\Debug\Test.dll" });
}
}
}

Просмотреть файл

@ -1,6 +1,7 @@
using System;
using System.IO;
using System.Text;
using NUnit.Framework;
namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
{
@ -78,6 +79,27 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
return path;
}
public static void DeleteDirectoryRobust(string directory)
{
for (var attempt = 1; ; attempt++)
{
try
{
Directory.Delete(directory, recursive: true);
break;
}
catch (IOException ex) when (attempt < 3 && (WinErrorCode)ex.HResult == WinErrorCode.DirNotEmpty)
{
TestContext.WriteLine("Another process added files to the directory while its contents were being deleted. Retrying...");
}
}
}
private enum WinErrorCode : ushort
{
DirNotEmpty = 145
}
/// <summary>
/// Removes any indentation that is common to all lines. If the first line has no indentation,
/// indentation common to all other lines is removed. If the first line is empty, it is removed.

Просмотреть файл

@ -0,0 +1,46 @@
using System;
using System.Collections.Generic;
namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
{
partial class IsolatedWorkspace
{
private sealed class RunSettings
{
private readonly List<string> arguments = new List<string>();
public string WorkingDirectory { get; }
public string FileName { get; }
public RunSettings(string workingDirectory, string fileName)
{
WorkingDirectory = workingDirectory;
FileName = fileName;
}
public void Run()
{
ProcessUtils.Run(WorkingDirectory, FileName, arguments);
}
public RunSettings Add(string argument)
{
arguments.Add(argument);
return this;
}
public RunSettings AddRange(IEnumerable<string> arguments)
{
if (arguments is null) throw new ArgumentNullException(nameof(arguments));
this.arguments.AddRange(arguments);
return this;
}
public RunSettings AddIf(bool condition, string argument)
{
if (condition) arguments.Add(argument);
return this;
}
}
}
}

Просмотреть файл

@ -6,7 +6,7 @@ using System.IO;
namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
{
[DebuggerDisplay("{Directory,nq}")]
public sealed class IsolatedWorkspace
public sealed partial class IsolatedWorkspace
{
private readonly List<string> projectPaths = new List<string>();
@ -38,7 +38,50 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
public void DotNetRestore()
{
ProcessUtils.Run(Directory, "dotnet", new[] { "restore" });
ConfigureRun("dotnet")
.Add("restore")
.Run();
}
public void DotNetBuild(bool noRestore = false)
{
ConfigureRun("dotnet")
.Add("build")
.AddIf(noRestore, "--no-restore")
.Run();
}
public void DotNetTest(bool noBuild = false)
{
ConfigureRun("dotnet")
.Add("test")
.AddIf(noBuild, "--no-build")
.Run();
}
public void DotNetVSTest(IEnumerable<string> testAssemblyPaths)
{
ConfigureRun("dotnet")
.Add("vstest")
.AddRange(testAssemblyPaths)
.Run();
}
public void MSBuild(string target = null, bool restore = false)
{
ConfigureRun(ToolLocationFacts.MSBuild)
.AddIf(target != null, "/t:" + target)
.AddIf(restore, "/restore")
.Run();
}
public void VSTest(IEnumerable<string> testAssemblyPaths)
{
ConfigureRun(ToolLocationFacts.VSTest)
.AddRange(testAssemblyPaths)
.Run();
}
private RunSettings ConfigureRun(string filename) => new RunSettings(Directory, filename);
}
}

Просмотреть файл

@ -31,7 +31,7 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
public void Dispose()
{
reasonFile.Dispose();
if (!keep) Directory.Delete(workspaceDirectory, recursive: true);
if (!keep) Utils.DeleteDirectoryRobust(workspaceDirectory);
}
public IsolatedWorkspace CreateWorkspace(string name)

Просмотреть файл

@ -8,7 +8,7 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
{
public static class ProcessUtils
{
public static void Run(string workingDirectory, string fileName, IEnumerable<string> arguments = null)
public static string Run(string workingDirectory, string fileName, IEnumerable<string> arguments = null)
{
if (!Path.IsPathRooted(workingDirectory))
throw new ArgumentException(nameof(workingDirectory), "Working directory must not be relative.");
@ -29,31 +29,35 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
}
})
{
var output = (StringBuilder)null;
var error = (StringBuilder)null;
// This is inherited if the test runner was started by the Visual Studio process.
// It breaks MSBuild 15s targets when it tries to build legacy csprojs and vbprojs.
process.StartInfo.EnvironmentVariables.Remove("VisualStudioVersion");
var stdout = (StringBuilder)null;
var stderr = (StringBuilder)null;
process.OutputDataReceived += (sender, e) =>
{
if (e.Data is null) return;
if (output is null)
output = new StringBuilder();
if (stdout is null)
stdout = new StringBuilder();
else
output.AppendLine();
stdout.AppendLine();
output.Append(e.Data);
stdout.Append(e.Data);
};
process.ErrorDataReceived += (sender, e) =>
{
if (e.Data is null) return;
if (error is null)
error = new StringBuilder();
if (stderr is null)
stderr = new StringBuilder();
else
error.AppendLine();
stderr.AppendLine();
error.Append(e.Data);
stderr.Append(e.Data);
};
process.Start();
@ -61,14 +65,16 @@ namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
process.BeginOutputReadLine();
process.WaitForExit();
if (process.ExitCode != 0 || error != null)
if (process.ExitCode != 0 || stderr != null)
{
throw new ProcessErrorException(
Path.GetFileName(fileName),
process.ExitCode,
output?.ToString(),
error?.ToString());
stdout?.ToString(),
stderr?.ToString());
}
return stdout?.ToString() ?? string.Empty;
}
}

Просмотреть файл

@ -0,0 +1,48 @@
using System;
using System.IO;
namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
{
public static class ToolLocationFacts
{
private static readonly Lazy<string> vsWhere = new Lazy<string>(() =>
{
return Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86),
@"Microsoft Visual Studio\Installer\vswhere.exe");
});
public static string VSWhere => vsWhere.Value;
private static readonly Lazy<string> msBuild = new Lazy<string>(() =>
{
var vsInstallation = ProcessUtils.Run(
Environment.CurrentDirectory,
VSWhere,
new[] { "-latest", "-products", "*", "-requires", "Microsoft.Component.MSBuild", "-property", "installationPath" });
if (string.IsNullOrWhiteSpace(vsInstallation))
throw new InvalidOperationException("MSBuild is not installed with Visual Studio on this machine.");
return Path.Combine(vsInstallation, @"MSBuild\15.0\Bin\MSBuild.exe");
});
public static string MSBuild => msBuild.Value;
private static readonly Lazy<string> vsTest = new Lazy<string>(() =>
{
var vsInstallation = ProcessUtils.Run(
Environment.CurrentDirectory,
VSWhere,
// https://github.com/Microsoft/vswhere/issues/126#issuecomment-360542783
new[] { "-latest", "-products", "*", "-requires", "Microsoft.VisualStudio.TestTools.TestPlatform.V1.CLI", "-property", "installationPath" });
if (string.IsNullOrWhiteSpace(vsInstallation))
throw new InvalidOperationException("VSTest is not installed with Visual Studio on this machine.");
return Path.Combine(vsInstallation, @"Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe");
});
public static string VSTest => vsTest.Value;
}
}