Added acceptance test project

.gitignore

@@ -1,3 +1,6 @@
+/.acceptanceworkspace
+
+
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 

NUnit3TestAdapter.sln

@@ -1,6 +1,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.27130.2026
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.28606.126
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{7CE30108-5D81-4850-BE6B-C8BCA35D3592}"
 	ProjectSection(SolutionItems) = preProject
@@ -40,6 +40,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NUnit3TestAdapterInstall",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit3AdapterExternalTests", "src\NUnit3AdapterExternalTests\NUnit3AdapterExternalTests.csproj", "{A4EA819A-D77D-46D3-B2B7-2C754DBD2BC7}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NUnit.TestAdapter.Tests.Acceptance", "src\NUnit.TestAdapter.Tests.Acceptance\NUnit.TestAdapter.Tests.Acceptance.csproj", "{3FAC7EE0-664F-4B11-918B-8E0FF865EE4C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -70,6 +72,10 @@ Global
 		{A4EA819A-D77D-46D3-B2B7-2C754DBD2BC7}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A4EA819A-D77D-46D3-B2B7-2C754DBD2BC7}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A4EA819A-D77D-46D3-B2B7-2C754DBD2BC7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3FAC7EE0-664F-4B11-918B-8E0FF865EE4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3FAC7EE0-664F-4B11-918B-8E0FF865EE4C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3FAC7EE0-664F-4B11-918B-8E0FF865EE4C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3FAC7EE0-664F-4B11-918B-8E0FF865EE4C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/NUnit.TestAdapter.Tests.Acceptance/AcceptanceTests.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+using NUnit.Framework.Interfaces;
+using NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools;
+
+namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
+{
+    [Category("Acceptance")]
+    public abstract class AcceptanceTests
+    {
+        protected static string NuGetPackageId { get; } = "NUnit3TestAdapter";
+
+        private readonly static Lazy<(IsolatedWorkspaceManager manager, string nupkgVersion)> Initialization = new Lazy<(IsolatedWorkspaceManager, string)>(() =>
+        {
+            var directory = TestContext.Parameters["ProjectWorkspaceDirectory"]
+                ?? TryAutoDetectProjectWorkspaceDirectory()
+                ?? throw new InvalidOperationException("The test parameter ProjectWorkspaceDirectory must be set in order to run this test.");
+
+            var nupkgDirectory = TestContext.Parameters["TestNupkgDirectory"]
+                ?? TryAutoDetectTestNupkgDirectory(NuGetPackageId)
+                ?? throw new InvalidOperationException("The test parameter TestNupkgDirectory must be set in order to run this test.");
+
+            var nupkgVersion = TryGetTestNupkgVersion(nupkgDirectory, packageId: NuGetPackageId)
+                ?? throw new InvalidOperationException($"No NuGet package with the ID {NuGetPackageId} was found in {nupkgDirectory}.");
+
+            return (
+                new IsolatedWorkspaceManager(
+                    reason: string.Join(Environment.NewLine,
+                        "Test assembly: " + typeof(AcceptanceTests).Assembly.Location,
+                        "Runner process: " + Process.GetCurrentProcess().MainModule.FileName),
+                    directory,
+                    nupkgDirectory,
+                    Path.Combine(directory, ".isolatednugetcache")),
+                nupkgVersion);
+        });
+
+        private readonly static Dictionary<string, List<IsolatedWorkspace>> WorkspacesByTestId = new Dictionary<string, List<IsolatedWorkspace>>();
+
+        protected static string NuGetPackageVersion => Initialization.Value.nupkgVersion;
+
+        protected static IsolatedWorkspace CreateWorkspace()
+        {
+            var test = TestContext.CurrentContext?.Test ?? throw new InvalidOperationException("There is no current test.");
+
+            var workspace = Initialization.Value.manager.CreateWorkspace(test.Name);
+
+            if (!WorkspacesByTestId.TryGetValue(test.ID, out var workspaces))
+                WorkspacesByTestId.Add(test.ID, workspaces = new List<IsolatedWorkspace>());
+            workspaces.Add(workspace);
+
+            return workspace;
+        }
+
+        [TearDown]
+        public static void TearDown()
+        {
+            var test = TestContext.CurrentContext?.Test ?? throw new InvalidOperationException("There is no current test.");
+
+            if (WorkspacesByTestId.TryGetValue(test.ID, out var workspaces))
+            {
+                WorkspacesByTestId.Remove(test.ID);
+
+                if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed)
+                {
+                    Initialization.Value.manager.PreserveDirectory(
+                        test.FullName + " failed:" + Environment.NewLine
+                        + TestContext.CurrentContext.Result.Message.TrimEnd() + Environment.NewLine);
+                }
+                else
+                {
+                    foreach (var workspace in workspaces)
+                        Directory.Delete(workspace.Directory, recursive: true);
+                }
+            }
+        }
+
+        internal static void OnGlobalTeardown()
+        {
+            if (!Initialization.IsValueCreated) return;
+
+            Initialization.Value.manager.Dispose();
+        }
+
+        private static string TryAutoDetectProjectWorkspaceDirectory()
+        {
+            for (var directory = TestContext.CurrentContext.TestDirectory; directory != null; directory = Path.GetDirectoryName(directory))
+            {
+                if (File.Exists(Path.Combine(directory, "build.cake")))
+                {
+                    return Path.Combine(directory, ".acceptanceworkspace");
+                }
+            }
+
+            return null;
+        }
+
+        private static string TryAutoDetectTestNupkgDirectory(string packageId)
+        {
+            // Keep in sync with build.cake.
+
+            for (var directory = TestContext.CurrentContext.TestDirectory; directory != null; directory = Path.GetDirectoryName(directory))
+            {
+                var packagePath = Path.Combine(directory, "package");
+
+                try
+                {
+                    if (Directory.EnumerateFiles(Path.Combine(directory, "package"), packageId + ".*.nupkg").Any())
+                        return packagePath;
+                }
+                catch (DirectoryNotFoundException)
+                {
+                }
+            }
+
+            return null;
+        }
+
+        private static string TryGetTestNupkgVersion(string directory, string packageId)
+        {
+            var path = Directory.GetFileSystemEntries(directory, packageId + ".*.nupkg").SingleOrDefault();
+
+            return path is null ? null :
+                Path.GetFileNameWithoutExtension(path).Substring(packageId.Length + 1);
+        }
+    }
+}

src/NUnit.TestAdapter.Tests.Acceptance/AcceptanceTestsTeardownFixture.cs

@@ -0,0 +1,15 @@
+using NUnit.Framework;
+
+namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
+{
+    // https://github.com/nunit/nunit/issues/3166
+    [SetUpFixture]
+    public sealed class AcceptanceTestsTeardownFixture
+    {
+        [OneTimeTearDown]
+        public static void OneTimeTearDown()
+        {
+            AcceptanceTests.OnGlobalTeardown();
+        }
+    }
+}

src/NUnit.TestAdapter.Tests.Acceptance/NUnit.TestAdapter.Tests.Acceptance.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Import Project="..\Common.props" />
+  <PropertyGroup>
+    <TargetFramework>net472</TargetFramework>
+
+    <RootNamespace>NUnit.VisualStudio.TestAdapter.Tests.Acceptance</RootNamespace>
+    <AssemblyName>NUnit.VisualStudio.TestAdapter.Tests.Acceptance</AssemblyName>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="nunit" Version="3.11.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="3.12.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.9.0" />
+  </ItemGroup>
+
+</Project>

src/NUnit.TestAdapter.Tests.Acceptance/NuGetRestoreTests.cs

@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using NUnit.Framework;
+
+namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
+{
+    public sealed class NuGetRestoreTests : AcceptanceTests
+    {
+        public static IEnumerable<string> TargetFrameworks => new[]
+        {
+            "net35",
+            "netcoreapp1.0"
+        };
+
+        [TestCaseSource(nameof(TargetFrameworks))]
+        public static void NuGet_package_can_be_restored_for_single_target_csproj(string targetFramework)
+        {
+            var workspace = CreateWorkspace()
+                .AddProject("Test.csproj", $@"
+                    <Project Sdk='Microsoft.NET.Sdk'>
+
+                      <PropertyGroup>
+                        <TargetFramework>{targetFramework}</TargetFramework>
+                      </PropertyGroup>
+
+                      <ItemGroup>
+                        <PackageReference Include='Microsoft.NET.Test.Sdk' Version='*' />
+                        <PackageReference Include='NUnit' Version='*' />
+                        <PackageReference Include='NUnit3TestAdapter' Version='{NuGetPackageVersion}' />
+                      </ItemGroup>
+
+                    </Project>")
+                .AddFile("SimpleTests.cs", @"
+                    using NUnit.Framework;
+
+                    namespace Simple
+                    {
+                        public class SimpleTests
+                        {
+                            [Test]
+                            public void PassingTest()
+                            {
+                                Assert.Pass();
+                            }
+                        }
+                    }");
+
+            Assert.That(1, Is.EqualTo(2));
+        }
+
+        [TestCaseSource(nameof(TargetFrameworks))]
+        public static void NuGet_package_can_be_restored_for_single_target_vbproj(string targetFramework)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

src/NUnit.TestAdapter.Tests.Acceptance/Utils.cs

@@ -0,0 +1,189 @@
+using System;
+using System.IO;
+using System.Text;
+
+namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
+{
+    internal static class Utils
+    {
+        public static string GetSafeFilename(string arbitraryString, bool allowDirectorySeparators = false)
+        {
+            var invalidChars = Path.GetInvalidFileNameChars();
+            var replaceIndex = arbitraryString.IndexOfAny(invalidChars, 0);
+            if (replaceIndex == -1) return arbitraryString;
+
+            var r = new StringBuilder();
+            var i = 0;
+
+            do
+            {
+                r.Append(arbitraryString, i, replaceIndex - i);
+
+                switch (arbitraryString[replaceIndex])
+                {
+                    case '"':
+                        r.Append("''");
+                        break;
+                    case '<':
+                    case '>':
+                    case '|':
+                    case ':':
+                    case '*':
+                    case '\\':
+                    case '/':
+                        r.Append(allowDirectorySeparators ? Path.DirectorySeparatorChar : '-');
+                        break;
+                    case '\0':
+                    case '\f':
+                    case '?':
+                        break;
+                    case '\t':
+                    case '\n':
+                    case '\r':
+                    case '\v':
+                        r.Append(' ');
+                        break;
+                    default:
+                        r.Append('_');
+                        break;
+                }
+
+                i = replaceIndex + 1;
+                replaceIndex = arbitraryString.IndexOfAny(invalidChars, i);
+            } while (replaceIndex != -1);
+
+            r.Append(arbitraryString, i, arbitraryString.Length - i);
+
+            return r.ToString();
+        }
+
+        public static string CreateUniqueDirectory(string parentDirectory, string name = null)
+        {
+            parentDirectory = Path.GetFullPath(parentDirectory);
+
+            var safeName = name is null ? null : GetSafeFilename(name);
+
+            var path = Path.Combine(parentDirectory, safeName ?? "1");
+
+            if (Directory.Exists(path))
+            {
+                for (var id = 2; ; id++)
+                {
+                    path = Path.Combine(parentDirectory, safeName is null ? id.ToString() : safeName + "_" + id);
+                    if (!Directory.Exists(path)) break;
+                }
+            }
+
+            Directory.CreateDirectory(path);
+            return path;
+        }
+
+        /// <summary>
+        /// Removes any indentation that is common to all lines. If the first line has no indentation,
+        /// indentation common to all other lines is removed. If the first line is empty, it is removed.
+        /// Empty lines are not considered when calculating indentation.
+        /// </summary>
+        public static string RemoveIndent(string indented)
+        {
+            if (indented is null) return null;
+
+            var reader = new StringReader(indented);
+            var firstLine = reader.ReadLine();
+
+            var firstLineStartsWithoutIndent = firstLine.Length != 0 && firstLine[0] != ' ';
+
+            var readerForCountingPass = new StringReader(indented);
+
+            // Skip the first line when determining common indentation if it has no indentation
+            if (firstLineStartsWithoutIndent)
+                _ = readerForCountingPass.ReadLine();
+
+            var indentationCharCount = CountCommonIndentationChars(readerForCountingPass);
+
+            var builder = new StringBuilder();
+
+            var previousLineHasEnded = true;
+
+            if (firstLine.Length != 0)
+            {
+                if (firstLineStartsWithoutIndent)
+                {
+                    builder.Append(firstLine);
+                    previousLineHasEnded = false;
+                }
+                else
+                {
+                    // Start at beginning
+                    reader = new StringReader(indented);
+                }
+            }
+
+            var remainingIndentationChars = indentationCharCount;
+
+            while (reader.Read() is var next && next != -1)
+            {
+                if (next == ' ' && remainingIndentationChars > 0)
+                {
+                    remainingIndentationChars--;
+                    continue;
+                }
+
+                if (!previousLineHasEnded) builder.AppendLine();
+
+                if (next == '\r')
+                {
+                    next = reader.Read();
+                    if (next != '\n') throw new NotImplementedException("Carriage return without line feed");
+                }
+
+                if (next != '\n')
+                {
+                    builder.Append((char)next).Append(reader.ReadLine());
+                }
+
+                previousLineHasEnded = false;
+                remainingIndentationChars = indentationCharCount;
+            }
+
+            return builder.ToString();
+        }
+
+        private static int CountCommonIndentationChars(StringReader reader)
+        {
+            var maxCount = 0;
+            var currentLineCount = 0;
+
+            while (true)
+            {
+                switch (reader.Read())
+                {
+                    case -1:
+                        return maxCount;
+
+                    case ' ':
+                        currentLineCount++;
+                        break;
+
+                    case '\t':
+                        throw new NotImplementedException("Tabs");
+
+                    case '\r':
+                    case '\n':
+                        currentLineCount = 0;
+                        break;
+
+                    default:
+                        if (currentLineCount == 0) return 0;
+
+                        if (maxCount == 0 || maxCount > currentLineCount)
+                            maxCount = currentLineCount;
+
+                        currentLineCount = 0;
+
+                        _ = reader.ReadLine();
+                        break;
+                }
+            }
+        }
+    }
+}

src/NUnit.TestAdapter.Tests.Acceptance/UtilsTests.cs

@@ -0,0 +1,59 @@
+using System;
+using NUnit.Framework;
+
+namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance
+{
+    public static class UtilsTests
+    {
+        [Test]
+        public static void RemoveIndent_removes_indentation_of_line_with_least_indentation_from_all_lines()
+        {
+            var result = Utils.RemoveIndent(string.Join(Environment.NewLine,
+                "    First line",
+                "  Second line",
+                "    "));
+
+            Assert.That(result, Is.EqualTo(string.Join(Environment.NewLine,
+                "  First line",
+                "Second line",
+                "  ")));
+        }
+
+        [Test]
+        public static void RemoveIndent_ignores_whitespace_lines()
+        {
+            var result = Utils.RemoveIndent(string.Join(Environment.NewLine,
+                "    First line",
+                " ",
+                "    Second line"));
+
+            Assert.That(result, Is.EqualTo(string.Join(Environment.NewLine,
+                "First line",
+                "",
+                "Second line")));
+        }
+
+        [Test]
+        public static void RemoveIndent_ignores_first_line_if_it_begins_without_indent()
+        {
+            var result = Utils.RemoveIndent(string.Join(Environment.NewLine,
+                "First line",
+                "    Second line"));
+
+            Assert.That(result, Is.EqualTo(string.Join(Environment.NewLine,
+                "First line",
+                "Second line")));
+        }
+
+        [Test]
+        public static void RemoveIndent_removes_first_line_if_is_empty()
+        {
+            var result = Utils.RemoveIndent(string.Join(Environment.NewLine,
+                "",
+                "    Second line"));
+
+            Assert.That(result, Is.EqualTo(string.Join(Environment.NewLine,
+                "Second line")));
+        }
+    }
+}

src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspace.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+
+namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
+{
+    [DebuggerDisplay("{Directory,nq}")]
+    public sealed class IsolatedWorkspace
+    {
+        private readonly List<string> projectPaths = new List<string>();
+
+        public string Directory { get; }
+
+        public IsolatedWorkspace(string directory)
+        {
+            Directory = directory;
+        }
+
+        public IsolatedWorkspace AddProject(string path, string contents)
+        {
+            AddFile(path, contents);
+            projectPaths.Add(path);
+            return this;
+        }
+
+        public IsolatedWorkspace AddFile(string path, string contents)
+        {
+            if (string.IsNullOrWhiteSpace(path))
+                throw new ArgumentException("File path must be specified.", nameof(path));
+
+            if (Path.IsPathRooted(path))
+                throw new ArgumentException("File path must not be rooted.", nameof(path));
+
+            File.WriteAllText(Path.Combine(Directory, path), Utils.RemoveIndent(contents));
+            return this;
+        }
+    }
+}

src/NUnit.TestAdapter.Tests.Acceptance/WorkspaceTools/IsolatedWorkspaceManager.cs

@@ -0,0 +1,89 @@
+using System;
+using System.IO;
+using System.Xml;
+
+namespace NUnit.VisualStudio.TestAdapter.Tests.Acceptance.WorkspaceTools
+{
+    public sealed class IsolatedWorkspaceManager : IDisposable
+    {
+        private readonly string workspaceDirectory;
+        private readonly StreamWriter reasonFile;
+        private bool keep;
+
+        public IsolatedWorkspaceManager(string reason, string baseDirectory, string testNupkgDirectory, string packageCachePath)
+        {
+            baseDirectory = Path.GetFullPath(baseDirectory);
+            testNupkgDirectory = Path.GetFullPath(testNupkgDirectory);
+            packageCachePath = Path.GetFullPath(packageCachePath);
+
+            Directory.CreateDirectory(baseDirectory);
+
+            workspaceDirectory = Utils.CreateUniqueDirectory(baseDirectory);
+
+            reasonFile = File.CreateText(Path.Combine(workspaceDirectory, "Reason.txt"));
+            reasonFile.WriteLine(reason);
+            reasonFile.WriteLine();
+            reasonFile.Flush();
+
+            WriteNuGetConfig(baseDirectory, testNupkgDirectory, packageCachePath);
+        }
+
+        public void Dispose()
+        {
+            reasonFile.Dispose();
+            if (!keep) Directory.Delete(workspaceDirectory, recursive: true);
+        }
+
+        public IsolatedWorkspace CreateWorkspace(string name)
+        {
+            return new IsolatedWorkspace(Utils.CreateUniqueDirectory(workspaceDirectory, name));
+        }
+
+        /// <summary>
+        /// Prevents the workspace directory from being deleted when <see cref="Dispose"/> is called.
+        /// </summary>
+        public void PreserveDirectory(string reason)
+        {
+            if (!keep)
+            {
+                keep = true;
+                reasonFile.WriteLine("Preserving workspace after cleanup, due to:");
+                reasonFile.WriteLine();
+            }
+
+            reasonFile.WriteLine(reason);
+            reasonFile.Flush();
+        }
+
+        private static void WriteNuGetConfig(string directory, string testNupkgDirectory, string packageCachePath)
+        {
+            using (var file = File.CreateText(Path.Combine(directory, "nuget.config")))
+            using (var writer = XmlWriter.Create(file, new XmlWriterSettings { Indent = true }))
+            {
+                writer.WriteStartElement("configuration");
+                writer.WriteStartElement("config");
+
+                writer.WriteStartElement("add");
+                writer.WriteAttributeString("key", "globalPackagesFolder");
+                writer.WriteAttributeString("value", packageCachePath);
+                writer.WriteEndElement();
+
+                writer.WriteEndElement();
+                writer.WriteStartElement("packageSources");
+
+                writer.WriteStartElement("add");
+                writer.WriteAttributeString("key", "Build script package output");
+                writer.WriteAttributeString("value", testNupkgDirectory);
+                writer.WriteEndElement();
+
+                writer.WriteStartElement("add");
+                writer.WriteAttributeString("key", "Pre-downloaded packages");
+                writer.WriteAttributeString("value", @"C:\Program Files\dotnet\sdk\NuGetFallbackFolder");
+                writer.WriteEndElement();
+
+                writer.WriteEndElement();
+                writer.WriteEndElement();
+            }
+        }
+    }
+}