[xharness] Add support for running the generator tests in .NET mode.

This also meant adding support for running .NET tests in xharness. Some
refactoring was done to extract common code to shared members, in order to
avoid duplicating a lot of code.

tests/xharness/Jenkins/Jenkins.cs

@@ -740,6 +740,7 @@ namespace Xharness.Jenkins
 				"src/generator.cs",
 				"src/generator-",
 				"src/Makefile.generator",
+				"tests/bgen",
 				"tests/generator",
 				"tests/common",
 			};
@@ -1093,11 +1094,28 @@ namespace Xharness.Jenkins
 				TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "generator", "generator-tests.csproj"))),
 				Platform = TestPlatform.iOS,
 				TestName = "Generator tests",
+				Mode = "NUnit",
 				Timeout = TimeSpan.FromMinutes (10),
 				Ignored = !IncludeBtouch,
 			};
 			Tasks.Add (runGenerator);
 
+			var buildDotNetGenerator = new DotNetBuildTask {
+				Jenkins = this,
+				TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "bgen", "bgen-tests.csproj"))),
+				SpecifyPlatform = false,
+				SpecifyConfiguration = false,
+				Platform = TestPlatform.iOS,
+			};
+			var runDotNetGenerator = new DotNetTestTask (buildDotNetGenerator) {
+				TestProject = buildDotNetGenerator.TestProject,
+				Platform = TestPlatform.iOS,
+				TestName = "Generator tests",
+				Mode = ".NET",
+				Ignored = !IncludeBtouch,
+			};
+			Tasks.Add (runDotNetGenerator);
+
 			var run_mmp = new MakeTask
 			{
 				Jenkins = this,
@@ -1748,6 +1766,7 @@ namespace Xharness.Jenkins
 			var allNUnitTasks = new List<NUnitExecuteTask> ();
 			var allMakeTasks = new List<MakeTask> ();
 			var allDeviceTasks = new List<RunDeviceTask> ();
+			var allDotNetTestTasks = new List<DotNetTestTask> ();
 			foreach (var task in Tasks) {
 				var aggregated = task as AggregatedRunSimulatorTask;
 				if (aggregated != null) {
@@ -1779,6 +1798,11 @@ namespace Xharness.Jenkins
 					continue;
 				}
 
+				if (task is DotNetTestTask dotnet) {
+					allDotNetTestTasks.Add (dotnet);
+					continue;
+				}
+
 				throw new NotImplementedException ();
 			}
 
@@ -1789,6 +1813,7 @@ namespace Xharness.Jenkins
 				allTasks.AddRange (allNUnitTasks);
 				allTasks.AddRange (allMakeTasks);
 				allTasks.AddRange (allDeviceTasks);
+				allTasks.AddRange (allDotNetTestTasks);
 			}
 
 			var failedTests = allTasks.Where ((v) => v.Failed);

tests/xharness/Jenkins/TestTasks/DotNetBuildTask.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+
+namespace Xharness.Jenkins.TestTasks {
+	class DotNetBuildTask : MSBuildTask {
+		protected override string ToolName {
+			get { return "dotnet"; }
+		}
+
+		protected override List<string> ToolArguments {
+			get {
+				var args = base.ToolArguments;
+				// 'dotnet build' takes almost the same arguments as 'msbuild', so just massage a little bit.
+				args.Remove ("--");
+				args.Insert (0, "build");
+				return args;
+			}
+		}
+	}
+}

tests/xharness/Jenkins/TestTasks/DotNetTestTask.cs

@@ -0,0 +1,36 @@
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Threading.Tasks;
+using Xamarin.Utils;
+using Xharness.Execution;
+using Xharness.Logging;
+
+namespace Xharness.Jenkins.TestTasks {
+	class DotNetTestTask : RunTestTask {
+		public DotNetTestTask (DotNetBuildTask build_task)
+			: base (build_task)
+		{
+		}
+
+		protected override async Task RunTestAsync ()
+		{
+			using (var resource = await NotifyAndAcquireDesktopResourceAsync ()) {
+				var trx = Logs.Create ($"results-{Timestamp}.trx", LogType.TrxLog.ToString ());
+				var html = Logs.Create ($"results-{Timestamp}.html", LogType.HtmlLog.ToString ());
+
+				var args = new List<string> {
+					"test",
+					BuildTask.ProjectFile,
+					"--results-directory:" + Logs.Directory,
+					"--logger:console;verbosity=detailed",
+					"--logger:trx;LogFileName=" + Path.GetFileName (trx.FullPath),
+					"--logger:html;LogFileName=" + Path.GetFileName (html.FullPath)
+				};
+
+				await ExecuteProcessAsync ("dotnet", args);
+			}
+		}
+	}
+}

tests/xharness/Jenkins/TestTasks/MSBuildTask.cs

@@ -13,26 +13,37 @@ namespace Xharness.Jenkins.TestTasks
 	{
 		public ILog BuildLog;
 
+		protected virtual string ToolName {
+			get { return Harness.XIBuildPath; }
+		}
+
+		protected virtual List<string> ToolArguments {
+			get {
+				var binlogPath = BuildLog.FullPath.Replace (".txt", ".binlog");
+
+				var args = new List<string> ();
+				args.Add ("--");
+				args.Add ("/verbosity:diagnostic");
+				args.Add ($"/bl:{binlogPath}");
+				if (SpecifyPlatform)
+					args.Add ($"/p:Platform={ProjectPlatform}");
+				if (SpecifyConfiguration)
+					args.Add ($"/p:Configuration={ProjectConfiguration}");
+				args.Add (ProjectFile);
+				return args;
+			}
+		}
+
 		protected override async Task ExecuteAsync ()
 		{
 			using (var resource = await NotifyAndAcquireDesktopResourceAsync ()) {
 				BuildLog = Logs.Create ($"build-{Platform}-{Timestamp}.txt", LogType.BuildLog.ToString ());
-				var binlogPath = BuildLog.FullPath.Replace (".txt", ".binlog");
 
 				await RestoreNugetsAsync (BuildLog, resource, useXIBuild: true);
 
 				using (var xbuild = new Process ()) {
-					xbuild.StartInfo.FileName = Harness.XIBuildPath;
+					xbuild.StartInfo.FileName = ToolName;
-					var args = new List<string> ();
+					xbuild.StartInfo.Arguments = StringUtils.FormatArguments (ToolArguments);
-					args.Add ("--");
-					args.Add ("/verbosity:diagnostic");
-					args.Add ($"/bl:{binlogPath}");
-					if (SpecifyPlatform)
-						args.Add ($"/p:Platform={ProjectPlatform}");
-					if (SpecifyConfiguration)
-						args.Add ($"/p:Configuration={ProjectConfiguration}");
-					args.Add (ProjectFile);
-					xbuild.StartInfo.Arguments = StringUtils.FormatArguments (args);
 					SetEnvironmentVariables (xbuild);
 					xbuild.StartInfo.EnvironmentVariables ["MSBuildExtensionsPath"] = null;
 					LogEvent (BuildLog, "Building {0} ({1})", TestName, Mode);

tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs

@@ -17,11 +17,8 @@ namespace Xharness.Jenkins.TestTasks
 	{
 		public string TestLibrary;
 		public string TestExecutable;
-		public string WorkingDirectory;
 		public bool ProduceHtmlReport = true;
 		public bool InProcess;
-		public TimeSpan Timeout = TimeSpan.FromMinutes (10);
-		IProcessManager ProcessManager { get; set; } = new ProcessManager ();
 
 		public NUnitExecuteTask (BuildToolTask build_task)
 			: base (build_task)
@@ -120,45 +117,22 @@ namespace Xharness.Jenkins.TestTasks
 				var xmlLog = Logs.CreateFile ($"log-{Timestamp}.xml", LogType.XmlLog.ToString ());
 				var log = Logs.Create ($"execute-{Timestamp}.txt", LogType.ExecutionLog.ToString ());
 				FindNUnitConsoleExecutable (log);
-				using (var proc = new Process ()) {
 
-					proc.StartInfo.WorkingDirectory = WorkingDirectory;
+				var args = new List<string> ();
-					proc.StartInfo.FileName = Harness.XIBuildPath;
+				args.Add ("-t");
-					var args = new List<string> ();
+				args.Add ("--");
-					args.Add ("-t");
+				args.Add (Path.GetFullPath (TestExecutable));
-					args.Add ("--");
+				args.Add (Path.GetFullPath (TestLibrary));
-					args.Add (Path.GetFullPath (TestExecutable));
+				if (IsNUnit3) {
-					args.Add (Path.GetFullPath (TestLibrary));
+					args.Add ("-result=" + xmlLog + ";format=nunit2");
-					if (IsNUnit3) {
+					args.Add ("--labels=All");
-						args.Add ("-result=" + xmlLog + ";format=nunit2");
+					if (InProcess)
-						args.Add ("--labels=All");
+						args.Add ("--inprocess");
-						if (InProcess)
+				} else {
-							args.Add ("--inprocess");
+					args.Add ("-xml=" + xmlLog);
-					} else {
+					args.Add ("-labels");
-						args.Add ("-xml=" + xmlLog);
-						args.Add ("-labels");
-					}
-					proc.StartInfo.Arguments = StringUtils.FormatArguments (args);
-					SetEnvironmentVariables (proc);
-					foreach (DictionaryEntry de in proc.StartInfo.EnvironmentVariables)
-						log.WriteLine ($"export {de.Key}={de.Value}");
-					Jenkins.MainLog.WriteLine ("Executing {0} ({1})", TestName, Mode);
-					if (!Harness.DryRun) {
-						ExecutionResult = TestExecutingResult.Running;
-						var result = await ProcessManager.RunAsync (proc, log, Timeout);
-						if (result.TimedOut) {
-							FailureMessage = $"Execution timed out after {Timeout.TotalMinutes} minutes.";
-							log.WriteLine (FailureMessage);
-							ExecutionResult = TestExecutingResult.TimedOut;
-						} else if (result.Succeeded) {
-							ExecutionResult = TestExecutingResult.Succeeded;
-						} else {
-							ExecutionResult = TestExecutingResult.Failed;
-							FailureMessage = $"Execution failed with exit code {result.ExitCode}";
-						}
-					}
-					Jenkins.MainLog.WriteLine ("Executed {0} ({1})", TestName, Mode);
 				}
+				await ExecuteProcessAsync (log, Harness.XIBuildPath, args);
 
 				if (ProduceHtmlReport) {
 					try {

tests/xharness/Jenkins/TestTasks/RunTestTask.cs

@@ -1,6 +1,11 @@
-using System.Collections.Generic;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
+using Xamarin.Utils;
+using Xharness.Execution;
 using Xharness.Logging;
 
 namespace Xharness.Jenkins.TestTasks
@@ -8,7 +13,10 @@ namespace Xharness.Jenkins.TestTasks
 	internal abstract class RunTestTask : TestTask
 	{
 		public readonly BuildToolTask BuildTask;
+		public TimeSpan Timeout = TimeSpan.FromMinutes (10);
 		public double TimeoutMultiplier { get; set; } = 1;
+		IProcessManager ProcessManager { get; } = new ProcessManager ();
+		public string WorkingDirectory;
 
 		public RunTestTask (BuildToolTask build_task)
 		{
@@ -105,5 +113,41 @@ namespace Xharness.Jenkins.TestTasks
 			base.Reset ();
 			BuildTask.Reset ();
 		}
+
+		protected Task ExecuteProcessAsync (string filename, List<string> arguments)
+		{ 
+			return ExecuteProcessAsync (null, filename, arguments);
+		}
+
+		protected async Task ExecuteProcessAsync (ILog log, string filename, List<string> arguments)
+		{
+			if (log == null)
+				log = Logs.Create ($"execute-{Timestamp}.txt", LogType.ExecutionLog.ToString ());
+
+			using var proc = new Process ();
+			proc.StartInfo.FileName = filename;
+			proc.StartInfo.Arguments = StringUtils.FormatArguments (arguments);
+			if (!string.IsNullOrEmpty (WorkingDirectory))
+				proc.StartInfo.WorkingDirectory = WorkingDirectory;
+			SetEnvironmentVariables (proc);
+			foreach (DictionaryEntry de in proc.StartInfo.EnvironmentVariables)
+				log.WriteLine ($"export {de.Key}={de.Value}");
+			Jenkins.MainLog.WriteLine ("Executing {0} ({1})", TestName, Mode);
+			if (!Harness.DryRun) {
+				ExecutionResult = TestExecutingResult.Running;
+				var result = await ProcessManager.RunAsync (proc, log, Timeout);
+				if (result.TimedOut) {
+					FailureMessage = $"Execution timed out after {Timeout.TotalMinutes} minutes.";
+					log.WriteLine (FailureMessage);
+					ExecutionResult = TestExecutingResult.TimedOut;
+				} else if (result.Succeeded) {
+					ExecutionResult = TestExecutingResult.Succeeded;
+				} else {
+					ExecutionResult = TestExecutingResult.Failed;
+					FailureMessage = $"Execution failed with exit code {result.ExitCode}";
+				}
+			}
+			Jenkins.MainLog.WriteLine ("Executed {0} ({1})", TestName, Mode);
+		}
 	}
 }

tests/xharness/Jenkins/TestTasks/RunXtroTask.cs

@@ -7,9 +7,6 @@ namespace Xharness.Jenkins.TestTasks
 {
 	class RunXtroTask : MacExecuteTask
 	{
-
-		public string WorkingDirectory;
-
 		public RunXtroTask (BuildToolTask build_task) : base (build_task)
 		{
 		}

tests/xharness/Logging/ILog.cs

@@ -14,6 +14,8 @@ namespace Xharness.Logging {
 		TestLog,
 		ExtensionTestLog,
 		ExecutionLog,
+		TrxLog,
+		HtmlLog,
 	}
 
 	public static class LogTypeExtensions {
@@ -37,6 +39,10 @@ namespace Xharness.Logging {
 				return "Extension test log";
 			case LogType.ExecutionLog:
 				return "Execution log";
+			case LogType.TrxLog:
+				return "Test log (trx)";
+			case LogType.HtmlLog:
+				return "Test log (html)";
 			default:
 				throw new ArgumentException ($"Unknown type for {nameof (type)}");
 			}

tests/xharness/xharness.csproj

@@ -10,6 +10,7 @@
     <RootNamespace>xharness</RootNamespace>
     <AssemblyName>xharness</AssemblyName>
     <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
+    <LangVersion>8.0</LangVersion>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
@@ -145,6 +146,8 @@
     <Compile Include="Listeners\SimpleListenerFactory.cs" />
     <Compile Include="Execution\IProcessManager.cs" />
     <Compile Include="Execution\ProcessManager.cs" />
+    <Compile Include="Jenkins\TestTasks\DotNetTestTask.cs" />
+    <Compile Include="Jenkins\TestTasks\DotNetBuildTask.cs" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="BCLTestImporter\" />