[xharness] Bump to the latest version of Microsoft.DotNet.XHarness.iOS.Shared (1.0.0-prerelease.21527.2). (#12899)

* [xharness] Bump to the latest version of Microsoft.DotNet.XHarness.iOS.Shared (1.0.0-prerelease.21479.1).

* [xharness] Handle an exceptional condition when we fail to enumerate simulators.

* [xharness] Use new xharness API to select which simulators to use.

This also required bumping xharness.

* Bump again.

* Bump yet again.

* [xharness] Bump and fix according to API break.

* [xharness] Stop the listener before cancelling it.

Cancelling it doesn't do anything if the listener has connected, and that's
usually the case. This prevents subsequent logic from reading incomplete logs.

Co-authored-by: Manuel de la Pena <mandel@microsoft.com>

tests/xharness/AppRunner.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.DotNet.XHarness.Common.CLI;
 using Microsoft.DotNet.XHarness.Common.Execution;
 using Microsoft.DotNet.XHarness.Common.Logging;
 using Microsoft.DotNet.XHarness.Common.Utilities;
@@ -128,7 +129,7 @@ namespace Xharness {
 
 			var sims = simulatorsLoaderFactory.CreateLoader ();
 			await sims.LoadDevices (Logs.Create ($"simulator-list-{Harness.Helpers.Timestamp}.log", "Simulator list"), false, false);
-			(simulator, companionSimulator) = await sims.FindSimulators (target, MainLog);
+			(simulator, companionSimulator) = await sims.FindSimulators (target.GetTargetOs (false), MainLog);
 
 			return simulator != null;
 		}
@@ -366,8 +367,8 @@ namespace Xharness {
 
 				MainLog.WriteLine ("Starting test run");
 
-				await testReporter.CollectSimulatorResult (
+				var testRunResult = await processManager.ExecuteCommandAsync (args, MainLog, testReporterTimeout, cancellationToken: testReporter.CancellationToken);
-					processManager.ExecuteCommandAsync (args, MainLog, testReporterTimeout, cancellationToken: testReporter.CancellationToken));
+				await testReporter.CollectSimulatorResult (testRunResult);
 
 				// cleanup after us
 				if (EnsureCleanSimulatorState)
@@ -406,13 +407,13 @@ namespace Xharness {
 
 					// We need to check for MT1111 (which means that mlaunch won't wait for the app to exit).
 					var aggregatedLog = Log.CreateAggregatedLog (testReporter.CallbackLog, MainLog);
-					Task<ProcessExecutionResult> runTestTask = processManager.ExecuteCommandAsync (
+					ProcessExecutionResult runTestResults = await processManager.ExecuteCommandAsync (
 						args,
 						aggregatedLog,
 						testReporterTimeout,
 						cancellationToken: testReporter.CancellationToken);
 
-					await testReporter.CollectDeviceResult (runTestTask);
+					await testReporter.CollectDeviceResult (runTestResults);
 				} finally {
 					deviceLogCapturer.StopCapture ();
 					deviceSystemLog.Dispose ();
@@ -425,7 +426,10 @@ namespace Xharness {
 				}
 			}
 
-			listener.Cancel ();
+			if (!listener.StopAsync ().Wait (TimeSpan.FromSeconds (5))) {
+				MainLog.WriteLine ("Failed to stop listener within 5 seconds. Will cancel it.");
+				listener.Cancel ();
+			}
 			listener.Dispose ();
 
 			// close a tunnel if it was created

tests/xharness/Jenkins/ErrorKnowledgeBase.cs

@@ -8,17 +8,17 @@ using Microsoft.DotNet.XHarness.iOS.Shared;
 
 namespace Xharness.Jenkins {
 	public class ErrorKnowledgeBase : IErrorKnowledgeBase {
-		static readonly Dictionary<string, (string HumanMessage, string IssueLink)> testErrorMaps = new Dictionary<string, (string HumanMessage, string IssueLink)> {
+		static readonly Dictionary<string, KnownIssue> testErrorMaps = new Dictionary<string, KnownIssue> {
-			["error HE0038: Failed to launch the app"] = (HumanMessage: "HE0038", IssueLink: "https://github.com/xamarin/maccore/issues/581)"),
+			["error HE0038: Failed to launch the app"] = new KnownIssue ("HE0038", issueLink: "https://github.com/xamarin/maccore/issues/581)"),
-			["Couldn't establish a TCP connection with any of the hostnames"] = (HumanMessage: "Tcp Connection Error: Tests are reported as crashes when they succeeded.", IssueLink: "https://github.com/xamarin/maccore/issues/1741"),
+			["Couldn't establish a TCP connection with any of the hostnames"] = new KnownIssue ("Tcp Connection Error: Tests are reported as crashes when they succeeded.", issueLink: "https://github.com/xamarin/maccore/issues/1741"),
-			["BCLTests.TestRunner.Core.TcpTextWriter..ctor"] = (HumanMessage: "Tcp Connection Error: Tests are reported as crashes when they succeeded.", IssueLink: "https://github.com/xamarin/maccore/issues/1741"),
+			["BCLTests.TestRunner.Core.TcpTextWriter..ctor"] = new KnownIssue ("Tcp Connection Error: Tests are reported as crashes when they succeeded.", issueLink: "https://github.com/xamarin/maccore/issues/1741"),
 		};
 		
-		static readonly Dictionary<string, (string HumanMessage, string IssueLink)> buildErrorMaps = new Dictionary<string, (string HumanMessage, string IssueLink)> {
+		static readonly Dictionary<string, KnownIssue> buildErrorMaps = new Dictionary<string, KnownIssue> {
-			["error MT5210: Native linking failed, undefined symbol: ___multi3"] = (HumanMessage: "Undefined symbol ___multi3 on Release Mode.", IssueLink:"https://github.com/mono/mono/issues/18560"),
+			["error MT5210: Native linking failed, undefined symbol: ___multi3"] = new KnownIssue ("Undefined symbol ___multi3 on Release Mode.", issueLink: "https://github.com/mono/mono/issues/18560"),
 		};
 		
-		static bool TryFindErrors (IFileBackedLog? log, Dictionary<string, (string HumanMessage, string IssueLink)> errorMap, [NotNullWhen (true)] out (string HumanMessage, string? IssueLink)? failureMessage)
+		static bool TryFindErrors (IFileBackedLog? log, Dictionary<string, KnownIssue> errorMap, [NotNullWhen (true)] out KnownIssue? failureMessage)
 		{
 			failureMessage = null;
 			if (log == null) {
@@ -45,13 +45,13 @@ namespace Xharness.Jenkins {
 			return false;
 		}
 
-		public bool IsKnownBuildIssue (IFileBackedLog buildLog, [NotNullWhen (true)] out (string HumanMessage, string? IssueLink)? knownFailureMessage) => 
+		public bool IsKnownBuildIssue (IFileBackedLog buildLog, [NotNullWhen (true)] out KnownIssue? knownFailureMessage) => 
 			TryFindErrors (buildLog, buildErrorMaps, out knownFailureMessage);
 
-		public bool IsKnownTestIssue (IFileBackedLog runLog, [NotNullWhen (true)] out (string HumanMessage, string? IssueLink)? knownFailureMessage) =>
+		public bool IsKnownTestIssue (IFileBackedLog runLog, [NotNullWhen (true)] out KnownIssue? knownFailureMessage) =>
 			TryFindErrors (runLog, testErrorMaps, out knownFailureMessage);
 
-		public bool IsKnownInstallIssue (IFileBackedLog installLog, [NotNullWhen (true)] out (string HumanMessage, string? IssueLink)? knownFailureMessage)
+		public bool IsKnownInstallIssue (IFileBackedLog installLog, [NotNullWhen (true)] out KnownIssue? knownFailureMessage)
 		{
 			// nothing yet that we are aware of
 			knownFailureMessage = null;

tests/xharness/Jenkins/Jenkins.cs

@@ -100,7 +100,7 @@ namespace Xharness.Jenkins {
 			this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager));
 			this.TunnelBore = tunnelBore ?? throw new ArgumentNullException (nameof (tunnelBore));
 			Harness = harness ?? throw new ArgumentNullException (nameof (harness));
-			Simulators = new SimulatorLoader (processManager);
+			Simulators = new SimulatorLoader (processManager, new SimulatorSelector ());
 			Devices = new HardwareDeviceLoader (processManager);
 			testSelector = new TestSelector (this, processManager, new GitHub (harness, processManager));
 			testVariationsFactory = new TestVariationsFactory (this, processManager);

tests/xharness/Jenkins/Reports/HtmlReportWriter.cs

@@ -310,7 +310,7 @@ namespace Xharness.Jenkins.Reports {
 					writer.Write ($"<span id='button_container2_{groupId}' class='expander' onclick='javascript: toggleContainerVisibility2 (\"{groupId}\");'>{defaultExpander}</span>");
 					writer.Write ($"<span id='x{id_counter++}' class='p1 autorefreshable' onclick='javascript: toggleContainerVisibility2 (\"{groupId}\");'>{group.Key}{RenderTextStates (group)}</span>");
 					if (jenkins.IsServerMode) {
-						var groupIds = string.Join (",", group.Where ((v) => !v.KnownFailure.HasValue).Select ((v) => v.ID.ToString ()));
+						var groupIds = string.Join (",", group.Where ((v) => v.KnownFailure is null).Select ((v) => v.ID.ToString ()));
 						writer.Write ($" <span class='runall'><a href='javascript: runtest (\"{groupIds}\");'>Run all</a> <a href='javascript: buildtest (\"{groupIds}\");'>Build all</a></span>");
 					}
 					writer.WriteLine ("</div>");
@@ -331,7 +331,7 @@ namespace Xharness.Jenkins.Reports {
 						writer.Write ($"<span id='button_container2_{modeGroupId}' class='expander' onclick='javascript: toggleContainerVisibility2 (\"{modeGroupId}\");'>{defaultExpander}</span>");
 						writer.Write ($"<span id='x{id_counter++}' class='p2 autorefreshable' onclick='javascript: toggleContainerVisibility2 (\"{modeGroupId}\");'>{modeGroup.Key}{RenderTextStates (modeGroup)}</span>");
 						if (jenkins.IsServerMode) {
-							var modeGroupIds = string.Join (",", modeGroup.Where ((v) => !v.KnownFailure.HasValue).Select ((v) => v.ID.ToString ()));
+							var modeGroupIds = string.Join (",", modeGroup.Where ((v) => v.KnownFailure is null).Select ((v) => v.ID.ToString ()));
 							writer.Write ($" <span class='runall'><a href='javascript: runtest (\"{modeGroupIds}\");'>Run all</a> <a href='javascript: buildtest (\"{modeGroupIds}\");'>Build all</a></span>");
 						}
 						writer.WriteLine ("</div>");
@@ -363,8 +363,8 @@ namespace Xharness.Jenkins.Reports {
 						writer.Write ($"<span id='button_{log_id}' class='expander' onclick='javascript: toggleLogVisibility (\"{log_id}\");'>{defaultExpander}</span>");
 						// we have a very common error we want to make this easier for the person that is dealing with the results
 						var knownFailure = string.Empty;
-						if (test.KnownFailure.HasValue)
+						if (test.KnownFailure is not null)
-							knownFailure = $" <a href='{test.KnownFailure.Value.IssueLink}'>{test.KnownFailure.Value.HumanMessage}</a>";
+							knownFailure = $" <a href='{test.KnownFailure.IssueLink}'>{test.KnownFailure.HumanMessage}</a>";
 						writer.Write ($"<span id='x{id_counter++}' class='p3 autorefreshable' onclick='javascript: toggleLogVisibility (\"{log_id}\");'>{title} (<span style='color: {test.GetTestColor ()}'>{state}{knownFailure}</span>{buildOnly}) </span>");
 						if (jenkins.IsServerMode) {
 							writer.Write ($" <span id='x{id_counter++}' class='autorefreshable'>");
@@ -385,8 +385,8 @@ namespace Xharness.Jenkins.Reports {
 						if (testAssemblies.Any ())
 							writer.WriteLine ($"Test assemblies:<br/>- {String.Join ("<br/>- ", testAssemblies)}<br />");
 
-						if (test.KnownFailure.HasValue)
+						if (test.KnownFailure is not null)
-							writer.WriteLine ($"Known failure: <a href='{test.KnownFailure.Value.IssueLink}'>{test.KnownFailure.Value.HumanMessage}</a> <br />");
+							writer.WriteLine ($"Known failure: <a href='{test.KnownFailure.IssueLink}'>{test.KnownFailure.HumanMessage}</a> <br />");
 
 						if (!string.IsNullOrEmpty (test.FailureMessage)) {
 							var msg = test.FailureMessage.AsHtml ();

tests/xharness/Jenkins/Reports/MarkdownReportWriter.cs

@@ -27,8 +27,8 @@ namespace Xharness.Jenkins.Reports {
 						writer.Write ($": {test.ExecutionResult}");
 						if (!string.IsNullOrEmpty (test.FailureMessage))
 							writer.Write ($" ({test.FailureMessage})");
-						if (test.KnownFailure.HasValue)
+						if (test.KnownFailure is not null)
-							writer.Write ($" Known issue: [{test.KnownFailure.Value.HumanMessage}]({test.KnownFailure.Value.IssueLink})");
+							writer.Write ($" Known issue: [{test.KnownFailure.HumanMessage}]({test.KnownFailure.IssueLink})");
 						writer.WriteLine ();
 					}
 					continue;

tests/xharness/Jenkins/RunSimulatorTasksFactory.cs

@@ -137,7 +137,7 @@ namespace Xharness.Jenkins {
 			}
 
 			for (int i = 0; i < targets.Length; i++) {
-				var sims = jenkins.Simulators.SelectDevices (targets [i], jenkins.SimulatorLoadLog, false);
+				var sims = jenkins.Simulators.SelectDevices (targets [i].GetTargetOs (false), jenkins.SimulatorLoadLog, false);
 				runtasks.Add (new RunSimulatorTask (
 					jenkins: jenkins,
 					simulators: jenkins.Simulators,

tests/xharness/Jenkins/TestData.cs

@@ -9,7 +9,7 @@ namespace Xharness.Jenkins {
 		public string Variation;
 		public string MTouchExtraArgs;
 		public string MonoBundlingExtraArgs; // mmp
-		public (string HumanMessage, string IssueLink)? KnownFailure;
+		public KnownIssue KnownFailure;
 		public bool Debug;
 		public bool Profiling;
 		public string LinkMode;

tests/xharness/Jenkins/TestTasks/ITestTask.cs

@@ -36,7 +36,7 @@ namespace Xharness.Jenkins.TestTasks {
 		bool BuildOnly { get; set; }
 		bool Ignored { get; set; }
 
-		(string HumanMessage, string IssueLink)? KnownFailure { get; set; }
+		KnownIssue KnownFailure { get; set; }
 		string ProjectConfiguration { get; set; }
 		string ProjectPlatform { get; set; }
 		string ProjectFile { get; }

tests/xharness/Jenkins/TestTasks/MSBuild.cs

@@ -48,7 +48,7 @@ namespace Xharness.Jenkins.TestTasks {
 			this.errorKnowledgeBase = errorKnowledgeBase ?? throw new ArgumentNullException (nameof (errorKnowledgeBase));
 		}
 
-		public async Task<(TestExecutingResult ExecutionResult, (string HumanMessage, string IssueLink)? KnownFailure)> ExecuteAsync (
+		public async Task<(TestExecutingResult ExecutionResult, KnownIssue KnownFailure)> ExecuteAsync (
 			string projectPlatform,
 			string projectConfiguration,
 			string projectFile,
@@ -58,7 +58,7 @@ namespace Xharness.Jenkins.TestTasks {
 			ILog mainLog)
 		{
 			BuildLog = buildLog;
-			(TestExecutingResult ExecutionResult, (string HumanMessage, string IssueLink)? KnownFailure) result = (TestExecutingResult.NotStarted, ((string HumanMessage, string IssueLink)?) null);
+			(TestExecutingResult ExecutionResult, KnownIssue KnownFailure) result = (TestExecutingResult.NotStarted, (KnownIssue) null);
 			var restoreResult = await RestoreNugetsAsync (buildLog, resource);
 			if ((restoreResult & TestExecutingResult.Failed) == TestExecutingResult.Failed) {
 				BuildLog.WriteLine ($"Failed to restore nugets: {restoreResult}");

tests/xharness/Jenkins/TestTasks/RunSimulator.cs

@@ -60,15 +60,19 @@ namespace Xharness.Jenkins.TestTasks {
 			if (asyncEnumerable != null)
 				await asyncEnumerable.ReadyTask;
 
-			if (!testTask.Candidates.Any ()) {
+			try {
+				if (!testTask.Candidates.Any ()) {
+					testTask.ExecutionResult = TestExecutingResult.DeviceNotFound;
+					testTask.FailureMessage = "No applicable devices found.";
+				} else {
+					testTask.Device = testTask.Candidates.First ();
+					if (testTask.Platform == TestPlatform.watchOS)
+						testTask.CompanionDevice = simulators.FindCompanionDevice (simulatorLoadLog, testTask.Device);
+				}
+			} catch (Exception e) {
 				testTask.ExecutionResult = TestExecutingResult.DeviceNotFound;
-				testTask.FailureMessage = "No applicable devices found.";
+				testTask.FailureMessage = $"No applicable devices found ({e.Message})";
-			} else {
-				testTask.Device = testTask.Candidates.First ();
-				if (testTask.Platform == TestPlatform.watchOS)
-					testTask.CompanionDevice = simulators.FindCompanionDevice (simulatorLoadLog, testTask.Device);
 			}
-
 		}
 
 		public async Task SelectSimulatorAsync ()

tests/xharness/Jenkins/TestTasks/RunTest.cs

@@ -67,7 +67,7 @@ namespace Xharness.Jenkins.TestTasks {
 					testTask.ExecutionResult = TestExecutingResult.BuildFailure;
 				}
 				testTask.FailureMessage = BuildTask.FailureMessage;
-				if (BuildTask.KnownFailure.HasValue)
+				if (BuildTask.KnownFailure is not null)
 					testTask.KnownFailure = BuildTask.KnownFailure;
 				if (generateXmlFailures && BuildTask.BuildLog != null && File.Exists (BuildTask.BuildLog.FullPath)) {
 					try {

tests/xharness/Jenkins/TestTasks/TestTask.cs

@@ -31,7 +31,7 @@ namespace Xharness.Jenkins.TestTasks {
 
 		public int ID { get; private set; }
 		public bool BuildOnly { get; set; }
-		public (string HumanMessage, string IssueLink)? KnownFailure { get; set; }
+		public KnownIssue KnownFailure { get; set; }
 		public string ProjectConfiguration { get; set; }
 		public string ProjectPlatform { get; set; }
 

tests/xharness/Jenkins/TestVariationsFactory.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.DotNet.XHarness.Common.Execution;
+using Microsoft.DotNet.XHarness.iOS.Shared;
 using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
 using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
 using Xharness.Jenkins.TestTasks;
@@ -92,10 +93,10 @@ namespace Xharness.Jenkins {
 						yield return new TestData { Variation = "Debug: SGenConc", MTouchExtraArgs = "", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, EnableSGenConc = true};
 					if (supports_interpreter) {
 						if (supports_debug) {
-							yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = (HumanMessage: "#1683", IssueLink: "https://github.com/xamarin/maccore/issues/1683") };
+							yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = new KnownIssue ("#1683", issueLink: "https://github.com/xamarin/maccore/issues/1683") };
-							yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = (HumanMessage:"#1682", IssueLink:"https://github.com/xamarin/maccore/issues/1682") };
+							yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = new KnownIssue ("#1682", issueLink: "https://github.com/xamarin/maccore/issues/1682") };
 						}
-						yield return new TestData { Variation = "Release (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = false, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = (HumanMessage:"#1682", IssueLink:"https://github.com/xamarin/maccore/issues/1682") };
+						yield return new TestData { Variation = "Release (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = false, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = new KnownIssue ("#1682", issueLink: "https://github.com/xamarin/maccore/issues/1682") };
 					}
 					break;
 				}
@@ -117,7 +118,7 @@ namespace Xharness.Jenkins {
 						yield return new TestData {
 							Variation = $"Debug ({test.Platform.GetSimulatorMinVersion ()})",
 							Debug = true,
-							Candidates = jenkins.Simulators.SelectDevices (target, jenkins.SimulatorLoadLog, true),
+							Candidates = jenkins.Simulators.SelectDevices (target.GetTargetOs (true), jenkins.SimulatorLoadLog, true),
 							Ignored = ignore ?? !jenkins.IncludeOldSimulatorTests, 
 						};
 					break;
@@ -189,7 +190,7 @@ namespace Xharness.Jenkins {
 					if (task.TestProject.IsDotNetProject)
 						variation += " [dotnet]";
 
-					if (known_failure.HasValue)
+					if (known_failure is not null)
 						ignored = true;
 
 					var clone = task.TestProject.Clone ();

tests/xharness/SimulatorLoaderFactory.cs

@@ -1,4 +1,5 @@
 using System;
+using Microsoft.DotNet.XHarness.iOS.Shared;
 using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
 using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
 
@@ -16,6 +17,20 @@ namespace Xharness {
 			this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager));
 		}
 
-		public ISimulatorLoader CreateLoader () => new SimulatorLoader (processManager);
+		public ISimulatorLoader CreateLoader () => new SimulatorLoader (processManager, new SimulatorSelector ());
+	}
+
+	public class SimulatorSelector : DefaultSimulatorSelector {
+		public override string GetDeviceType (TestTargetOs target, bool minVersion)
+		{
+			return target.Platform switch {
+				TestTarget.Simulator_iOS => "com.apple.CoreSimulator.SimDeviceType.iPhone-5s",
+				TestTarget.Simulator_iOS32 => "com.apple.CoreSimulator.SimDeviceType.iPhone-5s",
+				TestTarget.Simulator_iOS64 => "com.apple.CoreSimulator.SimDeviceType." + (minVersion ? "iPhone-6" : "iPhone-X"),
+				TestTarget.Simulator_tvOS => "com.apple.CoreSimulator.SimDeviceType.Apple-TV-1080p",
+				TestTarget.Simulator_watchOS => "com.apple.CoreSimulator.SimDeviceType.Apple-Watch-Series-3-38mm",
+				_ => throw new Exception (string.Format ("Invalid simulator target: {0}", target))
+			};
+		}
 	}
 }

tests/xharness/TestTargetExtensions.cs

@@ -25,5 +25,18 @@ namespace Xharness {
 				throw new NotImplementedException (platform.ToString ());
 			}
 		}
+
+		public static TestTargetOs GetTargetOs (this TestTarget target, bool minVersion)
+		{
+			return target switch {
+				TestTarget.Simulator_iOS32 => new TestTargetOs (target, minVersion ? SdkVersions.MiniOSSimulator : "10.3"),
+				TestTarget.Simulator_iOS64 => new TestTargetOs (target, minVersion ? SdkVersions.MiniOSSimulator : SdkVersions.MaxiOSSimulator),
+				TestTarget.Simulator_iOS => new TestTargetOs (target, minVersion ? SdkVersions.MiniOSSimulator : SdkVersions.MaxiOSSimulator),
+				TestTarget.Simulator_tvOS => new TestTargetOs (target, minVersion ? SdkVersions.MinTVOSSimulator : SdkVersions.MaxTVOSSimulator),
+				TestTarget.Simulator_watchOS => new TestTargetOs (target, minVersion ? SdkVersions.MinWatchOSSimulator : SdkVersions.MaxWatchOSSimulator),
+				_ => throw new Exception (string.Format ("Invalid simulator target: {0}", target))
+			};
+
+		}
 	}
 }

tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs

@@ -6,6 +6,7 @@ using System.Reflection;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.DotNet.XHarness.Common.CLI;
 using Microsoft.DotNet.XHarness.Common.Execution;
 using Microsoft.DotNet.XHarness.Common.Logging;
 using Microsoft.DotNet.XHarness.Common.Utilities;
@@ -261,6 +262,7 @@ namespace Xharness.Tests {
 				mainLog.Object,
 				TimeSpan.FromHours (1),
 				null,
+				5,
 				cancellationToken));
 		}
 
@@ -304,6 +306,7 @@ namespace Xharness.Tests {
 				mainLog.Object,
 				TimeSpan.FromMinutes (1),
 				null,
+				5,
 				null));
 		}
 
@@ -320,7 +323,7 @@ namespace Xharness.Tests {
 
 			// Mock finding simulators
 			simulators
-				.Setup (x => x.LoadDevices (It.IsAny<ILog> (), false, false, false))
+				.Setup (x => x.LoadDevices (It.IsAny<ILog> (), false, false, false, true))
 				.Returns (Task.FromResult(true));
 
 			string simulatorLogPath = Path.Combine (Path.GetTempPath (), "simulator-logs");
@@ -403,7 +406,7 @@ namespace Xharness.Tests {
 
 			// Mock finding simulators
 			simulators
-				.Setup (x => x.LoadDevices (It.IsAny<ILog> (), false, false, false))
+				.Setup (x => x.LoadDevices (It.IsAny<ILog> (), false, false, false, true))
 				.Returns (Task.FromResult(true));
 
 			string simulatorLogPath = Path.Combine (Path.GetTempPath (), "simulator-logs");
@@ -460,6 +463,7 @@ namespace Xharness.Tests {
 					mainLog.Object,
 					TimeSpan.FromMinutes (harness.Timeout * 2),
 					null,
+					5,
 					It.IsAny<CancellationToken> ()))
 				.ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 });
 
@@ -505,7 +509,7 @@ namespace Xharness.Tests {
 			simulators.VerifyAll ();
 
 			captureLog.Verify (x => x.StartCapture (), Times.AtLeastOnce);
-			captureLog.Verify (x => x.StopCapture (), Times.AtLeastOnce);
+			captureLog.Verify (x => x.StopCapture (null), Times.AtLeastOnce);
 
 			// When ensureCleanSimulatorState == true
 			simulator.Verify (x => x.PrepareSimulator (It.IsAny<ILog> (), appName));
@@ -624,6 +628,7 @@ namespace Xharness.Tests {
 					It.IsAny<ILog> (),
 					TimeSpan.FromMinutes (harness.Timeout * 2),
 					null,
+					5,
 					It.IsAny<CancellationToken> ()))
 				.ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 });
 
@@ -732,6 +737,7 @@ namespace Xharness.Tests {
 					It.IsAny<ILog> (),
 					TimeSpan.FromMinutes (harness.Timeout * 2),
 					null,
+					5,
 					It.IsAny<CancellationToken> ()))
 				.ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 });
 

tests/xharness/Xharness.Tests/Xharness.Tests.csproj

@@ -1,7 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <Import Project="..\..\..\packages\NUnit3TestAdapter.3.16.1\build\net35\NUnit3TestAdapter.props" Condition="Exists('..\..\..\packages\NUnit3TestAdapter.3.16.1\build\net35\NUnit3TestAdapter.props')" />
-  <Import Project="..\..\..\packages\NUnit.3.12.0\build\NUnit.props" Condition="Exists('..\..\..\packages\NUnit.3.12.0\build\NUnit.props')" />
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@@ -31,114 +29,10 @@
     <LangVersion>8.0</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Castle.Core, Version=4.0.0.0, Culture=neutral, PublicKeyToken=407dd0808d44fbdc, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Castle.Core.4.4.0\lib\net45\Castle.Core.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Bcl.AsyncInterfaces, Version=5.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Bcl.AsyncInterfaces.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Bcl.AsyncInterfaces.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.DotNet.XHarness.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.DotNet.XHarness.Common.1.0.0-prerelease.20621.4\lib\netstandard2.0\Microsoft.DotNet.XHarness.Common.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.DotNet.XHarness.iOS.Shared, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.DotNet.XHarness.iOS.Shared.1.0.0-prerelease.20621.4\lib\netstandard2.0\Microsoft.DotNet.XHarness.iOS.Shared.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Configuration, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Configuration.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Configuration.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Configuration.Abstractions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Configuration.Abstractions.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Configuration.Abstractions.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Configuration.Binder, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Configuration.Binder.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Configuration.Binder.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.DependencyInjection, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.DependencyInjection.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.DependencyInjection.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.DependencyInjection.Abstractions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.DependencyInjection.Abstractions.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.DependencyInjection.Abstractions.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Logging, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Logging.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Logging.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Logging.Abstractions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Logging.Abstractions.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Logging.Configuration, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Logging.Configuration.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Logging.Configuration.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Logging.Console, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Logging.Console.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Logging.Console.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Options, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Options.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Options.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Options.ConfigurationExtensions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Options.ConfigurationExtensions.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Options.ConfigurationExtensions.dll</HintPath>
-    </Reference>
-    <Reference Include="Microsoft.Extensions.Primitives, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Extensions.Primitives.5.0.0-rc.1.20451.14\lib\net461\Microsoft.Extensions.Primitives.dll</HintPath>
-    </Reference>
-    <Reference Include="Mono.Cecil, Version=0.11.2.0, Culture=neutral, PublicKeyToken=50cebf1cceb9d05e, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Mono.Cecil.0.11.2\lib\net40\Mono.Cecil.dll</HintPath>
-    </Reference>
-    <Reference Include="Mono.Cecil.Mdb, Version=0.11.2.0, Culture=neutral, PublicKeyToken=50cebf1cceb9d05e, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Mono.Cecil.0.11.2\lib\net40\Mono.Cecil.Mdb.dll</HintPath>
-    </Reference>
-    <Reference Include="Mono.Cecil.Pdb, Version=0.11.2.0, Culture=neutral, PublicKeyToken=50cebf1cceb9d05e, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Mono.Cecil.0.11.2\lib\net40\Mono.Cecil.Pdb.dll</HintPath>
-    </Reference>
-    <Reference Include="Mono.Cecil.Rocks, Version=0.11.2.0, Culture=neutral, PublicKeyToken=50cebf1cceb9d05e, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Mono.Cecil.0.11.2\lib\net40\Mono.Cecil.Rocks.dll</HintPath>
-    </Reference>
-    <Reference Include="Mono.Options, Version=6.0.0.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Mono.Options.6.6.0.161\lib\net40\Mono.Options.dll</HintPath>
-    </Reference>
-    <Reference Include="Moq, Version=4.13.0.0, Culture=neutral, PublicKeyToken=69f491c39445e920, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Moq.4.13.1\lib\net45\Moq.dll</HintPath>
-    </Reference>
-    <Reference Include="nunit.framework, Version=3.12.0.0, Culture=neutral, PublicKeyToken=2638cd05610744eb, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\NUnit.3.12.0\lib\net45\nunit.framework.dll</HintPath>
-    </Reference>
-    <Reference Include="System" />
-    <Reference Include="System.Buffers, Version=4.0.3.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Buffers.4.5.1\lib\net461\System.Buffers.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Diagnostics.DiagnosticSource, Version=5.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Diagnostics.DiagnosticSource.5.0.0-rc.1.20451.14\lib\net46\System.Diagnostics.DiagnosticSource.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Numerics" />
-    <Reference Include="System.Numerics.Vectors, Version=4.1.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=5.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Runtime.CompilerServices.Unsafe.5.0.0-rc.1.20451.14\lib\net45\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Runtime.InteropServices.RuntimeInformation, Version=4.0.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Runtime.InteropServices.RuntimeInformation.4.3.0\lib\net45\System.Runtime.InteropServices.RuntimeInformation.dll</HintPath>
-      <Private>True</Private>
-      <Private>True</Private>
-    </Reference>
-    <Reference Include="System.Text.Encodings.Web, Version=5.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Text.Encodings.Web.5.0.0-rc.1.20451.14\lib\net461\System.Text.Encodings.Web.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Text.Json, Version=5.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Text.Json.5.0.0-rc.1.20451.14\lib\net461\System.Text.Json.dll</HintPath>
-    </Reference>
-    <Reference Include="System.Threading.Tasks.Extensions, Version=4.2.0.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll</HintPath>
-    </Reference>
-    <Reference Include="System.ValueTuple, Version=4.0.3.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\System.ValueTuple.4.5.0\lib\net47\System.ValueTuple.dll</HintPath>
-    </Reference>
     <Reference Include="System.Web" />
-    <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Xml" />
-    <Reference Include="System.Runtime.Serialization" />
+    <PackageReference Include="Moq" Version="4.13.0.0" />
-    <Reference Include="System.Configuration" />
+    <PackageReference Include="NUnit" Version="3.12.0" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Jenkins\ErrorKnowledgeBaseTests.cs" />
@@ -189,11 +83,4 @@
     </ProjectReference>
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
-  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
-    <PropertyGroup>
-      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
-    </PropertyGroup>
-    <Error Condition="!Exists('..\..\..\packages\NUnit.3.12.0\build\NUnit.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\NUnit.3.12.0\build\NUnit.props'))" />
-    <Error Condition="!Exists('..\..\..\packages\NUnit3TestAdapter.3.16.1\build\net35\NUnit3TestAdapter.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\packages\NUnit3TestAdapter.3.16.1\build\net35\NUnit3TestAdapter.props'))" />
-  </Target>
 </Project>

tests/xharness/xharness.csproj

@@ -22,7 +22,7 @@
     <WarningLevel>4</WarningLevel>
     <ConsolePause>false</ConsolePause>
     <PlatformTarget>x64</PlatformTarget>
-    <LangVersion>8.0</LangVersion>
+    <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
     <DebugType>full</DebugType>
@@ -31,7 +31,7 @@
     <WarningLevel>4</WarningLevel>
     <Externalconsole>true</Externalconsole>
     <OutputPath>bin\Release</OutputPath>
-    <LangVersion>8.0</LangVersion>
+    <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(RunConfiguration)' == 'Default' ">
     <StartAction>Project</StartAction>
@@ -62,13 +62,10 @@
     <Reference Include="System.Web" />
     <Reference Include="Mono.Posix" />
     <Reference Include="System.Runtime.Serialization" />
-    <Reference Include="mscorlib" />
     <Reference Include="System.Xml.Linq" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.DotNet.XHarness.iOS.Shared">
+    <PackageReference Include="Microsoft.DotNet.XHarness.iOS.Shared" Version="1.0.0-prerelease.21527.2" />
-      <Version>1.0.0-branch-xamarin-xcode-12.5.21168.1</Version>
-    </PackageReference>
     <PackageReference Include="Mono.Cecil" Version="0.11.4" />
     <PackageReference Include="Mono.Options" Version="6.6.0.161" />
   </ItemGroup>