From 29e978e0be8fcb2008ba853a2b891b3cb26051a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 20 Mar 2020 19:18:28 +0100 Subject: [PATCH] [Harness] Add AppRunner.RunAsync unit tests (#8159) Co-authored-by: Premek Vysoky --- tests/xharness/AppRunner.cs | 57 +- tests/xharness/Harness.cs | 3 + .../Jenkins/TestTasks/RunDeviceTask.cs | 8 +- .../Jenkins/TestTasks/RunSimulatorTask.cs | 4 +- .../Xharness.Tests/Tests/AppRunnerTests.cs | 604 +++++++++++++++--- 5 files changed, 572 insertions(+), 104 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 6ce2bd147d..e361da0d85 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -64,11 +64,10 @@ namespace Xharness { readonly AppRunnerTarget target; readonly string projectFilePath; readonly IHarness harness; - readonly string configuration; + readonly string buildConfiguration; readonly string variation; readonly double timeoutMultiplier; readonly BuildToolTask buildTask; - readonly string logDirectory; string deviceName; string companionDeviceName; @@ -104,9 +103,9 @@ namespace Xharness { AppRunnerTarget target, IHarness harness, ILog mainLog, + ILogs logs, string projectFilePath, - string configuration, - string logDirectory = null, + string buildConfiguration, ISimulatorDevice [] simulators = null, string deviceName = null, string companionDeviceName = null, @@ -126,9 +125,8 @@ namespace Xharness { this.harness = harness ?? throw new ArgumentNullException (nameof (harness)); this.MainLog = mainLog ?? throw new ArgumentNullException (nameof (mainLog)); this.projectFilePath = projectFilePath ?? throw new ArgumentNullException (nameof (projectFilePath)); - this.logDirectory = logDirectory ?? harness.LogDirectory; - this.Logs = new Logs (this.logDirectory); - this.configuration = configuration; + this.Logs = logs ?? throw new ArgumentNullException (nameof (logs)); + this.buildConfiguration = buildConfiguration ?? throw new ArgumentNullException (nameof (buildConfiguration)); this.timeoutMultiplier = timeoutMultiplier; this.deviceName = deviceName; this.companionDeviceName = companionDeviceName; @@ -162,7 +160,7 @@ namespace Xharness { extension = extensionPointIdentifier.ParseFromNSExtensionPointIdentifier (); string appPath = Path.Combine (Path.GetDirectoryName (projectFilePath), - csproj.GetOutputPath (isSimulator ? "iPhoneSimulator" : "iPhone", configuration).Replace ('\\', Path.DirectorySeparatorChar), + csproj.GetOutputPath (isSimulator ? "iPhoneSimulator" : "iPhone", buildConfiguration).Replace ('\\', Path.DirectorySeparatorChar), appName + (extension != null ? ".appex" : ".app")); if (!Directory.Exists (appPath)) @@ -430,21 +428,16 @@ namespace Xharness { public async Task RunAsync () { - ILog listener_log = null; - ILog run_log = MainLog; - if (!isSimulator) FindDevice (); - var crashLogs = new Logs (Logs.Directory); - - ICrashSnapshotReporter crash_reports = snapshotReporterFactory.Create (MainLog, crashLogs, isDevice: !isSimulator, deviceName); - var args = new List (); + if (!string.IsNullOrEmpty (harness.XcodeRoot)) { args.Add ("--sdkroot"); args.Add (harness.XcodeRoot); } + for (int i = -1; i < harness.Verbosity; i++) args.Add ("-v"); args.Add ("-argument=-connection-mode"); @@ -487,7 +480,7 @@ namespace Xharness { args.Add ($"-setenv=NUNIT_HOSTNAME={ips}"); } - listener_log = Logs.Create ($"test-{mode.ToString().ToLower()}-{Helpers.Timestamp}.log", LogType.TestLog.ToString (), timestamp: !useXmlOutput); + var listener_log = Logs.Create ($"test-{mode.ToString().ToLower()}-{Helpers.Timestamp}.log", LogType.TestLog.ToString (), timestamp: !useXmlOutput); var (transport, listener, listenerTmpFile) = listenerFactory.Create (mode, MainLog, listener_log, isSimulator, true, useXmlOutput); args.Add ($"-argument=-app-arg:-transport:{transport}"); @@ -548,12 +541,16 @@ namespace Xharness { args.Add ("--disable-memory-limits"); var timeout = TimeSpan.FromMinutes (harness.Timeout * timeoutMultiplier); + + var crashLogs = new Logs (Logs.Directory); + ICrashSnapshotReporter crashReporter = snapshotReporterFactory.Create (MainLog, crashLogs, isDevice: !isSimulator, deviceName); + if (isSimulator) { if (!await FindSimulatorAsync ()) return 1; if (mode != RunMode.WatchOS) { - var stderr_tty = harness.GetStandardErrorTty(); + var stderr_tty = harness.GetStandardErrorTty (); if (!string.IsNullOrEmpty (stderr_tty)) { args.Add ($"--stdout={stderr_tty}"); args.Add ($"--stderr={stderr_tty}"); @@ -573,10 +570,11 @@ namespace Xharness { var logDescription = isCompanion ? LogType.CompanionSystemLog.ToString () : LogType.SystemLog.ToString (); var log = captureLogFactory.Create (Logs, - Path.Combine (logDirectory, sim.Name + ".log"), + Path.Combine (Logs.Directory, sim.Name + ".log"), sim.SystemLog, harness.Action != HarnessAction.Jenkins, logDescription); + log.StartCapture (); Logs.Add (log); systemLogs.Add (log); @@ -592,11 +590,17 @@ namespace Xharness { args.Add ($"--device=:v2:udid={simulator.UDID}"); - await crash_reports.StartCaptureAsync (); + await crashReporter.StartCaptureAsync (); MainLog.WriteLine ("Starting test run"); - var result = await processManager.ExecuteCommandAsync (harness.MlaunchPath, args, run_log, timeout, cancellation_token: cancellation_source.Token); + ILog run_log = MainLog; + var result = await processManager.ExecuteCommandAsync (harness.MlaunchPath, + args, + run_log, + timeout, + cancellation_token: cancellation_source.Token); + if (result.TimedOut) { timed_out = true; success = false; @@ -646,7 +650,7 @@ namespace Xharness { foreach (var log in systemLogs) log.StopCapture (); - + } else { MainLog.WriteLine ("*** Executing {0}/{1} on device '{2}' ***", AppInformation.AppName, mode, deviceName); @@ -655,15 +659,15 @@ namespace Xharness { } else { args.Add ("--wait-for-exit"); } - + AddDeviceName (args); var deviceSystemLog = Logs.Create ($"device-{deviceName}-{Helpers.Timestamp}.log", "Device log"); - var deviceLogCapturer = deviceLogCapturerFactory.Create (harness.HarnessLog,deviceSystemLog, deviceName); + var deviceLogCapturer = deviceLogCapturerFactory.Create (harness.HarnessLog, deviceSystemLog, deviceName); deviceLogCapturer.StartCapture (); try { - await crash_reports.StartCaptureAsync (); + await crashReporter.StartCaptureAsync (); MainLog.WriteLine ("Starting test run"); @@ -675,6 +679,7 @@ namespace Xharness { if (line?.Contains ("error MT1007") == true) launch_failure = true; }); + var runLog = Log.CreateAggregatedLog (callbackLog, MainLog); var timeoutWatch = Stopwatch.StartNew (); var result = await processManager.ExecuteCommandAsync (harness.MlaunchPath, args, runLog, timeout, cancellation_token: cancellation_source.Token); @@ -717,7 +722,7 @@ namespace Xharness { var crashed = false; if (File.Exists (listener_log.FullPath)) { WrenchLog.WriteLine ("AddFile: {0}", listener_log.FullPath); - success = TestsSucceeded (this.AppInformation, listener_log.FullPath, timed_out, out crashed); + success = TestsSucceeded (AppInformation, listener_log.FullPath, timed_out, out crashed); } else if (timed_out) { WrenchLog.WriteLine ("AddSummary: {0} never launched
", mode); MainLog.WriteLine ("Test run never launched"); @@ -742,7 +747,7 @@ namespace Xharness { if (crashed) crashLogWaitTime = 30; - await crash_reports.EndCaptureAsync (TimeSpan.FromSeconds (crashLogWaitTime)); + await crashReporter.EndCaptureAsync (TimeSpan.FromSeconds (crashLogWaitTime)); if (timed_out) { Result = TestExecutingResult.TimedOut; diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index 688a935c04..950bc544e6 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -615,6 +615,7 @@ namespace Xharness target, this, HarnessLog, + new Logs (LogDirectory), project.Path, buildConfiguration); @@ -644,6 +645,7 @@ namespace Xharness target, this, HarnessLog, + new Logs (LogDirectory), project.Path, buildConfiguration); @@ -671,6 +673,7 @@ namespace Xharness target, this, HarnessLog, + new Logs (LogDirectory), project.Path, buildConfiguration); diff --git a/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs b/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs index db81733ecf..5206126599 100644 --- a/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunDeviceTask.cs @@ -93,8 +93,8 @@ namespace Xharness.Jenkins.TestTasks Harness, projectFilePath: ProjectFile, mainLog: uninstall_log, - configuration: ProjectConfiguration, - logDirectory: LogDirectory, + logs: new Logs (LogDirectory ?? Harness.LogDirectory), + buildConfiguration: ProjectConfiguration, deviceName: Device.Name, companionDeviceName: CompanionDevice?.Name, timeoutMultiplier: TimeoutMultiplier, @@ -159,8 +159,8 @@ namespace Xharness.Jenkins.TestTasks Harness, projectFilePath: ProjectFile, mainLog: Logs.Create ($"extension-run-{Device.UDID}-{Timestamp}.log", "Extension run log"), - configuration: ProjectConfiguration, - logDirectory: LogDirectory, + logs: new Logs (LogDirectory ?? Harness.LogDirectory), + buildConfiguration: ProjectConfiguration, deviceName: Device.Name, companionDeviceName: CompanionDevice?.Name, timeoutMultiplier: TimeoutMultiplier, diff --git a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs index 30b9a264c7..ff0d517467 100644 --- a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs @@ -88,10 +88,10 @@ namespace Xharness.Jenkins.TestTasks AppRunnerTarget, Harness, mainLog: Logs.Create ($"run-{Device.UDID}-{Timestamp}.log", "Run log"), + logs: Logs, projectFilePath: ProjectFile, ensureCleanSimulatorState: clean_state, - logDirectory: LogDirectory, - configuration: ProjectConfiguration, + buildConfiguration: ProjectConfiguration, timeoutMultiplier: TimeoutMultiplier, variation: Variation, buildTask: BuildTask, diff --git a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs index 858f8d58b8..5d94087ceb 100644 --- a/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs +++ b/tests/xharness/Xharness.Tests/Tests/AppRunnerTests.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; +using System.Text; using System.Threading; using System.Threading.Tasks; using Moq; @@ -17,10 +18,13 @@ namespace Xharness.Tests { public class AppRunnerTests { const string appName = "com.xamarin.bcltests.SystemXunit"; + const string xcodePath = "/path/to/xcode"; + const string mlaunchPath = "/path/to/mlaunch"; - static readonly string outputPath = Path.GetDirectoryName (Assembly.GetAssembly (typeof(AppRunnerTests)).Location); + static readonly string outputPath = Path.GetDirectoryName (Assembly.GetAssembly (typeof (AppRunnerTests)).Location); static readonly string sampleProjectPath = Path.Combine (outputPath, "Samples", "TestProject"); static readonly string appPath = Path.Combine (sampleProjectPath, "bin", appName + ".app"); + static readonly string projectFilePath = Path.Combine (sampleProjectPath, "SystemXunit.csproj"); static readonly IHardwareDevice [] mockDevices = new IHardwareDevice [] { new Device() { @@ -52,8 +56,9 @@ namespace Xharness.Tests { Mock devices; Mock simpleListener; Mock snapshotReporter; + Mock logs; + Mock mainLog; - ILog mainLog; ISimulatorsLoaderFactory simulatorsFactory; IDeviceLoaderFactory devicesFactory; ISimpleListenerFactory listenerFactory; @@ -62,6 +67,9 @@ namespace Xharness.Tests { [SetUp] public void SetUp () { + logs = new Mock (); + logs.SetupGet (x => x.Directory).Returns (Path.Combine (outputPath, "logs")); + processManager = new Mock (); simulators = new Mock (); devices = new Mock (); @@ -78,15 +86,16 @@ namespace Xharness.Tests { var mock3 = new Mock (); mock3 - .Setup (m => m.Create (It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns ((ListenerTransport.Tcp, simpleListener.Object, null)); + .Setup (m => m.Create (It.IsAny (), It.IsAny (), It.IsAny (), It.IsAny (), It.IsAny (), It.IsAny ())) + .Returns ((ListenerTransport.Tcp, simpleListener.Object, "listener-temp-file")); listenerFactory = mock3.Object; + simpleListener.SetupGet (x => x.Port).Returns (1020); var mock4 = new Mock (); - mock4.Setup (m => m.Create (It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns (snapshotReporter.Object); + mock4.Setup (m => m.Create (It.IsAny (), It.IsAny (), It.IsAny (), It.IsAny ())).Returns (snapshotReporter.Object); snapshotReporterFactory = mock4.Object; - mainLog = new Mock ().Object; + mainLog = new Mock (); Directory.CreateDirectory (appPath); } @@ -103,12 +112,12 @@ namespace Xharness.Tests { Mock.Of (), Mock.Of (), AppRunnerTarget.Simulator_iOS64, - new Mock ().Object, - new Mock().Object, - Path.Combine (sampleProjectPath, "SystemXunit.csproj"), - "Debug", - Path.Combine (outputPath, "logs")); - + Mock.Of (), + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug"); + Assert.AreEqual (appName, appRunner.AppInformation.AppName); Assert.AreEqual (appPath, appRunner.AppInformation.AppPath); Assert.AreEqual (appPath, appRunner.AppInformation.LaunchAppPath); @@ -127,11 +136,11 @@ namespace Xharness.Tests { Mock.Of (), Mock.Of (), AppRunnerTarget.Simulator_iOS64, - new Mock ().Object, - new Mock().Object, - Path.Combine (sampleProjectPath, "SystemXunit.csproj"), - "Debug", - Path.Combine (outputPath, "logs")); + Mock.Of (), + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug"); var exception = Assert.ThrowsAsync ( async () => await appRunner.InstallAsync (new CancellationToken ()), @@ -150,11 +159,11 @@ namespace Xharness.Tests { Mock.Of (), Mock.Of (), AppRunnerTarget.Simulator_iOS64, - new Mock ().Object, - new Mock().Object, - Path.Combine (sampleProjectPath, "SystemXunit.csproj"), - "Debug", - Path.Combine (outputPath, "logs")); + Mock.Of (), + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug"); var exception = Assert.ThrowsAsync ( async () => await appRunner.UninstallAsync (), @@ -173,38 +182,28 @@ namespace Xharness.Tests { Mock.Of (), Mock.Of (), AppRunnerTarget.Device_iOS, - new Mock ().Object, - new Mock().Object, - Path.Combine (sampleProjectPath, "SystemXunit.csproj"), - "Debug", - Path.Combine (outputPath, "logs")); + Mock.Of (), + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug"); devices.Setup (x => x.ConnectedDevices).Returns (new IHardwareDevice [0]); - var exception = Assert.ThrowsAsync ( + Assert.ThrowsAsync ( async () => await appRunner.InstallAsync (new CancellationToken ()), - "Install requires a connected devices"); + "Install requires connected devices"); } [Test] public async Task InstallOnDeviceTest () { - Mock harnessMock = new Mock (); - harnessMock.SetupGet (x => x.XcodeRoot).Returns ("/path/to/xcode"); - harnessMock.SetupGet (x => x.MlaunchPath).Returns ("/path/to/mlaunch"); - harnessMock.SetupGet (x => x.Verbosity).Returns (2); + var harness = Mock.Of (x => x.XcodeRoot == "/path/to/xcode" + && x.MlaunchPath == "/path/to/mlaunch" + && x.Verbosity == 2); var processResult = new ProcessExecutionResult () { ExitCode = 1, TimedOut = false }; - - processManager - .Setup (x => x.ExecuteCommandAsync ( - It.IsAny (), - It.IsAny> (), - It.IsAny (), - It.IsAny (), - It.IsAny> (), - It.IsAny ())) - .ReturnsAsync(processResult); + processManager.SetReturnsDefault (Task.FromResult (processResult)); var appRunner = new AppRunner (processManager.Object, simulatorsFactory, @@ -215,19 +214,21 @@ namespace Xharness.Tests { Mock.Of (), Mock.Of (), AppRunnerTarget.Device_iOS, - harnessMock.Object, - mainLog, - Path.Combine (sampleProjectPath, "SystemXunit.csproj"), - "Debug", - Path.Combine (outputPath, "logs")); + harness, + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug"); devices.Setup (x => x.ConnectedDevices).Returns (mockDevices); + // Act CancellationToken cancellationToken = new CancellationToken (); var result = await appRunner.InstallAsync (cancellationToken); + // Verify Assert.AreEqual (1, result.ExitCode); - + processManager.Verify (x => x.ExecuteCommandAsync ( "/path/to/mlaunch", new List () { @@ -241,7 +242,7 @@ namespace Xharness.Tests { "--devname", "Test iPad" }, - mainLog, + mainLog.Object, TimeSpan.FromHours (1), null, cancellationToken)); @@ -250,22 +251,12 @@ namespace Xharness.Tests { [Test] public async Task UninstallFromDeviceTest () { - Mock harnessMock = new Mock (); - harnessMock.SetupGet (x => x.XcodeRoot).Returns ("/path/to/xcode"); - harnessMock.SetupGet (x => x.MlaunchPath).Returns ("/path/to/mlaunch"); - harnessMock.SetupGet (x => x.Verbosity).Returns (1); + var harness = Mock.Of (x => x.XcodeRoot == "/path/to/xcode" + && x.MlaunchPath == "/path/to/mlaunch" + && x.Verbosity == 1); var processResult = new ProcessExecutionResult () { ExitCode = 3, TimedOut = false }; - - processManager - .Setup (x => x.ExecuteCommandAsync ( - It.IsAny (), - It.IsAny> (), - It.IsAny (), - It.IsAny (), - null, - null)) - .ReturnsAsync(processResult); + processManager.SetReturnsDefault (Task.FromResult (processResult)); var appRunner = new AppRunner (processManager.Object, simulatorsFactory, @@ -276,18 +267,18 @@ namespace Xharness.Tests { Mock.Of (), Mock.Of (), AppRunnerTarget.Device_iOS, - harnessMock.Object, - mainLog, - Path.Combine (sampleProjectPath, "SystemXunit.csproj"), - "Debug", - Path.Combine (outputPath, "logs")); + harness, + mainLog.Object, + logs.Object, + projectFilePath: Path.Combine (sampleProjectPath, "SystemXunit.csproj"), + buildConfiguration: "Debug"); - devices.Setup (x => x.ConnectedDevices).Returns (mockDevices.Reverse()); + devices.Setup (x => x.ConnectedDevices).Returns (mockDevices.Reverse ()); var result = await appRunner.UninstallAsync (); Assert.AreEqual (3, result.ExitCode); - + processManager.Verify (x => x.ExecuteCommandAsync ( "/path/to/mlaunch", new List () { @@ -300,10 +291,479 @@ namespace Xharness.Tests { "--devname", "Test iPad" }, - mainLog, + mainLog.Object, TimeSpan.FromMinutes (1), null, null)); } + + [Test] + public async Task RunOnSimulatorWithNoAvailableSimulatorTest () + { + devices.Setup (x => x.ConnectedDevices).Returns (mockDevices.Reverse ()); + + // Crash reporter + var crashReporterFactory = new Mock (); + crashReporterFactory + .Setup (x => x.Create (mainLog.Object, It.IsAny (), false, null)) + .Returns (snapshotReporter.Object); + + // Mock finding simulators + simulators + .Setup (x => x.LoadAsync (It.IsAny (), false, false)) + .Returns (Task.CompletedTask); + + string simulatorLogPath = Path.Combine (Path.GetTempPath (), "simulator-logs"); + + simulators + .Setup (x => x.FindAsync (AppRunnerTarget.Simulator_tvOS, mainLog.Object, true, false)) + .ReturnsAsync ((ISimulatorDevice []) null); + + var listenerLogFile = new Mock (); + + logs + .Setup (x => x.Create (It.IsAny (), "TestLog", It.IsAny ())) + .Returns (listenerLogFile.Object); + + simpleListener.SetupGet (x => x.ConnectedTask).Returns (Task.CompletedTask); + + var captureLog = new Mock (); + captureLog.SetupGet (x => x.FullPath).Returns (simulatorLogPath); + + var captureLogFactory = new Mock (); + captureLogFactory + .Setup (x => x.Create ( + logs.Object, + Path.Combine (logs.Object.Directory, "tvos.log"), + "/path/to/simulator.log", + true, + It.IsAny ())) + .Returns (captureLog.Object); + + // Act + var appRunner = new AppRunner (processManager.Object, + simulatorsFactory, + listenerFactory, + devicesFactory, + crashReporterFactory.Object, + captureLogFactory.Object, + Mock.Of (), + Mock.Of (), + AppRunnerTarget.Simulator_tvOS, + GetMockedHarness (), + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug", + timeoutMultiplier: 2); + + var result = await appRunner.RunAsync (); + + // Verify + Assert.AreEqual (1, result); + + mainLog.Verify (x => x.WriteLine ("Test run completed"), Times.Never); + + simpleListener.Verify (x => x.Initialize (), Times.AtLeastOnce); + simpleListener.Verify (x => x.StartAsync (), Times.AtLeastOnce); + + simulators.VerifyAll (); + } + + [Test] + public async Task RunOnSimulatorSuccessfullyTest () + { + var harness = GetMockedHarness (); + + devices.Setup (x => x.ConnectedDevices).Returns (mockDevices.Reverse ()); + + // Crash reporter + var crashReporterFactory = new Mock (); + crashReporterFactory + .Setup (x => x.Create (mainLog.Object, It.IsAny (), false, null)) + .Returns (snapshotReporter.Object); + + // Mock finding simulators + simulators + .Setup (x => x.LoadAsync (It.IsAny (), false, false)) + .Returns (Task.CompletedTask); + + string simulatorLogPath = Path.Combine (Path.GetTempPath (), "simulator-logs"); + + var simulator = new Mock (); + simulator.SetupGet (x => x.Name).Returns ("Test iPhone simulator"); + simulator.SetupGet (x => x.UDID).Returns ("58F21118E4D34FD69EAB7860BB9B38A0"); + simulator.SetupGet (x => x.LogPath).Returns (simulatorLogPath); + simulator.SetupGet (x => x.SystemLog).Returns (Path.Combine (simulatorLogPath, "system.log")); + + simulators + .Setup (x => x.FindAsync (AppRunnerTarget.Simulator_iOS64, mainLog.Object, true, false)) + .ReturnsAsync (new ISimulatorDevice [] { simulator.Object }); + + var testResultFilePath = Path.GetTempFileName (); + var listenerLogFile = Mock.Of (x => x.FullPath == testResultFilePath); + File.WriteAllLines (testResultFilePath, new [] { "Some result here", "Tests run: 124", "Some result there" }); + + logs + .Setup (x => x.Create (It.Is (s => s.StartsWith ("test-sim64-")), "TestLog", It.IsAny ())) + .Returns (listenerLogFile); + + simpleListener.SetupGet (x => x.ConnectedTask).Returns (Task.CompletedTask); + + var captureLog = new Mock (); + captureLog.SetupGet (x => x.FullPath).Returns (simulatorLogPath); + + var captureLogFactory = new Mock (); + captureLogFactory + .Setup (x => x.Create ( + logs.Object, + Path.Combine (logs.Object.Directory, simulator.Object.Name + ".log"), + simulator.Object.SystemLog, + true, + It.IsAny ())) + .Returns (captureLog.Object); + + var expectedArgs = $"--sdkroot {xcodePath} -v -v -argument=-connection-mode -argument=none " + + $"-argument=-app-arg:-autostart -setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit " + + $"-setenv=NUNIT_AUTOEXIT=true -argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " + + $"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:127.0.0.1 " + + $"-setenv=NUNIT_HOSTNAME=127.0.0.1 -argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP " + + $"-argument=-app-arg:-hostport:{simpleListener.Object.Port} -setenv=NUNIT_HOSTPORT={simpleListener.Object.Port} " + + $"-setenv=env1=value1 -setenv=env2=value2 --launchsim {appPath} --stdout=tty1 --stderr=tty1 " + + $"--device=:v2:udid={simulator.Object.UDID}"; + + processManager + .Setup (x => x.ExecuteCommandAsync ( + mlaunchPath, + It.Is> (args => string.Join (" ", args) == expectedArgs), + mainLog.Object, + TimeSpan.FromMinutes (harness.Timeout * 2), + null, + It.IsAny ())) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + var xmlResultFile = Path.ChangeExtension (testResultFilePath, "xml"); + var resultParser = new Mock (); + resultParser + .Setup (x => x.CleanXml (testResultFilePath, xmlResultFile)) + .Callback (() => File.Copy (testResultFilePath, xmlResultFile)); + + // Act + var appRunner = new AppRunner (processManager.Object, + simulatorsFactory, + listenerFactory, + devicesFactory, + crashReporterFactory.Object, + captureLogFactory.Object, + Mock.Of (), // Use for devices only + resultParser.Object, + AppRunnerTarget.Simulator_iOS64, + harness, + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug", + timeoutMultiplier: 2, + ensureCleanSimulatorState: true); + + var result = await appRunner.RunAsync (); + + // Verify + Assert.AreEqual (0, result); + + mainLog.Verify (x => x.WriteLine ("Test run started")); + mainLog.Verify (x => x.WriteLine ("Test run completed")); + mainLog.Verify (x => x.WriteLine ("Test run succeeded")); + + simpleListener.Verify (x => x.Initialize (), Times.AtLeastOnce); + simpleListener.Verify (x => x.StartAsync (), Times.AtLeastOnce); + simpleListener.Verify (x => x.Cancel (), Times.AtLeastOnce); + simpleListener.Verify (x => x.Dispose (), Times.AtLeastOnce); + + simulators.VerifyAll (); + + captureLog.Verify (x => x.StartCapture (), Times.AtLeastOnce); + captureLog.Verify (x => x.StopCapture (), Times.AtLeastOnce); + + // When ensureCleanSimulatorState == true + simulator.Verify (x => x.PrepareSimulatorAsync (mainLog.Object, appName)); + simulator.Verify (x => x.KillEverythingAsync (mainLog.Object)); + + resultParser.VerifyAll (); + } + + [Test] + public void RunOnDeviceWithNoAvailableSimulatorTest () + { + devices.Setup (x => x.ConnectedDevices).Returns (mockDevices.Reverse ()); + + // Crash reporter + var crashReporterFactory = new Mock (); + crashReporterFactory + .Setup (x => x.Create (mainLog.Object, It.IsAny (), false, null)) + .Returns (snapshotReporter.Object); + + var listenerLogFile = new Mock (); + + logs + .Setup (x => x.Create (It.IsAny (), "TestLog", It.IsAny ())) + .Returns (listenerLogFile.Object); + + simpleListener.SetupGet (x => x.ConnectedTask).Returns (Task.CompletedTask); + + // Act + var appRunner = new AppRunner (processManager.Object, + simulatorsFactory, + listenerFactory, + devicesFactory, + crashReporterFactory.Object, + Mock.Of (), + Mock.Of (), + Mock.Of (), + AppRunnerTarget.Device_tvOS, + GetMockedHarness (), + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug", + timeoutMultiplier: 2); + + Assert.ThrowsAsync ( + async () => await appRunner.RunAsync (), + "Running requires connected devices"); + } + + [Test] + public async Task RunOnDeviceSuccessfullyTest () + { + var harness = GetMockedHarness (); + + devices.Setup (x => x.ConnectedDevices).Returns (mockDevices.Reverse ()); + + // Crash reporter + var crashReporterFactory = new Mock (); + crashReporterFactory + .Setup (x => x.Create (mainLog.Object, It.IsAny (), true, "Test iPad")) + .Returns (snapshotReporter.Object); + + var deviceSystemLog = new Mock (); + deviceSystemLog.SetupGet (x => x.FullPath).Returns (Path.GetTempFileName ()); + + var testResultFilePath = Path.GetTempFileName (); + var listenerLogFile = Mock.Of (x => x.FullPath == testResultFilePath); + File.WriteAllLines (testResultFilePath, new [] { "Some result here", "Some result there", "Tests run: 3" }); + + logs + .Setup (x => x.Create (It.Is (s => s.StartsWith ("test-ios-")), "TestLog", It.IsAny ())) + .Returns (listenerLogFile); + + logs + .Setup (x => x.Create (It.Is (s => s.StartsWith ("device-Test iPad-")), "Device log", It.IsAny ())) + .Returns (deviceSystemLog.Object); + + simpleListener.SetupGet (x => x.ConnectedTask).Returns (Task.CompletedTask); + + var deviceLogCapturer = new Mock (); + + var deviceLogCapturerFactory = new Mock (); + deviceLogCapturerFactory + .Setup (x => x.Create (mainLog.Object, deviceSystemLog.Object, "Test iPad")) + .Returns (deviceLogCapturer.Object); + + var ips = new StringBuilder (); + var ipAddresses = System.Net.Dns.GetHostEntry (System.Net.Dns.GetHostName ()).AddressList; + for (int i = 0; i < ipAddresses.Length; i++) { + if (i > 0) + ips.Append (','); + ips.Append (ipAddresses [i].ToString ()); + } + + var expectedArgs = $"--sdkroot {xcodePath} -v -v -argument=-connection-mode -argument=none " + + $"-argument=-app-arg:-autostart -setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit " + + $"-setenv=NUNIT_AUTOEXIT=true -argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " + + $"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:{ips} -setenv=NUNIT_HOSTNAME={ips} " + + $"-argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{simpleListener.Object.Port} " + + $"-setenv=NUNIT_HOSTPORT={simpleListener.Object.Port} -setenv=env1=value1 -setenv=env2=value2 " + + $"--launchdev {appPath} --disable-memory-limits --wait-for-exit --devname Test iPad"; + + processManager + .Setup (x => x.ExecuteCommandAsync ( + mlaunchPath, + It.Is> (args => string.Join (" ", args) == expectedArgs), + It.IsAny (), + TimeSpan.FromMinutes (harness.Timeout * 2), + null, + It.IsAny ())) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + var xmlResultFile = Path.ChangeExtension (testResultFilePath, "xml"); + var resultParser = new Mock (); + resultParser + .Setup (x => x.CleanXml (testResultFilePath, xmlResultFile)) + .Callback (() => File.Copy (testResultFilePath, xmlResultFile)); + + // Act + var appRunner = new AppRunner (processManager.Object, + simulatorsFactory, + listenerFactory, + devicesFactory, + crashReporterFactory.Object, + Mock.Of (), // Used for simulators only + deviceLogCapturerFactory.Object, + resultParser.Object, + AppRunnerTarget.Device_iOS, + harness, + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug", + timeoutMultiplier: 2); + + var result = await appRunner.RunAsync (); + + // Verify + Assert.AreEqual (0, result); + + processManager.VerifyAll (); + + mainLog.Verify (x => x.WriteLine ("Test run started")); + mainLog.Verify (x => x.WriteLine ("Test run completed")); + mainLog.Verify (x => x.WriteLine ("Test run succeeded")); + + simpleListener.Verify (x => x.Initialize (), Times.AtLeastOnce); + simpleListener.Verify (x => x.StartAsync (), Times.AtLeastOnce); + simpleListener.Verify (x => x.Cancel (), Times.AtLeastOnce); + simpleListener.Verify (x => x.Dispose (), Times.AtLeastOnce); + + snapshotReporter.Verify (x => x.StartCaptureAsync (), Times.AtLeastOnce); + snapshotReporter.Verify (x => x.StartCaptureAsync (), Times.AtLeastOnce); + + deviceSystemLog.Verify (x => x.Dispose (), Times.AtLeastOnce); + } + + [Test] + public async Task RunOnDeviceWithFailedTestsTest () + { + var harness = GetMockedHarness (); + + devices.Setup (x => x.ConnectedDevices).Returns (mockDevices.Reverse ()); + + // Crash reporter + var crashReporterFactory = new Mock (); + crashReporterFactory + .Setup (x => x.Create (mainLog.Object, It.IsAny (), true, "Test iPad")) + .Returns (snapshotReporter.Object); + + var deviceSystemLog = new Mock (); + deviceSystemLog.SetupGet (x => x.FullPath).Returns (Path.GetTempFileName ()); + + var testResultFilePath = Path.GetTempFileName (); + var listenerLogFile = Mock.Of (x => x.FullPath == testResultFilePath); + File.WriteAllLines (testResultFilePath, new [] { "Some result here", "[FAIL] This test failed", "Some result there", "Tests run: 3" }); + + logs + .Setup (x => x.Create (It.Is (s => s.StartsWith ("test-ios-")), "TestLog", It.IsAny ())) + .Returns (listenerLogFile); + + logs + .Setup (x => x.Create (It.Is (s => s.StartsWith ("device-Test iPad-")), "Device log", It.IsAny ())) + .Returns (deviceSystemLog.Object); + + simpleListener.SetupGet (x => x.ConnectedTask).Returns (Task.CompletedTask); + + var deviceLogCapturer = new Mock (); + + var deviceLogCapturerFactory = new Mock (); + deviceLogCapturerFactory + .Setup (x => x.Create (mainLog.Object, deviceSystemLog.Object, "Test iPad")) + .Returns (deviceLogCapturer.Object); + + var ips = new StringBuilder (); + var ipAddresses = System.Net.Dns.GetHostEntry (System.Net.Dns.GetHostName ()).AddressList; + for (int i = 0; i < ipAddresses.Length; i++) { + if (i > 0) + ips.Append (','); + ips.Append (ipAddresses [i].ToString ()); + } + + var expectedArgs = $"--sdkroot {xcodePath} -v -v -argument=-connection-mode -argument=none " + + $"-argument=-app-arg:-autostart -setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit " + + $"-setenv=NUNIT_AUTOEXIT=true -argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " + + $"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:{ips} -setenv=NUNIT_HOSTNAME={ips} " + + $"-argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{simpleListener.Object.Port} " + + $"-setenv=NUNIT_HOSTPORT={simpleListener.Object.Port} -setenv=env1=value1 -setenv=env2=value2 " + + $"--launchdev {appPath} --disable-memory-limits --wait-for-exit --devname Test iPad"; + + processManager + .Setup (x => x.ExecuteCommandAsync ( + mlaunchPath, + It.Is> (args => string.Join (" ", args) == expectedArgs), + It.IsAny (), + TimeSpan.FromMinutes (harness.Timeout * 2), + null, + It.IsAny ())) + .ReturnsAsync (new ProcessExecutionResult () { ExitCode = 0 }); + + var xmlResultFile = Path.ChangeExtension (testResultFilePath, "xml"); + var resultParser = new Mock (); + resultParser + .Setup (x => x.CleanXml (testResultFilePath, xmlResultFile)) + .Callback (() => File.Copy (testResultFilePath, xmlResultFile)); + + // Act + var appRunner = new AppRunner (processManager.Object, + simulatorsFactory, + listenerFactory, + devicesFactory, + crashReporterFactory.Object, + Mock.Of (), // Used for simulators only + deviceLogCapturerFactory.Object, + resultParser.Object, + AppRunnerTarget.Device_iOS, + harness, + mainLog.Object, + logs.Object, + projectFilePath: projectFilePath, + buildConfiguration: "Debug", + timeoutMultiplier: 2); + + var result = await appRunner.RunAsync (); + + // Verify + Assert.AreEqual (1, result); + + processManager.VerifyAll (); + + mainLog.Verify (x => x.WriteLine ("Test run started")); + mainLog.Verify (x => x.WriteLine ("Test run completed")); + mainLog.Verify (x => x.WriteLine ("Test run failed")); + + simpleListener.Verify (x => x.Initialize (), Times.AtLeastOnce); + simpleListener.Verify (x => x.StartAsync (), Times.AtLeastOnce); + simpleListener.Verify (x => x.Cancel (), Times.AtLeastOnce); + simpleListener.Verify (x => x.Dispose (), Times.AtLeastOnce); + + snapshotReporter.Verify (x => x.StartCaptureAsync (), Times.AtLeastOnce); + snapshotReporter.Verify (x => x.StartCaptureAsync (), Times.AtLeastOnce); + + deviceSystemLog.Verify (x => x.Dispose (), Times.AtLeastOnce); + } + + IHarness GetMockedHarness () + { + return Mock.Of (x => x.Action == HarnessAction.Run + && x.XcodeRoot == xcodePath + && x.MlaunchPath == mlaunchPath + && x.Verbosity == 1 + && x.HarnessLog == mainLog.Object + && x.LogDirectory == logs.Object.Directory + && x.InCI == false + && x.EnvironmentVariables == new Dictionary () { + { "env1", "value1" }, + { "env2", "value2" }, + } + && x.Timeout == 1d + && x.GetStandardErrorTty () == "tty1"); + } } }