diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 6d66053baf..61835dea69 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -12,6 +12,7 @@ using Xharness.Jenkins.TestTasks; using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; using Microsoft.DotNet.XHarness.iOS.Shared; using Microsoft.DotNet.XHarness.iOS.Shared.Hardware; +using Xharness.TestTasks; namespace Xharness.Jenkins { public class Jenkins @@ -76,7 +77,7 @@ namespace Xharness.Jenkins { } } - List Tasks = new List (); + List Tasks = new List (); Dictionary DependencyTasks = new Dictionary (); internal static Resource DesktopResource = new Resource ("Desktop", Environment.ProcessorCount); @@ -502,7 +503,7 @@ namespace Xharness.Jenkins { return rv; } - async Task> CreateRunSimulatorTasksAsync () + async Task> CreateRunSimulatorTasksAsync () { var runSimulatorTasks = new List (); @@ -568,7 +569,7 @@ namespace Xharness.Jenkins { return rv; } - Task> CreateRunDeviceTasksAsync () + Task> CreateRunDeviceTasksAsync () { var rv = new List (); var projectTasks = new List (); @@ -662,7 +663,7 @@ namespace Xharness.Jenkins { rv.AddRange (projectTasks); } - return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, processManager, candidates?.Cast () ?? test.Candidates))); + return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, processManager, candidates?.Cast () ?? test.Candidates))); } static string AddSuffixToPath (string path, string suffix) @@ -1371,7 +1372,7 @@ namespace Xharness.Jenkins { try { var allTasks = Tasks.SelectMany ((v) => { - var rv = new List (); + var rv = new List (); var runsim = v as AggregatedRunSimulatorTask; if (runsim != null) rv.AddRange (runsim.Tasks); @@ -1379,9 +1380,9 @@ namespace Xharness.Jenkins { return rv; }); - IEnumerable find_tasks (StreamWriter writer, string ids) + IEnumerable find_tasks (StreamWriter writer, string ids) { - IEnumerable tasks; + IEnumerable tasks; switch (request.Url.Query) { case "?all": tasks = Tasks; @@ -1394,10 +1395,10 @@ namespace Xharness.Jenkins { break; case "?": writer.WriteLine ("No tasks specified"); - return Array.Empty (); + return Array.Empty (); default: var id_inputs = ids.Substring (1).Split (','); - var rv = new List (id_inputs.Length); + var rv = new List (id_inputs.Length); foreach (var id_input in id_inputs) { if (int.TryParse (id_input, out var id)) { var task = Tasks.FirstOrDefault ((t) => t.ID == id); @@ -1677,7 +1678,7 @@ namespace Xharness.Jenkins { return tcs.Task; } - string GetTestColor (IEnumerable tests) + string GetTestColor (IEnumerable tests) { if (!tests.Any ()) return "black"; @@ -1707,7 +1708,7 @@ namespace Xharness.Jenkins { return "black"; } - string GetTestColor (TestTask test) + string GetTestColor (AppleTestTask test) { if (test.NotStarted) { return "black"; @@ -1861,7 +1862,7 @@ namespace Xharness.Jenkins { throw new NotImplementedException (); } - var allTasks = new List (); + var allTasks = new List (); if (!populating) { allTasks.AddRange (allExecuteTasks); allTasks.AddRange (allSimulatorTasks); @@ -2115,7 +2116,7 @@ namespace Xharness.Jenkins { writer.WriteLine ("
"); writer.WriteLine ("
"); - var orderedTasks = allTasks.GroupBy ((TestTask v) => v.TestName); + var orderedTasks = allTasks.GroupBy ((AppleTestTask v) => v.TestName); if (IsServerMode) { // In server mode don't take into account anything that can change during a test run @@ -2404,7 +2405,7 @@ namespace Xharness.Jenkins { writer.WriteLine ("
"); if (failedTests.Count () == 0) { foreach (var group in failedTests.GroupBy ((v) => v.TestName)) { - var enumerableGroup = group as IEnumerable; + var enumerableGroup = group as IEnumerable; if (enumerableGroup != null) { writer.WriteLine ("{0} ({1})
", group.Key, string.Join (", ", enumerableGroup.Select ((v) => string.Format ("{1}", GetTestColor (v), string.IsNullOrEmpty (v.Mode) ? v.ExecutionResult.ToString () : v.Mode)).ToArray ()), group.Key.Replace (' ', '-')); continue; @@ -2467,11 +2468,11 @@ namespace Xharness.Jenkins { return System.Web.HttpUtility.UrlEncode (path).Replace ("%2f", "/").Replace ("+", "%20"); } - string RenderTextStates (IEnumerable tests) + string RenderTextStates (IEnumerable tests) { // Create a collection of all non-ignored tests in the group (unless all tests were ignored). var allIgnored = tests.All ((v) => v.ExecutionResult == TestExecutingResult.Ignored); - IEnumerable relevantGroup; + IEnumerable relevantGroup; if (allIgnored) { relevantGroup = tests; } else { diff --git a/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs index e83e31905f..58aec21e8b 100644 --- a/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs @@ -9,7 +9,7 @@ namespace Xharness.Jenkins.TestTasks { // This class groups simulator run tasks according to the // simulator they'll run from, so that we minimize switching // between different simulators (which is slow). - class AggregatedRunSimulatorTask : TestTask + class AggregatedRunSimulatorTask : AppleTestTask { public IEnumerable Tasks; diff --git a/tests/xharness/Jenkins/TestTasks/AppleTestTask.cs b/tests/xharness/Jenkins/TestTasks/AppleTestTask.cs new file mode 100644 index 0000000000..902f4005a9 --- /dev/null +++ b/tests/xharness/Jenkins/TestTasks/AppleTestTask.cs @@ -0,0 +1,88 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.DotNet.XHarness.iOS.Shared.Logging; +using Xharness.TestTasks; + +namespace Xharness.Jenkins.TestTasks { + public abstract class AppleTestTask : Xharness.TestTasks.TestTasks + { + public Jenkins Jenkins; + public Harness Harness { get { return Jenkins.Harness; } } + + + public override string LogDirectory { + get { + var rv = Path.Combine (Jenkins.LogDirectory, TestName, ID.ToString ()); + Directory.CreateDirectory (rv); + return rv; + } + } + + public override void GenerateReport () => Jenkins.GenerateReport (); + + protected override void WriteLineToRunnerLog (string message) => Harness.HarnessLog.WriteLine (message); + + protected override void SetEnvironmentVariables (Process process) + { + var xcodeRoot = Harness.XcodeRoot; + + switch (Platform) { + case TestPlatform.iOS: + case TestPlatform.iOS_Unified: + case TestPlatform.iOS_Unified32: + case TestPlatform.iOS_Unified64: + case TestPlatform.iOS_TodayExtension64: + case TestPlatform.tvOS: + case TestPlatform.watchOS: + case TestPlatform.watchOS_32: + case TestPlatform.watchOS_64_32: + process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; + process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current"); + process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks"); + process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild"); + break; + case TestPlatform.Mac: + case TestPlatform.Mac_Modern: + case TestPlatform.Mac_Full: + case TestPlatform.Mac_System: + process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; + process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks"); + process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild"); + process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); + process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); + break; + case TestPlatform.All: + // Don't set: + // MSBuildExtensionsPath + // TargetFrameworkFallbackSearchPaths + // because these values used by both XM and XI and we can't set it to two different values at the same time. + // Any test that depends on these values should not be using 'TestPlatform.All' + process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; + process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current"); + process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); + process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); + break; + default: + throw new NotImplementedException (); + } + + foreach (var kvp in Environment) + process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value; + } + + + protected override void LogEvent (ILog log, string text, params object [] args) + { + base.LogEvent (log, text, args); + Jenkins.MainLog.WriteLine (text, args); + } + + protected override Task NotifyAndAcquireDesktopResourceAsync () + { + return NotifyBlockingWaitAsync (SupportsParallelExecution ? Jenkins.DesktopResource.AcquireConcurrentAsync () : Jenkins.DesktopResource.AcquireExclusiveAsync ()); + } + + } +} diff --git a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs index 10ae547098..7a99b315b3 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs @@ -8,6 +8,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared; using Microsoft.DotNet.XHarness.iOS.Shared.Execution; using Microsoft.DotNet.XHarness.iOS.Shared.Logging; using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; +using Xharness.TestTasks; namespace Xharness.Jenkins.TestTasks { abstract class BuildProjectTask : BuildToolTask diff --git a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs index 2211c59764..ea7b0ce718 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs @@ -4,7 +4,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Execution; namespace Xharness.Jenkins.TestTasks { - public abstract class BuildToolTask : TestTask + public abstract class BuildToolTask : AppleTestTask { protected readonly IProcessManager ProcessManager; diff --git a/tests/xharness/Jenkins/TestTasks/MakeTask.cs b/tests/xharness/Jenkins/TestTasks/MakeTask.cs index ec563feb36..26455ca114 100644 --- a/tests/xharness/Jenkins/TestTasks/MakeTask.cs +++ b/tests/xharness/Jenkins/TestTasks/MakeTask.cs @@ -39,7 +39,7 @@ namespace Xharness.Jenkins.TestTasks { } } using (var reader = log.GetReader ()) - AddWrenchLogFiles (reader); + AddCILogFiles (reader); Jenkins.MainLog.WriteLine ("Made {0} ({1})", TestName, Mode); } } diff --git a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs index abc1d31108..856919fdbd 100644 --- a/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs +++ b/tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs @@ -1,7 +1,5 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Threading.Tasks; diff --git a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs index c36e21f7ac..c9fbd0de3f 100644 --- a/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs @@ -9,6 +9,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared; using Microsoft.DotNet.XHarness.iOS.Shared.Listeners; using Microsoft.DotNet.XHarness.iOS.Shared.Collections; using Microsoft.DotNet.XHarness.iOS.Shared.Hardware; +using Xharness.TestTasks; namespace Xharness.Jenkins.TestTasks { class RunSimulatorTask : RunXITask diff --git a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs index e933470aab..21349fa7dd 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs @@ -10,7 +10,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Logging; using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; namespace Xharness.Jenkins.TestTasks { - internal abstract class RunTestTask : TestTask + internal abstract class RunTestTask : AppleTestTask { protected IProcessManager ProcessManager { get; } IResultParser ResultParser { get; } = new XmlResultParser (); diff --git a/tests/xharness/TestProject.cs b/tests/xharness/TestProject.cs index cb01f1bc36..1166d1b1f8 100644 --- a/tests/xharness/TestProject.cs +++ b/tests/xharness/TestProject.cs @@ -112,14 +112,14 @@ namespace Xharness { return rv; } - internal async Task CreateCloneAsync (TestTask test) + internal async Task CreateCloneAsync (AppleTestTask test) { var rv = Clone (); await rv.CreateCopyAsync (test); return rv; } - internal async Task CreateCopyAsync (TestTask test = null) + internal async Task CreateCopyAsync (AppleTestTask test = null) { var directory = DirectoryUtilities.CreateTemporaryDirectory (test?.TestName ?? System.IO.Path.GetFileNameWithoutExtension (Path)); Directory.CreateDirectory (directory); diff --git a/tests/xharness/TestTasks/IAcquiredResource.cs b/tests/xharness/TestTasks/IAcquiredResource.cs new file mode 100644 index 0000000000..4f26038be7 --- /dev/null +++ b/tests/xharness/TestTasks/IAcquiredResource.cs @@ -0,0 +1,7 @@ +using System; +namespace Xharness.TestTasks { + public interface IAcquiredResource : IDisposable + { + Resource Resource { get; } + } +} diff --git a/tests/xharness/Jenkins/Resource.cs b/tests/xharness/TestTasks/Resource.cs similarity index 96% rename from tests/xharness/Jenkins/Resource.cs rename to tests/xharness/TestTasks/Resource.cs index 96ce94394e..fb667b5393 100644 --- a/tests/xharness/Jenkins/Resource.cs +++ b/tests/xharness/TestTasks/Resource.cs @@ -2,7 +2,7 @@ using System.Collections.Concurrent; using System.Threading.Tasks; -namespace Xharness.Jenkins +namespace Xharness.TestTasks { // This is a very simple class to manage the general concept of 'resource'. // Performance isn't important, so this is very simple. @@ -90,9 +90,4 @@ namespace Xharness.Jenkins public Resource Resource { get; } } } - - public interface IAcquiredResource : IDisposable - { - Resource Resource { get; } - } } diff --git a/tests/xharness/Jenkins/Resources.cs b/tests/xharness/TestTasks/Resources.cs similarity index 97% rename from tests/xharness/Jenkins/Resources.cs rename to tests/xharness/TestTasks/Resources.cs index c63ee5c044..6e12a254f0 100644 --- a/tests/xharness/Jenkins/Resources.cs +++ b/tests/xharness/TestTasks/Resources.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -namespace Xharness.Jenkins +namespace Xharness.TestTasks { class Resources { diff --git a/tests/xharness/Jenkins/TestTasks/TestTask.cs b/tests/xharness/TestTasks/TestTask.cs similarity index 58% rename from tests/xharness/Jenkins/TestTasks/TestTask.cs rename to tests/xharness/TestTasks/TestTask.cs index 2836e4ccda..6088489f6b 100644 --- a/tests/xharness/Jenkins/TestTasks/TestTask.cs +++ b/tests/xharness/TestTasks/TestTask.cs @@ -10,70 +10,61 @@ using Microsoft.DotNet.XHarness.iOS.Shared; using Microsoft.DotNet.XHarness.iOS.Shared.Logging; using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; -namespace Xharness.Jenkins.TestTasks { - public abstract class TestTask +namespace Xharness.TestTasks { + public abstract class TestTasks { static int counter; + static DriveInfo RootDrive; + + #region Public vars + public readonly int ID; - - bool? supports_parallel_execution; - - public Jenkins Jenkins; - public Harness Harness { get { return Jenkins.Harness; } } - public TestProject TestProject; - public string ProjectFile { get { return TestProject?.Path; } } + public bool BuildOnly; + public string KnownFailure; public string ProjectConfiguration; public string ProjectPlatform; public Dictionary Environment = new Dictionary (); - public Func Dependency; // a task that's feteched and awaited before this task's ExecuteAsync method public Task InitialTask; // a task that's executed before this task's ExecuteAsync method. public Task CompletedTask; // a task that's executed after this task's ExecuteAsync method. + public TestProject TestProject; + public List Resources = new List (); - public bool BuildOnly; - public string KnownFailure; + #endregion - public TestTask () - { - ID = Interlocked.Increment (ref counter); - } + #region Properties - // VerifyRun is called in RunInternalAsync/ExecuteAsync to verify that the task can be executed/run. - // Typically used to fail tasks that don't have an available device, or if there's not enough disk space. - public virtual Task VerifyRunAsync () - { - return VerifyDiskSpaceAsync (); - } + protected static string Timestamp => Helpers.Timestamp; + public string ProjectFile => TestProject?.Path; + public bool HasCustomTestName => test_name != null; + public TestPlatform Platform { get; set; } - static DriveInfo RootDrive; - protected Task VerifyDiskSpaceAsync () - { - if (Finished) - return Task.CompletedTask; + public bool NotStarted => (ExecutionResult & TestExecutingResult.StateMask) == TestExecutingResult.NotStarted; + public bool InProgress => (ExecutionResult & TestExecutingResult.InProgress) == TestExecutingResult.InProgress; + public bool Waiting => (ExecutionResult & TestExecutingResult.Waiting) == TestExecutingResult.Waiting; + public bool Finished => (ExecutionResult & TestExecutingResult.Finished) == TestExecutingResult.Finished; - if (RootDrive == null) - RootDrive = new DriveInfo ("/"); - var afs = RootDrive.AvailableFreeSpace; - const long minSpaceRequirement = 1024 * 1024 * 1024; /* 1 GB */ - if (afs < minSpaceRequirement) { - FailureMessage = $"Not enough space on the root drive '{RootDrive.Name}': {afs / (1024.0 * 1024):#.##} MB left of {minSpaceRequirement / (1024.0 * 1024):#.##} MB required"; - ExecutionResult = TestExecutingResult.Failed; + public bool Building => (ExecutionResult & TestExecutingResult.Building) == TestExecutingResult.Building; + public bool Built => (ExecutionResult & TestExecutingResult.Built) == TestExecutingResult.Built; + public bool Running => (ExecutionResult & TestExecutingResult.Running) == TestExecutingResult.Running; + + public bool BuildSucceeded => (ExecutionResult & TestExecutingResult.BuildSucceeded) == TestExecutingResult.BuildSucceeded; + public bool Succeeded => (ExecutionResult & TestExecutingResult.Succeeded) == TestExecutingResult.Succeeded; + public bool Failed => (ExecutionResult & TestExecutingResult.Failed) == TestExecutingResult.Failed; + public bool Ignored { + get => ExecutionResult == TestExecutingResult.Ignored; + set { + if (ExecutionResult != TestExecutingResult.NotStarted && ExecutionResult != TestExecutingResult.Ignored) + throw new InvalidOperationException (); + ExecutionResult = value ? TestExecutingResult.Ignored : TestExecutingResult.NotStarted; } - return Task.CompletedTask; } + public bool DeviceNotFound => ExecutionResult == TestExecutingResult.DeviceNotFound; - public void CloneTestProject (TestProject project) - { - // Don't build in the original project directory - // We can build multiple projects in parallel, and if some of those - // projects have the same project dependencies, then we may end up - // building the same (dependent) project simultaneously (and they can - // stomp on eachother). - // So we clone the project file to a separate directory and build there instead. - // This is done asynchronously to speed to the initial test load. - TestProject = project.Clone (); - InitialTask = TestProject.CreateCopyAsync (); - } + public bool Crashed => (ExecutionResult & TestExecutingResult.Crashed) == TestExecutingResult.Crashed; + public bool TimedOut => (ExecutionResult & TestExecutingResult.TimedOut) == TestExecutingResult.TimedOut; + public bool BuildFailure => (ExecutionResult & TestExecutingResult.BuildFailure) == TestExecutingResult.BuildFailure; + public bool HarnessException => (ExecutionResult & TestExecutingResult.HarnessException) == TestExecutingResult.HarnessException; protected Stopwatch duration = new Stopwatch (); public TimeSpan Duration { @@ -82,19 +73,6 @@ namespace Xharness.Jenkins.TestTasks { } } - protected Stopwatch waitingDuration = new Stopwatch (); - public TimeSpan WaitingDuration => waitingDuration.Elapsed; - - TestExecutingResult execution_result; - public virtual TestExecutingResult ExecutionResult { - get { - return execution_result; - } - set { - execution_result = value; - } - } - string failure_message; public string FailureMessage { get { return failure_message; } @@ -104,48 +82,79 @@ namespace Xharness.Jenkins.TestTasks { } } - public virtual string ProgressMessage { get; } - public bool NotStarted { get { return (ExecutionResult & TestExecutingResult.StateMask) == TestExecutingResult.NotStarted; } } - public bool InProgress { get { return (ExecutionResult & TestExecutingResult.InProgress) == TestExecutingResult.InProgress; } } - public bool Waiting { get { return (ExecutionResult & TestExecutingResult.Waiting) == TestExecutingResult.Waiting; } } - public bool Finished { get { return (ExecutionResult & TestExecutingResult.Finished) == TestExecutingResult.Finished; } } - - public bool Building { get { return (ExecutionResult & TestExecutingResult.Building) == TestExecutingResult.Building; } } - public bool Built { get { return (ExecutionResult & TestExecutingResult.Built) == TestExecutingResult.Built; } } - public bool Running { get { return (ExecutionResult & TestExecutingResult.Running) == TestExecutingResult.Running; } } - - public bool BuildSucceeded { get { return (ExecutionResult & TestExecutingResult.BuildSucceeded) == TestExecutingResult.BuildSucceeded; } } - public bool Succeeded { get { return (ExecutionResult & TestExecutingResult.Succeeded) == TestExecutingResult.Succeeded; } } - public bool Failed { get { return (ExecutionResult & TestExecutingResult.Failed) == TestExecutingResult.Failed; } } - public bool Ignored { - get { return ExecutionResult == TestExecutingResult.Ignored; } - set { - if (ExecutionResult != TestExecutingResult.NotStarted && ExecutionResult != TestExecutingResult.Ignored) - throw new InvalidOperationException (); - ExecutionResult = value ? TestExecutingResult.Ignored : TestExecutingResult.NotStarted; + ILog test_log; + public ILog MainLog { + get { + if (test_log == null) + test_log = Logs.Create ($"main-{Timestamp}.log", "Main log"); + return test_log; } } - public bool DeviceNotFound { get { return ExecutionResult == TestExecutingResult.DeviceNotFound; } } - public bool Crashed { get { return (ExecutionResult & TestExecutingResult.Crashed) == TestExecutingResult.Crashed; } } - public bool TimedOut { get { return (ExecutionResult & TestExecutingResult.TimedOut) == TestExecutingResult.TimedOut; } } - public bool BuildFailure { get { return (ExecutionResult & TestExecutingResult.BuildFailure) == TestExecutingResult.BuildFailure; } } - public bool HarnessException { get { return (ExecutionResult & TestExecutingResult.HarnessException) == TestExecutingResult.HarnessException; } } + ILogs logs; + public ILogs Logs { + get { + return logs ?? (logs = new Logs (LogDirectory)); + } + } + IEnumerable referencedNunitAndXunitTestAssemblies; + public IEnumerable ReferencedNunitAndXunitTestAssemblies { + get { + if (referencedNunitAndXunitTestAssemblies != null) + return referencedNunitAndXunitTestAssemblies; + + if (TestName.Contains ("BCL tests group")) { // avoid loading unrelated projects + if (!File.Exists (ProjectFile)) + return Enumerable.Empty (); + + var csproj = new XmlDocument (); + try { + csproj.LoadWithoutNetworkAccess (ProjectFile.Replace ("\\", "/")); + referencedNunitAndXunitTestAssemblies = csproj.GetNunitAndXunitTestReferences (); + } catch (Exception e) { + referencedNunitAndXunitTestAssemblies = new string [] { $"Exception: {e.Message}", $"Filename: {ProjectFile}" }; + } + } else { + referencedNunitAndXunitTestAssemblies = Enumerable.Empty (); + } + return referencedNunitAndXunitTestAssemblies; + } + } + + #endregion + + #region Abstract + + public abstract void GenerateReport (); + public abstract string LogDirectory { get; } + protected abstract Task ExecuteAsync (); + protected abstract void SetEnvironmentVariables (Process process); + protected abstract Task NotifyAndAcquireDesktopResourceAsync (); + protected abstract void WriteLineToRunnerLog (string message); + + #endregion + + #region Virtual + + public virtual string ProgressMessage { get; } public virtual string Mode { get; set; } public virtual string Variation { get; set; } - protected static string Timestamp { - get { - return Helpers.Timestamp; - } + + bool? supports_parallel_execution; + public virtual bool SupportsParallelExecution { + get => supports_parallel_execution ?? true; + set => supports_parallel_execution = value; } - public bool HasCustomTestName { - get { - return test_name != null; - } + public virtual IEnumerable AggregatedLogs => Logs; + + TestExecutingResult execution_result; + public virtual TestExecutingResult ExecutionResult { + get => execution_result; + set => execution_result = value; } string test_name; @@ -185,64 +194,67 @@ namespace Xharness.Jenkins.TestTasks { } } - public TestPlatform Platform { get; set; } + protected virtual void PropagateResults () { } - public List Resources = new List (); + protected virtual void LogEvent (ILog log, string text, params object [] args) => log.WriteLine (text, args); - ILog test_log; - public ILog MainLog { - get { - if (test_log == null) - test_log = Logs.Create ($"main-{Timestamp}.log", "Main log"); - return test_log; - } + public virtual void Reset () + { + test_log = null; + failure_message = null; + logs = null; + duration.Reset (); + execution_result = TestExecutingResult.NotStarted; + execute_task = null; } - public virtual IEnumerable AggregatedLogs { - get { - return Logs; - } + + #endregion + + public TestTasks () + { + ID = Interlocked.Increment (ref counter); } - public string LogDirectory { - get { - var rv = Path.Combine (Jenkins.LogDirectory, TestName, ID.ToString ()); - Directory.CreateDirectory (rv); - return rv; - } + // VerifyRun is called in RunInternalAsync/ExecuteAsync to verify that the task can be executed/run. + // Typically used to fail tasks that don't have an available device, or if there's not enough disk space. + public virtual Task VerifyRunAsync () + { + return VerifyDiskSpaceAsync (); } - ILogs logs; - public ILogs Logs { - get { - return logs ?? (logs = new Logs (LogDirectory)); + protected Task VerifyDiskSpaceAsync () + { + if (Finished) + return Task.CompletedTask; + + if (RootDrive == null) + RootDrive = new DriveInfo ("/"); + var afs = RootDrive.AvailableFreeSpace; + const long minSpaceRequirement = 1024 * 1024 * 1024; /* 1 GB */ + if (afs < minSpaceRequirement) { + FailureMessage = $"Not enough space on the root drive '{RootDrive.Name}': {afs / (1024.0 * 1024):#.##} MB left of {minSpaceRequirement / (1024.0 * 1024):#.##} MB required"; + ExecutionResult = TestExecutingResult.Failed; } + return Task.CompletedTask; } - IEnumerable referencedNunitAndXunitTestAssemblies; - public IEnumerable ReferencedNunitAndXunitTestAssemblies { - get { - if (referencedNunitAndXunitTestAssemblies != null) - return referencedNunitAndXunitTestAssemblies; - - if (TestName.Contains ("BCL tests group")) { // avoid loading unrelated projects - if (!File.Exists (ProjectFile)) - return Enumerable.Empty (); - - var csproj = new XmlDocument (); - try { - csproj.LoadWithoutNetworkAccess (ProjectFile.Replace ("\\", "/")); - referencedNunitAndXunitTestAssemblies = csproj.GetNunitAndXunitTestReferences (); - } catch (Exception e) { - referencedNunitAndXunitTestAssemblies = new string [] { $"Exception: {e.Message}", $"Filename: {ProjectFile}" }; - } - } else { - referencedNunitAndXunitTestAssemblies = Enumerable.Empty (); - } - return referencedNunitAndXunitTestAssemblies; - } + public void CloneTestProject (TestProject project) + { + // Don't build in the original project directory + // We can build multiple projects in parallel, and if some of those + // projects have the same project dependencies, then we may end up + // building the same (dependent) project simultaneously (and they can + // stomp on eachother). + // So we clone the project file to a separate directory and build there instead. + // This is done asynchronously to speed to the initial test load. + TestProject = project.Clone (); + InitialTask = TestProject.CreateCopyAsync (); } + protected Stopwatch waitingDuration = new Stopwatch (); + public TimeSpan WaitingDuration => waitingDuration.Elapsed; + Task execute_task; async Task RunInternalAsync () { @@ -288,21 +300,7 @@ namespace Xharness.Jenkins.TestTasks { duration.Stop (); } - Jenkins.GenerateReport (); - } - - protected virtual void PropagateResults () - { - } - - public virtual void Reset () - { - test_log = null; - failure_message = null; - logs = null; - duration.Reset (); - execution_result = TestExecutingResult.NotStarted; - execute_task = null; + GenerateReport (); } public Task RunAsync () @@ -312,62 +310,14 @@ namespace Xharness.Jenkins.TestTasks { return execute_task; } - protected abstract Task ExecuteAsync (); public override string ToString () { return ExecutionResult.ToString (); } - protected void SetEnvironmentVariables (Process process) - { - var xcodeRoot = Harness.XcodeRoot; - switch (Platform) { - case TestPlatform.iOS: - case TestPlatform.iOS_Unified: - case TestPlatform.iOS_Unified32: - case TestPlatform.iOS_Unified64: - case TestPlatform.iOS_TodayExtension64: - case TestPlatform.tvOS: - case TestPlatform.watchOS: - case TestPlatform.watchOS_32: - case TestPlatform.watchOS_64_32: - process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; - process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current"); - process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks"); - process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild"); - break; - case TestPlatform.Mac: - case TestPlatform.Mac_Modern: - case TestPlatform.Mac_Full: - case TestPlatform.Mac_System: - process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; - process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks"); - process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild"); - process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); - process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); - break; - case TestPlatform.All: - // Don't set: - // MSBuildExtensionsPath - // TargetFrameworkFallbackSearchPaths - // because these values used by both XM and XI and we can't set it to two different values at the same time. - // Any test that depends on these values should not be using 'TestPlatform.All' - process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot; - process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current"); - process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); - process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current"); - break; - default: - throw new NotImplementedException (); - } - - foreach (var kvp in Environment) - process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value; - } - - protected void AddWrenchLogFiles (StreamReader stream) + protected void AddCILogFiles (StreamReader stream) { string line; while ((line = stream.ReadLine ()) != null) { @@ -385,18 +335,12 @@ namespace Xharness.Jenkins.TestTasks { Logs.AddFile (src); break; default: - Harness.HarnessLog.WriteLine ("Unknown @MonkeyWrench command in {0}: {1}", TestName, name); + WriteLineToRunnerLog ($"Unknown @MonkeyWrench command in {TestName}: {name}"); break; } } } - protected void LogEvent (ILog log, string text, params object [] args) - { - Jenkins.MainLog.WriteLine (text, args); - log.WriteLine (text, args); - } - public string GuessFailureReason (ILog log) { try { @@ -410,7 +354,7 @@ namespace Xharness.Jenkins.TestTasks { } } } catch (Exception e) { - Harness.Log ("Failed to guess failure reason: {0}", e.Message); + WriteLineToRunnerLog ($"Failed to guess failure reason: {e.Message}"); } return null; @@ -434,20 +378,6 @@ namespace Xharness.Jenkins.TestTasks { return rv; } - public virtual bool SupportsParallelExecution { - get { - return supports_parallel_execution ?? true; - } - set { - supports_parallel_execution = value; - } - } - - protected Task NotifyAndAcquireDesktopResourceAsync () - { - return NotifyBlockingWaitAsync (SupportsParallelExecution ? Jenkins.DesktopResource.AcquireConcurrentAsync () : Jenkins.DesktopResource.AcquireExclusiveAsync ()); - } - class BlockingWait : IAcquiredResource, IDisposable { public IAcquiredResource Wrapped; diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index 2a06e5ca0c..bb8f401be6 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -81,8 +81,6 @@ - - @@ -98,7 +96,7 @@ - + @@ -117,6 +115,10 @@ + + + +