xamarin
/
xamarin-macios
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[Harness] Move to create a more general TestTasks class to be reused. (#8286) 

Some of the Jenkins test tasks are very useful and do A LOT of stuff. So
we try to generalize the base class, to later be able to share the most
usebul ones in the shared lib.

Co-authored-by: Rolf Bjarne Kvinge <rolf@xamarin.com>
This commit is contained in:
Manuel de la Pena 2020-04-07 20:20:38 -04:00 коммит произвёл GitHub
Родитель c5ade80b2e
Коммит f891076bd8
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
15 изменённых файлов: 280 добавлений и 257 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

33
tests/xharness/Jenkins/Jenkins.cs
Просмотреть файл

@ -12,6 +12,7 @@ using Xharness.Jenkins.TestTasks;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
using Xharness.TestTasks;
namespace Xharness.Jenkins {
public class Jenkins
@ -76,7 +77,7 @@ namespace Xharness.Jenkins {
}
}
List<TestTask> Tasks = new List<TestTask> ();
List<AppleTestTask> Tasks = new List<AppleTestTask> ();
Dictionary<string, MakeTask> DependencyTasks = new Dictionary<string, MakeTask> ();
internal static Resource DesktopResource = new Resource ("Desktop", Environment.ProcessorCount);
@ -502,7 +503,7 @@ namespace Xharness.Jenkins {
return rv;
}
async Task<IEnumerable<TestTask>> CreateRunSimulatorTasksAsync ()
async Task<IEnumerable<AppleTestTask>> CreateRunSimulatorTasksAsync ()
{
var runSimulatorTasks = new List<RunSimulatorTask> ();
@ -568,7 +569,7 @@ namespace Xharness.Jenkins {
return rv;
}
Task<IEnumerable<TestTask>> CreateRunDeviceTasksAsync ()
Task<IEnumerable<AppleTestTask>> CreateRunDeviceTasksAsync ()
{
var rv = new List<RunDeviceTask> ();
var projectTasks = new List<RunDeviceTask> ();
@ -662,7 +663,7 @@ namespace Xharness.Jenkins {
rv.AddRange (projectTasks);
}
return Task.FromResult<IEnumerable<TestTask>> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, processManager, candidates?.Cast<IHardwareDevice> () ?? test.Candidates)));
return Task.FromResult<IEnumerable<AppleTestTask>> (CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask (devices, buildTask, processManager, candidates?.Cast<IHardwareDevice> () ?? test.Candidates)));
}
static string AddSuffixToPath (string path, string suffix)
@ -1371,7 +1372,7 @@ namespace Xharness.Jenkins {
try {
var allTasks = Tasks.SelectMany ((v) =>
{
var rv = new List<TestTask> ();
var rv = new List<AppleTestTask> ();
var runsim = v as AggregatedRunSimulatorTask;
if (runsim != null)
rv.AddRange (runsim.Tasks);
@ -1379,9 +1380,9 @@ namespace Xharness.Jenkins {
return rv;
});
IEnumerable<TestTask> find_tasks (StreamWriter writer, string ids)
IEnumerable<AppleTestTask> find_tasks (StreamWriter writer, string ids)
{
IEnumerable<TestTask> tasks;
IEnumerable<AppleTestTask> tasks;
switch (request.Url.Query) {
case "?all":
tasks = Tasks;
@ -1394,10 +1395,10 @@ namespace Xharness.Jenkins {
break;
case "?":
writer.WriteLine ("No tasks specified");
return Array.Empty<TestTask> ();
return Array.Empty<AppleTestTask> ();
default:
var id_inputs = ids.Substring (1).Split (',');
var rv = new List<TestTask> (id_inputs.Length);
var rv = new List<AppleTestTask> (id_inputs.Length);
foreach (var id_input in id_inputs) {
if (int.TryParse (id_input, out var id)) {
var task = Tasks.FirstOrDefault ((t) => t.ID == id);
@ -1677,7 +1678,7 @@ namespace Xharness.Jenkins {
return tcs.Task;
}
string GetTestColor (IEnumerable<TestTask> tests)
string GetTestColor (IEnumerable<AppleTestTask> tests)
{
if (!tests.Any ())
return "black";
@ -1707,7 +1708,7 @@ namespace Xharness.Jenkins {
return "black";
}
string GetTestColor (TestTask test)
string GetTestColor (AppleTestTask test)
{
if (test.NotStarted) {
return "black";
@ -1861,7 +1862,7 @@ namespace Xharness.Jenkins {
throw new NotImplementedException ();
}
var allTasks = new List<TestTask> ();
var allTasks = new List<AppleTestTask> ();
if (!populating) {
allTasks.AddRange (allExecuteTasks);
allTasks.AddRange (allSimulatorTasks);
@ -2115,7 +2116,7 @@ namespace Xharness.Jenkins {
writer.WriteLine ("<div id='test-table' style='width: 100%; display: flex;'>");
writer.WriteLine ("<div id='test-list'>");
var orderedTasks = allTasks.GroupBy ((TestTask v) => v.TestName);
var orderedTasks = allTasks.GroupBy ((AppleTestTask v) => v.TestName);
if (IsServerMode) {
// In server mode don't take into account anything that can change during a test run
@ -2404,7 +2405,7 @@ namespace Xharness.Jenkins {
writer.WriteLine ("<div id='test-status' style='margin-left: 100px;' class='autorefreshable'>");
if (failedTests.Count () == 0) {
foreach (var group in failedTests.GroupBy ((v) => v.TestName)) {
var enumerableGroup = group as IEnumerable<TestTask>;
var enumerableGroup = group as IEnumerable<AppleTestTask>;
if (enumerableGroup != null) {
writer.WriteLine ("<a href='#test_{2}'>{0}</a> ({1})<br />", group.Key, string.Join (", ", enumerableGroup.Select ((v) => string.Format ("<span style='color: {0}'>{1}</span>", GetTestColor (v), string.IsNullOrEmpty (v.Mode) ? v.ExecutionResult.ToString () : v.Mode)).ToArray ()), group.Key.Replace (' ', '-'));
continue;
@ -2467,11 +2468,11 @@ namespace Xharness.Jenkins {
return System.Web.HttpUtility.UrlEncode (path).Replace ("%2f", "/").Replace ("+", "%20");
}
string RenderTextStates (IEnumerable<TestTask> tests)
string RenderTextStates (IEnumerable<AppleTestTask> tests)
{
// Create a collection of all non-ignored tests in the group (unless all tests were ignored).
var allIgnored = tests.All ((v) => v.ExecutionResult == TestExecutingResult.Ignored);
IEnumerable<TestTask> relevantGroup;
IEnumerable<AppleTestTask> relevantGroup;
if (allIgnored) {
relevantGroup = tests;
} else {

2
tests/xharness/Jenkins/TestTasks/AggregatedRunSimulatorTask.cs
Просмотреть файл

@ -9,7 +9,7 @@ namespace Xharness.Jenkins.TestTasks {
// This class groups simulator run tasks according to the
// simulator they'll run from, so that we minimize switching
// between different simulators (which is slow).
class AggregatedRunSimulatorTask : TestTask
class AggregatedRunSimulatorTask : AppleTestTask
{
public IEnumerable<RunSimulatorTask> Tasks;

88
tests/xharness/Jenkins/TestTasks/AppleTestTask.cs Normal file
Просмотреть файл

@ -0,0 +1,88 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Xharness.TestTasks;
namespace Xharness.Jenkins.TestTasks {
public abstract class AppleTestTask : Xharness.TestTasks.TestTasks
{
public Jenkins Jenkins;
public Harness Harness { get { return Jenkins.Harness; } }
public override string LogDirectory {
get {
var rv = Path.Combine (Jenkins.LogDirectory, TestName, ID.ToString ());
Directory.CreateDirectory (rv);
return rv;
}
}
public override void GenerateReport () => Jenkins.GenerateReport ();
protected override void WriteLineToRunnerLog (string message) => Harness.HarnessLog.WriteLine (message);
protected override void SetEnvironmentVariables (Process process)
{
var xcodeRoot = Harness.XcodeRoot;
switch (Platform) {
case TestPlatform.iOS:
case TestPlatform.iOS_Unified:
case TestPlatform.iOS_Unified32:
case TestPlatform.iOS_Unified64:
case TestPlatform.iOS_TodayExtension64:
case TestPlatform.tvOS:
case TestPlatform.watchOS:
case TestPlatform.watchOS_32:
case TestPlatform.watchOS_64_32:
process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot;
process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current");
process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks");
process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild");
break;
case TestPlatform.Mac:
case TestPlatform.Mac_Modern:
case TestPlatform.Mac_Full:
case TestPlatform.Mac_System:
process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot;
process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks");
process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild");
process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
break;
case TestPlatform.All:
// Don't set:
// MSBuildExtensionsPath
// TargetFrameworkFallbackSearchPaths
// because these values used by both XM and XI and we can't set it to two different values at the same time.
// Any test that depends on these values should not be using 'TestPlatform.All'
process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot;
process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current");
process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
break;
default:
throw new NotImplementedException ();
}
foreach (var kvp in Environment)
process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value;
}
protected override void LogEvent (ILog log, string text, params object [] args)
{
base.LogEvent (log, text, args);
Jenkins.MainLog.WriteLine (text, args);
}
protected override Task<IAcquiredResource> NotifyAndAcquireDesktopResourceAsync ()
{
return NotifyBlockingWaitAsync (SupportsParallelExecution ? Jenkins.DesktopResource.AcquireConcurrentAsync () : Jenkins.DesktopResource.AcquireExclusiveAsync ());
}
}
}

1
tests/xharness/Jenkins/TestTasks/BuildProjectTask.cs
Просмотреть файл

@ -8,6 +8,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
using Xharness.TestTasks;
namespace Xharness.Jenkins.TestTasks {
abstract class BuildProjectTask : BuildToolTask

2
tests/xharness/Jenkins/TestTasks/BuildToolTask.cs
Просмотреть файл

@ -4,7 +4,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
namespace Xharness.Jenkins.TestTasks
{
public abstract class BuildToolTask : TestTask
public abstract class BuildToolTask : AppleTestTask
{
protected readonly IProcessManager ProcessManager;

2
tests/xharness/Jenkins/TestTasks/MakeTask.cs
Просмотреть файл

@ -39,7 +39,7 @@ namespace Xharness.Jenkins.TestTasks {
}
}
using (var reader = log.GetReader ())
AddWrenchLogFiles (reader);
AddCILogFiles (reader);
Jenkins.MainLog.WriteLine ("Made {0} ({1})", TestName, Mode);
}
}

2
tests/xharness/Jenkins/TestTasks/NUnitExecuteTask.cs
Просмотреть файл

@ -1,7 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;

1
tests/xharness/Jenkins/TestTasks/RunSimulatorTask.cs
Просмотреть файл

@ -9,6 +9,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Listeners;
using Microsoft.DotNet.XHarness.iOS.Shared.Collections;
using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
using Xharness.TestTasks;
namespace Xharness.Jenkins.TestTasks {
class RunSimulatorTask : RunXITask<ISimulatorDevice>

2
tests/xharness/Jenkins/TestTasks/RunTestTask.cs
Просмотреть файл

@ -10,7 +10,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
namespace Xharness.Jenkins.TestTasks {
internal abstract class RunTestTask : TestTask
internal abstract class RunTestTask : AppleTestTask
{
protected IProcessManager ProcessManager { get; }
IResultParser ResultParser { get; } = new XmlResultParser ();

4
tests/xharness/TestProject.cs
Просмотреть файл

@ -112,14 +112,14 @@ namespace Xharness {
return rv;
}
internal async Task<TestProject> CreateCloneAsync (TestTask test)
internal async Task<TestProject> CreateCloneAsync (AppleTestTask test)
{
var rv = Clone ();
await rv.CreateCopyAsync (test);
return rv;
}
internal async Task CreateCopyAsync (TestTask test = null)
internal async Task CreateCopyAsync (AppleTestTask test = null)
{
var directory = DirectoryUtilities.CreateTemporaryDirectory (test?.TestName ?? System.IO.Path.GetFileNameWithoutExtension (Path));
Directory.CreateDirectory (directory);

7
tests/xharness/TestTasks/IAcquiredResource.cs Normal file
Просмотреть файл

@ -0,0 +1,7 @@
using System;
namespace Xharness.TestTasks {
public interface IAcquiredResource : IDisposable
{
Resource Resource { get; }
}
}

7
tests/xharness/Jenkins/Resource.cs → tests/xharness/TestTasks/Resource.cs
Просмотреть файл

@ -2,7 +2,7 @@
using System.Collections.Concurrent;
using System.Threading.Tasks;
namespace Xharness.Jenkins
namespace Xharness.TestTasks
{
// This is a very simple class to manage the general concept of 'resource'.
// Performance isn't important, so this is very simple.
@ -90,9 +90,4 @@ namespace Xharness.Jenkins
public Resource Resource { get; }
}
}
public interface IAcquiredResource : IDisposable
{
Resource Resource { get; }
}
}

2
tests/xharness/Jenkins/Resources.cs → tests/xharness/TestTasks/Resources.cs
Просмотреть файл

@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace Xharness.Jenkins
namespace Xharness.TestTasks
{
class Resources
{

390
tests/xharness/Jenkins/TestTasks/TestTask.cs → tests/xharness/TestTasks/TestTask.cs
Просмотреть файл

@ -10,70 +10,61 @@ using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
namespace Xharness.Jenkins.TestTasks {
public abstract class TestTask
namespace Xharness.TestTasks {
public abstract class TestTasks
{
static int counter;
static DriveInfo RootDrive;
#region Public vars
public readonly int ID;
bool? supports_parallel_execution;
public Jenkins Jenkins;
public Harness Harness { get { return Jenkins.Harness; } }
public TestProject TestProject;
public string ProjectFile { get { return TestProject?.Path; } }
public bool BuildOnly;
public string KnownFailure;
public string ProjectConfiguration;
public string ProjectPlatform;
public Dictionary<string, string> Environment = new Dictionary<string, string> ();
public Func<Task> Dependency; // a task that's feteched and awaited before this task's ExecuteAsync method
public Task InitialTask; // a task that's executed before this task's ExecuteAsync method.
public Task CompletedTask; // a task that's executed after this task's ExecuteAsync method.
public TestProject TestProject;
public List<Resource> Resources = new List<Resource> ();
public bool BuildOnly;
public string KnownFailure;
#endregion
public TestTask ()
{
ID = Interlocked.Increment (ref counter);
#region Properties
protected static string Timestamp => Helpers.Timestamp;
public string ProjectFile => TestProject?.Path;
public bool HasCustomTestName => test_name != null;
public TestPlatform Platform { get; set; }
public bool NotStarted => (ExecutionResult & TestExecutingResult.StateMask) == TestExecutingResult.NotStarted;
public bool InProgress => (ExecutionResult & TestExecutingResult.InProgress) == TestExecutingResult.InProgress;
public bool Waiting => (ExecutionResult & TestExecutingResult.Waiting) == TestExecutingResult.Waiting;
public bool Finished => (ExecutionResult & TestExecutingResult.Finished) == TestExecutingResult.Finished;
public bool Building => (ExecutionResult & TestExecutingResult.Building) == TestExecutingResult.Building;
public bool Built => (ExecutionResult & TestExecutingResult.Built) == TestExecutingResult.Built;
public bool Running => (ExecutionResult & TestExecutingResult.Running) == TestExecutingResult.Running;
public bool BuildSucceeded => (ExecutionResult & TestExecutingResult.BuildSucceeded) == TestExecutingResult.BuildSucceeded;
public bool Succeeded => (ExecutionResult & TestExecutingResult.Succeeded) == TestExecutingResult.Succeeded;
public bool Failed => (ExecutionResult & TestExecutingResult.Failed) == TestExecutingResult.Failed;
public bool Ignored {
get => ExecutionResult == TestExecutingResult.Ignored;
set {
if (ExecutionResult != TestExecutingResult.NotStarted && ExecutionResult != TestExecutingResult.Ignored)
throw new InvalidOperationException ();
ExecutionResult = value ? TestExecutingResult.Ignored : TestExecutingResult.NotStarted;
}
// VerifyRun is called in RunInternalAsync/ExecuteAsync to verify that the task can be executed/run.
// Typically used to fail tasks that don't have an available device, or if there's not enough disk space.
public virtual Task VerifyRunAsync ()
{
return VerifyDiskSpaceAsync ();
}
public bool DeviceNotFound => ExecutionResult == TestExecutingResult.DeviceNotFound;
static DriveInfo RootDrive;
protected Task VerifyDiskSpaceAsync ()
{
if (Finished)
return Task.CompletedTask;
if (RootDrive == null)
RootDrive = new DriveInfo ("/");
var afs = RootDrive.AvailableFreeSpace;
const long minSpaceRequirement = 1024 * 1024 * 1024; /* 1 GB */
if (afs < minSpaceRequirement) {
FailureMessage = $"Not enough space on the root drive '{RootDrive.Name}': {afs / (1024.0 * 1024):#.##} MB left of {minSpaceRequirement / (1024.0 * 1024):#.##} MB required";
ExecutionResult = TestExecutingResult.Failed;
}
return Task.CompletedTask;
}
public void CloneTestProject (TestProject project)
{
// Don't build in the original project directory
// We can build multiple projects in parallel, and if some of those
// projects have the same project dependencies, then we may end up
// building the same (dependent) project simultaneously (and they can
// stomp on eachother).
// So we clone the project file to a separate directory and build there instead.
// This is done asynchronously to speed to the initial test load.
TestProject = project.Clone ();
InitialTask = TestProject.CreateCopyAsync ();
}
public bool Crashed => (ExecutionResult & TestExecutingResult.Crashed) == TestExecutingResult.Crashed;
public bool TimedOut => (ExecutionResult & TestExecutingResult.TimedOut) == TestExecutingResult.TimedOut;
public bool BuildFailure => (ExecutionResult & TestExecutingResult.BuildFailure) == TestExecutingResult.BuildFailure;
public bool HarnessException => (ExecutionResult & TestExecutingResult.HarnessException) == TestExecutingResult.HarnessException;
protected Stopwatch duration = new Stopwatch ();
public TimeSpan Duration {
@ -82,19 +73,6 @@ namespace Xharness.Jenkins.TestTasks {
}
}
protected Stopwatch waitingDuration = new Stopwatch ();
public TimeSpan WaitingDuration => waitingDuration.Elapsed;
TestExecutingResult execution_result;
public virtual TestExecutingResult ExecutionResult {
get {
return execution_result;
}
set {
execution_result = value;
}
}
string failure_message;
public string FailureMessage {
get { return failure_message; }
@ -104,48 +82,79 @@ namespace Xharness.Jenkins.TestTasks {
}
}
ILog test_log;
public ILog MainLog {
get {
if (test_log == null)
test_log = Logs.Create ($"main-{Timestamp}.log", "Main log");
return test_log;
}
}
ILogs logs;
public ILogs Logs {
get {
return logs ?? (logs = new Logs (LogDirectory));
}
}
IEnumerable<string> referencedNunitAndXunitTestAssemblies;
public IEnumerable<string> ReferencedNunitAndXunitTestAssemblies {
get {
if (referencedNunitAndXunitTestAssemblies != null)
return referencedNunitAndXunitTestAssemblies;
if (TestName.Contains ("BCL tests group")) { // avoid loading unrelated projects
if (!File.Exists (ProjectFile))
return Enumerable.Empty<string> ();
var csproj = new XmlDocument ();
try {
csproj.LoadWithoutNetworkAccess (ProjectFile.Replace ("\\", "/"));
referencedNunitAndXunitTestAssemblies = csproj.GetNunitAndXunitTestReferences ();
} catch (Exception e) {
referencedNunitAndXunitTestAssemblies = new string [] { $"Exception: {e.Message}", $"Filename: {ProjectFile}" };
}
} else {
referencedNunitAndXunitTestAssemblies = Enumerable.Empty<string> ();
}
return referencedNunitAndXunitTestAssemblies;
}
}
#endregion
#region Abstract
public abstract void GenerateReport ();
public abstract string LogDirectory { get; }
protected abstract Task ExecuteAsync ();
protected abstract void SetEnvironmentVariables (Process process);
protected abstract Task<IAcquiredResource> NotifyAndAcquireDesktopResourceAsync ();
protected abstract void WriteLineToRunnerLog (string message);
#endregion
#region Virtual
public virtual string ProgressMessage { get; }
public bool NotStarted { get { return (ExecutionResult & TestExecutingResult.StateMask) == TestExecutingResult.NotStarted; } }
public bool InProgress { get { return (ExecutionResult & TestExecutingResult.InProgress) == TestExecutingResult.InProgress; } }
public bool Waiting { get { return (ExecutionResult & TestExecutingResult.Waiting) == TestExecutingResult.Waiting; } }
public bool Finished { get { return (ExecutionResult & TestExecutingResult.Finished) == TestExecutingResult.Finished; } }
public bool Building { get { return (ExecutionResult & TestExecutingResult.Building) == TestExecutingResult.Building; } }
public bool Built { get { return (ExecutionResult & TestExecutingResult.Built) == TestExecutingResult.Built; } }
public bool Running { get { return (ExecutionResult & TestExecutingResult.Running) == TestExecutingResult.Running; } }
public bool BuildSucceeded { get { return (ExecutionResult & TestExecutingResult.BuildSucceeded) == TestExecutingResult.BuildSucceeded; } }
public bool Succeeded { get { return (ExecutionResult & TestExecutingResult.Succeeded) == TestExecutingResult.Succeeded; } }
public bool Failed { get { return (ExecutionResult & TestExecutingResult.Failed) == TestExecutingResult.Failed; } }
public bool Ignored {
get { return ExecutionResult == TestExecutingResult.Ignored; }
set {
if (ExecutionResult != TestExecutingResult.NotStarted && ExecutionResult != TestExecutingResult.Ignored)
throw new InvalidOperationException ();
ExecutionResult = value ? TestExecutingResult.Ignored : TestExecutingResult.NotStarted;
}
}
public bool DeviceNotFound { get { return ExecutionResult == TestExecutingResult.DeviceNotFound; } }
public bool Crashed { get { return (ExecutionResult & TestExecutingResult.Crashed) == TestExecutingResult.Crashed; } }
public bool TimedOut { get { return (ExecutionResult & TestExecutingResult.TimedOut) == TestExecutingResult.TimedOut; } }
public bool BuildFailure { get { return (ExecutionResult & TestExecutingResult.BuildFailure) == TestExecutingResult.BuildFailure; } }
public bool HarnessException { get { return (ExecutionResult & TestExecutingResult.HarnessException) == TestExecutingResult.HarnessException; } }
public virtual string Mode { get; set; }
public virtual string Variation { get; set; }
protected static string Timestamp {
get {
return Helpers.Timestamp;
}
bool? supports_parallel_execution;
public virtual bool SupportsParallelExecution {
get => supports_parallel_execution ?? true;
set => supports_parallel_execution = value;
}
public bool HasCustomTestName {
get {
return test_name != null;
}
public virtual IEnumerable<ILog> AggregatedLogs => Logs;
TestExecutingResult execution_result;
public virtual TestExecutingResult ExecutionResult {
get => execution_result;
set => execution_result = value;
}
string test_name;
@ -185,63 +194,66 @@ namespace Xharness.Jenkins.TestTasks {
}
}
public TestPlatform Platform { get; set; }
protected virtual void PropagateResults () { }
public List<Resource> Resources = new List<Resource> ();
protected virtual void LogEvent (ILog log, string text, params object [] args) => log.WriteLine (text, args);
ILog test_log;
public ILog MainLog {
get {
if (test_log == null)
test_log = Logs.Create ($"main-{Timestamp}.log", "Main log");
return test_log;
}
public virtual void Reset ()
{
test_log = null;
failure_message = null;
logs = null;
duration.Reset ();
execution_result = TestExecutingResult.NotStarted;
execute_task = null;
}
public virtual IEnumerable<ILog> AggregatedLogs {
get {
return Logs;
}
#endregion
public TestTasks ()
{
ID = Interlocked.Increment (ref counter);
}
public string LogDirectory {
get {
var rv = Path.Combine (Jenkins.LogDirectory, TestName, ID.ToString ());
Directory.CreateDirectory (rv);
return rv;
}
// VerifyRun is called in RunInternalAsync/ExecuteAsync to verify that the task can be executed/run.
// Typically used to fail tasks that don't have an available device, or if there's not enough disk space.
public virtual Task VerifyRunAsync ()
{
return VerifyDiskSpaceAsync ();
}
ILogs logs;
public ILogs Logs {
get {
return logs ?? (logs = new Logs (LogDirectory));
protected Task VerifyDiskSpaceAsync ()
{
if (Finished)
return Task.CompletedTask;
if (RootDrive == null)
RootDrive = new DriveInfo ("/");
var afs = RootDrive.AvailableFreeSpace;
const long minSpaceRequirement = 1024 * 1024 * 1024; /* 1 GB */
if (afs < minSpaceRequirement) {
FailureMessage = $"Not enough space on the root drive '{RootDrive.Name}': {afs / (1024.0 * 1024):#.##} MB left of {minSpaceRequirement / (1024.0 * 1024):#.##} MB required";
ExecutionResult = TestExecutingResult.Failed;
}
return Task.CompletedTask;
}
IEnumerable<string> referencedNunitAndXunitTestAssemblies;
public IEnumerable<string> ReferencedNunitAndXunitTestAssemblies {
get {
if (referencedNunitAndXunitTestAssemblies != null)
return referencedNunitAndXunitTestAssemblies;
public void CloneTestProject (TestProject project)
{
// Don't build in the original project directory
// We can build multiple projects in parallel, and if some of those
// projects have the same project dependencies, then we may end up
// building the same (dependent) project simultaneously (and they can
// stomp on eachother).
// So we clone the project file to a separate directory and build there instead.
// This is done asynchronously to speed to the initial test load.
TestProject = project.Clone ();
InitialTask = TestProject.CreateCopyAsync ();
}
if (TestName.Contains ("BCL tests group")) { // avoid loading unrelated projects
if (!File.Exists (ProjectFile))
return Enumerable.Empty<string> ();
var csproj = new XmlDocument ();
try {
csproj.LoadWithoutNetworkAccess (ProjectFile.Replace ("\\", "/"));
referencedNunitAndXunitTestAssemblies = csproj.GetNunitAndXunitTestReferences ();
} catch (Exception e) {
referencedNunitAndXunitTestAssemblies = new string [] { $"Exception: {e.Message}", $"Filename: {ProjectFile}" };
}
} else {
referencedNunitAndXunitTestAssemblies = Enumerable.Empty<string> ();
}
return referencedNunitAndXunitTestAssemblies;
}
}
protected Stopwatch waitingDuration = new Stopwatch ();
public TimeSpan WaitingDuration => waitingDuration.Elapsed;
Task execute_task;
async Task RunInternalAsync ()
@ -288,21 +300,7 @@ namespace Xharness.Jenkins.TestTasks {
duration.Stop ();
}
Jenkins.GenerateReport ();
}
protected virtual void PropagateResults ()
{
}
public virtual void Reset ()
{
test_log = null;
failure_message = null;
logs = null;
duration.Reset ();
execution_result = TestExecutingResult.NotStarted;
execute_task = null;
GenerateReport ();
}
public Task RunAsync ()
@ -312,62 +310,14 @@ namespace Xharness.Jenkins.TestTasks {
return execute_task;
}
protected abstract Task ExecuteAsync ();
public override string ToString ()
{
return ExecutionResult.ToString ();
}
protected void SetEnvironmentVariables (Process process)
{
var xcodeRoot = Harness.XcodeRoot;
switch (Platform) {
case TestPlatform.iOS:
case TestPlatform.iOS_Unified:
case TestPlatform.iOS_Unified32:
case TestPlatform.iOS_Unified64:
case TestPlatform.iOS_TodayExtension64:
case TestPlatform.tvOS:
case TestPlatform.watchOS:
case TestPlatform.watchOS_32:
case TestPlatform.watchOS_64_32:
process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot;
process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current");
process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks");
process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild");
break;
case TestPlatform.Mac:
case TestPlatform.Mac_Modern:
case TestPlatform.Mac_Full:
case TestPlatform.Mac_System:
process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot;
process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks");
process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPathFallbackPathsOverride"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild");
process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
break;
case TestPlatform.All:
// Don't set:
// MSBuildExtensionsPath
// TargetFrameworkFallbackSearchPaths
// because these values used by both XM and XI and we can't set it to two different values at the same time.
// Any test that depends on these values should not be using 'TestPlatform.All'
process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot;
process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current");
process.StartInfo.EnvironmentVariables ["XamarinMacFrameworkRoot"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
break;
default:
throw new NotImplementedException ();
}
foreach (var kvp in Environment)
process.StartInfo.EnvironmentVariables [kvp.Key] = kvp.Value;
}
protected void AddWrenchLogFiles (StreamReader stream)
protected void AddCILogFiles (StreamReader stream)
{
string line;
while ((line = stream.ReadLine ()) != null) {
@ -385,18 +335,12 @@ namespace Xharness.Jenkins.TestTasks {
Logs.AddFile (src);
break;
default:
Harness.HarnessLog.WriteLine ("Unknown @MonkeyWrench command in {0}: {1}", TestName, name);
WriteLineToRunnerLog ($"Unknown @MonkeyWrench command in {TestName}: {name}");
break;
}
}
}
protected void LogEvent (ILog log, string text, params object [] args)
{
Jenkins.MainLog.WriteLine (text, args);
log.WriteLine (text, args);
}
public string GuessFailureReason (ILog log)
{
try {
@ -410,7 +354,7 @@ namespace Xharness.Jenkins.TestTasks {
}
}
} catch (Exception e) {
Harness.Log ("Failed to guess failure reason: {0}", e.Message);
WriteLineToRunnerLog ($"Failed to guess failure reason: {e.Message}");
}
return null;
@ -434,20 +378,6 @@ namespace Xharness.Jenkins.TestTasks {
return rv;
}
public virtual bool SupportsParallelExecution {
get {
return supports_parallel_execution ?? true;
}
set {
supports_parallel_execution = value;
}
}
protected Task<IAcquiredResource> NotifyAndAcquireDesktopResourceAsync ()
{
return NotifyBlockingWaitAsync (SupportsParallelExecution ? Jenkins.DesktopResource.AcquireConcurrentAsync () : Jenkins.DesktopResource.AcquireExclusiveAsync ());
}
class BlockingWait : IAcquiredResource, IDisposable
{
public IAcquiredResource Wrapped;

8
tests/xharness/xharness.csproj
Просмотреть файл

@ -81,8 +81,6 @@
<Compile Include="Harness.cs" />
<Compile Include="TestReporterFactory.cs" />
<Compile Include="Jenkins\Jenkins.cs" />
<Compile Include="Jenkins\Resource.cs" />
<Compile Include="Jenkins\Resources.cs" />
<Compile Include="Jenkins\TestTasks\AggregatedRunSimulatorTask.cs" />
<Compile Include="Jenkins\TestTasks\BuildProjectTask.cs" />
<Compile Include="Jenkins\TestTasks\BuildToolTask.cs" />
@ -98,7 +96,7 @@
<Compile Include="Jenkins\TestTasks\RunTestTask.cs" />
<Compile Include="Jenkins\TestTasks\RunXITask.cs" />
<Compile Include="Jenkins\TestTasks\RunXtroTask.cs" />
<Compile Include="Jenkins\TestTasks\TestTask.cs" />
<Compile Include="Jenkins\TestTasks\AppleTestTask.cs" />
<Compile Include="MakefileGenerator.cs" />
<Compile Include="MonoNativeInfo.cs" />
<Compile Include="Program.cs" />
@ -117,6 +115,10 @@
<Compile Include="TestImporter\Xamarin\ProjectFilter.cs" />
<Compile Include="TestImporter\Xamarin\AssemblyDefinitionFactory.cs" />
<Compile Include="TestImporter\Xamarin\TestAssemblyDefinition.cs" />
<Compile Include="TestTasks\TestTask.cs" />
<Compile Include="TestTasks\IAcquiredResource.cs" />
<Compile Include="TestTasks\Resource.cs" />
<Compile Include="TestTasks\Resources.cs" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\tools\mtouch\SdkVersions.cs">