[Harness] Generalize the RunTestTask to allow it to be used in the CLI. (#8380)

Move all the logic outside and use it as a Composition pattern, later
this class can be used in the CLI so that we share the logic of building
and tested.
This commit is contained in:
Manuel de la Pena 2020-04-14 18:39:58 -04:00 коммит произвёл GitHub
Родитель af7da4b233
Коммит 969d5d92ed
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
17 изменённых файлов: 237 добавлений и 120 удалений

Просмотреть файл

@ -1,6 +1,7 @@
using System;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Xharness.TestTasks;
namespace Xharness.Jenkins.TestTasks
@ -11,6 +12,11 @@ namespace Xharness.Jenkins.TestTasks
public IProcessManager ProcessManager { get; }
public ILog BuildLog {
get => buildToolTask.BuildLog;
set => buildToolTask.BuildLog = value;
}
public override string TestName {
get => base.TestName;
set {
@ -55,7 +61,7 @@ namespace Xharness.Jenkins.TestTasks
set => buildToolTask.Mode = value;
}
protected virtual void InitializeTool () => buildToolTask = new Xharness.TestTasks.BuildTool (ProcessManager);
protected virtual void InitializeTool () => buildToolTask = new BuildTool (ProcessManager);
public virtual Task CleanAsync () => buildToolTask.CleanAsync ();
}
}

Просмотреть файл

@ -12,7 +12,7 @@ namespace Xharness.Jenkins.TestTasks {
DotNetBuildTask.SetDotNetEnvironmentVariables (Environment);
}
protected override async Task RunTestAsync ()
public override async Task RunTestAsync ()
{
using (var resource = await NotifyAndAcquireDesktopResourceAsync ()) {
var trx = Logs.Create ($"results-{Timestamp}.trx", LogType.TrxLog.ToString ());

Просмотреть файл

@ -7,8 +7,6 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
namespace Xharness.Jenkins.TestTasks {
class MSBuildTask : BuildProjectTask
{
public ILog BuildLog;
protected virtual string ToolName => Harness.XIBuildPath;
protected virtual List<string> ToolArguments =>

Просмотреть файл

@ -50,7 +50,7 @@ namespace Xharness.Jenkins.TestTasks {
}
}
protected override async Task RunTestAsync ()
public override async Task RunTestAsync ()
{
var projectDir = System.IO.Path.GetDirectoryName (ProjectFile);
var name = System.IO.Path.GetFileName (projectDir);

Просмотреть файл

@ -108,7 +108,7 @@ namespace Xharness.Jenkins.TestTasks
}
}
protected override async Task RunTestAsync ()
public override async Task RunTestAsync ()
{
using (var resource = await NotifyAndAcquireDesktopResourceAsync ()) {
var xmlLog = Logs.CreateFile ($"log-{Timestamp}.xml", LogType.XmlLog.ToString ());

Просмотреть файл

@ -68,7 +68,7 @@ namespace Xharness.Jenkins.TestTasks {
this.devices = devices ?? throw new ArgumentNullException (nameof (devices));
}
protected override async Task RunTestAsync ()
public override async Task RunTestAsync ()
{
Jenkins.MainLog.WriteLine ("Running '{0}' on device (candidates: '{1}')", ProjectFile, string.Join ("', '", Candidates.Select ((v) => v.Name).ToArray ()));

Просмотреть файл

@ -109,7 +109,7 @@ namespace Xharness.Jenkins.TestTasks {
}
}
protected override async Task RunTestAsync ()
public override async Task RunTestAsync ()
{
Jenkins.MainLog.WriteLine ("Running XI on '{0}' ({2}) for {1}", Device?.Name, ProjectFile, Device?.UDID);

Просмотреть файл

@ -1,30 +1,46 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
using Xharness.TestTasks;
namespace Xharness.Jenkins.TestTasks {
internal abstract class RunTestTask : AppleTestTask
internal abstract class RunTestTask : AppleTestTask, IRunTestTask
{
protected IProcessManager ProcessManager { get; }
IResultParser ResultParser { get; } = new XmlResultParser ();
protected RunTest runTest;
protected IProcessManager ProcessManager => runTest.ProcessManager;
public IBuildToolTask BuildTask => runTest.BuildTask;
public readonly IBuildToolTask BuildTask;
public TimeSpan Timeout = TimeSpan.FromMinutes (10);
public double TimeoutMultiplier { get; set; } = 1;
public string WorkingDirectory;
public double TimeoutMultiplier {
get => runTest.TimeoutMultiplier;
set => runTest.TimeoutMultiplier = value;
}
public string WorkingDirectory {
get => runTest.WorkingDirectory;
set => runTest.WorkingDirectory = value;
}
public TimeSpan Timeout {
get => runTest.Timeout;
set => runTest.Timeout = value;
}
public RunTestTask (Jenkins jenkins, IBuildToolTask build_task, IProcessManager processManager) : base (jenkins)
{
this.BuildTask = build_task;
this.ProcessManager = processManager ?? throw new ArgumentNullException (nameof (processManager));
runTest = new RunTest (
testTask: this,
buildTask: build_task,
processManager: processManager,
envManager: this,
mainLog: Jenkins.MainLog,
generateXmlFailures: Jenkins.Harness.InCI,
xmlResultJargon: Jenkins.Harness.XmlJargon,
dryRun: Jenkins.Harness.DryRun
);
TestProject = build_task.TestProject;
Platform = build_task.Platform;
@ -37,8 +53,8 @@ namespace Xharness.Jenkins.TestTasks {
public override IEnumerable<ILog> AggregatedLogs {
get {
var rv = base.AggregatedLogs;
if (BuildTask != null)
rv = rv.Union (BuildTask.AggregatedLogs);
if (runTest.BuildAggregatedLogs != null)
rv = rv.Union (runTest.BuildAggregatedLogs);
return rv;
}
}
@ -46,8 +62,8 @@ namespace Xharness.Jenkins.TestTasks {
public override TestExecutingResult ExecutionResult {
get {
// When building, the result is the build result.
if ((BuildTask.ExecutionResult & (TestExecutingResult.InProgress | TestExecutingResult.Waiting)) != 0)
return BuildTask.ExecutionResult & ~TestExecutingResult.InProgressMask | TestExecutingResult.Building;
if ((runTest.BuildResult & (TestExecutingResult.InProgress | TestExecutingResult.Waiting)) != 0)
return runTest.BuildResult & ~TestExecutingResult.InProgressMask | TestExecutingResult.Building;
return base.ExecutionResult;
}
set {
@ -55,57 +71,12 @@ namespace Xharness.Jenkins.TestTasks {
}
}
public async Task<bool> BuildAsync ()
{
if (Finished)
return true;
public Task<bool> BuildAsync () => runTest.BuildAsync ();
await VerifyBuildAsync ();
if (Finished)
return BuildTask.Succeeded;
protected override Task ExecuteAsync () => runTest.ExecuteAsync ();
ExecutionResult = TestExecutingResult.Building;
await BuildTask.RunAsync ();
if (!BuildTask.Succeeded) {
if (BuildTask.TimedOut) {
ExecutionResult = TestExecutingResult.TimedOut;
} else {
ExecutionResult = TestExecutingResult.BuildFailure;
}
FailureMessage = BuildTask.FailureMessage;
if (!string.IsNullOrEmpty (BuildTask.KnownFailure))
KnownFailure = BuildTask.KnownFailure;
if (Harness.InCI && BuildTask is MSBuildTask projectTask)
ResultParser.GenerateFailure (Logs, "build", projectTask.TestName, projectTask.Variation, $"App Build {projectTask.TestName} {projectTask.Variation}", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, Harness.XmlJargon);
} else {
ExecutionResult = TestExecutingResult.Built;
}
return BuildTask.Succeeded;
}
public abstract Task RunTestAsync ();
protected override async Task ExecuteAsync ()
{
if (Finished)
return;
await VerifyRunAsync ();
if (Finished)
return;
if (!await BuildAsync ())
return;
if (BuildOnly) {
ExecutionResult = TestExecutingResult.BuildSucceeded;
return;
}
ExecutionResult = TestExecutingResult.Running;
duration.Restart (); // don't count the build time.
await RunTestAsync ();
}
protected abstract Task RunTestAsync ();
// VerifyBuild is called in BuildAsync to verify that the task can be built.
// Typically used to fail tasks if there's not enough disk space.
public virtual Task VerifyBuildAsync ()
@ -116,7 +87,7 @@ namespace Xharness.Jenkins.TestTasks {
public override void Reset ()
{
base.Reset ();
BuildTask.Reset ();
runTest.Reset ();
}
protected Task ExecuteProcessAsync (string filename, List<string> arguments)
@ -124,35 +95,12 @@ namespace Xharness.Jenkins.TestTasks {
return ExecuteProcessAsync (null, filename, arguments);
}
protected async Task ExecuteProcessAsync (ILog log, string filename, List<string> arguments)
protected Task ExecuteProcessAsync (ILog log, string filename, List<string> arguments)
{
if (log == null)
log = Logs.Create ($"execute-{Timestamp}.txt", LogType.ExecutionLog.ToString ());
using var proc = new Process ();
proc.StartInfo.FileName = filename;
proc.StartInfo.Arguments = StringUtils.FormatArguments (arguments);
if (!string.IsNullOrEmpty (WorkingDirectory))
proc.StartInfo.WorkingDirectory = WorkingDirectory;
SetEnvironmentVariables (proc);
foreach (DictionaryEntry de in proc.StartInfo.EnvironmentVariables)
log.WriteLine ($"export {de.Key}={de.Value}");
Jenkins.MainLog.WriteLine ("Executing {0} ({1})", TestName, Mode);
if (!Harness.DryRun) {
ExecutionResult = TestExecutingResult.Running;
var result = await ProcessManager.RunAsync (proc, log, Timeout);
if (result.TimedOut) {
FailureMessage = $"Execution timed out after {Timeout.TotalMinutes} minutes.";
log.WriteLine (FailureMessage);
ExecutionResult = TestExecutingResult.TimedOut;
} else if (result.Succeeded) {
ExecutionResult = TestExecutingResult.Succeeded;
} else {
ExecutionResult = TestExecutingResult.Failed;
FailureMessage = $"Execution failed with exit code {result.ExitCode}";
}
}
Jenkins.MainLog.WriteLine ("Executed {0} ({1})", TestName, Mode);
return runTest.ExecuteProcessAsync (log, filename, arguments);
}
}
}

Просмотреть файл

@ -13,7 +13,7 @@ namespace Xharness.Jenkins.TestTasks {
{
}
protected override async Task RunTestAsync ()
public override async Task RunTestAsync ()
{
var projectDir = System.IO.Path.GetDirectoryName (ProjectFile);
var name = System.IO.Path.GetFileName (projectDir);

Просмотреть файл

@ -1,6 +1,7 @@
using System;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
namespace Xharness.TestTasks {
@ -10,6 +11,7 @@ namespace Xharness.TestTasks {
public IProcessManager ProcessManager { get; }
public TestPlatform Platform { get; set; }
public TestProject TestProject { get; set; }
public ILog BuildLog { get; set; }
public bool SpecifyPlatform { get; set; } = true;
public bool SpecifyConfiguration { get; set; } = true;

Просмотреть файл

@ -1,5 +1,6 @@
using System.Threading.Tasks;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
namespace Xharness.TestTasks {
@ -7,13 +8,15 @@ namespace Xharness.TestTasks {
/// Interface to be implemented by those tasks that represent a build execution.
/// </summary>
public interface IBuildToolTask : ITestTask {
IProcessManager ProcessManager { get; }
string TestName { get; set; }
ILog BuildLog { get; }
bool SpecifyPlatform { get; set; }
bool SpecifyConfiguration { get; set; }
TestProject TestProject { get; set; }
IProcessManager ProcessManager { get; }
TestPlatform Platform { get; set; }
string Mode { get; set; }
public Task CleanAsync ();
TestProject TestProject { get; set; }
Task CleanAsync ();
}
}

Просмотреть файл

@ -0,0 +1,9 @@
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
namespace Xharness.TestTasks {
public interface IRunTestTask : ITestTask {
Task RunTestAsync ();
Task VerifyBuildAsync ();
}
}

Просмотреть файл

@ -1,5 +1,6 @@
using System;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
@ -12,13 +13,17 @@ namespace Xharness.TestTasks {
bool Failed { get; }
bool Ignored { get; set; }
bool TimedOut { get; }
bool Finished { get; }
bool BuildOnly { get; set; }
string KnownFailure { get; set; }
string ProjectConfiguration { get; set; }
string ProjectPlatform { get; set; }
string ProjectFile { get; }
public string FailureMessage { get; set; }
string Mode { get; set; }
string Variation { get; set; }
string TestName { get; }
string FailureMessage { get; set; }
TimeSpan Duration { get; }
@ -26,9 +31,11 @@ namespace Xharness.TestTasks {
TestExecutingResult ExecutionResult { get; set; }
IEnumerable<ILog> AggregatedLogs { get; }
ILogs Logs { get; }
Stopwatch DurationStopWatch { get; }
Task RunAsync ();
Task VerifyRunAsync ();
void Reset ();
}
}

Просмотреть файл

@ -48,6 +48,7 @@ namespace Xharness.TestTasks {
ILog buildLog,
ILog mainLog)
{
BuildLog = buildLog;
(TestExecutingResult ExecutionResult, string KnownFailure) result = (TestExecutingResult.NotStarted, (string) null);
await RestoreNugetsAsync (buildLog, resource, useXIBuild: true);

Просмотреть файл

@ -0,0 +1,140 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
using Microsoft.DotNet.XHarness.iOS.Shared;
using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
namespace Xharness.TestTasks {
public class RunTest {
public IProcessManager ProcessManager { get; }
public IBuildToolTask BuildTask { get; private set; }
IResultParser ResultParser { get; } = new XmlResultParser ();
readonly IRunTestTask testTask;
readonly IEnvManager envManager;
readonly ILog mainLog;
readonly bool generateXmlFailures;
readonly bool dryRun;
readonly XmlResultJargon xmlResultJargon;
public TimeSpan Timeout { get; set; } = TimeSpan.FromMinutes (10);
public double TimeoutMultiplier { get; set; } = 1;
public string WorkingDirectory;
public RunTest (IRunTestTask testTask,
IBuildToolTask buildTask,
IProcessManager processManager,
IEnvManager envManager,
ILog mainLog,
bool generateXmlFailures,
XmlResultJargon xmlResultJargon, bool dryRun)
{
this.testTask = testTask ?? throw new ArgumentNullException (nameof (testTask));
this.BuildTask = buildTask ?? throw new ArgumentNullException (nameof (buildTask));
this.ProcessManager = processManager ?? throw new ArgumentNullException (nameof (processManager));
this.envManager = envManager ?? throw new ArgumentNullException (nameof (envManager));
this.mainLog = mainLog ?? throw new ArgumentNullException (nameof (mainLog));
this.generateXmlFailures = generateXmlFailures;
this.dryRun = dryRun;
this.xmlResultJargon = xmlResultJargon;
}
public IEnumerable<ILog> BuildAggregatedLogs => BuildTask.AggregatedLogs;
public TestExecutingResult BuildResult => BuildTask.ExecutionResult;
public async Task<bool> BuildAsync ()
{
if (testTask.Finished)
return true;
await testTask.VerifyBuildAsync ();
if (testTask.Finished)
return BuildTask.Succeeded;
testTask.ExecutionResult = TestExecutingResult.Building;
await BuildTask.RunAsync ();
if (!BuildTask.Succeeded) {
if (BuildTask.TimedOut) {
testTask.ExecutionResult = TestExecutingResult.TimedOut;
} else {
testTask.ExecutionResult = TestExecutingResult.BuildFailure;
}
testTask.FailureMessage = BuildTask.FailureMessage;
if (!string.IsNullOrEmpty (BuildTask.KnownFailure))
testTask.KnownFailure = BuildTask.KnownFailure;
if (generateXmlFailures)
ResultParser.GenerateFailure (
logs: testTask.Logs,
source: "build",
appName: testTask.TestName,
variation: testTask.Variation,
title: $"App Build {testTask.TestName} {testTask.Variation}",
message: $"App could not be built {testTask.FailureMessage}.",
stderrPath: BuildTask.BuildLog.FullPath,
jargon: xmlResultJargon);
} else {
testTask.ExecutionResult = TestExecutingResult.Built;
}
return BuildTask.Succeeded;
}
public async Task ExecuteAsync ()
{
if (testTask.Finished)
return;
await testTask.VerifyRunAsync ();
if (testTask.Finished)
return;
if (!await BuildAsync ())
return;
if (testTask.BuildOnly) {
testTask.ExecutionResult = TestExecutingResult.BuildSucceeded;
return;
}
testTask.ExecutionResult = TestExecutingResult.Running;
testTask.DurationStopWatch.Restart (); // don't count the build time.
await testTask.RunTestAsync ();
}
public void Reset () => BuildTask.Reset ();
public async Task ExecuteProcessAsync (ILog log, string filename, List<string> arguments)
{
if (log == null)
throw new ArgumentNullException (nameof (log));
using var proc = new Process ();
proc.StartInfo.FileName = filename;
proc.StartInfo.Arguments = StringUtils.FormatArguments (arguments);
if (!string.IsNullOrEmpty (WorkingDirectory))
proc.StartInfo.WorkingDirectory = WorkingDirectory;
envManager.SetEnvironmentVariables (proc);
foreach (DictionaryEntry de in proc.StartInfo.EnvironmentVariables)
log.WriteLine ($"export {de.Key}={de.Value}");
mainLog.WriteLine ("Executing {0} ({1})", testTask.TestName, testTask.Mode);
if (!dryRun) {
testTask.ExecutionResult = TestExecutingResult.Running;
var result = await ProcessManager.RunAsync (proc, log, Timeout);
if (result.TimedOut) {
testTask.FailureMessage = $"Execution timed out after {Timeout.TotalMinutes} minutes.";
log.WriteLine (testTask.FailureMessage);
testTask.ExecutionResult = TestExecutingResult.TimedOut;
} else if (result.Succeeded) {
testTask.ExecutionResult = TestExecutingResult.Succeeded;
} else {
testTask.ExecutionResult = TestExecutingResult.Failed;
testTask.FailureMessage = $"Execution failed with exit code {result.ExitCode}";
}
}
mainLog.WriteLine ("Executed {0} ({1})", testTask.TestName, testTask.Mode);
}
}
}

Просмотреть файл

@ -19,10 +19,6 @@ namespace Xharness.TestTasks {
#region Public vars
public readonly int ID;
public bool BuildOnly;
public string KnownFailure { get; set; }
public string ProjectConfiguration { get; set; }
public string ProjectPlatform { get; set; }
public Dictionary<string, string> Environment = new Dictionary<string, string> ();
public Func<Task> Dependency; // a task that's feteched and awaited before this task's ExecuteAsync method
public Task InitialTask { get; set; } // a task that's executed before this task's ExecuteAsync method.
@ -33,6 +29,11 @@ namespace Xharness.TestTasks {
#region Properties
public bool BuildOnly { get; set; }
public string KnownFailure { get; set; }
public string ProjectConfiguration { get; set; }
public string ProjectPlatform { get; set; }
protected static string Timestamp => Helpers.Timestamp;
public string ProjectFile => TestProject?.Path;
public bool HasCustomTestName => test_name != null;
@ -64,10 +65,10 @@ namespace Xharness.TestTasks {
public bool BuildFailure => (ExecutionResult & TestExecutingResult.BuildFailure) == TestExecutingResult.BuildFailure;
public bool HarnessException => (ExecutionResult & TestExecutingResult.HarnessException) == TestExecutingResult.HarnessException;
protected Stopwatch duration = new Stopwatch ();
public Stopwatch DurationStopWatch { get; } = new Stopwatch ();
public TimeSpan Duration {
get {
return duration.Elapsed;
return DurationStopWatch.Elapsed;
}
}
@ -204,7 +205,7 @@ namespace Xharness.TestTasks {
test_log = null;
failure_message = null;
logs = null;
duration.Reset ();
DurationStopWatch.Reset ();
execution_result = TestExecutingResult.NotStarted;
execute_task = null;
}
@ -275,7 +276,7 @@ namespace Xharness.TestTasks {
if (Finished)
return;
duration.Start ();
DurationStopWatch.Start ();
execute_task = ExecuteAsync ();
await execute_task;
@ -298,7 +299,7 @@ namespace Xharness.TestTasks {
PropagateResults ();
} finally {
logs?.Dispose ();
duration.Stop ();
DurationStopWatch.Stop ();
}
GenerateReport ();
@ -368,14 +369,14 @@ namespace Xharness.TestTasks {
var rv = new BlockingWait ();
// Stop the timer while we're waiting for a resource
duration.Stop ();
DurationStopWatch.Stop ();
waitingDuration.Start ();
ExecutionResult = ExecutionResult | TestExecutingResult.Waiting;
rv.Wrapped = await task;
ExecutionResult = ExecutionResult & ~TestExecutingResult.Waiting;
waitingDuration.Stop ();
duration.Start ();
rv.OnDispose = duration.Stop;
DurationStopWatch.Start ();
rv.OnDispose = DurationStopWatch.Stop;
return rv;
}

Просмотреть файл

@ -129,6 +129,8 @@
<Compile Include="TestTasks\DotNetBuild.cs" />
<Compile Include="TestTasks\IBuildToolTask.cs" />
<Compile Include="TestTasks\ITestTask.cs" />
<Compile Include="TestTasks\RunTest.cs" />
<Compile Include="TestTasks\IRunTestTask.cs" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\tools\common\SdkVersions.cs">