[Harness] Generalize XmlResultParser. (#8156)

First step to try and clean up the psring of results and ensure that we
are doing the correct thing. We have several issue there. We want to
move all the complicated logic out of AppRunner and test the different
outputs and the full result matrix.

This commit first allows to use an instance class for the parsing, later
we will move things out. Step by step
This commit is contained in:
Manuel de la Pena 2020-03-19 17:42:51 -04:00 коммит произвёл GitHub
Родитель a4b71841df
Коммит 79a089cd0d
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
12 изменённых файлов: 107 добавлений и 37 удалений

Просмотреть файл

@ -56,6 +56,7 @@ namespace Xharness {
readonly IDeviceLoaderFactory devicesLoaderFactory;
readonly ICaptureLogFactory captureLogFactory;
readonly IDeviceLogCapturerFactory deviceLogCapturerFactory;
readonly IResultParser resultParser;
readonly RunMode mode;
readonly bool isSimulator;
readonly AppRunnerTarget target;
@ -96,6 +97,7 @@ namespace Xharness {
IDeviceLoaderFactory devicesFactory,
ICaptureLogFactory captureLogFactory,
IDeviceLogCapturerFactory deviceLogCapturerFactory,
IResultParser resultParser,
AppRunnerTarget target,
IHarness harness,
ILog mainLog,
@ -116,6 +118,7 @@ namespace Xharness {
this.devicesLoaderFactory = devicesFactory ?? throw new ArgumentNullException (nameof (devicesFactory));
this.captureLogFactory = captureLogFactory ?? throw new ArgumentNullException (nameof (captureLogFactory));
this.deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException (nameof (deviceLogCapturerFactory));
this.resultParser = resultParser ?? throw new ArgumentNullException (nameof (resultParser));
this.harness = harness ?? throw new ArgumentNullException (nameof (harness));
this.MainLog = mainLog ?? throw new ArgumentNullException (nameof (mainLog));
this.projectFilePath = projectFilePath ?? throw new ArgumentNullException (nameof (projectFilePath));
@ -309,13 +312,13 @@ namespace Xharness {
// from the TCP connection, we are going to fail when trying to read it and not parse it. Therefore, we are not only
// going to check if we are in CI, but also if the listener_log is valid.
var path = Path.ChangeExtension (test_log_path, "xml");
XmlResultParser.CleanXml (test_log_path, path);
resultParser.CleanXml (test_log_path, path);
if (harness.InCI && XmlResultParser.IsValidXml (path, out var xmlType)) {
if (harness.InCI && resultParser.IsValidXml (path, out var xmlType)) {
(string resultLine, bool failed, bool crashed) parseResult = (null, false, false);
crashed = false;
try {
var newFilename = XmlResultParser.GetXmlFilePath (path, xmlType);
var newFilename = resultParser.GetXmlFilePath (path, xmlType);
// at this point, we have the test results, but we want to be able to have attachments in vsts, so if the format is
// the right one (NUnitV3) add the nodes. ATM only TouchUnit uses V3.
@ -330,7 +333,7 @@ namespace Xharness {
// add a final prefix to the file name to make sure that the VSTS test uploaded just pick
// the final version, else we will upload tests more than once
newFilename = XmlResultParser.GetVSTSFilename (newFilename);
XmlResultParser.UpdateMissingData (path, newFilename, testRunName, logFiles);
resultParser.UpdateMissingData (path, newFilename, testRunName, logFiles);
} else {
// rename the path to the correct value
File.Move (path, newFilename);
@ -339,7 +342,7 @@ namespace Xharness {
// write the human readable results in a tmp file, which we later use to step on the logs
var tmpFile = Path.Combine (Path.GetTempPath (), Guid.NewGuid ().ToString ());
(parseResult.resultLine, parseResult.failed) = XmlResultParser.GenerateHumanReadableResults (path, tmpFile, xmlType);
(parseResult.resultLine, parseResult.failed) = resultParser.GenerateHumanReadableResults (path, tmpFile, xmlType);
File.Copy (tmpFile, test_log_path, true);
File.Delete (tmpFile);
@ -787,7 +790,7 @@ namespace Xharness {
if (crash_reason != null) {
// if in CI, do write an xml error that will be picked as a failure by VSTS
if (harness.InCI) {
XmlResultParser.GenerateFailure (Logs,
resultParser.GenerateFailure (Logs,
"crash",
AppInformation.AppName,
variation,
@ -810,7 +813,7 @@ namespace Xharness {
FailureMessage = $"Killed by the OS ({crash_reason})";
}
if (harness.InCI) {
XmlResultParser.GenerateFailure (
resultParser.GenerateFailure (
Logs,
"crash",
AppInformation.AppName,
@ -824,7 +827,7 @@ namespace Xharness {
// same as with a crash
FailureMessage = $"Launch failure";
if (harness.InCI) {
XmlResultParser.GenerateFailure (
resultParser.GenerateFailure (
Logs,
"launch",
AppInformation.AppName,
@ -850,7 +853,7 @@ namespace Xharness {
}
if (isTcp) {
XmlResultParser.GenerateFailure (Logs,
resultParser.GenerateFailure (Logs,
"tcp-connection",
AppInformation.AppName,
variation,
@ -860,7 +863,7 @@ namespace Xharness {
harness.XmlJargon);
}
} else if (timed_out && harness.InCI) {
XmlResultParser.GenerateFailure (Logs,
resultParser.GenerateFailure (Logs,
"timeout",
AppInformation.AppName,
variation,

Просмотреть файл

@ -112,6 +112,7 @@ namespace Xharness
public HashSet<string> Labels { get; }
public XmlResultJargon XmlJargon { get; }
public IProcessManager ProcessManager { get; }
public IResultParser ResultParser { get; }
// This is the maccore/tests directory.
static string root_directory;
@ -185,9 +186,10 @@ namespace Xharness
public string GetStandardErrorTty () => Helpers.GetTerminalName (2);
public Harness (IProcessManager processManager, HarnessAction action, HarnessConfiguration configuration)
public Harness (IProcessManager processManager, IResultParser resultParser, HarnessAction action, HarnessConfiguration configuration)
{
ProcessManager = processManager ?? throw new ArgumentNullException (nameof (processManager));
ResultParser = resultParser ?? throw new ArgumentNullException (nameof (resultParser));
Action = action;
if (configuration is null)
@ -601,6 +603,7 @@ namespace Xharness
new DeviceLoaderFactory (this, ProcessManager),
new CaptureLogFactory (),
new DeviceLogCapturerFactory (ProcessManager, XcodeRoot, MlaunchPath),
new XmlResultParser (),
target,
this,
HarnessLog,
@ -628,6 +631,7 @@ namespace Xharness
new DeviceLoaderFactory (this, ProcessManager),
new CaptureLogFactory (),
new DeviceLogCapturerFactory (ProcessManager, XcodeRoot, MlaunchPath),
new XmlResultParser (),
target,
this,
HarnessLog,
@ -653,6 +657,7 @@ namespace Xharness
new DeviceLoaderFactory (this, ProcessManager),
new CaptureLogFactory (),
new DeviceLogCapturerFactory (ProcessManager, XcodeRoot, MlaunchPath),
new XmlResultParser (),
target,
this,
HarnessLog,
@ -732,7 +737,7 @@ namespace Xharness
AutoConfigureMac (false);
}
var jenkins = new Jenkins.Jenkins (this, ProcessManager);
var jenkins = new Jenkins.Jenkins (this, ProcessManager, ResultParser);
return jenkins.Run ();
}

Просмотреть файл

@ -0,0 +1,35 @@
using System;
using System.Collections.Generic;
using System.IO;
using Xharness.Logging;
namespace Xharness {
// Interface that represents an object that know how to parse results and generate timeout/crash/build errors so
// that CIs like VSTS and helix can parse them.
public interface IResultParser {
// generates a xml result that will consider to be an error by the CI. Allows to catch errors in cases in which we are not talking about a test
// failure perse but the situation in which the app could not be built, timeout or crashed.
void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, string stderrPath, XmlResultJargon jargon);
// updates a given xml result to contain a list of attachments. This is useful for CI to be able to add logs as part of the attachments of a failing test.
void UpdateMissingData (string source, string destination, string applicationName, IEnumerable<string> attachments);
// ensures that the given path contains a valid xml result and set the type of xml jargon found in the file.
bool IsValidXml (string path, out XmlResultJargon type);
// takes a xml file and removes any extra data that makes the test result not to be a pure xml result for the given jargon.
void CleanXml (string source, string destination);
// Returns the path to be used for the given jargon.
string GetXmlFilePath (string path, XmlResultJargon xmlType);
// parses the xml of the given jargon, create a human readable result and returns a result line with the summary of what was
// parsed.
(string resultLine, bool failed) GenerateHumanReadableResults (string source, string destination, XmlResultJargon xmlType);
// generated a human readable test report.
void GenerateTestReport (StreamWriter writer, string resultsPath, XmlResultJargon xmlType);
}
}

Просмотреть файл

@ -20,6 +20,7 @@ namespace Xharness.Jenkins
readonly ISimulatorsLoader simulators;
readonly IDeviceLoader devices;
readonly IProcessManager processManager;
readonly IResultParser resultParser;
bool populating = true;
public Harness Harness { get; }
@ -97,9 +98,10 @@ namespace Xharness.Jenkins
return new Resources (resources);
}
public Jenkins (Harness harness, IProcessManager processManager)
public Jenkins (Harness harness, IProcessManager processManager, IResultParser resultParser)
{
this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager));
this.resultParser = resultParser ?? throw new ArgumentNullException (nameof (resultParser));
Harness = harness ?? throw new ArgumentNullException (nameof (harness));
simulators = new Simulators (harness, processManager);
devices = new Devices (harness, processManager);
@ -2356,8 +2358,8 @@ namespace Xharness.Jenkins
} else if (log.Description == LogType.NUnitResult.ToString () || log.Description == LogType.XmlLog.ToString () ) {
try {
if (File.Exists (log.FullPath) && new FileInfo (log.FullPath).Length > 0) {
if (XmlResultParser.IsValidXml (log.FullPath, out var jargon)) {
XmlResultParser.GenerateTestReport (writer, log.FullPath, jargon);
if (resultParser.IsValidXml (log.FullPath, out var jargon)) {
resultParser.GenerateTestReport (writer, log.FullPath, jargon);
}
}
} catch (Exception ex) {

Просмотреть файл

@ -12,6 +12,7 @@ namespace Xharness.Jenkins.TestTasks
class RunDeviceTask : RunXITask<IHardwareDevice>
{
readonly IProcessManager processManager = new ProcessManager ();
readonly IResultParser resultParser = new XmlResultParser ();
readonly IDeviceLoader devices;
AppInstallMonitorLog install_log;
@ -86,6 +87,7 @@ namespace Xharness.Jenkins.TestTasks
new DeviceLoaderFactory (Harness, processManager),
new CaptureLogFactory (),
new DeviceLogCapturerFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath),
new XmlResultParser (),
AppRunnerTarget,
Harness,
projectFilePath: ProjectFile,
@ -118,7 +120,7 @@ namespace Xharness.Jenkins.TestTasks
FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
ExecutionResult = TestExecutingResult.Failed;
if (Harness.InCI)
XmlResultParser.GenerateFailure (Logs, "install", runner.AppInformation.AppName, Variation,
resultParser.GenerateFailure (Logs, "install", runner.AppInformation.AppName, Variation,
$"AppInstallation on {Device.Name}", $"Install failed on {Device.Name}, exit code: {install_result.ExitCode}",
install_log.FullPath, Harness.XmlJargon);
}
@ -150,6 +152,7 @@ namespace Xharness.Jenkins.TestTasks
new DeviceLoaderFactory (Harness, processManager),
new CaptureLogFactory (),
new DeviceLogCapturerFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath),
new XmlResultParser (),
AppRunnerTarget,
Harness,
projectFilePath: ProjectFile,

Просмотреть файл

@ -83,6 +83,7 @@ namespace Xharness.Jenkins.TestTasks
new DeviceLoaderFactory (Harness, processManager),
new CaptureLogFactory (),
new DeviceLogCapturerFactory (processManager, Harness.XcodeRoot, Harness.MlaunchPath),
new XmlResultParser (),
AppRunnerTarget,
Harness,
mainLog: Logs.Create ($"run-{Device.UDID}-{Timestamp}.log", "Run log"),

Просмотреть файл

@ -16,6 +16,7 @@ namespace Xharness.Jenkins.TestTasks
public TimeSpan Timeout = TimeSpan.FromMinutes (10);
public double TimeoutMultiplier { get; set; } = 1;
IProcessManager ProcessManager { get; } = new ProcessManager ();
IResultParser ResultParser { get; } = new XmlResultParser ();
public string WorkingDirectory;
public RunTestTask (BuildToolTask build_task)
@ -71,7 +72,7 @@ namespace Xharness.Jenkins.TestTasks
}
FailureMessage = BuildTask.FailureMessage;
if (Harness.InCI && BuildTask is MSBuildTask projectTask)
XmlResultParser.GenerateFailure (Logs, "build", projectTask.TestName, projectTask.Variation, $"App Build {projectTask.TestName} {projectTask.Variation}", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, Harness.XmlJargon);
ResultParser.GenerateFailure (Logs, "build", projectTask.TestName, projectTask.Variation, $"App Build {projectTask.TestName} {projectTask.Variation}", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, Harness.XmlJargon);
} else {
ExecutionResult = TestExecutingResult.Built;
}

Просмотреть файл

@ -105,7 +105,7 @@ namespace Xharness {
// XS sets this, which breaks pretty much everything if it doesn't match what was passed to --sdkroot.
Environment.SetEnvironmentVariable ("XCODE_DEVELOPER_DIR_PATH", null);
var harness = new Harness (new ProcessManager(), action, configuration);
var harness = new Harness (new ProcessManager(), new XmlResultParser (), action, configuration);
return harness.Execute ();
}

Просмотреть файл

@ -93,6 +93,7 @@ namespace Xharness.Tests {
devicesFactory,
Mock.Of<ICaptureLogFactory> (),
Mock.Of<IDeviceLogCapturerFactory> (),
Mock.Of<IResultParser> (),
AppRunnerTarget.Simulator_iOS64,
new Mock<IHarness> ().Object,
new Mock<ILog>().Object,
@ -115,6 +116,7 @@ namespace Xharness.Tests {
devicesFactory,
Mock.Of<ICaptureLogFactory> (),
Mock.Of<IDeviceLogCapturerFactory> (),
Mock.Of<IResultParser> (),
AppRunnerTarget.Simulator_iOS64,
new Mock<IHarness> ().Object,
new Mock<ILog>().Object,
@ -136,6 +138,7 @@ namespace Xharness.Tests {
devicesFactory,
Mock.Of<ICaptureLogFactory> (),
Mock.Of<IDeviceLogCapturerFactory> (),
Mock.Of<IResultParser> (),
AppRunnerTarget.Simulator_iOS64,
new Mock<IHarness> ().Object,
new Mock<ILog>().Object,
@ -157,6 +160,7 @@ namespace Xharness.Tests {
devicesFactory,
Mock.Of<ICaptureLogFactory> (),
Mock.Of<IDeviceLogCapturerFactory> (),
Mock.Of<IResultParser> (),
AppRunnerTarget.Device_iOS,
new Mock<IHarness> ().Object,
new Mock<ILog>().Object,
@ -197,6 +201,7 @@ namespace Xharness.Tests {
devicesFactory,
Mock.Of<ICaptureLogFactory> (),
Mock.Of<IDeviceLogCapturerFactory> (),
Mock.Of<IResultParser> (),
AppRunnerTarget.Device_iOS,
harnessMock.Object,
mainLog,
@ -256,6 +261,7 @@ namespace Xharness.Tests {
devicesFactory,
Mock.Of<ICaptureLogFactory> (),
Mock.Of<IDeviceLogCapturerFactory> (),
Mock.Of<IResultParser> (),
AppRunnerTarget.Device_iOS,
harnessMock.Object,
mainLog,

Просмотреть файл

@ -19,6 +19,20 @@ namespace Xharness.Tests
[XmlResultJargon.xUnit] = ValidatexUnitFailure,
};
XmlResultParser resultParser;
[SetUp]
public void SetUp ()
{
resultParser = new XmlResultParser ();
}
[TearDown]
public void TearDown ()
{
resultParser = null;
}
string CreateResultSample (XmlResultJargon jargon, bool includePing = false)
{
string sampleFileName = null;
@ -56,7 +70,7 @@ namespace Xharness.Tests
{
var path = Path.GetTempFileName ();
File.Delete (path);
Assert.IsFalse (XmlResultParser.IsValidXml (path, out var jargon), "missing file");
Assert.IsFalse (resultParser.IsValidXml (path, out var jargon), "missing file");
}
[TestCase (XmlResultJargon.NUnitV2)]
@ -66,7 +80,7 @@ namespace Xharness.Tests
public void IsValidXmlTest (XmlResultJargon jargon)
{
var path = CreateResultSample (jargon);
Assert.IsTrue (XmlResultParser.IsValidXml (path, out var resultJargon), "is valid");
Assert.IsTrue (resultParser.IsValidXml (path, out var resultJargon), "is valid");
Assert.AreEqual (jargon, resultJargon, "jargon");
File.Delete (path);
}
@ -79,7 +93,7 @@ namespace Xharness.Tests
public void GetXmlFilePathTest (string prefix, XmlResultJargon jargon)
{
var orignialPath = "/path/to/a/xml/result.xml";
var xmlPath = XmlResultParser.GetXmlFilePath (orignialPath, jargon);
var xmlPath = resultParser.GetXmlFilePath (orignialPath, jargon);
var fileName = Path.GetFileName (xmlPath);
StringAssert.StartsWith (prefix, fileName, "xml prefix");
}
@ -91,8 +105,8 @@ namespace Xharness.Tests
{
var path = CreateResultSample (jargon, includePing: true);
var cleanPath = path + "_clean";
XmlResultParser.CleanXml (path, cleanPath);
Assert.IsTrue (XmlResultParser.IsValidXml (cleanPath, out var resultJargon), "is valid");
resultParser.CleanXml (path, cleanPath);
Assert.IsTrue (resultParser.IsValidXml (cleanPath, out var resultJargon), "is valid");
Assert.AreEqual (jargon, resultJargon, "jargon");
File.Delete (path);
File.Delete (cleanPath);
@ -104,8 +118,8 @@ namespace Xharness.Tests
// similar to CleanXmlPingTest but using TouchUnit, so we do not want to see the extra nodes
var path = CreateResultSample (XmlResultJargon.TouchUnit, includePing: true);
var cleanPath = path + "_clean";
XmlResultParser.CleanXml (path, cleanPath);
Assert.IsTrue (XmlResultParser.IsValidXml (cleanPath, out var resultJargon), "is valid");
resultParser.CleanXml (path, cleanPath);
Assert.IsTrue (resultParser.IsValidXml (cleanPath, out var resultJargon), "is valid");
Assert.AreEqual (XmlResultJargon.NUnitV2, resultJargon, "jargon");
// load the xml, ensure we do not have the nodes we removed
var doc = XDocument.Load (cleanPath);
@ -121,10 +135,10 @@ namespace Xharness.Tests
string appName = "TestApp";
var path = CreateResultSample (XmlResultJargon.NUnitV3);
var cleanPath = path + "_clean";
XmlResultParser.CleanXml (path, cleanPath);
resultParser.CleanXml (path, cleanPath);
var updatedXml = path + "_updated";
var logs = new [] { "/first/path", "/second/path", "/last/path" };
XmlResultParser.UpdateMissingData (cleanPath, updatedXml, appName, logs);
resultParser.UpdateMissingData (cleanPath, updatedXml, appName, logs);
// assert that the required info was updated
Assert.IsTrue (File.Exists (updatedXml), "file exists");
var doc = XDocument.Load (updatedXml);
@ -273,7 +287,7 @@ namespace Xharness.Tests
// return the two temp files so that we can later validate that everything is present
_ = xmlLogMock.Setup (xmlLog => xmlLog.FullPath).Returns (finalPath);
XmlResultParser.GenerateFailure (logs.Object, src, appName, variation, title, message, stderrPath, jargon);
resultParser.GenerateFailure (logs.Object, src, appName, variation, title, message, stderrPath, jargon);
// actual assertions do happen in the validation functions
ValidationMap [jargon] (src, appName, variation, title, message, stderrMessage, finalPath, failureLogs.Length);

Просмотреть файл

@ -17,10 +17,10 @@ namespace Xharness {
Missing,
}
public static class XmlResultParser {
public class XmlResultParser : IResultParser {
// test if the file is valid xml, or at least, that can be read it.
public static bool IsValidXml (string path, out XmlResultJargon type)
public bool IsValidXml (string path, out XmlResultJargon type)
{
type = XmlResultJargon.Missing;
if (!File.Exists (path))
@ -280,7 +280,7 @@ namespace Xharness {
return (resultLine, total == 0 | errors != 0 || failed != 0);
}
public static string GetXmlFilePath (string path, XmlResultJargon xmlType)
public string GetXmlFilePath (string path, XmlResultJargon xmlType)
{
var fileName = Path.GetFileName (path);
switch (xmlType) {
@ -295,7 +295,7 @@ namespace Xharness {
}
}
public static void CleanXml (string source, string destination)
public void CleanXml (string source, string destination)
{
using (var reader = new StreamReader (source))
using (var writer = new StreamWriter (destination)) {
@ -313,7 +313,7 @@ namespace Xharness {
}
}
public static (string resultLine, bool failed) GenerateHumanReadableResults (string source, string destination, XmlResultJargon xmlType)
public (string resultLine, bool failed) GenerateHumanReadableResults (string source, string destination, XmlResultJargon xmlType)
{
(string resultLine, bool failed) parseData;
using (var reader = new StreamReader (source))
@ -475,7 +475,7 @@ namespace Xharness {
}
}
public static void GenerateTestReport (StreamWriter writer, string resultsPath, XmlResultJargon xmlType)
public void GenerateTestReport (StreamWriter writer, string resultsPath, XmlResultJargon xmlType)
{
using (var stream = new StreamReader (resultsPath))
using (var reader = XmlReader.Create (stream)) {
@ -498,7 +498,7 @@ namespace Xharness {
}
// get the file, parse it and add the attachments to the first node found
public static void UpdateMissingData (string source, string destination, string applicationName, IEnumerable<string> attachments)
public void UpdateMissingData (string source, string destination, string applicationName, IEnumerable<string> attachments)
{
// we could do this with a XmlReader and a Writer, but might be to complicated to get right, we pay with performance what we
// cannot pay with brain cells.
@ -715,7 +715,6 @@ namespace Xharness {
writer.WriteEndElement (); // collection
writer.WriteEndElement (); // assembly
writer.WriteEndElement (); // assemblies
}
static void GenerateFailureXml (string destination, string title, string message, string stderrPath, XmlResultJargon jargon)
@ -740,7 +739,7 @@ namespace Xharness {
}
}
public static void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, string stderrPath, XmlResultJargon jargon)
public void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, string stderrPath, XmlResultJargon jargon)
{
// VSTS does not provide a nice way to report build errors, create a fake
// test result with a failure in the case the build did not work

Просмотреть файл

@ -170,6 +170,7 @@
<Compile Include="BCLTestImporter\ITestAssemblyDefinition.cs" />
<Compile Include="BCLTestImporter\Xamarin\AssemblyDefinitionFactory.cs" />
<Compile Include="BCLTestImporter\Xamarin\TestAssemblyDefinition.cs" />
<Compile Include="IResultParser.cs" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\tools\mtouch\SdkVersions.cs">