[Xharness] We support diff xml outputs. Allow to pass the desired one. (#7971)
We support different outputs, lets add the avility for the caller to decide which one to use. We default to NUnit V3 due to or dependency to it in VSTS.
This commit is contained in:
Manuel de la Pena
GitHub
Родитель
a40f6e8804
Коммит
dc57f88e9f
|
|
@ -369,7 +369,7 @@ namespace xharness
|
|||
// at this point, we have the test results, but we want to be able to have attachments in vsts, so if the format is
|
||||
// the right one (NUnitV3) add the nodes. ATM only TouchUnit uses V3.
|
||||
var testRunName = $"{appName} {Variation}";
|
||||
if (xmlType == XmlResultParser.Jargon.NUnitV3) {
|
||||
if (xmlType == XmlResultJargon.NUnitV3) {
|
||||
var logFiles = new List<string> ();
|
||||
// add our logs AND the logs of the previous task, which is the build task
|
||||
logFiles.AddRange (Directory.GetFiles (Logs.Directory));
|
||||
|
|
@ -867,7 +867,7 @@ namespace xharness
|
|||
if (crash_reason != null) {
|
||||
// if in CI, do write an xml error that will be picked as a failure by VSTS
|
||||
if (Harness.InCI)
|
||||
XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3);
|
||||
XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, Harness.XmlJargon);
|
||||
break;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
|
|
@ -881,19 +881,19 @@ namespace xharness
|
|||
FailureMessage = $"Killed by the OS ({crash_reason})";
|
||||
}
|
||||
if (Harness.InCI)
|
||||
XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, XmlResultParser.Jargon.NUnitV3);
|
||||
XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, Harness.XmlJargon);
|
||||
} else if (launch_failure) {
|
||||
// same as with a crash
|
||||
FailureMessage = $"Launch failure";
|
||||
if (Harness.InCI)
|
||||
XmlResultParser.GenerateFailure (Logs, "launch", appName, Variation, $"AppLaunch on {device_name}", $"{FailureMessage} on {device_name}", main_log.FullPath, XmlResultParser.Jargon.NUnitV3);
|
||||
XmlResultParser.GenerateFailure (Logs, "launch", appName, Variation, $"AppLaunch on {device_name}", $"{FailureMessage} on {device_name}", main_log.FullPath, XmlResultJargon.NUnitV3);
|
||||
} else if (crashed && (!File.Exists (listener_log.FullPath) || string.IsNullOrEmpty (crash_reason)) && Harness.InCI) {
|
||||
// this happens more that what we would like on devices, the main reason most of the time is that we have had netwoking problems and the
|
||||
// tcp connection could not be stablished. We are going to report it as an error since we have not parsed the logs, evne when the app might have
|
||||
// not crashed.
|
||||
XmlResultParser.GenerateFailure (Logs, "tcp-connection", appName, Variation, $"TcpConnection on {device_name}", $"Device {device_name} could not reach the host over tcp.", main_log.FullPath, XmlResultParser.Jargon.NUnitV3);
|
||||
XmlResultParser.GenerateFailure (Logs, "tcp-connection", appName, Variation, $"TcpConnection on {device_name}", $"Device {device_name} could not reach the host over tcp.", main_log.FullPath, Harness.XmlJargon);
|
||||
} else if (timed_out && Harness.InCI) {
|
||||
XmlResultParser.GenerateFailure (Logs, "timeout", appName, Variation, "AppTimeout", $"Test run timed out after {timeout.TotalMinutes} minute(s).", main_log.FullPath, XmlResultParser.Jargon.NUnitV3);
|
||||
XmlResultParser.GenerateFailure (Logs, "timeout", appName, Variation, "AppTimeout", $"Test run timed out after {timeout.TotalMinutes} minute(s).", main_log.FullPath, Harness.XmlJargon);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ namespace xharness
|
|||
public Log HarnessLog { get; set; }
|
||||
public bool UseSystem { get; set; } // if the system XI/XM should be used, or the locally build XI/XM.
|
||||
public HashSet<string> Labels { get; } = new HashSet<string> ();
|
||||
public XmlResultJargon XmlJargon { get; set; } = XmlResultJargon.NUnitV3;
|
||||
|
||||
public string XIBuildPath {
|
||||
get { return Path.GetFullPath (Path.Combine (RootDirectory, "..", "tools", "xibuild", "xibuild")); }
|
||||
|
|
|
|||
|
|
@ -3523,7 +3523,7 @@ namespace xharness
|
|||
}
|
||||
FailureMessage = BuildTask.FailureMessage;
|
||||
if (Harness.InCI && BuildTask is XBuildTask projectTask)
|
||||
XmlResultParser.GenerateFailure (Logs, "build", projectTask.TestName, projectTask.Variation, "AppBuild", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, XmlResultParser.Jargon.NUnitV3);
|
||||
XmlResultParser.GenerateFailure (Logs, "build", projectTask.TestName, projectTask.Variation, "AppBuild", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, Harness.XmlJargon);
|
||||
} else {
|
||||
ExecutionResult = TestExecutingResult.Built;
|
||||
}
|
||||
|
|
@ -3765,7 +3765,7 @@ namespace xharness
|
|||
if (Harness.InCI)
|
||||
XmlResultParser.GenerateFailure (Logs, "install", runner.AppName, runner.Variation,
|
||||
$"AppInstallation on {runner.DeviceName}", $"Install failed on {runner.DeviceName}, exit code: {install_result.ExitCode}",
|
||||
install_log.FullPath, XmlResultParser.Jargon.NUnitV3);
|
||||
install_log.FullPath, Harness.XmlJargon);
|
||||
}
|
||||
} finally {
|
||||
this.install_log.Dispose ();
|
||||
|
|
|
|||
|
|
@ -72,6 +72,15 @@ namespace xharness
|
|||
{ "periodic-command-arguments=", "Arguments to the command to execute periodically.", (v) => harness.PeriodicCommandArguments = v },
|
||||
{ "periodic-interval=", "An interval (in minutes) between every attempt to execute the periodic command.", (v) => harness.PeriodicCommandInterval = TimeSpan.FromMinutes (double.Parse (v)) },
|
||||
{ "include-system-permission-tests:", "If tests that require system permissions (which could cause the OS to launch dialogs that hangs the test) should be executed or not. Default is to include such tests.", (v) => harness.IncludeSystemPermissionTests = ParseBool (v, "include-system-permission-tests") },
|
||||
{ "xml-jargon:", "The xml format to be used for test results. Values can be nunitv2, nunitv3, xunit.", (v) =>
|
||||
{
|
||||
if (Enum.TryParse<XmlResultJargon> (v, out var jargon))
|
||||
harness.XmlJargon = jargon;
|
||||
else
|
||||
harness.XmlJargon = XmlResultJargon.Missing;
|
||||
}
|
||||
},
|
||||
|
||||
};
|
||||
|
||||
showHelp = () => {
|
||||
|
|
@ -85,6 +94,10 @@ namespace xharness
|
|||
if (harness.Action == HarnessAction.None)
|
||||
showHelp ();
|
||||
|
||||
if (harness.XmlJargon == XmlResultJargon.Missing) {
|
||||
Console.WriteLine ("Unknown xml-jargon value provided. Values can be nunitv2, nunitv3, xunit");
|
||||
return 1;
|
||||
}
|
||||
// XS sets this, which breaks pretty much everything if it doesn't match what was passed to --sdkroot.
|
||||
Environment.SetEnvironmentVariable ("XCODE_DEVELOPER_DIR_PATH", null);
|
||||
|
||||
|
|
|
|||
|
|
@ -13,20 +13,20 @@ namespace Xharness.Tests {
|
|||
[TestFixture]
|
||||
public class XmlResultParserTests {
|
||||
|
||||
string CreateResultSample (Jargon jargon, bool includePing = false)
|
||||
string CreateResultSample (XmlResultJargon jargon, bool includePing = false)
|
||||
{
|
||||
string sampleFileName = null;
|
||||
switch (jargon) {
|
||||
case Jargon.NUnitV2:
|
||||
case XmlResultJargon.NUnitV2:
|
||||
sampleFileName = "NUnitV2Sample.xml";
|
||||
break;
|
||||
case Jargon.NUnitV3:
|
||||
case XmlResultJargon.NUnitV3:
|
||||
sampleFileName = "NUnitV3Sample.xml";
|
||||
break;
|
||||
case Jargon.TouchUnit:
|
||||
case XmlResultJargon.TouchUnit:
|
||||
sampleFileName = "TouchUnitSample.xml";
|
||||
break;
|
||||
case Jargon.xUnit:
|
||||
case XmlResultJargon.xUnit:
|
||||
sampleFileName = "xUnitSample.xml";
|
||||
break;
|
||||
}
|
||||
|
|
@ -53,11 +53,11 @@ namespace Xharness.Tests {
|
|||
Assert.IsFalse (XmlResultParser.IsValidXml (path, out var jargon), "missing file");
|
||||
}
|
||||
|
||||
[TestCase (Jargon.NUnitV2)]
|
||||
[TestCase (Jargon.NUnitV3)]
|
||||
[TestCase (Jargon.TouchUnit)]
|
||||
[TestCase (Jargon.xUnit)]
|
||||
public void IsValidXmlTest (Jargon jargon)
|
||||
[TestCase (XmlResultJargon.NUnitV2)]
|
||||
[TestCase (XmlResultJargon.NUnitV3)]
|
||||
[TestCase (XmlResultJargon.TouchUnit)]
|
||||
[TestCase (XmlResultJargon.xUnit)]
|
||||
public void IsValidXmlTest (XmlResultJargon jargon)
|
||||
{
|
||||
var path = CreateResultSample (jargon);
|
||||
Assert.IsTrue (XmlResultParser.IsValidXml (path, out var resultJargon), "is valid");
|
||||
|
|
@ -66,11 +66,11 @@ namespace Xharness.Tests {
|
|||
}
|
||||
|
||||
|
||||
[TestCase ("nunit-", Jargon.NUnitV2)]
|
||||
[TestCase ("nunit-", Jargon.NUnitV2)]
|
||||
[TestCase ("nunit-", Jargon.TouchUnit)]
|
||||
[TestCase ("xunit-", Jargon.xUnit)]
|
||||
public void GetXmlFilePathTest (string prefix, Jargon jargon)
|
||||
[TestCase ("nunit-", XmlResultJargon.NUnitV2)]
|
||||
[TestCase ("nunit-", XmlResultJargon.NUnitV2)]
|
||||
[TestCase ("nunit-", XmlResultJargon.TouchUnit)]
|
||||
[TestCase ("xunit-", XmlResultJargon.xUnit)]
|
||||
public void GetXmlFilePathTest (string prefix, XmlResultJargon jargon)
|
||||
{
|
||||
var orignialPath = "/path/to/a/xml/result.xml";
|
||||
var xmlPath = XmlResultParser.GetXmlFilePath (orignialPath, jargon);
|
||||
|
|
@ -78,10 +78,10 @@ namespace Xharness.Tests {
|
|||
StringAssert.StartsWith (prefix, fileName, "xml prefix");
|
||||
}
|
||||
|
||||
[TestCase (Jargon.NUnitV3)]
|
||||
[TestCase (Jargon.NUnitV2)]
|
||||
[TestCase (Jargon.xUnit)]
|
||||
public void CleanXmlPingTest (Jargon jargon)
|
||||
[TestCase (XmlResultJargon.NUnitV3)]
|
||||
[TestCase (XmlResultJargon.NUnitV2)]
|
||||
[TestCase (XmlResultJargon.xUnit)]
|
||||
public void CleanXmlPingTest (XmlResultJargon jargon)
|
||||
{
|
||||
var path = CreateResultSample (jargon, includePing: true);
|
||||
var cleanPath = path + "_clean";
|
||||
|
|
@ -96,11 +96,11 @@ namespace Xharness.Tests {
|
|||
public void CleanXmlTouchUnitTest ()
|
||||
{
|
||||
// similar to CleanXmlPingTest but using TouchUnit, so we do not want to see the extra nodes
|
||||
var path = CreateResultSample (Jargon.TouchUnit, includePing: true);
|
||||
var path = CreateResultSample (XmlResultJargon.TouchUnit, includePing: true);
|
||||
var cleanPath = path + "_clean";
|
||||
XmlResultParser.CleanXml (path, cleanPath);
|
||||
Assert.IsTrue (XmlResultParser.IsValidXml (cleanPath, out var resultJargon), "is valid");
|
||||
Assert.AreEqual (Jargon.NUnitV2, resultJargon, "jargon");
|
||||
Assert.AreEqual (XmlResultJargon.NUnitV2, resultJargon, "jargon");
|
||||
// load the xml, ensure we do not have the nodes we removed
|
||||
var doc = XDocument.Load (cleanPath);
|
||||
Assert.IsFalse (doc.Descendants ().Where (e => e.Name == "TouchUnitTestRun").Any (), "TouchUnitTestRun");
|
||||
|
|
@ -113,7 +113,7 @@ namespace Xharness.Tests {
|
|||
public void UpdateMissingDataTest () // only works with NUnitV3
|
||||
{
|
||||
string appName = "TestApp";
|
||||
var path = CreateResultSample (Jargon.NUnitV3);
|
||||
var path = CreateResultSample (XmlResultJargon.NUnitV3);
|
||||
var cleanPath = path + "_clean";
|
||||
XmlResultParser.CleanXml (path, cleanPath);
|
||||
var updatedXml = path + "_updated";
|
||||
|
|
|
|||
|
|
@ -46,9 +46,6 @@
|
|||
<Compile Include="..\AppRunner.cs">
|
||||
<Link>AppRunner.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="..\BCLTestInfo.cs">
|
||||
<Link>BCLTestInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="..\DeviceLogCapturer.cs">
|
||||
<Link>DeviceLogCapturer.cs</Link>
|
||||
</Compile>
|
||||
|
|
|
|||
|
|
@ -6,20 +6,21 @@ using System.Xml;
|
|||
using System.Xml.Linq;
|
||||
|
||||
namespace xharness {
|
||||
|
||||
public enum XmlResultJargon {
|
||||
TouchUnit,
|
||||
NUnitV2,
|
||||
NUnitV3,
|
||||
xUnit,
|
||||
Missing,
|
||||
}
|
||||
|
||||
public static class XmlResultParser {
|
||||
|
||||
public enum Jargon {
|
||||
TouchUnit,
|
||||
NUnitV2,
|
||||
NUnitV3,
|
||||
xUnit,
|
||||
Missing,
|
||||
}
|
||||
|
||||
// test if the file is valid xml, or at least, that can be read it.
|
||||
public static bool IsValidXml (string path, out Jargon type)
|
||||
public static bool IsValidXml (string path, out XmlResultJargon type)
|
||||
{
|
||||
type = Jargon.Missing;
|
||||
type = XmlResultJargon.Missing;
|
||||
if (!File.Exists (path))
|
||||
return false;
|
||||
|
||||
|
|
@ -29,19 +30,19 @@ namespace xharness {
|
|||
if (line.Contains ("ping"))
|
||||
continue;
|
||||
if (line.Contains ("test-run")) { // first element of the NUnitV3 test collection
|
||||
type = Jargon.NUnitV3;
|
||||
type = XmlResultJargon.NUnitV3;
|
||||
return true;
|
||||
}
|
||||
if (line.Contains ("TouchUnitTestRun")) {
|
||||
type = Jargon.TouchUnit;
|
||||
type = XmlResultJargon.TouchUnit;
|
||||
return true;
|
||||
}
|
||||
if (line.Contains ("test-results")) { // first element of the NUnitV3 test collection
|
||||
type = Jargon.NUnitV2;
|
||||
type = XmlResultJargon.NUnitV2;
|
||||
return true;
|
||||
}
|
||||
if (line.Contains ("<assemblies>")) { // first element of the xUnit test collection
|
||||
type = Jargon.xUnit;
|
||||
type = XmlResultJargon.xUnit;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -277,15 +278,15 @@ namespace xharness {
|
|||
return (resultLine, total == 0 | errors != 0 || failed != 0);
|
||||
}
|
||||
|
||||
public static string GetXmlFilePath (string path, Jargon xmlType)
|
||||
public static string GetXmlFilePath (string path, XmlResultJargon xmlType)
|
||||
{
|
||||
var fileName = Path.GetFileName (path);
|
||||
switch (xmlType) {
|
||||
case Jargon.TouchUnit:
|
||||
case Jargon.NUnitV2:
|
||||
case Jargon.NUnitV3:
|
||||
case XmlResultJargon.TouchUnit:
|
||||
case XmlResultJargon.NUnitV2:
|
||||
case XmlResultJargon.NUnitV3:
|
||||
return path.Replace (fileName, $"nunit-{fileName}");
|
||||
case Jargon.xUnit:
|
||||
case XmlResultJargon.xUnit:
|
||||
return path.Replace (fileName, $"xunit-{fileName}");
|
||||
default:
|
||||
return path;
|
||||
|
|
@ -310,22 +311,22 @@ namespace xharness {
|
|||
}
|
||||
}
|
||||
|
||||
public static (string resultLine, bool failed) GenerateHumanReadableResults (string source, string destination, Jargon xmlType)
|
||||
public static (string resultLine, bool failed) GenerateHumanReadableResults (string source, string destination, XmlResultJargon xmlType)
|
||||
{
|
||||
(string resultLine, bool failed) parseData;
|
||||
using (var reader = new StreamReader (source))
|
||||
using (var writer = new StreamWriter (destination, true)) {
|
||||
switch (xmlType) {
|
||||
case Jargon.TouchUnit:
|
||||
case XmlResultJargon.TouchUnit:
|
||||
parseData = ParseTouchUnitXml (reader, writer);
|
||||
break;
|
||||
case Jargon.NUnitV2:
|
||||
case XmlResultJargon.NUnitV2:
|
||||
parseData = ParseNUnitXml (reader, writer);
|
||||
break;
|
||||
case Jargon.NUnitV3:
|
||||
case XmlResultJargon.NUnitV3:
|
||||
parseData = ParseNUnitV3Xml (reader, writer);
|
||||
break;
|
||||
case Jargon.xUnit:
|
||||
case XmlResultJargon.xUnit:
|
||||
parseData = ParsexUnitXml (reader, writer);
|
||||
break;
|
||||
default:
|
||||
|
|
@ -472,19 +473,19 @@ namespace xharness {
|
|||
}
|
||||
}
|
||||
|
||||
public static void GenerateTestReport (StreamWriter writer, string resultsPath, Jargon xmlType)
|
||||
public static void GenerateTestReport (StreamWriter writer, string resultsPath, XmlResultJargon xmlType)
|
||||
{
|
||||
using (var stream = new StreamReader (resultsPath))
|
||||
using (var reader = XmlReader.Create (stream)) {
|
||||
switch (xmlType) {
|
||||
case Jargon.NUnitV2:
|
||||
case Jargon.TouchUnit:
|
||||
case XmlResultJargon.NUnitV2:
|
||||
case XmlResultJargon.TouchUnit:
|
||||
GenerateNUnitV2TestReport (writer, reader);
|
||||
break;
|
||||
case Jargon.xUnit:
|
||||
case XmlResultJargon.xUnit:
|
||||
GeneratexUnitTestReport (writer, reader);
|
||||
break;
|
||||
case Jargon.NUnitV3:
|
||||
case XmlResultJargon.NUnitV3:
|
||||
GenerateNUnitV3TestReport (writer, reader);
|
||||
break;
|
||||
default:
|
||||
|
|
@ -715,7 +716,7 @@ namespace xharness {
|
|||
|
||||
}
|
||||
|
||||
static void GenerateFailureXml (string destination, string title, string message, string stderrPath, Jargon jargon)
|
||||
static void GenerateFailureXml (string destination, string title, string message, string stderrPath, XmlResultJargon jargon)
|
||||
{
|
||||
XmlWriterSettings settings = new XmlWriterSettings { Indent = true };
|
||||
using (var stream = File.CreateText (destination))
|
||||
|
|
@ -723,13 +724,13 @@ namespace xharness {
|
|||
using (var xmlWriter = XmlWriter.Create (stream, settings)) {
|
||||
xmlWriter.WriteStartDocument ();
|
||||
switch (jargon) {
|
||||
case Jargon.NUnitV2:
|
||||
case XmlResultJargon.NUnitV2:
|
||||
GenerateNUnitV2Failure (xmlWriter, title, message, stderrReader);
|
||||
break;
|
||||
case Jargon.NUnitV3:
|
||||
case XmlResultJargon.NUnitV3:
|
||||
GenerateNUnitV3Failure (xmlWriter, title, message, stderrReader);
|
||||
break;
|
||||
case Jargon.xUnit:
|
||||
case XmlResultJargon.xUnit:
|
||||
GeneratexUnitFailure (xmlWriter, title, message, stderrReader);
|
||||
break;
|
||||
}
|
||||
|
|
@ -737,7 +738,7 @@ namespace xharness {
|
|||
}
|
||||
}
|
||||
|
||||
public static void GenerateFailure (Logs logs, string source, string appName, string variation, string title, string message, string stderrPath, Jargon jargon)
|
||||
public static void GenerateFailure (Logs logs, string source, string appName, string variation, string title, string message, string stderrPath, XmlResultJargon jargon)
|
||||
{
|
||||
// VSTS does not provide a nice way to report build errors, create a fake
|
||||
// test result with a failure in the case the build did not work
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче