[xharness] Fix rendering human-readable results for parameterized tests. Fixes #9400. (#9704)

We weren't probably parsing nested test-suite elements in the xml, so
implement parsing of test-suite elements using recursion, which is easier and
less error-prone (and turns out to work fine too).

Also use TextReader/Writer instead of StreamReader/Writer in a few places.
This makes it easier to write tests, since TextReader/Writer can use in-memory
streams (no need to mess with temporary files).

Fixes https://github.com/xamarin/xamarin-macios/issues/9400.
This commit is contained in:
Rolf Bjarne Kvinge 2020-09-25 19:18:37 +02:00 коммит произвёл GitHub
Родитель 764734d20a
Коммит 5aed65d45e
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
4 изменённых файлов: 3887 добавлений и 138 удалений

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@ -308,40 +308,44 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared.Tests {
Directory.Delete (logsDir, true);
}
/// <summary>
/// https://github.com/xamarin/xamarin-macios/issues/8214
/// </summary>
[Test]
public void Issue8214Test ()
[TestCase ("Issue8214.xml", true, "Tests run: 2376 Passed: 2301 Inconclusive: 13 Failed: 1 Ignored: 74")] // https://github.com/xamarin/xamarin-macios/issues/8214
[TestCase ("NUnitV2Sample.xml", true, "Tests run: 21 Passed: 4 Inconclusive: 1 Failed: 2 Ignored: 7")]
[TestCase ("NUnitV2SampleFailure.xml", true, "Tests run: 21 Passed: 4 Inconclusive: 1 Failed: 2 Ignored: 7")]
[TestCase ("NUnitV3Sample.xml", true, "Tests run: 25 Passed: 12 Inconclusive: 1 Failed: 2 Ignored: 4")]
[TestCase ("NUnitV3SampleFailure.xml", true, "Tests run: 5 Passed: 3 Inconclusive: 1 Failed: 2 Ignored: 4")]
[TestCase ("TestCaseFailures.xml", true, "Tests run: 440 Passed: 405 Inconclusive: 0 Failed: 23 Ignored: 6")]
[TestCase ("TouchUnitSample.xml", false, "Tests run: 2354 Passed: 2223 Inconclusive: 13 Failed: 0 Ignored: 59", new string [] { "Tests run: 2286 Passed: 2282 Inconclusive: 4 Failed: 0 Ignored: 47" })] // The counting is a bit off here, seems like that's in Touch.Unit
[TestCase ("xUnitSample.xml", false, "Tests run: 53821 Passed: 53801 Inconclusive: 0 Failed: 0 Ignored: 20")]
[TestCase ("NUnitV3SampleParameterizedFailure.xml", true, "Tests run: 2086 Passed: 2041 Inconclusive: 7 Failed: 2 Ignored: 43", new string [] { " [FAIL] GHIssue8342(OK,\"mandel\",\"12345678\",\"mandel\",\"12345678\") : Status not ok" })]
public void HumanReadableResultsTest (string xmlFile, bool expectedFailure, string expectedResultLine, string [] additionalLines = null)
{
string expectedResultLine = "Tests run: 2376 Passed: 2301 Inconclusive: 13 Failed: 1 Ignored: 74";
// get the sample that was added to the issue to validate that we do parse the resuls correctly and copy it to a local
// path to be parsed
var name = GetType ().Assembly.GetManifestResourceNames ().Where (a => a.EndsWith ("Issue8214.xml", StringComparison.Ordinal)).FirstOrDefault ();
var tempPath = Path.GetTempFileName ();
var destinationFile = Path.GetTempFileName ();
using (var outputStream = new StreamWriter (tempPath))
using (var sampleStream = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name))) {
string line;
while ((line = sampleStream.ReadLine ()) != null)
outputStream.WriteLine (line);
// get the sample xml to parse
var name = GetType ().Assembly.GetManifestResourceNames ().Where (a => a.EndsWith (xmlFile, StringComparison.Ordinal)).FirstOrDefault ();
using var validXmlSource = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name));
using var source = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name));
using var destination = new StringWriter ();
// Get the xml type
Assert.IsTrue (resultParser.IsValidXml (validXmlSource, out var type), "Valid Xml");
// generate the results
var (resultLine, failed) = resultParser.GenerateHumanReadableResults (source, destination, type);
var output = destination.ToString ();
Assert.AreEqual (expectedFailure, failed, "failed");
Assert.AreEqual (expectedResultLine, resultLine, "result line");
if (additionalLines != null) {
var lines = output.Split ('\n');
foreach (var line in additionalLines)
Assert.That (lines, Does.Contain (line), "Expected line");
}
var (resultLine, failed) = resultParser.GenerateHumanReadableResults (tempPath, destinationFile, XmlResultJargon.NUnitV3);
Assert.IsTrue (failed, "failed");
Assert.AreEqual (expectedResultLine, resultLine, "resultLine");
// verify that the destination does contain the result line
string resultLineInDestinationFile = null;
using (var resultReader = new StreamReader (destinationFile)) {
string line;
while ((line = resultReader.ReadLine ()) != null) {
if (line.Contains ("Tests run:")) {
resultLineInDestinationFile = line;
break;
}
}
if (expectedFailure) {
Assert.That (output, Does.Contain ("[FAIL]"), "FAIL");
} else {
Assert.That (output, Does.Not.Contain ("[FAIL]"), "Not FAIL");
}
Assert.IsNotNull (resultLineInDestinationFile, "result file result line");
Assert.AreEqual (expectedResultLine, resultLineInDestinationFile, "content result file result line");
}
[Test]

Просмотреть файл

@ -12,7 +12,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
// failure perse but the situation in which the app could not be built, timeout or crashed.
void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, string stderrPath, XmlResultJargon jargon);
void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, StreamReader stderrReader, XmlResultJargon jargon);
void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, TextReader stderrReader, XmlResultJargon jargon);
// updates a given xml result to contain a list of attachments. This is useful for CI to be able to add logs as part of the attachments of a failing test.
void UpdateMissingData (string source, string destination, string applicationName, IEnumerable<string> attachments);

Просмотреть файл

@ -28,37 +28,134 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
if (!File.Exists (path))
return false;
using (var stream = File.OpenText (path)) {
string line;
while ((line = stream.ReadLine ()) != null) { // special case when get got the tcp connection
if (line.Contains ("ping"))
continue;
if (line.Contains ("test-run")) { // first element of the NUnitV3 test collection
type = XmlResultJargon.NUnitV3;
return true;
}
if (line.Contains ("TouchUnitTestRun")) {
type = XmlResultJargon.TouchUnit;
return true;
}
if (line.Contains ("test-results")) { // first element of the NUnitV3 test collection
type = XmlResultJargon.NUnitV2;
return true;
}
if (line.Contains ("<assemblies>")) { // first element of the xUnit test collection
type = XmlResultJargon.xUnit;
return true;
}
if (line.Contains ("<TestRun")) {
type = XmlResultJargon.Trx;
return true;
}
using (var stream = File.OpenText (path))
return IsValidXml (stream, out type);
}
// test if the file is valid xml, or at least, that can be read it.
public bool IsValidXml (TextReader stream, out XmlResultJargon type)
{
type = XmlResultJargon.Missing;
string line;
while ((line = stream.ReadLine ()) != null) { // special case when get got the tcp connection
if (line.Contains ("ping"))
continue;
if (line.Contains ("test-run")) { // first element of the NUnitV3 test collection
type = XmlResultJargon.NUnitV3;
return true;
}
if (line.Contains ("TouchUnitTestRun")) {
type = XmlResultJargon.TouchUnit;
return true;
}
if (line.Contains ("test-results")) { // first element of the NUnitV3 test collection
type = XmlResultJargon.NUnitV2;
return true;
}
if (line.Contains ("<assemblies>")) { // first element of the xUnit test collection
type = XmlResultJargon.xUnit;
return true;
}
if (line.Contains ("<TestRun")) {
type = XmlResultJargon.Trx;
return true;
}
}
return false;
}
static (string resultLine, bool failed) ParseNUnitV3Xml (StreamReader stream, StreamWriter writer)
static void ParseNUnitV3XmlTestCase (TextWriter writer, XmlReader reader)
{
if (reader.NodeType != XmlNodeType.Element)
throw new InvalidOperationException ();
if (reader.Name != "test-case")
throw new InvalidOperationException ();
// read the test cases in the current node
var status = reader ["result"];
switch (status) {
case "Passed":
writer.Write ("\t[PASS] ");
break;
case "Skipped":
writer.Write ("\t[IGNORED] ");
break;
case "Error":
case "Failed":
writer.Write ("\t[FAIL] ");
break;
case "Inconclusive":
writer.Write ("\t[INCONCLUSIVE] ");
break;
default:
writer.Write ("\t[INFO] ");
break;
}
var name = reader ["name"];
writer.Write (name);
if (status == "Failed") { // we need to print the message
reader.ReadToDescendant ("failure");
reader.ReadToDescendant ("message");
writer.Write ($" : {reader.ReadElementContentAsString ()}");
if (reader.Name != "stack-trace")
reader.ReadToNextSibling ("stack-trace");
writer.Write ($" : {reader.ReadElementContentAsString ()}");
}
if (status == "Skipped") { // nice to have the skip reason
reader.ReadToDescendant ("reason");
reader.ReadToDescendant ("message");
writer.Write ($" : {reader.ReadElementContentAsString ()}");
}
// add a new line
writer.WriteLine ();
}
static void ParseNUnitV3XmlTestSuite (TextWriter writer, XmlReader reader, bool nested, int nesting = 0)
{
if (reader.NodeType != XmlNodeType.Element)
throw new InvalidOperationException ();
if (reader.Name != "test-suite")
throw new InvalidOperationException ();
var type = reader ["type"];
var isFixture = type == "TestFixture" || type == "ParameterizedFixture";
var testCaseName = reader ["fullname"];
var time = reader.GetAttribute ("time") ?? "0"; // some nodes might not have the time :/
if (isFixture)
writer.WriteLine (testCaseName);
if (reader.IsEmptyElement) {
if (isFixture)
writer.WriteLine ($"{testCaseName} {time} ms");
return;
}
while (reader.Read ()) {
switch (reader.NodeType ) {
case XmlNodeType.Element:
if (reader.Name == "test-case") {
ParseNUnitV3XmlTestCase (writer, reader);
} else if (reader.Name == "test-suite") {
ParseNUnitV3XmlTestSuite (writer, reader, nested || isFixture);
}
break;
case XmlNodeType.EndElement:
if (reader.Name == "test-suite") {
if (isFixture)
writer.WriteLine ($"{testCaseName} {time} ms");
return;
}
break;
}
}
throw new InvalidOperationException ("Invalid XML: no test-suite end element");
}
static (string resultLine, bool failed) ParseNUnitV3Xml (TextReader stream, TextWriter writer)
{
long testcasecount, passed, failed, inconclusive, skipped;
bool failedTestRun = false; // result = "Failed"
@ -79,52 +176,8 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
long.TryParse (reader ["skipped"], out skipped);
failedTestRun = failed != 0;
}
if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-suite" && (reader ["type"] == "TestFixture" || reader ["type"] == "ParameterizedFixture")) {
var testCaseName = reader ["fullname"];
writer.WriteLine (testCaseName);
var time = reader.GetAttribute ("time") ?? "0"; // some nodes might not have the time :/
// get the first node and then move in the siblings of the same type
reader.ReadToDescendant ("test-case");
do {
if (reader.Name != "test-case")
break;
// read the test cases in the current node
var status = reader ["result"];
switch (status) {
case "Passed":
writer.Write ("\t[PASS] ");
break;
case "Skipped":
writer.Write ("\t[IGNORED] ");
break;
case "Error":
case "Failed":
writer.Write ("\t[FAIL] ");
break;
case "Inconclusive":
writer.Write ("\t[INCONCLUSIVE] ");
break;
default:
writer.Write ("\t[INFO] ");
break;
}
writer.Write (reader ["name"]);
if (status == "Failed") { // we need to print the message
reader.ReadToDescendant ("failure");
reader.ReadToDescendant ("message");
writer.Write ($" : {reader.ReadElementContentAsString ()}");
reader.ReadToNextSibling ("stack-trace");
writer.Write ($" : {reader.ReadElementContentAsString ()}");
}
if (status == "Skipped") { // nice to have the skip reason
reader.ReadToDescendant ("reason");
reader.ReadToDescendant ("message");
writer.Write ($" : {reader.ReadElementContentAsString ()}");
}
// add a new line
writer.WriteLine ();
} while (reader.ReadToNextSibling ("test-case"));
writer.WriteLine ($"{testCaseName} {time} ms");
if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-suite") {
ParseNUnitV3XmlTestSuite (writer, reader, false);
}
}
}
@ -133,7 +186,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
return (resultLine, failedTestRun);
}
static (string resultLine, bool failed) ParseTouchUnitXml (StreamReader stream, StreamWriter writer)
static (string resultLine, bool failed) ParseTouchUnitXml (TextReader stream, TextWriter writer)
{
long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid;
total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L;
@ -162,7 +215,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
return (resultLine, total == 0 || errors != 0 || failed != 0);
}
static (string resultLine, bool failed) ParseNUnitXml (StreamReader stream, StreamWriter writer)
static (string resultLine, bool failed) ParseNUnitXml (TextReader stream, TextWriter writer)
{
long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid;
total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L;
@ -230,7 +283,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
return (resultLine, total == 0 | errors != 0 || failed != 0);
}
static (string resultLine, bool failed) ParsexUnitXml (StreamReader stream, StreamWriter writer)
static (string resultLine, bool failed) ParsexUnitXml (TextReader stream, TextWriter writer)
{
long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid;
total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L;
@ -292,7 +345,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
return (resultLine, total == 0 | errors != 0 || failed != 0);
}
static (string resultLine, bool failed) ParseTrxXml (StreamReader stream, StreamWriter writer)
static (string resultLine, bool failed) ParseTrxXml (TextReader stream, TextWriter writer)
{
using (var reader = XmlReader.Create (stream)) {
var tests = ParseTrxXml (reader);
@ -437,31 +490,27 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
public (string resultLine, bool failed) GenerateHumanReadableResults (string source, string destination, XmlResultJargon xmlType)
{
(string resultLine, bool failed) parseData;
using (var reader = new StreamReader (source))
using (var writer = new StreamWriter (destination, true)) {
switch (xmlType) {
case XmlResultJargon.TouchUnit:
parseData = ParseTouchUnitXml (reader, writer);
break;
case XmlResultJargon.NUnitV2:
parseData = ParseNUnitXml (reader, writer);
break;
case XmlResultJargon.NUnitV3:
parseData = ParseNUnitV3Xml (reader, writer);
break;
case XmlResultJargon.xUnit:
parseData = ParsexUnitXml (reader, writer);
break;
case XmlResultJargon.Trx:
parseData = ParseTrxXml (reader, writer);
break;
default:
parseData = ("", true);
break;
}
using (var writer = new StreamWriter (destination, true))
return GenerateHumanReadableResults (reader, writer, xmlType);
}
public (string resultLine, bool failed) GenerateHumanReadableResults (TextReader reader, TextWriter writer, XmlResultJargon xmlType)
{
switch (xmlType) {
case XmlResultJargon.TouchUnit:
return ParseTouchUnitXml (reader, writer);
case XmlResultJargon.NUnitV2:
return ParseNUnitXml (reader, writer);
case XmlResultJargon.NUnitV3:
return ParseNUnitV3Xml (reader, writer);
case XmlResultJargon.xUnit:
return ParsexUnitXml (reader, writer);
case XmlResultJargon.Trx:
return ParseTrxXml (reader, writer);
default:
return ("", true);
}
return parseData;
}
static void GenerateTouchUnitTestReport (TextWriter writer, XmlReader reader)
@ -770,7 +819,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
("time", "0"),
("asserts", "1"));
static void WriteNUnitV2TestCase (XmlWriter writer, string title, string message, StreamReader stderr)
static void WriteNUnitV2TestCase (XmlWriter writer, string title, string message, TextReader stderr)
{
writer.WriteStartElement ("test-case");
WriteAttributes (writer,
@ -792,7 +841,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
writer.WriteEndElement (); // test-case
}
static void GenerateNUnitV2Failure (XmlWriter writer, string title, string message, StreamReader stderr)
static void GenerateNUnitV2Failure (XmlWriter writer, string title, string message, TextReader stderr)
{
writer.WriteStartElement ("test-results");
WriteAttributes (writer,
@ -837,7 +886,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
("skipped", "0"),
("asserts", "0"));
static void WriteFailure (XmlWriter writer, string message, StreamReader? stderr = null)
static void WriteFailure (XmlWriter writer, string message, TextReader? stderr = null)
{
writer.WriteStartElement ("failure");
writer.WriteStartElement ("message");
@ -851,7 +900,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
writer.WriteEndElement (); // failure
}
static void GenerateNUnitV3Failure (XmlWriter writer, string title, string message, StreamReader stderr)
static void GenerateNUnitV3Failure (XmlWriter writer, string title, string message, TextReader stderr)
{
var date = DateTime.Now;
writer.WriteStartElement ("test-run");
@ -903,7 +952,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
writer.WriteEndElement (); // test-run
}
static void GeneratexUnitFailure (XmlWriter writer, string title, string message, StreamReader stderr)
static void GeneratexUnitFailure (XmlWriter writer, string title, string message, TextReader stderr)
{
writer.WriteStartElement ("assemblies");
writer.WriteStartElement ("assembly");
@ -940,7 +989,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
writer.WriteEndElement (); // assemblies
}
static void GenerateFailureXml (string destination, string title, string message, StreamReader stderrReader, XmlResultJargon jargon)
static void GenerateFailureXml (string destination, string title, string message, TextReader stderrReader, XmlResultJargon jargon)
{
XmlWriterSettings settings = new XmlWriterSettings { Indent = true };
using (var stream = File.CreateText (destination))
@ -962,7 +1011,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared {
}
public void GenerateFailure (ILogs logs, string source, string appName, string variation, string title,
string message, StreamReader stderr, XmlResultJargon jargon)
string message, TextReader stderr, XmlResultJargon jargon)
{
// VSTS does not provide a nice way to report build errors, create a fake
// test result with a failure in the case the build did not work