From 5aed65d45ea0fd2370cc0def6dedca839834a054 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 25 Sep 2020 19:18:37 +0200 Subject: [PATCH] [xharness] Fix rendering human-readable results for parameterized tests. Fixes #9400. (#9704) We weren't probably parsing nested test-suite elements in the xml, so implement parsing of test-suite elements using recursion, which is easier and less error-prone (and turns out to work fine too). Also use TextReader/Writer instead of StreamReader/Writer in a few places. This makes it easier to write tests, since TextReader/Writer can use in-memory streams (no need to mess with temporary files). Fixes https://github.com/xamarin/xamarin-macios/issues/9400. --- .../NUnitV3SampleParameterizedFailure.xml | 3696 +++++++++++++++++ .../XmlResultParserTests.cs | 66 +- .../IResultParser.cs | 2 +- .../XmlResultParser.cs | 261 +- 4 files changed, 3887 insertions(+), 138 deletions(-) create mode 100644 tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/NUnitV3SampleParameterizedFailure.xml diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/NUnitV3SampleParameterizedFailure.xml b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/NUnitV3SampleParameterizedFailure.xml new file mode 100644 index 0000000000..49fb7fcf34 --- /dev/null +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/NUnitV3SampleParameterizedFailure.xml @@ -0,0 +1,3696 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/execute-Mac_Modern-20200923_052627.txt + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/main-20200923_052836.log + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627-clean.xml + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627.xml + + + + + + + + + + + + + + + + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/execute-Mac_Modern-20200923_052627.txt + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/main-20200923_052836.log + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627-clean.xml + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/execute-Mac_Modern-20200923_052627.txt + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/main-20200923_052836.log + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627-clean.xml + + + /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/jenkins-results/tests/xammac-tests/311/test-Mac_Modern-20200923_052627.xml + + + + + + diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs index edd6ec7cf4..fd520b89b3 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs @@ -308,40 +308,44 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared.Tests { Directory.Delete (logsDir, true); } - /// - /// https://github.com/xamarin/xamarin-macios/issues/8214 - /// - [Test] - public void Issue8214Test () + [TestCase ("Issue8214.xml", true, "Tests run: 2376 Passed: 2301 Inconclusive: 13 Failed: 1 Ignored: 74")] // https://github.com/xamarin/xamarin-macios/issues/8214 + [TestCase ("NUnitV2Sample.xml", true, "Tests run: 21 Passed: 4 Inconclusive: 1 Failed: 2 Ignored: 7")] + [TestCase ("NUnitV2SampleFailure.xml", true, "Tests run: 21 Passed: 4 Inconclusive: 1 Failed: 2 Ignored: 7")] + [TestCase ("NUnitV3Sample.xml", true, "Tests run: 25 Passed: 12 Inconclusive: 1 Failed: 2 Ignored: 4")] + [TestCase ("NUnitV3SampleFailure.xml", true, "Tests run: 5 Passed: 3 Inconclusive: 1 Failed: 2 Ignored: 4")] + [TestCase ("TestCaseFailures.xml", true, "Tests run: 440 Passed: 405 Inconclusive: 0 Failed: 23 Ignored: 6")] + [TestCase ("TouchUnitSample.xml", false, "Tests run: 2354 Passed: 2223 Inconclusive: 13 Failed: 0 Ignored: 59", new string [] { "Tests run: 2286 Passed: 2282 Inconclusive: 4 Failed: 0 Ignored: 47" })] // The counting is a bit off here, seems like that's in Touch.Unit + [TestCase ("xUnitSample.xml", false, "Tests run: 53821 Passed: 53801 Inconclusive: 0 Failed: 0 Ignored: 20")] + [TestCase ("NUnitV3SampleParameterizedFailure.xml", true, "Tests run: 2086 Passed: 2041 Inconclusive: 7 Failed: 2 Ignored: 43", new string [] { " [FAIL] GHIssue8342(OK,\"mandel\",\"12345678\",\"mandel\",\"12345678\") : Status not ok" })] + public void HumanReadableResultsTest (string xmlFile, bool expectedFailure, string expectedResultLine, string [] additionalLines = null) { - string expectedResultLine = "Tests run: 2376 Passed: 2301 Inconclusive: 13 Failed: 1 Ignored: 74"; - // get the sample that was added to the issue to validate that we do parse the resuls correctly and copy it to a local - // path to be parsed - var name = GetType ().Assembly.GetManifestResourceNames ().Where (a => a.EndsWith ("Issue8214.xml", StringComparison.Ordinal)).FirstOrDefault (); - var tempPath = Path.GetTempFileName (); - var destinationFile = Path.GetTempFileName (); - using (var outputStream = new StreamWriter (tempPath)) - using (var sampleStream = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name))) { - string line; - while ((line = sampleStream.ReadLine ()) != null) - outputStream.WriteLine (line); + // get the sample xml to parse + var name = GetType ().Assembly.GetManifestResourceNames ().Where (a => a.EndsWith (xmlFile, StringComparison.Ordinal)).FirstOrDefault (); + using var validXmlSource = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name)); + using var source = new StreamReader (GetType ().Assembly.GetManifestResourceStream (name)); + using var destination = new StringWriter (); + + // Get the xml type + Assert.IsTrue (resultParser.IsValidXml (validXmlSource, out var type), "Valid Xml"); + + // generate the results + var (resultLine, failed) = resultParser.GenerateHumanReadableResults (source, destination, type); + var output = destination.ToString (); + + Assert.AreEqual (expectedFailure, failed, "failed"); + Assert.AreEqual (expectedResultLine, resultLine, "result line"); + + if (additionalLines != null) { + var lines = output.Split ('\n'); + foreach (var line in additionalLines) + Assert.That (lines, Does.Contain (line), "Expected line"); } - var (resultLine, failed) = resultParser.GenerateHumanReadableResults (tempPath, destinationFile, XmlResultJargon.NUnitV3); - Assert.IsTrue (failed, "failed"); - Assert.AreEqual (expectedResultLine, resultLine, "resultLine"); - // verify that the destination does contain the result line - string resultLineInDestinationFile = null; - using (var resultReader = new StreamReader (destinationFile)) { - string line; - while ((line = resultReader.ReadLine ()) != null) { - if (line.Contains ("Tests run:")) { - resultLineInDestinationFile = line; - break; - } - } + + if (expectedFailure) { + Assert.That (output, Does.Contain ("[FAIL]"), "FAIL"); + } else { + Assert.That (output, Does.Not.Contain ("[FAIL]"), "Not FAIL"); } - Assert.IsNotNull (resultLineInDestinationFile, "result file result line"); - Assert.AreEqual (expectedResultLine, resultLineInDestinationFile, "content result file result line"); } [Test] diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/IResultParser.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/IResultParser.cs index cbbc272135..39b45b2b23 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/IResultParser.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/IResultParser.cs @@ -12,7 +12,7 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared { // failure perse but the situation in which the app could not be built, timeout or crashed. void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, string stderrPath, XmlResultJargon jargon); - void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, StreamReader stderrReader, XmlResultJargon jargon); + void GenerateFailure (ILogs logs, string source, string appName, string variation, string title, string message, TextReader stderrReader, XmlResultJargon jargon); // updates a given xml result to contain a list of attachments. This is useful for CI to be able to add logs as part of the attachments of a failing test. void UpdateMissingData (string source, string destination, string applicationName, IEnumerable attachments); diff --git a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs index fa15f10076..7d9acdb752 100644 --- a/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs +++ b/tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs @@ -28,37 +28,134 @@ namespace Microsoft.DotNet.XHarness.iOS.Shared { if (!File.Exists (path)) return false; - using (var stream = File.OpenText (path)) { - string line; - while ((line = stream.ReadLine ()) != null) { // special case when get got the tcp connection - if (line.Contains ("ping")) - continue; - if (line.Contains ("test-run")) { // first element of the NUnitV3 test collection - type = XmlResultJargon.NUnitV3; - return true; - } - if (line.Contains ("TouchUnitTestRun")) { - type = XmlResultJargon.TouchUnit; - return true; - } - if (line.Contains ("test-results")) { // first element of the NUnitV3 test collection - type = XmlResultJargon.NUnitV2; - return true; - } - if (line.Contains ("")) { // first element of the xUnit test collection - type = XmlResultJargon.xUnit; - return true; - } - if (line.Contains ("")) { // first element of the xUnit test collection + type = XmlResultJargon.xUnit; + return true; + } + if (line.Contains ("