From 2aad661b4cfe08c072065162453a4af2f66160b3 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 23 Jan 2020 18:28:12 -0500 Subject: [PATCH 1/5] [XHarness] If in CI, no matter which, use the Xml output. (#7745) We used to test only on Jenkins, and if the build version had jenkins on it, rather than doing so, just check if we are in the CI by looking if BUILD_REVISION is present in the env. --- tests/xharness/AppRunner.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index ec552bd531..8e9cc677e6 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -457,7 +457,7 @@ namespace xharness // wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it. // // On the other hand, the nunit and xunit do not have that data and have to be parsed. - if (Harness.InJenkins) { + if (Harness.InCI) { (string resultLine, bool failed, bool crashed) parseResult = (null, false, false); // move the xml to a tmp path, that path will be use to read the xml // in the reader, and the writer will use the stream from the logger to @@ -614,7 +614,7 @@ namespace xharness args.Add ("-argument=-app-arg:-enablenetwork"); args.Add ("-setenv=NUNIT_ENABLE_NETWORK=true"); // detect if we are using a jenkins bot. - var useXmlOutput = Harness.InJenkins; + var useXmlOutput = Harness.InCI; if (useXmlOutput) { args.Add ("-setenv=NUNIT_ENABLE_XML_OUTPUT=true"); args.Add ("-setenv=NUNIT_ENABLE_XML_MODE=wrapped"); From fc0dd8812ba78cc19639353469129add149bfe32 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 24 Jan 2020 07:47:44 -0500 Subject: [PATCH 2/5] [Harness] Unify the CI properties. (#7750) Unify the harness properties to just look at InCI and remove all the other ones. There is no real need to have differences between jenkins and VSTS and Wrench is gone. --- .../BCLTestImportTargetFactory.cs | 2 +- tests/xharness/Harness.cs | 18 ++---------------- tests/xharness/Jenkins.cs | 6 +++--- tests/xharness/MakefileGenerator.cs | 2 +- 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs b/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs index 29057d1ac9..187c65488a 100644 --- a/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs +++ b/tests/xharness/BCLTestImporter/BCLTestImportTargetFactory.cs @@ -26,7 +26,7 @@ namespace xharness.BCLTestImporter { MacMonoSDKPath = Harness.MONO_MAC_SDK_DESTDIR, Override = true, GuidGenerator = Harness.NewStableGuid, - GroupTests = Harness.InJenkins || Harness.UseGroupedApps, + GroupTests = Harness.InCI || Harness.UseGroupedApps, }; } diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index bfb417dcfd..348bb6f91d 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -115,7 +115,7 @@ namespace xharness public Harness () { - LaunchTimeout = InWrench ? 3 : 120; + LaunchTimeout = InCI ? 3 : 120; } public bool GetIncludeSystemPermissionTests (TestPlatform platform, bool device) @@ -667,25 +667,11 @@ namespace xharness public void LogWrench (string message) { - if (!InWrench) + if (!InCI) return; Console.WriteLine (message); } - - public bool InWrench { - get { - var buildRev = Environment.GetEnvironmentVariable ("BUILD_REVISION"); - return !string.IsNullOrEmpty (buildRev) && buildRev != "jenkins"; - } - } - - public bool InJenkins { - get { - var buildRev = Environment.GetEnvironmentVariable ("BUILD_REVISION"); - return !string.IsNullOrEmpty (buildRev) && buildRev == "jenkins"; - } - } public bool InCI { get { diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs index bc551e1c27..15521fef92 100644 --- a/tests/xharness/Jenkins.cs +++ b/tests/xharness/Jenkins.cs @@ -1153,7 +1153,7 @@ namespace xharness try { Directory.CreateDirectory (LogDirectory); Log log = Logs.Create ($"Harness-{Harness.Timestamp}.log", "Harness log"); - if (Harness.InWrench) + if (Harness.InCI) log = Log.CreateAggregatedLog (log, new ConsoleLog ()); Harness.HarnessLog = MainLog = log; @@ -1161,7 +1161,7 @@ namespace xharness if (IsServerMode) tasks.Add (RunTestServer ()); - if (Harness.InJenkins) { + if (Harness.InCI) { Task.Factory.StartNew (async () => { while (true) { await Task.Delay (TimeSpan.FromMinutes (10)); @@ -3735,7 +3735,7 @@ namespace xharness } // Also clean up after us locally. - if (Harness.InJenkins || Harness.InWrench || (Jenkins.CleanSuccessfulTestRuns && Succeeded)) + if (Harness.InCI || (Jenkins.CleanSuccessfulTestRuns && Succeeded)) await BuildTask.CleanAsync (); } } diff --git a/tests/xharness/MakefileGenerator.cs b/tests/xharness/MakefileGenerator.cs index c929767508..676b14644c 100644 --- a/tests/xharness/MakefileGenerator.cs +++ b/tests/xharness/MakefileGenerator.cs @@ -211,7 +211,7 @@ namespace xharness public static void CreateMakefile (Harness harness, IEnumerable unified_targets, IEnumerable tvos_targets, IEnumerable watchos_targets, IEnumerable today_targets) { - var executeSim32 = !harness.InWrench; // Waiting for iOS 10.3 simulator to be installed on wrench + var executeSim32 = !harness.InCI; // Waiting for iOS 10.3 simulator to be installed on wrench var makefile = Path.Combine (Harness.RootDirectory, "Makefile.inc"); using (var writer = new StreamWriter (makefile, false, new UTF8Encoding (false))) { writer.WriteLine (".stamp-configure-projects: Makefile xharness/xharness.exe"); From d9bbdcf06563e926b8b0157ef6e0fa5cc4d1d98b Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 28 Jan 2020 14:44:34 -0500 Subject: [PATCH 3/5] [Xharness] Add workaround when we could not share the internet in bots. (#7755) Looks like we have issues with the internet sharing in the VSTS bots, this means that now that we always try to parse the XML on CI, we get an exception, catch it and do not show the results. The workaround simply tries to read the xml, if possible, we will parse it, else deal with the text only log. The fix will show the results, but it is a workaround for a configuration issue in the CI. --- tests/xharness/AppRunner.cs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 8e9cc677e6..f55064cb22 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -334,6 +334,19 @@ namespace xharness } } + // test if the file is valid xml, or at least, that can be read. The reader will throw an + // exception if it cannot read the file. This is not ideal :/ + bool IsXml (string filePath) + { + if (!File.Exists (filePath)) + return false; + + using (var stream = File.OpenText (filePath)) { + var firstLine = stream.ReadLine (); + return firstLine.StartsWith ("<", StringComparison.Ordinal); + } + } + bool IsTouchUnitResult (StreamReader stream) { // TouchUnitTestRun is the very first node in the TouchUnit xml result @@ -457,7 +470,12 @@ namespace xharness // wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it. // // On the other hand, the nunit and xunit do not have that data and have to be parsed. - if (Harness.InCI) { + // + // This if statement has a small trick, we found out that internet sharing in some of the bots (VSTS) does not work, in + // that case, we cannot do a TCP connection to xharness to get the log, this is a problem since if we did not get the xml + // from the TCP connection, we are going to fail when trying to read it and not parse it. Therefore, we are not only + // going to check if we are in CI, but also if the listener_log is valid. + if (Harness.InCI && IsXml (listener_log.FullPath)) { (string resultLine, bool failed, bool crashed) parseResult = (null, false, false); // move the xml to a tmp path, that path will be use to read the xml // in the reader, and the writer will use the stream from the logger to From 8acfe3859e862ba2cf7029f6eba1b316eadf0796 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 4 Feb 2020 11:03:38 -0500 Subject: [PATCH 4/5] [Xharness] Fix all xml parsing issues and ensure that the xml can be consumed by VSTS. (#7768) We ping the tcp listener to know that we have a tcp connection, that is written in the xml logs, which means that parsing will not work. Ignore the ping, parse xml, and make sure that the xml that will be consume by vsts is valid. This PR also fixes https://github.com/xamarin/maccore/issues/827 which does not longer happen. --- tests/bcl-test/BCLTests-mac.csproj.in | 11 +- tests/bcl-test/BCLTests-tv.csproj.in | 19 +- .../BCLTests-watchos-extension.csproj.in | 19 +- tests/bcl-test/BCLTests.csproj.in | 19 +- .../TestRunner.NUnit/NUnitTestRunner.cs | 2 - .../TestRunner.NUnit/XmlOutputWriter.cs | 349 ++++++++++++++++++ .../TestRunner.xUnit/XUnitTestRunner.cs | 11 +- tests/xharness/AppRunner.cs | 256 +++++++++---- tests/xharness/SimpleTcpListener.cs | 1 + 9 files changed, 580 insertions(+), 107 deletions(-) create mode 100644 tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs diff --git a/tests/bcl-test/BCLTests-mac.csproj.in b/tests/bcl-test/BCLTests-mac.csproj.in index 19541ac495..70a59e6206 100644 --- a/tests/bcl-test/BCLTests-mac.csproj.in +++ b/tests/bcl-test/BCLTests-mac.csproj.in @@ -48,14 +48,16 @@ - - - + + + + + @@ -99,6 +101,9 @@ TestRunner.NUnit\ClassOrNamespaceFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.NUnit\TestMethodFilter.cs diff --git a/tests/bcl-test/BCLTests-tv.csproj.in b/tests/bcl-test/BCLTests-tv.csproj.in index 92cc86c1f0..9ad5fb0a1b 100644 --- a/tests/bcl-test/BCLTests-tv.csproj.in +++ b/tests/bcl-test/BCLTests-tv.csproj.in @@ -122,15 +122,15 @@ cjk,mideast,other,rare,west - - - - - + + + + + - - - + + + @@ -176,6 +176,9 @@ TestRunner.NUnit\TestMethodFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.Core\Extensions.Bool.cs diff --git a/tests/bcl-test/BCLTests-watchos-extension.csproj.in b/tests/bcl-test/BCLTests-watchos-extension.csproj.in index 02b661163c..f5e04663a8 100644 --- a/tests/bcl-test/BCLTests-watchos-extension.csproj.in +++ b/tests/bcl-test/BCLTests-watchos-extension.csproj.in @@ -128,15 +128,15 @@ SdkOnly - - - - - + + + + + - - - + + + @@ -206,6 +206,9 @@ TestRunner.NUnit\TestMethodFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.xUnit\XUnitFilter.cs diff --git a/tests/bcl-test/BCLTests.csproj.in b/tests/bcl-test/BCLTests.csproj.in index 1d60c5faae..8d66f9a838 100644 --- a/tests/bcl-test/BCLTests.csproj.in +++ b/tests/bcl-test/BCLTests.csproj.in @@ -135,15 +135,15 @@ cjk,mideast,other,rare,west - - - - - + + + + + - - - + + + @@ -189,6 +189,9 @@ TestRunner.NUnit\TestMethodFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.Core\Extensions.Bool.cs diff --git a/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs index bb94b352ce..9ce39af845 100644 --- a/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs @@ -9,10 +9,8 @@ using System.Threading.Tasks; using Foundation; -using NUnitLite.Runner; using NUnit.Framework.Api; using NUnit.Framework.Internal; -using NUnit.Framework.Internal.WorkItems; using NUnit.Framework.Internal.Filters; using NUnitTest = NUnit.Framework.Internal.Test; diff --git a/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs b/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs new file mode 100644 index 0000000000..4476fd90d6 --- /dev/null +++ b/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs @@ -0,0 +1,349 @@ +// *********************************************************************** +// Copyright (c) 2011 Charlie Poole +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// *********************************************************************** + +using System; +using System.Globalization; +using System.Reflection; +using System.Xml; +using System.IO; +using NUnit.Framework.Api; +using NUnit.Framework.Internal; +using NUnitLite.Runner; +#if CLR_2_0 || CLR_4_0 +using System.Collections.Generic; +#else +using System.Collections.Specialized; +#endif + +namespace Xamarin.iOS.UnitTests.NUnit { + /// + /// NUnit2XmlOutputWriter is able to create an xml file representing + /// the result of a test run in NUnit 2.x format. + /// + public class NUnit2XmlOutputWriter : OutputWriter { + private XmlWriter xmlWriter; + private DateTime startTime; + +#if CLR_2_0 || CLR_4_0 + private static Dictionary resultStates = new Dictionary(); +#else + private static StringDictionary resultStates = new StringDictionary (); +#endif + + static NUnit2XmlOutputWriter () + { + resultStates ["Passed"] = "Success"; + resultStates ["Failed"] = "Failure"; + resultStates ["Failed:Error"] = "Error"; + resultStates ["Failed:Cancelled"] = "Cancelled"; + resultStates ["Inconclusive"] = "Inconclusive"; + resultStates ["Skipped"] = "Skipped"; + resultStates ["Skipped:Ignored"] = "Ignored"; + resultStates ["Skipped:Invalid"] = "NotRunnable"; + } + + public NUnit2XmlOutputWriter (DateTime startTime) + { + this.startTime = startTime; + } + + /// + /// Writes the result of a test run to a specified TextWriter. + /// + /// The test result for the run + /// The TextWriter to which the xml will be written + public override void WriteResultFile (ITestResult result, TextWriter writer) + { + // NOTE: Under .NET 1.1, XmlTextWriter does not implement IDisposable, + // but does implement Close(). Hence we cannot use a 'using' clause. + //using (XmlTextWriter xmlWriter = new XmlTextWriter(writer)) +#if SILVERLIGHT + XmlWriter xmlWriter = XmlWriter.Create(writer); +#else + XmlTextWriter xmlWriter = new XmlTextWriter (writer); + xmlWriter.Formatting = Formatting.Indented; +#endif + + try { + WriteXmlOutput (result, xmlWriter); + } finally { + writer.Close (); + } + } + + private void WriteXmlOutput (ITestResult result, XmlWriter xmlWriter) + { + this.xmlWriter = xmlWriter; + + InitializeXmlFile (result); + WriteResultElement (result); + TerminateXmlFile (); + } + + private void InitializeXmlFile (ITestResult result) + { + ResultSummary summaryResults = new ResultSummary (result); + + xmlWriter.WriteStartDocument (false); + xmlWriter.WriteComment ("This file represents the results of running a test suite"); + + xmlWriter.WriteStartElement ("test-results"); + + xmlWriter.WriteAttributeString ("name", result.FullName); + xmlWriter.WriteAttributeString ("total", summaryResults.TestCount.ToString ()); + xmlWriter.WriteAttributeString ("errors", summaryResults.ErrorCount.ToString ()); + xmlWriter.WriteAttributeString ("failures", summaryResults.FailureCount.ToString ()); + xmlWriter.WriteAttributeString ("not-run", summaryResults.NotRunCount.ToString ()); + xmlWriter.WriteAttributeString ("inconclusive", summaryResults.InconclusiveCount.ToString ()); + xmlWriter.WriteAttributeString ("ignored", summaryResults.IgnoreCount.ToString ()); + xmlWriter.WriteAttributeString ("skipped", summaryResults.SkipCount.ToString ()); + xmlWriter.WriteAttributeString ("invalid", summaryResults.InvalidCount.ToString ()); + + xmlWriter.WriteAttributeString ("date", XmlConvert.ToString (startTime, "yyyy-MM-dd")); + xmlWriter.WriteAttributeString ("time", XmlConvert.ToString (startTime, "HH:mm:ss")); + WriteEnvironment (); + WriteCultureInfo (); + xmlWriter.Flush (); + } + + private void WriteCultureInfo () + { + xmlWriter.WriteStartElement ("culture-info"); + xmlWriter.WriteAttributeString ("current-culture", + CultureInfo.CurrentCulture.ToString ()); + xmlWriter.WriteAttributeString ("current-uiculture", + CultureInfo.CurrentUICulture.ToString ()); + xmlWriter.WriteEndElement (); + xmlWriter.Flush (); + } + + private void WriteEnvironment () + { + xmlWriter.WriteStartElement ("environment"); + AssemblyName assemblyName = AssemblyHelper.GetAssemblyName (Assembly.GetExecutingAssembly ()); + xmlWriter.WriteAttributeString ("nunit-version", + assemblyName.Version.ToString ()); + xmlWriter.WriteAttributeString ("clr-version", + Environment.Version.ToString ()); + xmlWriter.WriteAttributeString ("os-version", + Environment.OSVersion.ToString ()); + xmlWriter.WriteAttributeString ("platform", + Environment.OSVersion.Platform.ToString ()); +#if !NETCF + xmlWriter.WriteAttributeString ("cwd", + Environment.CurrentDirectory); +#if !SILVERLIGHT + xmlWriter.WriteAttributeString ("machine-name", + Environment.MachineName); + xmlWriter.WriteAttributeString ("user", + Environment.UserName); + xmlWriter.WriteAttributeString ("user-domain", + Environment.UserDomainName); +#endif +#endif + xmlWriter.WriteEndElement (); + xmlWriter.Flush (); + } + + private void WriteResultElement (ITestResult result) + { + StartTestElement (result); + + WriteProperties (result); + + switch (result.ResultState.Status) { + case TestStatus.Skipped: + WriteReasonElement (result.Message); + break; + case TestStatus.Failed: + WriteFailureElement (result.Message, result.StackTrace); + break; + } + + if (result.Test is TestSuite) + WriteChildResults (result); + + xmlWriter.WriteEndElement (); // test element + xmlWriter.Flush (); + } + + private void TerminateXmlFile () + { + xmlWriter.WriteEndElement (); // test-results + xmlWriter.WriteEndDocument (); + xmlWriter.Flush (); + xmlWriter.Close (); + } + + + #region Element Creation Helpers + + private void StartTestElement (ITestResult result) + { + ITest test = result.Test; + TestSuite suite = test as TestSuite; + + if (suite != null) { + xmlWriter.WriteStartElement ("test-suite"); + xmlWriter.WriteAttributeString ("type", suite.TestType); + xmlWriter.WriteAttributeString ("name", suite.TestType == "Assembly" + ? result.Test.FullName + : result.Test.Name); + } else { + xmlWriter.WriteStartElement ("test-case"); + xmlWriter.WriteAttributeString ("name", result.Name); + } + + if (test.Properties.ContainsKey (PropertyNames.Description)) { + string description = (string)test.Properties.Get (PropertyNames.Description); + xmlWriter.WriteAttributeString ("description", description); + } + + TestStatus status = result.ResultState.Status; + string translatedResult = resultStates [result.ResultState.ToString ()]; + + if (status != TestStatus.Skipped) { + xmlWriter.WriteAttributeString ("executed", "True"); + xmlWriter.WriteAttributeString ("result", translatedResult); + xmlWriter.WriteAttributeString ("success", status == TestStatus.Passed ? "True" : "False"); + xmlWriter.WriteAttributeString ("time", result.Duration.TotalSeconds.ToString ()); + xmlWriter.WriteAttributeString ("asserts", result.AssertCount.ToString ()); + } else { + xmlWriter.WriteAttributeString ("executed", "False"); + xmlWriter.WriteAttributeString ("result", translatedResult); + } + } + + private void WriteProperties (ITestResult result) + { + IPropertyBag properties = result.Test.Properties; + int nprops = 0; + + foreach (string key in properties.Keys) { + if (key != PropertyNames.Category) { + if (nprops++ == 0) + xmlWriter.WriteStartElement ("properties"); + + foreach (object prop in properties [key]) { + xmlWriter.WriteStartElement ("property"); + xmlWriter.WriteAttributeString ("name", key); + xmlWriter.WriteAttributeString ("value", prop.ToString ()); + xmlWriter.WriteEndElement (); + } + } + } + + if (nprops > 0) + xmlWriter.WriteEndElement (); + } + + private void WriteReasonElement (string message) + { + xmlWriter.WriteStartElement ("reason"); + xmlWriter.WriteStartElement ("message"); + xmlWriter.WriteCData (message); + xmlWriter.WriteEndElement (); + xmlWriter.WriteEndElement (); + } + + private void WriteFailureElement (string message, string stackTrace) + { + xmlWriter.WriteStartElement ("failure"); + xmlWriter.WriteStartElement ("message"); + WriteCData (message); + xmlWriter.WriteEndElement (); + xmlWriter.WriteStartElement ("stack-trace"); + if (stackTrace != null) + WriteCData (stackTrace); + xmlWriter.WriteEndElement (); + xmlWriter.WriteEndElement (); + } + + private void WriteChildResults (ITestResult result) + { + xmlWriter.WriteStartElement ("results"); + + foreach (ITestResult childResult in result.Children) + WriteResultElement (childResult); + + xmlWriter.WriteEndElement (); + } + + #endregion + + #region Output Helpers + ///// + ///// Makes string safe for xml parsing, replacing control chars with '?' + ///// + ///// string to make safe + ///// xml safe string + //private static string CharacterSafeString(string encodedString) + //{ + // /*The default code page for the system will be used. + // Since all code pages use the same lower 128 bytes, this should be sufficient + // for finding uprintable control characters that make the xslt processor error. + // We use characters encoded by the default code page to avoid mistaking bytes as + // individual characters on non-latin code pages.*/ + // char[] encodedChars = System.Text.Encoding.Default.GetChars(System.Text.Encoding.Default.GetBytes(encodedString)); + + // System.Collections.ArrayList pos = new System.Collections.ArrayList(); + // for (int x = 0; x < encodedChars.Length; x++) + // { + // char currentChar = encodedChars[x]; + // //unprintable characters are below 0x20 in Unicode tables + // //some control characters are acceptable. (carriage return 0x0D, line feed 0x0A, horizontal tab 0x09) + // if (currentChar < 32 && (currentChar != 9 && currentChar != 10 && currentChar != 13)) + // { + // //save the array index for later replacement. + // pos.Add(x); + // } + // } + // foreach (int index in pos) + // { + // encodedChars[index] = '?';//replace unprintable control characters with ?(3F) + // } + // return System.Text.Encoding.Default.GetString(System.Text.Encoding.Default.GetBytes(encodedChars)); + //} + + private void WriteCData (string text) + { + int start = 0; + while (true) { + int illegal = text.IndexOf ("]]>", start); + if (illegal < 0) + break; + xmlWriter.WriteCData (text.Substring (start, illegal - start + 2)); + start = illegal + 2; + if (start >= text.Length) + return; + } + + if (start > 0) + xmlWriter.WriteCData (text.Substring (start)); + else + xmlWriter.WriteCData (text); + } + + #endregion + } +} \ No newline at end of file diff --git a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs index 7199b5114b..ffcd4da80e 100644 --- a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs @@ -23,7 +23,7 @@ namespace Xamarin.iOS.UnitTests.XUnit List filters = new List (); bool runAssemblyByDefault; - public XUnitResultFileFormat ResultFileFormat { get; set; } = XUnitResultFileFormat.NUnit; + public XUnitResultFileFormat ResultFileFormat { get; set; } = XUnitResultFileFormat.XunitV2; public AppDomainSupport AppDomainSupport { get; set; } = AppDomainSupport.Denied; protected override string ResultsFileName { get; set; } = "TestResults.xUnit.xml"; @@ -795,9 +795,10 @@ namespace Xamarin.iOS.UnitTests.XUnit { if (assembliesElement == null) return String.Empty; - + // remove all the empty nodes + assembliesElement.Descendants ().Where (e => e.Name == "collection" && !e.Descendants ().Any ()).Remove (); string outputFilePath = GetResultsFilePath (); - var settings = new XmlWriterSettings { Indent = true }; + var settings = new XmlWriterSettings { Indent = true}; using (var xmlWriter = XmlWriter.Create (outputFilePath, settings)) { switch (ResultFileFormat) { case XUnitResultFileFormat.XunitV2: @@ -818,6 +819,8 @@ namespace Xamarin.iOS.UnitTests.XUnit { if (assembliesElement == null) return; + // remove all the empty nodes + assembliesElement.Descendants ().Where (e => e.Name == "collection" && !e.Descendants ().Any ()).Remove (); var settings = new XmlWriterSettings { Indent = true }; using (var xmlWriter = XmlWriter.Create (writer, settings)) { switch (ResultFileFormat) { @@ -837,7 +840,7 @@ namespace Xamarin.iOS.UnitTests.XUnit } } } - + void Transform_Results (string xsltResourceName, XElement element, XmlWriter writer) { var xmlTransform = new System.Xml.Xsl.XslCompiledTransform (); diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index f55064cb22..733ba7f591 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -34,6 +34,13 @@ namespace xharness TodayExtension, } + public enum XmlResultType + { + TouchUnit, + NUnit, + xUnit, + } + public class AppRunner { public Harness Harness; @@ -342,35 +349,46 @@ namespace xharness return false; using (var stream = File.OpenText (filePath)) { - var firstLine = stream.ReadLine (); - return firstLine.StartsWith ("<", StringComparison.Ordinal); - } - } - - bool IsTouchUnitResult (StreamReader stream) - { - // TouchUnitTestRun is the very first node in the TouchUnit xml result - // which is not preset in the xunit xml, therefore we know the runner - // quite quickly - bool isTouchUnit = false; - using (var reader = XmlReader.Create (stream)) { - while (reader.Read ()) { - if (reader.NodeType == XmlNodeType.Element && reader.Name == "TouchUnitTestRun") { - isTouchUnit = true; - break; - } + string line; + while ((line = stream.ReadLine ()) != null) { + if (line.Contains ("ping")) + continue; + return line.StartsWith ("<", StringComparison.Ordinal); } } + return false; + } + + XmlResultType GetXmlType (StreamReader stream) + { + var resultType = XmlResultType.TouchUnit; // default + // more fun, the first like of the stream, is a ping from the application to the tcp server, and that will not be parsable as + // xml, advance the reader one line. + string line; + while ((line = stream.ReadLine ()) != null) { + if (line.Contains ("TouchUnitTestRun")) { + resultType = XmlResultType.TouchUnit; + break; + } + if (line.Contains ("nunit-version")) + resultType = XmlResultType.NUnit; + if (line.Contains ("xUnit")) { + resultType = XmlResultType.xUnit; + break; + } + } + // we want to reuse the stream (and we are sync) stream.BaseStream.Position = 0; stream.DiscardBufferedData (); - return isTouchUnit; + return resultType; } (string resultLine, bool failed) ParseTouchUnitXml (StreamReader stream, StreamWriter writer) { long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; + using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { @@ -399,7 +417,9 @@ namespace xharness { long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; - using (var reader = XmlReader.Create (stream)) { + XmlReaderSettings settings = new XmlReaderSettings (); + settings.ValidationType = ValidationType.None; + using (var reader = XmlReader.Create (stream, settings)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { total = long.Parse (reader ["total"]); @@ -442,6 +462,9 @@ namespace xharness } writer.Write (reader ["name"]); if (status == "Failure" || status == "Error") { // we need to print the message + reader.ReadToDescendant ("message"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + reader.ReadToNextSibling ("stack-trace"); writer.Write ($" : {reader.ReadElementContentAsString ()}"); } // add a new line @@ -457,10 +480,83 @@ namespace xharness return (resultLine, total == 0 | errors != 0 || failed != 0); } - - (string resultLine, bool failed, bool crashed) ParseResult (Log listener_log, bool timed_out, bool crashed) + + (string resultLine, bool failed) ParsexUnitXml (StreamReader stream, StreamWriter writer) { + long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; + total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; + using (var reader = XmlReader.Create (stream)) { + while (reader.Read ()) { + if (reader.NodeType == XmlNodeType.Element && reader.Name == "assembly") { + total += long.Parse (reader ["total"]); + errors += long.Parse (reader ["errors"]); + failed += long.Parse (reader ["failed"]); + skipped += long.Parse (reader ["skipped"]); + } + if (reader.NodeType == XmlNodeType.Element && reader.Name == "collection") { + var testCaseName = reader ["name"].Replace ("Test collection for ", ""); + writer.WriteLine (testCaseName); + var time = reader.GetAttribute ("time") ?? "0"; // some nodes might not have the time :/ + // get the first node and then move in the siblings of the same type + reader.ReadToDescendant ("test"); + do { + if (reader.Name != "test") + break; + // read the test cases in the current node + var status = reader ["result"]; + switch (status) { + case "Pass": + writer.Write ("\t[PASS] "); + break; + case "Skip": + writer.Write ("\t[IGNORED] "); + break; + case "Fail": + writer.Write ("\t[FAIL] "); + break; + default: + writer.Write ("\t[FAIL] "); + break; + } + writer.Write (reader ["name"]); + if (status == "Fail") { // we need to print the message + reader.ReadToDescendant ("message"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + reader.ReadToNextSibling ("stack-trace"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + } + // add a new line + writer.WriteLine (); + } while (reader.ReadToNextSibling ("test")); + writer.WriteLine ($"{testCaseName} {time} ms"); + } + } + } + var passed = total - errors - failed - notRun - inconclusive - ignored - skipped - invalid; + string resultLine = $"Tests run: {total} Passed: {passed} Inconclusive: {inconclusive} Failed: {failed + errors} Ignored: {ignored + skipped + invalid}"; + writer.WriteLine (resultLine); + + return (resultLine, total == 0 | errors != 0 || failed != 0); + } + + void CleanXml (string source, string destination) { - if (!File.Exists (listener_log.FullPath)) + using (var reader = new StreamReader (source)) + using (var writer = new StreamWriter (destination)) { + string line; + while ((line = reader.ReadLine ()) != null) { + if (line.StartsWith ("ping", StringComparison.InvariantCulture) || line.Contains ("TouchUnitTestRun") || line.Contains ("NUnitOutput") || line.Contains ("