From af389bda1f5901dfebb1936c8cdb3765a120dc90 Mon Sep 17 00:00:00 2001
From: Rolf Bjarne Kvinge <rolf@xamarin.com>
Date: Thu, 23 Nov 2017 19:16:02 +0100
Subject: [PATCH] [tests] Improve how test failures show up in the html report
 for Xamarin.Mac tests. (#3033)

* [tests] Add support for passing arguments to XM unit tests from the command line.

* [xharness] Get xml results for mac unit tests and parse it to show failures inline in the html report.
---
 tests/common/mac/MacTestMain.cs |  9 +++++++--
 tests/xharness/Jenkins.cs       | 29 +++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/tests/common/mac/MacTestMain.cs b/tests/common/mac/MacTestMain.cs
index c04f3b3f46..96e53040bd 100644
--- a/tests/common/mac/MacTestMain.cs
+++ b/tests/common/mac/MacTestMain.cs
@@ -25,10 +25,10 @@ namespace Xamarin.Mac.Tests
 #if !NO_GUI_TESTING
 			NSApplication.Init();
 #endif
-			RunTests ();
+			RunTests (args);
 		}
 
-		static void RunTests ()
+		static void RunTests (string [] original_args)
 		{
 			TestRunner.MainLoop = new NSRunLoopIntegration ();
 			List<string> args = new List<string> () { typeof (MainClass).Assembly.Location, "-labels", "-noheader" };
@@ -41,6 +41,11 @@ namespace Xamarin.Mac.Tests
 			args.Add ("-exclude=MacNotWorking,MobileNotWorking,NotOnMac,NotWorking,ValueAdd,CAS,InetAccess,NotWorkingInterpreter");
 #endif
 
+			// Skip arguments added by VSfM/macOS when running from the IDE
+			foreach (var arg in original_args)
+				if (!arg.StartsWith ("-psn_", StringComparison.Ordinal))
+					args.Add (arg);
+
 			TestRunner.Main (args.ToArray ());
 
 #if NO_GUI_TESTING
diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs
index 6cfb9dcb77..bcfc078273 100644
--- a/tests/xharness/Jenkins.cs
+++ b/tests/xharness/Jenkins.cs
@@ -549,6 +549,7 @@ namespace xharness
 							Ignored = ignored || !IncludeClassicMac,
 							BCLTest = project.IsBclTest,
 							TestName = project.Name,
+							IsUnitTest = true,
 						};
 					}
 					exec.Variation = configurations.Length > 1 ? config : project.TargetFrameworkFlavor.ToString ();
@@ -663,6 +664,7 @@ namespace xharness
 				return new MacExecuteTask (build) {
 					Ignored = ignore,
 					TestName = task.TestName,
+					IsUnitTest = macExec.IsUnitTest,
 				};
 			}
 			var nunit = task as NUnitExecuteTask;
@@ -1830,6 +1832,28 @@ function oninitialload ()
 										} catch (Exception ex) {
 											writer.WriteLine ("<span style='padding-left: 15px;'>Could not parse log file: {0}</span><br />", System.Web.HttpUtility.HtmlEncode (ex.Message));
 										}
+									} else if (log.Description == "NUnit results") {
+										try {
+											var doc = new System.Xml.XmlDocument ();
+											doc.LoadWithoutNetworkAccess (log.FullPath);
+											var failures = doc.SelectNodes ("//test-case[@result='Error' or @result='Failure']").Cast<System.Xml.XmlNode> ().ToArray ();
+											if (failures.Length > 0) {
+												writer.WriteLine ("<div style='padding-left: 15px;'>");
+												foreach (var failure in failures) {
+													var test_name = failure.Attributes ["name"]?.Value;
+													var message = failure.SelectSingleNode ("failure/message")?.InnerText;
+													writer.Write (System.Web.HttpUtility.HtmlEncode (test_name));
+													if (!string.IsNullOrEmpty (message)) {
+														writer.Write (": ");
+														writer.Write (System.Web.HttpUtility.HtmlEncode (message));
+													}
+													writer.WriteLine ("<br />");
+												}
+												writer.WriteLine ("</div>");
+											}
+										} catch (Exception ex) {
+											writer.WriteLine ($"<span style='padding-left: 15px;'>Could not parse {log.Description}: {System.Web.HttpUtility.HtmlEncode (ex.Message)}</span><br />");
+										}
 									}
 								}
 							}
@@ -2584,6 +2608,7 @@ function oninitialload ()
 	{
 		public string Path;
 		public bool BCLTest;
+		public bool IsUnitTest;
 
 		public MacExecuteTask (BuildToolTask build_task)
 			: base (build_task)
@@ -2650,6 +2675,10 @@ function oninitialload ()
 			using (var resource = await NotifyAndAcquireDesktopResourceAsync ()) {
 				using (var proc = new Process ()) {
 					proc.StartInfo.FileName = Path;
+					if (IsUnitTest) {
+						using (var xml = Logs.CreateFile ("NUnit results", System.IO.Path.Combine (LogDirectory, $"test-{Platform}-{Timestamp}.xml"), false))
+							proc.StartInfo.Arguments = $"-result={StringUtils.Quote (xml.FullPath)}";
+					}
 					Jenkins.MainLog.WriteLine ("Executing {0} ({1})", TestName, Mode);
 					var log = Logs.CreateStream (LogDirectory, $"execute-{Platform}-{Timestamp}.txt", "Execution log");
 					log.WriteLine ("{0} {1}", proc.StartInfo.FileName, proc.StartInfo.Arguments);