[XHarness] Add XUnit category filtering. (#5455)

Provide the ability to filter per categories in the xunit runner and add the default categories for Mac, iOS, TvOS and WatchOS.

tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunner.cs

@@ -34,6 +34,7 @@ namespace Xamarin.iOS.UnitTests
 		public abstract string WriteResultsToFile ();
 		public abstract void WriteResultsToFile (TextWriter writer);
 		public abstract void SkipTests (IEnumerable<string> tests);
+		public abstract void SkipCategories (IEnumerable<string> categories);
 
 		protected void OnError (string message)
 		{

tests/bcl-test/BCLTests/templates/common/TestRunner.NUnit/NUnitTestRunner.cs

@@ -271,26 +271,42 @@ namespace Xamarin.iOS.UnitTests.NUnit
 			resultsXml.WriteResultFile (results, writer);
 		}
 		
+		void AppendFilter (ITestFilter filter)
+		{
+			if (filter == null)
+				throw new ArgumentNullException (nameof (filter));
+			if (Filter.IsEmpty) {
+				Filter = filter;
+			} else {
+				AndFilter andFilter;
+				if (Filter is AndFilter) {
+					// add a new filter
+					andFilter = Filter as AndFilter;
+					andFilter.Add (filter);
+				} else {
+					andFilter = new AndFilter (Filter); 
+					andFilter.Add (filter);
+				}
+				Filter = andFilter;
+			}
+		}
+
 		public override void SkipTests (IEnumerable<string> tests)
 		{
 			// grab the tests and create a filter for them
 			if (tests.Any ()) {
-				if (Filter.IsEmpty) {
-					Filter = new TestMethodFilter (tests);
-				} else {
-					// create a special group filter with the previous one
-					// and the new one
-					AndFilter andFilter;
-					if (Filter is AndFilter) {
-						// add a new filter
-						andFilter = Filter as AndFilter;
-						andFilter.Add (new TestMethodFilter (tests));
-					} else {
-						andFilter = new AndFilter (Filter); 
-						andFilter.Add (new TestMethodFilter (tests));
-					}
-					Filter = andFilter;
-				}
+				AppendFilter (new TestMethodFilter (tests));
+			}
+		}
+
+		public override void SkipCategories (IEnumerable<string> categories)
+		{
+			if (categories.Any ()) {
+				// build a category expression and trust the nunit lib
+				var expression = categories.Aggregate (
+					(current, next) => current + "," + next);
+				var categoriesFilter = new NotFilter (new CategoryExpression (expression).Filter);
+				AppendFilter (categoriesFilter);
 			}
 		}
 	}

tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitTestRunner.cs

@@ -73,6 +73,12 @@ namespace Xamarin.iOS.UnitTests.XUnit
 			messageSink.Execution.TestStartingEvent += (MessageHandlerArgs<ITestStarting> args) => HandleEvent ("TestStartingEvent", args, HandleTestStarting);
 		}
 
+		public void AddFilter (XUnitFilter filter)
+		{
+			if (filter != null) {
+				filters.Add (filter);
+			}
+		}
 		public void SetFilters (List<XUnitFilter> newFilters)
 		{
 			if (newFilters == null) {
@@ -1015,5 +1021,14 @@ namespace Xamarin.iOS.UnitTests.XUnit
 				}
 			}
 		}
+
+		public override void SkipCategories (IEnumerable<string> categories)
+		{
+			if (categories.Any ()) {
+				foreach (var c in categories) {
+					filters.Add (XUnitFilter.CreateTraitFilter ("category", c, true));
+				}
+			}
+		}
 	}
 }

tests/bcl-test/BCLTests/templates/iOSApp/ViewController.cs

@@ -75,17 +75,36 @@ namespace BCLTests {
 			var logger = (writer == null || options.EnableXml) ? new LogWriter () : new LogWriter (writer);
 			logger.MinimumLogLevel = MinimumLogLevel.Info;
 			var testAssemblies = GetTestAssemblies ();
-			Xamarin.iOS.UnitTests.TestRunner runner;
-			if (RegisterType.IsXUnit)
-				runner = new XUnitTestRunner (logger);
-			else {
-				runner = new NUnitTestRunner (logger) {
-					// add known ignored categories.
-					Filter = new NotFilter (new CategoryExpression ("MobileNotWorking,NotOnMac,NotWorking,ValueAdd,CAS,InetAccess,NotWorkingLinqInterpreter").Filter)
+			var runner = RegisterType.IsXUnit ? (Xamarin.iOS.UnitTests.TestRunner) new XUnitTestRunner (logger) : new NUnitTestRunner (logger);
+			var categories = RegisterType.IsXUnit ?
+				new List<string> { 
+					"failing",
+					"nonmonotests",
+					"outerloop",
+					"nonosxtests"
+				} :
+				new List<string> {
+					"MobileNotWorking",
+					"NotOnMac",
+					"NotWorking",
+					"ValueAdd",
+					"CAS",
+					"InetAccess",
+					"NotWorkingLinqInterpreter",
 				};
+
+			if (RegisterType.IsXUnit) {
+				// special case when we are using the xunit runner,
+				// there is a trait we are not interested in which is 
+				// the Benchmark one
+				var xunitRunner = runner as XUnitTestRunner;
+				xunitRunner.AddFilter (XUnitFilter.CreateTraitFilter ("Benchmark", "true", true));
 			}
-			
+
 			// add category filters if they have been added
+			runner.SkipCategories (categories);
+			
+			// if we have ignore files, ignore those tests
 			var skippedTests = await IgnoreFileParser.ParseContentFilesAsync (NSBundle.MainBundle.BundlePath);
 			if (skippedTests.Any ()) {
 				// ensure that we skip those tests that have been passed via the ignore files

tests/bcl-test/BCLTests/templates/macOS/MacTestMain.cs

@@ -49,21 +49,42 @@ namespace Xamarin.Mac.Tests
 			var logger = new LogWriter (Console.Out); 
 			logger.MinimumLogLevel = MinimumLogLevel.Info;
 			var testAssemblies = GetTestAssemblies ();
-			TestRunner runner;
-			if (RegisterType.IsXUnit)
-				runner = new XUnitTestRunner (logger);
-			else {
-				runner = new NUnitTestRunner (logger) {
-					Filter = new NotFilter (new CategoryExpression ("MacNotWorking,MobileNotWorking,NotOnMac,NotWorking,ValueAdd,CAS,InetAccess,NotWorkingLinqInterpreter").Filter)
+			var runner = RegisterType.IsXUnit ? (TestRunner) new XUnitTestRunner (logger) : new NUnitTestRunner (logger);
+			var categories = RegisterType.IsXUnit ?
+				new List<string> { 
+					"failing",
+					"nonmonotests",
+					"outerloop",
+					"nonosxtests"
+				} :
+				new List<string> {
+					"MacNotWorking",
+					"MobileNotWorking",
+					"NotOnMac",
+					"NotWorking",
+					"ValueAdd",
+					"CAS",
+					"InetAccess",
+					"NotWorkingLinqInterpreter"
 				};
+
+			if (RegisterType.IsXUnit) {
+				// special case when we are using the xunit runner,
+				// there is a trait we are not interested in which is
+				// the Benchmark one
+				var xunitRunner = runner as XUnitTestRunner;
+				xunitRunner.AddFilter (XUnitFilter.CreateTraitFilter ("Benchmark", "true", true));
 			}
 			
+			runner.SkipCategories (categories);
 			runner.Run (testAssemblies.ToList ());
-			
-			using (var writer = new StreamWriter(options.ResultFile)) {
-				runner.WriteResultsToFile (writer);
+
+			if (options.ResultFile != null) {
+				using (var writer = new StreamWriter (options.ResultFile)) {
+					runner.WriteResultsToFile (writer);
+				}
+				logger.Info ($"Xml result can be found {options.ResultFile}");
 			}
-			logger.Info ($"Xml result can be found {options.ResultFile}");
 			
 			logger.Info ($"Tests run: {runner.TotalTests} Passed: {runner.PassedTests} Inconclusive: {runner.InconclusiveTests} Failed: {runner.FailedTests} Ignored: {runner.SkippedTests}");
 			return runner.FailedTests != 0 ? 1 : 0;

tests/bcl-test/BCLTests/templates/tvOSApp/ViewController.cs

@@ -73,15 +73,36 @@ namespace BCLTests {
 			var logger = (writer == null || options.EnableXml) ? new LogWriter () : new LogWriter (writer);
 			logger.MinimumLogLevel = MinimumLogLevel.Info;
 			var testAssemblies = GetTestAssemblies ();
-			Xamarin.iOS.UnitTests.TestRunner runner;
-			if (RegisterType.IsXUnit)
-				runner = new XUnitTestRunner (logger);
-			else {
-				runner = new NUnitTestRunner (logger) { 
-					Filter = new NotFilter (new CategoryExpression ("MobileNotWorking,NotOnMac,NotWorking,ValueAdd,CAS,InetAccess,NotWorkingLinqInterpreter").Filter)
+			var runner = RegisterType.IsXUnit ? (Xamarin.iOS.UnitTests.TestRunner) new XUnitTestRunner (logger) : new NUnitTestRunner (logger);
+			var categories = RegisterType.IsXUnit ?
+				new List<string> { 
+					"failing",
+					"nonmonotests",
+					"outerloop",
+					"nonosxtests"
+				} :
+				new List<string> {
+					"MobileNotWorking",
+					"NotOnMac",
+					"NotWorking",
+					"ValueAdd",
+					"CAS",
+					"InetAccess",
+					"NotWorkingLinqInterpreter",
 				};
+				
+			if (RegisterType.IsXUnit) {
+				// special case when we are using the xunit runner,
+				// there is a trait we are not interested in which is 
+				// the Benchmark one
+				var xunitRunner = runner as XUnitTestRunner;
+				xunitRunner.AddFilter (XUnitFilter.CreateTraitFilter ("Benchmark", "true", true));
 			}
 
+			// add category filters if they have been added
+			runner.SkipCategories (categories);
+			
+			// if we have ignore files, ignore those tests
 			var skippedTests = await IgnoreFileParser.ParseContentFilesAsync (NSBundle.MainBundle.BundlePath);
 			if (skippedTests.Any ()) {
 				// ensure that we skip those tests that have been passed via the ignore files

tests/bcl-test/BCLTests/templates/watchOS/Extension/InterfaceController.cs

@@ -104,14 +104,35 @@ namespace monotouchtestWatchKitExtension
 			var logger = (writer == null || options.EnableXml) ? new LogWriter () : new LogWriter (writer);
 			logger.MinimumLogLevel = MinimumLogLevel.Info;
 			var testAssemblies = GetTestAssemblies ();
-			if (RegisterType.IsXUnit)
-				runner = new XUnitTestRunner (logger);
-			else {
-				runner = new NUnitTestRunner (logger) { 
-					Filter = new NotFilter (new CategoryExpression ("MobileNotWorking,NotOnMac,NotWorking,ValueAdd,CAS,InetAccess,NotWorkingLinqInterpreter,RequiresBSDSockets").Filter)
+			runner = RegisterType.IsXUnit ? (Xamarin.iOS.UnitTests.TestRunner) new XUnitTestRunner (logger) : new NUnitTestRunner (logger);
+			var categories = RegisterType.IsXUnit ?
+				new List<string> { 
+					"failing",
+					"nonmonotests",
+					"outerloop",
+					"nonosxtests"
+				} :
+				new List<string> {
+					"MobileNotWorking",
+					"NotOnMac",
+					"NotWorking",
+					"ValueAdd",
+					"CAS",
+					"InetAccess",
+					"NotWorkingLinqInterpreter",
 				};
+
+			if (RegisterType.IsXUnit) {
+				// special case when we are using the xunit runner,
+				// there is a trait we are not interested in which is 
+				// the Benchmark one
+				var xunitRunner = runner as XUnitTestRunner;
+				xunitRunner.AddFilter (XUnitFilter.CreateTraitFilter ("Benchmark", "true", true));
 			}
+			// add category filters if they have been added
+			runner.SkipCategories (categories);
 			
+			// if we have ignore files, ignore those tests
 			var skippedTests = IgnoreFileParser.ParseContentFiles (NSBundle.MainBundle.BundlePath);
 			if (skippedTests.Any ()) {
 				// ensure that we skip those tests that have been passed via the ignore files