diff --git a/tests/bcl-test/BCLTests/BCLTests-mac.csproj.in b/tests/bcl-test/BCLTests/BCLTests-mac.csproj.in index 55e151b5cf..a18b749a96 100644 --- a/tests/bcl-test/BCLTests/BCLTests-mac.csproj.in +++ b/tests/bcl-test/BCLTests/BCLTests-mac.csproj.in @@ -139,6 +139,9 @@ TestRunner.NUnit\NUnitTestRunner.cs + + TestRunner.NUnit\ClassOrNamespaceFilter.cs + TestRunner.Core\Extensions.Bool.cs @@ -166,6 +169,12 @@ TestRunner.Core\TestRunner.cs + + TestRunner.Core\TestRunSelector.cs + + + TestRunner.Core\TestRunSelectorType.cs + TestRunner.xUnit\XUnitFilter.cs diff --git a/tests/bcl-test/BCLTests/BCLTests-tv.csproj.in b/tests/bcl-test/BCLTests/BCLTests-tv.csproj.in index 3194f9b064..9b5e3e2db5 100644 --- a/tests/bcl-test/BCLTests/BCLTests-tv.csproj.in +++ b/tests/bcl-test/BCLTests/BCLTests-tv.csproj.in @@ -210,6 +210,9 @@ TestRunner.NUnit\NUnitTestRunner.cs + + TestRunner.NUnit\ClassOrNamespaceFilter.cs + TestRunner.Core\Extensions.Bool.cs @@ -237,6 +240,12 @@ TestRunner.Core\TestRunner.cs + + TestRunner.Core\TestRunSelector.cs + + + TestRunner.Core\TestRunSelectorType.cs + TestRunner.xUnit\XUnitFilter.cs diff --git a/tests/bcl-test/BCLTests/BCLTests-watchos-extension.csproj.in b/tests/bcl-test/BCLTests/BCLTests-watchos-extension.csproj.in index 3b20fdcd2a..713e59941e 100644 --- a/tests/bcl-test/BCLTests/BCLTests-watchos-extension.csproj.in +++ b/tests/bcl-test/BCLTests/BCLTests-watchos-extension.csproj.in @@ -184,9 +184,18 @@ TestRunner.Core\TestRunner.cs + + TestRunner.Core\TestRunSelector.cs + + + TestRunner.Core\TestRunSelectorType.cs + TestRunner.NUnit\NUnitTestRunner.cs + + TestRunner.NUnit\ClassOrNamespaceFilter.cs + TestRunner.xUnit\XUnitFilter.cs diff --git a/tests/bcl-test/BCLTests/BCLTests.csproj.in b/tests/bcl-test/BCLTests/BCLTests.csproj.in index c11614cc13..62b25cd06c 100644 --- a/tests/bcl-test/BCLTests/BCLTests.csproj.in +++ b/tests/bcl-test/BCLTests/BCLTests.csproj.in @@ -222,6 +222,9 @@ TestRunner.NUnit\NUnitTestRunner.cs + + TestRunner.NUnit\ClassOrNamespaceFilter.cs + TestRunner.Core\Extensions.Bool.cs @@ -249,6 +252,12 @@ TestRunner.Core\TestRunner.cs + + TestRunner.Core\TestRunSelector.cs + + + TestRunner.Core\TestRunSelectorType.cs + TestRunner.xUnit\XUnitFilter.cs diff --git a/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunSelector.cs b/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunSelector.cs new file mode 100644 index 0000000000..f371375d49 --- /dev/null +++ b/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunSelector.cs @@ -0,0 +1,12 @@ +using System; + +namespace Xamarin.iOS.UnitTests +{ + public class TestRunSelector + { + public string Assembly { get; set; } + public string Value { get; set; } + public TestRunSelectorType Type { get; set; } + public bool Include { get; set; } + } +} diff --git a/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunSelectorType.cs b/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunSelectorType.cs new file mode 100644 index 0000000000..67180421c0 --- /dev/null +++ b/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunSelectorType.cs @@ -0,0 +1,12 @@ +using System; + +namespace Xamarin.iOS.UnitTests +{ + public enum TestRunSelectorType + { + Assembly, + Namespace, + Class, + Single, + } +} diff --git a/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunner.cs b/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunner.cs index c080ab2ac0..02372bbd7c 100644 --- a/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunner.cs +++ b/tests/bcl-test/BCLTests/templates/common/TestRunner.Core/TestRunner.cs @@ -17,6 +17,8 @@ namespace Xamarin.iOS.UnitTests public long FilteredTests { get; protected set; } = 0; public bool RunInParallel { get; set; } = false; public string TestsRootDirectory { get; set; } + public bool RunAllTestsByDefault { get; set; } = true; + public bool LogExcludedTests { get; set; } public TextWriter Writer { get; set; } public List FailureInfos { get; } = new List (); diff --git a/tests/bcl-test/BCLTests/templates/common/TestRunner.NUnit/ClassOrNamespaceFilter.cs b/tests/bcl-test/BCLTests/templates/common/TestRunner.NUnit/ClassOrNamespaceFilter.cs new file mode 100644 index 0000000000..7e5552f2ee --- /dev/null +++ b/tests/bcl-test/BCLTests/templates/common/TestRunner.NUnit/ClassOrNamespaceFilter.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; + +using NUnit.Framework.Api; +using NUnit.Framework.Internal; +using NUnit.Framework.Internal.Filters; + +namespace Xamarin.iOS.UnitTests.NUnit +{ + public class ClassOrNamespaceFilter : TestFilter + { + bool isClassFilter; + List names; + + public ClassOrNamespaceFilter (string name, bool isClassFilter) + { + AddName (name); + this.isClassFilter = isClassFilter; + } + + public ClassOrNamespaceFilter (IEnumerable names, bool isClassFilter) + { + if (names == null) + throw new ArgumentNullException (nameof (names)); + + this.isClassFilter = isClassFilter; + foreach (string n in names) { + string name = n?.Trim (); + if (String.IsNullOrEmpty (name)) + continue; + + AddName (name); + } + } + + public void AddName (string name) + { + if (String.IsNullOrEmpty (name)) + throw new ArgumentException ("must not be null or empty", nameof (name)); + + if (names == null) + names = new List (); + if (names.Contains (name)) + return; + + names.Add (name); + } + + public override bool Match (ITest test) + { + if (test == null || names == null || names.Count == 0) + return false; + + if (test.FixtureType == null) + return false; // It's probably an assembly name, all tests will have a fixture + + if (isClassFilter) + return NameMatches (test.FixtureType.FullName); + + int dot = test.FixtureType.FullName.LastIndexOf ('.'); + if (dot < 1) + return false; + + return NameMatches (test.FixtureType.FullName.Substring (0, dot)); + } + + bool NameMatches (string name) + { + foreach (string n in names) { + if (n == null) + continue; + if (String.Compare (name, n, StringComparison.Ordinal) == 0) + return true; + } + + return false; + } + } +} diff --git a/tests/bcl-test/BCLTests/templates/common/TestRunner.NUnit/NUnitTestRunner.cs b/tests/bcl-test/BCLTests/templates/common/TestRunner.NUnit/NUnitTestRunner.cs index 6f57573f7d..0f8ccf75d4 100644 --- a/tests/bcl-test/BCLTests/templates/common/TestRunner.NUnit/NUnitTestRunner.cs +++ b/tests/bcl-test/BCLTests/templates/common/TestRunner.NUnit/NUnitTestRunner.cs @@ -1,7 +1,8 @@ -using System; -using System.IO; +using System; using System.Collections; using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Reflection; using System.Text; @@ -21,9 +22,11 @@ namespace Xamarin.iOS.UnitTests.NUnit { Dictionary builderSettings; TestSuiteResult results; + bool runAssemblyByDefault; public ITestFilter Filter { get; set; } = TestFilter.Empty; public bool GCAfterEachFixture { get; set; } + public Dictionary AssemblyFilters { get; set; } protected override string ResultsFileName { get; set; } = "TestResults.NUnit.xml"; @@ -36,23 +39,31 @@ namespace Xamarin.iOS.UnitTests.NUnit { if (testAssemblies == null) throw new ArgumentNullException (nameof (testAssemblies)); - + + if (AssemblyFilters == null || AssemblyFilters.Count == 0) + runAssemblyByDefault = true; + else + runAssemblyByDefault = AssemblyFilters.Values.Any (v => !v); + var builder = new NUnitLiteTestAssemblyBuilder (); var runner = new NUnitLiteTestAssemblyRunner (builder, new FinallyDelegate ()); var testSuite = new TestSuite (NSBundle.MainBundle.BundleIdentifier); results = new TestSuiteResult (testSuite); + TotalTests = 0; foreach (TestAssemblyInfo assemblyInfo in testAssemblies) { - if (assemblyInfo == null || assemblyInfo.Assembly == null) + if (assemblyInfo == null || assemblyInfo.Assembly == null || !ShouldRunAssembly (assemblyInfo)) continue; - + if (!runner.Load (assemblyInfo.Assembly, builderSettings)) { OnWarning ($"Failed to load tests from assembly '{assemblyInfo.Assembly}"); continue; } - if (runner.LoadedTest is NUnitTest tests) + if (runner.LoadedTest is NUnitTest tests) { + TotalTests += tests.TestCaseCount; testSuite.Add (tests); - + } + // Messy API. .Run returns ITestResult which is, in reality, an instance of TestResult since that's // what WorkItem returns and we need an instance of TestResult to add it to TestSuiteResult. So, cast // the return to TestResult and hope for the best. @@ -73,9 +84,45 @@ namespace Xamarin.iOS.UnitTests.NUnit results.AddResult (testResult); } + // NUnitLite doesn't report filtered tests at all, but we can calculate here + FilteredTests = TotalTests - ExecutedTests; LogFailureSummary (); } + bool ShouldRunAssembly (TestAssemblyInfo assemblyInfo) + { + if (assemblyInfo == null) + return false; + + if (AssemblyFilters == null || AssemblyFilters.Count == 0) + return true; + + bool include; + if (AssemblyFilters.TryGetValue (assemblyInfo.FullPath, out include)) + return ReportFilteredAssembly (assemblyInfo, include); + + string fileName = Path.GetFileName (assemblyInfo.FullPath); + if (AssemblyFilters.TryGetValue (fileName, out include)) + return ReportFilteredAssembly (assemblyInfo, include); + + fileName = Path.GetFileNameWithoutExtension (assemblyInfo.FullPath); + if (AssemblyFilters.TryGetValue (fileName, out include)) + return ReportFilteredAssembly (assemblyInfo, include); + + return runAssemblyByDefault; + } + + bool ReportFilteredAssembly (TestAssemblyInfo assemblyInfo, bool include) + { + if (LogExcludedTests) { + const string included = "Included"; + const string excluded = "Excluded"; + + OnInfo ($"[FILTER] {(include ? included : excluded)} assembly: {assemblyInfo.FullPath}"); + } + return include; + } + public bool Pass (ITest test) { return true; diff --git a/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitFilter.cs b/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitFilter.cs index 48e1337479..e4731e453d 100644 --- a/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitFilter.cs +++ b/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitFilter.cs @@ -1,28 +1,128 @@ -using System; +using System; +using System.Text; namespace Xamarin.iOS.UnitTests.XUnit { public class XUnitFilter { - public string TraitName { get; } - public string TraitValue { get; } - public string TestCaseName { get; } - public bool Exclude { get; } - public XUnitFilterType FilterType { get; } + public string AssemblyName { get; private set; } + public string SelectorName { get; private set; } + public string SelectorValue { get; private set; } - public XUnitFilter (string testCaseName, bool exclude) + public bool Exclude { get; private set; } + public XUnitFilterType FilterType { get; private set; } + + public static XUnitFilter CreateSingleFilter (string singleTestName, bool exclude, string assemblyName = null) { - FilterType = XUnitFilterType.TypeName; - TestCaseName = testCaseName; - Exclude = exclude; + if (String.IsNullOrEmpty (singleTestName)) + throw new ArgumentException("must not be null or empty", nameof (singleTestName)); + + return new XUnitFilter { + AssemblyName = assemblyName, + SelectorValue = singleTestName, + FilterType = XUnitFilterType.Single, + Exclude = exclude + }; } - public XUnitFilter (string traitName, string traitValue, bool exclude) + public static XUnitFilter CreateAssemblyFilter (string assemblyName, bool exclude) { - FilterType = XUnitFilterType.Trait; - TraitName = traitName; - TraitValue = traitValue; - Exclude = exclude; + if (String.IsNullOrEmpty (assemblyName)) + throw new ArgumentException("must not be null or empty", nameof (assemblyName)); + + return new XUnitFilter { + AssemblyName = assemblyName, + FilterType = XUnitFilterType.Assembly, + Exclude = exclude + }; + } + + public static XUnitFilter CreateNamespaceFilter (string namespaceName, bool exclude, string assemblyName = null) + { + if (String.IsNullOrEmpty (namespaceName)) + throw new ArgumentException("must not be null or empty", nameof (namespaceName)); + + return new XUnitFilter { + AssemblyName = assemblyName, + SelectorValue = namespaceName, + FilterType = XUnitFilterType.Namespace, + Exclude = exclude + }; + } + + public static XUnitFilter CreateClassFilter (string className, bool exclude, string assemblyName = null) + { + if (String.IsNullOrEmpty (className)) + throw new ArgumentException("must not be null or empty", nameof (className)); + + return new XUnitFilter { + AssemblyName = assemblyName, + SelectorValue = className, + FilterType = XUnitFilterType.TypeName, + Exclude = exclude + }; + } + + public static XUnitFilter CreateTraitFilter (string traitName, string traitValue, bool exclude) + { + if (String.IsNullOrEmpty (traitName)) + throw new ArgumentException("must not be null or empty", nameof (traitName)); + + return new XUnitFilter { + AssemblyName = null, + SelectorName = traitName, + SelectorValue = traitValue ?? String.Empty, + FilterType = XUnitFilterType.Trait, + Exclude = exclude + }; + } + + public override string ToString () + { + var sb = new StringBuilder ("XUnitFilter ["); + + sb.Append ($"Type: {FilterType}; "); + sb.Append (Exclude ? "exclude" : "include"); + + if (!String.IsNullOrEmpty (AssemblyName)) + sb.Append ($"; AssemblyName: {AssemblyName}"); + + switch (FilterType) { + case XUnitFilterType.Assembly: + break; + + case XUnitFilterType.Namespace: + AppendDesc ("Namespace", SelectorValue); + break; + + case XUnitFilterType.Single: + AppendDesc ("Method", SelectorValue); + break; + + case XUnitFilterType.Trait: + AppendDesc ("Trait name", SelectorName); + AppendDesc ("Trait value", SelectorValue); + break; + + case XUnitFilterType.TypeName: + AppendDesc ("Class", SelectorValue); + break; + + default: + sb.Append ("; Unknown filter type"); + break; + } + sb.Append (']'); + + return sb.ToString (); + + void AppendDesc (string name, string value) + { + if (String.IsNullOrEmpty (value)) + return; + + sb.Append ($"; {name}: {value}"); + } } } -} \ No newline at end of file +} diff --git a/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitFilterType.cs b/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitFilterType.cs index 2d1d7070ea..4cd3021208 100644 --- a/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitFilterType.cs +++ b/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitFilterType.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace Xamarin.iOS.UnitTests.XUnit { @@ -6,5 +6,8 @@ namespace Xamarin.iOS.UnitTests.XUnit { Trait, TypeName, + Assembly, + Single, + Namespace, } -} \ No newline at end of file +} diff --git a/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitTestRunner.cs b/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitTestRunner.cs index 37e44a8295..b84f69df41 100644 --- a/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitTestRunner.cs +++ b/tests/bcl-test/BCLTests/templates/common/TestRunner.xUnit/XUnitTestRunner.cs @@ -19,6 +19,7 @@ namespace Xamarin.iOS.UnitTests.XUnit XElement assembliesElement; List filters; + bool runAssemblyByDefault; public XUnitResultFileFormat ResultFileFormat { get; set; } = XUnitResultFileFormat.NUnit; public AppDomainSupport AppDomainSupport { get; set; } = AppDomainSupport.Denied; @@ -677,9 +678,23 @@ namespace Xamarin.iOS.UnitTests.XUnit if (testAssemblies == null) throw new ArgumentNullException (nameof (testAssemblies)); + if (filters != null && filters.Count > 0) { + do_log ("Configured filters:"); + foreach (XUnitFilter filter in filters) { + do_log ($" {filter}"); + } + } + + List assemblyFilters = filters?.Where (sel => sel != null && sel.FilterType == XUnitFilterType.Assembly)?.ToList (); + if (assemblyFilters == null || assemblyFilters.Count == 0) { + runAssemblyByDefault = true; + assemblyFilters = null; + } else + runAssemblyByDefault = assemblyFilters.Any (f => f != null && f.Exclude); + assembliesElement = new XElement ("assemblies"); foreach (TestAssemblyInfo assemblyInfo in testAssemblies) { - if (assemblyInfo == null || assemblyInfo.Assembly == null) + if (assemblyInfo == null || assemblyInfo.Assembly == null || !ShouldRunAssembly (assemblyInfo)) continue; if (String.IsNullOrEmpty (assemblyInfo.FullPath)) { @@ -703,6 +718,50 @@ namespace Xamarin.iOS.UnitTests.XUnit } LogFailureSummary (); + + bool ShouldRunAssembly (TestAssemblyInfo assemblyInfo) + { + if (assemblyInfo == null) + return false; + + if (assemblyFilters == null) + return true; + + foreach (XUnitFilter filter in assemblyFilters) { + if (String.Compare (filter.AssemblyName, assemblyInfo.FullPath, StringComparison.Ordinal) == 0) + return ReportFilteredAssembly (assemblyInfo, filter); + + string fileName = Path.GetFileName (assemblyInfo.FullPath); + if (String.Compare (fileName, filter.AssemblyName, StringComparison.Ordinal) == 0) + return ReportFilteredAssembly (assemblyInfo, filter); + + string filterExtension = Path.GetExtension (filter.AssemblyName); + if (String.IsNullOrEmpty (filterExtension) || + (String.Compare (filterExtension, ".exe", StringComparison.OrdinalIgnoreCase) != 0 && + String.Compare (filterExtension, ".dll", StringComparison.OrdinalIgnoreCase) != 0)) { + string asmName = $"{filter.AssemblyName}.dll"; + if (String.Compare (asmName, fileName, StringComparison.Ordinal) == 0) + return ReportFilteredAssembly (assemblyInfo, filter); + + asmName = $"{filter.AssemblyName}.exe"; + if (String.Compare (asmName, fileName, StringComparison.Ordinal) == 0) + return ReportFilteredAssembly (assemblyInfo, filter); + } + } + + return runAssemblyByDefault; + } + + bool ReportFilteredAssembly (TestAssemblyInfo assemblyInfo, XUnitFilter filter) + { + if (LogExcludedTests) { + const string included = "Included"; + const string excluded = "Excluded"; + + OnInfo ($"[FILTER] {(filter.Exclude ? excluded : included)} assembly: {assemblyInfo.FullPath}"); + } + return !filter.Exclude; + } } public override string WriteResultsToFile () @@ -863,42 +922,88 @@ namespace Xamarin.iOS.UnitTests.XUnit bool IsIncluded (ITestCase testCase) { - if (testCase.Traits == null || testCase.Traits.Count == 0) - return true; - + if (testCase == null) + return false; + + bool haveTraits = testCase.Traits != null && testCase.Traits.Count > 0; foreach (XUnitFilter filter in filters) { List values; if (filter == null) continue; if (filter.FilterType == XUnitFilterType.Trait) { - if (!testCase.Traits.TryGetValue (filter.TraitName, out values)) + if (!haveTraits || !testCase.Traits.TryGetValue (filter.SelectorName, out values)) continue; if (values == null || values.Count == 0) { // We have no values and the filter doesn't specify one - that means we match on // the trait name only. - if (String.IsNullOrEmpty (filter.TraitValue)) - return !filter.Exclude; + if (String.IsNullOrEmpty (filter.SelectorValue)) + return ReportFilteredTest (filter); continue; } - if (values.Contains (filter.TraitValue, StringComparer.OrdinalIgnoreCase)) - return !filter.Exclude; + if (values.Contains (filter.SelectorValue, StringComparer.OrdinalIgnoreCase)) + return ReportFilteredTest (filter); continue; } if (filter.FilterType == XUnitFilterType.TypeName) { - Logger.Info ($"IsIncluded: filter: '{filter.TestCaseName}', test case name: {testCase.DisplayName}"); - if (String.Compare (testCase.DisplayName, filter.TestCaseName, StringComparison.OrdinalIgnoreCase) == 0) - return !filter.Exclude; + string testClassName = testCase.TestMethod?.TestClass?.Class?.Name?.Trim (); + if (String.IsNullOrEmpty (testClassName)) + continue; + + if (String.Compare (testClassName, filter.SelectorValue, StringComparison.OrdinalIgnoreCase) == 0) + return ReportFilteredTest (filter); continue; } + if (filter.FilterType == XUnitFilterType.Single) { + if (String.Compare (testCase.DisplayName, filter.SelectorValue, StringComparison.OrdinalIgnoreCase) == 0) + return ReportFilteredTest (filter); + continue; + } + + if (filter.FilterType == XUnitFilterType.Namespace) { + string testClassName = testCase.TestMethod?.TestClass?.Class?.Name?.Trim (); + if (String.IsNullOrEmpty (testClassName)) + continue; + + int dot = testClassName.LastIndexOf ('.'); + if (dot <= 0) + continue; + + string testClassNamespace = testClassName.Substring (0, dot); + if (String.Compare (testClassNamespace, filter.SelectorValue, StringComparison.OrdinalIgnoreCase) == 0) + return ReportFilteredTest (filter); + continue; + } + + if (filter.FilterType == XUnitFilterType.Assembly) { + continue; // Ignored: handled elsewhere + } + throw new InvalidOperationException ($"Unsupported filter type {filter.FilterType}"); } - return true; + return RunAllTestsByDefault; + + bool ReportFilteredTest (XUnitFilter filter) + { + if (LogExcludedTests) { + const string included = "Included"; + const string excluded = "Excluded"; + + string selector; + if (filter.FilterType == XUnitFilterType.Trait) + selector = $"'{filter.SelectorName}':'{filter.SelectorValue}'"; + else + selector = $"'{filter.SelectorValue}'"; + + do_log ($"[FILTER] {(filter.Exclude ? excluded : included)} test (filtered by {filter.FilterType}; {selector}): {testCase.DisplayName}"); + } + return !filter.Exclude; + } } } }