Add mini test framework and tests of result file contents

build.cake

@@ -9,7 +9,7 @@
 var target = Argument("target", "Default");
 var configuration = Argument("configuration", "Debug");
 
-#load scripts/parameters.cake
+#load cake/parameters.cake
 
 //////////////////////////////////////////////////////////////////////
 // SETUP AND TEARDOWN

cake/assertions.cake

@@ -0,0 +1,42 @@
+public static class Assert
+{
+    internal static List<string> FailureMessages = new List<string>();
+    internal static int AssertCount { get; set; }
+    internal static int FailureCount { get; set; }
+    internal static int SuccessCount => AssertCount - FailureCount;
+
+    public static void That<T>(T actual, Constraint<T> constraint, string message = null)
+    {
+        Assert.AssertCount++;
+        
+        if (!constraint.Matches(actual))
+        {
+            if (message != null)
+                FailureMessages.Add(message);
+
+            ReportFailure(constraint.Message);
+        }
+    }
+
+    public static void That(bool condition, string message)
+    {
+        Assert.AssertCount++;
+        if (!condition)
+            ReportFailure(message);
+    }
+
+    public static void Fail(string message = null)
+    {
+        throw new System.Exception(message);
+    }
+
+    internal static void ReportFailure(string message)
+    {
+        // Note that this version does not automatically
+        // terminate the test upon failure of an Assert.
+        // Use Assert.Fail if you want the test to end.
+        if (!string.IsNullOrEmpty(message))
+            FailureMessages.Add(message);
+        FailureCount++;
+    }
+}

cake/constraints.cake

@@ -0,0 +1,122 @@
+public abstract class Constraint<TActual>
+{
+    public abstract bool Matches(TActual actual);
+    public string Message { get; protected set; }
+
+    internal static string VAL<T>(T arg) => arg is string ? $"\"{arg}\"" : arg.ToString();
+}
+
+public class EqualConstraint<TActual> : Constraint<TActual>
+{
+    // Expected and actual types must match, possibly through conversion
+    private TActual _expected;
+
+    public EqualConstraint(TActual expected)
+    {
+        _expected = expected;
+    }
+
+    public override bool Matches(TActual actual)
+    {
+        if (_expected.Equals(actual))
+            return true;
+
+        Message = $"Expected: {VAL(_expected)} But was: {VAL(actual)}";
+
+        return false;
+    }
+}
+
+public class XmlElementConstraint : Constraint<XmlNode>
+{
+    private string _name;
+    private int _expectedCount;
+
+    public XmlElementConstraint(string name, int expectedCount = -1)
+    {
+        _name = name;
+        _expectedCount = expectedCount;
+    }
+
+    public override bool Matches(XmlNode actual)
+    {
+        var elements = actual.SelectNodes(_name);
+
+        if (_expectedCount < 0) // No count specified
+        {
+            if (elements.Count == 0)
+            {
+                Message = $"Expected element <{_name}> was not found.";
+                return false;
+            }
+        }
+        else // Count was specified
+        {
+            if (elements.Count != _expectedCount)
+            {
+                Message = $"Expected {_expectedCount} <{_name}> elements but found {elements.Count}.";
+                return false;
+            }
+        }
+
+        return true;
+    }
+}
+
+public class XmlAttributeConstraint : Constraint<XmlNode>
+{
+    private string _name;
+    private string _value;
+
+    public XmlAttributeConstraint(string name)
+    {
+        _name = name;
+    }
+
+    public override bool Matches(XmlNode actual)
+    {
+        var attr = actual.Attributes[_name];
+
+        if (attr == null)
+        {
+            var xml = actual.OuterXml;
+            int end = xml.IndexOf('>');
+            if (end > 0) xml = xml.Substring(0, end + 1);
+            Message = $"Expected: XmlNode with attribute {VAL(_name)}\r\nBut was: {xml}";
+            return false;
+        }
+
+        if (_value != null && attr.Value != _value)
+        {
+            Message = $"Expected: {_name}={VAL(_value)} But was: {VAL(attr.Value)}";
+            return false;
+        }
+
+        return true;
+    }
+
+    public XmlAttributeConstraint EqualTo(string value)
+    {
+        _value = value;
+        return this;
+    }
+}
+
+public static class Is
+{
+    public static EqualConstraint<T> EqualTo<T>(T expected) => new EqualConstraint<T>(expected);
+}
+
+public static class Has
+{
+    public static XmlElementConstraint Element(string name, int expectedCount = -1) => new XmlElementConstraint(name, expectedCount);
+    public static XmlAttributeConstraint Attribute(string name) => new XmlAttributeConstraint(name);
+    public static HasExactly One => new HasExactly(1);
+
+    public class HasExactly
+    {
+        private int _count;
+        public HasExactly(int count) { _count = count; }
+        public XmlElementConstraint Element(string name) => new XmlElementConstraint(name, _count);
+    }
+}

cake/packaging.cake

@@ -62,9 +62,11 @@ public abstract class PackageTester
     {
 		foreach (string consoleVersion in _parameters.SupportedConsoleVersions)
 		{
+			Banner($"Testing {MOCK_ASSEMBLY} under NUnit3-Console {consoleVersion}");
 			RunMockAssemblyTests(consoleVersion);
 
-			VerifyResultFile(NUNIT2_RESULT_FILE);
+			Banner($"Verifying {NUNIT2_RESULT_FILE}");
+			TestRunner.Run(typeof(ResultWriterTests));
 		}
 	}
 
@@ -75,10 +77,6 @@ public abstract class PackageTester
 
 	private void RunMockAssemblyTests(string consoleVersion)
     {
-		_context.Information("=======================================================");
-		_context.Information($"Testing {MOCK_ASSEMBLY} under NUnit3-Console {consoleVersion}");
-		_context.Information("=======================================================");
-
 		string runner = _parameters.GetPathToConsoleRunner(consoleVersion);
 
 		if (InstallDirectory.EndsWith(CHOCO_ID + "/"))
@@ -116,28 +114,15 @@ public abstract class PackageTester
 		}
 	}
 
-	private void VerifyResultFile(string resultFile)
+	private void Banner(string message)
-    {
+	{
+		_context.Information("\n=======================================================");
+		_context.Information(message);
 		_context.Information("=======================================================");
-		_context.Information($"Verifying {resultFile}");
+	}
-		_context.Information("=======================================================");
-
-		var doc = new XmlDocument();
-		doc.Load(resultFile);
-
-		_context.Information("Verifying <test-results> node...");
-		XmlNode testResult = doc.DocumentElement;
-		if (testResult.Name != "test-results")
-			throw new System.Exception($"Expected <test-results> but was <{testResult.Name}>");
-
-		_context.Information("Verifying top-level <test-suite>...");
-		if (testResult.FirstChild.Name != "test-suite")
-			throw new System.Exception("Top level test suite not found");
-
-		_context.Information("Verification was successful!");
-    }
 }
 
+
 public class NuGetPackageTester : PackageTester
 {
     public NuGetPackageTester(BuildParameters parameters) : base(parameters) { }

cake/parameters.cake

@@ -1,5 +1,7 @@
 #load "./constants.cake"
 #load "./packaging.cake"
+#load "./test-runner.cake"
+#load "./tests.cake"
 
 using System;
 

cake/test-runner.cake

@@ -0,0 +1,71 @@
+#load ./assertions.cake
+#load ./constraints.cake
+
+using System.Reflection;
+
+//////////////////////////////////////////////////////////////////////
+// A tiny test framework for use in cake scripts.
+//////////////////////////////////////////////////////////////////////
+
+public static class TestRunner
+{
+    public static void Run(params Type[] types)
+    {
+        Assert.FailureCount = 0;
+
+        foreach (Type type in types)
+        {
+            Console.WriteLine($"\n=> {type.Name}");
+
+            int testCount = 0;
+            int failCount = 0;
+
+            foreach (var method in type.GetMethods())
+            {
+                if (!method.IsDefined(typeof(TestAttribute)))
+                    continue;
+
+                Assert.FailureMessages.Clear();
+                testCount++;
+
+                try
+                {
+                    var obj = !type.IsStatic() ? System.Activator.CreateInstance(type) : null;
+                    method.Invoke(obj, new object[0]);
+                }
+                catch (Exception ex)
+                {
+                    if (ex is TargetInvocationException)
+                        ex = ex.InnerException;
+                    Assert.ReportFailure(ex.Message);
+                }
+
+                if (Assert.FailureMessages.Count == 0)
+                {
+                    Console.WriteLine($"  => {method.Name}");
+                }
+                else
+                {
+                    failCount++;
+                    Console.WriteLine($"  => {method.Name} FAILED!");
+                    foreach (string message in Assert.FailureMessages)
+                        Console.WriteLine($"     {message}");
+                    Assert.FailureMessages.Clear();
+                }
+            }
+
+            bool failed = Assert.FailureCount > 0;
+            string runResult = failed ? "FAILED" : "PASSED";
+
+            Console.WriteLine($"\nTest Run Summary - {runResult}");
+            Console.WriteLine($"  Tests: {testCount}, Passed: {testCount - failCount}, Failed: {failCount}");
+            Console.WriteLine($"  Asserts: {Assert.AssertCount}, Passed: {Assert.SuccessCount}, Failed: {Assert.FailureCount}\n");
+
+            if (failed)
+                throw new System.Exception();
+        }
+    }
+}
+
+[AttributeUsage(AttributeTargets.Method, Inherited = false)]
+public class TestAttribute : Attribute { }

cake/tests.cake

@@ -0,0 +1,93 @@
+public static class ResultWriterTests
+{
+    static XmlNode Fixture;
+
+    static ResultWriterTests()
+    {
+        var doc = new XmlDocument();
+        doc.Load("NUnit2TestResult.xml");
+        Fixture = doc.DocumentElement;
+    }
+
+    [Test]
+    public static void TopLevelHierarchy()
+    {
+        Assert.That(Fixture.Name, Is.EqualTo("test-results"));
+        Assert.That(Fixture, Has.One.Element("environment"));
+        Assert.That(Fixture, Has.One.Element("culture-info"));
+        Assert.That(Fixture, Has.One.Element("test-suite"));
+    }
+
+    [Test]
+    public static void TestSuitesHaveOneResultsElement()
+    {
+        var suites = Fixture.SelectNodes("//test-suite");
+        Assert.That(suites.Count > 0, "No <test-suite> elements found in file");
+
+        var resultElements = Fixture.SelectNodes("//results");
+        Assert.That(resultElements.Count, Is.EqualTo(suites.Count),
+            "Number of <results> elements should equal number of <test-suite> elements");
+
+        foreach (XmlNode suite in suites)
+            Assert.That(suite, Has.One.Element("results"));
+    }
+
+    [Test]
+    public static void TestCasesHaveResultsElementAsParent()
+    {
+        var testCases = Fixture.SelectNodes("//test-case");
+        foreach (XmlNode testCase in testCases)
+            Assert.That(testCase.ParentNode.Name, Is.EqualTo("results"));
+    }
+
+    [Test]
+    public static void TestResultsElement()
+    {
+        Assert.That(Fixture, Has.Attribute("name").EqualTo("mock-assembly.dll"));
+        Assert.That(Fixture, Has.Attribute("total").EqualTo("31"));
+        Assert.That(Fixture, Has.Attribute("errors").EqualTo("1"));
+        Assert.That(Fixture, Has.Attribute("failures").EqualTo("1"));
+        Assert.That(Fixture, Has.Attribute("not-run").EqualTo("10"));
+        Assert.That(Fixture, Has.Attribute("inconclusive").EqualTo("1"));
+        Assert.That(Fixture, Has.Attribute("ignored").EqualTo("4"));
+        Assert.That(Fixture, Has.Attribute("skipped").EqualTo("3"));
+        Assert.That(Fixture, Has.Attribute("invalid").EqualTo("3"));
+        Assert.That(Fixture, Has.Attribute("date"));
+        Assert.That(Fixture, Has.Attribute("time"));
+    }
+
+    [Test]
+    public static void TopLevelTestSuite()
+    {
+        XmlNode suite = Fixture.SelectSingleNode("test-suite");
+        Assert.That(suite, Has.Attribute("type").EqualTo("Assembly"));
+        Assert.That(suite, Has.Attribute("name").EqualTo("mock-assembly.dll"));
+        Assert.That(suite, Has.Attribute("executed").EqualTo("True"));
+        Assert.That(suite, Has.Attribute("result").EqualTo("Failure"));
+        Assert.That(suite, Has.Attribute("success").EqualTo("False"));
+        Assert.That(suite, Has.Attribute("time"));
+        Assert.That(suite, Has.Attribute("asserts").EqualTo("2"));
+    }
+
+    [Test]
+    public static void EnvironmentElement()
+    {
+        XmlNode environment = Fixture.SelectSingleNode("environment");
+        Assert.That(environment, Has.Attribute("nunit-version").EqualTo("3.11.0.0"));
+        Assert.That(environment, Has.Attribute("clr-version"));
+        Assert.That(environment, Has.Attribute("os-version"));
+        Assert.That(environment, Has.Attribute("platform"));
+        Assert.That(environment, Has.Attribute("cwd"));
+        Assert.That(environment, Has.Attribute("machine-name"));
+        Assert.That(environment, Has.Attribute("user"));
+        Assert.That(environment, Has.Attribute("user-domain"));
+    }
+
+    [Test]
+    public static void CultureInfoElement()
+    {
+        XmlNode cultureInfo = Fixture.SelectSingleNode("culture-info");
+        Assert.That(cultureInfo, Has.Attribute("current-culture"));
+        Assert.That(cultureInfo, Has.Attribute("current-uiculture"));
+    }
+}

nunit-v2-result-writer.sln

@@ -22,11 +22,15 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "mock-assembly", "src\mock-assembly\mock-assembly.csproj", "{6D160D73-AB70-47F3-9837-BB3A06125FFB}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{CE645163-A565-4C45-A9C5-4F66D67C13CC}"
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cake", "cake", "{C60D66FA-F4D4-46BE-9B1C-EFBCB344CA21}"
 	ProjectSection(SolutionItems) = preProject
-		scripts\constants.cake = scripts\constants.cake
+		cake\assert.cake = cake\assert.cake
-		scripts\packaging.cake = scripts\packaging.cake
+		cake\constants.cake = cake\constants.cake
-		scripts\parameters.cake = scripts\parameters.cake
+		cake\constraints.cake = cake\constraints.cake
+		cake\packaging.cake = cake\packaging.cake
+		cake\parameters.cake = cake\parameters.cake
+		cake\test-runner.cake = cake\test-runner.cake
+		cake\tests.cake = cake\tests.cake
 	EndProjectSection
 EndProject
 Global
@@ -52,7 +56,7 @@ Global
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
-		{CE645163-A565-4C45-A9C5-4F66D67C13CC} = {5F2D8428-EF74-449E-ADD6-505D20751D12}
+		{C60D66FA-F4D4-46BE-9B1C-EFBCB344CA21} = {5F2D8428-EF74-449E-ADD6-505D20751D12}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F9258C05-1EEA-485D-BC8E-69A1D6EFE5DC}

src/extension/NUnit2XmlResultWriter.cs

@@ -104,8 +104,8 @@ namespace NUnit.Engine.Addins
             DateTime start = result.GetAttribute("start-time", DateTime.UtcNow);
             _xmlWriter.WriteAttributeString("date", start.ToString("yyyy-MM-dd"));
             _xmlWriter.WriteAttributeString("time", start.ToString("HH:mm:ss"));
-            //WriteEnvironment(topLevelAssembly.SelectSingleNode("environment").GetAttribute("framework-version"));
+            WriteEnvironment(topLevelAssembly.SelectSingleNode("environment").GetAttribute("framework-version"));
-            //WriteCultureInfo();
+            WriteCultureInfo();
         }
 
         private void WriteCultureInfo()