From c9999cd9df41cf3b8549dc8d30979f6318073553 Mon Sep 17 00:00:00 2001 From: "N. Taylor Mullen" Date: Tue, 8 Sep 2015 16:32:18 -0700 Subject: [PATCH] Add stress reporting infrastructure to conditionally output stress results. - The default reporting mechanisms do not output stress data. - Can be overridden by configuration file `stressConfig.json` and running the tests with a `-stressconfig` option. - Updated runner to pass through metric information. - Updated `.gitignore` with `TestOutput` --- .gitignore | 1 + .../Collectors/Metric.cs | 1 + .../Collectors/MetricsRecordedMessage.cs | 7 +- .../DefaultStressTestMessageVisitor.cs | 38 +++++++++++ .../DefaultStressTestRunnerReporter.cs | 17 +++++ .../StressTestCase.cs | 2 +- .../StressTestIterationContext.cs | 4 -- .../StressTestRunner.cs | 1 + .../StressTestRunnerCSVReporter.cs | 66 ++++++++++++++++++ .../StressTestRunnerConfigReporter.cs | 67 +++++++++++++++++++ .../project.json | 11 ++- .../project.json | 2 +- .../project.json | 2 +- 13 files changed, 207 insertions(+), 12 deletions(-) create mode 100644 src/Microsoft.AspNet.StressFramework/DefaultStressTestMessageVisitor.cs create mode 100644 src/Microsoft.AspNet.StressFramework/DefaultStressTestRunnerReporter.cs create mode 100644 src/Microsoft.AspNet.StressFramework/StressTestRunnerCSVReporter.cs create mode 100644 src/Microsoft.AspNet.StressFramework/StressTestRunnerConfigReporter.cs diff --git a/.gitignore b/.gitignore index 986b55c..86d5e62 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ _ReSharper.*/ packages/ artifacts/ PublishProfiles/ +TestOutput/ *.user *.suo *.cache diff --git a/src/Microsoft.AspNet.StressFramework/Collectors/Metric.cs b/src/Microsoft.AspNet.StressFramework/Collectors/Metric.cs index 47b6a2e..9349893 100644 --- a/src/Microsoft.AspNet.StressFramework/Collectors/Metric.cs +++ b/src/Microsoft.AspNet.StressFramework/Collectors/Metric.cs @@ -1,4 +1,5 @@ using System; +using Xunit.Abstractions; namespace Microsoft.AspNet.StressFramework.Collectors { diff --git a/src/Microsoft.AspNet.StressFramework/Collectors/MetricsRecordedMessage.cs b/src/Microsoft.AspNet.StressFramework/Collectors/MetricsRecordedMessage.cs index 8046009..479cfd7 100644 --- a/src/Microsoft.AspNet.StressFramework/Collectors/MetricsRecordedMessage.cs +++ b/src/Microsoft.AspNet.StressFramework/Collectors/MetricsRecordedMessage.cs @@ -6,13 +6,14 @@ namespace Microsoft.AspNet.StressFramework.Collectors { public class MetricsRecordedMessage : IMessageSinkMessage { - private ITest Test { get; } - private IReadOnlyList Metrics { get; } - public MetricsRecordedMessage(ITest test, IEnumerable metrics) { Test = test; Metrics = metrics.ToList().AsReadOnly(); } + + public ITest Test { get; } + + public IReadOnlyList Metrics { get; } } } \ No newline at end of file diff --git a/src/Microsoft.AspNet.StressFramework/DefaultStressTestMessageVisitor.cs b/src/Microsoft.AspNet.StressFramework/DefaultStressTestMessageVisitor.cs new file mode 100644 index 0000000..b87f69d --- /dev/null +++ b/src/Microsoft.AspNet.StressFramework/DefaultStressTestMessageVisitor.cs @@ -0,0 +1,38 @@ +using System; +using Microsoft.AspNet.StressFramework.Collectors; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.AspNet.StressFramework +{ + public abstract class DefaultStressTestMessageVisitor : DefaultRunnerReporterMessageHandler + { + public DefaultStressTestMessageVisitor(IRunnerLogger logger) + : base(logger) + { + } + + public override bool OnMessage(IMessageSinkMessage message) + { + return DoVisit(message, (t, m) => t.Visit(m)) && + base.OnMessage(message); + } + + protected virtual bool Visit(MetricsRecordedMessage metric) + { + return true; + } + + bool DoVisit(IMessageSinkMessage message, Func callback) + where TMessage : class, IMessageSinkMessage + { + var castMessage = message as TMessage; + if (castMessage != null) + { + return callback(this, castMessage); + } + + return true; + } + } +} diff --git a/src/Microsoft.AspNet.StressFramework/DefaultStressTestRunnerReporter.cs b/src/Microsoft.AspNet.StressFramework/DefaultStressTestRunnerReporter.cs new file mode 100644 index 0000000..5d2fbcc --- /dev/null +++ b/src/Microsoft.AspNet.StressFramework/DefaultStressTestRunnerReporter.cs @@ -0,0 +1,17 @@ +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.AspNet.StressFramework +{ + public class DefaultStressTestRunnerReporter : DefaultRunnerReporter + { + public override string RunnerSwitch => "stress"; + + public override string Description => "doesn't output stress test results."; + + public override IMessageSink CreateMessageHandler(IRunnerLogger logger) + { + return new DefaultRunnerReporterMessageHandler(new ConsoleRunnerLogger(useColors: true)); + } + } +} diff --git a/src/Microsoft.AspNet.StressFramework/StressTestCase.cs b/src/Microsoft.AspNet.StressFramework/StressTestCase.cs index d405134..c2afb60 100644 --- a/src/Microsoft.AspNet.StressFramework/StressTestCase.cs +++ b/src/Microsoft.AspNet.StressFramework/StressTestCase.cs @@ -49,7 +49,7 @@ namespace Microsoft.AspNet.StressFramework DisplayName = suppliedDisplayName ?? BaseDisplayName; Iterations = iterations; WarmupIterations = warmupIterations; - + TestMethodArguments = testMethodArguments?.ToArray(); } diff --git a/src/Microsoft.AspNet.StressFramework/StressTestIterationContext.cs b/src/Microsoft.AspNet.StressFramework/StressTestIterationContext.cs index 442cb22..b47c79a 100644 --- a/src/Microsoft.AspNet.StressFramework/StressTestIterationContext.cs +++ b/src/Microsoft.AspNet.StressFramework/StressTestIterationContext.cs @@ -13,10 +13,6 @@ namespace Microsoft.AspNet.StressFramework private readonly IMessageBus _bus; private readonly Process _me; - private MemoryUsage _startMemory; - private MemoryUsage _endMemory; - private CpuTime _startCpu; - public IReadOnlyList Recordings { get; } public int Iteration { get; } diff --git a/src/Microsoft.AspNet.StressFramework/StressTestRunner.cs b/src/Microsoft.AspNet.StressFramework/StressTestRunner.cs index 55eb605..028082f 100644 --- a/src/Microsoft.AspNet.StressFramework/StressTestRunner.cs +++ b/src/Microsoft.AspNet.StressFramework/StressTestRunner.cs @@ -9,6 +9,7 @@ using Xunit.Abstractions; using Xunit.Sdk; using System.Collections.Generic; using Microsoft.AspNet.StressFramework.Collectors; +using System.Linq; using System.Reflection; #if DNXCORE50 || DNX451 diff --git a/src/Microsoft.AspNet.StressFramework/StressTestRunnerCSVReporter.cs b/src/Microsoft.AspNet.StressFramework/StressTestRunnerCSVReporter.cs new file mode 100644 index 0000000..59d24be --- /dev/null +++ b/src/Microsoft.AspNet.StressFramework/StressTestRunnerCSVReporter.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using Microsoft.AspNet.StressFramework.Collectors; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.AspNet.StressFramework +{ + public class StressTestRunnerCSVReporter : DefaultRunnerReporter + { + public override string RunnerSwitch => "stresscsv"; + + public override string Description => "output stress test results to CSV files. 1 CSV file per test."; + + public override IMessageSink CreateMessageHandler(IRunnerLogger logger) + { + return new CSVRunnerReporterMessageHandler(new ConsoleRunnerLogger(useColors: true)); + } + + private class CSVRunnerReporterMessageHandler : DefaultStressTestMessageVisitor + { + public CSVRunnerReporterMessageHandler(IRunnerLogger logger) + : base(logger) + { + } + + protected override bool Visit(MetricsRecordedMessage metricsRecordedMessage) + { + Debug.Assert(metricsRecordedMessage.Test.TestCase is StressTestCase); + + WriteCSV(metricsRecordedMessage); + + return true; + } + + private static void WriteCSV(MetricsRecordedMessage metricsRecordedMessage) + { + var test = metricsRecordedMessage.Test; + var metrics = metricsRecordedMessage.Metrics; + var testOutputDir = Path.Combine(Directory.GetCurrentDirectory(), "TestOutput"); + if (!Directory.Exists(testOutputDir)) + { + Directory.CreateDirectory(testOutputDir); + } + + var outputFile = Path.Combine(testOutputDir, $"{test.TestCase.DisplayName}.{test.TestCase.UniqueID}.metrics.csv"); + + using (var writer = new StreamWriter(new FileStream(outputFile, FileMode.Create))) + { + writer.WriteLine("Iteration,Heap,WorkingSet,PrivateBytes,ElapsedTicks"); + + foreach (var iteration in metrics.GroupBy(g => g.Iteration).OrderBy(g => g.Key)) + { + var elapsed = (ElapsedTime)iteration.Single(m => m.Value is ElapsedTime).Value; + var mem = (MemoryUsage)iteration.Single(m => m.Value is MemoryUsage).Value; + writer.WriteLine($"{iteration.Key},{mem.HeapMemoryBytes},{mem.WorkingSet},{mem.PrivateBytes},{elapsed.Elapsed.Ticks}"); + } + } + } + } + } +} diff --git a/src/Microsoft.AspNet.StressFramework/StressTestRunnerConfigReporter.cs b/src/Microsoft.AspNet.StressFramework/StressTestRunnerConfigReporter.cs new file mode 100644 index 0000000..edf113c --- /dev/null +++ b/src/Microsoft.AspNet.StressFramework/StressTestRunnerConfigReporter.cs @@ -0,0 +1,67 @@ +using System; +using System.IO; +using Microsoft.Framework.Configuration; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.AspNet.StressFramework +{ + public class StressTestRunnerConfigReporter : DefaultRunnerReporter + { + private const string StressConfigFileName = "stressConfig.json"; + private const string StressOutputReporterConfigName = "StressOutputReporter"; + private static readonly Lazy _defaultReporter = new Lazy( + () => new DefaultStressTestRunnerReporter()); + private static readonly Lazy _csvReporter = new Lazy( + () => new StressTestRunnerCSVReporter()); + private IRunnerReporter _activeReporter; + + private IRunnerReporter DefaultReporter => _defaultReporter.Value; + private IRunnerReporter CSVReporter => _csvReporter.Value; + + public override string RunnerSwitch => "stressconfig"; + + public override string Description => "determine stress output mechanism based on "; + + public override IMessageSink CreateMessageHandler(IRunnerLogger logger) + { + if (_activeReporter == null) + { + var stressConfigFile = new FileInfo(StressConfigFileName); + if (!stressConfigFile.Exists) + { + throw new InvalidOperationException( + $"Cannot determine stress configuration. {StressConfigFileName} does not exist."); + } + + var config = new ConfigurationBuilder(".") + .AddJsonFile(StressConfigFileName) + .Build(); + var stressTestOutputReporterName = config[StressOutputReporterConfigName]; + + if (!string.IsNullOrEmpty(stressTestOutputReporterName) || + string.Equals( + stressTestOutputReporterName, + DefaultReporter.RunnerSwitch, + StringComparison.OrdinalIgnoreCase)) + { + _activeReporter = DefaultReporter; + } + else if (string.Equals( + stressTestOutputReporterName, + CSVReporter.RunnerSwitch, + StringComparison.OrdinalIgnoreCase)) + { + _activeReporter = CSVReporter; + } + else + { + throw new InvalidOperationException( + $"Unknown {StressOutputReporterConfigName} value in {StressConfigFileName}."); + } + } + + return _activeReporter.CreateMessageHandler(logger); + } + } +} diff --git a/src/Microsoft.AspNet.StressFramework/project.json b/src/Microsoft.AspNet.StressFramework/project.json index c94b9c6..13fa12c 100644 --- a/src/Microsoft.AspNet.StressFramework/project.json +++ b/src/Microsoft.AspNet.StressFramework/project.json @@ -1,10 +1,17 @@ { "version": "1.0.0-*", "dependencies": { - "xunit.runner.aspnet": "2.0.0-aspnet-*" + "Microsoft.Framework.Configuration.Json": "1.0.0-*", + "xunit.assert": "2.1.0-*", + "xunit.core": "2.1.0-*", + "xunit.runner.dnx": "2.1.0-*" }, "frameworks": { - "dnx451": { }, + "dnx451": { + "frameworkAssemblies": { + "System.Reflection": "4.0.0.0" + } + }, "dnxcore50": { "dependencies": { "System.Runtime": "4.0.21-beta-*" diff --git a/test/Microsoft.AspNet.Stress.Mvc.Tests/project.json b/test/Microsoft.AspNet.Stress.Mvc.Tests/project.json index d5ac688..1643b3a 100644 --- a/test/Microsoft.AspNet.Stress.Mvc.Tests/project.json +++ b/test/Microsoft.AspNet.Stress.Mvc.Tests/project.json @@ -1,7 +1,7 @@ { "version": "1.0.0-*", "commands": { - "test": "xunit.runner.aspnet" + "test": "xunit.runner.dnx" }, "dependencies": { "Microsoft.AspNet.StressFramework": "", diff --git a/test/Microsoft.AspNet.Stress.Tests/project.json b/test/Microsoft.AspNet.Stress.Tests/project.json index 3eb527a..53fe6ef 100644 --- a/test/Microsoft.AspNet.Stress.Tests/project.json +++ b/test/Microsoft.AspNet.Stress.Tests/project.json @@ -1,7 +1,7 @@ { "version": "1.0.0-*", "commands": { - "test": "xunit.runner.aspnet" + "test": "xunit.runner.dnx" }, "dependencies": { "Microsoft.AspNet.StressFramework": "",