dotnet
/
BenchmarkDotNet
зеркало из https://github.com/dotnet/BenchmarkDotNet.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Improved "Scaled" column

This commit is contained in:
Andrey Akinshin 2016-07-07 15:15:39 +03:00
Родитель c6405a91be
Коммит f9f7481deb
10 изменённых файлов: 298 добавлений и 345 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

232
BenchmarkDotNet.IntegrationTests/BaselineDiffColumnTest.cs
Просмотреть файл

@ -1,232 +0,0 @@
using BenchmarkDotNet.Jobs;
using System;
using System.Linq;
using System.Threading;
using Xunit;
using BenchmarkDotNet.Reports;
using System.Collections.Generic;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Helpers;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Running;
namespace BenchmarkDotNet.IntegrationTests
{
public class BaselineDiffColumnTest
{
[Params(1, 2)]
public int ParamProperty { get; set; }
[Fact]
public void Test()
{
// This is the common way to run benchmarks, it should wire up the BenchmarkBaselineDeltaResultExtender for us
// BenchmarkTestExecutor.CanExecute(..) calls BenchmarkRunner.Run(..) under the hood
var summary = BenchmarkTestExecutor.CanExecute<BaselineDiffColumnTest>();
var table = summary.Table;
var headerRow = table.FullHeader;
var column = summary.Config.GetColumns().OfType<BaselineDiffColumn>().FirstOrDefault();
Assert.NotNull(column);
Assert.Equal(column.ColumnName, headerRow.Last());
var testNameColumn = Array.FindIndex(headerRow, c => c == "Method");
var extraColumn = Array.FindIndex(headerRow, c => c == column.ColumnName);
foreach (var row in table.FullContent)
{
Assert.Equal(row.Length, extraColumn + 1);
if (row[testNameColumn] == "BenchmarkSlow") // This is our baseline
Assert.Equal("1.00", row[extraColumn]);
else if (row[testNameColumn] == "BenchmarkFast") // This should have been compared to the baseline
Assert.Contains(".", row[extraColumn]);
}
}
[Benchmark(Baseline = true)]
public void BenchmarkSlow() => Thread.Sleep(20);
[Benchmark]
public void BenchmarkFast() => Thread.Sleep(5);
}
// TODO: repair
[Config(typeof(SingleRunFastConfig))]
public class BaselineDeltaResultExtenderNoBaselineTest
{
[Fact]
public void Test()
{
var testExporter = new TestExporter();
var config = DefaultConfig.Instance.With(testExporter);
BenchmarkTestExecutor.CanExecute<BaselineDeltaResultExtenderNoBaselineTest>(config);
// Ensure that when the TestBenchmarkExporter() was run, it wasn't passed an instance of "BenchmarkBaselineDeltaResultExtender"
Assert.False(testExporter.ExportCalled);
Assert.True(testExporter.ExportToFileCalled);
}
[Benchmark]
public void BenchmarkSlow()
{
Thread.Sleep(50);
}
[Benchmark]
public void BenchmarkFast()
{
Thread.Sleep(10);
}
public class TestExporter : IExporter
{
public bool ExportCalled { get; private set; }
public bool ExportToFileCalled { get; private set; }
public string Description => "For Testing Only!";
public string Name => "TestBenchmarkExporter";
public void ExportToLog(Summary summary, ILogger logger) => ExportCalled = true;
public IEnumerable<string> ExportToFiles(Summary summary)
{
ExportToFileCalled = true;
return Enumerable.Empty<string>();
}
}
}
// Todo: use
public class BaselineDeltaResultExtenderErrorTest
{
[Fact]
public void Test()
{
var summary = BenchmarkTestExecutor.CanExecute<BaselineDeltaResultExtenderErrorTest>(fullValidation: false);
// You can't have more than 1 method in a class with [Benchmark(Baseline = true)]
Assert.True(summary.HasCriticalValidationErrors);
}
[Benchmark(Baseline = true)]
public void BenchmarkSlow()
{
Thread.Sleep(50);
}
[Benchmark(Baseline = true)]
public void BenchmarkFast()
{
Thread.Sleep(5);
}
}
public class BaselineDeltaResultExtenderHandlesBenchmarkErrorTest
{
[Fact]
public void Test()
{
var summary = BenchmarkTestExecutor.CanExecute<BaselineDeltaResultExtenderHandlesBenchmarkErrorTest>(fullValidation: false);
var table = summary.Table;
var headerRow = table.FullHeader;
var column = summary.Config.GetColumns().OfType<BaselineDiffColumn>().FirstOrDefault();
Assert.NotNull(column);
Assert.Equal(column.ColumnName, headerRow.Last());
var testNameColumn = Array.FindIndex(headerRow, c => c == "Method");
var extraColumn = Array.FindIndex(headerRow, c => c == column.ColumnName);
foreach (var row in table.FullContent)
{
Assert.Equal(row.Length, extraColumn + 1);
if (row[testNameColumn] == "BenchmarkSlow") // This is our baseline
Assert.Equal("1.00", row[extraColumn]);
else if (row[testNameColumn] == "BenchmarkThrows") // This should have "?" as it threw an error
Assert.Contains("?", row[extraColumn]);
}
}
[Benchmark(Baseline = true)]
public void BenchmarkSlow()
{
Thread.Sleep(50);
}
[Benchmark]
public void BenchmarkThrows()
{
// Check that BaselineDiffColumn can handle Benchmarks that throw
// See https://github.com/PerfDotNet/BenchmarkDotNet/issues/151
// and https://github.com/PerfDotNet/BenchmarkDotNet/issues/158
throw new InvalidOperationException("Part of a Unit test - This is expected");
}
}
[Config(typeof(SingleRunFastConfig))]
public class BaselineScaledColumnsTest
{
[Params(1, 2)]
public int ParamProperty { get; set; }
[Fact]
public void Test()
{
// This is the common way to run benchmarks, it should wire up the BenchmarkBaselineDeltaResultExtender for us
var config = DefaultConfig.Instance
.With(Job.Dry.WithTargetCount(5))
.With(BaselineDiffColumn.Scaled50)
.With(BaselineDiffColumn.Scaled85)
.With(BaselineDiffColumn.Scaled95);
var summary = BenchmarkTestExecutor.CanExecute<BaselineScaledColumnsTest>(config);
var table = summary.Table;
var headerRow = table.FullHeader;
var columns = summary.Config.GetColumns().OfType<BaselineDiffColumn>().ToArray();
Assert.Equal(columns.Length, 4);
Assert.Equal(columns[0].ColumnName, headerRow[headerRow.Length - 4]);
Assert.Equal(columns[1].ColumnName, headerRow[headerRow.Length - 3]);
Assert.Equal(columns[2].ColumnName, headerRow[headerRow.Length - 2]);
Assert.Equal(columns[3].ColumnName, headerRow[headerRow.Length - 1]);
var testNameColumn = Array.FindIndex(headerRow, c => c == "Method");
var parseCulture = HostEnvironmentInfo.MainCultureInfo;
foreach (var row in table.FullContent)
{
Assert.Equal(row.Length, headerRow.Length);
if (row[testNameColumn] == "BenchmarkFast") // This is our baseline
{
Assert.Equal("1.00", row[headerRow.Length - 4]); // Scaled
Assert.Equal("1.00", row[headerRow.Length - 3]); // S50
Assert.Equal("1.00", row[headerRow.Length - 2]); // S85
Assert.Equal("1.00", row[headerRow.Length - 1]); // S95
}
else if (row[testNameColumn] == "BenchmarkSlow") // This should have been compared to the baseline
{
// This code fails on appveyor
// See also: https://github.com/PerfDotNet/BenchmarkDotNet/issues/204
// var min = 3.0; // 3.7
// var max = 5.0; // 4.3
// var scaled = double.Parse(row[headerRow.Length - 4], parseCulture);
// Assert.InRange(scaled, min, max);
// var s50 = double.Parse(row[headerRow.Length - 3], parseCulture);
// Assert.InRange(s50, min, max);
// var s85 = double.Parse(row[headerRow.Length - 2], parseCulture);
// Assert.InRange(s85, min, max);
// var s95 = double.Parse(row[headerRow.Length - 1], parseCulture);
// Assert.InRange(s95, min, max);
}
}
}
[Benchmark(Baseline = true)]
public void BenchmarkFast() => Thread.Sleep(5);
[Benchmark]
public void BenchmarkSlow() => Thread.Sleep(20);
}
}

129
BenchmarkDotNet.IntegrationTests/BaselineScaledColumnTest.cs Normal file
Просмотреть файл

@ -0,0 +1,129 @@
using BenchmarkDotNet.Jobs;
using System;
using System.Linq;
using System.Threading;
using Xunit;
using BenchmarkDotNet.Reports;
using System.Collections.Generic;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Helpers;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Running;
namespace BenchmarkDotNet.IntegrationTests
{
public class BaselineScaledColumnTest
{
[Params(1, 2)]
public int ParamProperty { get; set; }
[Fact]
public void Test()
{
// This is the common way to run benchmarks, it should wire up the BenchmarkBaselineDeltaResultExtender for us
// BenchmarkTestExecutor.CanExecute(..) calls BenchmarkRunner.Run(..) under the hood
var summary = BenchmarkTestExecutor.CanExecute<BaselineScaledColumnTest>();
var table = summary.Table;
var headerRow = table.FullHeader;
var column = summary.Config.GetColumns()
.OfType<BaselineScaledColumn>()
.FirstOrDefault(c => c.Kind == BaselineScaledColumn.DiffKind.Mean);
Assert.NotNull(column);
Assert.Equal(column.ColumnName, headerRow.Penult());
var testNameColumn = Array.FindIndex(headerRow, c => c == "Method");
var extraColumn = Array.FindIndex(headerRow, c => c == column.ColumnName);
foreach (var row in table.FullContent)
{
Assert.Equal(row.Length, extraColumn + 2);
if (row[testNameColumn] == "BenchmarkSlow") // This is our baseline
Assert.Equal("1.00", row[extraColumn]);
else if (row[testNameColumn] == "BenchmarkFast") // This should have been compared to the baseline
Assert.Contains(".", row[extraColumn]);
}
}
[Benchmark(Baseline = true)]
public void BenchmarkSlow() => Thread.Sleep(20);
[Benchmark]
public void BenchmarkFast() => Thread.Sleep(5);
}
// TODO: repair
[Config(typeof(SingleRunFastConfig))]
public class BaselineScaledResultExtenderNoBaselineTest
{
[Fact]
public void Test()
{
var testExporter = new TestExporter();
var config = DefaultConfig.Instance.With(testExporter);
BenchmarkTestExecutor.CanExecute<BaselineScaledResultExtenderNoBaselineTest>(config);
// Ensure that when the TestBenchmarkExporter() was run, it wasn't passed an instance of "BenchmarkBaselineDeltaResultExtender"
Assert.False(testExporter.ExportCalled);
Assert.True(testExporter.ExportToFileCalled);
}
[Benchmark]
public void BenchmarkSlow()
{
Thread.Sleep(50);
}
[Benchmark]
public void BenchmarkFast()
{
Thread.Sleep(10);
}
public class TestExporter : IExporter
{
public bool ExportCalled { get; private set; }
public bool ExportToFileCalled { get; private set; }
public string Description => "For Testing Only!";
public string Name => "TestBenchmarkExporter";
public void ExportToLog(Summary summary, ILogger logger) => ExportCalled = true;
public IEnumerable<string> ExportToFiles(Summary summary)
{
ExportToFileCalled = true;
return Enumerable.Empty<string>();
}
}
}
public class BaselineScaledResultExtenderErrorTest
{
[Fact]
public void Test()
{
var summary = BenchmarkTestExecutor.CanExecute<BaselineScaledResultExtenderErrorTest>(fullValidation: false);
// You can't have more than 1 method in a class with [Benchmark(Baseline = true)]
Assert.True(summary.HasCriticalValidationErrors);
}
[Benchmark(Baseline = true)]
public void BenchmarkSlow()
{
Thread.Sleep(50);
}
[Benchmark(Baseline = true)]
public void BenchmarkFast()
{
Thread.Sleep(5);
}
}
}

20
BenchmarkDotNet.Samples/Algorithms/Algo_Md5VsSha256.cs
Просмотреть файл

@ -6,13 +6,13 @@ using BenchmarkDotNet.Jobs;
namespace BenchmarkDotNet.Samples.Algorithms
{
// you can target all runtimes that you support with single config
internal class AllWindowsRuntimesConfig : ManualConfig
{
public AllWindowsRuntimesConfig()
{
Add(Job.Default.With(Runtime.Clr).With(Jit.RyuJit));
Add(Job.Default.With(Runtime.Core).With(Jit.RyuJit));
Add(Job.Default.With(Runtime.Clr));
Add(Job.Default.With(Runtime.Mono));
Add(Job.Default.With(Runtime.Core));
}
}
@ -22,8 +22,8 @@ namespace BenchmarkDotNet.Samples.Algorithms
private const int N = 10000;
private readonly byte[] data;
private readonly SHA256 sha256 = SHA256.Create();
private readonly MD5 md5 = MD5.Create();
private readonly SHA256 sha256 = SHA256.Create();
public Algo_Md5VsSha256()
{
@ -31,16 +31,16 @@ namespace BenchmarkDotNet.Samples.Algorithms
new Random(42).NextBytes(data);
}
[Benchmark(Baseline = true)]
public byte[] Md5()
{
return md5.ComputeHash(data);
}
[Benchmark]
public byte[] Sha256()
{
return sha256.ComputeHash(data);
}
[Benchmark]
public byte[] Md5()
{
return md5.ComputeHash(data);
}
}
}

30
BenchmarkDotNet.Samples/Intro/IntroBaseline.cs
Просмотреть файл

@ -1,20 +1,39 @@
using System.Threading;
using System;
using System.Threading;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Jobs;
namespace BenchmarkDotNet.Samples.Intro
{
[DryConfig]
[Config(typeof(Config))]
public class IntroBaseline
{
private class Config : ManualConfig
{
public Config()
{
Add(Job.Default.WithLaunchCount(0).WithWarmupCount(0).WithTargetCount(5));
}
}
private readonly Random random = new Random(42);
[Params(100, 200)]
public int BaselineTime { get; set; }
[Benchmark(Baseline = true)]
public void BaselineMethod()
public void Baseline()
{
Thread.Sleep(BaselineTime);
}
[Benchmark]
public void Slow()
{
Thread.Sleep(BaselineTime * 2);
}
[Benchmark]
public void Fast()
{
@ -22,9 +41,10 @@ namespace BenchmarkDotNet.Samples.Intro
}
[Benchmark]
public void Slow()
public void Unstable()
{
Thread.Sleep(BaselineTime * 2);
var diff = (int)((random.NextDouble() - 0.5) * 2 * BaselineTime);
Thread.Sleep(BaselineTime + diff);
}
}
}

5
BenchmarkDotNet.Samples/Intro/IntroPercentiles.cs
Просмотреть файл

@ -35,10 +35,7 @@ namespace BenchmarkDotNet.Samples.Intro
StatisticColumn.P85,
StatisticColumn.P90,
StatisticColumn.P95,
StatisticColumn.P100,
BaselineDiffColumn.Scaled50,
BaselineDiffColumn.Scaled85,
BaselineDiffColumn.Scaled95);
StatisticColumn.P100);
}
}

91
BenchmarkDotNet/Columns/BaselineDiffColumn.cs
Просмотреть файл

@ -1,91 +0,0 @@
using System.Linq;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Mathematics;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
namespace BenchmarkDotNet.Columns
{
public class BaselineDiffColumn : IColumn
{
public enum DiffKind
{
Delta,
Scaled
}
public static readonly IColumn Delta = new BaselineDiffColumn(DiffKind.Delta);
public static readonly IColumn Scaled = new BaselineDiffColumn(DiffKind.Scaled);
public static readonly IColumn Scaled50 = new BaselineDiffColumn(DiffKind.Scaled, 50);
public static readonly IColumn Scaled85 = new BaselineDiffColumn(DiffKind.Scaled, 85);
public static readonly IColumn Scaled95 = new BaselineDiffColumn(DiffKind.Scaled, 95);
public DiffKind Kind { get; set; }
public int? Percentile { get; set; }
private BaselineDiffColumn(DiffKind kind, int? percentile = null)
{
Kind = kind;
Percentile = percentile;
}
public string ColumnName => Percentile == null ?
Kind.ToString() :
Kind.ToString() + "P" + Percentile?.ToString();
public string GetValue(Summary summary, Benchmark benchmark)
{
var baselineBenchmark = summary.Benchmarks.
Where(b => b.Job.GetFullInfo() == benchmark.Job.GetFullInfo()).
Where(b => b.Parameters.FullInfo == benchmark.Parameters.FullInfo).
FirstOrDefault(b => b.Target.Baseline);
var invalidResults = baselineBenchmark == null ||
summary[baselineBenchmark] == null ||
summary[baselineBenchmark].ResultStatistics == null ||
summary[benchmark] == null ||
summary[benchmark].ResultStatistics == null;
if (invalidResults)
return "?";
double baselineMetric;
double currentMetric;
var resultStatistics = summary[baselineBenchmark].ResultStatistics;
var statistics = summary[benchmark].ResultStatistics;
if (Percentile == null)
{
baselineMetric = resultStatistics.Median;
currentMetric = statistics.Median;
}
else
{
baselineMetric = resultStatistics.Percentiles.Percentile(Percentile.GetValueOrDefault());
currentMetric = statistics.Percentiles.Percentile(Percentile.GetValueOrDefault());
}
if (baselineMetric == 0)
return "?";
switch (Kind)
{
case DiffKind.Delta:
if (benchmark.Target.Baseline)
return "Baseline";
var diff = (currentMetric - baselineMetric) / baselineMetric * 100.0;
return diff.ToStr("N1") + "%";
case DiffKind.Scaled:
var scale = currentMetric / baselineMetric;
return scale.ToStr("N2");
default:
return "?";
}
}
public bool IsAvailable(Summary summary) => summary.Benchmarks.Any(b => b.Target.Baseline);
public bool AlwaysShow => true;
public ColumnCategory Category => ColumnCategory.Statistics;
public override string ToString() => ColumnName;
}
}

82
BenchmarkDotNet/Columns/BaselineScaledColumn.cs Normal file
Просмотреть файл

@ -0,0 +1,82 @@
using System;
using System.Linq;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Mathematics;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
namespace BenchmarkDotNet.Columns
{
public class BaselineScaledColumn : IColumn
{
public enum DiffKind
{
Mean,
StdDev
}
public static readonly IColumn Scaled = new BaselineScaledColumn(DiffKind.Mean);
public static readonly IColumn ScaledStdDev = new BaselineScaledColumn(DiffKind.StdDev);
public DiffKind Kind { get; set; }
private BaselineScaledColumn(DiffKind kind)
{
Kind = kind;
}
public string ColumnName
{
get
{
switch (Kind)
{
case DiffKind.Mean:
return "Scaled";
case DiffKind.StdDev:
return "Scaled-SD";
}
throw new NotSupportedException();
}
}
public string GetValue(Summary summary, Benchmark benchmark)
{
var baseline = summary.Benchmarks.
Where(b => b.Job.GetFullInfo() == benchmark.Job.GetFullInfo()).
Where(b => b.Parameters.FullInfo == benchmark.Parameters.FullInfo).
FirstOrDefault(b => b.Target.Baseline);
var invalidResults = baseline == null ||
summary[baseline] == null ||
summary[baseline].ResultStatistics == null ||
summary[baseline].ResultStatistics.Invert() == null ||
summary[benchmark] == null ||
summary[benchmark].ResultStatistics == null;
if (invalidResults)
return "?";
var baselineStat = summary[baseline].ResultStatistics;
var targetStat = summary[benchmark].ResultStatistics;
var mean = benchmark.Target.Baseline ? 1 : Statistics.DivMean(targetStat, baselineStat);
var stdDev = benchmark.Target.Baseline ? 0 : Math.Sqrt(Statistics.DivVariance(targetStat, baselineStat));
switch (Kind)
{
case DiffKind.Mean:
return mean.ToStr("N2");
case DiffKind.StdDev:
return stdDev.ToStr("N2");
default:
throw new NotSupportedException();
}
}
public bool IsAvailable(Summary summary) => summary.Benchmarks.Any(b => b.Target.Baseline);
public bool AlwaysShow => true;
public ColumnCategory Category => ColumnCategory.Statistics;
public override string ToString() => ColumnName;
}
}

3
BenchmarkDotNet/Configs/DefaultConfig.cs
Просмотреть файл

@ -40,7 +40,8 @@ namespace BenchmarkDotNet.Configs
yield return StatisticColumn.Median;
yield return StatisticColumn.StdDev;
yield return BaselineDiffColumn.Scaled;
yield return BaselineScaledColumn.Scaled;
yield return BaselineScaledColumn.ScaledStdDev;
}
public IEnumerable<IExporter> GetExporters()

48
BenchmarkDotNet/Mathematics/Statistics.cs
Просмотреть файл

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using BenchmarkDotNet.Extensions;
namespace BenchmarkDotNet.Mathematics
{
@ -31,7 +32,7 @@ namespace BenchmarkDotNet.Mathematics
}
public Statistics(IEnumerable<int> values) :
this(values.Select(value => (double)value))
this(values.Select(value => (double) value))
{
}
@ -76,5 +77,50 @@ namespace BenchmarkDotNet.Mathematics
public double[] WithoutOutliers() => list.Where(value => !IsOutlier(value)).ToArray();
public override string ToString() => $"{Mean} +- {StandardError} (N = {N})";
/// <summary>
/// Statistics for [1/X]. If Min is less then or equal to 0, returns null.
/// </summary>
public Statistics Invert() => Min < 1e-9 ? null : new Statistics(list.Select(x => 1 / x));
/// <summary>
/// Statistics for [X^2].
/// </summary>
public Statistics Sqr() => new Statistics(list.Select(x => x * x));
/// <summary>
/// Mean for [X*Y].
/// </summary>
public static double MulMean(Statistics x, Statistics y) => x.Mean * y.Mean;
/// <summary>
/// Mean for [X/Y].
/// </summary>
public static double DivMean(Statistics x, Statistics y)
{
var yInvert = y.Invert();
if (yInvert == null)
throw new DivideByZeroException();
return MulMean(x, yInvert);
}
/// <summary>
/// Variance for [X*Y].
/// </summary>
public static double MulVariance(Statistics x, Statistics y)
{
return x.Sqr().Mean * y.Sqr().Mean - x.Mean.Sqr() * y.Mean.Sqr();
}
/// <summary>
/// Variance for [X/Y].
/// </summary>
public static double DivVariance(Statistics x, Statistics y)
{
var yInvert = y.Invert();
if (yInvert == null)
throw new DivideByZeroException();
return MulVariance(x, yInvert);
}
}
}

3
BenchmarkDotNet/Reports/Summary.cs
Просмотреть файл

@ -61,7 +61,7 @@ namespace BenchmarkDotNet.Reports
{
Benchmarks = benchmarks;
Table = new SummaryTable(this);
Reports = reports;
Reports = reports ?? new BenchmarkReport[0];
}
private Summary(string title, HostEnvironmentInfo hostEnvironmentInfo, IConfig config, string resultsDirectoryPath, TimeSpan totalTime, ValidationError[] validationErrors)
@ -72,6 +72,7 @@ namespace BenchmarkDotNet.Reports
ResultsDirectoryPath = resultsDirectoryPath;
TotalTime = totalTime;
ValidationErrors = validationErrors;
Reports = new BenchmarkReport[0];
}
internal static Summary CreateFailed(Benchmark[] benchmarks, string title, HostEnvironmentInfo hostEnvironmentInfo, IConfig config, string resultsDirectoryPath, ValidationError[] validationErrors)