dotnet
/
BenchmarkDotNet
зеркало из https://github.com/dotnet/BenchmarkDotNet.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Respect LogicalGroup order in DefaultOrderer (see #1864) (#1866)

This commit is contained in:
Andrey Akinshin 2021-12-09 17:20:26 +03:00 коммит произвёл GitHub
Родитель bb180a9bd9
Коммит f188f764ff
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
12 изменённых файлов: 194 добавлений и 121 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
samples/BenchmarkDotNet.Samples/IntroOrderManual.cs
Просмотреть файл

@ -21,7 +21,8 @@ namespace BenchmarkDotNet.Samples
private class FastestToSlowestOrderer : IOrderer
{
public IEnumerable<BenchmarkCase> GetExecutionOrder(ImmutableArray<BenchmarkCase> benchmarksCase) =>
public IEnumerable<BenchmarkCase> GetExecutionOrder(ImmutableArray<BenchmarkCase> benchmarksCase,
IEnumerable<BenchmarkLogicalGroupRule> order = null) =>
from benchmark in benchmarksCase
orderby benchmark.Parameters["X"] descending,
benchmark.Descriptor.WorkloadMethodDisplayInfo
@ -37,7 +38,8 @@ namespace BenchmarkDotNet.Samples
public string GetLogicalGroupKey(ImmutableArray<BenchmarkCase> allBenchmarksCases, BenchmarkCase benchmarkCase) =>
benchmarkCase.Job.DisplayInfo + "_" + benchmarkCase.Parameters.DisplayInfo;
public IEnumerable<IGrouping<string, BenchmarkCase>> GetLogicalGroupOrder(IEnumerable<IGrouping<string, BenchmarkCase>> logicalGroups) =>
public IEnumerable<IGrouping<string, BenchmarkCase>> GetLogicalGroupOrder(IEnumerable<IGrouping<string, BenchmarkCase>> logicalGroups,
IEnumerable<BenchmarkLogicalGroupRule> order = null) =>
logicalGroups.OrderBy(it => it.Key);
public bool SeparateLogicalGroups => true;

4
src/BenchmarkDotNet/Configs/ImmutableConfig.cs
Просмотреть файл

@ -30,7 +30,7 @@ namespace BenchmarkDotNet.Configs
private readonly ImmutableHashSet<Job> jobs;
private readonly ImmutableHashSet<HardwareCounter> hardwareCounters;
private readonly ImmutableHashSet<IFilter> filters;
private readonly ImmutableHashSet<BenchmarkLogicalGroupRule> rules;
private readonly ImmutableArray<BenchmarkLogicalGroupRule> rules;
internal ImmutableConfig(
ImmutableArray<IColumnProvider> uniqueColumnProviders,
@ -41,7 +41,7 @@ namespace BenchmarkDotNet.Configs
ImmutableHashSet<IAnalyser> uniqueAnalyzers,
ImmutableHashSet<IValidator> uniqueValidators,
ImmutableHashSet<IFilter> uniqueFilters,
ImmutableHashSet<BenchmarkLogicalGroupRule> uniqueRules,
ImmutableArray<BenchmarkLogicalGroupRule> uniqueRules,
ImmutableHashSet<Job> uniqueRunnableJobs,
ConfigUnionRule unionRule,
string artifactsPath,

2
src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs
Просмотреть файл

@ -45,7 +45,7 @@ namespace BenchmarkDotNet.Configs
var uniqueValidators = GetValidators(source.GetValidators(), MandatoryValidators, source.Options);
var uniqueFilters = source.GetFilters().ToImmutableHashSet();
var uniqueRules = source.GetLogicalGroupRules().ToImmutableHashSet();
var uniqueRules = source.GetLogicalGroupRules().ToImmutableArray();
var uniqueRunnableJobs = GetRunnableJobs(source.GetJobs()).ToImmutableHashSet();

9
src/BenchmarkDotNet/Configs/ManualConfig.cs
Просмотреть файл

@ -29,7 +29,7 @@ namespace BenchmarkDotNet.Configs
private readonly List<Job> jobs = new List<Job>();
private readonly HashSet<HardwareCounter> hardwareCounters = new HashSet<HardwareCounter>();
private readonly List<IFilter> filters = new List<IFilter>();
private readonly HashSet<BenchmarkLogicalGroupRule> logicalGroupRules = new HashSet<BenchmarkLogicalGroupRule>();
private readonly List<BenchmarkLogicalGroupRule> logicalGroupRules = new List<BenchmarkLogicalGroupRule>();
public IEnumerable<IColumnProvider> GetColumnProviders() => columnProviders;
public IEnumerable<IExporter> GetExporters() => exporters;
@ -191,7 +191,12 @@ namespace BenchmarkDotNet.Configs
public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rules)
{
logicalGroupRules.AddRange(rules);
foreach (var rule in rules)
{
if (logicalGroupRules.Contains(rule))
logicalGroupRules.Remove(rule);
logicalGroupRules.Add(rule);
}
return this;
}

110
src/BenchmarkDotNet/Order/DefaultOrderer.cs
Просмотреть файл

@ -1,3 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.CodeAnalysis;
@ -19,8 +20,7 @@ namespace BenchmarkDotNet.Order
private readonly IComparer<string[]> categoryComparer = CategoryComparer.Instance;
private readonly IComparer<ParameterInstances> paramsComparer = ParameterComparer.Instance;
private readonly IComparer<Job> jobComparer = JobComparer.Instance;
private readonly IComparer<BenchmarkCase> benchmarkComparer;
private readonly IComparer<IGrouping<string, BenchmarkCase>> logicalGroupComparer;
private readonly IComparer<Descriptor> targetComparer;
public SummaryOrderPolicy SummaryOrderPolicy { get; }
public MethodOrderPolicy MethodOrderPolicy { get; }
@ -31,14 +31,15 @@ namespace BenchmarkDotNet.Order
{
SummaryOrderPolicy = summaryOrderPolicy;
MethodOrderPolicy = methodOrderPolicy;
IComparer<Descriptor> targetComparer = new DescriptorComparer(methodOrderPolicy);
benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer);
logicalGroupComparer = new LogicalGroupComparer(benchmarkComparer);
targetComparer = new DescriptorComparer(methodOrderPolicy);
}
[PublicAPI]
public virtual IEnumerable<BenchmarkCase> GetExecutionOrder(ImmutableArray<BenchmarkCase> benchmarkCases)
public virtual IEnumerable<BenchmarkCase> GetExecutionOrder(
ImmutableArray<BenchmarkCase> benchmarkCases,
IEnumerable<BenchmarkLogicalGroupRule> order = null)
{
var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order);
var list = benchmarkCases.ToList();
list.Sort(benchmarkComparer);
return list;
@ -47,7 +48,7 @@ namespace BenchmarkDotNet.Order
public virtual IEnumerable<BenchmarkCase> GetSummaryOrder(ImmutableArray<BenchmarkCase> benchmarksCases, Summary summary)
{
var benchmarkLogicalGroups = benchmarksCases.GroupBy(b => GetLogicalGroupKey(benchmarksCases, b));
foreach (var logicalGroup in GetLogicalGroupOrder(benchmarkLogicalGroups))
foreach (var logicalGroup in GetLogicalGroupOrder(benchmarkLogicalGroups, benchmarksCases.FirstOrDefault()?.Config.GetLogicalGroupRules()))
foreach (var benchmark in GetSummaryOrderForGroup(logicalGroup.ToImmutableArray(), summary))
yield return benchmark;
}
@ -65,7 +66,7 @@ namespace BenchmarkDotNet.Order
case SummaryOrderPolicy.Declared:
return benchmarksCase;
default:
return GetExecutionOrder(benchmarksCase);
return GetExecutionOrder(benchmarksCase, benchmarksCase.FirstOrDefault()?.Config.GetLogicalGroupRules());
}
}
@ -84,41 +85,62 @@ namespace BenchmarkDotNet.Order
public string GetLogicalGroupKey(ImmutableArray<BenchmarkCase> allBenchmarksCases, BenchmarkCase benchmarkCase)
{
var rules = new HashSet<BenchmarkLogicalGroupRule>(benchmarkCase.Config.GetLogicalGroupRules());
var explicitRules = benchmarkCase.Config.GetLogicalGroupRules().ToList();
var implicitRules = new List<BenchmarkLogicalGroupRule>();
bool hasJobBaselines = allBenchmarksCases.Any(b => b.Job.Meta.Baseline);
bool hasDescriptorBaselines = allBenchmarksCases.Any(b => b.Descriptor.Baseline);
if (hasJobBaselines)
{
rules.Add(BenchmarkLogicalGroupRule.ByMethod);
rules.Add(BenchmarkLogicalGroupRule.ByParams);
implicitRules.Add(BenchmarkLogicalGroupRule.ByParams);
implicitRules.Add(BenchmarkLogicalGroupRule.ByMethod);
}
if (hasDescriptorBaselines)
{
rules.Add(BenchmarkLogicalGroupRule.ByJob);
rules.Add(BenchmarkLogicalGroupRule.ByParams);
implicitRules.Add(BenchmarkLogicalGroupRule.ByParams);
implicitRules.Add(BenchmarkLogicalGroupRule.ByJob);
}
if (hasJobBaselines && hasDescriptorBaselines)
{
rules.Remove(BenchmarkLogicalGroupRule.ByMethod);
rules.Remove(BenchmarkLogicalGroupRule.ByJob);
implicitRules.Remove(BenchmarkLogicalGroupRule.ByMethod);
implicitRules.Remove(BenchmarkLogicalGroupRule.ByJob);
}
var rules = new List<BenchmarkLogicalGroupRule>(explicitRules);
foreach (var rule in implicitRules.Where(rule => !rules.Contains(rule)))
rules.Add(rule);
var keys = new List<string>();
if (rules.Contains(BenchmarkLogicalGroupRule.ByCategory))
keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories));
if (rules.Contains(BenchmarkLogicalGroupRule.ByMethod))
keys.Add(benchmarkCase.Descriptor.DisplayInfo);
if (rules.Contains(BenchmarkLogicalGroupRule.ByJob))
keys.Add(benchmarkCase.Job.DisplayInfo);
if (rules.Contains(BenchmarkLogicalGroupRule.ByParams))
keys.Add(benchmarkCase.Parameters.ValueInfo);
foreach (var rule in rules)
{
switch (rule)
{
case BenchmarkLogicalGroupRule.ByMethod:
keys.Add(benchmarkCase.Descriptor.DisplayInfo);
break;
case BenchmarkLogicalGroupRule.ByJob:
keys.Add(benchmarkCase.Job.DisplayInfo);
break;
case BenchmarkLogicalGroupRule.ByParams:
keys.Add(benchmarkCase.Parameters.ValueInfo);
break;
case BenchmarkLogicalGroupRule.ByCategory:
keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories));
break;
default:
throw new ArgumentOutOfRangeException(nameof(rule), rule, $"Not supported {nameof(BenchmarkLogicalGroupRule)}");
}
}
string logicalGroupKey = string.Join("-", keys.Where(key => key != string.Empty));
return logicalGroupKey == string.Empty ? "*" : logicalGroupKey;
}
public virtual IEnumerable<IGrouping<string, BenchmarkCase>> GetLogicalGroupOrder(IEnumerable<IGrouping<string, BenchmarkCase>> logicalGroups)
public virtual IEnumerable<IGrouping<string, BenchmarkCase>> GetLogicalGroupOrder(
IEnumerable<IGrouping<string, BenchmarkCase>> logicalGroups,
IEnumerable<BenchmarkLogicalGroupRule> order = null)
{
var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order);
var logicalGroupComparer = new LogicalGroupComparer(benchmarkComparer);
var list = logicalGroups.ToList();
list.Sort(logicalGroupComparer);
return list;
@ -128,21 +150,36 @@ namespace BenchmarkDotNet.Order
private class BenchmarkComparer : IComparer<BenchmarkCase>
{
private static readonly BenchmarkLogicalGroupRule[] DefaultOrder =
{
BenchmarkLogicalGroupRule.ByCategory,
BenchmarkLogicalGroupRule.ByParams,
BenchmarkLogicalGroupRule.ByJob,
BenchmarkLogicalGroupRule.ByMethod
};
private readonly IComparer<string[]> categoryComparer;
private readonly IComparer<ParameterInstances> paramsComparer;
private readonly IComparer<Job> jobComparer;
private readonly IComparer<Descriptor> targetComparer;
private readonly List<BenchmarkLogicalGroupRule> order;
public BenchmarkComparer(
IComparer<string[]> categoryComparer,
IComparer<ParameterInstances> paramsComparer,
IComparer<Job> jobComparer,
IComparer<Descriptor> targetComparer)
IComparer<Descriptor> targetComparer,
IEnumerable<BenchmarkLogicalGroupRule> order)
{
this.categoryComparer = categoryComparer;
this.targetComparer = targetComparer;
this.jobComparer = jobComparer;
this.paramsComparer = paramsComparer;
this.order = new List<BenchmarkLogicalGroupRule>();
foreach (var rule in (order ?? ImmutableArray<BenchmarkLogicalGroupRule>.Empty).Concat(DefaultOrder))
if (!this.order.Contains(rule))
this.order.Add(rule);
}
public int Compare(BenchmarkCase x, BenchmarkCase y)
@ -150,14 +187,21 @@ namespace BenchmarkDotNet.Order
if (x == null && y == null) return 0;
if (x != null && y == null) return 1;
if (x == null) return -1;
return new[]
foreach (var rule in order)
{
categoryComparer?.Compare(x.Descriptor.Categories, y.Descriptor.Categories) ?? 0,
paramsComparer?.Compare(x.Parameters, y.Parameters) ?? 0,
jobComparer?.Compare(x.Job, y.Job) ?? 0,
targetComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0,
string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo)
}.FirstOrDefault(c => c != 0);
int compare = rule switch
{
BenchmarkLogicalGroupRule.ByMethod => targetComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0,
BenchmarkLogicalGroupRule.ByJob => jobComparer?.Compare(x.Job, y.Job) ?? 0,
BenchmarkLogicalGroupRule.ByParams => paramsComparer?.Compare(x.Parameters, y.Parameters) ?? 0,
BenchmarkLogicalGroupRule.ByCategory => categoryComparer?.Compare(x.Descriptor.Categories, y.Descriptor.Categories) ?? 0,
_ => throw new ArgumentOutOfRangeException()
};
if (compare != 0)
return compare;
}
return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo);
}
}
@ -176,4 +220,4 @@ namespace BenchmarkDotNet.Order
}
}
}
}
}

6
src/BenchmarkDotNet/Order/IOrderer.cs
Просмотреть файл

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
using JetBrains.Annotations;
@ -10,7 +11,7 @@ namespace BenchmarkDotNet.Order
public interface IOrderer
{
[PublicAPI, NotNull]
IEnumerable<BenchmarkCase> GetExecutionOrder(ImmutableArray<BenchmarkCase> benchmarksCase);
IEnumerable<BenchmarkCase> GetExecutionOrder(ImmutableArray<BenchmarkCase> benchmarksCase, IEnumerable<BenchmarkLogicalGroupRule> order = null);
[PublicAPI, NotNull]
IEnumerable<BenchmarkCase> GetSummaryOrder(ImmutableArray<BenchmarkCase> benchmarksCases, [NotNull] Summary summary);
@ -22,7 +23,8 @@ namespace BenchmarkDotNet.Order
string GetLogicalGroupKey(ImmutableArray<BenchmarkCase> allBenchmarksCases, [NotNull] BenchmarkCase benchmarkCase);
[PublicAPI, NotNull]
IEnumerable<IGrouping<string, BenchmarkCase>> GetLogicalGroupOrder(IEnumerable<IGrouping<string, BenchmarkCase>> logicalGroups);
IEnumerable<IGrouping<string, BenchmarkCase>> GetLogicalGroupOrder(IEnumerable<IGrouping<string, BenchmarkCase>> logicalGroups,
IEnumerable<BenchmarkLogicalGroupRule> order = null);
[PublicAPI]
bool SeparateLogicalGroups { get; }

24
tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt
Просмотреть файл

@ -10,22 +10,22 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- |
Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Base-[Param=2] | Yes | ^
Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | JobBaseline_MethodsParamsJobs.Base-[Param=2] | No |
Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^
Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No |
| | | | | | | | | | |
Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Foo-[Param=2] | Yes |
Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | JobBaseline_MethodsParamsJobs.Foo-[Param=2] | No |
Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes |
Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No |
| | | | | | | | | | |
Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Bar-[Param=2] | Yes |
Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | JobBaseline_MethodsParamsJobs.Bar-[Param=2] | No |
Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes |
Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No |
| | | | | | | | | | |
Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Base-[Param=10] | Yes | ^
Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | JobBaseline_MethodsParamsJobs.Base-[Param=10] | No |
Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^
Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No |
| | | | | | | | | | |
Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Foo-[Param=10] | Yes |
Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | JobBaseline_MethodsParamsJobs.Foo-[Param=10] | No |
Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes |
Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No |
| | | | | | | | | | |
Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsParamsJobs.Bar-[Param=10] | Yes |
Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | JobBaseline_MethodsParamsJobs.Bar-[Param=10] | No |
Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes |
Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No |
Errors: 0

12
tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParams.approved.txt
Просмотреть файл

@ -9,12 +9,12 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
Method | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |------ |---------:|--------:|--------:|------:|--------:|-----:|---------------------- |--------- |
Base | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | DefaultJob-[Param=2] | Yes | ^
Foo | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | DefaultJob-[Param=2] | No |
Bar | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | DefaultJob-[Param=2] | No |
Base | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-DefaultJob | Yes | ^
Foo | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2]-DefaultJob | No |
Bar | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2]-DefaultJob | No |
| | | | | | | | | |
Base | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | DefaultJob-[Param=10] | Yes | ^
Foo | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | DefaultJob-[Param=10] | No |
Bar | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | DefaultJob-[Param=10] | No |
Base | 10 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-DefaultJob | Yes | ^
Foo | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | [Param=10]-DefaultJob | No |
Bar | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | [Param=10]-DefaultJob | No |
Errors: 0

24
tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.MethodBaseline_MethodsParamsJobs.approved.txt
Просмотреть файл

@ -10,20 +10,20 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------- |--------- |
Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job1-[Param=2] | Yes | ^
Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | Job1-[Param=2] | No |
Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | Job1-[Param=2] | No |
Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Job1 | Yes | ^
Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | [Param=2]-Job1 | No |
Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 3 | [Param=2]-Job1 | No |
| | | | | | | | | | |
Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job2-[Param=2] | Yes |
Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | Job2-[Param=2] | No |
Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | Job2-[Param=2] | No |
Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Job2 | Yes |
Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 1.25 | 0.00 | 2 | [Param=2]-Job2 | No |
Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.50 | 0.00 | 3 | [Param=2]-Job2 | No |
| | | | | | | | | | |
Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job1-[Param=10] | Yes | ^
Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | Job1-[Param=10] | No |
Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.28 | 0.00 | 3 | Job1-[Param=10] | No |
Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Job1 | Yes | ^
Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | [Param=10]-Job1 | No |
Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.28 | 0.00 | 3 | [Param=10]-Job1 | No |
| | | | | | | | | | |
Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Job2-[Param=10] | Yes |
Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.10 | 0.00 | 2 | Job2-[Param=10] | No |
Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 3 | Job2-[Param=10] | No |
Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Job2 | Yes |
Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.10 | 0.00 | 2 | [Param=10]-Job2 | No |
Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 3 | [Param=10]-Job2 | No |
Errors: 0

32
tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByAll.approved.txt
Просмотреть файл

@ -10,36 +10,36 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------------------------- |--------- |
A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2] | Yes | ^
A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^
| | | | | | | | | | |
A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2] | No |
A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^
| | | | | | | | | | |
A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2] | Yes |
A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^
| | | | | | | | | | |
A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2] | No |
A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^
| | | | | | | | | | |
A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10] | Yes | ^
A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^
| | | | | | | | | | |
A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10] | No |
A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^
| | | | | | | | | | |
A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10] | Yes |
A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^
| | | | | | | | | | |
A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatA-NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10] | No |
A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^
| | | | | | | | | | |
B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2] | Yes | ^
B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^
| | | | | | | | | | |
B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2] | No |
B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^
| | | | | | | | | | |
B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2] | Yes |
B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^
| | | | | | | | | | |
B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2] | No |
B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^
| | | | | | | | | | |
B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10] | Yes | ^
B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^
| | | | | | | | | | |
B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10] | No |
B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^
| | | | | | | | | | |
B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10] | Yes |
B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^
| | | | | | | | | | |
B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | CatB-NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10] | No |
B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^
Errors: 0

32
tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/MarkdownExporterApprovalTests.GroupExporterTest.NoBaseline_MethodsParamsJobs_GroupByCategory.approved.txt
Просмотреть файл

@ -10,28 +10,28 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|--------------------- |--------- |
A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-Job1-[Param=2] | Yes | ^
A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | CatA-Job1-[Param=2] | No |
A1 | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=2]-Job1 | Yes | ^
A2 | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.98 | 0.02 | 2 | CatA-[Param=2]-Job1 | No |
| | | | | | | | | | |
A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-Job2-[Param=2] | Yes |
A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | CatA-Job2-[Param=2] | No |
A1 | Job2 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=2]-Job2 | Yes |
A2 | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | CatA-[Param=2]-Job2 | No |
| | | | | | | | | | |
A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-Job1-[Param=10] | Yes | ^
A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 2 | CatA-Job1-[Param=10] | No |
A1 | Job1 | 10 | 502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=10]-Job1 | Yes | ^
A2 | Job1 | 10 | 602.0 ns | 6.09 ns | 1.58 ns | 1.20 | 0.00 | 2 | CatA-[Param=10]-Job1 | No |
| | | | | | | | | | |
A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-Job2-[Param=10] | Yes |
A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | CatA-Job2-[Param=10] | No |
A1 | Job2 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatA-[Param=10]-Job2 | Yes |
A2 | Job2 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.14 | 0.00 | 2 | CatA-[Param=10]-Job2 | No |
| | | | | | | | | | |
B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-Job1-[Param=2] | Yes | ^
B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.11 | 0.00 | 2 | CatB-Job1-[Param=2] | No |
B1 | Job1 | 2 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=2]-Job1 | Yes | ^
B2 | Job1 | 2 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.11 | 0.00 | 2 | CatB-[Param=2]-Job1 | No |
| | | | | | | | | | |
B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-Job2-[Param=2] | Yes |
B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.09 | 0.00 | 2 | CatB-Job2-[Param=2] | No |
B1 | Job2 | 2 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=2]-Job2 | Yes |
B2 | Job2 | 2 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.09 | 0.00 | 2 | CatB-[Param=2]-Job2 | No |
| | | | | | | | | | |
B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-Job1-[Param=10] | Yes | ^
B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | 1.08 | 0.00 | 2 | CatB-Job1-[Param=10] | No |
B1 | Job1 | 10 | 1,302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=10]-Job1 | Yes | ^
B2 | Job1 | 10 | 1,402.0 ns | 6.09 ns | 1.58 ns | 1.08 | 0.00 | 2 | CatB-[Param=10]-Job1 | No |
| | | | | | | | | | |
B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-Job2-[Param=10] | Yes |
B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | 1.07 | 0.00 | 2 | CatB-Job2-[Param=10] | No |
B1 | Job2 | 10 | 1,502.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | CatB-[Param=10]-Job2 | Yes |
B2 | Job2 | 10 | 1,602.0 ns | 6.09 ns | 1.58 ns | 1.07 | 0.00 | 2 | CatB-[Param=10]-Job2 | No |
Errors: 0

54
tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs
Просмотреть файл

@ -17,24 +17,25 @@ namespace BenchmarkDotNet.Tests.Order
{
public class DefaultOrdererTests
{
private static Summary CreateMockSummary() => new("", ImmutableArray<BenchmarkReport>.Empty, HostEnvironmentInfo.GetCurrent(),
"", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray<ValidationError>.Empty);
private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new(
new Descriptor(null, null, categories: new[] { category }),
new Job(),
new ParameterInstances(new[]
{
new ParameterInstance(new ParameterDefinition("P", false, null, false, null, 0), parameter, SummaryStyle.Default)
}),
DefaultConfig.Instance.AddLogicalGroupRules(rules).CreateImmutableConfig()
);
private static string GetId(BenchmarkCase benchmarkCase) => benchmarkCase.Descriptor.Categories.First() + benchmarkCase.Parameters.Items.First().Value;
[Fact]
public void CategoriesHasHigherPriorityThanParameters()
{
var summary = new Summary("", ImmutableArray<BenchmarkReport>.Empty, HostEnvironmentInfo.GetCurrent(),
"", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray<ValidationError>.Empty);
BenchmarkCase CreateBenchmarkCase(string category, int parameter) => new(
new Descriptor(null, null, categories: new[] { category }),
new Job(),
new ParameterInstances(new[]
{
new ParameterInstance(new ParameterDefinition("P", false, null, false, null, 0), parameter, SummaryStyle.Default)
}),
DefaultConfig.Instance.CreateImmutableConfig()
);
string GetId(BenchmarkCase benchmarkCase) => benchmarkCase.Descriptor.Categories.First() + benchmarkCase.Parameters.Items.First().Value;
var summary = CreateMockSummary();
var benchmarkCases = new List<BenchmarkCase>
{
CreateBenchmarkCase("A", 1),
@ -42,9 +43,28 @@ namespace BenchmarkDotNet.Tests.Order
CreateBenchmarkCase("A", 2),
CreateBenchmarkCase("B", 2)
}.ToImmutableArray();
string[] sortedBenchmarkCases = DefaultOrderer.Instance.GetSummaryOrder(benchmarkCases, summary).Select(GetId).ToArray();
Assert.Equal(new[] {"A1", "A2", "B1", "B2"}, sortedBenchmarkCases);
Assert.Equal(new[] { "A1", "A2", "B1", "B2" }, sortedBenchmarkCases);
}
[Fact]
public void OrderCanBeOverriden()
{
BenchmarkLogicalGroupRule[] rules =
{
BenchmarkLogicalGroupRule.ByParams,
BenchmarkLogicalGroupRule.ByCategory,
};
var summary = CreateMockSummary();
var benchmarkCases = new List<BenchmarkCase>
{
CreateBenchmarkCase("A", 1, rules),
CreateBenchmarkCase("B", 1, rules),
CreateBenchmarkCase("A", 2, rules),
CreateBenchmarkCase("B", 2, rules)
}.ToImmutableArray();
string[] sortedBenchmarkCases = DefaultOrderer.Instance.GetSummaryOrder(benchmarkCases, summary).Select(GetId).ToArray();
Assert.Equal(new[] { "A1", "B1", "A2", "B2" }, sortedBenchmarkCases);
}
}
}