dotnet
/
BenchmarkDotNet
зеркало из https://github.com/dotnet/BenchmarkDotNet.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Engine.Run() should return the full list of performed measurements (fixes #2187) (#2188) 

* Engine.Run() should return the full list of performed measurements (fixes #2187)

* Code review fixes
This commit is contained in:
Andrey Akinshin 2022-11-03 09:55:02 +04:00 коммит произвёл GitHub
Родитель 9e759f9bc5
Коммит e75bdd8507
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
12 изменённых файлов: 186 добавлений и 80 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

21
src/BenchmarkDotNet/Engines/Engine.cs
Просмотреть файл

@ -41,6 +41,7 @@ namespace BenchmarkDotNet.Engines
private bool EvaluateOverhead { get; }
private bool MemoryRandomization { get; }
private readonly List<Measurement> jittingMeasurements = new (10);
private readonly EnginePilotStage pilotStage;
private readonly EngineWarmupStage warmupStage;
private readonly EngineActualStage actualStage;
@ -104,8 +105,10 @@ namespace BenchmarkDotNet.Engines
public RunResults Run()
{
var measurements = new List<Measurement>();
measurements.AddRange(jittingMeasurements);
long invokeCount = TargetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver, 1);
IReadOnlyList<Measurement> idle = null;
if (EngineEventSource.Log.IsEnabled())
EngineEventSource.Log.BenchmarkStart(BenchmarkName);
@ -114,21 +117,23 @@ namespace BenchmarkDotNet.Engines
{
if (Strategy != RunStrategy.Monitoring)
{
invokeCount = pilotStage.Run();
var pilotStageResult = pilotStage.Run();
invokeCount = pilotStageResult.PerfectInvocationCount;
measurements.AddRange(pilotStageResult.Measurements);
if (EvaluateOverhead)
{
warmupStage.RunOverhead(invokeCount, UnrollFactor);
idle = actualStage.RunOverhead(invokeCount, UnrollFactor);
measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor));
measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor));
}
}
warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy);
measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy));
}
Host.BeforeMainRun();
var main = actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring);
measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring));
Host.AfterMainRun();
@ -141,7 +146,7 @@ namespace BenchmarkDotNet.Engines
var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver);
return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
}
public Measurement RunIteration(IterationData data)
@ -183,6 +188,8 @@ namespace BenchmarkDotNet.Engines
// Results
var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds());
WriteLine(measurement.ToString());
if (measurement.IterationStage == IterationStage.Jitting)
jittingMeasurements.Add(measurement);
Consume(stackMemory);

38
src/BenchmarkDotNet/Engines/EnginePilotStage.cs
Просмотреть файл

@ -1,6 +1,9 @@
using System;
using System.Collections.Generic;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using JetBrains.Annotations;
using Perfolizer.Horology;
namespace BenchmarkDotNet.Engines
@ -8,6 +11,25 @@ namespace BenchmarkDotNet.Engines
// TODO: use clockResolution
internal class EnginePilotStage : EngineStage
{
public readonly struct PilotStageResult
{
public long PerfectInvocationCount { get; }
[NotNull]
public IReadOnlyList<Measurement> Measurements { get; }
public PilotStageResult(long perfectInvocationCount, [NotNull] List<Measurement> measurements)
{
PerfectInvocationCount = perfectInvocationCount;
Measurements = measurements;
}
public PilotStageResult(long perfectInvocationCount)
{
PerfectInvocationCount = perfectInvocationCount;
Measurements = Array.Empty<Measurement>();
}
}
internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2;
private readonly int unrollFactor;
@ -30,11 +52,11 @@ namespace BenchmarkDotNet.Engines
}
/// <returns>Perfect invocation count</returns>
public long Run()
public PilotStageResult Run()
{
// If InvocationCount is specified, pilot stage should be skipped
if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic))
return TargetJob.Run.InvocationCount;
return new PilotStageResult(TargetJob.Run.InvocationCount);
// Here we want to guess "perfect" amount of invocation
return TargetJob.HasValue(RunMode.IterationTimeCharacteristic)
@ -45,15 +67,17 @@ namespace BenchmarkDotNet.Engines
/// <summary>
/// A case where we don't have specific iteration time.
/// </summary>
private long RunAuto()
private PilotStageResult RunAuto()
{
long invokeCount = Autocorrect(minInvokeCount);
var measurements = new List<Measurement>();
int iterationCounter = 0;
while (true)
{
iterationCounter++;
var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor);
measurements.Add(measurement);
double iterationTime = measurement.Nanoseconds;
double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision
@ -75,15 +99,16 @@ namespace BenchmarkDotNet.Engines
}
WriteLine();
return invokeCount;
return new PilotStageResult(invokeCount, measurements);
}
/// <summary>
/// A case where we have specific iteration time.
/// </summary>
private long RunSpecific()
private PilotStageResult RunSpecific()
{
long invokeCount = Autocorrect(Engine.MinInvokeCount);
var measurements = new List<Measurement>();
int iterationCounter = 0;
@ -92,6 +117,7 @@ namespace BenchmarkDotNet.Engines
{
iterationCounter++;
var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor);
measurements.Add(measurement);
double actualIterationTime = measurement.Nanoseconds;
long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long)Math.Round(invokeCount * targetIterationTime / actualIterationTime)));
@ -105,7 +131,7 @@ namespace BenchmarkDotNet.Engines
}
WriteLine();
return invokeCount;
return new PilotStageResult(invokeCount, measurements);
}
private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor;

6
src/BenchmarkDotNet/Engines/EngineWarmupStage.cs
Просмотреть файл

@ -9,13 +9,13 @@ namespace BenchmarkDotNet.Engines
public EngineWarmupStage(IEngine engine) : base(engine) => this.engine = engine;
public void RunOverhead(long invokeCount, int unrollFactor)
public IReadOnlyList<Measurement> RunOverhead(long invokeCount, int unrollFactor)
=> Run(invokeCount, IterationMode.Overhead, unrollFactor, RunStrategy.Throughput);
public void RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy)
public IReadOnlyList<Measurement> RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy)
=> Run(invokeCount, IterationMode.Workload, unrollFactor, runStrategy);
internal List<Measurement> Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy)
internal IReadOnlyList<Measurement> Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy)
{
var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy);
return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor);

51
src/BenchmarkDotNet/Engines/RunResults.cs
Просмотреть файл

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Mathematics;
using BenchmarkDotNet.Reports;
using JetBrains.Annotations;
@ -13,11 +14,20 @@ namespace BenchmarkDotNet.Engines
{
private readonly OutlierMode outlierMode;
[NotNull, PublicAPI]
public IReadOnlyList<Measurement> EngineMeasurements { get; }
[CanBeNull, PublicAPI]
public IReadOnlyList<Measurement> Overhead { get; }
public IReadOnlyList<Measurement> Overhead
=> EngineMeasurements
.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual))
.ToArray();
[NotNull, PublicAPI]
public IReadOnlyList<Measurement> Workload { get; }
public IReadOnlyList<Measurement> Workload
=> EngineMeasurements
.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual))
.ToArray();
public GcStats GCStats { get; }
@ -25,27 +35,30 @@ namespace BenchmarkDotNet.Engines
public double ExceptionFrequency { get; }
public RunResults([CanBeNull] IReadOnlyList<Measurement> overhead,
[NotNull] IReadOnlyList<Measurement> workload,
OutlierMode outlierMode,
GcStats gcStats,
ThreadingStats threadingStats,
double exceptionFrequency)
public RunResults([NotNull] IReadOnlyList<Measurement> engineMeasurements,
OutlierMode outlierMode,
GcStats gcStats,
ThreadingStats threadingStats,
double exceptionFrequency)
{
this.outlierMode = outlierMode;
Overhead = overhead;
Workload = workload;
EngineMeasurements = engineMeasurements;
GCStats = gcStats;
ThreadingStats = threadingStats;
ExceptionFrequency = exceptionFrequency;
}
public IEnumerable<Measurement> GetMeasurements()
public IEnumerable<Measurement> GetWorkloadResultMeasurements()
{
double overhead = Overhead == null ? 0.0 : new Statistics(Overhead.Select(m => m.Nanoseconds)).Median;
var mainStats = new Statistics(Workload.Select(m => m.Nanoseconds));
var overheadActualMeasurements = Overhead ?? Array.Empty<Measurement>();
var workloadActualMeasurements = Workload;
if (workloadActualMeasurements.IsEmpty())
yield break;
double overhead = overheadActualMeasurements.IsEmpty() ? 0.0 : new Statistics(overheadActualMeasurements.Select(m => m.Nanoseconds)).Median;
var mainStats = new Statistics(workloadActualMeasurements.Select(m => m.Nanoseconds));
int resultIndex = 0;
foreach (var measurement in Workload)
foreach (var measurement in workloadActualMeasurements)
{
if (mainStats.IsActualOutlier(measurement.Nanoseconds, outlierMode))
continue;
@ -63,9 +76,17 @@ namespace BenchmarkDotNet.Engines
}
}
public IEnumerable<Measurement> GetAllMeasurements()
{
foreach (var measurement in EngineMeasurements)
yield return measurement;
foreach (var measurement in GetWorkloadResultMeasurements())
yield return measurement;
}
public void Print(TextWriter outWriter)
{
foreach (var measurement in GetMeasurements())
foreach (var measurement in GetWorkloadResultMeasurements())
outWriter.WriteLine(measurement.ToString());
if (!GCStats.Equals(GcStats.Empty))

2
src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs
Просмотреть файл

@ -67,7 +67,7 @@ namespace BenchmarkDotNet.Toolchains.Results
internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode)
=> exitCode != 0
? CreateFailed(exitCode)
: new ExecuteResult(runResults.GetMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency);
: new ExecuteResult(runResults.GetAllMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency);
internal static ExecuteResult CreateFailed(int exitCode = -1)
=> new ExecuteResult(false, exitCode, default, Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), 0);

7
tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs
Просмотреть файл

@ -72,8 +72,11 @@ namespace BenchmarkDotNet.IntegrationTests
Console.WriteLine(EngineRunMessage);
return new RunResults(
new List<Measurement> { new Measurement(1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1) },
new List<Measurement> { new Measurement(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) },
new List<Measurement>
{
new (1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1),
new (1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1)
},
OutlierMode.DontRemove,
default,
default,

78
tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs Normal file
Просмотреть файл

@ -0,0 +1,78 @@
using System.Linq;
using System.Threading;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using Xunit;
using Xunit.Abstractions;
namespace BenchmarkDotNet.IntegrationTests
{
public class EngineTests : BenchmarkTestExecutor
{
public EngineTests(ITestOutputHelper output) : base(output) { }
[Fact]
public void ZeroWarmupCountIsApplied()
{
var job = Job.InProcess
.WithEvaluateOverhead(false)
.WithWarmupCount(0)
.WithIterationCount(1)
.WithInvocationCount(1)
.WithUnrollFactor(1);
var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator);
var summary = CanExecute<FooBench>(config);
var report = summary.Reports.Single();
int workloadWarmupCount = report.AllMeasurements
.Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup));
Assert.Equal(0, workloadWarmupCount);
}
[Fact]
public void AllMeasurementsArePerformedDefault() => AllMeasurementsArePerformed(Job.Default);
[Fact]
public void AllMeasurementsArePerformedInProcess() => AllMeasurementsArePerformed(Job.InProcess);
private void AllMeasurementsArePerformed(Job baseJob)
{
var job = baseJob
.WithWarmupCount(1)
.WithIterationCount(1)
.WithInvocationCount(1)
.WithUnrollFactor(1);
var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator);
var summary = CanExecute<FooBench>(config);
var measurements = summary.Reports.Single().AllMeasurements;
Output.WriteLine("*** AllMeasurements ***");
foreach (var measurement in measurements)
Output.WriteLine(measurement.ToString());
Output.WriteLine("-----");
void Check(IterationMode mode, IterationStage stage)
{
int count = measurements.Count(m => m.Is(mode, stage));
Output.WriteLine($"Count({mode}{stage}) = {count}");
Assert.True(count > 0, $"AllMeasurements don't contain {mode}{stage}");
}
Check(IterationMode.Overhead, IterationStage.Jitting);
Check(IterationMode.Workload, IterationStage.Jitting);
Check(IterationMode.Overhead, IterationStage.Warmup);
Check(IterationMode.Overhead, IterationStage.Actual);
Check(IterationMode.Workload, IterationStage.Warmup);
Check(IterationMode.Workload, IterationStage.Actual);
Check(IterationMode.Workload, IterationStage.Result);
}
public class FooBench
{
[Benchmark]
public void Foo() => Thread.Sleep(10);
}
}
}

6
tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs
Просмотреть файл

@ -5,6 +5,7 @@ using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.IntegrationTests.InProcess.EmitTests;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
@ -91,7 +92,10 @@ namespace BenchmarkDotNet.IntegrationTests
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
// Operations + GlobalSetup + GlobalCleanup
long expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2);
long expectedCount = summary.Reports
.SelectMany(r => r.AllMeasurements)
.Where(m => m.IterationStage != IterationStage.Result)
.Sum(m => m.Operations + 2);
Assert.Equal(expectedCount, BenchmarkAllCases.Counter);
}
finally

11
tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs
Просмотреть файл

@ -8,6 +8,7 @@ using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
@ -213,7 +214,10 @@ namespace BenchmarkDotNet.IntegrationTests
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
// Operations + GlobalSetup + GlobalCleanup
var expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2);
long expectedCount = summary.Reports
.SelectMany(r => r.AllMeasurements)
.Where(m => m.IterationStage != IterationStage.Result)
.Sum(m => m.Operations + 2);
Assert.Equal(expectedCount, BenchmarkAllCases.Counter);
}
finally
@ -239,7 +243,10 @@ namespace BenchmarkDotNet.IntegrationTests
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
// Operations + GlobalSetup + GlobalCleanup
var expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2);
long expectedCount = summary.Reports
.SelectMany(r => r.AllMeasurements)
.Where(m => m.IterationStage != IterationStage.Result)
.Sum(m => m.Operations + 2);
Assert.Equal(expectedCount, BenchmarkAllCases.Counter);
}
finally

40
tests/BenchmarkDotNet.IntegrationTests/JobTests.cs
Просмотреть файл

@ -1,40 +0,0 @@
using System.Linq;
using System.Threading;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using Xunit;
using Xunit.Abstractions;
namespace BenchmarkDotNet.IntegrationTests
{
public class JobTests : BenchmarkTestExecutor
{
public JobTests(ITestOutputHelper output) : base(output) { }
[Fact]
public void ZeroWarmupCountIsApplied()
{
var job = Job.Default
.WithEvaluateOverhead(false)
.WithWarmupCount(0)
.WithIterationCount(1)
.WithInvocationCount(1)
.WithUnrollFactor(1);
var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator);
var summary = CanExecute<ZeroWarmupBench>(config);
var report = summary.Reports.Single();
int workloadWarmupCount = report.AllMeasurements
.Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup));
Assert.Equal(0, workloadWarmupCount);
}
public class ZeroWarmupBench
{
[Benchmark]
public void Foo() => Thread.Sleep(10);
}
}
}

4
tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs
Просмотреть файл

@ -47,7 +47,7 @@ namespace BenchmarkDotNet.Tests.Engine
Accuracy = { MaxRelativeError = maxRelativeError }
}.Freeze();
var stage = CreateStage(job, data => data.InvokeCount * operationTime);
long invokeCount = stage.Run();
long invokeCount = stage.Run().PerfectInvocationCount;
output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {MaxPossibleInvokeCount})");
Assert.InRange(invokeCount, minInvokeCount, MaxPossibleInvokeCount);
}
@ -60,7 +60,7 @@ namespace BenchmarkDotNet.Tests.Engine
Run = { IterationTime = iterationTime }
}.Freeze();
var stage = CreateStage(job, data => data.InvokeCount * operationTime);
long invokeCount = stage.Run();
long invokeCount = stage.Run().PerfectInvocationCount;
output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {maxInvokeCount})");
Assert.InRange(invokeCount, minInvokeCount, maxInvokeCount);
}

2
tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs
Просмотреть файл

@ -31,7 +31,7 @@ namespace BenchmarkDotNet.Tests.Engine
[AssertionMethod]
private static void CheckResults(int expectedResultCount, List<Measurement> measurements, OutlierMode outlierMode)
{
Assert.Equal(expectedResultCount, new RunResults(null, measurements, outlierMode, default, default, 0).GetMeasurements().Count());
Assert.Equal(expectedResultCount, new RunResults(measurements, outlierMode, default, default, 0).GetWorkloadResultMeasurements().Count());
}
private static void Add(List<Measurement> measurements, int time)