From e75bdd8507b4c3042091dd7aee158ae0238b8d4d Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Thu, 3 Nov 2022 09:55:02 +0400 Subject: [PATCH] Engine.Run() should return the full list of performed measurements (fixes #2187) (#2188) * Engine.Run() should return the full list of performed measurements (fixes #2187) * Code review fixes --- src/BenchmarkDotNet/Engines/Engine.cs | 21 +++-- .../Engines/EnginePilotStage.cs | 38 +++++++-- .../Engines/EngineWarmupStage.cs | 6 +- src/BenchmarkDotNet/Engines/RunResults.cs | 51 ++++++++---- .../Toolchains/Results/ExecuteResult.cs | 2 +- .../CustomEngineTests.cs | 7 +- .../EngineTests.cs | 78 +++++++++++++++++++ .../InProcessEmitTest.cs | 6 +- .../InProcessTest.cs | 11 ++- .../JobTests.cs | 40 ---------- .../Engine/EnginePilotStageTests.cs | 4 +- .../Engine/EngineResultStageTests.cs | 2 +- 12 files changed, 186 insertions(+), 80 deletions(-) create mode 100644 tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs delete mode 100644 tests/BenchmarkDotNet.IntegrationTests/JobTests.cs diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 708a126b9..cdf8adceb 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -41,6 +41,7 @@ namespace BenchmarkDotNet.Engines private bool EvaluateOverhead { get; } private bool MemoryRandomization { get; } + private readonly List jittingMeasurements = new (10); private readonly EnginePilotStage pilotStage; private readonly EngineWarmupStage warmupStage; private readonly EngineActualStage actualStage; @@ -104,8 +105,10 @@ namespace BenchmarkDotNet.Engines public RunResults Run() { + var measurements = new List(); + measurements.AddRange(jittingMeasurements); + long invokeCount = TargetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver, 1); - IReadOnlyList idle = null; if (EngineEventSource.Log.IsEnabled()) EngineEventSource.Log.BenchmarkStart(BenchmarkName); @@ -114,21 +117,23 @@ namespace BenchmarkDotNet.Engines { if (Strategy != RunStrategy.Monitoring) { - invokeCount = pilotStage.Run(); + var pilotStageResult = pilotStage.Run(); + invokeCount = pilotStageResult.PerfectInvocationCount; + measurements.AddRange(pilotStageResult.Measurements); if (EvaluateOverhead) { - warmupStage.RunOverhead(invokeCount, UnrollFactor); - idle = actualStage.RunOverhead(invokeCount, UnrollFactor); + measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor)); + measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor)); } } - warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy); + measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy)); } Host.BeforeMainRun(); - var main = actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring); + measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring)); Host.AfterMainRun(); @@ -141,7 +146,7 @@ namespace BenchmarkDotNet.Engines var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver); - return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats, exceptionFrequency); + return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency); } public Measurement RunIteration(IterationData data) @@ -183,6 +188,8 @@ namespace BenchmarkDotNet.Engines // Results var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds()); WriteLine(measurement.ToString()); + if (measurement.IterationStage == IterationStage.Jitting) + jittingMeasurements.Add(measurement); Consume(stackMemory); diff --git a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs index 9abda4077..1bf3e34d5 100644 --- a/src/BenchmarkDotNet/Engines/EnginePilotStage.cs +++ b/src/BenchmarkDotNet/Engines/EnginePilotStage.cs @@ -1,6 +1,9 @@ using System; +using System.Collections.Generic; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using JetBrains.Annotations; using Perfolizer.Horology; namespace BenchmarkDotNet.Engines @@ -8,6 +11,25 @@ namespace BenchmarkDotNet.Engines // TODO: use clockResolution internal class EnginePilotStage : EngineStage { + public readonly struct PilotStageResult + { + public long PerfectInvocationCount { get; } + [NotNull] + public IReadOnlyList Measurements { get; } + + public PilotStageResult(long perfectInvocationCount, [NotNull] List measurements) + { + PerfectInvocationCount = perfectInvocationCount; + Measurements = measurements; + } + + public PilotStageResult(long perfectInvocationCount) + { + PerfectInvocationCount = perfectInvocationCount; + Measurements = Array.Empty(); + } + } + internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2; private readonly int unrollFactor; @@ -30,11 +52,11 @@ namespace BenchmarkDotNet.Engines } /// Perfect invocation count - public long Run() + public PilotStageResult Run() { // If InvocationCount is specified, pilot stage should be skipped if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic)) - return TargetJob.Run.InvocationCount; + return new PilotStageResult(TargetJob.Run.InvocationCount); // Here we want to guess "perfect" amount of invocation return TargetJob.HasValue(RunMode.IterationTimeCharacteristic) @@ -45,15 +67,17 @@ namespace BenchmarkDotNet.Engines /// /// A case where we don't have specific iteration time. /// - private long RunAuto() + private PilotStageResult RunAuto() { long invokeCount = Autocorrect(minInvokeCount); + var measurements = new List(); int iterationCounter = 0; while (true) { iterationCounter++; var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); + measurements.Add(measurement); double iterationTime = measurement.Nanoseconds; double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision @@ -75,15 +99,16 @@ namespace BenchmarkDotNet.Engines } WriteLine(); - return invokeCount; + return new PilotStageResult(invokeCount, measurements); } /// /// A case where we have specific iteration time. /// - private long RunSpecific() + private PilotStageResult RunSpecific() { long invokeCount = Autocorrect(Engine.MinInvokeCount); + var measurements = new List(); int iterationCounter = 0; @@ -92,6 +117,7 @@ namespace BenchmarkDotNet.Engines { iterationCounter++; var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor); + measurements.Add(measurement); double actualIterationTime = measurement.Nanoseconds; long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long)Math.Round(invokeCount * targetIterationTime / actualIterationTime))); @@ -105,7 +131,7 @@ namespace BenchmarkDotNet.Engines } WriteLine(); - return invokeCount; + return new PilotStageResult(invokeCount, measurements); } private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor; diff --git a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs index 0a4d841aa..08ec7ec72 100644 --- a/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs +++ b/src/BenchmarkDotNet/Engines/EngineWarmupStage.cs @@ -9,13 +9,13 @@ namespace BenchmarkDotNet.Engines public EngineWarmupStage(IEngine engine) : base(engine) => this.engine = engine; - public void RunOverhead(long invokeCount, int unrollFactor) + public IReadOnlyList RunOverhead(long invokeCount, int unrollFactor) => Run(invokeCount, IterationMode.Overhead, unrollFactor, RunStrategy.Throughput); - public void RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy) + public IReadOnlyList RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy) => Run(invokeCount, IterationMode.Workload, unrollFactor, runStrategy); - internal List Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy) + internal IReadOnlyList Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy) { var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy); return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor); diff --git a/src/BenchmarkDotNet/Engines/RunResults.cs b/src/BenchmarkDotNet/Engines/RunResults.cs index 5a7651f46..ff4948870 100644 --- a/src/BenchmarkDotNet/Engines/RunResults.cs +++ b/src/BenchmarkDotNet/Engines/RunResults.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -13,11 +14,20 @@ namespace BenchmarkDotNet.Engines { private readonly OutlierMode outlierMode; + [NotNull, PublicAPI] + public IReadOnlyList EngineMeasurements { get; } + [CanBeNull, PublicAPI] - public IReadOnlyList Overhead { get; } + public IReadOnlyList Overhead + => EngineMeasurements + .Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)) + .ToArray(); [NotNull, PublicAPI] - public IReadOnlyList Workload { get; } + public IReadOnlyList Workload + => EngineMeasurements + .Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)) + .ToArray(); public GcStats GCStats { get; } @@ -25,27 +35,30 @@ namespace BenchmarkDotNet.Engines public double ExceptionFrequency { get; } - public RunResults([CanBeNull] IReadOnlyList overhead, - [NotNull] IReadOnlyList workload, - OutlierMode outlierMode, - GcStats gcStats, - ThreadingStats threadingStats, - double exceptionFrequency) + public RunResults([NotNull] IReadOnlyList engineMeasurements, + OutlierMode outlierMode, + GcStats gcStats, + ThreadingStats threadingStats, + double exceptionFrequency) { this.outlierMode = outlierMode; - Overhead = overhead; - Workload = workload; + EngineMeasurements = engineMeasurements; GCStats = gcStats; ThreadingStats = threadingStats; ExceptionFrequency = exceptionFrequency; } - public IEnumerable GetMeasurements() + public IEnumerable GetWorkloadResultMeasurements() { - double overhead = Overhead == null ? 0.0 : new Statistics(Overhead.Select(m => m.Nanoseconds)).Median; - var mainStats = new Statistics(Workload.Select(m => m.Nanoseconds)); + var overheadActualMeasurements = Overhead ?? Array.Empty(); + var workloadActualMeasurements = Workload; + if (workloadActualMeasurements.IsEmpty()) + yield break; + + double overhead = overheadActualMeasurements.IsEmpty() ? 0.0 : new Statistics(overheadActualMeasurements.Select(m => m.Nanoseconds)).Median; + var mainStats = new Statistics(workloadActualMeasurements.Select(m => m.Nanoseconds)); int resultIndex = 0; - foreach (var measurement in Workload) + foreach (var measurement in workloadActualMeasurements) { if (mainStats.IsActualOutlier(measurement.Nanoseconds, outlierMode)) continue; @@ -63,9 +76,17 @@ namespace BenchmarkDotNet.Engines } } + public IEnumerable GetAllMeasurements() + { + foreach (var measurement in EngineMeasurements) + yield return measurement; + foreach (var measurement in GetWorkloadResultMeasurements()) + yield return measurement; + } + public void Print(TextWriter outWriter) { - foreach (var measurement in GetMeasurements()) + foreach (var measurement in GetWorkloadResultMeasurements()) outWriter.WriteLine(measurement.ToString()); if (!GCStats.Equals(GcStats.Empty)) diff --git a/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs b/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs index dcd68000d..76d9c41cd 100644 --- a/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs +++ b/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs @@ -67,7 +67,7 @@ namespace BenchmarkDotNet.Toolchains.Results internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode) => exitCode != 0 ? CreateFailed(exitCode) - : new ExecuteResult(runResults.GetMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency); + : new ExecuteResult(runResults.GetAllMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency); internal static ExecuteResult CreateFailed(int exitCode = -1) => new ExecuteResult(false, exitCode, default, Array.Empty(), Array.Empty(), Array.Empty(), 0); diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index 5f423c259..43a7bac9e 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -72,8 +72,11 @@ namespace BenchmarkDotNet.IntegrationTests Console.WriteLine(EngineRunMessage); return new RunResults( - new List { new Measurement(1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1) }, - new List { new Measurement(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) }, + new List + { + new (1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1), + new (1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) + }, OutlierMode.DontRemove, default, default, diff --git a/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs new file mode 100644 index 000000000..624877f40 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs @@ -0,0 +1,78 @@ +using System.Linq; +using System.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class EngineTests : BenchmarkTestExecutor + { + public EngineTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void ZeroWarmupCountIsApplied() + { + var job = Job.InProcess + .WithEvaluateOverhead(false) + .WithWarmupCount(0) + .WithIterationCount(1) + .WithInvocationCount(1) + .WithUnrollFactor(1); + var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator); + var summary = CanExecute(config); + var report = summary.Reports.Single(); + int workloadWarmupCount = report.AllMeasurements + .Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup)); + Assert.Equal(0, workloadWarmupCount); + } + + [Fact] + public void AllMeasurementsArePerformedDefault() => AllMeasurementsArePerformed(Job.Default); + + [Fact] + public void AllMeasurementsArePerformedInProcess() => AllMeasurementsArePerformed(Job.InProcess); + + private void AllMeasurementsArePerformed(Job baseJob) + { + var job = baseJob + .WithWarmupCount(1) + .WithIterationCount(1) + .WithInvocationCount(1) + .WithUnrollFactor(1); + var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator); + var summary = CanExecute(config); + var measurements = summary.Reports.Single().AllMeasurements; + + Output.WriteLine("*** AllMeasurements ***"); + foreach (var measurement in measurements) + Output.WriteLine(measurement.ToString()); + Output.WriteLine("-----"); + + void Check(IterationMode mode, IterationStage stage) + { + int count = measurements.Count(m => m.Is(mode, stage)); + Output.WriteLine($"Count({mode}{stage}) = {count}"); + Assert.True(count > 0, $"AllMeasurements don't contain {mode}{stage}"); + } + + Check(IterationMode.Overhead, IterationStage.Jitting); + Check(IterationMode.Workload, IterationStage.Jitting); + Check(IterationMode.Overhead, IterationStage.Warmup); + Check(IterationMode.Overhead, IterationStage.Actual); + Check(IterationMode.Workload, IterationStage.Warmup); + Check(IterationMode.Workload, IterationStage.Actual); + Check(IterationMode.Workload, IterationStage.Result); + } + + public class FooBench + { + [Benchmark] + public void Foo() => Thread.Sleep(10); + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index 227e6411e..1d5619a27 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.IntegrationTests.InProcess.EmitTests; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -91,7 +92,10 @@ namespace BenchmarkDotNet.IntegrationTests Assert.DoesNotContain("No benchmarks found", logger.GetLog()); // Operations + GlobalSetup + GlobalCleanup - long expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2); + long expectedCount = summary.Reports + .SelectMany(r => r.AllMeasurements) + .Where(m => m.IterationStage != IterationStage.Result) + .Sum(m => m.Operations + 2); Assert.Equal(expectedCount, BenchmarkAllCases.Counter); } finally diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 6a2a0e2dd..a5de2462f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -213,7 +214,10 @@ namespace BenchmarkDotNet.IntegrationTests Assert.DoesNotContain("No benchmarks found", logger.GetLog()); // Operations + GlobalSetup + GlobalCleanup - var expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2); + long expectedCount = summary.Reports + .SelectMany(r => r.AllMeasurements) + .Where(m => m.IterationStage != IterationStage.Result) + .Sum(m => m.Operations + 2); Assert.Equal(expectedCount, BenchmarkAllCases.Counter); } finally @@ -239,7 +243,10 @@ namespace BenchmarkDotNet.IntegrationTests Assert.DoesNotContain("No benchmarks found", logger.GetLog()); // Operations + GlobalSetup + GlobalCleanup - var expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2); + long expectedCount = summary.Reports + .SelectMany(r => r.AllMeasurements) + .Where(m => m.IterationStage != IterationStage.Result) + .Sum(m => m.Operations + 2); Assert.Equal(expectedCount, BenchmarkAllCases.Counter); } finally diff --git a/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs b/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs deleted file mode 100644 index b3c8f1580..000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/JobTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System.Linq; -using System.Threading; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Reports; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class JobTests : BenchmarkTestExecutor - { - public JobTests(ITestOutputHelper output) : base(output) { } - - [Fact] - public void ZeroWarmupCountIsApplied() - { - var job = Job.Default - .WithEvaluateOverhead(false) - .WithWarmupCount(0) - .WithIterationCount(1) - .WithInvocationCount(1) - .WithUnrollFactor(1); - var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator); - var summary = CanExecute(config); - var report = summary.Reports.Single(); - int workloadWarmupCount = report.AllMeasurements - .Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup)); - Assert.Equal(0, workloadWarmupCount); - } - - public class ZeroWarmupBench - { - [Benchmark] - public void Foo() => Thread.Sleep(10); - } - } -} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs index 19d419f71..d5b7b8c65 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EnginePilotStageTests.cs @@ -47,7 +47,7 @@ namespace BenchmarkDotNet.Tests.Engine Accuracy = { MaxRelativeError = maxRelativeError } }.Freeze(); var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run(); + long invokeCount = stage.Run().PerfectInvocationCount; output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {MaxPossibleInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, MaxPossibleInvokeCount); } @@ -60,7 +60,7 @@ namespace BenchmarkDotNet.Tests.Engine Run = { IterationTime = iterationTime } }.Freeze(); var stage = CreateStage(job, data => data.InvokeCount * operationTime); - long invokeCount = stage.Run(); + long invokeCount = stage.Run().PerfectInvocationCount; output.WriteLine($"InvokeCount = {invokeCount} (Min= {minInvokeCount}, Max = {maxInvokeCount})"); Assert.InRange(invokeCount, minInvokeCount, maxInvokeCount); } diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs index 468ef8762..89c011e60 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs @@ -31,7 +31,7 @@ namespace BenchmarkDotNet.Tests.Engine [AssertionMethod] private static void CheckResults(int expectedResultCount, List measurements, OutlierMode outlierMode) { - Assert.Equal(expectedResultCount, new RunResults(null, measurements, outlierMode, default, default, 0).GetMeasurements().Count()); + Assert.Equal(expectedResultCount, new RunResults(measurements, outlierMode, default, default, 0).GetWorkloadResultMeasurements().Count()); } private static void Add(List measurements, int time)