Warmup improvements

BenchmarkDotNet.Samples/Intro/IntroUnstable.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Threading;
+using BenchmarkDotNet.Attributes;
+
+namespace BenchmarkDotNet.Samples.Intro
+{
+    public class IntroUnstable
+    {
+        private int counter = 0;
+
+        [Benchmark]
+        public void Foo()
+        {
+            Thread.Sleep(100 - (int) (Math.Log(++counter) * 10));
+        }
+    }
+}

BenchmarkDotNet/Mathematics/WelchTTest.cs

@@ -8,6 +8,7 @@ namespace BenchmarkDotNet.Mathematics
         public double T { get; }
         public double Df { get; }
         public double PValue { get; }
+        public bool NullHypothesisIsRejected => PValue < 0.05;
 
         public WelchTTest(double x, double df, double pValue)
         {

BenchmarkDotNet/Reports/Measurement.cs

@@ -10,8 +10,10 @@ namespace BenchmarkDotNet.Reports
     /// <summary>
     /// The basic captured statistics for a benchmark.
     /// </summary>
-    public sealed class Measurement
+    public struct Measurement
     {
+        private static readonly Measurement Error = new Measurement(-1, IterationMode.Unknown, 0, 0, 0);
+
         public IterationMode IterationMode { get; }
 
         public int LaunchIndex { get; }
@@ -100,7 +102,7 @@ namespace BenchmarkDotNet.Reports
             {
                 logger.WriteLineError("Parse error in the following line:");
                 logger.WriteLineError(line);
-                return null;
+                return Error;
             }
         }
 

BenchmarkDotNet/Running/BenchmarkRunner.cs

@@ -187,7 +187,7 @@ namespace BenchmarkDotNet.Running
                 for (int index = 0; index < executeResults.Count; index++)
                 {
                     var executeResult = executeResults[index];
-                    runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r != null));
+                    runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r.IterationMode != IterationMode.Unknown));
                 }
 
                 return new BenchmarkReport(benchmark, generateResult, buildResult, executeResults, runs);
@@ -259,7 +259,7 @@ namespace BenchmarkDotNet.Running
                 var measurements = executeResults
                         .SelectMany(r => r.Data)
                         .Select(line => Measurement.Parse(logger, line, 0))
-                        .Where(r => r != null).
+                        .Where(r => r.IterationMode != IterationMode.Unknown).
                         ToArray();
 
                 if (!measurements.Any())
@@ -285,7 +285,7 @@ namespace BenchmarkDotNet.Running
                 logger.WriteLineInfo($"// Run, Diagnostic");
                 config.GetCompositeDiagnoser().Start(benchmark);
                 var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, config.GetCompositeDiagnoser());
-                var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r != null).ToList();
+                var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList();
                 var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns);
                 config.GetCompositeDiagnoser().Stop(benchmark, report);
 

BenchmarkDotNet/Running/MethodInvoker.cs

@@ -147,17 +147,13 @@ namespace BenchmarkDotNet.Running
             if (iterationCount.IsAuto)
             {
                 int iterationCounter = 0;
-                Measurement previousMeasurement = null;
-                int upCount = 0;
+                var measurements = new List<Measurement>(50);
                 while (true)
                 {
                     iterationCounter++;
-                    var measurement = multiInvoke(new MultiInvokeInput(iterationMode, iterationCounter, invokeCount));
-                    if (previousMeasurement != null && measurement.Nanoseconds > previousMeasurement.Nanoseconds - 0.1)
-                        upCount++;
-                    if (iterationCounter >= WarmupAutoMinIterationCount && upCount >= 3)
+                    measurements.Add(multiInvoke(new MultiInvokeInput(iterationMode, iterationCounter, invokeCount)));
+                    if (IsWarmupFinished(measurements))
                         break;
-                    previousMeasurement = measurement;
                 }
             }
             else
@@ -168,6 +164,29 @@ namespace BenchmarkDotNet.Running
             Console.WriteLine();
         }
 
+        private static bool IsWarmupFinished(List<Measurement> measurements)
+        {
+            int n = measurements.Count;
+            if (n < 6 || n >= 50)
+                return false;
+            
+            int dir = -1, changeCount = 0;
+            for (int i = 1; i < n; i++)
+            {
+                var nextDir = Math.Sign(measurements[i].Nanoseconds - measurements[i - 1].Nanoseconds);
+                if (nextDir != dir)
+                {
+                    dir = nextDir;
+                    changeCount++;
+                }
+            }
+            if (changeCount < 4)
+                return false;
+
+            // TODO: add t-test
+            return true;
+        }
+
         private static IList<Measurement> RunTarget(Func<MultiInvokeInput, Measurement> multiInvoke, long invokeCount, IterationMode iterationMode, Count iterationCount)
         {
             var measurements = new List<Measurement>();
@@ -297,7 +316,7 @@ namespace BenchmarkDotNet.Running
 
         [MethodImpl(MethodImplOptions.NoInlining)]
         private void RunAction<T>(Func<T> action, int count, T returnHolder = default(T))
-        {            
+        {
             for (int i = 0; i < count; i++)
                 returnHolder = action();
             multiInvokeReturnHolder = returnHolder;