Add Baseline feature (#3)

2022-09-06 11:07:53 +02:00 · 2022-09-06 11:07:53 +02:00 · 84cfd68b7b
--- a/README.md
+++ b/README.md
@ -123,6 +123,18 @@ BenchmarkAttribute is used to specify which method should be invoked as benchmar

 Note, that benchmark method must be public and without parameters.

+#### Baseline
+
+BaselineAttribute is used to specify which method should be considered as baseline for calculation. Add new column "Ratio" to output.
+```text
+Console export: CompareObjectTypesBenchmark benchmark class.
+| MethodName                 | ItterationCount | Mean   | Ratio  | Min   | Max   |
+| ------------------------------------------------------------------------------ |
+| CompareByString            | 100             | 10 ms  | 1.0    | 10 ms | 10 ms |
+| CompareUsingTypeofIf       | 100             | 5.9 ms | 0.5900 | 0 ms  | 10 ms |
+| CompareUsingTypeofIfReturn | 100             | 5.5 ms | 0.5500 | 0 ms  | 10 ms |
+```
+
 ### Parsers

 You can specify parsers as attributes on class. Every parsers is invoked after benchmark run, so you can get results in multiple formats.  
--- a/nanoFramework.Benchmark.Sample/CompareObjectTypesBenchmark.cs
+++ b/nanoFramework.Benchmark.Sample/CompareObjectTypesBenchmark.cs
@ -27,6 +27,7 @@ namespace nanoFramework.Benchmark.Sample
                    };
        }

+        [Baseline]
        [Benchmark]
        public void CompareByString()
        {
--- a/nanoFramework.Benchmark.Sample/CreateRadomCollectionBenchmark.cs
+++ b/nanoFramework.Benchmark.Sample/CreateRadomCollectionBenchmark.cs
@ -6,7 +6,6 @@
 using System;
 using System.Collections;
 using nanoFramework.Benchmark.Attributes;
-using nanoFramework.Logging.Debug;

 namespace nanoFramework.Benchmark.Sample
 {
--- a/nanoFramework.Benchmark.Sample/nanoFramework.Benchmark.Sample.nfproj
+++ b/nanoFramework.Benchmark.Sample/nanoFramework.Benchmark.Sample.nfproj
@ -28,27 +28,9 @@
    <Reference Include="mscorlib">
      <HintPath>..\packages\nanoFramework.CoreLibrary.1.12.0\lib\mscorlib.dll</HintPath>
    </Reference>
-    <Reference Include="nanoFramework.Logging">
-      <HintPath>..\packages\nanoFramework.Logging.1.1.2\lib\nanoFramework.Logging.dll</HintPath>
-    </Reference>
-    <Reference Include="nanoFramework.Logging.Serial">
-      <HintPath>..\packages\nanoFramework.Logging.Serial.1.1.2\lib\nanoFramework.Logging.Serial.dll</HintPath>
-    </Reference>
-    <Reference Include="nanoFramework.Runtime.Events">
-      <HintPath>..\packages\nanoFramework.Runtime.Events.1.10.0\lib\nanoFramework.Runtime.Events.dll</HintPath>
-    </Reference>
    <Reference Include="nanoFramework.System.Collections">
      <HintPath>..\packages\nanoFramework.System.Collections.1.4.0\lib\nanoFramework.System.Collections.dll</HintPath>
    </Reference>
-    <Reference Include="nanoFramework.System.Text">
-      <HintPath>..\packages\nanoFramework.System.Text.1.2.3\lib\nanoFramework.System.Text.dll</HintPath>
-    </Reference>
-    <Reference Include="System.IO.Ports">
-      <HintPath>..\packages\nanoFramework.System.IO.Ports.1.1.6\lib\System.IO.Ports.dll</HintPath>
-    </Reference>
-    <Reference Include="System.IO.Streams">
-      <HintPath>..\packages\nanoFramework.System.IO.Streams.1.1.9\lib\System.IO.Streams.dll</HintPath>
-    </Reference>
  </ItemGroup>
  <ItemGroup>
    <None Include="packages.config" />
@ -56,6 +38,9 @@
  <ItemGroup>
    <ProjectReference Include="..\nanoFramework.Benchmark\nanoFramework.Benchmark.nfproj" />
  </ItemGroup>
+  <ItemGroup>
+    <Content Include="packages.lock.json" />
+  </ItemGroup>
  <Import Project="$(NanoFrameworkProjectSystemPath)NFProjectSystem.CSharp.targets" Condition="Exists('$(NanoFrameworkProjectSystemPath)NFProjectSystem.CSharp.targets')" />
  <ProjectExtensions>
    <ProjectCapabilities>
--- a/nanoFramework.Benchmark.Sample/packages.config
+++ b/nanoFramework.Benchmark.Sample/packages.config
@ -1,11 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
  <package id="nanoFramework.CoreLibrary" version="1.12.0" targetFramework="netnano1.0" />
-  <package id="nanoFramework.Logging" version="1.1.2" targetFramework="netnano1.0" />
-  <package id="nanoFramework.Logging.Serial" version="1.1.2" targetFramework="netnano1.0" />
-  <package id="nanoFramework.Runtime.Events" version="1.10.0" targetFramework="netnano1.0" />
  <package id="nanoFramework.System.Collections" version="1.4.0" targetFramework="netnano1.0" />
-  <package id="nanoFramework.System.IO.Ports" version="1.1.6" targetFramework="netnano1.0" />
-  <package id="nanoFramework.System.IO.Streams" version="1.1.9" targetFramework="netnano1.0" />
-  <package id="nanoFramework.System.Text" version="1.2.3" targetFramework="netnano1.0" />
 </packages>
--- a/nanoFramework.Benchmark.Sample/packages.lock.json
+++ b/nanoFramework.Benchmark.Sample/packages.lock.json
@ -8,47 +8,11 @@
        "resolved": "1.12.0",
        "contentHash": "qQrFNXmJiStMC4VXk5cVMOJp23/qlT9FW5i9i+igwQVwraQTtvpkam8yK1hj992jqrbjoCIFZP4Hw9E8H0pB7w=="
      },
-      "nanoFramework.Logging": {
-        "type": "Direct",
-        "requested": "[1.1.2, 1.1.2]",
-        "resolved": "1.1.2",
-        "contentHash": "3Sxz5BusVmnnlScEZaEtYCXGW3EW19lSaM3QWutwc2tsmZWLcSr6D0dCUvhEXbINqJEqpW9qI58Tje7aTCf9CQ=="
-      },
-      "nanoFramework.Logging.Serial": {
-        "type": "Direct",
-        "requested": "[1.1.2, 1.1.2]",
-        "resolved": "1.1.2",
-        "contentHash": "2OY7TwkHWnQ8JYszbLzdtO54ufEMAev04RcDNGycdPX8HQi7WWi0M+CpilMW9ePfKJovkLXhaLoee8Dp4f50qQ=="
-      },
-      "nanoFramework.Runtime.Events": {
-        "type": "Direct",
-        "requested": "[1.10.0, 1.10.0]",
-        "resolved": "1.10.0",
-        "contentHash": "ZRDjzVoTdc6+3sUNk9vORz4HHonYYGM4ED2V8VcfkYC8xQ8VFoaP8U6mXLd70ocmyhGaP9GK86sxHUQykSK7vA=="
-      },
      "nanoFramework.System.Collections": {
        "type": "Direct",
        "requested": "[1.4.0, 1.4.0]",
        "resolved": "1.4.0",
        "contentHash": "/yFwxtCFzi+24NuyxcwlH1YyBGOxRX4oHGLwVmFbgbvOyx3ny/Mwyk2YjHTzmTSgUg9C2XxPF+EkXWwCOAkytw=="
-      },
-      "nanoFramework.System.IO.Ports": {
-        "type": "Direct",
-        "requested": "[1.1.6, 1.1.6]",
-        "resolved": "1.1.6",
-        "contentHash": "M22BwMuT11q9RPQCqCwysrU+43Cd+cEgTLGMLN/dKq9IZcJVh9cUsZzu6U9I6jdn7L50/DbVpAedieTlTGNkkQ=="
-      },
-      "nanoFramework.System.IO.Streams": {
-        "type": "Direct",
-        "requested": "[1.1.9, 1.1.9]",
-        "resolved": "1.1.9",
-        "contentHash": "hWgCiA/9ju1pqFD7UZUHBBkLz+Wp7eKHHNU9z6Td6zhZxZZMxNhG3kHlz24pgHxBA7ntyN2MkSmHmYTMBsVWKw=="
-      },
-      "nanoFramework.System.Text": {
-        "type": "Direct",
-        "requested": "[1.2.3, 1.2.3]",
-        "resolved": "1.2.3",
-        "contentHash": "cQw6iVISN5Qdq0vnH3wLaCW/N1pFAImD6xA9Gsr8s0CZ28xkwkd8ZVj6nq9SJ/3k/kkZrxD5eFtfX0lEjXkT/w=="
      }
    }
  }
--- a/nanoFramework.Benchmark/Attributes/BaselineAttribute.cs
+++ b/nanoFramework.Benchmark/Attributes/BaselineAttribute.cs
@ -0,0 +1,12 @@
+using System;
+
+namespace nanoFramework.Benchmark.Attributes
+{
+    /// <summary>
+    /// Specifies method which should be considered as baseline for calculation. Add new column "Ratio" to output.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public sealed class BaselineAttribute : Attribute
+    {
+    }
+}
--- a/nanoFramework.Benchmark/BenchmarkMethodInfo.cs
+++ b/nanoFramework.Benchmark/BenchmarkMethodInfo.cs
@ -0,0 +1,18 @@
+using System;
+using System.Reflection;
+using System.Text;
+
+namespace nanoFramework.Benchmark
+{
+    internal sealed class BenchmarkMethodInfo
+    {
+        public MethodInfo MethodInfo { get; }
+        public bool IsBaseline { get; }
+
+        public BenchmarkMethodInfo(MethodInfo methodInfo, bool isBaseline)
+        {
+            MethodInfo = methodInfo;
+            IsBaseline = isBaseline;
+        }
+    }
+}
--- a/nanoFramework.Benchmark/BenchmarkRunner.cs
+++ b/nanoFramework.Benchmark/BenchmarkRunner.cs
@ -14,6 +14,9 @@ using System.Reflection;

 namespace nanoFramework.Benchmark
 {
+    /// <summary>
+    /// Main runner class.
+    /// </summary>
    public static class BenchmarkRunner
    {
        private static string[] ExcludedClassNamesFromBenchmarks =
@ -90,16 +93,19 @@ namespace nanoFramework.Benchmark
            setupMethod?.Invoke(classToInvokeMethodOn, null);

            var methodResults = new ArrayList();
-            foreach (MethodInfo method in ReflectionHelpers.GetBenchmarkMethods(allMethods, logger))
+            var allBenchmarkMethods = ReflectionHelpers.GetBenchmarkMethods(allMethods, logger);
+            foreach (BenchmarkMethodInfo method in allBenchmarkMethods)
            {
-                var result = Run(classToInvokeMethodOn, method, iterationCount);
-                methodResults.Add(new MethodResult(method.Name, result));
+                var result = Run(classToInvokeMethodOn, method.MethodInfo, iterationCount);
+                methodResults.Add(new MethodResult(method.MethodInfo.Name, result, method.IsBaseline));
            }

-            var benchmarkResult = new SingleBenchmarkResult(
-                ArrayListHelper.ConvertFromArrayListToMethodResultArray(methodResults), 
+            var methodResultsAsArray = ArrayListHelper.ConvertFromArrayListToMethodResultArray(methodResults);
+            var hasBaseline = ArrayHelper.FindBaseLine(methodResultsAsArray) != null;
+            var benchmarkResult = new SingleBenchmarkResult(methodResultsAsArray, 
                classType.Name, 
-                iterationCount);
+                iterationCount,
+                hasBaseline);

            InvokeParses(benchmarkResult, parsers, logger);
        }
--- a/nanoFramework.Benchmark/Helpers/ArrayHelper.cs
+++ b/nanoFramework.Benchmark/Helpers/ArrayHelper.cs
@ -0,0 +1,23 @@
+using nanoFramework.Benchmark.Result;
+using System;
+using System.Reflection;
+using System.Text;
+
+namespace nanoFramework.Benchmark.Helpers
+{
+    internal static class ArrayHelper
+    {
+        internal static MethodResult FindBaseLine(MethodResult[] methodResults)
+        {
+            foreach (var item in methodResults)
+            {
+                if (item.IsBaseline)
+                {
+                    return item;
+                }
+            }
+
+            return null;
+        }
+    }
+}
--- a/nanoFramework.Benchmark/Helpers/ArrayListHelper.cs
+++ b/nanoFramework.Benchmark/Helpers/ArrayListHelper.cs
@ -35,6 +35,18 @@ namespace nanoFramework.Benchmark.Helpers
            return result;
        }

+
+        internal static BenchmarkMethodInfo[] ConvertFromArrayListToBenchmarkMethodInfoArray(ArrayList arrayList)
+        {
+            var result = new BenchmarkMethodInfo[arrayList.Count];
+            for (int i = 0; i < arrayList.Count; i++)
+            {
+                result[i] = (BenchmarkMethodInfo)arrayList[i];
+            }
+
+            return result;
+        }
+
        internal static IResultParser[] ConvertFromArrayListToIResultParserArray(ArrayList arrayList)
        {
            var result = new IResultParser[arrayList.Count];
--- a/nanoFramework.Benchmark/Helpers/ReflectionHelpers.cs
+++ b/nanoFramework.Benchmark/Helpers/ReflectionHelpers.cs
@ -4,6 +4,7 @@
 ////

 using Microsoft.Extensions.Logging;
+using nanoFramework.Benchmark.Attributes;
 using System;
 using System.Collections;
 using System.Reflection;
@ -130,7 +131,7 @@ namespace nanoFramework.Benchmark.Helpers
            return false;
        }

-        internal static MethodInfo[] GetBenchmarkMethods(MethodInfo[] methodInfos, ILogger logger)
+        internal static BenchmarkMethodInfo[] GetBenchmarkMethods(MethodInfo[] methodInfos, ILogger logger)
        {
            var returnList = new ArrayList();
            foreach (var method in methodInfos)
@ -153,10 +154,23 @@ namespace nanoFramework.Benchmark.Helpers
                    continue;
                }

-                returnList.Add(method);
+                var isMethodBaseline = CheckIfMethodIsBenchmark(method);
+                var benchmarkMethodInfo = new BenchmarkMethodInfo(method, isMethodBaseline);
+                returnList.Add(benchmarkMethodInfo);
            }

-            return ArrayListHelper.ConvertFromArrayListToMethodInfoArray(returnList);
+            return ArrayListHelper.ConvertFromArrayListToBenchmarkMethodInfoArray(returnList);
+        }
+
+        private static bool CheckIfMethodIsBenchmark(MethodInfo method)
+        {
+            var baselineAttrib = GetFirstOrDefaultAttribute(method, typeof(BaselineAttribute));
+            if (baselineAttrib == null)
+            {
+                return false;
+            }
+
+            return true;
        }

        internal static object GetObjectInstanceFromPropertyIfExists(object classObject, Type typeToGet)
--- a/nanoFramework.Benchmark/Parser/Abstract/BaseParser.cs
+++ b/nanoFramework.Benchmark/Parser/Abstract/BaseParser.cs
@ -1,36 +0,0 @@
-////
-// Copyright (c) .NET Foundation and Contributors
-// See LICENSE file in the project root for full license information.
-////
-
-using nanoFramework.Benchmark.Helpers;
-using nanoFramework.Benchmark.Result.Attributes;
-using nanoFramework.Benchmark.Result;
-using System.Collections;
-using System.Reflection;
-
-namespace nanoFramework.Benchmark.Parser.Abstract
-{
-    internal abstract class BaseParser : IResultParser
-    {
-        public abstract void Parse(SingleBenchmarkResult benchmarkResult);
-
-        protected static MethodInfo[] GetMethodsToDisplay()
-        {
-            var tempList = new ArrayList();
-            var allMethods = typeof(SingleBenchmarkResult).GetMethods(BindingFlags.Public | BindingFlags.Instance);
-            foreach (var method in allMethods)
-            {
-                var displayAttrib = ReflectionHelpers.GetFirstOrDefaultAttribute(method, typeof(DisplayAttribute));
-                if (displayAttrib == null)
-                {
-                    continue;
-                }
-
-                tempList.Add(method);
-            }
-
-            return ArrayListHelper.ConvertFromArrayListToMethodInfoArray(tempList);
-        }
-    }
-}
--- a/nanoFramework.Benchmark/Parser/Abstract/BaseTextParser.cs
+++ b/nanoFramework.Benchmark/Parser/Abstract/BaseTextParser.cs
@ -12,7 +12,7 @@ using System.Text;

 namespace nanoFramework.Benchmark.Parser.Abstract
 {
-    internal abstract class BaseTextParser : BaseParser
+    internal abstract class BaseTextParser : IResultParser
    {
        protected abstract string GetParserName(SingleBenchmarkResult benchmarkResult);
        protected abstract string GetHeader(MethodInfo[] dataToDisplay);
@ -62,13 +62,13 @@ namespace nanoFramework.Benchmark.Parser.Abstract
            throw new InvalidOperationException();
        }

-        public override void Parse(SingleBenchmarkResult benchmarkResult)
+        public virtual void Parse(SingleBenchmarkResult benchmarkResult)
        {
            PrintLine(string.Empty);
            PrintLine(string.Empty);
            PrintLine(GetParserName(benchmarkResult));

-            var dataToDisplay = GetMethodsToDisplay();
+            var dataToDisplay = benchmarkResult.GetDataToDisplay();
            PrintLine(GetHeader(dataToDisplay));

            for (int i = 0; i < benchmarkResult.MethodResults.Length; i++)
--- a/nanoFramework.Benchmark/Parser/ConsoleParser.cs
+++ b/nanoFramework.Benchmark/Parser/ConsoleParser.cs
@ -26,7 +26,7 @@ namespace nanoFramework.Benchmark.Parser

        private int[] CalculateColumnSize(SingleBenchmarkResult benchmarkResult)
        {
-            var dataToDisplay = GetMethodsToDisplay();
+            var dataToDisplay = benchmarkResult.GetDataToDisplay();
            int[] columnSizes = new int[dataToDisplay.Length];
            for (int i = 0; i < dataToDisplay.Length; i++)
            {
--- a/nanoFramework.Benchmark/Result/MethodResult.cs
+++ b/nanoFramework.Benchmark/Result/MethodResult.cs
@ -13,13 +13,16 @@ namespace nanoFramework.Benchmark.Result
        private string _meanExecutionTime;
        private string _maxExecutionTime;
        private string _minExecutionTime;
+        private TimeSpan _meanExecutionTimeRaw;

        public SingleTestResult[] SingleTestResults { get; }
-        
-        public MethodResult(string methodName, SingleTestResult[] singleTestResults)
+        internal bool IsBaseline { get; }
+
+        public MethodResult(string methodName, SingleTestResult[] singleTestResults, bool isBaseline)
        {
            _methodName = methodName;
            SingleTestResults = singleTestResults;
+            IsBaseline = isBaseline;
        }

        internal string GetMethodName()
@ -28,6 +31,16 @@ namespace nanoFramework.Benchmark.Result
        }

        internal string GetMeanExecutionTime()
+        {
+            if (string.IsNullOrEmpty(_meanExecutionTime))
+            {
+                _meanExecutionTime = $"{GetMeanExecutionTimeRaw().TotalMilliseconds} ms";
+            }
+
+            return _meanExecutionTime;
+        }
+
+        internal TimeSpan GetMeanExecutionTimeRaw()
        {
            if (string.IsNullOrEmpty(_meanExecutionTime))
            {
@ -45,10 +58,10 @@ namespace nanoFramework.Benchmark.Result
                    elemetsCount++;
                }

-                _meanExecutionTime = $"{TimeSpan.FromTicks(sumTicks / elemetsCount).TotalMilliseconds} ms";
+                _meanExecutionTimeRaw = TimeSpan.FromTicks(sumTicks / elemetsCount);
            }

-            return _meanExecutionTime;
+            return _meanExecutionTimeRaw;
        }

        internal string GetMinExecutionTime()
--- a/nanoFramework.Benchmark/Result/SingleBenchmarkResult.cs
+++ b/nanoFramework.Benchmark/Result/SingleBenchmarkResult.cs
@ -4,6 +4,9 @@
 ////

 using nanoFramework.Benchmark.Result.Attributes;
+using nanoFramework.Benchmark.Helpers;
+using System.Collections;
+using System.Reflection;

 namespace nanoFramework.Benchmark.Result
 {
@ -12,12 +15,14 @@ namespace nanoFramework.Benchmark.Result
        public MethodResult[] MethodResults { get; }
        public string ClassName { get; }
        public int ItterationCount { get; }
+        public bool ShouldCalculateRatio { get; }

-        public SingleBenchmarkResult(MethodResult[] methodResults, string className, int itterationCount)
+        public SingleBenchmarkResult(MethodResult[] methodResults, string className, int itterationCount, bool shouldCalculateRatio)
        {
            MethodResults = methodResults;
            ClassName = className;
            ItterationCount = itterationCount;
+            ShouldCalculateRatio = shouldCalculateRatio;
        }

        [Display("MethodName")]
@ -38,6 +43,21 @@ namespace nanoFramework.Benchmark.Result
            return MethodResults[index].GetMeanExecutionTime();
        }

+        [Display("Ratio")]
+        public string Ratio(int index)
+        {
+            var isBaseline = MethodResults[index].IsBaseline;
+            if (isBaseline)
+            {
+                return "1.0";
+            }
+
+            var baseLineResult = ArrayHelper.FindBaseLine(MethodResults).GetMeanExecutionTimeRaw();
+            var currentResult = MethodResults[index].GetMeanExecutionTimeRaw();
+
+            return $"{((float)currentResult.Ticks / baseLineResult.Ticks):N4}";
+        }
+
        [Display("Min")]
        public string MinExecutionTime(int index)
        {
@ -49,5 +69,32 @@ namespace nanoFramework.Benchmark.Result
        {
            return MethodResults[index].GetMaxExecutionTime();
        }
+
+        /// <summary>
+        /// Gets methods with data based on configuration.
+        /// </summary>
+        /// <returns>All methods which will be used in parsers.</returns>
+        internal MethodInfo[] GetDataToDisplay()
+        {
+            var tempList = new ArrayList();
+            var allMethods = typeof(SingleBenchmarkResult).GetMethods(BindingFlags.Public | BindingFlags.Instance);
+            foreach (var method in allMethods)
+            {
+                var displayAttrib = ReflectionHelpers.GetFirstOrDefaultAttribute(method, typeof(DisplayAttribute));
+                if (displayAttrib == null)
+                {
+                    continue;
+                }
+
+                if (method.Name == nameof(Ratio) && !ShouldCalculateRatio)
+                {
+                    continue;
+                }
+
+                tempList.Add(method);
+            }
+
+            return ArrayListHelper.ConvertFromArrayListToMethodInfoArray(tempList);
+        }
    }
 }
--- a/nanoFramework.Benchmark/nanoFramework.Benchmark.nfproj
+++ b/nanoFramework.Benchmark/nanoFramework.Benchmark.nfproj
@ -31,17 +31,19 @@
  <Import Project="$(NanoFrameworkProjectSystemPath)NFProjectSystem.props" Condition="Exists('$(NanoFrameworkProjectSystemPath)NFProjectSystem.props')" />
  <ItemGroup>
    <Compile Include="Attributes\Abstract\ParserAttribute.cs" />
+    <Compile Include="Attributes\BaselineAttribute.cs" />
    <Compile Include="Attributes\CsvParserAttribute.cs" />
    <Compile Include="Attributes\ConsoleParserAttribute.cs" />
    <Compile Include="BenchmarkAttribute.cs" />
+    <Compile Include="BenchmarkMethodInfo.cs" />
    <Compile Include="BenchmarkRunner.cs" />
    <Compile Include="Attributes\DebugLoggerAttribute.cs" />
+    <Compile Include="Helpers\ArrayHelper.cs" />
    <Compile Include="Helpers\ArrayListHelper.cs" />
    <Compile Include="Helpers\SettingsHelper.cs" />
    <Compile Include="Helpers\StringArrayExtensions.cs" />
    <Compile Include="Attributes\IterationCountAttribute.cs" />
    <Compile Include="Attributes\Abstract\LoggerAttribute.cs" />
-    <Compile Include="Parser\Abstract\BaseParser.cs" />
    <Compile Include="Parser\Abstract\BaseTextParser.cs" />
    <Compile Include="Parser\ConsoleParser.cs" />
    <Compile Include="Parser\CsvParser.cs" />