Support Mono/LLVM as a runtime/jit, fixes #226

BenchmarkDotNet.Tests/ConfigParserTests.cs

@@ -54,8 +54,8 @@ namespace BenchmarkDotNet.Tests
             var parser = new ConfigParser();
             var config = parser.Parse(new[] { "jobs=all" });
 
-            // TODO How to make this robust, 11 is only valid when there are 11 items in "availableJobs" in ConfigParser.cs
-            Assert.Equal(11, config.GetJobs().Count());
+            // TODO How to make this robust, 12 is only valid when there are 12 items in "availableJobs" in ConfigParser.cs
+            Assert.Equal(12, config.GetJobs().Count());
 
             Assert.Equal(0, config.GetColumns().Count());
             Assert.Equal(0, config.GetExporters().Count());

BenchmarkDotNet.sln.DotSettings

@@ -1,2 +1,3 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=GC/@EntryIndexedValue">GC</s:String></wpf:ResourceDictionary>
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=GC/@EntryIndexedValue">GC</s:String>
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=LLVM/@EntryIndexedValue">LLVM</s:String></wpf:ResourceDictionary>

BenchmarkDotNet/Attributes/Jobs/MonoLLVMJobAttribute.cs

@@ -0,0 +1,11 @@
+using BenchmarkDotNet.Jobs;
+
+namespace BenchmarkDotNet.Attributes.Jobs
+{
+    public class MonoLLVMJobAttribute : JobConfigBaseAttribute
+    {
+        public MonoLLVMJobAttribute() : base(Job.MonoLLVM)
+        {
+        }
+    }
+}

BenchmarkDotNet/Configs/ConfigParser.cs

@@ -70,6 +70,7 @@ namespace BenchmarkDotNet.Configs
                 { "alljits", Job.AllJits },
                 { "clr", new[] { Job.Clr } },
                 { "mono", new[] { Job.Mono } },
+                { "monollvm", new[] { Job.MonoLLVM } },
                 { "longrun", new[] { Job.LongRun } }
             };
         private static Lazy<IJob[]> allJobs = new Lazy<IJob[]>(() => availableJobs.SelectMany(e => e.Value).ToArray());

BenchmarkDotNet/Jobs/Jit.cs

@@ -2,6 +2,12 @@
 {
     public enum Jit
     {
-        Host, LegacyJit, RyuJit
+        Host,
+        LegacyJit,
+        RyuJit,
+        /// <summary>
+        /// supported only for Mono
+        /// </summary>
+        LLVM
     }
 }

BenchmarkDotNet/Jobs/Job.cs

@@ -13,6 +13,7 @@ namespace BenchmarkDotNet.Jobs
         public static readonly IJob[] AllJits = { LegacyJitX86, LegacyJitX64, RyuJitX64 };
         public static readonly IJob Clr = new Job { Runtime = Runtime.Clr };
         public static readonly IJob Mono = new Job { Runtime = Runtime.Mono };
+        public static readonly IJob MonoLLVM = new Job { Runtime = Runtime.Mono, Jit = Jit.LLVM};
         public static readonly IJob Core = new Job { Runtime = Runtime.Core };
         public static readonly IJob MediumRun = new Job { LaunchCount = 3, WarmupCount = 15, TargetCount = 100 };
         public static readonly IJob LongRun = new Job { LaunchCount = 3, WarmupCount = 30, TargetCount = 1000 };

BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -112,6 +112,8 @@ namespace BenchmarkDotNet.Jobs
                 shortInfo = "Dry";
             if (job.Equals(Job.Mono))
                 shortInfo = "Mono";
+            if (job.Equals(Job.MonoLLVM))
+                shortInfo = "MonoLLVM";
             if (job.Equals(Job.Clr))
                 shortInfo = "Clr";
             if (job.Equals(Job.Core))

BenchmarkDotNet/Toolchains/Classic/ClassicExecutor.cs

@@ -2,6 +2,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Extensions;
@@ -102,7 +103,7 @@ namespace BenchmarkDotNet.Toolchains.Classic
                     break;
                 case Runtime.Mono:
                     start.FileName = "mono";
-                    start.Arguments = exeName + " " + args;
+                    start.Arguments = GetMonoArguments(benchmark.Job, exeName, args);
                     break;
                 default:
                     throw new NotSupportedException("Runtime = " + benchmark.Job.Runtime);
@@ -110,6 +111,18 @@ namespace BenchmarkDotNet.Toolchains.Classic
             return start;
         }
 
+        private string GetMonoArguments(IJob job, string exeName, string args)
+        {
+            // from mono --help: "Usage is: mono [options] program [program-options]"
+            return new StringBuilder(30)
+                .Append(job.Jit == Jit.LLVM ? "--llvm" : "--nollvm")
+                .Append(' ')
+                .Append(exeName)
+                .Append(' ')
+                .Append(args)
+                .ToString();
+        }
+
         private class ConsoleHandler
         {
             public ConsoleCancelEventHandler EventHandler { get; private set; }

BenchmarkDotNet/Toolchains/Core/CoreToolchain.cs

@@ -29,6 +29,11 @@ namespace BenchmarkDotNet.Toolchains.Core
 
         public override bool IsSupported(Benchmark benchmark, ILogger logger)
         {
+            if(!base.IsSupported(benchmark, logger))
+            {
+                return false;
+            }
+
             if (!HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled())
             {
                 logger.WriteLineError($"BenchmarkDotNet requires dotnet cli toolchain to be installed, benchmark {benchmark.ShortInfo} will not be executed");

BenchmarkDotNet/Toolchains/Toolchain.cs

@@ -24,7 +24,17 @@ namespace BenchmarkDotNet.Toolchains
             Executor = executor;
         }
 
-        public virtual bool IsSupported(Benchmark benchmark, ILogger logger) => true;
+        public virtual bool IsSupported(Benchmark benchmark, ILogger logger)
+        {
+            var runtime = benchmark.Job.Runtime == Runtime.Host ? RuntimeInformation.GetCurrent() : benchmark.Job.Runtime;
+            if (runtime != Runtime.Mono && benchmark.Job.Jit == Jit.LLVM)
+            {
+                logger.WriteLineError($"LLVM is supported only for Mono, benchmark {benchmark.ShortInfo} will not be executed");
+                return false;
+            }
+
+            return true;
+        }
 
         public override string ToString() => Name;