Move compiler options test to separate net461 process (#310)

# problem
The Azure DevOps VSTest runner is not handling the Compiler Options test well, because it takes so long. Also the test takes a very long time to run.

# solution
Run different options in parallel, and in a separate process to finish reliably and in a reasonable time.

Infer.sln

@@ -104,6 +104,7 @@ Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "TestPython", "test\TestPyth
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{5759CC85-729E-479A-BF7E-8A0B8937B123}"
 	ProjectSection(SolutionItems) = preProject
+		build\all-compiler-options.yml = build\all-compiler-options.yml
 		build\common.props = build\common.props
 		build\copyassemblies.sh = build\copyassemblies.sh
 		build\evaluator-netcore.yml = build\evaluator-netcore.yml
@@ -124,6 +125,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{5759CC85
 		build\windows-msbuild.yml = build\windows-msbuild.yml
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestAllCompilerOptions", "test\TestAllCompilerOptions\TestAllCompilerOptions.csproj", "{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -544,6 +547,16 @@ Global
 		{79B39D08-DF06-47B8-B7EE-15E30EB9A209}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{79B39D08-DF06-47B8-B7EE-15E30EB9A209}.ReleaseCore|Any CPU.ActiveCfg = Release|Any CPU
 		{79B39D08-DF06-47B8-B7EE-15E30EB9A209}.ReleaseFull|Any CPU.ActiveCfg = Release|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.DebugCore|Any CPU.ActiveCfg = Debug|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.DebugFull|Any CPU.ActiveCfg = DebugFull|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.DebugFull|Any CPU.Build.0 = DebugFull|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.ReleaseCore|Any CPU.ActiveCfg = Release|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.ReleaseFull|Any CPU.ActiveCfg = ReleaseFull|Any CPU
+		{5A3A6D71-03FF-44CC-92CD-C0977B592F8E}.ReleaseFull|Any CPU.Build.0 = ReleaseFull|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

build/all-compiler-options.yml

@@ -0,0 +1,91 @@
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the MIT license.
+# See the LICENSE file in the project root for more information.
+
+# Run all-compiler-options-test.
+
+pool:
+  name: InferNet
+  demands:
+  - msbuild
+  - visualstudio
+  - vstest
+
+trigger: none # disable CI build
+
+variables:
+  Configuration: Release
+
+schedules:
+- cron: "30 23 * * 3"
+  branches:
+    include:
+    - master
+
+jobs:
+- job: Test
+  timeoutInMinutes: 40000
+  steps:
+  - checkout: self
+    clean: true
+    submodules: recursive
+  
+  - task: MSBuild@1
+    displayName: 'Restore solution Infer.sln'
+    inputs:
+      solution: Infer.sln
+      platform: 'Any CPU'
+      configuration: '$(Configuration)'
+      msbuildArguments: '/t:Restore'
+  
+  - task: VSBuild@1
+    displayName: 'Build solution Infer'
+    inputs:
+      solution: Infer.sln
+      platform: 'Any CPU'
+      configuration: '$(Configuration)'
+  
+  - script: |
+      test\TestAllCompilerOptions\bin\$(Configuration)\net461\TestAllCompilerOptions.exe
+    displayName: 'Run test'
+
+  # Clean up the process if the script timeout out.
+  - powershell: |
+      # ErrorAction is required because if the process is not found, gps logs
+      # an error.
+      # The test app outputs build variables in this format so that we can kill them all here.
+      dir "Env:OptionsBuild_${Env:BUILD_ID}_*" | % { $_.Value } | % { Get-Process -Id $_ } | kill
+    condition: always()
+  
+  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+    displayName: 'Component Detection'
+  
+  - powershell: |
+     # Each agent does its work in directories named after small unsigned numbers.
+     # These directories are not reliably cleaned up and over time the consumed
+     # space eventually grows without bound.
+     # Note that SilentlyContinue suppresses any deletion errors. This is because
+     # errors in deleting the directory currently in use are expected and there
+     # doesn't seem to be any way around it.
+     dir .\_work -Directory
+     dir .\_work -Directory |
+      ? { [UInt32]::TryParse($_.Name, [ref] 0) } |
+      % { del $_.FullName -Force -Recurse -ErrorAction SilentlyContinue }
+    errorActionPreference: continue
+    ignoreLASTEXITCODE: true
+    workingDirectory: '$(Agent.HomeDirectory)'
+    displayName: 'Clean-up agent working directories'
+    condition: always()
+  
+  - script: |
+     rem Deleting the tasks directory must be done by 'script' because
+     rem this is the only task that does not require code in the tasks
+     rem directory itself.
+     rem The reason to delete the tasks directory is that it is never cleaned
+     rem up otherwise, and each version of each task has a separate directory,
+     rem and some tasks are published daily -- so over time this directory
+     rem grows in size without bound.
+     del $(Agent.HomeDirectory)\_work\_tasks /S /F /Q
+    workingDirectory: '$(Agent.HomeDirectory)'
+    displayName: 'Clean-up agent tasks directory'
+    condition: always()

test/TestAllCompilerOptions/CompilerOptionsTestUtils.cs

@@ -0,0 +1,230 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace Microsoft.ML.Probabilistic.Tests.TestAllCompilerOptions
+{
+    using System;
+    using System.Collections.Concurrent;
+    using System.Collections.Generic;
+    using System.Diagnostics;
+    using System.IO;
+    using System.Linq;
+    using System.Reflection;
+    using System.Threading;
+    using System.Threading.Tasks;
+
+    using Xunit;
+
+    public static class CompilerOptionsTestUtils
+    {
+        public static async Task<int> RunProcessAsync(
+            string fileName,
+            string arguments,
+            string workingDirectory,
+            int instance,
+            CancellationToken cancellationToken)
+        {
+            using (var process = new Process
+            {
+                EnableRaisingEvents = true,
+                StartInfo = new ProcessStartInfo
+                {
+                    Arguments = arguments,
+                    CreateNoWindow = true,
+                    FileName = fileName,
+                    RedirectStandardError = true,
+                    RedirectStandardOutput = true,
+                    WorkingDirectory = workingDirectory,
+                    UseShellExecute = false,
+                },
+            })
+            {
+                process.ErrorDataReceived += (o, e) =>
+                {
+                    if (!string.IsNullOrWhiteSpace(e.Data))
+                    {
+                        Console.Error.WriteLine($"{instance} error: {e.Data}");
+                        Console.WriteLine($"{instance} error: {e.Data}");
+                    }
+                };
+
+                process.OutputDataReceived += (o, e) =>
+                {
+                    if (!string.IsNullOrWhiteSpace(e.Data))
+                    {
+                        Console.WriteLine($"{instance}: {e.Data}");
+                    }
+                };
+
+                var processExited = new TaskCompletionSource<int>();
+                process.Exited += (o, e) =>
+                    processExited.TrySetResult(process.ExitCode);
+
+                cancellationToken.Register(() =>
+                {
+                    try
+                    {
+                        process.Kill();
+                    }
+                    catch (InvalidOperationException)
+                    {
+                        // The process already exited.
+                    }
+
+                    // After the result has been set, the process object
+                    // is disposed of -- so it is important to use it
+                    // (in the call to Kill above) before setting the result
+                    // here.
+                    processExited.TrySetCanceled();
+                });
+
+                Console.WriteLine($"{instance}: {fileName} {process.StartInfo.Arguments}");
+
+                var startedProcess = process.Start();
+
+                // We set the ID so that we can stop the process if it is cancelled.
+                var buildId = Environment.GetEnvironmentVariable("BUILD_BUILDID");
+                Console.WriteLine($"##vso[task.setvariable variable=OptionsBuild_{buildId}_{instance}]{process.Id}");
+                Assert.True(startedProcess, $"Could not start: {fileName} {arguments}");
+
+                process.BeginOutputReadLine();
+                process.BeginErrorReadLine();
+
+                var exitCode = await processExited.Task.ConfigureAwait(false);
+
+                return exitCode;
+            }
+        }
+
+        private async static Task<Task<T>[]> ForEachParallelWithLimit<T>(int limit, Func<Task<T>>[] tasks)
+        {
+            var outputTaskSources = tasks.Select(_ => new TaskCompletionSource<T>()).ToArray();
+            var outputTasks = outputTaskSources.Select(task => task.Task).ToArray();
+            var semaphore = new SemaphoreSlim(limit, limit);
+            for (int i = 0; i < tasks.Length; i++)
+            {
+                var index = i;
+                new Thread(() =>
+                {
+                    semaphore.Wait();
+                    try
+                    {
+                        var pending = tasks[index]();
+                        try
+                        {
+                            outputTaskSources[index].SetResult(pending.Result);
+                        }
+                        catch (Exception e)
+                        {
+                            outputTaskSources[index].SetException(e);
+                        }
+                    }
+                    finally
+                    {
+                        semaphore.Release();
+                    }
+                }).Start();
+            }
+
+            await Task.WhenAll(outputTasks);
+            return outputTasks;
+        }
+
+        private static void CopyDirectory(string source, string destination)
+        {
+            var directoryInfo = new DirectoryInfo(source);
+            Directory.CreateDirectory(Path.Combine(destination, directoryInfo.Name));
+
+            foreach (var file in directoryInfo.GetFiles())
+            {
+                File.Copy(file.FullName, Path.Combine(destination, directoryInfo.Name, file.Name));
+            }
+
+            foreach (var directory in directoryInfo.GetDirectories())
+            {
+                CopyDirectory(directory.FullName, Path.Combine(destination, directoryInfo.Name));
+            }
+        }
+
+        public static async Task LaunchTestAllCompilerOptionsProcesses()
+        {
+            var assembly = Assembly.GetExecutingAssembly().Location;
+
+            var temporaryDirectories = new ConcurrentBag<string>();
+
+            // This cancellation token is used to kill left-over processes.
+            var cancellationTokenSource = new CancellationTokenSource();
+
+            var runners = new List<Func<Task<int>>>();
+
+            try
+            {
+                var processLimit = Environment.ProcessorCount / 2;
+                var semaphore = new Semaphore(processLimit, processLimit);
+                var instance = 0;
+                foreach (var loops in new[] { true, false })
+                {
+                    foreach (var free in new[] { true, false })
+                    {
+                        foreach (var copies in new[] { true, false })
+                        {
+                            foreach (var optimise in new[] { true, false })
+                            {
+                                var instanceIndex = instance++;
+                                runners.Add(() =>
+                                    Task.Run(async () =>
+                                    {
+                                        var temp = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+                                        temporaryDirectories.Add(temp);
+                                        Directory.CreateDirectory(temp);
+
+                                        var dataDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Data");
+                                        CopyDirectory(dataDirectory, temp);
+
+                                        return await RunProcessAsync(
+                                            assembly,
+                                            $"\"{temp}\" {loops} {free} {copies} {optimise}",
+                                            temp,
+                                            instanceIndex,
+                                            cancellationTokenSource.Token);
+                                    }));
+                            }
+                        }
+                    }
+                }
+
+                var results = await ForEachParallelWithLimit(processLimit, runners.ToArray());
+
+                foreach (var result in results)
+                {
+                    Assert.True(result.IsCompleted, "A process has not completed.");
+                    Assert.True(result.Result == 0, "A process has failed.");
+                }
+            }
+            finally
+            {
+                cancellationTokenSource.Cancel();
+
+                // Wait a few seconds for processes to exit
+                // before deleting working directories.
+                await Task.Delay(3000);
+
+                foreach (var item in temporaryDirectories)
+                {
+                    if (Directory.Exists(item))
+                    {
+                        try
+                        {
+                            Directory.Delete(item, recursive: true);
+                        }
+                        catch (Exception e)
+                        {
+                            Console.WriteLine($"Could not delete temporary directory: {e}");
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

test/TestAllCompilerOptions/Program.cs

@@ -0,0 +1,55 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace Microsoft.ML.Probabilistic.Tests.TestAllCompilerOptions
+{
+    using System;
+    using System.Diagnostics;
+    using System.IO;
+    using System.Threading.Tasks;
+    using Xunit;
+
+    class Program
+    {
+        static async Task<int> Main(string[] args)
+        {
+            try
+            {
+                // This is the process that launches and monitors other processes.
+                if (args.Length == 0)
+                {
+                    // Output the process ID so that the process can be cancelled.
+                    Console.WriteLine("Creating processes...");
+                    var buildId = Environment.GetEnvironmentVariable("BUILD_BUILDID");
+                    var processId = Process.GetCurrentProcess().Id;
+                    Console.WriteLine($"##vso[task.setvariable variable=OptionsBuild_{buildId}_Host]{processId}");
+
+                    await CompilerOptionsTestUtils.LaunchTestAllCompilerOptionsProcesses();
+                    return 0;
+                }
+
+                // This is a worker process.
+                var workingDirectory = args[0];
+                var loops = bool.Parse(args[1]);
+                var free = bool.Parse(args[2]);
+                var copies = bool.Parse(args[3]);
+                var optimise = bool.Parse(args[4]);
+                var result = TestUtils.TestAllCompilerOptions(
+                    workingDirectory,
+                    loadTestAssembly: true,
+                    loops: loops,
+                    free: free,
+                    copies: copies,
+                    optimise: optimise);
+                Assert.True(result == 0, "There was a failed test.");
+                return 0;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine(e);
+                return 1;
+            }
+        }
+    }
+}

test/TestAllCompilerOptions/TestAllCompilerOptions.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="$(MSBuildThisFileDirectory)..\..\build\common.props" />
+
+  <PropertyGroup>
+    <Configurations>Debug;Release;DebugFull;ReleaseFull</Configurations>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net461</TargetFrameworks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Tests\Tests.csproj" />
+  </ItemGroup>
+</Project>

test/TestApp/Program.cs

@@ -92,7 +92,7 @@ namespace TestApp
                 InferenceEngine.DefaultEngine.Compiler.RecommendedQuality = QualityBand.Preview;
                 InferenceEngine.DefaultEngine.Compiler.GenerateInMemory = true;
                 InferenceEngine.DefaultEngine.Compiler.WriteSourceFiles = false;
-                TestUtils.TestAllCompilerOptions(true);
+                TestUtils.TestAllCompilerOptions();
                 //TestUtils.TestAllCompilerOptions(path);
                 //TestUtils.RunAllTests(path);
                 //using(TextWriter writer = new StreamWriter(@"..\..\RunAllTests.cs")) {

test/Tests/CompilerOptionsTest.cs

@@ -1,20 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-// See the LICENSE file in the project root for more information.
-
-using Xunit;
-
-namespace Microsoft.ML.Probabilistic.Tests
-{
-    
-    public class CompilerOptionsTest
-    {
-        [Fact]
-        [Trait("Category", "CompilerOptionsTest")]
-        public void TestAllCompilerOptions()
-        {
-            var failed = TestUtils.TestAllCompilerOptions();
-            Assert.Equal(0, failed);
-        }
-    }
-}

test/Tests/TestUtils.cs

@@ -26,6 +26,7 @@ namespace Microsoft.ML.Probabilistic.Tests
     using Xunit.Sdk;
     using Microsoft.ML.Probabilistic.Models;
     using Microsoft.ML.Probabilistic.Factors;
+    using System.Threading;
 
     public static class TestUtils
     {
@@ -223,33 +224,31 @@ namespace Microsoft.ML.Probabilistic.Tests
             Console.WriteLine($"mscorlib version {coreAssemblyInfo.ProductVersion}");
         }
 
-        public static int TestAllCompilerOptions(bool loadTestAssembly = false)
+        public static int TestAllCompilerOptions(
+            string workingDirectory,
+            bool loadTestAssembly,
+            bool loops,
+            bool free,
+            bool copies,
+            bool optimise,
+            string path = null)
         {
+            Directory.SetCurrentDirectory(workingDirectory);
+
             var assembly = Assembly.GetCallingAssembly();
             if (loadTestAssembly)
                 assembly = Assembly.Load("Microsoft.ML.Probabilistic.Tests");
-            var tests = FindTestMethods(assembly);
-
-            ////tests = tests.Where(m => m.Name.Contains("ChainWithTransitionParameterTest3")).ToArray();
+            var tests = TestUtils.FindTestMethods(assembly, path);
 
             Console.WriteLine($"Running {tests.Length} tests with all compiler options");
-            WriteEnvironment();
+            TestUtils.WriteEnvironment();
 
             TestUtils.SetBrowserMode(BrowserMode.Never);
-            var stdout = Console.Out;
-#if MONO_SUPPORT
-            Console.SetOut(NullWriter.Instance); 
-#else
-            Console.SetOut(StreamWriter.Null);
-#endif
-            var failed = TestAllCompilerOptions(state =>
-            {
-                Stopwatch watch = Stopwatch.StartNew();
-                var list = RunAllTests(tests).Select(m => new Tuple<string, MethodInfo>(state, m)).ToList();
-                Trace.WriteLine($"{state} completed in {watch.Elapsed}");
-                return list;
-            });
-            Console.SetOut(stdout);
+            var failed = RunTests(tests,
+                loops: loops,
+                free: free,
+                copies: copies,
+                optimise: optimise);
 
             Console.WriteLine("Executed {0} tests", tests.Length);
             foreach (var g in failed.GroupBy(t => t.Item2))
@@ -263,20 +262,13 @@ namespace Microsoft.ML.Probabilistic.Tests
             return failed.Count();
         }
 
-        private class TestRunner : MarshalByRefObject
+        public static MethodInfo[] FindTestMethods(
+            Assembly assembly,
+            string filter = null,
+            string path = null)
         {
-            public bool RunTest(MethodInfo mi)
-            {
-                Console.SetOut(StreamWriter.Null);
-                InferenceEngine.DefaultEngine.Compiler.GeneratedSourceFolder = "GeneratedSource" + AppDomain.CurrentDomain.Id;
-                InferenceEngine.DefaultEngine.Compiler.WriteSourceFiles = false;
-                return !RunAllTests(new[] { mi }).Any();
-            }
-        }
+            var methods = GetTests(assembly, path);
 
-        internal static MethodInfo[] FindTestMethods(Assembly assembly, string filter = null)
-        {
-            var methods = assembly.GetTypes().SelectMany(t => t.GetMethods()).ToArray();
             var excludeCategories = new[] { "BadTest", "OpenBug", "CompilerOptionsTest", "x86" };
             var testMethods = methods.Where(method =>
             {
@@ -290,43 +282,47 @@ namespace Microsoft.ML.Probabilistic.Tests
             return testMethods;
         }
 
-        public static void TestAllCompilerOptions(string path)
+        public static List<Tuple<string, MethodInfo>> RunTests(
+            MethodInfo[] tests,
+            bool loops,
+            bool free,
+            bool copies,
+            bool optimise)
         {
-            TestAllCompilerOptions<string>(state =>
-            {
-                Console.WriteLine(state);
-                RunAllTests(path);
-                return null;
-            });
-        }
+            InferenceEngine.DefaultEngine.Compiler.UseParallelForLoops = loops;
+            InferenceEngine.DefaultEngine.Compiler.FreeMemory = free;
+            InferenceEngine.DefaultEngine.Compiler.ReturnCopies = copies;
+            InferenceEngine.DefaultEngine.Compiler.OptimiseInferenceCode = optimise;
 
-        private static List<T> TestAllCompilerOptions<T>(Func<string, IEnumerable<T>> operation)
-        {
-            var output = new List<T>();
-            foreach (var loops in new[] { true, false })
+            var testsDone = 0;
+            var totalTests = tests.Length;
+
+            var startTime = DateTime.UtcNow;
+            var lastTime = DateTime.UtcNow;
+
+            var stdout = Console.Out;
+            try
             {
-                InferenceEngine.DefaultEngine.Compiler.UseParallelForLoops = loops;
-                foreach (var free in new[] { true, false })
+                Console.SetOut(StreamWriter.Null);
+
+                void TestFinished(string name, TimeSpan testDuration)
                 {
-                    InferenceEngine.DefaultEngine.Compiler.FreeMemory = free;
-                    foreach (var copies in new[] { true, false })
-                    {
-                        InferenceEngine.DefaultEngine.Compiler.ReturnCopies = copies;
-                        foreach (var optimise in new[] { true, false })
-                        {
-                            InferenceEngine.DefaultEngine.Compiler.OptimiseInferenceCode = optimise;
-                            var state = string.Format("UseParallelForLoops={0} FreeMemory={1} ReturnCopies={2} OptimiseInferenceCode={3}", loops, free, copies, optimise);
-                            output.AddRange(operation(state));
-                        }
-                    }
+                    Interlocked.Increment(ref testsDone);
+                    stdout.WriteLine($"{100.0 * testsDone / totalTests}% Elapsed: {DateTime.UtcNow.Subtract(startTime)} Name: {name} Duration: {DateTime.UtcNow.Subtract(lastTime)}");
+                    lastTime = DateTime.UtcNow;
                 }
+
+                return RunAllTests(TestFinished, tests, runInParallel: false).Select(m => new Tuple<string, MethodInfo>($"UseParallelForLoops={loops} FreeMemory={free} ReturnCopies={copies} OptimiseInferenceCode={optimise}: {m.error}", m.test)).ToList();
+            }
+            finally
+            {
+                Console.SetOut(stdout);
             }
-            return output;
         }
 
-        public static void RunAllTests(string path)
+        public static void RunAllTests(Action<string, TimeSpan> testFinished, string path)
         {
-            List<string> testNames = ReadTestNames(path);
+            List<string> testNames = TestUtils.ReadTestNames(path);
             var testMethods = testNames.Select(name =>
             {
                 int commaPos = name.IndexOf(',');
@@ -337,15 +333,15 @@ namespace Microsoft.ML.Probabilistic.Tests
                 MethodInfo method = type.GetMethod(methodName, Type.EmptyTypes);
                 return method;
             }).ToArray();
-            RunAllTests(testMethods);
+            RunAllTests(testFinished, testMethods, runInParallel: false);
         }
 
-        private static IEnumerable<MethodInfo> RunAllTests(MethodInfo[] tests)
+        private static IEnumerable<(MethodInfo test, Exception error)> RunAllTests(Action<string, TimeSpan> testFinished, MethodInfo[] tests, bool runInParallel)
         {
-            var failed = new ConcurrentQueue<MethodInfo>();
+            var failed = new ConcurrentQueue<(MethodInfo, Exception)>();
             var safeTests = new ConcurrentQueue<MethodInfo>();
             var unsafeTests = new ConcurrentQueue<MethodInfo>();
-            Parallel.ForEach(tests, test =>
+            ForEach(tests, test =>
             {
                 var testCategories = TraitHelper.GetTraits(test)
                     .Where(trait => trait.Key == "Category")
@@ -356,33 +352,59 @@ namespace Microsoft.ML.Probabilistic.Tests
                 else safeTests.Enqueue(test);
             });
             // unsafe tests must run sequentially.
-            Trace.WriteLine($"Running {unsafeTests.Count} tests sequentially");
+            Trace.WriteLine($"Running {unsafeTests.Count} unsafe tests sequentially");
             foreach (var test in unsafeTests)
             {
+                var sw = Stopwatch.StartNew();
                 RunTest(test, failed);
+                testFinished(test.Name, sw.Elapsed);
             }
             var safeTestsArray = safeTests.ToArray();
             Array.Sort(safeTestsArray, (a, b) => a.Name.CompareTo(b.Name));
-            Trace.WriteLine($"Running {safeTests.Count} tests in parallel");
-            try
+            if (runInParallel)
             {
-                Parallel.ForEach(safeTestsArray, test =>
+                Trace.WriteLine($"Running {safeTests.Count} safe tests in parallel");
+                try
                 {
-                    RunTest(test, failed);
-                });
+                    Parallel.ForEach(safeTestsArray, test =>
+                    {
+                        var sw = Stopwatch.StartNew();
+                        RunTest(test, failed);
+                        testFinished(test.Name, sw.Elapsed);
+                    });
+                }
+                catch (AggregateException ex)
+                {
+                    // To make the Visual Studio debugger stop at the inner exception, check "Enable Just My Code" in Debug->Options.
+                    // throw InnerException while preserving stack trace
+                    // https://stackoverflow.com/questions/57383/in-c-how-can-i-rethrow-innerexception-without-losing-stack-trace
+                    System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(ex.InnerException).Throw();
+                    throw;
+                }
             }
-            catch (AggregateException ex)
+            else
             {
-                // To make the Visual Studio debugger stop at the inner exception, check "Enable Just My Code" in Debug->Options.
-                // throw InnerException while preserving stack trace
-                // https://stackoverflow.com/questions/57383/in-c-how-can-i-rethrow-innerexception-without-losing-stack-trace
-                System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(ex.InnerException).Throw();
-                throw;
+                Trace.WriteLine($"Running {unsafeTests.Count} safe tests sequentially");
+                foreach (var test in safeTestsArray)
+                {
+                    var sw = Stopwatch.StartNew();
+                    RunTest(test, failed);
+                    testFinished(test.Name, sw.Elapsed);
+                }
             }
+
             return failed;
         }
 
-        private static void RunTest(MethodInfo test, ConcurrentQueue<MethodInfo> failed)
+        private static void ForEach<T>(IEnumerable<T> tests, Action<T> action)
+        {
+            foreach (var item in tests)
+            {
+                action(item);
+            }
+        }
+
+        private static void RunTest(MethodInfo test, ConcurrentQueue<(MethodInfo, Exception)> failed)
         {
             object obj = Activator.CreateInstance(test.DeclaringType);
             BindingFlags flags = BindingFlags.Public | BindingFlags.Instance | BindingFlags.InvokeMethod;
@@ -392,12 +414,65 @@ namespace Microsoft.ML.Probabilistic.Tests
             }
             catch (Exception ex)
             {
-                Trace.WriteLine(test);
-                Trace.WriteLine(ex);
-                failed.Enqueue(test);
+                failed.Enqueue((test, ex));
             }
         }
 
+        private static MethodInfo[] GetTests(
+            Assembly assembly,
+            string path = null)
+        {
+            if (path == null)
+            {
+                return assembly.GetTypes().SelectMany(t => t.GetMethods()).ToArray();
+            }
+
+            List<string> testNames = ReadTestNames(path);
+            return testNames.Select(name =>
+            {
+                int commaPos = name.IndexOf(',');
+                string methodName = name.Substring(0, commaPos);
+                string className = name.Substring(commaPos + 1);
+                Console.WriteLine(name);
+                Type type = Type.GetType(className, true);
+                MethodInfo method = type.GetMethod(methodName, Type.EmptyTypes);
+                return method;
+            }).ToArray();
+        }
+
+        public static int TestAllCompilerOptions(string path = null)
+        {
+            var workingDirectory = Directory.GetCurrentDirectory();
+
+            var failures = 0;
+            var loadTestAssembly = true;
+            foreach (var loops in new[] { false, true })
+            {
+                foreach (var free in new[] { false, true })
+                {
+                    foreach (var copies in new[] { false, true })
+                    {
+                        foreach (var optimise in new[] { false, true })
+                        {
+                            failures += TestAllCompilerOptions(
+                                workingDirectory,
+                                loadTestAssembly: loadTestAssembly,
+                                loops: loops,
+                                free: free,
+                                copies: copies,
+                                optimise: optimise,
+                                path: path);
+
+                            // Only do this the first time.
+                            loadTestAssembly = false;
+                        }
+                    }
+                }
+            }
+
+            return failures;
+        }
+
         public static void WriteCodeToRunAllTests(TextWriter writer, string path)
         {
             writer.WriteLine("InferenceEngine.DefaultEngine.BrowserMode = BrowserMode.Never;");