Adding BenchmarkDb (#9)

2017-11-13 11:48:29 -08:00 · 2017-11-13 11:48:29 -08:00 · 099b7874c9
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,286 @@
+# Custom
+.build/
+.vscode/
+
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
+
+# User-specific files
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+
+# Visual Studio 2015 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUNIT
+*.VisualState.xml
+TestResult.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# .NET Core
+global.json
+project.lock.json
+project.fragment.lock.json
+artifacts/
+**/Properties/launchSettings.json
+
+*_i.c
+*_p.c
+*_i.h
+*.ilk
+*.meta
+*.obj
+*.pch
+*.pdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# JustCode is a .NET coding add-in
+.JustCode
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# TODO: Comment the next line if you want to checkin your web deploy settings
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# The packages folder can be ignored because of Package Restore
+**/packages/*
+# except build/, which is used as an MSBuild target.
+!**/packages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/packages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+
+# SQL Server files
+*.mdf
+*.ldf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Typescript v1 declaration files
+typings/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# JetBrains Rider
+.idea/
+*.sln.iml
+
+# CodeRush
+.cr/
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+korebuild-lock.txt
+/src/BenchmarkDb/results.csv
+/src/BenchmarkDb/results.md
--- a/src/BenchmarkDb/BenchmarkDb.csproj
+++ b/src/BenchmarkDb/BenchmarkDb.csproj
@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="MySqlConnector" Version="0.28.2" />
+    <PackageReference Include="Npgsql" Version="3.2.5" />
+    <PackageReference Include="System.Data.SqlClient" Version="4.4.0" />
+  </ItemGroup>
+
+</Project>
--- a/src/BenchmarkDb/Fortune.cs
+++ b/src/BenchmarkDb/Fortune.cs
@ -0,0 +1,8 @@
+namespace BenchmarkDb
+{
+    public class Fortune
+    {
+        public int Id { get; set; }
+        public string Message { get; set; }
+    }
+}
--- a/src/BenchmarkDb/Program.cs
+++ b/src/BenchmarkDb/Program.cs
@ -0,0 +1,364 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Common;
+using System.Data.SqlClient;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MySql.Data.MySqlClient;
+using Npgsql;
+
+namespace BenchmarkDb
+{
+    public class Program
+    {
+        static volatile int _counter = 0;
+        static volatile int _stopping = 0;
+
+        const int Warmup = 3;
+        const string SqlQuery = "SELECT id,message FROM fortune";
+        const string PostgreSql = nameof(PostgreSql);
+        const string MySql = nameof(MySql);
+        const string SqlServer = nameof(SqlServer);
+
+        static object synlock = new object();
+
+        public static async Task Main(string[] args)
+        {
+            if (args.Length < 3)
+            {
+                Console.WriteLine("usage: database connectionstring threads mode time(s) desc");
+                Environment.Exit(1);
+            }
+
+            var database = args[0];
+            var connectionString = args[1];
+
+            if (!int.TryParse(args[2], out int numThreads))
+            {
+                Console.WriteLine($"Invalid threads value: {args[2]}");
+                Environment.Exit(1);
+            }
+
+            string mode = args[3];
+
+            if (mode != "async" && mode != "sync")
+            {
+                Console.WriteLine("Accepted mode values: sync, async, sync+conn, async+conn");
+                Environment.Exit(1);
+            }
+
+            if (!int.TryParse(args[4], out int time))
+            {
+                Console.WriteLine($"Invalid time value: {args[4]}");
+                Environment.Exit(1);
+            }
+
+            DbProviderFactory factory = null;
+
+            switch (database)
+            {
+                case PostgreSql:
+                    factory = NpgsqlFactory.Instance;
+                    break;
+
+                case MySql:
+                    factory = MySqlClientFactory.Instance;
+                    break;
+
+                case SqlServer:
+                    factory = SqlClientFactory.Instance;
+                    break;
+
+                default:
+                    Console.WriteLine($"Accepted database values: {SqlServer}, {MySql}, {PostgreSql}");
+                    Environment.Exit(2);
+                    break;
+            }
+
+            var stopwatch = new Stopwatch();
+            var startTime = DateTime.UtcNow;
+            var stopTime = DateTime.UtcNow;
+            var lastDisplay = DateTime.UtcNow;
+            var totalTransactions = 0;
+            var listOfResults = new List<double>();
+
+            List<Task> tasks = null;
+            switch (mode)
+            {
+                case "sync":
+                    tasks = Enumerable.Range(1, numThreads).Select(_ => Task.Factory.StartNew(DoWorkSync, TaskCreationOptions.LongRunning)).ToList();
+                    break;
+
+                case "async":
+                    tasks = Enumerable.Range(1, numThreads).Select(_ => Task.Factory.StartNew(DoWorkAsync, TaskCreationOptions.LongRunning).Unwrap()).ToList();
+                    break;
+
+                case "sync+conn":
+                    tasks = Enumerable.Range(1, numThreads).Select(_ => Task.Factory.StartNew(DoWorkSyncReuseConnection, TaskCreationOptions.LongRunning)).ToList();
+                    break;
+
+                case "async+conn":
+                    tasks = Enumerable.Range(1, numThreads).Select(_ => Task.Factory.StartNew(DoWorkAsyncReuseConnection, TaskCreationOptions.LongRunning).Unwrap()).ToList();
+                    break;
+            }
+
+            // Displays and records the current status of the test
+            tasks.Add(
+                Task.Run(async () =>
+                {
+                    // Warmup
+                    Console.Write($"Warming up... {Warmup}s");
+                    await Task.Delay(TimeSpan.FromSeconds(Warmup));
+                    Console.SetCursorPosition(0, Console.CursorTop);
+
+                    startTime = DateTime.UtcNow;
+                    Interlocked.Exchange(ref _counter, 0);
+
+                    while (_stopping != 1)
+                    {
+                        await Task.Delay(200);
+                        var now = DateTime.UtcNow;
+                        var tps = (int) (_counter / (now - lastDisplay).TotalSeconds);
+                        var remaining = (int) (time - (now - startTime).TotalSeconds);
+                        listOfResults.Add(tps);
+
+                        Console.Write($"{tps} tps, {remaining}s                     ");
+                        Console.SetCursorPosition(0, Console.CursorTop);
+                        lastDisplay = now;
+                        totalTransactions += Interlocked.Exchange(ref _counter, 0);
+                    }
+                })
+            );
+
+            // Signals the beginning and the end of the test
+            tasks.Add(
+                Task.Run(async () =>
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(time + Warmup));
+
+                    Interlocked.Exchange(ref _stopping, 1);
+                    stopTime = DateTime.UtcNow;
+                })
+            );
+
+            await Task.WhenAll(tasks);
+
+            var totalTps = (int) (totalTransactions / (stopTime - startTime).TotalSeconds);
+
+
+            listOfResults.Remove(listOfResults.Max());
+            listOfResults.Remove(listOfResults.Min());
+            var stddev = CalculateStdDev(listOfResults, listOfResults.Count);
+
+            Console.SetCursorPosition(0, Console.CursorTop);
+            Console.WriteLine($"{numThreads:D2} Threads, tps: {totalTps:F2}, stddev(w/o best+worst): {stddev:F2}");
+
+            var desc = $"{database}+{mode}+{numThreads}";
+
+            using (var sw = File.AppendText("results.md"))
+            {
+                sw.WriteLine($"|{desc}|{mode}|{numThreads:D2}|{totalTps:F0}|{stddev:F0}|");
+            }
+
+            using (var sw = File.AppendText("results.csv"))
+            {
+                sw.WriteLine($"{desc},{mode},{numThreads:D2},{totalTps:F0},{stddev:F0}");
+            }
+
+            double CalculateStdDev(IEnumerable<double> values, int count)
+            {
+                var avg = values.Sum() / count;
+                double sum = values.Sum(d => Math.Pow(d - avg, 2));
+                return Math.Sqrt((sum) / count);
+            }
+
+            async Task DoWorkAsync()
+            {
+                while (_stopping != 1)
+                {
+                    Interlocked.Increment(ref _counter);
+
+                    try
+                    {
+                        var results = new List<Fortune>();
+
+                        using (var connection = factory.CreateConnection())
+                        {
+                            connection.ConnectionString = connectionString;
+                            await connection.OpenAsync();
+
+                            using (var command = connection.CreateCommand())
+                            {
+                                command.CommandText = SqlQuery;
+                                command.Prepare();
+
+                                using (var reader = await command.ExecuteReaderAsync())
+                                {
+                                    while (await reader.ReadAsync())
+                                    {
+                                        results.Add(new Fortune
+                                        {
+                                            Id = reader.GetInt32(0),
+                                            Message = reader.GetString(1)
+                                        });
+                                    }
+                                }
+                            }
+                        }
+
+                        if (results.Count != 12)
+                        {
+                            throw new InvalidDataException("Not 12");
+                        }
+
+                    }
+                    catch (Exception e)
+                    {
+                        Console.WriteLine(e.Message);
+                    }
+                }
+            }
+
+            async Task DoWorkAsyncReuseConnection()
+            {
+                using (var connection = factory.CreateConnection())
+                {
+                    using (var command = connection.CreateCommand())
+                    {
+                        connection.ConnectionString = connectionString;
+                        await connection.OpenAsync();
+
+                        command.CommandText = SqlQuery;
+                        command.Prepare();
+
+                        while (_stopping != 1)
+                        {
+                            Interlocked.Increment(ref _counter);
+
+                            try
+                            {
+                                var results = new List<Fortune>();
+
+                                using (var reader = await command.ExecuteReaderAsync())
+                                {
+                                    while (await reader.ReadAsync())
+                                    {
+                                        results.Add(new Fortune
+                                        {
+                                            Id = reader.GetInt32(0),
+                                            Message = reader.GetString(1)
+                                        });
+                                    }
+                                }
+
+                                if (results.Count != 12)
+                                {
+                                    throw new InvalidDataException("Not 12");
+                                }
+                            }
+                            catch (Exception e)
+                            {
+                                Console.WriteLine(e.Message);
+                            }
+                        }
+                    }
+                }
+            }
+
+            void DoWorkSync()
+            {
+                while (_stopping != 1)
+                {
+                    Interlocked.Increment(ref _counter);
+
+                    try
+                    {
+                        var results = new List<Fortune>();
+
+                        using (var connection = factory.CreateConnection())
+                        {
+                            connection.ConnectionString = connectionString;
+                            connection.Open();
+
+                            using (var command = connection.CreateCommand())
+                            {
+                                command.CommandText = SqlQuery;
+                                command.Prepare();
+
+                                using (var reader = command.ExecuteReader())
+                                {
+                                    while (reader.Read())
+                                    {
+                                        results.Add(new Fortune
+                                        {
+                                            Id = reader.GetInt32(0),
+                                            Message = reader.GetString(1)
+                                        });
+                                    }
+                                }
+                            }
+                        }
+
+                        if (results.Count != 12)
+                        {
+                            throw new InvalidDataException("Not 12");
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        Console.WriteLine(e.Message);
+                    }
+                }
+            }
+
+            void DoWorkSyncReuseConnection()
+            {
+                using (var connection = factory.CreateConnection())
+                {
+                    using (var command = connection.CreateCommand())
+                    {
+                        connection.ConnectionString = connectionString;
+                        connection.Open();
+
+                        command.CommandText = SqlQuery;
+                        command.Prepare();
+
+                        while (_stopping != 1)
+                        {
+                            Interlocked.Increment(ref _counter);
+
+                            try
+                            {
+                                var results = new List<Fortune>();
+
+                                using (var reader = command.ExecuteReader())
+                                {
+                                    while (reader.Read())
+                                    {
+                                        results.Add(new Fortune
+                                        {
+                                            Id = reader.GetInt32(0),
+                                            Message = reader.GetString(1)
+                                        });
+                                    }
+                                }
+
+                                if (results.Count != 12)
+                                {
+                                    throw new InvalidDataException("Not 12");
+                                }
+                            }
+                            catch (Exception e)
+                            {
+                                Console.WriteLine(e.Message);
+                            }
+                        }
+                    }
+                }
+            }
+        }        
+    }
+}
--- a/src/DataAccessPerformance.sln
+++ b/src/DataAccessPerformance.sln
@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.27108.1
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkDb", "BenchmarkDb\BenchmarkDb.csproj", "{7FA56803-EFBD-4028-8067-33F9608E7821}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{7FA56803-EFBD-4028-8067-33F9608E7821}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7FA56803-EFBD-4028-8067-33F9608E7821}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7FA56803-EFBD-4028-8067-33F9608E7821}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7FA56803-EFBD-4028-8067-33F9608E7821}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {29C6A361-AEF7-4B3C-BD61-6225B47CBF91}
+	EndGlobalSection
+EndGlobal
--- a/src/NuGet.config
+++ b/src/NuGet.config
@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="NuGet" value="https://api.nuget.org/v3/index.json" />
+    <add key="npgsql-unstable" value="https://www.myget.org/F/npgsql-unstable/api/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources />
+</configuration>