зеркало из https://github.com/microsoft/Power-Fx.git
Improve Benchmark (#968)
Improve benchmark project Add reference tests Add script to run locally easily Add script to read results quickly Add script to write results to SQL database
This commit is contained in:
Luc Genetier
GitHub
Родитель
075c0d1395
Коммит
e0e5c5427c
|
|
@ -0,0 +1,314 @@
|
|||
param(
|
||||
[Parameter (Mandatory = $true)] [String]$ConnectionString
|
||||
)
|
||||
|
||||
## $env:BUILD_SOURCESDIRECTORY = "C:\Data\2\Power-Fx"
|
||||
## $env:BUILD_DEFINITIONNAME = "Test"
|
||||
## $env:BUILD_BUILDID = "69992023"
|
||||
## $env:BUILD_BUILDNUMBER = "2023.X1"
|
||||
## $env:BUILDCONFIGURATION = "TestRelease"
|
||||
|
||||
function ConvertToMs([string]$str)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (($str.Trim() -eq '-') -or [string]::IsNullOrEmpty($str.Trim()))
|
||||
{
|
||||
$val = [double]0
|
||||
}
|
||||
else
|
||||
{
|
||||
$parts = $str.Split(@(' '), [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
$val = [double]$parts[0]
|
||||
|
||||
## Convert to milliseconds
|
||||
if ($parts[1] -eq "s") { $val *= 1000 }
|
||||
elseif ($parts[1] -eq "ms") { } ## Do nothing
|
||||
elseif ($parts[1] -eq "µs") { $val /= 1000 }
|
||||
elseif ($parts[1] -eq "ns") { $val /= 1000000 }
|
||||
else { throw ("Unknown unit: " + $parts[1]) }
|
||||
}
|
||||
|
||||
$val
|
||||
}
|
||||
catch
|
||||
{
|
||||
Write-Error $_
|
||||
}
|
||||
}
|
||||
|
||||
function ConvertToBytes([string]$str)
|
||||
{
|
||||
try
|
||||
{
|
||||
$parts = $str.Split(@(' '), [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
$val = [double]$parts[0]
|
||||
|
||||
## Convert to milliseconds
|
||||
if ($parts[1] -eq "B") { } ## Do nothing
|
||||
elseif ($parts[1] -eq "kB") { $val *= 1024 }
|
||||
elseif ($parts[1] -eq "MB") { $val *= 1048576 }
|
||||
elseif ($parts[1] -eq "GB") { $val *= 1073741824 }
|
||||
else { throw ("Unknown unit: " + $parts[1]) }
|
||||
|
||||
$val
|
||||
}
|
||||
catch
|
||||
{
|
||||
Write-Error $_
|
||||
}
|
||||
}
|
||||
|
||||
## Retrieve Power Fx latest Git hash and Git remote branch
|
||||
cd ($env:BUILD_SOURCESDIRECTORY)
|
||||
$pfxHash = (git log -n 1 --pretty=%H).ToString()
|
||||
|
||||
$pfxBranch = (git log -n 1 --pretty=%D).Split(", /")
|
||||
$pfxBranch = $pfxBranch[$pfxBranch.Length-1]
|
||||
|
||||
Write-Host "------ Git context ------"
|
||||
Write-Host "PFX Hash : $pfxHash"
|
||||
Write-Host "PFX Branch: $pfxBranch"
|
||||
Write-Host
|
||||
|
||||
cd src
|
||||
|
||||
if (-not (Test-Path "BenchmarkDotNet.Artifacts"))
|
||||
{
|
||||
Write-Host -ForegroundColor Yellow "No BenchmarkDotNet.Artifacts folder!"
|
||||
Write-Host -ForegroundColor DarkYellow "Run benchmark first."
|
||||
dir
|
||||
exit
|
||||
}
|
||||
|
||||
Write-Host "------ Test context ------"
|
||||
|
||||
$cpuFile = "BenchmarkDotNet.Artifacts\cpu.csv"
|
||||
$memoryFile = "BenchmarkDotNet.Artifacts\memory.csv"
|
||||
$referenceFile = "BenchmarkDotNet.Artifacts\results\Microsoft.PowerFx.Performance.Tests.Reference-report-github.md"
|
||||
|
||||
## Read CPU file, fixed size columns
|
||||
$index = 0; $x = 0
|
||||
foreach ($line in (Get-Content $cpuFile))
|
||||
{
|
||||
## Get headers and identify their positon
|
||||
if ($index -eq 0)
|
||||
{
|
||||
$headers = $line.Split(@(' '), [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
$hdrs = [System.Collections.ArrayList]::new()
|
||||
$l = $line
|
||||
for ($i = 0; $i -lt $headers.Length; $i++)
|
||||
{
|
||||
$y = $line.IndexOf($headers[$i])
|
||||
$x += $y
|
||||
[void]$hdrs.Add($x)
|
||||
$line = $line.Substring($headers[$i].Length + $y )
|
||||
$x += $headers[$i].Length
|
||||
}
|
||||
|
||||
[void]$hdrs.Add($x)
|
||||
}
|
||||
|
||||
if ($index -eq 1)
|
||||
{
|
||||
$cpuModel = $line.Substring($hdrs[0], $hdrs[1] - $hdrs[0]).Trim()
|
||||
$cpuSpeed = [int]::Parse($line.Substring($hdrs[1], $hdrs[2] - $hdrs[1]).Trim())
|
||||
$cpuName = $line.Substring($hdrs[2], $hdrs[3] - $hdrs[2]).Trim()
|
||||
$numberCores = [int]::Parse($line.Substring($hdrs[3], $hdrs[4] - $hdrs[3]).Trim())
|
||||
$numberLogicalProcs = [int]::Parse($line.Substring($hdrs[4], $hdrs[5] - $hdrs[4]).Trim())
|
||||
}
|
||||
|
||||
$index++
|
||||
}
|
||||
|
||||
Write-Host "CPU Model : $cpuModel"
|
||||
Write-Host "CPU Speed (MHz) : $cpuSpeed"
|
||||
Write-Host "CPU Name : $cpuName"
|
||||
Write-Host "Number Cores : $numberCores"
|
||||
Write-Host "Logical processors: $numberLogicalProcs"
|
||||
|
||||
## Read CPU file, fixed size columns
|
||||
$index = 0; $x = 0; $memory = [double]0;
|
||||
foreach ($line in (Get-Content $memoryFile))
|
||||
{
|
||||
## Get headers and identify their positon
|
||||
if ($index -eq 0)
|
||||
{
|
||||
$headers = $line.Split(@(' '), [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
$hdrs = [System.Collections.ArrayList]::new()
|
||||
$l = $line
|
||||
for ($i = 0; $i -lt $headers.Length; $i++)
|
||||
{
|
||||
$y = $line.IndexOf($headers[$i])
|
||||
$x += $y
|
||||
[void]$hdrs.Add($x)
|
||||
$line = $line.Substring($headers[$i].Length + $y )
|
||||
$x += $headers[$i].Length
|
||||
}
|
||||
|
||||
[void]$hdrs.Add($x)
|
||||
}
|
||||
|
||||
if ($index -eq 1)
|
||||
{
|
||||
$memory = [double]::Parse($line.Substring($hdrs[0], $hdrs[1] - $hdrs[0]).Trim());
|
||||
}
|
||||
|
||||
$index++
|
||||
}
|
||||
|
||||
$memoryGB = $memory / [double]1024 / [double]1024 / [double]1024;
|
||||
|
||||
Write-Host "Memory (GB) : $memoryGB"
|
||||
|
||||
$index = 0
|
||||
foreach ($line in (Get-Content $referenceFile))
|
||||
{
|
||||
if ($index -in @(2, 4))
|
||||
{
|
||||
foreach ($a in ($line.Split(@(','), [System.StringSplitOptions]::RemoveEmptyEntries) | % { $_.Trim() }))
|
||||
{
|
||||
$b = $a.Split(@('='), [System.StringSplitOptions]::RemoveEmptyEntries) | % { $_.Trim() }
|
||||
if ($b[0] -eq 'BenchmarkDotNet') { $bmdnVersion = $b[1] }
|
||||
if ($b[0] -eq 'OS') { $osVersion = $b[1] }
|
||||
if ($b[0] -eq 'VM') { $vmType = $b[1] }
|
||||
if ($b[0] -in @('.Net Core SDK', '.NET SDK')) { $dnRTVersion = $b[1] }
|
||||
}
|
||||
}
|
||||
if ($index -eq 6)
|
||||
{
|
||||
$a = $line.Split(@(':'), [System.StringSplitOptions]::RemoveEmptyEntries) | % { $_.Trim() }
|
||||
$dnVersion = $a[1]
|
||||
}
|
||||
|
||||
$index++
|
||||
}
|
||||
|
||||
Write-Host "BenchmarkDotNet : $bmdnVersion"
|
||||
Write-Host "OS Version : $osVersion"
|
||||
Write-Host "VM Type : $vmType"
|
||||
Write-Host ".Net RT Version : $dnRTVersion"
|
||||
Write-Host ".Net Version : $dnVersion"
|
||||
|
||||
Write-Host
|
||||
|
||||
$Connection = New-Object System.Data.SQLClient.SQLConnection
|
||||
$Connection.ConnectionString = $ConnectionString
|
||||
$Connection.Open()
|
||||
|
||||
$Command = New-Object System.Data.SQLClient.SQLCommand
|
||||
$Command.Connection = $Connection
|
||||
|
||||
$insertQuery = "insert into Runs ([TimeStamp], [Hash], [Pipeline], [BuildId], [BuildNumber], [BuildConfiguration], [Branch]) "
|
||||
$insertQuery += "values (getutcdate(), '$pfxHash', '$env:BUILD_DEFINITIONNAME', '$env:BUILD_BUILDID', '$env:BUILD_BUILDNUMBER', '$env:BUILDCONFIGURATION', '$pfxBranch');"
|
||||
$insertQuery += "select scope_identity() as 'Id'"
|
||||
|
||||
$Command.CommandText = $insertquery
|
||||
$runId = $Command.ExecuteScalar()
|
||||
|
||||
Write-Host "RunId: $runId"
|
||||
|
||||
$insertQuery = "insert into Contexts ([RunId], [CPUModel], [CPUSpeedMHz], [CPUName], [NumberCores], [LogicalProcessors], [MemoryGB], [BenchMarkDotNetVersion], [OS], [VM], [DotNetRuntime], [DotNetVersion]) "
|
||||
$insertQuery += "values ('$runId', '$cpuModel', '$cpuSpeed', '$cpuName', '$numberCores', '$numberLogicalProcs', '$memoryGB', '$bmdnVersion', '$osVersion', '$vmType', '$dnRTVersion', '$dnVersion');"
|
||||
$insertQuery += "select scope_identity() as 'Id'"
|
||||
|
||||
$Command.CommandText = $insertquery
|
||||
$contextId = $Command.ExecuteScalar()
|
||||
|
||||
Write-Host "ContextId: $contextId"
|
||||
Write-Host
|
||||
|
||||
## Always start the list of results with the Reference CSV file
|
||||
$list = (Get-Item -Filter *.csv -Path '.\BenchmarkDotNet.Artifacts\results\*' | % { $_.FullName })
|
||||
foreach ($file in [System.Linq.Enumerable]::OrderBy($list, [Func[object, string]] { param($s) if ($s -match 'Reference-report\.csv') { "" } else { $s } }))
|
||||
{
|
||||
$t = [System.IO.Path]::GetFileNameWithoutExtension($file).Split(@('.'))[-1]
|
||||
$testCategory = $t.Substring(0, $t.Length - 7)
|
||||
|
||||
Write-Host "------ [TEST] $testCategory ------"
|
||||
|
||||
$table = [System.Data.DataTable]::new()
|
||||
[void]$table.Columns.Add("TestName", [string]); $table.Columns["TestName"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("N", [int]); $table.Columns["N"].AllowDBNull = $true ## Optional column, depends on the test
|
||||
[void]$table.Columns.Add("Mean", [double]); $table.Columns["Mean"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("StdDev", [double]); $table.Columns["StdDev"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Min", [double]); $table.Columns["Min"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Q1", [double]); $table.Columns["Q1"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Median", [double]); $table.Columns["Median"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Q3", [double]); $table.Columns["Q3"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Max", [double]); $table.Columns["Max"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Gen0", [double]); $table.Columns["Gen0"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Gen1", [double]); $table.Columns["Gen1"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Allocated", [double]); $table.Columns["Allocated"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("AllocatedNativeMemory", [double]); $table.Columns["AllocatedNativeMemory"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("NativeMemoryLeak", [double]); $table.Columns["NativeMemoryLeak"].AllowDBNull = $false
|
||||
|
||||
|
||||
foreach ($row in (Import-Csv $file | Select-Object Method, Runtime, N, Mean, StdDev, Min, Q1, Median, Q3, Max, Gen0, Gen1, Allocated, 'Allocated Native Memory', 'Native Memory Leak'))
|
||||
{
|
||||
$mean = ConvertToMs($row.Mean)
|
||||
$stddev = ConvertToMs($row.StdDev)
|
||||
$min = ConvertToMs($row.Min)
|
||||
$q1 = ConvertToMs($row.Q1)
|
||||
$median = ConvertToMs($row.Median)
|
||||
$q3 = ConvertToMs($row.Q3)
|
||||
$max = ConvertToMs($row.Max)
|
||||
|
||||
$gen0 = $row.Gen0 ## Gen X means number of GC collections per 1000 operations for that generation
|
||||
$gen1 = $row.Gen1
|
||||
$alloc = ConvertToBytes($row.Allocated)
|
||||
$native = ConvertToBytes($row.'Allocated Native Memory')
|
||||
$leak = ConvertToBytes($row.'Native Memory Leak')
|
||||
|
||||
if (($gen0 -eq $null) -or ($gen0.Trim() -eq '-') -or [string]::IsNullOrEmpty($gen0.Trim())) { $gen0 = [double]0 }
|
||||
if (($gen1 -eq $null) -or ($gen1.Trim() -eq '-') -or [string]::IsNullOrEmpty($gen1.Trim())) { $gen1 = [double]0 }
|
||||
|
||||
[void]$table.Rows.Add($row.Method, $row.N, $mean, $stddev, $min, $q1, $median, $q3, $max, $gen0, $gen1, $alloc, $native, $leak)
|
||||
}
|
||||
|
||||
$table | Sort-Object TestName, N | ft TestName, N, Mean, StdDev, Min, Q1, Median, Q3, Max, Gen0, Gen1, Allocated, AllocatedNativeMemory, NativeMemoryLeak
|
||||
|
||||
$ctxids = [System.Collections.ArrayList]::new()
|
||||
foreach ($row in $table)
|
||||
{
|
||||
$testName = $row.TestName
|
||||
$n = $row.N
|
||||
$mean = $row.Mean
|
||||
$stdDev = $row.StdDev
|
||||
$min = $row.Min
|
||||
$q1 = $row.Q1
|
||||
$median = $row.Median
|
||||
$q3 = $row.Q3
|
||||
$max = $row.Max
|
||||
$gen0 = $row.Gen0
|
||||
$gen1 = $row.Gen1
|
||||
$alloc = $row.Allocated
|
||||
$native = $row.AllocatedNativeMemory
|
||||
$leak = $row.NativeMemoryLeak
|
||||
|
||||
$insertQuery = "insert into Tests ([RunId], [ContextId], [TestName], [N], [MeanMs], [StdDevMs], [MinMs], [Q1Ms], [MedianMs], [Q3Ms], [MaxMs], [Gen0], [Gen1], [Allocated], [AllocatedNativeMemory], [NativeMemoryLeak]) "
|
||||
$insertQuery += "values ('$runId', '$contextId', '$testName', "
|
||||
|
||||
if ($n.GetType() -eq [DBNull])
|
||||
{
|
||||
$insertQuery += "null"
|
||||
}
|
||||
else
|
||||
{
|
||||
$insertQuery += "'$n'"
|
||||
}
|
||||
|
||||
$insertQuery += ", '$mean', '$stdDev', '$min', '$q1', '$median', '$q3', '$max', '$gen0', '$gen1', '$alloc', '$native', '$leak'); "
|
||||
$insertQuery += "select scope_identity() as 'Id'"
|
||||
|
||||
$Command.CommandText = $insertquery
|
||||
$id = $Command.ExecuteScalar()
|
||||
|
||||
[void]$ctxids.Add($id)
|
||||
}
|
||||
|
||||
Write-Host "TestIds: " ([String]::Join(', ', $ctxids.ToArray()))
|
||||
Write-Host
|
||||
}
|
||||
|
||||
Write-Host "--- End of script ---"
|
||||
|
|
@ -73,7 +73,7 @@
|
|||
<Copyright>© Microsoft Corporation. All rights reserved.</Copyright>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup Condition="'$(Project)'!='Microsoft.PowerFx.Performance.Tests'">
|
||||
<PropertyGroup Condition=" '$(ProjectName)' != 'Microsoft.PowerFx.Performance.Tests' ">
|
||||
<AssemblyOriginatorKeyFile Condition="EXISTS('..\..\..\src\build\35MSSharedLib1024.snk')">..\..\..\src\build\35MSSharedLib1024.snk</AssemblyOriginatorKeyFile>
|
||||
<SignAssembly>true</SignAssembly>
|
||||
<DelaySign>true</DelaySign>
|
||||
|
|
|
|||
|
|
@ -44,7 +44,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PowerFx.Performan
|
|||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{5EEC2873-35B5-4364-A2A0-97AAD7BF960D}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Repl", "tools\Repl\Repl.csproj", "{142854D7-409C-4C87-A5B8-39B51477F5B2}"
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Repl", "tools\Repl\Repl.csproj", "{142854D7-409C-4C87-A5B8-39B51477F5B2}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Benchmark", "Benchmark", "{4167C30B-F9D2-4187-87DA-85864AD2AA94}"
|
||||
ProjectSection(SolutionItems) = preProject
|
||||
ReadBenchmarkData.ps1 = ReadBenchmarkData.ps1
|
||||
RunBenchmark.cmd = RunBenchmark.cmd
|
||||
..\.scripts\WriteBenchmark.ps1 = ..\.scripts\WriteBenchmark.ps1
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.PowerFx.Json", "libraries\Microsoft.PowerFx.Json\Microsoft.PowerFx.Json.csproj", "{D4CC6660-8C30-41E5-A28A-C018DDDF8D96}"
|
||||
EndProject
|
||||
|
|
|
|||
|
|
@ -0,0 +1,220 @@
|
|||
if (-not (Test-Path "BenchmarkDotNet.Artifacts"))
|
||||
{
|
||||
Write-Host -ForegroundColor Yellow "No BenchmarkDotNet.Artifacts folder!"
|
||||
Write-Host -ForegroundColor DarkYellow "Run benchmark first."
|
||||
exit
|
||||
}
|
||||
|
||||
function ConvertToMs([string]$str)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (($str.Trim() -eq '-') -or [string]::IsNullOrEmpty($str.Trim()))
|
||||
{
|
||||
$val = [double]0
|
||||
}
|
||||
else
|
||||
{
|
||||
$parts = $str.Split(@(' '), [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
$val = [double]$parts[0]
|
||||
|
||||
## Convert to milliseconds
|
||||
if ($parts[1] -eq "s") { $val *= 1000 }
|
||||
elseif ($parts[1] -eq "ms") { } ## Do nothing
|
||||
elseif ($parts[1] -eq "µs") { $val /= 1000 }
|
||||
elseif ($parts[1] -eq "ns") { $val /= 1000000 }
|
||||
else { throw ("Unknown unit: " + $parts[1]) }
|
||||
}
|
||||
|
||||
$val
|
||||
}
|
||||
catch
|
||||
{
|
||||
Write-Error $_
|
||||
}
|
||||
}
|
||||
|
||||
function ConvertToBytes([string]$str)
|
||||
{
|
||||
try
|
||||
{
|
||||
$parts = $str.Split(@(' '), [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
$val = [double]$parts[0]
|
||||
|
||||
## Convert to milliseconds
|
||||
if ($parts[1] -eq "B") { } ## Do nothing
|
||||
elseif ($parts[1] -eq "kB") { $val *= 1024 }
|
||||
elseif ($parts[1] -eq "MB") { $val *= 1048576 }
|
||||
elseif ($parts[1] -eq "GB") { $val *= 1073741824 }
|
||||
else { throw ("Unknown unit: " + $parts[1]) }
|
||||
|
||||
$val
|
||||
}
|
||||
catch
|
||||
{
|
||||
Write-Error $_
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "------ Test context ------"
|
||||
|
||||
$cpuFile = "BenchmarkDotNet.Artifacts\cpu.csv"
|
||||
$memoryFile = "BenchmarkDotNet.Artifacts\memory.csv"
|
||||
$referenceFile = "BenchmarkDotNet.Artifacts\results\Microsoft.PowerFx.Performance.Tests.Reference-report-github.md"
|
||||
|
||||
## Read CPU file, fixed size columns
|
||||
$index = 0; $x = 0
|
||||
foreach ($line in (Get-Content $cpuFile))
|
||||
{
|
||||
## Get headers and identify their positon
|
||||
if ($index -eq 0)
|
||||
{
|
||||
$headers = $line.Split(@(' '), [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
$hdrs = [System.Collections.ArrayList]::new()
|
||||
$l = $line
|
||||
for ($i = 0; $i -lt $headers.Length; $i++)
|
||||
{
|
||||
$y = $line.IndexOf($headers[$i])
|
||||
$x += $y
|
||||
[void]$hdrs.Add($x)
|
||||
$line = $line.Substring($headers[$i].Length + $y )
|
||||
$x += $headers[$i].Length
|
||||
}
|
||||
|
||||
[void]$hdrs.Add($x)
|
||||
}
|
||||
|
||||
if ($index -eq 1)
|
||||
{
|
||||
$cpuModel = $line.Substring($hdrs[0], $hdrs[1] - $hdrs[0]).Trim()
|
||||
$cpuSpeed = [int]::Parse($line.Substring($hdrs[1], $hdrs[2] - $hdrs[1]).Trim())
|
||||
$cpuName = $line.Substring($hdrs[2], $hdrs[3] - $hdrs[2]).Trim()
|
||||
$numberCores = [int]::Parse($line.Substring($hdrs[3], $hdrs[4] - $hdrs[3]).Trim())
|
||||
$numberLogicalProcs = [int]::Parse($line.Substring($hdrs[4], $hdrs[5] - $hdrs[4]).Trim())
|
||||
}
|
||||
|
||||
$index++
|
||||
}
|
||||
|
||||
Write-Host "CPU Model : $cpuModel"
|
||||
Write-Host "CPU Speed (MHz) : $cpuSpeed"
|
||||
Write-Host "CPU Name : $cpuName"
|
||||
Write-Host "Number Cores : $numberCores"
|
||||
Write-Host "Logical processors: $numberLogicalProcs"
|
||||
|
||||
## Read CPU file, fixed size columns
|
||||
$index = 0; $x = 0; $memory = [double]0;
|
||||
foreach ($line in (Get-Content $memoryFile))
|
||||
{
|
||||
## Get headers and identify their positon
|
||||
if ($index -eq 0)
|
||||
{
|
||||
$headers = $line.Split(@(' '), [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
$hdrs = [System.Collections.ArrayList]::new()
|
||||
$l = $line
|
||||
for ($i = 0; $i -lt $headers.Length; $i++)
|
||||
{
|
||||
$y = $line.IndexOf($headers[$i])
|
||||
$x += $y
|
||||
[void]$hdrs.Add($x)
|
||||
$line = $line.Substring($headers[$i].Length + $y )
|
||||
$x += $headers[$i].Length
|
||||
}
|
||||
|
||||
[void]$hdrs.Add($x)
|
||||
}
|
||||
|
||||
if ($index -eq 1)
|
||||
{
|
||||
$memory = [double]::Parse($line.Substring($hdrs[0], $hdrs[1] - $hdrs[0]).Trim());
|
||||
}
|
||||
|
||||
$index++
|
||||
}
|
||||
|
||||
$memoryGB = $memory / [double]1024 / [double]1024 / [double]1024;
|
||||
|
||||
Write-Host "Memory (GB) : $memoryGB"
|
||||
|
||||
$index = 0
|
||||
foreach ($line in (Get-Content $referenceFile))
|
||||
{
|
||||
if ($index -in @(2, 4))
|
||||
{
|
||||
foreach ($a in ($line.Split(@(','), [System.StringSplitOptions]::RemoveEmptyEntries) | % { $_.Trim() }))
|
||||
{
|
||||
$b = $a.Split(@('='), [System.StringSplitOptions]::RemoveEmptyEntries) | % { $_.Trim() }
|
||||
if ($b[0] -eq 'BenchmarkDotNet') { $bmdnVersion = $b[1] }
|
||||
if ($b[0] -eq 'OS') { $osVersion = $b[1] }
|
||||
if ($b[0] -eq 'VM') { $vmType = $b[1] }
|
||||
if ($b[0] -in @('.Net Core SDK', '.NET SDK')) { $dnRTVersion = $b[1] }
|
||||
}
|
||||
}
|
||||
if ($index -eq 6)
|
||||
{
|
||||
$a = $line.Split(@(':'), [System.StringSplitOptions]::RemoveEmptyEntries) | % { $_.Trim() }
|
||||
$dnVersion = $a[1]
|
||||
}
|
||||
|
||||
$index++
|
||||
}
|
||||
|
||||
Write-Host "BenchmarkDotNet : $bmdnVersion"
|
||||
Write-Host "OS Version : $osVersion"
|
||||
Write-Host "VM Type : $vmType"
|
||||
Write-Host ".Net RT Version : $dnRTVersion"
|
||||
Write-Host ".Net Version : $dnVersion"
|
||||
|
||||
Write-Host
|
||||
|
||||
## Always start the list of results with the Reference CSV file
|
||||
$list = (Get-Item -Filter *.csv -Path '.\BenchmarkDotNet.Artifacts\results\*' | % { $_.FullName })
|
||||
foreach ($file in [System.Linq.Enumerable]::OrderBy($list, [Func[object, string]] { param($s) if ($s -match 'Reference-report\.csv') { "" } else { $s } }))
|
||||
{
|
||||
$t = [System.IO.Path]::GetFileNameWithoutExtension($file).Split(@('.'))[-1]
|
||||
$testCategory = $t.Substring(0, $t.Length - 7)
|
||||
|
||||
Write-Host "------ [TEST] $testCategory ------"
|
||||
|
||||
$table = [System.Data.DataTable]::new()
|
||||
[void]$table.Columns.Add("TestName", [string]); $table.Columns["TestName"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("N", [int]); $table.Columns["N"].AllowDBNull = $true ## Optional column, depends on the test
|
||||
[void]$table.Columns.Add("Mean", [double]); $table.Columns["Mean"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("StdDev", [double]); $table.Columns["StdDev"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Min", [double]); $table.Columns["Min"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Q1", [double]); $table.Columns["Q1"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Median", [double]); $table.Columns["Median"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Q3", [double]); $table.Columns["Q3"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Max", [double]); $table.Columns["Max"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Gen0", [double]); $table.Columns["Gen0"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Gen1", [double]); $table.Columns["Gen1"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("Allocated", [double]); $table.Columns["Allocated"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("AllocatedNativeMemory", [double]); $table.Columns["AllocatedNativeMemory"].AllowDBNull = $false
|
||||
[void]$table.Columns.Add("NativeMemoryLeak", [double]); $table.Columns["NativeMemoryLeak"].AllowDBNull = $false
|
||||
|
||||
foreach ($row in (Import-Csv $file | Select-Object Method, Runtime, N, Mean, StdDev, Min, Q1, Median, Q3, Max, Gen0, Gen1, Allocated, 'Allocated Native Memory', 'Native Memory Leak'))
|
||||
{
|
||||
$mean = ConvertToMs($row.Mean)
|
||||
$stddev = ConvertToMs($row.StdDev)
|
||||
$min = ConvertToMs($row.Min)
|
||||
$q1 = ConvertToMs($row.Q1)
|
||||
$median = ConvertToMs($row.Median)
|
||||
$q3 = ConvertToMs($row.Q3)
|
||||
$max = ConvertToMs($row.Max)
|
||||
$gen0 = $row.Gen0 ## Gen X means number of GC collections per 1000 operations for that generation
|
||||
$gen1 = $row.Gen1
|
||||
$alloc = ConvertToBytes($row.Allocated)
|
||||
$native = ConvertToBytes($row.'Allocated Native Memory')
|
||||
$leak = ConvertToBytes($row.'Native Memory Leak')
|
||||
|
||||
if (($gen0 -eq $null) -or ($gen0.Trim() -eq '-') -or [string]::IsNullOrEmpty($gen0.Trim())) { $gen0 = [double]0 }
|
||||
if (($gen1 -eq $null) -or ($gen1.Trim() -eq '-') -or [string]::IsNullOrEmpty($gen1.Trim())) { $gen1 = [double]0 }
|
||||
|
||||
[void]$table.Rows.Add($row.Method, $row.N, $mean, $stddev, $min, $q1, $median, $q3, $max, $gen0, $gen1, $alloc, $native, $leak)
|
||||
}
|
||||
|
||||
$table | Sort-Object TestName, N | ft TestName, N, Mean, StdDev, Min, Q1, Median, Q3, Max, Gen0, Gen1, Allocated, AllocatedNativeMemory, NativeMemoryLeak
|
||||
Write-Host
|
||||
}
|
||||
|
||||
Write-Host '--- End of script ---'
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
@echo off
|
||||
if not exist "BenchmarkDotNet.Artifacts" md "BenchmarkDotNet.Artifacts"
|
||||
wmic cpu get Caption, CurrentClockSpeed, Name, NumberOfCores, NumberOfLogicalProcessors > BenchmarkDotNet.Artifacts\cpu.csv
|
||||
wmic computersystem get TotalPhysicalMemory > BenchmarkDotNet.Artifacts\memory.csv
|
||||
for /f %%a in ('dotnet --list-sdks ^| findstr 3\.1') do set dotnetver=%%a
|
||||
set MSBuildSDKsPath=C:\Program Files\dotnet\sdk\%dotnetver%\sdks
|
||||
if not exist global.json dotnet new globaljson --sdk-version %dotnetver%
|
||||
"tests\Microsoft.PowerFx.Performance.Tests\bin\Release\netcoreapp3.1\Microsoft.PowerFx.Performance.Tests.exe" --list tree
|
||||
"tests\Microsoft.PowerFx.Performance.Tests\bin\Release\netcoreapp3.1\Microsoft.PowerFx.Performance.Tests.exe" -f *
|
||||
del global.json
|
||||
|
|
@ -7,19 +7,24 @@ using System.Linq;
|
|||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Diagnostics.Windows;
|
||||
using BenchmarkDotNet.Diagnostics.Windows.Configs;
|
||||
using BenchmarkDotNet.Jobs;
|
||||
using Microsoft.PowerFx.Syntax;
|
||||
using Microsoft.PowerFx.Types;
|
||||
|
||||
namespace Microsoft.PowerFx.Performance.Tests
|
||||
{
|
||||
[MemoryDiagnoser]
|
||||
[NativeMemoryProfiler]
|
||||
[EtwProfiler] // https://benchmarkdotnet.org/articles/features/etwprofiler.html
|
||||
[CsvExporter] // https://benchmarkdotnet.org/articles/configs/exporters.html
|
||||
[MinColumn]
|
||||
[Q1Column]
|
||||
[MeanColumn]
|
||||
[MedianColumn]
|
||||
[Q3Column]
|
||||
[MaxColumn]
|
||||
public class PerformanceTest1
|
||||
[SimpleJob(runtimeMoniker: RuntimeMoniker.NetCoreApp31)]
|
||||
public class BasicPerformance
|
||||
{
|
||||
private PowerFxConfig _powerFxConfig;
|
||||
private Engine _engine;
|
||||
|
|
@ -8,7 +8,9 @@
|
|||
<GenerateDocumentationFile>false</GenerateDocumentationFile>
|
||||
<StartupObject>Microsoft.PowerFx.Performance.Tests.Program</StartupObject>
|
||||
<LangVersion>9.0</LangVersion>
|
||||
<AssemblyName>Microsoft.PowerFx.Performance.Benchmarks</AssemblyName>
|
||||
<AssemblyName>Microsoft.PowerFx.Performance.Tests</AssemblyName>
|
||||
<SignAssembly>False</SignAssembly>
|
||||
<DelaySign>False</DelaySign>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
|||
|
|
@ -9,16 +9,30 @@ namespace Microsoft.PowerFx.Performance.Tests
|
|||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
/*
|
||||
*
|
||||
* --> This program needs to be run ELEVATED
|
||||
*
|
||||
* --> Also set the following env. variable before starting VS
|
||||
* SET MSBuildSDKsPath=C:\Program Files\dotnet\sdk\7.0.100\Sdks
|
||||
*
|
||||
/*
|
||||
1. Build in Release mode
|
||||
|
||||
2. Then, execute the following commands (from cmd.exe, ELEVATED [required for ETW generation])
|
||||
set MSBuildSDKsPath=C:\Program Files\dotnet\sdk\3.1.426\sdks
|
||||
cd <repo root>\src
|
||||
dotnet new globaljson --sdk-version 3.1.426
|
||||
"tests\Microsoft.PowerFx.Performance.Tests\bin\Release\netcoreapp3.1\Microsoft.PowerFx.Performance.Tests.exe" -f *
|
||||
|
||||
Notice that version 3.1.426 or other versions are obtained with "dotnet --list-sdks"
|
||||
"dotnet new globaljson --sdk-version 3.1.426" shouldn't be needed
|
||||
|
||||
Results are in
|
||||
- <repo root>\src\BenchmarkDotNet.Artifacts
|
||||
Microsoft.PowerFx.Performance.Tests.<test class>-<date>-<time>.log
|
||||
Microsoft.PowerFx.Performance.Tests.<test class and name (+opt. N)>-<date>-<time>.etl
|
||||
- <repo root>\src\BenchmarkDotNet.Artifacts\results
|
||||
Microsoft.PowerFx.Performance.Tests.<test class>-report-github.md
|
||||
Microsoft.PowerFx.Performance.Tests.<test class>-report.html
|
||||
Microsoft.PowerFx.Performance.Tests.<test class>-report.csv
|
||||
|
||||
*/
|
||||
|
||||
var summary = BenchmarkRunner.Run<PerformanceTest1>();
|
||||
_ = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using BenchmarkDotNet.Diagnostics.Windows.Configs;
|
||||
using BenchmarkDotNet.Engines;
|
||||
using BenchmarkDotNet.Jobs;
|
||||
using Microsoft.PowerFx.Syntax;
|
||||
|
||||
namespace Microsoft.PowerFx.Performance.Tests
|
||||
{
|
||||
[MemoryDiagnoser]
|
||||
[NativeMemoryProfiler]
|
||||
[EtwProfiler] // https://benchmarkdotnet.org/articles/features/etwprofiler.html
|
||||
[CsvExporter] // https://benchmarkdotnet.org/articles/configs/exporters.html
|
||||
[MinColumn]
|
||||
[Q1Column]
|
||||
[MeanColumn]
|
||||
[MedianColumn]
|
||||
[Q3Column]
|
||||
[MaxColumn]
|
||||
[SimpleJob(runtimeMoniker: RuntimeMoniker.NetCoreApp31)]
|
||||
|
||||
// The objective of this set of tests is to serve as a reference in order to
|
||||
// compare results between them if the server on which they are run have
|
||||
// different performances
|
||||
// - FixedDuration is supposed to take 100ms, always
|
||||
// - FixedLoop will always have a fixed set of instructions and its duration
|
||||
// should only depend on the CPU speed
|
||||
public class Reference
|
||||
{
|
||||
[GlobalSetup]
|
||||
public void GlobalSetup()
|
||||
{
|
||||
CultureInfo.CurrentCulture = new CultureInfo("en-US");
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
[MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)]
|
||||
public int FixedDuration()
|
||||
{
|
||||
// Fixed duration, independant of CPU speed
|
||||
Thread.Sleep(100);
|
||||
return 0;
|
||||
}
|
||||
|
||||
[Benchmark]
|
||||
[MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)]
|
||||
public long FixedLoop()
|
||||
{
|
||||
var j = 0;
|
||||
for (var i = 0; i < 10000000; i++)
|
||||
{
|
||||
j += i;
|
||||
}
|
||||
|
||||
return j;
|
||||
}
|
||||
}
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче