Add `jit-tp-analyze` (#385)
This is a C# port of the `analyze-pin-trace-diff.ps1` script used to produce the summary of detailed (trace) PIN diffs. It has been noted in the past that such tooling would be beneficial to have in jitutils; this is a start. --------- Co-authored-by: Bruce Forstall <brucefo@microsoft.com>
This commit is contained in:
SingleAccretion
GitHub
Родитель
13e8beeff7
Коммит
597e75000c
|
|
@ -7,7 +7,7 @@ automate tasks when working on CoreCLR.
|
|||
|
||||
Current tools include:
|
||||
|
||||
1. [Assembly diffs](doc/diffs.md): jit-diff, jit-dasm, jit-dasm-pmi, jit-analyze.
|
||||
1. [Assembly diffs](doc/diffs.md): jit-diff, jit-dasm, jit-dasm-pmi, jit-analyze, jit-tp-analyze.
|
||||
2. [CI jobs information](doc/cijobs.md): cijobs.
|
||||
3. [JIT source code formatting](doc/formatting.md): jit-format.
|
||||
4. [General tools](doc/tools.md): pmi
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ REM Do as many builds as possible; don't stop on first failure (if any).
|
|||
set __ExitCode=0
|
||||
|
||||
REM Declare the list of projects
|
||||
set projects=jit-diff jit-dasm jit-analyze jit-format pmi jit-dasm-pmi jit-decisions-analyze performance-explorer instructions-retired-explorer
|
||||
set projects=jit-diff jit-dasm jit-analyze jit-tp-analyze jit-format pmi jit-dasm-pmi jit-decisions-analyze performance-explorer instructions-retired-explorer
|
||||
|
||||
REM Build each project
|
||||
for %%p in (%projects%) do (
|
||||
|
|
|
|||
2
build.sh
2
build.sh
|
|
@ -48,7 +48,7 @@ while getopts "hpb:" opt; do
|
|||
done
|
||||
|
||||
# declare the array of projects
|
||||
declare -a projects=(jit-dasm jit-diff jit-analyze jit-format pmi jit-dasm-pmi jit-decisions-analyze performance-explorer instructions-retired-explorer)
|
||||
declare -a projects=(jit-dasm jit-diff jit-analyze jit-tp-analyze jit-format pmi jit-dasm-pmi jit-decisions-analyze performance-explorer instructions-retired-explorer)
|
||||
|
||||
# for each project either build or publish
|
||||
for proj in "${projects[@]}"
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ See the [CoreCLR](https://github.com/dotnet/coreclr) GitHub repo for directions
|
|||
* jit-analyze - Compare and analyze `*.dasm` files from baseline/diff.
|
||||
Produces a report on diffs, total size regression/improvement, and
|
||||
size regression/improvement by file and method.
|
||||
* jit-tp-analyze - Compare trace files with per-function instruction
|
||||
counts from baseline/diff. See [jit-tp-analyze.md](../src/jit-tp-analyze/README.md).
|
||||
|
||||
## Dependencies
|
||||
|
||||
|
|
|
|||
19
jitutils.sln
19
jitutils.sln
|
|
@ -1,7 +1,7 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio Version 16
|
||||
VisualStudioVersion = 16.0.30011.22
|
||||
# Visual Studio Version 17
|
||||
VisualStudioVersion = 17.7.34018.315
|
||||
MinimumVisualStudioVersion = 15.0.26124.0
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{69D45FDC-948D-464B-BF14-5675F291FC62}"
|
||||
EndProject
|
||||
|
|
@ -23,6 +23,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "jit-decisions-analyze", "sr
|
|||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AnalyzeAsm", "src\AnalyzeAsm\AnalyzeAsm.csproj", "{94C5BAFE-F285-45A7-85E8-6A1E9A8C1745}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "jit-tp-analyze", "src\jit-tp-analyze\jit-tp-analyze.csproj", "{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
|
|
@ -141,6 +143,18 @@ Global
|
|||
{94C5BAFE-F285-45A7-85E8-6A1E9A8C1745}.Release|x64.Build.0 = Release|Any CPU
|
||||
{94C5BAFE-F285-45A7-85E8-6A1E9A8C1745}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{94C5BAFE-F285-45A7-85E8-6A1E9A8C1745}.Release|x86.Build.0 = Release|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|x64.Build.0 = Release|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
@ -155,6 +169,7 @@ Global
|
|||
{2C846294-79D4-40EA-8ED3-0F01E033BE32} = {69D45FDC-948D-464B-BF14-5675F291FC62}
|
||||
{59FB3663-C16C-4C3B-90A7-AF7259C8A39B} = {69D45FDC-948D-464B-BF14-5675F291FC62}
|
||||
{94C5BAFE-F285-45A7-85E8-6A1E9A8C1745} = {69D45FDC-948D-464B-BF14-5675F291FC62}
|
||||
{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897} = {69D45FDC-948D-464B-BF14-5675F291FC62}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {678D84CC-4243-4F8C-84DB-80BE89656529}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,175 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.CommandLine;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace ManagedCodeGen;
|
||||
|
||||
internal class Program
|
||||
{
|
||||
private static readonly Regex _traceLineRegex = new("(\\d+) +: (.*)", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
private readonly string _baseTracePath;
|
||||
private readonly string _diffTracePath;
|
||||
private readonly double _noise;
|
||||
|
||||
public Program(JitTpAnalyzeRootCommand command, ParseResult result)
|
||||
{
|
||||
_baseTracePath = result.GetValue(command.BasePath);
|
||||
_diffTracePath = result.GetValue(command.DiffPath);
|
||||
_noise = result.GetValue(command.Noise);
|
||||
}
|
||||
|
||||
public void Run()
|
||||
{
|
||||
TextWriter output = Console.Out;
|
||||
Dictionary<string, long> baseTrace = ParseTrace(_baseTracePath);
|
||||
Dictionary<string, long> diffTrace = ParseTrace(_diffTracePath);
|
||||
HashSet<string> allRecordedFunctions = new();
|
||||
foreach (var function in baseTrace)
|
||||
{
|
||||
allRecordedFunctions.Add(function.Key);
|
||||
}
|
||||
foreach (var function in diffTrace)
|
||||
{
|
||||
allRecordedFunctions.Add(function.Key);
|
||||
}
|
||||
|
||||
long baseTotalInsCount = baseTrace.Sum(x => x.Value);
|
||||
long diffTotalInsCount = diffTrace.Sum(x => x.Value);
|
||||
double totalPercentageDiff = GetPercentageDiff(baseTotalInsCount, diffTotalInsCount);
|
||||
output.WriteLine($"Base: {baseTotalInsCount}, Diff: {diffTotalInsCount}, {FormatPercentageDiff(totalPercentageDiff, "0000")}");
|
||||
output.WriteLine();
|
||||
|
||||
// Now create a list of functions which contributed to the difference.
|
||||
long totalAbsInsCountDiff = 0;
|
||||
List<FunctionDiff> diffs = new();
|
||||
foreach (string functionName in allRecordedFunctions)
|
||||
{
|
||||
long diffInsCount = diffTrace.GetValueOrDefault(functionName);
|
||||
long baseInsCount = baseTrace.GetValueOrDefault(functionName);
|
||||
long insCountDiff = diffInsCount - baseInsCount;
|
||||
if (insCountDiff == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
diffs.Add(new()
|
||||
{
|
||||
Name = functionName,
|
||||
InsCountDiff = insCountDiff,
|
||||
InsPercentageDiff = GetPercentageDiff(baseInsCount, diffInsCount),
|
||||
TotalInsPercentageDiff = (double)insCountDiff / baseTotalInsCount * 100
|
||||
});
|
||||
|
||||
totalAbsInsCountDiff += Math.Abs(insCountDiff);
|
||||
}
|
||||
|
||||
foreach (ref FunctionDiff diff in CollectionsMarshal.AsSpan(diffs))
|
||||
{
|
||||
diff.ContributionPercentage = (double)Math.Abs(diff.InsCountDiff) / totalAbsInsCountDiff * 100;
|
||||
}
|
||||
|
||||
// Filter out functions below the noise level.
|
||||
diffs = diffs.Where(d => d.ContributionPercentage > _noise).ToList();
|
||||
diffs.Sort((x, y) => y.InsCountDiff.CompareTo(x.InsCountDiff));
|
||||
|
||||
int maxNameLength = 0;
|
||||
int maxInsCountDiffLength = 0;
|
||||
int maxInsPercentageDiffLength = 0;
|
||||
foreach (ref FunctionDiff diff in CollectionsMarshal.AsSpan(diffs))
|
||||
{
|
||||
maxNameLength = Math.Max(maxNameLength, diff.Name.Length);
|
||||
maxInsCountDiffLength = Math.Max(maxInsCountDiffLength, $"{diff.InsCountDiff}".Length);
|
||||
maxInsPercentageDiffLength = Math.Max(maxInsPercentageDiffLength, FormatPercentageDiff(diff.InsPercentageDiff).Length);
|
||||
}
|
||||
|
||||
foreach (ref FunctionDiff diff in CollectionsMarshal.AsSpan(diffs))
|
||||
{
|
||||
output.WriteLine(
|
||||
$"{{0,-{maxNameLength}}} : {{1,-{maxInsCountDiffLength}}} : {{2,-{maxInsPercentageDiffLength}}} : {{3,-6:P2}} : {{4}}",
|
||||
diff.Name,
|
||||
diff.InsCountDiff,
|
||||
double.IsInfinity(diff.InsPercentageDiff) ? "NA" : FormatPercentageDiff(diff.InsPercentageDiff),
|
||||
diff.ContributionPercentage / 100,
|
||||
FormatPercentageDiff(diff.TotalInsPercentageDiff, "0000"));
|
||||
}
|
||||
}
|
||||
|
||||
private Dictionary<string, long> ParseTrace(string path)
|
||||
{
|
||||
Dictionary<string, long> trace = new();
|
||||
foreach (string line in File.ReadLines(path))
|
||||
{
|
||||
Match match = _traceLineRegex.Match(line);
|
||||
if (match.Success)
|
||||
{
|
||||
trace.Add(match.Groups[2].Value, long.Parse(match.Groups[1].Value));
|
||||
}
|
||||
}
|
||||
|
||||
return trace;
|
||||
}
|
||||
|
||||
private static double GetPercentageDiff(double baseValue, double diffValue) =>
|
||||
(diffValue - baseValue) / baseValue * 100;
|
||||
|
||||
private static string FormatPercentageDiff(double value, string precision = "00") =>
|
||||
(value > 0 ? "+" : "") + value.ToString($"0.{precision}") + "%";
|
||||
|
||||
private static void Main(string[] args) =>
|
||||
new CliConfiguration(new JitTpAnalyzeRootCommand().UseVersion()).Invoke(args);
|
||||
|
||||
private struct FunctionDiff
|
||||
{
|
||||
public string Name;
|
||||
public long InsCountDiff;
|
||||
public double InsPercentageDiff;
|
||||
public double ContributionPercentage;
|
||||
public double TotalInsPercentageDiff;
|
||||
}
|
||||
}
|
||||
|
||||
internal class JitTpAnalyzeRootCommand : CliRootCommand
|
||||
{
|
||||
public JitTpAnalyzeRootCommand() : base("Compare PIN-based throughput traces")
|
||||
{
|
||||
Options.Add(BasePath);
|
||||
Options.Add(DiffPath);
|
||||
Options.Add(Noise);
|
||||
|
||||
SetAction(result =>
|
||||
{
|
||||
try
|
||||
{
|
||||
Program jitTpDiff = new(this, result);
|
||||
jitTpDiff.Run();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.ResetColor();
|
||||
Console.ForegroundColor = ConsoleColor.Red;
|
||||
Console.Error.WriteLine("Error: " + e.Message);
|
||||
Console.Error.WriteLine(e.ToString());
|
||||
Console.ResetColor();
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
public CliOption<string> BasePath { get; } =
|
||||
new("--base", "-b") { Description = "Base trace file", DefaultValueFactory = (_) => "basetp.txt" };
|
||||
public CliOption<string> DiffPath { get; } =
|
||||
new("--diff", "-d") { Description = "Diff trace file", DefaultValueFactory = (_) => "difftp.txt" };
|
||||
public CliOption<double> Noise { get; } =
|
||||
new("--noise", "-n") { Description = "Minimal contribution percentage for inclusion into the summary", DefaultValueFactory = (_) => 0.1 };
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
# jit-tp-analyze - throughput difference analysis tool
|
||||
|
||||
jit-tp-analyze is a utility to parse traces generated by PIN-based
|
||||
instrumentation over runs of the JIT. The tool reads all lines in
|
||||
the following format from the two input files:
|
||||
```
|
||||
<Exclusive instruction count> : <Method name>
|
||||
```
|
||||
The tool ignores all lines that do not match this pattern and so can be
|
||||
run directly against superpmi.exe's usual output.
|
||||
|
||||
The tool produces the following summary:
|
||||
```
|
||||
Base: 1039322782, Diff: 1040078986, +0.0728%
|
||||
|
||||
`Compiler::optCopyPropPushDef'::`2'::<lambda_1>::operator() : 1073512 : NA : 18.17% : +0.1033%
|
||||
SsaBuilder::RenamePushDef : 911022 : NA : 15.42% : +0.0877%
|
||||
`Compiler::fgValueNumberLocalStore'::`2'::<lambda_1>::operator() : 584435 : NA : 9.89% : +0.0562%
|
||||
Compiler::lvaLclExactSize : 244692 : +60.09% : 4.14% : +0.0235%
|
||||
ValueNumStore::VNForMapSelectWork : 87006 : +2.78% : 1.47% : +0.0084%
|
||||
GenTree::DefinesLocal : 82633 : +1.63% : 1.40% : +0.0080%
|
||||
Rationalizer::DoPhase : -91104 : -6.36% : 1.54% : -0.0088%
|
||||
Compiler::gtCallGetDefinedRetBufLclAddr : -115926 : -98.78% : 1.96% : -0.0112%
|
||||
Compiler::optBlockCopyProp : -272450 : -5.75% : 4.61% : -0.0262%
|
||||
Compiler::fgValueNumberLocalStore : -313540 : -50.82% : 5.31% : -0.0302%
|
||||
Compiler::GetSsaNumForLocalVarDef : -322826 : -100.00% : 5.46% : -0.0311%
|
||||
SsaBuilder::RenameDef : -478441 : -28.33% : 8.10% : -0.0460%
|
||||
Compiler::optCopyPropPushDef : -711380 : -55.34% : 12.04% : -0.0684%
|
||||
```
|
||||
The columns, in order:
|
||||
1. Method name.
|
||||
2. The instruction count difference for the given function.
|
||||
3. Same as `1`, but relative. May be `NA`, indicating the base didn't contain the given function, or `-100%` indicating the diff didn't.
|
||||
4. Relative contribution to the diff. Calculated as `abs(instruction diff count) / sum-over-all-functions(abs(instruction diff count))`.
|
||||
5. Relative difference, calculated as `instruction diff count / total base instruction count`.
|
||||
|
||||
To use:
|
||||
1. Obtain the base and diff traces, by compiling and running a PIN tool that counts instructions retired for each function.
|
||||
2. Invoke `./jit-tp-analyze --base base-trace.txt --diff diff-trace.txt`.
|
||||
|
||||
For convenience, both arguments have default values: `basetp.txt` for `--base`, `difftp.txt` for `--diff`, and so can be omitted.
|
||||
|
||||
By default, the tool will hide functions that contributed less than `0.1%` to the difference. You can change this value with the `--noise` argument.
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<Import Project="$([MSBuild]::GetPathOfFileAbove(target-framework.props))" />
|
||||
<Import Project="$([MSBuild]::GetPathOfFileAbove(jit-include.props))" />
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
Загрузка…
Ссылка в новой задаче