diff --git a/README.md b/README.md index c707199..72ead85 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ automate tasks when working on CoreCLR. Current tools include: -1. [Assembly diffs](doc/diffs.md): jit-diff, jit-dasm, jit-dasm-pmi, jit-analyze. +1. [Assembly diffs](doc/diffs.md): jit-diff, jit-dasm, jit-dasm-pmi, jit-analyze, jit-tp-analyze. 2. [CI jobs information](doc/cijobs.md): cijobs. 3. [JIT source code formatting](doc/formatting.md): jit-format. 4. [General tools](doc/tools.md): pmi diff --git a/build.cmd b/build.cmd index ab94725..02badd0 100644 --- a/build.cmd +++ b/build.cmd @@ -44,7 +44,7 @@ REM Do as many builds as possible; don't stop on first failure (if any). set __ExitCode=0 REM Declare the list of projects -set projects=jit-diff jit-dasm jit-analyze jit-format pmi jit-dasm-pmi jit-decisions-analyze performance-explorer instructions-retired-explorer +set projects=jit-diff jit-dasm jit-analyze jit-tp-analyze jit-format pmi jit-dasm-pmi jit-decisions-analyze performance-explorer instructions-retired-explorer REM Build each project for %%p in (%projects%) do ( diff --git a/build.sh b/build.sh index c2ee4ad..f5813c2 100755 --- a/build.sh +++ b/build.sh @@ -48,7 +48,7 @@ while getopts "hpb:" opt; do done # declare the array of projects -declare -a projects=(jit-dasm jit-diff jit-analyze jit-format pmi jit-dasm-pmi jit-decisions-analyze performance-explorer instructions-retired-explorer) +declare -a projects=(jit-dasm jit-diff jit-analyze jit-tp-analyze jit-format pmi jit-dasm-pmi jit-decisions-analyze performance-explorer instructions-retired-explorer) # for each project either build or publish for proj in "${projects[@]}" diff --git a/doc/diffs.md b/doc/diffs.md index b52c8e4..9c90716 100644 --- a/doc/diffs.md +++ b/doc/diffs.md @@ -22,6 +22,8 @@ See the [CoreCLR](https://github.com/dotnet/coreclr) GitHub repo for directions * jit-analyze - Compare and analyze `*.dasm` files from baseline/diff. Produces a report on diffs, total size regression/improvement, and size regression/improvement by file and method. +* jit-tp-analyze - Compare trace files with per-function instruction + counts from baseline/diff. See [jit-tp-analyze.md](../src/jit-tp-analyze/README.md). ## Dependencies diff --git a/jitutils.sln b/jitutils.sln index 3a95093..540a098 100644 --- a/jitutils.sln +++ b/jitutils.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30011.22 +# Visual Studio Version 17 +VisualStudioVersion = 17.7.34018.315 MinimumVisualStudioVersion = 15.0.26124.0 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{69D45FDC-948D-464B-BF14-5675F291FC62}" EndProject @@ -23,6 +23,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "jit-decisions-analyze", "sr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AnalyzeAsm", "src\AnalyzeAsm\AnalyzeAsm.csproj", "{94C5BAFE-F285-45A7-85E8-6A1E9A8C1745}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "jit-tp-analyze", "src\jit-tp-analyze\jit-tp-analyze.csproj", "{DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -141,6 +143,18 @@ Global {94C5BAFE-F285-45A7-85E8-6A1E9A8C1745}.Release|x64.Build.0 = Release|Any CPU {94C5BAFE-F285-45A7-85E8-6A1E9A8C1745}.Release|x86.ActiveCfg = Release|Any CPU {94C5BAFE-F285-45A7-85E8-6A1E9A8C1745}.Release|x86.Build.0 = Release|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|x64.ActiveCfg = Debug|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|x64.Build.0 = Debug|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|x86.ActiveCfg = Debug|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Debug|x86.Build.0 = Debug|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|Any CPU.Build.0 = Release|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|x64.ActiveCfg = Release|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|x64.Build.0 = Release|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|x86.ActiveCfg = Release|Any CPU + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -155,6 +169,7 @@ Global {2C846294-79D4-40EA-8ED3-0F01E033BE32} = {69D45FDC-948D-464B-BF14-5675F291FC62} {59FB3663-C16C-4C3B-90A7-AF7259C8A39B} = {69D45FDC-948D-464B-BF14-5675F291FC62} {94C5BAFE-F285-45A7-85E8-6A1E9A8C1745} = {69D45FDC-948D-464B-BF14-5675F291FC62} + {DC0B9E30-DE1B-4C8A-BC2E-3653941F9897} = {69D45FDC-948D-464B-BF14-5675F291FC62} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {678D84CC-4243-4F8C-84DB-80BE89656529} diff --git a/src/jit-tp-analyze/Program.cs b/src/jit-tp-analyze/Program.cs new file mode 100644 index 0000000..8177e2a --- /dev/null +++ b/src/jit-tp-analyze/Program.cs @@ -0,0 +1,175 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; + +namespace ManagedCodeGen; + +internal class Program +{ + private static readonly Regex _traceLineRegex = new("(\\d+) +: (.*)", RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private readonly string _baseTracePath; + private readonly string _diffTracePath; + private readonly double _noise; + + public Program(JitTpAnalyzeRootCommand command, ParseResult result) + { + _baseTracePath = result.GetValue(command.BasePath); + _diffTracePath = result.GetValue(command.DiffPath); + _noise = result.GetValue(command.Noise); + } + + public void Run() + { + TextWriter output = Console.Out; + Dictionary baseTrace = ParseTrace(_baseTracePath); + Dictionary diffTrace = ParseTrace(_diffTracePath); + HashSet allRecordedFunctions = new(); + foreach (var function in baseTrace) + { + allRecordedFunctions.Add(function.Key); + } + foreach (var function in diffTrace) + { + allRecordedFunctions.Add(function.Key); + } + + long baseTotalInsCount = baseTrace.Sum(x => x.Value); + long diffTotalInsCount = diffTrace.Sum(x => x.Value); + double totalPercentageDiff = GetPercentageDiff(baseTotalInsCount, diffTotalInsCount); + output.WriteLine($"Base: {baseTotalInsCount}, Diff: {diffTotalInsCount}, {FormatPercentageDiff(totalPercentageDiff, "0000")}"); + output.WriteLine(); + + // Now create a list of functions which contributed to the difference. + long totalAbsInsCountDiff = 0; + List diffs = new(); + foreach (string functionName in allRecordedFunctions) + { + long diffInsCount = diffTrace.GetValueOrDefault(functionName); + long baseInsCount = baseTrace.GetValueOrDefault(functionName); + long insCountDiff = diffInsCount - baseInsCount; + if (insCountDiff == 0) + { + continue; + } + + diffs.Add(new() + { + Name = functionName, + InsCountDiff = insCountDiff, + InsPercentageDiff = GetPercentageDiff(baseInsCount, diffInsCount), + TotalInsPercentageDiff = (double)insCountDiff / baseTotalInsCount * 100 + }); + + totalAbsInsCountDiff += Math.Abs(insCountDiff); + } + + foreach (ref FunctionDiff diff in CollectionsMarshal.AsSpan(diffs)) + { + diff.ContributionPercentage = (double)Math.Abs(diff.InsCountDiff) / totalAbsInsCountDiff * 100; + } + + // Filter out functions below the noise level. + diffs = diffs.Where(d => d.ContributionPercentage > _noise).ToList(); + diffs.Sort((x, y) => y.InsCountDiff.CompareTo(x.InsCountDiff)); + + int maxNameLength = 0; + int maxInsCountDiffLength = 0; + int maxInsPercentageDiffLength = 0; + foreach (ref FunctionDiff diff in CollectionsMarshal.AsSpan(diffs)) + { + maxNameLength = Math.Max(maxNameLength, diff.Name.Length); + maxInsCountDiffLength = Math.Max(maxInsCountDiffLength, $"{diff.InsCountDiff}".Length); + maxInsPercentageDiffLength = Math.Max(maxInsPercentageDiffLength, FormatPercentageDiff(diff.InsPercentageDiff).Length); + } + + foreach (ref FunctionDiff diff in CollectionsMarshal.AsSpan(diffs)) + { + output.WriteLine( + $"{{0,-{maxNameLength}}} : {{1,-{maxInsCountDiffLength}}} : {{2,-{maxInsPercentageDiffLength}}} : {{3,-6:P2}} : {{4}}", + diff.Name, + diff.InsCountDiff, + double.IsInfinity(diff.InsPercentageDiff) ? "NA" : FormatPercentageDiff(diff.InsPercentageDiff), + diff.ContributionPercentage / 100, + FormatPercentageDiff(diff.TotalInsPercentageDiff, "0000")); + } + } + + private Dictionary ParseTrace(string path) + { + Dictionary trace = new(); + foreach (string line in File.ReadLines(path)) + { + Match match = _traceLineRegex.Match(line); + if (match.Success) + { + trace.Add(match.Groups[2].Value, long.Parse(match.Groups[1].Value)); + } + } + + return trace; + } + + private static double GetPercentageDiff(double baseValue, double diffValue) => + (diffValue - baseValue) / baseValue * 100; + + private static string FormatPercentageDiff(double value, string precision = "00") => + (value > 0 ? "+" : "") + value.ToString($"0.{precision}") + "%"; + + private static void Main(string[] args) => + new CliConfiguration(new JitTpAnalyzeRootCommand().UseVersion()).Invoke(args); + + private struct FunctionDiff + { + public string Name; + public long InsCountDiff; + public double InsPercentageDiff; + public double ContributionPercentage; + public double TotalInsPercentageDiff; + } +} + +internal class JitTpAnalyzeRootCommand : CliRootCommand +{ + public JitTpAnalyzeRootCommand() : base("Compare PIN-based throughput traces") + { + Options.Add(BasePath); + Options.Add(DiffPath); + Options.Add(Noise); + + SetAction(result => + { + try + { + Program jitTpDiff = new(this, result); + jitTpDiff.Run(); + } + catch (Exception e) + { + Console.ResetColor(); + Console.ForegroundColor = ConsoleColor.Red; + Console.Error.WriteLine("Error: " + e.Message); + Console.Error.WriteLine(e.ToString()); + Console.ResetColor(); + return 1; + } + + return 0; + }); + } + + public CliOption BasePath { get; } = + new("--base", "-b") { Description = "Base trace file", DefaultValueFactory = (_) => "basetp.txt" }; + public CliOption DiffPath { get; } = + new("--diff", "-d") { Description = "Diff trace file", DefaultValueFactory = (_) => "difftp.txt" }; + public CliOption Noise { get; } = + new("--noise", "-n") { Description = "Minimal contribution percentage for inclusion into the summary", DefaultValueFactory = (_) => 0.1 }; +} diff --git a/src/jit-tp-analyze/README.md b/src/jit-tp-analyze/README.md new file mode 100644 index 0000000..f2ee545 --- /dev/null +++ b/src/jit-tp-analyze/README.md @@ -0,0 +1,43 @@ +# jit-tp-analyze - throughput difference analysis tool + +jit-tp-analyze is a utility to parse traces generated by PIN-based +instrumentation over runs of the JIT. The tool reads all lines in +the following format from the two input files: +``` + : +``` +The tool ignores all lines that do not match this pattern and so can be +run directly against superpmi.exe's usual output. + +The tool produces the following summary: +``` +Base: 1039322782, Diff: 1040078986, +0.0728% + +`Compiler::optCopyPropPushDef'::`2'::::operator() : 1073512 : NA : 18.17% : +0.1033% +SsaBuilder::RenamePushDef : 911022 : NA : 15.42% : +0.0877% +`Compiler::fgValueNumberLocalStore'::`2'::::operator() : 584435 : NA : 9.89% : +0.0562% +Compiler::lvaLclExactSize : 244692 : +60.09% : 4.14% : +0.0235% +ValueNumStore::VNForMapSelectWork : 87006 : +2.78% : 1.47% : +0.0084% +GenTree::DefinesLocal : 82633 : +1.63% : 1.40% : +0.0080% +Rationalizer::DoPhase : -91104 : -6.36% : 1.54% : -0.0088% +Compiler::gtCallGetDefinedRetBufLclAddr : -115926 : -98.78% : 1.96% : -0.0112% +Compiler::optBlockCopyProp : -272450 : -5.75% : 4.61% : -0.0262% +Compiler::fgValueNumberLocalStore : -313540 : -50.82% : 5.31% : -0.0302% +Compiler::GetSsaNumForLocalVarDef : -322826 : -100.00% : 5.46% : -0.0311% +SsaBuilder::RenameDef : -478441 : -28.33% : 8.10% : -0.0460% +Compiler::optCopyPropPushDef : -711380 : -55.34% : 12.04% : -0.0684% +``` +The columns, in order: +1. Method name. +2. The instruction count difference for the given function. +3. Same as `1`, but relative. May be `NA`, indicating the base didn't contain the given function, or `-100%` indicating the diff didn't. +4. Relative contribution to the diff. Calculated as `abs(instruction diff count) / sum-over-all-functions(abs(instruction diff count))`. +5. Relative difference, calculated as `instruction diff count / total base instruction count`. + +To use: +1. Obtain the base and diff traces, by compiling and running a PIN tool that counts instructions retired for each function. +2. Invoke `./jit-tp-analyze --base base-trace.txt --diff diff-trace.txt`. + +For convenience, both arguments have default values: `basetp.txt` for `--base`, `difftp.txt` for `--diff`, and so can be omitted. + +By default, the tool will hide functions that contributed less than `0.1%` to the difference. You can change this value with the `--noise` argument. diff --git a/src/jit-tp-analyze/jit-tp-analyze.csproj b/src/jit-tp-analyze/jit-tp-analyze.csproj new file mode 100644 index 0000000..fe8dbbe --- /dev/null +++ b/src/jit-tp-analyze/jit-tp-analyze.csproj @@ -0,0 +1,10 @@ + + + + + + + Exe + + +