From 2dadfbbd2ecee637c4479878f2f4737aa8da08d2 Mon Sep 17 00:00:00 2001 From: Simon Nattress Date: Fri, 7 Dec 2018 16:06:42 -0800 Subject: [PATCH] Improvements to R2R Test Harness (#6653) To support multi-core testing, associate the test process's PID with the ETW filter so that only assemblies loaded by the runtime the harness started are considered. Emit the assembly name for methods that are jitted to allow grouping of methods by their assembly. Add a new parameter, `--include` which configures the ETW filter to include all assemblies in a given folder, instead of having to specify a large number of assemblies individually. Bug fixes around path casing causing events to be dropped. --- .../ReadyToRun.TestHarness/PathHelpers.cs | 34 ++++++++++++++ .../tools/ReadyToRun.TestHarness/Program.cs | 47 +++++++++++++++---- .../ReadyToRunEtlMethodFilter.cs | 41 +++++++++++++--- 3 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 tests/src/tools/ReadyToRun.TestHarness/PathHelpers.cs diff --git a/tests/src/tools/ReadyToRun.TestHarness/PathHelpers.cs b/tests/src/tools/ReadyToRun.TestHarness/PathHelpers.cs new file mode 100644 index 000000000..155ca2623 --- /dev/null +++ b/tests/src/tools/ReadyToRun.TestHarness/PathHelpers.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +using System; +using System.IO; + +namespace ReadyToRun.TestHarness +{ + /// + /// A set of helper to manipulate paths into a canonicalized form to ensure user-provided paths + /// match those in the ETW log. + /// + static class PathExtensions + { + internal static string ToAbsolutePath(this string argValue) => Path.GetFullPath(argValue); + + internal static string ToAbsoluteDirectoryPath(this string argValue) => argValue.ToAbsolutePath().StripTrailingDirectorySeparators(); + + internal static string StripTrailingDirectorySeparators(this string str) + { + if (String.IsNullOrWhiteSpace(str)) + { + return str; + } + + while (str.Length > 0 && str[str.Length - 1] == Path.DirectorySeparatorChar) + { + str = str.Remove(str.Length - 1); + } + + return str; + } + } +} diff --git a/tests/src/tools/ReadyToRun.TestHarness/Program.cs b/tests/src/tools/ReadyToRun.TestHarness/Program.cs index d2d066c55..6ffe766f1 100644 --- a/tests/src/tools/ReadyToRun.TestHarness/Program.cs +++ b/tests/src/tools/ReadyToRun.TestHarness/Program.cs @@ -39,13 +39,14 @@ namespace ReadyToRun.TestHarness private static string _coreRunExePath; private static string _testExe; private static IReadOnlyList _referenceFilenames = Array.Empty(); + private static IReadOnlyList _referenceFolders = Array.Empty(); private static string _whitelistFilename; private static IReadOnlyList _testargs = Array.Empty(); private static bool _noEtl; static void ShowUsage() { - Console.WriteLine("dotnet ReadyToRun.TestHarness --corerun --in --ref [ReferencedBinaries] --whitelist [MethodWhiteListFile] --testargs [TestArgs]"); + Console.WriteLine("dotnet ReadyToRun.TestHarness --corerun --in --ref [ReferencedBinaries] --whitelist [MethodWhiteListFile] --testargs [TestArgs] --include [FoldersContainingAssemblies]"); } private static ArgumentSyntax ParseCommandLine(string[] args) @@ -55,11 +56,13 @@ namespace ReadyToRun.TestHarness syntax.ApplicationName = "ReadyToRun.TestHarness"; syntax.HandleHelp = false; syntax.HandleErrors = true; + syntax.HandleResponseFiles = true; syntax.DefineOption("h|help", ref _help, "Help message for R2RDump"); syntax.DefineOption("c|corerun", ref _coreRunExePath, "Path to CoreRun"); syntax.DefineOption("i|in", ref _testExe, "Path to test exe"); syntax.DefineOptionList("r|ref", ref _referenceFilenames, "Paths to referenced assemblies"); + syntax.DefineOptionList("include", ref _referenceFolders, "Folders containing assemblies to monitor"); syntax.DefineOption("w|whitelist", ref _whitelistFilename, "Path to method whitelist file"); syntax.DefineOptionList("testargs", ref _testargs, "Args to pass into test"); syntax.DefineOption ("noetl", ref _noEtl, "Run the test without ETL enabled"); @@ -100,24 +103,39 @@ namespace ReadyToRun.TestHarness } var testModules = new HashSet(); - testModules.Add(_testExe); + var testFolders = new HashSet(); + testModules.Add(_testExe.ToLower()); foreach (string reference in _referenceFilenames) { - testModules.Add(reference); + // CoreCLR generates ETW events with all lower case native image files that break our string comparison. + testModules.Add(reference.ToLower()); + } + + foreach (string reference in _referenceFolders) + { + string absolutePath = reference.ToAbsoluteDirectoryPath(); + + if (!Directory.Exists(reference)) + { + Console.WriteLine($"Error: {reference} does not exist."); + ShowUsage(); + return exitCode; + } + testFolders.Add(reference.ToLower()); } if (_noEtl) { - RunTest(null, passThroughArguments, out exitCode); + RunTest(null, null, passThroughArguments, out exitCode); } else { using (var session = new TraceEventSession("ReadyToRunTestSession")) { - var r2rMethodFilter = new ReadyToRunJittedMethods(session, testModules); + var r2rMethodFilter = new ReadyToRunJittedMethods(session, testModules, testFolders); session.EnableProvider(ClrTraceEventParser.ProviderGuid, TraceEventLevel.Verbose, (ulong)(ClrTraceEventParser.Keywords.Jit | ClrTraceEventParser.Keywords.Loader)); - Task.Run(() => RunTest(session, passThroughArguments, out exitCode)); + Task.Run(() => RunTest(session, r2rMethodFilter, passThroughArguments, out exitCode)); // Block, processing callbacks for events we subscribed to session.Source.Process(); @@ -183,9 +201,10 @@ namespace ReadyToRun.TestHarness int jittedMethodCount = 0; foreach (var jittedMethod in jittedMethods.JittedMethods) { - if (!whiteListedMethods.Contains(jittedMethod.MethodName)) + string fullName = GetFullModuleName(jittedMethod.assemblyName, jittedMethod.MethodName); + if (!whiteListedMethods.Contains(jittedMethod.MethodName) && !whiteListedMethods.Contains(fullName)) { - Console.WriteLine(jittedMethod.MethodName); + Console.WriteLine(fullName); ++jittedMethodCount; } } @@ -203,7 +222,12 @@ namespace ReadyToRun.TestHarness return StatusTestPassed; } - private static void RunTest(TraceEventSession session, string testArguments, out int exitCode) + private static string GetFullModuleName(string moduleName, string methodName) + { + return $"[{moduleName}]{methodName}"; + } + + private static void RunTest(TraceEventSession session, ReadyToRunJittedMethods r2rMethodFilter, string testArguments, out int exitCode) { exitCode = -100; @@ -217,6 +241,11 @@ namespace ReadyToRun.TestHarness process.Start(); + if (r2rMethodFilter != null) + { + r2rMethodFilter.SetProcessId(process.Id); + } + process.OutputDataReceived += delegate(object sender, DataReceivedEventArgs args) { Console.WriteLine(args.Data); diff --git a/tests/src/tools/ReadyToRun.TestHarness/ReadyToRunEtlMethodFilter.cs b/tests/src/tools/ReadyToRun.TestHarness/ReadyToRunEtlMethodFilter.cs index 7fd8dd75e..9a0010f50 100644 --- a/tests/src/tools/ReadyToRun.TestHarness/ReadyToRunEtlMethodFilter.cs +++ b/tests/src/tools/ReadyToRun.TestHarness/ReadyToRunEtlMethodFilter.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.IO; using System.Collections.Generic; using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Parsers.Clr; @@ -19,33 +20,61 @@ namespace ReadyToRun.TestHarness class ReadyToRunJittedMethods { private ICollection _testModuleNames; + private ICollection _testFolderNames; private List _testModuleIds = new List(); - private List<(string, bool)> _methodsJitted = new List<(string name, bool readyToRunRejected)>(); + private Dictionary _testModuleIdToName = new Dictionary(); + private List<(string, string, bool)> _methodsJitted = new List<(string name, string moduleName, bool readyToRunRejected)>(); + private int _pid = -1; - public ReadyToRunJittedMethods(TraceEventSession session, ICollection testModuleNames) + public ReadyToRunJittedMethods(TraceEventSession session, ICollection testModuleNames, ICollection testFolderNames) { _testModuleNames = testModuleNames; + _testFolderNames = testFolderNames; session.Source.Clr.LoaderModuleLoad += delegate(ModuleLoadUnloadTraceData data) { - if (_testModuleNames.Contains(data.ModuleILPath) || _testModuleNames.Contains(data.ModuleNativePath)) + if (ShouldMonitorModule(data)) { Console.WriteLine($"Tracking module {data.ModuleILFileName} with Id {data.ModuleID}"); _testModuleIds.Add(data.ModuleID); + _testModuleIdToName[data.ModuleID] = Path.GetFileNameWithoutExtension(data.ModuleILFileName); } }; session.Source.Clr.MethodLoadVerbose += delegate (MethodLoadUnloadVerboseTraceData data) { - if (_testModuleIds.Contains(data.ModuleID) && data.IsJitted) + if (data.ProcessID == _pid && _testModuleIds.Contains(data.ModuleID) && data.IsJitted) { Console.WriteLine($"Method loaded {GetName(data)} - {data}"); - _methodsJitted.Add((GetName(data), ((int)data.MethodFlags & 0x40) != 0)); + _methodsJitted.Add((GetName(data), _testModuleIdToName[data.ModuleID], ((int)data.MethodFlags & 0x40) != 0)); } }; } - public IEnumerable<(string MethodName, bool ReadyToRunRejected)> JittedMethods => _methodsJitted; + private bool ShouldMonitorModule(ModuleLoadUnloadTraceData data) + { + if (data.ProcessID != _pid) + return false; + + if (File.Exists(data.ModuleILPath) && _testFolderNames.Contains(Path.GetDirectoryName(data.ModuleILPath).ToAbsoluteDirectoryPath().ToLower())) + return true; + + if (_testModuleNames.Contains(data.ModuleILPath.ToLower()) || _testModuleNames.Contains(data.ModuleNativePath.ToLower())) + return true; + + return false; + } + + /// + /// Set the process to monitor events for given its Id. This should be set immediately after + /// calling Process.Start to ensure no module load events are missed for the runtime instance. + /// + public void SetProcessId(int pid) + { + _pid = pid; + } + + public IEnumerable<(string MethodName, string assemblyName, bool ReadyToRunRejected)> JittedMethods => _methodsJitted; /// /// Returns the number of test assemblies that were loaded by the runtime