diff --git a/AppInspector.CLI/CLICmdOptions.cs b/AppInspector.CLI/CLICmdOptions.cs index 1cda651..142a356 100644 --- a/AppInspector.CLI/CLICmdOptions.cs +++ b/AppInspector.CLI/CLICmdOptions.cs @@ -74,6 +74,9 @@ namespace Microsoft.ApplicationInspector.CLI [Option('A', "allow-all-tags-in-build-files", Required = false, HelpText = "Allow all tags (not just Metadata tags) in files of type Build.")] public bool AllowAllTagsInBuildFiles { get; internal set; } + + [Option('M', "max-num-matches-per-tag", Required = false, HelpText = "If non-zero, and TagsOnly is not set, will ignore rules based on if all of their tags have been found the set value number of times.")] + public int MaxNumMatchesPerTag { get; set; } = 0; } [Verb("tagdiff", HelpText = "Compares unique tag values between two source paths")] diff --git a/AppInspector.CLI/Program.cs b/AppInspector.CLI/Program.cs index 64f1946..d2e6a0c 100644 --- a/AppInspector.CLI/Program.cs +++ b/AppInspector.CLI/Program.cs @@ -262,7 +262,8 @@ namespace Microsoft.ApplicationInspector.CLI ScanUnknownTypes = cliOptions.ScanUnknownTypes, TagsOnly = cliOptions.TagsOnly, NoFileMetadata = cliOptions.NoFileMetadata, - AllowAllTagsInBuildFiles = cliOptions.AllowAllTagsInBuildFiles + AllowAllTagsInBuildFiles = cliOptions.AllowAllTagsInBuildFiles, + MaxNumMatchesPerTag = cliOptions.MaxNumMatchesPerTag }); if (!cliOptions.NoShowProgressBar) diff --git a/AppInspector/Commands/AnalyzeCommand.cs b/AppInspector/Commands/AnalyzeCommand.cs index 18a8125..43995d4 100644 --- a/AppInspector/Commands/AnalyzeCommand.cs +++ b/AppInspector/Commands/AnalyzeCommand.cs @@ -44,6 +44,10 @@ namespace Microsoft.ApplicationInspector.Commands public int ContextLines { get; set; } = 3; public bool ScanUnknownTypes { get; set; } public bool NoFileMetadata { get; set; } + /// + /// If non-zero, and is not set, will ignore rules based on if all of their tags have been found the set value number of times. + /// + public int MaxNumMatchesPerTag { get; set; } = 0; } /// @@ -294,6 +298,7 @@ namespace Microsoft.ApplicationInspector.Commands /// /// /// + [Obsolete("Instead PopulateRecords with no options argument and set the options when creating the AnalyzeCommand.")] public AnalyzeResult.ExitCode PopulateRecords(CancellationToken cancellationToken, AnalyzeOptions opts, IEnumerable populatedEntries) { WriteOnce.SafeLog("AnalyzeCommand::PopulateRecords", LogLevel.Trace); @@ -372,7 +377,7 @@ namespace Microsoft.ApplicationInspector.Commands { _metaDataHelper.AddLanguage("Unknown"); languageInfo = new LanguageInfo() { Extensions = new string[] { Path.GetExtension(file.FullPath) }, Name = "Unknown" }; - if (!_options.ScanUnknownTypes) + if (!opts.ScanUnknownTypes) { fileRecord.Status = ScanState.Skipped; } @@ -391,6 +396,10 @@ namespace Microsoft.ApplicationInspector.Commands { results = _rulesProcessor.AnalyzeFile(file, languageInfo, _metaDataHelper.UniqueTags.Keys, -1); } + else if (opts.MaxNumMatchesPerTag > 0) + { + results = _rulesProcessor.AnalyzeFile(file, languageInfo, _metaDataHelper.UniqueTags.Where(x => x.Value < opts.MaxNumMatchesPerTag).Select(x => x.Key), opts.ContextLines); + } else { results = _rulesProcessor.AnalyzeFile(file, languageInfo, null, opts.ContextLines); @@ -421,6 +430,10 @@ namespace Microsoft.ApplicationInspector.Commands { results = _rulesProcessor.AnalyzeFile(file, languageInfo, _metaDataHelper.UniqueTags.Keys, -1); } + else if (opts.MaxNumMatchesPerTag > 0) + { + results = _rulesProcessor.AnalyzeFile(file, languageInfo, _metaDataHelper.UniqueTags.Where(x => x.Value < opts.MaxNumMatchesPerTag).Select(x => x.Key), opts.ContextLines); + } else { results = _rulesProcessor.AnalyzeFile(file, languageInfo, null, opts.ContextLines); @@ -439,6 +452,13 @@ namespace Microsoft.ApplicationInspector.Commands { _metaDataHelper.AddTagsFromMatchRecord(matchRecord); } + else if (opts.MaxNumMatchesPerTag > 0) + { + if (matchRecord.Tags?.Any(x => _metaDataHelper.UniqueTags.TryGetValue(x, out int value) is bool foundValue && (!foundValue || foundValue && value < opts.MaxNumMatchesPerTag)) ?? true) + { + _metaDataHelper.AddMatchRecord(matchRecord); + } + } else { _metaDataHelper.AddMatchRecord(matchRecord); @@ -459,6 +479,14 @@ namespace Microsoft.ApplicationInspector.Commands } } + /// + /// Populate the MetaDataHelper with the data from the FileEntries + /// + /// + /// + /// + public AnalyzeResult.ExitCode PopulateRecords(CancellationToken cancellationToken, IEnumerable populatedEntries) => PopulateRecords(cancellationToken, _options, populatedEntries); + /// /// Populate the records in the metadata asynchronously. /// @@ -525,9 +553,9 @@ namespace Microsoft.ApplicationInspector.Commands if (fileRecord.Status != ScanState.Skipped) { - var results = _options.TagsOnly ? - await _rulesProcessor.AnalyzeFileAsync(file, languageInfo, cancellationToken, _metaDataHelper.UniqueTags.Keys, -1) : - await _rulesProcessor.AnalyzeFileAsync(file, languageInfo, cancellationToken, null, _options.ContextLines); + var contextLines = _options.TagsOnly ? -1 : _options.ContextLines; + var ignoredTags = _options.TagsOnly ? _metaDataHelper.UniqueTags.Keys : _options.MaxNumMatchesPerTag > 0 ? _metaDataHelper.UniqueTags.Where(x => x.Value < _options.MaxNumMatchesPerTag).Select(x => x.Key) : null; + var results = await _rulesProcessor.AnalyzeFileAsync(file, languageInfo, cancellationToken, ignoredTags, contextLines); fileRecord.Status = ScanState.Analyzed; if (results.Any()) @@ -541,6 +569,13 @@ namespace Microsoft.ApplicationInspector.Commands { _metaDataHelper.AddTagsFromMatchRecord(matchRecord); } + else if (_options.MaxNumMatchesPerTag > 0) + { + if (matchRecord.Tags?.Any(x => _metaDataHelper.UniqueTags.TryGetValue(x, out int value) is bool foundValue && (!foundValue || foundValue && value < _options.MaxNumMatchesPerTag)) ?? true) + { + _metaDataHelper.AddMatchRecord(matchRecord); + } + } else { _metaDataHelper.AddMatchRecord(matchRecord); @@ -827,7 +862,7 @@ namespace Microsoft.ApplicationInspector.Commands if (_options.ProcessingTimeOut > 0) { using var cts = new CancellationTokenSource(); - var t = Task.Run(() => PopulateRecords(cts.Token, _options, fileEntries), cts.Token); + var t = Task.Run(() => PopulateRecords(cts.Token, fileEntries), cts.Token); if (!t.Wait(new TimeSpan(0, 0, 0, 0, _options.ProcessingTimeOut))) { timedOut = true; @@ -845,7 +880,7 @@ namespace Microsoft.ApplicationInspector.Commands } else { - PopulateRecords(new CancellationToken(), _options, fileEntries); + PopulateRecords(new CancellationToken(), fileEntries); } } } diff --git a/AppInspector/MetaDataHelper.cs b/AppInspector/MetaDataHelper.cs index ccd844a..ef027ba 100644 --- a/AppInspector/MetaDataHelper.cs +++ b/AppInspector/MetaDataHelper.cs @@ -23,7 +23,7 @@ namespace Microsoft.ApplicationInspector.Commands internal ConcurrentDictionary UniqueDependencies { get; set; } = new ConcurrentDictionary(); private ConcurrentDictionary AppTypes { get; set; } = new ConcurrentDictionary(); - internal ConcurrentDictionary UniqueTags { get; set; } = new ConcurrentDictionary(); + internal ConcurrentDictionary UniqueTags { get; set; } = new ConcurrentDictionary(); private ConcurrentDictionary Outputs { get; set; } = new ConcurrentDictionary(); private ConcurrentDictionary Targets { get; set; } = new ConcurrentDictionary(); private ConcurrentDictionary CPUTargets { get; set; } = new ConcurrentDictionary(); @@ -122,7 +122,10 @@ namespace Microsoft.ApplicationInspector.Commands //update list of unique tags as we go foreach (string tag in matchRecord.Tags ?? Array.Empty()) { - UniqueTags.TryAdd(tag, 0); + if (!UniqueTags.TryAdd(tag, 1)) + { + UniqueTags[tag]++; + } } } } @@ -196,7 +199,10 @@ namespace Microsoft.ApplicationInspector.Commands //update list of unique tags as we go foreach (string tag in nonCounters) { - UniqueTags.TryAdd(tag, 0); + if (!UniqueTags.TryAdd(tag, 1)) + { + UniqueTags[tag]++; + } } Matches.Add(matchRecord); diff --git a/UnitTest.Commands/Tests_NuGet/TestAnalyzeCmd.cs b/UnitTest.Commands/Tests_NuGet/TestAnalyzeCmd.cs index 14c2c10..88ccbf7 100644 --- a/UnitTest.Commands/Tests_NuGet/TestAnalyzeCmd.cs +++ b/UnitTest.Commands/Tests_NuGet/TestAnalyzeCmd.cs @@ -66,6 +66,76 @@ namespace ApplicationInspector.Unitprocess.Commands Assert.IsTrue(exitCode == AnalyzeResult.ExitCode.CriticalError);//test fails even when values match unless this case run individually -mstest bug? } + [TestMethod] + public void MaxNumMatches_Pass() + { + AnalyzeOptions options = new AnalyzeOptions() + { + SourcePath = new string[1] { Path.Combine(Helper.GetPath(Helper.AppPath.testSource), @"unzipped\simple\main.cpp") }, + FilePathExclusions = Array.Empty(), //allow source under unittest path, + MaxNumMatchesPerTag = 1 + }; + + AnalyzeResult.ExitCode exitCode = AnalyzeResult.ExitCode.CriticalError; + AnalyzeCommand command = new AnalyzeCommand(options); + AnalyzeResult result = command.GetResult(); + Assert.AreEqual(1, result.Metadata.Matches.Count(x => x.Tags.Contains("Platform.OS.Microsoft.WindowsStandard"))); + exitCode = result.ResultCode; + Assert.IsTrue(exitCode == AnalyzeResult.ExitCode.Success); + } + + [TestMethod] + public void MaxNumMatchesDisabled_Pass() + { + AnalyzeOptions options = new AnalyzeOptions() + { + SourcePath = new string[1] { Path.Combine(Helper.GetPath(Helper.AppPath.testSource), @"unzipped\simple\main.cpp") }, + FilePathExclusions = Array.Empty(), //allow source under unittest path, + }; + + AnalyzeResult.ExitCode exitCode = AnalyzeResult.ExitCode.CriticalError; + AnalyzeCommand command = new AnalyzeCommand(options); + AnalyzeResult result = command.GetResult(); + Assert.AreEqual(3, result.Metadata.Matches.Count(x => x.Tags.Contains("Platform.OS.Microsoft.WindowsStandard"))); + exitCode = result.ResultCode; + Assert.IsTrue(exitCode == AnalyzeResult.ExitCode.Success); + } + + [TestMethod] + public async Task MaxNumMatchesAsync_Pass() + { + AnalyzeOptions options = new AnalyzeOptions() + { + SourcePath = new string[1] { Path.Combine(Helper.GetPath(Helper.AppPath.testSource), @"unzipped\simple\main.cpp") }, + FilePathExclusions = Array.Empty(), //allow source under unittest path, + MaxNumMatchesPerTag = 1 + }; + + AnalyzeResult.ExitCode exitCode = AnalyzeResult.ExitCode.CriticalError; + AnalyzeCommand command = new AnalyzeCommand(options); + AnalyzeResult result = await command.GetResultAsync(new CancellationTokenSource().Token); + Assert.AreEqual(1, result.Metadata.Matches.Count(x => x.Tags.Contains("Platform.OS.Microsoft.WindowsStandard"))); + exitCode = result.ResultCode; + Assert.IsTrue(exitCode == AnalyzeResult.ExitCode.Success); + } + + [TestMethod] + public async Task MaxNumMatchesAsyncDisabled_Pass() + { + AnalyzeOptions options = new AnalyzeOptions() + { + SourcePath = new string[1] { Path.Combine(Helper.GetPath(Helper.AppPath.testSource), @"unzipped\simple\main.cpp") }, + FilePathExclusions = Array.Empty(), //allow source under unittest path, + }; + + AnalyzeResult.ExitCode exitCode = AnalyzeResult.ExitCode.CriticalError; + AnalyzeCommand command = new AnalyzeCommand(options); + AnalyzeResult result = await command.GetResultAsync(new CancellationTokenSource().Token); + Assert.AreEqual(3, result.Metadata.Matches.Count(x => x.Tags.Contains("Platform.OS.Microsoft.WindowsStandard"))); + exitCode = result.ResultCode; + Assert.IsTrue(exitCode == AnalyzeResult.ExitCode.Success); + } + [TestMethod] public void BasicAnalyze_Pass() {