Fix #480 (#481)

* Fix #480

Fixes case sensitivity of Enum based arguments
Fixes default Confidence argument not being respected
Fixes severity not being respected
Fixes custom rules not being validated if provided as a directory
Adds new option to skip validation of custom rules
Adds a TextContainer based API for analyze.

* Simplify post validation action

AppInspector.CLI/CLICmdOptions.cs

@@ -38,6 +38,9 @@ namespace Microsoft.ApplicationInspector.CLI
 
     public record CLIAnalysisSharedCommandOptions : CLICustomRulesCommandOptions
     {
+        [Option("disable-custom-rule-validation", Required = false, HelpText = "By default when providing custom rules they are validated. When set, validation will be skipped.")]
+        public bool DisableCustomRuleValidation { get; set; } = false;
+        
         [Option('i', "ignore-default-rules", Required = false, HelpText = "Exclude default rules bundled with application", Default = false)]
         public bool IgnoreDefaultRules { get; set; }
 
@@ -62,11 +65,13 @@ namespace Microsoft.ApplicationInspector.CLI
         [Option('u', "scan-unknown-filetypes", Required = false, HelpText = "Scan files of unknown types.")]
         public bool ScanUnknownTypes { get; set; }
 
-        [Option('c', "confidence-filters", Required = false, Separator = ',',  HelpText = "Output only matches with specified confidence <value>,<value>. Default: Medium,High. [High|Medium|Low]")]
+        [Option('c', "confidence-filters", Required = false, Separator = ',', HelpText = "Output only matches with specified confidence <value>,<value>. Default: Medium,High. [High|Medium|Low]", Default = new Confidence[]{Confidence.High, Confidence.Medium})]
         public IEnumerable<Confidence> ConfidenceFilters { get; set; } = new Confidence[] { Confidence.High, Confidence.Medium };
 
-        [Option("severity-filters", Required = false, Separator = ',', HelpText = "Output only matches with specified severity <value>,<value>. Default: All are enabled. [Critical|Important|Moderate|BestPractice|ManualReview]")]
-        public IEnumerable<Severity> SeverityFilters { get; set; } = (IEnumerable<Severity>)Enum.GetValues(typeof(Severity));
+        [Option("severity-filters", Required = false, Separator = ',',
+            HelpText =
+                "Output only matches with specified severity <value>,<value>. Default: All are enabled. [Critical|Important|Moderate|BestPractice|ManualReview]", Default = new Severity[] { Severity.Critical, Severity.Important, Severity.Moderate, Severity.BestPractice, Severity.ManualReview })]
+        public IEnumerable<Severity> SeverityFilters { get; set; } = new Severity[] { Severity.Critical, Severity.Important, Severity.Moderate, Severity.BestPractice, Severity.ManualReview };
     }
 
     /// <summary>

AppInspector.CLI/Program.cs

@@ -7,14 +7,14 @@ namespace Microsoft.ApplicationInspector.CLI
 {
     using CommandLine;
     using Commands;
+    using Microsoft.ApplicationInspector.Common;
+    using Microsoft.Extensions.Logging;
     using ShellProgressBar;
     using System;
     using System.IO;
     using System.Linq;
     using System.Threading;
     using System.Threading.Tasks;
-    using Microsoft.ApplicationInspector.Common;
-    using Microsoft.Extensions.Logging;
     using ILogger = Extensions.Logging.ILogger;
 
     public static class Program
@@ -33,7 +33,7 @@ namespace Microsoft.ApplicationInspector.CLI
             Exception? exception = null;
             try
             {
-                var argsResult = Parser.Default.ParseArguments<CLIAnalyzeCmdOptions,
+                var argsResult = new Parser(settings => settings.CaseInsensitiveEnumValues = true).ParseArguments<CLIAnalyzeCmdOptions,
                     CLITagDiffCmdOptions,
                     CLIExportTagsCmdOptions,
                     CLIVerifyRulesCmdOptions,
@@ -277,6 +277,7 @@ namespace Microsoft.ApplicationInspector.CLI
                 CustomLanguagesPath = cliOptions.CustomLanguagesPath,
                 IgnoreDefaultRules = cliOptions.IgnoreDefaultRules,
                 ConfidenceFilters = cliOptions.ConfidenceFilters,
+                SeverityFilters = cliOptions.SeverityFilters,
                 FilePathExclusions = cliOptions.FilePathExclusions,
                 SingleThread = cliOptions.SingleThread,
                 NoShowProgress = cliOptions.NoShowProgressBar,
@@ -289,7 +290,8 @@ namespace Microsoft.ApplicationInspector.CLI
                 AllowAllTagsInBuildFiles = cliOptions.AllowAllTagsInBuildFiles,
                 MaxNumMatchesPerTag = cliOptions.MaxNumMatchesPerTag,
                 DisableCrawlArchives = cliOptions.DisableArchiveCrawling,
-                EnumeratingTimeout = cliOptions.EnumeratingTimeout
+                EnumeratingTimeout = cliOptions.EnumeratingTimeout,
+                DisableCustomRuleVerification = cliOptions.DisableCustomRuleValidation,
             }, adjustedFactory);
 
             AnalyzeResult analyzeResult = command.GetResult();
@@ -355,6 +357,7 @@ namespace Microsoft.ApplicationInspector.CLI
                 ProcessingTimeOut = cliOptions.ProcessingTimeOut,
                 ScanUnknownTypes = cliOptions.ScanUnknownTypes,
                 SingleThread = cliOptions.SingleThread,
+                DisableCustomRuleValidation = cliOptions.DisableCustomRuleValidation,
             }, loggerFactory);
 
             TagDiffResult tagDiffResult = command.GetResult();

AppInspector.RulesEngine/RuleProcessor.cs

@@ -125,12 +125,21 @@ namespace Microsoft.ApplicationInspector.RulesEngine
             return rawResult;
         }
 
-        public List<MatchRecord> AnalyzeFile(string contents, FileEntry fileEntry, LanguageInfo languageInfo, IEnumerable<string>? tagsToIgnore = null, int numLinesContext = 3)
+        /// <summary>
+        /// Analyzes a file and returns a list of <see cref="MatchRecord"/>
+        /// </summary>
+        /// <param name="textContainer">TextContainer which holds the text to analyze</param>
+        /// <param name="fileEntry">FileEntry which has the name of the file being analyzed.</param>
+        /// <param name="languageInfo">The LanguageInfo for the file</param>
+        /// <param name="tagsToIgnore">Ignore rules that match tags that are only in the tags to ignore list</param>
+        /// <param name="numLinesContext">Number of lines of text to extract for the sample. Set to 0 to disable context gathering. Set to -1 to also disable sampling the match.</param>
+        /// <returns>A List of the matches against the Rules the processor is configured with.</returns>
+        public List<MatchRecord> AnalyzeFile(TextContainer textContainer, FileEntry fileEntry,
+            LanguageInfo languageInfo, IEnumerable<string>? tagsToIgnore = null, int numLinesContext = 3)
         {
             var rules = GetRulesForFile(languageInfo, fileEntry, tagsToIgnore);
             List<MatchRecord> resultsList = new();
 
-            TextContainer textContainer = new(contents, languageInfo.Name, _languages);
             var caps = analyzer.GetCaptures(rules, textContainer);
             foreach (var ruleCapture in caps)
             {
@@ -220,6 +229,28 @@ namespace Microsoft.ApplicationInspector.RulesEngine
             return resultsList;
         }
 
+        /// <summary>
+        /// Analyzes a file and returns a list of <see cref="MatchRecord"/>
+        /// </summary>
+        /// <param name="contents">A string containing the text to analyze</param>
+        /// <param name="fileEntry">FileEntry which has the name of the file being analyzed</param>
+        /// <param name="languageInfo">The LanguageInfo for the file</param>
+        /// <param name="tagsToIgnore">Ignore rules that match tags that are only in the tags to ignore list</param>
+        /// <param name="numLinesContext">Number of lines of text to extract for the sample. Set to 0 to disable context gathering. Set to -1 to also disable sampling the match.</param>
+        /// <returns>A List of the matches against the Rules the processor is configured with.</returns>
+        public List<MatchRecord> AnalyzeFile(string contents, FileEntry fileEntry, LanguageInfo languageInfo, IEnumerable<string>? tagsToIgnore = null, int numLinesContext = 3)
+        {
+            TextContainer textContainer = new(contents, languageInfo.Name, _languages);
+            return AnalyzeFile(textContainer, fileEntry, languageInfo, tagsToIgnore, numLinesContext);
+        }
+
+        /// <summary>
+        /// Get the Rules which apply to the FileName of the FileEntry provided.
+        /// </summary>
+        /// <param name="languageInfo"></param>
+        /// <param name="fileEntry"></param>
+        /// <param name="tagsToIgnore"></param>
+        /// <returns></returns>
         public IEnumerable<ConvertedOatRule> GetRulesForFile(LanguageInfo languageInfo, FileEntry fileEntry, IEnumerable<string>? tagsToIgnore)
         {
             return GetRulesByLanguage(languageInfo.Name)
@@ -229,6 +260,14 @@ namespace Microsoft.ApplicationInspector.RulesEngine
                 .Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
         }
 
+        /// <summary>
+        /// Analyzes a file and returns a list of <see cref="MatchRecord"/>
+        /// </summary>
+        /// <param name="fileEntry">FileEntry which holds the name of the file being analyzed as well as a Stream containing the contents to analyze</param>
+        /// <param name="languageInfo">The LanguageInfo for the file</param>
+        /// <param name="tagsToIgnore">Ignore rules that match tags that are only in the tags to ignore list</param>
+        /// <param name="numLinesContext">Number of lines of text to extract for the sample. Set to 0 to disable context gathering. Set to -1 to also disable sampling the match.</param>
+        /// <returns>A List of the matches against the Rules the processor is configured with.</returns>
         public List<MatchRecord> AnalyzeFile(FileEntry fileEntry, LanguageInfo languageInfo, IEnumerable<string>? tagsToIgnore = null, int numLinesContext = 3)
         {
             using var sr = new StreamReader(fileEntry.Content);

AppInspector/Commands/AnalyzeCommand.cs

@@ -69,6 +69,8 @@ namespace Microsoft.ApplicationInspector.Commands
         /// If <see cref="DisableCrawlArchives"/> is not set, will restrict the amount of time allowed to extract each archive. Not supported in async operations.
         /// </summary>
         public int EnumeratingTimeout { get; set; }
+
+        public bool DisableCustomRuleVerification { get; set; }
     }
 
     /// <summary>
@@ -205,32 +207,55 @@ namespace Microsoft.ApplicationInspector.Commands
             if (!string.IsNullOrEmpty(_options.CustomRulesPath))
             {
                 rulesSet ??= new RuleSet(_loggerFactory);
-
+                RulesVerifierOptions rulesVerifierOptions = new()
+                {
+                    FailFast = false,
+                    LanguageSpecs = _languages,
+                    LoggerFactory = _loggerFactory,
+                };
+                RulesVerifier verifier = new(rulesVerifierOptions);
+                bool anyFails = false;
                 if (Directory.Exists(_options.CustomRulesPath))
                 {
-                    rulesSet.AddDirectory(_options.CustomRulesPath);
+                    foreach (string filename in Directory.EnumerateFileSystemEntries(_options.CustomRulesPath, "*.json",
+                                 SearchOption.AllDirectories))
+                    {
+                        VerifyFile(filename);
+                    }
                 }
                 else if (File.Exists(_options.CustomRulesPath)) //verify custom rules before use
                 {
-                    RulesVerifierOptions rulesVerifierOptions = new()
-                    {
-                        FailFast = false,
-                        LanguageSpecs = _languages,
-                        LoggerFactory = _loggerFactory,
-                    };
-                    RulesVerifier verifier = new(rulesVerifierOptions);
-                    RulesVerifierResult verification = verifier.Verify(_options.CustomRulesPath);
-                    if (!verification.Verified)
-                    {
-                        throw new OpException(MsgHelp.FormatString(MsgHelp.ID.VERIFY_RULE_LOADFILE_FAILED, _options.CustomRulesPath));
-                    }
-
-                    rulesSet.AddRange(verification.CompiledRuleSet.GetAppInspectorRules());
+                    VerifyFile(_options.CustomRulesPath);
                 }
                 else
                 {
                     throw new OpException(MsgHelp.FormatString(MsgHelp.ID.CMD_INVALID_RULE_PATH, _options.CustomRulesPath));
                 }
+
+                if (anyFails)
+                {
+                    throw new OpException(MsgHelp.FormatString(MsgHelp.ID.VERIFY_RULE_LOADFILE_FAILED, _options.CustomRulesPath));
+                }
+
+                void VerifyFile(string filename)
+                {
+                    if (!_options.DisableCustomRuleVerification)
+                    {
+                        RulesVerifierResult verification = verifier.Verify(_options.CustomRulesPath);
+                        if (!verification.Verified)
+                        {
+                            anyFails = true;
+                        }
+                        else
+                        {
+                            rulesSet.AddFile(filename);
+                        }
+                    }
+                    else
+                    {
+                        rulesSet.AddFile(filename);
+                    }
+                }
             }
 
             //error check based on ruleset not path enumeration

AppInspector/Commands/TagDiffCommand.cs

@@ -28,6 +28,7 @@ namespace Microsoft.ApplicationInspector.Commands
         public IEnumerable<Severity> SeverityFilters { get; set; } = new Severity[] { Severity.Critical | Severity.Important | Severity.Moderate | Severity.BestPractice | Severity.ManualReview };
         public string? CustomCommentsPath { get; set; }
         public string? CustomLanguagesPath { get; set; }
+        public bool DisableCustomRuleValidation { get; set; }
     }
 
     /// <summary>
@@ -151,6 +152,7 @@ namespace Microsoft.ApplicationInspector.Commands
                     NoShowProgress = true,
                     ScanUnknownTypes = _options.ScanUnknownTypes,
                     SingleThread = _options.SingleThread,
+                    DisableCustomRuleVerification = _options.DisableCustomRuleValidation
                 }, _factory);
                 AnalyzeCommand cmd2 = new(new AnalyzeOptions()
                 {