Complete work on Test Case Reporting (#504)

AppInspector.CLI/CLICmdOptions.cs

@@ -41,6 +41,12 @@ namespace Microsoft.ApplicationInspector.CLI
         /// </summary>
         [Option("success-error-code-with-no-matches", Required = false, HelpText = "When processing is apparently successful but there are no matches return a success error code - useful for CI.")]
         public bool SuccessErrorCodeOnNoMatches { get; set; }
+        
+        [Option("require-must-match", Required = false, HelpText = "When validating, require rules to have MustMatch self-tests.")]
+        public bool RequireMustMatch { get; set; }
+        
+        [Option("require-must-not-match", Required = false, HelpText = "When validating, require rules to have MustNotMatch self-tests.")]
+        public bool RequireMustNotMatch { get; set; }
 
     }
 

AppInspector.CLI/Program.cs

@@ -292,7 +292,9 @@ namespace Microsoft.ApplicationInspector.CLI
                 EnumeratingTimeout = cliOptions.EnumeratingTimeout,
                 DisableCustomRuleVerification = cliOptions.DisableCustomRuleValidation,
                 DisableRequireUniqueIds = cliOptions.DisableRequireUniqueIds,
-                SuccessErrorCodeOnNoMatches = cliOptions.SuccessErrorCodeOnNoMatches
+                SuccessErrorCodeOnNoMatches = cliOptions.SuccessErrorCodeOnNoMatches,
+                RequireMustMatch = cliOptions.RequireMustMatch,
+                RequireMustNotMatch = cliOptions.RequireMustNotMatch
             }, adjustedFactory);
 
             AnalyzeResult analyzeResult = command.GetResult();
@@ -360,7 +362,9 @@ namespace Microsoft.ApplicationInspector.CLI
                 SingleThread = cliOptions.SingleThread,
                 DisableCustomRuleValidation = cliOptions.DisableCustomRuleValidation,
                 DisableRequireUniqueIds = cliOptions.DisableRequireUniqueIds,
-                SuccessErrorCodeOnNoMatches = cliOptions.SuccessErrorCodeOnNoMatches
+                SuccessErrorCodeOnNoMatches = cliOptions.SuccessErrorCodeOnNoMatches,
+                RequireMustMatch = cliOptions.RequireMustMatch,
+                RequireMustNotMatch = cliOptions.RequireMustNotMatch
             }, loggerFactory);
 
             TagDiffResult tagDiffResult = command.GetResult();
@@ -395,7 +399,9 @@ namespace Microsoft.ApplicationInspector.CLI
                 CustomRulesPath = cliOptions.CustomRulesPath,
                 CustomCommentsPath = cliOptions.CustomCommentsPath,
                 CustomLanguagesPath = cliOptions.CustomLanguagesPath,
-                DisableRequireUniqueIds = cliOptions.DisableRequireUniqueIds
+                DisableRequireUniqueIds = cliOptions.DisableRequireUniqueIds,
+                RequireMustMatch = cliOptions.RequireMustMatch,
+                RequireMustNotMatch = cliOptions.RequireMustNotMatch
             }, loggerFactory);
 
             VerifyRulesResult exportTagsResult = command.GetResult();

AppInspector.RulesEngine/RulesVerifier.cs

@@ -264,6 +264,23 @@ namespace Microsoft.ApplicationInspector.RulesEngine
                 errors.Add($"Rule must specify tags. {rule.Id}");
             }
 
+            if (_options.RequireMustMatch)
+            {
+                if (rule.MustMatch?.Any() is not true)
+                {
+                    _logger?.LogError("Rule must specify MustMatch when `RequireMustMatch` is set. {0}", rule.Id);
+                    errors.Add($"Rule must specify MustMatch when `RequireMustMatch` is set. {rule.Id}");
+                }
+            }
+            if (_options.RequireMustNotMatch)
+            {
+                if (rule.MustNotMatch?.Any() is not true)
+                {
+                    _logger?.LogError("Rule must specify MustNotMatch when `RequireMustNotMatch` is set. {0}", rule.Id);
+                    errors.Add($"Rule must specify MustNotMatch when `RequireMustNotMatch` is set. {rule.Id}");
+                }
+            }
+
             return new RuleStatus()
             {
                 RulesId = rule.Id,

AppInspector.RulesEngine/RulesVerifierOptions.cs

@@ -22,4 +22,14 @@ public class RulesVerifierOptions
     /// By default rules must have unique IDs, this disables that validation check
     /// </summary>
     public bool DisableRequireUniqueIds { get; set; }
+    
+    /// <summary>
+    /// By default, the <see cref="RuleStatus.HasPositiveSelfTests"/> property informs if <see cref="Rule.MustMatch"/> is populated. Enabling this will cause an error to be raised during rule validation if it is not populated.
+    /// </summary>
+    public bool RequireMustMatch { get; set; }
+    
+    /// <summary>
+    /// By default, the <see cref="RuleStatus.HasNegativeSelfTests"/> property informs if <see cref="Rule.MustNotMatch"/> is populated. Enabling this will cause an error to be raised during rule validation if it is not populated.
+    /// </summary>
+    public bool RequireMustNotMatch { get; set; }
 }

AppInspector/Commands/AnalyzeCommand.cs

@@ -83,6 +83,9 @@ namespace Microsoft.ApplicationInspector.Commands
         /// Return a success error code when no matches were found but operation was apparently successful. Useful for CI scenarios
         /// </summary>
         public bool SuccessErrorCodeOnNoMatches { get; set; }
+
+        public bool RequireMustMatch { get; set; }
+        public bool RequireMustNotMatch { get; set; }
     }
 
     /// <summary>
@@ -207,7 +210,9 @@ namespace Microsoft.ApplicationInspector.Commands
                 {
                     LanguageSpecs = _languages,
                     LoggerFactory = _loggerFactory,
-                    DisableRequireUniqueIds = _options.DisableRequireUniqueIds
+                    DisableRequireUniqueIds = _options.DisableRequireUniqueIds,
+                    RequireMustMatch = _options.RequireMustMatch,
+                    RequireMustNotMatch = _options.RequireMustNotMatch
                 };
                 RulesVerifier verifier = new(rulesVerifierOptions);
                 bool anyFails = false;

AppInspector/Commands/ExportTagsCommand.cs

@@ -112,41 +112,20 @@ namespace Microsoft.ApplicationInspector.Commands
                 AppVersion = Common.Utils.GetVersionString()
             };
 
-            SortedDictionary<string, string> uniqueTags = new();
-
-            try
+            HashSet<string> tags = new();
+            foreach (var rule in _rules.GetAppInspectorRules())
             {
-                foreach (Rule? r in _rules)
+                foreach (var tag in rule.Tags ?? Array.Empty<string>())
                 {
-                    //builds a list of unique tags
-                    foreach (string t in r?.Tags ?? Array.Empty<string>())
-                    {
-                        if (uniqueTags.ContainsKey(t))
-                        {
-                            continue;
-                        }
-                        else
-                        {
-                            uniqueTags.Add(t, t);
-                        }
-                    }
+                    tags.Add(tag);
                 }
-
-                //generate results list
-                foreach (string s in uniqueTags.Values)
-                {
-                    exportTagsResult.TagsList.Add(s);
-                }
-
-                exportTagsResult.ResultCode = ExportTagsResult.ExitCode.Success;
-            }
-            catch (OpException e)
-            {
-                _logger.LogError(e.Message);
-                //caught for CLI callers with final exit msg about checking log or throws for DLL callers
-                throw;
             }
 
+            exportTagsResult.TagsList = tags.ToList();
+            exportTagsResult.TagsList.Sort();
+
+            exportTagsResult.ResultCode = ExportTagsResult.ExitCode.Success;
+
             return exportTagsResult;
         }
     }

AppInspector/Commands/TagDiffCommand.cs

@@ -35,6 +35,8 @@ namespace Microsoft.ApplicationInspector.Commands
         /// </summary>
         public bool SuccessErrorCodeOnNoMatches { get; set; }
 
+        public bool RequireMustMatch { get; set; }
+        public bool RequireMustNotMatch { get; set; }
     }
 
     /// <summary>
@@ -162,7 +164,9 @@ namespace Microsoft.ApplicationInspector.Commands
                     CustomLanguagesPath = _options.CustomLanguagesPath,
                     DisableCustomRuleVerification = _options.DisableCustomRuleValidation,
                     DisableRequireUniqueIds = _options.DisableRequireUniqueIds,
-                    SuccessErrorCodeOnNoMatches = _options.SuccessErrorCodeOnNoMatches
+                    SuccessErrorCodeOnNoMatches = _options.SuccessErrorCodeOnNoMatches,
+                    RequireMustMatch = _options.RequireMustMatch,
+                    RequireMustNotMatch = _options.RequireMustNotMatch
                 }, _factory);
                 AnalyzeCommand cmd2 = new(new AnalyzeOptions()
                 {

AppInspector/Commands/VerifyRulesCommand.cs

@@ -23,6 +23,8 @@ namespace Microsoft.ApplicationInspector.Commands
         public string? CustomCommentsPath { get; set; }
         public string? CustomLanguagesPath { get; set; }
         public bool DisableRequireUniqueIds { get; set; }
+        public bool RequireMustMatch { get; set; }
+        public bool RequireMustNotMatch { get; set; }
     }
 
     public class VerifyRulesResult : Result
@@ -98,6 +100,8 @@ namespace Microsoft.ApplicationInspector.Commands
                 {
                     Analyzer = analyzer,
                     DisableRequireUniqueIds = _options.DisableRequireUniqueIds,
+                    RequireMustMatch = _options.RequireMustMatch,
+                    RequireMustNotMatch = _options.RequireMustNotMatch,
                     LoggerFactory = _loggerFactory,
                     LanguageSpecs = Languages.FromConfigurationFiles(_loggerFactory, _options.CustomCommentsPath, _options.CustomLanguagesPath)
                 };

Benchmarks/DistinctBenchmarks.cs

@@ -0,0 +1,88 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+using Microsoft.ApplicationInspector.Commands;
+using Microsoft.ApplicationInspector.Common;
+using Microsoft.ApplicationInspector.RulesEngine;
+
+namespace Benchmarks;
+
+[MemoryDiagnoser()]
+public class DistinctBenchmarks
+{
+    public DistinctBenchmarks()
+    {
+    }
+    IEnumerable<Rule> ruleSet = RuleSetUtils.GetDefaultRuleSet().GetAppInspectorRules();
+    
+    [Benchmark(Baseline = true)]
+    public List<string> OldCode()
+    {
+        SortedDictionary<string, string> uniqueTags = new();
+        List<string> outList = new();
+        try
+        {
+            foreach (Rule? r in ruleSet)
+            {
+                //builds a list of unique tags
+                foreach (string t in r?.Tags ?? Array.Empty<string>())
+                {
+                    if (uniqueTags.ContainsKey(t))
+                    {
+                        continue;
+                    }
+                    else
+                    {
+                        uniqueTags.Add(t, t);
+                    }
+                }
+            }
+
+            //generate results list
+            foreach (string s in uniqueTags.Values)
+            {
+                outList.Add(s);
+            }
+        }
+        catch (OpException e)
+        {
+            throw;
+        }
+
+        return outList;
+    }
+
+    [Benchmark]
+    public List<string> HashSet()
+    {
+        HashSet<string> hashSet = new();
+        foreach (Rule? r in ruleSet)
+        {
+            //builds a list of unique tags
+            foreach (string t in r?.Tags ?? Array.Empty<string>())
+            {
+                hashSet.Add(t);
+            }
+        }
+
+        List<string> theList = hashSet.ToList();
+        theList.Sort();
+        return theList;
+    }
+    
+    [Benchmark]
+    public List<string> WithLinq()
+    {
+        return ruleSet.SelectMany(x => x.Tags ?? Array.Empty<string>()).Distinct().OrderBy(x => x)
+            .ToList();
+    }
+    
+    [Benchmark]
+    public List<string> WithLinqAndHashSet()
+    {
+        var theList = ruleSet.SelectMany(x => x.Tags ?? Array.Empty<string>()).ToHashSet().ToList();
+        theList.Sort();
+        return theList;
+    }
+}