microsoft
/
ApplicationInspector
зеркало из https://github.com/microsoft/ApplicationInspector.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Refactor Conditions (#495) 

* wip checkin

Partial refactor. Requires updated OAT just merged.

* wip checkin

Partial refactor. Requires updated OAT just merged.

* Fix issues with refactored within

* WIP

* Fix inversion in conditions

* Fix regex json rule offset

* Fix off by one

* Validate subclause of within clause.
This commit is contained in:
Gabe Stocco 2022-08-10 12:42:41 -07:00 коммит произвёл GitHub
Родитель 175de2aaef
Коммит dce5493d93
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
14 изменённых файлов: 292 добавлений и 331 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

218
AppInspector.RulesEngine/AbstractRuleSet.cs
Просмотреть файл

@ -73,62 +73,13 @@ namespace Microsoft.ApplicationInspector.RulesEngine
var expression = new StringBuilder("(");
foreach (var pattern in rule.Patterns)
{
if (pattern.Pattern != null)
clauses.Add(GenerateClause(pattern, clauseNumber));
if (clauseNumber > 0)
{
var scopes = pattern.Scopes ?? new PatternScope[] { PatternScope.All };
var modifiers = pattern.Modifiers?.ToList() ?? new List<string>();
if (pattern.PatternType is PatternType.String or PatternType.Substring)
{
clauses.Add(new OatSubstringIndexClause(scopes, useWordBoundaries: pattern.PatternType == PatternType.String, xPaths: pattern.XPaths, jsonPaths:pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { pattern.Pattern },
Capture = true,
Arguments = pattern.Modifiers?.ToList() ?? new List<string>()
});
if (clauseNumber > 0)
{
expression.Append(" OR ");
}
expression.Append(clauseNumber);
clauseNumber++;
}
else if (pattern.PatternType == PatternType.Regex)
{
clauses.Add(new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { pattern.Pattern },
Capture = true,
Arguments = modifiers,
CustomOperation = "RegexWithIndex"
});
if (clauseNumber > 0)
{
expression.Append(" OR ");
}
expression.Append(clauseNumber);
clauseNumber++;
}
else if (pattern.PatternType == PatternType.RegexWord)
{
clauses.Add(new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { $"\\b({pattern.Pattern})\\b" },
Capture = true,
Arguments = pattern.Modifiers?.ToList() ?? new List<string>(),
CustomOperation = "RegexWithIndex"
});
if (clauseNumber > 0)
{
expression.Append(" OR ");
}
expression.Append(clauseNumber);
clauseNumber++;
}
expression.Append(" OR ");
}
expression.Append(clauseNumber);
clauseNumber++;
}
if (clauses.Count > 0)
@ -142,24 +93,43 @@ namespace Microsoft.ApplicationInspector.RulesEngine
foreach (var condition in rule.Conditions ?? Array.Empty<SearchCondition>())
{
if (condition.Pattern?.Pattern != null)
Clause? clause = GenerateCondition(condition, clauseNumber);
if (clause is { })
{
clauses.Add(clause);
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
}
}
return new ConvertedOatRule(rule.Id, rule)
{
Clauses = clauses,
Expression = expression.ToString()
};
}
private Clause? GenerateCondition(SearchCondition condition, int clauseNumber)
{
if (condition.Pattern is {} conditionPattern)
{
var subClause = GenerateClause(conditionPattern);
if (subClause is null)
{
_logger.LogWarning("SubClause for condition could not be generated");
}
else
{
List<string> conditionModifiers = condition.Pattern.Modifiers?.ToList() ?? new();
if (condition.SearchIn?.Equals("finding-only", StringComparison.InvariantCultureIgnoreCase) != false)
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = conditionModifiers,
FindingOnly = true,
CustomOperation = "Within",
Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All }
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else if (condition.SearchIn.StartsWith("finding-region", StringComparison.InvariantCultureIgnoreCase))
{
@ -181,93 +151,111 @@ namespace Microsoft.ApplicationInspector.RulesEngine
}
if (argList.Count == 2)
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = conditionModifiers,
FindingRegion = true,
CustomOperation = "Within",
Before = argList[0],
After = argList[1],
Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All }
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
SubClause = subClause,
Invert = condition.NegateFinding
};
}
}
else if (condition.SearchIn.Equals("same-line", StringComparison.InvariantCultureIgnoreCase))
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = conditionModifiers,
SameLineOnly = true,
CustomOperation = "Within",
Scopes = condition.Pattern.Scopes ?? new PatternScope[] { PatternScope.All }
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else if (condition.SearchIn.Equals("same-file", StringComparison.InvariantCultureIgnoreCase))
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = condition.Pattern.Modifiers?.ToList() ?? new List<string>(),
SameFile = true
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
SameFile = true,
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else if (condition.SearchIn.Equals("only-before", StringComparison.InvariantCultureIgnoreCase))
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = condition.Pattern.Modifiers?.ToList() ?? new List<string>(),
OnlyBefore = true
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
OnlyBefore = true,
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else if (condition.SearchIn.Equals("only-after", StringComparison.InvariantCultureIgnoreCase))
{
clauses.Add(new WithinClause()
return new WithinClause()
{
Data = new List<string>() { condition.Pattern.Pattern },
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),
Invert = condition.NegateFinding,
Arguments = condition.Pattern.Modifiers?.ToList() ?? new List<string>(),
OnlyAfter = true
});
expression.Append(" AND ");
expression.Append(clauseNumber);
clauseNumber++;
OnlyAfter = true,
SubClause = subClause,
Invert = condition.NegateFinding
};
}
else
{
_logger.LogWarning("Search condition {Condition} is not one of the accepted values and this condition will be ignored", condition.SearchIn);
return null;
}
}
}
return new ConvertedOatRule(rule.Id, rule)
return null;
}
private Clause? GenerateClause(SearchPattern pattern, int clauseNumber = -1)
{
if (pattern.Pattern != null)
{
Clauses = clauses,
Expression = expression.ToString()
};
var scopes = pattern.Scopes ?? new PatternScope[] { PatternScope.All };
var modifiers = pattern.Modifiers?.ToList() ?? new List<string>();
if (pattern.PatternType is PatternType.String or PatternType.Substring)
{
return new OatSubstringIndexClause(scopes, useWordBoundaries: pattern.PatternType == PatternType.String, xPaths: pattern.XPaths, jsonPaths:pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { pattern.Pattern },
Capture = true,
Arguments = pattern.Modifiers?.ToList() ?? new List<string>()
};
}
else if (pattern.PatternType == PatternType.Regex)
{
return new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo
.InvariantCulture), //important to pattern index identification
Data = new List<string>() { pattern.Pattern },
Capture = true,
Arguments = modifiers,
CustomOperation = "RegexWithIndex"
};
}
else if (pattern.PatternType == PatternType.RegexWord)
{
return new OatRegexWithIndexClause(scopes, null, pattern.XPaths, pattern.JsonPaths)
{
Label = clauseNumber.ToString(CultureInfo.InvariantCulture),//important to pattern index identification
Data = new List<string>() { $"\\b({pattern.Pattern})\\b" },
Capture = true,
Arguments = pattern.Modifiers?.ToList() ?? new List<string>(),
CustomOperation = "RegexWithIndex"
};
}
}
return null;
}
/// <summary>

2
AppInspector.RulesEngine/AppInspector.RulesEngine.csproj
Просмотреть файл

@ -31,7 +31,7 @@
<ItemGroup>
<PackageReference Include="JsonCons.JsonPath" Version="1.1.0" />
<PackageReference Include="Microsoft.CST.OAT" Version="1.2.19" />
<PackageReference Include="Microsoft.CST.OAT" Version="1.2.24" />
<PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.11" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />

17
AppInspector.RulesEngine/OatExtensions/GetBoundaryResult.cs Normal file
Просмотреть файл

@ -0,0 +1,17 @@
namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions;
internal class GetBoundaryResult
{
internal GetBoundaryResult(bool validBoundary, int start, int end)
{
ValidBoundary = validBoundary;
Start = start;
End = end;
}
internal int End { get; }
internal int Start { get; }
internal bool ValidBoundary { get; }
}

26
AppInspector.RulesEngine/OatExtensions/OatRegexWithIndexOperation.cs
Просмотреть файл

@ -73,6 +73,7 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
if (state1 is TextContainer tc && clause is OatRegexWithIndexClause src && clause.Data is List<string> RegexList && RegexList.Count > 0)
{
RegexOptions regexOpts = new();
Boundary? subBoundary = state2 is Boundary s2 ? s2 : null;
if (src.Arguments.Contains("i"))
{
@ -98,10 +99,9 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
var targets = tc.GetStringFromXPath(xmlPath);
foreach (var target in targets)
{
var matches = GetMatches(regex, target.Item1, tc, clause, src.Scopes);
var matches = GetMatches(regex, tc, clause, src.Scopes, target.Item2);
foreach (var match in matches)
{
match.Item2.Index += target.Item2.Index;
outmatches.Add(match);
}
}
@ -114,18 +114,20 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
var targets = tc.GetStringFromJsonPath(jsonPath);
foreach (var target in targets)
{
var matches = GetMatches(regex, target.Item1, tc, clause, src.Scopes);
foreach (var match in matches)
{
match.Item2.Index += target.Item2.Index;
outmatches.Add(match);
}
outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, target.Item2));
}
}
}
if (src.JsonPaths is null && src.XPaths is null)
{
outmatches.AddRange(GetMatches(regex, tc.FullContent, tc, clause, src.Scopes));
if (subBoundary is not null)
{
outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, subBoundary));
}
else
{
outmatches.AddRange(GetMatches(regex, tc, clause, src.Scopes, null));
}
}
}
}
@ -137,16 +139,16 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
return new OperationResult(false, null);
}
private IEnumerable<(int, Boundary)> GetMatches(Regex regex, string content, TextContainer tc, Clause clause, PatternScope[] scopes)
private IEnumerable<(int, Boundary)> GetMatches(Regex regex, TextContainer tc, Clause clause, PatternScope[] scopes, Boundary? boundary)
{
foreach (var match in regex.Matches(content))
foreach (var match in regex.Matches(boundary is null ? tc.FullContent : tc.GetBoundaryText(boundary)))
{
if (match is Match m)
{
Boundary translatedBoundary = new()
{
Length = m.Length,
Index = m.Index
Index = m.Index + (boundary?.Index ?? 0)
};
//regex patterns will be indexed off data while string patterns result in N clauses

12
AppInspector.RulesEngine/OatExtensions/OatSubstringIndexOperation.cs
Просмотреть файл

@ -97,8 +97,16 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
}
if (src.JsonPaths is null && src.XPaths is null)
{
var matches = GetMatches(tc.FullContent, stringList[i], comparisonType, tc, src);
outmatches.AddRange(matches.Select(x => (i, x)));
// If state 2 is a boundary, restrict the text provided to check to match the boundary
if (state2 is Boundary boundary)
{
var matches = GetMatches(tc.GetBoundaryText(boundary), stringList[i], comparisonType, tc, src);
outmatches.AddRange(matches.Select(x => (i, x))); }
else
{
var matches = GetMatches(tc.FullContent, stringList[i], comparisonType, tc, src);
outmatches.AddRange(matches.Select(x => (i, x))); }
}
}

2
AppInspector.RulesEngine/OatExtensions/WithinClause.cs
Просмотреть файл

@ -18,7 +18,7 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
public bool SameFile { get; set; }
public bool FindingOnly { get; set; }
public bool SameLineOnly { get; set; }
public PatternScope[] Scopes { get; set; } = new PatternScope[1] { PatternScope.All };
public bool FindingRegion { get; set; }
public Clause SubClause { get; set; }
}
}

261
AppInspector.RulesEngine/OatExtensions/WithinOperation.cs
Просмотреть файл

@ -1,4 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
@ -16,6 +17,7 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
{
_loggerFactory = loggerFactory ?? new NullLoggerFactory();
_regexEngine = new RegexOperation(analyzer);
_analyzer = analyzer;
CustomOperation = "Within";
OperationDelegate = WithinOperationDelegate;
ValidationDelegate = WithinValidationDelegate;
@ -25,159 +27,108 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
{
if (c is WithinClause wc && state1 is TextContainer tc)
{
var regexOpts = RegexOptions.Compiled;
if (wc.Arguments.Contains("i"))
{
regexOpts |= RegexOptions.IgnoreCase;
}
if (wc.Arguments.Contains("m"))
{
regexOpts |= RegexOptions.Multiline;
}
var passed = new List<Boundary>();
foreach (var captureHolder in captures ?? Array.Empty<ClauseCapture>())
{
if (captureHolder is TypedClauseCapture<List<(int, Boundary)>> tcc)
{
List<(int, Boundary)> toRemove = new();
foreach ((int clauseNum, Boundary capture) in tcc.Result)
{
if (wc.FindingOnly)
{
var res = ProcessLambda(tc.GetBoundaryText(capture), capture);
if (res.Result)
{
if (res.Capture is TypedClauseCapture<List<Boundary>> boundaryList)
{
passed.AddRange(boundaryList.Result);
}
}
else
{
toRemove.Add((clauseNum, capture));
}
}
else if (wc.SameLineOnly)
{
var start = tc.LineStarts[tc.GetLocation(capture.Index).Line];
var end = tc.LineEnds[tc.GetLocation(start + capture.Length).Line];
var res = ProcessLambda(tc.FullContent[start..end], capture);
if (res.Result)
{
if (res.Capture is TypedClauseCapture<List<Boundary>> boundaryList)
{
passed.AddRange(boundaryList.Result);
}
}
else
{
toRemove.Add((clauseNum, capture));
}
}
else if (wc.FindingRegion)
{
var startLine = tc.GetLocation(capture.Index).Line;
// Before is already a negative number
var start = tc.LineStarts[Math.Max(1, startLine + wc.Before)];
var end = tc.LineEnds[Math.Min(tc.LineEnds.Count - 1, startLine + wc.After)];
var res = ProcessLambda(tc.FullContent[start..(end+1)], capture);
if (res.Result)
{
if (res.Capture is TypedClauseCapture<List<Boundary>> boundaryList)
{
passed.AddRange(boundaryList.Result);
}
}
else
{
toRemove.Add((clauseNum, capture));
}
}
else if (wc.SameFile)
{
var start = tc.LineStarts[0];
var end = tc.LineEnds[^1];
var res = ProcessLambda(tc.FullContent[start..end], capture);
if (res.Result)
{
if (res.Capture is TypedClauseCapture<List<Boundary>> boundaryList)
{
passed.AddRange(boundaryList.Result);
}
}
else
{
toRemove.Add((clauseNum, capture));
}
}
else if (wc.OnlyBefore)
{
var start = tc.LineStarts[0];
var end = capture.Index;
var res = ProcessLambda(tc.FullContent[start..end], capture);
if (res.Result)
{
if (res.Capture is TypedClauseCapture<List<Boundary>> boundaryList)
{
passed.AddRange(boundaryList.Result);
}
}
else
{
toRemove.Add((clauseNum, capture));
} }
else if (wc.OnlyAfter)
{
var start = capture.Index + capture.Length;
var end = tc.LineEnds[^1];
var res = ProcessLambda(tc.FullContent[start..end], capture);
if (res.Result)
{
if (res.Capture is TypedClauseCapture<List<Boundary>> boundaryList)
{
passed.AddRange(boundaryList.Result);
}
}
else
{
toRemove.Add((clauseNum, capture));
}
}
}
tcc.Result.RemoveAll(x => toRemove.Contains(x));
}
}
// In the case that we have inverted the lambda, the captures are null and thus the passed list will be empty. We thus need to invert this again to get true correctly in that case.
return new OperationResult(passed.Any() ^ wc.Invert, passed.Any() ? new TypedClauseCapture<List<Boundary>>(wc, passed) : null);
List<(int, Boundary)> passed =
new List<(int, Boundary)>();
List<(int, Boundary)> failed =
new List<(int, Boundary)>();
OperationResult ProcessLambda(string target, Boundary targetBoundary)
foreach (var capture in captures)
{
var boundaries = new List<Boundary>();
foreach (var pattern in wc.Data.Select(x => _regexEngine.StringToRegex(x, regexOpts)))
if (capture is TypedClauseCapture<List<(int, Boundary)>> tcc)
{
if (pattern is Regex r)
foreach ((int clauseNum, Boundary boundary) in tcc.Result)
{
var matches = r.Matches(target);
foreach (var match in matches)
var boundaryToCheck = GetBoundaryToCheck();
if (boundaryToCheck is not null)
{
if (match is Match m)
var operationResult = ProcessLambda(boundaryToCheck);
if (operationResult.Result)
{
Boundary translatedBoundary = new()
{
Length = m.Length,
Index = targetBoundary.Index + m.Index
};
// Should return only scoped matches
if (tc.ScopeMatch(wc.Scopes, translatedBoundary))
{
boundaries.Add(translatedBoundary);
}
passed.Add((clauseNum, boundary));
}
else
{
failed.Add((clauseNum, boundary));
}
}
Boundary? GetBoundaryToCheck()
{
if (wc.FindingOnly)
{
return boundary;
}
if (wc.SameLineOnly)
{
var startInner = tc.LineStarts[tc.GetLocation(boundary.Index).Line];
var endInner = tc.LineEnds[tc.GetLocation(startInner + boundary.Length).Line];
return new Boundary()
{
Index = startInner,
Length = (endInner - startInner) + 1
};
}
if (wc.FindingRegion)
{
var startLine = tc.GetLocation(boundary.Index).Line;
// Before is already a negative number
var startInner = tc.LineStarts[Math.Max(1, startLine + wc.Before)];
var endInner = tc.LineEnds[Math.Min(tc.LineEnds.Count - 1, startLine + wc.After)];
var bound = new Boundary()
{
Index = startInner,
Length = (endInner - startInner) + 1
};
var theText = tc.GetBoundaryText(bound);
return bound;
}
if (wc.SameFile)
{
var startInner = tc.LineStarts[0];
var endInner = tc.LineEnds[^1];
return new Boundary()
{
Index = startInner,
Length = (endInner - startInner) + 1
};
}
if (wc.OnlyBefore)
{
var startInner = tc.LineStarts[0];
var endInner = boundary.Index;
return new Boundary()
{
Index = startInner,
Length = (endInner - startInner) + 1
};
}
if (wc.OnlyAfter)
{
var startInner = boundary.Index + boundary.Length;
var endInner = tc.LineEnds[^1];
return new Boundary()
{
Index = startInner,
Length = (endInner - startInner) + 1
};
}
return null;
}
}
}
// Invert the result of the operation if requested
return new OperationResult(boundaries.Any() ^ wc.Invert, boundaries.Any() ? new TypedClauseCapture<List<Boundary>>(wc, boundaries) : null);
var passedOrFailed = wc.Invert ? failed : passed;
return new OperationResult(passedOrFailed.Any(), passedOrFailed.Any() ? new TypedClauseCapture<List<(int, Boundary)>>(wc, passedOrFailed.ToList()) : null);
}
OperationResult ProcessLambda(Boundary target)
{
return _analyzer.GetClauseCapture(wc.SubClause, tc, target, captures);
}
}
return new OperationResult(false, null);
@ -215,18 +166,31 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
rule, clause);
}
}
if (!wc.Data?.Any() ?? true)
if (wc.Data?.Any() ?? false)
{
yield return new Violation($"Must provide some regexes as data.", rule, clause);
yield break;
yield return new Violation($"Don't provide data directly. Instead use SubClause..", rule, clause);
}
foreach (var datum in wc.Data ?? new List<string>())
{
if (_regexEngine.StringToRegex(datum, RegexOptions.None) is null)
yield return new Violation($"Data in WithinClause is ignored. Use SubClause. Data {datum} found in Rule {rule.Name} Clause {clause.Label ?? rule.Clauses.IndexOf(clause).ToString(CultureInfo.InvariantCulture)} is not a valid regex.", rule, clause);
}
var subOp = _analyzer
.GetOperation(wc.SubClause.Key.Operation, wc.SubClause.Key.CustomOperation);
if (subOp is null)
{
yield return new Violation($"SubClause in Rule {rule.Name} Clause {clause.Label ?? rule.Clauses.IndexOf(clause).ToString(CultureInfo.InvariantCulture)} is of type '{wc.SubClause.Key.Operation},{wc.SubClause.Key.CustomOperation}' is not present in the analyzer.", rule, clause);
}
else
{
foreach (var violation in subOp.ValidationDelegate.Invoke(rule, wc.SubClause))
{
yield return new Violation($"Regex {datum} in Rule {rule.Name} Clause {clause.Label ?? rule.Clauses.IndexOf(clause).ToString(CultureInfo.InvariantCulture)} is not a valid regex.", rule, clause);
yield return violation;
}
}
}
else
{
@ -236,5 +200,6 @@ namespace Microsoft.ApplicationInspector.RulesEngine.OatExtensions
private readonly RegexOperation _regexEngine;
private readonly ILoggerFactory _loggerFactory;
private readonly Analyzer _analyzer;
}
}

10
AppInspector.RulesEngine/RuleProcessor.cs
Просмотреть файл

@ -143,7 +143,10 @@ namespace Microsoft.ApplicationInspector.RulesEngine
var caps = analyzer.GetCaptures(rules, textContainer);
foreach (var ruleCapture in caps)
{
foreach (var cap in ruleCapture.Captures)
// If we had a WithinClause we only want the captures that passed the within filter.
var filteredCaptures = ruleCapture.Captures.Any(x => x.Clause is WithinClause)
? ruleCapture.Captures.Where(x => x.Clause is WithinClause) : ruleCapture.Captures;
foreach (var cap in filteredCaptures)
{
resultsList.AddRange(ProcessBoundary(cap));
}
@ -294,11 +297,14 @@ namespace Microsoft.ApplicationInspector.RulesEngine
TextContainer textContainer = new(await sr.ReadToEndAsync().ConfigureAwait(false), languageInfo.Name, _languages, _opts.LoggerFactory?.CreateLogger<TextContainer>() ?? NullLogger<TextContainer>.Instance);
foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
{
// If we had a WithinClause we only want the captures that passed the within filter.
var filteredCaptures = ruleCapture.Captures.Any(x => x.Clause is WithinClause)
? ruleCapture.Captures.Where(x => x.Clause is WithinClause) : ruleCapture.Captures;
if (cancellationToken?.IsCancellationRequested is true)
{
return resultsList;
}
foreach (var cap in ruleCapture.Captures)
foreach (var cap in filteredCaptures)
{
resultsList.AddRange(ProcessBoundary(cap));
}

5
AppInspector.RulesEngine/TextContainer.cs
Просмотреть файл

@ -56,8 +56,11 @@ namespace Microsoft.ApplicationInspector.RulesEngine
prefix = languages.GetCommentPrefix(Language);
suffix = languages.GetCommentSuffix(Language);
inline = languages.GetCommentInline(Language);
Languages = languages;
}
public Languages Languages { get; set; }
private bool _triedToConstructJsonDocument;
private JsonDocument? _jsonDocument;
internal IEnumerable<(string, Boundary)> GetStringFromJsonPath(string Path)

15
AppInspector.Tests/DefaultRules/TestDefaultRules.cs
Просмотреть файл

@ -1,4 +1,5 @@
using System.Diagnostics.CodeAnalysis;
using System;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using Microsoft.ApplicationInspector.Commands;
using Microsoft.ApplicationInspector.Logging;
@ -26,7 +27,19 @@ namespace AppInspector.Tests.DefaultRules
var loggerFactory = new LogOptions() {ConsoleVerbosityLevel = LogEventLevel.Verbose}.GetLoggerFactory();
VerifyRulesCommand command = new(options, loggerFactory);
VerifyRulesResult result = command.GetResult();
foreach (var unverified in result.Unverified)
{
Console.WriteLine("Failed to validate {0}",unverified.RulesId);
foreach (var error in unverified.Errors)
{
Console.WriteLine(error);
}
foreach (var oatError in unverified.OatIssues)
{
Console.WriteLine(oatError.Description);
}
}
Assert.AreEqual(VerifyRulesResult.ExitCode.Verified, result.ResultCode);
Assert.AreNotEqual(0, result.RuleStatusList.Count);
}

48
AppInspector.Tests/RuleProcessor/WithinClauseTests.cs
Просмотреть файл

@ -121,50 +121,6 @@ namespace AppInspector.Tests.RuleProcessor
}
Assert.AreEqual(expectedNumIssues, verifier.CheckIntegrity(rules).Sum(x => x.OatIssues.Count()));
}
[TestMethod]
public void ValidateRuleHasData()
{
RuleSet rules = new(_loggerFactory);
rules.AddString(validationRule, "TestRules");
IEnumerable<WithinClause> withinClauses = rules
.GetOatRules()
.SelectMany(x => x.Clauses)
.OfType<WithinClause>();
foreach (WithinClause clause in withinClauses)
{
clause.Data = new List<string>();
}
RulesVerifier verifier = new(new RulesVerifierOptions() {LoggerFactory = _loggerFactory});
var oatIssues = verifier.CheckIntegrity(rules).SelectMany(x => x.OatIssues);
foreach (var violation in oatIssues)
{
_logger.LogDebug(violation.Description);
}
Assert.AreEqual(1, verifier.CheckIntegrity(rules).Sum(x => x.OatIssues.Count()));
}
[TestMethod]
public void ValidateRegexAreValid()
{
RuleSet rules = new(_loggerFactory);
rules.AddString(validationRule, "TestRules");
IEnumerable<WithinClause> withinClauses = rules
.GetOatRules()
.SelectMany(x => x.Clauses)
.OfType<WithinClause>();
foreach (WithinClause clause in withinClauses)
{
clause.Data = new List<string>(){"^($"};
}
RulesVerifier verifier = new(new RulesVerifierOptions() {LoggerFactory = _loggerFactory});
var oatIssues = verifier.CheckIntegrity(rules).SelectMany(x => x.OatIssues);
foreach (var violation in oatIssues)
{
_logger.LogDebug(violation.Description);
}
Assert.AreEqual(1, verifier.CheckIntegrity(rules).Sum(x => x.OatIssues.Count()));
}
[DataRow(true, 1, new[] { 2 })]
[DataRow(false, 1, new[] { 3 })]
@ -174,7 +130,7 @@ namespace AppInspector.Tests.RuleProcessor
RuleSet rules = new(_loggerFactory);
var newRule = findingRangeZeroRule.Replace("REPLACE_NEGATE", invert.ToString().ToLowerInvariant());
rules.AddString(newRule, "TestRules");
Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules, new RuleProcessorOptions());
Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules, new RuleProcessorOptions(){Parallel = false});
if (_languages.FromFileNameOut("test.c", out LanguageInfo info))
{
List<MatchRecord> matches = processor.AnalyzeFile(insideFindingData, new Microsoft.CST.RecursiveExtractor.FileEntry("test.cs", new MemoryStream()), info);
@ -197,7 +153,7 @@ namespace AppInspector.Tests.RuleProcessor
{
RuleSet rules = new(_loggerFactory);
rules.AddString(multiLineRule, "multiline-tests");
Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules, new RuleProcessorOptions());
Microsoft.ApplicationInspector.RulesEngine.RuleProcessor processor = new(rules, new RuleProcessorOptions(){Parallel = false});
if (_languages.FromFileNameOut("test.c", out LanguageInfo info))
{
List<MatchRecord> matches = processor.AnalyzeFile(multiLineData, new Microsoft.CST.RecursiveExtractor.FileEntry("test.cs", new MemoryStream()), info);

2
AppInspector/AppInspector.Commands.csproj
Просмотреть файл

@ -54,7 +54,7 @@
<PackageReference Include="CommandLineParser" Version="2.9.1" />
<PackageReference Include="DotLiquid" Version="2.2.656" />
<PackageReference Include="Glob" Version="1.1.9" />
<PackageReference Include="Microsoft.CST.OAT" Version="1.2.19" />
<PackageReference Include="Microsoft.CST.OAT" Version="1.2.24" />
<PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.11" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />

3
AppInspector/Commands/VerifyRulesCommand.cs
Просмотреть файл

@ -1,6 +1,7 @@
// Copyright (C) Microsoft. All rights reserved.
// Licensed under the MIT License. See LICENSE.txt in the project root for license information.
using System.Linq;
using Microsoft.ApplicationInspector.RulesEngine.OatExtensions;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
@ -40,6 +41,8 @@ namespace Microsoft.ApplicationInspector.Commands
[JsonProperty(PropertyName ="ruleStatusList")]
public List<RuleStatus> RuleStatusList { get; set; }
[JsonIgnore] public IEnumerable<RuleStatus> Unverified => RuleStatusList.Where(x => !x.Verified);
public VerifyRulesResult()
{
RuleStatusList = new List<RuleStatus>();

2
Directory.Build.props
Просмотреть файл

@ -2,7 +2,7 @@
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<PackageReference Include="Nerdbank.GitVersioning">
<Version>3.5.108</Version>
<Version>3.5.109</Version>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>