зеркало из https://github.com/microsoft/rudder.git
Put a limit in the number of iterations in the fix point (a sort of wideniing). Now we only give top if it does not get traceables after a minimun number of iterations (need to check)
Add support for a couple of more Json methods.
This commit is contained in:
Diego
Родитель
d871ca4410
Коммит
0089f0420c
|
|
@ -512,6 +512,8 @@ namespace Backend.Analyses
|
|||
|
||||
internal class MoveNextVisitorForDependencyAnalysis : InstructionVisitor
|
||||
{
|
||||
private static readonly int MAX_ITERATIONS = 3;
|
||||
|
||||
private IDictionary<IVariable, IExpression> equalities;
|
||||
private IteratorDependencyAnalysis iteratorDependencyAnalysis;
|
||||
|
||||
|
|
@ -523,9 +525,13 @@ namespace Backend.Analyses
|
|||
private PTAVisitor visitorPTA;
|
||||
private VariableRangeDomain variableRanges;
|
||||
private bool validBlock;
|
||||
// Used to check if all predecessors where traversed at least once. Maybe no longer needed
|
||||
private bool predecessorsVisited;
|
||||
|
||||
int numberOfVisits;
|
||||
|
||||
public MoveNextVisitorForDependencyAnalysis(IteratorDependencyAnalysis iteratorDependencyAnalysis,
|
||||
CFGNode cfgNode, DependencyPTGDomain oldInput)
|
||||
CFGNode cfgNode, DependencyPTGDomain oldInput, int numberOfVisits ,bool predecessorsVisited = true)
|
||||
{
|
||||
|
||||
// A visitor for the points-to graph
|
||||
|
|
@ -540,7 +546,10 @@ namespace Backend.Analyses
|
|||
this.method = iteratorDependencyAnalysis.method;
|
||||
this.visitorPTA = visitorPTA;
|
||||
this.variableRanges = this.iteratorDependencyAnalysis.rangeAnalysis.Result[cfgNode.Id].Output;
|
||||
}
|
||||
|
||||
this.numberOfVisits = numberOfVisits;
|
||||
this.predecessorsVisited = predecessorsVisited;
|
||||
}
|
||||
|
||||
private bool IsClousureType(IVariable instance)
|
||||
{
|
||||
|
|
@ -1074,7 +1083,7 @@ namespace Backend.Analyses
|
|||
if (!isScopeRowMethod)
|
||||
{
|
||||
// Analyze methods that parte JsonObjects (that can be accessed as "fields" in columns)
|
||||
var isJsonMethod = AnalyzeJsonMethod(methodCallStmt, methodInvoked);
|
||||
var isJsonMethod = HandleJsonRelatedMethod(methodCallStmt, methodInvoked);
|
||||
if (!isJsonMethod)
|
||||
{
|
||||
// Analyze collection handling methods (lists, sets, dictionaries)
|
||||
|
|
@ -1178,9 +1187,15 @@ namespace Backend.Analyses
|
|||
}
|
||||
}
|
||||
|
||||
private bool AnalyzeJsonMethod(MethodCallInstruction methodCallStmt, IMethodReference methodInvoked)
|
||||
private bool HandleJsonRelatedMethod(MethodCallInstruction methodCallStmt, IMethodReference methodInvoked)
|
||||
{
|
||||
if (methodInvoked.ContainingType.GetFullName() == "Newtonsoft.Json.JsonConvert")
|
||||
if (methodInvoked.Name.Value == "ToString" && methodCallStmt.Arguments[0].Type.GetFullName().Contains("Newtonsoft.Json"))
|
||||
{
|
||||
this.State.CopyTraceables(methodCallStmt.Result, methodCallStmt.Arguments[0]);
|
||||
UpdatePTAForScopeMethod(methodCallStmt);
|
||||
return true;
|
||||
}
|
||||
else if (methodInvoked.ContainingType.GetFullName() == "Newtonsoft.Json.JsonConvert")
|
||||
{
|
||||
if (methodInvoked.Name.Value == "DeserializeObject")
|
||||
{
|
||||
|
|
@ -1192,13 +1207,15 @@ namespace Backend.Analyses
|
|||
|
||||
return true;
|
||||
}
|
||||
// DIEGODIEGO: Should I handle this as a Pure?
|
||||
else if (methodInvoked.Name.Value == "SerializeObject")
|
||||
{
|
||||
var arg = methodCallStmt.Arguments[0];
|
||||
AddJsonColumnFieldToTraceables(methodCallStmt, arg, "*");
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
}
|
||||
}
|
||||
else if (methodInvoked.ContainingType.GetFullName() == "Newtonsoft.Json.Linq.JObject")
|
||||
{
|
||||
|
|
@ -1233,6 +1250,10 @@ namespace Backend.Analyses
|
|||
|
||||
return true; ;
|
||||
}
|
||||
else
|
||||
{
|
||||
}
|
||||
|
||||
}
|
||||
else if (methodInvoked.ContainingType.GetFullName() == "Newtonsoft.Json.Linq.JToken" || methodInvoked.ContainingType.GetFullName() == "ScopeRuntime.StringColumnData")
|
||||
{
|
||||
|
|
@ -1242,6 +1263,7 @@ namespace Backend.Analyses
|
|||
UpdatePTAForScopeMethod(methodCallStmt);
|
||||
return true; ;
|
||||
}
|
||||
else { }
|
||||
}
|
||||
else if (methodInvoked.ContainingType.GetFullName() == "Microsoft.DataMap.Common.Tag")
|
||||
{
|
||||
|
|
@ -1252,6 +1274,9 @@ namespace Backend.Analyses
|
|||
AddJsonColumnFieldToTraceables(methodCallStmt, arg, columName);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
@ -1394,7 +1419,10 @@ namespace Backend.Analyses
|
|||
return true;
|
||||
}
|
||||
|
||||
var containingType = metodCallStmt.Method.ContainingType;
|
||||
if (metodCallStmt.Method.Name.Value == "ToString")
|
||||
return true;
|
||||
|
||||
var containingType = metodCallStmt.Method.ContainingType;
|
||||
|
||||
if (containingType.IsString())
|
||||
{
|
||||
|
|
@ -1404,12 +1432,15 @@ namespace Backend.Analyses
|
|||
{
|
||||
return true;
|
||||
}
|
||||
if ( containingType.IsValueType())
|
||||
if (containingType.IsValueType())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var specialMethods = new Tuple<string, string>[] { Tuple.Create("System.IDisposable", "Dispose") };
|
||||
if (metodCallStmt.Method.Name.Value == "ToString")
|
||||
return true;
|
||||
|
||||
var specialMethods = new Tuple<string, string>[] { Tuple.Create("System.IDisposable", "Dispose"), };
|
||||
result = specialMethods.Any(sm => sm.Item1 == containingType.GetFullName()
|
||||
&& sm.Item2 == metodCallStmt.Method.Name.Value);
|
||||
|
||||
|
|
@ -1632,15 +1663,18 @@ namespace Backend.Analyses
|
|||
{
|
||||
if (!traceables.Any())
|
||||
{
|
||||
// DIEGODIEGO: We need to check this
|
||||
// DIEGODIEGODIEGO: We need to check this
|
||||
// When the analysis fail I'm giving a new opportunity
|
||||
// and I mark the block as invalid.
|
||||
// We need to actually check whether if the block has
|
||||
// all predecesors analyzed at least once or
|
||||
// (if we can) check if we are at the right iterator state
|
||||
// this.State.SetTOP();
|
||||
AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, instruction, "We are expecting a traceable and there isn't any"));
|
||||
this.validBlock = false;
|
||||
if (this.numberOfVisits > MAX_ITERATIONS)
|
||||
{
|
||||
this.State.SetTOP();
|
||||
AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, instruction, "We are expecting a traceable and there isn't any"));
|
||||
}
|
||||
else
|
||||
this.validBlock = false;
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2024,11 +2058,15 @@ namespace Backend.Analyses
|
|||
return result;
|
||||
}
|
||||
}
|
||||
// DIEGODIEGO: We need to check this
|
||||
// Check comment about having analyzed the previous predecessors
|
||||
//this.State.SetTOP();
|
||||
this.validBlock = false;
|
||||
AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, methodCallStmt, "Scope Table mapping not available. Could not get schema"));
|
||||
// DIEGODIEGODIEGO: We need to check this
|
||||
// Check comment about having analyzed enough
|
||||
if (this.numberOfVisits > MAX_ITERATIONS)
|
||||
{
|
||||
this.State.SetTOP();
|
||||
AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, methodCallStmt, "Scope Table mapping not available. Could not get schema"));
|
||||
}
|
||||
else
|
||||
this.validBlock = false;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -2464,18 +2502,31 @@ namespace Backend.Analyses
|
|||
return elem.Clone();
|
||||
}
|
||||
|
||||
IDictionary<CFGNode,int> numberOfVisits = new Dictionary<CFGNode,int>();
|
||||
|
||||
protected override DependencyPTGDomain Flow(CFGNode node, DependencyPTGDomain input)
|
||||
{
|
||||
if (input.IsTop)
|
||||
return input;
|
||||
|
||||
var oldInput = input.Clone();
|
||||
// var currentPTG = pta.Result[node.Id].Output;
|
||||
// var currentPTG = pta.Result[node.Id].Output;
|
||||
|
||||
// A visitor for the dependency analysis
|
||||
var visitor = new MoveNextVisitorForDependencyAnalysis(this, node, oldInput);
|
||||
|
||||
// var predecessorsVisited = node.Predecessors.All(n => visited.Contains(n));
|
||||
|
||||
int count = 0;
|
||||
if (!numberOfVisits.TryGetValue(node, out count))
|
||||
{
|
||||
numberOfVisits[node] = 0;
|
||||
}
|
||||
|
||||
// A visitor for the dependency analysis
|
||||
var visitor = new MoveNextVisitorForDependencyAnalysis(this, node, oldInput, count);
|
||||
visitor.Visit(node);
|
||||
|
||||
numberOfVisits[node] = count + 1;
|
||||
|
||||
return visitor.State;
|
||||
}
|
||||
public void CopyRow(DependencyPTGDomain state, IVariable arg1,
|
||||
|
|
|
|||
|
|
@ -26,8 +26,9 @@ namespace ScopeProgramAnalysis
|
|||
{
|
||||
var useScopeFactory = true;
|
||||
var scopeKind = ScopeMethodKind.All;
|
||||
var interProcAnalysis = false;
|
||||
|
||||
string input;
|
||||
string input;
|
||||
|
||||
//const string root = @"c:\users\t-diga\source\repos\scopeexamples\metting\";
|
||||
//const string input = root + @"__ScopeCodeGen__.dll";
|
||||
|
|
@ -150,8 +151,24 @@ namespace ScopeProgramAnalysis
|
|||
input = @"C:\Temp\Scope\InterestingScopeProjects\0ce5ea59-dec8-4f6f-be08-0e0746e12515\__ScopeCodeGen__.dll";
|
||||
input = @"C:\Temp\Scope\NewtonSoftMethodSurvey_E18FC06FBAF9E44\__ScopeCodeGen__.dll";
|
||||
|
||||
input = @"C:\Temp\Scope\JsonSurvey\0195e2b3-3fb3-4f36-bc10-cadbfd76c8cd\__ScopeCodeGen__.dll";
|
||||
input = @"C:\Temp\Scope\JsonSurvey\SimpleJsonProcessors\bin\Debug\6BEA71AFD72D97FF\UDO1_C79877185261167E\__ScopeCodeGen__.dll";
|
||||
//input = @"C:\Temp\Scope\JsonSurvey\0195e2b3-3fb3-4f36-bc10-cadbfd76c8cd\__ScopeCodeGen__.dll";
|
||||
// Example that uses JsonConvert
|
||||
//var jsonObj = JsonConvert.DeserializeObject<MyType>(s);
|
||||
// string a = jsonObj.A;
|
||||
//input = @"C:\Temp\Scope\JsonSurvey\SimpleJsonProcessors\bin\Debug\6BEA71AFD72D97FF\UDO1_C79877185261167E\__ScopeCodeGen__.dll";
|
||||
|
||||
// Example that uses o = JObject.Parse and o[field]
|
||||
// This one is too complex. I uses complex getters and setters to access the row columns
|
||||
// input = @"C:\Temp\Scope\JsonSurvey\0195e2b3-3fb3-4f36-bc10-cadbfd76c8cd\__ScopeCodeGen__.dll";
|
||||
|
||||
// Simpler example made my Mike
|
||||
input = @"C:\Temp\Scope\JsonSurvey\ProcessorWithJObject\__ScopeCodeGen__.dll";
|
||||
|
||||
// UDO with JSon extracted by Mike
|
||||
input = @"C:\Temp\Scope\JsonSurvey\JobsWithUDOs\0a671f56-c4c1-47da-a0c0-09c5701854c8\__ScopeCodeGen__.dll";
|
||||
|
||||
input = @"C:\Temp\Scope\JsonSurvey\JobsWithUDOs\0ab0c6d5-731f-5029-8943-32ba867897c1\__ScopeCodeGen__.dll";
|
||||
|
||||
|
||||
string[] directories = Path.GetDirectoryName(input).Split(Path.DirectorySeparatorChar);
|
||||
var outputPath = Path.Combine(@"c:\Temp\", directories.Last()) + "_" + Path.ChangeExtension(Path.GetFileName(input), ".sarif");
|
||||
|
|
@ -159,7 +176,7 @@ namespace ScopeProgramAnalysis
|
|||
var logPath = Path.Combine(@"c:\Temp\", "analysis.log");
|
||||
var outputStream = File.CreateText(logPath);
|
||||
|
||||
var log = AnalyzeDll(input, scopeKind, useScopeFactory, false);
|
||||
var log = AnalyzeDll(input, scopeKind, useScopeFactory, interProcAnalysis);
|
||||
SarifLogger.WriteSarifOutput(log, outputPath);
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ namespace ScopeAnalysisBulkScripts
|
|||
var outputAnalyzer = Path.Combine(rudderPath, @"CompareAnalysisOutput\Compare\bin\Debug\Compare.exe");
|
||||
|
||||
var inputFolder =// @"C:\Temp\Scope\ScopeJobs-2017-07-07-Cosmos11";
|
||||
//@"C:\Temp\Scope\First100JobsFromMadan";
|
||||
@"C:\Temp\Scope\InterestingScopeProjects";
|
||||
@"C:\Temp\Scope\First100JobsFromMadan";
|
||||
//@"C:\Temp\Scope\InterestingScopeProjects";
|
||||
// @"\\madanm2\parasail2\TFS\parasail\ScopeSurvey\AutoDownloader\bin\Debug";
|
||||
if (doOnlyPassthrough)
|
||||
{
|
||||
|
|
@ -40,7 +40,7 @@ namespace ScopeAnalysisBulkScripts
|
|||
|
||||
var inputList = @"C:\Temp\Zvo\inputDlls.txt";
|
||||
//var inputList = @"C:\Temp\Zvo\sampleDlls.txt";
|
||||
var outputFolder = @"C:\Temp\Scope\out";
|
||||
var outputFolder = @"C:\Temp\Scope\OutNew";
|
||||
//outputFolder = @"C:\Temp\Mike100";
|
||||
//outputFolder = @"C:\temp\ZvoList";
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче