0) I discover name clashing in the PT analysis. One option was to add context to each variable and field in the PTA and dependency analysis. I rolled back and added ImethodDefinition to all variables to get better context.  For these reason I need to add a method evry time the analysis creates a variable (e.g, return value).  Need to also update the analysis-net fork.
1) Improve support for Json. Now partially  understands access to deserialized dictionaries
2) Some minor fixes and refactors
This commit is contained in:
Diego 2018-02-10 18:37:51 -08:00
Родитель 8d1bb842e1
Коммит 41c17e921e
13 изменённых файлов: 109 добавлений и 67 удалений

Просмотреть файл

@ -120,8 +120,7 @@ namespace AnalysisClient
}
try
{
var sarifLog = ScopeProgramAnalysis.Program.AnalyzeDll(inputDll, ScopeProgramAnalysis.ScopeProgramAnalysis.ScopeMethodKind.All, true);
var sarifLog = ScopeProgramAnalysis.Program.AnalyzeDll(inputDll, ScopeProgramAnalysis.ScopeProgramAnalysis.ScopeMethodKind.All, true, true);
if (sarifLog == null)
{

Просмотреть файл

@ -351,7 +351,7 @@ namespace ScopeAnalyzer
var domFrontierAnalysis = new DominanceFrontierAnalysis(cfg);
domFrontierAnalysis.Analyze();
var splitter = new WebAnalysis(cfg);
var splitter = new WebAnalysis(cfg, methodDefinition);
splitter.Analyze();
splitter.Transform();

Просмотреть файл

@ -127,7 +127,7 @@ namespace ScopeProgramAnalysis
var ptgOfEntry = entryResult[cfgEntry.Exit.Id].Output;
// 2) Call the GetEnumerator that may create a new clousure and polulate it
var myGetEnumResult = new LocalVariable("$_temp_it") { Type = processToAnalyze.GetIteratorMethod.Type };
var myGetEnumResult = new LocalVariable("$_temp_it", processToAnalyze.EntryMethod) { Type = processToAnalyze.GetIteratorMethod.Type };
ptgOfEntry.Add(myGetEnumResult);
var ptgAfterEnum = this.interprocManager.PTAInterProcAnalysis(ptgOfEntry, new List<IVariable> { pointsToEntry.ReturnVariable }, myGetEnumResult, processToAnalyze.GetIteratorMethod);

Просмотреть файл

@ -338,8 +338,9 @@ namespace ScopeProgramAnalysis
public bool HasTraceables(IVariable arg)
{
return Dependencies.A2_Variables.ContainsKey(arg)
&& Dependencies.A2_Variables[arg].Count > 0;
return (Dependencies.A2_Variables.ContainsKey(arg)
&& Dependencies.A2_Variables[arg].Any())
|| GetTraceables(arg).Any();
}
@ -519,9 +520,11 @@ namespace ScopeProgramAnalysis
public ISet<IVariable> ControlVariables { get; set; }
//public SimplePointsToGraph PTG { get; set; }
//public ISet<Traceable> ControlTraceables { get; set; }
public DependencyDomain()
//public SimplePointsToGraph PTG { get; set; }
public DependencyDomain()
{
A2_References = new MapSet<SimplePTGNode, Traceable>();
@ -533,8 +536,9 @@ namespace ScopeProgramAnalysis
A1_Escaping = new HashSet<Traceable>();
ControlVariables = new HashSet<IVariable>();
//ControlTraceables = new HashSet<Traceable>();
IsTop = false;
IsTop = false;
}
private bool MapLessEqual<K, V>(MapSet<K, V> left, MapSet<K, V> right)
@ -583,15 +587,16 @@ namespace ScopeProgramAnalysis
if (oth.IsTop) return true;
else if (this.isTop) return false;
return oth != null
&& this.A1_Escaping.IsSubsetOf(oth.A1_Escaping)
&& MapLessEqual(A2_References, oth.A2_References)
&& MapLessEqual(A2_Variables, oth.A2_Variables)
&& MapLessEqual(A3_Fields, oth.A3_Fields)
&& MapLessEqual(A4_Ouput, oth.A4_Ouput)
&& MapLessEqual(A4_Ouput_Control, oth.A4_Ouput_Control)
&& ControlVariables.IsSubsetOf(oth.ControlVariables);
}
return oth != null
&& this.A1_Escaping.IsSubsetOf(oth.A1_Escaping)
&& MapLessEqual(A2_References, oth.A2_References)
&& MapLessEqual(A2_Variables, oth.A2_Variables)
&& MapLessEqual(A3_Fields, oth.A3_Fields)
&& MapLessEqual(A4_Ouput, oth.A4_Ouput)
&& MapLessEqual(A4_Ouput_Control, oth.A4_Ouput_Control)
&& ControlVariables.IsSubsetOf(oth.ControlVariables);
//&& ControlTraceables.IsSubsetOf(oth.ControlTraceables);
}
public override bool Equals(object obj)
{
// Add ControlVariables
@ -609,14 +614,15 @@ namespace ScopeProgramAnalysis
}
public override int GetHashCode()
{
// Add ControlVariables
return A1_Escaping.GetHashCode()
+ A2_Variables.GetHashCode()
+ A3_Fields.GetHashCode()
+ A4_Ouput.GetHashCode()
+ ControlVariables.GetHashCode();
// Add ControlVariables
return A1_Escaping.GetHashCode()
+ A2_Variables.GetHashCode()
+ A3_Fields.GetHashCode()
+ A4_Ouput.GetHashCode()
+ ControlVariables.GetHashCode();
//+ControlTraceables.GetHashCode();
}
}
public DependencyDomain Clone()
{
var result = new DependencyDomain();
@ -629,7 +635,8 @@ namespace ScopeProgramAnalysis
result.A4_Ouput = new MapSet<IVariable, Traceable>(this.A4_Ouput);
result.A4_Ouput_Control = new MapSet<IVariable, Traceable>(this.A4_Ouput_Control);
result.ControlVariables = new HashSet<IVariable>(this.ControlVariables);
return result;
//result.ControlTraceables = new HashSet<Traceable>(this.ControlTraceables);
return result;
}
public DependencyDomain Join(DependencyDomain right)
{
@ -659,8 +666,9 @@ namespace ScopeProgramAnalysis
result.A4_Ouput_Control = new MapSet<IVariable, Traceable>(this.A4_Ouput_Control);
result.ControlVariables = new HashSet<IVariable>(this.ControlVariables);
//result.ControlTraceables = new HashSet<Traceable>(this.ControlTraceables);
result.isTop = result.isTop || right.isTop;
result.isTop = result.isTop || right.isTop;
result.A2_References.UnionWith(right.A2_References);
result.A1_Escaping.UnionWith(right.A1_Escaping);
@ -670,7 +678,8 @@ namespace ScopeProgramAnalysis
result.A4_Ouput_Control.UnionWith(right.A4_Ouput_Control);
result.ControlVariables.UnionWith(right.ControlVariables);
}
//result.ControlTraceables.UnionWith(right.ControlTraceables);
}
return result;
}
@ -732,7 +741,8 @@ namespace ScopeProgramAnalysis
&& oth.A4_Ouput.MapEquals(A4_Ouput)
&& oth.A4_Ouput_Control.MapEquals(A4_Ouput_Control)
&& oth.ControlVariables.SetEquals(ControlVariables);
}
//&& oth.ControlTraceables.SetEquals(ControlTraceables);
}
}

Просмотреть файл

@ -81,7 +81,7 @@ namespace ScopeProgramAnalysis
private int stackDepth;
private IMetadataHost host;
private ScopeProcessorInfo processToAnalyze;
private const int MaxStackDepth = 100;
private const int MaxStackDepth = 5;
private Stack<IMethodDefinition> callStack;
public MethodCFGCache CFGCache { get; set; }
@ -154,9 +154,13 @@ namespace ScopeProgramAnalysis
/// <param name="calleeCFG"></param>
private InterProceduralReturnInfo InterproceduralAnalysis(InterProceduralCallInfo callInfo, ControlFlowGraph calleeCFG)
{
if (stackDepth > InterproceduralManager.MaxStackDepth)
return new InterProceduralReturnInfo(callInfo.CallerState);
if (stackDepth > InterproceduralManager.MaxStackDepth)
{
callInfo.CallerState.Dependencies.IsTop = true;
AnalysisStats.AddAnalysisReason(new AnalysisReason(callInfo.Caller, callInfo.Instruction, String.Format(CultureInfo.InvariantCulture, "Reach maximum call depth {0}", callInfo.Callee.Name)));
return new InterProceduralReturnInfo(callInfo.CallerState);
}
stackDepth++;
// I currently do not support recursive calls
// Will add support for this in the near future
@ -275,9 +279,11 @@ namespace ScopeProgramAnalysis
calleeDepDomain.Dependencies.A3_Fields = callInfo.CallerState.Dependencies.A3_Fields;
calleeDepDomain.Dependencies.ControlVariables = callInfo.CallerState.Dependencies.ControlVariables;
//calleeDepDomain.Dependencies.A1_Escaping.UnionWith(callInfo.CallerState.Dependencies.A1_Escaping);
//calleeDepDomain.Dependencies.A3_Clousures.UnionWith(callInfo.CallerState.Dependencies.A3_Clousures);
return calleeDepDomain;
//calleeDepDomain.Dependencies.ControlTraceables = callInfo.CallerState.Dependencies.ControlTraceables;
//calleeDepDomain.Dependencies.A1_Escaping.UnionWith(callInfo.CallerState.Dependencies.A1_Escaping);
//calleeDepDomain.Dependencies.A3_Clousures.UnionWith(callInfo.CallerState.Dependencies.A3_Clousures);
return calleeDepDomain;
}
private static IVariable AdaptIsReference(IVariable arg)
@ -324,7 +330,7 @@ namespace ScopeProgramAnalysis
{
//if (exitResult.HasOutputTraceables(outputVar))
{
var newVar = new LocalVariable(callInfo.Callee.Name + "_" + outputVar.Name) { Type = outputVar.Type };
var newVar = new LocalVariable(callInfo.Callee.Name + "_" + outputVar.Name, callInfo.Callee) { Type = outputVar.Type };
callInfo.CallerState.AddTraceables(newVar, exitResult.GetTraceables(outputVar));
callInfo.CallerState.AddOutputTraceables(newVar, exitResult.GetOutputTraceables(outputVar));
callInfo.CallerState.AddOutputControlTraceables(newVar, exitResult.GetOutputControlTraceables(outputVar));
@ -334,7 +340,7 @@ namespace ScopeProgramAnalysis
{
//if (exitResult.HasOutputControlTraceables(outputVar))
{
var newVar = new LocalVariable(callInfo.Callee.Name + "_" + outputVar.Name) { Type = outputVar.Type };
var newVar = new LocalVariable(callInfo.Callee.Name + "_" + outputVar.Name, callInfo.Callee) { Type = outputVar.Type };
callInfo.CallerState.AddTraceables(newVar, exitResult.GetTraceables(outputVar));
callInfo.CallerState.AddOutputControlTraceables(newVar, exitResult.GetOutputControlTraceables(outputVar));
}
@ -346,8 +352,10 @@ namespace ScopeProgramAnalysis
callInfo.CallerState.Dependencies.A2_References.UnionWith(exitResult.Dependencies.A2_References);
callInfo.CallerState.Dependencies.A3_Fields.UnionWith(exitResult.Dependencies.A3_Fields);
callInfo.CallerState.Dependencies.IsTop |= exitResult.Dependencies.IsTop;
//callInfo.CallerState.Dependencies.ControlTraceables.UnionWith(exitResult.Dependencies.ControlTraceables);
callInfo.CallerState.Dependencies.IsTop |= exitResult.Dependencies.IsTop;
if (callInfo.CallLHS != null)
{

Просмотреть файл

@ -812,10 +812,14 @@ namespace Backend.Analyses
var itState = this.State.IteratorState;
}
var validHeap = true;
if (fieldAccess.Type.IsClassOrStruct())
{
var nodes = currentPTG.GetTargets(fieldAccess.Instance);
validHeap = nodes.Any();
}
// this is a[loc(o.f)]
var nodes = currentPTG.GetTargets(fieldAccess.Instance);
if (nodes.Any())
if (validHeap)
{
// TODO: SHould I only consider the clousure fields?
traceables.UnionWith(this.State.GetHeapTraceables(fieldAccess.Instance, fieldAccess.Field));
@ -1045,9 +1049,9 @@ namespace Backend.Analyses
public override void Visit(ConditionalBranchInstruction instruction)
{
instruction.Accept(visitorPTA);
this.State.Dependencies.ControlVariables.UnionWith(instruction.UsedVariables.Where( v => this.State.GetTraceables(v).Any()));
}
//this.State.Dependencies.ControlTraceables.UnionWith(instruction.UsedVariables.SelectMany(v => this.State.GetTraceables(v)));
}
public override void Visit(ReturnInstruction instruction)
{
instruction.Accept(visitorPTA);
@ -1616,8 +1620,15 @@ namespace Backend.Analyses
{
if (methodInvoked.ContainingType.IsDictionary())
{
var itemField = this.iteratorDependencyAnalysis.pta.GetItemforCollection(this.State.PTG, methodCallStmt.Offset, methodCallStmt.Arguments[0], methodCallStmt.Result);
this.State.AssignTraceables(methodCallStmt.Result, this.State.GetHeapTraceables(methodCallStmt.Arguments[0], itemField));
if (methodCallStmt.Arguments.Count>1 && this.State.GetTraceables(methodCallStmt.Arguments[0]).OfType<TraceableJson>().Any())
{
var columLiteral = variableRanges.GetValue(methodCallStmt.Arguments[1]).Literal;
if (columLiteral != null)
AddJsonColumnFieldToTraceables(methodCallStmt, methodCallStmt.Arguments[0], String.Format("[{0}]",columLiteral));
}
var itemField = this.iteratorDependencyAnalysis.pta.GetItemforCollection(this.State.PTG, methodCallStmt.Offset, methodCallStmt.Arguments[0], methodCallStmt.Result);
this.State.AddTraceables(methodCallStmt.Result, this.State.GetHeapTraceables(methodCallStmt.Arguments[0], itemField));
// this.State.AddTraceables(methodCallStmt.Result, this.State.GetTraceables(methodCallStmt.Arguments[0]));
}
else
@ -1758,8 +1769,8 @@ namespace Backend.Analyses
&& (methodInvoked.ContainingType.IsIEnumeratorRow()
|| methodInvoked.ContainingType.IsIEnumeratorScopeMapUsage()))
{
if (this.iteratorDependencyAnalysis.processToAnalyze.ProcessorClass.GetName() == "ResourceDataTagFlattener")
{ }
//if (this.iteratorDependencyAnalysis.processToAnalyze.ProcessorClass.GetName() == "ResourceDataTagFlattener")
//{ }
var arg = methodCallStmt.Arguments[0];
var traceables = this.State.GetTraceables(arg);
@ -1808,6 +1819,7 @@ namespace Backend.Analyses
UpdatePTAForScopeMethod(methodCallStmt);
this.State.AddOutputTraceables(arg0, traceables);
//var controlTraceables = this.State.Dependencies.ControlTraceables; // this.State.Dependencies.ControlVariables.SelectMany(controlVar => this.State.GetTraceables(controlVar));
var controlTraceables = this.State.Dependencies.ControlVariables.SelectMany(controlVar => this.State.GetTraceables(controlVar));
this.State.AddOutputControlTraceables(arg0, controlTraceables);
@ -1823,6 +1835,7 @@ namespace Backend.Analyses
UpdatePTAForScopeMethod(methodCallStmt);
this.State.AddOutputTraceables(arg1, traceables);
//var controlTraceables = this.State.Dependencies.ControlTraceables; //this.State.Dependencies.ControlVariables.SelectMany(controlVar => this.State.GetTraceables(controlVar));
var controlTraceables = this.State.Dependencies.ControlVariables.SelectMany(controlVar => this.State.GetTraceables(controlVar));
this.State.AddOutputControlTraceables(arg1, controlTraceables);
@ -1856,8 +1869,13 @@ namespace Backend.Analyses
// We need to check somehow at the end if the information has not propagated
// One option: remenber arg0 and arg1 and check at the end if they have traceables.
// If they have (because of SSA) they will be assigned only here
this.State.SetTOP();
AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, methodCallStmt, "Could not determine the input or output table"));
if (this.numberOfVisits > MAX_ITERATIONS)
{
this.State.SetTOP();
AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, methodCallStmt, "Could not determine the input or output table"));
}
else
this.validBlock = false;
}
}
@ -2435,7 +2453,7 @@ namespace Backend.Analyses
this.protectedNodes = protectedNodes;
this.interproceduralManager = interprocManager;
this.initValue = null;
this.ReturnVariable = new LocalVariable(method.Name + "_$RV") { Type = Types.Instance.PlatformType.SystemObject };
this.ReturnVariable = new LocalVariable(method.Name + "_$RV", method) { Type = Types.Instance.PlatformType.SystemObject };
this.InterProceduralAnalysisEnabled = AnalysisOptions.DoInterProcAnalysis;
this.pta = pta;
this.rangeAnalysis = rangeAnalysis;
@ -2594,13 +2612,15 @@ namespace Backend.Analyses
var traceableInputColumn = new TraceableColumn(inputTable, column);
var traceableOutputColumn = new TraceableColumn(outputTable, column);
var outputColumnVar = new TemporalVariable(arg1.Name + "_$" + column.Name, 1) { Type = Types.Instance.PlatformType.SystemVoid };
var outputColumnVar = new TemporalVariable(arg1.Name + "_$" + column.Name, 1, method) { Type = Types.Instance.PlatformType.SystemVoid };
state.AssignTraceables(outputColumnVar, new Traceable[] { traceableOutputColumn });
state.AddOutputTraceables(outputColumnVar, new Traceable[] { traceableInputColumn });
var traceables = state.Dependencies.ControlVariables.SelectMany(controlVar => state.GetTraceables(controlVar));
state.AddOutputControlTraceables(outputColumnVar, traceables);
//var traceables = state.Dependencies.ControlTraceables;
var traceables = state.Dependencies.ControlVariables.SelectMany(controlVar => state.GetTraceables(controlVar));
state.AddOutputControlTraceables(outputColumnVar, traceables);
this.InputColumns.Add(traceableInputColumn);
this.OutputColumns.Add(traceableOutputColumn);

Просмотреть файл

@ -257,7 +257,7 @@ namespace ScopeProgramAnalysis
public RangeAnalysis(ControlFlowGraph cfg, IMethodDefinition method): base(cfg)
{
this.ReturnVariable = new LocalVariable(method.Name + "_" + "$RV") { Type = method.Type.PlatformType.SystemObject };
this.ReturnVariable = new LocalVariable(method.Name + "_" + "$RV", method) { Type = method.Type.PlatformType.SystemObject};
this.methodUnderAnalysis = method;
}
public RangeAnalysis(ControlFlowGraph cfg, IMethodDefinition method, VariableRangeDomain initRange) : this(cfg, method)

Просмотреть файл

@ -851,9 +851,9 @@ namespace ScopeProgramAnalysis
if (reducerClassDefinition.FullName().Contains(@"ScoperTransformer_4") || reducerClassDefinition.FullName().Contains(@"ScopeFilterTransformer_17"))
{
}
//else
// if (isCompilerGenerated)
// continue;
else
if (isCompilerGenerated)
continue;
if (processorsToAnalyze.Contains(reducerClassDefinition))
continue;

Просмотреть файл

@ -46,7 +46,7 @@ namespace ScopeProgramAnalysis.Framework
var domFrontierAnalysis = new DominanceFrontierAnalysis(cfg);
domFrontierAnalysis.Analyze();
var splitter = new WebAnalysis(cfg);
var splitter = new WebAnalysis(cfg, method);
splitter.Analyze();
splitter.Transform();

Просмотреть файл

@ -265,7 +265,7 @@ namespace Backend.Analyses
this.method = method;
if (GlobalVariable==null)
GlobalVariable = new LocalVariable("$Global") { Type = method.Type.PlatformType.SystemObject };
GlobalVariable = new LocalVariable("$Global", Dummy.MethodDefinition) { Type = method.Type.PlatformType.SystemObject };
// this.specialFields = specialFields;
this.CreateInitialGraph();
@ -345,7 +345,7 @@ namespace Backend.Analyses
private void CreateInitialGraph(bool createNodeForParams = true, SimplePointsToGraph initialGraph = null)
{
this.ReturnVariable = new LocalVariable(this.method.Name + "_" + "$RV") { Type = this.method.Type.PlatformType.SystemObject };
this.ReturnVariable = new LocalVariable(this.method.Name + "_" + "$RV", method) { Type = this.method.Type.PlatformType.SystemObject };
//IteratorPointsToAnalysis.GlobalVariable= new LocalVariable("$Global");
//IteratorPointsToAnalysis.GlobalVariable.Type = MyLoader.PlatformTypes.SystemObject;

Просмотреть файл

@ -169,6 +169,10 @@ namespace ScopeProgramAnalysis
input = @"C:\Temp\Scope\JsonSurvey\JobsWithUDOs\0ab0c6d5-731f-5029-8943-32ba867897c1\__ScopeCodeGen__.dll";
// 1 UDO with Json
input = @"C:\Temp\Scope\JsonSurvey\JobsWithUDOs\0a671f56-c4c1-47da-a0c0-09c5701854c8\__ScopeCodeGen__.dll";
// 1 UDO with Json + compiler generated process that generates a dictionary from JSon...
input = @"C:\Temp\Scope\JsonSurvey\JobsWithUDOs\0a671f56-c4c1-47da-a0c0-09c5701854c8\__ScopeCodeGen__.dll";
string[] directories = Path.GetDirectoryName(input).Split(Path.DirectorySeparatorChar);
var outputPath = Path.Combine(@"c:\Temp\", directories.Last()) + "_" + Path.ChangeExtension(Path.GetFileName(input), ".sarif");

Просмотреть файл

@ -27,9 +27,10 @@ namespace ScopeAnalysisBulkScripts
var outputAnalyzer = Path.Combine(rudderPath, @"CompareAnalysisOutput\Compare\bin\Debug\Compare.exe");
var inputFolder =// @"C:\Temp\Scope\ScopeJobs-2017-07-07-Cosmos11";
@"C:\Temp\Scope\First100JobsFromMadan";
//@"C:\Temp\Scope\InterestingScopeProjects";
// @"\\madanm2\parasail2\TFS\parasail\ScopeSurvey\AutoDownloader\bin\Debug";
//@"C:\Temp\Scope\InterestingScopeProjects";
// @"\\madanm2\parasail2\TFS\parasail\ScopeSurvey\AutoDownloader\bin\Debug";
//@"C:\Temp\Scope\First100JobsFromMadan";
@"C:\Temp\Scope\JsonSurvey\JobsWithUDOs";
if (doOnlyPassthrough)
{
inputFolder = @"C:\Temp\Scope\out-cosmos11";
@ -40,7 +41,7 @@ namespace ScopeAnalysisBulkScripts
var inputList = @"C:\Temp\Zvo\inputDlls.txt";
//var inputList = @"C:\Temp\Zvo\sampleDlls.txt";
var outputFolder = @"C:\Temp\Scope\OutNew";
var outputFolder = @"C:\Temp\Scope\OutJSon";
//outputFolder = @"C:\Temp\Mike100";
//outputFolder = @"C:\temp\ZvoList";

@ -1 +1 @@
Subproject commit 8c65ab73c8c85f8b863ea2380c396645ad075ae0
Subproject commit 058dececd7aaf0e00119ca9f6003448243ad73e5