diff --git a/ScopeProgramAnalysis/Analysis/IteratorAnalysis.cs b/ScopeProgramAnalysis/Analysis/IteratorAnalysis.cs index 75aca6a..c08f560 100644 --- a/ScopeProgramAnalysis/Analysis/IteratorAnalysis.cs +++ b/ScopeProgramAnalysis/Analysis/IteratorAnalysis.cs @@ -512,6 +512,8 @@ namespace Backend.Analyses internal class MoveNextVisitorForDependencyAnalysis : InstructionVisitor { + private static readonly int MAX_ITERATIONS = 3; + private IDictionary equalities; private IteratorDependencyAnalysis iteratorDependencyAnalysis; @@ -523,9 +525,13 @@ namespace Backend.Analyses private PTAVisitor visitorPTA; private VariableRangeDomain variableRanges; private bool validBlock; + // Used to check if all predecessors where traversed at least once. Maybe no longer needed + private bool predecessorsVisited; + + int numberOfVisits; public MoveNextVisitorForDependencyAnalysis(IteratorDependencyAnalysis iteratorDependencyAnalysis, - CFGNode cfgNode, DependencyPTGDomain oldInput) + CFGNode cfgNode, DependencyPTGDomain oldInput, int numberOfVisits ,bool predecessorsVisited = true) { // A visitor for the points-to graph @@ -540,7 +546,10 @@ namespace Backend.Analyses this.method = iteratorDependencyAnalysis.method; this.visitorPTA = visitorPTA; this.variableRanges = this.iteratorDependencyAnalysis.rangeAnalysis.Result[cfgNode.Id].Output; - } + + this.numberOfVisits = numberOfVisits; + this.predecessorsVisited = predecessorsVisited; + } private bool IsClousureType(IVariable instance) { @@ -1074,7 +1083,7 @@ namespace Backend.Analyses if (!isScopeRowMethod) { // Analyze methods that parte JsonObjects (that can be accessed as "fields" in columns) - var isJsonMethod = AnalyzeJsonMethod(methodCallStmt, methodInvoked); + var isJsonMethod = HandleJsonRelatedMethod(methodCallStmt, methodInvoked); if (!isJsonMethod) { // Analyze collection handling methods (lists, sets, dictionaries) @@ -1178,9 +1187,15 @@ namespace Backend.Analyses } } - private bool AnalyzeJsonMethod(MethodCallInstruction methodCallStmt, IMethodReference methodInvoked) + private bool HandleJsonRelatedMethod(MethodCallInstruction methodCallStmt, IMethodReference methodInvoked) { - if (methodInvoked.ContainingType.GetFullName() == "Newtonsoft.Json.JsonConvert") + if (methodInvoked.Name.Value == "ToString" && methodCallStmt.Arguments[0].Type.GetFullName().Contains("Newtonsoft.Json")) + { + this.State.CopyTraceables(methodCallStmt.Result, methodCallStmt.Arguments[0]); + UpdatePTAForScopeMethod(methodCallStmt); + return true; + } + else if (methodInvoked.ContainingType.GetFullName() == "Newtonsoft.Json.JsonConvert") { if (methodInvoked.Name.Value == "DeserializeObject") { @@ -1192,13 +1207,15 @@ namespace Backend.Analyses return true; } - // DIEGODIEGO: Should I handle this as a Pure? else if (methodInvoked.Name.Value == "SerializeObject") { var arg = methodCallStmt.Arguments[0]; AddJsonColumnFieldToTraceables(methodCallStmt, arg, "*"); return true; } + else + { + } } else if (methodInvoked.ContainingType.GetFullName() == "Newtonsoft.Json.Linq.JObject") { @@ -1233,6 +1250,10 @@ namespace Backend.Analyses return true; ; } + else + { + } + } else if (methodInvoked.ContainingType.GetFullName() == "Newtonsoft.Json.Linq.JToken" || methodInvoked.ContainingType.GetFullName() == "ScopeRuntime.StringColumnData") { @@ -1242,6 +1263,7 @@ namespace Backend.Analyses UpdatePTAForScopeMethod(methodCallStmt); return true; ; } + else { } } else if (methodInvoked.ContainingType.GetFullName() == "Microsoft.DataMap.Common.Tag") { @@ -1252,6 +1274,9 @@ namespace Backend.Analyses AddJsonColumnFieldToTraceables(methodCallStmt, arg, columName); return true; } + else + { + } } return false; } @@ -1394,7 +1419,10 @@ namespace Backend.Analyses return true; } - var containingType = metodCallStmt.Method.ContainingType; + if (metodCallStmt.Method.Name.Value == "ToString") + return true; + + var containingType = metodCallStmt.Method.ContainingType; if (containingType.IsString()) { @@ -1404,12 +1432,15 @@ namespace Backend.Analyses { return true; } - if ( containingType.IsValueType()) + if (containingType.IsValueType()) { return true; } - var specialMethods = new Tuple[] { Tuple.Create("System.IDisposable", "Dispose") }; + if (metodCallStmt.Method.Name.Value == "ToString") + return true; + + var specialMethods = new Tuple[] { Tuple.Create("System.IDisposable", "Dispose"), }; result = specialMethods.Any(sm => sm.Item1 == containingType.GetFullName() && sm.Item2 == metodCallStmt.Method.Name.Value); @@ -1632,15 +1663,18 @@ namespace Backend.Analyses { if (!traceables.Any()) { - // DIEGODIEGO: We need to check this + // DIEGODIEGODIEGO: We need to check this // When the analysis fail I'm giving a new opportunity // and I mark the block as invalid. - // We need to actually check whether if the block has - // all predecesors analyzed at least once or - // (if we can) check if we are at the right iterator state - // this.State.SetTOP(); - AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, instruction, "We are expecting a traceable and there isn't any")); - this.validBlock = false; + if (this.numberOfVisits > MAX_ITERATIONS) + { + this.State.SetTOP(); + AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, instruction, "We are expecting a traceable and there isn't any")); + } + else + this.validBlock = false; + + } } @@ -2024,11 +2058,15 @@ namespace Backend.Analyses return result; } } - // DIEGODIEGO: We need to check this - // Check comment about having analyzed the previous predecessors - //this.State.SetTOP(); - this.validBlock = false; - AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, methodCallStmt, "Scope Table mapping not available. Could not get schema")); + // DIEGODIEGODIEGO: We need to check this + // Check comment about having analyzed enough + if (this.numberOfVisits > MAX_ITERATIONS) + { + this.State.SetTOP(); + AnalysisStats.AddAnalysisReason(new AnalysisReason(this.method, methodCallStmt, "Scope Table mapping not available. Could not get schema")); + } + else + this.validBlock = false; } return result; } @@ -2464,18 +2502,31 @@ namespace Backend.Analyses return elem.Clone(); } + IDictionary numberOfVisits = new Dictionary(); + protected override DependencyPTGDomain Flow(CFGNode node, DependencyPTGDomain input) { if (input.IsTop) return input; var oldInput = input.Clone(); - // var currentPTG = pta.Result[node.Id].Output; + // var currentPTG = pta.Result[node.Id].Output; - // A visitor for the dependency analysis - var visitor = new MoveNextVisitorForDependencyAnalysis(this, node, oldInput); + + // var predecessorsVisited = node.Predecessors.All(n => visited.Contains(n)); + + int count = 0; + if (!numberOfVisits.TryGetValue(node, out count)) + { + numberOfVisits[node] = 0; + } + + // A visitor for the dependency analysis + var visitor = new MoveNextVisitorForDependencyAnalysis(this, node, oldInput, count); visitor.Visit(node); + numberOfVisits[node] = count + 1; + return visitor.State; } public void CopyRow(DependencyPTGDomain state, IVariable arg1, diff --git a/ScopeProgramAnalysis/Program.cs b/ScopeProgramAnalysis/Program.cs index a896009..dd7eeb7 100644 --- a/ScopeProgramAnalysis/Program.cs +++ b/ScopeProgramAnalysis/Program.cs @@ -26,8 +26,9 @@ namespace ScopeProgramAnalysis { var useScopeFactory = true; var scopeKind = ScopeMethodKind.All; + var interProcAnalysis = false; - string input; + string input; //const string root = @"c:\users\t-diga\source\repos\scopeexamples\metting\"; //const string input = root + @"__ScopeCodeGen__.dll"; @@ -150,8 +151,24 @@ namespace ScopeProgramAnalysis input = @"C:\Temp\Scope\InterestingScopeProjects\0ce5ea59-dec8-4f6f-be08-0e0746e12515\__ScopeCodeGen__.dll"; input = @"C:\Temp\Scope\NewtonSoftMethodSurvey_E18FC06FBAF9E44\__ScopeCodeGen__.dll"; - input = @"C:\Temp\Scope\JsonSurvey\0195e2b3-3fb3-4f36-bc10-cadbfd76c8cd\__ScopeCodeGen__.dll"; - input = @"C:\Temp\Scope\JsonSurvey\SimpleJsonProcessors\bin\Debug\6BEA71AFD72D97FF\UDO1_C79877185261167E\__ScopeCodeGen__.dll"; + //input = @"C:\Temp\Scope\JsonSurvey\0195e2b3-3fb3-4f36-bc10-cadbfd76c8cd\__ScopeCodeGen__.dll"; + // Example that uses JsonConvert + //var jsonObj = JsonConvert.DeserializeObject(s); + // string a = jsonObj.A; + //input = @"C:\Temp\Scope\JsonSurvey\SimpleJsonProcessors\bin\Debug\6BEA71AFD72D97FF\UDO1_C79877185261167E\__ScopeCodeGen__.dll"; + + // Example that uses o = JObject.Parse and o[field] + // This one is too complex. I uses complex getters and setters to access the row columns + // input = @"C:\Temp\Scope\JsonSurvey\0195e2b3-3fb3-4f36-bc10-cadbfd76c8cd\__ScopeCodeGen__.dll"; + + // Simpler example made my Mike + input = @"C:\Temp\Scope\JsonSurvey\ProcessorWithJObject\__ScopeCodeGen__.dll"; + + // UDO with JSon extracted by Mike + input = @"C:\Temp\Scope\JsonSurvey\JobsWithUDOs\0a671f56-c4c1-47da-a0c0-09c5701854c8\__ScopeCodeGen__.dll"; + + input = @"C:\Temp\Scope\JsonSurvey\JobsWithUDOs\0ab0c6d5-731f-5029-8943-32ba867897c1\__ScopeCodeGen__.dll"; + string[] directories = Path.GetDirectoryName(input).Split(Path.DirectorySeparatorChar); var outputPath = Path.Combine(@"c:\Temp\", directories.Last()) + "_" + Path.ChangeExtension(Path.GetFileName(input), ".sarif"); @@ -159,7 +176,7 @@ namespace ScopeProgramAnalysis var logPath = Path.Combine(@"c:\Temp\", "analysis.log"); var outputStream = File.CreateText(logPath); - var log = AnalyzeDll(input, scopeKind, useScopeFactory, false); + var log = AnalyzeDll(input, scopeKind, useScopeFactory, interProcAnalysis); SarifLogger.WriteSarifOutput(log, outputPath); diff --git a/ScopeProgramAnalysis/ScopeAnalysisBulkScripts/BulkAnalysis.cs b/ScopeProgramAnalysis/ScopeAnalysisBulkScripts/BulkAnalysis.cs index 5f72cc5..d5f8ee0 100644 --- a/ScopeProgramAnalysis/ScopeAnalysisBulkScripts/BulkAnalysis.cs +++ b/ScopeProgramAnalysis/ScopeAnalysisBulkScripts/BulkAnalysis.cs @@ -27,8 +27,8 @@ namespace ScopeAnalysisBulkScripts var outputAnalyzer = Path.Combine(rudderPath, @"CompareAnalysisOutput\Compare\bin\Debug\Compare.exe"); var inputFolder =// @"C:\Temp\Scope\ScopeJobs-2017-07-07-Cosmos11"; - //@"C:\Temp\Scope\First100JobsFromMadan"; - @"C:\Temp\Scope\InterestingScopeProjects"; + @"C:\Temp\Scope\First100JobsFromMadan"; + //@"C:\Temp\Scope\InterestingScopeProjects"; // @"\\madanm2\parasail2\TFS\parasail\ScopeSurvey\AutoDownloader\bin\Debug"; if (doOnlyPassthrough) { @@ -40,7 +40,7 @@ namespace ScopeAnalysisBulkScripts var inputList = @"C:\Temp\Zvo\inputDlls.txt"; //var inputList = @"C:\Temp\Zvo\sampleDlls.txt"; - var outputFolder = @"C:\Temp\Scope\out"; + var outputFolder = @"C:\Temp\Scope\OutNew"; //outputFolder = @"C:\Temp\Mike100"; //outputFolder = @"C:\temp\ZvoList";