Merge pull request #5 from unity/JsonParserUpdate

Update PBR to support Json format performance test run files

UnityPerformanceBenchmarkReporter/ESupportedFileTypes.cs

@@ -0,0 +1,8 @@
+namespace UnityPerformanceBenchmarkReporter.Entities
+{
+    public enum ESupportedFileTypes
+    {
+        json,
+        xml
+    }
+}

UnityPerformanceBenchmarkReporter/Entities/Data.cs

@@ -18,6 +18,7 @@ namespace UnityPerformanceBenchmarkReporter.Entities.New
         public string Name;
         public SampleUnit Unit;
         public bool IncreaseIsBetter;
+        public double Threshold = 0.15;
         public List<double> Samples = new List<double>();
         public double Min;
         public double Max;
@@ -65,6 +66,9 @@ namespace UnityPerformanceBenchmarkReporter.Entities.New
         public int ProcessorCount;
         public string GraphicsDeviceName;
         public int SystemMemorySizeMB;
+
+        public string XrModel ;
+        public string XrDevice;
     }
 
     [Serializable]

UnityPerformanceBenchmarkReporter/Entities/PerformanceTestRun.cs

@@ -12,10 +12,25 @@ namespace UnityPerformanceBenchmarkReporter.Entities
         public ScreenSettings ScreenSettings;
         public QualitySettings QualitySettings;
         public PlayerSettings PlayerSettings;
+        public ProjectVersion ProjectVersion;
+
+        public string TestProject;
         public string TestSuite;
         public double StartTime;
         public double EndTime;
         public List<PerformanceTestResult> Results  = new List<PerformanceTestResult>();
+
+        public JobMetaData JobMetaData ;
+        public object Dependencies;
+    }
+
+    [Serializable]    
+        public class ProjectVersion
+    {
+        public string ProjectName { get; set; }
+        public string Branch { get; set; }
+        public string Changeset { get; set; }
+        public DateTime Date { get; set; }
     }
 
     [Serializable]
@@ -73,13 +88,12 @@ namespace UnityPerformanceBenchmarkReporter.Entities
     public class PlayerSettings
     {
         public string ScriptingBackend;
-        public string ScriptingRuntimeVersion;
         public bool VrSupported;
         public bool MtRendering;
         public bool GraphicsJobs;
         public bool GpuSkinning;
         public string GraphicsApi;
-        //public string Batchmode; TODO
+        public string Batchmode; 
         //public int StaticBatching; TODO
         //public int DynamicBatching; TODO
         public string StereoRenderingPath;
@@ -87,5 +101,27 @@ namespace UnityPerformanceBenchmarkReporter.Entities
         public string AndroidMinimumSdkVersion;
         public string AndroidTargetSdkVersion;
         public List<string> EnabledXrTargets;
+
+        public string ScriptingRuntimeVersion;
+    }
+    
+
+    [Serializable]
+     public class Yamato
+    {
+        public string JobFriendlyName { get; set; }
+        public string JobName { get; set; }
+        public string JobId { get; set; }
+        public string ProjectId { get; set; }
+        public string ProjectName { get; set; }
+        public object WorkDir { get; set; }
+        public object JobOwnerEmail { get; set; }
+    }
+
+    [Serializable]
+    public class JobMetaData
+    {
+        public Yamato Yamato { get; set; }
+        public object Bokken { get; set; }
     }
 }

UnityPerformanceBenchmarkReporter/Entities/SampleGroup.cs

@@ -28,6 +28,8 @@ namespace UnityPerformanceBenchmarkReporter.Entities
         public double Threshold;
         public bool IncreaseIsBetter;
         public double Percentile;
+
+        public bool FailOnBaseline;
     }
 
     public enum AggregationType

UnityPerformanceBenchmarkReporter/Entities/SampleGroupResult.cs

@@ -12,6 +12,7 @@ namespace UnityPerformanceBenchmarkReporter.Entities
         public string AggregationType;
         public double Percentile;
         public bool Regressed;
+        public bool Progressed;
         public double Min;
         public double Max;
         public double Median;

UnityPerformanceBenchmarkReporter/IParser.cs

@@ -0,0 +1,9 @@
+using UnityPerformanceBenchmarkReporter.Entities;
+
+namespace UnityPerformanceBenchmarkReporter
+{
+    public interface IParser
+    {
+        public PerformanceTestRun Parse(string path,int version);
+    }
+}

UnityPerformanceBenchmarkReporter/OptionsParser.cs

@@ -41,7 +41,7 @@ namespace UnityPerformanceBenchmarkReporter
                     ShowHelp(string.Empty, os);
                 }
 
-                if (!performanceBenchmark.ResultXmlFilePaths.Any() && !performanceBenchmark.ResultXmlDirectoryPaths.Any())
+                if (!performanceBenchmark.ResultFilePaths.Any() && !performanceBenchmark.ResultDirectoryPaths.Any())
                 {
                     ShowHelp("Missing required option --results=(filePath|directoryPath)", os);
                 }
@@ -61,20 +61,22 @@ namespace UnityPerformanceBenchmarkReporter
         private OptionSet GetOptions(PerformanceBenchmark performanceBenchmark)
         {
             var optionsSet = new OptionSet();
-                optionsSet.Add("?|help|h", "Prints out the options.", option => help = option != null);
-                optionsSet.Add(
-                        "results|testresultsxmlsource=", 
-                        "REQUIRED - Path to a test result XML filename OR directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
-                    xmlsource => performanceBenchmark.AddXmlSourcePath(xmlsource, "results", ResultType.Test));
-                optionsSet.Add(
-                        "baseline|baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename.",
-                        xmlsource => performanceBenchmark.AddXmlSourcePath(xmlsource, "baseline", ResultType.Baseline));
-                optionsSet.Add(
-                        "report|reportdirpath:", "OPTIONAL - Path to where the report will be written. Default is current working directory.",
-                        performanceBenchmark.AddReportDirPath);
-                optionsSet.Add("failonbaseline",
-                    "Enable return '1' by the reporter if a baseline is passed in and one or more matching configs is out of threshold. Disabled is default. Use option to enable, or use option and append '-' to explicitly disable.",
-                    option => performanceBenchmark.FailOnBaseline = option != null);
+            optionsSet.Add("?|help|h", "Prints out the options.", option => help = option != null);
+            optionsSet.Add("dataversion|version=", "Sets Expected Perf Data Version for Results and Baseline Files (1 = V1 2 = V2). Versions of Unity Perf Framework 2.0 or newer will use the V2 data format. If no arg is provided we assume the format is V2", version => performanceBenchmark.SetDataVersion(version));
+            optionsSet.Add("fileformat|format=", "Sets Expected File Format for Results and Baseline Files. If no arg is provided we assume the format is XML", filtype => performanceBenchmark.SetFileType(filtype));
+            optionsSet.Add(
+                    "results|testresultsxmlsource=",
+                    "REQUIRED - Path to a test result XML filename OR directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
+                xmlsource => performanceBenchmark.AddSourcePath(xmlsource, "results", ResultType.Test));
+            optionsSet.Add(
+                    "baseline|baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename.",
+                    xmlsource => performanceBenchmark.AddSourcePath(xmlsource, "baseline", ResultType.Baseline));
+            optionsSet.Add(
+                    "report|reportdirpath:", "OPTIONAL - Path to where the report will be written. Default is current working directory.",
+                    performanceBenchmark.AddReportDirPath);
+            optionsSet.Add("failonbaseline",
+                "Enable return '1' by the reporter if a baseline is passed in and one or more matching configs is out of threshold. Disabled is default. Use option to enable, or use option and append '-' to explicitly disable.",
+                option => performanceBenchmark.FailOnBaseline = option != null);
             return optionsSet;
         }
 

UnityPerformanceBenchmarkReporter/PerformanceBenchmark.cs

@@ -12,8 +12,10 @@ namespace UnityPerformanceBenchmarkReporter
         private readonly PerformanceTestRunProcessor performanceTestRunProcessor = new PerformanceTestRunProcessor();
 
         public readonly TestRunMetadataProcessor TestRunMetadataProcessor;
-        private readonly string xmlFileExtension = ".xml";
+        private ESupportedFileTypes fileExtension = ESupportedFileTypes.xml;
+        public ESupportedFileTypes FileType { get { return fileExtension; } }
 
+        public int DataVersion { get; private set; } = 2;
         public PerformanceBenchmark(Dictionary<Type, string[]> configFieldNames = null)
         {
             // Default significant figures to use for non-integer metrics if user doesn't specify another value.
@@ -24,70 +26,71 @@ namespace UnityPerformanceBenchmarkReporter
             TestRunMetadataProcessor = new TestRunMetadataProcessor(configFieldNames);
         }
 
-        public HashSet<string> ResultXmlFilePaths { get; } = new HashSet<string>();
-        public HashSet<string> ResultXmlDirectoryPaths { get; } = new HashSet<string>();
-        public HashSet<string> BaselineXmlFilePaths { get; } = new HashSet<string>();
+        public HashSet<string> ResultFilePaths { get; } = new HashSet<string>();
+        public HashSet<string> ResultDirectoryPaths { get; } = new HashSet<string>();
+        public HashSet<string> BaselineFilePaths { get; } = new HashSet<string>();
         public uint SigFig { get; }
         public string ReportDirPath { get; private set; }
         public bool FailOnBaseline { get; set; }
 
 
-        public bool BaselineResultFilesExist => BaselineXmlFilePaths.Any();
+        public bool BaselineResultFilesExist => BaselineFilePaths.Any();
+
+        public bool ResultFilesExist => ResultFilePaths.Any() || ResultDirectoryPaths.Any();
 
-        public bool ResultFilesExist => ResultXmlFilePaths.Any() || ResultXmlDirectoryPaths.Any();
 
         public void AddPerformanceTestRunResults(
-            TestResultXmlParser testResultXmlParser,
+            IParser testResultParser,
             List<PerformanceTestRunResult> performanceTestRunResults,
             List<TestResult> testResults,
             List<TestResult> baselineTestResults)
         {
-            AddTestResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults,
-                ResultXmlDirectoryPaths, ResultXmlFilePaths);
+            AddTestResults(testResultParser, performanceTestRunResults, testResults, baselineTestResults,
+                ResultDirectoryPaths, ResultFilePaths);
         }
 
         public void AddBaselinePerformanceTestRunResults(
-            TestResultXmlParser testResultXmlParser,
+            IParser testResultParser,
             List<PerformanceTestRunResult> baselinePerformanceTestRunResults,
             List<TestResult> baselineTestResults)
         {
-            AddTestResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults,
-                baselineTestResults, null, BaselineXmlFilePaths, true);
+            AddTestResults(testResultParser, baselinePerformanceTestRunResults, baselineTestResults,
+                baselineTestResults, null, BaselineFilePaths, true);
         }
 
         private void AddTestResults(
-            TestResultXmlParser testResultXmlParser,
+            IParser testResultParser,
             List<PerformanceTestRunResult> testRunResults,
             List<TestResult> testResults,
             List<TestResult> baselineTestResults,
-            HashSet<string> xmlDirectoryPaths,
-            HashSet<string> xmlFileNamePaths,
+            HashSet<string> directoryPaths,
+            HashSet<string> fileNamePaths,
             bool isBaseline = false)
         {
-            if (!isBaseline && xmlDirectoryPaths != null && xmlDirectoryPaths.Any())
+            if (!isBaseline && directoryPaths != null && directoryPaths.Any())
             {
-                foreach (var xmlDirectory in xmlDirectoryPaths)
+                foreach (var directory in directoryPaths)
                 {
-                    var xmlFileNames = GetAllXmlFileNames(xmlDirectory);
+                    var fileNames = GetAllFileNames(directory);
 
-                    foreach (var xmlFileName in xmlFileNames)
+                    foreach (var fileName in fileNames)
                     {
-                        xmlFileNamePaths.Add(xmlFileName);
+                        fileNamePaths.Add(fileName);
                     }
                 }
             }
 
-            if (xmlFileNamePaths.Any())
+            if (fileNamePaths.Any())
             {
                 var perfTestRuns = new List<KeyValuePair<string, PerformanceTestRun>>();
 
-                foreach (var xmlFileNamePath in xmlFileNamePaths)
+                foreach (var fileNamePath in fileNamePaths)
                 {
-                    var performanceTestRun = testResultXmlParser.Parse(xmlFileNamePath);
+                    var performanceTestRun = testResultParser.Parse(fileNamePath, DataVersion);
                     if (performanceTestRun != null && performanceTestRun.Results.Any())
                     {
                         perfTestRuns.Add(
-                            new KeyValuePair<string, PerformanceTestRun>(xmlFileNamePath, performanceTestRun));
+                            new KeyValuePair<string, PerformanceTestRun>(fileNamePath, performanceTestRun));
                     }
                 }
 
@@ -97,7 +100,7 @@ namespace UnityPerformanceBenchmarkReporter
                 for (var i = 0; i < resultFilesOrderedByResultName.Length; i++)
                 {
                     var performanceTestRun =
-                        testResultXmlParser.Parse(resultFilesOrderedByResultName[i].Key);
+                        testResultParser.Parse(resultFilesOrderedByResultName[i].Key, DataVersion);
 
                     if (performanceTestRun != null && performanceTestRun.Results.Any())
                     {
@@ -128,19 +131,52 @@ namespace UnityPerformanceBenchmarkReporter
             }
         }
 
-        private IEnumerable<string> GetAllXmlFileNames(string xmlDirectory)
+        public void SetDataVersion(string version)
         {
-            var dir = new DirectoryInfo(xmlDirectory);
-            var xmlFileNames = dir.GetFiles("*" + xmlFileExtension, SearchOption.AllDirectories)
-                .Select(f => f.FullName);
-            return xmlFileNames;
+            if (int.TryParse(version, out int result))
+            {
+                if (result > 0 && result < 3)
+                    DataVersion = result;
+                else
+                    throw new ArgumentException($"{version} is not a valid data format version. Please pass 1 or 2");
+            }
+            else
+            {
+                throw new ArgumentException($"{version} is not a valid data format version");
+            }
         }
 
-        public void AddXmlSourcePath(string xmlSourcePath, string optionName, OptionsParser.ResultType resultType)
+        public void SetFileType(string filetype)
         {
-            if (string.IsNullOrEmpty(xmlSourcePath))
+            if (String.IsNullOrEmpty(filetype))
+                return;
+
+            if (Enum.TryParse<ESupportedFileTypes>(filetype, true, out ESupportedFileTypes result))
             {
-                throw new ArgumentNullException(xmlSourcePath);
+                fileExtension = result;
+            }
+            else
+            {
+                throw new ArgumentException($"{filetype} is not a valid file format");
+            }
+        }
+
+        private IEnumerable<string> GetAllFileNames(string directory)
+        {
+            var dir = new DirectoryInfo(directory);
+            var FileNames = dir.GetFiles("*" + fileExtension, SearchOption.AllDirectories)
+                .Select(f => f.FullName);
+            return FileNames;
+        }
+
+        public void AddSourcePath(string sourcePath, string optionName, OptionsParser.ResultType resultType)
+        {
+            System.Console.WriteLine($" Adding Source Path : {sourcePath}");
+            System.Console.WriteLine($"");
+
+            if (string.IsNullOrEmpty(sourcePath))
+            {
+                throw new ArgumentNullException(sourcePath);
             }
 
             if (string.IsNullOrEmpty(optionName))
@@ -148,50 +184,67 @@ namespace UnityPerformanceBenchmarkReporter
                 throw new ArgumentNullException(optionName);
             }
 
-            //If has .xml file extension
-            if (xmlSourcePath.EndsWith(xmlFileExtension))
+            if (sourcePath.EndsWith(fileExtension.ToString()))
             {
-                ProcessAsXmlFile(xmlSourcePath, optionName, resultType);
+                ProcessAsFile(sourcePath, optionName, resultType);
             }
             else
             {
-                ProcessAsXmlDirectory(xmlSourcePath, optionName, resultType);
+                ProcessAsDirectory(sourcePath, optionName, resultType);
             }
+
+
+
+
         }
 
-        private void ProcessAsXmlDirectory(string xmlSourcePath, string optionName, OptionsParser.ResultType resultType)
+        private void ProcessAsDirectory(string sourcePath, string optionName, OptionsParser.ResultType resultType)
         {
-            if (!Directory.Exists(xmlSourcePath))
+            if (!Directory.Exists(sourcePath))
             {
                 throw new ArgumentException(string.Format("{0} directory `{1}` cannot be found", optionName,
-                    xmlSourcePath));
+                    sourcePath));
             }
 
-            var xmlFileNames = GetAllXmlFileNames(xmlSourcePath).ToArray();
-            if (!xmlFileNames.Any())
+            var fileNames = GetAllFileNames(sourcePath).ToArray();
+            if (!fileNames.Any())
             {
-                throw new ArgumentException(string.Format("{0} directory `{1}` doesn't contain any .xml files.",
+                throw new ArgumentException(string.Format("{0} directory `{1}` doesn't contain any {2} files.",
                     optionName,
-                    xmlSourcePath));
-            }
-
-            ResultXmlDirectoryPaths.Add(xmlSourcePath);
-        }
-
-        private void ProcessAsXmlFile(string xmlSourcePath, string optionName, OptionsParser.ResultType resultType)
-        {
-            if (!File.Exists(xmlSourcePath))
-            {
-                throw new ArgumentException(string.Format("{0} file `{1}` cannot be found", optionName, xmlSourcePath));
+                    sourcePath, FileType));
             }
 
             switch (resultType)
             {
                 case OptionsParser.ResultType.Test:
-                    ResultXmlFilePaths.Add(xmlSourcePath);
+                    ResultDirectoryPaths.Add(sourcePath);
                     break;
                 case OptionsParser.ResultType.Baseline:
-                    BaselineXmlFilePaths.Add(xmlSourcePath);
+                    foreach (var filename in fileNames)
+                    {
+                        BaselineFilePaths.Add(filename);
+                    }
+
+                    break;
+                default:
+                    throw new InvalidEnumArgumentException(resultType.ToString());
+            }
+        }
+
+        private void ProcessAsFile(string sourcePath, string optionName, OptionsParser.ResultType resultType)
+        {
+            if (!File.Exists(sourcePath))
+            {
+                throw new ArgumentException(string.Format("{0} file `{1}` cannot be found", optionName, sourcePath));
+            }
+
+            switch (resultType)
+            {
+                case OptionsParser.ResultType.Test:
+                    ResultFilePaths.Add(sourcePath);
+                    break;
+                case OptionsParser.ResultType.Baseline:
+                    BaselineFilePaths.Add(sourcePath);
                     break;
                 default:
                     throw new InvalidEnumArgumentException(resultType.ToString());

UnityPerformanceBenchmarkReporter/PerformanceTestRunProcessor.cs

@@ -26,7 +26,7 @@ namespace UnityPerformanceBenchmarkReporter
                     TestName = testName,
                     TestCategories = performanceTestRun.Results.First(r => r.TestName == testName).TestCategories,
                     TestVersion = performanceTestRun.Results.First(r => r.TestName == testName).TestVersion,
-                    State = (int) TestState.Success,
+                    State = (int)TestState.Success,
                     SampleGroupResults = new List<SampleGroupResult>()
                 };
                 foreach (var sampleGroup in mergedTestExecutions[testName])
@@ -83,13 +83,24 @@ namespace UnityPerformanceBenchmarkReporter
                         sampleGroupResult.BaselineValue = baselineSampleGroupResult.AggregatedValue;
                         sampleGroupResult.Threshold = baselineSampleGroupResult.Threshold;
 
-                        sampleGroupResult.Regressed = DeterminePerformanceResult(sampleGroupResult, sigfig) == MeasurementResult.Regression;
+                        var res = DeterminePerformanceResult(sampleGroupResult, sigfig);
+
+                        if (res == MeasurementResult.Regression)
+                        {
+                            sampleGroupResult.Regressed = true;
+                            sampleGroupResult.Progressed = false;
+                        }
+                        else if (res == MeasurementResult.Progression)
+                        {
+                            sampleGroupResult.Regressed = false;
+                            sampleGroupResult.Progressed = true;
+                        }
                     }
                 }
 
                 if (testResult.SampleGroupResults.Any(r => r.Regressed))
                 {
-                    testResult.State = (int) TestState.Failure;
+                    testResult.State = (int)TestState.Failure;
                 }
             }
         }
@@ -97,7 +108,7 @@ namespace UnityPerformanceBenchmarkReporter
         private Dictionary<string, List<SampleGroup>> MergeTestExecutions(PerformanceTestRun performanceTestRun)
         {
             var mergedTestExecutions = new Dictionary<string, List<SampleGroup>>();
-            var testNames = performanceTestRun.Results.Select(te => te.TestName).Distinct().ToList();
+            var testNames = performanceTestRun.Results.Select(te => te.TestName).Where(t => !String.IsNullOrEmpty(t)).Distinct().ToList();
             foreach (var testName in testNames)
             {
                 var executions = performanceTestRun.Results.Where(te => te.TestName == testName);
@@ -152,8 +163,11 @@ namespace UnityPerformanceBenchmarkReporter
         private MeasurementResult DeterminePerformanceResult(SampleGroupResult sampleGroup, uint sigFig)
         {
             var measurementResult = MeasurementResult.Neutral;
-            var positiveThresholdValue = sampleGroup.BaselineValue + sampleGroup.BaselineValue * sampleGroup.Threshold;
-            var negativeThresholdValue = sampleGroup.BaselineValue - sampleGroup.BaselineValue * sampleGroup.Threshold;
+            var positiveThresholdValue = sampleGroup.BaselineValue + (sampleGroup.BaselineValue * sampleGroup.Threshold);
+            var negativeThresholdValue = sampleGroup.BaselineValue - (sampleGroup.BaselineValue * sampleGroup.Threshold);
+            positiveThresholdValue += sampleGroup.StandardDeviation;
+            negativeThresholdValue -= sampleGroup.StandardDeviation;
+
             if (sampleGroup.IncreaseIsBetter)
             {
                 if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) < negativeThresholdValue.TruncToSigFig(sigFig))

UnityPerformanceBenchmarkReporter/Program.cs

@@ -46,11 +46,21 @@ namespace UnityPerformanceBenchmarkReporter
             var optionsParser = new OptionsParser();
 
             optionsParser.ParseOptions(performanceBenchmark, args);
-            var testResultXmlParser = new TestResultXmlParser();
+
+            IParser testResultParser = null;
+
+            if (performanceBenchmark.FileType == ESupportedFileTypes.json)
+            {
+                testResultParser = new TestResultJsonParser();
+            }
+            else if (performanceBenchmark.FileType == ESupportedFileTypes.xml)
+            {
+                testResultParser = new TestResultXmlParser();
+            }
 
             if (performanceBenchmark.BaselineResultFilesExist)
             {
-                performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults);
+                performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultParser, baselinePerformanceTestRunResults, baselineTestResults);
 
                 if (baselinePerformanceTestRunResults.Any())
                 {
@@ -64,7 +74,7 @@ namespace UnityPerformanceBenchmarkReporter
 
             if (performanceBenchmark.ResultFilesExist)
             {
-                performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults);
+                performanceBenchmark.AddPerformanceTestRunResults(testResultParser, performanceTestRunResults, testResults, baselineTestResults);
 
                 if (performanceTestRunResults.Any())
                 {
@@ -76,7 +86,7 @@ namespace UnityPerformanceBenchmarkReporter
                 }
             }
 
-            var performanceTestResults = new PerformanceTestRunResult[0]; 
+            var performanceTestResults = new PerformanceTestRunResult[0];
 
             // If we have a baseline
             if (aggregateTestRunResults.Any(a => a.IsBaseline))
@@ -106,14 +116,16 @@ namespace UnityPerformanceBenchmarkReporter
                 performanceBenchmark.SigFig,
                 performanceBenchmark.ReportDirPath,
                 performanceBenchmark.BaselineResultFilesExist);
-
-            return WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
+            WriteProgressedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
+            int result = WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
+            WriteLine($"Finished with Result {result}");
+            return result;
         }
 
         private static int WriteFailedTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
         {
             var failedTestsExist = performanceTestResults.SelectMany(ptr => ptr.TestResults)
-                .Any(tr => tr.State == (int) TestState.Failure);
+                .Any(tr => tr.State == (int)TestState.Failure);
             if (failedTestsExist)
             {
                 WriteLine("FAILURE: One ore more performance test metric aggregations is out of threshold from the baseline value.");
@@ -147,10 +159,59 @@ namespace UnityPerformanceBenchmarkReporter
                     }
                 }
             }
-            
             return performanceBenchmark.FailOnBaseline && failedTestsExist ? 1 : 0;
         }
 
+        private static void WriteProgressedTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
+        {
+            bool loggedHeader = false;
+            var passedTestsExist = performanceTestResults.SelectMany(ptr => ptr.TestResults)
+                .Any(tr => tr.State == (int)TestState.Success);
+            if (passedTestsExist)
+            {
+
+                foreach (var performanceTestRunResult in performanceTestResults)
+                {
+                    var passedTests = performanceTestRunResult.TestResults.Where(tr => tr.State == (int)TestState.Success);
+                    if (passedTests.Any())
+                    {
+                        foreach (var tests in passedTests)
+                        {
+                            if (tests.SampleGroupResults.Any(sgr => sgr.Progressed))
+                            {
+                                if (!loggedHeader)
+                                {
+                                    loggedHeader = true;
+                                    WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value.");
+                                    WriteLine("-------------------------------------");
+                                    WriteLine(" Performance tests with Progressed metrics");
+                                    WriteLine("-------------------------------------");
+                                }
+
+                                ++indentLevel;
+                                WriteLine("{0}", tests.TestName);
+
+                                var progressedSgs = tests.SampleGroupResults.Where(sgr => sgr.Progressed);
+                                foreach (var sampleGroupResult in progressedSgs)
+                                {
+                                    WriteLine("----");
+                                    WriteLine("Metric        : {0}", sampleGroupResult.SampleGroupName);
+                                    WriteLine("Aggregation   : {0}", sampleGroupResult.AggregationType);
+                                    WriteLine("New Value  : {0,8:F2}", sampleGroupResult.AggregatedValue);
+                                    WriteLine("Baseline Value: {0,8:F2}", sampleGroupResult.BaselineValue);
+                                    WriteLine("Threshold %   : {0,8:F2}", sampleGroupResult.Threshold);
+                                    WriteLine("Actual Diff % : {0,8:F2}", Math.Abs(sampleGroupResult.BaselineValue - sampleGroupResult.AggregatedValue) / sampleGroupResult.BaselineValue);
+                                }
+                                --indentLevel;
+                                WriteLine("\r\n");
+                            }
+                        }
+                    }
+                }
+            }
+
+        }
+
         private static void WriteLine(string format, params object[] args)
         {
             Console.Write(new string('\t', indentLevel));

UnityPerformanceBenchmarkReporter/Report/ReportWriter.cs

@@ -63,6 +63,8 @@ namespace UnityPerformanceBenchmarkReporter.Report
                 var benchmarkReportFile = GetBenchmarkReportFile(reportDirectory);
                 using (var rw = new StreamWriter(benchmarkReportFile))
                 {
+                    System.Console.WriteLine($"Writing Report To: {reportDirectory.FullName}");
+                    System.Console.WriteLine($"");
                     WriteHtmlReport(rw);
                 }
             }
@@ -217,7 +219,7 @@ namespace UnityPerformanceBenchmarkReporter.Report
                 {
                     streamWriter.WriteLine("<input type=\"checkbox\" onclick=\"toggleCanvasWithNoFailures()\">");
                 }
-                
+
             }
             else
             {
@@ -365,7 +367,7 @@ namespace UnityPerformanceBenchmarkReporter.Report
 
         private void WriteStatMethodButtonEventListeners(StreamWriter rw)
         {
-            var statisticalMethods = new List<string> {"Min", "Max", "Median", "Average"};
+            var statisticalMethods = new List<string> { "Min", "Max", "Median", "Average" };
             foreach (var thisStatMethod in statisticalMethods)
             {
                 rw.WriteLine("	document.getElementById('{0}Button').addEventListener('click', function()",
@@ -683,7 +685,7 @@ namespace UnityPerformanceBenchmarkReporter.Report
                         var minDefaultValue = nullString;
                         var maxDefaultValue = nullString;
                         var avgDefaultValue = nullString;
-                        var stdevDefaultValue =  nullString;
+                        var stdevDefaultValue = nullString;
                         var baselineDefaultValue = nullString;
 
                         if (performanceTestRunResult.TestResults.Any(r =>
@@ -915,7 +917,7 @@ namespace UnityPerformanceBenchmarkReporter.Report
         {
             var sampleGroupHasSamples = resultsForThisTest.SelectMany(r => r.SampleGroupResults).Any(sg =>
                 ScrubStringForSafeForVariableUse(sg.SampleGroupName) == distinctSampleGroupName);
-                return sampleGroupHasSamples;
+            return sampleGroupHasSamples;
         }
 
         private bool SampleGroupHasRegressions(IEnumerable<TestResult> resultsForThisTest,

UnityPerformanceBenchmarkReporter/TestResultJsonParser.cs

@@ -0,0 +1,181 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Xml.Linq;
+using Newtonsoft.Json;
+using UnityPerformanceBenchmarkReporter.Entities;
+using UnityPerformanceBenchmarkReporter.Entities.New;
+
+namespace UnityPerformanceBenchmarkReporter
+{
+    public class TestResultJsonParser : IParser
+    {
+        public PerformanceTestRun Parse(string path, int version)
+        {
+            string report = "";
+            try
+            {
+                using (StreamReader reader = new StreamReader(path))
+                {
+                    string stream = reader.ReadToEnd().Trim();
+
+                    // json wrawrapped in invalid [], removed for valid json format 
+                    if (stream[0] == '[' && stream[stream.Length - 1] == ']')
+                        report = stream.Substring(1, stream.Length - 2);
+                    else
+                        report = stream;
+                }
+            }
+            catch (System.Exception)
+            {
+
+                throw;
+            }
+
+            switch (version)
+            {
+                case 1:
+                    return ParseJsonV1(report);
+                case 2:
+                    return ParseJsonV2(report);
+                default:
+                    return null;
+            }
+        }
+
+        private static PerformanceTestRun ParseJsonV1(string json)
+        {
+
+            PerformanceTestRun result;
+
+            try
+            {
+                result = JsonConvert.DeserializeObject<PerformanceTestRun>(json);
+            }
+            catch (System.Exception)
+            {
+
+                throw;
+            }
+
+
+            return result;
+        }
+
+        private static PerformanceTestRun ParseJsonV2(string json)
+        {
+
+            Run run = null;
+            try
+            {
+                run = JsonConvert.DeserializeObject<Run>(json);
+            }
+            catch (System.Exception)
+            {
+                throw;
+            }
+
+            if (run != null)
+            {
+                var testRun = new PerformanceTestRun()
+                {
+                    BuildSettings = new BuildSettings()
+                    {
+                        Platform = run.Player.Platform,
+                        BuildTarget = run.Player.BuildTarget,
+                        DevelopmentPlayer = true,
+                        AndroidBuildSystem = run.Player.AndroidBuildSystem
+                    },
+                    EditorVersion = new EditorVersion()
+                    {
+                        Branch = run.Editor.Branch,
+                        DateSeconds = run.Editor.Date,
+                        FullVersion = $"{run.Editor.Version} ({run.Editor.Changeset})",
+                        RevisionValue = 0
+                    },
+                    PlayerSettings = new PlayerSettings()
+                    {
+                        GpuSkinning = run.Player.GpuSkinning,
+                        GraphicsApi = run.Player.GraphicsApi,
+                        RenderThreadingMode = run.Player.RenderThreadingMode,
+                        ScriptingBackend = run.Player.ScriptingBackend,
+                        AndroidTargetSdkVersion = run.Player.AndroidTargetSdkVersion,
+                        EnabledXrTargets = new List<string>(),
+                        ScriptingRuntimeVersion = "",
+                        StereoRenderingPath = run.Player.StereoRenderingPath
+                    },
+                    QualitySettings = new QualitySettings()
+                    {
+                        Vsync = run.Player.Vsync,
+                        AntiAliasing = run.Player.AntiAliasing,
+                        AnisotropicFiltering = run.Player.AnisotropicFiltering,
+                        BlendWeights = run.Player.BlendWeights,
+                        ColorSpace = run.Player.ColorSpace
+                    },
+                    ScreenSettings = new ScreenSettings()
+                    {
+                        Fullscreen = run.Player.Fullscreen,
+                        ScreenHeight = run.Player.ScreenHeight,
+                        ScreenWidth = run.Player.ScreenWidth,
+                        ScreenRefreshRate = run.Player.ScreenRefreshRate
+                    },
+                    PlayerSystemInfo = new Entities.PlayerSystemInfo()
+                    {
+                        DeviceModel = run.Hardware.DeviceModel,
+                        DeviceName = run.Hardware.DeviceName,
+                        OperatingSystem = run.Hardware.OperatingSystem,
+                        ProcessorCount = run.Hardware.ProcessorCount,
+                        ProcessorType = run.Hardware.ProcessorType,
+                        GraphicsDeviceName = run.Hardware.GraphicsDeviceName,
+                        SystemMemorySize = run.Hardware.SystemMemorySizeMB,
+                        XrDevice = run.Hardware.XrDevice,
+                        XrModel = run.Hardware.XrModel
+                    },
+                    StartTime = run.Date,
+                    TestSuite = run.TestSuite,
+                    Results = new List<PerformanceTestResult>()
+                };
+
+                testRun.EndTime = DateTime.Now.ToUniversalTime()
+                .Subtract(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))
+                .TotalMilliseconds;
+
+                foreach (var res in run.Results)
+                {
+                    var pt = new PerformanceTestResult()
+                    {
+                        TestCategories = res.Categories,
+                        TestName = res.Name,
+                        TestVersion = res.Version,
+                        SampleGroups = res.SampleGroups.Select(sg => new Entities.SampleGroup
+                        {
+                            Samples = sg.Samples,
+                            Average = sg.Average,
+                            Max = sg.Max,
+                            Median = sg.Median,
+                            Min = sg.Min,
+                            Sum = sg.Sum,
+                            StandardDeviation = sg.StandardDeviation,
+                            SampleCount = sg.Samples.Count,
+                            Definition = new SampleGroupDefinition()
+                            {
+                                Name = sg.Name,
+                                SampleUnit = (Entities.SampleUnit)sg.Unit,
+                                IncreaseIsBetter = sg.IncreaseIsBetter,
+                                Threshold = sg.Threshold
+                            }
+                        }).ToList()
+                    };
+                    testRun.Results.Add(pt);
+                }
+
+                return testRun;
+            }
+
+            return null;
+        }
+
+    }
+}

UnityPerformanceBenchmarkReporter/TestResultXmlParser.cs

@@ -10,9 +10,9 @@ using UnityPerformanceBenchmarkReporter.Entities.New;
 
 namespace UnityPerformanceBenchmarkReporter
 {
-    public class TestResultXmlParser
+    public class TestResultXmlParser : IParser
     {
-        public PerformanceTestRun Parse(string path)
+        public PerformanceTestRun Parse(string path,int version)
         {
             var xmlDocument = XDocument.Load(path);
             return Parse(xmlDocument);
@@ -90,11 +90,13 @@ namespace UnityPerformanceBenchmarkReporter
                                 Sum = sg.Sum,
                                 StandardDeviation = sg.StandardDeviation,
                                 SampleCount = sg.Samples.Count,
+                                
                                 Definition = new SampleGroupDefinition()
                                 {
                                     Name = sg.Name,
                                     SampleUnit = (Entities.SampleUnit)sg.Unit,
-                                    IncreaseIsBetter = sg.IncreaseIsBetter
+                                    IncreaseIsBetter = sg.IncreaseIsBetter,
+                                    Threshold = sg.Threshold
                                 }
                             }).ToList()
                         };
@@ -178,7 +180,7 @@ namespace UnityPerformanceBenchmarkReporter
                         ScreenWidth = result.Player.ScreenWidth,
                         ScreenRefreshRate = result.Player.ScreenRefreshRate
                     },
-                    PlayerSystemInfo = new PlayerSystemInfo()
+                    PlayerSystemInfo = new Entities.PlayerSystemInfo()
                     {
                         DeviceModel = result.Hardware.DeviceModel,
                         DeviceName = result.Hardware.DeviceName,

UnityPerformanceBenchmarkReporterTests/PerformanceBenchmarkTestsJson.cs

@@ -0,0 +1,385 @@
+using System.Collections.Generic;
+using System.Linq;
+using NUnit.Framework;
+using UnityPerformanceBenchmarkReporter;
+using UnityPerformanceBenchmarkReporter.Entities;
+
+namespace UnityPerformanceBenchmarkReporterTests
+{
+    public class PerformanceBenchmarkTestsJson : PerformanceBenchmarkTestsBase
+    {
+        private OptionsParser optionsParser;
+        private IParser testResultJsonParser;
+        private List<PerformanceTestRunResult> performanceTestRunResults;
+        private List<TestResult> testResults;
+        private List<PerformanceTestRunResult> baselinePerformanceTestRunResults;
+        private List<TestResult> baselineTestResults;
+
+        [SetUp]
+        public void Setup()
+        {
+            optionsParser = new OptionsParser();
+            PerformanceBenchmark = new PerformanceBenchmark();
+            testResultJsonParser = new TestResultJsonParser();
+            performanceTestRunResults = new List<PerformanceTestRunResult>();
+            testResults = new List<TestResult>();
+            baselinePerformanceTestRunResults = new List<PerformanceTestRunResult>();
+            baselineTestResults = new List<TestResult>();
+        }
+
+
+        [Test]
+        public void VerifyV1_AddPerformanceTestRunResults()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results.json");
+            var args = new[] { "--format=json", string.Format("--testresultsxmlsource={0}", resultJsonFilePath), "--version=1" };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddPerformanceTestRunResults(testResultJsonParser, performanceTestRunResults, testResults, new List<TestResult>());
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath });
+            Assert.NotNull(testResults);
+            Assert.IsTrue(testResults.Count > 0);
+            Assert.NotNull(performanceTestRunResults);
+            Assert.IsTrue(performanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV1_AddPerformanceTestRunResults_TwoResultFiles()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results.json");
+            var resultFileName2 = EnsureFullPath("results.json");
+            var args = new[]
+            {
+                "--format=json",
+                string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                string.Format("--testresultsxmlsource={0}", resultFileName2)
+                , "--version=1"
+            };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddPerformanceTestRunResults(testResultJsonParser, performanceTestRunResults, testResults, new List<TestResult>());
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath, resultFileName2 });
+            Assert.NotNull(testResults);
+            Assert.IsTrue(testResults.Count > 0);
+            Assert.NotNull(performanceTestRunResults);
+            Assert.IsTrue(performanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV1_AddPerformanceTestRunResults_OneResultFiles_OneResultDirectory()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results.json");
+            var resultsJsonDir = EnsureFullPath("ResultsJson");
+            var args = new[]
+            {
+                "--format=json",
+                    string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                    string.Format("--testresultsxmlsource={0}", resultsJsonDir)
+                    , "--version=1"
+            };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddPerformanceTestRunResults(testResultJsonParser, performanceTestRunResults, testResults, new List<TestResult>());
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
+            AssertCorrectResultsJsonDirectoryPaths(new[] { resultsJsonDir });
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath });
+            Assert.NotNull(testResults);
+            Assert.IsTrue(testResults.Count > 0);
+            Assert.NotNull(performanceTestRunResults);
+            Assert.IsTrue(performanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV1_AddBaselinePerformanceTestRunResults()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results.json");
+            var baselineJsonFilePath = EnsureFullPath("baseline.json");
+            var args = new[]
+            {
+                "--format=Json",
+                string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                string.Format("--baselinexmlsource={0}", baselineJsonFilePath)
+                , "--version=1"
+            };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddBaselinePerformanceTestRunResults(testResultJsonParser, baselinePerformanceTestRunResults, baselineTestResults);
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.BaselineResultFilesExist);
+            AssertCorrectBaselineJsonFilePaths(new[] { baselineJsonFilePath });
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath });
+            Assert.NotNull(baselineTestResults);
+            Assert.IsTrue(baselineTestResults.Count > 0);
+            Assert.NotNull(baselinePerformanceTestRunResults);
+            Assert.IsTrue(baselinePerformanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV1_AddBaselinePerformanceTestRunResultsDirectory()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("ResultsJson");
+            var baselineJsonFilePath = EnsureFullPath("BaselineJson");
+            var args = new[]
+            {
+                "--format=Json",
+                string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                string.Format("--baselinexmlsource={0}", baselineJsonFilePath)
+                , "--version=1"
+            };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddBaselinePerformanceTestRunResults(testResultJsonParser, baselinePerformanceTestRunResults, baselineTestResults);
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.BaselineResultFilesExist);
+            Assert.NotNull(baselineTestResults);
+            Assert.IsTrue(baselineTestResults.Count > 0);
+            Assert.NotNull(baselinePerformanceTestRunResults);
+            Assert.IsTrue(baselinePerformanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV1_Verify_AddBaselineAndNonBaselinePerformanceTestRunResults()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results.json");
+            var baselineJsonFilePath = EnsureFullPath("baseline.json");
+            var args = new[]
+            {
+                "--format=Json",
+                    string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                    string.Format("--baselinexmlsource={0}", baselineJsonFilePath)
+                    , "--version=1"
+                };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddBaselinePerformanceTestRunResults(testResultJsonParser, baselinePerformanceTestRunResults, baselineTestResults);
+            PerformanceBenchmark.AddPerformanceTestRunResults(testResultJsonParser, performanceTestRunResults, testResults, new List<TestResult>());
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath });
+            Assert.NotNull(testResults);
+            Assert.IsTrue(testResults.Count > 0);
+            Assert.NotNull(performanceTestRunResults);
+            Assert.IsTrue(performanceTestRunResults.Count > 0);
+
+            Assert.IsTrue(PerformanceBenchmark.BaselineResultFilesExist);
+            AssertCorrectBaselineJsonFilePaths(new[] { baselineJsonFilePath });
+            Assert.NotNull(baselineTestResults);
+            Assert.IsTrue(baselineTestResults.Count > 0);
+            Assert.NotNull(baselinePerformanceTestRunResults);
+            Assert.IsTrue(baselinePerformanceTestRunResults.Count > 0);
+        }
+
+        public void VerifyV2_AddPerformanceTestRunResults()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("baseline2.json");
+            var args = new[] { string.Format("--format=Json", "--testresultsxmlsource={0}", resultJsonFilePath), "--version=2" };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddPerformanceTestRunResults(testResultJsonParser, performanceTestRunResults, testResults, new List<TestResult>());
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath });
+            Assert.NotNull(testResults);
+            Assert.IsTrue(testResults.Count > 0);
+            Assert.NotNull(performanceTestRunResults);
+            Assert.IsTrue(performanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV2_AddPerformanceTestRunResults_TwoResultFiles()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("ResultsJson2/results2.json");
+            var resultFileName2 = EnsureFullPath("results2.json");
+            var args = new[]
+            {
+                "--format=Json",
+                string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                string.Format("--testresultsxmlsource={0}", resultFileName2)
+                , "--version=2"
+            };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddPerformanceTestRunResults(testResultJsonParser, performanceTestRunResults, testResults, new List<TestResult>());
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath, resultFileName2 });
+            Assert.NotNull(testResults);
+            Assert.IsTrue(testResults.Count > 0);
+            Assert.NotNull(performanceTestRunResults);
+            Assert.IsTrue(performanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV2_AddPerformanceTestRunResults_OneResultFiles_OneResultDirectory()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results2.json");
+            var resultsJsonDir = EnsureFullPath("ResultsJson2");
+            var args = new[]
+            {
+                "--format=Json",
+                    string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                    string.Format("--testresultsxmlsource={0}", resultsJsonDir)
+                    , "--version=2"
+            };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddPerformanceTestRunResults(testResultJsonParser, performanceTestRunResults, testResults, new List<TestResult>());
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
+            AssertCorrectResultsJsonDirectoryPaths(new[] { resultsJsonDir });
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath });
+            Assert.NotNull(testResults);
+            Assert.IsTrue(testResults.Count > 0);
+            Assert.NotNull(performanceTestRunResults);
+            Assert.IsTrue(performanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV2_AddBaselinePerformanceTestRunResults()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results2.json");
+            var baselineJsonFilePath = EnsureFullPath("baseline2.json");
+            var args = new[]
+            {
+                "--format=Json",
+                string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                string.Format("--baselinexmlsource={0}", baselineJsonFilePath)
+                , "--version=2"
+            };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddBaselinePerformanceTestRunResults(testResultJsonParser, baselinePerformanceTestRunResults, baselineTestResults);
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.BaselineResultFilesExist);
+            AssertCorrectBaselineJsonFilePaths(new[] { baselineJsonFilePath });
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath });
+            Assert.NotNull(baselineTestResults);
+            Assert.IsTrue(baselineTestResults.Count > 0);
+            Assert.NotNull(baselinePerformanceTestRunResults);
+            Assert.IsTrue(baselinePerformanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV2_AddBaselinePerformanceTestRunResultsDirectory()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results2.json");
+            var baselineJsonFilePath = EnsureFullPath("baseline2.json");
+            var args = new[]
+            {
+                "--format=Json",
+                string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                string.Format("--baselinexmlsource={0}", baselineJsonFilePath)
+                , "--version=2"
+            };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddBaselinePerformanceTestRunResults(testResultJsonParser, baselinePerformanceTestRunResults, baselineTestResults);
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.BaselineResultFilesExist);
+            Assert.NotNull(baselineTestResults);
+            Assert.IsTrue(baselineTestResults.Count > 0);
+            Assert.NotNull(baselinePerformanceTestRunResults);
+            Assert.IsTrue(baselinePerformanceTestRunResults.Count > 0);
+        }
+
+        [Test]
+        public void VerifyV2_Verify_AddBaselineAndNonBaselinePerformanceTestRunResults()
+        {
+            // Arrange
+            var resultJsonFilePath = EnsureFullPath("results2.json");
+            var baselineJsonFilePath = EnsureFullPath("baseline2.json");
+            var args = new[]
+            {
+                "--format=Json",
+                    string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
+                    string.Format("--baselinexmlsource={0}", baselineJsonFilePath)
+                    , "--version=2"
+                };
+            optionsParser.ParseOptions(PerformanceBenchmark, args);
+
+            // Act
+            PerformanceBenchmark.AddBaselinePerformanceTestRunResults(testResultJsonParser, baselinePerformanceTestRunResults, baselineTestResults);
+            PerformanceBenchmark.AddPerformanceTestRunResults(testResultJsonParser, performanceTestRunResults, testResults, new List<TestResult>());
+
+            // Assert
+            Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
+            AssertCorrectResultJsonFilePaths(new[] { resultJsonFilePath });
+            Assert.NotNull(testResults);
+            Assert.IsTrue(testResults.Count > 0);
+            Assert.NotNull(performanceTestRunResults);
+            Assert.IsTrue(performanceTestRunResults.Count > 0);
+
+            Assert.IsTrue(PerformanceBenchmark.BaselineResultFilesExist);
+            AssertCorrectBaselineJsonFilePaths(new[] { baselineJsonFilePath });
+            Assert.NotNull(baselineTestResults);
+            Assert.IsTrue(baselineTestResults.Count > 0);
+            Assert.NotNull(baselinePerformanceTestRunResults);
+            Assert.IsTrue(baselinePerformanceTestRunResults.Count > 0);
+        }
+
+        private void AssertCorrectBaselineJsonFilePaths(string[] baselineJsonFilePaths)
+        {
+            foreach (var baselineJsonFilePath in baselineJsonFilePaths)
+            {
+                Assert.IsFalse(PerformanceBenchmark.ResultFilePaths.Any(f => f.Equals(baselineJsonFilePath)));
+                Assert.IsTrue(PerformanceBenchmark.BaselineFilePaths.Any(f => f.Equals(baselineJsonFilePath)));
+            }
+        }
+
+        private void AssertCorrectResultJsonFilePaths(string[] resultFileNames)
+        {
+            foreach (var resultJsonFilePath in resultFileNames)
+            {
+                Assert.IsTrue(PerformanceBenchmark.ResultFilePaths.Contains(resultJsonFilePath));
+                Assert.IsFalse(PerformanceBenchmark.BaselineFilePaths.Contains(resultJsonFilePath));
+            }
+        }
+
+        private void AssertCorrectResultsJsonDirectoryPaths(string[] resultsJsonDirPaths)
+        {
+            foreach (var resultJsonDirPath in resultsJsonDirPaths)
+            {
+                Assert.IsTrue(PerformanceBenchmark.ResultDirectoryPaths.Contains(resultJsonDirPath));
+            }
+        }
+    }
+
+}

UnityPerformanceBenchmarkReporterTests/PerformanceBenchmarkTestsXML.cs

@@ -6,13 +6,13 @@ using UnityPerformanceBenchmarkReporter.Entities;
 
 namespace UnityPerformanceBenchmarkReporterTests
 {
-    public class PerformanceBenchmarkTests : PerformanceBenchmarkTestsBase
+    public class PerformanceBenchmarkTestsXML : PerformanceBenchmarkTestsBase
     {
         private OptionsParser optionsParser;
-        private TestResultXmlParser testResultXmlParser;
+        private IParser testResultXmlParser;
         private List<PerformanceTestRunResult> performanceTestRunResults;
         private List<TestResult> testResults;
-        private List<PerformanceTestRunResult>  baselinePerformanceTestRunResults;
+        private List<PerformanceTestRunResult> baselinePerformanceTestRunResults;
         private List<TestResult> baselineTestResults;
 
         [SetUp]
@@ -130,7 +130,7 @@ namespace UnityPerformanceBenchmarkReporterTests
         {
             // Arrange
             var resultXmlFilePath = EnsureFullPath("results.xml");
-            var baselineXmlFilePath = EnsureFullPath("baseline.xml");
+            var baselineXmlFilePath = EnsureFullPath("Baselines");
             var args = new[]
             {
                 string.Format("--testresultsxmlsource={0}", resultXmlFilePath),
@@ -186,17 +186,17 @@ namespace UnityPerformanceBenchmarkReporterTests
         {
             foreach (var baselineXmlFilePath in baselineXmlFilePaths)
             {
-                Assert.IsFalse(PerformanceBenchmark.ResultXmlFilePaths.Any(f => f.Equals(baselineXmlFilePath)));
-                Assert.IsTrue(PerformanceBenchmark.BaselineXmlFilePaths.Any(f => f.Equals(baselineXmlFilePath)));
+                Assert.IsFalse(PerformanceBenchmark.ResultFilePaths.Any(f => f.Equals(baselineXmlFilePath)));
+                Assert.IsTrue(PerformanceBenchmark.BaselineFilePaths.Any(f => f.Equals(baselineXmlFilePath)));
             }
         }
-        
+
         private void AssertCorrectResultXmlFilePaths(string[] resultFileNames)
         {
             foreach (var resultXmlFilePath in resultFileNames)
             {
-                Assert.IsTrue(PerformanceBenchmark.ResultXmlFilePaths.Contains(resultXmlFilePath));
-                Assert.IsFalse(PerformanceBenchmark.BaselineXmlFilePaths.Contains(resultXmlFilePath));
+                Assert.IsTrue(PerformanceBenchmark.ResultFilePaths.Contains(resultXmlFilePath));
+                Assert.IsFalse(PerformanceBenchmark.BaselineFilePaths.Contains(resultXmlFilePath));
             }
         }
 
@@ -204,7 +204,7 @@ namespace UnityPerformanceBenchmarkReporterTests
         {
             foreach (var resultXmlDirPath in resultsXmlDirPaths)
             {
-                Assert.IsTrue(PerformanceBenchmark.ResultXmlDirectoryPaths.Contains(resultXmlDirPath));
+                Assert.IsTrue(PerformanceBenchmark.ResultDirectoryPaths.Contains(resultXmlDirPath));
             }
         }
     }

UnityPerformanceBenchmarkReporterTests/UnityPerformanceBenchmarkReporterTests.csproj

@@ -44,6 +44,30 @@
     <None Update="TestData\Results\results2.xml">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="TestData\baseline.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestData\baseline2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestData\BaselineJson2\baseline2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestData\BaselineJson\baseline.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestData\results.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestData\results2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestData\ResultsJson\results.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="TestData\ResultsJson2\results2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
 
 </Project>