Merge pull request #1 from unity/new-data-format

Add support for 2.x package data format

UnityPerformanceBenchmarkReporter/Entities/Data.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+
+namespace UnityPerformanceBenchmarkReporter.Entities.New
+{
+    [Serializable]
+    public class TestResult
+    {
+        public string Name;
+        public string Version;
+        public List<string> Categories = new List<string>();
+        public List<SampleGroup> SampleGroups = new List<SampleGroup>();
+    }
+
+    [Serializable]
+    public class SampleGroup
+    {
+        public string Name;
+        public SampleUnit Unit;
+        public bool IncreaseIsBetter;
+        public List<double> Samples = new List<double>();
+        public double Min;
+        public double Max;
+        public double Median;
+        public double Average;
+        public double StandardDeviation;
+        public double Sum;
+
+        public SampleGroup(string name, SampleUnit unit, bool increaseIsBetter)
+        {
+            Name = name;
+            Unit = unit;
+            IncreaseIsBetter = increaseIsBetter;
+        }
+    }
+
+    [Serializable]
+    public class Run
+    {
+        public string TestSuite;
+        public int Date;
+        public Player Player;
+        public Hardware Hardware;
+        public Editor Editor;
+        public List<string> Dependencies = new List<string>();
+        public List<TestResult> Results = new List<TestResult>();
+    }
+
+    [Serializable]
+    public class Editor
+    {
+        public string Version;
+        public string Branch;
+        public string Changeset;
+        public int Date;
+    }
+
+    [Serializable]
+    public class Hardware
+    {
+        public string OperatingSystem;
+        public string DeviceModel;
+        public string DeviceName;
+        public string ProcessorType;
+        public int ProcessorCount;
+        public string GraphicsDeviceName;
+        public int SystemMemorySizeMB;
+    }
+
+    [Serializable]
+    public class Player
+    {
+        public string Platform;
+        public bool Development;
+        public int ScreenWidth;
+        public int ScreenHeight;
+        public int ScreenRefreshRate;
+        public bool Fullscreen;
+        public int Vsync;
+        public int AntiAliasing;
+        public string ColorSpace;
+        public string AnisotropicFiltering;
+        public string BlendWeights;
+        public string GraphicsApi;
+        public bool Batchmode;
+        public string RenderThreadingMode;
+        public bool GpuSkinning;
+
+        // strings because values are editor only enums
+        public string ScriptingBackend;
+        public string AndroidTargetSdkVersion;
+        public string AndroidBuildSystem;
+        public string BuildTarget;
+        public string StereoRenderingPath;
+    }
+
+    public enum SampleUnit
+    {
+        Nanosecond,
+        Microsecond,
+        Millisecond,
+        Second,
+        Byte,
+        Kilobyte,
+        Megabyte,
+        Gigabyte,
+        Undefined
+    }
+}

UnityPerformanceBenchmarkReporter/PerformanceBenchmark.cs

@@ -82,7 +82,7 @@ namespace UnityPerformanceBenchmarkReporter
 
                 foreach (var xmlFileNamePath in xmlFileNamePaths)
                 {
-                    var performanceTestRun = testResultXmlParser.GetPerformanceTestRunFromXml(xmlFileNamePath);
+                    var performanceTestRun = testResultXmlParser.Parse(xmlFileNamePath);
                     if (performanceTestRun != null && performanceTestRun.Results.Any())
                     {
                         perfTestRuns.Add(
@@ -96,7 +96,7 @@ namespace UnityPerformanceBenchmarkReporter
                 for (var i = 0; i < resultFilesOrderedByResultName.Length; i++)
                 {
                     var performanceTestRun =
-                        testResultXmlParser.GetPerformanceTestRunFromXml(resultFilesOrderedByResultName[i].Key);
+                        testResultXmlParser.Parse(resultFilesOrderedByResultName[i].Key);
 
                     if (performanceTestRun != null && performanceTestRun.Results.Any())
                     {

UnityPerformanceBenchmarkReporter/TestResultXmlParser.cs

@@ -2,71 +2,71 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
 using System.Xml.Linq;
 using Newtonsoft.Json;
 using UnityPerformanceBenchmarkReporter.Entities;
+using UnityPerformanceBenchmarkReporter.Entities.New;
 
 namespace UnityPerformanceBenchmarkReporter
 {
     public class TestResultXmlParser
     {
-        public PerformanceTestRun GetPerformanceTestRunFromXml(string resultXmlFileName)
+        public PerformanceTestRun Parse(string path)
         {
-            ValidateInput(resultXmlFileName);
+            var xmlDocument = XDocument.Load(path);
-            var xmlDocument = TryLoadResultXmlFile(resultXmlFileName);
+            return Parse(xmlDocument);
-            var performanceTestRun = TryParseXmlToPerformanceTestRun(xmlDocument);
-            return performanceTestRun;
         }
 
-        private void ValidateInput(string resultXmlFileName)
+        private static PerformanceTestRun Parse(XDocument xmlDocument)
         {
-            if (string.IsNullOrEmpty(resultXmlFileName))
+            var output = xmlDocument.Descendants("output");
+            var xElements = output as XElement[] ?? output.ToArray();
+
+            if (!xElements.Any())
             {
-                throw new ArgumentNullException(resultXmlFileName, nameof(resultXmlFileName));
+                return null;
             }
 
-            if (!File.Exists(resultXmlFileName))
+            var run = DeserializeMetadata(xElements) ?? DeserializeMetadataV2(xElements);
-            {
-                throw new FileNotFoundException("Result file not found; {0}", resultXmlFileName);
-            }
-        }
 
-        private XDocument TryLoadResultXmlFile(string resultXmlFileName)
+            if (run == null)
-        {
-            try
             {
-                return XDocument.Load(resultXmlFileName);
+                return null;
-            }
-            catch (Exception e)
-            {
-                var errMsg = string.Format("Failed to load xml result file: {0}", resultXmlFileName);
-                WriteExceptionConsoleErrorMessage(errMsg, e);
-                throw;
-            }
-        }
-
-        private PerformanceTestRun TryParseXmlToPerformanceTestRun(XDocument xmlDocument)
-        {
-            var output = xmlDocument.Descendants("output").ToArray();
-            if (output == null || !output.Any())
-            {
-                throw new Exception("The xmlDocument passed to the TryParseXmlToPerformanceTestRun method does not have any \'ouput\' xml tags needed for correct parsing.");
             }
 
-            var run = new PerformanceTestRun();
+            run.EndTime = DateTime.Now.ToUniversalTime()
-            DeserializeTestResults(output, run);
+                .Subtract(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))
-            DeserializeMetadata(output, run);
+                .TotalMilliseconds;
-            
+
+            DeserializeTestResults(xElements, run);
+            DeserializeTestResultsV2(xElements, run);
+
             return run;
         }
 
-        private void DeserializeTestResults(IEnumerable<XElement> output, PerformanceTestRun run)
+        private static void DeserializeTestResults(IEnumerable<XElement> output, PerformanceTestRun run)
         {
             foreach (var element in output)
             {
                 foreach (var line in element.Value.Split('\n'))
                 {
                     var json = GetJsonFromHashtag("performancetestresult", line);
+                    if (json == null) continue;
+
+                    var result = JsonConvert.DeserializeObject<PerformanceTestResult>(json);
+                    run.Results.Add(result);
+                }
+            }
+        }
+
+        private static void DeserializeTestResultsV2(IEnumerable<XElement> output, PerformanceTestRun run)
+        {
+            foreach (var element in output)
+            {
+                foreach (var line in element.Value.Split('\n'))
+                {
+                    var json = GetJsonFromHashtag("performancetestresult2", line);
                     if (json == null)
                     {
                         continue;
@@ -75,102 +75,161 @@ namespace UnityPerformanceBenchmarkReporter
                     var result = TryDeserializePerformanceTestResultJsonObject(json);
                     if (result != null)
                     {
-                        run.Results.Add(result);
+                        var pt = new PerformanceTestResult()
-                    }                    
+                        {
+                            TestCategories = result.Categories,
+                            TestName = result.Name,
+                            TestVersion = result.Version,
+                            SampleGroups = result.SampleGroups.Select(sg => new Entities.SampleGroup
+                            {
+                                Samples = sg.Samples,
+                                Average = sg.Average,
+                                Max = sg.Max,
+                                Median = sg.Median,
+                                Min = sg.Min,
+                                Sum = sg.Sum,
+                                StandardDeviation = sg.StandardDeviation,
+                                SampleCount = sg.Samples.Count,
+                                Definition = new SampleGroupDefinition()
+                                {
+                                    Name = sg.Name,
+                                    SampleUnit = (Entities.SampleUnit)sg.Unit,
+                                    IncreaseIsBetter = sg.IncreaseIsBetter
+                                }
+                            }).ToList()
+                        };
+                        run.Results.Add(pt);
+                    }
                 }
             }
         }
 
-        private void DeserializeMetadata(IEnumerable<XElement> output, PerformanceTestRun run)
+        private static PerformanceTestRun DeserializeMetadata(IEnumerable<XElement> output)
+        {
+            return (output
+                .SelectMany(element => element.Value.Split('\n'),
+                    (element, line) => GetJsonFromHashtag("performancetestruninfo", line))
+                .Where(json => json != null)
+                .Select(JsonConvert.DeserializeObject<PerformanceTestRun>)).FirstOrDefault();
+        }
+
+        private static PerformanceTestRun DeserializeMetadataV2(IEnumerable<XElement> output)
         {
             foreach (var element in output)
             {
-                var elements = element.Value.Split('\n');
+                var pattern = @"##performancetestruninfo2:(.+)\n";
-                if(elements.Any(e => e.Length > 0 && e.Substring(0, 2).Equals("##")))
+                var regex = new Regex(pattern);
+                var matches = regex.Match(element.Value);
+                if (!matches.Success) continue;
+                if (matches.Groups.Count == 0) continue;
+
+                if (matches.Groups[1].Captures.Count > 1)
                 {
-                    var line = elements.First(e => e.Length > 0 && e.Substring(0, 2).Equals("##"));
+                    throw new Exception("Performance test run had multiple hardware and player settings, there should only be one.");
+                }
 
-                    var json = GetJsonFromHashtag("performancetestruninfo", line);
+                var json = matches.Groups[1].Value;
+                if (string.IsNullOrEmpty(json))
+                {
+                    throw new Exception("Performance test run has incomplete hardware and player settings.");
+                }
 
-                    // This is the happy case where we have a performancetestruninfo json object
+                var result = TryDeserializePerformanceTestRunJsonObject(json);
-                    if (json != null)
+
+                var run = new PerformanceTestRun()
+                {
+                    BuildSettings = new BuildSettings()
                     {
-                        var result = TryDeserializePerformanceTestRunJsonObject(json);
+                        Platform = result.Player.Platform,
-                        if (result != null)
+                        BuildTarget = result.Player.BuildTarget,
-                        {
+                        DevelopmentPlayer = true,
-                            run.TestSuite = result.TestSuite;
+                        AndroidBuildSystem = result.Player.AndroidBuildSystem
-                            run.EditorVersion = result.EditorVersion;
+                    },
-                            run.QualitySettings = result.QualitySettings;
+                    EditorVersion = new EditorVersion()
-                            run.ScreenSettings = result.ScreenSettings;
-                            run.BuildSettings = result.BuildSettings;
-                            run.PlayerSettings = result.PlayerSettings;
-                            run.PlayerSystemInfo = result.PlayerSystemInfo;
-                            run.StartTime = result.StartTime;
-                            // @TODO fix end time, does it matter for now?
-                            run.EndTime = run.StartTime;
-                        }
-                    }
-                    // Unhappy case where we couldn't find a performancetestruninfo object
-                    // This could be because we have missing metadata for the test run
-                    // In this case, we try to look for a performancetestresult json object
-                    // We should have at least startime metadata  that we can use to correctly
-                    // display the test results on the x-axis of the chart
-                    else
                     {
-                        json = GetJsonFromHashtag("performancetestresult", line);
+                        Branch = result.Editor.Branch,
-                        if (json != null)
+                        DateSeconds = result.Editor.Date,
-                        {
+                        FullVersion = $"{result.Editor.Version} ({result.Editor.Changeset})",
-                            var result = TryDeserializePerformanceTestRunJsonObject(json);
+                        RevisionValue = 0
-                            run.StartTime = result.StartTime;
+                    },
-                            // @TODO fix end time, does it matter for now?
+                    PlayerSettings = new PlayerSettings()
-                            run.EndTime = run.StartTime;
+                    {
-                        }
+                        GpuSkinning = result.Player.GpuSkinning,
-                    }
+                        GraphicsApi = result.Player.GraphicsApi,
-                } 
+                        RenderThreadingMode = result.Player.RenderThreadingMode,
+                        ScriptingBackend = result.Player.ScriptingBackend,
+                        AndroidTargetSdkVersion = result.Player.AndroidTargetSdkVersion,
+                        EnabledXrTargets = new List<string>(),
+                        ScriptingRuntimeVersion = "",
+                        StereoRenderingPath = result.Player.StereoRenderingPath
+                    },
+                    QualitySettings = new QualitySettings()
+                    {
+                        Vsync = result.Player.Vsync,
+                        AntiAliasing = result.Player.AntiAliasing,
+                        AnisotropicFiltering = result.Player.AnisotropicFiltering,
+                        BlendWeights = result.Player.BlendWeights,
+                        ColorSpace = result.Player.ColorSpace
+                    },
+                    ScreenSettings = new ScreenSettings()
+                    {
+                        Fullscreen = result.Player.Fullscreen,
+                        ScreenHeight = result.Player.ScreenHeight,
+                        ScreenWidth = result.Player.ScreenWidth,
+                        ScreenRefreshRate = result.Player.ScreenRefreshRate
+                    },
+                    PlayerSystemInfo = new PlayerSystemInfo()
+                    {
+                        DeviceModel = result.Hardware.DeviceModel,
+                        DeviceName = result.Hardware.DeviceName,
+                        OperatingSystem = result.Hardware.OperatingSystem,
+                        ProcessorCount = result.Hardware.ProcessorCount,
+                        ProcessorType = result.Hardware.ProcessorType,
+                        GraphicsDeviceName = result.Hardware.GraphicsDeviceName,
+                        SystemMemorySize = result.Hardware.SystemMemorySizeMB,
+                        XrDevice = "",
+                        XrModel = ""
+                    },
+                    StartTime = result.Date,
+                    TestSuite = result.TestSuite,
+                    Results = new List<PerformanceTestResult>()
+                };
+
+                return run;
             }
+
+            return null;
         }
 
-        private PerformanceTestResult TryDeserializePerformanceTestResultJsonObject(string json)
+        private static Run TryDeserializePerformanceTestRunJsonObject(string json)
         {
-            PerformanceTestResult performanceTestResult;
             try
             {
-                performanceTestResult = JsonConvert.DeserializeObject<PerformanceTestResult>(json);
+                return JsonConvert.DeserializeObject<Run>(json);
             }
             catch (Exception e)
             {
-                var errMsg = string.Format("Exception thrown while deserializing json string to PerformanceTestResult: {0}", json);
+                Console.WriteLine(e.Message);
-                WriteExceptionConsoleErrorMessage(errMsg, e);
-                throw;
             }
 
-            return performanceTestResult;
+            return null;
         }
 
-        private void WriteExceptionConsoleErrorMessage(string errMsg, Exception e)
+        private static Entities.New.TestResult TryDeserializePerformanceTestResultJsonObject(string json)
         {
-            Console.Error.WriteLine("{0}\r\nException: {1}\r\nInnerException: {2}", errMsg, e.Message,
-                e.InnerException.Message);
-        }
-
-        private PerformanceTestRun TryDeserializePerformanceTestRunJsonObject(string json)
-        {
-            PerformanceTestRun performanceTestRun;
             try
             {
-                performanceTestRun = JsonConvert.DeserializeObject<PerformanceTestRun>(json);
+                return JsonConvert.DeserializeObject<Entities.New.TestResult>(json);
             }
             catch (Exception e)
             {
-                var errMsg = string.Format("Exception thrown while deserializing json string to PerformanceTestRun: {0}", json);
+                Console.WriteLine(e.Message);
-                WriteExceptionConsoleErrorMessage(errMsg, e);
-                throw;
             }
 
-            return performanceTestRun;
+            return null;
         }
 
-        private string GetJsonFromHashtag(string tag, string line)
+        private static string GetJsonFromHashtag(string tag, string line)
         {
             if (!line.Contains($"##{tag}:")) return null;
             var jsonStart = line.IndexOf('{');
@@ -179,18 +238,19 @@ namespace UnityPerformanceBenchmarkReporter
             while (openBrackets > 0 || stringIndex == jsonStart)
             {
                 var character = line[stringIndex];
-                switch (character)
+                if (character == '{')
                 {
-                    case '{':
+                    openBrackets++;
-                        openBrackets++;
+                }
-                        break;
+
-                    case '}':
+                if (character == '}')
-                        openBrackets--;
+                {
-                        break;
+                    openBrackets--;
                 }
 
                 stringIndex++;
             }
+
             var jsonEnd = stringIndex;
             return line.Substring(jsonStart, jsonEnd - jsonStart);
         }