Xr metadata extraction (#3)
Update metadata processor to extract xr specific metadata
This commit is contained in:
Sean Stolberg
GitHub Enterprise
Родитель
d6722faec7
Коммит
95bbad6fcb
|
|
@ -60,20 +60,22 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
|
||||
private OptionSet GetOptions(PerformanceBenchmark performanceBenchmark)
|
||||
{
|
||||
return new OptionSet()
|
||||
.Add("?|help|h", "Prints out the options.", option => help = option != null)
|
||||
.Add("results|testresultsxmlsource=", "REQUIRED - Path to a test result XML filename OR directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
|
||||
xmlsource =>
|
||||
{
|
||||
performanceBenchmark.AddXmlSourcePath(xmlsource, "results", ResultType.Test);
|
||||
})
|
||||
.Add("baseline|baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename.",
|
||||
xmlsource =>
|
||||
{
|
||||
performanceBenchmark.AddXmlSourcePath(xmlsource, "baseline", ResultType.Baseline);
|
||||
})
|
||||
.Add("report|reportdirpath:", "OPTIONAL - Path to where the report will be written. Default is current working directory.",
|
||||
performanceBenchmark.AddReportDirPath);
|
||||
var optionsSet = new OptionSet();
|
||||
optionsSet.Add("?|help|h", "Prints out the options.", option => help = option != null);
|
||||
optionsSet.Add(
|
||||
"results|testresultsxmlsource=",
|
||||
"REQUIRED - Path to a test result XML filename OR directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
|
||||
xmlsource => performanceBenchmark.AddXmlSourcePath(xmlsource, "results", ResultType.Test));
|
||||
optionsSet.Add(
|
||||
"baseline|baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename.",
|
||||
xmlsource => performanceBenchmark.AddXmlSourcePath(xmlsource, "baseline", ResultType.Baseline));
|
||||
optionsSet.Add(
|
||||
"report|reportdirpath:", "OPTIONAL - Path to where the report will be written. Default is current working directory.",
|
||||
performanceBenchmark.AddReportDirPath);
|
||||
optionsSet.Add("failonbaseline",
|
||||
"Enable return '1' by the reporter if a baseline is passed in and one or more matching configs is out of threshold. Disabled is default. Use option to enable, or use option and append '-' to explicitly disable.",
|
||||
option => performanceBenchmark.FailOnBaseline = option != null);
|
||||
return optionsSet;
|
||||
}
|
||||
|
||||
private void ShowHelp(string message, OptionSet optionSet)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
public HashSet<string> BaselineXmlFilePaths { get; } = new HashSet<string>();
|
||||
public uint SigFig { get; }
|
||||
public string ReportDirPath { get; private set; }
|
||||
public bool FailOnBaseline { get; set; }
|
||||
|
||||
|
||||
public bool BaselineResultFilesExist => BaselineXmlFilePaths.Any();
|
||||
|
|
@ -112,19 +113,16 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
|
||||
testResults.AddRange(results);
|
||||
|
||||
performanceTestRunProcessor.UpdateTestResultsBasedOnBaselineResults(baselineTestResults,
|
||||
testResults, SigFig);
|
||||
performanceTestRunProcessor.UpdateTestResultsBasedOnBaselineResults(baselineTestResults, testResults, SigFig);
|
||||
|
||||
TestRunMetadataProcessor.ProcessMetadata(performanceTestRun,
|
||||
resultFilesOrderedByResultName[i].Key);
|
||||
TestRunMetadataProcessor.ProcessMetadata(performanceTestRun, resultFilesOrderedByResultName[i].Key);
|
||||
|
||||
testRunResults.Add(performanceTestRunProcessor.CreateTestRunResult
|
||||
(
|
||||
performanceTestRun,
|
||||
results,
|
||||
Path.GetFileNameWithoutExtension(resultFilesOrderedByResultName[i].Key),
|
||||
isBaseline)
|
||||
);
|
||||
var performanceTestRunResult = performanceTestRunProcessor.CreateTestRunResult(
|
||||
performanceTestRun,
|
||||
results,
|
||||
Path.GetFileNameWithoutExtension(resultFilesOrderedByResultName[i].Key),
|
||||
isBaseline);
|
||||
testRunResults.Add(performanceTestRunResult);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,15 +64,25 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
{
|
||||
foreach (var testResult in testResults)
|
||||
{
|
||||
// If the baseline results doesn't have a matching TestName for this result, skip it
|
||||
if (baselineTestResults.All(r => r.TestName != testResult.TestName)) continue;
|
||||
|
||||
// Get the corresponding baseline testname samplegroupresults for this result's testname
|
||||
var baselineSampleGroupResults = baselineTestResults.First(r => r.TestName == testResult.TestName).SampleGroupResults;
|
||||
|
||||
foreach (var sampleGroupResult in testResult.SampleGroupResults)
|
||||
{
|
||||
// if we have a corresponding baseline samplegroupname in this sampleGroupResult, compare them
|
||||
if (baselineSampleGroupResults.Any(sg => sg.SampleGroupName == sampleGroupResult.SampleGroupName))
|
||||
{
|
||||
var baselineSampleGroupResult = baselineSampleGroupResults.First(sg =>
|
||||
sg.SampleGroupName == sampleGroupResult.SampleGroupName);
|
||||
// Get the baselineSampleGroupResult that corresponds to this SampleGroupResults sample group name
|
||||
var baselineSampleGroupResult = baselineSampleGroupResults.First(sg => sg.SampleGroupName == sampleGroupResult.SampleGroupName);
|
||||
|
||||
// update this samplegroupresults baselinevalue and threshold to be that of the baselinesamplegroup so we can perform an accurate assessement of
|
||||
// whether or not a regression has occurred.
|
||||
sampleGroupResult.BaselineValue = baselineSampleGroupResult.AggregatedValue;
|
||||
sampleGroupResult.Threshold = baselineSampleGroupResult.Threshold;
|
||||
|
||||
sampleGroupResult.Regressed = DeterminePerformanceResult(sampleGroupResult, sigfig) == MeasurementResult.Regression;
|
||||
}
|
||||
}
|
||||
|
|
@ -84,7 +94,7 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
}
|
||||
}
|
||||
|
||||
public Dictionary<string, List<SampleGroup>> MergeTestExecutions(PerformanceTestRun performanceTestRun)
|
||||
private Dictionary<string, List<SampleGroup>> MergeTestExecutions(PerformanceTestRun performanceTestRun)
|
||||
{
|
||||
var mergedTestExecutions = new Dictionary<string, List<SampleGroup>>();
|
||||
var testNames = performanceTestRun.Results.Select(te => te.TestName).Distinct().ToList();
|
||||
|
|
|
|||
|
|
@ -8,23 +8,34 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
{
|
||||
internal class Program
|
||||
{
|
||||
private static int indentLevel;
|
||||
|
||||
private static readonly Dictionary<Type, string[]> ExcludedConfigFieldNames = new Dictionary<Type, string[]>
|
||||
{
|
||||
{typeof(EditorVersion), new []
|
||||
{
|
||||
"DateSeconds",
|
||||
"RevisionValue",
|
||||
"Branch"
|
||||
"RevisionValue"
|
||||
}},
|
||||
{typeof(BuildSettings), new []
|
||||
{
|
||||
"DevelopmentPlayer"
|
||||
}},
|
||||
{typeof(PlayerSystemInfo), new []
|
||||
{
|
||||
"XrModel"
|
||||
}},
|
||||
{typeof(PlayerSettings), new []
|
||||
{
|
||||
"MtRendering", // Hidden because we have a calculated field, RenderThreadingMode, that provides a more succinct value (SingleThreaded, MultiThreaded, GfxJobs)
|
||||
"GraphicsJobs", // Hidden because we have a calculated field, RenderThreadingMode, that provides a more succinct value (SingleThreaded, MultiThreaded, GfxJobs)
|
||||
"VrSupported" // Hidden because this value doesn't seem to be coming through as 'True' when it should be true.
|
||||
"VrSupported", // Hidden because this value doesn't seem to be coming through as 'True' when it should be true.
|
||||
"AndroidMinimumSdkVersion",
|
||||
"AndroidTargetSdkVersion"
|
||||
}}
|
||||
};
|
||||
|
||||
private static void Main(string[] args)
|
||||
private static int Main(string[] args)
|
||||
{
|
||||
var aggregateTestRunResults = new List<PerformanceTestRunResult>();
|
||||
var baselinePerformanceTestRunResults = new List<PerformanceTestRunResult>();
|
||||
|
|
@ -67,8 +78,10 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
|
||||
var performanceTestResults = new PerformanceTestRunResult[0];
|
||||
|
||||
// If we have a baseline
|
||||
if (aggregateTestRunResults.Any(a => a.IsBaseline))
|
||||
{
|
||||
// Insert the baseline in the front of the array results; this way we can display the baseline first in the report
|
||||
Array.Resize(ref performanceTestResults, 1);
|
||||
performanceTestResults[0] = aggregateTestRunResults.First(a => a.IsBaseline);
|
||||
}
|
||||
|
|
@ -84,13 +97,64 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
performanceTestResults[performanceTestResults.Length - 1] = performanceTestRunResult;
|
||||
}
|
||||
|
||||
performanceBenchmark.TestRunMetadataProcessor.PerformFinalMetadataUpdate(performanceBenchmark);
|
||||
|
||||
var reportWriter = new ReportWriter(performanceBenchmark.TestRunMetadataProcessor);
|
||||
|
||||
reportWriter.WriteReport(
|
||||
performanceTestResults,
|
||||
performanceBenchmark.SigFig,
|
||||
performanceBenchmark.ReportDirPath,
|
||||
performanceBenchmark.BaselineResultFilesExist);
|
||||
|
||||
return WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
|
||||
}
|
||||
|
||||
private static int WriteFailedTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
|
||||
{
|
||||
var failedTestsExist = performanceTestResults.SelectMany(ptr => ptr.TestResults)
|
||||
.Any(tr => tr.State == (int) TestState.Failure);
|
||||
if (failedTestsExist)
|
||||
{
|
||||
WriteLine("FAILURE: One ore more performance test metric aggregations is out of threshold from the baseline value.");
|
||||
WriteLine("-------------------------------------");
|
||||
WriteLine(" Performance tests with failed metrics");
|
||||
WriteLine("-------------------------------------");
|
||||
foreach (var performanceTestRunResult in performanceTestResults)
|
||||
{
|
||||
var failedTests = performanceTestRunResult.TestResults.Where(tr => tr.State == (int)TestState.Failure);
|
||||
if (failedTests.Any())
|
||||
{
|
||||
foreach (var failedTest in failedTests)
|
||||
{
|
||||
++indentLevel;
|
||||
WriteLine("{0}", failedTest.TestName);
|
||||
|
||||
var regressedSgs = failedTest.SampleGroupResults.Where(sgr => sgr.Regressed);
|
||||
foreach (var sampleGroupResult in regressedSgs)
|
||||
{
|
||||
WriteLine("----");
|
||||
WriteLine("Metric : {0}", sampleGroupResult.SampleGroupName);
|
||||
WriteLine("Aggregation : {0}", sampleGroupResult.AggregationType);
|
||||
WriteLine("Failed Value : {0,8:F2}", sampleGroupResult.AggregatedValue);
|
||||
WriteLine("Baseline Value: {0,8:F2}", sampleGroupResult.BaselineValue);
|
||||
WriteLine("Threshold % : {0,8:F2}", sampleGroupResult.Threshold);
|
||||
WriteLine("Actual Diff % : {0,8:F2}", Math.Abs(sampleGroupResult.AggregatedValue - sampleGroupResult.BaselineValue) / sampleGroupResult.BaselineValue);
|
||||
}
|
||||
--indentLevel;
|
||||
WriteLine("\r\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return performanceBenchmark.FailOnBaseline && failedTestsExist ? 1 : 0;
|
||||
}
|
||||
|
||||
private static void WriteLine(string format, params object[] args)
|
||||
{
|
||||
Console.Write(new string('\t', indentLevel));
|
||||
Console.WriteLine(format, args);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ using System.Reflection;
|
|||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using UnityPerformanceBenchmarkReporter.Entities;
|
||||
|
||||
namespace UnityPerformanceBenchmarkReporter.Report
|
||||
|
|
@ -35,6 +36,8 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
private PerformanceTestRunResult[] perfTestRunResults = { };
|
||||
private bool thisHasBenchmarkResults;
|
||||
private uint thisSigFig;
|
||||
private bool anyTestFailures;
|
||||
private readonly string nullString = "null";
|
||||
|
||||
public ReportWriter(TestRunMetadataProcessor metadataProcessor)
|
||||
{
|
||||
|
|
@ -46,6 +49,8 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
{
|
||||
if (results != null && results.Length > 0)
|
||||
{
|
||||
Thread.CurrentThread.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture;
|
||||
|
||||
thisSigFig = sigFig;
|
||||
thisHasBenchmarkResults = hasBenchmarkResults;
|
||||
perfTestRunResults = results;
|
||||
|
|
@ -162,6 +167,10 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
WriteTestConfig(streamWriter);
|
||||
WriteStatMethodTable(streamWriter);
|
||||
WriteTestTableWithVisualizations(streamWriter);
|
||||
if (anyTestFailures)
|
||||
{
|
||||
streamWriter.WriteLine("<script>toggleCanvasWithNoFailures();</script>");
|
||||
}
|
||||
streamWriter.WriteLine("</body>");
|
||||
}
|
||||
|
||||
|
|
@ -197,7 +206,7 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
if (thisHasBenchmarkResults)
|
||||
{
|
||||
streamWriter.WriteLine("<label id=\"hidefailed\" class=\"containerLabel\">Show failed tests only");
|
||||
streamWriter.WriteLine("<input type=\"checkbox\" onclick=\"toggleCanvasWithNoFailures()\">");
|
||||
streamWriter.WriteLine("<input type=\"checkbox\" onclick=\"toggleCanvasWithNoFailures()\" checked>");
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -395,7 +404,6 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
var canvasId = GetCanvasId(distinctTestName, distinctSampleGroupName);
|
||||
|
||||
rw.WriteLine("Chart.defaults.global.elements.rectangle.borderColor = \'#fff\';");
|
||||
|
||||
rw.WriteLine("var ctx{0} = document.getElementById('{0}').getContext('2d');", canvasId);
|
||||
rw.WriteLine("window.{0} = new Chart(ctx{0}, {{", canvasId);
|
||||
rw.WriteLine("type: 'bar',");
|
||||
|
|
@ -443,6 +451,14 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
rw.WriteLine(" }]");
|
||||
rw.WriteLine("},");
|
||||
rw.WriteLine("responsive: true,");
|
||||
rw.WriteLine("animation:");
|
||||
rw.WriteLine("{");
|
||||
rw.WriteLine(" duration: 0 // general animation time");
|
||||
rw.WriteLine("},");
|
||||
rw.WriteLine("hover:");
|
||||
rw.WriteLine("{");
|
||||
rw.WriteLine(" animationDuration: 0 // general animation time");
|
||||
rw.WriteLine("},");
|
||||
rw.WriteLine("responsiveAnimationDuration: 0,");
|
||||
rw.WriteLine("title: {");
|
||||
rw.WriteLine(" display: true,");
|
||||
|
|
@ -531,6 +547,7 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
{
|
||||
var resultsForThisTest = GetResultsForThisTest(distinctTestName);
|
||||
var noTestRegressions = IsNoTestFailures(resultsForThisTest);
|
||||
anyTestFailures = anyTestFailures || !noTestRegressions;
|
||||
rw.WriteLine("<tr {0}>", noTestRegressions ? "class=\"nofailures\"" : string.Empty);
|
||||
rw.WriteLine(
|
||||
"<td class=\"testnamecell\"><div class=\"testname {0}\"><p><h5>Test Name:</h5></p><p><h3>{1}</h3></p></div></td></tr>",
|
||||
|
|
@ -647,8 +664,17 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
var baselineValuesArrayString = new StringBuilder();
|
||||
baselineValuesArrayString.Append(string.Format("var {0} = [", baselineValuesArrayName));
|
||||
|
||||
var benchmarkResults = perfTestRunResults[0];
|
||||
foreach (var performanceTestRunResult in perfTestRunResults)
|
||||
{
|
||||
var aggregatedDefaultValue = nullString;
|
||||
var medianDefaultValue = nullString;
|
||||
var minDefaultValue = nullString;
|
||||
var maxDefaultValue = nullString;
|
||||
var avgDefaultValue = nullString;
|
||||
var stdevDefaultValue = nullString;
|
||||
var baselineDefaultValue = nullString;
|
||||
|
||||
if (performanceTestRunResult.TestResults.Any(r =>
|
||||
ScrubStringForSafeForVariableUse(r.TestName).Equals(distinctTestName)))
|
||||
{
|
||||
|
|
@ -660,19 +686,50 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
ScrubStringForSafeForVariableUse(r.SampleGroupName)
|
||||
.Equals(distinctSampleGroupName));
|
||||
aggregatedValuesArrayString.Append(string.Format("'{0}', ",
|
||||
sgResult != null ? sgResult.AggregatedValue.ToString("F" + thisSigFig) : ""));
|
||||
sgResult != null ? sgResult.AggregatedValue.ToString("F" + thisSigFig) : aggregatedDefaultValue));
|
||||
medianValuesArrayString.Append(string.Format("'{0}', ",
|
||||
sgResult != null ? sgResult.Median.ToString("F" + thisSigFig) : ""));
|
||||
sgResult != null ? sgResult.Median.ToString("F" + thisSigFig) : medianDefaultValue));
|
||||
minValuesArrayString.Append(string.Format("'{0}', ",
|
||||
sgResult != null ? sgResult.Min.ToString("F" + thisSigFig) : ""));
|
||||
sgResult != null ? sgResult.Min.ToString("F" + thisSigFig) : minDefaultValue));
|
||||
maxValuesArrayString.Append(string.Format("'{0}' ,",
|
||||
sgResult != null ? sgResult.Max.ToString("F" + thisSigFig) : ""));
|
||||
sgResult != null ? sgResult.Max.ToString("F" + thisSigFig) : maxDefaultValue));
|
||||
avgValuesArrayString.Append(string.Format("'{0}' ,",
|
||||
sgResult != null ? sgResult.Average.ToString("F" + thisSigFig) : ""));
|
||||
sgResult != null ? sgResult.Average.ToString("F" + thisSigFig) : avgDefaultValue));
|
||||
stdevValuesArrayString.Append(string.Format("'{0}' ,",
|
||||
sgResult != null ? sgResult.StandardDeviation.ToString("F" + thisSigFig) : ""));
|
||||
baselineValuesArrayString.Append(string.Format("'{0}' ,",
|
||||
sgResult != null ? sgResult.BaselineValue.ToString("F" + thisSigFig) : ""));
|
||||
sgResult != null ? sgResult.StandardDeviation.ToString("F" + thisSigFig) : stdevDefaultValue));
|
||||
|
||||
if (benchmarkResults.TestResults
|
||||
.Any(r => ScrubStringForSafeForVariableUse(r.TestName)
|
||||
.Equals(distinctTestName)))
|
||||
{
|
||||
var resultMatch = benchmarkResults.TestResults
|
||||
.First(r => ScrubStringForSafeForVariableUse(r.TestName)
|
||||
.Equals(distinctTestName));
|
||||
var benchmarkSampleGroup = resultMatch.SampleGroupResults.FirstOrDefault(r =>
|
||||
ScrubStringForSafeForVariableUse(r.SampleGroupName)
|
||||
.Equals(distinctSampleGroupName));
|
||||
|
||||
var value = thisHasBenchmarkResults && benchmarkSampleGroup != null
|
||||
? benchmarkSampleGroup.BaselineValue.ToString("F" + thisSigFig)
|
||||
: baselineDefaultValue;
|
||||
|
||||
baselineValuesArrayString.Append(string.Format("'{0}' ,",
|
||||
value));
|
||||
}
|
||||
else
|
||||
{
|
||||
baselineValuesArrayString.Append(string.Format("'{0}' ,", baselineDefaultValue));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
aggregatedValuesArrayString.Append(string.Format("'{0}', ", aggregatedDefaultValue));
|
||||
medianValuesArrayString.Append(string.Format("'{0}', ", medianDefaultValue));
|
||||
minValuesArrayString.Append(string.Format("'{0}', ", minDefaultValue));
|
||||
maxValuesArrayString.Append(string.Format("'{0}' ,", maxDefaultValue));
|
||||
avgValuesArrayString.Append(string.Format("'{0}' ,", avgDefaultValue));
|
||||
stdevValuesArrayString.Append(string.Format("'{0}' ,", stdevDefaultValue));
|
||||
baselineValuesArrayString.Append(string.Format("'{0}' ,", baselineDefaultValue));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -845,8 +902,9 @@ namespace UnityPerformanceBenchmarkReporter.Report
|
|||
|
||||
private bool SampleGroupHasSamples(IEnumerable<TestResult> resultsForThisTest, string distinctSampleGroupName)
|
||||
{
|
||||
return resultsForThisTest.First().SampleGroupResults.Any(sg =>
|
||||
var sampleGroupHasSamples = resultsForThisTest.SelectMany(r => r.SampleGroupResults).Any(sg =>
|
||||
ScrubStringForSafeForVariableUse(sg.SampleGroupName) == distinctSampleGroupName);
|
||||
return sampleGroupHasSamples;
|
||||
}
|
||||
|
||||
private bool SampleGroupHasRegressions(IEnumerable<TestResult> resultsForThisTest,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ using System.Linq;
|
|||
using System.Reflection;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using UnityPerformanceBenchmarkReporter.Entities;
|
||||
|
||||
namespace UnityPerformanceBenchmarkReporter
|
||||
|
|
@ -78,13 +79,16 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
{
|
||||
private readonly string[] androidOnlyMetadata =
|
||||
{
|
||||
"AndroidMinimumSdkVersion",
|
||||
"AndroidTargetSdkVersion",
|
||||
"AndroidBuildSystem"
|
||||
};
|
||||
|
||||
private readonly string[] builtInXrOnlyMetadata =
|
||||
{
|
||||
"EnabledXrTargets"
|
||||
};
|
||||
|
||||
private readonly Dictionary<Type, string[]> excludedConfigFieldNames;
|
||||
private readonly string metadataNotAvailable = "Metadata not available";
|
||||
private static readonly string MetadataNotAvailable = "Metadata not available";
|
||||
|
||||
private readonly Type[] metadataTypes =
|
||||
{
|
||||
|
|
@ -98,18 +102,142 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
|
||||
public readonly List<TypeMetadata> TypeMetadata = new List<TypeMetadata>();
|
||||
|
||||
private readonly string[] xrOnlyMetadata =
|
||||
{
|
||||
"XrModel",
|
||||
"XrDevice",
|
||||
"EnabledXrTargets",
|
||||
"StereoRenderingPath"
|
||||
};
|
||||
|
||||
private bool isAndroid;
|
||||
|
||||
private bool isVrSupported;
|
||||
public bool BuiltInVrExists { get; private set; }
|
||||
|
||||
public class ExtractField
|
||||
{
|
||||
public string ExtractedFieldName;
|
||||
public Regex ExtractionRegex;
|
||||
public string ValueExtracted;
|
||||
}
|
||||
|
||||
private readonly Dictionary<string, List<ExtractField>> extraMetadataExtractFields =
|
||||
new Dictionary<string, List<ExtractField>>
|
||||
{
|
||||
{
|
||||
"ScriptingRuntimeVersion",
|
||||
new List<ExtractField>
|
||||
{
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "OculusPluginVersion",
|
||||
ExtractionRegex = new Regex("OculusPluginVersion\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "DeviceRuntimeVersion",
|
||||
ExtractionRegex =
|
||||
new Regex("deviceruntimeversion\\|[^/]*/[^/]*/[^/]*/[^/]*/([0-9]*\\.[0-9]*\\.[0-9]*):",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "XrSdkName",
|
||||
ExtractionRegex = new Regex("XrsdkName\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "XrSdkVersion",
|
||||
ExtractionRegex = new Regex("XrSdkVersion\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "XrSdkRevision",
|
||||
ExtractionRegex = new Regex("XrSdkRevision\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "XrSdkRevisionDate",
|
||||
ExtractionRegex = new Regex("XrSdkRevisionDate\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "XrSdkBranch",
|
||||
ExtractionRegex = new Regex("XrSdkBranch\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "XrManagementVersion",
|
||||
ExtractionRegex = new Regex("XrManagementVersion\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "XrManagementRevision",
|
||||
ExtractionRegex = new Regex("XrManagementRevision\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "DeviceUniqueId",
|
||||
ExtractionRegex = new Regex("deviceuniqueid\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "Username",
|
||||
ExtractionRegex = new Regex("username\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "RenderPipeline",
|
||||
ExtractionRegex = new Regex("renderpipeline\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "FfrLevel",
|
||||
ExtractionRegex = new Regex("ffrlevel\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "TestsBranch",
|
||||
ExtractionRegex = new Regex("testsbranch\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "TestsRev",
|
||||
ExtractionRegex = new Regex("testsrev\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "TestsRevDate",
|
||||
ExtractionRegex = new Regex("testsrevdate\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "PerfTestsPackageName",
|
||||
ExtractionRegex = new Regex("PerfTestsPackageName\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "PerfTestsVersion",
|
||||
ExtractionRegex = new Regex("PerfTestsVersion\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
},
|
||||
new ExtractField
|
||||
{
|
||||
ExtractedFieldName = "PerfTestsRevision",
|
||||
ExtractionRegex = new Regex("PerfTestsRevision\\|([^|]*)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
public TestRunMetadataProcessor(Dictionary<Type, string[]> excludedFieldNames)
|
||||
{
|
||||
|
|
@ -123,7 +251,7 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
/// <param name="xmlFileNamePath"></param>
|
||||
public void ProcessMetadata(PerformanceTestRun performanceTestRun, string xmlFileNamePath)
|
||||
{
|
||||
SetIsVrSupported(new[] {performanceTestRun});
|
||||
SetIsBuiltInVr(new[] {performanceTestRun});
|
||||
SetIsAndroid(new[] {performanceTestRun});
|
||||
|
||||
foreach (var metadataType in metadataTypes)
|
||||
|
|
@ -140,6 +268,7 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
}
|
||||
|
||||
var fieldInfos = performanceTestRun.GetType().GetFields();
|
||||
|
||||
// If this metadataType is completely missing from the perf test run, mark it as such and move on
|
||||
if (fieldInfos.Any(f => f.FieldType == metadataType))
|
||||
{
|
||||
|
|
@ -166,47 +295,122 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
continue;
|
||||
}
|
||||
|
||||
var fields = obj.GetType().GetFields();
|
||||
var fieldsToProcess = GetFieldsToProcess(metadataType, fields);
|
||||
var fieldsToProcess = GetFieldsToProcess(metadataType, obj.GetType().GetFields());
|
||||
|
||||
// if we have valid field metadata to process
|
||||
if (fieldsToProcess.Length > 0)
|
||||
{
|
||||
foreach (var field in fieldsToProcess)
|
||||
{
|
||||
if (!typeMetadata.FieldGroups.Any(fg => fg.FieldName.Equals(field.Name)))
|
||||
{
|
||||
typeMetadata.FieldGroups.Add(new FieldGroup(field.Name));
|
||||
}
|
||||
|
||||
var thisFieldGroup = typeMetadata.FieldGroups.First(fg => fg.FieldName.Equals(field.Name));
|
||||
|
||||
// We want to keep the values array length consistent with the number of results, even for results
|
||||
// that are missing metadata. We do that here.
|
||||
BackfillFieldGroupValuesForMissingMetadata(xmlFileNamePath, thisFieldGroup, typeMetadata);
|
||||
|
||||
// Add this field value
|
||||
var value = GetValueBasedOnType(metadataType, field, obj);
|
||||
|
||||
Array.Resize(ref thisFieldGroup.Values, thisFieldGroup.Values.Length + 1);
|
||||
thisFieldGroup.Values[thisFieldGroup.Values.Length - 1] =
|
||||
new FieldValue(xmlFileNamePath, value);
|
||||
|
||||
// fieldGroup.Values is sorted by result name; the first element in this array
|
||||
// is considered to be the reference point, regardless if it's a "baseline" or not.
|
||||
if (thisFieldGroup.Values[thisFieldGroup.Values.Length - 1].Value !=
|
||||
thisFieldGroup.Values[0].Value)
|
||||
{
|
||||
thisFieldGroup.Values[thisFieldGroup.Values.Length - 1].IsMismatched = true;
|
||||
}
|
||||
}
|
||||
|
||||
typeMetadata.ValidResultCount++;
|
||||
ProcessMetaData(xmlFileNamePath, fieldsToProcess, typeMetadata, metadataType, obj);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ProcessMetaData(string xmlFileNamePath, FieldInfo[] fieldsToProcess, TypeMetadata typeMetadata,
|
||||
Type metadataType, object obj)
|
||||
{
|
||||
foreach (var field in fieldsToProcess)
|
||||
{
|
||||
var fieldName = field.Name;
|
||||
if (!typeMetadata.FieldGroups.Any(fg => fg.FieldName.Equals(fieldName)))
|
||||
{
|
||||
typeMetadata.FieldGroups.Add(new FieldGroup(fieldName));
|
||||
}
|
||||
|
||||
var thisFieldGroup = typeMetadata.FieldGroups.First(fg => fg.FieldName.Equals(fieldName));
|
||||
|
||||
|
||||
if (extraMetadataExtractFields.ContainsKey(fieldName))
|
||||
{
|
||||
foreach (var extractField in extraMetadataExtractFields[fieldName])
|
||||
{
|
||||
FieldGroup newFieldGroup;
|
||||
if (!typeMetadata.FieldGroups.Any(fg => fg.FieldName.Equals(extractField.ExtractedFieldName)))
|
||||
{
|
||||
newFieldGroup = new FieldGroup(extractField.ExtractedFieldName);
|
||||
typeMetadata.FieldGroups.Add(newFieldGroup);
|
||||
}
|
||||
else
|
||||
{
|
||||
newFieldGroup =
|
||||
typeMetadata.FieldGroups.First(fg =>
|
||||
fg.FieldName.Equals(extractField.ExtractedFieldName));
|
||||
}
|
||||
|
||||
var value = GetValueBasedOnType(metadataType, field, obj);
|
||||
|
||||
extractField.ValueExtracted = ExtractValue(extractField.ExtractionRegex, value);
|
||||
|
||||
InsertFieldValueWithBackfill(xmlFileNamePath, newFieldGroup, typeMetadata,
|
||||
extractField.ValueExtracted);
|
||||
DetermineIfMismatchExists(typeMetadata, newFieldGroup);
|
||||
}
|
||||
|
||||
InsertFieldValueWithBackfill(xmlFileNamePath, thisFieldGroup, typeMetadata, MetadataNotAvailable);
|
||||
DetermineIfMismatchExists(typeMetadata, thisFieldGroup);
|
||||
}
|
||||
else
|
||||
{
|
||||
var thisValue = GetValueBasedOnType(metadataType, field, obj);
|
||||
InsertFieldValueWithBackfill(xmlFileNamePath, thisFieldGroup, typeMetadata, thisValue);
|
||||
DetermineIfMismatchExists(typeMetadata, thisFieldGroup);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var fieldGroup in typeMetadata.FieldGroups.Where(fg =>
|
||||
fg.Values.Length < typeMetadata.ValidResultCount + 1))
|
||||
{
|
||||
while (fieldGroup.Values.Length < typeMetadata.ValidResultCount + 1)
|
||||
{
|
||||
InsertFieldValue(xmlFileNamePath, fieldGroup, MetadataNotAvailable, isMismatched: true);
|
||||
}
|
||||
}
|
||||
|
||||
typeMetadata.ValidResultCount++;
|
||||
}
|
||||
|
||||
private static void DetermineIfMismatchExists(TypeMetadata typeMetadata, FieldGroup thisFieldGroup)
|
||||
{
|
||||
// fieldGroup.Values is sorted by result name; the first element in this array
|
||||
// is considered to be the reference point, regardless if it's a "baseline" or not.
|
||||
if (typeMetadata.FieldGroups.Any(fg => fg.FieldName.Equals(thisFieldGroup.FieldName)) &&
|
||||
thisFieldGroup.Values.Length > 0 && thisFieldGroup.Values[thisFieldGroup.Values.Length - 1].Value !=
|
||||
thisFieldGroup.Values[0].Value)
|
||||
{
|
||||
thisFieldGroup.Values[thisFieldGroup.Values.Length - 1].IsMismatched = true;
|
||||
}
|
||||
}
|
||||
|
||||
private string ExtractValue(Regex regex, string value)
|
||||
{
|
||||
var matches = regex.Matches(value);
|
||||
var matchValue = matches.Count > 0 ? matches[0].Groups[1].Value : MetadataNotAvailable;
|
||||
|
||||
return matchValue;
|
||||
}
|
||||
|
||||
private void InsertFieldValueWithBackfill(string xmlFileNamePath, FieldGroup thisFieldGroup,
|
||||
TypeMetadata typeMetadata,
|
||||
string value)
|
||||
{
|
||||
// We want to keep the values array length consistent with the number of results, even for results
|
||||
// that are missing metadata. We do that here.
|
||||
BackfillFieldGroupValuesForMissingMetadata(xmlFileNamePath, thisFieldGroup, typeMetadata);
|
||||
|
||||
InsertFieldValue(xmlFileNamePath, thisFieldGroup, value);
|
||||
}
|
||||
|
||||
private static void InsertFieldValue(string xmlFileNamePath, FieldGroup thisFieldGroup, string value,
|
||||
bool isMismatched = false)
|
||||
{
|
||||
Array.Resize(ref thisFieldGroup.Values, thisFieldGroup.Values.Length + 1);
|
||||
thisFieldGroup.Values[thisFieldGroup.Values.Length - 1] =
|
||||
new FieldValue(xmlFileNamePath, value)
|
||||
{
|
||||
IsMismatched = isMismatched
|
||||
};
|
||||
}
|
||||
|
||||
private void BackfillFieldGroupValuesForMissingMetadata(string xmlFileNamePath, FieldGroup fieldGroup,
|
||||
TypeMetadata typeMetadata)
|
||||
{
|
||||
|
|
@ -216,7 +420,7 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
{
|
||||
Array.Resize(ref fieldGroup.Values, fieldGroup.Values.Length + 1);
|
||||
fieldGroup.Values[fieldGroup.Values.Length - 1] =
|
||||
new FieldValue(xmlFileNamePath, metadataNotAvailable);
|
||||
new FieldValue(xmlFileNamePath, MetadataNotAvailable);
|
||||
|
||||
// fieldGroup.Values is sorted by result name; the first element in this array
|
||||
// is considered to be the reference point, regardless if it's a "baseline" or not.
|
||||
|
|
@ -305,13 +509,24 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
var value = field.GetValue((T) obj);
|
||||
if (value == null)
|
||||
{
|
||||
value = metadataNotAvailable;
|
||||
value = MetadataNotAvailable;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (value.GetType() != typeof(string))
|
||||
{
|
||||
value = Convert.ToString(value);
|
||||
if ((string) value == string.Empty)
|
||||
{
|
||||
value = MetadataNotAvailable;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if ((string) value == string.Empty)
|
||||
{
|
||||
value = MetadataNotAvailable;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -321,9 +536,10 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
private FieldInfo[] GetFieldsToProcess(Type metadataType, FieldInfo[] fields)
|
||||
{
|
||||
// Derive a subset of fields to process from validFieldNames
|
||||
var excludedFieldNames = excludedConfigFieldNames != null && excludedConfigFieldNames.ContainsKey(metadataType)
|
||||
? excludedConfigFieldNames[metadataType]
|
||||
: null;
|
||||
var excludedFieldNames =
|
||||
excludedConfigFieldNames != null && excludedConfigFieldNames.ContainsKey(metadataType)
|
||||
? excludedConfigFieldNames[metadataType]
|
||||
: null;
|
||||
var validFieldNames = GetValidFieldNames(excludedFieldNames, fields.Select(f => f.Name).ToArray());
|
||||
var fieldsToProcess = fields.Join(validFieldNames, f => f.Name, s => s, (field, vField) => field).ToArray();
|
||||
return fieldsToProcess;
|
||||
|
|
@ -369,9 +585,9 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
validFieldNames = validFieldNames.Where(k1 => excludedFieldNames.All(k2 => k2 != k1)).ToArray();
|
||||
}
|
||||
|
||||
if (!isVrSupported)
|
||||
if (!BuiltInVrExists)
|
||||
{
|
||||
validFieldNames = validFieldNames.Where(k1 => xrOnlyMetadata.All(k2 => k2 != k1)).ToArray();
|
||||
validFieldNames = validFieldNames.Where(k1 => builtInXrOnlyMetadata.All(k2 => k2 != k1)).ToArray();
|
||||
}
|
||||
|
||||
if (!isAndroid)
|
||||
|
|
@ -382,11 +598,11 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
return validFieldNames;
|
||||
}
|
||||
|
||||
private void SetIsVrSupported(PerformanceTestRun[] performanceTestRuns)
|
||||
private void SetIsBuiltInVr(PerformanceTestRun[] performanceTestRuns)
|
||||
{
|
||||
foreach (var performanceTestRun in performanceTestRuns)
|
||||
{
|
||||
isVrSupported = isVrSupported || performanceTestRun.PlayerSettings?.EnabledXrTargets != null && performanceTestRun.PlayerSettings.EnabledXrTargets.Any();
|
||||
BuiltInVrExists = BuiltInVrExists || performanceTestRun.PlayerSettings.EnabledXrTargets != null && performanceTestRun.PlayerSettings.EnabledXrTargets.Any();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -398,5 +614,19 @@ namespace UnityPerformanceBenchmarkReporter
|
|||
performanceTestRun.BuildSettings.Platform.Equals("Android");
|
||||
}
|
||||
}
|
||||
|
||||
public void PerformFinalMetadataUpdate(PerformanceBenchmark performanceBenchmark)
|
||||
{
|
||||
// The keys in the extraMetadataExtractFields structure have additional embedded metadata that we extract out.
|
||||
// This renders the raw, unextracted, value of this field unusable, so we discard it
|
||||
foreach (var metadataName in extraMetadataExtractFields.Keys)
|
||||
{
|
||||
foreach (var typeMetadata in performanceBenchmark.TestRunMetadataProcessor.TypeMetadata)
|
||||
{
|
||||
typeMetadata.FieldGroups.RemoveAll(fg => fg.FieldName.Equals(metadataName));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче