AvaloniaUI
/
angle
зеркало из https://github.com/AvaloniaUI/angle.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Retool perf test running. 

This change does a few things involving the way we run the perf tests:

- Perf test runner can capture different metrics

Useful for capturing the new "nanoSecPerIteration" metric.

- Removes the "score" metric

We'll move to the new time-based metrics. These new metrics are scaled
correctly with iteration counts.

- Runs three trials per perf test

This gives more measurements per test. Each trial is approximately one
second. First the perf tests set a fixed number of iterations after
calibrating the number of steps that we can run in one second. After
that the three trials are run. This should give more stable results.

- Apply more CPU stabilization on Windows

Use SetPriorityClass to apply more CPU priority. Also upgrade
SetThreadPriority to the highest level.

- Always build the Vulkan command buffer test

This catches build regressions more easily. We still skip the test on
non-Android platforms.

Bug: angleproject:2923
Change-Id: I7da234c5af07775ba4a232bb8d65e0138ee7073f
Reviewed-on: https://chromium-review.googlesource.com/c/1330262
Commit-Queue: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
This commit is contained in:
Jamie Madill 2018-11-14 16:24:25 -05:00 коммит произвёл Commit Bot
Родитель 1934b78614
Коммит f3acb8c133
14 изменённых файлов: 173 добавлений и 104 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

62
scripts/perf_test_runner.py
Просмотреть файл

@ -20,7 +20,8 @@ base_path = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file_
# Might have to add lower case "release" in some configurations.
perftests_paths = glob.glob('out/*Release*')
metric = 'score'
metric = 'wall_time'
max_experiments = 10
binary_name = 'angle_perftests'
if sys.platform == 'win32':
@ -95,45 +96,44 @@ if len(sys.argv) >= 2:
print('Using test executable: ' + perftests_path)
print('Test name: ' + test_name)
# Infinite loop of running the tests.
while True:
process = subprocess.Popen([perftests_path, '--gtest_filter=' + test_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def get_results(metric, extra_args=[]):
process = subprocess.Popen([perftests_path, '--gtest_filter=' + test_name] + extra_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = process.communicate()
start_index = output.find(metric + "=")
if start_index == -1:
print("Did not find the score of the specified test in output:")
print(output)
sys.exit(1)
start_index += len(metric) + 2
end_index = output[start_index:].find(" ")
if end_index == -1:
print("Error parsing output:")
print(output)
sys.exit(2)
m = re.search('Running (\d+) tests', output)
m = re.search(r'Running (\d+) tests', output)
if m and int(m.group(1)) > 1:
print("Found more than one test result in output:")
print(output)
sys.exit(3)
end_index += start_index
pattern = metric + r'= ([0-9.]+)'
m = re.findall(pattern, output)
if m is None:
print("Did not find the metric '%s' in the test output:" % metric)
print(output)
sys.exit(1)
score = int(output[start_index:end_index])
sys.stdout.write("score: " + str(score))
return [float(value) for value in m]
scores.append(score)
sys.stdout.write(", mean: %.2f" % mean(scores))
# Calibrate the number of steps
steps = get_results("steps", ["--calibration"])[0]
print("running with %d steps." % steps)
if (len(scores) > 1):
sys.stdout.write(", variation: %.2f%%" % (coefficient_of_variation(scores) * 100.0))
# Loop 'max_experiments' times, running the tests.
for experiment in range(max_experiments):
experiment_scores = get_results(metric, ["--steps", str(steps)])
if (len(scores) > 7):
trucation_n = len(scores) >> 3
sys.stdout.write(", truncated mean: %.2f" % truncated_mean(scores, trucation_n))
sys.stdout.write(", variation: %.2f%%" % (truncated_cov(scores, trucation_n) * 100.0))
for score in experiment_scores:
sys.stdout.write("%s: %.2f" % (metric, score))
scores.append(score)
print("")
if (len(scores) > 1):
sys.stdout.write(", mean: %.2f" % mean(scores))
sys.stdout.write(", variation: %.2f%%" % (coefficient_of_variation(scores) * 100.0))
if (len(scores) > 7):
truncation_n = len(scores) >> 3
sys.stdout.write(", truncated mean: %.2f" % truncated_mean(scores, trucation_n))
sys.stdout.write(", variation: %.2f%%" % (truncated_cov(scores, trucation_n) * 100.0))
print("")

3
src/tests/BUILD.gn
Просмотреть файл

@ -255,9 +255,6 @@ if (is_win || is_linux || is_android || is_mac) {
if (angle_enable_vulkan) {
sources += angle_perf_tests_vulkan_sources
if (is_android) {
sources += angle_perf_tests_vulkan_command_buffer_sources
}
deps += [ "$angle_root/third_party/glslang:glslang" ]
public_configs = [ "$angle_root/third_party/glslang:glslang_config" ]
}

8
src/tests/angle_perftests.gni
Просмотреть файл

@ -38,12 +38,12 @@ angle_perf_tests_sources = [
"test_utils/draw_call_perf_utils.cpp",
"test_utils/draw_call_perf_utils.h",
]
angle_perf_tests_win_sources = [ "perf_tests/IndexDataManagerTest.cpp" ]
angle_perf_tests_vulkan_sources = [ "perf_tests/VulkanPipelineCachePerf.cpp" ]
# Currently Vulkan Command Buffer Perf Tests compile on Android/Linux
angle_perf_tests_vulkan_command_buffer_sources = [
angle_perf_tests_win_sources = [ "perf_tests/IndexDataManagerTest.cpp" ]
angle_perf_tests_vulkan_sources = [
"perf_tests/VulkanCommandBufferPerf.cpp",
"perf_tests/VulkanPipelineCachePerf.cpp",
"test_utils/third_party/vulkan_command_buffer_utils.cpp",
"test_utils/third_party/vulkan_command_buffer_utils.h",
]

21
src/tests/angle_perftests_main.cpp
Просмотреть файл

@ -9,28 +9,11 @@
#include <gtest/gtest.h>
extern bool g_OnlyOneRunFrame;
extern bool gEnableTrace;
extern const char *gTraceFile;
void ANGLEProcessPerfTestArgs(int *argc, char **argv);
int main(int argc, char **argv)
{
for (int i = 0; i < argc; ++i)
{
if (strcmp("--one-frame-only", argv[i]) == 0)
{
g_OnlyOneRunFrame = true;
}
if (strcmp("--enable-trace", argv[i]) == 0)
{
gEnableTrace = true;
}
if (strcmp("--trace-file", argv[i]) == 0 && i < argc - 1)
{
gTraceFile = argv[++i];
}
}
ANGLEProcessPerfTestArgs(&argc, argv);
testing::InitGoogleTest(&argc, argv);
testing::AddGlobalTestEnvironment(new testing::Environment());
int rt = RUN_ALL_TESTS();

121
src/tests/perf_tests/ANGLEPerfTest.cpp
Просмотреть файл

@ -17,15 +17,21 @@
#include <cmath>
#include <fstream>
#include <iostream>
#include <sstream>
#include <json/json.h>
namespace
{
constexpr size_t kInitialTraceEventBufferSize = 50000;
constexpr size_t kWarmupIterations = 3;
constexpr double kMicroSecondsPerSecond = 1e6;
constexpr double kNanoSecondsPerSecond = 1e9;
constexpr double kCalibrationRunTimeSeconds = 1.0;
constexpr double kMaximumRunTimeSeconds = 10.0;
constexpr unsigned int kNumTrials = 3;
bool gCalibration = false;
Optional<unsigned int> gStepsToRunOverride;
struct TraceCategory
{
@ -154,7 +160,6 @@ ANGLEPerfTest::ANGLEPerfTest(const std::string &name,
: mName(name),
mSuffix(suffix),
mTimer(CreateTimer()),
mRunTimeSeconds(2.0),
mSkipTest(false),
mNumStepsPerformed(0),
mIterationsPerStep(iterationsPerStep),
@ -174,17 +179,52 @@ void ANGLEPerfTest::run()
return;
}
// Calibrate to a fixed number of steps during an initial set time.
if (!gStepsToRunOverride.valid())
{
doRunLoop(kCalibrationRunTimeSeconds);
// Calibration allows the perf test runner script to save some time.
if (gCalibration)
{
printResult("steps", static_cast<size_t>(mNumStepsPerformed), "count", false);
return;
}
gStepsToRunOverride = mNumStepsPerformed;
}
// Do another warmup run. Seems to consistently improve results.
doRunLoop(kMaximumRunTimeSeconds);
for (unsigned int trial = 0; trial < kNumTrials; ++trial)
{
doRunLoop(kMaximumRunTimeSeconds);
printResults();
}
}
void ANGLEPerfTest::doRunLoop(double maxRunTime)
{
mNumStepsPerformed = 0;
mRunning = true;
mTimer->start();
while (mRunning)
{
step();
if (mRunning)
{
++mNumStepsPerformed;
}
if (mTimer->getElapsedTime() > mRunTimeSeconds || g_OnlyOneRunFrame)
{
mRunning = false;
if (mTimer->getElapsedTime() > maxRunTime)
{
mRunning = false;
}
else if (gStepsToRunOverride.valid() &&
mNumStepsPerformed >= gStepsToRunOverride.value())
{
mRunning = false;
}
}
}
finishTest();
@ -207,11 +247,10 @@ void ANGLEPerfTest::SetUp()
void ANGLEPerfTest::TearDown()
{
if (mSkipTest)
{
return;
}
}
void ANGLEPerfTest::printResults()
{
double elapsedTimeSeconds = mTimer->getElapsedTime();
double secondsPerStep = elapsedTimeSeconds / static_cast<double>(mNumStepsPerformed);
@ -221,16 +260,13 @@ void ANGLEPerfTest::TearDown()
if (secondsPerIteration > 1e-3)
{
double microSecondsPerIteration = secondsPerIteration * kMicroSecondsPerSecond;
printResult("microSecPerIteration", microSecondsPerIteration, "us", true);
printResult("wall_time", microSecondsPerIteration, "us", true);
}
else
{
double nanoSecPerIteration = secondsPerIteration * kNanoSecondsPerSecond;
printResult("nanoSecPerIteration", nanoSecPerIteration, "ns", true);
printResult("wall_time", nanoSecPerIteration, "ns", true);
}
double relativeScore = static_cast<double>(mNumStepsPerformed) / elapsedTimeSeconds;
printResult("score", static_cast<size_t>(std::round(relativeScore)), "score", true);
}
double ANGLEPerfTest::normalizedTime(size_t value) const
@ -269,9 +305,11 @@ ANGLERenderTest::ANGLERenderTest(const std::string &name, const RenderTestParams
mOSWindow(nullptr)
{
// Force fast tests to make sure our slowest bots don't time out.
// TODO(jmadill): Remove this flag once rolled into Chromium. http://anglebug.com/2923
if (g_OnlyOneRunFrame)
{
const_cast<RenderTestParams &>(testParams).iterationsPerStep = 1;
gStepsToRunOverride = 1;
}
// Try to ensure we don't trigger allocation during execution.
@ -341,15 +379,6 @@ void ANGLERenderTest::SetUp()
abortTest();
return;
}
// Warm up the benchmark to reduce variance.
if (!g_OnlyOneRunFrame)
{
for (size_t iteration = 0; iteration < kWarmupIterations; ++iteration)
{
drawBenchmark();
}
}
}
void ANGLERenderTest::TearDown()
@ -453,3 +482,47 @@ EGLWindow *ANGLERenderTest::createEGLWindow(const RenderTestParams &testParams)
return new EGLWindow(testParams.majorVersion, testParams.minorVersion,
testParams.eglParameters);
}
void ANGLEProcessPerfTestArgs(int *argc, char **argv)
{
int argcOutCount = 0;
for (int argIndex = 0; argIndex < *argc; argIndex++)
{
if (strcmp("--one-frame-only", argv[argIndex]) == 0)
{
g_OnlyOneRunFrame = true;
gStepsToRunOverride = 1;
}
else if (strcmp("--enable-trace", argv[argIndex]) == 0)
{
gEnableTrace = true;
}
else if (strcmp("--trace-file", argv[argIndex]) == 0 && argIndex < *argc - 1)
{
gTraceFile = argv[argIndex];
// Skip an additional argument.
argIndex++;
}
else if (strcmp("--calibration", argv[argIndex]) == 0)
{
gCalibration = true;
}
else if (strcmp("--steps", argv[argIndex]) == 0 && argIndex < *argc - 1)
{
unsigned int stepsToRun = 0;
std::stringstream strstr;
strstr << argv[argIndex + 1];
strstr >> stepsToRun;
gStepsToRunOverride = stepsToRun;
// Skip an additional argument.
argIndex++;
}
else
{
argv[argcOutCount++] = argv[argIndex];
}
}
*argc = argcOutCount;
}

4
src/tests/perf_tests/ANGLEPerfTest.h
Просмотреть файл

@ -84,14 +84,16 @@ class ANGLEPerfTest : public testing::Test, angle::NonCopyable
void abortTest() { mRunning = false; }
unsigned int getNumStepsPerformed() const { return mNumStepsPerformed; }
void doRunLoop(double maxRunTime);
std::string mName;
std::string mSuffix;
Timer *mTimer;
double mRunTimeSeconds;
bool mSkipTest;
private:
void printResults();
unsigned int mNumStepsPerformed;
unsigned int mIterationsPerStep;
bool mRunning;

2
src/tests/perf_tests/CompilerPerf.cpp
Просмотреть файл

@ -162,7 +162,7 @@ void main()
const char *kTrickyESSL300Id = "TrickyESSL300";
constexpr int kNumIterationsPerStep = 10;
constexpr int kNumIterationsPerStep = 4;
struct CompilerPerfParameters final : public angle::CompilerParameters
{

1
src/tests/perf_tests/DrawCallPerf.cpp
Просмотреть файл

@ -110,7 +110,6 @@ class DrawCallPerfBenchmark : public ANGLERenderTest,
DrawCallPerfBenchmark::DrawCallPerfBenchmark() : ANGLERenderTest("DrawCallPerf", GetParam())
{
mRunTimeSeconds = GetParam().runTimeSeconds;
}
void DrawCallPerfBenchmark::initializeBenchmark()

2
src/tests/perf_tests/DrawElementsPerf.cpp
Просмотреть файл

@ -90,8 +90,6 @@ class DrawElementsPerfBenchmark : public ANGLERenderTest,
DrawElementsPerfBenchmark::DrawElementsPerfBenchmark()
: ANGLERenderTest("DrawElementsPerf", GetParam())
{
mRunTimeSeconds = GetParam().runTimeSeconds;
if (GetParam().type == GL_UNSIGNED_INT)
{
addExtensionPrerequisite("GL_OES_element_index_uint");

1
src/tests/perf_tests/IndexConversionPerf.cpp
Просмотреть файл

@ -72,7 +72,6 @@ IndexConversionPerfTest::IndexConversionPerfTest()
mVertexBuffer(0),
mIndexBuffer(0)
{
mRunTimeSeconds = 3.0;
}
void IndexConversionPerfTest::initializeBenchmark()

1
src/tests/perf_tests/InstancingPerf.cpp
Просмотреть файл

@ -103,7 +103,6 @@ class InstancingPerfBenchmark : public ANGLERenderTest,
InstancingPerfBenchmark::InstancingPerfBenchmark()
: ANGLERenderTest("InstancingPerf", GetParam()), mProgram(0), mNumPoints(75000)
{
mRunTimeSeconds = GetParam().runTimeSeconds;
}
void InstancingPerfBenchmark::initializeBenchmark()

5
src/tests/perf_tests/VulkanCommandBufferPerf.cpp
Просмотреть файл

@ -79,6 +79,11 @@ VulkanCommandBufferPerfTest::VulkanCommandBufferPerfTest()
mCBImplementation = GetParam().CBImplementation;
mFrames = GetParam().frames;
mBuffers = GetParam().buffers;
// This test appears to be flaky on multiple platforms.
#if !defined(ANGLE_PLATFORM_ANDROID)
mSkipTest = true;
#endif // !defined(ANGLE_PLATFORM_ANDROID)
}
void VulkanCommandBufferPerfTest::SetUp()

40
src/tests/test_utils/third_party/vulkan_command_buffer_utils.cpp поставляемый
Просмотреть файл

@ -397,7 +397,7 @@ LRESULT CALLBACK WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
void init_window(struct sample_info &info)
{
WNDCLASSEX win_class;
WNDCLASSEXA win_class;
assert(info.width > 0);
assert(info.height > 0);
@ -418,7 +418,7 @@ void init_window(struct sample_info &info)
win_class.lpszClassName = info.name;
win_class.hIconSm = LoadIcon(NULL, IDI_WINLOGO);
// Register window class:
if (!RegisterClassEx(&win_class))
if (!RegisterClassExA(&win_class))
{
// It didn't work, so try to give a useful error:
printf("Unexpected error trying to start the application!\n");
@ -428,18 +428,18 @@ void init_window(struct sample_info &info)
// Create window with the registered class:
RECT wr = {0, 0, info.width, info.height};
AdjustWindowRect(&wr, WS_OVERLAPPEDWINDOW, FALSE);
info.window = CreateWindowEx(0,
info.name, // class name
info.name, // app name
WS_OVERLAPPEDWINDOW | // window style
WS_VISIBLE | WS_SYSMENU,
100, 100, // x/y coords
wr.right - wr.left, // width
wr.bottom - wr.top, // height
NULL, // handle to parent
NULL, // handle to menu
info.connection, // hInstance
NULL); // no extra parameters
info.window = CreateWindowExA(0,
info.name, // class name
info.name, // app name
WS_OVERLAPPEDWINDOW | // window style
WS_VISIBLE | WS_SYSMENU,
100, 100, // x/y coords
wr.right - wr.left, // width
wr.bottom - wr.top, // height
NULL, // handle to parent
NULL, // handle to menu
info.connection, // hInstance
NULL); // no extra parameters
if (!info.window)
{
// It didn't work, so try to give a useful error:
@ -454,6 +454,7 @@ void destroy_window(struct sample_info &info)
{
vkDestroySurfaceKHR(info.inst, info.surface, NULL);
DestroyWindow(info.window);
UnregisterClassA(info.name, GetModuleHandle(NULL));
}
#elif defined(__ANDROID__)
@ -846,7 +847,16 @@ void init_swap_chain(struct sample_info &info, VkImageUsageFlags usageFlags)
// The FIFO present mode is guaranteed by the spec to be supported
// Also note that current Android driver only supports FIFO
VkPresentModeKHR swapchainPresentMode = VK_PRESENT_MODE_IMMEDIATE_KHR;
VkPresentModeKHR swapchainPresentMode = VK_PRESENT_MODE_FIFO_KHR;
for (uint32_t presentModeIndex = 0; presentModeIndex < presentModeCount; ++presentModeIndex)
{
if (presentModes[presentModeIndex] == VK_PRESENT_MODE_IMMEDIATE_KHR)
{
swapchainPresentMode = VK_PRESENT_MODE_IMMEDIATE_KHR;
break;
}
}
// Determine the number of VkImage's to use in the swap chain.
// We need to acquire only 1 presentable image at at time.

6
util/windows/win32/Win32_system_utils.cpp
Просмотреть файл

@ -25,7 +25,11 @@ bool StabilizeCPUForBenchmarking()
{
return false;
}
if (SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST) == FALSE)
if (SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) == FALSE)
{
return false;
}
if (SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL) == FALSE)
{
return false;
}