From 77defeb04693452a4227da1c04e88cc72f690a64 Mon Sep 17 00:00:00 2001 From: Jamie Madill Date: Tue, 27 Oct 2020 18:31:53 -0400 Subject: [PATCH] Add the ability to re-trace existing traces. This new script runs the ANGLE trace tests with capture enabled to generate updated replay cpp files. This allows us to update our traces files to a new file format in one step. Trace metadata (currently only the default FBO format) is preserved between re-traces. Currently only desktop-based retracing is supported. This means a couple traces that require specific extensions don't run on all platforms. Bug: angleproject:5134 Change-Id: I7c923d89e33c18285ab36a7cee91f2fb735758eb Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2488130 Commit-Queue: Jamie Madill Reviewed-by: Courtney Goeltzenleuchter Reviewed-by: Cody Northrop --- .../restricted_traces.json | 2 +- src/libANGLE/FrameCapture.cpp | 4 +- src/tests/perf_tests/ANGLEPerfTest.cpp | 4 + src/tests/perf_tests/ANGLEPerfTestArgs.cpp | 5 + src/tests/perf_tests/ANGLEPerfTestArgs.h | 1 + src/tests/perf_tests/TracePerfTest.cpp | 9 + .../gen_restricted_traces.py | 6 +- .../retrace_restricted_traces.py | 170 ++++++++++++++++++ 8 files changed, 195 insertions(+), 6 deletions(-) create mode 100755 src/tests/restricted_traces/retrace_restricted_traces.py diff --git a/scripts/code_generation_hashes/restricted_traces.json b/scripts/code_generation_hashes/restricted_traces.json index d249645b9..b2eb70999 100644 --- a/scripts/code_generation_hashes/restricted_traces.json +++ b/scripts/code_generation_hashes/restricted_traces.json @@ -20,7 +20,7 @@ "src/tests/restricted_traces/free_fire.tar.gz.sha1": "a95efee5e5c6d85bac2d8c2ab09dc34c", "src/tests/restricted_traces/gen_restricted_traces.py": - "a520ef50175b1b9a20383a884e293b54", + "5c95022fc335f608f8bc9daf305c8bf3", "src/tests/restricted_traces/kartrider_rush.tar.gz.sha1": "64079b7406be62a9b04062b8790f3313", "src/tests/restricted_traces/manhattan_10.tar.gz.sha1": diff --git a/src/libANGLE/FrameCapture.cpp b/src/libANGLE/FrameCapture.cpp index 5fb7d6945..339285fcf 100644 --- a/src/libANGLE/FrameCapture.cpp +++ b/src/libANGLE/FrameCapture.cpp @@ -199,7 +199,7 @@ std::string GetCaptureTrigger() #if defined(ANGLE_PLATFORM_ANDROID) return AndroidGetEnvFromProp(kAndroidCaptureTrigger); #else - return std::string(); + return GetEnvironmentVar(kCaptureTriggerVarName); #endif // defined(ANGLE_PLATFORM_ANDROID) } @@ -4352,7 +4352,7 @@ void FrameCapture::checkForCaptureTrigger() // Use the original trigger value as the frame count mCaptureEndFrame = mCaptureStartFrame + (mCaptureTrigger - 1); - INFO() << "Capture triggered at frame " << mCaptureStartFrame << " for " << mCaptureTrigger + INFO() << "Capture triggered after frame " << mFrameIndex << " for " << mCaptureTrigger << " frames"; // Stop polling diff --git a/src/tests/perf_tests/ANGLEPerfTest.cpp b/src/tests/perf_tests/ANGLEPerfTest.cpp index 52b8aaccf..a3bff3c83 100644 --- a/src/tests/perf_tests/ANGLEPerfTest.cpp +++ b/src/tests/perf_tests/ANGLEPerfTest.cpp @@ -210,6 +210,10 @@ void ANGLEPerfTest::run() } uint32_t numTrials = OneFrame() ? 1 : gTestTrials; + if (gVerboseLogging) + { + printf("Test Trials: %d\n", static_cast(numTrials)); + } for (uint32_t trial = 0; trial < numTrials; ++trial) { diff --git a/src/tests/perf_tests/ANGLEPerfTestArgs.cpp b/src/tests/perf_tests/ANGLEPerfTestArgs.cpp index 158c9d9f4..bd4e1fb98 100644 --- a/src/tests/perf_tests/ANGLEPerfTestArgs.cpp +++ b/src/tests/perf_tests/ANGLEPerfTestArgs.cpp @@ -23,6 +23,7 @@ double gTestTimeSeconds = 1.0; int gTestTrials = 3; bool gNoFinish = false; bool gEnableAllTraceTests = false; +bool gStartTraceAfterSetup = false; // Default to three warmup loops. There's no science to this. More than two loops was experimentally // helpful on a Windows NVIDIA setup when testing with Vulkan and native trace tests. @@ -121,6 +122,10 @@ void ANGLEProcessPerfTestArgs(int *argc, char **argv) { gEnableAllTraceTests = true; } + else if (strcmp("--start-trace-after-setup", argv[argIndex]) == 0) + { + gStartTraceAfterSetup = true; + } else { argv[argcOutCount++] = argv[argIndex]; diff --git a/src/tests/perf_tests/ANGLEPerfTestArgs.h b/src/tests/perf_tests/ANGLEPerfTestArgs.h index ba7d3b307..cd2bcdcb6 100644 --- a/src/tests/perf_tests/ANGLEPerfTestArgs.h +++ b/src/tests/perf_tests/ANGLEPerfTestArgs.h @@ -25,6 +25,7 @@ extern double gTestTimeSeconds; extern int gTestTrials; extern bool gNoFinish; extern bool gEnableAllTraceTests; +extern bool gStartTraceAfterSetup; inline bool OneFrame() { diff --git a/src/tests/perf_tests/TracePerfTest.cpp b/src/tests/perf_tests/TracePerfTest.cpp index ee48d7dca..a24886d61 100644 --- a/src/tests/perf_tests/TracePerfTest.cpp +++ b/src/tests/perf_tests/TracePerfTest.cpp @@ -343,12 +343,21 @@ void TracePerfTest::initializeBenchmark() // Potentially slow. Can load a lot of resources. SetupReplay(params.testID); + glFinish(); ASSERT_TRUE(mEndFrame > mStartFrame); getWindow()->ignoreSizeEvents(); getWindow()->setVisible(true); + + // If we're re-tracing, trigger capture start after setup. This ensures the Setup function gets + // recaptured into another Setup function and not merged with the first frame. + if (angle::gStartTraceAfterSetup) + { + angle::SetEnvironmentVar("ANGLE_CAPTURE_TRIGGER", "0"); + getGLWindow()->swap(); + } } #undef TRACE_TEST_CASE diff --git a/src/tests/restricted_traces/gen_restricted_traces.py b/src/tests/restricted_traces/gen_restricted_traces.py index 90dd09f21..7f0b15cb7 100644 --- a/src/tests/restricted_traces/gen_restricted_traces.py +++ b/src/tests/restricted_traces/gen_restricted_traces.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/python # # Copyright 2020 The ANGLE Project Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -305,9 +305,9 @@ def main(): outputs = [gni_file, header_file, source_file, '.gitignore'] if sys.argv[1] == 'inputs': - print ','.join(inputs) + print(','.join(inputs)) elif sys.argv[1] == 'outputs': - print ','.join(outputs) + print(','.join(outputs)) else: print('Invalid script parameters.') return 1 diff --git a/src/tests/restricted_traces/retrace_restricted_traces.py b/src/tests/restricted_traces/retrace_restricted_traces.py new file mode 100755 index 000000000..281bb8f40 --- /dev/null +++ b/src/tests/restricted_traces/retrace_restricted_traces.py @@ -0,0 +1,170 @@ +#! /usr/bin/env python3 +# +# Copyright 2020 The ANGLE Project Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +''' +Script that re-captures the traces in the restricted trace folder. We can +use this to update traces without needing to re-run the app on a device. +''' + +import argparse +import fnmatch +import json +import logging +import os +import re +import subprocess +import sys + +from gen_restricted_traces import get_context as get_context + +DEFAULT_TEST_SUITE = 'angle_perftests' +DEFAULT_TEST_JSON = 'restricted_traces.json' +DEFAULT_LOG_LEVEL = 'info' + +# We preserve select metadata in the trace header that can't be re-captured properly. +# Currently this is just the set of default framebuffer surface config bits. +METADATA_KEYWORDS = ['kDefaultFramebuffer'] + + +def src_trace_path(trace): + script_dir = os.path.dirname(sys.argv[0]) + return os.path.join(script_dir, trace) + + +def context_header(trace, trace_path): + context_id = get_context(trace_path) + header = '%s_capture_context%s.h' % (trace, context_id) + return os.path.join(trace_path, header) + + +def get_num_frames(trace): + + trace_path = src_trace_path(trace) + + lo = 99999999 + hi = 0 + + for file in os.listdir(trace_path): + match = re.match(r'.+_capture_context\d_frame(\d+)\.cpp', file) + if match: + frame = int(match.group(1)) + if frame < lo: + lo = frame + if frame > hi: + hi = frame + + return hi - lo + 1 + + +def get_trace_metadata(trace): + trace_path = src_trace_path(trace) + header_file = context_header(trace, trace_path) + metadata = [] + with open(header_file, 'rt') as f: + for line in f.readlines(): + for keyword in METADATA_KEYWORDS: + if keyword in line: + metadata += [line] + return metadata + + +def replace_metadata(header_file, metadata): + lines = [] + replaced = False + with open(header_file, 'rt') as f: + for line in f.readlines(): + found_keyword = False + for keyword in METADATA_KEYWORDS: + if keyword in line: + found_keyword = True + break + + if found_keyword: + if not replaced: + replaced = True + lines += metadata + else: + lines += [line] + + with open(header_file, 'wt') as f: + f.writelines(lines) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('gn_path', help='GN build path') + parser.add_argument('out_path', help='Output directory') + parser.add_argument('-f', '--filter', help='Trace filter. Defaults to all.', default='*') + parser.add_argument('-l', '--log', help='Logging level.', default=DEFAULT_LOG_LEVEL) + args, extra_flags = parser.parse_known_args() + + logging.basicConfig(level=args.log.upper()) + + script_dir = os.path.dirname(sys.argv[0]) + + # Load trace names + with open(os.path.join(script_dir, DEFAULT_TEST_JSON)) as f: + traces = json.loads(f.read()) + + traces = traces['traces'] + + binary = os.path.join(args.gn_path, DEFAULT_TEST_SUITE) + if os.name == 'nt': + binary += '.exe' + + failures = [] + + for trace in fnmatch.filter(traces, args.filter): + logging.debug('Tracing %s' % trace) + + trace_path = os.path.abspath(os.path.join(args.out_path, trace)) + if not os.path.isdir(trace_path): + os.makedirs(trace_path) + + num_frames = get_num_frames(trace) + metadata = get_trace_metadata(trace) + + logging.debug('Read metadata: %s' % str(metadata)) + + env = os.environ.copy() + env['ANGLE_CAPTURE_OUT_DIR'] = trace_path + env['ANGLE_CAPTURE_LABEL'] = trace + env['ANGLE_CAPTURE_TRIGGER'] = str(num_frames) + + trace_filter = '--gtest_filter=TracePerfTest.Run/vulkan_swiftshader_%s' % trace + run_args = [ + binary, + trace_filter, + '--no-warmup', + '--trials', + '1', + '--start-trace-after-setup', + '--steps', + str(num_frames), + '--enable-all-trace-tests', + ] + + print('Capturing %s (%d frames)...' % (trace, num_frames)) + logging.debug('Running %s with capture environment' % ' '.join(run_args)) + subprocess.check_call(run_args, env=env) + + header_file = context_header(trace, trace_path) + + if not os.path.exists(header_file): + logging.warning('There was a problem tracing %s, could not find header file' % trace) + failures += [trace] + else: + replace_metadata(header_file, metadata) + + if failures: + print('The following traces failed to re-trace:\n') + print('\n'.join([' ' + trace for trace in failures])) + + return 0 + + +if __name__ == '__main__': + sys.exit(main())