AvaloniaUI
/
angle
зеркало из https://github.com/AvaloniaUI/angle.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Re-land "Add new test runner harness." 

Re-land changes:

 * Unit test is suppressed in ASAN
 * --deqp-case is fixed
 * Debug layer errors should correctly work with failure expectations

Original message:

The ANGLE test harness is a harness around GoogleTest that provides
functionality similar to the Chromium test harness. It supports:

 * splitting a test set into shards
 * catching and reporting crashes and timeouts
 * outputting to the Chromium JSON test results format
 * multi-process execution

Unit tests are added in test_utils_unittest.cpp.

Bug: angleproject:3162
Change-Id: I841f2b5dfe51f7f44dac68324bdf6afd418b8bfb
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1948240
Reviewed-by: Jonah Ryan-Davis <jonahr@google.com>
Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
Commit-Queue: Jamie Madill <jmadill@chromium.org>
This commit is contained in:
Jamie Madill 2019-12-04 13:18:36 -05:00 коммит произвёл Commit Bot
Родитель af52f9c692
Коммит e20560faf1
22 изменённых файлов: 1733 добавлений и 148 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

70
gni/angle.gni
Просмотреть файл

@ -187,11 +187,14 @@ set_defaults("angle_test") {
public_deps = []
sources = []
data = []
defines = []
main = ""
suppressed_configs = angle_remove_configs
# TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162
if (build_with_chromium) {
# By default use the Chromium harness in Chromium. Can be overriden in a target.
standalone_harness = !build_with_chromium
if (!standalone_harness) {
suppressed_configs -= [ "//build/config/compiler:default_include_dirs" ]
}
@ -284,18 +287,6 @@ template("angle_static_library") {
}
template("angle_test") {
_googletest_deps = [
"//testing/gmock",
"//testing/gtest",
"//third_party/googletest:gmock",
"//third_party/googletest:gtest",
]
# TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162
if (build_with_chromium) {
_googletest_deps += [ "//base/test:test_support" ]
}
test(target_name) {
forward_variables_from(invoker,
"*",
@ -309,9 +300,10 @@ template("angle_test") {
forward_variables_from(invoker, [ "visibility" ])
configs += invoker.configs
configs -= invoker.suppressed_configs
configs -= [ angle_root + ":constructor_and_destructor_warnings" ]
configs -= [ angle_root + ":extra_warnings" ]
configs -= invoker.suppressed_configs + [
"$angle_root:constructor_and_destructor_warnings",
"$angle_root:extra_warnings",
]
if (is_linux && !is_component_build) {
# Set rpath to find shared libs in a non-component build.
@ -319,21 +311,39 @@ template("angle_test") {
}
if (is_android) {
configs += [ angle_root + ":build_id_config" ]
if (build_with_chromium) {
configs += [ "$angle_root:build_id_config" ]
}
deps += [
"$angle_root:angle_common",
"$angle_root:includes",
"$angle_root/third_party/rapidjson:rapidjson",
"//testing/gmock",
"//testing/gtest",
"//third_party/googletest:gmock",
"//third_party/googletest:gtest",
]
sources += [
"$angle_root/src/tests/test_utils/runner/TestSuite.cpp",
"$angle_root/src/tests/test_utils/runner/TestSuite.h",
]
# To use the Chromium test infrastructure we must currently use the //base test launcher.
# Eventually we could switch to using standalone testing. See http://crbug.com/837741
if (standalone_harness) {
if (invoker.main != "") {
sources += [ "${invoker.main}.cpp" ]
}
} else {
if (invoker.main != "") {
sources += [ "//gpu/${invoker.main}.cc" ]
}
deps += [ "//base/test:test_support" ]
if (is_android) {
configs -= [ "//build/config/android:hide_all_but_jni" ]
}
}
deps += _googletest_deps + [
"$angle_root:angle_common",
"$angle_root:includes",
]
if (build_with_chromium) {
sources += [ "//gpu/${invoker.main}.cc" ]
} else {
sources += [ "${invoker.main}.cpp" ]
}
}
}

7
src/common/platform.h
Просмотреть файл

@ -123,4 +123,11 @@
# endif
#endif
// Define ANGLE_WITH_ASAN macro.
#if defined(__has_feature)
# if __has_feature(address_sanitizer)
# define ANGLE_WITH_ASAN 1
# endif
#endif
#endif // COMMON_PLATFORM_H_

2
src/common/system_utils.h
Просмотреть файл

@ -17,6 +17,8 @@ namespace angle
std::string GetExecutablePath();
std::string GetExecutableDirectory();
const char *GetSharedLibraryExtension();
const char *GetExecutableExtension();
char GetPathSeparator();
Optional<std::string> GetCWD();
bool SetCWD(const char *dirName);
bool SetEnvironmentVar(const char *variableName, const char *value);

10
src/common/system_utils_posix.cpp
Просмотреть файл

@ -125,4 +125,14 @@ void BreakDebugger()
// See https://cs.chromium.org/chromium/src/base/debug/debugger_posix.cc
abort();
}
const char *GetExecutableExtension()
{
return "";
}
char GetPathSeparator()
{
return '/';
}
} // namespace angle

9
src/common/system_utils_win.cpp
Просмотреть файл

@ -166,4 +166,13 @@ void BreakDebugger()
__debugbreak();
}
const char *GetExecutableExtension()
{
return ".exe";
}
char GetPathSeparator()
{
return '\\';
}
} // namespace angle

14
src/tests/BUILD.gn
Просмотреть файл

@ -13,11 +13,19 @@ declare_args() {
build_angle_gles1_conform_tests = false
}
angle_executable("test_utils_unittest_helper") {
sources = test_utils_unittest_helper_sources
angle_test("test_utils_unittest_helper") {
standalone_harness = true
sources = [
"../../util/test_utils_unittest_helper.cpp",
"../../util/test_utils_unittest_helper.h",
"test_utils/angle_test_instantiate.h",
"test_utils/runner/TestSuite_unittest.cpp",
]
deps = [
"${angle_root}:angle_common",
"$angle_root:angle_common",
"$angle_root:angle_util",
]
}

4
src/tests/angle_deqp_tests_main.cpp
Просмотреть файл

@ -8,6 +8,8 @@
#include <gtest/gtest.h>
#include "test_utils/runner/TestSuite.h"
// Defined in angle_deqp_gtest.cpp. Declared here so we don't need to make a header that we import
// in Chromium.
namespace angle
@ -18,7 +20,7 @@ void InitTestHarness(int *argc, char **argv);
int main(int argc, char **argv)
{
angle::InitTestHarness(&argc, argv);
testing::InitGoogleTest(&argc, argv);
angle::TestSuite testSuite(&argc, argv);
int rt = RUN_ALL_TESTS();
return rt;
}

3
src/tests/angle_end2end_tests_main.cpp
Просмотреть файл

@ -5,13 +5,14 @@
//
#include "gtest/gtest.h"
#include "test_utils/runner/TestSuite.h"
void ANGLEProcessTestArgs(int *argc, char *argv[]);
int main(int argc, char **argv)
{
angle::TestSuite testSuite(&argc, argv);
ANGLEProcessTestArgs(&argc, argv);
testing::InitGoogleTest(&argc, argv);
int rt = RUN_ALL_TESTS();
return rt;
}

5
src/tests/angle_perftests_main.cpp
Просмотреть файл

@ -9,13 +9,14 @@
#include <gtest/gtest.h>
#include "test_utils/runner/TestSuite.h"
void ANGLEProcessPerfTestArgs(int *argc, char **argv);
int main(int argc, char **argv)
{
angle::TestSuite testSuite(&argc, argv);
ANGLEProcessPerfTestArgs(&argc, argv);
testing::InitGoogleTest(&argc, argv);
testing::AddGlobalTestEnvironment(new testing::Environment());
int rt = RUN_ALL_TESTS();
return rt;
}

6
src/tests/angle_unittest_main.cpp
Просмотреть файл

@ -6,6 +6,7 @@
#include "GLSLANG/ShaderLang.h"
#include "gtest/gtest.h"
#include "test_utils/runner/TestSuite.h"
class CompilerTestEnvironment : public testing::Environment
{
@ -29,8 +30,7 @@ class CompilerTestEnvironment : public testing::Environment
int main(int argc, char **argv)
{
testing::InitGoogleTest(&argc, argv);
angle::TestSuite testSuite(&argc, argv);
testing::AddGlobalTestEnvironment(new CompilerTestEnvironment());
int rt = RUN_ALL_TESTS();
return rt;
return testSuite.run();
}

181
src/tests/angle_unittests.gni
Просмотреть файл

@ -44,91 +44,92 @@ angle_unittests_sources = [
"../libANGLE/renderer/ImageImpl_mock.h",
"../libANGLE/renderer/TextureImpl_mock.h",
"../libANGLE/renderer/TransformFeedbackImpl_mock.h",
"../tests/angle_unittests_utils.h",
"../tests/compiler_tests/API_test.cpp",
"../tests/compiler_tests/AppendixALimitations_test.cpp",
"../tests/compiler_tests/ARB_texture_rectangle_test.cpp",
"../tests/compiler_tests/AtomicCounter_test.cpp",
"../tests/compiler_tests/BufferVariables_test.cpp",
"../tests/compiler_tests/CollectVariables_test.cpp",
"../tests/compiler_tests/ConstantFolding_test.cpp",
"../tests/compiler_tests/ConstantFoldingNaN_test.cpp",
"../tests/compiler_tests/ConstantFoldingOverflow_test.cpp",
"../tests/compiler_tests/ConstructCompiler_test.cpp",
"../tests/compiler_tests/DebugShaderPrecision_test.cpp",
"../tests/compiler_tests/EmulateGLBaseVertexBaseInstance_test.cpp",
"../tests/compiler_tests/EmulateGLDrawID_test.cpp",
"../tests/compiler_tests/EmulateGLFragColorBroadcast_test.cpp",
"../tests/compiler_tests/ExpressionLimit_test.cpp",
"../tests/compiler_tests/EXT_YUV_target_test.cpp",
"../tests/compiler_tests/EXT_blend_func_extended_test.cpp",
"../tests/compiler_tests/EXT_frag_depth_test.cpp",
"../tests/compiler_tests/EXT_shader_texture_lod_test.cpp",
"../tests/compiler_tests/ExtensionDirective_test.cpp",
"../tests/compiler_tests/FloatLex_test.cpp",
"../tests/compiler_tests/FragDepth_test.cpp",
"../tests/compiler_tests/GLSLCompatibilityOutput_test.cpp",
"../tests/compiler_tests/GlFragDataNotModified_test.cpp",
"../tests/compiler_tests/GeometryShader_test.cpp",
"../tests/compiler_tests/ImmutableString_test.cpp",
"../tests/compiler_tests/InitOutputVariables_test.cpp",
"../tests/compiler_tests/IntermNode_test.cpp",
"../tests/compiler_tests/NV_draw_buffers_test.cpp",
"../tests/compiler_tests/OES_standard_derivatives_test.cpp",
"../tests/compiler_tests/Pack_Unpack_test.cpp",
"../tests/compiler_tests/PruneEmptyCases_test.cpp",
"../tests/compiler_tests/PruneEmptyDeclarations_test.cpp",
"../tests/compiler_tests/PrunePureLiteralStatements_test.cpp",
"../tests/compiler_tests/PruneUnusedFunctions_test.cpp",
"../tests/compiler_tests/QualificationOrderESSL31_test.cpp",
"../tests/compiler_tests/QualificationOrder_test.cpp",
"../tests/compiler_tests/RecordConstantPrecision_test.cpp",
"../tests/compiler_tests/RegenerateStructNames_test.cpp",
"../tests/compiler_tests/RemovePow_test.cpp",
"../tests/compiler_tests/RemoveUnreferencedVariables_test.cpp",
"../tests/compiler_tests/RewriteDoWhile_test.cpp",
"../tests/compiler_tests/SamplerMultisample_test.cpp",
"../tests/compiler_tests/ScalarizeVecAndMatConstructorArgs_test.cpp",
"../tests/compiler_tests/ShaderImage_test.cpp",
"../tests/compiler_tests/ShaderValidation_test.cpp",
"../tests/compiler_tests/ShaderVariable_test.cpp",
"../tests/compiler_tests/ShCompile_test.cpp",
"../tests/compiler_tests/TextureFunction_test.cpp",
"../tests/compiler_tests/Type_test.cpp",
"../tests/compiler_tests/TypeTracking_test.cpp",
"../tests/compiler_tests/UnfoldShortCircuitAST_test.cpp",
"../tests/compiler_tests/VariablePacker_test.cpp",
"../tests/compiler_tests/VectorizeVectorScalarArithmetic_test.cpp",
"../tests/compiler_tests/OVR_multiview_test.cpp",
"../tests/compiler_tests/OVR_multiview2_test.cpp",
"../tests/compiler_tests/WorkGroupSize_test.cpp",
"../tests/test_expectations/GPUTestExpectationsParser_unittest.cpp",
"../tests/preprocessor_tests/char_test.cpp",
"../tests/preprocessor_tests/comment_test.cpp",
"../tests/preprocessor_tests/define_test.cpp",
"../tests/preprocessor_tests/error_test.cpp",
"../tests/preprocessor_tests/extension_test.cpp",
"../tests/preprocessor_tests/identifier_test.cpp",
"../tests/preprocessor_tests/if_test.cpp",
"../tests/preprocessor_tests/input_test.cpp",
"../tests/preprocessor_tests/location_test.cpp",
"../tests/preprocessor_tests/MockDiagnostics.h",
"../tests/preprocessor_tests/MockDirectiveHandler.h",
"../tests/preprocessor_tests/number_test.cpp",
"../tests/preprocessor_tests/operator_test.cpp",
"../tests/preprocessor_tests/pragma_test.cpp",
"../tests/preprocessor_tests/PreprocessorTest.cpp",
"../tests/preprocessor_tests/PreprocessorTest.h",
"../tests/preprocessor_tests/space_test.cpp",
"../tests/preprocessor_tests/token_test.cpp",
"../tests/preprocessor_tests/version_test.cpp",
"../tests/test_utils/compiler_test.cpp",
"../tests/test_utils/compiler_test.h",
"../tests/test_utils/ConstantFoldingTest.h",
"../tests/test_utils/ConstantFoldingTest.cpp",
"../tests/test_utils/ShaderCompileTreeTest.h",
"../tests/test_utils/ShaderCompileTreeTest.cpp",
"../tests/test_utils/ShaderExtensionTest.h",
"angle_unittests_utils.h",
"compiler_tests/API_test.cpp",
"compiler_tests/AppendixALimitations_test.cpp",
"compiler_tests/ARB_texture_rectangle_test.cpp",
"compiler_tests/AtomicCounter_test.cpp",
"compiler_tests/BufferVariables_test.cpp",
"compiler_tests/CollectVariables_test.cpp",
"compiler_tests/ConstantFolding_test.cpp",
"compiler_tests/ConstantFoldingNaN_test.cpp",
"compiler_tests/ConstantFoldingOverflow_test.cpp",
"compiler_tests/ConstructCompiler_test.cpp",
"compiler_tests/DebugShaderPrecision_test.cpp",
"compiler_tests/EmulateGLBaseVertexBaseInstance_test.cpp",
"compiler_tests/EmulateGLDrawID_test.cpp",
"compiler_tests/EmulateGLFragColorBroadcast_test.cpp",
"compiler_tests/ExpressionLimit_test.cpp",
"compiler_tests/EXT_YUV_target_test.cpp",
"compiler_tests/EXT_blend_func_extended_test.cpp",
"compiler_tests/EXT_frag_depth_test.cpp",
"compiler_tests/EXT_shader_texture_lod_test.cpp",
"compiler_tests/ExtensionDirective_test.cpp",
"compiler_tests/FloatLex_test.cpp",
"compiler_tests/FragDepth_test.cpp",
"compiler_tests/GLSLCompatibilityOutput_test.cpp",
"compiler_tests/GlFragDataNotModified_test.cpp",
"compiler_tests/GeometryShader_test.cpp",
"compiler_tests/ImmutableString_test.cpp",
"compiler_tests/InitOutputVariables_test.cpp",
"compiler_tests/IntermNode_test.cpp",
"compiler_tests/NV_draw_buffers_test.cpp",
"compiler_tests/OES_standard_derivatives_test.cpp",
"compiler_tests/Pack_Unpack_test.cpp",
"compiler_tests/PruneEmptyCases_test.cpp",
"compiler_tests/PruneEmptyDeclarations_test.cpp",
"compiler_tests/PrunePureLiteralStatements_test.cpp",
"compiler_tests/PruneUnusedFunctions_test.cpp",
"compiler_tests/QualificationOrderESSL31_test.cpp",
"compiler_tests/QualificationOrder_test.cpp",
"compiler_tests/RecordConstantPrecision_test.cpp",
"compiler_tests/RegenerateStructNames_test.cpp",
"compiler_tests/RemovePow_test.cpp",
"compiler_tests/RemoveUnreferencedVariables_test.cpp",
"compiler_tests/RewriteDoWhile_test.cpp",
"compiler_tests/SamplerMultisample_test.cpp",
"compiler_tests/ScalarizeVecAndMatConstructorArgs_test.cpp",
"compiler_tests/ShaderImage_test.cpp",
"compiler_tests/ShaderValidation_test.cpp",
"compiler_tests/ShaderVariable_test.cpp",
"compiler_tests/ShCompile_test.cpp",
"compiler_tests/TextureFunction_test.cpp",
"compiler_tests/Type_test.cpp",
"compiler_tests/TypeTracking_test.cpp",
"compiler_tests/UnfoldShortCircuitAST_test.cpp",
"compiler_tests/VariablePacker_test.cpp",
"compiler_tests/VectorizeVectorScalarArithmetic_test.cpp",
"compiler_tests/OVR_multiview_test.cpp",
"compiler_tests/OVR_multiview2_test.cpp",
"compiler_tests/WorkGroupSize_test.cpp",
"test_expectations/GPUTestExpectationsParser_unittest.cpp",
"preprocessor_tests/char_test.cpp",
"preprocessor_tests/comment_test.cpp",
"preprocessor_tests/define_test.cpp",
"preprocessor_tests/error_test.cpp",
"preprocessor_tests/extension_test.cpp",
"preprocessor_tests/identifier_test.cpp",
"preprocessor_tests/if_test.cpp",
"preprocessor_tests/input_test.cpp",
"preprocessor_tests/location_test.cpp",
"preprocessor_tests/MockDiagnostics.h",
"preprocessor_tests/MockDirectiveHandler.h",
"preprocessor_tests/number_test.cpp",
"preprocessor_tests/operator_test.cpp",
"preprocessor_tests/pragma_test.cpp",
"preprocessor_tests/PreprocessorTest.cpp",
"preprocessor_tests/PreprocessorTest.h",
"preprocessor_tests/space_test.cpp",
"preprocessor_tests/token_test.cpp",
"preprocessor_tests/version_test.cpp",
"test_utils/angle_test_instantiate.h",
"test_utils/compiler_test.cpp",
"test_utils/compiler_test.h",
"test_utils/ConstantFoldingTest.h",
"test_utils/ConstantFoldingTest.cpp",
"test_utils/ShaderCompileTreeTest.h",
"test_utils/ShaderCompileTreeTest.cpp",
"test_utils/ShaderExtensionTest.h",
"../../util/test_utils_unittest.cpp",
"../../util/test_utils_unittest_helper.h",
]
@ -139,11 +140,6 @@ angle_unittests_hlsl_sources = [
"../tests/compiler_tests/UnrollFlatten_test.cpp",
]
test_utils_unittest_helper_sources = [
"../../util/test_utils_unittest_helper.cpp",
"../../util/test_utils_unittest_helper.h",
]
if (is_android) {
angle_unittests_sources +=
[ "../tests/compiler_tests/ImmutableString_test_ESSL_autogen.cpp" ]
@ -151,3 +147,8 @@ if (is_android) {
angle_unittests_sources +=
[ "../tests/compiler_tests/ImmutableString_test_autogen.cpp" ]
}
if (!is_android && !is_fuchsia) {
angle_unittests_sources +=
[ "../tests/test_utils/runner/TestSuite_unittest.cpp" ]
}

3
src/tests/angle_white_box_tests_main.cpp
Просмотреть файл

@ -6,10 +6,11 @@
#include "gtest/gtest.h"
#include "test_utils/ANGLETest.h"
#include "test_utils/runner/TestSuite.h"
int main(int argc, char **argv)
{
testing::InitGoogleTest(&argc, argv);
angle::TestSuite testSuite(&argc, argv);
testing::AddGlobalTestEnvironment(new ANGLETestEnvironment());
int rt = RUN_ALL_TESTS();
return rt;

6
src/tests/test_utils/ANGLETest.cpp
Просмотреть файл

@ -58,9 +58,7 @@ void TestPlatform_logError(PlatformMethods *platform, const char *errorMessage)
GTEST_NONFATAL_FAILURE_(errorMessage);
// Print the stack and stop any crash handling to prevent duplicate reports.
PrintStackBacktrace();
TerminateCrashHandler();
}
void TestPlatform_logWarning(PlatformMethods *platform, const char *warningMessage)
@ -481,8 +479,6 @@ void ANGLETestBase::ANGLETestSetUp()
{
mSetUpCalled = true;
InitCrashHandler(nullptr);
gDefaultPlatformMethods.overrideWorkaroundsD3D = TestPlatform_overrideWorkaroundsD3D;
gDefaultPlatformMethods.overrideFeaturesVk = TestPlatform_overrideFeaturesVk;
gDefaultPlatformMethods.logError = TestPlatform_logError;
@ -616,8 +612,6 @@ void ANGLETestBase::ANGLETestTearDown()
mFixture->eglWindow->destroySurface();
}
TerminateCrashHandler();
// Check for quit message
Event myEvent;
while (mFixture->osWindow->popEvent(&myEvent))

10
src/tests/test_utils/ANGLETest.h
Просмотреть файл

@ -625,14 +625,4 @@ bool IsGLExtensionRequestable(const std::string &extName);
extern angle::PlatformMethods gDefaultPlatformMethods;
#define ANGLE_SKIP_TEST_IF(COND) \
do \
{ \
if (COND) \
{ \
std::cout << "Test skipped: " #COND "." << std::endl; \
return; \
} \
} while (0)
#endif // ANGLE_TESTS_ANGLE_TEST_H_

21
src/tests/test_utils/angle_test_instantiate.h
Просмотреть файл

@ -12,6 +12,8 @@
#include <gtest/gtest.h>
#include "common/platform.h"
namespace angle
{
struct SystemInfo;
@ -40,6 +42,15 @@ bool IsIntel();
bool IsAMD();
bool IsNVIDIA();
inline bool IsASan()
{
#if defined(ANGLE_WITH_ASAN)
return true;
#else
return false;
#endif // defined(ANGLE_WITH_ASAN)
}
bool IsPlatformAvailable(const PlatformParameters &param);
// This functions is used to filter which tests should be registered,
@ -208,4 +219,14 @@ extern std::string gSelectedConfig;
extern bool gSeparateProcessPerConfig;
} // namespace angle
#define ANGLE_SKIP_TEST_IF(COND) \
do \
{ \
if (COND) \
{ \
std::cout << "Test skipped: " #COND "." << std::endl; \
return; \
} \
} while (0)
#endif // ANGLE_TEST_INSTANTIATE_H_

47
src/tests/test_utils/runner/README.md Normal file
Просмотреть файл

@ -0,0 +1,47 @@
# ANGLE Test Harness
The ANGLE test harness is a harness around GoogleTest that provides functionality similar to the
[Chromium test harness][BaseTest]. It features:
* splitting a test set into shards
* catching and reporting crashes and timeouts
* outputting to the Chromium [JSON test results format][JSONFormat]
* multi-process execution
## Command-Line Arguments
The ANGLE test harness accepts all standard GoogleTest arguments. The harness also accepts the
following additional command-line arguments:
* `--shard-count` and `--shard-index` control the test sharding
* `--bot-mode` enables multi-process execution and test batching
* `--batch-size` limits the number of tests to run in each batch
* `--batch-timeout` limits the amount of time spent in each batch
* `--max-processes` limits the number of simuntaneous processes
* `--test-timeout` limits the amount of time spent in each test
* `--results-file` specifies a location for the JSON test result output
* `--results-directory` specifies a directory to write test results to
* `--filter-file` allows passing a larget `gtest_filter` via a file
## Implementation Notes
* The test harness only requires `angle_common` and `angle_util`.
* It does not depend on any Chromium browser code. This allows us to compile on other non-Clang platforms.
* It uses rapidjson to read and write JSON files.
* Timeouts are detected via a watchdog thread.
* Crashes are handled via ANGLE's test crash handling code.
* Currently it does not entirely support Android or Fuchsia.
* Test execution is not currently deterministic in multi-process mode.
* We capture stdout to output test failure reasons.
See the source code for more details: [TestSuite.h](TestSuite.h) and [TestSuite.cpp](TestSuite.cpp).
## Potential Areas of Improvement
* Deterministic test execution.
* Using sockets to communicate with test children. Similar to dEQP's test harness.
* Closer integration with ANGLE's test expectations and system config libraries.
* Supporting a GoogleTest-free integration.
[BaseTest]: https://chromium.googlesource.com/chromium/src/+/refs/heads/master/base/test/
[JSONFormat]: https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md

1187
src/tests/test_utils/runner/TestSuite.cpp Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

153
src/tests/test_utils/runner/TestSuite.h Normal file
Просмотреть файл

@ -0,0 +1,153 @@
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite:
// Basic implementation of a test harness in ANGLE.
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include "util/test_utils.h"
namespace angle
{
struct TestIdentifier
{
TestIdentifier();
TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn);
TestIdentifier(const TestIdentifier &other);
~TestIdentifier();
TestIdentifier &operator=(const TestIdentifier &other);
static bool ParseFromString(const std::string &str, TestIdentifier *idOut);
bool valid() const { return !testName.empty(); }
void sprintfName(char *outBuffer) const;
std::string testSuiteName;
std::string testName;
};
inline bool operator<(const TestIdentifier &a, const TestIdentifier &b)
{
return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName);
}
inline bool operator==(const TestIdentifier &a, const TestIdentifier &b)
{
return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName);
}
inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id)
{
return os << id.testSuiteName << "." << id.testName;
}
enum class TestResultType
{
Crash,
Fail,
Skip,
Pass,
Timeout,
Unknown,
};
const char *TestResultTypeToString(TestResultType type);
struct TestResult
{
TestResultType type = TestResultType::Skip;
double elapsedTimeSeconds = 0.0;
};
inline bool operator==(const TestResult &a, const TestResult &b)
{
return a.type == b.type;
}
inline std::ostream &operator<<(std::ostream &os, const TestResult &result)
{
return os << TestResultTypeToString(result.type);
}
struct TestResults
{
TestResults();
~TestResults();
std::map<TestIdentifier, TestResult> results;
std::mutex currentTestMutex;
TestIdentifier currentTest;
Timer currentTestTimer;
bool allDone = false;
};
struct FileLine
{
const char *file;
int line;
};
struct ProcessInfo : angle::NonCopyable
{
ProcessInfo();
~ProcessInfo();
ProcessInfo(ProcessInfo &&other);
ProcessInfo &operator=(ProcessInfo &&rhs);
ProcessHandle process;
std::vector<TestIdentifier> testsInBatch;
std::string resultsFileName;
std::string filterFileName;
std::string commandLine;
};
class TestSuite
{
public:
TestSuite(int *argc, char **argv);
~TestSuite();
int run();
void onCrashOrTimeout(TestResultType crashOrTimeout);
private:
bool parseSingleArg(const char *argument);
bool launchChildTestProcess(const std::vector<TestIdentifier> &testsInBatch);
bool finishProcess(ProcessInfo *processInfo);
int printFailuresAndReturnCount() const;
void startWatchdog();
std::string mTestExecutableName;
std::string mTestSuiteName;
std::vector<TestIdentifier> mTestQueue;
std::string mFilterString;
std::string mFilterFile;
std::string mResultsDirectory;
std::string mResultsFile;
int mShardCount;
int mShardIndex;
angle::CrashCallback mCrashCallback;
TestResults mTestResults;
bool mBotMode;
int mBatchSize;
int mCurrentResultCount;
int mTotalResultCount;
int mMaxProcesses;
int mTestTimeout;
int mBatchTimeout;
std::vector<std::string> mGoogleTestCommandLineArgs;
std::map<TestIdentifier, FileLine> mTestFileLines;
std::vector<ProcessInfo> mCurrentProcesses;
std::thread mWatchdogThread;
};
bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);
} // namespace angle

115
src/tests/test_utils/runner/TestSuite_unittest.cpp Normal file
Просмотреть файл

@ -0,0 +1,115 @@
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// TestSuite_unittest.cpp: Unit tests for ANGLE's test harness.
//
#include <gtest/gtest.h>
#include "../angle_test_instantiate.h"
#include "TestSuite.h"
#include "common/debug.h"
#include "common/system_utils.h"
#include "util/test_utils.h"
#include "util/test_utils_unittest_helper.h"
#include <rapidjson/document.h>
using namespace angle;
namespace js = rapidjson;
namespace
{
constexpr char kTestHelperExecutable[] = "test_utils_unittest_helper";
class TestSuiteTest : public testing::Test
{
protected:
void TearDown() override
{
if (!mTempFileName.empty())
{
angle::DeleteFile(mTempFileName.c_str());
}
}
std::string mTempFileName;
};
// Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions.
// Verifies that Pass, Fail, Crash and Timeout are all handled correctly.
TEST_F(TestSuiteTest, RunMockTests)
{
// Fails in ASan when trying to communicate with child processes. http://crbug.com/1030192
ANGLE_SKIP_TEST_IF(IsASan());
std::string executablePath = GetExecutableDirectory();
EXPECT_NE(executablePath, "");
executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension();
constexpr uint32_t kMaxTempDirLen = 100;
char tempFileName[kMaxTempDirLen * 2];
ASSERT_TRUE(GetTempDir(tempFileName, kMaxTempDirLen));
std::stringstream tempFNameStream;
tempFNameStream << tempFileName << "/test_temp_" << rand() << ".json";
mTempFileName = tempFNameStream.str();
std::string resultsFileName = "--results-file=" + mTempFileName;
std::vector<const char *> args = {executablePath.c_str(),
kRunTestSuite,
"--gtest_filter=MockTestSuiteTest.DISABLED_*",
"--gtest_also_run_disabled_tests",
"--bot-mode",
"--test-timeout=10",
resultsFileName.c_str()};
ProcessHandle process(args, true, true);
EXPECT_TRUE(process->started());
EXPECT_TRUE(process->finish());
EXPECT_TRUE(process->finished());
EXPECT_EQ(process->getStderr(), "");
TestResults actual;
ASSERT_TRUE(GetTestResultsFromFile(mTempFileName.c_str(), &actual));
EXPECT_TRUE(DeleteFile(mTempFileName.c_str()));
mTempFileName.clear();
std::map<TestIdentifier, TestResult> expectedResults = {
{{"MockTestSuiteTest", "DISABLED_Pass"}, {TestResultType::Pass, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Fail"}, {TestResultType::Fail, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Timeout"}, {TestResultType::Timeout, 0.0}},
{{"MockTestSuiteTest", "DISABLED_Crash"}, {TestResultType::Crash, 0.0}},
};
EXPECT_EQ(expectedResults, actual.results);
}
// Normal passing test.
TEST(MockTestSuiteTest, DISABLED_Pass)
{
EXPECT_TRUE(true);
}
// Normal failing test.
TEST(MockTestSuiteTest, DISABLED_Fail)
{
EXPECT_TRUE(false);
}
// Trigger a test timeout.
TEST(MockTestSuiteTest, DISABLED_Timeout)
{
angle::Sleep(30000);
}
// Trigger a test crash.
TEST(MockTestSuiteTest, DISABLED_Crash)
{
ANGLE_CRASH();
}
} // namespace

17
util/posix/crash_handler_posix.cpp
Просмотреть файл

@ -35,7 +35,6 @@
namespace angle
{
#if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_FUCHSIA)
void PrintStackBacktrace()
@ -54,6 +53,10 @@ void TerminateCrashHandler()
}
#else
namespace
{
CrashCallback *gCrashHandlerCallback;
} // namespace
# if defined(ANGLE_PLATFORM_APPLE)
@ -85,6 +88,11 @@ void PrintStackBacktrace()
static void Handler(int sig)
{
if (gCrashHandlerCallback)
{
(*gCrashHandlerCallback)();
}
printf("\nSignal %d:\n", sig);
PrintStackBacktrace();
@ -127,6 +135,11 @@ void PrintStackBacktrace()
static void Handler(int sig)
{
if (gCrashHandlerCallback)
{
(*gCrashHandlerCallback)();
}
printf("\nSignal %d [%s]:\n", sig, strsignal(sig));
PrintStackBacktrace();
@ -142,6 +155,7 @@ static constexpr int kSignals[] = {
void InitCrashHandler(CrashCallback *callback)
{
gCrashHandlerCallback = callback;
for (int sig : kSignals)
{
// Register our signal handler unless something's already done so (e.g. catchsegv).
@ -155,6 +169,7 @@ void InitCrashHandler(CrashCallback *callback)
void TerminateCrashHandler()
{
gCrashHandlerCallback = nullptr;
for (int sig : kSignals)
{
void (*prev)(int) = signal(sig, SIG_DFL);

10
util/test_utils_unittest_helper.cpp
Просмотреть файл

@ -7,12 +7,22 @@
#include "test_utils_unittest_helper.h"
#include "../src/tests/test_utils/runner/TestSuite.h"
#include "common/system_utils.h"
#include <string.h>
int main(int argc, char **argv)
{
for (int argIndex = 1; argIndex < argc; ++argIndex)
{
if (strcmp(argv[argIndex], kRunTestSuite) == 0)
{
angle::TestSuite testSuite(&argc, argv);
return testSuite.run();
}
}
if (argc != 3 || strcmp(argv[1], kRunAppTestArg1) != 0 || strcmp(argv[2], kRunAppTestArg2) != 0)
{
fprintf(stderr, "Expected command line:\n%s %s %s\n", argv[0], kRunAppTestArg1,

1
util/test_utils_unittest_helper.h
Просмотреть файл

@ -16,6 +16,7 @@ constexpr char kRunAppTestStdout[] = "RunAppTest stdout test\n";
constexpr char kRunAppTestStderr[] = "RunAppTest stderr test\n .. that expands multiple lines\n";
constexpr char kRunAppTestArg1[] = "--expected-arg1";
constexpr char kRunAppTestArg2[] = "expected_arg2";
constexpr char kRunTestSuite[] = "--run-test-suite";
} // anonymous namespace
#endif // COMMON_SYSTEM_UTILS_UNITTEST_HELPER_H_