From 25e1ded8c2ae2acfb02fa7595ff0f780d0abf958 Mon Sep 17 00:00:00 2001 From: pmbrown1055 Date: Fri, 19 Apr 2019 15:33:42 -0700 Subject: [PATCH] Add per TEST_METHOD names to the output directory structure, so parallel unit test runs won't overwrite one another. (#219) --- Testing/WinMLRunnerTest/WinMLRunnerTest.cpp | 202 +++++++++++--------- 1 file changed, 113 insertions(+), 89 deletions(-) diff --git a/Testing/WinMLRunnerTest/WinMLRunnerTest.cpp b/Testing/WinMLRunnerTest/WinMLRunnerTest.cpp index 7f284e93..a86c3bf6 100644 --- a/Testing/WinMLRunnerTest/WinMLRunnerTest.cpp +++ b/Testing/WinMLRunnerTest/WinMLRunnerTest.cpp @@ -32,13 +32,16 @@ static HRESULT RunProc(LPWSTR commandLine) return HRESULT_FROM_WIN32(exitCode); } +// Use this test method definition, if the test needs access to the METHOD_NAME +#define TEST_METHOD_WITH_NAME(methodName) TEST_METHOD(methodName) { const std::wstring METHOD_NAME(L#methodName); + namespace WinMLRunnerTest { static const std::wstring CURRENT_PATH = FileHelper::GetModulePath(); static const std::wstring EXE_PATH = CURRENT_PATH + L"WinMLRunner.exe"; static const std::wstring INPUT_FOLDER_PATH = CURRENT_PATH + L"test_folder_input"; static const std::wstring OUTPUT_PATH = CURRENT_PATH + L"test_output.csv"; - static const std::wstring TENSOR_DATA_PATH = CURRENT_PATH + L"TestResult"; + static const std::wstring TENSOR_DATA_PATH = CURRENT_PATH + L"TestResults"; static std::wstring BuildCommand(std::initializer_list&& arguments) { @@ -217,8 +220,7 @@ namespace WinMLRunnerTest return true; } - TEST_CLASS(GarbageInputTest) { -public: TEST_CLASS_INITIALIZE(SetupClass) { + TEST_CLASS(GarbageInputTest) { public: TEST_CLASS_INITIALIZE(SetupClass) { // Make test_folder_input folder before starting the tests std::string mkFolderCommand = "mkdir " + std::string(INPUT_FOLDER_PATH.begin(), INPUT_FOLDER_PATH.end()); system(mkFolderCommand.c_str()); @@ -241,6 +243,13 @@ public: TEST_CLASS_INITIALIZE(SetupClass) { std::string copyCommand = "rd /s /q "; copyCommand += std::string(INPUT_FOLDER_PATH.begin(), INPUT_FOLDER_PATH.end()); system(copyCommand.c_str()); + try + { + std::filesystem::remove_all(std::string(TENSOR_DATA_PATH.begin(), TENSOR_DATA_PATH.end()).c_str()); + } + catch (const std::filesystem::filesystem_error &) + { + } } TEST_METHOD_CLEANUP(CleanupMethod) @@ -521,6 +530,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) { } catch (const std::filesystem::filesystem_error &) {} } + TEST_METHOD(ProvidedImageInputCpuAndGpu) { const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; @@ -552,78 +562,84 @@ public: TEST_CLASS_INITIALIZE(SetupClass) { BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, L"-autoScale", L"Cubic" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); } - TEST_METHOD(ProvidedImageInputOnlyGpuSaveTensor) - { + + TEST_METHOD_WITH_NAME(ProvidedImageInputOnlyGpuSaveTensor) const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; const std::wstring inputPath = CURRENT_PATH + L"fish.png"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" }); + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-GPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_GPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv")); - } - TEST_METHOD(ProvidedImageInputOnlyCpuSaveTensor) - { - const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; - const std::wstring inputPath = CURRENT_PATH + L"fish.png"; - const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" }); - Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); - Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_CPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv")); + tensorDataPath + L"\\softmaxout_1GpuIteration1.csv")); } - TEST_METHOD(ProvidedImageInputOnlyGpuSaveTensorImageDenotation) - { + TEST_METHOD_WITH_NAME(ProvidedImageInputOnlyCpuSaveTensor) + const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; + const std::wstring inputPath = CURRENT_PATH + L"fish.png"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; + const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-CPU" }); + Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); + Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_CPU.csv", + tensorDataPath + L"\\softmaxout_1CpuIteration1.csv")); + } + + TEST_METHOD_WITH_NAME(ProvidedImageInputOnlyGpuSaveTensorImageDenotation) const std::wstring modelPath = CURRENT_PATH + L"mnist.onnx"; const std::wstring inputPath = CURRENT_PATH + L"mnist_28.png"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" }); + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-GPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Mnist_8_input_GPU.csv", - TENSOR_DATA_PATH + L"\\Plus214_Output_0GpuIteration1.csv")); + tensorDataPath + L"\\Plus214_Output_0GpuIteration1.csv")); } - TEST_METHOD(ProvidedImageInputOnlyCpuSaveTensorImageDenotation) - { + + TEST_METHOD_WITH_NAME(ProvidedImageInputOnlyCpuSaveTensorImageDenotation) const std::wstring modelPath = CURRENT_PATH + L"mnist.onnx"; const std::wstring inputPath = CURRENT_PATH + L"mnist_28.png"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" }); + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-CPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Mnist_8_input_CPU.csv", - TENSOR_DATA_PATH + L"\\Plus214_Output_0CpuIteration1.csv")); + tensorDataPath + L"\\Plus214_Output_0CpuIteration1.csv")); } - TEST_METHOD(ProvidedImageInputOnlyGpuSaveTensorFp16) - { + + TEST_METHOD_WITH_NAME(ProvidedImageInputOnlyGpuSaveTensorFp16) const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx"; const std::wstring inputPath = CURRENT_PATH + L"fish.png"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" }); + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-GPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv")); + tensorDataPath + L"\\softmaxout_1GpuIteration1.csv")); } - TEST_METHOD(ProvidedImageInputOnlyCpuSaveTensorFp16) - { + + TEST_METHOD_WITH_NAME(ProvidedImageInputOnlyCpuSaveTensorFp16) const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx"; const std::wstring inputPath = CURRENT_PATH + L"fish.png"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" }); + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-CPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv")); + tensorDataPath + L"\\softmaxout_1CpuIteration1.csv")); } - TEST_METHOD(ProvidedImageInputOnlyCpuPerIterationPerformance) - { + + TEST_METHOD_WITH_NAME(ProvidedImageInputOnlyCpuPerIterationPerformance) const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-PerfOutput", OUTPUT_PATH, L"-perf", - L"-SavePerIterationPerf", L"-BaseOutputPath", TENSOR_DATA_PATH, + L"-SavePerIterationPerf", L"-BaseOutputPath", tensorDataPath, L"-PerIterationPath PerIterationData", L"-CPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); // We need to expect one more line because of the header - Assert::AreEqual(static_cast(2), GetOutputCSVLineCount(TENSOR_DATA_PATH + L"\\PerIterationData\\Summary.csv")); + Assert::AreEqual(static_cast(2), GetOutputCSVLineCount(tensorDataPath + L"\\PerIterationData\\Summary.csv")); } }; @@ -641,6 +657,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) { } catch (const std::filesystem::filesystem_error &) {} } + TEST_METHOD(ProvidedCSVInput) { const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; @@ -648,6 +665,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) { const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath }); Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str())); } + TEST_METHOD(ProvidedCSVBadBinding) { const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; @@ -655,76 +673,82 @@ public: TEST_CLASS_INITIALIZE(SetupClass) { const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath }); Assert::AreEqual(HRESULT_FROM_WIN32(ERROR_INVALID_PARAMETER), RunProc((wchar_t *)command.c_str())); } - TEST_METHOD(ProvidedCSVInputGPUSaveCpuBoundTensor) - { + + TEST_METHOD_WITH_NAME(ProvidedCSVInputGPUSaveCpuBoundTensor) const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" }); + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-GPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_GPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv")); - } - TEST_METHOD(ProvidedCSVInputGPUSaveGpuBoundTensor) - { - const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; - const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; - const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU", L"-GPUBoundInput" }); - Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); - Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_GPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv")); - } - TEST_METHOD(ProvidedCSVInputCPUSaveCpuBoundTensor) - { - const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; - const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; - const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" }); - Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); - Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_CPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv")); - } - TEST_METHOD(ProvidedCSVInputGPUSaveCpuBoundTensorFp16) - { - const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx"; - const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; - const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" }); - Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); - Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv")); - } - TEST_METHOD(ProvidedCSVInputCPUSaveCpuBoundTensorFp16) - { - const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx"; - const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; - const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" }); - Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); - Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv", - TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv")); + tensorDataPath + L"\\softmaxout_1GpuIteration1.csv")); } - TEST_METHOD(ProvidedCSVInputOnlyGpuSaveCpuBoundTensorImageDenotation) - { + TEST_METHOD_WITH_NAME(ProvidedCSVInputGPUSaveGpuBoundTensor) + const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; + const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; + const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-GPU", L"-GPUBoundInput" }); + Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); + Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_GPU.csv", + tensorDataPath + L"\\softmaxout_1GpuIteration1.csv")); + } + + TEST_METHOD_WITH_NAME(ProvidedCSVInputCPUSaveCpuBoundTensor) + const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx"; + const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; + const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-CPU" }); + Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); + Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_CPU.csv", + tensorDataPath + L"\\softmaxout_1CpuIteration1.csv")); + } + + TEST_METHOD_WITH_NAME(ProvidedCSVInputGPUSaveCpuBoundTensorFp16) + const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx"; + const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; + const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-GPU" }); + Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); + Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv", + tensorDataPath + L"\\softmaxout_1GpuIteration1.csv")); + } + + TEST_METHOD_WITH_NAME(ProvidedCSVInputCPUSaveCpuBoundTensorFp16) + const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx"; + const std::wstring inputPath = CURRENT_PATH + L"fish.csv"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; + const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-CPU" }); + Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); + Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv", + tensorDataPath + L"\\softmaxout_1CpuIteration1.csv")); + } + + TEST_METHOD_WITH_NAME(ProvidedCSVInputOnlyGpuSaveCpuBoundTensorImageDenotation) const std::wstring modelPath = CURRENT_PATH + L"mnist.onnx"; const std::wstring inputPath = CURRENT_PATH + L"mnist_28.csv"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" }); + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-GPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Mnist_8_input_GPU.csv", - TENSOR_DATA_PATH + L"\\Plus214_Output_0GpuIteration1.csv")); + tensorDataPath + L"\\Plus214_Output_0GpuIteration1.csv")); } - TEST_METHOD(ProvidedCSVInputOnlyCpuSaveCpuBoundTensorImageDenotation) - { + + TEST_METHOD_WITH_NAME(ProvidedCSVInputOnlyCpuSaveCpuBoundTensorImageDenotation) const std::wstring modelPath = CURRENT_PATH + L"mnist.onnx"; const std::wstring inputPath = CURRENT_PATH + L"mnist_28.csv"; + const std::wstring tensorDataPath = TENSOR_DATA_PATH + L"\\" + METHOD_NAME; const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, - L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" }); + L"-SaveTensorData", L"First", L"-PerIterationPath", tensorDataPath, L"-CPU" }); Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str())); Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Mnist_8_input_CPU.csv", - TENSOR_DATA_PATH + L"\\Plus214_Output_0CpuIteration1.csv")); + tensorDataPath + L"\\Plus214_Output_0CpuIteration1.csv")); } };