diff --git a/Tools/WinMLRunner/src/CommandLineArgs.cpp b/Tools/WinMLRunner/src/CommandLineArgs.cpp index d0362cc8..8b7f4fdb 100644 --- a/Tools/WinMLRunner/src/CommandLineArgs.cpp +++ b/Tools/WinMLRunner/src/CommandLineArgs.cpp @@ -218,13 +218,7 @@ CommandLineArgs::CommandLineArgs(const std::vector& args) } catch (...) { - // set to default path - auto time = std::time(nullptr); - struct tm localTime; - localtime_s(&localTime, &time); - std::wostringstream oss; - oss << std::put_time(&localTime, L"%Y-%m-%d_%H.%M.%S"); - SetTensorOutputPath(L"\\PerIterationRun[" + oss.str() + L"]"); + // Will Set Default Path after argument checks } } else if (_wcsicmp(args[i].c_str(), L"-version") == 0) @@ -318,6 +312,17 @@ CommandLineArgs::CommandLineArgs(const std::vector& args) throw hresult_invalid_argument(msg.c_str()); } } + + // set default path for per iteration / tensor output + if (this->TensorOutputPath().empty()) + { + auto time = std::time(nullptr); + struct tm localTime; + localtime_s(&localTime, &time); + std::wostringstream oss; + oss << std::put_time(&localTime, L"%Y-%m-%d_%H.%M.%S"); + SetTensorOutputPath(L".\\PerIterationRun[" + oss.str() + L"]"); + } CheckForInvalidArguments(); } diff --git a/Tools/WinMLRunner/src/OutputHelper.h b/Tools/WinMLRunner/src/OutputHelper.h index 74362a96..70a8c966 100644 --- a/Tools/WinMLRunner/src/OutputHelper.h +++ b/Tools/WinMLRunner/src/OutputHelper.h @@ -596,7 +596,7 @@ public: void SetCSVFileName(const std::wstring& fileName) { m_csvFileName = fileName; } - void WritePerIterationPerformance(const CommandLineArgs& args, std::wstring model, std::wstring img) + void WritePerIterationPerformance(const CommandLineArgs& args, const std::wstring& model, const std::wstring& img) { if (m_csvFileNamePerIterationSummary.length() > 0) { diff --git a/Tools/WinMLRunner/src/Run.cpp b/Tools/WinMLRunner/src/Run.cpp index 9f06b8d0..f6f5bec9 100644 --- a/Tools/WinMLRunner/src/Run.cpp +++ b/Tools/WinMLRunner/src/Run.cpp @@ -80,12 +80,12 @@ HRESULT BindInputFeatures(const LearningModel& model, const LearningModelBinding return S_OK; } -HRESULT LoadModel(LearningModel &model, const std::wstring& path, bool capturePerf, OutputHelper& output, const CommandLineArgs& args, - uint32_t iterationNum, Profiler& profiler) +HRESULT LoadModel(LearningModel& model, const std::wstring& path, bool capturePerf, OutputHelper& output, + const CommandLineArgs& args, uint32_t iterationNum, Profiler& profiler) { try { - output.PrintLoadingInfo(path); + output.PrintLoadingInfo(path); if (capturePerf) { WINML_PROFILING_START(profiler, WINML_MODEL_TEST_PERF::LOAD_MODEL); @@ -111,9 +111,9 @@ HRESULT LoadModel(LearningModel &model, const std::wstring& path, bool capturePe return S_OK; } -HRESULT CreateSession(LearningModelSession& session, IDirect3DDevice& winrtDevice, LearningModel& model, CommandLineArgs& args, OutputHelper& output, - DeviceType deviceType, DeviceCreationLocation deviceCreationLocation, - Profiler& profiler) +HRESULT CreateSession(LearningModelSession& session, IDirect3DDevice& winrtDevice, LearningModel& model, + CommandLineArgs& args, OutputHelper& output, DeviceType deviceType, + DeviceCreationLocation deviceCreationLocation, Profiler& profiler) { if (model == nullptr) { @@ -209,12 +209,14 @@ HRESULT CreateSession(LearningModelSession& session, IDirect3DDevice& winrtDevic return S_OK; } -HRESULT BindInputs(LearningModelBinding &context, const LearningModel& model, const LearningModelSession& session, OutputHelper& output, - DeviceType deviceType, const CommandLineArgs& args, InputBindingType inputBindingType, InputDataType inputDataType, - const IDirect3DDevice& winrtDevice, DeviceCreationLocation deviceCreationLocation, uint32_t iteration, +HRESULT BindInputs(LearningModelBinding& context, const LearningModel& model, const LearningModelSession& session, + OutputHelper& output, DeviceType deviceType, const CommandLineArgs& args, + InputBindingType inputBindingType, InputDataType inputDataType, const IDirect3DDevice& winrtDevice, + DeviceCreationLocation deviceCreationLocation, uint32_t iteration, Profiler& profiler) { - if (deviceType == DeviceType::CPU && inputDataType == InputDataType::Tensor && inputBindingType == InputBindingType::GPU) + if (deviceType == DeviceType::CPU && inputDataType == InputDataType::Tensor && + inputBindingType == InputBindingType::GPU) { std::cout << "Cannot create D3D12 device on client if CPU device type is selected." << std::endl; return E_INVALIDARG; @@ -238,7 +240,6 @@ HRESULT BindInputs(LearningModelBinding &context, const LearningModel& model, co return hr.code(); } - HRESULT bindInputResult = BindInputFeatures(model, context, inputFeatures, args, output, captureIterationPerf, iteration, profiler); @@ -277,16 +278,16 @@ std::vector GetModelsInDirectory(CommandLineArgs& args, OutputHelp return modelPaths; } - HRESULT CheckIfModelAndConfigurationsAreSupported(LearningModel& model, const std::wstring& modelPath, const DeviceType deviceType, const std::vector& inputDataTypes, const std::vector& deviceCreationLocations) { // Does user want image as input binding - bool hasInputBindingImage = std::any_of(inputDataTypes.begin(), inputDataTypes.end(), [](const InputDataType inputDataType){ + bool hasInputBindingImage = + std::any_of(inputDataTypes.begin(), inputDataTypes.end(), [](const InputDataType inputDataType) { return inputDataType == InputDataType::ImageBGR || inputDataType == InputDataType::ImageRGB; - }); + }); for (auto inputFeature : model.InputFeatures()) { @@ -345,7 +346,6 @@ HRESULT EvaluateModel(LearningModelEvaluationResult& result, const LearningModel if (args.IsPerIterationCapture()) { output.SaveEvalPerformance(profiler, iterationNum); - } } } @@ -496,18 +496,19 @@ int run(CommandLineArgs& args, Profiler& profiler) try ConcurrentLoadModel(modelPaths, args.NumThreads(), args.ThreadInterval(), true); return 0; } - for (const auto& path: modelPaths) + for (const auto& path : modelPaths) { LearningModel model = nullptr; - - LoadModel(model, path, args.IsPerformanceCapture() || args.IsPerIterationCapture(), output, args, 0, profiler); + + LoadModel(model, path, args.IsPerformanceCapture() || args.IsPerIterationCapture(), output, args, 0, + profiler); for (auto deviceType : deviceTypes) { lastHr = CheckIfModelAndConfigurationsAreSupported(model, path, deviceType, inputDataTypes, - deviceCreationLocations); + deviceCreationLocations); if (FAILED(lastHr)) { - continue; + continue; } for (auto deviceCreationLocation : deviceCreationLocations) { @@ -523,27 +524,28 @@ int run(CommandLineArgs& args, Profiler& profiler) try LearningModelSession session = nullptr; IDirect3DDevice winrtDevice = nullptr; lastHr = CreateSession(session, winrtDevice, model, args, output, deviceType, - deviceCreationLocation, profiler); + deviceCreationLocation, profiler); if (FAILED(lastHr)) { continue; } for (auto inputDataType : inputDataTypes) { - for (auto inputBindingType : inputBindingTypes) - { - // Clear up bind, eval performance metrics after iteration - if (args.IsPerformanceCapture() || args.IsPerIterationCapture()) - { - // Resets all values from profiler for bind and evaluate. - profiler.Reset(WINML_MODEL_TEST_PERF::BIND_VALUE, WINML_MODEL_TEST_PERF::COUNT); - } + for (auto inputBindingType : inputBindingTypes) + { + // Clear up bind, eval performance metrics after iteration + if (args.IsPerformanceCapture() || args.IsPerIterationCapture()) + { + // Resets all values from profiler for bind and evaluate. + profiler.Reset(WINML_MODEL_TEST_PERF::BIND_VALUE, WINML_MODEL_TEST_PERF::COUNT); + } for (uint32_t i = 0; i < args.NumIterations(); i++) { #if defined(_AMD64_) // PIX markers only work on AMD64 - // If PIX tool was attached then capture already began for the first iteration before session creation. - // This is to begin PIX capture for each iteration after the first iteration. + // If PIX tool was attached then capture already began for the first iteration before + // session creation. This is to begin PIX capture for each iteration after the first + // iteration. if (i > 0 && output.GetGraphicsAnalysis()) { output.GetGraphicsAnalysis()->BeginCapture(); @@ -551,15 +553,15 @@ int run(CommandLineArgs& args, Profiler& profiler) try #endif LearningModelBinding context(session); lastHr = BindInputs(context, model, session, output, deviceType, args, inputBindingType, - inputDataType, winrtDevice, deviceCreationLocation, i, profiler); + inputDataType, winrtDevice, deviceCreationLocation, i, profiler); if (FAILED(lastHr)) { break; } LearningModelEvaluationResult result = nullptr; bool capture_perf = args.IsPerformanceCapture() || args.IsPerIterationCapture(); - lastHr = EvaluateModel(result, model, context, session, args, output, - capture_perf, i, profiler); + lastHr = EvaluateModel(result, model, context, session, args, output, capture_perf, i, + profiler); if (FAILED(lastHr)) { output.PrintEvaluatingInfo(i + 1, deviceType, inputBindingType, inputDataType, @@ -604,8 +606,12 @@ int run(CommandLineArgs& args, Profiler& profiler) try inputBindingTypeStringified, deviceCreationLocationStringified); } + else if (args.IsPerIterationCapture() && args.IsImageInput()) + { + output.WritePerIterationPerformance(args, args.ModelPath(), args.ImagePath()); + } } - } + } } } }