From 35986ae22a27fc47974d3d030856576cb4061c8c Mon Sep 17 00:00:00 2001 From: Clemens Marschner Date: Wed, 25 May 2016 13:30:38 +0200 Subject: [PATCH] Extend checks and tests for extended eval interface Removed m_numberOfSamples, which is redundant Added more checks Fixed and extended test cases --- Source/Common/Include/Eval.h | 6 +- Source/EvalDll/CNTKEval.cpp | 34 ++++- .../UnitTests/EvalTests/EvalExtendedTests.cpp | 132 ++++++++++++------ Tests/UnitTests/EvalTests/EvalTests.vcxproj | 28 +--- 4 files changed, 123 insertions(+), 77 deletions(-) diff --git a/Source/Common/Include/Eval.h b/Source/Common/Include/Eval.h index 4160b8d9c..3a2b25dfd 100644 --- a/Source/Common/Include/Eval.h +++ b/Source/Common/Include/Eval.h @@ -176,10 +176,12 @@ public: template struct VariableBuffer { - size_t m_numberOfSamples = 0; - // // All elements of a sequence, concatenated. + // For dense inputs, the number of samples is given by the the length of + // this vector / product of tensor dimensions. E.g. for a tensor of dimension + // [2,2] and 12 elements in the buffer, the number of samples is 3. + // For sparse inputs, the number of samples is indicated by the m_colIndices field. // std::vector m_buffer; diff --git a/Source/EvalDll/CNTKEval.cpp b/Source/EvalDll/CNTKEval.cpp index 86a30f7d6..235aca1e7 100644 --- a/Source/EvalDll/CNTKEval.cpp +++ b/Source/EvalDll/CNTKEval.cpp @@ -304,11 +304,39 @@ void CNTKEvalExtended::ForwardPass(const Variables& inputs, for (auto& input : m_inputMatrices) { VariableBuffer buffer = inputs[i]; - int numRows = input.second.sampleLayout.GetNumElements(); - int numCols = buffer.m_numberOfSamples; shared_ptr> matrix = dynamic_pointer_cast>(input.second.matrix); - auto type = matrix->GetMatrixType(); + auto type = matrix->GetMatrixType(); + int numRows = input.second.sampleLayout.GetNumElements(); + if (type == MatrixType::DENSE) + { + if (buffer.m_buffer.size() % numRows != 0) + { + RuntimeError("Input %ls: Expected input data to be a multiple of %ld, but it is %ld", m_inputNodes[i]->GetName().c_str(), numRows, buffer.m_buffer.size()); + } + if (buffer.m_buffer.size() == 0) + { + RuntimeError("Input %ls: Expected at least one element.", m_inputNodes[i]->GetName().c_str()); + } + } + else if (type == MatrixType::SPARSE) + { + if (buffer.m_colIndices.size() < 2) + { + RuntimeError("Input %ls: Expected at least one element.", m_inputNodes[i]->GetName().c_str()); + } + if (buffer.m_colIndices[0] != 0) + { + RuntimeError("Input %ls: First element of column indices must be 0", m_inputNodes[i]->GetName().c_str()); + } + if (buffer.m_colIndices[buffer.m_colIndices.size()-1] != buffer.m_indices.size()) + { + RuntimeError("Input %ls: Last element of column indices must be equal to the size of indices (%ld), but was %d", m_inputNodes[i]->GetName().c_str(), buffer.m_indices.size(), buffer.m_colIndices[buffer.m_colIndices.size() - 1]); + } + } + + int numCols = type == MatrixType::DENSE ? buffer.m_buffer.size() / numRows : buffer.m_colIndices.size() - 1; + assert(numCols >= 1); input.second.pMBLayout->Init(1, numCols); input.second.pMBLayout->AddSequence(0, 0, 0, numCols); diff --git a/Tests/UnitTests/EvalTests/EvalExtendedTests.cpp b/Tests/UnitTests/EvalTests/EvalExtendedTests.cpp index 254f33ba7..375ee685f 100644 --- a/Tests/UnitTests/EvalTests/EvalExtendedTests.cpp +++ b/Tests/UnitTests/EvalTests/EvalExtendedTests.cpp @@ -5,6 +5,9 @@ #include "stdafx.h" #include "EvalTestHelper.h" +#include "fileutil.h" +#include "ExceptionWithCallStack.h" +#include "ScriptableObjects.h" using namespace Microsoft::MSR::CNTK; @@ -87,13 +90,16 @@ BOOST_AUTO_TEST_CASE(EvalConstantPlusTest) // Allocate the output values layer std::vector> outputBuffer(1); - // Allocate the input values layer (empty) + // Allocate the input values layer (empty)> EvalTests.exe!Microsoft::MSR::CNTK::Test::EvalTestSuite::EvalConstantPlusTest::test_method() Line 85 C++ + std::vector> inputBuffer; // We can call the evaluate method and get back the results... eval->ForwardPass(inputBuffer, outputBuffer); - BOOST_CHECK_EQUAL(outputBuffer[0].m_buffer[0], 3 /* 1 + 2 */); + std::vector expected{ 3 /* 1 + 2 */ }; + auto buf = outputBuffer[0].m_buffer; + BOOST_CHECK_EQUAL_COLLECTIONS(buf.begin(), buf.end(), expected.begin(), expected.end()); eval->Destroy(); } @@ -121,7 +127,6 @@ BOOST_AUTO_TEST_CASE(EvalScalarTimesTest) // Allocate the input values layer std::vector> inputBuffer(1); - inputBuffer[0].m_numberOfSamples = 1; inputBuffer[0].m_buffer.push_back(2); inputBuffer[0].m_indices.push_back(0); inputBuffer[0].m_colIndices.push_back(0); @@ -129,7 +134,9 @@ BOOST_AUTO_TEST_CASE(EvalScalarTimesTest) // We can call the evaluate method and get back the results... eval->ForwardPass(inputBuffer, outputBuffer); - BOOST_CHECK_EQUAL(outputBuffer[0].m_buffer[0], 6); + std::vector expected{ 6 }; + auto buf = outputBuffer[0].m_buffer; + BOOST_CHECK_EQUAL_COLLECTIONS(buf.begin(), buf.end(), expected.begin(), expected.end()); eval->Destroy(); } @@ -159,7 +166,6 @@ BOOST_AUTO_TEST_CASE(EvalScalarTimesDualOutputTest) // Allocate the input values layer std::vector> inputBuffer(1); - inputBuffer[0].m_numberOfSamples = 1; inputBuffer[0].m_buffer.push_back(2); inputBuffer[0].m_indices.push_back(0); inputBuffer[0].m_colIndices.push_back(0); @@ -168,48 +174,68 @@ BOOST_AUTO_TEST_CASE(EvalScalarTimesDualOutputTest) // TODO: Indicate to ForwardPass that we want output o2 only eval->ForwardPass(inputBuffer, outputBuffer); - BOOST_CHECK_EQUAL(outputBuffer[0].m_buffer[0], 6); + std::vector expected{ 6 }; + auto buf = outputBuffer[0].m_buffer; + BOOST_CHECK_EQUAL_COLLECTIONS(buf.begin(), buf.end(), expected.begin(), expected.end()); eval->Destroy(); } BOOST_AUTO_TEST_CASE(EvalDenseTimesTest) { - std::string modelDefinition = - "deviceId = -1 \n" - "precision = \"float\" \n" - "traceLevel = 1 \n" - "run=NDLNetworkBuilder \n" - "NDLNetworkBuilder=[ \n" - "i1 = Input(4) \n" - "o1 = Times(i1, Constant(2), tag=\"output\") \n" - "FeatureNodes = (i1) \n" - "] \n"; + try + { + std::string modelDefinition = + "deviceId = -1 \n" + "precision = \"float\" \n" + "traceLevel = 1 \n" + "run=BrainScriptNetworkBuilder \n" + "BrainScriptNetworkBuilder=[ \n" + "i1 = Input(4) \n" + "o1 = Times(ConstantTensor(2, 1:4), i1, tag=\"output\") \n" + "FeatureNodes = (i1) \n" + "] \n"; - VariableSchema inputLayouts; - VariableSchema outputLayouts; - IEvaluateModelExtended *eval; - eval = SetupNetworkAndGetLayouts(modelDefinition, inputLayouts, outputLayouts); + VariableSchema inputLayouts; + VariableSchema outputLayouts; + IEvaluateModelExtended *eval; + eval = SetupNetworkAndGetLayouts(modelDefinition, inputLayouts, outputLayouts); - // Allocate the output values layer - std::vector> outputBuffer(1); - - // Allocate the input values layer - std::vector> inputBuffer(1); - inputBuffer[0].m_numberOfSamples = 4; - inputBuffer[0].m_buffer = {1, 2, 3, 4}; - inputBuffer[0].m_indices.push_back(0); - inputBuffer[0].m_colIndices.push_back(0); + // Allocate the output values layer + std::vector> outputBuffer(1); - // We can call the evaluate method and get back the results... - eval->ForwardPass(inputBuffer, outputBuffer); + // Number of inputs must adhere to the schema + std::vector> inputBuffer1(0); + BOOST_REQUIRE_THROW(eval->ForwardPass(inputBuffer1, outputBuffer), std::exception); // Not enough inputs - BOOST_CHECK_EQUAL(outputBuffer[0].m_buffer[0], 2); - BOOST_CHECK_EQUAL(outputBuffer[0].m_buffer[1], 4); - BOOST_CHECK_EQUAL(outputBuffer[0].m_buffer[2], 6); - BOOST_CHECK_EQUAL(outputBuffer[0].m_buffer[3], 8); + // Number of elements in the input must adhere to the schema + std::vector> inputBuffer(1); + inputBuffer[0].m_buffer = { 1, 2, 3 }; + BOOST_REQUIRE_THROW(eval->ForwardPass(inputBuffer, outputBuffer), std::exception); // Not enough elements in the sample - eval->Destroy(); + // Output values and shape must be correct. + inputBuffer[0].m_buffer = { 1, 2, 3, 4 }; + eval->ForwardPass(inputBuffer, outputBuffer); + + std::vector expected{ 20 }; + auto buf = outputBuffer[0].m_buffer; + BOOST_CHECK_EQUAL_COLLECTIONS(buf.begin(), buf.end(), expected.begin(), expected.end()); + + eval->Destroy(); + } + catch (const ScriptableObjects::ScriptingException& err) + { + fprintf(stderr, "\n"); + err.PrintError(L"EXCEPTION occurred"); + throw; + } + catch (const IExceptionWithCallStackBase& err) + { + fprintf(stderr, "\n"); + fprintf(stderr, "%s", err.CallStack()); + fprintf(stderr, "EXCEPTION occurred: %s\n", dynamic_cast(err).what()); + throw; + } } BOOST_AUTO_TEST_CASE(EvalSparseTimesTest) @@ -220,8 +246,8 @@ BOOST_AUTO_TEST_CASE(EvalSparseTimesTest) "traceLevel = 1 \n" "run=NDLNetworkBuilder \n" "NDLNetworkBuilder=[ \n" - "i1 = SparseInput(9) \n" - "o1 = Times(i1, Constant(2), tag=\"output\") \n" + "i1 = SparseInput(3) \n" + "o1 = Times(Constant(2, rows=1, cols=3), i1, tag=\"output\") \n" "FeatureNodes = (i1) \n" "] \n"; @@ -235,16 +261,30 @@ BOOST_AUTO_TEST_CASE(EvalSparseTimesTest) // Allocate the input values layer std::vector> inputBuffer(1); - inputBuffer[0].m_numberOfSamples = 9; - inputBuffer[0].m_buffer = {1, 2, 3, 4, 5, 6}; - inputBuffer[0].m_indices = {0, 2, 3, 6}; - inputBuffer[0].m_colIndices = {0, 2, 2, 0, 1, 2}; + inputBuffer[0].m_buffer = {1, 2, 3, 5, 6}; + inputBuffer[0].m_indices = {0, 2, 2, 1, 2}; + + inputBuffer[0].m_colIndices = {}; + BOOST_REQUIRE_THROW(eval->ForwardPass(inputBuffer, outputBuffer), std::exception); // Empty input + + inputBuffer[0].m_colIndices = { 0 }; + BOOST_REQUIRE_THROW(eval->ForwardPass(inputBuffer, outputBuffer), std::exception); // Empty input + + inputBuffer[0].m_colIndices = { 1, 0 }; + BOOST_REQUIRE_THROW(eval->ForwardPass(inputBuffer, outputBuffer), std::exception); // Illegal: First entry must be 0 + + inputBuffer[0].m_colIndices = { 0, 2, 2, 4 }; + BOOST_REQUIRE_THROW(eval->ForwardPass(inputBuffer, outputBuffer), std::exception); // Illegal: Last entry must be indices.size() + + inputBuffer[0].m_colIndices = { 0, 2, 2, 5 }; // We can call the evaluate method and get back the results... - // TODO: Enable when SparseInput is supported - //eval->ForwardPass(inputBuffer, outputBuffer); - //BOOST_CHECK_EQUAL(outputBuffer[0].m_buffer[0], 2); - + eval->ForwardPass(inputBuffer, outputBuffer); + + // [2,2,2] * [1,2,3]^T etc. + std::vector expected{ 6, 0, 28 }; + auto buf = outputBuffer[0].m_buffer; + BOOST_CHECK_EQUAL_COLLECTIONS(buf.begin(), buf.end(), expected.begin(), expected.end()); eval->Destroy(); } diff --git a/Tests/UnitTests/EvalTests/EvalTests.vcxproj b/Tests/UnitTests/EvalTests/EvalTests.vcxproj index c2ac6ec52..ac67d0a17 100644 --- a/Tests/UnitTests/EvalTests/EvalTests.vcxproj +++ b/Tests/UnitTests/EvalTests/EvalTests.vcxproj @@ -123,35 +123,11 @@ %(AdditionalLibraryDirectories);$(CudaLibPath) %(DelayLoadDLLs);nvml.dll;$(CudaRuntimeDll) - - copy "$(OutDir)..\evaldll.dll" "$(TargetDir)" - - - Copy EvalDLL to the UnitTest output directory - - - copy "$(OutDir)..\evaldll.dll" "$(TargetDir)" - - - Copy EvalDLL to the UnitTest output directory - CPUONLY;%(PreprocessorDefinitions) - - copy "$(OutDir)..\evaldll.dll" "$(TargetDir)" - - - Copy EvalDLL to the UnitTest output directory - - - copy "$(OutDir)..\evaldll.dll" "$(TargetDir)" - - - Copy EvalDLL to the UnitTest output directory - @@ -177,10 +153,10 @@ $(OutDir)..\cudnn64_4.dll - + - +