Make the reader skip over extra inputs even when their name exceeds
  the maximum expected input name length.
This commit is contained in:
Alexey Reznichenko 2017-03-27 13:54:42 +02:00
Родитель a3a941500b
Коммит 0099706d2d
2 изменённых файлов: 30 добавлений и 4 удалений

Просмотреть файл

@ -697,13 +697,16 @@ bool TextParser<ElemType>::TryGetInputId(size_t& id, size_t& bytesToRead)
{
// the current string length is already equal to the maximum expected length,
// yet it's not followed by a delimiter.
if (ShouldWarn())
if (m_traceLevel >= Info)
{
string namePrefix(m_scratch.get(), m_maxAliasLength);
fprintf(stderr,
"WARNING: Did not find a valid input name %ls.\n",
GetFileInfo().c_str());
"INFO: Skipping unknown input %ls. "
"Input name (with the %" PRIu64 "-character prefix '%s') "
"exceeds the maximum expected length (%" PRIu64 ").\n",
GetFileInfo().c_str(), m_maxAliasLength, namePrefix.c_str(), m_maxAliasLength);
}
break;
return false;
}
++m_pos;

Просмотреть файл

@ -698,7 +698,30 @@ BOOST_AUTO_TEST_CASE(CNTKTextFormatReader_no_trailing_newline_invalid_input)
" while reading the input file (no_trailing_newline.txt).") == ex.what();
});;
}
};
BOOST_AUTO_TEST_CASE(CNTKTextFormatReader_extra_input_should_be_ignored)
{
vector<StreamDescriptor> streams(1);
streams[0].m_alias = "A";
streams[0].m_name = L"A";
streams[0].m_storageType = StorageType::dense;
streams[0].m_sampleDimension = 1;
string filename = "extra_input.txt";
for (auto& input : { "|A 1 |B 1 2 3", "|A 2 |this_input_is_supposed_to_be_also_ignored 1 2 3" })
{
{
boost::filesystem::remove(filename);
std::ofstream file;
file.open(filename, std::ofstream::out);
file << input;
}
CNTKTextFormatReaderTestRunner<double> testRunner(filename, streams, 0);
testRunner.LoadChunk();
}
};
// 100 sequences with N samples for each of 3 inputs, where N is chosen at random