microsoft
/
CNTK
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge branch 'master' of https://git01.codeplex.com/cntk

This commit is contained in:
Mike Seltzer 2014-10-31 14:28:11 -07:00
Родитель e6248be2a9 c576fc6f2c
Коммит 9f85e4554a
84 изменённых файлов: 4855 добавлений и 3669 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

62
CNTKSolution/CNTKSolution.sln
Просмотреть файл

@ -14,7 +14,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cn", "..\MachineLearning\cn
{E6646FFE-3588-4276-8A15-8D65C22711C1} = {E6646FFE-3588-4276-8A15-8D65C22711C1}
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "UnitTests", "UnitTests", "{D45DF403-6781-444E-B654-A96868C5BE68}"
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Unit Tests", "Unit Tests", "{D45DF403-6781-444E-B654-A96868C5BE68}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CNTKMathTest", "..\Math\CNTKMathTest\CNTKMathTest.vcxproj", "{6CEE834A-8104-46A8-8902-64C81BD7928F}"
EndProject
@ -31,8 +31,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "UCIFastReader", "..\DataRea
{1D5787D4-52E4-45DB-951B-82F220EE0C6A} = {1D5787D4-52E4-45DB-951B-82F220EE0C6A}
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{2326995F-9ABE-4DEE-BAD0-147541548B5A}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BinaryReader", "..\DataReader\BinaryReader\BinaryReader.vcxproj", "{1D5787D4-52E4-45DB-951B-82F220EE0C6A}"
ProjectSection(ProjectDependencies) = postProject
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5} = {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}
@ -59,98 +57,56 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CNTKEvalTest", "..\MachineL
{482999D1-B7E2-466E-9F8D-2119F93EAFD9} = {482999D1-B7E2-466E-9F8D-2119F93EAFD9}
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Reader Plugins", "Reader Plugins", "{33EBFE78-A1A8-4961-8938-92A271941F94}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CNTK", "CNTK", "{DD043083-71A4-409A-AA91-F9C548DCF7EC}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Mixed Platforms = Debug|Mixed Platforms
Debug|x64 = Debug|x64
Release|Mixed Platforms = Release|Mixed Platforms
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Debug|Mixed Platforms.Build.0 = Debug|x64
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Debug|x64.ActiveCfg = Debug|x64
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Debug|x64.Build.0 = Debug|x64
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Release|Mixed Platforms.ActiveCfg = Release|x64
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Release|Mixed Platforms.Build.0 = Release|x64
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Release|x64.ActiveCfg = Release|x64
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Release|x64.Build.0 = Release|x64
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Debug|Mixed Platforms.Build.0 = Debug|x64
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Debug|x64.ActiveCfg = Debug|x64
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Debug|x64.Build.0 = Debug|x64
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Release|Mixed Platforms.ActiveCfg = Release|x64
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Release|Mixed Platforms.Build.0 = Release|x64
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Release|x64.ActiveCfg = Release|x64
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Release|x64.Build.0 = Release|x64
{6CEE834A-8104-46A8-8902-64C81BD7928F}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{6CEE834A-8104-46A8-8902-64C81BD7928F}.Debug|Mixed Platforms.Build.0 = Debug|x64
{6CEE834A-8104-46A8-8902-64C81BD7928F}.Debug|x64.ActiveCfg = Debug|x64
{6CEE834A-8104-46A8-8902-64C81BD7928F}.Debug|x64.Build.0 = Debug|x64
{6CEE834A-8104-46A8-8902-64C81BD7928F}.Release|Mixed Platforms.ActiveCfg = Release|x64
{6CEE834A-8104-46A8-8902-64C81BD7928F}.Release|Mixed Platforms.Build.0 = Release|x64
{6CEE834A-8104-46A8-8902-64C81BD7928F}.Release|x64.ActiveCfg = Release|x64
{33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Debug|Mixed Platforms.Build.0 = Debug|x64
{33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Debug|x64.ActiveCfg = Debug|x64
{33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Debug|x64.Build.0 = Debug|x64
{33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Release|Mixed Platforms.ActiveCfg = Release|x64
{33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Release|Mixed Platforms.Build.0 = Release|x64
{33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Release|x64.ActiveCfg = Release|x64
{33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Release|x64.Build.0 = Release|x64
{668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Debug|Mixed Platforms.Build.0 = Debug|x64
{668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Debug|x64.ActiveCfg = Debug|x64
{668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Debug|x64.Build.0 = Debug|x64
{668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Release|Mixed Platforms.ActiveCfg = Release|x64
{668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Release|Mixed Platforms.Build.0 = Release|x64
{668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Release|x64.ActiveCfg = Release|x64
{E6646FFE-3588-4276-8A15-8D65C22711C1}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{E6646FFE-3588-4276-8A15-8D65C22711C1}.Debug|Mixed Platforms.Build.0 = Debug|x64
{E6646FFE-3588-4276-8A15-8D65C22711C1}.Debug|x64.ActiveCfg = Debug|x64
{E6646FFE-3588-4276-8A15-8D65C22711C1}.Debug|x64.Build.0 = Debug|x64
{E6646FFE-3588-4276-8A15-8D65C22711C1}.Release|Mixed Platforms.ActiveCfg = Release|x64
{E6646FFE-3588-4276-8A15-8D65C22711C1}.Release|Mixed Platforms.Build.0 = Release|x64
{E6646FFE-3588-4276-8A15-8D65C22711C1}.Release|x64.ActiveCfg = Release|x64
{E6646FFE-3588-4276-8A15-8D65C22711C1}.Release|x64.Build.0 = Release|x64
{1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Debug|Mixed Platforms.Build.0 = Debug|x64
{1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Debug|x64.ActiveCfg = Debug|x64
{1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Debug|x64.Build.0 = Debug|x64
{1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Release|Mixed Platforms.ActiveCfg = Release|x64
{1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Release|Mixed Platforms.Build.0 = Release|x64
{1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Release|x64.ActiveCfg = Release|x64
{1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Release|x64.Build.0 = Release|x64
{9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Debug|Mixed Platforms.Build.0 = Debug|x64
{9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Debug|x64.ActiveCfg = Debug|x64
{9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Debug|x64.Build.0 = Debug|x64
{9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Release|Mixed Platforms.ActiveCfg = Release|x64
{9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Release|Mixed Platforms.Build.0 = Release|x64
{9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Release|x64.ActiveCfg = Release|x64
{9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Release|x64.Build.0 = Release|x64
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Debug|Mixed Platforms.Build.0 = Debug|x64
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Debug|x64.ActiveCfg = Debug|x64
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Debug|x64.Build.0 = Debug|x64
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Release|Mixed Platforms.ActiveCfg = Release|x64
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Release|Mixed Platforms.Build.0 = Release|x64
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Release|x64.ActiveCfg = Release|x64
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Release|x64.Build.0 = Release|x64
{482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Debug|Mixed Platforms.Build.0 = Debug|x64
{482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Debug|x64.ActiveCfg = Debug|x64
{482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Debug|x64.Build.0 = Debug|x64
{482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Release|Mixed Platforms.ActiveCfg = Release|x64
{482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Release|Mixed Platforms.Build.0 = Release|x64
{482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Release|x64.ActiveCfg = Release|x64
{482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Release|x64.Build.0 = Release|x64
{0F30EBCF-09F3-4EED-BF54-4214BCE53FEC}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
{0F30EBCF-09F3-4EED-BF54-4214BCE53FEC}.Debug|Mixed Platforms.Build.0 = Debug|x64
{0F30EBCF-09F3-4EED-BF54-4214BCE53FEC}.Debug|x64.ActiveCfg = Debug|x64
{0F30EBCF-09F3-4EED-BF54-4214BCE53FEC}.Debug|x64.Build.0 = Debug|x64
{0F30EBCF-09F3-4EED-BF54-4214BCE53FEC}.Release|Mixed Platforms.ActiveCfg = Release|Win32
{0F30EBCF-09F3-4EED-BF54-4214BCE53FEC}.Release|Mixed Platforms.Build.0 = Release|Win32
{0F30EBCF-09F3-4EED-BF54-4214BCE53FEC}.Release|x64.ActiveCfg = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
@ -160,5 +116,13 @@ Global
{6CEE834A-8104-46A8-8902-64C81BD7928F} = {D45DF403-6781-444E-B654-A96868C5BE68}
{668BEED5-AC07-4F35-B3AE-EE65A7F9C976} = {D45DF403-6781-444E-B654-A96868C5BE68}
{0F30EBCF-09F3-4EED-BF54-4214BCE53FEC} = {D45DF403-6781-444E-B654-A96868C5BE68}
{1D5787D4-52E4-45DB-951B-82F220EE0C6A} = {33EBFE78-A1A8-4961-8938-92A271941F94}
{33D2FD22-DEF2-4507-A58A-368F641AEBE5} = {33EBFE78-A1A8-4961-8938-92A271941F94}
{62836DC1-DF77-4B98-BF2D-45C943B7DDC6} = {33EBFE78-A1A8-4961-8938-92A271941F94}
{9A2F2441-5972-4EA8-9215-4119FCE0FB68} = {33EBFE78-A1A8-4961-8938-92A271941F94}
{E6646FFE-3588-4276-8A15-8D65C22711C1} = {33EBFE78-A1A8-4961-8938-92A271941F94}
{E6F26F9A-FF64-4F0A-B749-CD309EE357EE} = {DD043083-71A4-409A-AA91-F9C548DCF7EC}
{482999D1-B7E2-466E-9F8D-2119F93EAFD9} = {DD043083-71A4-409A-AA91-F9C548DCF7EC}
{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5} = {DD043083-71A4-409A-AA91-F9C548DCF7EC}
EndGlobalSection
EndGlobal

21
Common/BestGpu.cpp
Просмотреть файл

@ -6,14 +6,28 @@
#define _CRT_SECURE_NO_WARNINGS // "secure" CRT not available on all platforms --add this at the top of all CPP files that give "function or variable may be unsafe" warnings
#include "BestGpu.h"
#include "CommonMatrix.h" // for CPUDEVICE and AUTOPLACEMATRIX
#ifdef CPUONLY
namespace Microsoft {
namespace MSR {
namespace CNTK {
short DeviceFromConfig(const ConfigParameters& config)
{
return CPUDEVICE;
}
}
}
}
#else
// CUDA-C includes
#include <cuda.h>
#include <windows.h>
#include <delayimp.h>
#include <Shlobj.h>
#include <stdio.h>
#include "BestGpu.h"
#include "CommonMatrix.h" // for CPUDEVICE and AUTOPLACEMATRIX
// The "notify hook" gets called for every call to the
// delay load helper. This allows a user to hook every call and
@ -507,4 +521,5 @@ void BestGpu::QueryNvmlData()
m_nvmlData = true;
return;
}
}}}
}}}
#endif

4
Common/ConfigFile.cpp
Просмотреть файл

@ -53,7 +53,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
configString += config.ReadConfigFile(filePath);
}
else
Error("Cannot specify same config file multiple times at the command line.");
RuntimeError("Cannot specify same config file multiple times at the command line.");
}
}
}
@ -85,7 +85,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
std::string filePaths = line.substr(includeKeywordSize, line.size() - includeKeywordSize);
if (filePaths.find(openBraceVar) != std::string::npos)
{
Error("Variable usage (eg, \"$varName$\") not supported in \"include\" statements. Explicit path to config file must be provided");
RuntimeError("Variable usage (eg, \"$varName$\") not supported in \"include\" statements. Explicit path to config file must be provided");
}
std::vector<std::string> filePathVec = msra::strfun::split (filePaths, "+");

21
Common/DataReader.cpp
Просмотреть файл

@ -8,6 +8,7 @@
#include "stdafx.h"
#define DATAREADER_LOCAL
#include "basetypes.h"
#include "DataReader.h"
namespace Microsoft { namespace MSR { namespace CNTK {
@ -22,7 +23,7 @@ template<> std::string GetReaderName(double) {std::string name = "GetReaderD"; r
template<class ElemType>
void DataReader<ElemType>::Init(const ConfigParameters& /*config*/)
{
Error("Init shouldn't be called, use constructor");
RuntimeError("Init shouldn't be called, use constructor");
// not implemented, calls the underlying class instead
}
@ -43,23 +44,12 @@ void DataReader<ElemType>::GetDataReader(const ConfigParameters& config)
typedef void (*GetReaderProc)(IDataReader<ElemType>** preader);
// initialize just in case
m_hModule = NULL;
m_dataReader = NULL;
// get the name for the reader we want to use, default to UCIFastReader
m_dllName = msra::strfun::utf16(config("readerType", "UCIFastReader"));
m_dllName += L".dll";
m_hModule = LoadLibrary(m_dllName.c_str());
if (m_hModule == NULL)
{
std::string message = "Reader not found: ";
message += msra::strfun::utf8(m_dllName);
Error((char*) message.c_str());
}
// create a variable of each type just to call the proper templated version
ElemType elemType = ElemType();
GetReaderProc getReaderProc = (GetReaderProc)GetProcAddress(m_hModule, GetReaderName(elemType).c_str());
GetReaderProc getReaderProc = (GetReaderProc)Plugin::Load(config("readerType", "UCIFastReader"), GetReaderName(elemType).c_str());
getReaderProc(&m_dataReader);
}
@ -86,11 +76,6 @@ DataReader<ElemType>::~DataReader()
m_dataReader->Destroy();
m_dataReader = NULL;
}
if (m_hModule != NULL)
{
FreeLibrary(m_hModule);
m_hModule = NULL;
}
}
//StartMinibatchLoop - Startup a minibatch loop

39
Common/DataWriter.cpp
Просмотреть файл

@ -22,7 +22,7 @@ template<> std::string GetWriterName(double) {std::string name = "GetWriterD"; r
template<class ElemType>
void DataWriter<ElemType>::Init(const ConfigParameters& /*config*/)
{
Error("Init shouldn't be called, use constructor");
RuntimeError("Init shouldn't be called, use constructor");
// not implemented, calls the underlying class instead
}
@ -43,37 +43,25 @@ void DataWriter<ElemType>::GetDataWriter(const ConfigParameters& config)
typedef void (*GetWriterProc)(IDataWriter<ElemType>** pwriter);
// initialize just in case
m_hModule = NULL;
m_dataWriter = NULL;
// get the name for the writer we want to use, default to BinaryWriter (which is in BinaryReader.dll)
string writerType = config("writerType","BinaryReader");
if (writerType == "HTKMLFWriter" || writerType == "HTKMLFReader")
{
writerType = "HTKMLFReader";
}
else if (writerType == "BinaryWriter" || writerType == "BinaryReader")
{
writerType = "BinaryReader";
}
string writerType = config("writerType", "BinaryReader");
if (writerType == "HTKMLFWriter" || writerType == "HTKMLFReader")
{
writerType = "HTKMLFReader";
}
else if (writerType == "BinaryWriter" || writerType == "BinaryReader")
{
writerType = "BinaryReader";
}
else if (writerType == "LUSequenceWriter" || writerType == "LUSequenceReader")
{
writerType = "LUSequenceReader";
}
m_dllName = msra::strfun::utf16(writerType);
m_dllName += L".dll";
m_hModule = LoadLibrary(m_dllName.c_str());
if (m_hModule == NULL)
{
std::string message = "Writer not found: ";
message += msra::strfun::utf8(m_dllName);
Error((char*)message.c_str());
}
// create a variable of each type just to call the proper templated version
ElemType elemType = ElemType();
GetWriterProc getWriterProc = (GetWriterProc)GetProcAddress(m_hModule, GetWriterName(elemType).c_str());
GetWriterProc getWriterProc = (GetWriterProc)Plugin::Load(writerType, GetWriterName(elemType).c_str());
getWriterProc(&m_dataWriter);
}
@ -97,11 +85,6 @@ DataWriter<ElemType>::~DataWriter()
m_dataWriter->Destroy();
m_dataWriter = NULL;
}
if (m_hModule != NULL)
{
FreeLibrary(m_hModule);
m_hModule = NULL;
}
}
// GetSections - Get the sections of the file

27
Common/File.cpp
Просмотреть файл

@ -9,10 +9,16 @@
#include "basetypes.h"
#define FORMAT_SPECIALIZE // to get the specialized version of the format routines
#include "fileutil.h"
#include "message.h"
#include "File.h"
#include <string>
#include <stdint.h>
#include <locale>
#ifdef _WIN32
#include <Windows.h>
#endif
#ifdef __unix__
#include <unistd.h>
#endif
namespace Microsoft{ namespace MSR { namespace CNTK {
@ -77,7 +83,7 @@ void File::goToDelimiter(int delim)
ch=fgetc(m_file);
if (feof(m_file)) {
printf("Unexpected end of file\n");
throw std::logic_error("Unexpected end of file\n");
throw std::logic_error("Unexpected end of file\n");
}
}
}
@ -461,9 +467,7 @@ File& File::operator>>(FileMarker marker)
break;
case fileMarkerEndFile: // end of file marker, should we throw if it's not the end of the file?
if (!IsEOF())
{
ERROR("fileMarkerEndFile not found");
}
throw std::runtime_error("fileMarkerEndFile not found");
break;
case fileMarkerBeginList: // Beginning of list marker
// no marker written unless an list with a count header
@ -477,7 +481,7 @@ File& File::operator>>(FileMarker marker)
{
int found = EndOfLineOrEOF(true);
if (found != (int)true) // EOF can also be returned
ERROR("Newline not found");
throw std::runtime_error("Newline not found");
}
break;
case fileMarkerBeginSection: // beginning of section
@ -550,9 +554,7 @@ File& File::GetMarker(FileMarker marker, const std::string& section)
string str;
*this >> str;
if (str != section)
{
ERROR("section name mismatch %s != %s", str.c_str(), section.c_str());
}
throw std::runtime_error(std::string("section name mismatch ") + str + " != " + section);
return *this;
}
@ -565,9 +567,7 @@ File& File::GetMarker(FileMarker marker, const std::wstring& section)
wstring str;
*this >> str;
if (str != section)
{
ERROR("section name mismatch %ls != %ls", str.c_str(), section.c_str());
}
throw std::runtime_error(std::string("section name mismatch ") + msra::strfun::utf8(str) + " != " + msra::strfun::utf8(section));
return *this;
}
@ -627,4 +627,5 @@ void File::SetPosition(uint64_t pos)
{
fsetpos (m_file, pos);
}
}}}
}}}

7
Common/Include/BestGpu.h
Просмотреть файл

@ -4,15 +4,18 @@
// </copyright>
//
#pragma once
#ifndef CPUONLY
#include <cuda_runtime.h>
#include <nvml.h>
#include <vector>
#endif
#include "commandArgUtil.h"
namespace Microsoft { namespace MSR { namespace CNTK {
short DeviceFromConfig(const ConfigParameters& config);
#ifndef CPUONLY
struct ProcessorData
{
int cores;
@ -69,4 +72,6 @@ public:
};
extern BestGpu* g_bestGpu;
}}}
#endif
}}}

14
Common/Include/DataReader.h
Просмотреть файл

@ -18,9 +18,10 @@
#else
#define DATAREADER_API __declspec(dllimport)
#endif
#include "matrix.h"
#include "Matrix.h"
#include <map>
#include <string>
#include "basetypes.h"
#include "commandArgUtil.h"
namespace Microsoft { namespace MSR { namespace CNTK {
@ -56,8 +57,7 @@ public:
virtual void SetLabelMapping(const std::wstring& sectionName, const std::map<typename LabelIdType, typename LabelType>& labelMapping) = 0;
virtual bool GetData(const std::wstring& sectionName, size_t numRecords, void* data, size_t& dataBufferSize, size_t recordStart) = 0;
virtual bool DataEnd(EndDataType endDataType) = 0;
virtual void SetSentenceEndInBatch(vector<size_t> &sentenceEnd)=0;
virtual void SetSentenceEndInBatch(vector<size_t> &sentenceEnd) = 0;
};
// GetReader - get a reader type from the DLL
@ -72,12 +72,10 @@ extern "C" DATAREADER_API void GetReaderD(IDataReader<double>** preader);
// interface for clients of the Data Reader
// mirrors the IDataReader interface, except the Init method is private (use the constructor)
template<class ElemType>
class DataReader : public IDataReader<ElemType>
class DataReader : public IDataReader<ElemType>, public Plugin
{
private:
IDataReader *m_dataReader; // reader
HMODULE m_hModule; // module handle for the reader DLL
std::wstring m_dllName; // name of the reader DLL
IDataReader<ElemType> *m_dataReader; // reader
// Init - Reader Initialize for multiple data sets
// config - [in] configuration parameters for the datareader
@ -155,4 +153,4 @@ public:
void SetSentenceEndInBatch(std::vector<size_t> &sentenceEnd);
};
}}}
}}}

13
Common/Include/DataWriter.h
Просмотреть файл

@ -18,9 +18,10 @@
#else
#define DATAWRITER_API __declspec(dllimport)
#endif
#include "matrix.h"
#include "Matrix.h"
#include <map>
#include <string>
#include "basetypes.h"
#include "commandArgUtil.h"
@ -52,7 +53,7 @@ public:
virtual void Destroy() = 0;
virtual void GetSections(std::map<std::wstring, SectionType, nocase_compare>& sections) = 0;
virtual bool SaveData(size_t recordStart, const std::map<std::wstring, void*, nocase_compare>& matrices, size_t numRecords, size_t datasetSize, size_t byteVariableSized) = 0;
virtual void SaveMapping(std::wstring saveId, const std::map<typename LabelIdType, typename LabelType>& labelMapping) = 0;
virtual void SaveMapping(std::wstring saveId, const std::map<LabelIdType, LabelType>& labelMapping) = 0;
};
@ -68,12 +69,10 @@ extern "C" DATAWRITER_API void GetWriterD(IDataWriter<double>** pwriter);
// interface for clients of the Data Writer
// mirrors the IDataWriter interface, except the Init method is private (use the constructor)
template<class ElemType>
class DataWriter : public IDataWriter<ElemType>
class DataWriter : public IDataWriter<ElemType>, public Plugin
{
private:
IDataWriter *m_dataWriter; // writer
HMODULE m_hModule; // module handle for the writer DLL
std::wstring m_dllName; // name of the writer DLL
IDataWriter<ElemType> *m_dataWriter; // writer
// Init - Writer Initialize for multiple data sets
// config - [in] configuration parameters for the datawriter
@ -146,7 +145,7 @@ public:
// SaveMapping - save a map into the file
// saveId - name of the section to save into (section:subsection format)
// labelMapping - map we are saving to the file
virtual void SaveMapping(std::wstring saveId, const std::map<typename LabelIdType, typename LabelType>& labelMapping);
virtual void SaveMapping(std::wstring saveId, const std::map<LabelIdType, LabelType>& labelMapping);
};
}}}

35
Common/Include/File.h
Просмотреть файл

@ -8,6 +8,12 @@
#include <string>
#include <vector>
#include <stdint.h>
#ifdef _WIN32
#include <Windows.h>
#endif
#ifdef __unix__
#include <unistd.h>
#endif
#include "fileutil.h" // for f{ge,pu}t{,Text}()
namespace Microsoft{ namespace MSR { namespace CNTK {
@ -40,8 +46,9 @@ enum FileMarker
};
// attempt a given operation (lambda) and retry multiple times
// body - the lambda to retry
template<typename FUNCTION> static void attempt (int retries, const FUNCTION & body)
// body - the lambda to retry, must be restartable
template<typename FUNCTION> static void attempt(int retries, const FUNCTION & body)
{
for (int attempt = 1; ; attempt++)
{
@ -53,16 +60,20 @@ template<typename FUNCTION> static void attempt (int retries, const FUNCTION & b
}
catch (const std::exception & e)
{
void sleep(size_t ms);
if (attempt >= retries)
throw; // failed N times --give up and rethrow the error
fprintf (stderr, "attempt: %s, retrying %d-th time out of %d...\n", e.what(), attempt+1, retries);
::Sleep (1000); // wait a little, then try again
#ifdef _DEBUG
DebugBreak();
// wait a little, then try again
#ifdef _WIN32
::Sleep(1000);
#else // assuming __unix__
sleep(1);
#endif
}
}
}
template<typename FUNCTION> static void attempt (const FUNCTION & body)
{
static const int retries = 5;
@ -112,13 +123,18 @@ public:
template <typename T>
File& operator<<(T val)
{
#ifndef __CUDACC__ // TODO: CUDA compiler blows up, fix this
attempt([=]()
#endif
{
if (IsTextBased())
fputText(m_file, val);
else
fput(m_file, val);
});
}
#ifndef __CUDACC__
);
#endif
return *this;
}
File& operator<<(const std::wstring& val);
@ -145,13 +161,18 @@ public:
template <typename T>
File& operator>>(T& val)
{
#ifndef __CUDACC__ // TODO: CUDA compiler blows up, fix this
attempt([&]()
#endif
{
if (IsTextBased())
fgetText(m_file, val);
else
fget(m_file, val);
});
}
#ifndef __CUDACC__
);
#endif
return *this;
}

1928
Common/Include/basetypes.h
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

33
Common/Include/commandArgUtil.h
Просмотреть файл

@ -108,25 +108,29 @@ namespace Microsoft { namespace MSR { namespace CNTK {
return value;
}
operator float () const { return (float) (double) *this; }
operator long () const
private:
long tolong() const
{
char * ep; // will be set to point to first character that failed parsing
long value = strtol (c_str(), &ep, 10);
long value = strtol(c_str(), &ep, 10);
if (empty() || *ep != 0)
throw std::runtime_error ("ConfigValue (long): invalid input string");
throw std::runtime_error("ConfigValue (long): invalid input string");
return value;
}
operator unsigned long () const
unsigned long toulong() const
{
char * ep; // will be set to point to first character that failed parsing
unsigned long value = strtoul (c_str(), &ep, 10);
unsigned long value = strtoul(c_str(), &ep, 10);
if (empty() || *ep != 0)
throw std::runtime_error ("ConfigValue (unsigned long): invalid input string");
throw std::runtime_error("ConfigValue (unsigned long): invalid input string");
return value;
}
public:
operator long() const { return tolong(); }
operator unsigned long() const { return toulong(); }
operator short () const
{
long val = (long) *this;
long val = tolong();
short ival = (short) val;
if (val != ival)
throw std::runtime_error ("ConfigValue (short): integer argument expected");
@ -134,7 +138,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
operator unsigned short () const
{
unsigned long val = (unsigned long) *this;
unsigned long val = toulong();
unsigned short ival = (unsigned short) val;
if (val != ival)
throw std::runtime_error ("ConfigValue (unsigned short): integer argument expected");
@ -142,7 +146,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
operator int () const
{
long val = (long) *this;
long val = tolong();
int ival = (int) val;
if (val != ival)
throw std::runtime_error ("ConfigValue (int): integer argument expected");
@ -150,7 +154,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
operator unsigned int () const
{
unsigned long val = (unsigned long) *this;
unsigned long val = toulong();
unsigned int ival = (unsigned int) val;
if (val != ival)
throw std::runtime_error ("ConfigValue (unsigned int): integer argument expected");
@ -159,7 +163,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
operator int64_t () const
{
char * ep; // will be set to point to first character that failed parsing
int64_t value = _strtoui64 (c_str(), &ep, 10);
int64_t value = _strtoi64 (c_str(), &ep, 10);
if (empty() || *ep != 0)
throw std::runtime_error ("ConfigValue (int64_t): invalid input string");
return value;
@ -240,6 +244,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_separator = configParser.m_separator;
m_configName = move(configParser.m_configName);
}
ConfigParser& operator=(const ConfigParser& configParser) = default;
public:
// FindBraces - find matching braces in a string starting at the current position
@ -869,7 +874,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_repeatAsterisk = repeatAsterisk;
}
// copy and move constructors
// copy and move constructors and assignment
ConfigArray(const ConfigArray& configValue) : ConfigParser(configValue)
{
m_repeatAsterisk = true;
@ -880,6 +885,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_repeatAsterisk = true;
*this = move(configValue);
}
ConfigArray& operator=(const ConfigArray& configValue) = default;
// cast a configArray back to a string so we can return it as a ConfigValue
operator ConfigValue()
@ -947,7 +953,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
for (int i=0;i < count;++i)
{
char buf[10];
_itoa_s((int)size(), buf, 10);
sprintf (buf, "%d", (int)size()); // TODO: left-over of Linux compat, can be done nicer
std::string name = m_configName + '[' + buf + ']' ;
push_back(ConfigValue(value, name));
}
@ -976,6 +982,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<typename T>
class argvector : public std::vector<T>
{
typedef std::vector<T> B; using B::clear; using B::reserve;
static void parse (const std::wstring & in, float & val) { val = (float) msra::strfun::todouble (in); }
static void parse (const std::wstring & in, size_t & val) // convert wstring toks2[0] to T val and check type
{

268
Common/Include/fileutil.h
Просмотреть файл

@ -3,234 +3,19 @@
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
// $Log: /Speech_To_Speech_Translation/dbn/dbn/fileutil.h $
//
// 71 1/03/13 8:53p Kaisheny
// Asynchronous SGD using data pipe.
//
// 70 9/30/12 10:46a Fseide
// new optional parameter to fuptodate()--caller can now choose whether a
// missing input file, with target file present, will cause a failure or
// considers the target up-to-date
//
// 69 11/09/11 10:01 Fseide
// added a new overload for fgetfilelines() that returns an array of char*
// instead of strings, to avoid mem alloc
//
// 68 6/10/11 9:50 Fseide
// (fixed a missing 'inline')
//
// 67 6/10/11 9:49 Fseide
// new function fgetfilelines() for reading text files
//
// 66 6/09/11 15:18 Fseide
// added overloads to fexists() that accept STL strings
//
// 65 3/07/11 12:13 Fseide
// actually implemented unlinkOrDie() (was a dummy)
//
// 64 11/17/10 15:00 Fseide
// new function fuptodate();
// make_intermediate_dirs() moved to namespace msra::files (all new
// functions should be put in there)
//
// 63 11/15/10 7:04p Fseide
// added an overload for freadOrDie (vector) that takes size as a size_t
// instead of an int, to pleasr the x64 compiler
//
// 62 11/08/10 17:07 Fseide
// new function make_intermediate_dirs()
//
// 61 11/08/10 11:43 Fseide
// (minor cleanup)
//
// 60 2/05/09 19:06 Fseide
// fgetline() now returns a non-const pointer, because user may want to
// post-process the line, and the returned value is a user-specified
// buffer anyway
//
// 59 1/16/09 17:34 Fseide
// relpath() and splitpath() moved to fileutil.h
//
// 58 1/16/09 8:59 Fseide
// exported fskipspace()
//
// 57 1/15/09 7:38 Fseide
// some magic to unify fgetstring() for char and wchar_t to a single
// template function
//
// 56 1/15/09 7:26 Fseide
// corrected the #include order of basetypes.h
//
// 55 1/14/09 19:26 Fseide
// new functions fsetpos() and fgetpos();
// new fixed-buffer size overload for fgetstring() and fgettoken()
//
// 54 1/08/09 16:14 Fseide
// fopenOrDie() now supports "-" as the pathname, referring to stdin or
// stdout
//
// 53 1/08/09 15:32 Fseide
// new funtion expand_wildcards()
//
// 52 1/05/09 8:44 Fseide
// (added comments)
//
// 51 11/11/08 6:04p Qiluo
// recover the old fputstring functions
//
// 50 10/31/08 5:09p Qiluo
// remove banned APIs
//
// 49 7/17/08 7:22p V-spwang
// undid changes - back to version 47
//
// 47 6/24/08 19:03 Fseide
// added fgetwstring() and fputstring() for wstrings;
// added templates for freadOrDie() and fwriteOrDie() for STL vectors
//
// 46 6/18/08 11:41 Fseide
// added #pragma once
//
// 45 08-05-29 18:18 Llu
// fix the interface of fputwav
//
// 44 08-05-29 13:54 Llu
// add fputwav revise fgetwav using stl instead of short *
//
// 43 11/27/06 11:40 Fseide
// new methods fgetwfx() and fputwfx() for direct access to simple PCM WAV
// files
//
// 42 10/14/06 18:31 Fseide
// added char* version of fexists()
//
// 41 5/22/06 9:34 Fseide
// (experimental auto_file class checked in)
//
// 40 5/14/06 19:59 Fseide
// new function fsetmode()
//
// 39 3/29/06 15:36 Fseide
// changed to reading entire file instead of line-by-line, not changing
// newlines anymore
//
// 38 2/21/06 12:39p Kit
// Added filesize64 function
//
// 37 1/09/06 7:12p Rogeryu
// wide version of fgetline
//
// 36 12/19/05 21:52 Fseide
// fputfile() added in 8-bit string version
//
// 35 12/15/05 20:25 Fseide
// added getfiletime(), setfiletime(), and fputfile() for strings
//
// 34 9/27/05 12:22 Fseide
// added wstring version of renameOrDie()
//
// 33 9/22/05 12:26 Fseide
// new method fexists()
//
// 32 9/15/05 11:33 Fseide
// new version of fgetline() that avoids buffer allocations, since this
// seems very expensive esp. when reading a file line by line with
// fgetline()
//
// 31 9/05/05 4:57p F-xyzhao
// added #include <windows.h> for #include <mmreg.h> -- ugh
//
// 30 9/05/05 11:00 Fseide
// new method renameOrDie()
//
// 29 8/24/05 5:45p Kjchen
// merge changes in OneNote
//
// 28 8/19/05 17:56 Fseide
// extended WAVEHEADER with write() and update()
//
// 27 8/13/05 15:37 Fseide
// added new version of fgetline that takes a buffer
//
// 26 7/26/05 18:54 Fseide
// new functions fgetint24() and fputint24()
//
// 25 2/12/05 15:21 Fseide
// fgetdouble() and fputdouble() added
//
// 24 2/05/05 12:38 Fseide
// new methods fputfile(), fgetfile();
// new overload for filesize()
//
// 23 2/03/05 22:34 Fseide
// added new version of fgetline() that returns an STL string
//
// 22 5/31/04 10:06 Fseide
// new methods fseekOrDie(), ftellOrDie(), unlinkOrDie(), renameOrDie()
//
// 21 3/19/04 4:01p Fseide
// fwriteOrDie(): first argument changed to const
//
// 20 2/27/04 10:04a V-xlshi
//
// 19 2/19/04 3:45p V-xlshi
// fgetraw function is added.
//
// 18 2/19/04 1:49p V-xlshi
//
// 17 2/03/04 8:17p V-xlshi
//
// 16 2/03/04 6:20p V-xlshi
// WAVEHEADER.prepare() added
//
// 15 2/03/04 5:58p V-xlshi
// WAVEHEADER structure added
//
// 14 8/15/03 15:40 Fseide
// new method filesize()
//
// 13 8/13/03 21:06 Fseide
// new function fputbyte()
//
// 12 8/13/03 15:37 Fseide
// prototype of fOpenOrDie() Unicode version changed
//
// 11 8/07/03 22:04 Fseide
// fprintfOrDie() now really dies in case of error
//
// 10 03-07-30 12:06 I-rogery
// enable both unicode and non-unicode version
//
// 9 7/25/03 6:07p Fseide
// new functions fgetbyte() and fgetwav()
//
// 8 7/03/02 9:25p Fseide
// fcompareTag() now uses string type for both of its arguments (before,
// it used const char * for one of them)
//
// 7 6/10/02 3:14p Fseide
// new functions fgettoken(), fgetfloat_ascii(), fskipNewline()
//
// 6 6/07/02 7:26p Fseide
// new functions fcheckTag_ascii() and fgetint_ascii()
//
// 5 4/15/02 1:12p Fseide
// void fputstring (FILE * f, const TSTRING & str) and fpad() added
//
// 4 4/03/02 3:58p Fseide
// VSS keyword and copyright added
//
// F. Seide 5 Mar 2002
//
#pragma once
#ifndef _FILEUTIL_
#define _FILEUTIL_
#include <stdio.h>
#include <windows.h> // for mmreg.h and FILETIME
#include <mmreg.h>
#ifdef _WIN32
#define isfinite(x) _finite(x)
#define isnan(x) _isnan(x)
#endif
#ifdef __unix__
#include <sys/types.h>
#include <sys/stat.h>
#endif
#include <algorithm> // for std::find
#include <vector>
#include <map>
@ -239,6 +24,7 @@
#include <errno.h>
#include <stdint.h>
#include <assert.h>
#include <string.h> // for strerror()
using namespace std;
@ -313,11 +99,9 @@ size_t filesize (FILE * f);
int64_t filesize64 (const wchar_t * pathname);
// ----------------------------------------------------------------------------
// fseekOrDie(),ftellOrDie(), fget/setpos(): seek functions with error handling
// fget/setpos(): seek functions with error handling
// ----------------------------------------------------------------------------
size_t fseekOrDie (FILE * f, size_t offset, int mode = SEEK_SET);
#define ftellOrDie _ftelli64
uint64_t fgetpos (FILE * f);
void fsetpos (FILE * f, uint64_t pos);
@ -670,13 +454,6 @@ namespace msra { namespace files {
vector<char*> fgetfilelines (const wstring & pathname, vector<char> & readbuffer);
};};
// ----------------------------------------------------------------------------
// getfiletime(), setfiletime(): access modification time
// ----------------------------------------------------------------------------
bool getfiletime (const std::wstring & path, FILETIME & time);
void setfiletime (const std::wstring & path, const FILETIME & time);
// ----------------------------------------------------------------------------
// expand_wildcards() -- expand a path with wildcards (also intermediate ones)
// ----------------------------------------------------------------------------
@ -699,10 +476,34 @@ namespace msra { namespace files {
bool fuptodate (const wstring & target, const wstring & input, bool inputrequired = true);
};};
#if 0
// ----------------------------------------------------------------------------
// simple support for WAV file I/O
// ----------------------------------------------------------------------------
// define the header if we haven't seen it yet
#ifndef _WAVEFORMATEX_
#define _WAVEFORMATEX_
/*
* extended waveform format structure used for all non-PCM formats. this
* structure is common to all non-PCM formats.
*/
typedef unsigned short WORD; // in case not defined yet (i.e. linux)
typedef struct tWAVEFORMATEX
{
WORD wFormatTag; /* format type */
WORD nChannels; /* number of channels (i.e. mono, stereo...) */
DWORD nSamplesPerSec; /* sample rate */
DWORD nAvgBytesPerSec; /* for buffer estimation */
WORD nBlockAlign; /* block size of data */
WORD wBitsPerSample; /* number of bits per sample of mono data */
WORD cbSize; /* the count in bytes of the size of */
/* extra information (after cbSize) */
} WAVEFORMATEX, *PWAVEFORMATEX;
#endif /* _WAVEFORMATEX_ */
typedef struct wavehder{
char riffchar[4];
unsigned int RiffLength;
@ -738,6 +539,7 @@ void fputwfx (FILE *f, const WAVEFORMATEX & wfx, unsigned int numSamples);
// channel. j is sample index.
// ----------------------------------------------------------------------------
void fgetraw (FILE *f,std::vector< std::vector<short> > & data,const WAVEHEADER & wavhd);
#endif
// ----------------------------------------------------------------------------
// temp functions -- clean these up

675
Common/Include/message.h
Просмотреть файл

@ -1,675 +0,0 @@
//
// message.h - class for simple I/O of log messages
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
// $Log: /Speech_To_Speech_Translation/dbn/dbn/message.h $
//
// 66 7/05/11 8:17 Fseide
// error() now prints the Win32 error code as well, maybe we can now track
// down the unreliable-server problem
//
// 65 11/30/09 1:33p Kit
// updated to compile under winCE
//
// 64 6/07/09 0:00 Fseide
// (added a comment)
//
// 63 5/18/09 15:28 Fseide
// minor bug fix in an error message in __flush()
//
// 62 1/08/09 16:14 Fseide
// moved timeDateStamp() here, as it is often used in logging
//
// 61 1/08/09 9:23 Fseide
// moved _CHECKED_ASSERT_error() to message.h, finally getting rid of
// message.cpp
//
// 60 12/09/08 6:59p Qiluo
// reverted stringerror => strerror
//
// 59 12/09/08 6:29p Qiluo
// strerror => stringerror
//
// 58 11/14/08 7:43p Qiluo
// mark banned APIs
//
// 57 11/11/08 18:19 Fseide
// no longer disables 4996
//
// 56 11/11/08 17:56 Fseide
// (a comment added)
//
// 55 11/11/08 17:55 Fseide
// replaced strbXXX() calls with safe fixed-buffer overloads
//
// 54 11/11/08 17:47 Fseide
// fixed use of Xprintf() functions to use fixed-size overloads
//
// 53 11/11/08 15:08 Fseide
// replaced safe(r) _vsnprintf() by unsafer vsprintf() assuming there is
// an overload to make it safe again...
//
// 52 11/11/08 14:52 Fseide
// (added a comment)
//
// 51 6/18/08 11:41 Fseide
// added #pragma once
//
// 50 29/05/08 4:58p Kit
// G.muted semantics changed - now only controls whether logging goes to
// stderr or not
//
// 49 29/05/08 3:19p Kit
// changed semantics of G.muted - now G.muted only controls logging to
// stderr - so if G.muted is one but log file is specified, it will still
// log to the file
//
// 48 10/01/07 13:19 Fseide
// added setHeavyLogging() and noFlush flag for cases where a lot of info
// is written to the log file
//
// 47 27/06/07 5:11p Kit
// rolled back to version 45
//
// 45 27/06/07 4:58p Kit
// changed a few more methods to inline to avoid linker errors
//
// 44 27/06/07 4:54p Kit
// changed a few methods to inline to avoid multiple include linker errors
//
// 43 5/08/07 16:29 Fseide
// increased output buffer size to 30K
//
// 42 4/11/07 17:24 Fseide
// fixed a bug for string overflow for vsnprintf(), 0-terminator missing
//
// 41 07-04-11 14:57 Qfyin
// added a std::
//
// 40 3/28/07 11:57 Fseide
// fixed the C4702 problem with error() and mem_error()
//
// 39 3/27/07 20:54 Fseide
// silly compiler warning again (inconsistent warning between Release and
// Debug about unreachable code)
//
// 38 3/27/07 20:49 Fseide
// fixed a compiler warning
//
// 37 3/27/07 17:58 Fseide
// added namespace qualifiers
//
// 36 3/27/07 15:59 Fseide
// changed struct back to namespace, uniqueness problem of static g solved
// by moving _glob() into __globals
//
// 35 3/27/07 15:23 Fseide
// added private/public qualifiers back in
//
// 34 3/27/07 15:19 Fseide
// changed namespace into struct (namespace has problems with the shared
// state)
//
// 33 3/27/07 15:14 Fseide
// fixed compiler warnings
//
// 32 3/27/07 13:53 Fseide
// removed 'static' markers as they led to warnings
//
// 31 3/27/07 13:49 Fseide
// changed from class HDM_CLog to namespace msra::logging;
// now does not require message.cpp anymore
//
// 30 2/14/07 15:38 Fseide
// (fixed compiler warnings when compiling managed)
//
// 29 11/22/06 6:39p Rogeryu
// new function getLogFile
//
// 28 5/30/06 6:42p Rogeryu
// refine the log handle reconnection
//
// 27 5/24/06 2:51p Rogeryu
//
// 26 3/24/06 4:40p Rogeryu
// workaround a VC 2003 header bug (va_start macro for references) in
// MESSAGE/ERROR functions
//
// 25 3/24/06 13:33 Fseide
// cleaned up C4996 (back to file level to keep code tidy)
//
// 24 3/22/06 5:44p Rogeryu
// change to strbxxx macros
//
// 23 3/22/06 4:57p Rogeryu
// refine comments
//
// 22 3/21/06 5:21p Rogeryu
// review and fix level2_security OACR warnings
//
// 21 06-03-14 14:32 Yushli
//
// 20 06-03-14 11:58 Yushli
// Suppress C4996 Warning on strerror per function
//
// 19 06-03-14 11:44 Yushli
// Suppress C4996 Warning on strcpy per function
//
// 18 2/24/06 8:03p Kjchen
// depress oacr warnings
//
// 17 9/25/05 12:04p Kjchen
// merge OneNote's change
//
// 16 9/21/05 11:26 Fseide
// output changed from << to fwrite
//
// 15 5/09/05 11:09p Kjchen
// add: using namespace std;
//
// 14 2/17/05 10:32 Fseide
// added muted mode and new method shutUp()
//
// 13 2/03/05 19:37 Fseide
// removed unnecessary dependence on fileutil.h;
// removed dependence on critsec.h (CCritSec now in basetypes.h)
//
// 12 4/19/04 18:58 Fseide
// showbuf() now does not anymore use "-orDie" functions to avoid infinite
// recursion...
//
// 11 2/21/04 10:26 Fseide
// (compiler warnings eliminated)
//
// 10 7/31/03 12:37p Fseide
// ERROR() can now throw an exception instead of exit()-ing;
// new method HDM_CLog::setExceptionFlag();
// new class message_exception
//
// 9 8/16/02 7:14p Fseide
// now thread-safe
//
// 8 8/01/02 7:48p Fseide
// new function (macro) memError (MEMERROR) to display an error in case of
// out-of-memory (ERROR allocates memory itself)
//
// 7 7/31/02 10:13a Fseide
// implemented logging to file (accessed through SETLOGFILE() macro)
//
// 6 4/03/02 3:58p Fseide
// VSS keyword and copyright added
//
// F. Seide 5 Mar 2002
//
#pragma once
#ifndef _MESSAGE_
#define _MESSAGE_
#ifndef UNDER_CE // fixed-buffer overloads not available for wince
#ifdef _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES // fixed-buffer overloads for strcpy() etc.
#undef _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES
#endif
#define _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES 1
#endif
#include "basetypes.h"
#include <stdarg.h>
#include <iostream>
#include <exception>
#include <time.h> // for _time64 in timeDateStamp()
#pragma warning (push)
#pragma warning (disable : 4793) // caused by varargs
namespace msra { namespace logging
{
// -----------------------------------------------------------------------
// message_exception - exception thrown by this error module
// -----------------------------------------------------------------------
class message_exception : public std::runtime_error
{
char buf[1024]; // local buffer for message
const char * dup_what (const char * what)
{
strcpy (buf, "message_exception:");
strcat (buf, what);
return &buf[0];
}
public:
message_exception (const char * what) : runtime_error (dup_what (what))
{
}
};
// -----------------------------------------------------------------------
// global state (hidden as a static struct in a variable)
// -----------------------------------------------------------------------
struct __globals
{
msra::basetypes::CCritSec lock;
FILE * logFile;
bool noFlush; // heavy logging: don't flush
bool throwExceptionFlag; // true: don't exit but throw exception
// G.muted semantics is as follows
// - G.muted && !G.logFile => don't show anything
// - G.muted && G.logFile => write to log file
// - !G.muted && !G.logFile => write to stderr only
// - !G.muted && G.logFile => write to log file and stderr
bool muted;
std::string filename; // log file name
char buf[30000]; // for _vsnprintf()
__globals() : logFile (NULL), throwExceptionFlag (true), muted (false), noFlush (false)
{ buf[0] = 0; buf[sizeof (buf) / sizeof (*buf) -1] = 0; }
static __globals & get() { static __globals g; return g; }
};
#pragma push_macro ("G")
#define G (__globals::get()) // access global XYZ as G.XYZ
// ------------------------------------------------------------------------
// setLogFile(): set the log file
// if non-NULL then every message will be written both to stderr and this
// log file.
// multi-threading: not thread-safe, set this before starting
// ------------------------------------------------------------------------
static inline FILE * setLogFile (FILE * newLogFile)
{
FILE * oldLogFile;
oldLogFile = G.logFile;
if (newLogFile != stderr)
{
G.logFile = newLogFile;
}
else
{
G.logFile = NULL;
}
return oldLogFile;
}
// ------------------------------------------------------------------------
// setLogFileByName(): set the log file by file name
// in this mode, log file will be re-connected when disconnected
// filename == NULL indicates an attempt to reconnect
// WARNING: if the filename is invalid, it will try to reconnect every time
// ------------------------------------------------------------------------
static inline void setLogFileByName (const char * p_filename)
{
FILE * newLogFile = NULL;
if (p_filename == NULL)
{
// for reconnection
ASSERT (G.filename != "");
newLogFile = fopen (G.filename.c_str (), "ab");
}
else
{
ASSERT (p_filename[0]);
G.filename = p_filename; // remember filename
newLogFile = fopen (p_filename, "wb");
}
// handle open failure
if (newLogFile == NULL)
{
if (G.logFile != NULL)
{
fprintf (G.logFile, "ERROR: setLogFileByName: error opening log file %s: %s\n",
G.filename.c_str (), strerror (errno));
// in case of a reconnect, this ^^ will obviously fail, we ignore this
}
fprintf (stderr, "ERROR: setLogFileByName: error opening log file %s: %s\n",
G.filename.c_str (), strerror (errno));
return;
}
// set new handle
FILE * oldLogFile = setLogFile (newLogFile);
// close old handle
if (oldLogFile != NULL && oldLogFile != stderr && oldLogFile != stdin)
{
int rc = fclose (oldLogFile);
if (rc != 0)
{
if (G.logFile != NULL)
{ // note: this goes to the new log file
fprintf (G.logFile, "ERROR: setLogFileByName: error closing old log file: %s\n",
strerror (errno));
}
fprintf (stderr, "ERROR: setLogFileByName: error closing old log file: %s\n",
strerror (errno));
}
}
}
// ------------------------------------------------------------------------
// setExceptionFlag(): set flag whether to throw an exception (true) or exit() (false, default)
// ------------------------------------------------------------------------
static inline bool setExceptionFlag (bool throwExceptionFlag = true)
{
bool oldFlag = G.throwExceptionFlag;
G.throwExceptionFlag = throwExceptionFlag;
return oldFlag;
}
// ------------------------------------------------------------------------
// timeDateStamp() -- often needed for logging
// ------------------------------------------------------------------------
static inline std::string timeDateStamp (void)
{
#ifdef _MSC_VER
__time64_t localtime; _time64 (&localtime); // get current time and date
struct tm now = *_localtime64 (&localtime); // convert
char buf[20];
sprintf (buf, "%04d/%02d/%02d %02d:%02d:%02d",
now.tm_year + 1900, now.tm_mon + 1, now.tm_mday,
now.tm_hour, now.tm_min, now.tm_sec);
return buf;
#else
return "(now)"; // TODO: fix this for GCC builds
#endif
}
// ------------------------------------------------------------------------
// __flush(): flush output
// ------------------------------------------------------------------------
static inline void __flush()
{
int rc = fflush (G.logFile);
if (rc != 0)
{
fprintf (stderr, "ERROR: __flush: error flushing to log file %s\n",
strerror (errno));
}
}
// ------------------------------------------------------------------------
// setHeavyLogging(): we are heavily logging: don't flush & increase out buf
// ------------------------------------------------------------------------
static inline void setHeavyLogging (bool isHeavy)
{
__flush(); // flush the current buffer
if (!isHeavy)
{
G.noFlush = false;
}
else
{
G.noFlush = true;
if (G.logFile)
setvbuf (G.logFile, NULL, _IOFBF, 16384); // flush every 16K
}
}
// ------------------------------------------------------------------------
// shutUp(): set muted mode (true: no output will be generated anymore)
//
// multi-threading: retrieving the previous state is not thread-safe,
// if you want, do this before starting
// ------------------------------------------------------------------------
static inline bool shutUp (bool quiet = true)
{
bool oldFlag = G.muted;
G.muted = quiet;
return oldFlag;
}
// ------------------------------------------------------------------------
// getLogFile(): get log file handle
// ------------------------------------------------------------------------
static inline FILE * getLogFile (void)
{
return G.logFile;
}
// ------------------------------------------------------------------------
// __showbuf(): output contents of buf[] with prefix prepended
// multi-threading: must be called from within critical section
// ------------------------------------------------------------------------
static inline void __showbuf (const std::string & prefix, bool nl)
{
ASSERT (strlen (G.buf) < sizeof (G.buf) / sizeof (*G.buf));
std::string outtext = prefix + G.buf;
if (nl) outtext += "\n";
// write out; first to screen in case we can't write to log file
#ifndef ONENOTE_COMPILER
// OneNote treats it as an error if stderr is not empty.
// and in OneNote, we can't see message printed to stderr
// So, in OneNote, don't put it into stderr
// G.muted semantics is as follows
// - G.muted && !G.logFile => don't show anything
// - G.muted && G.logFile => write to log file
// - !G.muted && !G.logFile => write to stderr only
// - !G.muted && G.logFile => write to log file and stderr
if (!G.muted)
{
fwrite ((void*) outtext.c_str(), sizeof (*outtext.c_str()),
outtext.length(), stderr);
if (!G.noFlush)
fflush (stderr);
}
#endif
// write to log file
// check whether the log file has been disconnected or not
if (G.filename != "") // with known filename, suppose to reconnect
{
if (G.logFile == NULL || ferror (G.logFile) != 0)
{
setLogFileByName (NULL); // attempt to re-open the log file
if (G.logFile)
{
fprintf (G.logFile, "ERROR: __showbuf: log file handle lost, reconnected\n");
}
}
}
if (G.logFile)
{
size_t n = fwrite ((void*) outtext.c_str(), sizeof (*outtext.c_str()),
outtext.length(), G.logFile);
if (n != outtext.length() * sizeof (*outtext.c_str()))
{ // write error
fprintf (stderr, "ERROR: __showbuf: error writing this to log file: %s\n", strerror (errno));
fwrite ((void*) outtext.c_str(), sizeof (*outtext.c_str()),
outtext.length(), stderr);
}
else if (!G.noFlush) // flush logFile
{
__flush();
}
}
}
// ------------------------------------------------------------------------
// noLoggingReqd: function to determine if any logging reqd
// at all - used so that we can exit early if none reqd
// ------------------------------------------------------------------------
static inline bool noLoggingReqd()
{
return G.muted && !G.logFile;
}
// ------------------------------------------------------------------------
// message(): like printf(), writing to log output
// multi-threading: this function is thread-safe
// ------------------------------------------------------------------------
static inline void message (const char * fmt, ...)
{
if (noLoggingReqd()) return; // muted: all output is suppressed
msra::basetypes::CAutoLock autoLock (G.lock);
va_list arg_ptr;
va_start (arg_ptr, fmt);
vsprintf (G.buf, fmt, arg_ptr);
__showbuf ("", true);
}
static void message_nolf (const char * fmt, ...)
{
if (noLoggingReqd()) return; // muted: all output is suppressed
msra::basetypes::CAutoLock autoLock (G.lock);
va_list arg_ptr;
va_start (arg_ptr, fmt);
vsprintf (G.buf, fmt, arg_ptr);
__showbuf ("", false);
}
// ------------------------------------------------------------------------
// warning(): like message(), with text "WARNING: " prepended
// multi-threading: this function is thread-safe
// ------------------------------------------------------------------------
static void warning (const char * fmt, ...)
{
if (noLoggingReqd()) return; // muted: all output is suppressed
msra::basetypes::CAutoLock autoLock (G.lock);
va_list arg_ptr;
va_start (arg_ptr, fmt);
vsprintf (G.buf, fmt, arg_ptr);
__showbuf ("WARNING: ", true);
__flush();
}
// ------------------------------------------------------------------------
// __throw_or_exit(): exit() or throw exception depending on throwExceptionFlag
// ------------------------------------------------------------------------
static inline void __throw_or_exit (void)
{
__flush();
if (G.throwExceptionFlag)
{
throw message_exception (G.buf);
}
exit (1);
}
// ------------------------------------------------------------------------
// error(): like warning() but terminates program afterwards
// multi-threading: this function is thread-safe
// ------------------------------------------------------------------------
#pragma warning (push)
#pragma warning (disable : 4702) // the 'return 0;' causes this in Release
static int error (const char * fmt, ...)
{
#if 0 // special test code to determine the Windows error in case of a network error
DWORD winErr = GetLastError();
try
{
msra::basetypes::CAutoLock autoLock (G.lock);
sprintf (G.buf, "%d (\"%S\")", winErr, FormatWin32Error(winErr).c_str());
if (!noLoggingReqd())
__showbuf ("Win32 error of subsequent error message: ", true);
}
catch(...){}
#endif
msra::basetypes::CAutoLock autoLock (G.lock);
va_list arg_ptr;
va_start (arg_ptr, fmt);
vsprintf (G.buf, fmt, arg_ptr);
if (!noLoggingReqd())
{ // if muted, we format the msg (for __throw_or_exit) but don't print it
__showbuf ("ERROR: ", true);
}
__throw_or_exit();
return 0;
}
// ------------------------------------------------------------------------
// mem_error(): similar to error() but without any memory allocations
// (only one string argument allowed)
// multi-threading: this function is thread-safe
// ------------------------------------------------------------------------
static int mem_error (const char * fmt, int arg)
{
msra::basetypes::CAutoLock autoLock (G.lock);
if (!noLoggingReqd())
{ // if muted, we format the msg (for __throw_or_exit) but don't print it
fprintf (stderr, fmt, arg);
fprintf (stderr, "\n");
if (G.logFile)
{
fprintf (G.logFile, fmt, arg);
fprintf (G.logFile, "\n");
int rc = fflush (G.logFile);
if (rc != 0)
{
fprintf (stderr, "error flushing log message to file: %s\n",
strerror (errno));
}
}
}
// format msg for __throw_or_exit()
sprintf (G.buf, fmt, arg);
strcat (G.buf, "\n");
__throw_or_exit();
return 0;
}
#pragma warning (pop)
static inline void __avoid_C4505 (void)
{ message (""); message_nolf (""); warning (""); error (""); mem_error ("", 0); }
#pragma pop_macro ("G")
};};
#pragma warning(pop)
// ===========================================================================
// compatibility macros (for older source code)
// ===========================================================================
#undef ERROR // defined in wingdi.h... aargh!
#define WARNING msra::logging::warning
#define ERROR msra::logging::error
#define MESSAGE msra::logging::message
#define MESSAGE_NOLF msra::logging::message_nolf
#define MEMERROR msra::logging::mem_error
#define SETLOGFILE msra::logging::setLogFile
// ===========================================================================
// special function for basetypes.h's ASSERT() macro
// ===========================================================================
#ifdef _CHECKED
void inline _CHECKED_ASSERT_error(const char * file, int line, const char * exp)
{ ERROR ("%s:%d:assertion failure: %s", file, line, exp); }
#endif
#endif // _MESSAGE_

5
Common/Include/stdafx.h Normal file
Просмотреть файл

@ -0,0 +1,5 @@
// dummy stdafx.h file for Linux version
// In the Windows build, there are several stdafx.h files which are used for project-specific precompilation of headers.
// In Linux, this is not used. By placing this dummy file ahead of all other stdafx.h in the include search path,
// we make sure it compiles while not getting confused by Windows-specific content of those files.

255
Common/fileutil.cpp
Просмотреть файл

@ -6,6 +6,8 @@
#define _CRT_SECURE_NO_WARNINGS // "secure" CRT not available on all platforms --add this at the top of all CPP files that give "function or variable may be unsafe" warnings
#define _CRT_NONSTDC_NO_DEPRECATE // make VS accept POSIX functions without _
#pragma warning (disable: 4996) // ^^ this does not seem to work--TODO: make it work
#define _FILE_OFFSET_BITS = 64 // for ftell64() in Linux
#ifndef UNDER_CE // fixed-buffer overloads not available for wince
#ifdef _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES // fixed-buffer overloads for strcpy() etc.
@ -15,8 +17,9 @@
#endif
#include "basetypes.h"
#include "fileutil.h"
#include "message.h"
#ifdef __unix__
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#endif
#include <stdio.h>
@ -24,7 +27,9 @@
#include <ctype.h>
#include <stdint.h>
#include <assert.h>
#ifdef _WIN32
#include "Windows.h" // for FILETIME
#endif
#include <algorithm> // for std::find
#include <limits.h>
@ -39,6 +44,15 @@
using namespace std;
// ----------------------------------------------------------------------------
// some mappings for non-Windows builds
// ----------------------------------------------------------------------------
#ifndef _MSC_VER // add some functions that are VS-only
static int _wunlink (const wchar_t * p) { return unlink (charpath (p)); }
static int _wmkdir (const wchar_t * p) { return mkdir (charpath (p), 0777/*correct?*/); }
#endif
template <> const wchar_t* GetScanFormatString(char) {return L" %hc";}
template <> const wchar_t* GetScanFormatString(wchar_t) {return L" %lc";}
template <> const wchar_t* GetScanFormatString(short) {return L" %hi";}
@ -46,12 +60,10 @@ template <> const wchar_t* GetScanFormatString(int) {return L" %i";
template <> const wchar_t* GetScanFormatString(long) {return L" %li";}
template <> const wchar_t* GetScanFormatString(unsigned short) {return L" %hu";}
template <> const wchar_t* GetScanFormatString(unsigned int) {return L" %u";}
template <> const wchar_t* GetScanFormatString(unsigned long) {return L" %lu";}
//template <> const wchar_t* GetScanFormatString(unsigned long) {return L" %lu";}
template <> const wchar_t* GetScanFormatString(float) {return L" %g";}
template <> const wchar_t* GetScanFormatString(double) {return L" %lg";}
#if (SIZE_MAX != UINT_MAX) // on 32 bit platforms, the following will be flagged as a redefinition
template <> const wchar_t* GetScanFormatString(size_t) {return L" %llu";}
#endif
template <> const wchar_t* GetScanFormatString(long long) {return L" %lli";}
template <> const wchar_t* GetFormatString(char) {return L" %hc";}
@ -61,12 +73,10 @@ template <> const wchar_t* GetFormatString(int) {return L" %i";}
template <> const wchar_t* GetFormatString(long) {return L" %li";}
template <> const wchar_t* GetFormatString(unsigned short) {return L" %hu";}
template <> const wchar_t* GetFormatString(unsigned int) {return L" %u";}
template <> const wchar_t* GetFormatString(unsigned long) {return L" %lu";}
//template <> const wchar_t* GetFormatString(unsigned long) {return L" %lu";}
template <> const wchar_t* GetFormatString(float) {return L" %.9g";}
template <> const wchar_t* GetFormatString(double) {return L" %.17g";}
#if (SIZE_MAX != UINT_MAX)
template <> const wchar_t* GetFormatString(size_t) { return L" %llu"; }
#endif
template <> const wchar_t* GetFormatString(long long) {return L" %lli";}
// ----------------------------------------------------------------------------
@ -77,9 +87,9 @@ void fgetText(FILE * f, char& v)
const wchar_t* formatString = GetFormatString(v);
int rc = fwscanf(f, formatString, &v);
if (rc == 0)
ERROR ("error reading value from file (invalid format): %s", formatString);
RuntimeError ("error reading value from file (invalid format): %s", formatString);
else if (rc == EOF)
ERROR ("error reading from file: %s", strerror (errno));
RuntimeError ("error reading from file: %s", strerror (errno));
assert(rc == 1);
}
void fgetText(FILE * f, wchar_t& v)
@ -87,9 +97,9 @@ void fgetText(FILE * f, wchar_t& v)
const wchar_t* formatString = GetFormatString(v);
int rc = fwscanf(f, formatString, &v);
if (rc == 0)
ERROR ("error reading value from file (invalid format): %s", formatString);
RuntimeError ("error reading value from file (invalid format): %s", formatString);
else if (rc == EOF)
ERROR ("error reading from file: %s", strerror (errno));
RuntimeError ("error reading from file: %s", strerror (errno));
assert(rc == 1);
}
@ -116,7 +126,7 @@ FILE * fopenOrDie (const string & pathname, const char * mode)
FILE * f = (pathname[0] == '-') ? fopenStdHandle (mode) : fopen (pathname.c_str(), mode);
if (f == NULL)
{
ERROR ("error opening file '%s': %s", pathname.c_str(), strerror (errno));
RuntimeError ("error opening file '%s': %s", pathname.c_str(), strerror (errno));
return NULL; // keep OACR happy
}
if (strchr (mode, 'S'))
@ -131,7 +141,7 @@ FILE * fopenOrDie (const wstring & pathname, const wchar_t * mode)
FILE * f = (pathname[0] == '-') ? fopenStdHandle (mode) : _wfopen (pathname.c_str(), mode);
if (f == NULL)
{
ERROR ("error opening file '%S': %s", pathname.c_str(), strerror (errno));
RuntimeError ("error opening file '%S': %s", pathname.c_str(), strerror (errno));
return NULL; // keep OACR happy
}
if (strchr (mode, 'S'))
@ -153,7 +163,7 @@ void fsetmode(FILE * f, char type)
{
if (type != 'b' && type != 't')
{
ERROR ("fsetmode: invalid type '%c'");
RuntimeError ("fsetmode: invalid type '%c'", type);
}
#ifdef UNDER_CE // winCE and win32 have different return types for _fileno
FILE *fd = fileno (f); // note: no error check possible
@ -164,7 +174,7 @@ void fsetmode(FILE * f, char type)
int rc = setmode (fd, mode);
if (rc == -1)
{
ERROR ("error changing file mode: %s", strerror (errno));
RuntimeError ("error changing file mode: %s", strerror (errno));
}
}
@ -180,7 +190,7 @@ void freadOrDie (void * ptr, size_t size, size_t count, FILE * f)
size_t chunkn = min (count, (size_t)15*1024*1024); // BUGBUG: I surely meant this limit to be bytes, not units of 'size'...
size_t n = fread (ptr, size, chunkn, f);
if (n != chunkn)
ERROR ("error reading from file: %s", strerror (errno));
RuntimeError ("error reading from file: %s", strerror (errno));
count -= n;
ptr = n * size + (char*) ptr;
}
@ -207,7 +217,7 @@ void fwriteOrDie (const void * ptr, size_t size, size_t count, FILE * f)
size_t n = fwrite ((const void *) p1, 1, wantWrite, f);
if (n != wantWrite)
{
ERROR ("error writing to file (ptr=0x%08lx, size=%d,"
RuntimeError ("error writing to file (ptr=0x%08lx, size=%d,"
" count=%d, writing %d bytes after %d): %s",
ptr, size, count, (int) wantWrite,
(int) (size * count - totalBytes),
@ -232,7 +242,7 @@ void fprintfOrDie (FILE * f, const char * fmt, ...)
int rc = vfprintf (f, fmt, arg_ptr);
if (rc < 0)
{
ERROR ("error writing to file: %s", strerror (errno));
RuntimeError ("error writing to file: %s", strerror (errno));
}
}
#pragma warning(pop)
@ -246,7 +256,7 @@ void fflushOrDie (FILE * f)
int rc = fflush (f);
if (rc != 0)
{
ERROR ("error flushing to file: %s", strerror (errno));
RuntimeError ("error flushing to file: %s", strerror (errno));
}
}
@ -255,27 +265,30 @@ void fflushOrDie (FILE * f)
// ----------------------------------------------------------------------------
size_t filesize (FILE * f)
{
#ifdef _WIN32
size_t curPos = _ftelli64(f);
if (curPos == -1L)
{
ERROR ("error determining file position: %s", strerror (errno));
RuntimeError ("error determining file position: %s", strerror (errno));
}
int rc = _fseeki64 (f, 0, SEEK_END);
if (rc != 0)
{
ERROR ("error seeking to end of file: %s", strerror (errno));
}
RuntimeError ("error seeking to end of file: %s", strerror (errno));
size_t len = _ftelli64 (f);
if (len == -1L)
{
ERROR ("error determining file position: %s", strerror (errno));
}
RuntimeError ("error determining file position: %s", strerror (errno));
rc = _fseeki64 (f, curPos, SEEK_SET);
if (rc != 0)
{
ERROR ("error resetting file position: %s", strerror (errno));
}
RuntimeError ("error resetting file position: %s", strerror (errno));
return len;
#else // TODO: test this
struct stat stat_buf;
int rc = fstat(fileno(f), &stat_buf);
if (rc != 0)
RuntimeError("error determining length of file: %s", strerror(errno));
static_assert (sizeof(stat_buf.st_size)>=sizeof(uint64_t), "struct stat not compiled for 64-bit mode");
return stat_buf.st_size;
#endif
}
// filesize(): determine size of the file in bytes (with pathname)
@ -300,39 +313,28 @@ size_t filesize (const wchar_t * pathname)
// filesize64(): determine size of the file in bytes (with pathname)
int64_t filesize64 (const wchar_t * pathname)
{
__stat64 fileinfo;
#ifdef _WIN32
struct _stat64 fileinfo;
if (_wstat64 (pathname,&fileinfo) == -1)
return 0;
else
return fileinfo.st_size;
#else
return filesize (pathname);
#endif
}
#endif
// ----------------------------------------------------------------------------
// fseekOrDie(),ftellOrDie(), fget/setpos(): seek functions with error handling
// fget/setpos(): seek functions with error handling
// ----------------------------------------------------------------------------
size_t fseekOrDie (FILE * f, size_t offset, int mode)
{
size_t curPos = _ftelli64 (f);
if (curPos == -1L)
{
ERROR ("error seeking: %s", strerror (errno));
}
int rc = _fseeki64 (f, offset, mode);
if (rc != 0)
{
ERROR ("error seeking: %s", strerror (errno));
}
return curPos;
}
uint64_t fgetpos (FILE * f)
{
fpos_t post;
int rc = ::fgetpos (f, &post);
if (rc != 0)
ERROR ("error getting file position: %s", strerror (errno));
RuntimeError ("error getting file position: %s", strerror (errno));
return post;
}
@ -367,7 +369,7 @@ void fsetpos (FILE * f, uint64_t reqpos)
fpos_t post = reqpos;
int rc = ::fsetpos (f, &post);
if (rc != 0)
ERROR ("error setting file position: %s", strerror (errno));
RuntimeError ("error setting file position: %s", strerror (errno));
}
// ----------------------------------------------------------------------------
@ -377,15 +379,12 @@ void fsetpos (FILE * f, uint64_t reqpos)
void unlinkOrDie (const std::string & pathname)
{
if (unlink (pathname.c_str()) != 0 && errno != ENOENT) // if file is missing that's what we want
ERROR ("error deleting file '%s': %s", pathname.c_str(), strerror (errno));
RuntimeError ("error deleting file '%s': %s", pathname.c_str(), strerror (errno));
}
#ifndef _MSC_VER
static int _wunlink (const wchar_t * p) { return unlink (msra::strfun::wcstombs (p).c_str()); }
#endif
void unlinkOrDie (const std::wstring & pathname)
{
if (_wunlink (pathname.c_str()) != 0 && errno != ENOENT) // if file is missing that's what we want
ERROR ("error deleting file '%S': %s", pathname.c_str(), strerror (errno));
RuntimeError ("error deleting file '%S': %s", pathname.c_str(), strerror (errno));
}
// ----------------------------------------------------------------------------
@ -394,14 +393,23 @@ void unlinkOrDie (const std::wstring & pathname)
void renameOrDie (const std::string & from, const std::string & to)
{
#ifdef _WIN32
if (!MoveFileA (from.c_str(),to.c_str()))
ERROR ("error renaming: %s", GetLastError());
RuntimeError("error renaming: %s", GetLastError());
#else // TODO: test this
if (!rename (from.c_str(), to.c_str()))
RuntimeError("error renaming file '%s': %s", from.c_str(), strerror(errno));
#endif
}
void renameOrDie (const std::wstring & from, const std::wstring & to)
{
if (!MoveFileW (from.c_str(),to.c_str()))
ERROR ("error renaming: %s", GetLastError());
#ifdef _WIN32
if (!MoveFileW(from.c_str(), to.c_str()))
RuntimeError ("error renaming: %s", GetLastError());
#else
renameOrDie (charpath(from), charpath(to));
#endif
}
// ----------------------------------------------------------------------------
@ -423,7 +431,7 @@ bool fexists (const wchar_t * pathname)
return false;
}
#else
auto_file_ptr f = _wfopen (pathname, L"r");
auto_file_ptr f (_wfopen (pathname, L"r"));
return f != nullptr;
#endif
}
@ -443,7 +451,7 @@ bool fexists (const char * pathname)
return false;
}
#else
auto_file_ptr f = fopen (pathname, "r");
auto_file_ptr f (fopen (pathname, "r"));
return f != nullptr;
#endif
}
@ -469,12 +477,6 @@ bool funicode (FILE * f)
// Returns 'buf' (always). buf guaranteed to be 0-terminated.
// ----------------------------------------------------------------------------
// TODO: we should redefine this to write UTF-16 (which matters on GCC which defines wchar_t as 32 bit)
static inline wchar_t * fgets(wchar_t * buf, int n, FILE * f) { return fgetws(buf, n, f); }
static inline string _utf8 (const string & s) { return s; }
static inline string _utf8 (const wstring & s) { return msra::strfun::utf8 (s); }
static inline size_t strnlen (wchar_t * s, size_t n) { return wcsnlen (s, n); }
#ifndef _MSC_VER // strnlen is VS proprietary
static inline size_t strnlen(const char * s, size_t /*n*/) { return strlen(s); }
#endif
@ -483,6 +485,9 @@ static inline size_t strnlen(const char * s, size_t /*n*/) { return strlen(s); }
static inline size_t strnlen (const char *s, size_t n) { return std::find (s,s+n,'\0') - s; }
#endif
static inline wchar_t * fgets(wchar_t * buf, int n, FILE * f) { return fgetws(buf, n, f); }
static inline size_t strnlen(wchar_t * s, size_t n) { return wcsnlen(s, n); }
template<class CHAR>
CHAR * fgetline (FILE * f, CHAR * buf, int size)
{
@ -491,7 +496,7 @@ CHAR * fgetline (FILE * f, CHAR * buf, int size)
if (p == NULL) // EOF reached: next time feof() = true
{
if (ferror (f))
ERROR ("error reading line: %s", strerror (errno));
RuntimeError ("error reading line: %s", strerror (errno));
buf[0] = 0;
return buf;
}
@ -502,9 +507,8 @@ CHAR * fgetline (FILE * f, CHAR * buf, int size)
if (n >= (size_t) size -1)
{
basic_string<CHAR> example (p, n < 100 ? n : 100);
uint64_t filepos = fgetpos(f); // (for error message only)
ERROR("input line too long at file offset %I64d (max. %d characters allowed) [%s ...]",
filepos, size -1, _utf8 (example).c_str());
uint64_t filepos = fgetpos(f); // (for error message only)
RuntimeError("input line too long at file offset %I64d (max. %d characters allowed) [%s ...]", filepos, size - 1, msra::strfun::utf8(example).c_str());
}
// remove newline at end
@ -535,7 +539,7 @@ const wchar_t * fgetline (FILE * f, wchar_t * buf, int size)
if (p == NULL) // EOF reached: next time feof() = true
{
if (ferror (f))
ERROR ("error reading line: %s", strerror (errno));
RuntimeError ("error reading line: %s", strerror (errno));
buf[0] = 0;
return buf;
}
@ -546,7 +550,7 @@ const wchar_t * fgetline (FILE * f, wchar_t * buf, int size)
if (n >= (size_t) size -1)
{
wstring example (buf, min (n, 100));
ERROR ("input line too long at file offset %U64d (max. %d characters allowed) [%S ...]",
RuntimeError ("input line too long at file offset %U64d (max. %d characters allowed) [%S ...]",
fgetpos (f), size -1, example.c_str());
}
@ -626,12 +630,10 @@ const char * fgetstring (FILE * f, __out_z_cap(size) char * buf, int size)
{
int c = fgetc(f);
if (c == EOF)
ERROR("error reading string or missing 0: %s", strerror(errno));
RuntimeError ("error reading string or missing 0: %s", strerror(errno));
if (c == 0) break;
if (i >= size - 1)
{
ERROR("input line too long (max. %d characters allowed)", size - 1);
}
RuntimeError ("input line too long (max. %d characters allowed)", size - 1);
buf[i] = (char)c;
}
assert (i < size);
@ -647,7 +649,7 @@ string fgetstring (FILE * f)
{
char c = (char)fgetc (f);
if (c == EOF)
ERROR ("error reading string or missing 0: %s", strerror (errno));
RuntimeError ("error reading string or missing 0: %s", strerror (errno));
if (c == 0) break;
res.push_back (c);
}
@ -663,11 +665,11 @@ const wchar_t * fgetstring (FILE * f, __out_z_cap(size) wchar_t * buf, int size)
// TODO: we should redefine this to write UTF-16 (which matters on GCC which defines wchar_t as 32 bit)
wint_t c = fgetwc(f);
if (c == WEOF)
ERROR("error reading string or missing 0: %s", strerror(errno));
RuntimeError ("error reading string or missing 0: %s", strerror(errno));
if (c == 0) break;
if (i >= size - 1)
{
ERROR("input line too long (max. %d wchar_tacters allowed)", size - 1);
RuntimeError ("input line too long (max. %d wchar_tacters allowed)", size - 1);
}
buf[i] = (wchar_t)c;
}
@ -698,7 +700,7 @@ wstring fgetwstring (FILE * f)
// note the order below works only for little endian
wint_t c = (wint_t)((c2 << 8) | c1);
if (c == WEOF)
ERROR ("error reading string or missing 0: %s", strerror (errno));
RuntimeError ("error reading string or missing 0: %s", strerror (errno));
if (c == 0) break;
res.push_back ((wchar_t) c);
}
@ -715,7 +717,7 @@ wstring fgetwstring (FILE * f)
{
wint_t c = fgetwc(f);
if (c == WEOF)
ERROR("error reading string or missing 0: %s", strerror(errno));
RuntimeError ("error reading string or missing 0: %s", strerror(errno));
if (c == 0) break;
res.push_back((wchar_t)c);
}
@ -732,14 +734,14 @@ bool fskipspace (FILE * f)
if (c == EOF) // hit the end
{
if (ferror(f))
ERROR("error reading from file: %s", strerror(errno));
RuntimeError ("error reading from file: %s", strerror(errno));
break;
}
if (!isspace (c)) // end of space: undo getting that character
{
int rc = ungetc(c, f);
if (rc != c)
ERROR("error in ungetc(): %s", strerror(errno));
RuntimeError ("error in ungetc(): %s", strerror(errno));
break;
}
}
@ -756,14 +758,14 @@ bool fskipwspace (FILE * f)
if (c == WEOF) // hit the end
{
if (ferror (f))
ERROR ("error reading from file: %s", strerror (errno));
RuntimeError ("error reading from file: %s", strerror (errno));
break;
}
if (!iswspace (c)) // end of space: undo getting that character
{
wint_t rc = ungetwc (c, f);
if (rc != c)
ERROR ("error in ungetc(): %s", strerror (errno));
RuntimeError ("error in ungetc(): %s", strerror (errno));
break;
}
}
@ -799,7 +801,7 @@ int fskipNewline (FILE * f, bool skip)
return found?(int)true:EOF;
int rc = ungetc (c, f);
if (rc != c)
ERROR ("error in ungetc(): %s", strerror (errno));
RuntimeError ("error in ungetc(): %s", strerror (errno));
return (int)found;
}
// if we get here we saw a newline
@ -834,7 +836,7 @@ int fskipwNewline (FILE * f, bool skip)
return found?(int)true:EOF;
wint_t rc = ungetwc (c, f);
if (rc != c)
ERROR ("error in ungetwc(): %s", strerror (errno));
RuntimeError ("error in ungetwc(): %s", strerror (errno));
return (int)found;
}
// if we get here we saw a double newline
@ -854,7 +856,7 @@ const char * fgettoken (FILE * f, __out_z_cap(size) char * buf, int size)
if (c == EOF) break;
if (isspace (c)) break;
if (i >= size -1)
ERROR ("input token too long (max. %d characters allowed)", size -1);
RuntimeError ("input token too long (max. %d characters allowed)", size -1);
buf[i] = (char) c;
}
// ... TODO: while (IsWhiteSpace (c)) c = fgetc (f); // skip trailing space
@ -862,7 +864,7 @@ const char * fgettoken (FILE * f, __out_z_cap(size) char * buf, int size)
{
int rc = ungetc (c, f);
if (rc != c)
ERROR ("error in ungetc(): %s", strerror (errno));
RuntimeError ("error in ungetc(): %s", strerror (errno));
}
assert (i < size);
buf[i] = 0;
@ -888,7 +890,7 @@ const wchar_t * fgettoken (FILE * f, __out_z_cap(size) wchar_t * buf, int size)
if (c == WEOF) break;
if (iswspace (c)) break;
if (i >= size -1)
ERROR ("input token too long (max. %d wchar_tacters allowed)", size -1);
RuntimeError ("input token too long (max. %d wchar_tacters allowed)", size -1);
buf[i] = (wchar_t) c;
}
// ... TODO: while (IsWhiteSpace (c)) c = fgetc (f); // skip trailing space
@ -896,7 +898,7 @@ const wchar_t * fgettoken (FILE * f, __out_z_cap(size) wchar_t * buf, int size)
{
int rc = ungetwc (c, f);
if (rc != c)
ERROR ("error in ungetwc(): %s", strerror (errno));
RuntimeError ("error in ungetwc(): %s", strerror (errno));
}
assert (i < size);
buf[i] = 0;
@ -981,7 +983,7 @@ void fcheckTag_ascii (FILE * f, const string & expectedTag)
fgettoken (f, buf, sizeof(buf)/sizeof(*buf));
if (expectedTag != buf)
{
ERROR ("invalid tag '%s' found; expected '%s'", buf, expectedTag.c_str());
RuntimeError ("invalid tag '%s' found; expected '%s'", buf, expectedTag.c_str());
}
}
@ -993,7 +995,7 @@ void fcompareTag (const string & readTag, const string & expectedTag)
{
if (readTag != expectedTag)
{
ERROR ("invalid tag '%s' found; expected '%s'",
RuntimeError ("invalid tag '%s' found; expected '%s'",
readTag.c_str(), expectedTag.c_str());
}
}
@ -1033,7 +1035,7 @@ void fpad (FILE * f, int n)
int pos = ftell (f);
if (pos == -1)
{
ERROR ("error in ftell(): %s", strerror (errno));
RuntimeError ("error in ftell(): %s", strerror (errno));
}
// determine how many bytes are needed (at least 1 for the 0-terminator)
// and create a dummy string of that length incl. terminator
@ -1119,7 +1121,7 @@ int fgetint_ascii (FILE * f)
int rc = ungetc (c, f);
if (rc != c)
{
ERROR ("error in ungetc(): %s", strerror (errno));
RuntimeError ("error in ungetc(): %s", strerror (errno));
}
return res;
}
@ -1158,9 +1160,9 @@ float fgetfloat_ascii (FILE * f)
fskipspace (f);
int rc = fscanf (f, "%f", &val); // security hint: safe overloads
if (rc == 0)
ERROR ("error reading float value from file (invalid format): %s");
RuntimeError ("error reading float value from file (invalid format): %s");
else if (rc == EOF)
ERROR ("error reading from file: %s", strerror (errno));
RuntimeError ("error reading from file: %s", strerror (errno));
assert (rc == 1);
return val;
}
@ -1325,7 +1327,7 @@ void fgetfile (FILE * f, std::vector<char> & buffer)
size_t n = fread (&inbuf[0], sizeof (inbuf[0]), inbuf.size(), f);
if (ferror (f))
{
ERROR ("fgetfile: error reading from file: %s", strerror (errno));
RuntimeError ("fgetfile: error reading from file: %s", strerror (errno));
}
buffer.insert (buffer.end(), inbuf.begin(), inbuf.begin() + n);
}
@ -1335,7 +1337,7 @@ void fgetfile (FILE * f, std::vector<char> & buffer)
// load it into RAM in one huge chunk
static size_t fgetfilechars (const std::wstring & path, vector<char> & buffer)
{
auto_file_ptr f = fopenOrDie (path, L"rb");
auto_file_ptr f (fopenOrDie (path, L"rb"));
size_t len = filesize (f);
buffer.reserve (len +1);
freadOrDie (buffer, len, f);
@ -1374,11 +1376,23 @@ vector<char*> msra::files::fgetfilelines (const wstring & path, vector<char> & b
}
// ----------------------------------------------------------------------------
// getfiletime(), setfiletime(): access modification time
// getfiletime(): access modification time
// ----------------------------------------------------------------------------
#ifndef _FILETIME_
//typedef struct _FILETIME { DWORD dwLowDateTime; DWORD dwHighDateTime; }; // from minwindef.h
typedef time_t FILETIME;
#else
bool operator>= (const FILETIME & targettime, const FILETIME & inputtime) // for use in fuptodate()
{
return (targettime.dwHighDateTime > inputtime.dwHighDateTime) ||
(targettime.dwHighDateTime == inputtime.dwHighDateTime && targettime.dwLowDateTime >= inputtime.dwLowDateTime);
}
#endif
bool getfiletime (const wstring & path, FILETIME & time)
{ // return file modification time, false if cannot be determined
#ifdef _WIN32
WIN32_FIND_DATAW findFileData;
auto_handle hFind (FindFirstFileW (path.c_str(), &findFileData), ::FindClose);
if (hFind != INVALID_HANDLE_VALUE)
@ -1387,11 +1401,23 @@ bool getfiletime (const wstring & path, FILETIME & time)
return true;
}
else
{
return false;
}
#else // TODO: test this; e.g. does st_mtime have the desired resolution?
struct stat buf;
int result;
// Get data associated with "crt_stat.c":
result = stat(charpath(path), &buf);
// Check if statistics are valid:
if (result != 0)
return false;
time = buf.st_mtime;
return true;
#endif
}
#if 0
void setfiletime (const wstring & path, const FILETIME & time)
{ // update the file modification time of an existing file
auto_handle h (CreateFileW (path.c_str(), FILE_WRITE_ATTRIBUTES,
@ -1399,14 +1425,15 @@ void setfiletime (const wstring & path, const FILETIME & time)
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL));
if (h == INVALID_HANDLE_VALUE)
{
ERROR ("setfiletime: error opening file: %d", GetLastError());
RuntimeError ("setfiletime: error opening file: %d", GetLastError());
}
BOOL rc = SetFileTime (h, NULL, NULL, &time);
if (!rc)
{
ERROR ("setfiletime: error setting file time information: %d", GetLastError());
RuntimeError ("setfiletime: error setting file time information: %d", GetLastError());
}
}
#endif
#if 0
// ----------------------------------------------------------------------------
@ -1483,7 +1510,7 @@ void expand_wildcards (const wstring & path, vector<wstring> & paths)
{
BOOL rc = ExpandWildcards (path, paths);
if (!rc)
ERROR("error in expanding wild cards '%S': %S", path.c_str(), FormatWin32Error(::GetLastError()).c_str());
RuntimeError ("error in expanding wild cards '%S': %S", path.c_str(), FormatWin32Error(::GetLastError()).c_str());
}
#endif
@ -1491,31 +1518,21 @@ void expand_wildcards (const wstring & path, vector<wstring> & paths)
// make_intermediate_dirs() -- make all intermediate dirs on a path
// ----------------------------------------------------------------------------
#ifndef _MSC_VER // _wmkdir() is VS proprietary
static int _wmkdir (const wchar_t * p)
{
return mkdir (msra::strfun::wcstombs (p).c_str(), 0777/*correct?*/);
}
#endif
static void mkdir (const wstring & path)
{
int rc = _wmkdir (path.c_str());
if (rc >= 0 || errno == EEXIST)
return; // no error or already existing --ok
#ifdef _WIN32 // bug in _wmkdir(): returns access_denied if folder exists but read-only --check existence
if (errno == EACCES)
{
// bug in _wmkdir(): returns access_denied if folder exists but read-only --check existence
DWORD att = ::GetFileAttributesW (path.c_str());
if (att != INVALID_FILE_ATTRIBUTES || (att & FILE_ATTRIBUTE_DIRECTORY) != 0)
return; // ok
}
ERROR ("make_intermediate_dirs: error creating intermediate directory %S", path.c_str());
}
#ifndef _MSC_VER
wchar_t* wcstok_s(wchar_t* s, const wchar_t* delim, wchar_t** ptr) { return wcstok (s, delim, ptr); }
#endif
RuntimeError ("mkdir: error creating intermediate directory %S", path.c_str());
}
// make subdir of a file including parents
void msra::files::make_intermediate_dirs (const wstring & filepath)
@ -1559,10 +1576,8 @@ bool msra::files::fuptodate (const wstring & target, const wstring & input, bool
if (!getfiletime (target, targettime)) return false; // target missing: need to update
FILETIME inputtime;
if (!getfiletime (input, inputtime)) return !inputrequired; // input missing: if required, pretend to be out of date as to force caller to fail
ULARGE_INTEGER targett, inputt;
memcpy (&targett, &targettime, sizeof (targett));
memcpy (&inputt, &inputtime, sizeof (inputt));
return !(targett.QuadPart < inputt.QuadPart); // up to date if target not older than input
// up to date if target has higher time stamp
return targettime >= inputtime; // note: uses an overload for WIN32 FILETIME (in Linux, FILETIME=time_t=size_t)
}
/// separate string by separator

2
DataReader/BinaryReader/BinaryFile.cpp
Просмотреть файл

@ -432,7 +432,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
return m_name;
// if name is not set yet, get it from the description header
msra::strfun::utf16 nameDescription(m_sectionHeader->nameDescription);
std::wstring nameDescription (msra::strfun::utf16(m_sectionHeader->nameDescription));
auto firstColon = nameDescription.find_first_of(L':');
if (firstColon != npos && nameDescription.size() >= firstColon)
{

4
DataReader/BinaryReader/BinaryReader.vcxproj
Просмотреть файл

@ -171,7 +171,11 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\Include\basetypes.h" />
<ClInclude Include="..\..\Common\Include\DataReader.h" />
<ClInclude Include="..\..\Common\Include\DataWriter.h" />
<ClInclude Include="..\..\Common\Include\File.h" />
<ClInclude Include="..\..\Common\Include\fileutil.h" />
<ClInclude Include="BinaryReader.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />

54
DataReader/BinaryReader/BinaryReader.vcxproj.filters Normal file
Просмотреть файл

@ -0,0 +1,54 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="BinaryFile.cpp" />
<ClCompile Include="BinaryReader.cpp" />
<ClCompile Include="BinaryWriter.cpp" />
<ClCompile Include="Exports.cpp" />
<ClCompile Include="dllmain.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="..\..\Common\DataWriter.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\DataReader.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\ConfigFile.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\File.cpp">
<Filter>Common</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="BinaryReader.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="..\..\Common\Include\DataWriter.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\basetypes.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataReader.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\File.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\fileutil.h">
<Filter>Common\Include</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Filter Include="Common">
<UniqueIdentifier>{7a294e4e-7a00-4bd3-8e0a-86f08db10a98}</UniqueIdentifier>
</Filter>
<Filter Include="Common\Include">
<UniqueIdentifier>{ecc1d590-3964-477c-9487-9a78e9a497ba}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>

1
DataReader/HTKMLFReader/HTKMLFReader.vcxproj
Просмотреть файл

@ -178,7 +178,6 @@
<ClInclude Include="HTKMLFWriter.h" />
<ClInclude Include="latticearchive.h" />
<ClInclude Include="latticestorage.h" />
<ClInclude Include="message.h" />
<ClInclude Include="minibatchiterator.h" />
<ClInclude Include="minibatchsourcehelpers.h" />
<ClInclude Include="msra_mgram.h" />

66
DataReader/HTKMLFReader/HTKMLFReader.vcxproj.filters Normal file
Просмотреть файл

@ -0,0 +1,66 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="DataReader.cpp" />
<ClCompile Include="DataWriter.cpp" />
<ClCompile Include="dllmain.cpp" />
<ClCompile Include="HTKMLFReader.cpp" />
<ClCompile Include="HTKMLFWriter.cpp" />
<ClCompile Include="latticearchive.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="fileutil.cpp">
<Filter>Duplicates to remove</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="biggrowablevectors.h" />
<ClInclude Include="chunkevalsource.h" />
<ClInclude Include="htkfeatio.h" />
<ClInclude Include="HTKMLFReader.h" />
<ClInclude Include="HTKMLFWriter.h" />
<ClInclude Include="latticearchive.h" />
<ClInclude Include="latticestorage.h" />
<ClInclude Include="minibatchiterator.h" />
<ClInclude Include="minibatchsourcehelpers.h" />
<ClInclude Include="msra_mgram.h" />
<ClInclude Include="numahelpers.h" />
<ClInclude Include="pplhelpers.h" />
<ClInclude Include="readaheadsource.h" />
<ClInclude Include="rollingwindowsource.h" />
<ClInclude Include="simplesenonehmm.h" />
<ClInclude Include="simplethread.h" />
<ClInclude Include="simple_checked_arrays.h" />
<ClInclude Include="ssefloat4.h" />
<ClInclude Include="ssematrix.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="utterancesource.h" />
<ClInclude Include="utterancesourcemulti.h" />
<ClInclude Include="basetypes.h">
<Filter>Duplicates to remove</Filter>
</ClInclude>
<ClInclude Include="fileutil.h">
<Filter>Duplicates to remove</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataReader.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataWriter.h">
<Filter>Common\Include</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Text Include="ReadMe.txt" />
</ItemGroup>
<ItemGroup>
<Filter Include="Duplicates to remove">
<UniqueIdentifier>{9a77edee-47ee-45ec-9b35-d22cc73ea64e}</UniqueIdentifier>
</Filter>
<Filter Include="Common">
<UniqueIdentifier>{5e80057c-a54a-4d57-a856-dc1d23a63aea}</UniqueIdentifier>
</Filter>
<Filter Include="Common\Include">
<UniqueIdentifier>{812d1247-04eb-4693-a0d2-5399a94e6235}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>

45
DataReader/HTKMLFReader/ReadMe.txt
Просмотреть файл

@ -1,40 +1,5 @@
========================================================================
DYNAMIC LINK LIBRARY : HTKMLFReader Project Overview
========================================================================
AppWizard has created this HTKMLFReader DLL for you.
This file contains a summary of what you will find in each of the files that
make up your HTKMLFReader application.
HTKMLFReader.vcxproj
This is the main project file for VC++ projects generated using an Application Wizard.
It contains information about the version of Visual C++ that generated the file, and
information about the platforms, configurations, and project features selected with the
Application Wizard.
HTKMLFReader.vcxproj.filters
This is the filters file for VC++ projects generated using an Application Wizard.
It contains information about the association between the files in your project
and the filters. This association is used in the IDE to show grouping of files with
similar extensions under a specific node (for e.g. ".cpp" files are associated with the
"Source Files" filter).
HTKMLFReader.cpp
This is the main DLL source file.
/////////////////////////////////////////////////////////////////////////////
Other standard files:
StdAfx.h, StdAfx.cpp
These files are used to build a precompiled header (PCH) file
named HTKMLFReader.pch and a precompiled types file named StdAfx.obj.
/////////////////////////////////////////////////////////////////////////////
Other notes:
AppWizard uses "TODO:" comments to indicate parts of the source code you
should add to or customize.
/////////////////////////////////////////////////////////////////////////////
This is the reader for HTK-formatted speech files.
Large parts of this code deviate from the coding style of the rest of CNTK and also include some unused code and complexity not really needed for CNTK.
The reason is that this code was ported from another, older tool. We hope to gradually clean this out.

49
DataReader/HTKMLFReader/basetypes.h
Просмотреть файл

@ -1,3 +1,5 @@
// TODO: This is a dup, we should get back to the shared one. But this one has some stuff the other doesn't.
//
// <copyright file="basetypes.h" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
@ -959,4 +961,51 @@ using namespace msra::basetypes; // for compatibility
#pragma warning (pop)
// RuntimeError - throw a std::runtime_error with a formatted error string
static inline void RuntimeError(const char * format, ...)
{
va_list args;
char buffer[1024];
va_start(args, format);
vsprintf(buffer, format, args);
throw std::runtime_error(buffer);
};
// ----------------------------------------------------------------------------
// dynamic loading of modules
// ----------------------------------------------------------------------------
#ifdef _WIN32
class Plugin
{
HMODULE m_hModule; // module handle for the writer DLL
std::wstring m_dllName; // name of the writer DLL
public:
Plugin() { m_hModule = NULL; }
FARPROC Load(const std::string & plugin, const std::string & proc)
{
m_dllName = msra::strfun::utf16(plugin);
m_dllName += L".dll";
m_hModule = LoadLibrary(m_dllName.c_str());
if (m_hModule == NULL)
RuntimeError("Plugin not found: %s", msra::strfun::utf8(m_dllName));
// create a variable of each type just to call the proper templated version
return GetProcAddress(m_hModule, proc.c_str());
}
~Plugin() { if (m_hModule) FreeLibrary(m_hModule); }
};
#else
class Plugin
{
public:
void * Load(const std::string & plugin, const std::string & proc)
{
RuntimeError("Plugins not implemented on Linux yet");
return NULL;
}
};
#endif
#endif // _BASETYPES_

117
DataReader/HTKMLFReader/fileutil.cpp
Просмотреть файл

@ -16,7 +16,6 @@
#include "basetypes.h"
#include "fileutil.h"
#include "message.h"
#include <stdio.h>
#include <string.h>
#include <ctype.h>
@ -50,7 +49,7 @@ template<class _T> FILE * fopenStdHandle (const _T * mode)
// switch to binary mode if not yet (in case it is stdin)
int rc = _setmode (_fileno (f), strchr (mode, 'b') ? _O_BINARY : _O_TEXT);
if (rc == -1)
ERROR ("error switching stream to binary mode: %s", strerror (errno));
RuntimeError ("error switching stream to binary mode: %s", strerror (errno));
}
return f;
}
@ -60,7 +59,7 @@ FILE * fopenOrDie (const STRING & pathname, const char * mode)
FILE * f = (pathname[0] == '-') ? fopenStdHandle (mode) : fopen (pathname.c_str(), mode);
if (f == NULL)
{
ERROR ("error opening file '%s': %s", pathname.c_str(), strerror (errno));
RuntimeError ("error opening file '%s': %s", pathname.c_str(), strerror (errno));
return NULL; // keep OACR happy
}
if (strchr (mode, 'S'))
@ -75,7 +74,7 @@ FILE * fopenOrDie (const WSTRING & pathname, const wchar_t * mode)
FILE * f = (pathname[0] == '-') ? fopenStdHandle (mode) : _wfopen (pathname.c_str(), mode);
if (f == NULL)
{
ERROR ("error opening file '%S': %s", pathname.c_str(), strerror (errno));
RuntimeError ("error opening file '%S': %s", pathname.c_str(), strerror (errno));
return NULL; // keep OACR happy
}
if (strchr (mode, 'S'))
@ -93,7 +92,7 @@ void fsetmode (FILE * f, char type)
{
if (type != 'b' && type != 't')
{
ERROR ("fsetmode: invalid type '%c'");
RuntimeError ("fsetmode: invalid type '%c'");
}
#ifdef UNDER_CE // winCE and win32 have different return types for _fileno
FILE *fd = _fileno (f); // note: no error check possible
@ -104,7 +103,7 @@ void fsetmode (FILE * f, char type)
int rc = _setmode (fd, mode);
if (rc == -1)
{
ERROR ("error changing file mode: %s", strerror (errno));
RuntimeError ("error changing file mode: %s", strerror (errno));
}
}
@ -120,7 +119,7 @@ void freadOrDie (void * ptr, size_t size, size_t count, FILE * f)
size_t chunkn = min (count, 15*1024*1024); // BUGBUG: I surely meant this limit to be bytes, not units of 'size'...
size_t n = fread (ptr, size, chunkn, f);
if (n != chunkn)
ERROR ("error reading from file: %s", strerror (errno));
RuntimeError ("error reading from file: %s", strerror (errno));
count -= n;
ptr = n * size + (char*) ptr;
}
@ -135,7 +134,7 @@ void freadOrDie (void * ptr, size_t size, size_t count, const HANDLE f)
DWORD n ;
ReadFile(f, ptr, (DWORD) chunkn, &n, NULL);
if (n != chunkn)
ERROR ("error number for reading from file: %s", GetLastError());
RuntimeError ("error number for reading from file: %s", GetLastError());
count -= (size_t) (n / size);
ptr = n + (char*) ptr;
}
@ -162,7 +161,7 @@ void fwriteOrDie (const void * ptr, size_t size, size_t count, FILE * f)
size_t n = fwrite ((const void *) p1, 1, wantWrite, f);
if (n != wantWrite)
{
ERROR ("error writing to file (ptr=0x%08lx, size=%d,"
RuntimeError ("error writing to file (ptr=0x%08lx, size=%d,"
" count=%d, writing %d bytes after %d): %s",
ptr, size, count, (int) wantWrite,
(int) (size * count - totalBytes),
@ -188,7 +187,7 @@ void fwriteOrDie (const void * ptr, size_t size, size_t count, const HANDLE f)
DWORD byteWritten = 0 ;
if (WriteFile(f, (const void *) p1, wantWrite, &byteWritten, NULL) == false)
{
ERROR ("error writing to file (ptr=0x%08lx, size=%d,"
RuntimeError ("error writing to file (ptr=0x%08lx, size=%d,"
" count=%d, writing %d bytes after %d): %s",
ptr, size, count, (int) wantWrite,
(int) (size * count - totalBytes),
@ -213,7 +212,7 @@ void fprintfOrDie (FILE * f, const char * fmt, ...)
int rc = vfprintf (f, fmt, arg_ptr);
if (rc < 0)
{
ERROR ("error writing to file: %s", strerror (errno));
RuntimeError ("error writing to file: %s", strerror (errno));
}
}
#pragma warning(pop)
@ -227,7 +226,7 @@ void fflushOrDie (FILE * f)
int rc = fflush (f);
if (rc != 0)
{
ERROR ("error flushing to file: %s", strerror (errno));
RuntimeError ("error flushing to file: %s", strerror (errno));
}
}
@ -240,22 +239,22 @@ size_t filesize (FILE * f)
long curPos = ftell (f);
if (curPos == -1L)
{
ERROR ("error determining file position: %s", strerror (errno));
RuntimeError ("error determining file position: %s", strerror (errno));
}
int rc = fseek (f, 0, SEEK_END);
if (rc != 0)
{
ERROR ("error seeking to end of file: %s", strerror (errno));
RuntimeError ("error seeking to end of file: %s", strerror (errno));
}
long len = ftell (f);
if (len == -1L)
{
ERROR ("error determining file position: %s", strerror (errno));
RuntimeError ("error determining file position: %s", strerror (errno));
}
rc = fseek (f, curPos, SEEK_SET);
if (rc != 0)
{
ERROR ("error resetting file position: %s", strerror (errno));
RuntimeError ("error resetting file position: %s", strerror (errno));
}
return (size_t) len;
}
@ -299,12 +298,12 @@ long fseekOrDie (FILE * f, long offset, int mode)
long curPos = ftell (f);
if (curPos == -1L)
{
ERROR ("error seeking: %s", strerror (errno));
RuntimeError ("error seeking: %s", strerror (errno));
}
int rc = fseek (f, offset, mode);
if (rc != 0)
{
ERROR ("error seeking: %s", strerror (errno));
RuntimeError ("error seeking: %s", strerror (errno));
}
return curPos;
}
@ -314,7 +313,7 @@ uint64_t fgetpos (FILE * f)
fpos_t post;
int rc = ::fgetpos (f, &post);
if (rc != 0)
ERROR ("error getting file position: %s", strerror (errno));
RuntimeError ("error getting file position: %s", strerror (errno));
return post;
}
@ -347,7 +346,7 @@ void fsetpos (FILE * f, uint64_t reqpos)
fpos_t post = reqpos;
int rc = ::fsetpos (f, &post);
if (rc != 0)
ERROR ("error setting file position: %s", strerror (errno));
RuntimeError ("error setting file position: %s", strerror (errno));
}
// ----------------------------------------------------------------------------
@ -357,12 +356,12 @@ void fsetpos (FILE * f, uint64_t reqpos)
void unlinkOrDie (const std::string & pathname)
{
if (_unlink (pathname.c_str()) != 0 && errno != ENOENT) // if file is missing that's what we want
ERROR ("error deleting file '%s': %s", pathname.c_str(), strerror (errno));
RuntimeError ("error deleting file '%s': %s", pathname.c_str(), strerror (errno));
}
void unlinkOrDie (const std::wstring & pathname)
{
if (_wunlink (pathname.c_str()) != 0 && errno != ENOENT) // if file is missing that's what we want
ERROR ("error deleting file '%S': %s", pathname.c_str(), strerror (errno));
RuntimeError ("error deleting file '%S': %s", pathname.c_str(), strerror (errno));
}
// ----------------------------------------------------------------------------
@ -373,14 +372,14 @@ void unlinkOrDie (const std::wstring & pathname)
void renameOrDie (const std::string & from, const std::string & to)
{
if (!MoveFileA (from.c_str(),to.c_str()))
ERROR ("error renaming: %s", GetLastError());
RuntimeError ("error renaming: %s", GetLastError());
}
#endif
void renameOrDie (const std::wstring & from, const std::wstring & to)
{
if (!MoveFileW (from.c_str(),to.c_str()))
ERROR ("error renaming: %s", GetLastError());
RuntimeError ("error renaming: %s", GetLastError());
}
// ----------------------------------------------------------------------------
@ -458,7 +457,7 @@ CHAR * fgetline (FILE * f, CHAR * buf, int size)
if (p == NULL) // EOF reached: next time feof() = true
{
if (ferror (f))
ERROR ("error reading line: %s", strerror (errno));
RuntimeError ("error reading line: %s", strerror (errno));
buf[0] = 0;
return buf;
}
@ -469,7 +468,7 @@ CHAR * fgetline (FILE * f, CHAR * buf, int size)
if (n >= (size_t) size -1)
{
basic_string<CHAR> example (p, n < 100 ? n : 100);
ERROR ("input line too long at file offset %I64d (max. %d characters allowed) [%s ...]",
RuntimeError ("input line too long at file offset %I64d (max. %d characters allowed) [%s ...]",
filepos, size -1, _utf8 (example).c_str());
}
@ -501,7 +500,7 @@ const wchar_t * fgetline (FILE * f, wchar_t * buf, int size)
if (p == NULL) // EOF reached: next time feof() = true
{
if (ferror (f))
ERROR ("error reading line: %s", strerror (errno));
RuntimeError ("error reading line: %s", strerror (errno));
buf[0] = 0;
return buf;
}
@ -512,7 +511,7 @@ const wchar_t * fgetline (FILE * f, wchar_t * buf, int size)
if (n >= (size_t) size -1)
{
wstring example (buf, min (n, 100));
ERROR ("input line too long at file offset %U64d (max. %d characters allowed) [%S ...]",
RuntimeError ("input line too long at file offset %U64d (max. %d characters allowed) [%S ...]",
fgetpos (f), size -1, example.c_str());
}
@ -592,11 +591,11 @@ const char * fgetstring (FILE * f, __out_z_cap(size) char * buf, int size)
{
int c = fgetc (f);
if (c == EOF)
ERROR ("error reading string or missing 0: %s", strerror (errno));
RuntimeError ("error reading string or missing 0: %s", strerror (errno));
if (c == 0) break;
if (i >= size -1)
{
ERROR ("input line too long (max. %d characters allowed)", size -1);
RuntimeError ("input line too long (max. %d characters allowed)", size -1);
}
buf[i] = (char) c;
}
@ -615,7 +614,7 @@ const char * fgetstring (const HANDLE f, __out_z_cap(size) char * buf, int size)
if (c == (char) 0) break;
if (i >= size -1)
{
ERROR ("input line too long (max. %d characters allowed)", size -1);
RuntimeError ("input line too long (max. %d characters allowed)", size -1);
}
buf[i] = (char) c;
}
@ -632,7 +631,7 @@ wstring fgetwstring (FILE * f)
{
int c = fgetwc (f);
if (c == EOF)
ERROR ("error reading string or missing 0: %s", strerror (errno));
RuntimeError ("error reading string or missing 0: %s", strerror (errno));
if (c == 0) break;
res.push_back ((wchar_t) c);
}
@ -647,14 +646,14 @@ void fskipspace (FILE * f)
if (c == EOF) // hit the end
{
if (ferror (f))
ERROR ("error reading from file: %s", strerror (errno));
RuntimeError ("error reading from file: %s", strerror (errno));
break;
}
if (!isspace (c)) // end of space: undo getting that character
{
int rc = ungetc (c, f);
if (rc != c)
ERROR ("error in ungetc(): %s", strerror (errno));
RuntimeError ("error in ungetc(): %s", strerror (errno));
break;
}
}
@ -679,7 +678,7 @@ void fskipNewline (FILE * f)
if (c != '\n')
{
ERROR ("unexpected garbage at end of line");
RuntimeError ("unexpected garbage at end of line");
}
}
@ -696,7 +695,7 @@ const char * fgettoken (FILE * f, __out_z_cap(size) char * buf, int size)
if (c == EOF) break;
if (isspace (c)) break;
if (i >= size -1)
ERROR ("input token too long (max. %d characters allowed)", size -1);
RuntimeError ("input token too long (max. %d characters allowed)", size -1);
buf[i] = (char) c;
}
// ... TODO: while (isspace (c)) c = fgetc (f); // skip trailing space
@ -704,7 +703,7 @@ const char * fgettoken (FILE * f, __out_z_cap(size) char * buf, int size)
{
int rc = ungetc (c, f);
if (rc != c)
ERROR ("error in ungetc(): %s", strerror (errno));
RuntimeError ("error in ungetc(): %s", strerror (errno));
}
ASSERT (i < size);
buf[i] = 0;
@ -789,7 +788,7 @@ void fcheckTag_ascii (FILE * f, const STRING & expectedTag)
fgettoken (f, buf, sizeof(buf)/sizeof(*buf));
if (expectedTag != buf)
{
ERROR ("invalid tag '%s' found; expected '%s'", buf, expectedTag.c_str());
RuntimeError ("invalid tag '%s' found; expected '%s'", buf, expectedTag.c_str());
}
}
@ -801,7 +800,7 @@ void fcompareTag (const STRING & readTag, const STRING & expectedTag)
{
if (readTag != expectedTag)
{
ERROR ("invalid tag '%s' found; expected '%s'",
RuntimeError ("invalid tag '%s' found; expected '%s'",
readTag.c_str(), expectedTag.c_str());
}
}
@ -848,7 +847,7 @@ void fpad (FILE * f, int n)
int pos = ftell (f);
if (pos == -1)
{
ERROR ("error in ftell(): %s", strerror (errno));
RuntimeError ("error in ftell(): %s", strerror (errno));
}
// determine how many bytes are needed (at least 1 for the 0-terminator)
// and create a dummy string of that length incl. terminator
@ -940,7 +939,7 @@ int fgetint_ascii (FILE * f)
int rc = ungetc (c, f);
if (rc != c)
{
ERROR ("error in ungetc(): %s", strerror (errno));
RuntimeError ("error in ungetc(): %s", strerror (errno));
}
return res;
}
@ -968,9 +967,9 @@ float fgetfloat_ascii (FILE * f)
fskipspace (f);
int rc = fscanf (f, "%f", &val); // security hint: safe overloads
if (rc == 0)
ERROR ("error reading float value from file (invalid format): %s");
RuntimeError ("error reading float value from file (invalid format): %s");
else if (rc == EOF)
ERROR ("error reading from file: %s", strerror (errno));
RuntimeError ("error reading from file: %s", strerror (errno));
ASSERT (rc == 1);
return val;
}
@ -1073,7 +1072,7 @@ void WAVEHEADER::write (FILE * f)
long curPos = ftell (f);
if (curPos == -1L)
{
ERROR ("error determining file position: %s", strerror (errno));
RuntimeError ("error determining file position: %s", strerror (errno));
}
unsigned int len = (unsigned int) filesize (f);
unsigned int RiffLength = len - 8;
@ -1085,6 +1084,7 @@ void WAVEHEADER::write (FILE * f)
fseekOrDie (f, curPos, SEEK_SET);
}
#if 0
unsigned int WAVEHEADER::read (FILE * f, signed short & wRealFormatTag, int & bytesPerSample)
{
// read header
@ -1099,17 +1099,17 @@ unsigned int WAVEHEADER::read (FILE * f, signed short & wRealFormatTag, int & by
wRealFormatTag = 1; // Workaround: pretend it is 1 (seems safe)
}
(wRealFormatTag == 1 || wRealFormatTag == 7)
|| ERROR ("WAVEHEADER::read: wFormatTag=%d not supported for now", wRealFormatTag);
|| RuntimeError ("WAVEHEADER::read: wFormatTag=%d not supported for now", wRealFormatTag);
unsigned short wChannels = fgetshort (f);
unsigned long dwSamplesPerSec = fgetint (f);
unsigned int sampleRate = dwSamplesPerSec;
/*unsigned long dwAvgBytesPerSec = */ fgetint (f);
unsigned short wBlockAlign = fgetshort (f);
unsigned short wBitsPerSample = fgetshort (f);
(wBitsPerSample <= 16) || ERROR ("WAVEHEADER::read: invalid wBitsPerSample %d", wBitsPerSample);
(wBitsPerSample <= 16) || RuntimeError ("WAVEHEADER::read: invalid wBitsPerSample %d", wBitsPerSample);
bytesPerSample = wBitsPerSample / 8;
(wBlockAlign == wChannels * bytesPerSample)
|| ERROR ("WAVEHEADER::read: wBlockAlign != wChannels*bytesPerSample not supported");
|| RuntimeError ("WAVEHEADER::read: wBlockAlign != wChannels*bytesPerSample not supported");
while (fmtLen > 16) // unused extra garbage in header
{
fgetbyte (f);
@ -1117,7 +1117,7 @@ unsigned int WAVEHEADER::read (FILE * f, signed short & wRealFormatTag, int & by
}
if (wRealFormatTag == 7)
{
(bytesPerSample == 1) || ERROR ("WAVEHEADER::read: invalid wBitsPerSample %d for mulaw", wBitsPerSample);
(bytesPerSample == 1) || RuntimeError ("WAVEHEADER::read: invalid wBitsPerSample %d for mulaw", wBitsPerSample);
fcheckTag (f, "fact");
unsigned int factLen = fgetint (f);
while (factLen > 0)
@ -1160,7 +1160,7 @@ static void fgetwavraw(FILE * f, ARRAY<short> & wav, const WAVEHEADER & wavhd)
wav.resize (wavhd.DataLength / bytesPerSample);
if (wavhd.wFormatTag == 7) // mulaw
{
(wavhd.nChannels == 1) || ERROR ("fgetwav: wChannels=%d not supported for mulaw", wavhd.nChannels);
(wavhd.nChannels == 1) || RuntimeError ("fgetwav: wChannels=%d not supported for mulaw", wavhd.nChannels);
ARRAY<unsigned char> data;
int numSamples = wavhd.DataLength/wavhd.nBlockAlign;
data.resize (numSamples);
@ -1177,7 +1177,7 @@ static void fgetwavraw(FILE * f, ARRAY<short> & wav, const WAVEHEADER & wavhd)
// ... TODO: support 8 bit linear PCM samples (implement when needed; samples scaled to 'short')
else
{
ERROR ("bytesPerSample != 2 is not supported except mulaw format!\n");
RuntimeError ("bytesPerSample != 2 is not supported except mulaw format!\n");
}
}
@ -1218,7 +1218,7 @@ void fgetwav (FILE * f, ARRAY<short> & wav, int & sampleRate)
}
else
{
ERROR ("bytesPerSample/wChannels != 2 needs to be implemented");
RuntimeError ("bytesPerSample/wChannels != 2 needs to be implemented");
}
}
@ -1276,7 +1276,7 @@ unsigned int fgetwfx (FILE * f, WAVEFORMATEX & wfx)
wfx.wFormatTag = 1; // Workaround: pretend it is 1 (seems safe)
}
(wfx.wFormatTag == 1 || wfx.wFormatTag == 3 || wfx.wFormatTag == 7)
|| ERROR ("WAVEHEADER::read: wFormatTag=%d not supported for now", wfx.wFormatTag);
|| RuntimeError ("WAVEHEADER::read: wFormatTag=%d not supported for now", wfx.wFormatTag);
wfx.nChannels = fgetshort (f);
wfx.nSamplesPerSec = fgetint (f);
wfx.nAvgBytesPerSec = fgetint (f);
@ -1294,7 +1294,7 @@ void fputwfx (FILE *f, const WAVEFORMATEX & wfx, unsigned int numSamples)
{
unsigned int DataLength = numSamples * wfx.nBlockAlign;
(DataLength / wfx.nBlockAlign == numSamples)
|| ERROR ("fputwfx: data size exceeds WAV header 32-bit range");
|| RuntimeError ("fputwfx: data size exceeds WAV header 32-bit range");
unsigned int RiffLength = 36 + DataLength;
unsigned int FmtLength = 16;
// file header
@ -1345,6 +1345,7 @@ void fputwav (const wstring & fn, const vector<short> & wav, int sampleRate, int
fputwav (f, wav, sampleRate, nChannels);
fflushOrDie (f); // after this, fclose() (in destructor of f) cannot fail
}
#endif
// ----------------------------------------------------------------------------
// fputbyte(): write a byte value
@ -1491,7 +1492,7 @@ void fgetfile (FILE * f, ARRAY<char> & buffer)
size_t n = fread (&inbuf[0], sizeof (inbuf[0]), inbuf.size(), f);
if (ferror (f))
{
ERROR ("fgetfile: error reading from file: %s", strerror (errno));
RuntimeError ("fgetfile: error reading from file: %s", strerror (errno));
}
buffer.insert (buffer.end(), inbuf.begin(), inbuf.begin() + n);
}
@ -1566,12 +1567,12 @@ void setfiletime (const wstring & path, const FILETIME & time)
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL));
if (h == INVALID_HANDLE_VALUE)
{
ERROR ("setfiletime: error opening file: %d", GetLastError());
RuntimeError ("setfiletime: error opening file: %d", GetLastError());
}
BOOL rc = SetFileTime (h, NULL, NULL, &time);
if (!rc)
{
ERROR ("setfiletime: error setting file time information: %d", GetLastError());
RuntimeError ("setfiletime: error setting file time information: %d", GetLastError());
}
}
@ -1649,7 +1650,7 @@ void expand_wildcards (const wstring & path, vector<wstring> & paths)
{
BOOL rc = ExpandWildcards (path, paths);
if (!rc)
ERROR ("error in expanding wild cards '%S': %S", path.c_str(), FormatWin32Error (::GetLastError()).c_str());
RuntimeError ("error in expanding wild cards '%S': %S", path.c_str(), FormatWin32Error (::GetLastError()).c_str());
}
// ----------------------------------------------------------------------------
@ -1668,7 +1669,7 @@ static void mkdir (const wstring & path)
if (att != INVALID_FILE_ATTRIBUTES || (att & FILE_ATTRIBUTE_DIRECTORY) != 0)
return; // ok
}
ERROR ("make_intermediate_dirs: error creating intermediate directory %S", path.c_str());
RuntimeError ("make_intermediate_dirs: error creating intermediate directory %S", path.c_str());
}
// make subdir of a file including parents

473
DataReader/HTKMLFReader/message.h
Просмотреть файл

@ -1,473 +0,0 @@
//
// message.h - class for simple I/O of log messages
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
#pragma once
#ifndef _MESSAGE_
#define _MESSAGE_
#ifndef UNDER_CE // fixed-buffer overloads not available for wince
#ifdef _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES // fixed-buffer overloads for strcpy() etc.
#undef _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES
#endif
#define _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES 1
#endif
#include "basetypes.h"
#include <stdarg.h>
#include <iostream>
#include <exception>
#include <time.h> // for _time64 in timeDateStamp()
#pragma warning (push)
#pragma warning (disable : 4793) // caused by varargs
#pragma warning(disable : 4996) // strcpy and other safety stuff disabled
namespace msra { namespace logging
{
// -----------------------------------------------------------------------
// message_exception - exception thrown by this error module
// -----------------------------------------------------------------------
class message_exception : public std::exception
{
char buf[1024]; // local buffer for message
const char * dup_what (const char * what)
{
strcpy (buf, "message_exception:"); // security hint: safe overloads
strcat (buf, what);
return &buf[0];
}
public:
message_exception (const char * what) : exception (dup_what (what))
{
}
};
// -----------------------------------------------------------------------
// global state (hidden as a static struct in a variable)
// -----------------------------------------------------------------------
struct __globals
{
msra::basetypes::CCritSec lock;
FILE * logFile;
bool noFlush; // heavy logging: don't flush
bool throwExceptionFlag; // true: don't exit but throw exception
// G.muted semantics is as follows
// - G.muted && !G.logFile => don't show anything
// - G.muted && G.logFile => write to log file
// - !G.muted && !G.logFile => write to stderr only
// - !G.muted && G.logFile => write to log file and stderr
bool muted;
std::string filename; // log file name
char buf[30000]; // for _vsnprintf()
__globals() : logFile (NULL), throwExceptionFlag (false), muted (false), noFlush (false)
{ buf[0] = 0; buf[sizeof (buf) / sizeof (*buf) -1] = 0; }
static __globals & get() { static __globals g; return g; }
};
#pragma push_macro ("G")
#define G (__globals::get()) // access global XYZ as G.XYZ
// ------------------------------------------------------------------------
// setLogFile(): set the log file
// if non-NULL then every message will be written both to stderr and this
// log file.
// multi-threading: not thread-safe, set this before starting
// ------------------------------------------------------------------------
static inline FILE * setLogFile (FILE * newLogFile)
{
FILE * oldLogFile;
oldLogFile = G.logFile;
if (newLogFile != stderr)
{
G.logFile = newLogFile;
}
else
{
G.logFile = NULL;
}
return oldLogFile;
}
// ------------------------------------------------------------------------
// setLogFileByName(): set the log file by file name
// in this mode, log file will be re-connected when disconnected
// filename == NULL indicates an attempt to reconnect
// WARNING: if the filename is invalid, it will try to reconnect every time
// ------------------------------------------------------------------------
static inline void setLogFileByName (const char * p_filename)
{
FILE * newLogFile = NULL;
if (p_filename == NULL)
{
// for reconnection
ASSERT (G.filename != "");
newLogFile = fopen (G.filename.c_str (), "ab");
}
else
{
ASSERT (p_filename[0]);
G.filename = p_filename; // remember filename
newLogFile = fopen (p_filename, "wb");
}
// handle open failure
if (newLogFile == NULL)
{
if (G.logFile != NULL)
{
fprintf (G.logFile, "ERROR: setLogFileByName: error opening log file %s: %s\n",
G.filename.c_str (), strerror (errno));
// in case of a reconnect, this ^^ will obviously fail, we ignore this
}
fprintf (stderr, "ERROR: setLogFileByName: error opening log file %s: %s\n",
G.filename.c_str (), strerror (errno));
return;
}
// set new handle
FILE * oldLogFile = setLogFile (newLogFile);
// close old handle
if (oldLogFile != NULL && oldLogFile != stderr && oldLogFile != stdin)
{
int rc = fclose (oldLogFile);
if (rc != 0)
{
if (G.logFile != NULL)
{ // note: this goes to the new log file
fprintf (G.logFile, "ERROR: setLogFileByName: error closing old log file: %s\n",
strerror (errno));
}
fprintf (stderr, "ERROR: setLogFileByName: error closing old log file: %s\n",
strerror (errno));
}
}
}
// ------------------------------------------------------------------------
// setExceptionFlag(): set flag whether to throw an exception (true) or exit() (false, default)
// ------------------------------------------------------------------------
static inline bool setExceptionFlag (bool throwExceptionFlag = true)
{
bool oldFlag = G.throwExceptionFlag;
G.throwExceptionFlag = throwExceptionFlag;
return oldFlag;
}
// ------------------------------------------------------------------------
// timeDateStamp() -- often needed for logging
// ------------------------------------------------------------------------
static inline std::string timeDateStamp (void)
{
__time64_t localtime; _time64 (&localtime); // get current time and date
struct tm now; _localtime64_s (&now, &localtime); // convert
char buf[20];
sprintf (buf, "%04d/%02d/%02d %02d:%02d:%02d", // security hint: this is an overload
now.tm_year + 1900, now.tm_mon + 1, now.tm_mday,
now.tm_hour, now.tm_min, now.tm_sec);
return buf;
}
// ------------------------------------------------------------------------
// __flush(): flush output
// ------------------------------------------------------------------------
static inline void __flush()
{
int rc = fflush (G.logFile);
if (rc != 0)
{
fprintf (stderr, "ERROR: __flush: error flushing to log file %s\n",
strerror (errno));
}
}
// ------------------------------------------------------------------------
// setHeavyLogging(): we are heavily logging: don't flush & increase out buf
// ------------------------------------------------------------------------
static inline void setHeavyLogging (bool isHeavy)
{
__flush(); // flush the current buffer
if (!isHeavy)
{
G.noFlush = false;
}
else
{
G.noFlush = true;
if (G.logFile)
setvbuf (G.logFile, NULL, _IOFBF, 16384); // flush every 16K
}
}
// ------------------------------------------------------------------------
// shutUp(): set muted mode (true: no output will be generated anymore)
//
// multi-threading: retrieving the previous state is not thread-safe,
// if you want, do this before starting
// ------------------------------------------------------------------------
static inline bool shutUp (bool quiet = true)
{
bool oldFlag = G.muted;
G.muted = quiet;
return oldFlag;
}
// ------------------------------------------------------------------------
// getLogFile(): get log file handle
// ------------------------------------------------------------------------
static inline FILE * getLogFile (void)
{
return G.logFile;
}
// ------------------------------------------------------------------------
// __showbuf(): output contents of buf[] with prefix prepended
// multi-threading: must be called from within critical section
// ------------------------------------------------------------------------
static inline void __showbuf (const std::string & prefix, bool nl)
{
ASSERT (strlen (G.buf) < sizeof (G.buf) / sizeof (*G.buf)); // security hint: safe overloads
std::string outtext = prefix + G.buf;
if (nl) outtext += "\n";
// write out; first to screen in case we can't write to log file
#ifndef ONENOTE_COMPILER
// OneNote treats it as an error if stderr is not empty.
// and in OneNote, we can't see message printed to stderr
// So, in OneNote, don't put it into stderr
// G.muted semantics is as follows
// - G.muted && !G.logFile => don't show anything
// - G.muted && G.logFile => write to log file
// - !G.muted && !G.logFile => write to stderr only
// - !G.muted && G.logFile => write to log file and stderr
if (!G.muted)
{
fwrite ((void*) outtext.c_str(), sizeof (*outtext.c_str()),
outtext.length(), stderr);
if (!G.noFlush)
fflush (stderr);
}
#endif
// write to log file
// check whether the log file has been disconnected or not
if (G.filename != "") // with known filename, suppose to reconnect
{
if (G.logFile == NULL || ferror (G.logFile) != 0)
{
setLogFileByName (NULL); // attempt to re-open the log file
if (G.logFile)
{
fprintf (G.logFile, "ERROR: __showbuf: log file handle lost, reconnected\n");
}
}
}
if (G.logFile)
{
size_t n = fwrite ((void*) outtext.c_str(), sizeof (*outtext.c_str()),
outtext.length(), G.logFile);
if (n != outtext.length() * sizeof (*outtext.c_str()))
{ // write error
fprintf (stderr, "ERROR: __showbuf: error writing this to log file: %s\n", strerror (errno));
fwrite ((void*) outtext.c_str(), sizeof (*outtext.c_str()),
outtext.length(), stderr);
}
else if (!G.noFlush) // flush logFile
{
__flush();
}
}
}
// ------------------------------------------------------------------------
// noLoggingReqd: function to determine if any logging reqd
// at all - used so that we can exit early if none reqd
// ------------------------------------------------------------------------
static inline bool noLoggingReqd()
{
return G.muted && !G.logFile;
}
// ------------------------------------------------------------------------
// message(): like printf(), writing to log output
// multi-threading: this function is thread-safe
// ------------------------------------------------------------------------
static inline void message (const char * fmt, ...)
{
if (noLoggingReqd()) return; // muted: all output is suppressed
msra::basetypes::CAutoLock autoLock (G.lock);
va_list arg_ptr;
va_start (arg_ptr, fmt);
vsprintf (G.buf, fmt, arg_ptr); // security hint: this is an overload
__showbuf ("", true);
}
static void message_nolf (const char * fmt, ...)
{
if (noLoggingReqd()) return; // muted: all output is suppressed
msra::basetypes::CAutoLock autoLock (G.lock);
va_list arg_ptr;
va_start (arg_ptr, fmt);
vsprintf (G.buf, fmt, arg_ptr); // security hint: this is an overload
__showbuf ("", false);
}
// ------------------------------------------------------------------------
// warning(): like message(), with text "WARNING: " prepended
// multi-threading: this function is thread-safe
// ------------------------------------------------------------------------
static void warning (const char * fmt, ...)
{
if (noLoggingReqd()) return; // muted: all output is suppressed
msra::basetypes::CAutoLock autoLock (G.lock);
va_list arg_ptr;
va_start (arg_ptr, fmt);
vsprintf (G.buf, fmt, arg_ptr); // security hint: this is an overload
__showbuf ("WARNING: ", true);
__flush();
}
// ------------------------------------------------------------------------
// __throw_or_exit(): exit() or throw exception depending on throwExceptionFlag
// ------------------------------------------------------------------------
static inline void __throw_or_exit (void)
{
__flush();
if (G.throwExceptionFlag)
{
throw message_exception (G.buf);
}
exit (1);
}
// ------------------------------------------------------------------------
// error(): like warning() but terminates program afterwards
// multi-threading: this function is thread-safe
// ------------------------------------------------------------------------
#pragma warning (push)
#pragma warning (disable : 4702) // the 'return 0;' causes this in Release
static int error (const char * fmt, ...)
{
#if 1 // special test code to determine the Windows error in case of a network error
DWORD winErr = GetLastError();
try
{
msra::basetypes::CAutoLock autoLock (G.lock);
sprintf (G.buf, "%d (\"%S\")", winErr, FormatWin32Error(winErr).c_str());
if (!noLoggingReqd())
__showbuf ("Win32 error of subsequent error message: ", true);
}
catch(...){}
#endif
msra::basetypes::CAutoLock autoLock (G.lock);
va_list arg_ptr;
va_start (arg_ptr, fmt);
vsprintf (G.buf, fmt, arg_ptr); // security hint: this is an overload
if (!noLoggingReqd())
{ // if muted, we format the msg (for __throw_or_exit) but don't print it
__showbuf ("ERROR: ", true);
}
__throw_or_exit();
return 0;
}
// ------------------------------------------------------------------------
// mem_error(): similar to error() but without any memory allocations
// (only one string argument allowed)
// multi-threading: this function is thread-safe
// ------------------------------------------------------------------------
static int mem_error (const char * fmt, int arg)
{
msra::basetypes::CAutoLock autoLock (G.lock);
if (!noLoggingReqd())
{ // if muted, we format the msg (for __throw_or_exit) but don't print it
fprintf (stderr, fmt, arg);
fprintf (stderr, "\n");
if (G.logFile)
{
fprintf (G.logFile, fmt, arg);
fprintf (G.logFile, "\n");
int rc = fflush (G.logFile);
if (rc != 0)
{
fprintf (stderr, "error flushing log message to file: %s\n",
strerror (errno));
}
}
}
// format msg for __throw_or_exit()
sprintf (G.buf, fmt, arg); // security hint: this is an overload
strcat (G.buf, "\n"); // security hint: this is an overload
__throw_or_exit();
return 0;
}
#pragma warning (pop)
static inline void __avoid_C4505 (void)
{ message (""); message_nolf (""); warning (""); error (""); mem_error ("", 0); }
#pragma pop_macro ("G")
};};
#pragma warning(pop)
// ===========================================================================
// compatibility macros (for older source code)
// ===========================================================================
#undef ERROR // defined in wingdi.h... aargh!
#define WARNING msra::logging::warning
#define ERROR msra::logging::error
#define MESSAGE msra::logging::message
#define MESSAGE_NOLF msra::logging::message_nolf
#define MEMERROR msra::logging::mem_error
#define SETLOGFILE msra::logging::setLogFile
// ===========================================================================
// special function for basetypes.h's ASSERT() macro
// ===========================================================================
#ifdef _CHECKED
void inline _CHECKED_ASSERT_error(const char * file, int line, const char * exp)
{ ERROR ("%s:%d:assertion failure: %s", file, line, exp); }
#endif
#endif // _MESSAGE_

133
DataReader/HTKMLFReader/msra_mgram.h
Просмотреть файл

@ -9,7 +9,6 @@
#pragma once
#include "basetypes.h"
#include "message.h" // for logging and throwing
#include "fileutil.h" // for opening/reading the ARPA file
#include <vector>
#include <string>
@ -173,7 +172,7 @@ public:
fgetstring (f, buf);
int id = (*this)[buf];
if (id != k)
throw logic_error ("plsa: sequence error while reading vocabulary");
RuntimeError ("plsa: sequence error while reading vocabulary");
}
}
};
@ -1027,7 +1026,7 @@ public:
const double log10 = log (10.0);
for (int m = 1; m <= M; m++)
{
MESSAGE_NOLF ("estimate: writing %d %d-grams..", map.size (m), m);
fprintf (stderr, "estimate: writing %d %d-grams..", map.size (m), m);
int step = (int) logP.size (m) / 100;
if (step == 0) step = 1;
int numMGramsWritten = 0;
@ -1061,13 +1060,13 @@ public:
// progress
if (numMGramsWritten % step == 0)
{
MESSAGE_NOLF (".");
fprintf (stderr, ".");
}
numMGramsWritten++;
}
fflushOrDie (outf);
ASSERT (numMGramsWritten == map.size (m));
MESSAGE ("");
fprintf (stderr, "\n");
}
fprintfOrDie (outf, "\n\\end\\\n");
@ -1126,7 +1125,7 @@ public:
{
int lineNo = 0;
msra::basetypes::auto_file_ptr f = fopenOrDie (pathname, L"rbS");
MESSAGE_NOLF ("read: reading %S", pathname.c_str());
fprintf (stderr, "read: reading %S", pathname.c_str());
filename = pathname; // (keep this info for debugging)
// --- read header information
@ -1154,7 +1153,7 @@ public:
M = (int) dims.size() -1;
if (M == 0)
ERROR ("read: mal-formed LM file, no dimension information (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, no dimension information (%d): %S", lineNo, pathname.c_str());
int fileM = M;
if (M > maxM)
M = maxM;
@ -1188,7 +1187,7 @@ public:
lineNo++, fgetline (f, buf);
if (sscanf (buf, "\\%d-grams:", &n) != 1 || n != m)
ERROR ("read: mal-formed LM file, bad section header (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, bad section header (%d): %S", lineNo, pathname.c_str());
lineNo++, fgetline (f, buf);
std::vector<int> mgram (m +1, -1); // current mgram being read ([0]=dummy)
@ -1207,7 +1206,7 @@ public:
// -- parse the line
tokens = &buf[0];
if ((int) tokens.size() != ((m < fileM) ? m + 2 : m + 1))
ERROR ("read: mal-formed LM file, incorrect number of tokens (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, incorrect number of tokens (%d): %S", lineNo, pathname.c_str());
double scoreVal = atof (tokens[0]); // ... use sscanf() instead for error checking?
double thisLogP = scoreVal * ln10xLMF; // convert to natural log
@ -1239,7 +1238,7 @@ public:
{
id = symbolToId (tok);
if (id == -1)
ERROR ("read: mal-formed LM file, m-gram contains unknown word (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, m-gram contains unknown word (%d): %S", lineNo, pathname.c_str());
}
}
mgram[n] = id; // that's our id
@ -1289,9 +1288,9 @@ skipMGram:
}
}
MESSAGE_NOLF (", %d %d-grams", map.size (m), m);
fprintf (stderr, ", %d %d-grams", map.size (m), m);
}
MESSAGE ("");
fprintf (stderr, "\n");
// check end tag
if (M == fileM)
@ -1299,7 +1298,7 @@ skipMGram:
while (buf[0] == 0 && !feof (f))
lineNo++, fgetline (f, buf);
if (strcmp (buf, "\\end\\") != 0)
ERROR ("read: mal-formed LM file, no \\end\\ tag (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, no \\end\\ tag (%d): %S", lineNo, pathname.c_str());
}
// update zerogram score by one appropriate for OOVs
@ -1534,7 +1533,7 @@ protected:
if (seenMass > 1.0)
{
if (seenMass > 1.0001) // (a minor round-off error is acceptable)
WARNING ("estimate: seen mass > 1.0: %8.5f --oops??", seenMass);
fprintf (stderr, "estimate: seen mass > 1.0: %8.5f --oops??\n", seenMass);
seenMass = 1.0; // oops?
}
@ -1543,7 +1542,7 @@ protected:
if (coveredBackoffMass > 1.0)
{
if (coveredBackoffMass > 1.0001) // 1.0 for unigrams, sometimes flags this
WARNING ("estimate: unseen backoff mass < 0: %8.5f --oops??", 1.0 - coveredBackoffMass);
fprintf (stderr, "estimate: unseen backoff mass < 0: %8.5f --oops??\n", 1.0 - coveredBackoffMass);
coveredBackoffMass = 1.0; // oops?
}
@ -1640,11 +1639,11 @@ public:
{// first time initial
minObs.resize(M, 0);
if (M > 2) minObs[2] = 2; // GangLi: prune trigram if Obs < 2, this is default value
MESSAGE("Set miniObs to 0 0 2.");
fprintf (stderr, "Set miniObs to 0 0 2.\n");
}
else
{
MESSAGE("Not reset miniObs because it has already been set.");
fprintf (stderr, "Not reset miniObs because it has already been set.\n");
}
for (int m = 1; m <= M; m++) counts.reserve (m, 1000000); // something to start with
@ -1654,7 +1653,7 @@ public:
void setMinObs(const std::vector<unsigned int> & setMinObs)
{
if (minObs.size() != setMinObs.size())
ERROR("In setMinObs: setMinObs size (%d) is not for %d-gram.", setMinObs.size(), minObs.size());
RuntimeError("In setMinObs: setMinObs size (%d) is not for %d-gram.", setMinObs.size(), minObs.size());
minObs = setMinObs;
}
@ -1688,7 +1687,7 @@ protected:
mcounts.push_back (mgram_map::coord(), ntoks); // zerogram count
std::vector<int> keybuf (M+1);
// do one order after another (to save memory)
MESSAGE_NOLF ("merge: adding %d tokens...", ntoks);
fprintf (stderr, "merge: adding %d tokens...", ntoks);
for (int m = 1; m <= M; m++)
{
mgram_map::cache_t mmapCache;
@ -1770,7 +1769,7 @@ protected:
mcounts.push_back (mmap.create (newkey, mmapCache), count); // store 'count' under 'key'
}
}
MESSAGE_NOLF (" %d %d-grams", mcounts.size (m), m);
fprintf (stderr, " %d %d-grams", mcounts.size (m), m);
}
// remove used up tokens from the buffer
@ -1788,7 +1787,7 @@ protected:
map.swap (mmap);
counts.swap (mcounts);
MESSAGE ("");
fprintf (stderr, "\n");
// destructor will delete previous counts and map (now in mcount/mmap)
}
@ -1889,7 +1888,7 @@ protected:
int id = dropId == -1 ? userSymMap[p] : constSymMap[p];
ids.push_back (id);
if (totalTokens++ % 100000 == 0) MESSAGE_NOLF (".");
if (totalTokens++ % 100000 == 0) fprintf (stderr, ".");
}
ids.push_back (endId);
totalTokens += 2;
@ -1928,7 +1927,7 @@ public:
int dropId = filterVocabulary ? unkId != -1 ? unkId : userSymMap.size() : -1;
if (filterVocabulary)
ERROR ("CMGramLMEstimator::read() not tested for filterVocabulary==true");
RuntimeError ("CMGramLMEstimator::read() not tested for filterVocabulary==true");
// reset adaptation
adapt (NULL, maxM); // pass dimension here
@ -1947,14 +1946,14 @@ public:
string thispath = fgetline (f);
if (thispath.empty() || thispath[0] == '#') continue; // comment
msra::basetypes::auto_file_ptr thisf = fopenOrDie (thispath, "rbS");
MESSAGE_NOLF ("read: ingesting training text from %s ..", thispath.c_str());
fprintf (stderr, "read: ingesting training text from %s ..", thispath.c_str());
int numTokens = read (thisf, userSymMap, startId, endId, dropId);
MESSAGE ("%d tokens", numTokens);
fprintf (stderr, "%d tokens\n", numTokens);
}
}
else if (!tag.empty() && tag[0] == '#')
{
ERROR ("read: unknown tag '%s'", tag.c_str());
RuntimeError ("read: unknown tag '%s'", tag.c_str());
}
else // no tag: just load the file directly
{
@ -2028,7 +2027,7 @@ public:
while (M > 0 && counts.size (M) == 0) resize (M-1);
for (int m = 1; m <= M; m++)
MESSAGE ("estimate: read %d %d-grams", counts.size (m), m);
fprintf (stderr, "estimate: read %d %d-grams\n", counts.size (m), m);
// === Kneser-Ney smoothing
// This is a strange algorithm.
@ -2043,14 +2042,14 @@ public:
mgram_data<unsigned int> KNTotalCounts; // [shifted, shortened m-gram] (*,v,*)
if (M >= 2)
{
MESSAGE ("estimate: allocating Kneser-Ney counts...");
fprintf (stderr, "estimate: allocating Kneser-Ney counts...\n");
KNCounts.init (M-1);
for (int m = 0; m <= M-1; m++) KNCounts.assign (m, counts.size (m), 0);
KNTotalCounts.init (M-2);
for (int m = 0; m <= M-2; m++) KNTotalCounts.assign (m, counts.size (m), 0);
MESSAGE ("estimate: computing Kneser-Ney counts...");
fprintf (stderr, "estimate: computing Kneser-Ney counts...\n");
// loop over all m-grams to determine KN counts
for (mgram_map::deep_iterator iter (map); iter; ++iter)
@ -2082,7 +2081,7 @@ public:
std::vector<double> d1 (M+1, 0.0);
std::vector<double> d2 (M+1, 0.0);
std::vector<double> d3 (M+1, 0.0);
MESSAGE_NOLF ("estimate: discounting values:");
fprintf (stderr, "estimate: discounting values:");
{
// actually estimate discounting values
@ -2122,11 +2121,11 @@ public:
{
if (n1[m] == 0) throw runtime_error (msra::strfun::strprintf ("estimate: error estimating discounting values: n1[%d] == 0", m));
if (n2[m] == 0) throw runtime_error (msra::strfun::strprintf ("estimate: error estimating discounting values: n2[%d] == 0", m));
//if (n3[m] == 0) ERROR ("estimate: error estimating discounting values: n3[%d] == 0", m);
//if (n3[m] == 0) RuntimeError ("estimate: error estimating discounting values: n3[%d] == 0", m);
double Y = n1[m] / (n1[m] + 2.0 * n2[m]);
if (n3[m] ==0 || n4[m] == 0)
{
WARNING ("estimate: n3[%d] or n4[%d] is 0, falling back to unmodified discounting", m, m);
fprintf (stderr, "estimate: n3[%d] or n4[%d] is 0, falling back to unmodified discounting\n", m, m);
d1[m] = Y;
d2[m] = Y;
d3[m] = Y;
@ -2138,16 +2137,16 @@ public:
d3[m] = 3.0 - 4.0 * Y * n4[m] / n3[m];
}
// ... can these be negative??
MESSAGE_NOLF (" (%.3f, %.3f, %.3f)", d1[m], d2[m], d3[m]);
fprintf (stderr, " (%.3f, %.3f, %.3f)", d1[m], d2[m], d3[m]);
}
MESSAGE ("");
fprintf (stderr, "\n");
}
// === threshold against minimum counts (set counts to 0)
// this is done to save memory, but it has no impact on the seen probabilities
// ...well, it does, as pruned mass get pushed to back-off distribution... ugh!
MESSAGE ("estimate: pruning against minimum counts...");
fprintf (stderr, "estimate: pruning against minimum counts...\n");
// prune unigrams first (unigram cut-off can be higher than m-gram cut-offs,
// as a means to decimate the vocabulary)
@ -2161,7 +2160,7 @@ public:
dropWord[wid] = true; // will throw out all related m-grams
removedWords++;
}
MESSAGE ("estimate: removing %d too rare vocabulary entries", removedWords);
fprintf (stderr, "estimate: removing %d too rare vocabulary entries\n", removedWords);
// now prune m-grams against count cut-off
@ -2189,7 +2188,7 @@ public:
if (m < M) histCoord[m] = iter;
mgram_map::coord j = histCoord[m-1]; // parent
if (counts[j] == 0)
ERROR ("estimate: invalid pruning: a parent m-gram got pruned away");
RuntimeError ("estimate: invalid pruning: a parent m-gram got pruned away");
//throw runtime_error ("estimate: invalid pruning: a parent m-gram got pruned away");
numMGrams[m]++;
}
@ -2197,7 +2196,7 @@ public:
for (int m = 1; m <= M; m++)
{
MESSAGE ("estimate: %d-grams after pruning: %d out of %d (%.1f%%)", m,
fprintf (stderr, "estimate: %d-grams after pruning: %d out of %d (%.1f%%)\n", m,
numMGrams[m], counts.size (m),
100.0 * numMGrams[m] / max (counts.size (m), 1));
}
@ -2212,7 +2211,7 @@ public:
// === estimate M-gram
MESSAGE ("estimate: estimating probabilities...");
fprintf (stderr, "estimate: estimating probabilities...\n");
// dimension the m-gram store
mgram_data<float> P (M); // [M+1][i] probabilities
@ -2231,7 +2230,7 @@ public:
P.push_back (mgram_map::coord(), 0.0f); // will be updated later
for (int m = 1; m <= M; m++)
{
MESSAGE ("estimate: estimating %d %d-gram probabilities...", numMGrams[m], m);
fprintf (stderr, "estimate: estimating %d %d-gram probabilities...\n", numMGrams[m], m);
// loop over all m-grams of level 'm'
msra::basetypes::fixed_vector<mgram_map::coord> histCoord (m);
@ -2270,7 +2269,7 @@ public:
{
count = KNCounts[iter]; // (u,v,w) -> count (*,v,w)
if (count == 0) // must exist
ERROR ("estimate: malformed data: back-off value not found (numerator)");
RuntimeError ("estimate: malformed data: back-off value not found (numerator)");
const mgram_map::key key_h = key.pop_w();
mgram_map::foundcoord c_h = map[key_h];
@ -2278,7 +2277,7 @@ public:
throw runtime_error ("estimate: invalid shortened KN history");
histCount = KNTotalCounts[c_h]; // (u,v,w) -> count (*,v,*)
if (histCount == 0) // must exist
ERROR ("estimate: malformed data: back-off value not found (denominator)");
RuntimeError ("estimate: malformed data: back-off value not found (denominator)");
ASSERT (histCount >= count);
}
}
@ -2323,7 +2322,7 @@ public:
throw runtime_error ("estimate: negative discounted count value");
if (histCount == 0)
ERROR ("estimate: unexpected 0 denominator");
RuntimeError ("estimate: unexpected 0 denominator");
double dP = dcount / histCount;
// and this is the discounted probability value
{
@ -2339,7 +2338,7 @@ skippruned:; // m-gram was pruned
}
}
// the distributions are not normalized --discount mass is missing
MESSAGE ("estimate: freeing memory for counts...");
fprintf (stderr, "estimate: freeing memory for counts...\n");
KNCounts.clear(); // free some memory
KNTotalCounts.clear();
@ -2372,7 +2371,7 @@ skippruned:; // m-gram was pruned
if (missingUnigramMass > 0.0)
{
float missingUnigramProb = (float) (missingUnigramMass * P[mgram_map::coord()]);
MESSAGE ("estimate: distributing missing unigram mass of %.2f to %d unigrams",
fprintf (stderr, "estimate: distributing missing unigram mass of %.2f to %d unigrams\n",
missingUnigramMass, vocabSize);
for (mgram_map::iterator iter (map, 1); iter; ++iter)
{
@ -2383,7 +2382,7 @@ skippruned:; // m-gram was pruned
// --- M-gram sections --back-off weights
MESSAGE ("estimate: determining back-off weights...");
fprintf (stderr, "estimate: determining back-off weights...\n");
computeBackoff (map, M, P, logB, false);
// now the LM is normalized assuming the ARPA back-off computation
@ -2412,9 +2411,9 @@ skippruned:; // m-gram was pruned
// desired OOV score.
updateOOVScore();
MESSAGE_NOLF ("estimate: done");
for (int m = 1; m <= M; m++) MESSAGE_NOLF (", %d %d-grams", logP.size (m), m);
MESSAGE ("");
fprintf (stderr, "estimate: done");
for (int m = 1; m <= M; m++) fprintf (stderr, ", %d %d-grams", logP.size (m), m);
fprintf (stderr, "\n");
}
};
@ -2509,10 +2508,10 @@ skipMGram:
template<class SYMMAP>
static void write (const ILM & lm, int M, FILE * outf, const SYMMAP & symbols)
{
MESSAGE ("write: cloning...");
fprintf (stderr, "write: cloning...\n");
CMGramLMClone outlm;
outlm.clone (lm, M);
MESSAGE ("write: saving...");
fprintf (stderr, "write: saving...\n");
((const CMGramLM&) outlm).write (outf, symbols);
}
@ -2808,7 +2807,7 @@ public:
{
int lineNo = 0;
msra::basetypes::auto_file_ptr f = fopenOrDie (pathname, L"rbS");
MESSAGE_NOLF ("read: reading %S", pathname.c_str());
fprintf (stderr, "read: reading %S", pathname.c_str());
filename = pathname; // (keep this info for debugging)
// --- read header information
@ -2836,7 +2835,7 @@ public:
M = (int) dims.size() -1;
if (M == 0)
ERROR ("read: mal-formed LM file, no dimension information (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, no dimension information (%d): %S", lineNo, pathname.c_str());
int fileM = M;
if (M > maxM)
M = maxM;
@ -2866,7 +2865,7 @@ public:
lineNo++, fgetline (f, buf);
if (sscanf (buf, "\\%d-grams:", &n) != 1 || n != m)
ERROR ("read: mal-formed LM file, bad section header (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, bad section header (%d): %S", lineNo, pathname.c_str());
lineNo++, fgetline (f, buf);
std::vector<int> mgram (m +1); // current mgram being read
@ -2888,7 +2887,7 @@ public:
const char * delim = " \t\n\r";
const char * score = strtok (&buf[0], delim);
if (score == NULL || score[0] == 0) // not checking whether it is numeric
ERROR ("read: mal-formed LM file, no score (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, no score (%d): %S", lineNo, pathname.c_str());
double scoreVal = atof (score);
double logP = scoreVal * ln10xLMF; // convert to natural log
@ -2897,7 +2896,7 @@ public:
{
/*const*/ char * tok = strtok (NULL, delim);
if (tok == NULL)
ERROR ("read: mal-formed LM file, not enough words in mgram (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, not enough words in mgram (%d): %S", lineNo, pathname.c_str());
// map to id
int id;
if (m == 1) // unigram: build vocab table
@ -2922,7 +2921,7 @@ public:
{
id = symbolToId (tok);
if (id == -1)
ERROR ("read: mal-formed LM file, m-gram contains unknown word (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, m-gram contains unknown word (%d): %S", lineNo, pathname.c_str());
}
}
mgram[n] = id; // that's our id
@ -2934,7 +2933,7 @@ public:
{
const char * bo = strtok (NULL, delim);
if (score == NULL || score[0] == 0) // not checking whether it is numeric
ERROR ("read: mal-formed LM file, no score (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, no score (%d): %S", lineNo, pathname.c_str());
double boVal = atof (bo);
logB = boVal * ln10xLMF; // convert to natural log
}
@ -2956,7 +2955,7 @@ public:
continue;
if (prevValid && mgram[n] < prevmgram[n])
ERROR ("read: mal-formed LM file, m-gram out of order (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, m-gram out of order (%d): %S", lineNo, pathname.c_str());
// a history token differs from previous mgram. That history must exist.
const std::vector<LMSCORE> & entries_n = entries[n];
@ -2965,14 +2964,14 @@ public:
int end = refs_h[histEntry[n -1] +1].firstEntry;
int i = findEntry (entries_n, beg, end, mgram[n]);
if (i == -1) // unknown history: fall back
ERROR ("read: mal-formed LM file, m-gram history not defined (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, m-gram history not defined (%d): %S", lineNo, pathname.c_str());
// found it: narrow down search range
histEntry[n] = i;
prevValid = false;
}
if (prevValid && mgram[m] <= prevmgram[m])
ERROR ("read: mal-formed LM file, m-gram out of order (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, m-gram out of order (%d): %S", lineNo, pathname.c_str());
if (m < M) // create history entry
refs[m].push_back (LMHIST (0, logB));
@ -3015,9 +3014,9 @@ skipMGram:
}
}
MESSAGE_NOLF (", %d %d-grams", entries[m].size(), m);
fprintf (stderr, ", %d %d-grams", entries[m].size(), m);
}
MESSAGE ("");
fprintf (stderr, "\n");
// check end tag
if (M == fileM)
@ -3025,7 +3024,7 @@ skipMGram:
while (buf[0] == 0 && !feof (f))
lineNo++, fgetline (f, buf);
if (strcmp (buf, "\\end\\") != 0)
ERROR ("read: mal-formed LM file, no \\end\\ tag (%d): %S", lineNo, pathname.c_str());
RuntimeError ("read: mal-formed LM file, no \\end\\ tag (%d): %S", lineNo, pathname.c_str());
}
// update zerogram score
@ -3107,7 +3106,7 @@ public:
if (logP <= -1e20)
{
#if 0 // should really not happen
MESSAGE ("skipping poor-scoring %s (%.2f)", symMap[buf[i]], logP);
fprintf (stderr, "skipping poor-scoring %s (%.2f)\n", symMap[buf[i]], logP);
#endif
numOOVTokens++;
continue;
@ -3134,14 +3133,14 @@ public:
strcat (seq, "_");
strcat (seq, symMap[buf[i1]]);
}
MESSAGE ("=%-22s\t%6.2f\t%s\t%s %s", seq+1, logP, pbuf +1, smseenhist, smseen);
fprintf (stderr, "=%-22s\t%6.2f\t%s\t%s %s\n", seq+1, logP, pbuf +1, smseenhist, smseen);
#else
symMap;
#endif
#if 0 // testing of optimization
double logP1 = lm.score_unoptimized (&buf[0], i +1); // use full history
if (fabs (logP - logP1) > 1e-3)
ERROR ("bug in optimized score()");
RuntimeError ("bug in optimized score()");
#endif
logPAcc += logP;
numTokensAcc++;

2
DataReader/LUSequenceReader/LUSequenceParser.cpp
Просмотреть файл

@ -272,7 +272,7 @@ size_t LUSequenceParser<NumType, LabelType>::UpdateBuffer()
size_t bytesToRead = min(m_bufferSize, m_fileSize-m_bufferStart)-saveBytes;
size_t bytesRead = fread(m_fileBuffer+saveBytes, 1, bytesToRead, m_pFile);
if (bytesRead == 0 && ferror(m_pFile))
Error("LUSequenceParser::UpdateBuffer - error reading file");
RuntimeError("LUSequenceParser::UpdateBuffer - error reading file");
return bytesRead;
}

6
DataReader/LUSequenceReader/LUSequenceParser.h
Просмотреть файл

@ -220,10 +220,10 @@ public:
errno_t err = _wfopen_s( &m_pFile, fileName, L"rb" );
if (err)
Error("LUSequenceParser::ParseInit - error opening file");
RuntimeError("LUSequenceParser::ParseInit - error opening file");
int rc = _fseeki64(m_pFile, 0, SEEK_END);
if (rc)
Error("LUSequenceParser::ParseInit - error seeking in file");
RuntimeError("LUSequenceParser::ParseInit - error seeking in file");
m_fileBuffer = new BYTE[m_bufferSize];
}
@ -271,7 +271,7 @@ public:
if (mFile) fclose(mFile);
if (_wfopen_s(&mFile, fileName, L"rt") != 0)
Error("cannot open file %s", fileName);
RuntimeError("cannot open file %s", fileName);
}
void ParseReset()

20
DataReader/LUSequenceReader/LUSequenceReader.cpp
Просмотреть файл

@ -90,7 +90,7 @@ bool LUSequenceReader<ElemType>::GetIdFromLabel(const vector<string>& labelValue
val.push_back(found->second);
}
else
Error("LUSequenceReader::GetIdFromLabel: cannot find value");
RuntimeError("LUSequenceReader::GetIdFromLabel: cannot find value");
}
return true;
}
@ -262,7 +262,7 @@ void LUSequenceReader<ElemType>::Init(const ConfigParameters& readerConfig)
}
}
else
Error("two label definitions (in and out) required for Sequence Reader");
RuntimeError("two label definitions (in and out) required for Sequence Reader");
ConfigParameters featureConfig = readerConfig(m_featuresName,"");
ConfigParameters labelConfig[2] = {readerConfig(m_labelsName[0],""),readerConfig(m_labelsName[1],"")};
@ -440,7 +440,7 @@ void LUSequenceReader<ElemType>::ChangeMaping(const map<string, string>& maplist
{
if (punk == word4idx.end())
{
Error("check unk list is missing ");
RuntimeError("check unk list is missing ");
}
idx = punk->second;
}
@ -468,7 +468,7 @@ void LUSequenceReader<ElemType>::ReadLabelInfo(const wstring & vocfile,
if (vin == nullptr)
{
Error("cannot open word class file");
RuntimeError("cannot open word class file");
}
b = 0;
while (!feof(vin)){
@ -720,7 +720,7 @@ bool LUSequenceReader<ElemType>::SentenceEnd()
{
LabelIdType index ;
if (CheckIdFromLabel(labelInfo.endSequence, labelInfo, index) == false)
Error("cannot find sentence begining label");
RuntimeError("cannot find sentence begining label");
if (m_labelIdData[jEnd] == index )
return true;
@ -752,7 +752,7 @@ void LUSequenceReader<ElemType>::SetLabelMapping(const std::wstring& /*sectionNa
{
if (m_cachingReader)
{
Error("Cannot set mapping table when the caching reader is being used");
RuntimeError("Cannot set mapping table when the caching reader is being used");
}
LabelInfo& labelInfo = m_labelInfo[( m_labelInfo[labelInfoOut].type == labelNextWord)?labelInfoIn:labelInfoOut];
@ -776,7 +776,7 @@ template<class ElemType>
bool LUSequenceReader<ElemType>::GetData(const std::wstring& sectionName, size_t numRecords, void* data, size_t& dataBufferSize, size_t recordStart)
{
if (!m_cachingReader)
Error("GetData not supported in LUSequenceReader");
RuntimeError("GetData not supported in LUSequenceReader");
return m_cachingReader->GetData(sectionName, numRecords, data, dataBufferSize, recordStart);
}
@ -832,7 +832,7 @@ void BatchLUSequenceReader<ElemType>::Init(const ConfigParameters& readerConfig)
}
}
else
Error("two label definitions (in and out) required for Sequence Reader");
RuntimeError("two label definitions (in and out) required for Sequence Reader");
ConfigParameters featureConfig = readerConfig(m_featuresName,"");
ConfigParameters labelConfig[2] = {readerConfig(m_labelsName[0],""),readerConfig(m_labelsName[1],"")};
@ -1185,7 +1185,7 @@ bool BatchLUSequenceReader<ElemType>::EnsureDataAvailable(size_t /*mbStartSample
}
else
{
Error("Input label expected to be a category label");
RuntimeError("Input label expected to be a category label");
}
}
@ -1238,7 +1238,7 @@ bool BatchLUSequenceReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix
// figure out the size of the next sequence
actualmbsize = m_labelIdData.size() ;
if (actualmbsize > m_mbSize * mToProcess.size()){
Error("specified minibatch size %d is smaller than the actual minibatch size %d. memory can crash!", m_mbSize, actualmbsize);
RuntimeError("specified minibatch size %d is smaller than the actual minibatch size %d. memory can crash!", m_mbSize, actualmbsize);
}
// now get the labels

5
DataReader/LUSequenceReader/LUSequenceReader.vcxproj
Просмотреть файл

@ -100,6 +100,11 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\Include\basetypes.h" />
<ClInclude Include="..\..\Common\Include\DataReader.h" />
<ClInclude Include="..\..\Common\Include\DataWriter.h" />
<ClInclude Include="..\..\Common\Include\File.h" />
<ClInclude Include="..\..\Common\Include\fileutil.h" />
<ClInclude Include="LUSequenceWriter.h" />
<ClInclude Include="minibatchsourcehelpers.h" />
<ClInclude Include="stdafx.h" />

65
DataReader/LUSequenceReader/LUSequenceReader.vcxproj.filters Normal file
Просмотреть файл

@ -0,0 +1,65 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="Exports.cpp" />
<ClCompile Include="dllmain.cpp" />
<ClCompile Include="LUSequenceWriter.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="LUSequenceReader.cpp" />
<ClCompile Include="LUSequenceParser.cpp" />
<ClCompile Include="..\..\Common\ConfigFile.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\DataReader.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="DataWriter.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\File.cpp">
<Filter>Common</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="LUSequenceWriter.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="LUSequenceReader.h" />
<ClInclude Include="LUSequenceParser.h" />
<ClInclude Include="..\..\Common\Include\basetypes.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataReader.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataWriter.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\File.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\fileutil.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="minibatchsourcehelpers.h">
<Filter>Duplicates to remove</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Text Include="SequenceTest.txt" />
</ItemGroup>
<ItemGroup>
<Filter Include="Common">
<UniqueIdentifier>{75463cd7-3094-4900-8dae-5b6835c23fc4}</UniqueIdentifier>
</Filter>
<Filter Include="Common\Include">
<UniqueIdentifier>{85d2fa50-2b95-4ec7-8f2c-c5c0b1cb493e}</UniqueIdentifier>
</Filter>
<Filter Include="Duplicates to remove">
<UniqueIdentifier>{10f819fb-8861-4607-9389-60ca80f968c2}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>

6
DataReader/LUSequenceReader/LUSequenceWriter.cpp
Просмотреть файл

@ -39,7 +39,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
ConfigArray outputNames = writerConfig("outputNodeNames","");
if (outputNames.size()<1)
Error("writer needs at least one outputNodeName specified in config");
RuntimeError("writer needs at least one outputNodeName specified in config");
foreach_index(i, outputNames) // inputNames should map to node names
@ -75,7 +75,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (vin == nullptr)
{
Error("cannot open word class file");
RuntimeError("cannot open word class file");
}
b = 0;
while (!feof(vin)){
@ -128,7 +128,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
string str(outputFile.begin(), outputFile.end());
ofs = fopen(str.c_str(), "wt");
if (ofs == nullptr)
Error("Cannot open open %s for writing", str.c_str());
RuntimeError("Cannot open open %s for writing", str.c_str());
outputFileIds[outputFile] = ofs;
fp = ofs;
}

6
DataReader/SequenceReader/SequenceParser.cpp
Просмотреть файл

@ -341,7 +341,7 @@ int64_t SequenceParser<NumType, LabelType>::GetFilePosition()
{
int64_t position = _ftelli64(m_pFile);
if (position == -1L)
Error("SequenceParser::GetFilePosition - error retrieving file position in file");
RuntimeError("SequenceParser::GetFilePosition - error retrieving file position in file");
return position;
}
@ -354,7 +354,7 @@ void SequenceParser<NumType, LabelType>::SetFilePosition(int64_t position)
{
int rc = _fseeki64(m_pFile, position, SEEK_SET);
if (rc)
Error("SequenceParser::SetFilePosition - error seeking in file");
RuntimeError("SequenceParser::SetFilePosition - error seeking in file");
// setup state machine to start at this position
PrepareStartPosition(position);
@ -407,7 +407,7 @@ size_t SequenceParser<NumType, LabelType>::UpdateBuffer()
size_t bytesToRead = min(m_bufferSize, m_fileSize-m_bufferStart)-saveBytes;
size_t bytesRead = fread(m_fileBuffer+saveBytes, 1, bytesToRead, m_pFile);
if (bytesRead == 0 && ferror(m_pFile))
Error("SequenceParser::UpdateBuffer - error reading file");
RuntimeError("SequenceParser::UpdateBuffer - error reading file");
return bytesRead;
}

2
DataReader/SequenceReader/SequenceParser.h
Просмотреть файл

@ -522,7 +522,7 @@ public:
if (mFile) fclose(mFile);
if (_wfopen_s(&mFile, fileName, L"rt") != 0)
Error("cannot open file %s", fileName);
RuntimeError("cannot open file %s", fileName);
}
void ParseReset()

44
DataReader/SequenceReader/SequenceReader.cpp
Просмотреть файл

@ -178,7 +178,7 @@ bool SequenceReader<ElemType>::EnsureDataAvailable(size_t mbStartSample, bool /*
if ((m_sequence.size() == 1 ? epochSample : epochSample - m_sequence[m_sequence.size()-2]) > m_mbSize)
{
fprintf(stderr, "read sentence length is longer than the minibatch size. should be smaller. increase the minibatch size to at least %d", epochSample);
Error("read sentence length is longer than the minibatch size. should be smaller. increase the minibatch size to at least %d", epochSample);
RuntimeError("read sentence length is longer than the minibatch size. should be smaller. increase the minibatch size to at least %d", epochSample);
}
if (!_strcmpi(labelValue.c_str(), m_labelInfo[labelInfoIn].endSequence.c_str()))
@ -198,25 +198,25 @@ bool SequenceReader<ElemType>::EnsureDataAvailable(size_t mbStartSample, bool /*
}
else
{
Error("Input label expected to be a category label");
RuntimeError("Input label expected to be a category label");
}
// if we have potential features
if (m_featureDim > 0)
{
Error("to-do. Assume sparse input feature. need to change the code from dense matrix");
RuntimeError("to-do. Assume sparse input feature. need to change the code from dense matrix");
// move the position up to the start of the additional features section
/* pos += labelIn.dim;
assert(pos + m_featureDim == m_featureData.size());
// this has to be an even number, a pair of index and value
if ((spos.numberPos&1) != 0)
Error("Features must be specified in pairs (index:value). Invalid features for label '%s'\n", labelValue);
RuntimeError("Features must be specified in pairs (index:value). Invalid features for label '%s'\n", labelValue);
while (feature < spos.numberPos)
{
int index = (int)featureTemp[feature++];
if (index < 0 || index >= m_featureDim)
Error("Invalid feature index: %d for label '%s', feature max dimension = %lld\n", index, labelValue, m_featureDim);
RuntimeError("Invalid feature index: %d for label '%s', feature max dimension = %lld\n", index, labelValue, m_featureDim);
ElemType value = featureTemp[feature++];
m_featureData[pos+index] = value;
@ -240,7 +240,7 @@ bool SequenceReader<ElemType>::EnsureDataAvailable(size_t mbStartSample, bool /*
}
else
{
Error("Invalid output label type, expected Category, or Next Word");
RuntimeError("Invalid output label type, expected Category, or Next Word");
}
// get the ID from the label
@ -260,11 +260,11 @@ bool SequenceReader<ElemType>::EnsureDataAvailable(size_t mbStartSample, bool /*
int jEnd = (int) m_labelIdData.size() - 1;
LabelIdType index ;
if (CheckIdFromLabel(labelInfo.endSequence, labelInfo, index) == false)
Error("cannot find sentence begining label");
RuntimeError("cannot find sentence begining label");
if (m_labelIdData[jEnd] != index )
/// for language model, the first word/letter has to be <s>
Error("SequenceReader: the last letter/word of a batch has to be the sentence ending symbol");
RuntimeError("SequenceReader: the last letter/word of a batch has to be the sentence ending symbol");
}
}
@ -436,7 +436,7 @@ void SequenceReader<ElemType>::Init(const ConfigParameters& readerConfig)
}
}
else
Error("two label definitions (in and out) required for Sequence Reader");
RuntimeError("two label definitions (in and out) required for Sequence Reader");
ConfigParameters featureConfig = readerConfig(m_featuresName,"");
ConfigParameters labelConfig[2] = {readerConfig(m_labelsName[0],""),readerConfig(m_labelsName[1],"")};
@ -616,7 +616,7 @@ void SequenceReader<ElemType>::ReadClassInfo(const wstring & vocfile, bool /*fla
if (vin == nullptr)
{
Error("cannot open word class file");
RuntimeError("cannot open word class file");
}
for (int a = 0; a < nwords; a++)
{
@ -948,7 +948,7 @@ bool SequenceReader<ElemType>::SentenceEnd()
{
LabelIdType index ;
if (CheckIdFromLabel(labelInfo.endSequence, labelInfo, index) == false)
Error("cannot find sentence begining label");
RuntimeError("cannot find sentence begining label");
if (m_labelIdData[jEnd] == index )
return true;
@ -1088,7 +1088,7 @@ bool SequenceReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix<ElemTy
}
if (actualmbsize > m_mbSize){
Error("specified minibatch size %d is smaller than the actual minibatch size %d. memory can crash!", m_mbSize, actualmbsize);
RuntimeError("specified minibatch size %d is smaller than the actual minibatch size %d. memory can crash!", m_mbSize, actualmbsize);
}
// hit the end of the dataset,
@ -1184,7 +1184,7 @@ bool SequenceReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix<ElemTy
}
}catch(...)
{
Error("features size might not be sufficiently large. The asked minibatch size is %s. check minibatchSize in the feature definition" ,actualmbsize);
RuntimeError("features size might not be sufficiently large. The asked minibatch size is %s. check minibatchSize in the feature definition" ,actualmbsize);
}
try
@ -1208,7 +1208,7 @@ bool SequenceReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix<ElemTy
}
}catch(...)
{
Error("cannot find matrices for %s", m_labelsName[labelInfoOut]);
RuntimeError("cannot find matrices for %s", m_labelsName[labelInfoOut]);
}
// we read some records, so process them
@ -1232,7 +1232,7 @@ void SequenceReader<ElemType>::OrganizeClass()
for (i=0; i<class_size; i++) {
if (class_cn[i] == 0) {
Error ("ERROR: class is empty");
RuntimeError ("class is empty");
}
}
}
@ -1259,7 +1259,7 @@ void SequenceReader<ElemType>::SetLabelMapping(const std::wstring& /*sectionName
{
if (m_cachingReader)
{
Error("Cannot set mapping table when the caching reader is being used");
RuntimeError("Cannot set mapping table when the caching reader is being used");
}
LabelInfo& labelInfo = m_labelInfo[( m_labelInfo[labelInfoOut].type == labelNextWord)?labelInfoIn:labelInfoOut];
@ -1283,7 +1283,7 @@ template<class ElemType>
bool SequenceReader<ElemType>::GetData(const std::wstring& sectionName, size_t numRecords, void* data, size_t& dataBufferSize, size_t recordStart)
{
if (!m_cachingReader)
Error("GetData not supported in SequenceReader");
RuntimeError("GetData not supported in SequenceReader");
return m_cachingReader->GetData(sectionName, numRecords, data, dataBufferSize, recordStart);
}
@ -1324,7 +1324,7 @@ void BatchSequenceReader<ElemType>::Init(const ConfigParameters& readerConfig)
}
}
else
Error("two label definitions (in and out) required for Sequence Reader");
RuntimeError("two label definitions (in and out) required for Sequence Reader");
ConfigParameters featureConfig = readerConfig(m_featuresName,"");
ConfigParameters labelConfig[2] = {readerConfig(m_labelsName[0],""),readerConfig(m_labelsName[1],"")};
@ -1681,7 +1681,7 @@ bool BatchSequenceReader<ElemType>::EnsureDataAvailable(size_t /*mbStartSample*/
}
else
{
Error("Input label expected to be a category label");
RuntimeError("Input label expected to be a category label");
}
// now get the output label
@ -1700,7 +1700,7 @@ bool BatchSequenceReader<ElemType>::EnsureDataAvailable(size_t /*mbStartSample*/
}
else
{
Error("Invalid output label type, expected Category, or Next Word");
RuntimeError("Invalid output label type, expected Category, or Next Word");
}
// get the ID from the label
@ -1747,7 +1747,7 @@ bool BatchSequenceReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix<E
// figure out the size of the next sequence
actualmbsize = m_labelIdData.size() ;
if (actualmbsize > m_mbSize * mToProcess.size()){
Error("specified minibatch size %d is smaller than the actual minibatch size %d. memory can crash!", m_mbSize, actualmbsize);
RuntimeError("specified minibatch size %d is smaller than the actual minibatch size %d. memory can crash!", m_mbSize, actualmbsize);
}
// now get the labels
@ -1834,7 +1834,7 @@ bool BatchSequenceReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix<E
}
}catch(...)
{
Error("features size might not be sufficiently large. The asked minibatch size is %s. check minibatchSize in the feature definition" ,actualmbsize);
RuntimeError("features size might not be sufficiently large. The asked minibatch size is %s. check minibatchSize in the feature definition" ,actualmbsize);
}
// we read some records, so process them

5
DataReader/SequenceReader/SequenceReader.vcxproj
Просмотреть файл

@ -100,6 +100,11 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\Include\basetypes.h" />
<ClInclude Include="..\..\Common\Include\DataReader.h" />
<ClInclude Include="..\..\Common\Include\DataWriter.h" />
<ClInclude Include="..\..\Common\Include\File.h" />
<ClInclude Include="..\..\Common\Include\fileutil.h" />
<ClInclude Include="minibatchsourcehelpers.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />

64
DataReader/SequenceReader/SequenceReader.vcxproj.filters Normal file
Просмотреть файл

@ -0,0 +1,64 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="Exports.cpp" />
<ClCompile Include="dllmain.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="SequenceReader.cpp" />
<ClCompile Include="SequenceParser.cpp" />
<ClCompile Include="..\..\Common\ConfigFile.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\DataReader.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\DataWriter.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\File.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<Filter>Common</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="SequenceReader.h" />
<ClInclude Include="SequenceParser.h" />
<ClInclude Include="minibatchsourcehelpers.h">
<Filter>Duplicates to remove</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\basetypes.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataReader.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataWriter.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\File.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\fileutil.h">
<Filter>Common\Include</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Text Include="SentenceTest.txt" />
<Text Include="SequenceTest.txt" />
</ItemGroup>
<ItemGroup>
<Filter Include="Common">
<UniqueIdentifier>{82bb574a-405b-43a0-a59b-52a60aef9d61}</UniqueIdentifier>
</Filter>
<Filter Include="Common\Include">
<UniqueIdentifier>{761d8c5d-2d9b-47a0-8409-9fc9a502801b}</UniqueIdentifier>
</Filter>
<Filter Include="Duplicates to remove">
<UniqueIdentifier>{616e954b-4068-45dc-a1d4-aaa9873983ec}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>

14
DataReader/UCIFastReader/UCIFastReader.cpp
Просмотреть файл

@ -148,7 +148,7 @@ bool UCIFastReader<ElemType>::EnsureDataAvailable(size_t mbStartSample, bool end
if (value == m_mapLabelToId.end())
{
if (m_labelFileToWrite.empty())
Error("label found in data not specified in label mapping file: %s", label.c_str());
RuntimeError("label found in data not specified in label mapping file: %s", label.c_str());
// new label so add it to the mapping tables
m_mapLabelToId[label] = m_labelIdMax;
m_mapIdToLabel[m_labelIdMax] = label;
@ -301,11 +301,11 @@ void UCIFastReader<ElemType>::Init(const ConfigParameters& readerConfig)
ConfigParameters configFeatures = readerConfig(m_featuresName,"");
ConfigParameters configLabels = readerConfig(m_labelsName,"");;
if (configFeatures.size() == 0)
Error("features file not found, required in configuration: i.e. 'features=[file=c:\\myfile.txt;start=1;dim=123]'");
RuntimeError("features file not found, required in configuration: i.e. 'features=[file=c:\\myfile.txt;start=1;dim=123]'");
if (configLabels.size() == 0)
fprintf(stderr, "Warning: labels are not specified.");
else if (configFeatures("file","") != configLabels("file",""))
Error("features and label files must be the same file, use separate readers to define single use files");
RuntimeError("features and label files must be the same file, use separate readers to define single use files");
size_t vdim = configFeatures("dim");
string name = configFeatures.Name();
@ -409,7 +409,7 @@ void UCIFastReader<ElemType>::Init(const ConfigParameters& readerConfig)
if (allowLabelCreation)
m_labelFileToWrite = labelPath;
else
Error("label mapping file %ws not found, can be created with a 'createLabelMap' command/action\n", labelPath.c_str());
RuntimeError("label mapping file %ws not found, can be created with a 'createLabelMap' command/action\n", labelPath.c_str());
}
}
@ -612,7 +612,7 @@ void UCIFastReader<ElemType>::SetupEpoch()
{
fprintf(stderr, "WARNING: file %ws NOT written to disk, label file will only be written when starting epochs at the beginning of the dataset\n", m_labelFileToWrite.c_str());
m_labelFileToWrite.clear();
Error("LabelMappingFile not provided in config, must be provided if not starting from epoch Zero (0)");
RuntimeError("LabelMappingFile not provided in config, must be provided if not starting from epoch Zero (0)");
}
}
m_epochStartSample = m_mbStartSample = mbStartSample;
@ -700,7 +700,7 @@ void UCIFastReader<ElemType>::StartMinibatchLoop(size_t mbSize, size_t epoch, si
if (m_randomizeRange != randomizeAuto)
{
if ((m_epochSize != requestDataSize && m_epochSize % m_randomizeRange != 0) || (m_randomizeRange % m_mbSize != 0))
Error("randomizeRange must be an even multiple of mbSize and an integral factor of epochSize");
RuntimeError("randomizeRange must be an even multiple of mbSize and an integral factor of epochSize");
m_randomordering.resize(m_randomizeRange, m_randomizeRange);
}
}
@ -741,7 +741,7 @@ bool UCIFastReader<ElemType>::GetMinibatch(std::map<std::wstring, Matrix<ElemTyp
}
// get the features array
if (matrices.find(m_featuresName) == matrices.end())
Error("Features matrix not found in config file, there should be a section '%ls=[...]' in the configuration file.", m_featuresName.c_str());
RuntimeError("Features matrix not found in config file, there should be a section '%ls=[...]' in the configuration file.", m_featuresName.c_str());
Matrix<ElemType>& features = *matrices[m_featuresName];

5
DataReader/UCIFastReader/UCIFastReader.vcxproj
Просмотреть файл

@ -171,6 +171,11 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\Include\basetypes.h" />
<ClInclude Include="..\..\Common\Include\DataReader.h" />
<ClInclude Include="..\..\Common\Include\DataWriter.h" />
<ClInclude Include="..\..\Common\Include\File.h" />
<ClInclude Include="..\..\Common\Include\fileutil.h" />
<ClInclude Include="minibatchsourcehelpers.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />

60
DataReader/UCIFastReader/UCIFastReader.vcxproj.filters Normal file
Просмотреть файл

@ -0,0 +1,60 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="Exports.cpp" />
<ClCompile Include="dllmain.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="UCIFastReader.cpp" />
<ClCompile Include="UCIParser.cpp" />
<ClCompile Include="..\..\Common\ConfigFile.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\DataReader.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\DataWriter.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\File.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<Filter>Common</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="UCIFastReader.h" />
<ClInclude Include="UCIParser.h" />
<ClInclude Include="minibatchsourcehelpers.h">
<Filter>Duplicates to remove</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\basetypes.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataReader.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataWriter.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\File.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\fileutil.h">
<Filter>Common\Include</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Filter Include="Common">
<UniqueIdentifier>{8e18fb2e-ab57-4862-ad16-5e322d6f2fbc}</UniqueIdentifier>
</Filter>
<Filter Include="Common\Include">
<UniqueIdentifier>{0dbad9d8-ce99-4f36-b871-3b98a27f58c2}</UniqueIdentifier>
</Filter>
<Filter Include="Duplicates to remove">
<UniqueIdentifier>{1032308c-b577-4b1e-9f49-9570b93800ec}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>

3
MachineLearning/CNTKEval/CNTKEval.vcxproj
Просмотреть файл

@ -103,7 +103,10 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\Include\basetypes.h" />
<ClInclude Include="..\..\Common\Include\Eval.h" />
<ClInclude Include="..\..\Common\Include\File.h" />
<ClInclude Include="..\..\Common\Include\fileutil.h" />
<ClInclude Include="EvalReader.h" />
<ClInclude Include="EvalWriter.h" />
<ClInclude Include="stdafx.h" />

50
MachineLearning/CNTKEval/CNTKEval.vcxproj.filters Normal file
Просмотреть файл

@ -0,0 +1,50 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="..\..\Common\BestGpu.cpp" />
<ClCompile Include="..\cn\ComputationNode.cpp" />
<ClCompile Include="..\cn\PTaskGraphBuilder.cpp" />
<ClCompile Include="dllmain.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="CNTKEval.cpp" />
<ClCompile Include="..\..\Common\ConfigFile.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\Eval.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\File.cpp">
<Filter>Common</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="EvalReader.h" />
<ClInclude Include="EvalWriter.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="CNTKEval.h" />
<ClInclude Include="..\..\Common\Include\Eval.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\basetypes.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\File.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\fileutil.h">
<Filter>Common\Include</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Filter Include="Common">
<UniqueIdentifier>{bed53b47-70b1-494c-824d-0748362003b2}</UniqueIdentifier>
</Filter>
<Filter Include="Common\Include">
<UniqueIdentifier>{f3bf0104-8a08-40c9-a4d9-af8411c49669}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>

4
MachineLearning/CNTKEval/EvalReader.h
Просмотреть файл

@ -45,7 +45,7 @@ public:
{
// record count must be the same for all the data
if (recordCount != m_recordCount)
Error("Error: Record Count of %ls (%lux%lu) does not match the record count of previous entries (%lu).", val.c_str(), rows, recordCount, m_recordCount);
RuntimeError("Record Count of %ls (%lux%lu) does not match the record count of previous entries (%lu).", val.c_str(), rows, recordCount, m_recordCount);
}
else
{
@ -136,7 +136,7 @@ public:
// allocate the matrix if we don't have one yet
if (iterIn == matrices.end())
{
Error("No matrix data found for key '%ls', cannot continue", val.c_str());
RuntimeError("No matrix data found for key '%ls', cannot continue", val.c_str());
}
Matrix<ElemType>* matrix = iterIn->second;

4
MachineLearning/CNTKEval/EvalWriter.h
Просмотреть файл

@ -42,7 +42,7 @@ public:
{
// record count must be the same for all the data
if (recordCount != m_recordCount)
Error("Error: Record Count of %ls (%lux%lu) does not match the record count of previous entries (%lu).", val.c_str(), rows, recordCount, m_recordCount);
RuntimeError("Record Count of %ls (%lux%lu) does not match the record count of previous entries (%lu).", val.c_str(), rows, recordCount, m_recordCount);
}
else
{
@ -97,7 +97,7 @@ public:
// allocate the matrix if we don't have one yet
if (iterIn == matrices.end())
{
Error("No matrix data found for key '%ls', cannot continue", val.c_str());
RuntimeError("No matrix data found for key '%ls', cannot continue", val.c_str());
}
Matrix<ElemType>* matrix = (Matrix<ElemType>*)iterIn->second;

2
MachineLearning/cn/ComputationNode.h
Просмотреть файл

@ -3047,7 +3047,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
static void WINAPI EvaluateThisNodeS(Matrix<ElemType>& functionValues, const Matrix<ElemType>& input0, const Matrix<ElemType>& input1)
{
functionValues.AssignProductOf(input0, false, input1, false);
functionValues.AssignProductOf(input0.Get00Element(), input1);
#if NANCHECK
functionValues.HasNan("Scale");
#endif

74
MachineLearning/cn/ModelEditLanguage.cpp
Просмотреть файл

@ -110,7 +110,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 0, numOptionalParams = 0;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: CreateModel(). newly created model always becomes the new default.");
RuntimeError("Invalid number of parameters. Valid parameters: CreateModel(). newly created model always becomes the new default.");
ComputationNetwork<ElemType>* cn = new ComputationNetwork<ElemType>(CPUDEVICE);
OverrideModelNameAndSetDefaultModel(cn);
@ -119,7 +119,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 1, numOptionalParams = 0;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: CreateModelWithName(modelName). newly created model always becomes the new default.");
RuntimeError("Invalid number of parameters. Valid parameters: CreateModelWithName(modelName). newly created model always becomes the new default.");
ComputationNetwork<ElemType>* cn = new ComputationNetwork<ElemType>(CPUDEVICE);
OverrideModelNameAndSetDefaultModel(cn, params[0]);
@ -128,7 +128,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 1, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: LoadModel(modelFileName, [format=cntk]). newly loaded model always becomes the new default.");
RuntimeError("Invalid number of parameters. Valid parameters: LoadModel(modelFileName, [format=cntk]). newly loaded model always becomes the new default.");
std::wstring modelFormat = GetOptionalModelFormat(params, numFixedParams);
@ -140,7 +140,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 2, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: LoadModelWithName(modelName, modelFileName, [format=cntk]). newly loaded model always becomes the new default.");
RuntimeError("Invalid number of parameters. Valid parameters: LoadModelWithName(modelName, modelFileName, [format=cntk]). newly loaded model always becomes the new default.");
std::wstring modelFormat = GetOptionalModelFormat(params, numFixedParams);
@ -152,7 +152,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 2, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: LoadNDLSnippet(modelName, ndlsnippet).");
RuntimeError("Invalid number of parameters. Valid parameters: LoadNDLSnippet(modelName, ndlsnippet).");
string modelName = params[0];
wstring ndlSnippetFileName = params[1];
@ -167,7 +167,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
if (!ndlScript.Exists(section))
{
Error("Section %s specified in optional parameter was not found in the %ls file\n", section.c_str(), ndlSnippetFileName.c_str());
RuntimeError("Section %s specified in optional parameter was not found in the %ls file\n", section.c_str(), ndlSnippetFileName.c_str());
}
ConfigValue ndlSnippet = ndlScript(section);
EvaluateNDLSnippet(ndlSnippet, cn);
@ -183,7 +183,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 1, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: SaveDefaultModel(modelFileName, [format=cntk]).");
RuntimeError("Invalid number of parameters. Valid parameters: SaveDefaultModel(modelFileName, [format=cntk]).");
std::wstring modelFormat = GetOptionalModelFormat(params, numFixedParams);
@ -191,7 +191,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
ComputationNetwork<ElemType>* cn = m_netNdlDefault->cn;
if (cn == NULL)
Error("SaveDefaultModel can only be called after a default name exists (i.e., at least one model is loaded.)");
RuntimeError("SaveDefaultModel can only be called after a default name exists (i.e., at least one model is loaded.)");
// validate the network before we save it out
ProcessNDLScript(m_netNdlDefault, ndlPassAll, true);
@ -202,7 +202,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 2, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: SaveModel(modelName, modelFileName, [format=cntk]).");
RuntimeError("Invalid number of parameters. Valid parameters: SaveModel(modelName, modelFileName, [format=cntk]).");
std::wstring modelFormat = GetOptionalModelFormat(params, numFixedParams);
@ -211,7 +211,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
NetNdl<ElemType>* netNdl = &m_mapNameToNetNdl[modelName];
if (netNdl->cn == NULL)
Error("SaveModel can only be called after a network has been setup, no active model named %ls.", modelName.c_str());
RuntimeError("SaveModel can only be called after a network has been setup, no active model named %ls.", modelName.c_str());
// validate and finish the second pass through NDL if any in-line NDL was defined
ProcessNDLScript(netNdl, ndlPassAll, true);
@ -221,7 +221,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 1, numOptionalParams = 0;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: SetDefaultModel(modelName)");
RuntimeError("Invalid number of parameters. Valid parameters: SetDefaultModel(modelName)");
SetExistingModelAsDefault(params[0]);
}
@ -248,7 +248,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 2, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: DumpNetwork(modelName, fileName, [includeData=false|true])");
RuntimeError("Invalid number of parameters. Valid parameters: DumpNetwork(modelName, fileName, [includeData=false|true])");
bool includeData = GetOptionalIncludeDataValue(params, numFixedParams);
@ -257,7 +257,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
auto found = m_mapNameToNetNdl.find(modelName);
if (found == m_mapNameToNetNdl.end())
Error("Model %s does not exist. Cannot dump non-existant model.", modelName);
RuntimeError("Model %s does not exist. Cannot dump non-existant model.", modelName);
else
{
NetNdl<ElemType>* netNdl = &found->second;
@ -269,7 +269,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 2, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters: DumpNode(nodeName, fileName, [includeData=false|true])");
RuntimeError("Invalid number of parameters. Valid parameters: DumpNode(nodeName, fileName, [includeData=false|true])");
bool includeData = GetOptionalIncludeDataValue(params, numFixedParams);
@ -284,7 +284,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 2, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters are: CopyNode(fromNode, toNode, [copy=all|value])");
RuntimeError("Invalid number of parameters. Valid parameters are: CopyNode(fromNode, toNode, [copy=all|value])");
CopyNodeFlags copyFlags = GetOptionalCopyNodeFlags(params, numFixedParams);
@ -296,7 +296,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 3, numOptionalParams = 1;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters are: CopySubTree(fromNode, toNetwork, toNodeNamePrefix, [copy=all|value])");
RuntimeError("Invalid number of parameters. Valid parameters are: CopySubTree(fromNode, toNetwork, toNodeNamePrefix, [copy=all|value])");
CopyNodeFlags copyFlags = GetOptionalCopyNodeFlags(params, numFixedParams);
@ -309,7 +309,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 2, numOptionalParams = 0;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters are: CopyNodeInputs(fromNode, toNode)");
RuntimeError("Invalid number of parameters. Valid parameters are: CopyNodeInputs(fromNode, toNode)");
// get the nodes
NetNdl<ElemType>* netNdlTo;
@ -317,7 +317,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
vector<GenNameValue> names = GenerateNames(params[0], params[1], netNdlFrom, netNdlTo);
if (netNdlFrom != netNdlTo)
Error("CopyInputs requires two symbols from the same network, %s and %s belong to different networks", params[0], params[1]);
RuntimeError("CopyInputs requires two symbols from the same network, %s and %s belong to different networks", params[0], params[1]);
ProcessNDLScript(netNdlFrom, ndlPassAll);
for (GenNameValue name : names)
@ -333,7 +333,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 3, numOptionalParams = 0;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters are: SetNodeInput(toNode, inputID(0-based), inputNodeName)");
RuntimeError("Invalid number of parameters. Valid parameters are: SetNodeInput(toNode, inputID(0-based), inputNodeName)");
// get the nodes
NetNdl<ElemType>* netNdlTo;
@ -343,12 +343,12 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
int inputNum = params[1];
if (netNdlTo != netNdlFrom)
Error("SetNodeInput() requires two symbols from the same network, %s and %s belong to different networks", params[0], params[2]);
RuntimeError("SetNodeInput() requires two symbols from the same network, %s and %s belong to different networks", params[0], params[2]);
if (nodeFrom.size() != 1)
Error("SetNodeInput() must have a single value input, %s doesn't represent one item",params[0]);
RuntimeError("SetNodeInput() must have a single value input, %s doesn't represent one item",params[0]);
if (nodeTo.size() < 1)
Error("SetNodeInput() must have at least one target, %s doesn't represent any items",params[2]);
RuntimeError("SetNodeInput() must have at least one target, %s doesn't represent any items",params[2]);
// process outstanding NDL scripts ensuring that the inputs have all been resolved
ProcessNDLScript(netNdlFrom, ndlPassResolve);
@ -360,13 +360,13 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
else if (EqualInsensitive(name, "SetNodeInputs", "SetInputs"))
{
if (params.size() > 4 || params.size() < 2)
Error("Invalid number of parameters. Valid parameters are: SetNodeInputs(toNode, inputNodeName1, [inputNodeName2, inputNodeName3])");
RuntimeError("Invalid number of parameters. Valid parameters are: SetNodeInputs(toNode, inputNodeName1, [inputNodeName2, inputNodeName3])");
// get the nodes
NetNdl<ElemType>* netNdlTo;
vector<ComputationNode<ElemType>*> nodeTo = FindSymbols(params[0], netNdlTo);
if (nodeTo.size() != 1)
Error("SetNodeInputs() must have exactly one target, %s doesn't represent any node.",params[0]);
RuntimeError("SetNodeInputs() must have exactly one target, %s doesn't represent any node.",params[0]);
vector<ComputationNode<ElemType>*> inputNodes;
inputNodes.resize(params.size()-1);
@ -380,10 +380,10 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
vector<ComputationNode<ElemType>*> nodeFrom = FindSymbols(params[i], netNdlFrom);
if (netNdlTo != netNdlFrom)
Error("SetNodeInputs() requires all symbols from the same network, %s and %s belong to different networks", params[0], params[i]);
RuntimeError("SetNodeInputs() requires all symbols from the same network, %s and %s belong to different networks", params[0], params[i]);
if (nodeFrom.size() != 1)
Error("SetNodeInputs() each input node should be translated to one node name. %s is translated to multiple node names.", params[i]);
RuntimeError("SetNodeInputs() each input node should be translated to one node name. %s is translated to multiple node names.", params[i]);
inputNodes[i-1] = nodeFrom[0];
}
@ -395,12 +395,12 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
else if (inputNodes.size() == 3)
nodeTo[0]->AttachInputs(inputNodes[0], inputNodes[1], inputNodes[2]);
else
Error("SetNodeInputs(): You specified more than 3 input nodes.");
RuntimeError("SetNodeInputs(): You specified more than 3 input nodes.");
}
else if (EqualInsensitive(name, "SetProperty"))
{
if (params.size() != 3)
Error("Invalid number of parameters: Valid parameters are: SetProperty(toNode, propertyName, propertyValue)");
RuntimeError("Invalid number of parameters: Valid parameters are: SetProperty(toNode, propertyName, propertyValue)");
std::string propName = params[1];
MELProperty prop=melPropNull;
@ -434,7 +434,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
}
else
{
Error("Invalid property, %s, is not supported", propName);
RuntimeError("Invalid property, %s, is not supported", propName);
}
// get the nodes
@ -491,7 +491,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
}
default:
{
Error("Invalid property, %s, is not supported", propName);
RuntimeError("Invalid property, %s, is not supported", propName);
break;
}
}
@ -501,7 +501,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 3, numOptionalParams = 0;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters are: SetPropertyForSubTree(rootNodeName, propertyName, propertyValue)");
RuntimeError("Invalid number of parameters. Valid parameters are: SetPropertyForSubTree(rootNodeName, propertyName, propertyValue)");
std::string propName = params[1];
MELProperty prop=melPropNull;
@ -511,7 +511,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
}
else
{
Error("Invalid property, %s, is not supported", propName);
RuntimeError("Invalid property, %s, is not supported", propName);
}
// get the nodes
@ -533,7 +533,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
}
default:
{
Error("Invalid property, %s, is not supported", propName);
RuntimeError("Invalid property, %s, is not supported", propName);
break;
}
}
@ -558,7 +558,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
}
if (nodes.size() < 1)
Error("Delete must have at least one target, %s doesn't represent any items",params[i]);
RuntimeError("Delete must have at least one target, %s doesn't represent any items",params[i]);
for (ComputationNode<ElemType>* node : nodes)
{
netNdl->cn->DeleteNode(node->NodeName());
@ -569,7 +569,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
{
size_t numFixedParams = 2, numOptionalParams = 0;
if (params.size() > numFixedParams + numOptionalParams || params.size() < numFixedParams)
Error("Invalid number of parameters. Valid parameters are Rename(oldNodeName, newNodeName)");
RuntimeError("Invalid number of parameters. Valid parameters are Rename(oldNodeName, newNodeName)");
// get the nodes
NetNdl<ElemType>* netNdlTo;
@ -577,7 +577,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
vector<GenNameValue> nodeNames = GenerateNames(params[0], params[1], netNdlFrom, netNdlTo);
if (netNdlFrom != netNdlTo)
Error("CopyInputs requires two symbols from the same network, %s and %s belong to different networks", params[0], params[1]);
RuntimeError("CopyInputs requires two symbols from the same network, %s and %s belong to different networks", params[0], params[1]);
// process everything in case these nodes may have tags on them
ProcessNDLScript(netNdlFrom, ndlPassAll);
@ -591,7 +591,7 @@ void MELScript<ElemType>::CallFunction(const std::string& p_name, const ConfigPa
}
else
{
Error("Unknown Editor function %s", name.c_str());
RuntimeError("Unknown Editor function %s", name.c_str());
}
}

36
MachineLearning/cn/ModelEditLanguage.h
Просмотреть файл

@ -160,7 +160,7 @@ public:
else
{
if (ndlNode->GetType() != ndlTypeConstant)
Error("Matching NDL name found for %s, but no corresponding computation node found\n", symbol);
RuntimeError("Matching NDL name found for %s, but no corresponding computation node found\n", symbol);
// probably a constant node, so make the ComputationNode that is equivalent
ComputationNode<ElemType>* nodePtr = cn->CreateLearnableParameter(name, 1, 1);
ndlNode->SetEvalValue(nodePtr);
@ -170,7 +170,7 @@ public:
}
}
if (nodes.empty())
Error("FindSymbols could not find a symbol for %s\n", symbol);
RuntimeError("FindSymbols could not find a symbol for %s\n", symbol);
return nodes;
}
@ -207,7 +207,7 @@ public:
vector<ComputationNode<ElemType>*> nodes = netNdlIn->cn->GetNodesFromName(name);
if (!nodes.size()) //found
Error("GenerateNames: Node name does not exist %ls.", name.c_str());
RuntimeError("GenerateNames: Node name does not exist %ls.", name.c_str());
size_t firstStartOut, firstCountOut, secondStartOut, secondCountOut;
netNdlOut = ParseName(symbolOut, firstStartOut, firstCountOut, secondStartOut, secondCountOut);
@ -236,7 +236,7 @@ public:
// make sure the patterns are the same
if (!(singleInputMultiOutput || ((!firstCount == !firstCountOut) && (!secondCount == !secondCountOut))))
{
Error("The input symbols and output symbols must match, when the input matches has more than one element. %s = %s not allowed", symbolOut.c_str(), symbolIn.c_str());
RuntimeError("The input symbols and output symbols must match, when the input matches has more than one element. %s = %s not allowed", symbolOut.c_str(), symbolIn.c_str());
}
// get the first and last "unchanged" portions
@ -254,7 +254,7 @@ public:
// make sure that there are some nodes to copy to
if (nodesOut.size() == 0)
Error("Setting a single input to multiple outputs requires the multiple outputs to exist. In %ls = %ls, %ls does not match any nodes.", nameOut.c_str(), name.c_str(), nameOut.c_str());
RuntimeError("Setting a single input to multiple outputs requires the multiple outputs to exist. In %ls = %ls, %ls does not match any nodes.", nameOut.c_str(), name.c_str(), nameOut.c_str());
// this is the *.W = L2.W case
// We want to find all the destination existing matches and then assign the in node to all of them
@ -397,7 +397,7 @@ public:
{
auto found = m_mapNameToNetNdl.find(modelName);
if (found == m_mapNameToNetNdl.end())
Error("Model %s does not exist. Cannot set it to default.", modelName);
RuntimeError("Model %s does not exist. Cannot set it to default.", modelName);
else
m_netNdlDefault = &found->second;
}
@ -417,7 +417,7 @@ public:
}
else
{
Error("Invalid optional parameter %s, valid optional parameters: includeData=(false|true)", propName.c_str());
RuntimeError("Invalid optional parameter %s, valid optional parameters: includeData=(false|true)", propName.c_str());
}
}
}
@ -441,12 +441,12 @@ public:
}
else
{
Error("Invalid optional parameter value %s, valid values are: format=(cntk)", value.c_str());
RuntimeError("Invalid optional parameter value %s, valid values are: format=(cntk)", value.c_str());
}
}
else
{
Error("Invalid optional parameter %s, valid optional parameters: format=(cntk)", propName.c_str());
RuntimeError("Invalid optional parameter %s, valid optional parameters: format=(cntk)", propName.c_str());
}
}
}
@ -466,12 +466,12 @@ public:
{
if (value.empty())
{
Error("Invalid optional parameter value <empty>, a section name must be specified: section=(sectionName)");
RuntimeError("Invalid optional parameter value <empty>, a section name must be specified: section=(sectionName)");
}
}
else
{
Error("Invalid optional parameter %s, valid optional parameters: section=(sectionName)", propName.c_str());
RuntimeError("Invalid optional parameter %s, valid optional parameters: section=(sectionName)", propName.c_str());
}
}
}
@ -501,12 +501,12 @@ public:
}
else
{
Error("Invalid optional parameter value %s in CopyNode(), valid values are copyFlag=(all|value)", value.c_str());
RuntimeError("Invalid optional parameter value %s in CopyNode(), valid values are copyFlag=(all|value)", value.c_str());
}
}
else
{
Error("Invalid optional parameter to Copy, %s\n valid optional parameters: copyFlag=(all|value)", propName.c_str());
RuntimeError("Invalid optional parameter to Copy, %s\n valid optional parameters: copyFlag=(all|value)", propName.c_str());
}
}
}
@ -556,7 +556,7 @@ public:
{
auto paramStart = token.find_first_of(OPENBRACES);
if (paramStart == npos)
Error("Invalid macro/function call can not be parsed: %s\n", token.c_str());
RuntimeError("Invalid macro/function call can not be parsed: %s\n", token.c_str());
nameFunction = token.substr(0, paramStart);
Trim(nameFunction);
params = token.substr(paramStart);
@ -625,7 +625,7 @@ public:
{
size_t tokenStartNew = keyEnd+1;
if (!(tokenStartNew < tokenEnd))
Error("Equal at the end of line not allowed");
RuntimeError("Equal at the end of line not allowed");
std::string rightValue = stringParse.substr(tokenStartNew,tokenEnd-tokenStartNew);
Trim(rightValue);
@ -634,7 +634,7 @@ public:
if (foundBrace == npos)
{
if (!m_netNdlDefault)
Error("NDL Command cannot be executed until default model is established, cannot set '%s' without a default mode\n Try calling SetDefaultModel(model) before any NDL statement are embedded\n", key.c_str());
RuntimeError("NDL Command cannot be executed until default model is established, cannot set '%s' without a default mode\n Try calling SetDefaultModel(model) before any NDL statement are embedded\n", key.c_str());
HandleNDLInline(stringParse, tokenStart, tokenEnd);
}
else //createModel, loadModel, or loadNDL
@ -668,7 +668,7 @@ public:
else
{ // not a MEL command, so pass it on to NDL
if (!m_netNdlDefault)
Error("NDL Command cannot be executed until default model is established, cannot set '%s' without a default mode\n Try calling SetDefaultModel(model) before any NDL statement are embedded\n", key.c_str());
RuntimeError("NDL Command cannot be executed until default model is established, cannot set '%s' without a default mode\n Try calling SetDefaultModel(model) before any NDL statement are embedded\n", key.c_str());
HandleNDLInline(stringParse, tokenStart, tokenEnd);
}
}
@ -680,7 +680,7 @@ public:
{
std::string value = stringParse.substr(tokenStart,tokenEnd-tokenStart);
if (keyEnd > tokenEnd)
Error("Invalid line, expecting function call, %s", value);
RuntimeError("Invalid line, expecting function call, %s", value);
std::string functionName;
std::string paramList;
// Function(x,y,z) - function with no return

8
MachineLearning/cn/NDLNetworkBuilder.h
Просмотреть файл

@ -85,7 +85,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
std::string name = config("load");
if (!config.Exists(name))
Error("the configuration parameter 'load=%s' doesn't specify another section in this configuration file.\n"
RuntimeError("the configuration parameter 'load=%s' doesn't specify another section in this configuration file.\n"
"No 'networkDescription' variable was defined if specifying a separate file was desired.\n ", name.c_str());
newConfig.Insert(name, config(name));
@ -93,11 +93,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
if (!config.Exists("run"))
Error("In NDLNetworkBuilder section either a 'networkDescription=filename' or 'run=sectionName' must exist.");
RuntimeError("In NDLNetworkBuilder section either a 'networkDescription=filename' or 'run=sectionName' must exist.");
std::string name = config("run");
if (!config.Exists(name))
Error("the configuration parameter 'run=%s' doesn't specify another section in this configuration file.\n"
RuntimeError("the configuration parameter 'run=%s' doesn't specify another section in this configuration file.\n"
"No 'networkDescription' variable was defined if specifying a separate file was desired.\n ", name.c_str());
newConfig.Insert(name, config(name));
@ -107,7 +107,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
std::string networkConfigString = networkConfig;
if (networkConfigString.find_first_of("+") != std::string::npos)
Error("\"+\" not allowed in \"networkDescription\" value. Multiple files cannot be specified via \"networkDescription\" parameter. "
RuntimeError("\"+\" not allowed in \"networkDescription\" value. Multiple files cannot be specified via \"networkDescription\" parameter. "
"In order to load multiple NDL files (eg, for loading several files of macros), use the \"ndlMacros\" parameter.");
// find the "run" and "load" keys and add them

4
MachineLearning/cn/NDLUtil.h
Просмотреть файл

@ -151,7 +151,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
valid = (outputType == ndlTypeArray || outputType == ndlTypeFunction || outputType == ndlTypeMacroCall);
}
if (!valid)
Error("Invalid network node definition for '%s', nonexistant or wrong type", symbolName.c_str());
RuntimeError("Invalid network node definition for '%s', nonexistant or wrong type", symbolName.c_str());
if (nodeArray)
{
vector<NDLNode<ElemType>*> nodes;
@ -168,7 +168,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
// if no evaluation value exists throw an error
if (cnNode == nullptr)
{
Error("Invalid node '%s' as an output node, nonexistant or wrong type", nodes[i]->GetName().c_str());
RuntimeError("Invalid node '%s' as an output node, nonexistant or wrong type", nodes[i]->GetName().c_str());
}
// see if it's already in the collection

30
MachineLearning/cn/NetworkDescriptionLanguage.h
Просмотреть файл

@ -287,7 +287,7 @@ public:
if (!node || node->GetType() != ndlTypeConstant)
{
std::string name = node->GetName();
Error("Scalar expected, '%s' must be a constant or variable that resolves to a constant\n", name.c_str());
RuntimeError("Scalar expected, '%s' must be a constant or variable that resolves to a constant\n", name.c_str());
}
return node->GetValue();
}
@ -307,7 +307,7 @@ public:
// make sure the actual parameters and expected parameters match
if (m_parameters.size() < m_paramMacro.size())
{
Error("Parameter mismatch, %d parameters provided, %d expected in call to %s\n",
RuntimeError("Parameter mismatch, %d parameters provided, %d expected in call to %s\n",
m_parameters.size(),m_paramMacro.size(),m_value.c_str());
}
@ -326,7 +326,7 @@ public:
else if (nodeParam->GetType() == ndlTypeOptionalParameter)
{
if (i < m_paramMacro.size())
Error("Parameter mismatch, parameter %d is an optional parameter, but should be a required parameter\n",i);
RuntimeError("Parameter mismatch, parameter %d is an optional parameter, but should be a required parameter\n",i);
// if no symbol yet, add it
if (!m_script->ExistsSymbol(paramName))
{
@ -420,7 +420,7 @@ public:
m_scriptString = configValue;
NDLNode<ElemType>* ndlNode = s_global.CheckName(macroName, true);
if (ndlNode == NULL)
Error("Invalid macro definition, %s not found", macroName.c_str());
RuntimeError("Invalid macro definition, %s not found", macroName.c_str());
// get and parse the parameters
ConfigArray parameters = ndlNode->GetParamMacro();
@ -435,7 +435,7 @@ public:
// in which case 'functionName' will get the default node name returned
if (CheckFunction<ElemType>(functionName))
{
Error("NDLScript: Macro %s includes a parameter %s, which is also the name of a function. Parameter names may not be the same as function names.", macroName.c_str(), param.c_str());
RuntimeError("NDLScript: Macro %s includes a parameter %s, which is also the name of a function. Parameter names may not be the same as function names.", macroName.c_str(), param.c_str());
}
NDLNode<ElemType>* paramNode = new NDLNode<ElemType>(param, param, this, ndlTypeParameter);
@ -549,7 +549,7 @@ public:
if (script != NULL)
{
if (node->GetType() != ndlTypeMacroCall || script == NULL)
Error("Symbol name not valid, %s is not a macro, so %s cannot be interpretted",search.c_str(),symbol.c_str() );
RuntimeError("Symbol name not valid, %s is not a macro, so %s cannot be interpretted",search.c_str(),symbol.c_str() );
return script->FindSymbol(symbol.substr(firstDot+1), searchForDotNames);
}
}
@ -598,7 +598,7 @@ public:
if (nodeFound->GetType() != ndlTypeUndetermined && nodeFound->GetType() != ndlTypeParameter)
{
std::string value = found->second->GetValue();
Error("Symbol '%s' currently assigned to '%s' reassigning to a different value not allowed\n", symbol.c_str(), value.c_str());
RuntimeError("Symbol '%s' currently assigned to '%s' reassigning to a different value not allowed\n", symbol.c_str(), value.c_str());
}
}
m_symbols[symbol] = node;
@ -612,7 +612,7 @@ public:
auto found = m_symbols.find(symbol);
if (found == m_symbols.end())
{
Error("Symbol '%s' currently does not exist, attempting to assigned value '%s' AssignSymbol() requires existing symbol\n", symbol.c_str(), node->GetValue());
RuntimeError("Symbol '%s' currently does not exist, attempting to assigned value '%s' AssignSymbol() requires existing symbol\n", symbol.c_str(), node->GetValue());
}
m_symbols[symbol] = node;
}
@ -726,7 +726,7 @@ public:
{
auto paramStart = token.find_first_of(OPENBRACES);
if (paramStart == npos)
Error("Invalid macro/function call can not be parsed: %s\n", token.c_str());
RuntimeError("Invalid macro/function call can not be parsed: %s\n", token.c_str());
nameFunction = token.substr(0, paramStart);
Trim(nameFunction);
params = token.substr(paramStart);
@ -763,7 +763,7 @@ public:
}
if (paramNode == NULL)
{
Error("variable name '%s' not found, must be previously defined\n", param.c_str());
RuntimeError("variable name '%s' not found, must be previously defined\n", param.c_str());
}
else
{
@ -832,7 +832,7 @@ public:
std::string nameFunction, params;
NDLNode<ElemType>* ndlNode = CallStringParse(token, nameFunction, params);
if (ndlNode)
Error("function '%s' already defined\n", nameFunction.c_str());
RuntimeError("function '%s' already defined\n", nameFunction.c_str());
ndlNode = new NDLNode<ElemType>(nameFunction, params, &s_global, ndlTypeMacro);
// now set the variables/parameters which will be parsed when the body shows up
@ -854,7 +854,7 @@ public:
NDLNode<ElemType>* ndlNode = CallStringParse(token, nameFunction, params);
if (ndlNode == NULL)
Error("Undefined function or macro '%s' in %s\n", nameFunction.c_str(), token.c_str());
RuntimeError("Undefined function or macro '%s' in %s\n", nameFunction.c_str(), token.c_str());
// now setup the variables/parameters
ConfigValue value = ConfigValue(params, nameFunction);
@ -890,7 +890,7 @@ public:
oneLineDefinition = true;
tokenStart = stringParse.find_first_not_of(" \t", tokenStart+1);
if (tokenStart == npos)
Error("Body of Macro missing");
RuntimeError("Body of Macro missing");
}
NDLScript<ElemType>* script = new NDLScript<ElemType>(ConfigValue(stringParse.substr(tokenStart, tokenEnd-tokenStart), macroNode->GetName()), macroNode->GetName(), oneLineDefinition);
@ -908,7 +908,7 @@ public:
{
keyEnd = stringParse.find_first_of(OPENBRACES, tokenStart);
if (keyEnd == npos || keyEnd >= tokenEnd)
Error("Invalid statement, does not contain an '=' sign: %s\n", stringParse.substr(tokenStart, tokenEnd-tokenStart).c_str());
RuntimeError("Invalid statement, does not contain an '=' sign: %s\n", stringParse.substr(tokenStart, tokenEnd-tokenStart).c_str());
m_macroNode = ParseDefinition(stringParse.substr(tokenStart, tokenEnd-tokenStart));
// the body of the macro will come through next time
return tokenEnd;
@ -930,7 +930,7 @@ public:
// check to make sure variable name isn't a valid function name as well
string strTemp = key;
if (CheckFunction<ElemType>(strTemp))
Error("variable %s is invalid, it is reserved because it is also the name of a function", key.c_str());
RuntimeError("variable %s is invalid, it is reserved because it is also the name of a function", key.c_str());
tokenStart = keyEnd;
if (stringParse[keyEnd] == '=')

18
MachineLearning/cn/SGD.h
Просмотреть файл

@ -1252,7 +1252,7 @@ protected:
{
const wstring prevEpochFile = GetModelNameForEpoch (e-1);
if (IsResultFileUpdateToDate (curEpochFile, prevEpochFile, false))
if (msra::files::fuptodate (curEpochFile, prevEpochFile, false))
{
firstEpoch = size_t(e)+1;
break;
@ -1264,22 +1264,6 @@ protected:
return firstEpoch;
}
//up to date if resultFile is older than srcFile or missing
bool IsResultFileUpdateToDate (const wstring & resultFile, const wstring & srcFile, const bool IsSrcFileNeeded)
{
FILETIME resultFileTime;
if (!getfiletime (resultFile, resultFileTime))
return false; // not up to date is resultFile is missing
FILETIME srcFileTime;
if (!getfiletime (srcFile, srcFileTime))
return !IsSrcFileNeeded; // srcFile missing: if required, the result file is not up to date
//up to date if resultFile has higher time stamp
return (resultFileTime.dwHighDateTime > srcFileTime.dwHighDateTime) ||
(resultFileTime.dwHighDateTime == srcFileTime.dwHighDateTime && resultFileTime.dwLowDateTime >= srcFileTime.dwLowDateTime);
}
AdaptationRegType ParseAdaptationRegType(wstring s)
{
transform(s.begin(), s.end(), s.begin(),tolower);

42
MachineLearning/cn/SynchronousExecutionEngine.h
Просмотреть файл

@ -67,7 +67,7 @@ public:
if (InputValue<ElemType>::TypeName() == cnNodeType)
{
if (parameter.size() < 1 || parameter.size() > 2)
Error("%ws should have 1 or 2 parameters[rows, [cols=1]].", cnNodeType);
RuntimeError("%ws should have 1 or 2 parameters[rows, [cols=1]].", cnNodeType);
if (pass == ndlPassInitial)
{
@ -86,7 +86,7 @@ public:
else if (SparseInputValue<ElemType>::TypeName() == cnNodeType)
{
if (parameter.size() < 1 || parameter.size() > 2)
Error("%ws should have 1 or 2 parameters[rows, [cols=1]].", cnNodeType);
RuntimeError("%ws should have 1 or 2 parameters[rows, [cols=1]].", cnNodeType);
if (pass == ndlPassInitial)
{
@ -105,7 +105,7 @@ public:
else if (cnNodeType == L"ImageInput")
{
if (parameter.size() < 3 || parameter.size() > 4)
Error("%ws should have 3 or 4 parameters[imageWidth, imageHeight, imageChannels, [numImages=1]].", cnNodeType);
RuntimeError("%ws should have 3 or 4 parameters[imageWidth, imageHeight, imageChannels, [numImages=1]].", cnNodeType);
if (pass == ndlPassInitial)
{
@ -122,7 +122,7 @@ public:
else if (LearnableParameter<ElemType>::TypeName() == cnNodeType)
{
if (parameter.size() < 1 || parameter.size() > 2)
Error("%ws should have 1 or 2 parameters[rows, [cols=1]] plus other optional parameters (needGradient=[true|false], init=[uniform|gaussian|fixedvalue], initValueScale=[1|float], value=[0|float]).", cnNodeType);
RuntimeError("%ws should have 1 or 2 parameters[rows, [cols=1]] plus other optional parameters (needGradient=[true|false], init=[uniform|gaussian|fixedvalue], initValueScale=[1|float], value=[0|float]).", cnNodeType);
if (pass == ndlPassInitial)
{
@ -155,22 +155,22 @@ public:
{
std::string initFromFilePath = node->GetOptionalParameter("initFromFilePath", "");
if (initFromFilePath == "")
Error("initFromFilePath must be set when using \"fromFile\" initialization method");
RuntimeError("initFromFilePath must be set when using \"fromFile\" initialization method");
if(initFromFilePath[0] == '\"' && initFromFilePath[initFromFilePath.size()-1] == '\"')
// remove the opening and closing double quotes
initFromFilePath = initFromFilePath.substr(1, initFromFilePath.size()-2);
if(!fexists(initFromFilePath))
Error("File pointed to by initFromFilePath does not exist: %s", initFromFilePath);
RuntimeError("File pointed to by initFromFilePath does not exist: %s", initFromFilePath);
m_net.InitLearnableParametersFromFile(nodePtr, initFromFilePath);
}
else
Error("init must be one of the values of [uniform|gaussian|fixedvalue]");
RuntimeError("init must be one of the values of [uniform|gaussian|fixedvalue]");
}
}
else if (SparseLearnableParameter<ElemType>::TypeName() == cnNodeType)
{
if (parameter.size() < 1 || parameter.size() > 2)
Error("%ws should have 1 or 2 parameters[rows, [cols=1]] plus other optional parameters (needGradient=[true|false], init=[uniform|gaussian|fixedvalue], initValueScale=[1|float], value=[0|float]).", cnNodeType);
RuntimeError("%ws should have 1 or 2 parameters[rows, [cols=1]] plus other optional parameters (needGradient=[true|false], init=[uniform|gaussian|fixedvalue], initValueScale=[1|float], value=[0|float]).", cnNodeType);
if (pass == ndlPassInitial)
{
@ -203,22 +203,22 @@ public:
{
std::string initFromFilePath = node->GetOptionalParameter("initFromFilePath", "");
if (initFromFilePath == "")
Error("initFromFilePath must be set when using \"fromFile\" initialization method");
RuntimeError("initFromFilePath must be set when using \"fromFile\" initialization method");
if(initFromFilePath[0] == '\"' && initFromFilePath[initFromFilePath.size()-1] == '\"')
// remove the opening and closing double quotes
initFromFilePath = initFromFilePath.substr(1, initFromFilePath.size()-2);
if(!fexists(initFromFilePath))
Error("File pointed to by initFromFilePath does not exist: %s", initFromFilePath);
RuntimeError("File pointed to by initFromFilePath does not exist: %s", initFromFilePath);
m_net.InitLearnableParametersFromFile(nodePtr, initFromFilePath);
}
else
Error("init must be one of the values of [uniform|gaussian|fixedvalue]");
RuntimeError("init must be one of the values of [uniform|gaussian|fixedvalue]");
}
}
else if (cnNodeType == L"Constant")
{
if (parameter.size() != 1)
Error("Constant should have 1 fixed parameter [val] and two optional parameters [rows=[1|yourvalue], cols=[1|yourvalue]].");
RuntimeError("Constant should have 1 fixed parameter [val] and two optional parameters [rows=[1|yourvalue], cols=[1|yourvalue]].");
if (pass == ndlPassInitial)
{
@ -281,7 +281,7 @@ public:
else if (cnNodeType == ConvolutionNode<ElemType>::TypeName())
{
if (parameter.size() != 7)
Error("%ws should have 7 fixed parameters[weightNodeName, inputValueNodeName, kernelWidth, kernelHeight, outputChannels,horizontalSubsample, verticalSubsample] and two optional parameters [zeroPadding = [false|yourvalue], maxTempMemSizeInSamples = [0|yourvalue]].", cnNodeType);
RuntimeError("%ws should have 7 fixed parameters[weightNodeName, inputValueNodeName, kernelWidth, kernelHeight, outputChannels,horizontalSubsample, verticalSubsample] and two optional parameters [zeroPadding = [false|yourvalue], maxTempMemSizeInSamples = [0|yourvalue]].", cnNodeType);
// setup the parameter position of children so we can hook them up later
nodeParamCount = 2;
@ -314,7 +314,7 @@ public:
else if (cnNodeType == MaxPoolingNode<ElemType>::TypeName())
{
if (parameter.size() != 5)
Error("%ws should have 5 parameters[inputValueNodeName, windowWidth, windowHeight, horizontalSubsample, verticalSubsample].", cnNodeType);
RuntimeError("%ws should have 5 parameters[inputValueNodeName, windowWidth, windowHeight, horizontalSubsample, verticalSubsample].", cnNodeType);
// setup the parameter position of children so we can hook them up later
nodeParamCount = 1;
@ -341,7 +341,7 @@ public:
else if (cnNodeType == AveragePoolingNode<ElemType>::TypeName())
{
if (parameter.size() != 5)
Error("%ws should have 5 parameters[inputValueNodeName, windowWidth, windowHeight, horizontalSubsample, verticalSubsample].", cnNodeType);
RuntimeError("%ws should have 5 parameters[inputValueNodeName, windowWidth, windowHeight, horizontalSubsample, verticalSubsample].", cnNodeType);
// setup the parameter position of children so we can hook them up later
nodeParamCount = 1;
@ -405,7 +405,7 @@ public:
break;
default:
if (nodeParamCount > 0)
Error("Invalid number of parameters name = '%s' call = '%s'\n", node->GetName().c_str(), node->GetValue().c_str());
RuntimeError("Invalid number of parameters name = '%s' call = '%s'\n", node->GetName().c_str(), node->GetValue().c_str());
break;
}
@ -461,7 +461,7 @@ public:
std::size_t firstDotPos = name.find_first_of(".");
if (firstDotPos == std::string::npos)
{
Error("Logic Error: nodeParam of type \"ndlTypeDotParameter\" doesn't have a dot in its name: %s", name.c_str());
LogicError("nodeParam of type \"ndlTypeDotParameter\" doesn't have a dot in its name: %s", name.c_str());
}
std::string nameBeforeDot = name.substr(0, firstDotPos);
@ -504,7 +504,7 @@ public:
// if we still didn't get a value, throw an error
if (nodeParam->GetEvalValue() == nullptr)
{
Error("Logic Error: Dot name could not be resolved '%s': should have a node named '%ls' in computational network\n", nodeParam->GetName().c_str(), name.c_str());
LogicError("Dot name could not be resolved '%s': should have a node named '%ls' in computational network\n", nodeParam->GetName().c_str(), name.c_str());
}
}
return nodeParam;
@ -525,7 +525,7 @@ public:
if (script == NULL)
{
std::wstring name = baseName + L"." + msra::strfun::utf16(node->GetName());
Error("Logic Error: no script for a parameter node in call to %ls\n", name.c_str());
LogicError("no script for a parameter node in call to %ls\n", name.c_str());
}
// evaluate the parameter if we haven't yet, or if we are in the resolve pass (need to set the inputs)
@ -567,7 +567,7 @@ public:
}
else
{
Error("Parameter name could not be resolved '%s'\n", name.c_str());
RuntimeError("Parameter name could not be resolved '%s'\n", name.c_str());
}
}
}
@ -590,7 +590,7 @@ public:
default:
{
std::wstring name = baseName + L"." + msra::strfun::utf16(node->GetName());
Error("Invalid parameter (macro definitions and arrays not allowed), see call to %ls\n", name.c_str());
RuntimeError("Invalid parameter (macro definitions and arrays not allowed), see call to %ls\n", name.c_str());
}
break;
}

2
MachineLearning/cn/TrainingCriterionNode.h
Просмотреть файл

@ -752,7 +752,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
Matrix<ElemType>& inputGradientValues, const Matrix<ElemType>& gradientValues, const Matrix<ElemType>& inputFunctionValues)
{
gradientOfL1Norm.AssignSignOf(inputFunctionValues);
inputGradientValues.AddElementProductOf(gradientValues, gradientOfL1Norm);
inputGradientValues.AddWithScaleOf(gradientValues.Get00Element(), gradientOfL1Norm);
}
// GetTaskDescriptor - Get a task descriptor for this node

7
MachineLearning/cn/cn.cpp
Просмотреть файл

@ -177,12 +177,11 @@ void DoCrossValidate(const ConfigParameters& config)
{
wstring cvModelPath = msra::strfun::wstrprintf (L"%ws.%lld", modelPath.c_str(), i);
FILETIME resultFileTime;
if (!getfiletime (cvModelPath, resultFileTime))
if (!fexists (cvModelPath))
{
fprintf(stderr, "model %ws does not exist.\n", cvModelPath.c_str());
if (finalModelEvaluated || !getfiletime (modelPath, resultFileTime))
continue; //file missing
if (finalModelEvaluated || !fexists (modelPath))
continue; // file missing
else
{
cvModelPath = modelPath;

564
MachineLearning/cn/cn.vcxproj
Просмотреть файл

@ -1,282 +1,284 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}</ProjectGuid>
<SccProjectName>
</SccProjectName>
<SccAuxPath>
</SccAuxPath>
<SccLocalPath>
</SccLocalPath>
<SccProvider>
</SccProvider>
<Keyword>Win32Proj</Keyword>
<RootNamespace>cn</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
<Import Project="$(VCTargetsPath)\BuildCustomizations\CUDA 6.5.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>..\..\Math\Math;..\..\Common\;..\..\Common\include;..\..\Common\PTask\include;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<LibraryPath>$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(SolutionDir)..\Common\PTask\lib\$(Configuration)\;$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform)</LibraryPath>
<CustomBuildAfterTargets>Build</CustomBuildAfterTargets>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>..\..\Math\Math;..\..\Common\;..\..\Common\include;..\..\Common\PTask\include;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<LibraryPath>$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(SolutionDir)..\Common\PTask\lib\$(Configuration)\;$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform)</LibraryPath>
<CustomBuildAfterTargets>Build</CustomBuildAfterTargets>
<ExecutablePath>$(SolutionDir)..\Common\PTask\bin\;$(ExecutablePath)</ExecutablePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<OpenMPSupport>true</OpenMPSupport>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>Delayimp.lib;nvml.lib;cudart.lib;cntkMath.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)\</AdditionalLibraryDirectories>
<DelayLoadDLLs>CNTKMath.dll;nvml.dll</DelayLoadDLLs>
</Link>
<PostBuildEvent>
<Command>
</Command>
<Message>
</Message>
</PostBuildEvent>
<CustomBuildStep>
</CustomBuildStep>
<CustomBuildStep>
<Outputs>$(TargetDir)config.txt;$(TargetDir)labels.txt;$(TargetDir)network.txt;$(TargetDir)NdlScript.txt</Outputs>
</CustomBuildStep>
<CustomBuildStep>
<TreatOutputAsContent>true</TreatOutputAsContent>
<Message>Copy content files to target directory</Message>
</CustomBuildStep>
<CudaCompile>
<GenerateLineInfo>true</GenerateLineInfo>
<CodeGeneration>compute_20,sm_20;compute_30,sm_30;compute_35,sm_35;compute_50,sm_50;</CodeGeneration>
</CudaCompile>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<Profile>true</Profile>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>Disabled</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<AdditionalOptions>/d2Zi+ %(AdditionalOptions)</AdditionalOptions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalDependencies>Delayimp.lib;nvml.lib;cudart.lib;cntkMath.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<Profile>true</Profile>
<DelayLoadDLLs>CNTKMath.dll;nvml.dll</DelayLoadDLLs>
</Link>
<PostBuildEvent>
<Command>copy $(SolutionDir)..\Common\PTask\bin\*.dll $(TargetDir)</Command>
<Message>Copy over the NVidia tools extention DLL</Message>
</PostBuildEvent>
<CustomBuildStep>
<Command>
</Command>
</CustomBuildStep>
<CustomBuildStep>
<Outputs>
</Outputs>
</CustomBuildStep>
<CustomBuildStep>
<TreatOutputAsContent>true</TreatOutputAsContent>
<Message>
</Message>
</CustomBuildStep>
<CudaCompile>
<GenerateLineInfo>true</GenerateLineInfo>
</CudaCompile>
</ItemDefinitionGroup>
<ItemGroup>
<Text Include="config.txt">
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
</Text>
<Text Include="DefaultMacros.txt" />
<Text Include="modelEditor.txt" />
<Text Include="modelEditorFromScratch.txt" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\hostname.h" />
<ClInclude Include="..\..\Common\Include\basetypes.h" />
<ClInclude Include="..\..\Common\Include\BestGpu.h" />
<ClInclude Include="..\..\Common\Include\commandArgUtil.h" />
<ClInclude Include="..\..\Common\Include\File.h" />
<ClInclude Include="..\..\Common\Include\fileutil.h" />
<ClInclude Include="..\..\Common\Include\nvml.h" />
<ClInclude Include="CompositeComputationNode.h" />
<ClInclude Include="ComputationNetwork.h" />
<ClInclude Include="ComputationNetworkHelper.h" />
<ClInclude Include="ComputationNode.h" />
<ClInclude Include="EvaluationCriterionNode.h" />
<ClInclude Include="IComputationNetBuilder.h" />
<ClInclude Include="IExecutionEngine.h" />
<ClInclude Include="ModelEditLanguage.h" />
<ClInclude Include="NDLNetworkBuilder.h" />
<ClInclude Include="NDLUtil.h" />
<ClInclude Include="NetworkDescriptionLanguage.h" />
<ClInclude Include="PTask.h" />
<ClInclude Include="PTaskGraphBuilder.h" />
<ClInclude Include="SimpleEvaluator.h" />
<ClInclude Include="SimpleOutputWriter.h" />
<ClInclude Include="SGD.h" />
<ClInclude Include="SimpleNetworkBuilder.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="SynchronousExecutionEngine.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="TrainingCriterionNode.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\Common\BestGpu.cpp" />
<ClCompile Include="..\..\Common\ConfigFile.cpp" />
<ClCompile Include="..\..\Common\DataReader.cpp" />
<ClCompile Include="..\..\Common\DataWriter.cpp" />
<ClCompile Include="..\..\Common\File.cpp">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader>
</ClCompile>
<ClCompile Include="cn.cpp" />
<ClCompile Include="ComputationNode.cpp" />
<ClCompile Include="ModelEditLanguage.cpp" />
<ClCompile Include="NetworkDescriptionLanguage.cpp" />
<ClCompile Include="PTaskGraphBuilder.cpp" />
<ClCompile Include="SimpleNetworkBuilder.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="tests.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Math\Math\Math.vcxproj">
<Project>{60bdb847-d0c4-4fd3-a947-0c15c08bcdb5}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="$(VCTargetsPath)\BuildCustomizations\CUDA 6.5.targets" />
</ImportGroup>
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}</ProjectGuid>
<SccProjectName>
</SccProjectName>
<SccAuxPath>
</SccAuxPath>
<SccLocalPath>
</SccLocalPath>
<SccProvider>
</SccProvider>
<Keyword>Win32Proj</Keyword>
<RootNamespace>cn</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v120</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
<Import Project="$(VCTargetsPath)\BuildCustomizations\CUDA 6.5.props" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>..\..\Math\Math;..\..\Common\;..\..\Common\include;..\..\Common\PTask\include;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<LibraryPath>$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(SolutionDir)..\Common\PTask\lib\$(Configuration)\;$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform)</LibraryPath>
<CustomBuildAfterTargets>Build</CustomBuildAfterTargets>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>..\..\Math\Math;..\..\Common\;..\..\Common\include;..\..\Common\PTask\include;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
<LibraryPath>$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(SolutionDir)..\Common\PTask\lib\$(Configuration)\;$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform)</LibraryPath>
<CustomBuildAfterTargets>Build</CustomBuildAfterTargets>
<ExecutablePath>$(SolutionDir)..\Common\PTask\bin\;$(ExecutablePath)</ExecutablePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<OpenMPSupport>true</OpenMPSupport>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>Delayimp.lib;nvml.lib;cudart.lib;cntkMath.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)\</AdditionalLibraryDirectories>
<DelayLoadDLLs>CNTKMath.dll;nvml.dll;nvcuda.dll</DelayLoadDLLs>
</Link>
<PostBuildEvent>
<Command>
</Command>
<Message>
</Message>
</PostBuildEvent>
<CustomBuildStep>
</CustomBuildStep>
<CustomBuildStep>
<Outputs>$(TargetDir)config.txt;$(TargetDir)labels.txt;$(TargetDir)network.txt;$(TargetDir)NdlScript.txt</Outputs>
</CustomBuildStep>
<CustomBuildStep>
<TreatOutputAsContent>true</TreatOutputAsContent>
<Message>Copy content files to target directory</Message>
</CustomBuildStep>
<CudaCompile>
<GenerateLineInfo>true</GenerateLineInfo>
<CodeGeneration>compute_20,sm_20;compute_30,sm_30;compute_35,sm_35;compute_50,sm_50;</CodeGeneration>
</CudaCompile>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<Profile>true</Profile>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>Disabled</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<AdditionalOptions>/d2Zi+ %(AdditionalOptions)</AdditionalOptions>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalDependencies>Delayimp.lib;nvml.lib;cudart.lib;cntkMath.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<Profile>true</Profile>
<DelayLoadDLLs>CNTKMath.dll;nvml.dll;nvcuda.dll</DelayLoadDLLs>
</Link>
<PostBuildEvent>
<Command>copy $(SolutionDir)..\Common\PTask\bin\*.dll $(TargetDir)</Command>
<Message>Copy over the NVidia tools extention DLL</Message>
</PostBuildEvent>
<CustomBuildStep>
<Command>
</Command>
</CustomBuildStep>
<CustomBuildStep>
<Outputs>
</Outputs>
</CustomBuildStep>
<CustomBuildStep>
<TreatOutputAsContent>true</TreatOutputAsContent>
<Message>
</Message>
</CustomBuildStep>
<CudaCompile>
<GenerateLineInfo>true</GenerateLineInfo>
</CudaCompile>
</ItemDefinitionGroup>
<ItemGroup>
<Text Include="config.txt">
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
</Text>
<Text Include="DefaultMacros.txt" />
<Text Include="modelEditor.txt" />
<Text Include="modelEditorFromScratch.txt" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\hostname.h" />
<ClInclude Include="..\..\Common\Include\basetypes.h" />
<ClInclude Include="..\..\Common\Include\BestGpu.h" />
<ClInclude Include="..\..\Common\Include\commandArgUtil.h" />
<ClInclude Include="..\..\Common\Include\DataReader.h" />
<ClInclude Include="..\..\Common\Include\DataWriter.h" />
<ClInclude Include="..\..\Common\Include\File.h" />
<ClInclude Include="..\..\Common\Include\fileutil.h" />
<ClInclude Include="..\..\Common\Include\nvml.h" />
<ClInclude Include="CompositeComputationNode.h" />
<ClInclude Include="ComputationNetwork.h" />
<ClInclude Include="ComputationNetworkHelper.h" />
<ClInclude Include="ComputationNode.h" />
<ClInclude Include="EvaluationCriterionNode.h" />
<ClInclude Include="IComputationNetBuilder.h" />
<ClInclude Include="IExecutionEngine.h" />
<ClInclude Include="ModelEditLanguage.h" />
<ClInclude Include="NDLNetworkBuilder.h" />
<ClInclude Include="NDLUtil.h" />
<ClInclude Include="NetworkDescriptionLanguage.h" />
<ClInclude Include="PTask.h" />
<ClInclude Include="PTaskGraphBuilder.h" />
<ClInclude Include="SimpleEvaluator.h" />
<ClInclude Include="SimpleOutputWriter.h" />
<ClInclude Include="SGD.h" />
<ClInclude Include="SimpleNetworkBuilder.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="SynchronousExecutionEngine.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="TrainingCriterionNode.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\Common\BestGpu.cpp" />
<ClCompile Include="..\..\Common\ConfigFile.cpp" />
<ClCompile Include="..\..\Common\DataReader.cpp" />
<ClCompile Include="..\..\Common\DataWriter.cpp" />
<ClCompile Include="..\..\Common\File.cpp">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader>
</ClCompile>
<ClCompile Include="cn.cpp" />
<ClCompile Include="ComputationNode.cpp" />
<ClCompile Include="ModelEditLanguage.cpp" />
<ClCompile Include="NetworkDescriptionLanguage.cpp" />
<ClCompile Include="PTaskGraphBuilder.cpp" />
<ClCompile Include="SimpleNetworkBuilder.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="tests.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Math\Math\Math.vcxproj">
<Project>{60bdb847-d0c4-4fd3-a947-0c15c08bcdb5}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="$(VCTargetsPath)\BuildCustomizations\CUDA 6.5.targets" />
</ImportGroup>
</Project>

97
MachineLearning/cn/cn.vcxproj.filters Normal file
Просмотреть файл

@ -0,0 +1,97 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="..\..\Common\BestGpu.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\ConfigFile.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\DataReader.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\DataWriter.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\File.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="NetworkDescriptionLanguage.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="cn.cpp" />
<ClCompile Include="ComputationNode.cpp" />
<ClCompile Include="ModelEditLanguage.cpp" />
<ClCompile Include="PTaskGraphBuilder.cpp" />
<ClCompile Include="SimpleNetworkBuilder.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="tests.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\Include\basetypes.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\BestGpu.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\hostname.h">
<Filter>Common</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\commandArgUtil.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\fileutil.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\File.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\nvml.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataReader.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\DataWriter.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="CompositeComputationNode.h" />
<ClInclude Include="ComputationNetwork.h" />
<ClInclude Include="ComputationNetworkHelper.h" />
<ClInclude Include="ComputationNode.h" />
<ClInclude Include="EvaluationCriterionNode.h" />
<ClInclude Include="IComputationNetBuilder.h" />
<ClInclude Include="IExecutionEngine.h" />
<ClInclude Include="ModelEditLanguage.h" />
<ClInclude Include="NDLNetworkBuilder.h" />
<ClInclude Include="NDLUtil.h" />
<ClInclude Include="NetworkDescriptionLanguage.h" />
<ClInclude Include="PTask.h" />
<ClInclude Include="PTaskGraphBuilder.h" />
<ClInclude Include="SGD.h" />
<ClInclude Include="SimpleEvaluator.h" />
<ClInclude Include="SimpleNetworkBuilder.h" />
<ClInclude Include="SimpleOutputWriter.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="SynchronousExecutionEngine.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="TrainingCriterionNode.h" />
</ItemGroup>
<ItemGroup>
<Text Include="config.txt" />
<Text Include="DefaultMacros.txt" />
<Text Include="modelEditor.txt" />
<Text Include="modelEditorFromScratch.txt" />
</ItemGroup>
<ItemGroup>
<Filter Include="Common">
<UniqueIdentifier>{b3d05c7b-7bcf-4b12-bcb5-dced86717202}</UniqueIdentifier>
</Filter>
<Filter Include="Common\Include">
<UniqueIdentifier>{85226dda-87ba-4da6-af04-563d0ce23b94}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>

4
Math/CNTKMathTest/CNTKMathTest.vcxproj
Просмотреть файл

@ -117,7 +117,7 @@
<PreprocessorDefinitions>WIN32;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<UseFullPaths>true</UseFullPaths>
<OpenMPSupport>true</OpenMPSupport>
<TreatWarningAsError>false</TreatWarningAsError>
<TreatWarningAsError>true</TreatWarningAsError>
</ClCompile>
<Link>
<SubSystem>Windows</SubSystem>
@ -159,7 +159,7 @@
<PreprocessorDefinitions>WIN32;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<UseFullPaths>true</UseFullPaths>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<TreatWarningAsError>false</TreatWarningAsError>
<TreatWarningAsError>true</TreatWarningAsError>
<OpenMPSupport>false</OpenMPSupport>
<AdditionalOptions>/d2Zi+ %(AdditionalOptions)</AdditionalOptions>
</ClCompile>

8
Math/CNTKMathTest/CPUMatrixUnitTests.cpp
Просмотреть файл

@ -24,14 +24,13 @@ namespace CNTKMathTest
static void DebugPrint(FILE* gpuDebugFile, Matrix M, const char* str, const bool colwiseVec = true)
{
fprintf(gpuDebugFile, "\n %s\n", str);
const int matNumRow = M.GetNumRows();
const int matNumCol = M.GetNumCols();
const int elemNum = M.GetNumElements();
const size_t matNumCol = M.GetNumCols();
const size_t elemNum = M.GetNumElements();
Matrix M1 = M.Transpose();
double* pArray = M1.GetArray();
if (colwiseVec)
{
for (int i = 0; i < elemNum; i++)
for (size_t i = 0; i < elemNum; i++)
{
fprintf(gpuDebugFile, "%3d ", (int)pArray[i]);
@ -39,6 +38,7 @@ namespace CNTKMathTest
fprintf(gpuDebugFile, "\n");
}
}
//const size_t matNumRow = M.GetNumRows();
//for (int i = 0; i < matNumRow; i++)
//{
// for (int j = 0; j < matNumCol; j++)

2
Math/CNTKMathTest/GPUMatrixUnitTests.cpp
Просмотреть файл

@ -12,6 +12,8 @@
#define epsilon 0.00001
#define IDX2C(i,j,ld) (((j)*(ld))+(i)) // 0 based indexing
#pragma warning (disable: 4244 4245 4305) // conversions and truncations; we don't care in this test project
#define DEBUG_FLAG 1
using namespace Microsoft::VisualStudio::CppUnitTestFramework;
using namespace Microsoft::MSR::CNTK;

6
Math/CNTKMathTest/GPUMatrixcuBLASTests.cpp
Просмотреть файл

@ -9,6 +9,8 @@
#include "..\Math\CPUMatrix.h"
#include "..\Math\GPUMatrix.cuh"
#pragma warning (disable: 4244 4245 4305) // conversions and truncations; we don't care in this test project
using namespace Microsoft::VisualStudio::CppUnitTestFramework;
using namespace Microsoft::MSR::CNTK;
@ -23,11 +25,11 @@ namespace CNTKMathTest
TEST_METHOD(GPU_MultiplyAndWeightedAdd_NoExceptionOnly_Test)
{
float alpha = 0.435;
float alpha = 0.4;
GPUMatrix<float> M0_GPU(12,5);
GPUMatrix<float> M1_GPU(5,11);
GPUMatrix<float> M2_GPU(12,11);
GPUMatrix<float>::MultiplyAndWeightedAdd(0.1,M0_GPU,false,M1_GPU,false,0.4,M2_GPU);
GPUMatrix<float>::MultiplyAndWeightedAdd(0.1,M0_GPU,false,M1_GPU,false,alpha,M2_GPU);
}

6
Math/CNTKMathTest/GPUSparseMatrixUnitTests.cpp
Просмотреть файл

@ -13,6 +13,8 @@
using namespace Microsoft::VisualStudio::CppUnitTestFramework;
using namespace Microsoft::MSR::CNTK;
#pragma warning (disable: 4244 4245 4305) // conversions and truncations; we don't care in this test project
#define ID_2C(i,j,ld) (((i)*(ld))+(j)) // 0 based indexing
namespace CNTKMathTest
@ -163,7 +165,7 @@ namespace CNTKMathTest
GPUSparseMatrix<float> ATs = A.Transpose();
GPUMatrix<float> ATd = ATs.CopyToDenseMatrix();
float* arrTd = ATd.CopyToArray();
float* arrTd = ATd.CopyToArray(); arrTd;
float arrA_times_AT[19] = {17,8,5,0,8,13,0,27,5,0,138,48,0,27,48,117};
GPUMatrix<float> Cet(4,4,arrA_times_AT,matrixFlagNormal);
@ -176,7 +178,7 @@ namespace CNTKMathTest
GPUMatrix<float> Cres1(5,5);
GPUSparseMatrix<float>::Multiply(ATd,A,Cres1); //Dense times sparse
float* arr = Cres1.CopyToArray();
float* arr = Cres1.CopyToArray(); arr;
Assert::IsTrue(Cres1.IsEqualTo(Cet1));

6
Math/CNTKMathTest/MatrixBLASTests.cpp
Просмотреть файл

@ -7,6 +7,8 @@
#include "CppUnitTest.h"
#include "..\Math\Matrix.h"
#pragma warning (disable: 4244 4245 4305) // conversions and truncations; we don't care in this test project
#define epsilon 0.000001
#define IDX2C(i,j,ld) (((j)*(ld))+(i)) // 0 based indexing
@ -137,7 +139,7 @@ namespace CNTKMathTest
{
float x = C2(i,j);
float y = (alpha*A2(i,0)+beta*B2(i,j));
Assert::IsTrue(fabsf(C2(i,j)-(alpha*A2(i,0)+beta*B2(i,j)))<precision);
Assert::IsTrue(fabsf(x-y)<precision);
}
}
@ -177,7 +179,7 @@ namespace CNTKMathTest
{
float x = C2(i,j);
float y = (alpha*A2(i,0)+beta*B2(i,j));
Assert::IsTrue(fabsf(C2(i,j)-(alpha*A2(i,0)+beta*B2(i,j)))<precision);
Assert::IsTrue(fabsf(x-y)<precision);
}
}

2
Math/CNTKMathTest/MatrixSparseDenseInteractionsTests.cpp
Просмотреть файл

@ -7,6 +7,8 @@
#include "CppUnitTest.h"
#include "..\Math\Matrix.h"
#pragma warning (disable: 4244 4245 4305) // conversions and truncations; we don't care in this test project
#define epsilon 0.000001
#define IDX2C(i,j,ld) (((j)*(ld))+(i)) // 0 based indexing

9
Math/CNTKMathTest/MatrixUnitTests.cpp
Просмотреть файл

@ -7,6 +7,8 @@
#include "CppUnitTest.h"
#include "..\Math\Matrix.h"
#pragma warning (disable: 4244 4245 4305) // conversions and truncations; we don't care in this test project
#define epsilon 0.000001
#define IDX2C(i,j,ld) (((j)*(ld))+(i)) // 0 based indexing
@ -86,12 +88,12 @@ namespace CNTKMathTest
if (i!=j)
{
float x = A1(i,j);
Assert::AreEqual<float>(0,A1(i,j));
Assert::AreEqual<float>(0,x);
}
else
{
float x = A1(i,j);
Assert::AreEqual<float>(1,A1(i,j));
Assert::AreEqual<float>(1,x);
}
}
@ -128,7 +130,6 @@ namespace CNTKMathTest
bool has_big=false;
foreach_coord(i,j,A4)
{
float x = A4(i,j);
Assert::IsTrue((A4(i,j)>=-26.3)&&(A4(i,j)<30.2));
if (A4(i,j)<-3)
has_small=true;
@ -577,7 +578,7 @@ namespace CNTKMathTest
C.AssignElementProductOf(A,B);
foreach_coord(i,j,C)
{
Assert::IsTrue(C(i,j)=A(i,j)*B(i,j));
Assert::IsTrue(C(i,j)==A(i,j)*B(i,j));
}
//AddElementProductOf

59
Math/Math/CPUMatrix.cpp
Просмотреть файл

@ -17,7 +17,17 @@
#include "CPUMatrix.h"
#include <random>
#include <chrono>
#include <exception>
#ifdef _WIN32
#include <Windows.h>
#else
#ifndef max
#define max(a,b) (((a) > (b)) ? (a) : (b))
#endif
#include <cfloat>
#endif
#ifdef LEAKDETECT
#include <vld.h>
#endif
@ -345,7 +355,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
throw std::logic_error("AddWithRowSliceValuesOf: input matrix a is empty.");
if (GetNumRows() != numRows)
throw std::logic_error("AddWithRowSliceValuesOf: this->GetNumRows() != numRows.");
throw std::logic_error("AddWithRowSliceValuesOf: GetNumRows() != numRows.");
if (startIndex + numRows > a.GetNumRows())
throw std::logic_error("AddWithRowSliceValuesOf: startIndex + numRows exceeds a.GetNumRows().");
@ -554,7 +564,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
if (IsEmpty())
throw std::logic_error("SetColumn: Matrix is empty.");
assert(valMat.GetNumRows() == this->GetNumRows() && valMat.GetNumCols() == 1) ;
assert(valMat.GetNumRows() == GetNumRows() && valMat.GetNumCols() == 1) ;
auto& us = *this;
long m=(long)GetNumRows();
@ -751,8 +761,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (IsEmpty())
throw std::logic_error("SetUniformRandomValue: Matrix is empty.");
#ifdef _MSC_VER // TODO: check if available under GCC/Linux
std::ranlux64_base_01 generator;
generator.seed(seed==USE_TIME_BASED_SEED ? (unsigned long) time(NULL) : seed);
#else
std::default_random_engine generator (seed);
#endif
std::uniform_real_distribution<ElemType> r(low, high);
long m=(long)GetNumElements();
@ -782,8 +796,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
throw std::logic_error("SetUniformRandomValue: Matrix is empty.");
auto& us = *this;
std::ranlux64_base_01 generator;
#ifdef _MSC_VER // TODO: check if available under GCC/Linux
std::ranlux64_base_01 generator;
generator.seed(seed==USE_TIME_BASED_SEED ? (unsigned long) time(NULL) : seed);
#else
std::default_random_engine generator (seed);
#endif
std::normal_distribution<ElemType> r(mean, sigma);
//#pragma omp parallel for //is it thread safe?
foreach_coord(i,j,us)
@ -802,8 +820,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
throw std::logic_error("SetUniformRandomValue: Matrix is empty.");
auto& us = *this;
std::ranlux64_base_01 generator;
#ifdef _MSC_VER // TODO: check if available under GCC/Linux
std::ranlux64_base_01 generator;
generator.seed(seed==USE_TIME_BASED_SEED ? (unsigned long) time(NULL) : seed);
#else
std::default_random_engine generator (seed);
#endif
std::normal_distribution<ElemType> r(mean, sigma);
long m=(long)GetNumRows(), n=(long)GetNumCols();
@ -825,17 +847,22 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
}
//maskRate: percentage of values masked out (similar to dropout rate)
//scaleValue: which scale value to set to the left ones (unmasked items).
template<class ElemType>
void CPUMatrix<ElemType>::SetUniformRandomMask(const ElemType maskRate, const ElemType scaleValue, unsigned long seed=USE_TIME_BASED_SEED)
void CPUMatrix<ElemType>::SetUniformRandomMask(const ElemType maskRate, const ElemType scaleValue, unsigned long seed)
{
if (IsEmpty())
throw std::logic_error("SetUniformRandomValue: Matrix is empty.");
auto& us = *this;
std::ranlux64_base_01 generator;
#ifdef _MSC_VER // TODO: check if available under GCC/Linux
std::ranlux64_base_01 generator;
generator.seed(seed==USE_TIME_BASED_SEED ? (unsigned long) time(NULL) : seed);
#else
std::default_random_engine generator (seed==USE_TIME_BASED_SEED ? (unsigned long) time(NULL) : seed);
#endif
std::uniform_real_distribution<ElemType> r(0, 1);
long m=(long)GetNumRows(), n=(long)GetNumCols();
@ -866,13 +893,13 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
void CPUMatrix<ElemType>::Adagrad(CPUMatrix<ElemType>& gradients)
{
if (this->IsEmpty())
if (IsEmpty())
{
this->Resize(gradients.GetNumRows(), gradients.GetNumCols());
this->SetValue(0.0);
Resize(gradients.GetNumRows(), gradients.GetNumCols());
SetValue(0.0);
}
assert(this->GetNumRows() == gradients.GetNumRows() && this->GetNumCols() == gradients.GetNumCols());
assert(GetNumRows() == gradients.GetNumRows() && GetNumCols() == gradients.GetNumCols());
ElemType *a=m_pArray, *d_v=gradients.m_pArray;
size_t n = GetNumElements();
@ -919,10 +946,10 @@ namespace Microsoft { namespace MSR { namespace CNTK {
size_t n = gradients.GetNumElements();
ElemType *curr_grad=gradients.m_pArray;
if (this->IsEmpty() || this->GetNumCols() < gradients.GetNumCols() * 3)
if (IsEmpty() || GetNumCols() < gradients.GetNumCols() * 3)
{
this->Resize(gradients.GetNumRows(), gradients.GetNumCols() * 3);
this->SetValue(0.0);
Resize(gradients.GetNumRows(), gradients.GetNumCols() * 3);
SetValue(0.0);
ElemType *avars=m_pArray; // accumulated variances for RMS scaling
ElemType *steps=m_pArray+2*n; // current step size
@ -940,7 +967,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
ElemType *signs=m_pArray+n; // sign of previous gradient
ElemType *steps=m_pArray+2*n; // current step size
assert(this->GetNumRows() == gradients.GetNumRows() && this->GetNumCols() == gradients.GetNumCols() * 3);
assert(GetNumRows() == gradients.GetNumRows() && GetNumCols() == gradients.GetNumCols() * 3);
ElemType ONE_MINUS_GAMMA = ElemType(1.0) - RMS_GAMMA;
//int upd[] = {
@ -3933,7 +3960,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (a.IsEmpty())
throw std::logic_error("Scale: Input matrix a is empty.");
if (alpha.GetNumElements()!=1)
#ifdef _MSC_VER // TODO: check if available under GCC/Linux
throw std::exception("Matrix alpha must be 1x1");
#else
throw std::exception();
#endif
CPUMatrix<ElemType>::Scale(alpha(0,0),a);
}

17
Math/Math/CPUMatrix.h
Просмотреть файл

@ -7,15 +7,20 @@
#include <vector>
#include <stdio.h>
#include <ctime>
#include <limits.h>
#include "File.h"
#include "Helpers.h"
#include "CommonMatrix.h"
#ifdef _WIN32
#ifdef MATH_EXPORTS
#define MATH_API __declspec(dllexport)
#else
#define MATH_API __declspec(dllimport)
#endif
#else // no DLLs on Linux
#define MATH_API
#endif
#ifndef USE_TIME_BASED_SEED
#define USE_TIME_BASED_SEED ULONG_MAX
@ -38,6 +43,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
class MATH_API CPUMatrix : public BaseMatrix<ElemType>
{
typedef BaseMatrix<ElemType> B; using B::m_numRows; using B::m_numCols; using B::m_pArray; using B::m_computeDevice; using B::m_elemSizeAllocated;
using B::m_externalBuffer; using B::m_format; using B::m_matrixName; // without this, base members would require to use thi-> in GCC
public:
CPUMatrix();
CPUMatrix(FILE* f, const char * matrixName); //matrixName is used to verify that correct matrix is read.
@ -51,7 +58,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
~CPUMatrix();
public:
size_t BufferSize() const {return m_numRows*m_numCols*sizeof(ElemType);}
using B::OwnBuffer; using B::GetNumElements; using B::IsEmpty; using B::GetNumRows; using B::GetNumCols; using B::SetOwnBuffer; using B::SetMatrixName;
size_t BufferSize() const { return m_numRows*m_numCols*sizeof(ElemType); }
ElemType* BufferPointer() const {return m_pArray;}
CPUMatrix<ElemType> ColumnSlice(size_t startColumn, size_t numCols) const;
@ -274,7 +283,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
static void SVD(const CPUMatrix<ElemType>& A, CPUMatrix<ElemType>& SIGMA, CPUMatrix<ElemType>& U, CPUMatrix<ElemType>& VT);
static void MultiplyAndWeightedAdd(ElemType alpha, const CPUMatrix<ElemType>& a, const bool transposeA, const CPUMatrix<ElemType>& b, const bool transposeB,
ElemType beta, CPUMatrix<ElemType>& c);
ElemType beta, CPUMatrix<ElemType>& c);
static void MultiplyAndAdd(const CPUMatrix<ElemType>& a, const bool transposeA, const CPUMatrix<ElemType>& b, const bool transposeB, CPUMatrix<ElemType>& c);
static void Multiply(const CPUMatrix<ElemType>& a, const bool transposeA, const CPUMatrix<ElemType>& b, const bool transposeB, CPUMatrix<ElemType>& c);
static void Multiply(const CPUMatrix<ElemType>& a, const CPUMatrix<ElemType>& b, CPUMatrix<ElemType>& c);
@ -347,8 +356,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
protected:
inline size_t LocateElement (const size_t i, const size_t j) const;
inline size_t LocateColumn (const size_t j) const;
size_t LocateElement (const size_t i, const size_t j) const;
size_t LocateColumn (const size_t j) const;
private:
void ZeroInit(); //should only be used by constructors.

28
Math/Math/CPUSparseMatrix.cpp
Просмотреть файл

@ -15,7 +15,9 @@
#include "CPUSparseMatrix.h"
#include <random>
#include <chrono>
#ifdef _WIN32
#include <Windows.h>
#endif
#ifdef LEAKDETECT
#include <vld.h>
#endif
@ -114,10 +116,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
}
//should only be used by constructors.
template<class ElemType>
CPUSparseMatrix<ElemType>::CPUSparseMatrix(const MatrixFormat format)
void CPUSparseMatrix<ElemType>::CheckInit(const MatrixFormat format)
{
if(format != MatrixFormat::matrixFormatSparseCSC && format != MatrixFormat::matrixFormatSparseCSR && format != MatrixFormat::matrixFormatSparseBlockCol && format != MatrixFormat::matrixFormatSparseBlockRow)
if (format != MatrixFormat::matrixFormatSparseCSC && format != MatrixFormat::matrixFormatSparseCSR && format != MatrixFormat::matrixFormatSparseBlockCol && format != MatrixFormat::matrixFormatSparseBlockRow)
{
throw std::logic_error("CPUSparseMatrix: unsupported sparse matrix format");
}
@ -125,9 +128,16 @@ namespace Microsoft { namespace MSR { namespace CNTK {
ZeroInit();
}
template<class ElemType>
CPUSparseMatrix<ElemType>::CPUSparseMatrix(const MatrixFormat format)
{
CheckInit(format);
}
template<class ElemType>
CPUSparseMatrix<ElemType>::CPUSparseMatrix(const MatrixFormat format, const size_t numRows, const size_t numCols, const size_t size)
{ CPUSparseMatrix<ElemType>::CPUSparseMatrix(format);
{
CheckInit(format);
Resize(numRows, numCols, size);
}
@ -489,7 +499,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else
{
throw std::exception("CPUSparseMatrix:: ScaleAndAdd() Not implemented");
throw std::runtime_error("CPUSparseMatrix:: ScaleAndAdd() Not implemented");
}
}
@ -509,7 +519,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
throw std::logic_error("AssignSoftmaxOf: Matrix a, class, idx2cls or label is empty.");
if(etp.GetFormat() != MatrixFormat::matrixFormatSparseCSC)
throw std::exception("CPUSparseMatrix:: ClassEntropy() only support CSC");
throw std::runtime_error("CPUSparseMatrix:: ClassEntropy() only support CSC");
size_t nC = cls.GetNumCols();
size_t nV = label.GetNumRows() - nC;
@ -678,7 +688,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
if (c.IsEmpty())
{
c.Resize(this->GetNumRows(), this->GetNumCols());
c.Resize(GetNumRows(), GetNumCols());
c.SetValue(0.0);
}
@ -701,7 +711,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else
{
throw std::exception("CPUSparseMatrix:: NormalGrad() only support block sparse format");
throw std::runtime_error("CPUSparseMatrix:: NormalGrad() only support block sparse format");
}
}
@ -711,7 +721,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
if (c.IsEmpty())
{
c.Resize(this->GetNumRows(), this->GetNumCols());
c.Resize(GetNumRows(), GetNumCols());
c.SetValue(0.0);
}
@ -787,7 +797,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else
{
throw std::exception("CPUSparseMatrix:: InplaceTruncate() only support block based sparse matrix");
throw std::runtime_error("CPUSparseMatrix:: InplaceTruncate() only support block based sparse matrix");
}
return *this;
}

229
Math/Math/CPUSparseMatrix.h
Просмотреть файл

@ -1,111 +1,118 @@
//
// <copyright file="GPUMatrixUnitTests.cpp" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//
#pragma once
#include <stdio.h>
#include "cpumatrix.h"
#include <map>
#include <unordered_map>
#ifdef MATH_EXPORTS
#define MATH_API __declspec(dllexport)
#else
#define MATH_API __declspec(dllimport)
#endif
namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
class MATH_API CPUSparseMatrix : public BaseMatrix<ElemType>
{
private:
void ZeroInit();
public:
CPUSparseMatrix(const MatrixFormat format);
CPUSparseMatrix(const MatrixFormat format, const size_t numRows, const size_t numCols, const size_t size);
~CPUSparseMatrix();
public:
void SetValue(const size_t rIdx, const size_t cIdx, ElemType val);
void SetValue(const CPUSparseMatrix& /*val*/) { NOT_IMPLEMENTED; }
void ShiftBy(int /*numShift*/) { NOT_IMPLEMENTED; }
size_t BufferSize() const {return m_elemSizeAllocated*sizeof(ElemType);}
ElemType* BufferPointer() const;
void SetGaussianRandomValue(const ElemType /*mean*/, const ElemType /*sigma*/, unsigned long /*seed*/) { NOT_IMPLEMENTED; }
static void ClassEntropy(const CPUMatrix<ElemType>& a, const CPUMatrix<ElemType>& weight,
const CPUSparseMatrix<ElemType> & label, const CPUMatrix<ElemType>& cls,
const CPUMatrix<ElemType>& idx2cls, CPUSparseMatrix<ElemType>& etp, CPUMatrix<ElemType>& entropyScore);
static void CPUSparseMatrix<ElemType>::ClassEntropyError(CPUSparseMatrix<ElemType>& a);
static void CPUSparseMatrix<ElemType>::ClassEntropyGradientOfInput(
const CPUSparseMatrix<ElemType>& error,
const CPUMatrix<ElemType>& weight,
CPUMatrix<ElemType>& grd);
static void CPUSparseMatrix<ElemType>::ClassEntropyGradientOfWeight(
const CPUSparseMatrix<ElemType>& error,
const CPUMatrix<ElemType>& input,
const CPUSparseMatrix<ElemType> & label,
const CPUMatrix<ElemType>& cls,
const CPUMatrix<ElemType>& idx2cls,
CPUSparseMatrix<ElemType>& grd);
static void MultiplyAndWeightedAdd(ElemType alpha, const CPUMatrix<ElemType>& lhs, const bool transposeA,
const CPUSparseMatrix<ElemType>& rhs, const bool transposeB, ElemType beta, CPUMatrix<ElemType>& c);
static void MultiplyAndAdd(ElemType alpha, const CPUMatrix<ElemType>& lhs, const bool transposeA,
const CPUSparseMatrix<ElemType>& rhs, const bool transposeB, CPUSparseMatrix<ElemType>& c);
static void ScaleAndAdd(const ElemType alpha, const CPUSparseMatrix<ElemType>& lhs, CPUMatrix<ElemType>& c);
/// sum(vec(a).*vec(b))
static ElemType InnerProductOfMatrices(const CPUSparseMatrix<ElemType>& /*a*/, const CPUMatrix<ElemType>& /*b*/) { NOT_IMPLEMENTED; }
static void AddScaledDifference(const ElemType /*alpha*/, const CPUSparseMatrix<ElemType>& /*a*/, const CPUMatrix<ElemType>& /*b*/, CPUMatrix<ElemType>& /*c*/,
bool /*bDefaultZero*/ ) { NOT_IMPLEMENTED; }
static void AddScaledDifference(const ElemType /*alpha*/, const CPUMatrix<ElemType>& /*a*/, const CPUSparseMatrix<ElemType>& /*b*/, CPUMatrix<ElemType>& /*c*/,
bool /*bDefaultZero*/ ) { NOT_IMPLEMENTED; }
int GetComputeDeviceId() const {return -1;}
void Resize(const size_t numRows, const size_t numCols, size_t size = 0);
void Reset();
public:
void NormalGrad(CPUMatrix<ElemType>& c, const ElemType momentum);
void Adagrad(CPUMatrix<ElemType>& c);
public:
CPUSparseMatrix<ElemType>& InplaceTruncateTop (const ElemType /*threshold*/) { NOT_IMPLEMENTED; }
CPUSparseMatrix<ElemType>& InplaceTruncateBottom (const ElemType /*threshold*/) { NOT_IMPLEMENTED; }
CPUSparseMatrix<ElemType>& InplaceTruncate (const ElemType /*threshold*/);
public:
void Print(const char* /*matrixName*/) const { NOT_IMPLEMENTED; }
int m_colIdx; //used to SetValue()
ElemType *m_val; // values
size_t *m_row; //row/col ids in CSC/CSR format
size_t *m_pb; //begin ids of col/row in CSC/CSR format
size_t m_blockSize; //block size
ElemType *m_blockVal; //block values
size_t *m_blockIds; //block ids
};
typedef CPUSparseMatrix<float> CPUSingleSparseMatrix;
typedef CPUSparseMatrix<double> CPUDoubleSparseMatrix;
}}}
//
// <copyright file="GPUMatrixUnitTests.cpp" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//
#pragma once
#include <stdio.h>
#include "CPUMatrix.h"
#include <map>
#include <unordered_map>
#ifdef _WIN32
#ifdef MATH_EXPORTS
#define MATH_API __declspec(dllexport)
#else
#define MATH_API __declspec(dllimport)
#endif
#endif /* Linux - already defined in CPUMatrix.h */
namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
class MATH_API CPUSparseMatrix : public BaseMatrix<ElemType>
{
typedef BaseMatrix<ElemType> B; using B::m_elemSizeAllocated; using B::m_computeDevice; using B::m_externalBuffer; using B::m_format; using B::m_matrixName;
using B::m_numCols; using B::m_numRows; using B::m_nz; using B::m_pArray; // without this, base members would require to use thi-> in GCC
private:
void ZeroInit();
void CheckInit(const MatrixFormat format);
public:
CPUSparseMatrix(const MatrixFormat format);
CPUSparseMatrix(const MatrixFormat format, const size_t numRows, const size_t numCols, const size_t size);
~CPUSparseMatrix();
public:
using B::GetNumCols; using B::GetNumRows;
void SetValue(const size_t rIdx, const size_t cIdx, ElemType val);
void SetValue(const CPUSparseMatrix& /*val*/) { NOT_IMPLEMENTED; }
void ShiftBy(int /*numShift*/) { NOT_IMPLEMENTED; }
size_t BufferSize() const {return m_elemSizeAllocated*sizeof(ElemType);}
ElemType* BufferPointer() const;
void SetGaussianRandomValue(const ElemType /*mean*/, const ElemType /*sigma*/, unsigned long /*seed*/) { NOT_IMPLEMENTED; }
static void ClassEntropy(const CPUMatrix<ElemType>& a, const CPUMatrix<ElemType>& weight,
const CPUSparseMatrix<ElemType> & label, const CPUMatrix<ElemType>& cls,
const CPUMatrix<ElemType>& idx2cls, CPUSparseMatrix<ElemType>& etp, CPUMatrix<ElemType>& entropyScore);
static void ClassEntropyError(CPUSparseMatrix<ElemType>& a);
static void ClassEntropyGradientOfInput(
const CPUSparseMatrix<ElemType>& error,
const CPUMatrix<ElemType>& weight,
CPUMatrix<ElemType>& grd);
static void ClassEntropyGradientOfWeight(
const CPUSparseMatrix<ElemType>& error,
const CPUMatrix<ElemType>& input,
const CPUSparseMatrix<ElemType> & label,
const CPUMatrix<ElemType>& cls,
const CPUMatrix<ElemType>& idx2cls,
CPUSparseMatrix<ElemType>& grd);
static void MultiplyAndWeightedAdd(ElemType alpha, const CPUMatrix<ElemType>& lhs, const bool transposeA,
const CPUSparseMatrix<ElemType>& rhs, const bool transposeB, ElemType beta, CPUMatrix<ElemType>& c);
static void MultiplyAndAdd(ElemType alpha, const CPUMatrix<ElemType>& lhs, const bool transposeA,
const CPUSparseMatrix<ElemType>& rhs, const bool transposeB, CPUSparseMatrix<ElemType>& c);
static void ScaleAndAdd(const ElemType alpha, const CPUSparseMatrix<ElemType>& lhs, CPUMatrix<ElemType>& c);
/// sum(vec(a).*vec(b))
static ElemType InnerProductOfMatrices(const CPUSparseMatrix<ElemType>& /*a*/, const CPUMatrix<ElemType>& /*b*/) { NOT_IMPLEMENTED; }
static void AddScaledDifference(const ElemType /*alpha*/, const CPUSparseMatrix<ElemType>& /*a*/, const CPUMatrix<ElemType>& /*b*/, CPUMatrix<ElemType>& /*c*/,
bool /*bDefaultZero*/ ) { NOT_IMPLEMENTED; }
static void AddScaledDifference(const ElemType /*alpha*/, const CPUMatrix<ElemType>& /*a*/, const CPUSparseMatrix<ElemType>& /*b*/, CPUMatrix<ElemType>& /*c*/,
bool /*bDefaultZero*/ ) { NOT_IMPLEMENTED; }
int GetComputeDeviceId() const {return -1;}
void Resize(const size_t numRows, const size_t numCols, size_t size = 0);
void Reset();
public:
void NormalGrad(CPUMatrix<ElemType>& c, const ElemType momentum);
void Adagrad(CPUMatrix<ElemType>& c);
public:
CPUSparseMatrix<ElemType>& InplaceTruncateTop (const ElemType /*threshold*/) { NOT_IMPLEMENTED; }
CPUSparseMatrix<ElemType>& InplaceTruncateBottom (const ElemType /*threshold*/) { NOT_IMPLEMENTED; }
CPUSparseMatrix<ElemType>& InplaceTruncate (const ElemType /*threshold*/);
public:
void Print(const char* /*matrixName*/) const { NOT_IMPLEMENTED; }
int m_colIdx; //used to SetValue()
ElemType *m_val; // values
size_t *m_row; //row/col ids in CSC/CSR format
size_t *m_pb; //begin ids of col/row in CSC/CSR format
size_t m_blockSize; //block size
ElemType *m_blockVal; //block values
size_t *m_blockIds; //block ids
};
typedef CPUSparseMatrix<float> CPUSingleSparseMatrix;
typedef CPUSparseMatrix<double> CPUDoubleSparseMatrix;
}}}

5
Math/Math/CommonMatrix.h
Просмотреть файл

@ -6,6 +6,7 @@
#pragma once
#include <string>
#include <stdint.h>
#define AUTOPLACEMATRIX 1000 // used in parameters only
#define MANAGEDEXTERN -2 // managed externally (i.e. PTask)
@ -79,7 +80,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
Clear();
if (s!=nullptr)
{
size_t n = wcsnlen_s(s, SIZE_MAX);
size_t n = wcslen(s);
m_matrixName = new wchar_t[n+1];
wmemcpy(m_matrixName,s,n+1);
}
@ -120,4 +121,4 @@ namespace Microsoft { namespace MSR { namespace CNTK {
size_t m_nz; //Number of non-zero elements for sparse matrices (unused in other formats)
wchar_t* m_matrixName;
};
}}}
}}}

1670
Math/Math/GPUDummy.cpp Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

329
Math/Math/GPUMatrix.cu
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

26
Math/Math/GPUMatrix.cuh
Просмотреть файл

@ -7,6 +7,7 @@
#include <string>
#include <vector>
#include <ctime>
#include <limits.h> // for ULONG_MAX
#include "File.h"
#include "Helpers.h"
#include "CommonMatrix.h"
@ -17,11 +18,17 @@ typedef struct cublasContext *cublasHandle_t;
struct CUstream_st;
typedef struct CUstream_st *cudaStream_t;
#ifdef _WIN32
#ifndef MATH_API
#ifdef MATH_EXPORTS
#define MATH_API __declspec(dllexport)
#else
#define MATH_API __declspec(dllimport)
#endif
#endif /* MATH_API */
#else // no DLLs in Linux
#define MATH_API
#endif
#ifndef USE_TIME_BASED_SEED
#define USE_TIME_BASED_SEED ULONG_MAX
@ -56,6 +63,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
class MATH_API GPUMatrix : public BaseMatrix<ElemType>
{
typedef BaseMatrix<ElemType> B; using B::m_numRows; using B::m_numCols; using B::m_pArray; // without this, base members would require to use thi-> in GCC
public:
static const int MaxGpus = 8; // support up to 8 GPUs
private:
@ -99,18 +107,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
ElemType* BufferPointer() const {return m_pArray;}
void Adagrad(GPUMatrix<ElemType>& gradients);
void RmsProp(GPUMatrix<ElemType>& gradients,
ElemType RMS_GAMMA,
ElemType RMS_WGT_INC,
ElemType RMS_WGT_MAX,
ElemType RMS_WGT_DEC,
ElemType RMS_WGT_MIN
);
void RmsProp(GPUMatrix<ElemType>& gradients, ElemType RMS_GAMMA, ElemType RMS_WGT_INC, ElemType RMS_WGT_MAX, ElemType RMS_WGT_DEC, ElemType RMS_WGT_MIN);
void Reshape(const size_t numRows, const size_t numCols);
void Resize(const size_t numRows, const size_t numCols, bool growOnly = true); //by default we only reallocate if need to grow
ElemType& operator() (const size_t /*row*/, const size_t /*col*/) { throw std::exception("GPUMatrix doesn't support this"); }
const ElemType& operator() (const size_t /*row*/, const size_t /*col*/) const { throw std::exception("GPUMatrix doesn't support this"); }
ElemType& operator() (const size_t /*row*/, const size_t /*col*/) { throw std::logic_error("GPUMatrix doesn't support this"); }
const ElemType& operator() (const size_t /*row*/, const size_t /*col*/) const { throw std::logic_error("GPUMatrix doesn't support this"); }
ElemType Get00Element() const;
void SetValue(const ElemType v);
@ -262,7 +264,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
GPUMatrix<ElemType>& AssignInnerProductOfMatrices(const GPUMatrix<ElemType>& a, const GPUMatrix<ElemType>& b);
void Print(const char* matrixName, size_t rowStart, size_t rowEnd, size_t colStart, size_t colEnd) const;
void Print(const char* matrixName = nullptr) const; //print whole matrix. can be expensive
void Print(const char* matrixName = NULL) const; //print whole matrix. can be expensive
void ReadFromFile(FILE* f, const char * matrixName); //matrixName is used to verify that correct matrix is read.
void WriteToFile(FILE* f, const char * matrixName); //matrixName is used to verify that correct matrix is read.
@ -334,7 +336,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
size_t elsize;
stream>>elsize;
if (sizeof(ElemType)!=elsize)
#ifndef LINUX
throw std::exception("Template argument size doesn't match those in file");
#else
throw std::exception();
#endif
std::wstring matrixName;
size_t numRows, numCols;
int format;

10
Math/Math/GPUMatrixCUDAKernels.cu
Просмотреть файл

@ -3,6 +3,9 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//
#ifndef CPU_ONLY
#include <float.h>
#include <cuda_runtime.h>
#include "CommonMatrix.h"
@ -18,6 +21,9 @@
#define MINLOGEXP -9.2103
#define LSMALL -0.5E10
// Predefine this for later.
static __inline__ __device__ double atomicAdd(double* address, double val);
//CUDA Kernels code
template<class ElemType>
__global__ void _elementWisePowerOnCuda(
@ -3223,4 +3229,6 @@ else
d_tmp[0] = max((ElemType)0, d_tmp[0]/max((ElemType)1.0e-10,sqrt(d_tmp[1]))/max((ElemType)1.0e-10,sqrt(d_tmp[2])));
}
}
*/
*/
#endif // !CPU_ONLY

207
Math/Math/GPUSparseMatrix.cu
Просмотреть файл

@ -17,8 +17,11 @@
#pragma warning (disable: 4267) // conversion from 'size_t' to 'unsigned int'; happens in CUDA <<<a,b>>> syntax if a and b are size_t
#pragma warning (disable: 4127) // conditional expression is constant; "if (sizeof(ElemType)==sizeof(float))" triggers this
#ifdef _WIN32
// thread local storage to access the current stream, initalize to default stream
extern __declspec( thread ) cudaStream_t t_stream;
extern __declspec (thread)
#endif
cudaStream_t t_stream;
void CUDACALL(cudaError_t x)
{
@ -27,7 +30,7 @@ void CUDACALL(cudaError_t x)
const char* errmsg = cudaGetErrorString(x);
std::cout<<"!!!!!!!!CUDA EXCEPTION: "<<errmsg<<std::endl;
throw std::exception(errmsg);
throw std::runtime_error(errmsg);
}
}
@ -36,7 +39,7 @@ void CUSPARSECALL(cusparseStatus_t x)
if(x!= CUSPARSE_STATUS_SUCCESS)
{
std::cout<<"!!!!!!!!CUSPARSE EXCEPTION: "<<std::endl;
throw std::exception("CUSPARSE EXCEPTION");
throw std::runtime_error("CUSPARSE EXCEPTION");
}
}
@ -100,7 +103,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
/*template<class ElemType>
void GPUSparseMatrix<ElemType>::Resize(const size_t nR, const size_t nC)
{
if (!this->IsEmpty())
if (!IsEmpty())
{
Clear();
}
@ -108,7 +111,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_numCols=nC;
m_nz=0;
m_elemSizeAllocated=m_nz;
m_pArray = nullptr;
m_pArray = NULL;
}*/
// PrepareDevice - Setup the correct cuda context for an operation
@ -166,7 +169,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
void GPUSparseMatrix<ElemType>::SetValue(const GPUSparseMatrix<ElemType>& deepCopy)
{
if (!this->IsEmpty())
if (!IsEmpty())
{
Clear();
}
@ -178,7 +181,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
GPUMatrix<ElemType> GPUSparseMatrix<ElemType>::CopyToDenseMatrix()
{
GPUMatrix<ElemType> res;
if (this->IsEmpty())
if (IsEmpty())
return res;
PrepareDevice();
@ -216,7 +219,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
void GPUSparseMatrix<ElemType>::SetValue(const GPUMatrix<ElemType>& denseMatrix)
{
if (!this->IsEmpty())
if (!IsEmpty())
{
Clear();
}
@ -278,7 +281,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
this->SetMatrixName(denseMatrix.GetMatrixName());
SetMatrixName(denseMatrix.GetMatrixName());
}
template<class ElemType>
@ -346,9 +349,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (m_matrixName!=NULL)
{
delete[] m_matrixName;
m_matrixName = nullptr;
m_matrixName = NULL;
}
if(m_format == MatrixFormat::matrixFormatSparseCSC || m_format == MatrixFormat::matrixFormatSparseCSR)
if(m_format == matrixFormatSparseCSC || m_format == matrixFormatSparseCSR)
{
if(m_val != NULL)
CUDACALL(cudaFree(m_val));
@ -357,7 +360,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if(m_pb != NULL)
CUDACALL(cudaFree(m_pb));
}
else if (m_format == MatrixFormat::matrixFormatSparseBlockCol || m_format == MatrixFormat::matrixFormatSparseBlockRow)
else if (m_format == matrixFormatSparseBlockCol || m_format == matrixFormatSparseBlockRow)
{
if(m_blockVal != NULL)
CUDACALL(cudaFree(m_blockVal));
@ -394,7 +397,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (reallocate)
{
if (!OwnBuffer())
throw runtime_error("cannot reallocate a buffer not owned by the matrix");
throw std::runtime_error("cannot reallocate a buffer not owned by the matrix");
if (m_pArray!=NULL)
CUDACALL(cudaFree(m_pArray));
CUDACALL(cudaMalloc((void **)&m_pArray,BufferSize()));
@ -422,7 +425,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_nz = 0;
m_matrixName = NULL;
if(m_format == MatrixFormat::matrixFormatSparseCSC || m_format == MatrixFormat::matrixFormatSparseCSR)
if(m_format == matrixFormatSparseCSC || m_format == matrixFormatSparseCSR)
{
m_colIdx = -1;
m_val = NULL;
@ -434,7 +437,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_block2Id = NULL;
m_block2UniqId = NULL;
}
else if (m_format == MatrixFormat::matrixFormatSparseBlockCol || m_format == MatrixFormat::matrixFormatSparseBlockRow)
else if (m_format == matrixFormatSparseBlockCol || m_format == matrixFormatSparseBlockRow)
{
m_blockSize = 0;
m_blockVal = NULL;
@ -445,7 +448,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>::GPUSparseMatrix(const MatrixFormat format, const int deviceId)
{
if(format != MatrixFormat::matrixFormatSparseCSC && format != MatrixFormat::matrixFormatSparseCSR && format != MatrixFormat::matrixFormatSparseBlockCol && format != MatrixFormat::matrixFormatSparseBlockRow)
if(format != matrixFormatSparseCSC && format != matrixFormatSparseCSR && format != matrixFormatSparseBlockCol && format != matrixFormatSparseBlockRow)
{
throw std::logic_error("GPUSparseMatrix: unsupported sparse matrix format");
}
@ -457,7 +460,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
ElemType* GPUSparseMatrix<ElemType>::BufferPointer() const
{
if(m_format == MatrixFormat::matrixFormatSparseCSC || m_format == MatrixFormat::matrixFormatSparseCSR)
if(m_format == matrixFormatSparseCSC || m_format == matrixFormatSparseCSR)
{
return m_val;
}
@ -477,7 +480,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if(m_elemSizeAllocated < size)
{
m_elemSizeAllocated = size;
if(m_format == MatrixFormat::matrixFormatSparseCSC || m_format == MatrixFormat::matrixFormatSparseCSR)
if(m_format == matrixFormatSparseCSC || m_format == matrixFormatSparseCSR)
{
if(m_val != NULL)
CUDACALL(cudaFree(m_val));
@ -497,14 +500,14 @@ namespace Microsoft { namespace MSR { namespace CNTK {
PrepareDevice();
CUDACALL(cudaMalloc((void **)&m_val,sizeof(ElemType)*size));
CUDACALL(cudaMalloc((void **)&m_row,sizeof(size_t)*size));
int len = m_format == MatrixFormat::matrixFormatSparseCSC ? numCols : numRows;
int len = m_format == matrixFormatSparseCSC ? numCols : numRows;
CUDACALL(cudaMalloc((void **)&m_pb,sizeof(size_t)*(len+1)));
CUDACALL(cudaMalloc((void **)&m_rowIdx,sizeof(size_t)*size));
CUDACALL(cudaMalloc((void **)&m_col,sizeof(size_t)*size));
CUDACALL(cudaMalloc((void **)&m_block2Id,sizeof(size_t)*(numCols*2)));
CUDACALL(cudaMalloc((void **)&m_block2UniqId,sizeof(size_t)*(numCols*2)));
}
else if(m_format == MatrixFormat::matrixFormatSparseBlockCol || m_format == MatrixFormat::matrixFormatSparseBlockRow)
else if(m_format == matrixFormatSparseBlockCol || m_format == matrixFormatSparseBlockRow)
{
if(m_blockVal != NULL)
CUDACALL(cudaFree(m_blockVal));
@ -535,7 +538,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
void GPUSparseMatrix<ElemType>::SetMatrixFromCSCFormat(size_t *h_row, size_t *h_rowIdx, size_t size, size_t blockSize)
{
if(m_format != MatrixFormat::matrixFormatSparseCSC)
if(m_format != matrixFormatSparseCSC)
{
throw std::logic_error("CPUSparseMatrix: unsupported SetValue() call.");
}
@ -556,7 +559,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
void GPUSparseMatrix<ElemType>::SetMatrixFromLabelAndClass(size_t *h_row, size_t *h_block2Id, size_t *h_block2UniqId, size_t labelSize, size_t expandedSize, size_t blockSize)
{
if(m_format != MatrixFormat::matrixFormatSparseCSC)
if(m_format != matrixFormatSparseCSC)
{
throw std::logic_error("CPUSparseMatrix: unsupported SetValue() call.");
}
@ -584,7 +587,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
if (lhs.GetComputeDeviceId()!=rhs.GetComputeDeviceId()||(lhs.GetComputeDeviceId()!=c.GetComputeDeviceId()))
throw std::exception("MultiplyAndWeightedAddStD: All matrices must be on the same GPU");
throw std::runtime_error("MultiplyAndWeightedAddStD: All matrices must be on the same GPU");
if (lhs.IsEmpty() || rhs.IsEmpty())
throw std::logic_error("LeftMultiplyAndAdd: one of the input matrix is empty.");
@ -653,7 +656,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
const GPUSparseMatrix<ElemType>& rhs, const bool transposeB, GPUSparseMatrix<ElemType>& c)
{
if (lhs.GetComputeDeviceId()!=rhs.GetComputeDeviceId())
throw std::exception("GPUSparseMatrix::MultiplyAndAdd: All matrices must be on the same GPU");
throw std::runtime_error("GPUSparseMatrix::MultiplyAndAdd: All matrices must be on the same GPU");
int m = transposeA? (int)lhs.GetNumCols(): (int)lhs.GetNumRows();
int k = transposeA? (int)lhs.GetNumRows(): (int)lhs.GetNumCols();
@ -714,12 +717,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
void GPUSparseMatrix<ElemType>::ScaleAndAdd(const ElemType alpha, const GPUSparseMatrix<ElemType>& lhs, GPUMatrix<ElemType>& rhs)
{
if (lhs.GetComputeDeviceId()!=rhs.GetComputeDeviceId())
throw std::exception("GPUSparseMatrix::ScaleAndAdd: All matrices must be on the same GPU");
throw std::runtime_error("GPUSparseMatrix::ScaleAndAdd: All matrices must be on the same GPU");
if (lhs.m_format == MatrixFormat::matrixFormatSparseBlockCol || lhs.m_format == MatrixFormat::matrixFormatSparseBlockRow)
if (lhs.m_format == matrixFormatSparseBlockCol || lhs.m_format == matrixFormatSparseBlockRow)
{
size_t len = (lhs.m_format == MatrixFormat::matrixFormatSparseBlockCol) ? lhs.GetNumRows(): lhs.GetNumCols();
bool blockCol = (lhs.m_format == MatrixFormat::matrixFormatSparseBlockCol);
size_t len = (lhs.m_format == matrixFormatSparseBlockCol) ? lhs.GetNumRows(): lhs.GetNumCols();
bool blockCol = (lhs.m_format == matrixFormatSparseBlockCol);
cudaEvent_t done = nullptr;
CUDACALL(cudaEventCreate(&done));
@ -738,7 +741,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else
{
throw std::exception("GPUSparseMatrix:: ScaleAndAdd() Not implemented");
throw std::runtime_error("GPUSparseMatrix:: ScaleAndAdd() Not implemented");
}
}
@ -756,7 +759,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
int deviceId = a.GetComputeDeviceId();
if (weight.GetComputeDeviceId()!=deviceId || label.GetComputeDeviceId()!=deviceId || cls.GetComputeDeviceId()!=deviceId
|| idx2cls.GetComputeDeviceId()!=deviceId || etp.GetComputeDeviceId()!=deviceId )
throw std::exception("GPUSparseMatrix:: ClassEntropy() All matrices must be on the same GPU");
throw std::runtime_error("GPUSparseMatrix:: ClassEntropy() All matrices must be on the same GPU");
size_t nC = cls.GetNumCols();
size_t nV = label.GetNumRows() - nC;
@ -831,7 +834,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
int deviceId = error.GetComputeDeviceId();
if (weight.GetComputeDeviceId()!=deviceId || grd.GetComputeDeviceId()!=deviceId )
throw std::exception("GPUSparseMatrix::ClassEntropyGradientOfInput() All matrices must be on the same GPU");
throw std::runtime_error("GPUSparseMatrix::ClassEntropyGradientOfInput() All matrices must be on the same GPU");
grd.SetValue((ElemType)0);
cudaEvent_t done = nullptr;
@ -858,7 +861,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
int deviceId = error.GetComputeDeviceId();
if (input.GetComputeDeviceId()!=deviceId || label.GetComputeDeviceId()!=deviceId || cls.GetComputeDeviceId()!=deviceId || idx2cls.GetComputeDeviceId()!=deviceId || grd.GetComputeDeviceId()!=deviceId )
throw std::exception("GPUSparseMatrix::ClassEntropyGradientOfWeight() All matrices must be on the same GPU");
throw std::runtime_error("GPUSparseMatrix::ClassEntropyGradientOfWeight() All matrices must be on the same GPU");
grd.SetFormat(matrixFormatSparseBlockRow);
size_t nz = label.m_blockSize * grd.GetNumCols();
@ -898,20 +901,20 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::InplaceTruncate (const ElemType threshold)
{
if(m_format == MatrixFormat::matrixFormatSparseBlockCol || m_format == MatrixFormat::matrixFormatSparseBlockRow)
if(m_format == matrixFormatSparseBlockCol || m_format == matrixFormatSparseBlockRow)
{
long N=(long)GetNZElements();
int blocksPerGrid =(int)ceil(N*1.0/threadsPerBlock);
cudaEvent_t done = nullptr;
CUDACALL(cudaEventCreate(&done));
_inplaceTruncate<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_blockVal,threshold,N);
_inplaceTruncate<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_blockVal,threshold,N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
CUDACALL(cudaEventDestroy(done));
}
else
{
throw std::exception("GPUSparseMatrix:: InplaceTruncate() only support block based sparse matrix");
throw std::runtime_error("GPUSparseMatrix:: InplaceTruncate() only support block based sparse matrix");
}
return *this;
}
@ -922,11 +925,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
if (c.IsEmpty())
{
c.Resize(this->GetNumRows(), this->GetNumCols());
c.Resize(GetNumRows(), GetNumCols());
c.SetValue(0.0);
}
if(m_format == MatrixFormat::matrixFormatSparseBlockCol || m_format == MatrixFormat::matrixFormatSparseBlockRow)
if(m_format == matrixFormatSparseBlockCol || m_format == matrixFormatSparseBlockRow)
{
size_t blocksPerGrid = m_blockSize;
bool isBlockCol = (m_format == MatrixFormat::matrixFormatSparseBlockCol);
@ -947,7 +950,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else
{
throw std::exception("GPUSparseMatrix:: NormalGrad() only support block sparse format");
throw std::runtime_error("GPUSparseMatrix:: NormalGrad() only support block sparse format");
}
}
@ -960,7 +963,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
const GPUMatrix<ElemType>& b, ElemType beta, GPUMatrix<ElemType>& c)
{
if (a.GetComputeDeviceId()!=b.GetComputeDeviceId()||(b.GetComputeDeviceId()!=a.GetComputeDeviceId()))
throw std::exception("MultiplyAndWeightedAddStD: All matrices must be on the same GPU");
throw std::runtime_error("MultiplyAndWeightedAddStD: All matrices must be on the same GPU");
a.PrepareDevice();
cusparseHandle_t cusparseHandle = 0;
CUSPARSECALL(cusparseCreate(&cusparseHandle));
@ -1054,7 +1057,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (cSize >= rowBufferRequired && c.NzLocation() != NULL && canReuseBuffer)
{
// determine the final location if we reuse the buffer
csrRowPtrC = (int*)((byte*)c.NzLocation() + nzBufSize);
csrRowPtrC = (int*)((char*)c.NzLocation() + nzBufSize);
}
else
{
@ -1105,7 +1108,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
void GPUSparseMatrix<ElemType>::Multiply(const GPUSparseMatrix<ElemType>& S1, bool transposeS1, const GPUSparseMatrix<ElemType>& S2, bool transposeS2, GPUSparseMatrix<ElemType> &c)
{
if (S1.GetComputeDeviceId()!=S2.GetComputeDeviceId())
throw std::exception("Sparse matrix multiply: both matrices must be on the same device");
throw std::runtime_error("Sparse matrix multiply: both matrices must be on the same device");
S1.PrepareDevice();
cusparseHandle_t cusparseHandle = 0;
@ -1122,7 +1125,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
int k = int(transposeS1 ? S1.GetNumRows() : S1.GetNumCols());
int l = int(transposeS2 ? S2.GetNumCols() : S2.GetNumRows());
if (k!=l)
throw std::exception("Sparse matrix multiply: dimensionality mismatch");
throw std::runtime_error("Sparse matrix multiply: dimensionality mismatch");
int nnzA = (int)S1.GetNZElements();
int nnzB = (int)S2.GetNZElements();
@ -1170,9 +1173,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
void GPUSparseMatrix<ElemType>::ScaleAndAdd(ElemType alpha,const GPUSparseMatrix<ElemType>& a, ElemType beta, const GPUSparseMatrix<ElemType>& b, GPUSparseMatrix<ElemType>& c)
{
if (a.GetNumCols()!=b.GetNumCols() || a.GetNumRows()!=b.GetNumRows())
throw new std::exception("Dimensions mismatch in ScaleAndAdd");
throw new std::runtime_error("Dimensions mismatch in ScaleAndAdd");
if (a.GetComputeDeviceId()!=b.GetComputeDeviceId())
throw new std::exception("ScaleAndAdd: matrices must be on the same device");
throw new std::runtime_error("ScaleAndAdd: matrices must be on the same device");
int m = (int)a.GetNumRows();
int n = (int)a.GetNumCols();
@ -1221,7 +1224,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (a.GetNumRows()!=b.GetNumRows()||a.GetNumRows()!=c.GetNumRows()||a.GetNumCols()!=b.GetNumCols()||a.GetNumCols()!=c.GetNumCols())
throw std::logic_error("ScaleAndAdd: dimension mismatch");
if (a.GetComputeDeviceId()!=b.GetComputeDeviceId()||a.GetComputeDeviceId()!=c.GetComputeDeviceId())
throw std::exception("ScaleAndAdd: matrices must be on the same device");
throw std::runtime_error("ScaleAndAdd: matrices must be on the same device");
b.PrepareDevice();
//copy b to c
CUDACALL(cudaMemcpy(c.BufferPointer(),b.BufferPointer(),sizeof(ElemType)*b.GetNumElements(),cudaMemcpyDeviceToDevice));
@ -1280,7 +1283,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
a.PrepareDevice();
long N=(long)a.GetNZElements();
int blocksPerGrid =(int)ceil(1.0*N/threadsPerBlock);
_elementWisePowerOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(alpha,a.NzLocation(),c.NzLocation(),N);
_elementWisePowerOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(alpha,a.NzLocation(),c.NzLocation(),N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
}
@ -1290,7 +1293,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
ElemType GPUSparseMatrix<ElemType>::InnerProductOfMatrices(const GPUSparseMatrix<ElemType>& a, const GPUMatrix<ElemType>& b)
{
if (a.GetComputeDeviceId()!=b.GetComputeDeviceId())
throw std::exception("a and b must be on the same device");
throw std::runtime_error("a and b must be on the same device");
//This implementation requires additional memory
//need to put a in ColumnMajor format
@ -1330,7 +1333,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
//int* h_vectArray= new int[a.m_nz];
int blocksPerGrid =(int)ceil(1.0*M/threadsPerBlock);
CUDACALL(cudaEventCreate(&done));
_getSparseVectorRepresntationForMatrix<ElemType><<<blocksPerGrid,threadsPerBlock>>>(cscColPtrA,cscRowIndA,vectArray,M,N);
_getSparseVectorRepresntationForMatrix<ElemType><<<blocksPerGrid,threadsPerBlock>>>(cscColPtrA,cscRowIndA,vectArray,M,N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
CUDACALL(cudaEventDestroy(done));
@ -1381,7 +1384,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
CUDACALL(cudaMemcpy(d_res,res,sizeof(long)*3,cudaMemcpyHostToDevice));
int blocksPerGrid =(int)ceil(1.0*a.GetNZElements()/threadsPerBlock);
_areEqual<ElemType><<<blocksPerGrid,threadsPerBlock>>>(a.NzLocation(),b.NzLocation(),(long)a.GetNZElements(),threshold,d_res);
_areEqual<ElemType><<<blocksPerGrid,threadsPerBlock>>>(a.NzLocation(),b.NzLocation(),(long)a.GetNZElements(),threshold,d_res);
_areEqual<int><<<blocksPerGrid,threadsPerBlock>>>(a.ColLocation(),b.ColLocation(),(long)a.GetNZElements(),(int)threshold,d_res+1);
blocksPerGrid =(int)ceil((1.0*a.GetNumRows()+1.0)/threadsPerBlock);
_areEqual<int><<<blocksPerGrid,threadsPerBlock>>>(a.RowLocation(),b.RowLocation(),(long)a.GetNumRows()+1,(int)threshold,d_res+2);
@ -1489,7 +1492,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::operator^=(ElemType alpha)
{
auto& us = *this;
GPUSparseMatrix<ElemType>& us = *this;
ElementWisePower(alpha, us, us);
return us;
}
@ -1506,7 +1509,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::operator*=(ElemType alpha)
{
auto& us = *this;
GPUSparseMatrix<ElemType>& us = *this;
if (alpha!=1)
Scale(alpha,us);
return us;
@ -1531,9 +1534,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType> GPUSparseMatrix<ElemType>::Transpose() const
{
int m = (int)this->GetNumRows();
int n = (int)this->GetNumCols();
int nnz = (int)this->GetNZElements();
int m = (int)GetNumRows();
int n = (int)GetNumCols();
int nnz = (int)GetNZElements();
cusparseAction_t cpVals = CUSPARSE_ACTION_NUMERIC;
cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
@ -1549,12 +1552,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
CUDACALL(cudaEventCreate(&done));
if (sizeof(ElemType)==sizeof(float))
{
CUSPARSECALL(cusparseScsr2csc(cusparseHandle,m,n,nnz,reinterpret_cast<const float*>(this->NzLocation()),this->CompressedIndexLocation(),this->IndexLocation(),
CUSPARSECALL(cusparseScsr2csc(cusparseHandle,m,n,nnz,reinterpret_cast<const float*>(NzLocation()),CompressedIndexLocation(),IndexLocation(),
reinterpret_cast<float*>(c.NzLocation()),c.IndexLocation(),c.CompressedIndexLocation(),cpVals,idxBase));
}
else
{
CUSPARSECALL(cusparseDcsr2csc(cusparseHandle,m,n,nnz,reinterpret_cast<const double*>(this->NzLocation()),this->CompressedIndexLocation(),this->IndexLocation(),
CUSPARSECALL(cusparseDcsr2csc(cusparseHandle,m,n,nnz,reinterpret_cast<const double*>(NzLocation()),CompressedIndexLocation(),IndexLocation(),
reinterpret_cast<double*>(c.NzLocation()),c.IndexLocation(),c.CompressedIndexLocation(),cpVals,idxBase));
}
CUDACALL(cudaEventRecord(done));
@ -1580,19 +1583,19 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
void GPUSparseMatrix<ElemType>::InplaceTranspose()
{
if (this->IsEmpty())
if (IsEmpty())
return;
// transfer converted block over to this pointer
*this = std::move(this->Transpose());
*this = std::move(Transpose());
}
template<class ElemType>
ElemType GPUSparseMatrix<ElemType>::SumOfAbsElements() const
{
if (this->IsEmpty())
if (IsEmpty())
throw std::logic_error("SumOfAbsElements: Matrix is empty");
cublasHandle_t cuHandle = GPUMatrix<ElemType>::GetCublasHandle(this->GetComputeDeviceId());
cublasHandle_t cuHandle = GPUMatrix<ElemType>::GetCublasHandle(GetComputeDeviceId());
if (sizeof(ElemType)==sizeof(float))
{
float res=0;
@ -1610,7 +1613,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
ElemType GPUSparseMatrix<ElemType>::SumOfElements() const
{
if (this->IsEmpty())
if (IsEmpty())
throw std::logic_error("SumOfElements: Matrix is empty");
PrepareDevice();
@ -1618,7 +1621,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
ElemType h_sum;
CUDACALL(cudaMalloc((void**)&d_sum,sizeof(ElemType)));
//WARNING: THIS kernel is not the most efficient way!
_reductionSum<ElemType><<<1,1024>>>(m_pArray,d_sum,(LONG64)this->GetNZElements());
_reductionSum<ElemType><<<1,1024>>>(m_pArray,d_sum,(LONG64)GetNZElements());
CUDACALL(cudaMemcpy(&h_sum,d_sum,sizeof(ElemType),cudaMemcpyDeviceToHost));
CUDACALL(cudaFree(d_sum));
return h_sum;
@ -1635,7 +1638,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
ElemType h_sum=0;
CUDACALL(cudaMalloc((void**)&d_sum,sizeof(ElemType)));
//WARNING: THIS kernel is not the most efficient way!
_reductionSum2<ElemType><<<1,1024>>>(m_pArray,d_sum,(int)this->GetNZElements());
_reductionSum2<ElemType><<<1,1024>>>(m_pArray,d_sum,(int)GetNZElements());
CUDACALL(cudaMemcpy(&h_sum,d_sum,sizeof(ElemType),cudaMemcpyDeviceToHost));
CUDACALL(cudaFree(d_sum));
if (sizeof(ElemType)==sizeof(float))
@ -1654,7 +1657,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
ElemType h_maxAbs=0;
CUDACALL(cudaMalloc((void**)&d_maxAbs,sizeof(ElemType)));
//WARNING: THIS kernel is not the most efficient way!
_reductionMatrixNormInf<ElemType><<<1,1024>>>(m_pArray,d_maxAbs,(int)this->GetNZElements());
_reductionMatrixNormInf<ElemType><<<1,1024>>>(m_pArray,d_maxAbs,(int)GetNZElements());
CUDACALL(cudaMemcpy(&h_maxAbs,d_maxAbs,sizeof(ElemType),cudaMemcpyDeviceToHost));
CUDACALL(cudaFree(d_maxAbs));
if (sizeof(ElemType)==sizeof(float))
@ -1668,7 +1671,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
if (IsEmpty())
throw std::logic_error("MatrixNorm1: Matrix is empty.");
return this->SumOfAbsElements();
return SumOfAbsElements();
}
#pragma endregion Member BLAS Functions
@ -1685,7 +1688,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
int blocksPerGrid =(int)ceil(1.0*N/threadsPerBlock);
cudaEvent_t done = nullptr;
CUDACALL(cudaEventCreate(&done));
_elemInverse<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
_elemInverse<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
return *this;
@ -1694,8 +1697,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::AssignElementInverseOf (const GPUSparseMatrix<ElemType>& a)
{
this->SetValue(a);
return this->ElementInverse();
SetValue(a);
return ElementInverse();
}
template<class ElemType>
@ -1708,8 +1711,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::AssignSigmoidOf (const GPUSparseMatrix<ElemType>& a)
{
this->SetValue(a);
this->InplaceSigmoid();
SetValue(a);
InplaceSigmoid();
return *this;
}
@ -1723,8 +1726,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::AssignLinearRectifierDerivativeOf (const GPUSparseMatrix<ElemType>& a)
{
this->SetValue(a);
this->InplaceLinearRectifierDerivative();
SetValue(a);
InplaceLinearRectifierDerivative();
return *this;
}
@ -1738,8 +1741,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::AssignTanhOf (const GPUSparseMatrix<ElemType>& a)
{
this->SetValue(a);
this->InplaceTanh();
SetValue(a);
InplaceTanh();
return *this;
}
@ -1753,8 +1756,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::AssignSqrtOf (const GPUSparseMatrix<ElemType>& a)
{
this->SetValue(a);
this->InplaceSqrt();
SetValue(a);
InplaceSqrt();
return *this;
}
@ -1768,8 +1771,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::AssignExpOf (const GPUSparseMatrix<ElemType>& a)
{
this->SetValue(a);
this->InplaceExp();
SetValue(a);
InplaceExp();
return *this;
}
@ -1783,8 +1786,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::AssignLogOf (const GPUSparseMatrix<ElemType>& a)
{
this->SetValue(a);
this->InplaceLog();
SetValue(a);
InplaceLog();
return *this;
}
@ -1798,8 +1801,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
GPUSparseMatrix<ElemType>& GPUSparseMatrix<ElemType>::AssignAbsOf (const GPUSparseMatrix<ElemType>& a)
{
this->SetValue(a);
this->InplaceAbs();
SetValue(a);
InplaceAbs();
return *this;
}
@ -1812,7 +1815,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
int blocksPerGrid =(int)ceil(N*1.0/threadsPerBlock);
cudaEvent_t done = nullptr;
CUDACALL(cudaEventCreate(&done));
_inplaceTruncateBottom<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,threshold,N);
_inplaceTruncateBottom<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,threshold,N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
return *this;
@ -1833,7 +1836,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
int blocksPerGrid =(int)ceil(N*1.0/threadsPerBlock);
cudaEvent_t done = nullptr;
CUDACALL(cudaEventCreate(&done));
_assignTruncateBottom<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,a.NzLocation(),threshold,N);
_assignTruncateBottom<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,a.NzLocation(),threshold,N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
return *this;
@ -1848,7 +1851,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
int blocksPerGrid =(int)ceil(N*1.0/threadsPerBlock);
cudaEvent_t done = nullptr;
CUDACALL(cudaEventCreate(&done));
_inplaceTruncateTop<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,threshold,N);
_inplaceTruncateTop<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,threshold,N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
return *this;
@ -1869,7 +1872,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
int blocksPerGrid =(int)ceil(N*1.0/threadsPerBlock);
cudaEvent_t done = nullptr;
CUDACALL(cudaEventCreate(&done));
_assignTruncateTop<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,a.NzLocation(),threshold,N);
_assignTruncateTop<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,a.NzLocation(),threshold,N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
return *this;
@ -1884,7 +1887,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
int blocksPerGrid =(int)ceil(N*1.0/threadsPerBlock);
cudaEvent_t done = nullptr;
CUDACALL(cudaEventCreate(&done));
_setToZeroIfAbsLessThan<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,threshold,N);
_setToZeroIfAbsLessThan<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,threshold,N);
CUDACALL(cudaEventRecord(done));
CUDACALL(cudaEventSynchronize(done));
return *this;
@ -1978,22 +1981,22 @@ namespace Microsoft { namespace MSR { namespace CNTK {
switch (kind)
{
case 0:
_inplaceSigmoidOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
_inplaceSigmoidOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
break;
case 1:
_inplaceTanhOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
_inplaceTanhOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
break;
case 2:
_inplaceSqrtOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
_inplaceSqrtOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
break;
case 3:
_inplaceExpOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
_inplaceExpOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
break;
case 4:
_inplaceLogOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
_inplaceLogOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
break;
case 5:
_inplaceAbsOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
_inplaceAbsOnCuda<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
break;
case 6:
_inplaceLinRectDerivative<ElemType><<<blocksPerGrid,threadsPerBlock>>>(m_pArray,N);
@ -2012,7 +2015,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_format=matrixFormatSparseCSR;
m_externalBuffer = false;
if (OwnBuffer() && m_pArray != nullptr)
if (OwnBuffer() && m_pArray != NULL)
{
CUDACALL(cudaFree(m_pArray));
}
@ -2031,12 +2034,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
void GPUSparseMatrix<ElemType>::GetMatrixFromCSRFormat(int*& h_CSRRow, int*& h_Col, ElemType*& h_Val, size_t &nz, size_t &numRows, size_t &numCols) const
{
if (h_CSRRow!=NULL || h_Col!=NULL || h_Val!=NULL)
throw std::exception("Passed pointers must be NULL");
nz = this->GetNZElements();
numRows = this->GetNumRows();
numCols = this->GetNumCols();
throw std::runtime_error("Passed pointers must be NULL");
nz = GetNZElements();
numRows = GetNumRows();
numCols = GetNumCols();
if (this->IsEmpty())
if (IsEmpty())
return;
else
{
@ -2063,7 +2066,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
size_t elsize;
stream>>elsize;
if (sizeof(ElemType)!=elsize)
throw std::exception("Template argument size doesn't match those in file");
throw std::runtime_error("Template argument size doesn't match those in file");
std::wstring matrixName;
// save off the buffer size being passed in
@ -2151,7 +2154,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
// What we would like to do here, is transfer to CPUSparse and save, do that when the format is the same
byte* hostBuffer = new byte[us.BufferSize()];
char* hostBuffer = new char[us.BufferSize()]; // TODO: use std::shared_ptr
GPUSparseMatrix<ElemType> hostSide(us.GetNumRows(), us.GetNumCols(), us.NzCount(), (ElemType*)hostBuffer, us.GetFormat());
CUDACALL(cudaMemcpy(hostBuffer, us.NzLocation(),us.BufferSize(),cudaMemcpyDeviceToHost));
@ -2186,4 +2189,4 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template MATH_API File& operator<<(File& stream, const GPUSparseMatrix<float>& us);
template MATH_API File& operator<<(File& stream, const GPUSparseMatrix<double>& us);
}}}
}}}

14
Math/Math/GPUSparseMatrix.cuh
Просмотреть файл

@ -19,6 +19,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
class MATH_API GPUSparseMatrix : public BaseMatrix<ElemType>
{
typedef BaseMatrix<ElemType> B; using B::m_numRows; using B::m_numCols; using B::m_pArray; using B::m_elemSizeAllocated; using B::m_nz; using B::m_format; // without this, base members would require to use thi-> in GCC
private:
void ZeroInit();
void Init();
@ -39,7 +40,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
GPUSparseMatrix(const GPUSparseMatrix<ElemType>&);
GPUSparseMatrix(const GPUMatrix<ElemType>&);
#ifndef LINUX
GPUSparseMatrix(GPUSparseMatrix<ElemType>&&);
#endif /* LINUX */
~GPUSparseMatrix();
public:
void Resize(const size_t numRows, const size_t numCols, size_t size = 0);
@ -89,7 +92,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
GPUMatrix<ElemType> CopyToDenseMatrix();
GPUSparseMatrix<ElemType>& operator=(const GPUSparseMatrix<ElemType>& deepCopy);
#ifndef LINUX
GPUSparseMatrix<ElemType>& operator=(GPUSparseMatrix<ElemType>&& moveFrom);
#endif /* LINUX */
GPUSparseMatrix<ElemType> operator+ (const GPUSparseMatrix<ElemType>& a) const;
GPUSparseMatrix<ElemType> operator- (const GPUSparseMatrix<ElemType>& a) const;
GPUSparseMatrix<ElemType>& operator^= (ElemType alpha); //element-wise power
@ -199,10 +204,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
public:
template <class ElemType>
friend MATH_API File& operator>>(File& stream, GPUSparseMatrix<ElemType>& us);
template <class ElemType>
friend MATH_API File& operator<<(File& stream, const GPUSparseMatrix<ElemType>& us);
// See: http://stackoverflow.com/questions/4660123/overloading-friend-operator-for-template-class/4661372#4661372
template <class ElemTypeDummy>
friend MATH_API File& operator>>(File& stream, GPUSparseMatrix<ElemTypeDummy>& us);
template <class ElemTypeDummy>
friend MATH_API File& operator<<(File& stream, const GPUSparseMatrix<ElemTypeDummy>& us);
bool m_legacy;
int m_colIdx; //used to SetValue()

6
Math/Math/InstantiateTemplates.cpp
Просмотреть файл

@ -8,8 +8,8 @@
#include "CPUMatrix.cpp"
#include "Matrix.cpp"
#include "..\..\common\include\fileutil.cpp"
#include "..\..\common\include\File.cpp"
#include "..\..\Common\Include\fileutil.cpp"
#include "..\..\Common\Include\File.cpp"
//don't treat it as sample code. some code does not make sense
//only used to force compiler to build the code
@ -343,4 +343,4 @@ namespace Microsoft { namespace MSR { namespace CNTK {
CallEverythingInMatrix<double>();
}
#pragma endregion instantiate all classes
}}}
}}}

33
Math/Math/Makefile Normal file
Просмотреть файл

@ -0,0 +1,33 @@
CSOURCES = CPUMatrix.cpp CPUSparseMatrix.cpp Matrix.cpp
OBJECTS = CPUMatrix.o CPUSparseMatrix.o Matrix.o \
GPUMatrixCUDAKernels.o GPUMatrix.o \
GPUWatcher.o GPUSparseMatrix.o
INCLUDES = -I../../Common/Include -I/opt/acml5.3.1/gfortran64_mp_int64/include
DEPS =
CFLAGS = $(INCLUDES) \
-D BASETYPES_NO_UNSAFECRTOVERLOAD -DBASETYPES_NO_STRPRINTF \
-DLINUX -Wnon-template-friend -std=c++11 # -D_FILEUTIL_
NVCFLAGS = -DLINUX -D BASETYPES_NO_UNSAFECRTOVERLOAD -DBASETYPES_NO_STRPRINTF \
-I../../Common/Include -arch=compute_20 -std=c++11 # -D_FILEUTIL_
CXX = gcc
NVCC = nvcc
all: libmatrixmat.so
libmatrixmat.so: $(OBJECTS)
$(CXX) -shared $(OBJECTS) -o libmatrixmat.so
libmatrixmat.so: $(OBJECTS)
%.o: %.cpp $(DEPS)
$(CXX) -c $(CFLAGS) $< -o $@
%.o: %.cu $(DEPS)
$(NVCC) -c $(NVCFLAGS) $< -o $@

6
Math/Math/Math.vcxproj
Просмотреть файл

@ -109,7 +109,7 @@
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>NO_SYNC;WIN32;NDEBUG;_WINDOWS;_USRDLL;MATH_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<SDLCheck>true</SDLCheck>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>..\..\common\include\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<FloatingPointModel>Fast</FloatingPointModel>
<OpenMPSupport>true</OpenMPSupport>
@ -147,6 +147,9 @@
</CudaLink>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="..\..\Common\Include\basetypes.h" />
<ClInclude Include="..\..\Common\Include\File.h" />
<ClInclude Include="..\..\Common\Include\fileutil.h" />
<ClInclude Include="CommonMatrix.h" />
<ClInclude Include="CPUMatrix.h" />
<None Include="GPUWatcher.cuh">
@ -179,6 +182,7 @@
<CudaCompile Include="GPUWatcher.cu">
<FileType>CppCode</FileType>
</CudaCompile>
<ClCompile Include="GPUDummy.cpp" />
<ClCompile Include="Matrix.cpp" />
<ClCompile Include="stdafx.cpp">
<PrecompiledHeader>Create</PrecompiledHeader>

84
Math/Math/Math.vcxproj.filters Normal file
Просмотреть файл

@ -0,0 +1,84 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<CudaCompile Include="GPUMatrix.cu">
<Filter>GPU</Filter>
</CudaCompile>
<CudaCompile Include="GPUMatrixCUDAKernels.cu">
<Filter>GPU</Filter>
</CudaCompile>
<CudaCompile Include="GPUSparseMatrix.cu">
<Filter>GPU</Filter>
</CudaCompile>
<CudaCompile Include="GPUWatcher.cu">
<Filter>GPU</Filter>
</CudaCompile>
</ItemGroup>
<ItemGroup>
<ClCompile Include="dllmain.cpp" />
<ClCompile Include="Matrix.cpp" />
<ClCompile Include="stdafx.cpp" />
<ClCompile Include="..\..\Common\File.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\fileutil.cpp">
<Filter>Common</Filter>
</ClCompile>
<ClCompile Include="CPUMatrix.cpp">
<Filter>CPU</Filter>
</ClCompile>
<ClCompile Include="CPUSparseMatrix.cpp">
<Filter>CPU</Filter>
</ClCompile>
<ClCompile Include="GPUDummy.cpp">
<Filter>GPU</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="CommonMatrix.h" />
<ClInclude Include="Helpers.h" />
<ClInclude Include="Matrix.h" />
<ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" />
<ClInclude Include="..\..\Common\Include\File.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\fileutil.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Include\basetypes.h">
<Filter>Common\Include</Filter>
</ClInclude>
<ClInclude Include="CPUMatrix.h">
<Filter>CPU</Filter>
</ClInclude>
<ClInclude Include="CPUSparseMatrix.h">
<Filter>CPU</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="GPUMatrix.cuh">
<Filter>GPU</Filter>
</None>
<None Include="GPUSparseMatrix.cuh">
<Filter>GPU</Filter>
</None>
<None Include="GPUWatcher.cuh">
<Filter>GPU</Filter>
</None>
</ItemGroup>
<ItemGroup>
<Filter Include="Common">
<UniqueIdentifier>{4d07e945-74fb-48fa-aa63-23f3a7763789}</UniqueIdentifier>
</Filter>
<Filter Include="Common\Include">
<UniqueIdentifier>{51b468dd-7e8a-4be8-ae6f-5e3f3d752b88}</UniqueIdentifier>
</Filter>
<Filter Include="CPU">
<UniqueIdentifier>{94878fe0-ecce-4868-b3be-2199f4ee7507}</UniqueIdentifier>
</Filter>
<Filter Include="GPU">
<UniqueIdentifier>{cc9a219d-d8ab-484a-b253-fd2a29ad7c7c}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>

100
Math/Math/Matrix.cpp
Просмотреть файл

@ -8,11 +8,17 @@
#include "fileutil.h"
#include "Matrix.h"
#include <assert.h>
#include <math.h>
#pragma warning (disable: 4127) // conditional expression is constant; "if (sizeof(ElemType)==sizeof(float))" triggers this
#pragma warning (disable: 4239) // nonstandard extension; triggered by this pattern: "auto& second = transposeB ? b.m_GPUMatrix->Transpose() : *b.m_GPUMatrix;"
#pragma warning (disable: 4702) // unreachable code; triggered for unknown reasons
#ifndef min
#define min(a,b) (((a) < (b)) ? (a) : (b))
#endif
//before calling the following macro the current matrix location and matrix type on MatrixPointerToCheck must have been set correctly
#define DISPATCH_MATRIX_ON_FLAG(MatrixPointerToCheck, MatrixPointerToSetFlag, CPUDense, GPUDense, CPUSparse, GPUSparse) \
{ \
@ -197,7 +203,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
m_GPUSparseMatrix = (GPUSparseMatrix<ElemType>*)baseMatrix;
SetDataLocation(GPU, SPARSE);
}
}
}
else
{
@ -210,15 +216,15 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
m_GPUMatrix = (GPUMatrix<ElemType>*)baseMatrix;
SetDataLocation(GPU, DENSE);
}
}
}
m_baseMatrix = baseMatrix;
m_baseMatrix->SetArray(pArray);
}
//matrixName is used to verify that correct matrix is read.
template<class ElemType>
Matrix<ElemType>::Matrix(FILE* f, const char * matrixName, short deviceId=AUTOPLACEMATRIX, const MatrixType matrixType = DENSE)
Matrix<ElemType>::Matrix(FILE* f, const char * matrixName, short deviceId, const MatrixType matrixType)
{
if (deviceId == MANAGEDEXTERN)
throw runtime_error("Externally Managed Matrix must use the basic constructor, then SetValue()\n");
@ -256,7 +262,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
template<class ElemType>
Matrix<ElemType>::Matrix(const size_t numRows, const size_t numCols, short deviceId=AUTOPLACEMATRIX, const MatrixType matrixType = DENSE)
Matrix<ElemType>::Matrix(const size_t numRows, const size_t numCols, short deviceId, const MatrixType matrixType)
{
if (deviceId == MANAGEDEXTERN)
throw runtime_error("Externally Managed Matrix must use the basic constructor, then SetValue(), or the full constructor\n");
@ -282,19 +288,19 @@ namespace Microsoft { namespace MSR { namespace CNTK {
{
if (m_preferredDeviceId == CPUDEVICE)
{
m_CPUMatrix = new CPUMatrix<ElemType>(numRows,numCols);
m_CPUMatrix = new CPUMatrix<ElemType>(numRows, numCols);
SetDataLocation(CPU, DENSE);
}
else
{
m_GPUMatrix = new GPUMatrix<ElemType>(numRows,numCols,m_preferredDeviceId);
}
else
{
m_GPUMatrix = new GPUMatrix<ElemType>(numRows, numCols, m_preferredDeviceId);
SetDataLocation(GPU, DENSE);
}
}
}
}
template<class ElemType>
Matrix<ElemType>::Matrix(const size_t numRows, const size_t numCols, ElemType *pArray, const size_t matrixFlags, short deviceId=AUTOPLACEMATRIX, const size_t nnz=0)
Matrix<ElemType>::Matrix(const size_t numRows, const size_t numCols, ElemType *pArray, const size_t matrixFlags, short deviceId, const size_t nnz)
{
Init(deviceId);
@ -332,7 +338,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
//copy constructor, deep copy
template<class ElemType>
Matrix<ElemType>::Matrix(const Matrix<ElemType>& deepCopyFrom, short deviceId=AUTOPLACEMATRIX)
Matrix<ElemType>::Matrix(const Matrix<ElemType>& deepCopyFrom, short deviceId)
{
if (deviceId == MANAGEDEXTERN)
throw runtime_error("Externally Managed Matrix must use the basic constructor, then SetValue(), or the full constructor\n");
@ -519,7 +525,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
NOT_IMPLEMENTED;
}
else
throw std::exception("Unknown matrix type");
{
throw std::runtime_error("Unknown matrix type");
}
}
template<class ElemType>
@ -605,10 +613,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else if (GetMatrixType() == MatrixType::SPARSE)
{
NOT_IMPLEMENTED;
NOT_IMPLEMENTED;
}
else
throw std::exception("Unknown matrix type");
{
throw std::runtime_error("Unknown matrix type");
}
return slice;
}
@ -820,7 +830,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
DISPATCH_MATRIX_ON_FLAG(this,
this,
m_CPUMatrix->SetValue(*db_number.ExposePointer2Value()),
if (GetDeviceId()!=db_number.GetDeviceId()) throw std::exception("Matrix and device bound number must be on the same device");
if (GetDeviceId()!=db_number.GetDeviceId())
throw std::runtime_error("Matrix and device bound number must be on the same device");
m_GPUMatrix->SetValue(db_number.ExposePointer2Value()),
NOT_IMPLEMENTED,
NOT_IMPLEMENTED
@ -889,7 +900,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
//WARNING: what's the exact meaning of MANAGEDEXTERN here? This is not handled currently
template<class ElemType>
void Matrix<ElemType>::SetValue(const size_t numRows, const size_t numCols, ElemType *pArray, const size_t matrixFlags, int deviceId=MANAGEDEXTERN)
void Matrix<ElemType>::SetValue(const size_t numRows, const size_t numCols, ElemType *pArray, const size_t matrixFlags, int deviceId)
{
if (pArray == nullptr)
throw std::invalid_argument("Invalid pArray.");
@ -1054,7 +1065,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
//maskRate: percentage of values masked out (similar to dropout rate)
//scaleValue: which scale value to set to the left ones (unmasked items).
template<class ElemType>
void Matrix<ElemType>::SetUniformRandomMask(const ElemType maskRate, const ElemType scaleValue, unsigned long seed=USE_TIME_BASED_SEED)
void Matrix<ElemType>::SetUniformRandomMask(const ElemType maskRate, const ElemType scaleValue, unsigned long seed)
{
if (IsEmpty())
throw std::logic_error("SetUniformRandomMask: Matrix is empty.");
@ -2308,24 +2319,24 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (sizeof(ElemType)==sizeof(float))
{
if (!_finitef((float)threshold))
if (!isfinite((float)threshold))
return *this;
}
else
{
if (!_finite(threshold))
if (!isfinite(threshold))
return *this;
}
DISPATCH_MATRIX_ON_FLAG(this,
this,
this->m_CPUMatrix->InplaceTruncate(threshold),
this->m_GPUMatrix->InplaceTruncateTop(abs(threshold)); this->m_GPUMatrix->InplaceTruncateBottom(-abs(threshold)),
this->m_GPUMatrix->InplaceTruncateTop(fabs(threshold)); this->m_GPUMatrix->InplaceTruncateBottom(-fabs(threshold)),
this->m_CPUSparseMatrix->InplaceTruncate(threshold),
if(this->m_GPUSparseMatrix->m_legacy)
{
this->m_GPUSparseMatrix->InplaceTruncateTop(abs(threshold));
this->m_GPUSparseMatrix->InplaceTruncateBottom(-abs(threshold));
this->m_GPUSparseMatrix->InplaceTruncateTop(fabs(threshold));
this->m_GPUSparseMatrix->InplaceTruncateBottom(-fabs(threshold));
}
else //new GPU Sparse matrix
{
@ -2345,12 +2356,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (sizeof(ElemType)==sizeof(float))
{
if (!_finitef((float)threshold))
if (!isfinite((float)threshold))
return *this;
}
else
{
if (!_finite(threshold))
if (!isfinite(threshold))
return *this;
}
@ -2374,7 +2385,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (sizeof(ElemType)==sizeof(float))
{
if (!_finitef((float)threshold))
if (!isfinite((float)threshold))
{
(*this) = a;
return *this;
@ -2382,7 +2393,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else
{
if (!_finite(threshold))
if (!isfinite(threshold))
{
(*this) = a;
return *this;
@ -2412,12 +2423,12 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (sizeof(ElemType)==sizeof(float))
{
if (!_finitef((float)threshold))
if (!isfinite((float)threshold))
return *this;
}
else
{
if (!_finite(threshold))
if (!isfinite(threshold))
return *this;
}
@ -2440,7 +2451,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
if (sizeof(ElemType)==sizeof(float))
{
if (!_finitef((float)threshold))
if (!isfinite((float)threshold))
{
(*this) = a;
return *this;
@ -2448,7 +2459,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else
{
if (!_finite(threshold))
if (!isfinite(threshold))
{
(*this) = a;
return *this;
@ -3396,7 +3407,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else if (a.m_matrixType==MatrixType::SPARSE && b.m_matrixType==c.m_matrixType && b.m_matrixType==MatrixType::DENSE) //Sparse*Dense+Dense
{
auto& second = transposeB ? b.m_GPUMatrix->Transpose() : *b.m_GPUMatrix;
GPUMatrix<ElemType> second = transposeB ? b.m_GPUMatrix->Transpose() : *b.m_GPUMatrix;
GPUSparseMatrix<ElemType>::MultiplyAndWeightedAdd(alpha,*a.m_GPUSparseMatrix,transposeA,second,beta,*c.m_GPUMatrix);
c.SetDataLocation(GPU, DENSE);
}
@ -3409,8 +3420,10 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else
{
auto& first = transposeA ? a.m_GPUMatrix->Transpose()*alpha : (*a.m_GPUMatrix)*alpha;
auto& second = transposeB ? b.m_GPUSparseMatrix->Transpose() : *b.m_GPUSparseMatrix;
GPUMatrix<ElemType> firstDummy = transposeA ? a.m_GPUMatrix->Transpose()*alpha : (*a.m_GPUMatrix)*alpha;
GPUMatrix<ElemType> & first= firstDummy; // GCC does not support mixing refs and non-refs
GPUSparseMatrix<ElemType> secondDummy = transposeB ? b.m_GPUSparseMatrix->Transpose() : *b.m_GPUSparseMatrix;
GPUSparseMatrix<ElemType> & second = secondDummy;
if (beta==0)
{
GPUSparseMatrix<ElemType>::Multiply(first,second,*c.m_GPUMatrix);
@ -3432,7 +3445,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
else if (a.m_matrixType==b.m_matrixType && b.m_matrixType==c.m_matrixType && a.m_matrixType==MatrixType::SPARSE)
{
auto& first = alpha==1 ? *a.m_GPUSparseMatrix : (*a.m_GPUSparseMatrix)*alpha;
GPUSparseMatrix<ElemType> firstDummy = alpha==1 ? *a.m_GPUSparseMatrix : (*a.m_GPUSparseMatrix)*alpha;
GPUSparseMatrix<ElemType> & first = firstDummy; // By Malcolm.. gcc doesn't support auto
if (beta==0)
{
GPUSparseMatrix<ElemType>::Multiply(first,transposeA,*b.m_GPUSparseMatrix,transposeB,*c.m_GPUSparseMatrix);
@ -3897,18 +3911,16 @@ namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
bool Matrix<ElemType>::HasNan (const char * name) const
{
#if 0
name;
return false;
#else
const auto & us = *this;
// const auto & us = *this;
const Matrix<ElemType> & us = *this;
foreach_coord (i, j, us)
if (_isnan (us(i,j)))
// if (isnan (us(i,j)))
if (isnan (us(i,j)))
{
fprintf (stderr, "hasnan: NaN detected at %s (%d,%d)\n", name, i, j);
fprintf (stderr, "hasnan: NaN detected at %s (%ld,%ld)\n", name, i, j);
return true;
}
#endif
return false;
}
#define CheckNan(m) m.HasNan (#m)
@ -3924,7 +3936,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
foreach_coord (i, j, us)
{
auto val = us(i,j);
if (_isnan (val) || !_finite (val))
if (isnan (val) || !isfinite (val))
n++;
}
return n;

2
Math/Math/Matrix.h
Просмотреть файл

@ -3,7 +3,9 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//
#pragma once
#include "CPUMatrix.h"
#include "CPUSparseMatrix.h"
#include "GPUMatrix.cuh"

6
Math/Math/stdafx.h
Просмотреть файл

@ -14,10 +14,10 @@
#define _CRT_SECURE_NO_WARNINGS // "secure" CRT not available on all platforms --add this at the top of all CPP files that give "function or variable may be unsafe" warnings
#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers
// Windows Header Files:
#ifdef _WIN32
#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers
#include <windows.h>
#endif
// TODO: reference additional headers your program requires here

2
Math/Math/targetver.h
Просмотреть файл

@ -10,4 +10,6 @@
// If you wish to build your application for a previous Windows platform, include WinSDKVer.h and
// set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h.
#ifdef _WIN32
#include <SDKDDKVer.h>
#endif

48
makefile Normal file
Просмотреть файл

@ -0,0 +1,48 @@
# WORK IN PROGRESS, not currently complete nor usable
# makefile for a Linux/GCC build of CNTK
# This needs ACML_PATH. E.g. in tcsh, say: setenv ACML_PATH C:/AMD/acml5.3.1/ifort64_mp
# This is work in progress and not at all complete or usable.
#
# The Linux and Windows versions are not different branches, but rather build off the same
# source files, using different makefiles. This current makefile has the purpose of enabling
# work to make all sources compile with GCC, and also to check for GCC-compat regressions due to
# modifications which are currently done under Windows.
#
# The planned steps are:
# - runnable non-GPU GCC-built version under Cygwin
# - get all CPU-only sources to compile with GCC/x64 under Cygwin --currently ongoing work
# - port the dynamic-loading mechanism
# - runnable non-GPU version on actual Linux
# - enable CUDA on Linux (=makefile code and figuring out the right compiler options)
#
# Any help is welcome, of course!
#
# This makefile will be extended/completed as we go.
.SUFFIXES:
#SRC := Common/File.cpp Math/Math/Matrix.cpp
#OBJ := $(SRC:%.cpp=%.obj)
#DEP := $(OBJ:%.obj=%.dep)
INCFLAGS = -I Common/Include -I Math/Math -I $(ACML_PATH)/include
COMMON_SRC = Common/fileutil.cpp Common/DataWriter.cpp Common/ConfigFile.cpp Common/DataReader.cpp \
Common/Eval.cpp Common/File.cpp Common/NetworkDescriptionLanguage.cpp Common/BestGpu.cpp
MATH_SRC = Math/Math/Matrix.obj Math/Math/CPUMatrix.obj Math/Math/CPUSparseMatrix.obj Math/Math/GPUDummy.obj
SRC = $(MATH_SRC) $(COMMON_SRC)
all: ${SRC:.cpp=.obj}
CFLAGS = -std=c++0x -std=c++11 -DCPUONLY -D_POSIX_SOURCE -D_XOPEN_SOURCE=600 -fpermissive
%.obj: %.cpp
gcc -c -o $@ $(CPPFLAGS) $(CFLAGS) $(INCFLAGS) -MD -MP -MF ${@:.obj=.dep} $<
# .dep files created by -MD option in the gcc call
-include $(DEP)