diff --git a/CNTK.sln b/CNTK.sln index d963f6f53..201248005 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -366,6 +366,7 @@ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "EvalDll", "Source\EvalDll\EvalDll.vcxproj", "{482999D1-B7E2-466E-9F8D-2119F93EAFD9}" ProjectSection(ProjectDependencies) = postProject {928ABD1B-4D3B-4017-AEF1-0FA1B4467513} = {928ABD1B-4D3B-4017-AEF1-0FA1B4467513} + {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5} = {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5} EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Math", "Source\Math\Math.vcxproj", "{60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}" @@ -505,8 +506,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CIFAR-10", "CIFAR-10", "{77 Examples\Image\Miscellaneous\CIFAR-10\32to64.txt = Examples\Image\Miscellaneous\CIFAR-10\32to64.txt Examples\Image\Miscellaneous\CIFAR-10\CIFAR_convert.py = Examples\Image\Miscellaneous\CIFAR-10\CIFAR_convert.py Examples\Image\Miscellaneous\CIFAR-10\CifarConverter.py = Examples\Image\Miscellaneous\CIFAR-10\CifarConverter.py - Examples\Image\Miscellaneous\CIFAR-10\Macros.ndl = Examples\Image\Miscellaneous\CIFAR-10\Macros.ndl Examples\Image\Miscellaneous\CIFAR-10\labelsmap.txt = Examples\Image\Miscellaneous\CIFAR-10\labelsmap.txt + Examples\Image\Miscellaneous\CIFAR-10\Macros.ndl = Examples\Image\Miscellaneous\CIFAR-10\Macros.ndl Examples\Image\Miscellaneous\CIFAR-10\readme.txt = Examples\Image\Miscellaneous\CIFAR-10\readme.txt EndProjectSection EndProject @@ -679,6 +680,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{83BFF5BF EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ResNet", "ResNet", "{06D2C644-AE5F-4C30-A1F6-C78E2845AAB1}" ProjectSection(SolutionItems) = preProject + Examples\Image\Miscellaneous\ImageNet\ResNet\create_eval_model.mel = Examples\Image\Miscellaneous\ImageNet\ResNet\create_eval_model.mel Examples\Image\Miscellaneous\ImageNet\ResNet\Macros.ndl = Examples\Image\Miscellaneous\ImageNet\ResNet\Macros.ndl Examples\Image\Miscellaneous\ImageNet\ResNet\ProjWeightsGen.py = Examples\Image\Miscellaneous\ImageNet\ResNet\ProjWeightsGen.py Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_152.config = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_152.config @@ -687,99 +689,210 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ResNet", "ResNet", "{06D2C6 Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_34.ndl = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_34.ndl Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_50.config = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_50.config Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_50.ndl = Examples\Image\Miscellaneous\ImageNet\ResNet\ResNet_50.ndl - Examples\Image\Miscellaneous\ImageNet\ResNet\create_eval_model.mel = Examples\Image\Miscellaneous\ImageNet\ResNet\create_eval_model.mel + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Extensibility", "Extensibility", "{60F87E25-BC87-4782-8E20-1621AAEBB113}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "EvalWrapper", "Source\Extensibility\EvalWrapper\EvalWrapper.vcxproj", "{EF766CAE-9CB1-494C-9153-0030631A6340}" + ProjectSection(ProjectDependencies) = postProject + {482999D1-B7E2-466E-9F8D-2119F93EAFD9} = {482999D1-B7E2-466E-9F8D-2119F93EAFD9} + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSEvalClient", "Source\Extensibility\CSEvalClient\CSEvalClient.csproj", "{41E11A59-62B2-4927-A4F8-F40B1B612C6C}" + ProjectSection(ProjectDependencies) = postProject + {EF766CAE-9CB1-494C-9153-0030631A6340} = {EF766CAE-9CB1-494C-9153-0030631A6340} EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug_CpuOnly|x64 = Debug_CpuOnly|x64 Debug|x64 = Debug|x64 + Release_CpuOnly|x64 = Release_CpuOnly|x64 Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Debug|x64.ActiveCfg = Debug|x64 {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Debug|x64.Build.0 = Debug|x64 + {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Release|x64.ActiveCfg = Release|x64 {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}.Release|x64.Build.0 = Release|x64 + {928ABD1B-4D3B-4017-AEF1-0FA1B4467513}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {928ABD1B-4D3B-4017-AEF1-0FA1B4467513}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {928ABD1B-4D3B-4017-AEF1-0FA1B4467513}.Debug|x64.ActiveCfg = Debug|x64 {928ABD1B-4D3B-4017-AEF1-0FA1B4467513}.Debug|x64.Build.0 = Debug|x64 + {928ABD1B-4D3B-4017-AEF1-0FA1B4467513}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {928ABD1B-4D3B-4017-AEF1-0FA1B4467513}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {928ABD1B-4D3B-4017-AEF1-0FA1B4467513}.Release|x64.ActiveCfg = Release|x64 {928ABD1B-4D3B-4017-AEF1-0FA1B4467513}.Release|x64.Build.0 = Release|x64 + {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}.Debug|x64.ActiveCfg = Debug|x64 {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}.Debug|x64.Build.0 = Debug|x64 + {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}.Release|x64.ActiveCfg = Release|x64 {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}.Release|x64.Build.0 = Release|x64 + {EAD17188-072C-4726-B840-A769C36DAD1B}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {EAD17188-072C-4726-B840-A769C36DAD1B}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {EAD17188-072C-4726-B840-A769C36DAD1B}.Debug|x64.ActiveCfg = Debug|x64 {EAD17188-072C-4726-B840-A769C36DAD1B}.Debug|x64.Build.0 = Debug|x64 + {EAD17188-072C-4726-B840-A769C36DAD1B}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {EAD17188-072C-4726-B840-A769C36DAD1B}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {EAD17188-072C-4726-B840-A769C36DAD1B}.Release|x64.ActiveCfg = Release|x64 {EAD17188-072C-4726-B840-A769C36DAD1B}.Release|x64.Build.0 = Release|x64 + {4701E678-5E6F-470D-B348-9CD1A2C095D1}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {4701E678-5E6F-470D-B348-9CD1A2C095D1}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {4701E678-5E6F-470D-B348-9CD1A2C095D1}.Debug|x64.ActiveCfg = Debug|x64 {4701E678-5E6F-470D-B348-9CD1A2C095D1}.Debug|x64.Build.0 = Debug|x64 + {4701E678-5E6F-470D-B348-9CD1A2C095D1}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {4701E678-5E6F-470D-B348-9CD1A2C095D1}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {4701E678-5E6F-470D-B348-9CD1A2C095D1}.Release|x64.ActiveCfg = Release|x64 {4701E678-5E6F-470D-B348-9CD1A2C095D1}.Release|x64.Build.0 = Release|x64 + {EB2BE26F-6BD4-4274-971F-86D080779DD1}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {EB2BE26F-6BD4-4274-971F-86D080779DD1}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {EB2BE26F-6BD4-4274-971F-86D080779DD1}.Debug|x64.ActiveCfg = Debug|x64 {EB2BE26F-6BD4-4274-971F-86D080779DD1}.Debug|x64.Build.0 = Debug|x64 + {EB2BE26F-6BD4-4274-971F-86D080779DD1}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {EB2BE26F-6BD4-4274-971F-86D080779DD1}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {EB2BE26F-6BD4-4274-971F-86D080779DD1}.Release|x64.ActiveCfg = Release|x64 {EB2BE26F-6BD4-4274-971F-86D080779DD1}.Release|x64.Build.0 = Release|x64 + {A4FC3467-4787-43E8-BBC0-D79AE56B468D}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {A4FC3467-4787-43E8-BBC0-D79AE56B468D}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {A4FC3467-4787-43E8-BBC0-D79AE56B468D}.Debug|x64.ActiveCfg = Debug|x64 {A4FC3467-4787-43E8-BBC0-D79AE56B468D}.Debug|x64.Build.0 = Debug|x64 + {A4FC3467-4787-43E8-BBC0-D79AE56B468D}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {A4FC3467-4787-43E8-BBC0-D79AE56B468D}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {A4FC3467-4787-43E8-BBC0-D79AE56B468D}.Release|x64.ActiveCfg = Release|x64 {A4FC3467-4787-43E8-BBC0-D79AE56B468D}.Release|x64.Build.0 = Release|x64 + {482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Debug|x64.ActiveCfg = Debug|x64 {482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Debug|x64.Build.0 = Debug|x64 + {482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Release|x64.ActiveCfg = Release|x64 {482999D1-B7E2-466E-9F8D-2119F93EAFD9}.Release|x64.Build.0 = Release|x64 + {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Debug|x64.ActiveCfg = Debug|x64 {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Debug|x64.Build.0 = Debug|x64 + {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Release|x64.ActiveCfg = Release|x64 {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5}.Release|x64.Build.0 = Release|x64 + {B3DD765E-694E-4494-BAD7-37BBF2942517}.Debug_CpuOnly|x64.ActiveCfg = Debug|x64 {B3DD765E-694E-4494-BAD7-37BBF2942517}.Debug|x64.ActiveCfg = Debug|x64 {B3DD765E-694E-4494-BAD7-37BBF2942517}.Debug|x64.Build.0 = Debug|x64 + {B3DD765E-694E-4494-BAD7-37BBF2942517}.Release_CpuOnly|x64.ActiveCfg = Release|x64 {B3DD765E-694E-4494-BAD7-37BBF2942517}.Release|x64.ActiveCfg = Release|x64 {B3DD765E-694E-4494-BAD7-37BBF2942517}.Release|x64.Build.0 = Release|x64 + {D667AF32-028A-4A5D-BE19-F46776F0F6B2}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {D667AF32-028A-4A5D-BE19-F46776F0F6B2}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {D667AF32-028A-4A5D-BE19-F46776F0F6B2}.Debug|x64.ActiveCfg = Debug|x64 {D667AF32-028A-4A5D-BE19-F46776F0F6B2}.Debug|x64.Build.0 = Debug|x64 + {D667AF32-028A-4A5D-BE19-F46776F0F6B2}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {D667AF32-028A-4A5D-BE19-F46776F0F6B2}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {D667AF32-028A-4A5D-BE19-F46776F0F6B2}.Release|x64.ActiveCfg = Release|x64 {D667AF32-028A-4A5D-BE19-F46776F0F6B2}.Release|x64.Build.0 = Release|x64 + {1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Debug|x64.ActiveCfg = Debug|x64 {1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Debug|x64.Build.0 = Debug|x64 + {1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Release|x64.ActiveCfg = Release|x64 {1D5787D4-52E4-45DB-951B-82F220EE0C6A}.Release|x64.Build.0 = Release|x64 + {014DA766-B37B-4581-BC26-963EA5507931}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {014DA766-B37B-4581-BC26-963EA5507931}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {014DA766-B37B-4581-BC26-963EA5507931}.Debug|x64.ActiveCfg = Debug|x64 {014DA766-B37B-4581-BC26-963EA5507931}.Debug|x64.Build.0 = Debug|x64 + {014DA766-B37B-4581-BC26-963EA5507931}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {014DA766-B37B-4581-BC26-963EA5507931}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {014DA766-B37B-4581-BC26-963EA5507931}.Release|x64.ActiveCfg = Release|x64 {014DA766-B37B-4581-BC26-963EA5507931}.Release|x64.Build.0 = Release|x64 + {33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Debug|x64.ActiveCfg = Debug|x64 {33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Debug|x64.Build.0 = Debug|x64 + {33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Release|x64.ActiveCfg = Release|x64 {33D2FD22-DEF2-4507-A58A-368F641AEBE5}.Release|x64.Build.0 = Release|x64 + {9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Debug|x64.ActiveCfg = Debug|x64 {9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Debug|x64.Build.0 = Debug|x64 + {9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Release|x64.ActiveCfg = Release|x64 {9A2F2441-5972-4EA8-9215-4119FCE0FB68}.Release|x64.Build.0 = Release|x64 + {62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Debug|x64.ActiveCfg = Debug|x64 {62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Debug|x64.Build.0 = Debug|x64 + {62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Release|x64.ActiveCfg = Release|x64 {62836DC1-DF77-4B98-BF2D-45C943B7DDC6}.Release|x64.Build.0 = Release|x64 + {CE429AA2-3778-4619-8FD1-49BA3B81197B}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {CE429AA2-3778-4619-8FD1-49BA3B81197B}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {CE429AA2-3778-4619-8FD1-49BA3B81197B}.Debug|x64.ActiveCfg = Debug|x64 {CE429AA2-3778-4619-8FD1-49BA3B81197B}.Debug|x64.Build.0 = Debug|x64 + {CE429AA2-3778-4619-8FD1-49BA3B81197B}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {CE429AA2-3778-4619-8FD1-49BA3B81197B}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {CE429AA2-3778-4619-8FD1-49BA3B81197B}.Release|x64.ActiveCfg = Release|x64 {CE429AA2-3778-4619-8FD1-49BA3B81197B}.Release|x64.Build.0 = Release|x64 + {E6646FFE-3588-4276-8A15-8D65C22711C1}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {E6646FFE-3588-4276-8A15-8D65C22711C1}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {E6646FFE-3588-4276-8A15-8D65C22711C1}.Debug|x64.ActiveCfg = Debug|x64 {E6646FFE-3588-4276-8A15-8D65C22711C1}.Debug|x64.Build.0 = Debug|x64 + {E6646FFE-3588-4276-8A15-8D65C22711C1}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {E6646FFE-3588-4276-8A15-8D65C22711C1}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {E6646FFE-3588-4276-8A15-8D65C22711C1}.Release|x64.ActiveCfg = Release|x64 {E6646FFE-3588-4276-8A15-8D65C22711C1}.Release|x64.Build.0 = Release|x64 + {9BD0A746-0BBD-45B6-B81C-053F03C26CFB}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {9BD0A746-0BBD-45B6-B81C-053F03C26CFB}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {9BD0A746-0BBD-45B6-B81C-053F03C26CFB}.Debug|x64.ActiveCfg = Debug|x64 {9BD0A746-0BBD-45B6-B81C-053F03C26CFB}.Debug|x64.Build.0 = Debug|x64 + {9BD0A746-0BBD-45B6-B81C-053F03C26CFB}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {9BD0A746-0BBD-45B6-B81C-053F03C26CFB}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {9BD0A746-0BBD-45B6-B81C-053F03C26CFB}.Release|x64.ActiveCfg = Release|x64 {9BD0A746-0BBD-45B6-B81C-053F03C26CFB}.Release|x64.Build.0 = Release|x64 + {731312A8-6DA3-4841-AFCD-57520BA1BF8E}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {731312A8-6DA3-4841-AFCD-57520BA1BF8E}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {731312A8-6DA3-4841-AFCD-57520BA1BF8E}.Debug|x64.ActiveCfg = Debug|x64 {731312A8-6DA3-4841-AFCD-57520BA1BF8E}.Debug|x64.Build.0 = Debug|x64 + {731312A8-6DA3-4841-AFCD-57520BA1BF8E}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {731312A8-6DA3-4841-AFCD-57520BA1BF8E}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {731312A8-6DA3-4841-AFCD-57520BA1BF8E}.Release|x64.ActiveCfg = Release|x64 {731312A8-6DA3-4841-AFCD-57520BA1BF8E}.Release|x64.Build.0 = Release|x64 + {668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Debug|x64.ActiveCfg = Debug|x64 {668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Debug|x64.Build.0 = Debug|x64 + {668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 {668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Release|x64.ActiveCfg = Release|x64 {668BEED5-AC07-4F35-B3AE-EE65A7F9C976}.Release|x64.Build.0 = Release|x64 + {EF766CAE-9CB1-494C-9153-0030631A6340}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {EF766CAE-9CB1-494C-9153-0030631A6340}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 + {EF766CAE-9CB1-494C-9153-0030631A6340}.Debug|x64.ActiveCfg = Debug|x64 + {EF766CAE-9CB1-494C-9153-0030631A6340}.Debug|x64.Build.0 = Debug|x64 + {EF766CAE-9CB1-494C-9153-0030631A6340}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {EF766CAE-9CB1-494C-9153-0030631A6340}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 + {EF766CAE-9CB1-494C-9153-0030631A6340}.Release|x64.ActiveCfg = Release|x64 + {EF766CAE-9CB1-494C-9153-0030631A6340}.Release|x64.Build.0 = Release|x64 + {41E11A59-62B2-4927-A4F8-F40B1B612C6C}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {41E11A59-62B2-4927-A4F8-F40B1B612C6C}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 + {41E11A59-62B2-4927-A4F8-F40B1B612C6C}.Debug|x64.ActiveCfg = Debug|x64 + {41E11A59-62B2-4927-A4F8-F40B1B612C6C}.Debug|x64.Build.0 = Debug|x64 + {41E11A59-62B2-4927-A4F8-F40B1B612C6C}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {41E11A59-62B2-4927-A4F8-F40B1B612C6C}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 + {41E11A59-62B2-4927-A4F8-F40B1B612C6C}.Release|x64.ActiveCfg = Release|x64 + {41E11A59-62B2-4927-A4F8-F40B1B612C6C}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -872,5 +985,7 @@ Global {850008BC-36B0-4A0A-BD0C-B6D5C2184227} = {6F4125B5-220F-4FB7-B6C4-85A966A0268C} {E6DC3B7D-303D-4A54-B040-D8DCF8C56E17} = {8C128B1D-87E0-4643-AB93-2581589AE425} {06D2C644-AE5F-4C30-A1F6-C78E2845AAB1} = {EF710C5A-E616-442A-889D-C997D39AF2E1} + {EF766CAE-9CB1-494C-9153-0030631A6340} = {60F87E25-BC87-4782-8E20-1621AAEBB113} + {41E11A59-62B2-4927-A4F8-F40B1B612C6C} = {60F87E25-BC87-4782-8E20-1621AAEBB113} EndGlobalSection EndGlobal diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook_ASRDecoder_Chapter.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook_ASRDecoder_Chapter.lyx index eb02b6818..8c9af1f69 100644 --- a/Documentation/CNTK-TechReport/lyx/CNTKBook_ASRDecoder_Chapter.lyx +++ b/Documentation/CNTK-TechReport/lyx/CNTKBook_ASRDecoder_Chapter.lyx @@ -743,7 +743,7 @@ To build the TIMIT graph, only three input files are needed: the model state \end_layout \begin_layout Standard -The scripts assume each context-indepenent phone is represented by a three +The scripts assume each context-independent phone is represented by a three state, left to right, hidden markov model. The names of these states should be in a \begin_inset Quotes eld @@ -756,7 +756,7 @@ model state map file that has one line for every model. The first column is the name of the model, and subsequent columns are the names of the states, in left to right order. - The transition probabilites between these states are stored in a separate + The transition probabilities between these states are stored in a separate \begin_inset Quotes eld \end_inset @@ -859,7 +859,7 @@ To decode, the following parameters to Argon should be specified: -graph, The decoder uses a Viterbi beam search algorithm, in which unlikely hypotheses are pruned at each frame. The -beam parameter prevents unlikely hypotheses from being pursued. - Any hypothesis that differes from the best hypothesis by more than this + Any hypothesis that differs from the best hypothesis by more than this amount will be be discarded. The -max-tokens parameter controls the number of active hypotheses. If the -beam parameter causes more than max-tokens hypotheses to be generated, @@ -872,7 +872,7 @@ The decoder uses a Viterbi beam search algorithm, in which unlikely hypotheses \begin_layout Standard The -graph parameter tells Argon which compiled decoding graph should be used. - The -lm should indicate an ARPA format ngram languag emodel. + The -lm should indicate an ARPA format ngram language model. \end_layout \begin_layout Standard diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Adv_Chapter.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Adv_Chapter.lyx index f5bcedbfa..5f783a745 100644 --- a/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Adv_Chapter.lyx +++ b/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Adv_Chapter.lyx @@ -705,7 +705,7 @@ After defining the network, it’s important to let CNTK know what the special It also needs to know the default output nodes, evaluation nodes and training criteria nodes. Note here the specification of the nodes that require special handling - (NodesReqMultiSeqHandling) when the network is evalauted or trained with + (NodesReqMultiSeqHandling) when the network is evaluated or trained with multiple sequences, e.g., when the network itself is an RNN or the model is trained with the sequence-level criterion. Since in these cases multiple sequences will be stitched together to improve @@ -2233,7 +2233,7 @@ RowStack \end_layout \begin_layout Standard -Concatnate rows of input matrices to form a bigger matrix. +Concatenate rows of input matrices to form a bigger matrix. The resulting matrix is a sumof(rows) by m1.cols matrix. It supports variable-length input. The syntax is @@ -2898,11 +2898,11 @@ labels - the ground truth labels. The first row is the ground truth output id. The second row is the ground truth class id. The third and fourth rows are the start (inclusive) and end (exclusive) - output ids corresponding to the ground trueth class id. + output ids corresponding to the ground truth class id. \end_layout \begin_layout Itemize -mainInputInfo - contains the main information to make the classfication +mainInputInfo - contains the main information to make the classification decision. It's an inputDim by T matrix. In language model, inputDim is often the hidden layer size. @@ -4422,7 +4422,7 @@ To integrate this new layer into the model, the inputs and outputs of the After the copy any node whose connected nodes were not copied will have those connections set to an invalid value. These need to be fixed in order to have a valid model. - Before a model can be saved CNTK first checkes to see if all nodes are + Before a model can be saved CNTK first checks to see if all nodes are correctly connected. \end_layout diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Chapter.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Chapter.lyx index 6695f1d13..398d81b24 100644 --- a/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Chapter.lyx +++ b/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Chapter.lyx @@ -965,7 +965,7 @@ CLASSLSTM : the class-based long short-term memory neural network. It uses sparse input, sparse parameter and sparse output. - This is often uesd for language modeling tasks. + This is often used for language modeling tasks. \end_layout \end_deeper @@ -1768,10 +1768,10 @@ numMiniBatch4LRSearch \end_inset -: the number of minibatches used to search the minibatch size whenin adaptive +: the number of minibatches used to search the minibatch size when in adaptive minibatch size mode. Default value is 500. - It's typically set to 10-20% of the total minibatches in an epochthis is + It's typically set to 10-20% of the total minibatches in an epoch. This is shared with the search for learning rate in SearchBeforeEpoch mode. \end_layout @@ -1792,10 +1792,10 @@ autoAdjustMinibatch \end_inset : enable or disable whether minibatch size is adaptively adjusted. - Default value is false.Adapative minibatch sizing will begin on epochs starting - after user minbatch sizes expcitilyspecified are complete. + Default value is false. Adaptive minibatch sizing will begin on epochs starting + after user minibatch sizes explicitly specified are complete. For example if the userspecifed minibatchSize=256:1024, then 256 and 1024are - used in the first 2 Epochs and adaptive minibatchsizing is used aferwards + used in the first 2 Epochs and adaptive minibatchsizing is used afterwards \end_layout @@ -1814,7 +1814,7 @@ minibatchSizeTuningFrequency \end_inset -: The number of epochs to skip, on a periodic basis, beforedynamically adjusting +: The number of epochs to skip, on a periodic basis, before dynamically adjusting the minibatch size. Default value is 1. @@ -1835,7 +1835,7 @@ minibatchSizeTuningMax \end_inset -: The maximum size allowed for anadaptively adjusted minibatch size. +: The maximum size allowed for an adaptively adjusted minibatch size. Default value is 1048576. \end_layout @@ -2669,10 +2669,10 @@ rollingWindow option reads in all feature files and stores them on disk in one large temporary binary file. - The data is randomized by running a large rollowing window over the data + The data is randomized by running a large rolling window over the data in this file and randomizing the data within the window. This method produces more thorough randomization of the data but requires - a large temprorary file written to disk. + a large temporary file written to disk. The other option is \begin_inset Quotes eld \end_inset @@ -2798,14 +2798,14 @@ labels \end_inset are the default names used by the SimpleNetworkBuilder but if the network - is designed using the Network Descrition Language (NDL), then any names + is designed using the Network Description Language (NDL), then any names can be used, as long as they each have a corresponding node in the network. \end_layout \begin_layout Standard To specify continuous-valued features, e.g. MFCC's or log mel filterbank coefficients, the following parameters should - be included in the a confguration block: + be included in the a configuration block: \end_layout \begin_layout Itemize @@ -3378,7 +3378,7 @@ nbruttsineachrecurrentiter The reader arranges same-length input sentences, up to the specified limit, into each minibatch. For recurrent networks, trainer resets hidden layer activities only at - the begining of sentences. + the beginning of sentences. Activities of hidden layers are carried over to the next minibatch if an end of sentence is not reached. Using multiple sentences in a minibatch can speed up training processes. @@ -3425,7 +3425,7 @@ wordclass This is used for class-based language modeling. An example of the class information is below. The first column is the word index. - The second column is the number of occurances, the third column is the + The second column is the number of occurrences, the third column is the word, and the last column is the class id of the word. \begin_inset listings @@ -3795,7 +3795,7 @@ nbrUttsInEachRecurrentIter The reader arranges same-length input sentences, up to the specified limit, into each minibatch. For recurrent networks, trainer resets hidden layer activities only at - the begining of sentences. + the beginning of sentences. Activities of hidden layers are carried over to the next minibatch if an end of sentence is not reached. Using multiple sentences in a minibatch can speed up training processes. @@ -4999,7 +4999,7 @@ section \end_inset – the encoderReader and decoderReader are the readers for encoder and decoder. - Similary for encoderCVReader and decoderCVReader for validation set. + Similarly for encoderCVReader and decoderCVReader for validation set. \end_layout @@ -5365,7 +5365,7 @@ deviceId \begin_layout Standard CNTK supports CPU and GPU computation. - Users can determine what device to use by setting the deviceId papameter. + Users can determine what device to use by setting the deviceId parameter. The possible values are \end_layout @@ -5509,7 +5509,7 @@ traceLevel=0 # larger values mean more output The default value is 0 and specifies minimal output. The higher the number the more output can be expected. - Currently 0 (limited output), 1 (medium ouput) and 2 (verbose output) are + Currently 0 (limited output), 1 (medium output) and 2 (verbose output) are the only values supported. \end_layout diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Programmer_Chapter.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Programmer_Chapter.lyx index 0b3d6899d..fbe07a008 100644 --- a/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Programmer_Chapter.lyx +++ b/Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Programmer_Chapter.lyx @@ -3186,7 +3186,7 @@ s().GetNumCols() != 1) \begin_layout Plain Layout - throw std::logic_error("The left value of ScaleNode must be a scarlar + throw std::logic_error("The left value of ScaleNode must be a scalar value."); \end_layout diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook_CN_Chapter.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook_CN_Chapter.lyx index a67f3cd95..78fbd8367 100644 --- a/Documentation/CNTK-TechReport/lyx/CNTKBook_CN_Chapter.lyx +++ b/Documentation/CNTK-TechReport/lyx/CNTKBook_CN_Chapter.lyx @@ -1816,11 +1816,11 @@ sly. In this algorithm, all the nodes whose children have not been computed are in the waiting set and those whose children are computed are in the ready set. - At the beginning, all non-leaf descendents of + At the beginning, all non-leaf descendants of \begin_inset Formula $root$ \end_inset - are in the waiting set and all leaf descendents are in the ready set. + are in the waiting set and all leaf descendants are in the ready set. The scheduler picks a node from the ready set based on some policy, removes it from the ready set, and dispatches it for computation. Popular policies include first-come/first-serve, shortest task first, and @@ -2015,7 +2015,7 @@ status open \begin_inset Formula $waiting$ \end_inset - is initialized to include all non-leaf descendents of + is initialized to include all non-leaf descendants of \begin_inset Formula $root$ \end_inset @@ -2061,7 +2061,7 @@ status open \begin_inset Formula $ready$ \end_inset - is initialized to include all leaf descendents of + is initialized to include all leaf descendants of \begin_inset Formula $root$ \end_inset @@ -3412,7 +3412,7 @@ status open \end_inset -Decide the order to compute the gradient at all descendents of +Decide the order to compute the gradient at all descendants of \begin_inset Formula $node$ \end_inset @@ -8892,7 +8892,7 @@ CRF \color none CRF stands for conditional random fields. This node does sequence-level training, using CRF criterion. - This node has three nputs. + This node has three inputs. The first is the label \family default \series bold @@ -10198,7 +10198,7 @@ reference "fig:CN-WithDelayNode" A simple way to do forward computation and backpropagation in a recurrent network is to unroll all samples in the sequence over time. Once unrolled, the graph is expanded into a DAG and the forward computation - and gradient calcalclation algorithms we just discussed can be directly + and gradient calculation algorithms we just discussed can be directly used. This means, however, all computation nodes in the CN need to be computed sample by sample and this significantly reduces the potential of parallelizatio @@ -10318,7 +10318,7 @@ key "StronglyConnectedComponents-Hopcroft+1983" in the CN and the CN is reduced to a DAG. All the nodes inside each loop (or composite node) can be unrolled over time and also reduced to a DAG. - For all these DAGs the forward computation and backprogation algorithms + For all these DAGs the forward computation and backpropagation algorithms we discussed in the previous sections can be applied. The detailed procedure in determining the forward computation order in the CN with arbitrary recurrent connections is described in Algorithm diff --git a/Documentation/CNTK-TechReport/lyx/CNTKBook_ExampleSetup_Chapter.lyx b/Documentation/CNTK-TechReport/lyx/CNTKBook_ExampleSetup_Chapter.lyx index d8a4cab35..a3aac49ce 100644 --- a/Documentation/CNTK-TechReport/lyx/CNTKBook_ExampleSetup_Chapter.lyx +++ b/Documentation/CNTK-TechReport/lyx/CNTKBook_ExampleSetup_Chapter.lyx @@ -97,7 +97,7 @@ ns. All examples are based on the TIMIT corpus for phonetic recognition but can easily be modified for use for large vocabulary continuous speech recogniti on. - The only significant change is that context-indepenent phonetic states + The only significant change is that context-independent phonetic states used in the TIMIT example would be replaced by context-dependent senone targets for large vocabulary tasks. We note that these examples are not meant to be representative of state @@ -146,10 +146,10 @@ SimpleNetworkBuilder will also be monitored during training using the evalCriterion parameter. The input data will be mean and variance normalized since applyMeanVarNorm has been set to true. - In addtion, if needPrior is set to true, the prior probablities of the + In addition, if needPrior is set to true, the prior probabilities of the labels will be computed and a ScaledLogLikelihood node in the network will be automatically created. - This is important if this netwok will be used to generate acoustic scores + This is important if this network will be used to generate acoustic scores in a speech recognition decoder. \end_layout @@ -838,7 +838,7 @@ SquareError Below is a snippet from the NDL file for this example. This autoencoder has three hidden layers including a middle bottleneck layer of 64 neurons. - A macro is defined to peform mean and variance normalization and it is + A macro is defined to perform mean and variance normalization and it is applied to both the input and target features. Also, \end_layout @@ -1123,7 +1123,7 @@ discriminative pre-training \begin_layout Standard It is well known that deep networks can be difficult to optimize, especially when a limited amount of training data is available. - As a result, a number of aproaches to initializing the parameters of these + As a result, a number of approaches to initializing the parameters of these networks have been proposed. One of these methods is known as discriminative pre-training. In this approach, a network with a single hidden layer is trained starting @@ -1526,7 +1526,7 @@ multi-task learning \begin_layout Standard One interesting approach to network training is multi-task learning, where - the network is trained to optmize two objective functions simultaneously. + the network is trained to optimize two objective functions simultaneously. This can be done in CNTK through the appropriate use of NDL. Let's assume that we have a network specified in NDL that has three hidden layers and output of the third hidden layer is defined as L3. @@ -1856,7 +1856,7 @@ TIMIT.statelist" \begin_layout Standard The NDL for constructing a network with these inputs and outputs can be done in a number of ways. - One way is to contruct a macro that constructs a layer that takes two inputs, + One way is to construct a macro that constructs a layer that takes two inputs, as follows: \end_layout @@ -1984,7 +1984,7 @@ L1 = SBFF2(featInput1, HiddenDim, FeatDim1, featInput2, FeatDim2) The rest of the hidden layers and the output layer with a cross entropy objective function would be the same as previous examples. - Notice that the names and dimensionality of the input adn output data have + Notice that the names and dimensionality of the input and output data have to the same in both the NDL model description and the reader configuration. \end_layout @@ -2623,7 +2623,7 @@ layerSizes \end_inset =10000:200:10000. - Sizes of input, hidden and ouput layers. + Sizes of input, hidden and output layers. Input layer size is equal to vocabulary size, hidden layer is normally in the range of 50 to 500, output layer size is the vocabulary size. \end_layout @@ -2640,7 +2640,7 @@ uniformInit \end_inset =true. - Whether to use uniformly randomizied values for initial paramter weights. + Whether to use uniformly randomized values for initial parameter weights. \end_layout \begin_layout Itemize @@ -2898,7 +2898,7 @@ learnRateDecreaseFactor \end_inset =0.5. - Learning rate decrese factor. + Learning rate decrease factor. \end_layout \end_deeper @@ -2963,7 +2963,7 @@ t word_class \end_layout \begin_layout Standard -word_id is a unique non-negative interger, frequency is the frequency of +word_id is a unique non-negative integer, frequency is the frequency of word (optional), word_string is the word string (low frequent words may be mapped to ), and word_class is the class id of word. Word class can be derived using frequency based heuristics @@ -4254,7 +4254,7 @@ wordContext \end_inset =0:1:2 : this specifies the time indices for forming a context window. - In this example, this setup coresponds to using the current input, the + In this example, this setup corresponds to using the current input, the next input, and the input after the next input for a context window of size 3. User can also use other cases such as wordcontext=0:-1:1 to form a context @@ -4344,7 +4344,7 @@ BOS \begin_inset Quotes erd \end_inset - : this specifies the symbol of sequence begining. + : this specifies the symbol of sequence beginning. \end_layout \begin_layout Itemize @@ -4634,7 +4634,7 @@ outputs:labels \end_inset : this specifies which nodes to output results. - These node names are pre-spefied in CNTK's simple network builder. + These node names are pre-specified in CNTK's simple network builder. The node \begin_inset Quotes eld \end_inset @@ -4643,7 +4643,7 @@ outputs \begin_inset Quotes erd \end_inset - is the node that output activies before softmax. + is the node that output activates before softmax. The node \begin_inset Quotes eld \end_inset @@ -4836,7 +4836,7 @@ output.rec.txt \begin_inset Quotes erd \end_inset - : the file name for writting decode results from LUSequenceWriter. + : the file name for writing decode results from LUSequenceWriter. \end_layout diff --git a/Documentation/Documents/Configuration Files.docx b/Documentation/Documents/Configuration Files.docx deleted file mode 100644 index 60803a771..000000000 Binary files a/Documentation/Documents/Configuration Files.docx and /dev/null differ diff --git a/Documentation/Documents/Configuration Files.md b/Documentation/Documents/Configuration Files.md new file mode 100644 index 000000000..ef447fc26 --- /dev/null +++ b/Documentation/Documents/Configuration Files.md @@ -0,0 +1,1265 @@ +# Configuration Files + +## Overview + + +The Computational Network ToolKit (CNTK) consists of a number of components to complete machine learning task. These include Network Builders, Network Trainers, Data Readers, and various other components. Most of these components need some configuration information available in order to function, and these configuration parameters are provided through configuration files in CNTK. + +### Configuration Data + +Configuration files are collections of name-value pairs. The configuration data can be one of the following types: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
config typeExamplesDescription
SimpledeviceId=autoA single value is assigned to the configuration parameter
Array + +
minibatchSize=256:512:512:512:1024
+minibatchSize=256:512*3:1024
+files=(;c:\data.txt;c:\labels.txt)
+
+ + 		+
    +
  • a configuration parameter is assigned an array of values which need not be of a uniform type
    • +
  • a ‘:’ is the default separator for arrays, but may be changed by enclosing the array values in parenthesis and placing the new separator character immediately following the open parenthesis
    • +
  • an ‘*’ repeat character allows a particular value to be repeated multiple times in the array
    • +
+
Parameter Set + +
section1=[id=1;size=256]
section2=[
  subsection=[string="hi";num=5]
  value=1e-10
  array=10:"this is a test":1.25
]
+
+ + 		+
    +
  • Parameter sets contain sets of configuration parameters of any type
    • +
  • Parameter sets can be nested
    • +
  • The default separator for parameter sets is the ‘;’ if multiple items are included on one line
    • +
  • Line separators also serve as separators for items
    • +
+
+ +### Organization + +In CNTK configuration files Parameter Sets are organized in a hierarchal fashion. The actual data values are not evaluated until a CNTK components requests the value. When a value is requested, by a component, it will search that components section of the configuration file, if the value is not found, it will continue looking in the parent parameter set and continue looking in parent parameter sets until the parameter is found, or the top level of the configuration hierarchy is reached without a match. + +### Default Values + +Most parameters in configuration files have a default value which will be used if no configuration value is specified. If there is no default value and the value cannot be found in a search, an exception will be displayed and the program will exit. + +### Repeated Values + +If a parameter name is specified more than once, the last value set to that value is the one that will be maintained. The only exception to this is in parameter sets, which are surrounded by ‘\[‘ square braces ‘\]’, in these cases the values inside the braces are considered to be a parameter set, and will be added to the currently existing parameter set. For example: + +``` +params=[a=1;b=2;c=3] +params=[c=5;d=6;e=7] +``` + +is effectively equal to: + +``` +params=[a=1;b=2;c=5;d=6;e=7] +``` + +this “append” processing is not used for arrays elements, and the entire array will be replaced if it is set multiple times. + +### Comments + +The ‘\#’ character signifies the beginning of a comment, everything that occurs after the ‘\#’ is ignored. The ‘\#’ must be preceded by whitespace or be at the beginning of the line to be interpreted as a comment. The following are valid comments, and an example of a value that will not be interpreted as a comment: + +``` +#commands used will be appended the stderr name to create a path +stderr=c:\cntk\log\cntk # "_mnistTrain_mnistTest.log" would be appended +# the parameter below is set to infinity, the ‘#’ in ‘1#INF’ is not a comment marker +var = 1#INF +``` + +## Example + +This section will go through a sample configuration file that creates a simple DNN (Deep Neural Network), trains the network and then tests the results. If you would rather see the Comprehensive documentation of configuration parameters skip to the Configuration Reference section. + +Here is a simple example of a configuration file: + +``` +# sample configuration file for CNTK command=mnistTrain:mnistTest #global parameters, all commands use these values unless overridden at a higher level precision=float deviceId=auto #commands used will be appended the stderr name to create a path stderr=c:\cntk\log\cntk # “_mnistTrain_mnistTest.log” would be appended traceLevel=0 # larger values mean more output ndlMacros=C:\cntk\config\DefaultMacros.ndl modelPath=c:\cntk\model\sample.dnn labelMappingFile=c:\cntk\data\mnist\labels.map mnistTrain=[ action=train minibatchSize=32 epochSize=60000 NDLNetworkBuilder=[ networkDescription=c:\cntk\config\sample.ndl run=ndlMacroUse ] SGD=[ #modelPath - moved to root level to share with mnistTest learningRatesPerMB=0.001 maxEpochs=50 ] reader=[ readerType=UCIFastReader file=c:\cntk\data\mnist\mnist_train.txt features=[ dim=784 start=1 ] labels=[ dim=1 start=0 labelDim=10 ] ] ] mnistTest=[ action=eval maxEpochs=1 epochSize=10000 minibatchSize=1000 reader=[ readerType=UCIFastReader randomize=None file=c:\data\mnist\mnist_test.txt features=[ dim=784 start=1 ] labels=[ dim=1 start=0 labelDim=10 ] ] ] +``` + +### Commands and actions + +The first thing at the top of the configuration is the command: + +``` +command=mnistTrain:mnistTest +``` + +This command instructs CNTK to execute the **mnistTrain** section of the config file, followed by mnistTest. Each of these Config sections has an action associated with it: + +``` +mnistTrain=[ action=train … +``` +The **mnistTrain** section will execute the **train** action, and the **mnistTest** section will execute **eval**. The names of the sections is arbitrary, but the configuration parameter names must be command and action. + +### Precision + +The accuracy and speed of obtaining results in any machine learning experiment will be affected by the precision of the floating point values that are used in creating the model. The precision is specified as follows: + +``` +precision=float +``` + +The other option is double and will be more precise, but slower depending on your hardware. Most GPU hardware is much faster using float precision, but some experiments require the added precision of double. + +### Device Identifier + +CNTK supports CPU and GPU computation, and the determination of which device should be used is based on the **deviceId** parameter. The default value and the value used in the example config is: + +``` +deviceId=auto +``` + +This setting will pick the best device available on the machine. If multiple GPUs are available, it will choose the fastest, least busy GPU, and a CPU if no usable GPUs can be found. The following settings can be used for **deviceId**: + +Value | Description +---|--- +auto | choose the best GPU, or CPU if no usable GPU is available. +cpu | use the CPU for all computation +0 | The device Identifier (as used in CUDA) for the GPU device you wish to use (1, 2, etc.) +all | Use all the available GPU devices (will use PTask engine if more than one GPU is present) +\*2 | Will use the 2 best GPUs, any number up to the number of available GPUs on the machine can be used. (will use PTask engine) + +### Log Files + +Log files are redirection of the normal standard error output. All log information is sent to standard error, and will appear on the console screen unless the stderr parameter is defined, or some other form of user redirection is active. The stderr parameter defines the directory and the prefix for the log file. The suffix is defined by what commands are being run. As an example if “abc” is the setting “abc\_mnistTrain.log” would be the log file name. It is important to note that this file is overwritten on subsequent executions if the stderr parameter and the command being run are identical. + +``` +#commands used will be appended the stderr name to create a path stderr=c:\cntk\log\cntk # “_mnistTrain_mnistTest.log” would be appended traceLevel=0 # larger values mean more output +``` + +The **traceLevel** parameter is uniformly used by the code in CNTK to specify how much extra output (verbosity) is desired. The default value is 0 (zero) and specifies minimal output, the higher the number the more output can be expected. Currently 0-limited output, 1-medium ouput, 2-verbose output are the only values supported. + +### Top Level Parameters + +It is often advantageous to set some values at the top level of the config file. This is because config searches start with the target section and continue the search to higher level sections. If the same parameter is used in multiple sections putting the parameter at a higher level where both sections can share it can be a good idea. In our example the following parameters are used by both the train and the test step: + +``` +ndlMacros=C:\cntk\config\DefaultMacros.ndl modelPath=c:\cntk\model\sample.dnn labelMappingFile=c:\cntk\data\mnist\labels.map +``` + +It can also be advantageous to specify parameters that often change all in one area, rather than separated into the sections to which the parameters belong. These commonly modified parameters can even be placed in a separate file if desired. See the layered config files in the reference section for more information. + +### Command Section + +A command section is a top level section of the configuration file that contains an action parameter. The command parameter at the root level determines which command sections will be executed, and in what order. The **mnistTrain** command section uses the **train** **action** to train a Deep Neural Network (DNN) using Stochastic Gradient Descent (SGD). It uses the Network Description Language (NDL) to define the network, and the UCIFastReader component to obtain its data. All the necessary information to complete this training is included in the command section. There are many other parameters that could be provided, but most have reasonable default values that have been found to provide good results for many datasets. + +Under the command section there may be one or more sub-sections. These sub-sections vary by the **action** specified in the command section. In our example we used the **train action**, which includes the following subsections: + +sub-section | Options | Description +---|---|--- +**Network Builder** | SimpleNetworkBuilder | Creates simple layer-based networks with various options + | NDLNetworkBuilder | Create a network defined in NDL (Network Description Language). This allows arbitrary networks to be created and offers more control over the network structure. +**Trainer** | SGD | Stochastic Gradient Descent trainer, currently this is the only option provided with CNTK +**Reader** | UCIFastReader | Reads the text-based UCI format, which contains labels and features combined in one file + | HTKMLFReader | Reads the HTK/MLF format files, often used in speech recognition applications + | BinaryReader | Reads files in a CNTK Binary format. It is also used by UCIFastReader to cache the dataset in a binary format for faster processing. + | SequenceReader | Reads text-based files that contain word sequences, for predicting word sequences. + | LUSequenceReader | Reads text-based files that contain word sequences, used for language understanding. + +For the Network Builder and the Trainer the existence of the sub-section name tells the train action which component to use. For example, **NDLNetworkBuilder** is specified in our example, so CNTK will use the NDL Network Builder to define the network. Similarly **SGD** is specified, so that trainer will be used. The reader sub-section is a little different, and is always called **reader**, the **readerType** parameter in the sub-section defines which reader will actually be used. Readers are implemented as separate DLLs, and the name of the reader is also the name of the DLL file that will be loaded. + +``` +mnistTrain=[ action=train minibatchSize=32 epochSize=60000 NDLNetworkBuilder=[ networkDescription=c:\cntk\config\sample.ndl run=ndlMacroUse ] SGD=[ #modelPath - moved to root level to share with mnistTest learningRatesPerMB=0.001 maxEpochs=50 ] reader=[ readerType=UCIFastReader file=c:\cntk\data\mnist\mnist_train.txt features=[ dim=784 start=1 ] labels=[ dim=1 start=0 labelDim=10 ] ] ] +``` + +The rest of the parameters in the mnistTrain Command Section are briefly explained here, more details about the parameters available for each component are available in the Configuration Reference section of this document. + +### SGD Parameters + +The parameters at the top of the command section are actually SGD parameters. These parameters have moved up a level in the configuration mainly to provide easier visibility to the parameters. Configuration searches continue searching parents until the root of the configuration is reached or the parameter is found. + +``` +minibatchSize=32 +epochSize=60000 +``` + +**minibatchSize** is the number of records that will be taken from the dataset and processed at once. There is often a tradeoff in training accuracy and the size of the minibatch. Particularly for GPUs, a larger minibatch is usually better, since the GPUs are most efficient when doing operations on large chunks of data. For large dataset values that are powers of 2 are most often specified, 512 and 1024 are often good choices to start with. Since the MNIST dataset is so small, we have chosen a smaller minibatch size for the example. + +**epochSize** is the number of dataset records that will be processed in a training pass. It is most often set to be the same as the dataset size, but can be smaller or larger that the dataset. It defaults to the size of the dataset if not present in the configuration file. It can also be set to zero for SGD, which has the same meaning. + +``` +SGD=[ #modelPath - moved to root level to share with mnistTest learningRatesPerMB=0.001 maxEpochs=50 ] +``` + +**modelPath** is the path to the model file, and will be the name used when a model is completely trained. For epochs prior to the final model a number will be appended to the end signifying the epoch that was saved (i.e. myModel.dnn.5). These intermediate files are important to allow the training process to restart after an interruption. Training will automatically resume at the first non-existent epoch when training is restarted. + +**learningRatesPerMB** is the learning rate per minibatch used by SGD to update parameter matrices. This parameter is actually an array, and can be used as follows: 0.001\*10:0.0005 to specify that for the first 10 epochs 0.001 should be used and then 0.0005 should be used for the remainder of the epochs. + +**maxEpochs** is the total number of epochs that will be run before the model is considered complete. + +### NDLNetworkBuilder Parameters + +The NDL (Network Description Language) Network Builder component will be used by this config because the NDLNetworkBuilder section is present. Had there been a SimpleNetworkBuilder section instead, that network builder would be used. + +**networkDescription** is the file path of the NDL script to execute. If there is no networkDescription file specified then the NDL is assumed to be in the same configuration file as the NDLNetworkBuilder subsection, specified with the “run” parameter. Note that only one file path may be specified via the “networkDescription” parameter; to load multiple files of macros, use the “ndlMacros” parameter. + +**run** is the section containing the NDL that will be executed. If using an external file via the “networkDescription” parameter, as in the example, the **run** parameter identifies the section in that file that will be executed as an NDL script. This parameter overrides any **run** parameters that may already exist in the file. If no **networkDescription** file is specified, **run** identifies a section in the current configuration file. It must exist where a regular configuration search will find it (peer or closer to the root of the hierarchy) + +**load** specifies what sections of NDL to load. Multiple sections can be specified via a “:” separated list. The sections specified by **load** are generally used to define macros, for use by the **run** section. If using an external file via the **networkDescription** parameter, as in the example, the **load** parameter identifies the section(s) in that file to load. This parameter overrides any **load** parameters that may already exist in the file. If no **networkDescription** file is specified, **load** identifies a section in the current configuration file. It must exist where a regular configuration search will find it (peer or closer to the root of the hierarchy) + +**ndlMacros** is the file path where NDL macros may be loaded. This parameter is usually used to load a default set of NDL macros that can be used by all NDL scripts. Multiple NDL files, each specifying different sets of macros, can be loaded by specifying a “+” separated list of file paths for this “ndlMacros” parameters. In order to share this parameter with other Command Sections which also expect an “ndlMacros” parameter (eg, for MEL scripts), one should define it at the root level of the configuration file. + +**randomSeedOffset** is a parameter which allows you to run an experiment with a different random initializations to the learnable parameters which are meant to be initialized randomly (either uniform or Gaussian). Whatever non-negative number is specified via “randomSeedOffset” will be added to the seed which would have otherwise been used. The default value is 0. + +### Reader Parameters + +The reader section is used for all types of readers and the **readerType** parameter identifies which reader will be used. For our example, we are using the UCIFastReader, which reads text-based UCI format data. The format of UCI data is a line of space-delimited floating point feature and label values for each data record. The label information is either at the beginning or the end of each line, if label information is included in the dataset. + +``` +readerType=UCIFastReader +``` + +Each of the readers uses the same interface into CNTK, and each reader is implemented in a separate DLL. There are many parameters in the reader section that are used by all the different types of readers, and some are specific to a particular reader. Our example reader section is as follows: + +``` +reader=[ readerType=UCIFastReader file=c:\cntk\data\mnist\mnist_train.txt features=[ dim=784 start=1 ] labels=[ dim=1 start=0 labelDim=10 ] ] +``` + +The two sub-sections in the reader section identify two different data sets. In our example they are named **features** and **labels**, though any names could be used. These names need to match the names used in the NDL network definition Inputs in our example, so the correct definition is used for each input dataset. Each of these sections for the UCIFastReader have the following parameters: + +**dim** is the number of columns of data that are contained in this dataset + +**start** is the column (zero-based) where the data columns start for this dataset + +**file** is the file that contains the dataset. This parameter has been moved up from the features and labels subsections, because UCIFastReader requires the file be the same, and moving up a level is an excellent way to make sure this restriction is met. + +**labelDim** is the number of possible label values that are possible for the dataset, and belongs in any label section. In MNIST this value is 10, because MNIST is a number recognition application, and there are only 10 possible digits that can be recognized + +**labelMappingFile** is the path to a file that lists all the possible label values, one per line, which might be text or numeric. The line number is the identifier that will be used by CNTK to identify that label. In our example this file has been moved to the root level to share with other Command Sections. In this case, it’s important that the Evaluation Command Section share the same label mapping as the trainer, otherwise, the evaluation results will not be accurate. + +The final Command section in the example is the **mnistTest** section. This section takes the trained model and tests it against a separate test dataset. All the parameters that appear in this section also appeared in the **mnistTrain** section. + +### Command Line syntax + +The config file can be specified on the command line when launching the Computational Network Process (cn.exe): + +``` +cn.exe configFile=config.txt +``` + +This will load the requested configuration file, and execute any command section listed in the **command** parameters in the configuration file. In our example it will execute **mnistTrain** followed by **mnistTest**. + +### Configuration override + +It is common to have a configuration that can be used as a base configuration, and modify only a few parameters for each experimental run. This can be done in a few different ways, one of which is to override settings on the command line. For example if I wanted to override the model file path, I could simply modify my command line: + +``` +cn.exe configFile=config.txt stderr=c:\temp\newpath +``` + +this will override the current setting for stderr, which is defined at the root level of the configuration file, with the new value. If a parameter inside a command section needs to be modified, the section also needs to be specified. For example, if I wanted to change the minibatchSize for an experiment from the command line: + +``` +cn.exe configFile=config.txt mnistTrain=[minibatchSize=256] +``` + +or to modify the data file used for an experiment: + +``` +cn.exe configFile=config.txt mnistTrain=[reader=[file=mynewfile.txt]] +``` + +Another way to do this is with layered configuration files: + +### Layered Configuration Files + +Instead of overriding some parts of a configuration file using command line parameters, one can also specify multiple configuration files, where the latter files override the earlier ones. This allows a user to have a “master” configuration file, and then specify, in a separate configuration file, which parameters of the master they would like to override for a given run of CNTK. This can be accomplished by either specifying a ‘+’ separated list of configuration files, or by using the “configFile=” tag multiple times. The following are equivalent: + +``` +cn.exe configFile=config1.txt+config2.txt +cn.exe configFile=config1.txt configFile=config2.txt +``` + +If config2.txt contains the string “mnistTrain=\[reader=\[file=mynewfile.txt\]\]”, then both of these commands would be equivalent to: + +``` +cn.exe configFile=config1.txt mnistTrain=[reader=[file=mynewfile.txt]] +``` + +Note that the value of a variable is always determined by the last time it is assigned. It is also possible to mix command-line parameters, and layered configuration files, in arbitrary combinations. For example: + +``` +cn.exe configFile=config1.txt+config2.txt var1=value configFile=config3.txt +``` + +This would process these configuration parameters in the order they appear on the command line. + +### Including Configuration Files + +In addition being able to specify multiple configuration files at the command line, a user can “include” one configuration file within another. For example, if the first line of config2.txt was “include=config1.txt”, then simply running “cn.exe configFile=config2.txt” would be equivalent to running “cn.exe configFile=config1.txt+config2.txt” (where in this latter case, config2.txt doesn’t contain the “include” statement). Note that these include statements can appear anywhere inside a configuration file; wherever the include statement appears, that is where the specified configuration file will be “included”. Including a configuration file is equivalent to pasting the contents of that file at the location of the include statement. Include statements are resolved recursively (using a depth-first search), meaning that if configFileA.txt includes configFileB.txt, and configFileB.txt includes configFileC.txt, then the full chain will be resolved, and configFileC.txt will effectively be included in configFileA.txt. If a configuration file is included multiple times (eg, ‘A’ includes ‘B’ and ‘C’, and ‘B’ also includes ‘C’), then it will effectively only be included the first time it is encountered. + +### Stringize variables + +While layered configuration files allow users to reuse configuration files across experiments, this can still be a cumbersome process. For each experiment, a user might have to override several parameters, some of which might be long file paths (eg, ‘stderr’, ‘modelPath’, ‘file’, etc). The “stringize” functionality can make this process much easier. It allows a user to specify configuration like the following: + +``` +command=SpeechTrain stderr=$Root$\$RunName$.log speechTrain=[ modelPath=$Root$\$RunName$.model SGD=[ reader=[ features=[ type=Real dim=$DataSet1_Dim$ file=$DataSet1_Features$ ]]]] +``` + +Here, “Root”,“RunName”, “DataSet1\_Dim”, and “DataSet1\_Features” are variables specified elsewhere in the configuration (at a scope visible from the point at which they are used). When interpreting this configuration file, the parser would replace every string of the form “$VarName$” with the string “VarValue”, where “VarValue” represents the value of the variable called “VarName”. The variable resolution process is recursive; for example, if A=$B$, B=$C$, and C=HelloWorld.txt, then A would be resolved as “HelloWorld.txt”. +Notice that because it is equivalent for a user to specify the value of a variable in a configuration file vs. at the command line, the values for these variables can be specified in either location. Recall that the value of a variable is determined by the last time it is assigned, whether that happens to be in a configuration file, or on the command line. Thus, if “Root” is defined in config1.txt, but overridden at the command-line, then the value specified at the command-line would be the one used to resolve instances of $Root$ in configFile1.txt. One useful feature is that if ‘stderr’ or ‘modelPath’ point to directories which do not exist, these directories will be created by CNTK; this allows you to specify something like: “stderr=$Root$\\$RunName$\\$RunName$.log”, even if the directory “$Root$\\$RunName$” doesn’t exist. + +## User Reference + +This section is intended as a reference to all the possible configuration settings used in CNTK. See the example section for usage by example. + +### Parameter Search + +Config files are hierarchal organizations of Configuration Parameter Sets, a collection of name-value pairs. Any value that is expected to occur in a parameter set can alternately be defined at a higher level. The search for a parameter name will continue through parent parameter sets until it is resolved, or not found. + +Default values are often assigned to parameters, these values will be used should the search for the parameter fail. If no default is available, the parameter is a required parameter, and an error will occur if it is not provided. + +If a parameter occurs more than once in a given parameter set, the last occurrence of that value will have precedence. + +### Commands and actions + +There must be a top-level command parameter, which defines the commands that will be executed in the configuration file. Each command references a Command section of the file, which must contain an action parameter defining the operation that section will perform: + +``` +command=mnistTrain:mnistTest mnistTrain=[ action=train … ] mnistTest=[ action=eval … ] +``` + +This snippet will execute the **mnistTrain** section which executes the **train** action, followed by the **mnistTest** section. + +### Command sections + +The following actions are currently supported in the CNTK. The command sections that contain these action properties also require other configuration settings. The names contained in square braces (i.e. \[reader\]) are configuration sections, and values in curly braces (i.e. {true}) are default values used when the parameter is not specified: + +- **train** – Train a model + + - \[Reader\] – reader configuration section to read the dataset + + - \[Trainer\] – trainer configuration section, currently SGD is the only trainer supported + + - \[Network Builder\] – network builder configuration section, the method of creating the network + + - \[cvReader\] – (optional) reader configuration section for cross-validation data + + - makeMode-\[{true},false\] – start from scratch even if an interrupted training session exists (default true) + +- **test, eval** – Evaluate/Test a model for accuracy, usually with a test dataset + + - \[Reader\] – reader configuration section to read the test dataset + +- **createLabelMap** – creates a label mapping file from the dataset for readers that support it. Currently UCIFastReader is the only reader that supports this action. + + - section – the section name (usually a *train* section) which has the reader sub-section that will be used to generate the label mapping file. The labelMappingFile property in this reader section will be written to with the results of the map file generation. + + - minibatchSize – the minibatch size to use when creating the label mapping file + +- **edit** – execute an Model Editing Language (MEL) script. + + - editPath – the path to the Model Editing Language (MEL) script to be executed + + - ndlMacros - the path to the Network Definition Language (NDL) macros file that will be loaded and usable in the MEL script. + +- **testUnroll** – Evaluate/Test a model for accuracy, by unrolling it + + - \[reader\] - reader configuration section to read the test dataset + + - minibatchSize – the minibatch size to use when reading and processing the dataset + + - epochSize – {0} size of epoch, if not specified or set to zero entire dataset will be read once. + + - modelPath – path to the model file to evaluate + + - path2EvalResults – optional, if provided evaluation results will be dumped to this file + +- **adapt** – adapt an already Trained model, supports KL divergence regularization + + - \[Reader\] – reader configuration section to read the dataset + + - \[Trainer\] – trainer configuration section, currently SGD is the only trainer supported + + - \[cvReader\] – (optional) reader configuration section for cross-validation data + + - makeMode-\[{true},false\] – start from scratch even if an interrupted training session exists (default true) + + - originalModelFileName – file name for the model that will be adapted + + - refNodeName – name of the node in the computational network which will be used for KL divergence regularization (see SGD section for additional parameters required) + +- **cv** – Use Cross Validation to evaluate a series of epoch model for the best results + + - \[reader\] - reader configuration section to read the test dataset + + - minibatchSize – the minibatch size to use when reading and processing the dataset + + - epochSize – {0} size of epoch, if not specified or set to zero entire dataset will be read once. + + - modelPath – path to the model file to evaluate, epoch files have the epoch number appended to the end of this path name + + - crossValidationInterval – array of 3 integers identifying the starting epoch, epoch increment and final epoch to evaluate. + + - sleepTimeBetweenRuns – how many seconds to wait between runs + + - numMBsToShowResult – after how many minibatches should intermediate results be shown? + + - evalNodeNames – an array of one or more node names to evaluate + +- **write** – Write the output of a network to a file + + - \[reader\] - reader configuration section to read the dataset + + - \[writer\] – writer configuration section to the data writer for output data. If this value is not specified the outputPath parameter will be used. + + - minibatchSize – the minibatch size to use when reading and processing the dataset + + - epochSize – {0} size of epoch, if not specified or set to zero entire dataset will be read once. + + - modelPath – path to the model file we are using to process the input data + + - outputPath – output path to write file in a text based format. Either the writer config, or the outputPath will be used. + + - outputNodeNames – an array of one or more output node names to be written to a file + +- **dumpnode** – Dump the node(s) to an output file. Note: this can also be accomplished in MEL with greater control. + + - modelPath – path to the model file containing the nodes to dump + + - nodeName – the name of the node to be written to a file, if not specified all nodes will be dumped + + - outputFile – path to the output file, will be generated in the same file as the modelPath if not specified. + + - printValues – \[{true}, false\] prints the values associated with a node if applicable. + +The following table identifies the options for sub-section types associated with each of the action types: + +sub-section | Options | Description +---|---|--- +**Network Builder** | SimpleNetworkBuilder | Creates simple layer-based networks with various options + | NDLNetworkBuilder | Create a network defined in NDL (Network Description Language). This allows arbitrary networks to be created and offers more control over the network structure. +**Trainer** | SGD | Stochastic Gradient Descent trainer, currently this is the only option provided with CNTK +**Reader** | UCIFastReader | Reads the text-based UCI format, which contains labels and features combined in one file + | HTKMLFReader | Reads the HTK/MLF format files, often used in speech recognition applications + | BinaryReader | Reads files in a CNTK Binary format. It is also used by UCIFastReader to cache the dataset in a binary format for faster processing. + | SequenceReader | Reads text-based files that contain word sequences, for predicting word sequences. + | LUSequenceReader | Reads text-based files that contain word sequences, used for language understanding. | + +### Top Level Parameters + +- **command** – an array of Command sections (which contain action parameters) that should be executed + +- **precision** – \[float, double\], required parameter. Float values (32-bit floating point) is usually faster on most hardware, but less precise. This applies to all floating point values in CNTK. + +- **deviceId** – \[{auto}, cpu, \#, all, \*\#\], default is auto. Which hardware device should be used for an action. This is used at lower levels but is often defined at the top level. + +Value | Description +---|--- +auto | choose the best GPU, or CPU if no usable GPU is available. +cpu | use the CPU for all computation +0 | The device Identifier (as used in CUDA) for the GPU device you wish to use (1, 2, etc.) +all | Use all the available GPU devices (will use PTask engine if more than one GPU is present) +\*2 | Will use the 2 best GPUs, any number up to the number of available GPUs on the machine can be used. (will use PTask engine) + +- **stderr** – optional path to where the log files will be stored. This is a redirection of stderr to a file, if not specified the output will be output to the normal stderr device (usually the console). The path here defines the directory and the prefix for the log file. The suffix is defined by what commands are being run and will be appended to create the file name. For example if “stderr=c:\\abc” and “command=mnistTrain” the log file would be named “c:\\abc\_mnistTrain.log”. + +- **traceLevel** – \[{0}\] the level of output to stderr that is desired. The higher the number the more output can be expected. Currently 0-limited output, 1-medium output, 2-verbose output are the only values supported. + +### Network Builders + +Network builders provide a way to create a network. There are two network builders currently supported, SimpleNetworkBuilder and NDLNetworkBuilder. The sub-sections with one of these names define which network builder will be used for the train action. + +#### SimpleNetworkBuilder + +#### NDLNetworkBuilder + +The NDL (Network Description Language) Network Builder component takes a config section written in NDL and interprets it to create a model. For more information on NDL please refer to the Network Description Language section of the document. + +- **networkDescription –** (optional) file path of the NDL script to execute, the run parameter in this subsection can be used to override the run parameter in the file if desired. If there is no networkDescription file specified then the NDL is assumed to be in the same configuration file as the NDLNetworkBuilder subsection. + +- **run** – (optional) the section containing the NDL that will be executed. If using an external file the run parameter may already exist in that file and identifies the sections in that file that will be executed as NDL scripts. This parameter in NDLNetworkBuilder section will override those settings. If no networkDescription file is specified, a section with the given name will be searched for in the current configuration file. It must exist where a regular configuration search will find it (peer or closer to the root of the hierarchy) + +- **load** – (optional) the section(s) in the same file that contain NDL macros to be loaded. If specified t must exist where a regular configuration search will find it (peer or closer to the root of the hierarchy) + +- **ndlMacros** – (optional) path to an NDL macros file, normally defined at the root level so it could be shared with other Command Sections. This parameter is usually used to load a default set of NDL macros that can be used by all NDL scripts. + +### Trainers + +#### SGD + +### Readers + +The readers all share the same section name, which is **reader**. The **readerType** parameter identifies which reader will be used. + +``` +readerType=UCIFastReader +``` + +Each of the readers uses the same interface into CNTK, and each reader is implemented in a separate DLL. CNTK takes the **readerType** parameter value and appends “.dll” and dynamically loads that DLL to read data. This interface to data readers allows for new data formats to be supported in CNTK simply by writing a new reader component. For more information on the reader interface, and how to write a reader, refer to the Programmer documentation section. + +There are many parameters in the reader section that are used by all the different types of readers, and others are specific to a particular reader. There are sub-sections under the reader section which are used to define the data records to be read. For UCIFastReader these look like: + +``` +reader=[ readerType=UCIFastReader file=c:\cntk\data\mnist\mnist_train.txt features=[ dim=784 start=1 ] labels=[ dim=1 start=0 labelDim=10 ] +] +``` + +The sub-sections in the reader section identify the different data records. In our example they are named **features** and **labels**, though any names could be used. If NDL was used to create the network, these section names should match the names of the input nodes in the NDL network definition. These names will be used as the matrix names passed back from the reader and name matching ensures the correct records are assigned to the correct inputs. + +#### UCIFastReader + +UCIFastReader reads text-based UCI format data. The format of UCI data is a line of space-delimited floating point feature and label values for each data record. The label information is either at the beginning or the end of each line, if label information is included in the dataset. + +The following parameters can be used to customize the behavior of the reader: + +- **randomize** – \[{Auto}, None, \#\] the randomization range (number of records to randomize across) for randomizing the input. This needs to be an integral factor of the epochSize and an integral multiple of minibatch size. Setting it to Auto will let CNTK find something that works. + +- **minibatchMode** – \[{Partial},Full\] the mode for minibatchs when the end of the epoch is reached. In partial minibatch mode, if the remaining records are less than a full minibatch, only those read will be returned (a partial minibatch). I Full minibatch mode, no partial minibatches will be returned, instead those records will be skipped. + +Each of the data record sub-sections have the following parameters: + +- **dim** - the number of columns of data that are contained in this data record + +- **start** -the column (zero-based) where the data columns start for this data record + +- **file** - the file that contains the dataset. This parameter may be moved up to the reader section level to ensure the file is the same for all the sections, as UCIFastReader requires. + +In addition if the data record sub-section is defining labels the following parameters need to be defined: + +- **labelDim** – the number of possible label values that are possible for the dataset. For example, for the MNIST dataset this value is 10, because MNIST is a number recognition application, and there are only 10 possible digits that can be recognized + +- **labelMappingFile** – the path to a file that lists all the possible label values, one per line, which might be text or numeric. The line number is the identifier that will be used by CNTK to identify that label. This parameter is often moved to the root level to share with other Command Sections. For example, it’s important that the Evaluation Command Section share the same label mapping as the trainer, otherwise, the evaluation results will not be accurate. + +- **labelType** – \[{Category},Regression,None\] the type of label + +The UCIFastReader is a text-based reader, and though it is optimized for speed, is still significantly slower than reading binary files. To improve training speed that may be limited by the data reader the content of the dataset can be cached the first time through the dataset, and subsequently read from a binary cache saved to disk. UCIFastReader uses BinaryReader and BinaryWriter to make this cache and to read from it. BinaryReader and BinaryWriter support multiple datasets in a single file, so data read from multiple files can all be cached in a single file. Please refer to BinaryWriter to see a sample of how to setup a UCIFastReader with caching. + +#### HTKMLFReader + +HTKMLFReader reads files in the HTK/MLF format, which is used for speech datasets. The reader has two different modes, one for training and evaluating datasets and another for processing a dataset and producing another dataset. + +For training and evaluation the following need to be defined: + +- **randomize** – \[{auto},None,\#\] randomization range used for randomizing data. Auto automatically picks a randomization range, None does no randomization, and a specific number sets the range to that number. + +- **readMethod** – \[{blockRandomize},rollingWindow\] the method of randomization that will occur. Block randomize randomizes in a block format and does not require extra disk storage. RollingWindow creates a temporary file and randomizes data anywhere within a rolling window around the current file location. + +- **framemode** – \[{true}, false\] is the reader reading frames, or utterances + +- **minibatchMode** – \[{Partial},Full\] the mode for minibatchs when the end of the epoch is reached. In partial minibatch mode, if the remaining records are less than a full minibatch, only those read will be returned (a partial minibatch). I Full minibatch mode, no partial minibatches will be returned, instead those records will be skipped. + +- **readAhead** – \[true,{false}\] have the reader read ahead in another thread. NOTE: some known issues with this feature + +- **verbosity** – \[0-9\] default is ‘2’. The amount of information that will be displayed while the reader is running. + +- **addEnergy** – {0} the number of energy elements that will be added to each frame (initialized to zero). This only functions if readMethod=rollingWindow. + +- **unigram** – (optional) path to unigram file + +- **\[input**\] – subsection that holds all the input subsections + subsections with arbitrary names occur under the input subsection, which will contain: + + - **dim** – dimension of the input data + + - **type** – \[{real},Category\] type of input + + - **file** – input file path + +- **\[output\]** – subsection that holds all the output subsections + subsections with arbitrary names occur under the output subsection, which will contain: + + - **dim** – dimension of the input data + + - **type** – \[{real},Category\] type of input + + - **file** – input file path + + - **labelMappingFile** – (optional) state list to the labelMappingFile + + - **labelToTargetMappingFile** – (optional) filename for the labelToTargetMappingFile + + - **targetDim** – dimension of the target if labelToTargetMapping is desired. + +The following two sections can currently be used instead of input and output subsections if there is only one input and one output. However, the previous syntax is recommended + +- **\[features\]** – subsection defining the features data, must use this name + + - **dim** – dimension of the features data + + - **file** – path to the “.scp” script file that lists the locations of feature data + +- **\[labels\]** – subsection defining labels data, must use this name + + - **labelDim** – dimension of the labels data + + - **file** – path to the MLF file describing the labels + + - **labelMappingFile** – (optional) state list to the labelMappingFile + + - **labelToTargetMappingFile** – (optional) filename for the labelToTargetMappingFile + + - **targetDim** – dimension of the target if labelToTargetMapping is desired. + +For dataset processing the following parameters are used: + +- **\[input**\] – subsection that holds all the input subsections + subsections with arbitrary names occur under the input subsection, which will contain: + + - **dim** – dimension of the input data + + - **file** – input file path + +- **\[write\]** – subsection that holds all the output subsections + subsections with arbitrary names occur under the output subsection, which will contain: + + - **dim** – dimension of the input data + + - **path** – output file path + + - **ext** – {mfc} file extention to use for output files + + - **type** – (optional) if type=scaledLogLikelihood output will be scaled by Prior + + - **nodeName** – node name for output to be captured + + - **scpFile** – (optional) file name for SCP file if desired + +#### SequenceReader + +SequenceReader is a reader that reads text string. It is mostly often used for language modeling tasks. An example of the text string is as follows: + +``` + pierre N years old will join the board as a nonexecutive director nov. N mr. is chairman of n.v. the dutch publishing group +``` + +Symbol </s> is used to denote both beginning and ending of a sentence. However, this symbol can be specified by beginSequence and endSequence. + +The parameters used for the SequenceReader are shared with other reader types. However, it has some unique parameters as follows: + +- randomize – \[None, Auto\] the mode for whether doing sentence randomization of the whole corpus. + +- Nbruttsineachrecurrentiter – this set the maximum number of allowed sentences in each minibatch. + +- Wordclass – word class information. This is used for class-based language modeling. + +- File – the corpus file. + +A subsection is for input label information. + +- lableIn – the section for input label. It contains the following setups + + - labelDim – the input vocabulary size + + - beginSequence – the sentence beginning symbol + + - endSequence – the sentence ending symbol + +- labels – the section for output label. In the language modeling case, it is the same as labelIn. + +#### LUSequenceReader + +LUSequenceReader is similar to SequenceReader. It however is used for language understanding tasks which have input and output strings that are different. The content of an example file is listed below + +``` +BOS O i O want O to O fly O from O boston B-fromloc.city_name at O 1110 B-arrive_time.time in O the O morning B-arrive_time.period_of_day EOS O +``` + +consists of some unique setups as follows: + +The LUSequenceReader assumes that the last column is the label and all other columns are inputs. The beginning and ending of a sentence are specified using beginning and ending symbols. In the above example, they are BOS and EOS, respectively, for the beginning and ending symbols. + +The LUSequenceReader has some unique setups as follows: + +- wordContext – this specifies a context window. For example, wordContext=0:1:2 specifies a context window of 3. In this context window, it reads input at a current time, the next time and the time after the next time. Another example would be wordContext=0:-1. In this example, LUSequencReader reads a context window of 2 that consist of the current input and the immediate last input. + +- Nbruttsineachrecurrentiter – this specifies the maximum number of sentences in a minibatch. + +- Unk – this specifies the symbol to represent unseen input symbols. Usually, this symbol is “unk”. + +- Wordmap – this specifies a file that maps inputs to other inputs. This is useful if the user wants to map some inputs to unknown symbols. For example: + +``` + buy buy trans +``` + +- File – the corpus file + +A subsection is for input label information. + +- lableIn – the section for input label. It contains the following setups + + - usewordmap – \[Ture, False\] specifies if using word map to map input words to other input words. + + - beginSequence – the sentence beginning symbol + + - endSequence – the sentence ending symbol + + - token – token file contains a list of input words. Their orders are not important. + +- labels – the section for output label. In the language modeling case, it is the same as labelIn. + + - Token – token file contains a list of output labels. Their order is not important as long as the tokens are unique. + +#### BinaryReader + +BinaryReader is a reader that uses a hierarchal file format to store potentially multiple datasets in an efficient way. It uses memory mapped files with a moving window of viewable data to support files larger than can be held in memory at once. More details about the file format are in the BinaryWriter Section. + +The parameters used for the binary reader are quite simple as most of the required information concerning the file is contained in the file headers. The binary reader will also be called when a configuration is setup to cache UCIFastReader if the binary cached file exists. + +The following parameters can be used to customize the behavior of the reader: + +- **minibatchMode** – \[{Partial},Full\] the mode for minibatchs when the end of the epoch is reached. In partial minibatch mode, if the remaining records are less than a full minibatch, only those read will be returned (a partial minibatch). I Full minibatch mode, no partial minibatches will be returned, instead those records will be skipped. + +- **file** – array of files paths to load. Each file may contain one or more datasets. The dataset names used when the file was created will be used when the file is read. + +### Writers + +Writers are used in a similar way to Readers in CNTK. Writers are often implemented in the same DLL as the Reader for the same format. Just as in the reader case, the writer is dynamically load based on the name in the writerType parameter: + +``` +writerType=BinaryReader # NOTE: BinaryReader.dll also implements BinaryWriter +``` + +#### BinaryWriter + +BinaryWriter is an implementation of a hierarchal file format the mirrors the configuration file. I uses memory mapped files to enable large files that do not fit in memory to be written and read using a moving window into the file. The binary writer is also used as a Cache mechanism for UCIFastReader to allow for much faster access to data after the dataset has been read once. + +The following is an example of a BinaryWriter definition. Since it is most commonly used as a cache for UCIFastReader, this definition is show as a UCIFastReader cache. The parameters needed for BinaryWriter are in bold type below: + +``` + # Parameter values for the reader with cache reader=[ # reader to use readerType=UCIFastReader # if writerType is set, we will cache to a binary file # if the binary file exists, we will use it instead of parsing this file writerType=BinaryReader miniBatchMode=Partial randomize=Auto windowSize=10000 #### write definition wfile=c:\data\mnist\mnist_train.bin #wsize - inital size of the file in MB # if calculated size would be bigger, that is used instead wsize=256 #wrecords - number of records we should allocate space for in the file # files cannot be expanded, so this should be large enough. wrecords=60000 features=[ dim=784 start=1 file=c:\data\mnist\mnist_train.txt ### write definition #wsize=200 #wfile=c:\data\mnist\mnist_train_features.bin sectionType=data ] labels=[ dim=1 start=0 file=c:\data\mnist\mnist_train.txt labelMappingFile=c:\temp\labels.txt labelDim=10 labelType=Category #### Write definition #### # sizeof(unsigned) which is the label index type #wsize=10 #wfile=c:\data\mnist\mnist_train_labels.bin elementSize=4 wref=features sectionType=labels mapping=[ #redefine number of records for this section, #since we don't need to save it for each data record wrecords=10 #variable size so use an average string size elementSize=10 sectionType=labelMapping ] category=[ dim=10 #elementSize=sizeof(ElemType) is default sectionType=categoryLabels ] ] ] +] +``` + +The example above shows all the parameters necessary to create a Binary file with BinaryWriter: + +- **writerType** – The definition of the writer to use. The CNTK code takes this name and appends “.DLL” to dynamically load the DLL and access the writer implementation. BinaryWriter is implemented in the same DLL as BinaryReader, so BinaryReader is the correct value for this setting. + +- **windowSize** – the size of the moving window that will be used to access the data in the file. BinaryReader and BinaryWriter both us memory mapped files to support extremely large datasets which cannot be contained in memory. This window size is the minimum amount that will be present in memory at one time. + +- **wfile** – the filename and path to the binary file. This can appear at the base of the hierarchy (as in this case) which means all sub-sections defined will be saved in a single file. Alternately, separate files can be defined for different sub-sections. The commented out sections in the feature and labels sections show how separate binary files could be saved. Simply comment out the based definition and uncomment the sub-section definitions and two separate files will be created. + +- **wsize** – used in conjunction with wfile, defines how large (in Megabytes) the initial filesize should be. It must be large enough to contain all data or an error will occur. Once the file is completely written it will shrink down to its actual size. + +- **wrecords** – the number of records that will be written to disk. Different subsections can override this value if the number of records for a subsection are different (as is the case for the label mapping table subsection) + +> Each subsection in the configuration will create a corresponding subsection in the binary file. The name used for the subsection in the configuration file will be saved for each subsection, and will be name used to refer to the data when it is read later. Each subsection must have a few parameters: + +- **sectionType** – the type of section this config section is describing. The possibilities are: + +**Section Type** | **Description** +---|--- +Data | data section (floating point values) +Labels | label data (floating point values). For category labels the integer part of this floating point value will be interpreted as an index into the label mapping table. +LabelMapping | label mapping table (array of strings) +Stats | data statistics. can compute the following statistics across the entire dataset: `sum:count:mean:variance:stdDev:max:min:range:rootMeanSquare` +CategoryLabels | labels in category format (floating point type - all zeros with a single 1.0 per column that matches the mapping table) This format is directly usable in this form for training, so it can be stored in this form. + +- **elementSize** – size in bytes of the elements. If this value is not specified, it will default to the sizeof(ElemType), where ElemType is either float or double based on the precision specified in the configuration file. + +- **wref** – A reference to the section that holds the data referenced by these labels. It is often best to store both the labels and the data in the same file so they remain associated with each other. If separate label and data binary files that were generated at different times are used together an error will occur (as they are likely not aligned to each other) + +- **dim** – used for categoryLabels format sections. It contains the number of columns of category data, which will contain all zeros except for a single 1.0 for each row. + +## Programmer Reference + +This section covers topics that are of interest to those who wish to modify the code and use the provided classes in there code. The first section covers configuration files and their use from a programmer’s perspective. The second section covers Reader/Writer interfaces for data input/output. The third section covers the CNTKMath library, and the last section covers using PTask to enable a computation node to participate in multi-GPU processing. + +### Configuration Files + +Configuration files, and easy manipulation of these files is a main feature of CNTK. The configuration files make the users life much easier, and the programmer interface is also quite simple to use. The programmer interface to the configuration files is contained in a few C++ classes and focuses on “just-in-time” evaluation of the parameter values. The idea is simple, leave the configuration values in string format until they actually need to be parsed into some other form. + +#### Configuration Formats + +The following is a summary of the different formats the configuration classes can support: + +Config Type | C++ type | Format | Notes +---|---|---|--- +integer | int, long, short, size_t | [-]###### | Optional sign and numeric digits. All signed and unsigned integer numeric types +floating=point | float, double | [-]####.#####[e{+-}###] | Numeric digits with a decimal point, optional sign,optional scientific notation +string | std::wstring, std::string | Any valid character | If contained in an array or dictionary and the default separator is contained in the string (i.e. c:\temp in an array) use alternate separator. +boolean | bool | T/True/1, F/False/0 | True or False values, may be specified by existence or absence of a boolean parameter with no ‘=’ or value after the parameter name +array | ConfigArray | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Config typeC++ typeFormatNotes
integerint, long, short, size_t[-]######Optional sign and numeric digits. All signed and unsigned integer numeric types
floating-pointfloat, double[-]####.#####[e{+-}###]Numeric digits with a decimal point, optional sign, optional scientific notation
stringstd::wstring, wtd::stringAny valid characterIf contained in an array or dictionary and the default separator is contained in the string (i.e. c:\temp in an array) use alternate separator.
booleanbool +
    +
  • T, True, 1
    • +
  • F, False, 0
    • +
+ 		True or False values, may be specified by existence or absence of a boolean parameter with no ‘=’ or value after the parameter name
arrayConfigArray +
    +
  • value:value:value
    • +
  • value:value*#:value
    • +
  • {|value|value|value}
    • +
  • {|value|value*#|valve}
    • + +
  • + +
    {
value
value
value*#
}
    + +
    • + +
+ 		Multiple values in an array are separated by colons ‘:’. A value may be repeated multiple times with the ‘*’ character followed by an integer (the # in the examples). Values in an array may be of any supported type and need not be uniform. The values in a vector can also be surrounded by curly braces ‘{}’, braces are required if new lines are used as separators. An alternate separation character can be specified immediately following the opening brace if desired.
dictionaryConfigParameters +
    +
  • + +
    parameter1=value1;
+parameter2=value2;
+boolparam
    + +
    • +
  • + +
    [#parameter1=value1#parameter2=value2#boolparam]
    + +
    • +
  • + +
    [
parameter1=value1
parameter2=value2
boolparam
]
    + +
    • +
+ 		Multiple parameters grouped together in a dictionary. The contents of the dictionary are each named values and can be of different types. Dictionaries can be used to create a configuration hierarchy. When specified on the same line a ‘;’ semicolon is used as the default separator. The values can optionally be surrounded by square braces ‘[]’. Braces are required when using newlines as separators in a config file. An unnamed dictionary is also allowed in the case of an array of dictionaries. An alternate separation character can be specified immediately following the opening brace if desired.
+ + + +#### Configuration classes + +There are three main classes that are used to access configuration files. *ConfigParameters* and *ConfigArray* contain instances of *ConfigValue*. The main definitions are as follows: + +``` +class ConfigValue : public std::string class ConfigParameters : public ConfigParser, public ConfigDictionary class ConfigArray:public ConfigParser, public std::vector +``` + +##### ConfigValue + +The key class that is used to allow the JIT evaluation of configuration strings is the *ConfigValue* class. This class inherits from std::string, and stores an optional configuration path string, which is mainly used for error messages. It contains many cast operators that do the actual parsing of the string value into the target type on demand. + +##### ConfigParameters + +Configuration files are mainly made up of a hierarchy of Configuration Sets (*ConfigParameters*), which are dictionaries of *ConfigValue*. This class provides access to the configuration values and automatically searches up the hierarchy of ConfigParameter classes if a value is not found on the current level. The hierarchy is maintained by the order of class instantiations on the stack. ConfigParameters should only be created on the stack. + +In configuration files the ‘name=value’ named pair are usually separated by newlines. However, they also can be separated by other characters and placed on the same line. The default separator for ConfigParmeters is a ‘;’ (semicolon). This can be overridden by placing the alternate separator character immediately following the opening brace. For example ‘\[|’ causes ‘|’ to be the separator for that ConfigParameter instance: + +``` +name=[|parameter1=value1|parameter2=value2|parameter3=value3] +``` + +There are two types of ConfigParameters type accessors: + +**value = config(“name”)** – which will return the named parameter cast to the type of the value parameter. If the named configuration parameter does not exist an exception will be thrown. + +**value = config(“name”, “defaultValue”)** – returns the named parameter, if it doesn’t exist returns defaultValue. + +**config.Exists(“name”)** – returns the existence of a named value in the *ConfigParameters* instance. + +To insert elements into a *ConfigParameters* variable the following methods can be used: + +**config.Insert(“name”, value)** – insert a new value into the existing *ConfigParameters* instance. If the value already exists, it will be replaced unless the value is itself another *ConfigParameters* instance, or string representation surrounded by ‘\[\]’ square braces, in which case the parameters are “merged”. + +**config.Insert(value)** – insert the passed string into the dictionary, it is expected to be in ‘name=value’ format. + +##### ConfigArray + +This type inherits from std::vector<ConfigValue> and can be used to hold arrays of values. Since ConfigValue is a just-in-time evaluation type. The values in the array need not be homogeneous, as long as the code that evaluates the array in the end knows how to interpret the values. + +In a ConfigArray the array values are normally separated by the default separator character, which is a ‘:’ (colon). However, they also can be separated by other characters and or place each value on a separate line. The default separator can be overridden by placing the alternate separator character immediately following the opening brace. For example ‘{|’ causes ‘|’ to be the separator for a ConfigArray instance: + +``` +array={|c:\temp\new.txt|12*3|1e-12} +``` + +A value may be repeated multiple times with the ‘\*’ character followed by an integer. In the above example, there are 5 elements in the array, with three ‘12’ values occupying the center 3 positions. + +The values in a ConfigArray can be accessed just like values in a normal std::vector type, but the automatic type conversion of ConfigValue will still be in affect. + +#### Other Useful Configuration Methods + +Another convenient method that exists for both ConfigParameters and ConfigArray types is a method to load a config file into an existing instance. It is implemented in ConfigParser, which both of these classes inherit from: + +``` +config.LoadConfigFile(path.c_str()); +``` + +To use this method with a ConfigArray, the file can simply contain a list of values each on their own line and they will be read into the ConfigArray. More complex types can also be contained in the array using the config syntax discussed earlier in this document. An array can contain other arrays as well as ConfigParameters. + +ConfigArray instances can also be converted to argvector<T> instances simply by assigning them. Care should be taken to assign to a local variable, and not just passing as a parameter due to lifetime issues, as follows: + +``` +ConfigArray configLearnRatesPerMB = config("learningRatesPerMB"); argvector learnRatesPerMB = configLearnRatesPerMB; +``` + +ConfigParameters and ConfigArray instances are very flexible, but require parsing every time a value is accessed. argvector<T> ,on the other hand, parses once and then accesses values as a standard vector. + +#### Configuration Program Example + +Some sample code that would parse the example configuration file given at the beginning of this document follows. This is a revised version of actual code in CNTK: + +``` +#include "commandArgUtil.h" // process the command void DoCommand(const ConfigParameters& config) { ConfigArray command = config("command"); for (int i=0; i < command.size(); i++) { //get the configuration parameters that match the command ConfigParameters commandParams=config(command[i]); ConfigArray action = commandParams("action","train"); // determine the action to perform, and do it for (int j=0; j < action.size(); j++) { if (action[j] == "train") DoTrain(commandParams); else if (action[j] == "test" || action[j] == "eval") DoEval(commandParams); else throw runtime_error("unknown action: " + action[j] + " in command set: " + command[i]); } } } void DoTrain(const ConfigParameters& config) { ConfigParameters configSGD=config("SGD"); ConfigParameters readerConfig = config("reader"); IComputationNetBuilder* netBuilder = NULL; ConfigParameters configNDL = config("NDLNetworkBuilder"); netBuilder = (IComputationNetBuilder*)new NDLBuilder(configNDL); DataReader* dataReader = new DataReader(readerConfig); ConfigArray learningRatesPerMBStr = configSGD("learningRatesPerMB", ""); floatargvector learningRatesPerMB = learningRatesPerMBStr; ConfigArray minibatchSize = configSGD("minibatchSize", "256"); size_t epochSize = configSGD("epochSize", "0"); if (epochSize == 0) { epochSize = requestDataSize; } size_t maxEpochs = configSGD("maxEpochs"); wstring modelPath = configSGD("modelPath"); int traceLevel = configSGD("traceLevel", "0"); SGD = sgd(learningRatesPerMB, minibatchSize, epochSize, maxEpochs, modelPath, traceLevel); sgd.Train(netBuilder, dataReader); delete netBuilder; delete dataReader; } +``` + +The code above is very easy to code, you simply delare a config, or basic type variable on the stack and assign something from a ConfigParameters class to that variable (i.e. int i = config(”setting”,”default”). Both parameters with defaults and those that don’t are used in the sample code above. The ConfigValue class takes care of parsing the value to be the correct type, and is returned by config() references above. + +The Config classes are meant to be used on the stack as shown in this example. Storing them in member variables or allocating using ‘new’ or other methods is not supported. The reason for this is an internal pointer is used to link to parent instances of config classes. This allows us to trace “up the stack” and look at all the config parameters that exist at a higher level. Since our search traverses up the stack, we need to ensure that all the parent configuration classes still exist, which is guaranteed if all config parameters are stack allocated and have lifetimes that extend past any children. + +### Data Interfaces + +CNTK was designed with the idea that data input and output would need to transpire in many different formats. So data interfaces were designed in an attempt to cover various data needs. Currently there are two data interfaces designed, one for input and the other for output called IDataReader and IDataWriter respectively. The reader/writer code is housed in separate DLLs that which are dynamically loaded to provide data services. This allows the user to simply change a configuration setting and have a different reader provide the data. + +Other possible scenarios are also enabled by using a common interface, for example one reader/writer can act as a cache for another slower reader. UCIFastReader is a text-based reader, and though it is very fast, there is still a significant amount of overhead to parsing, so BinaryWriter/BinaryReader can act as a cache for UCIFastReader. The caching code is currently implemented in UCIFastReader. + +The five readers and one writer provided with CNTK all use these same interfaces and each is housed in its own DLL. CNTK loads the DLL and looks for exported functions that will return the interface of interest. The functions are defined as follows: + +``` +extern "C" DATAREADER_API void GetReaderF(IDataReader** preader); extern "C" DATAREADER_API void GetReaderD(IDataReader** preader); extern "C" DATAWRITER_API void GetWriterF(IDataWriter** pwriter); extern "C" DATAWRITER_API void GetWriterD(IDataWriter** pwriter); +``` + +each reader or writer DLL exports the appropriate functions, and will return the interface when called. The following sections defined the interfaces: + +#### Reader Interface + +``` +/ Data Reader interface // implemented by DataReader and underlying classes template class DATAREADER_API IDataReader { public: typedef std::string LabelType; typedef unsigned LabelIdType; virtual void Init(const ConfigParameters& config) = 0; virtual void Destroy() = 0; virtual void StartMinibatchLoop(size_t mbSize, size_t epoch, size_t requestedEpochSamples=requestDataSize) = 0; virtual bool GetMinibatch(std::map*>& matrices) = 0; virtual const std::map& GetLabelMapping(const std::wstring& sectionName) = 0; virtual void SetLabelMapping(const std::wstring& sectionName, const std::map& labelMapping) = 0; virtual bool GetData(const std::wstring& sectionName, size_t numRecords, void* data, size_t& dataBufferSize, size_t recordStart) = 0; virtual bool DataEnd(EndDataType endDataType) = 0; // Recursive network specific methods virtual size_t NumberSlicesInEachRecurrentIter() = 0; virtual void SetNbrSlicesEachRecurrentIter(const size_t) = 0; virtual void ReloadLabels() = 0; virtual void SaveLabels() = 0; virtual void SetSentenceEndInBatch(vector &sentenceEnd)=0; }; +``` + +The methods are as follows: + +- **Init** – initialize the reader from a set of ConfigurationParameters. See the reader documentation for elements of readers that should be similar across all types. + +- **Destroy** – the “destructor” for the reader. Since we are being called from external code we use an explicit method rather than a normal c++ destructor, but the intent is the same. + +- **StartMinibatchLoop** – Starts the minibatch loop with the following parameters: + + - **mbSize** – minibatch size + + - **epoch** – epoch number we are currently processing + + - **requestedEpochSize –** the number of records in an epoch. This value is not required to be the same as the dataset size, it also could be larger or smaller. If the datasetSize is not known the constant requestDataSize can be used to request a single pass through the dataset equal the epochSize. + +- **GetMinibatch –** Get the minibatch data + + - **matrices –** a dictionary that maps from the matrix name to the actual matrix. The names of the matrices in the dictionary should be equal to the subsections in the reader configurations. + + - returns – true if there is more data, false if end of epoch is reached. + +- **GetLabelMapping –** Get the label map from the reader + + - **sectionName –** the section which contains the label map, if applicable. Some readers do not need a section name if only one label map is supported + + - **returns –** the label map from labelId (integer) to label (std::string) + +- **SetLabelMapping –** Set the label map for the reader + + - **sectionName –** the section which is assigned to the label mapping, if applicable. Some readers do not need a section name, they generally only support one label map. + + - **labelMapping –** the labelMap that is being set + +- **GetData –** Get some data from a predefined section + + - **sectionName –** the section which contains the data + + - **numRecords –** the number of records to read + + - **data –** pointer to the data buffer, must have enough room to hold the data + + - **dataBufferSize –** size of the buffer, if zero is passed in, or null is passed for the data pointer the number of bytes required in the buffer will returned in this variable + + - **recordStart –** the record to start reading from + +- **DataEnd** – Are we at the end of a data section? + + - **endDataType** – type of ending we are checking (Dataset, Epoch, Sentence) + + - **returns** – true or false + +- NumberSlicesInEachRecurrentIter + +- SetNbrSlicesEachRecurrentIter + +- ReloadLabels + +- SaveLabels + +- SetSentenceEndInBatch + +#### Writer Interface + +``` +// Data Writer interface // implemented by some DataWriters template class DATAWRITER_API IDataWriter { public: typedef std::string LabelType; typedef unsigned LabelIdType; virtual void Init(const ConfigParameters& config) = 0; virtual void Destroy() = 0; virtual void GetSections(std::map& sections) = 0; virtual bool SaveData(size_t recordStart, const std::map& matrices, size_t numRecords, size_t datasetSize, size_t byteVariableSized) = 0; virtual void SaveMapping(std::wstring saveId, const std::map& labelMapping) = 0; }; +``` + +The methods are as follows: + +- **Init** – initialize the writer from a set of ConfigurationParameters. See the writer documentation for an example of BinaryWriter. + +- **Destroy** – the “destructor” for the writer. Since we are being called from external code we use an explicit method rather than a normal c++ destructor, but the intent is the same. + +- **GetSections** – Gets the sections that are available in the file to write to: + + - **sections** – sections that are defined to write to + +- **SaveData** – Save data to the file + + - **recordStart –** the record to start reading from + + - **matrices –** a dictionary that maps from the section name to the data pointers. The names of the sections in the dictionary should be equal to the sections returned by GetSections(). + + - **numRecords –** number of records to write + + - **datasetSize –** size of the dataset + + - **byteVariableSized –** for variable sized data, the number of bytes used by the data + +- **SaveMapping –** save the mapping table + + - **saveId –** the section name or other id where the mapping will be saved + + - **labelMapping –** the label map from labelId (integer) to label (std::string) + +### CNTKMath Library + +The CNTK Math library is implemented in the DLL CNTKMath.dll and provides a library of math routines for dealing with matrix operations. The library supports CPU and GPU computation with sparse and dense matrix formats. + +The library contains a wrapper class called Matrix<ElemType> (where ElemType is float or double) that hides the differences between the multiple matrix implementations and takes care of data transfers between the GPU and CPU. GPUs and CPUs have different memory spaces, and copying data between them is necessary to access or modify the data from either device. The library attempts to keep data on the GPU as much as possible if a GPU is being used. + +When data is accessed or modified from the CPU, if the data is currently on the GPU the matrix will automatically be relocated to the CPU, and relocated back when the GPU attempts to access or modify the data. Currently the entire matrix object is transferred, so care should be taken when accessing matrix data from the CPU. + +The library uses BLAS libraries from NVidia for the GPU (CuBLAS) and AMD for the CPU (AMCL). Other third party libraries that are used include CuRand (for random number generation) and CuSparse (for sparse matrix implementations), + +### PTask support + +PTask is a library used in CTNK to enable multiple GPU computation on a single machine. PTask uses the concept of a “Tasks organized in a filter graph. It allows fully asynchronous operation of the tasks, each only depending on inputs being available to execute. PTask distributes the tasks across the available hardware and handles data transfers. + +CTNK is organized in a different fashion with Computation Nodes. However, each node has two methods that do all the computation work: EvaluateThisNode() and ComputeInputPartial(), which can be used as the “Tasks”. However, since Tasks can be executed asynchronously, they need to be stateless. To enable these methods as task a static version of each method that takes all inputs and outputs as parameters are created. The class methods simply call these “Task” functions with the class variables for their implementation. + +The PTaskGraphBuilder component takes a computation network and transforms it into a filter graph. In order to do this work it requires the parameter description for each of the tasks. Since C++ does not have a reflection mechanism as in available in C\# and some other languages, a class method has been introduced to ComputationNode to provide this information. The method GetPTaskDescriptor() provides this information to PTaskGraphBuilder so it can build the graph. + +The following is an example of a GetPTaskDescriptor() implementation. This function returns a TaskDescriptor class containing all the parameter and other information necessary to build the filter graph for a particular node. This node is the “TimesNode” and does a matrix multiply. The following implementation of the two important member functions are: + +``` +virtual void EvaluateThisNode() { EvaluateThisNodeS(FunctionValues(), Inputs(0)->FunctionValues(), Inputs(1)->FunctionValues()); } virtual void ComputeInputPartial(const size_t inputIndex) { if (inputIndex > 1) throw std::invalid_argument("Times operation only takes two inputs."); if (inputIndex == 0) //left derivative { ComputeInputPartialLeft(Inputs(1)->FunctionValues(), Inputs(0)->GradientValues(), GradientValues()); } else //right derivative { ComputeInputPartialRight(Inputs(0)->FunctionValues(), Inputs(1)->GradientValues(), GradientValues()); } } +``` + +The GPTaskDescriptor() method describes the necessary parameter information for each method. Each node has a FunctionValue matrix and a GradientValue matrix associated with it, and the descriptor methods identify which values are needed, and if they come from the current node or one of its inputs as follows: + +``` +// GetTaskDescriptor - Get a task descriptor for this node // taskType - task type we are generating a task for virtual TaskDescriptor* GetPTaskDescriptor(TaskType taskType, size_t inputIndex=0) const { TaskDescriptor* descriptor = new TaskDescriptor(this, taskType, inputIndex); switch(taskType) { case taskComputeInputPartial: descriptor->FunctionParam(1-inputIndex, paramOptionsInput); descriptor->GradientParam(inputIndex, paramOptionsInput | paramOptionsOutput | paramOptionsInitialize); descriptor->GradientParam(); descriptor->SetFunction( (inputIndex?(FARPROC)ComputeInputPartialRight:(FARPROC)ComputeInputPartialLeft)); break; case taskEvaluate: descriptor->FunctionParam(); descriptor->FunctionParam(0, paramOptionsInput); descriptor->FunctionParam(1, paramOptionsInput); descriptor->SetFunction((FARPROC)EvaluateThisNodeS); break; default: assert(false); throw std::logic_error("Unsupported task requested"); } return descriptor; } +``` + +For the Evaluate method, the first parameter is an output to the FunctionValue matrix of the current node. + +``` +descriptor->FunctionParam(); +``` + +The default value for this method is “current node, output” so no parameters are needed. The next two parameters are inputs and are the function values from the two inputs: + +``` +descriptor->FunctionParam(0, paramOptionsInput); descriptor->FunctionParam(1, paramOptionsInput); +``` + +The last call passes a pointer to the task function: + +``` +descriptor->SetFunction((FARPROC)EvaluateThisNodeS); +``` + +and the descriptor is complete. The two ComputeInputPartial task function parameters are very similar. Depending on the inputIndex, the values are switched. The first parameter is an input of the function value of one of the inputs, and the second is an output value to the gradient matrix of the other input: + +``` +descriptor->FunctionParam(1-inputIndex, paramOptionsInput); descriptor->GradientParam(inputIndex, paramOptionsInput | paramOptionsOutput | paramOptionsInitialize); +``` + +The second parameter is interesting because it is required to retain it value from one call to the next, this is done in a filter graph by having a parameter be input and output at the same time, meaning it updates itself. There is a clear distinction between values that need to be maintained and those that are transcient in a filter graph, and this idiom is how we instruct PTaskGraphBuilder to retain the value. The Initialize option is also necessary so on the first iteration the matrix will be cleared out (zeros). + +The last parameter is the gradient matrix of the current node, and is an input (defaults for this function). + +``` +descriptor->GradientParam(); +``` + +Lastly, the task functions must be set. They are different based on which input we are computing the gradient for: + +``` +descriptor->SetFunction((inputIndex ? (FARPROC)ComputeInputPartialRight : (FARPROC)ComputeInputPartialLeft)); +``` + +For reference the three task functions are as follows: + +``` +static void WINAPI ComputeInputPartialLeft(Matrix& inputFunctionValues, Matrix& inputGradientValues, const Matrix& gradientValues) static void WINAPI ComputeInputPartialRight(Matrix& inputFunctionValues, Matrix& inputGradientValues, const Matrix& gradientValues) static void WINAPI EvaluateThisNodeS(Matrix& functionValues, const Matrix& input0, const Matrix& input1) ``` + +### NDL classes and processing + +The ability to describe a network architecture in NDL (Network Description Language) is one of the major features of CNTK. However, it is not immediately obvious to the developer looking at the code how it all works. This is meant to shed a little light on the inner workings of NDL processing in CNTK. + +NDL is based on the same configuration parser that is used for config files and MEL (Model Editing Language). While this is convenient to share code, it also makes things a little less clear when viewing the code. The configuration file classes, MEL class, and NDL classes all inherit from ConfigParser, which provides the basic parsing, bracket matching, and other common features (quote handling, etc.). The parsing engine implemented in ConfigParser calls back to a virtual method called ParseValue() when it has a token that needs to be interpreted. So ParseValue() in the NDLScript class is the main location where interpretation of tokens takes place. + +NDL supports Macros, which makes things much more convenient, but a bit messier for the developer to deal with. All the macros are parsed and stored in a Global script so they can be accessed by any NDL script. It also means that you don’t want to load or define a set of macros more than once, or you will get “function already exists” errors. + +The processing of NDL proceeds through the following phases: + +1. Parsing the script + +2. Evaluation – initial pass – create ComputationNodes for all NDLNodes that require it + +3. Evaluation – second pass – connect the inputs of the ComputationNodes + +4. Validate the network – This also has the side effect or allocating all the matrix classes to their correct dimensions, and computing dimensions derived from input nodes + +5. Evaluation – final pass – All operations that must have the matrix values present occur here. For example, matrix initialization happens here + +There is a helper class in NDLUtil, which will take care of executing through these various phases. It also tracks how far along in the current processing phase a script has progressed. Processing can continue statement by statement as needed. This is the want in-line NDL is processed. + +Now a little more detail is in order for these layers: + +#### Parsing + +- The script in question is first parsed, and as each Macro Definition, macro call, parameter, variable, or function call is encountered an NDLNode is created. This NDLNode describes the entity and a reference is stored in the NDLScript class which owns it so it can be freed at some later point in time. + +- If the NDLNode is an executable statement, it will be added in order to the list of statements to execute + +- All variable names used in a script will be added to a symbol table in the NDLScript class that owns the NDLNode. + +- If the NDLNode is a macro or function call its parameters will be parsed and added to a parameter list in the NDLNode. Note that parameters may actually be other function and macro calls. The actual parameter names used in the call and the names used in the macro that will be called are recorded. + +- If the NDLNode is a macro, it will have its own NDLScript, and contain its own list of executable statements. It will also be stored in the Global Script repository (just a global NDLScript class) + +#### Evaluation – initial pass + +- The main purpose of this pass is to create a Computation Node for every NDL node that requires one. Effectively every “Function call” in NDL maps to a Computation Node. The full “dot path” will be the name of the node in the Computational Network. + +- Each pass evaluates the entire script, but only certain actions are performed based on what pass is being executed. So though all the parameters are evaluated in NDL, only Function Calls will create computation nodes. + +#### Evaluation – second pass + +- This pass goes through the entire evaluation process again, but this time all Computation Network nodes should already exist. The main purpose of this pass is to hook up all the inputs between nodes. + +- Doing this in a separate pass allows nodes to be referenced before they are actually defined in the NDL Script. This is a necessary feature for Recursive Neural Networks with a DelayNode. + +- Having a separate pass allows inline-NDL to support DelayNodes, and “Just-in-time” execution of inline-NDL can occur, where depending on the MEL command that is being executed, we can evaluate the NDL to the appropriate level. Some MEL commands need a “final-state” network to excute safely, others may only require the initial pass to be completed. The initial pass is executed when the inline-NDL is encountered, and how much evaluation has happened for each node is tracked. + +- At the end of this pass the computational network is fully connected and complete + +#### Validation + +Validation multiple purposes: + +- Ensure that the network is fully connected and that all necessary network nodes exist + +- Ensure that the dimensions of the matrices passed to nodes are compatible with the nodes + +- Calculates dimensions that depend on input dimensions. This is a feature of Convolutional Networks to make them easier to define. + +- Allocate the memory for the matrices + +- Ensure the special nodes (CriteriaNode, Features, Labels, Output, etc.) exist if required for this network + +#### Evaluation – final pass + +- This pass does special processing that requires the matrices to exist. As an example there is an optional parameter for the Parameter() function that allows a parameter to be initialized in various ways (zero, random values, from a file), this requires the matrix to be there, so it is done in the final pass. + +#### Evaluation Process + +Now that the entire has been explained, a little more detail on how each Evaluation pass occurs seems relavant. + +- First the NDLScript::Evaluate() method is called. This take a few parameters the nodeEvaluator (which creates the actual Computation Nodes), and a baseName, and which pass we are on. + +- NDLScript::Evaluate() loops through it’s list of NDL statements and calls NodeEvaluator::Evaluate() of NodeEvaluator::EvaluateMacro() on each of them with the current baseName + +- For Macros EvaluateMacro() is called. This takes care of getting the arguments into the target macros symbol table with the correct actual parameters names associated with them. And calls the NDLScript::Evaluate() on the macro script, it also appends to the baseName so it’s still correct. + +- In subsequent passes, the node is looked up using the same name instead of created + +- The parameters (if any) to the node are evaluated in the first two passes, and in the second pass are assigned as inputs to the computation nodes. + +- In the second pass optional parameters are also processed. The “tag” optional parameter must be processed to identify features, labels, etc. + +- “dotNames” are currently handled separately, whenever one is referenced the baseName of the parent is added as a prefix to the current scoped name and looked up directly in the computational network. + +- There is an “evalValue” that gets set whenever a computational node is resolved to be associated with an NDLNode. However, when macros are called, this will usually be the wrong value, since it will hold the last time the macro was called, not necessarily the instance desired. To alleviate this issue, evalValues are always cleared out on every macro call and the values are used immediately after evaluation, otherwise the values may be different than expected. + +Though this explanation may not be complete hopefully it has been instructive, and if all else fails, trace through it in the debugger, and it will likely help. diff --git a/Documentation/Documents/External Buffer Behavior.docx b/Documentation/Documents/External Buffer Behavior.docx deleted file mode 100644 index a28df3183..000000000 Binary files a/Documentation/Documents/External Buffer Behavior.docx and /dev/null differ diff --git a/Documentation/Documents/External Buffer Behavior.md b/Documentation/Documents/External Buffer Behavior.md new file mode 100644 index 000000000..2136845c8 --- /dev/null +++ b/Documentation/Documents/External Buffer Behavior.md @@ -0,0 +1,69 @@ +# ExternalBuffer in Matrix class + +There are at least 4 different implementations of the Matrix class that have over time diverged in their implementation in respect to how the external buffer case is handled. The external buffer case is when the matrix class does not actually own it's own memory and is pointing to an external buffer that is managed separately. A deviceID of MANAGEDEXTERN used to be the way this was done, however we have now moved to setting a flag m_externalBuffer in the common header to signify an eternal buffer. We have two instances of this in our code today: + +1. Column Slices were implemented using this feature. The idea is you only want to reference a portion of a full matrix, but don't want to copy the contents to a new matrix for efficiency reasons. In this case the slice can be modified just like a real matrix, and it is the programmers responsibility to ensure that the lifetime of the underlying matrix is longer than any of it's slices. NOTE: lifetime management is not taken care of for you, so be careful +2. PTask Buffers - PTask is our solution for using multiple GPUs. It uses a filter graph based approach for accelerating GPU applications. PTask executes a graph and calls each of it's tasks as the inputs are available. In CNTK most of these inputs are numeric arrays with an associated Matrix header metadata. We wrap the buffer in a Matrix shell with external buffers set, and call the normal processing methods. + +Both of these uses are similar, but slightly different as well. We believe that we can use the same implementations to satisfy both sets of needs. So here are the definitions: + +```c++ +Matrix(const size_t numRows, const size_t numCols, ElemType *pArray, const size_t matrixFlags=matrixFlagNormal, +short deviceId=AUTOPLACEMATRIX, const size_t nnz=0); +``` + +* Matrix constructor that constructs a matrix from a buffer pointer and some flags. The behavior depends on the flags. In all cases dimensions, format (from the matrixFlags), deviceId and nnz (for sparse representations) are copied: + * matrixFlagDontOwnBuffer - in this case the pArray pointer is set as the m_pArray of the matrix and m_externalBuffer = true + * matrixFlagSetValueOnDevice - if set this signifies that the buffer is on the proper device, but needs to be copied to newly allocated space for the m_pArray, m_externalBuffer = false + * neither set - the buffer is on the CPU and device memory is allocated and then the buffer is copied over, m_externalBuffer = false + +```c++ +Matrix(const Matrix& deepCopyFrom, short deviceId=AUTOPLACEMATRIX); //copy constructor, deep copy +``` + +* Matrix constructor that constructs a matrix from an existing matrix, Dimensions, format, and other elements are also copied: + * deepCopyFrom - regardless of if m_externalBuffer is set or not, a new buffer is allocated and the contents of the deepCopyFrom are copied to the new buffer. m_externalBuffer = false; + * NOTE: use move constructor or SetValue with matrixFlagDontOwnBuffer if an externalBuffer at the destination is desired + +```c++ +Matrix& operator=(const Matrix& deepCopyFrom); //assignment operator, deep copy +``` + +* assignment operator copies from one matrix to another. In all cases , dimensions, format, and other members are copied, m_externalBuffer is left unchanged, and copy of the buffer is buffer content only: + * destination normal, deepCopyFrom is external - destination is resized as necessary and then copy. + * destination is external, deepCopyFrom can be either - If the destination would require a resize, an exception is thrown, otherwise copy. + +```c++ +Matrix(Matrix&& moveFrom); //move constructor, shallow copy +``` + +* constructor with move semantics copies from one matrix to another: + * moveFrom is bitwise copied to the newly created matrix. So it is an exact copy of previous matrix (which is going to be discarded without destructors running) + +```c++ +Matrix& operator=(Matrix&& moveFrom); //move operator, shallow copy +``` + +* assignment operator with move semantics copies from one matrix to another: + * destination normal - In this case existing buffers are freed, and then everything is bitwise copied (including m_externalBuffer flag). + * destination is external - bitwise copy over everything (including m_externalBuffer flag) + +```c++ +void SetValue(const Matrix& deepCopyFrom); +``` + +* Straight copy from one buffer to another, irrespective of m_external flags, which remain unchanged. If the destination is not large enough, it will be resized. If buffer mismatch occurs and the destination is m_externalBuffer, it will throw an exception. + +```c++ +void SetValue(const size_t numRows, const size_t numCols, ElemType *pArray, +const size_t matrixFlags=matrixFlagNormal, int deviceId=MANAGEDEXTERN); +``` + +* SetValue with a buffer pointer copies the contents of that buffer to the matrix, resizing the destination as necessary. Also sets the format (through a mask of the matrixFlags) and deviceId of the matrix: + * matrixFlagDontOwnBuffer set, destination normal - Free the contents of the current array buffer, replace pointer, dimensions, m_externalBuffer = true + * matrixFlagDontOwnBuffer set, destination external - replace pointer and dimensions, m_externalBuffer = true + * matrixFlagSetValueOnDevice set, destination normal - the buffer is on the proper device, resize destination as necessary, set the dimensions and copy buffer to the current array, m_externalBuffer = false + * matrixFlagSetValueOnDevice set, destination external - the buffer is on the proper device, throw if dimensions are incompatible, set the dimensions and copy buffer content to the current array location, m_externalBuffer = false + * no flags set, destination normal - the buffer is on the CPU, resize destination as necessary, set the dimensions and copy buffer to the current array, m_externalBuffer = false + * no flags set, destination external - the buffer is on the CPU, throw if dimensions are incompatible, set the dimensions and copy buffer content to the current array location, m_externalBuffer = false + diff --git a/Documentation/Documents/Model Editing Language.docx b/Documentation/Documents/Model Editing Language.docx deleted file mode 100644 index f02b3eb7a..000000000 Binary files a/Documentation/Documents/Model Editing Language.docx and /dev/null differ diff --git a/Documentation/Documents/Model Editing Language.md b/Documentation/Documents/Model Editing Language.md new file mode 100644 index 000000000..f0aba5e77 --- /dev/null +++ b/Documentation/Documents/Model Editing Language.md @@ -0,0 +1,750 @@ +# Model Editing Language + +## Definition + +The Model Editing Language (MEL) of the Computational Network ToolKit (CNTK) provides a means to modify an existing trained network using a set of provided commands. It provides a number of functions to modify the network and can use Network Description Language (NDL) to define new elements. It looks similar to a scripting language in syntax, but is not a programming “language”, but a simple way to modify an existing network. This network must have been defined in a format that CNTK can read, currently only the CNTK computational network disk format is supported. This document assumes some knowledge of NDL, reading the NDL document prior to this document is recommended. + +## Example + +This section will cover the features of the MEL by example. If you would rather see the “programmer documentation” just skip to MEL Reference section. + +Here is a simple example of a MEL script: + +``` + model1 = LoadModel(“c:\models\mymodel.dnn”, format=cntk) + SetDefaultModel(model1) + DumpModel(model1, “c:\temp\originalModel.dmp”, includeData = true) + + #Let’s create another hidden layer + Copy(L3.*, L4.*, copy=all) + + #Now hook up the layer + SetInput(L4.*.T, 1, L3.RL) # Layer 3 output to Layer 4 input + SetInput(CE.*.T, 1, L4.RL) # Layer 4 output to Top layer input + + #Add mean variance normalization using in-line NDL + meanVal = Mean(features) + invstdVal = InvStdDev(features) + inputVal = PerDimMeanVarNormalization(features,meanVal,invstdVal) + + #make the features input now take the normalized input + SetInput(L1.BFF.FF.T, 1, inputVal) + + #save model + SaveModel(“c:\models\mymodel4HiddenWithMeanVarNorm.dnn”) +``` + +This MEL script is using a network that was defined originally by the following NDL script: + +``` + # constants defined + # Sample, Hidden, and Label dimensions + SDim=784 + HDim=256 + LDim=10 + + features=Input(SDim, tag=feature) + labels=Input(LDim, tag=label) + + # Layer operations + L1 = RBFF(features, HDim, SDim) + L2 = RBFF(L1, HDim, HDim) + L3 = RBFF(L2, HDim, HDim) + CE = SMBFF(L3, LDim, HDim, labels, tag=Criteria) + Err=ErrorPrediction(labels, CE.F, tag=Eval) + + # rootNodes defined here + OutputNodes=(CE.F) +``` + +### Loading a model + +The first thing command executed in a MEL script is usually a LoadModel() command. This function takes the name of a model file on disk, and an optional parameter specifying the format of the model file. Currently only CNTK format model files are accepted, and CNTK format is the default value. Programmers can write file converters to support more model formats. + +``` + model1 = LoadModel(“c:\models\mymodel.dnn”, format=cntk) + SetDefaultModel(model1) +``` + +‘model1’ is the identifying name this model is given for use in the MEL script. This identifier is used in the next line to this model as the default model. The default model defines what model will be assumed in all name references within the script, and the model to which any NDL (Network Definition Language) commands will apply. This line isn’t really necessary in this case, because the first model loaded will be the default model without explicitly calling the SetDefaultModel() function. + +### Viewing a model file + +It is often necessary to view a model file to determine the names used in the model file. MEL uses the node names in most commands, to specify which node(s) should be modified. The Dump() command dumps the node names and optionally values to a file. + +``` + DumpModel(model1, “c:\temp\originalModel.dmp”, includeData = true) +``` + +the parameters are the model name, the file name, and if the dump should include data. The includeData optional parameter defaults to false. The dump looks something like this: + +``` + … + features=InputValue [784,32] + L1.BFF.B=LearnableParameter [256,1] NeedGradient=true + 0.0127850091 + -0.00473949127 + 0.0156492535 + … + 0.00529919751 + #################################################################### + L1.BFF.FF.P=Plus ( L1.BFF.FF.T , L1.BFF.B ) + L1.BFF.FF.T=Times ( L1.BFF.W , normInput ) + L1.BFF.W=LearnableParameter [256,784] NeedGradient=true + 0.0174789988 0.0226208009 -0.00648776069 0.0346485041 -0.0449098013 -0.0233792514 + 0.0154407881 0.000157605857 0.0206625946 0.0491085015 0.00128563121 + … +``` + +These variables are set to scalar numeric values in this case and are used as parameters in the NDL Functions. These values are the dimensions of the data samples, hidden layers, and labels used in training. This particular setup is for the MNIST dataset, which is a collection of images that contain 784 pixels each. Each image is a handwritten digit (0-9), so there are 10 possible labels that can be applied to each image. The hidden matrix dimensions are determined by the user depending on their needs. + +### Copy + +The copy command will copy a node, or a group of nodes from one location to another location. This can be done within the same model, or between different models: + +``` + #Let’s create another hidden layer + Copy(L3.*, L4.*, copy=all) +``` + +The first parameter is the source of the copy and must exist, the second is the target and may or may not exist. If it does exist, those matching nodes will be overwritten by the copy. The optional parameter **copy** can be used to change this behavior, the options are: **all** the default which copies all node data and links to other nodes also copied, or **value** which copies the node values only, leaving the connections between nodes (if any) unchanged. + +In this command an entire layer is duplicated in the same model creating a new L4 layer in the model. The Copy() command will copy the nodes and connections between the nodes being copied by default so the optional parameter was not required in this case. + +The L3 used in this copy command was originally defined in NDL as follows: + +``` + L3 = RBFF(L2, HDim, HDim) +``` + +So the new L4 layer will contain all the nodes L3 contains (RectifiedLinear, Plus, Times, W and B Parameters) all connected just as they were in the L3 layer. + +### SetInput + +To integrate this new layer into the model, the inputs and outputs must still be set properly. After the copy any node whose connected nodes were not copied will have those connections set to an invalid value. These need to be fixed up in order to have a valid model. Attempts to Save a model will first validate the model in the case where some nodes were not reconnected. + +You can change connections between nodes with the SetInput() command. This takes a node to modify, the input number to modify (zero-based input\#), and the new value for that input. The following commands hook up the inputs and outputs for our copied nodes: + +``` + #Now hook up the layer + SetInput(L4.*.T, 1, L3.RL) # Layer 3 output to Layer 4 input + SetInput(CE.*.T, 1, L4.RL) # Layer 4 output to Top layer input +``` + +To connect our new L4 layer, we need to set the second input of the Times node (L4.BFF.FF.T) to L3.RL, which is the output of the L3 layer. The input number is zero-based, so the first input is zero and the second input would be '1'. +Likewise we need to hook the output of the L4 layer nodes to the input of the top layer. Once again this ends up being a Times node (CE.BFF.FF.T) + +### Name Matching + +You may have noticed the use of the ‘\*’ wildcard character in the commands presented to this point. Those are name matching wildcards, and are useful in matching a group of related nodes. Because of the hierarchal “dot naming” scheme used by NDL, it is easy to select all the nodes that a particular macro generated because they will all start with the same prefix. Nodes generated by NDL macros have the following structure: + +``` +\[name\]{.\[macroName\]}.\[nameNode\] +``` + +Where **name** is the name assigned in NDL, **macroName** is the name given to a macro called by the initial macro, and can be several layers deep, and **nameNode** is the name given to a single node in the final macro. For Example this macro in NDL: + +``` +L3 = RBFF(L2, HDim, HDim) +``` + +Generates the following nodes: + +L3.RL | RectifiedLinear node +---|--- +L3.BFF.B | Parameter node – used for bias | +L3.BFF.W | Parameter node – used for weight | +L3.BFF.FF.T | Times node | +L3.BFF.FF.P | Plus node | + +These patterns can be used to access these nodes: + +L3.\* | Select all the L3 nodes +---|--- +L3.\*.P | Select the L3.BFF.FF.P node +\*.W | Select L3.BFF.W and any other node named ‘W’ in the model +model1.L3.\* | All the L3 nodes in the ‘model1’ model +model1\[.\*\] | All the nodes in model1 (the ‘.\*’) is optional + +There are also methods that will copy nodes based on the structure of the graph. Look for CopySubTree() in the reference section for details. + +### Adding new nodes + +Adding new nodes to an existing model can be done just like a model can be originally defined, in NDL. There are two ways to do this, the simplest is to just type the NDL definitions into the MEL script, as if it was NDL, like so: + +``` + #Add mean variance normalization using in-line NDL + meanVal = Mean(features) + invstdVal = InvStdDev(features) + inputVal = PerDimMeanVarNormalization(features,meanVal,invstdVal) +``` + +This is called in-line NDL and can be used for most tasks. This sequence of nodes does a mean variance normalization on the dataset. The new nodes will be placed in the current default model in the MEL script. In our example script, we only use one model, and it was set as the default model using the SetDefaultModel() command. If no model has been explicitly set to be the default model, the last loaded model is used as the default. However, It is recommended that the SetDefaultModel() command be used to make it explicit. + +Notice the variable **features** that is used in the NDL is actually a node from the default model. It is legal to use nodes from the model in in-line NDL and vise-versa. However, no name matching '\*' patterns are allowed in NDL commands, and macros cannot be defined in in-line NDL. + +An NDL macro can also be used from in-line NDL, as long as it appears in the default macros defined for the editing script, or it is defined in an NDL Snippet (see below). + +### Connecting in-line NDL + +The sequence of nodes used to do mean variance normalization are now in the model. However, we have to use the output of these NDL nodes to replace the previous InputNode that provided the features. This node is called ‘features’ in this model, and we need to set the input to the L1 layer to be ‘inputVal’ (the output from the NDL nodes we just created) instead. This is done, again, using the SetInput() command: + +``` + #make the features input now take the normalized input instead + SetInput(L1.BFF.FF.T, 1, inputVal) +``` + +Now the nodes have all been connected and the model is valid, a mean variance normalization step has just been added to the model. The mean() and variance() nodes both execute before the any training begins and are called ‘pre-compute’ nodes. The mean and variance are calculated over the training data set, and then those values are used during training to normalize the data. + +## NDL Snippets + +NDL snippets are sections of NDL definitions that generate a new model. Any NDL construct that is legal in an NDL script can be used. This includes defining macros and other advanced NDL features. For example, instead of loading an existing NDL file, an NDL snippet could have been used to define the network structure. The NDL Snippet looks like: + +``` +model1=[ + # constants defined + # Sample, Hidden, and Label dimensions + SDim=784 + HDim=256 + LDim=10 + + features=Input(SDim, tag=feature) + labels=Input(LDim, tag=label) + + # Layer operations + L1 = RBFF(features, HDim, SDim) + L2 = RBFF(L1, HDim, HDim) + L3 = RBFF(L2, HDim, HDim) + CE = SMBFF(L3, LDim, HDim, labels, tag=Criteria) + Err=ErrorPrediction(labels, CE.F, tag=Eval) + + # rootNodes defined here + OutputNodes=(CE.F) +] +``` + +When snippets are used, wildcard naming, and use of symbols from another model are not allowed. The syntax rules are identical to creating an NDL script. + +### SaveModel + +After the model edits are complete, it’s time to save the model: + +``` + #save model + SaveModel("c:\models\mymodel4HiddenWithMeanVarNorm.dnn") +``` + +This command saves the default model (still ‘model1’) to the path name specified. ‘model1’ could have been specified as the first parameter with the path as the second to obtain the same affect. Before the save happens the model is validated to ensure it is a valid model before save can occur. Should there be an error in the model, an error message will be displayed on the console and the model edit will terminate. + +## MEL Reference + +Model Editing Language (MEL) is a language that provides a means to modify an existing CNTK network, or a trained model to create new networks and models. MEL allows nodes of a network to be copied, new nodes created, and node values to be duplicated to create new networks based on other previously done work. + +### Commands + +Commands in MEL are the operations that can be used to modify a network or model. The commands are represented in a function call like syntax: + +`Command(parameter1, parameter2, optionalParameter=value)` + +Commands do not return values, with the exception of the CreateModel() and LoadModel() commands, and some may have optional parameters. The parameters are delimited with a comma. The commands are: + +**Command Name** | **Example** | **Notes** +---|---|--- +CreateModel | m1=CreateModel() | Returns a value +CreateModelWithName | CreateModelWithName(model1) | Alternate no return value +LoadModel | m1=LoadModel(“new.dnn”, format=cntk) | Returns a value +LoadModelWithName | LoadModelWithName(m1, “new.dnn”, format=cntk) | Alternate no return value +LoadNDLSnippet | LoadNDLSnippet(mNDL, “net.ndl”) | +SaveDefaultModel | SaveDefaultModel(“new.dnn”, format=cntk) | +SaveModelWithName | SaveModelWithName(m1, “new.dnn”, format=cntk) | +SetDefaultModel | SetDefaultMode(m1) | +UnloadModel | UnloadModel(m1) | +Dump\[Model\] | Dump\[Network\](m1, “dump.txt”, includeData=false) | DumpModel is alternate name +DumpNode | DumpNode(node, “node.txt”, includeData=false) | +Copy\[Node\] | Copy(fromNode, toNode, copy=all) | CopyNode is alternate name +CopySubTree | CopySubTree(fromNode, toNetwork, toNodeNamePrefix, copy=all) | +Copy\[Node\]Inputs | CopyInputs(fromNode, toNode) | CopyNodeInputs is alternate name +Set\[Node\]Input | SetInput(fromNode, inputID, inputNode) | SetNodeInput is alternate name +Set\[Node\]Inputs | SetInputs(fromNode, inputNode1\[, inputNode2, inputNode3\]) | SetNodeInputs is alternate name, variable number of parameters | +SetProperty | SetProperty(toNode, propertyName, propertyValjue) | +SetPropertyForSubTree | SetPropertyForSubTree(rootNode, propertyName, propertyValue) | +Remove\[Node\] | Remove(node\[, node2, node3, …\]) | Same as DeleteNode() +Delete\[Node\] | Delete(node\[, node2, node3, …\]) | Same as RemoveNode() +Rename | Rename(nodeOld, nodeNew) | + +### Name Matching + +MEL provides a way to perform a command on more than one node at a time. This is done through wildcard name matching. Because of the hierarchal “dot naming” scheme used by NDL, related nodes are easy to select with a wildcard name matching scheme. Nodes generated by NDL macros have the following structure: + +`{[modelName].}[name]{.[macroName]}.[nameNode]` + +Element | Descriptions +---|--- +**modelName** | an optional prefix that defines which model should be applied to the rest of the name. If no **modelName** is specified, the current default model is assumed. +**name** | the name of the node in question, or if NDL was used to create the network, the top level symbol used to identify the node (i.e. L3 in the following example. +**macroName** | the name given to a macro called by the initial macro and can be several layers deep. Usually the names are the same as the macros called. A user is unlikely to know these names unless they dump the network nodes, so wildcard name matching can be used instead of the **macroName** (s) +**nameNode** | the name given to a single node in the final macro. + +For Example this macro in NDL: + +``` + L3 = RBFF(L2, HDim, HDim) +``` + +Generates the following nodes: + +Name | Descriptions +---|--- +L3.RL | RectifiedLinear node | +L3.BFF.B | Parameter node – used for bias | +L3.BFF.W | Parameter node – used for weight | +L3.BFF.FF.T | Times node | +L3.BFF.FF.P | Plus node | + +The following wildcard patterns can be used to select nodes within a model. If a \[model\] prefix is not specified the default model is assumed: + +Pattern | Example | Result +---|---|--- +\[prefix\]\* | L3.\* | Select all the nodes starting with \[prefix\] +\[prefix\]\*\[suffix\] | L3.\*.P | Select all nodes with \[prefix\] and \[suffix\] +\*\[suffix\] | \*.W | Select all the nodes with \[suffix\] +\[model\].\[pattern\] | model1.L3.\* | Select all the nodes matching a pattern in \[model\] +\[model\]{\*} | model1.\* | Select all nodes in the model, ‘\*’ is optional + +There are also methods that will copy nodes based on the structure of the graph. Look for CopySubTree() in the reference section for details. + +### Optional Parameters + +Many commands have optional parameters that will change the behavior of the command. For example: + +``` + Copy(L1.\*, L2.\*, copy=all) +``` + +In this example all the nodes starting with "L1." are copied to nodes starting with "L2.", the values of the nodes as well as any links between the nodes (the network structure) are copied. If the destination “L2.\*” nodes already exist, they will be overwritten. The other option is copy=value, which would be used when the network structure desired already exists, and the values contained in the node are all that are required to be copied. This can be used to copy over the values of Parameter() nodes to a new model with identical structure. + +Each command may have optional parameters, look in the Command reference section for details of the optional parameters that are accepted by a command. +Stringize variables +MEL supports a “stringize” feature similar to the one supported by configuration files. Anywhere in a MEL script file, you can specify “$VarName$”, and this entire string will be replaced by the value of the variable called “VarName”. Note that the variables that are considered in scope for this purpose are the configuration variables that are visible from the configuration section where the path to this MEL script is specified (via the “editPath” parameter). For example, if the variables “OldModelPath” and “NewModelPath” were defined at the root level of the configuration file, the following would be a proper MEL script: + +``` + m1=LoadModel("$OldModelPath$",format=cntk) + # make change to model here + SaveModel(m1,"$NewModelPath$",format=cntk) +``` + +## NDL Integration + +NDL (Network Description Language) can be used freely in MEL to create new nodes and integrate them into an existing model. Please refer to the NDL Section of the documentation to get the details on all the NDL Functions that are available. The NDL Functions can be used in two different ways in MEL. In-line and as a snippet. + +### In-line NDL + +In-line NDL is, as it sounds, NDL lines mixed in with MEL Command calls. This is an easy way to define new nodes in a MEL script. In-line NDL only works on the default model at the time the NDL function is encountered. The default model is set with the SetDefaultModel() command, or if no such command has been encountered the last LoadModel() or CreateModel() command. It is recommended that the SetDefaultModel() command appear before any In-line NDL to make it clear which model is being modified. + +In-line NDL may use node names from the default model as parameters, and MEL commands may use NDL symbols as parameters. There are a number of restrictions using in-line NDL: + +1. ‘\*’ names may not be used in In-line NDL, only fully quantified node names are accepted. +2. NDL symbols only apply to the default model at the time they were created when used in MEL commands +3. Macros may not be defined in in-line NDL (though they can in an NDL snippet) +4. Only macros defined in the default macro file referenced in the config file, or macros defined in an NDL snippet in the MEL Script may be used +5. NDL will be processed when the next MEL command that requires it to be processed is encountered. It is only at this time that the new nodes are fully created. If forward references are used to variables, they must be resolved before the next MEL command that requires the variables to be resolved. + +### NDL Snippets + +NDL snippets are sections of NDL definitions that generate a new model. Any NDL construct that is legal in an NDL script can be used. This includes defining macros and other advanced NDL features. The syntax for defining and NDL snippet are as follows: + +``` + [modelName]=[ + #ndl commands go here + ] +``` + +Upon the completion of the snippet, the modelName will be the name of the newly defined model. This model need not be fully defined, for example, the special nodes (i.e. criteria nodes) do not need to be defined in the model. However, all referenced variables must be defined in the snippet. It is often easier to use in-line NDL to define new nodes in MEL, and NDL Snippets to define any macros. Macros are defined in a global namespace and can be defined in any model and used from any other model. + +One possible use of an NDL snippet is to define an entirely new model, and then use MEL to populate the new model with values. Here is an example of how an NDL snippet could have been used to define the entire network structure: + +``` +model1=[ + # constants defined + # Sample, Hidden, and Label dimensions + SDim=784 + HDim=256 + LDim=10 + + features=Input(SDim, tag=feature) + labels=Input(LDim, tag=label) + + # Layer operations + L1 = RBFF(features, HDim, SDim) + L2 = RBFF(L1, HDim, HDim) + L3 = RBFF(L2, HDim, HDim) + CE = SMBFF(L3, LDim, HDim, labels, tag=Criteria) + Err=ErrorPrediction(labels, CE.F, tag=Eval) + + # rootNodes defined here + OutputNodes=(CE.F) +] +``` + +When snippets are used, wildcard naming, and use of symbols from another model are not allowed. The syntax rules are identical to creating an NDL script. Alternately, the LoadNDLSnippet() command can be used to load NDL from an external file. + +## Comments + +Comments in MEL are identical to those used in the NDL and configuration files. The ‘\#’ character signifies the beginning of a comment, everything that occurs after the ‘\#’ is ignored. The ‘\#’ must be preceded by whitespace or be at the beginning of the line to be interpreted as a comment. The following are valid comments: + +``` + # Layer operations + L1 = RBFF(features, HDim, SDim) # define the first layer + # the following variable is set to infinity and the ‘#’ in ‘1#INF’ is not interpreted as a comment marker + var = 1#INF +``` + +## MEL Commands + +This section contains the currently implemented MEL Command functions. + +### CreateModel + +Creates a new model which is empty. + +`m1=CreateModel()` + +#### Parameters + +none + +#### Returns + +the new model + +#### Notes + +This command is one of only a few that return a value. If you prefer to easily distinguish between NDL functions (which always return a value) and MEL commands (which normally do not) you may wish to use the alternate CreateModelWithName() call, which takes the new model identifier as a parameter instead of returning it as a return value. + +### CreateModelWithName + +Creates a new model which is empty. + +`CreateModelWithName(m1)` + +#### Parameters + +the identifier for the newly created model + +#### Notes + +The alternate form of the command is CreateModel() and returns a value. If you prefer to easily distinguish between NDL functions (which always return a value) and MEL commands (which normally do not) you may wish to use this version of the command. + +### LoadModel + +Load a model from a disk file and assign it a name. The format of the file may be specified as an optional parameter. + +`m1=LoadModel(modelFileName, [format=cntk])` + +#### Parameters + +`modelFileName` – name of the model file, can be a full path name. If it contains spaces, it must be enclosed in double quotes. + +#### Returns + +model identifier for the model that will be loaded + +#### Optional Parameters + +`format=[cntk]` – Specifies the format of a file, defaults to ‘cntk’. Currently only the native CNTK format of model file is accepted. Other formats may be supported in the future. + +#### Notes + +This command is one of only a few that return a value. If you prefer to easily distinguish between NDL functions (which always return a value) and MEL commands (which normally do not) you may wish to use the alternate LoadModelWithName() call, which takes the new model identifier as a parameter instead of returning it as a return value. + +### LoadModelWithName + +Load a model from a disk file and assign it a name. The format of the file may be specified as an optional parameter. + +`LoadModelWithName(model, modelFileName, [format=cntk])` + +#### Parameters + +`model`-identifier associated with the model that will be loaded. + +`modelFileName` – name of the model file, can be a full path name. If it contains spaces, it must be enclosed in double quotes. + +#### Optional Parameters + +`format=[cntk]` – Specifies the format of a file, defaults to ‘cntk’. Currently only the native CNTK format of model file is accepted. Other formats may be supported in the future. + +#### Notes + +The alternate form of the command is LoadModel() and returns a value. If you prefer to easily distinguish between NDL functions (which always return a value) and MEL commands (which normally do not) you may wish to use this version of the command. + +### LoadNDLSnippet + +Load an NDL Snippet from a file, and process it, assigning the results to a symbol + +`LoadNDLSnippet(model, nsdSnippetFileName[, section=first])` + +#### Parameters + +`model` – the identifier that will be used to reference this model. + +`ndlSnippetFileName` – name of the file that contains the snippet we want to load + +#### Optional Parameters + +`section=sectionName` – name of the section that contains the snippet we want to load. If the entire file is the snippet no section name should be specifiedmsmswscar cars Adam + +### SaveModel + +Save a model to disk in the specified model format + +`SaveModel(model, modelFileName[, format=cntk])` + +#### Parameters + +`model` – the identifier of the model which will be saved +`modelFileName` – the file name to save the model as + +#### Optional Parameters + +`format=cntk` – the format of file to save. The only valid value currently is CNTK format, which is the default. It is expected that different formats will be added in the future + +### SaveDefaultModel + +Save the current default model to a file. The format can be specified with an optional parameter + +`SaveDefaultModel(modelFileName, format=cntk)` + +#### Parameters + +`modelFileName` – name of the model file to save + +#### Optional Parameters + +`format=cntk` – the format of file to save. The only valid value currently is CNTK format, which is the default. It is expected that different formats will be added in the future + +### UnloadModel + +Unload the specified model from memory. + +`UnloadModel(model)` + +#### Parameters + +`model` – model identifier. + +#### Notes + +In general it is unnecessary to unload a model explicitly since it will happen automatically at the end of the MEL script. It is also not recommended that you reuse a model identifier after unloading a model. + +### Dump, DumpModel + +Create a text file that represents the contents and structure of the Computational network. + +`Dump(model, dumpFileName[, includeData=false])` +`DumpModel(model, dumpFileName[, includeData=false])` + +#### Parameters + +model – model Identifier +dumpFileName – file name to save the output + +#### Optional Parameters + +`includeData=[true,false]` – (default = false) Include the data contained in a node. This will output the contents of nodes that contain matrix values. + +### DumpNode + +Create a text file that represents the contents of a node. + +`DumpNode(node, dumpFileName[, includeData=false])` + +#### Parameters + +`node` – node Identifier, a wildcard name may be used to output multiple nodes in one call +`dumpFileName` – file name to save the output + +#### Optional Parameters + +`includeData=[true,false]` – (default = false) Include the data contained in a node. This will output the contents of nodes that contain matrix values. + +### Copy, CopyNode + +Copy a node, or a group of nodes from one location to another location. This can be done within the same model, or between different models. The copy can create new nodes or overwrite/update existing nodes. The network structure can be copied with multiple nodes, or just the values in the nodes. + +`Copy(fromNode, toNode[, copy=all])` +`CopyNode(fromNode, toNode[, copy=all])` + +#### Parameters + +`fromNode` – node identifier we are copying from. This can also be a wildcard pattern. + +`toNode` – node identifier we are copying to. This can also be a wildcard pattern, but must match the `fromNode` pattern. A copy from a single node to multiple nodes is also permitted. + +#### Optional Parameters + +`copy=[all,value]` – (default = all). Specifies how the copy will be performed: + + | If destination node exists | If destination node does not exist +---|---|--- +All | Copies over the values of the nodes and any links between them overwriting the existing node values. Any node inputs that are not included in the copy set will remain unchanged. | Copies over the values of the nodes and any links between them creating new nodes. All nodes that include inputs in the copy set will still be connected. All other nodes will have no inputs and will need to be set using SetInput() +Value | Copies over the node contents, the node inputs will remain unchanged | Not a valid option, the nodes must exist to copy only values. + +#### Examples + +`Copy(L1.*, L2.*)` – copies all the nodes and the inputs in the L1.\* copy set to L2.\*. If the L2.\* nodes did not exist, they will be created + +`Copy(L1.BFF.FF.W, model2.*.W, copy=value)` – copies values in the L1.BFF.FF.W node to all the nodes in model2 that are use the name ‘W’. + +#### Notes + +If an entire network is to be copied, it is easier to save the network first (possibly to a temporary location) and reload that model under a new name. + +### CopySubTree + +Copy all nodes in a subtree of a computational network from one location to another location. This can be done within the same model, or between different models. + +`CopySubTree(fromRootNode, toRootNode[, copy=all])` + +#### Parameters + +`fromRootNode` – node identifier we are copying from. This can also be a wildcard pattern. + +`toRootNode` – node identifier we are copying to. This can also be a wildcard pattern, but must match the fromRootNode pattern. + +#### Optional Parameters + +`copy=[all,value]` – (default = all). Specifies how the copy will be performed: + + | If destination node exists | If destination node does not exist +---|---|--- +All | Copies over the values of the nodes and any links between them overwriting the existing node values. Any node inputs that are not included in the copy set will remain unchanged. | Copies over the values of the nodes and any links between them creating new nodes. All nodes that include inputs in the copy set will still be connected. All other nodes will have no inputs and will need to be set using SetInput() +Value | Copies over the node contents, the node inputs will remain unchanged | Not a valid option, the nodes must exist to copy only values. + +#### Notes + +If the fromRootNode is a wildcard pattern then the toRootNode must also be a similar wildcard pattern. The CopySubTree() command will execute separately for each root node. + +### SetInput, SetNodeInput + +Set an input of a node to a value + +`SetInput(node, inputNumber, inputNode)` + +#### Parameters + +`node` – node whose input we are modifying . This can also be a wildcard pattern. + +`inputNumber` – a zero-based index to the input that will be set. + +`inputNode` – node identifier for input node. This must be a single node. + +#### Notes + +SetInput() or SetInputs() are often required after a Copy() command in order to hook up all the copied nodes into the network. + +### SetInputs, SetNodeInputs + +Set all the inputs of a node. If only one input needs to be set use the SetInput() command instead. + +`SetInputs(node, inputNode1[, inputNode2, inputNode3])` + +#### Parameters + +`node` – node whose input we are modifying . + +`inputNode1`, `inputNode2`, `inputNode3` – node identifier for input node. The number of input parameters must match the number of inputs **node** requires. + +#### Notes + +SetInput() or SetInputs() are often required after a Copy() command in order to hook up all the copied nodes into the network. + +### SetProperty + +Set the property of a node to a specific value. + +`SetProperty(node, propertyName, propertyValue)` + +#### Parameters + +`node` – the node whose properties will set + +`propertyName` – name of the property to modify. + +`propertyValue` – the value the Property will receive. + +The acceptable propertyNames and propertyValues are as follows: + +PropertyName | Description | PropertyValue +---|---|--- +ComputeGradient / NeedsGradient | A flag that determines if a node participates in gradient calculations. Applies to Parameter nodes | true / false +Feature | Sets the node as a feature input. Applies to Input nodes | true / false +Label | Set the node as a label input. Applies to Input nodes | true / false +FinalCriterion / Criteria | Sets the node as one of the Criteria nodes of the network | true / false +Evaluation / Eval | Set the node as one of the evaluation nodes | true / false +Output | Set the node as one of the output nodes | true / false + +#### Notes + +Most of these properties can be set on nodes through alternate methods. All of these properties except for the ComputeGradient can be added (but not removed) using the special node syntax in NDL. + +### SetPropertyForSubTree + +Set the property of a node to a specific value. + +`SetProperty(rootNode, propertyName, propertyValue)` + +#### Parameters + +`rootNode` – the node at the root of the subtree + +`propertyName` – name of the property to modify. + +`propertyValue` – the value the Property will receive. + +The acceptable propertyNames and propertyValues for this command are as follows: + +PropertyName | Description | PropertyValue +---|---|--- +ComputeGradient / NeedsGradient | A flag that determines if a node participates in gradient calculations. Applies to Parameter nodes | true / false + +#### Notes + +The ComputeGradient property only applies to Parameter nodes in the subtree. + +### Remove, RemoveNode, Delete, DeleteNode + +Delete or Remove node(s) from a model. All alternate command names provide the same option. + +`Remove(node[, node2, node3, …])` + +`Delete(node[, node2, node3, …])` + +`RemoveNode(node[, node2, node3, …])` + +`DeleteNode(node[, node2, node3, …])` + +#### Parameters + +`node` – the node to be removed. This can be a wildcard name. + +`node2`, `node3` – additional optional nodes that will also be removed, These can be wildcards + +#### Notes + +This command can leave unconnected nodes in a model which would need to be reconnected using the SetInput() or SetInputs() commands. + +### Rename + +Rename a node + +`Rename(oldNode, newNode)` + +#### Parameters + +`oldNode` – the node name of the old node, wildcard naming may be used. + +`newNode` – the node name for the new node, matching wildcard naming may be used if oldNode contains wildcards. + +#### Notes + +Renaming nodes has no effect on the node inputs, even if a name changes the association will remain intact. diff --git a/Documentation/Documents/Network Description Language.docx b/Documentation/Documents/Network Description Language.docx deleted file mode 100644 index 468b42efa..000000000 Binary files a/Documentation/Documents/Network Description Language.docx and /dev/null differ diff --git a/Documentation/Documents/Network Description Language.md b/Documentation/Documents/Network Description Language.md new file mode 100644 index 000000000..f83eae79a --- /dev/null +++ b/Documentation/Documents/Network Description Language.md @@ -0,0 +1,900 @@ +# Network Description Language + +## Definition + +The Network Description Language (NDL) of the Computational Network ToolKit (CNTK) provides a simple way to define a network in a code-like fashion. It contains variables, and Macros, and other well understood concepts. It looks similar to a scripting language in syntax, but is not a programming “language”, but a simple way to define a network. + +## Example + +This section will cover the features of the NDL by example. If you would rather see the “programmer documentation” just skip to NDL Reference section. + +Here is a simple example of a network definition: + +``` + SDim=784 + HDim=256 + LDim=10 + B0=Parameter(HDim) + W0=Parameter(HDim, SDim) + features=Input(SDim) + labels=Input(LDim) + Times1=Times(W0, features) + Plus1=Plus(Times1, B0) + RL1=RectifiedLinear(Plus1) + B1=Parameter(LDim, 1) + W1=Parameter(LDim, HDim) + Times2=Times(W1, RL1) + Plus2=Plus(Times2, B1) + CrossEntropy=CrossEntropyWithSoftmax(labels, Plus2) + ErrPredict=ErrorPrediction(labels, Plus2) + FeatureNodes=(features) + LabelNodes=(labels) + CriteriaNodes=(CrossEntropy) + EvalNodes=(ErrPredict) + OutputNodes=(Plus2) +``` + +This is a simple Neural Network that consist of two layers. + +### Variables + +The first thing you will notice is that the SDim, HDim and LDim variables. Variable names can be any alphanumeric string (starting with a letter) and are case-insensitive. + +``` + SDim=784 + HDim=256 + LDim=10 +``` + +These variables are set to scalar numeric values in this case and are used as parameters in the NDL Functions. These values are the dimensions of the data samples, hidden layers, and labels used in training. This particular setup is for the MNIST dataset, which is a collection of images that contain 784 pixels each. Each image is a handwritten digit (0-9), so there are 10 possible labels that can be applied to each image. The hidden matrix dimension is determined by the user depending on their needs. + +### Parameters + +Parameters are matrices that constitute the learned model upon completion of training. The model parameter matrices are used to modify the sample data into the desired output data and are updated as part of the learning process. + +``` + B0=Parameter(HDim) + W0=Parameter(HDim, SDim) +``` + +These lines setup the parameters that will be trained, W0 is the weight matrix and B0 is the bias matrix. Parameters are matrices, and have two dimension parameters. If only one dimension is given the other dimension is assumed to be a ‘1’. By default Parameters are initialized with uniform random numbers, but other options exist (see NDL Function definitions) + +### Inputs + +The inputs into the network are defined by the sample data and the labels associated with the samples. + +``` + features=Input(SDim) + labels=Input(LDim) +``` + +The ‘features’ input will have the dimensions of the sample data, and the ‘labels’ input will have the dimensions of the labels. The variables chosen here are for convenience and could be any valid variable name. + +### Computation + +The computation portion of the network gets the product of the weight matrix and the features matrix and adds on the bias. It uses the matrix operators Times() and Plus(). + +``` + Times1=Times(W0, features) + Plus1=Plus(Times1, B0) + RL1=RectifiedLinear(Plus1) +``` + +Following this computation we apply the energy function, in this case RectifiedLinear(), to the product. The Sigmoid() function is also available (see NDL Function definitions). + +### Top Layer + +The top layer in a network is where the neural network produces the probabilities that correspond to the labels provided in supervised learning. This network uses category labels, for the MNIST case these will appear as an array of 10 floating point values, all of which are zero except for the proper label category which is 1.0. + +``` + CrossEntropy=CrossEntropyWithSoftmax(labels, Plus2) +``` + +Networks will often use the SoftMax function to obtain the probabilities for each label. The error between the actual and the probability is then computed using CrossEntropy. In the CNTK these two actions can be combined in one function for efficiency. CrossEntropyWithSoftmax() takes the input, computes the SoftMax function, calculates the error from the actual value using CrossEntropy and that error signal is used to update the parameters in the network via back propagation. + +### Back Propagation + +CNTK does not require you to specify anything additional for the back propagation portion of the network. For this example Stochastic Gradient Descent (SGD) is used as the learning algorithm. Each function in CNTK also has a derivative counterpart function and the system automatically does the back propagation update of the network parameters. + +### Error Prediction + +Predicted error rates are often computed during the training phase to validate the system is getting better as the training progresses. This is handled in the CNTK using the following function: + +``` + ErrPredict=ErrorPrediction(labels, Plus2) +``` + +The probabilities produced by the network are compared to the actual label and an error rate is computed. This is generally displayed by the system. Though this is useful, it is not mandatory to use ErrorPrediction, and this can be left out of the network if desired. + +### Defining special nodes + +After defining the network, it’s important to let CNTK know where the special nodes are in the network. For example, the input nodes (which are features, and which are labels), the output nodes, evaluation nodes and Top Layer criteria nodes. CNTK supports multiple nodes for each type, so the values are arrays. The array syntax is comma separated variable names surrounded by parenthesis. + +``` + FeatureNodes=(features) + LabelNodes=(labels) + CriteriaNodes=(CrossEntropy) + EvalNodes=(ErrPredict) + OutputNodes=(Plus2) +``` + +## Macros + +While creating a network using the syntax shown above is not all that difficult, it can get wordy when creating deep neural networks with many layers. To alleviate this problem, common definitions can be combined into Macros. Macros can be defined as nested calls on a single line, or can be in a more function like syntax as can be seen in the following examples: + +### Examples + +Macro examples: + +``` + RFF(x1, w1, b1)=RectifiedLinear(Plus(Times(w1,x1),b1)) +``` + +This one macro is equivalent to the computation section in the previous section, but all in one line. The parameters used in macros are local to each macro. + +``` + FF(X1, W1, B1) + { + T=Times(W1,X1); + FF=Plus(T, B1); + } +``` + +This macro is a feed forward computation without the energy function. It shows the alternate format of macros. Semicolons are optional, but can be used if desired. The variables and parameters used inside the macros are local to the macro. The return value of a macro is defined by a local macro variable that has the same name as the macro. In this case the FF() macros return value will be the FF local variable. If no variables match, the last variable in the macro will be returned. + +``` + #Base Feed Forward network, includes Bias and weight parameters + BFF(in, rows, cols) + { + B=Parameter(rows) + W=Parameter(rows, cols) + BFF = FF(in, w, b) + } +``` + +This macro shows how parameters can be declared within a macro. It also shows the comment syntax using a ‘\#’ as the first character in a line, signifies a comment line. As in this example, a macro may call another macro, however recursion is not supported. + +``` + RBFF(input,rowCount,colCount) + { + F = BFF(input, rowCount, colCount); + RBFF = RectifiedLinear(F); + } +``` + +This macro calls the previous macro adding the RectifiedLinear() energy function for a complete layer. + +``` + SMBFF(x,r,c, labels) + { + F = BFF(x,r,c); + SM = CrossEntropyWithSoftmax(labels, F) + } +``` + +This macro defines a full Top layer, also using the BFF macro as in the other full layer macro. In this case no variables match the name of the macro, so the SM variable will be used as the return value, since it’s the last value defined in the macro. + +### Using Macros + +The following example uses the macros defined above + +``` + # constants defined + # Sample, Hidden, and Label dimensions + SDim=784 + HDim=256 + LDim=10 + + features=Input(SDim) + labels=Input(LDim) + + # Layer operations + L1 = RBFF(features, HDim, SDim) + CE = SMBFF(L1, LDim, HDim, labels) + Err=ErrorPrediction(labels, CE.F) +``` + +This shows the network definition equivalent to the original network shown but using the above macros. Much simpler to deal with, and understand. One new feature shown in this network definition is access to Macro variables. ErrorPrediction() needs to access the result of the FeedForward result before the CrossEntropyWithSoftmax() is applied to it. However the needed variable is local to the macro, but can still be accessed via “dot” syntax. The return value of the macro was CE, so to access the local F variable defined in the macro itself, CE.F can be used. In the single line version of macros, there are no user defined variable names, so this feature cannot be used. + +## Optional Parameters + +Optional Parameters are a feature that allows additional parameters to be specified on functions. While the optional parameters can be specified on any function or macro, they are limited to constant values and the underlying function must support the passed optional parameters, or there is no effect on the network. When used on a macro, the macro will have local variables defined that match the optional parameter name and value. + +### Parameter initialization + +One common use of these optional parameters is to define how parameters will be initialized: + +``` + B0=Parameter(HDim, init=zero) + W0=Parameter(HDim, SDim, init=uniform) +``` + +In this example the Bias matrix will be zero initialized, and the weight matrix will be initialized with uniform random numbers. Please consult the NDL Function reference to find which functions accept optional parameters + +### Tagging special values + +As an alternate to providing an array of special nodes that are used as features, labels, criteria, etc, optional parameters can be used. So instead of: + +``` + FeatureNodes=(features) + LabelNodes=(labels) + CriteriaNodes=(CrossEntropy) + EvalNodes=(ErrPredict) + OutputNodes=(Plus2) +``` + +The network can be defined as + +``` + # constants defined + # Sample, Hidden, and Label dimensions + SDim=784 + HDim=256 + LDim=10 + + features=Input(SDim, tag=feature) + labels=Input(LDim, tag=label) + + # Layer operations + L1 = RBFF(features, HDim, SDim) + L2 = RBFF(L1, HDim, HDim) + L3 = RBFF(L2, HDim, HDim) + CE = SMBFF(L3, LDim, HDim, labels, tag=Criteria) + Err=ErrorPrediction(labels, CE.F, tag=Eval) + + # rootNodes defined here + OutputNodes=(CE.F) +``` + +Which avoids adding elements to the node arrays, and instead sets the ‘tag’ optional parameter on the functions or macros that return the value that fits into a specified category. In this case, since the output node is actually computed inside a macro, we must specify it explicitly. + +## NDL Reference + +### Variables + +Variables are defined in NDL when they appear on the left of an equal sign (‘=’). From that point on that variable name will be associated with the value it was assigned. Variables are immutable, and assigning new values to an existing variable is not supported. + +Variable names may be any alphanumeric sequence that starts with a letter. The variables can contain a matrix or scalar value. + +#### Reserved words + +Any name that is also a function name is a reserved word and cannot be used for a variable. The special node names are also reserved and are as follows: + +* `FeatureNodes` +* `LabelNodes` +* `CriteriaNodes` +* `EvalNodes` +* `OutputNodes` + +These may not be used as variable names. + +#### Dot names + +When it is necessary to access a variable that is defined in a macro (see Macros below), it can be accessed using dot-names. If the following macro is called from code: + +``` + L1 = FF(features, HDim, SDim) +``` + +And the macro is defined as follows: + +``` + FF(X1, W1, B1) + { + T=Times(W1,X1); + FF=Plus(T, B1); + } +``` + +If I want to access the result of the Times() function before the Plus happened, I can with the following variable: + +``` + L1.T +``` + +The variable name used in the script followed by a dot and the local name in the macro. This does requires the user to know the name used in the macro, so having all macro definitions available is important. Since macros can be nested, dot names can be several layers deep if necessary. + +### Functions + +Functions are called using function call syntax similar to most programming languages: + +``` + Times1=Times(W0, features) +``` + +The function name is followed by parenthesis which contains the comma separated parameter list. Each function returns a single value, which is identified by a variable. + +### Macros + +Macros are a combination of multiple Functions combined in a block. This can be done in a single-line nested fashion: + +``` + RFF(x1, w1, b1)=RectifiedLinear(Plus(Times(w1,x1),b1)) +``` + +In this case the functions called will be evaluated from the innermost nested function call to the outermost. + +The other method of defining macros uses a “programming block” style: + +``` + FF(X1, W1, B1) + { + T=Times(W1,X1); + FF=Plus(T, B1); + } +``` + +In this case the intermediate variables, which are local to the macro, can still be accessed from the outside using the dot syntax for variables. + +### Optional Parameters + + +Many functions will have optional parameters that will change the behavior of the function. For example: + +``` + B0=Parameter(HDim, init=zero) +``` + +In this example the Bias vector will be zero initialized. The NDL Function reference will specify what optional parameters are accepted by each function + +#### Tags + +Tags are a special case of optional parameters, and are discussed in the Special Nodes section. + +### Special nodes + +Special nodes need to be identified for CNTK to automatically do back propagation updates of Learnable Parameters and identify inputs properly. These special nodes be specified in two different ways, the node arrays, or by use of special tags. If both methods are used the values are combined. + +#### Node Arrays + +CNTK supports multiple nodes for each type, so all these values are arrays. However, In many cases there will only be a single node for each node type. The array syntax (parenthesis) must be used when setting these special nodes, even if there is only one element. If more than one element is include, the entries are comma separated and surrounded by parenthesis. For example: + +``` + FeatureNodes=(features) + LabelNodes=(labels) + CriteriaNodes=(CrossEntropy) + EvalNodes=(ErrPredict) + OutputNodes=(Plus2) +``` + +#### Tags + +A special optional parameter is a “tag”. These can be used as a shortcut way to identify special values in the network. For example features and labels can be tagged as such when the inputs are defined, as follows: + +``` + F1=Input(SDim, tag=feature) + L1=Input(LDim, tag=label) +``` + +The acceptable tag names correspond to the special node types and are as follows: + +Tag name | Meaning +---|--- +feature | A feature input +label | A label input +criteria | criteria node, top level node +eval | evaluation node +Output | output node + +## NDL Functions + +This section contains the currently implemented NDL functions. The CNTK is being expanded and additional functions will be available as development continues. + +### Input, InputValue + +Defines input data for the network. This defines the input that will be read from a datasource. The datasource information is specified in the configuration file separately, allowing the same network to be used with multiple datasets easily. + + +`Input(rows, [cols=1])` + +`InputValue(rows, [cols=1])` + + +#### Parameters + +`rows` – row dimension of the data. + +`cols` – \[optional\] col dimension of the data. If this dimension is not specified, it is assumed to be 1 + +#### Notes + +Input nodes are normally tagged with their intended purpose so the CNTK can use the inputs appropriately. The following tags may be used as optional parameters, and specify feature values, and label values respectively: + +`tag=feature` + +`tag=label` + +### ImageInput + +Defines image input data for the network. This defines the input that will be read from a datasource. The datasource information is specified in the configuration file separately, allowing the same network to be used with multiple datasets easily. + +ImageInput(width, height, channels, \[numImages=1\]) + +#### Parameters + +`width` – width of the image data. + +`height` – height of the image data. + +`channels` – number of channels in the image data (i.e. RGB would have 3 channels) + +`numImages` – \[optional\] number of images in each sample, defaults to 1 + +#### Notes + +Each data element is expected to be in 16-bit (single) or 32-bit (double) floating point format. The order of the data from least frequently changing to most frequently changing is Image, Col, Row, Channel. + +Input nodes are normally tagged with their intended purpose so the CNTK can use the inputs appropriately. The following tags may be used as optional parameters, and specify feature values, and label values respectively: + +`tag=feature` + +`tag=label` + +### Parameter, LearnableParameter + +Defines a parameter in the network that will be trained. Normally used for weight and bias matrices/vectors. + + +`Parameter(row, \[cols\])` + +`LearnableParameter(rows, \[cols\])` + +#### Parameters + +`rows` – number of rows in the parameter, this will normally be determined by the Input size, a hidden weight/bias matrix size, or an output size. + +`cols` – (optional, defaults to 1) number of columns in the parameter data. This is often left at the default value to be determined by the minibatch size when processing the network. + +#### Optional Parameters + +`ComputeGradient=[true,false]` – Turns on (or off) automatic gradient calculation required for Stochastic Gradient Descent (SGD) training. Defaults to on. + +`InitValueScale=number` – Initialization value for the input. Depending on the initialization technique this number is used to determine the range of the random numbers used for initialization. Defaults to 0.05 producing random numbers in a range of \(\lbrack - 0.05 - 0.05\rbrack\) + +`Init = [None, Zero, Uniform, Gaussian]` – Form of initialization for inputs + +- None – No initialization is required, should only be used if the network will be initializing in some other way + +- Zero – zero initialize the parameter matrix + +- Uniform – Initializes the parameter matrix with random numbers based on the InitValueScale in the following range: \(\pm InitValueScale/\sqrt{\text{cols}}\) + +- Gaussian – Initializes the parameter matrix with random numbers using a Gaussian distribution in the range \(\pm (0.2)InitValueScale/\sqrt{\text{cols}}\) + +### Sum + +Calculate the sum of two matrices. + +`Sum(add1, add2)` + +#### Parameters + +`add1`, `add2` – matrix values, must be the same dimensions. + +#### Returns + +`add1`+`add2`, the element-wise matrix sum of the parameters. The result of the sum is stored in the `add1` matrix (`add1+=add2`) + +### Scale + +Scale a matrix by a scalar value + +`Scale(scaleFactor, matrix)` + +#### Parameters + +`scaleFactor` – floating point scalar scale factor + +`matrix` - matrix values, must be the same dimensions. + +#### Returns + +`scaleFactor * matrix`, the element-wise product of the scaleFactor and matrix + +### Times + +Calculate the sum of two matrices. + +`Times(mult1, mult2)` + +#### Parameters + +`mult1`, `mult2` – matrix values, the mult1.rows must equal mult2.cols. + +#### Returns + +`mult1 * mult2`, the matrix product of the parameters + +### Plus + +Calculate the sum of two matrices. + +`Plus(add1, add2)` + +#### Parameters + +`add1`, `add2` – matrix values, must be the same dimensions. + +#### Returns + +`add1+add2`, the element-wise matrix sum of the parameters + +### Minus + +Calculate the difference of two matrices. + +`Minus(sub1, sub2)` + +#### Parameters + +`sub1`, `sub2` – matrix values, must be the same dimensions. + +#### Returns + +`sub1 - sub2`, the element-wise matrix difference of the parameters + +### Negate + +Negate the matrix. + +`Negate(matrix)` + +#### Parameters + +`matrix` – matrix value. + +#### Returns + +`-(matrix)`, the element-wise negation of all elements of the matrix + +### RectifiedLinear + +Compute the RectifiedLinear operation on the matrix. + +`RectifiedLinear(matrix)` + +#### Parameters + +`matrix` – matrix value. + +#### Returns + +`RectifiedLinear(matrix)`, the element-wise rectified linear operation of all elements of the matrix + +### Sigmoid + +Compute the Sigmoid of the matrix. + +`Sigmoid(matrix)` + +#### Parameters + +`matrix` – matrix value. + +#### Returns + +`1 / (1 + (e ^ -t))`, the element-wise sigmoid of all elements of the matrix + +### Tanh + +Compute the Hyperbolic Tangent of the matrix elements. + +`Tanh(matrix)` + +#### Parameters + +`matrix` – matrix value. + +#### Returns + +`tanh(matrix)` the element-wise hyperbolic tangent of all elements of the matrix + +### Log + +Compute the Logarithm (base 10) of the matrix elements. + +`Log(matrix)` + +#### Parameters + +`matrix` – matrix value. + +#### Returns + +`log(matrix)` + +the element-wise logarithm of all elements of the matrix + +### Softmax + +Compute the Softmax of the matrix. + +`Softmax(matrix)` + +#### Parameters + +`matrix` – matrix value. + +#### Returns + +`softmax(matrix)` the softmax of the matrix + +### SquareError + +Compute the SquareError of the matrix. + +`SquareError(m1, m2)` + +#### Parameters + +`m1` – first matrix to compare. + +`m2` - second matrix to compare + +#### Returns + +The square error value of the matrix, returned in a 1x1 matrix + +### CrossEntropyWithSoftmax, CEWithSM + +Compute the Softmax of the matrix, compare against the ground truth labels and compute the CrossEntropy error matrix. + +`CrossEntropyWithSoftmax(labels, matrix)` + +`CEWithSM(labels, matrix)` + +#### Parameters + +`labels` – the ground truth labels + +`matrix` – matrix value. + +#### Returns + +the CrossEntropy error matrix + +#### Notes + +This node will often be tagged as a “Criteria” node to allow the CNTK to identify the node producing the error matrix. To tag appropriate node(s) the following optional parameter should be added to the call(s): + +`tag=Criteria` + +### MatrixL1Reg, L1Reg + +Compute the sum of the absolute value of the entire matrix. + +`MatrixL1Reg(matrix)` + +`L1Reg(matrix)` + +#### Parameters + +`matrix` – matrix to use in computation + +#### Returns + +the sum of the absolute value of the matrix elements, returned in a 1x1 matrix + +### MatrixL2Reg, L2Reg + +Compute the FrobeniusNorm of the matrix. + +`MatrixL2Reg(matrix)` + +`L2Reg(matrix)` + +#### Parameters + +`matrix` – matrix to compute the FrobeniusNorm on. + +#### Returns + +The FrobeniusNorm of the matrix, returned in a 1x1 matrix + +### PerDimMeanVarNormalization, PerDimMVNorm + +Compute the Mean-Variance Normalized Matrix + +`PerDimMeanVarNormalization(matrix, mean, invStdDev)` + +`PerDimMVNorm(matrix, mean, invStdDev)` + +#### Parameters + +`matrix` – matrix than needs to be normalized + +`mean` – the mean for each sample index (same row dimensions as “matrix”) + +`invStdDev` – 1/stddev for each sample index. (same row dimensions as “matrix”) + +#### Returns + +The mean variance normalized matrix + +#### Notes + +This function requires the Mean and InvStdDev to be already computed. They can either be loaded from a dataset, or computed in a pre-pass, before normalization is required. + +### ErrorPrediction + +Evaluate the accuracy of the current predictions made by the model. This is generally used to compute the training accuracy of a model. It finds the highest predicted probability from the model and compares it to the actual ground truth. + +`ErrorPrediction(labels, matrix)` + +#### Parameters + +`labels` – the ground truth labels + +`matrix` – matrix value. + +#### Returns + +The number of predicted values that do not match the labels in the current minibatch. Returns a 1x1 matrix + +#### Notes + +This node will often be tagged as an “Eval” node to allow the CNTK to print ongoing error statistics during training. To take appropriate node(s) the following optional parameter should be added to the call(s): + +`tag=Eval` + +### Dropout + +Compute a new matrix with *dropoutRate* perecent set to zero. The values that are set to zero are randomly chosen. This is commonly used to prevent overfitting during the training process. + +`Dropout(matrix)` + +#### Parameters + +`matrix` – source matrix + +#### Returns + +a new matrix with *dropoutRate* percent of the elements set to zero (dropped out). + +#### Optional Parameters + +`dropoutRate` – The percent (represented as a decimal 0.0-1.0) of values that will be dropped on each iteration. + +### Mean + +Compute the Per dim mean matrix for the entire dataset + +`Mean(matrix)` + +#### Parameters + +`matrix` – source matrix + +#### Returns + +_Note: Can't use LaTex on GitHub, so this is a patched together solution_ + +`mean(i) = (Sum from j=0 to j=n of matrix(i,j)) / n` + +Where 'n' is the size of the entire dataset + +#### Notes + +This function is a pre-pass function, will only be called during a pre-pass through the entire dataset before the first training pass. This allows the Mean to be computed before it is required for Mean-Variance Normalization. + +### Convolution, Convolve + +Compute the convolution of an image input + +`Convolution(cvweight, features, kernelWidth, kernelHeight, outputChannels, horizontalSubsample, verticalSubsample, zeroPadding=false)` + +#### Parameters + +`cvweight` – convolution weight matrix, it has the dimensions of \[outputChannels, kernelWidth \* kernelHeight \* inputChannels\] + +`kernelWidth` – width of the kernel + +`kernelHeight` – height of the kernel + +`outputChannels` – number of output channels + +`horizontalSubsample` – subsamples in the horizontal direction + +`verticalSubsample` – subsamples in the vertical direction + +#### Optional Parameters + +`zeroPadding` – \[default = false\] should the sides of the image be padded with zeros? + +`maxTempMemSizeInSamples` – \[default=0\] maximum amount of memory (in samples) that should be reserved as temporary space + +#### Returns + +The convolved matrix according to the parameters passed + +#### Notes + +The input to this node must be an ImageInput(). This node automatically determines image size on input and output based on the size of the original input and which nodes the input has passed through. This function is often followed by another Convolution() or a MaxPooling() or AveragePooling() node. + +### MaxPooling + +Computes a new matrix by selecting the maximum value in the pooling window. This is used to reduce the dimensions of a matrix. + +`MaxPooling(matrix, windowWidth, windowHeight, stepW, stepH)` + +#### Parameters + +`matrix` – input matrix + +`windowWidth` – width of the pooling window + +`windowHeight` – height of the pooling window + +`stepW` – step used in the width direction + +`stepH` – step used in the height direction + +#### Returns + +The dimension reduced matrix consisting of the maximum value within each pooling window. + +#### Notes + +This function is often associated with Convolution() operations. + +### AveragePooling + +Computes a new matrix by selecting the average value in the pooling window. This is used to reduce the dimensions of a matrix. + +`MaxPooling(matrix, windowWidth, windowHeight, stepW, stepH)` + +#### Parameters + +`matrix` – input matrix + +`windowWidth` – width of the pooling window + +`windowHeight` – height of the pooling window + +`stepW` – step used in the width direction + +`stepH` – step used in the height direction + +#### Returns + +The dimension reduced matrix consisting of the maximum value within each pooling window. + +#### Notes + +This function is often associated with Convolution() operations. + +### Delay + +Delay node used in recurrent networks, allows creation of a loop in the convolutional network that will repeat a specified number of times. + +`Delay(rows, [cols], delayNode, delayTime=1, needGradient=true, defaultHiddenActivity=0.1)` + +#### Parameters + +`cvweight` – convolution weight matrix, it has the dimensions of \[outputChannels, kernelWidth \* kernelHeight \* inputChannels\] + +`kernelWidth` – width of the kernel + +`kernelHeight` – height of the kernel + +`outputChannels` – number of output channels + +`horizontalSubsample` – subsamples in the horizontal direction + +`verticalSubsample` – subsamples in the vertical direction + +#### Optional Parameters + +`delayTime` – \[default = 1\] the amount of delay that will be introduced (number of times the loop will happen) + +`needGradient` – \[default = true\] does the gradient need to be computed for this node + +`defaultHiddenActivity` – \[default = 0.1\] the numerical amount for the defaultHiddenActivity + +#### Returns + +The results of the completed Delay loop + +#### Notes + +This node is used in recurrent networks, where a delay is introduced to examine values from a previous time, such as the prior value (t-1). This has the affect of creating a loop in the computational network that will repeat delayTime number of iterations. \ No newline at end of file diff --git a/Examples/Image/MNIST/AdditionalFiles/mnist_convert.py b/Examples/Image/MNIST/AdditionalFiles/mnist_convert.py index bf05a0702..08ffb3980 100644 --- a/Examples/Image/MNIST/AdditionalFiles/mnist_convert.py +++ b/Examples/Image/MNIST/AdditionalFiles/mnist_convert.py @@ -7,9 +7,9 @@ import struct import numpy as np def loadData(src, cimg): - print 'Downloading ' + src + print ('Downloading ' + src) gzfname, h = urllib.urlretrieve(src, './delete.me') - print 'Done.' + print ('Done.') try: with gzip.open(gzfname) as gz: n = struct.unpack('I', gz.read(4)) diff --git a/Examples/Image/Miscellaneous/CIFAR-10/CIFAR_convert.py b/Examples/Image/Miscellaneous/CIFAR-10/CIFAR_convert.py index fdd65f2c1..d0274c6b2 100644 --- a/Examples/Image/Miscellaneous/CIFAR-10/CIFAR_convert.py +++ b/Examples/Image/Miscellaneous/CIFAR-10/CIFAR_convert.py @@ -32,39 +32,39 @@ def readBatch(src, outFmt): feat[:, 1::3] = g feat[:, 2::3] = b else: - print 'Format not supported: ' + outFmt + print ('Format not supported: ' + outFmt) usage() sys.exit(1) return np.hstack((np.reshape(d['labels'], (len(d['labels']), 1)), feat)) def loadData(src, outFmt): - print 'Downloading ' + src + print ('Downloading ' + src) fname, h = urllib.urlretrieve(src, './delete.me') - print 'Done.' + print ('Done.') try: - print 'Extracting files...' + print ('Extracting files...') with tarfile.open(fname) as tar: tar.extractall() - print 'Done.' - print 'Preparing train set...' + print ('Done.') + print ('Preparing train set...') trn = np.empty((0, NumFeat + 1)) for i in range(5): batchName = './cifar-10-batches-py/data_batch_{0}'.format(i + 1) trn = np.vstack((trn, readBatch(batchName, outFmt))) - print 'Done.' - print 'Preparing test set...' + print ('Done.') + print ('Preparing test set...') tst = readBatch('./cifar-10-batches-py/test_batch', outFmt) - print 'Done.' + print ('Done.') finally: os.remove(fname) return (trn, tst) def usage(): - print 'Usage: CIFAR_convert.py [-f ] \n where format can be either cudnn or legacy. Default is cudnn.' + print ('Usage: CIFAR_convert.py [-f ] \n where format can be either cudnn or legacy. Default is cudnn.') def parseCmdOpt(argv): if len(argv) == 0: - print "Using cudnn output format." + print ("Using cudnn output format.") return "cudnn" try: opts, args = getopt.getopt(argv, 'hf:', ['help', 'outFormat=']) @@ -78,7 +78,7 @@ def parseCmdOpt(argv): elif opt in ('-f', '--outFormat'): fmt = arg if fmt != 'cudnn' and fmt != 'legacy': - print 'Invalid output format option.' + print ('Invalid output format option.') usage() sys.exit(1) return fmt @@ -86,9 +86,9 @@ def parseCmdOpt(argv): if __name__ == "__main__": fmt = parseCmdOpt(sys.argv[1:]) trn, tst = loadData('http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz', fmt) - print 'Writing train text file...' + print ('Writing train text file...') np.savetxt(r'./Train.txt', trn, fmt = '%u', delimiter='\t') - print 'Done.' - print 'Writing test text file...' + print ('Done.') + print ('Writing test text file...') np.savetxt(r'./Test.txt', tst, fmt = '%u', delimiter='\t') - print 'Done.' + print ('Done.') diff --git a/Examples/Speech/Miscellaneous/AMI/scripts/Convert_label_to_cntk.py b/Examples/Speech/Miscellaneous/AMI/scripts/Convert_label_to_cntk.py index ba7dc24e3..eb9ac03b5 100644 --- a/Examples/Speech/Miscellaneous/AMI/scripts/Convert_label_to_cntk.py +++ b/Examples/Speech/Miscellaneous/AMI/scripts/Convert_label_to_cntk.py @@ -5,7 +5,7 @@ import string import os def usage(): - print 'Convert_label_to_cntk.py -in [fea_list file] [label_list_file] delay_number' + print ('Convert_label_to_cntk.py -in [fea_list file] [label_list_file] delay_number') def createDir(d): if not os.path.isdir(d): @@ -22,12 +22,12 @@ else: delay = int(sys.argv[4]) lablines = [x.rstrip() for x in fr] fr.close() - print "#!MLF!#" + print ("#!MLF!#") frnum=0 for line in lines: linenew = line.split('.') tmp = linenew[0].split('/') - print "\""+tmp[-1]+"\"" + print ("\""+tmp[-1]+"\"") fr = open (lablines[frnum], 'r') labs = [x.rstrip() for x in fr] fr.close() @@ -37,8 +37,8 @@ else: k = i-delay if i-delay < 0: k = 0 - print i, j, labs[k], labs[k] + print (i, j, labs[k], labs[k]) i = i+1 - print "." + print (".") frnum = frnum + 1 diff --git a/Makefile b/Makefile index 1807079de..05b518096 100644 --- a/Makefile +++ b/Makefile @@ -178,9 +178,10 @@ ifeq ("$(BUILDTYPE)","release") GENCODE_FLAGS := $(GENCODE_SM20) $(GENCODE_SM30) $(GENCODE_SM35) $(GENCODE_SM50) endif - CXXFLAGS += -O4 + CXXFLAGS += -g -O4 + LDFLAGS += -rdynamic CPPFLAGS += -DNDEBUG - CUFLAGS += -O3 -use_fast_math -lineinfo $(GENCODE_FLAGS) + CUFLAGS += -O3 -g -use_fast_math -lineinfo $(GENCODE_FLAGS) endif ifdef CNTK_CUDA_DEVICE_DEBUGINFO diff --git a/README.md b/README.md index d57c75164..6315e42db 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ The figure below compares processing speed (frames processed per second) of CNTK If you used this toolkit or part of it to do your research, please cite the work as: -Amit Agarwal, Eldar Akchurin, Chris Basoglu, Guoguo Chen, Scott Cyphers, Jasha Droppo, Adam Eversole, Brian Guenter, Mark Hillebrand, Ryan Hoens, Xuedong Huang, Zhiheng Huang, Vladimir Ivanov, Alexey Kamenev, Philipp Kranen, Oleksii Kuchaiev, Wolfgang Manousek, Avner May, Bhaskar Mitra, Olivier Nano, Gaizka Navarro, Alexey Orlov, Hari Parthasarathi, Baolin Peng, Marko Radmilac, Alexey Reznichenko, Frank Seide, Michael L. Seltzer, Malcolm Slaney, Andreas Stolcke, Huaming Wang, Yongqiang Wang, Kaisheng Yao, Dong Yu, Yu Zhang, Geoffrey Zweig (in alphabetical order), ["An Introduction to Computational Networks and the Computational Network Toolkit"](http://research.microsoft.com/apps/pubs/?id=226641), Microsoft Technical Report MSR-TR-2014-112, 2014. +Amit Agarwal, Eldar Akchurin, Chris Basoglu, Guoguo Chen, Scott Cyphers, Jasha Droppo, Adam Eversole, Brian Guenter, Mark Hillebrand, T. Ryan Hoens, Xuedong Huang, Zhiheng Huang, Vladimir Ivanov, Alexey Kamenev, Philipp Kranen, Oleksii Kuchaiev, Wolfgang Manousek, Avner May, Bhaskar Mitra, Olivier Nano, Gaizka Navarro, Alexey Orlov, Hari Parthasarathi, Baolin Peng, Marko Radmilac, Alexey Reznichenko, Frank Seide, Michael L. Seltzer, Malcolm Slaney, Andreas Stolcke, Huaming Wang, Yongqiang Wang, Kaisheng Yao, Dong Yu, Yu Zhang, Geoffrey Zweig (in alphabetical order), ["An Introduction to Computational Networks and the Computational Network Toolkit"](http://research.microsoft.com/apps/pubs/?id=226641), Microsoft Technical Report MSR-TR-2014-112, 2014. ## Disclaimer -CNTK is in active use at Microsoft and constantly evolving. There will be bugs in places. +CNTK is in active use at Microsoft and constantly evolving. There will be bugs. diff --git a/Source/ActionsLib/ActionsLib.vcxproj b/Source/ActionsLib/ActionsLib.vcxproj index 9da302060..43d51748c 100644 --- a/Source/ActionsLib/ActionsLib.vcxproj +++ b/Source/ActionsLib/ActionsLib.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {EB2BE26F-6BD4-4274-971F-86D080779DD1} @@ -24,14 +32,26 @@ CNTK ActionsLib + + + + false + + + + + true + + + - + StaticLibrary true v120 Unicode - + StaticLibrary false v120 @@ -40,31 +60,35 @@ - + - + - + true - ..\SequenceTrainingLib;..\SGDLib;..\ComputationNetworkLib;..\CNTK;..\Math;..\Common\Include;..\CNTK\BrainScript;$(MSMPI_INC);$(CUDA_PATH)\include;$(VCInstallDir)include;..\..\DataReader\HTKMLFReader;$(WindowsSDK_IncludePath) - C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform) + ..\SequenceTrainingLib;..\SGDLib;..\ComputationNetworkLib;..\CNTK;..\Math;..\Common\Include;..\CNTK\BrainScript;$(MSMPI_INC);$(VCInstallDir)include;..\..\DataReader\HTKMLFReader;$(WindowsSDK_IncludePath) + $(MSMPI_LIB64);$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64) Build $(Platform)\$(Configuration)\$(ProjectName)\ false - + false - ..\SequenceTrainingLib;..\SGDLib;..\ComputationNetworkLib;..\CNTK;..\Math;..\Common\Include;..\CNTK\BrainScript;$(MSMPI_INC);$(CUDA_PATH)\include;$(VCInstallDir)include;..\..\DataReader\HTKMLFReader;$(WindowsSDK_IncludePath) - C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform) + ..\SequenceTrainingLib;..\SGDLib;..\ComputationNetworkLib;..\CNTK;..\Math;..\Common\Include;..\CNTK\BrainScript;$(MSMPI_INC);$(VCInstallDir)include;..\..\DataReader\HTKMLFReader;$(WindowsSDK_IncludePath) + $(MSMPI_LIB64);$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64) Build $(ExecutablePath) $(Platform)\$(Configuration)\$(ProjectName)\ false - + + $(IncludePath);$(CUDA_PATH)\include; + $(LibraryPath);$(CUDA_PATH)\lib\$(Platform) + + @@ -85,10 +109,6 @@ Math.dll; nvml.dll; cudart64_70.dll 100000000 - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - @@ -103,7 +123,15 @@ - + + + %(PreprocessorDefinitions);CPUONLY + + + Math.dll + + + Level4 @@ -128,10 +156,6 @@ Math.dll; nvml.dll; cudart64_70.dll "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - @@ -150,6 +174,22 @@ + + + %(PreprocessorDefinitions);CPUONLY + + + + Math.dll + + + + + + if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) + Copying NVidia GDK extension DLL to target folder + + @@ -164,10 +204,10 @@ - NotUsing + NotUsing - NotUsing + NotUsing diff --git a/Source/CNTK/CNTK.vcxproj b/Source/CNTK/CNTK.vcxproj index 1e68985b8..e55fbb31f 100644 --- a/Source/CNTK/CNTK.vcxproj +++ b/Source/CNTK/CNTK.vcxproj @@ -9,7 +9,27 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + + + + + false + + + + + true + + + {E6F26F9A-FF64-4F0A-B749-CD309EE357EE} @@ -25,13 +45,13 @@ CNTK - + Application true v120 Unicode - + Application false v120 @@ -40,29 +60,33 @@ - + - + - + true - ..\..\Multiverso\include;$(MSMPI_INC);..\ActionsLib;..\SequenceTrainingLib;..\SGDLib;..\ComputationNetworkLib;..\Math;..\Common\Include;..\CNTK\BrainScript;..\..\DataReader\HTKMLFReader;C:\Program Files (x86)\Microsoft SDKs\MPI\Include;$(CUDA_PATH)\include;$(VCInstallDir)include;$(WindowsSDK_IncludePath); - ..\..\Multiverso\x64\$(Configuration);$(MSMPI_LIB64);C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform) + ..\ActionsLib;..\SequenceTrainingLib;..\SGDLib;..\ComputationNetworkLib;..\Math;..\Common\Include;..\CNTK\BrainScript;..\..\DataReader\HTKMLFReader;$(MSMPI_INC);$(VCInstallDir)include;$(WindowsSDK_IncludePath); + $(MSMPI_LIB64);$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64) Build $(Platform)\$(Configuration)\$(ProjectName)\ - + false - ..\..\Multiverso\include;$(MSMPI_INC);..\ActionsLib;..\SequenceTrainingLib;..\SGDLib;..\ComputationNetworkLib;..\Math;..\Common\Include;..\CNTK\BrainScript;..\..\DataReader\HTKMLFReader;C:\Program Files (x86)\Microsoft SDKs\MPI\Include;$(CUDA_PATH)\include;$(VCInstallDir)include;$(WindowsSDK_IncludePath); - ..\..\Multiverso\x64\$(Configuration);$(MSMPI_LIB64);C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform) + ..\ActionsLib;..\SequenceTrainingLib;..\SGDLib;..\ComputationNetworkLib;..\Math;..\Common\Include;..\CNTK\BrainScript;..\..\DataReader\HTKMLFReader;$(MSMPI_INC);$(VCInstallDir)include;$(WindowsSDK_IncludePath); + $(MSMPI_LIB64);$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64) Build $(ExecutablePath) $(Platform)\$(Configuration)\$(ProjectName)\ - + + $(IncludePath);$(CUDA_PATH)\include + $(LibraryPath);$(CUDA_PATH)\lib\$(Platform) + + @@ -83,10 +107,6 @@ Math.dll; msmpi.dll; nvml.dll; cudart64_70.dll 100000000 - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - @@ -100,7 +120,15 @@ prebuild.bat - + + + %(PreprocessorDefinitions);CPUONLY + + + Math.dll; msmpi.dll; + + + Level4 @@ -126,10 +154,6 @@ "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" 100000000 - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - @@ -147,6 +171,20 @@ prebuild.bat + + + %(PreprocessorDefinitions);CPUONLY + + + Math.dll; msmpi.dll; + + + + + if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) + Copying NVidia GDK extension DLL to target folder + + @@ -201,20 +239,18 @@ - NotUsing - NotUsing + NotUsing - NotUsing + NotUsing - NotUsing + NotUsing - - + diff --git a/Source/ComputationNetworkLib/ComputationNetwork.h b/Source/ComputationNetworkLib/ComputationNetwork.h index dbf3ff0b2..c756e1ba3 100644 --- a/Source/ComputationNetworkLib/ComputationNetwork.h +++ b/Source/ComputationNetworkLib/ComputationNetwork.h @@ -922,6 +922,10 @@ private: }; typedef ComputationNetwork::ComputationNetworkPtr ComputationNetworkPtr; +// The following emits the class and enables the BaseMatrix to be available (used by EvalDll) +// The corresponding Matrix is emitted in the SetDeviceId function above. +template class Matrix; + // TODOs: // - automatic inference of time window w.r.t. delay nodes (and related nodes such as a temporal pooling) // - have overrides of RuntimeError etc. in ComputationNode, which prepend the error string with the node name and operation diff --git a/Source/ComputationNetworkLib/ComputationNetworkLib.vcxproj b/Source/ComputationNetworkLib/ComputationNetworkLib.vcxproj index b5224cf9b..01ac5787f 100644 --- a/Source/ComputationNetworkLib/ComputationNetworkLib.vcxproj +++ b/Source/ComputationNetworkLib/ComputationNetworkLib.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {928ABD1B-4D3B-4017-AEF1-0FA1B4467513} @@ -24,14 +32,26 @@ CNTK ComputationNetworkLib + + + + false + + + + + true + + + - + StaticLibrary true v120 Unicode - + StaticLibrary false v120 @@ -40,31 +60,35 @@ - + - + - + true - $(MSMPI_INC);..\SequenceTrainingLib;..\Math;..\Common\Include;..\CNTK\BrainScript;C:\Program Files (x86)\Microsoft SDKs\MPI\Include;$(CUDA_PATH)\include;$(VCInstallDir)include;..\..\DataReader\HTKMLFReader;$(WindowsSDK_IncludePath) - $(MSMPI_LIB64);C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform) + ..\SequenceTrainingLib;..\Math;..\Common\Include;..\CNTK\BrainScript;$(MSMPI_INC);$(VCInstallDir)include;..\..\DataReader\HTKMLFReader;$(WindowsSDK_IncludePath) + $(MSMPI_LIB64);$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64) Build $(Platform)\$(Configuration)\$(ProjectName)\ false - + false - $(MSMPI_INC);..\SequenceTrainingLib;..\Math;..\Common\Include;..\CNTK\BrainScript;C:\Program Files (x86)\Microsoft SDKs\MPI\Include;$(CUDA_PATH)\include;$(VCInstallDir)include;..\..\DataReader\HTKMLFReader;$(WindowsSDK_IncludePath) - $(MSMPI_LIB64);C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform) + ..\SequenceTrainingLib;..\Math;..\Common\Include;..\CNTK\BrainScript;$(MSMPI_INC);$(VCInstallDir)include;..\..\DataReader\HTKMLFReader;$(WindowsSDK_IncludePath) + $(MSMPI_LIB64);$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64) Build $(ExecutablePath) $(Platform)\$(Configuration)\$(ProjectName)\ false - + + $(IncludePath);$(CUDA_PATH)\include + $(LibraryPath);$(CUDA_PATH)\lib\$(Platform) + + @@ -85,10 +109,6 @@ Math.dll; nvml.dll; cudart64_70.dll 100000000 - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - @@ -103,7 +123,17 @@ - + + + CPUONLY;%(PreprocessorDefinitions) + + + + Math.dll + + + + Level4 @@ -128,10 +158,6 @@ Math.dll; nvml.dll; cudart64_70.dll "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - @@ -150,6 +176,22 @@ + + + CPUONLY;%(PreprocessorDefinitions) + + + + Math.dll + + + + + + if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) + Copying NVidia GDK extension DLL to target folder + + @@ -183,10 +225,10 @@ - NotUsing + NotUsing - NotUsing + NotUsing diff --git a/Source/EvalDll/EvalDll.vcxproj b/Source/EvalDll/EvalDll.vcxproj index e374d3d45..0d59a0fd1 100644 --- a/Source/EvalDll/EvalDll.vcxproj +++ b/Source/EvalDll/EvalDll.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {482999D1-B7E2-466E-9F8D-2119F93EAFD9} @@ -24,14 +32,26 @@ EvalDll EvalDll + + + + false + + + + + true + + + - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -41,35 +61,39 @@ - + - + - + true - $(MSMPI_INC);..\SGDLib;..\ComputationNetworkLib;..\SequenceTrainingLib;..\Math;..\Common\Include;..\CNTK\BrainScript;C:\Program Files (x86)\Microsoft SDKs\MPI\Include;$(CUDA_PATH)\include;$(VCInstallDir)include;$(WindowsSDK_IncludePath) - $(MSMPI_LIB64);..\ComputationNetworkLib;..\Math;C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(CUDA_PATH)\lib\$(Platform);$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(Platform) + ..\SGDLib;..\ComputationNetworkLib;..\SequenceTrainingLib;..\Math;..\Common\Include;..\CNTK\BrainScript;$(MSMPI_INC);$(VCInstallDir)include;$(WindowsSDK_IncludePath) + ..\ComputationNetworkLib;..\Math;$(MSMPI_LIB64);$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(Platform);$(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\$(ProjectName)\ EvalDll - + false - $(MSMPI_INC);..\SGDLib;..\ComputationNetworkLib;..\SequenceTrainingLib;..\Math;..\Common\Include;..\CNTK\BrainScript;C:\Program Files (x86)\Microsoft SDKs\MPI\Include;$(CUDA_PATH)\include;$(VCInstallDir)include;$(WindowsSDK_IncludePath) - $(MSMPI_LIB64);..\ComputationNetworkLib;..\Math;C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(CUDA_PATH)\lib\$(Platform);$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(Platform) + ..\SGDLib;..\ComputationNetworkLib;..\SequenceTrainingLib;..\Math;..\Common\Include;..\CNTK\BrainScript;$(MSMPI_INC);$(CUDA_PATH)\include;$(VCInstallDir)include;$(WindowsSDK_IncludePath) + ..\ComputationNetworkLib;..\Math;$(MSMPI_LIB64);$(CUDA_PATH)\lib\$(Platform);$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(Platform);$(SolutionDir)$(Platform)\$(Configuration)\ $(Platform)\$(Configuration)\$(ProjectName)\ EvalDll - + + $(IncludePath);$(CUDA_PATH)\include + $(LibraryPath);$(CUDA_PATH)\lib\$(Platform) + + NotUsing Level4 Disabled - EVALDLL;WIN32;_DEBUG;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + EVALDLL;WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true - ..\Common\include;..\Math\Math;"c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\include" + "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\include" true /bigobj %(AdditionalOptions) @@ -77,24 +101,30 @@ Console true ComputationNetworkLib.lib; Math.lib; kernel32.lib; user32.lib; shell32.lib; SequenceTrainingLib.lib; %(AdditionalDependencies) - $(SolutionDir)$(Platform)\$(Configuration)\; "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" + "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" Math.dll; nvml.dll; cudart64_70.dll - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - - + + + CPUONLY;%(PreprocessorDefinitions) + + + + Math.dll + + + + Level4 NotUsing MaxSpeed true true - EVALDLL;WIN32;NDEBUG;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + EVALDLL;WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true - ..\Common\include; ..\Math\Math; "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\include" + "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\include" false /d2Zi+ /bigobj %(AdditionalOptions) MultiThreadedDLL @@ -107,10 +137,22 @@ true true ComputationNetworkLib.lib; Math.lib; kernel32.lib; user32.lib; shell32.lib; SequenceTrainingLib.lib; %(AdditionalDependencies) - $(SolutionDir)$(Platform)\$(Configuration)\; "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" + "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" true Math.dll; nvml.dll; cudart64_70.dll + + + + CPUONLY;%(PreprocessorDefinitions) + + + + Math.dll + + + + if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) Copying NVidia GDK extension DLL to target folder diff --git a/Source/Extensibility/CSEvalClient/App.config b/Source/Extensibility/CSEvalClient/App.config new file mode 100644 index 000000000..8e1564635 --- /dev/null +++ b/Source/Extensibility/CSEvalClient/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Source/Extensibility/CSEvalClient/CSEvalClient.csproj b/Source/Extensibility/CSEvalClient/CSEvalClient.csproj new file mode 100644 index 000000000..927ae471f --- /dev/null +++ b/Source/Extensibility/CSEvalClient/CSEvalClient.csproj @@ -0,0 +1,80 @@ + + + + + Debug + AnyCPU + {41E11A59-62B2-4927-A4F8-F40B1B612C6C} + Exe + Properties + Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient + CSEvalClient + v4.5 + 512 + + + true + ..\..\..\x64\Debug\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + true + ..\..\..\x64\Debug_CpuOnly\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + ..\..\..\x64\Release\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + ..\..\..\x64\Release_CpuOnly\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + + + + + + + + + + + + + {ef766cae-9cb1-494c-9153-0030631a6340} + EvalWrapper + + + + + \ No newline at end of file diff --git a/Source/Extensibility/CSEvalClient/Program.cs b/Source/Extensibility/CSEvalClient/Program.cs new file mode 100644 index 000000000..f8605707b --- /dev/null +++ b/Source/Extensibility/CSEvalClient/Program.cs @@ -0,0 +1,133 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// +// Program.cs -- main C# file that contains client code to call the CLI Wrapper class. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient +{ + /// + /// Program for running model evaluations using the CLIWrapper + /// + /// + /// This program is a managed client using the CLIWrapper to run the model evaluator in CTNK. + /// It uses one of the examples provided in CNTK for evaluating the model associated with the example. + /// In order to run this program the model must already exist in the example. To create the model, + /// first run the example in /Examples/Image/MNIST. Once the model file 01_OneHidden is created, + /// you can run this client. + /// This client shows two methods for obtaining the output results from the evaluation, the first as + /// return values from the Evaluate method call (which only returns a single layer output), and the second + /// by passing the allocated output layers to the evaluate method. + /// + class Program + { + /// + /// Program entry point + /// + /// Program arguments (ignored) + private static void Main(string[] args) + { + try + { + // The examples assume the executable is running from the data folder + // We switch the current directory to the data folder (assuming the executable is in the /x64/Debug|Release folder + Environment.CurrentDirectory = Path.Combine(Environment.CurrentDirectory, @"..\..\Examples\Image\MNIST\Data\"); + + Dictionary> outputs; + + using (var model = new IEvaluateModelManagedF()) + { + // Initialize model evaluator + string config = GetConfig(); + model.Init(config); + + // Load model + string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\01_OneHidden"); + model.LoadModel(modelFilePath); + + // Generate random input values in the appropriate structure and size + var inputs = GetDictionary("features", 28*28, 255); + + // We can call the evaluate method and get back the results (single layer)... + // List outputList = model.Evaluate(inputs, "ol.z", 10); + + // ... or we can preallocate the structure and pass it in (multiple output layers) + outputs = GetDictionary("ol.z", 10, 1); + model.Evaluate(inputs, outputs); + } + + Console.WriteLine("--- Output results ---"); + foreach (var item in outputs) + { + Console.WriteLine("Output layer: {0}", item.Key); + foreach (var entry in item.Value) + { + Console.WriteLine(entry); + } + } + } + catch (Exception ex) + { + Console.WriteLine("Error: {0} \n {1}", ex, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception"); + } + + Console.WriteLine("Press to terminate."); + Console.ReadLine(); + } + + /// + /// Creates a Dictionary for input entries or output allocation + /// + /// The key for the mapping + /// The number of element entries associated to the key + /// The maximum value for random generation values + /// A dictionary with a single entry for the key/values + static Dictionary> GetDictionary(string key, int size, int maxValue) + { + var dict = new Dictionary>(); + if (key != string.Empty && size >= 0 && maxValue > 0) + { + dict.Add(key, GetFloatArray(size, maxValue)); + } + + return dict; + } + + /// + /// Reads the configuration file and returns the contents as a string + /// + /// The content of the configuration file + static string GetConfig() + { + string configFilePath = Path.Combine(Environment.CurrentDirectory, + @"..\Config\01_OneHidden.config"); + + var lines = System.IO.File.ReadAllLines(configFilePath); + return string.Join("\n", lines); + } + + /// + /// Creats a list of random numbers + /// + /// The size of the list + /// The maximum value for the generated values + /// A list of random numbers + static List GetFloatArray(int size, int maxValue) + { + List list = new List(); + if (size > 0 && maxValue >= 0) + { + Random rnd = new Random(); + list.AddRange(Enumerable.Range(1, size).Select(i => (float)rnd.Next(maxValue)).ToList()); + } + + return list; + } + } +} diff --git a/Source/Extensibility/CSEvalClient/Properties/AssemblyInfo.cs b/Source/Extensibility/CSEvalClient/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..518eae801 --- /dev/null +++ b/Source/Extensibility/CSEvalClient/Properties/AssemblyInfo.cs @@ -0,0 +1,41 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// +// AssemblyInfo.cs -- Assembly information +// +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("CSEvalClient")] +[assembly: AssemblyDescription("Managed client using managed wrapper for CNTK evaluation model")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Microsoft")] +[assembly: AssemblyProduct("CSEvalClient")] +[assembly: AssemblyCopyright("Copyright © 2016 Microsoft. All rights reserved.")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("6ec08331-7554-4ebd-b663-b64ab6e719e2")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj b/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj new file mode 100644 index 000000000..382ac57cc --- /dev/null +++ b/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj @@ -0,0 +1,98 @@ + + + + + Debug + x64 + + + Debug_CpuOnly + x64 + + + Release + x64 + + + Release_CpuOnly + x64 + + + + {EF766CAE-9CB1-494C-9153-0030631A6340} + v4.5 + ManagedCProj + EvalWrapper + + + + DynamicLibrary + true + v120 + true + Unicode + + + DynamicLibrary + false + v120 + true + Unicode + + + + + + + + + + + + + true + $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)Source\Common\Include + .dll + $(SolutionDir)$(Platform)\$(Configuration)\;$(LibraryPath) + + + false + $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)Source\Common\Include + .dll + $(SolutionDir)$(Platform)\$(Configuration)\;$(LibraryPath) + + + + Level3 + Disabled + WIN32;_DEBUG;%(PreprocessorDefinitions) + + + true + %(AdditionalDependencies) + + + + + + + Level3 + WIN32;NDEBUG;%(PreprocessorDefinitions) + + + true + %(AdditionalDependencies) + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj.filters b/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj.filters new file mode 100644 index 000000000..9e983c31c --- /dev/null +++ b/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj.filters @@ -0,0 +1,25 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {f89d33a1-2be0-4dee-b681-3630c6595c18} + + + {46d2ad19-e3a7-4619-afa2-cc3b6cd5f751} + + + + + Source Files + + + + + Common\Include + + + \ No newline at end of file diff --git a/Source/Extensibility/EvalWrapper/wrapper.cpp b/Source/Extensibility/EvalWrapper/wrapper.cpp new file mode 100644 index 000000000..336c7ba0a --- /dev/null +++ b/Source/Extensibility/EvalWrapper/wrapper.cpp @@ -0,0 +1,248 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// +// Wrapper.cpp -- Managed code wrapping the native EvaluateModel interface +// + +#include +#include +#include +#include +#include + +#include "Eval.h" + +#using +#using + +using namespace System; +using namespace System::Collections::Generic; +using namespace System::Collections; +using namespace Microsoft::MSR::CNTK; + +namespace Microsoft { +namespace MSR { +namespace CNTK { +namespace Extensibility { +namespace Managed { + +// Used for retrieving the model appropriate for the element type (float / double) +template +using GetEvalProc = void(*)(IEvaluateModel**); + +/// Managed wrapper for the native evaluation model +template +public ref class IEvaluateModelManaged : IDisposable +{ + typedef std::pair*> MapEntry; + +public: + /// Initializes a new instance of the class. + /// Factory function name for retrieving the native model from the dll. + IEvaluateModelManaged(String^ funcName) + { + pin_ptr dllname = PtrToStringChars("evaldll.dll"); + auto hModule = LoadLibrary(dllname); + + msclr::interop::marshal_context context; + const std::string func = context.marshal_as(funcName); + auto procAddress = GetProcAddress(hModule, func.c_str()); + auto getEvalProc = (GetEvalProc)procAddress; + pin_ptr *> p_eval = &m_eval; + getEvalProc(p_eval); + } + + /// Initializes the model evaluation library with a CNTK configuration + /// Model configuration entries + void Init(String^ config) + { + if (m_eval == nullptr) + { + throw gcnew ObjectDisposedException("Object has been disposed."); + } + + msclr::interop::marshal_context context; + const std::string stdConfig = context.marshal_as(config); + + m_eval->Init(stdConfig); + } + + /// Loads a model file + /// The model file name to load + void LoadModel(String^ modelFileName) + { + if (m_eval == nullptr) + { + throw gcnew ObjectDisposedException("Object has been disposed."); + } + + pin_ptr stdModelPath = PtrToStringChars(modelFileName); + m_eval->LoadModel(stdModelPath); + } + + /// Evaluates the model against input data and retrieves the output layer data + /// + /// + void Evaluate(Dictionary^>^ inputs, Dictionary^>^ outputs) + { + if (m_eval == nullptr) + { + throw gcnew ObjectDisposedException("Object has been disposed."); + } + + std::map*> stdInputs; + std::map*> stdOutputs; + + try + { + std::vector>> sharedInputVectors; + std::vector>> sharedOutputVectors; + + for each (auto item in inputs) + { + pin_ptr key = PtrToStringChars(item.Key); + shared_ptr> ptr = CopyList(item.Value); + sharedInputVectors.push_back(ptr); + stdInputs.insert(MapEntry(key, ptr.get())); + } + + for each (auto item in outputs) + { + pin_ptr key = PtrToStringChars(item.Key); + shared_ptr> ptr = CopyList(item.Value); + sharedOutputVectors.push_back(ptr); + stdOutputs.insert(MapEntry(key, ptr.get())); + } + + m_eval->Evaluate(stdInputs, stdOutputs); + + auto enumerator = outputs->Keys->GetEnumerator(); + for (std::map*>::iterator ii = stdOutputs.begin(), e = stdOutputs.end(); ii != e; ii++) + { + // Retrieve the layer key + enumerator.MoveNext(); + String^ key = enumerator.Current; + + std::vector &refVector = *((*ii).second); + int index = 0; + + // Copy output to CLI structure + for (std::vector::iterator ii = refVector.begin(), e = refVector.end(); ii != e; ii++) + { + outputs[key][index++] = *ii; + } + } + } + catch (Exception^) + { + throw; + } + } + + /// Evaluates the model against input data and retrieves the output layer data + /// + /// + /// + /// Results for specified layer + List^ Evaluate(Dictionary^>^ inputs, String^ outputKey, int outputSize) + { + List^ outputs = gcnew List(outputSize); + for (int i = 0; i < outputSize; i++) + { + outputs->Add(*(gcnew ElemType)); + } + + Dictionary^>^ outputMap = gcnew Dictionary^>(); + outputMap->Add(outputKey, outputs); + + Evaluate(inputs, outputMap); + + return outputMap[outputKey]; + } + + ~IEvaluateModelManaged() + { + if (m_eval == nullptr) + { + return; + } + + this->!IEvaluateModelManaged(); + } + +protected: + !IEvaluateModelManaged() + { + if (m_eval != nullptr) + { + m_eval->Destroy(); + m_eval = nullptr; + } + } + +private: + // Native model evaluation instance + IEvaluateModel *m_eval; + + /// Copies a list of element types from a CLI structure to a native structure + /// The CLI list of items + /// A native vector of items + shared_ptr> CopyList(List^ list) + { + shared_ptr> lower(new std::vector()); + if (list != nullptr) + { + for each (ElemType item in list) + { + lower->push_back(item); + } + } + return lower; + } +}; + +/// Managed float-specific model evaluation class +/// This class is necessary due to how generics and templates work in CLR +public ref class IEvaluateModelManagedF : IEvaluateModelManaged +{ +public: + IEvaluateModelManagedF::IEvaluateModelManagedF() + : IEvaluateModelManaged("GetEvalF") + { + } +}; + +/// Managed double-specific model evaluation class +/// This class is necessary due to how generics and templates work in CLR +public ref class IEvaluateModelManagedD : IEvaluateModelManaged +{ +public: + IEvaluateModelManagedD::IEvaluateModelManagedD() + : IEvaluateModelManaged("GetEvalD") + { + } +}; + +// This method tricks the compiler into emitting the methods of the classes +// Refer to https://msdn.microsoft.com/en-us/library/ms177213.aspx for an +// explanation to this behavior +void emit() +{ + IEvaluateModelManagedF f; + f.Init(""); + f.Evaluate(nullptr, nullptr); + f.Evaluate(nullptr, "", 0); + f.LoadModel(""); + + IEvaluateModelManagedD d; + d.Init(""); + d.Evaluate(nullptr, nullptr); + d.Evaluate(nullptr, "", 0); + d.LoadModel(""); +} +} +} +} +} +} \ No newline at end of file diff --git a/Source/Math/CUDAPageLockedMemAllocator.cpp b/Source/Math/CUDAPageLockedMemAllocator.cpp index 0637f8900..4d9cfbee8 100644 --- a/Source/Math/CUDAPageLockedMemAllocator.cpp +++ b/Source/Math/CUDAPageLockedMemAllocator.cpp @@ -60,8 +60,17 @@ void* CUDAPageLockedMemAllocator::Malloc(size_t) return nullptr; } +void* CUDAPageLockedMemAllocator::Malloc(size_t, int) +{ + return nullptr; +} + void CUDAPageLockedMemAllocator::Free(void*) { } + +void CUDAPageLockedMemAllocator::Free(void*, int) +{ +} #endif } } } diff --git a/Source/Math/ConvolutionEngine.cpp b/Source/Math/ConvolutionEngine.cpp index 75fee1ee2..7c3e1cab8 100644 --- a/Source/Math/ConvolutionEngine.cpp +++ b/Source/Math/ConvolutionEngine.cpp @@ -487,7 +487,7 @@ std::unique_ptr> ConvolutionEngineFactory>(); } - RuntimeError("Not supported convolution engine type: %d.", engType); + RuntimeError("Not supported convolution engine type: %d.", (int)engType); } template class ConvolutionEngineFactory; diff --git a/Source/Math/GPUDataTransferer.h b/Source/Math/GPUDataTransferer.h index 96be50357..1c57f9464 100644 --- a/Source/Math/GPUDataTransferer.h +++ b/Source/Math/GPUDataTransferer.h @@ -7,10 +7,22 @@ #include #include +#ifdef _WIN32 +#ifndef MATH_API +#ifdef MATH_EXPORTS +#define MATH_API __declspec(dllexport) +#else +#define MATH_API __declspec(dllimport) +#endif +#endif /* MATH_API */ +#else // no DLLs in Linux +#define MATH_API +#endif + namespace Microsoft { namespace MSR { namespace CNTK { template -class GPUDataTransferer +class MATH_API GPUDataTransferer { public: GPUDataTransferer(int deviceId, bool useConcurrentStreams); diff --git a/Source/Math/Math.vcxproj b/Source/Math/Math.vcxproj index 1ad021d30..0345cdbff 100644 --- a/Source/Math/Math.vcxproj +++ b/Source/Math/Math.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {60BDB847-D0C4-4FD3-A947-0C15C08BCDB5} @@ -24,15 +32,27 @@ Math + + + + false + + + + + true + + + - + DynamicLibrary true v120 Unicode No - + DynamicLibrary false v120 @@ -43,28 +63,25 @@ - + - + - + false - ..\Common\include;$(ACML_PATH)\include;$(CUDA_PATH)\include;$(IncludePath) - $(SolutionDir)$(Platform)\$(Configuration);$(ACML_PATH)\lib;$(CUDA_PATH)\lib\$(Platform);$(LibraryPath) + ..\Common\include;$(ACML_PATH)\include;$(IncludePath) + $(SolutionDir)$(Platform)\$(Configuration);$(ACML_PATH)\lib;$(LibraryPath) $(Platform)\$(Configuration)\$(ProjectName)\ Math - - false - ..\Common\include;$(ACML_PATH)\include;$(CUDA_PATH)\include;$(IncludePath) - $(SolutionDir)$(Platform)\$(Configuration);$(ACML_PATH)\lib;$(CUDA_PATH)\lib\$(Platform);$(LibraryPath) - $(Platform)\$(Configuration)\$(ProjectName)\ - Math + + $(IncludePath);$(CUDA_PATH)\include + $(LibraryPath);$(CUDA_PATH)\lib\$(Platform) - + NotUsing Level4 @@ -83,7 +100,7 @@ true libacml_mp_dll.lib;%(AdditionalDependencies) $(SolutionDir)$(Platform)\$(Configuration)\ - cublas64_70.dll; cusparse64_70.dll; curand64_70.dll; cudart64_70.dll; libacml_mp_dll.dll; %(DelayLoadDLLs) + libacml_mp_dll.dll; cublas64_70.dll; cusparse64_70.dll; curand64_70.dll; cudart64_70.dll; %(DelayLoadDLLs) true @@ -107,7 +124,15 @@ - + + + CPUONLY;%(PreprocessorDefinitions) + + + libacml_mp_dll.dll + + + Level4 Use @@ -134,7 +159,7 @@ $(SolutionDir)$(Platform)\$(Configuration)\ libacml_mp_dll.lib;%(AdditionalDependencies) true - cublas64_70.dll; cusparse64_70.dll; curand64_70.dll; cudart64_70.dll; libacml_dll.dll; libacml_mp_dll.dll; %(DelayLoadDLLs) + libacml_dll.dll; libacml_mp_dll.dll; cublas64_70.dll; cusparse64_70.dll; curand64_70.dll; cudart64_70.dll; %(DelayLoadDLLs) xcopy /D /I /Y "$(ACML_PATH)\lib\*.dll" $(OutputPath) @@ -155,6 +180,14 @@ true + + + CPUONLY; %(PreprocessorDefinitions) + + + libacml_dll.dll; libacml_mp_dll.dll + + diff --git a/Source/Math/NoGPU.cpp b/Source/Math/NoGPU.cpp index 0ce467f9b..ce2d8c52b 100644 --- a/Source/Math/NoGPU.cpp +++ b/Source/Math/NoGPU.cpp @@ -2145,33 +2145,33 @@ typename CuDnnConvolutionEngineFactory::Tensor4DPtr CuDnnConvolutionEn template typename CuDnnConvolutionEngineFactory::FilterPtr CuDnnConvolutionEngineFactory::CreateFilter(size_t, size_t, size_t, size_t) { - RuntimeError("The code is compiled without CPUONLY macro."); + RuntimeError("The code is compiled with CPUONLY macro."); } template typename CuDnnConvolutionEngineFactory::ConvDescPtr CuDnnConvolutionEngineFactory::CreateConvDescriptor( const Tensor4D&, const Filter&, size_t, size_t, bool) { - RuntimeError("The code is compiled without CPUONLY macro."); + RuntimeError("The code is compiled with CPUONLY macro."); } template typename CuDnnConvolutionEngineFactory::PoolDescPtr CuDnnConvolutionEngineFactory::CreatePoolDescriptor( typename PoolDesc::PoolKind, size_t, size_t, size_t, size_t, size_t, size_t) { - RuntimeError("The code is compiled without CPUONLY macro."); + RuntimeError("The code is compiled with CPUONLY macro."); } template typename CuDnnConvolutionEngineFactory::ConvEnginePtr CuDnnConvolutionEngineFactory::CreateConvEngine(DEVICEID_TYPE, size_t) { - RuntimeError("The code is compiled without CPUONLY macro."); + RuntimeError("The code is compiled with CPUONLY macro."); } template typename CuDnnConvolutionEngineFactory::PoolEnginePtr CuDnnConvolutionEngineFactory::CreatePoolEngine(DEVICEID_TYPE) { - RuntimeError("The code is compiled without CPUONLY macro."); + RuntimeError("The code is compiled with CPUONLY macro."); } template diff --git a/Source/Readers/BinaryReader/BinaryReader.vcxproj b/Source/Readers/BinaryReader/BinaryReader.vcxproj index 06c234763..35addb431 100644 --- a/Source/Readers/BinaryReader/BinaryReader.vcxproj +++ b/Source/Readers/BinaryReader/BinaryReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {1D5787D4-52E4-45DB-951B-82F220EE0C6A} @@ -24,13 +32,13 @@ UCIReader - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -40,26 +48,26 @@ - + - + - + true ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + Use Level4 @@ -76,7 +84,7 @@ $(SolutionDir)$(Platform)\$(Configuration)\;..\..\Math\$(Platform)\$(Configuration);..\$(Platform)\$(Configuration) - + Level4 Use @@ -114,36 +122,28 @@ - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - false - - - false - + false + - Create - Create + Create diff --git a/Source/Readers/DSSMReader/DSSMReader.vcxproj b/Source/Readers/DSSMReader/DSSMReader.vcxproj index 1427cdf81..ea053ee4b 100644 --- a/Source/Readers/DSSMReader/DSSMReader.vcxproj +++ b/Source/Readers/DSSMReader/DSSMReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {014DA766-B37B-4581-BC26-963EA5507931} @@ -24,13 +32,13 @@ DSSMReader - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -40,32 +48,31 @@ - + - + - - + true ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false c:\Program Files\Microsoft MPI\Inc;..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); c:\Program Files\Microsoft MPI\Lib\amd64;$(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + NotUsing Level4 Disabled - _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_WINDOWS;_USRDLL;DSSMREADER_EXPORTS;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math true @@ -78,14 +85,14 @@ $(SolutionDir)$(Platform)\$(Configuration)\;..\..\Math\$(Platform)\$(Configuration);..\$(Platform)\$(Configuration) - + Level4 NotUsing MaxSpeed true true - _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_WINDOWS;_USRDLL;DSSMREADER_EXPORTS;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math false @@ -105,16 +112,16 @@ - false + false - false + false - false + false - false + false @@ -124,19 +131,16 @@ - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing diff --git a/Source/Readers/HTKMLFReader/HTKMLFReader.vcxproj b/Source/Readers/HTKMLFReader/HTKMLFReader.vcxproj index d92d4d26a..003e0e43d 100644 --- a/Source/Readers/HTKMLFReader/HTKMLFReader.vcxproj +++ b/Source/Readers/HTKMLFReader/HTKMLFReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {33D2FD22-DEF2-4507-A58A-368F641AEBE5} @@ -24,13 +32,13 @@ - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -40,31 +48,31 @@ - + - + - + true ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + Use Level4 Disabled - WIN32;_DEBUG;_WINDOWS;_USRDLL;HTKMLFREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true true @@ -74,14 +82,14 @@ Math.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) - + Level4 Use MaxSpeed true true - WIN32;NDEBUG;_WINDOWS;_USRDLL;HTKMLFREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true /d2Zi+ %(AdditionalOptions) true @@ -125,28 +133,28 @@ - false - + false + - false - + false + NotUsing - + - + - Create - Create + Create + Create diff --git a/Source/Readers/ImageReader/ImageReader.vcxproj b/Source/Readers/ImageReader/ImageReader.vcxproj index 7c6806051..4f2667084 100644 --- a/Source/Readers/ImageReader/ImageReader.vcxproj +++ b/Source/Readers/ImageReader/ImageReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {9BD0A746-0BBD-45B6-B81C-053F03C26CFB} @@ -41,10 +49,10 @@ - + true - + false true @@ -59,17 +67,17 @@ $(Platform)\$(Configuration)\$(ProjectName)\ opencv_world300.lib - + true - + false Use Level4 - WIN32;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true true true @@ -83,13 +91,13 @@ xcopy /Y $(OPENCV_PATH)\x64\vc12\bin\opencv_world300.dll $(TargetDir) - + Disabled _DEBUG;%(PreprocessorDefinitions) - + MaxSpeed true @@ -123,8 +131,8 @@ NotUsing - NotUsing - NotUsing + NotUsing + NotUsing diff --git a/Source/Readers/LMSequenceReader/LMSequenceReader.vcxproj b/Source/Readers/LMSequenceReader/LMSequenceReader.vcxproj index 82f5df0eb..a515d14cf 100644 --- a/Source/Readers/LMSequenceReader/LMSequenceReader.vcxproj +++ b/Source/Readers/LMSequenceReader/LMSequenceReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {9A2F2441-5972-4EA8-9215-4119FCE0FB68} @@ -25,13 +33,13 @@ LMSequenceReader - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -41,31 +49,31 @@ - + - + - + true ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + Use Level4 Disabled - WIN32;_DEBUG;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math true @@ -77,14 +85,14 @@ $(SolutionDir)$(Platform)\$(Configuration)\;..\..\Math\$(Platform)\$(Configuration);..\$(Platform)\$(Configuration) - + Level4 Use MaxSpeed true true - WIN32;NDEBUG;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math false @@ -118,34 +126,29 @@ - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - false - + false + - false - + false + - Create - Create + Create diff --git a/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj b/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj index 29c2e4bac..df6187c07 100644 --- a/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj +++ b/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {62836DC1-DF77-4B98-BF2D-45C943B7DDC6} @@ -25,13 +33,13 @@ LUSequenceReader - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -41,31 +49,31 @@ - + - + - + true ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + Use Level4 Disabled - WIN32;_DEBUG;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math true @@ -77,14 +85,14 @@ $(SolutionDir)$(Platform)\$(Configuration)\;..\..\Math\$(Platform)\$(Configuration);..\$(Platform)\$(Configuration) - + Level4 Use MaxSpeed true true - WIN32;NDEBUG;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math false @@ -116,35 +124,30 @@ - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - false - + false + - false - + false + - Create - Create + Create diff --git a/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj b/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj index c48487f6e..8c582afec 100644 --- a/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj +++ b/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {D667AF32-028A-4A5D-BE19-F46776F0F6B2} @@ -24,13 +32,13 @@ LibSVMBinaryReader - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -40,32 +48,32 @@ - + - + - + true ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false c:\Program Files\Microsoft MPI\Inc;..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); c:\Program Files\Microsoft MPI\Lib\amd64;$(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + Use Level4 Disabled - _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_WINDOWS;_USRDLL;DSSMREADER_EXPORTS;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math false @@ -78,14 +86,14 @@ $(SolutionDir)$(Platform)\$(Configuration)\;..\..\Math\$(Platform)\$(Configuration);..\$(Platform)\$(Configuration) - + Level4 Use MaxSpeed true true - _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_WINDOWS;_USRDLL;LIBSVMBINARYREADER_EXPORTS;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math false @@ -115,39 +123,39 @@ - NotUsing + NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing + NotUsing - NotUsing - NotUsing + NotUsing + NotUsing - NotUsing - NotUsing + NotUsing + NotUsing - NotUsing - NotUsing + NotUsing + NotUsing - NotUsing + NotUsing - NotUsing + NotUsing - NotUsing + NotUsing - NotUsing - Create + NotUsing + Create diff --git a/Source/Readers/SparsePCReader/SparsePCReader.vcxproj b/Source/Readers/SparsePCReader/SparsePCReader.vcxproj index 5d03f7348..2a0a3c030 100644 --- a/Source/Readers/SparsePCReader/SparsePCReader.vcxproj +++ b/Source/Readers/SparsePCReader/SparsePCReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {CE429AA2-3778-4619-8FD1-49BA3B81197B} @@ -24,13 +32,13 @@ SparsePCReader - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -40,32 +48,32 @@ - + - + - + true ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false c:\Program Files\Microsoft MPI\Inc;..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); c:\Program Files\Microsoft MPI\Lib\amd64;$(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + NotUsing Level4 Disabled - _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_WINDOWS;_USRDLL;SparsePCREADER_EXPORTS;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math true @@ -78,14 +86,14 @@ $(SolutionDir)$(Platform)\$(Configuration)\;..\..\Math\$(Platform)\$(Configuration);..\$(Platform)\$(Configuration) - + Level4 Use MaxSpeed true true - _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_WINDOWS;_USRDLL;SparsePCREADER_EXPORTS;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math false @@ -105,16 +113,16 @@ - false + false - false + false - false + false - false + false @@ -124,30 +132,27 @@ - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing + NotUsing - Use + Use - Create + Create diff --git a/Source/Readers/UCIFastReader/UCIFastReader.vcxproj b/Source/Readers/UCIFastReader/UCIFastReader.vcxproj index aa899186b..b0d826a71 100644 --- a/Source/Readers/UCIFastReader/UCIFastReader.vcxproj +++ b/Source/Readers/UCIFastReader/UCIFastReader.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {E6646FFE-3588-4276-8A15-8D65C22711C1} @@ -24,13 +32,13 @@ UCIReader - + DynamicLibrary true v120 Unicode - + DynamicLibrary false v120 @@ -40,31 +48,31 @@ - + - + - + true ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false ..\..\common\include;..\..\Math;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + Use Level4 Disabled - WIN32;_DEBUG;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;_DEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math true @@ -76,14 +84,14 @@ $(SolutionDir)$(Platform)\$(Configuration)\;..\..\Math\$(Platform)\$(Configuration);..\$(Platform)\$(Configuration) - + Level4 Use MaxSpeed true true - WIN32;NDEBUG;_WINDOWS;_USRDLL;UCIREADER_EXPORTS;%(PreprocessorDefinitions) + WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) true ..\..\common\include;..\..\Math false @@ -116,33 +124,28 @@ - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - NotUsing - NotUsing + NotUsing - false - + false + - false - + false + - Create - Create + Create diff --git a/Source/SGDLib/SGDLib.vcxproj b/Source/SGDLib/SGDLib.vcxproj index d951d7240..9d36cb17c 100644 --- a/Source/SGDLib/SGDLib.vcxproj +++ b/Source/SGDLib/SGDLib.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {DE3C54E5-D7D0-47AF-A783-DFDCE59E7937} @@ -24,14 +32,26 @@ CNTK SGDLib + + + + false + + + + + true + + + - + StaticLibrary true v120 Unicode - + StaticLibrary false v120 @@ -40,35 +60,33 @@ - + - + - - true - ..\SequenceTrainingLib;..\ComputationNetworkLib;..\Math;..\Common\Include;..\CNTK\BrainScript;..\..\DataReader\HTKMLFReader;$(MSMPI_INC);$(CUDA_PATH)\include;$(VCInstallDir)include;$(WindowsSDK_IncludePath) - ..\1BitSGD;$(IncludePath) - ..\Multiverso\include;$(IncludePath) - C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform) + + ..\SequenceTrainingLib;..\ComputationNetworkLib;..\Math;..\Common\Include;..\CNTK\BrainScript;..\..\DataReader\HTKMLFReader;$(MSMPI_INC);$(VCInstallDir)include;$(WindowsSDK_IncludePath) + ..\1BitSGD;$(IncludePath) + $(MSMPI_LIB64);$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64) Build $(Platform)\$(Configuration)\$(ProjectName)\ false - + false - ..\SequenceTrainingLib;..\ComputationNetworkLib;..\Math;..\Common\Include;..\CNTK\BrainScript;..\..\DataReader\HTKMLFReader;$(MSMPI_INC);$(CUDA_PATH)\include;$(VCInstallDir)include;$(WindowsSDK_IncludePath) - ..\1BitSGD;$(IncludePath) - ..\Multiverso\include;$(IncludePath) - ..\Multiverso\x64\$(Configuration);$(MSMPI_LIB64);C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64;$(SolutionDir)$(Platform)\$(Configuration);$(SolutionDir)..\Common\lib;$(VCInstallDir)lib\amd64;$(WindowsSDK_LibraryPath_x64);$(CUDA_PATH)\lib\$(Platform) - Build $(ExecutablePath) - $(Platform)\$(Configuration)\$(ProjectName)\ - false - + + true + + + $(IncludePath);$(CUDA_PATH)\include + $(LibraryPaht);$(CUDA_PATH)\lib\$(Platform) + + @@ -90,10 +108,6 @@ Math.dll; nvml.dll; cudart64_70.dll 100000000 - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - @@ -108,7 +122,17 @@ - + + + CPUONLY;%(PreprocessorDefinitions) + + + + Math.dll + + + + Level4 @@ -134,10 +158,6 @@ Math.dll; nvml.dll; cudart64_70.dll "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" - - if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) - Copying NVidia GDK extension DLL to target folder - @@ -156,6 +176,22 @@ + + + CPUONLY;%(PreprocessorDefinitions) + + + + Math.dll + + + + + + if exist "%ProgramW6432%\NVIDIA Corporation\NVSMI" xcopy /I /D /Y "%ProgramW6432%\NVIDIA Corporation\NVSMI\nvml*.dll" $(TargetDir) + Copying NVidia GDK extension DLL to target folder + + @@ -167,7 +203,6 @@ - @@ -197,16 +232,15 @@ - NotUsing - NotUsing + NotUsing - NotUsing + NotUsing - NotUsing + NotUsing diff --git a/Source/SequenceTrainingLib/SequenceTrainingLib.vcxproj b/Source/SequenceTrainingLib/SequenceTrainingLib.vcxproj index 2bf2dca57..626e7072b 100644 --- a/Source/SequenceTrainingLib/SequenceTrainingLib.vcxproj +++ b/Source/SequenceTrainingLib/SequenceTrainingLib.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {EAD17188-072C-4726-B840-A769C36DAD1B} @@ -21,10 +29,10 @@ v120 Unicode - + true - + false true @@ -48,13 +56,18 @@ true - + + + CPUONLY;%(PreprocessorDefinitions) + + + Disabled _DEBUG;%(PreprocessorDefinitions) - + MaxSpeed true diff --git a/Source/SequenceTrainingLib/latticeNoGPU.cpp b/Source/SequenceTrainingLib/latticeNoGPU.cpp index 816e1abf1..2b6b6c346 100644 --- a/Source/SequenceTrainingLib/latticeNoGPU.cpp +++ b/Source/SequenceTrainingLib/latticeNoGPU.cpp @@ -1,4 +1,7 @@ #ifdef CPUONLY +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS // "secure" CRT not available on all platforms --add this at the top of all CPP files that give "function or variable may be unsafe" warnings +#endif #include "BestGpu.h" diff --git a/Tests/UnitTests/EvalTest/EvalTest.vcxproj b/Tests/UnitTests/EvalTest/EvalTest.vcxproj index 712f8130b..934ec8bdb 100644 --- a/Tests/UnitTests/EvalTest/EvalTest.vcxproj +++ b/Tests/UnitTests/EvalTest/EvalTest.vcxproj @@ -9,20 +9,40 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {731312A8-6DA3-4841-AFCD-57520BA1BF8E} Win32Proj EvalTest + + + + false + + + + + true + + + - + Application true v120 Unicode - + Application false v120 @@ -32,26 +52,26 @@ - + - + - + true ..\..\..\Source\Math;..\..\..\Source\Common\include;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + false ..\..\..\Source\Math;..\..\..\Source\Common\include;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(SolutionDir)$(Platform)\$(Configuration);$(VCInstallDir)lib\amd64;$(VCInstallDir)atlmfc\lib\amd64;$(WindowsSDK_LibraryPath_x64); $(Platform)\$(Configuration)\$(ProjectName)\ - + Use Level4 @@ -66,7 +86,7 @@ Math.lib; kernel32.lib; user32.lib; shell32.lib; %(AdditionalDependencies) - + Level4 Use @@ -85,33 +105,38 @@ Math.lib; kernel32.lib; user32.lib; shell32.lib; %(AdditionalDependencies) + + + CPUONLY;%(PreprocessorDefinitions) + + - NotUsing - NotUsing + NotUsing + NotUsing - NotUsing - NotUsing + NotUsing + NotUsing - NotUsing - NotUsing + NotUsing + NotUsing - NotUsing - NotUsing + NotUsing + NotUsing - Create - Create + Create + Create diff --git a/Tests/UnitTests/MathPerformanceTests/MathPerformanceTests.vcxproj b/Tests/UnitTests/MathPerformanceTests/MathPerformanceTests.vcxproj index f6756949c..a16575e04 100644 --- a/Tests/UnitTests/MathPerformanceTests/MathPerformanceTests.vcxproj +++ b/Tests/UnitTests/MathPerformanceTests/MathPerformanceTests.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {668BEED5-AC07-4F35-B3AE-EE65A7F9C976} @@ -23,14 +31,26 @@ Win32Proj MathPerformanceTests + + + + false + + + + + true + + + - + Application true v120 Unicode - + Application false v120 @@ -38,26 +58,23 @@ Unicode - - - - + - + - + true ..\..\..\Source\Common\include\;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(Platform)\$(Configuration)\$(ProjectName)\ - + false $(Platform)\$(Configuration)\$(ProjectName)\ - + Use Level4 @@ -65,7 +82,7 @@ WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - ..\..\..\Source\Math; ..\..\..\Source\Common\Include; $(CudaToolkitIncludeDir); %(AdditionalIncludeDirectories) + ..\..\..\Source\Math; ..\..\..\Source\Common\Include; %(AdditionalIncludeDirectories) Console @@ -78,7 +95,7 @@ compute_20,sm_20;compute_30,sm_30;%(CodeGeneration) - + Level4 Use @@ -88,7 +105,7 @@ WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true true - ..\..\..\Source\Math; ..\..\..\Source\Common\Include; $(CudaToolkitIncludeDir); %(AdditionalIncludeDirectories) + ..\..\..\Source\Math; ..\..\..\Source\Common\Include; %(AdditionalIncludeDirectories) true @@ -100,6 +117,22 @@ Math.lib;%(AdditionalDependencies) + + + CPUONLY;%(PreprocessorDefinitions) + + + + + $(CudaToolkitIncludeDir); %(AdditionalIncludeDirectories) + + + + + + + + @@ -108,11 +141,10 @@ Create + Create Create + Create - - - \ No newline at end of file diff --git a/Tests/UnitTests/MathTests/MathTests.vcxproj b/Tests/UnitTests/MathTests/MathTests.vcxproj index a7a19536b..fa030b106 100644 --- a/Tests/UnitTests/MathTests/MathTests.vcxproj +++ b/Tests/UnitTests/MathTests/MathTests.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {4701E678-5E6F-470D-B348-9CD1A2C095D1} @@ -16,13 +24,21 @@ MathTests - + Application true v120 Unicode false + + Application + false + v120 + true + Unicode + false + @@ -35,39 +51,43 @@ - - Application - false - v120 - true - Unicode - false - + + + + false + + + + + true + + + - + - + - + - + true $(IncludePath) $(Platform)\$(Configuration)\$(ProjectName)\ $(LibraryPath) $(SolutionDir)$(Platform)\$(Configuration)\UnitTests\ - + true $(IncludePath);$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath); $(Platform)\$(Configuration)\$(ProjectName)\ $(SolutionDir)$(Platform)\$(Configuration)\UnitTests\ - + NotUsing Level4 @@ -89,7 +109,7 @@ compute_20,sm_20;compute_30,sm_30;%(CodeGeneration) - + Level4 Use @@ -113,6 +133,11 @@ Math.lib;%(AdditionalDependencies) + + + CPUONLY;%(PreprocessorDefinitions) + + @@ -141,7 +166,7 @@ - + @@ -149,10 +174,13 @@ - $(OutDir)..\cudnn64_4.dll + $(OutDir)..\cudnn64_4.dll - + + + + diff --git a/Tests/UnitTests/ReaderTests/ReaderTests.vcxproj b/Tests/UnitTests/ReaderTests/ReaderTests.vcxproj index 65a818a0c..c7545b590 100644 --- a/Tests/UnitTests/ReaderTests/ReaderTests.vcxproj +++ b/Tests/UnitTests/ReaderTests/ReaderTests.vcxproj @@ -9,6 +9,14 @@ Release x64 + + Debug_CpuOnly + x64 + + + Release_CpuOnly + x64 + {A4FC3467-4787-43E8-BBC0-D79AE56B468D} @@ -17,7 +25,7 @@ ReaderTests - + Application true v120 @@ -36,7 +44,19 @@ - + + + + false + + + + + true + + + + Application false v120 @@ -46,28 +66,28 @@ - + - + - + false ..\..\..\Source\Common\include;..\..\..\Source\Math;$(IncludePath) $(OutDir);$(LibraryPath) $(SolutionDir)$(Platform)\$(Configuration)\UnitTests\ $(Platform)\$(Configuration)\$(ProjectName)\ - + false ..\..\..\Source\Common\include;..\..\..\Source\Math;$(IncludePath) $(OutDir);$(LibraryPath) $(SolutionDir)$(Platform)\$(Configuration)\UnitTests\ $(Platform)\$(Configuration)\$(ProjectName)\ - + NotUsing Level4 @@ -86,7 +106,7 @@ true - + Level4 NotUsing @@ -110,6 +130,11 @@ htkmlfreader.lib;Math.lib;%(AdditionalDependencies) + + + CPUONLY;%(PreprocessorDefinitions) + + @@ -125,8 +150,8 @@ - Create - Create + Create + Create