added a global test runner BAT file as a Build action, so far it is running, but CNTK output does not appear in the Build window;

fixed missing newline in stderr output when terminating with an exception message

CNTK.sln

@@ -156,6 +156,9 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Other", "Other", "{39E42C4B-A078-4CA4-9D92-B883D8129601}"
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CheckInSuites", "CheckInSuites\CheckInSuites.vcxproj", "{DBB3C106-B0B4-4059-8477-C89528CEC1B0}"
+	ProjectSection(ProjectDependencies) = postProject
+		{E6F26F9A-FF64-4F0A-B749-CD309EE357EE} = {E6F26F9A-FF64-4F0A-B749-CD309EE357EE}
+	EndProjectSection
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

CheckInSuites/ASR/config/README.txt

@@ -1,10 +1,8 @@
 
 These scripts are similar to those in the TIMIT folder of the ExampleSetups except they use much fewer files (100 utterances) and fewer minibatches. See the README.txt file there for more details about these configurations. 
 
-The globals_cpu.config and globals_gpu.config differ only in which device they use and where the results are stored. 
-
 To test on CPU:
-cntk configFile=globals_cpu.config+<DesiredConfigFile>
+CNTK.exe  WorkDir=...  ExpDir=...  LibDir=...  ScpDir=...  configFile=globals.config+select_cpu.config+<DesiredConfigFile>
 
 To test on GPU:
-cntk configFile=globals_gpu.config+<DesiredConfigFile>
+CNTK.exe  WorkDir=...  ExpDir=...  LibDir=...  ScpDir=...  configFile=globals.config+select_gpu.config+<DesiredConfigFile>

CheckInSuites/ASR/config/TIMIT_EvalSimpleNetwork.config

@@ -1,9 +1,9 @@
 stderr=$ExpDir$\EvalSimpleNetwork\log\log
-command=TIMIT_EvalSimple
+command=TIMIT_EvalSimpleNetwork
 
 precision=float
 
-TIMIT_EvalSimple=[
+TIMIT_EvalSimpleNetwork=[
     action=eval
 
     modelPath=$ExpDir$\TrainSimpleNetwork\model\cntkSpeech.dnn

CheckInSuites/ASR/config/TIMIT_TrainNDLNetwork.config

@@ -1,12 +1,12 @@
 stderr=$ExpDir$\TrainNDLNetwork\log\log
-command=TIMIT_TrainNDL
+command=TIMIT_TrainNDLNetwork
 
 precision=float
 
 #######################################
 #  TRAINING CONFIG (NDL, Fixed LR) #
 #######################################
-TIMIT_TrainNDL=[
+TIMIT_TrainNDLNetwork=[
     action=train
 
     modelPath=$ExpDir$\TrainNDLNetwork\model\cntkSpeech.dnn

CheckInSuites/ASR/config/TIMIT_TrainSimpleNetwork.config

@@ -1,5 +1,5 @@
 stderr=$ExpDir$\TrainSimpleNetwork\log\log
-command=TIMIT_TrainSimple
+command=TIMIT_TrainSimpleNetwork
 
 precision=float
 
@@ -7,7 +7,7 @@ precision=float
 #  TRAINING CONFIG (Simple, Fixed LR) #
 #######################################
 
-TIMIT_TrainSimple=[
+TIMIT_TrainSimpleNetwork=[
     action=train
 
     modelPath=$ExpDir$\TrainSimpleNetwork\model\cntkSpeech.dnn

CheckInSuites/ASR/config/runall.bat

@@ -1,25 +1,37 @@
+::: first argument is CPU or GPU
+set PROC=%1
+::: second argument is buildconfig (Release or Debug)
+set BUILD=%2
+echo running ASR test on %PROC%
+
 ::: the CNTK executable is found relative to this BAT file
 set  THIS=%~dp0
-set  ROOT=%THIS%\..\..\..
+set  ROOT=%THIS%..\..\..
 
-set  CNTK=%ROOT%\x64\Release\CNTK1.exe
+set  CNTK=%ROOT%\x64\%2\CNTK.exe
 
 ::: directories we pass in to CNTK config
 
 ::: example setups are here
 set WorkDir=%ROOT%\ExampleSetups\ASR\TIMIT
-set  ExpDir=%THIS%\..\test_out
+set  ExpDir=%THIS%..\test_out
 set  LibDir=%WorkDir%\lib
 set  ScpDir=%LibDir%\scp
 
-::: first argument is CPU or GPU
-set proc=%1
-
-%CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%\globals.config+%THIS%\select_%proc%.config+%THIS%\TIMIT_TrainSimpleNetwork.config
-%CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%\globals.config+%THIS%\select_%proc%.config+%THIS%\TIMIT_TrainNDLNetwork.config
-%CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%\globals.config+%THIS%\select_%proc%.config+%THIS%\TIMIT_TrainAutoEncoder.config
-%CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%\globals.config+%THIS%\select_%proc%.config+%THIS%\TIMIT_TrainMultiInput.config
-%CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%\globals.config+%THIS%\select_%proc%.config+%THIS%\TIMIT_TrainMultiTask.config
-%CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%\globals.config+%THIS%\select_%proc%.config+%THIS%\TIMIT_EvalSimpleNetwork.config
-%CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%\globals.config+%THIS%\select_%proc%.config+%THIS%\TIMIT_WriteScaledLogLike.config
-%CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%\globals.config+%THIS%\select_%proc%.config+%THIS%\TIMIT_WriteBottleneck.config
+::: run all tests
+::: TODO: fix the log path, it seems it cannot be passed to CNTK currently on the command line
+for %%t in (TrainSimpleNetwork TrainNDLNetwork TrainAutoEncoder TrainMultiInput TrainMultiTask EvalSimpleNetwork WriteScaledLogLike WriteBottleneck) do (
+  echo ------
+  echo running test TIMIT_%%t.config logging to %ExpDir%\%%t\log\log_TIMIT_%%t.log
+  %CNTK%  WorkDir=%WorkDir%  ExpDir=%ExpDir%  LibDir=%LibDir%  ScpDir=%ScpDir%  configFile=%THIS%globals.config+%THIS%select_%PROC%.config+%THIS%TIMIT_%%t.config
+  if ERRORLEVEL 1 (
+    echo CNTK FAILED:
+    findstr /I EXCEPTION %ExpDir%\%%t\log\log_TIMIT_%%t.log
+  ) else (
+    echo CNTK OUTPUT:
+    findstr /I Finished %ExpDir%\%%t\log\log_TIMIT_%%t.log
+    findstr /I EXCEPTION %ExpDir%\%%t\log\log_TIMIT_%%t.log
+    echo REFERENCE:
+    findstr /I Finished %THIS%..\%PROC%\%%t.output
+  )
+)

CheckInSuites/CheckInSuites.vcxproj

@@ -140,6 +140,7 @@
     <None Include="MNIST\GPU\expectedresults.txt" />
     <None Include="MNIST\mnistCheckIn.config" />
     <None Include="MNIST\mnistlabels.txt" />
+    <None Include="runall.bat" />
     <None Include="SLU\atis.dev.IOB.simple" />
     <None Include="SLU\atis.test.apos.pred.pos.head.IOB.simple" />
     <None Include="SLU\atis.train.apos.pred.pos.head.IOB.simple" />
@@ -180,9 +181,11 @@
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <NMakePreprocessorDefinitions>WIN32;_DEBUG;$(NMakePreprocessorDefinitions)</NMakePreprocessorDefinitions>
+    <NMakeBuildCommandLine>runall.bat $(Configuration)</NMakeBuildCommandLine>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <NMakePreprocessorDefinitions>WIN32;NDEBUG;$(NMakePreprocessorDefinitions)</NMakePreprocessorDefinitions>
+    <NMakeBuildCommandLine>runall.bat $(Configuration)</NMakeBuildCommandLine>
   </PropertyGroup>
   <ItemDefinitionGroup>
   </ItemDefinitionGroup>

CheckInSuites/CheckInSuites.vcxproj.filters

@@ -516,5 +516,8 @@
     <None Include="ASR\config\select_gpu.config">
       <Filter>CheckInSuites\ASR\config</Filter>
     </None>
+    <None Include="runall.bat">
+      <Filter>CheckInSuites</Filter>
+    </None>
   </ItemGroup>
 </Project>

CheckInSuites/runall.bat

@@ -0,0 +1,22 @@
+::: this runs all tests in this folder
+::: BUGBUG: so far only the ASR tests have updated pathnames etc., the others are non-functional stubs here that need to be updated
+::: TODO: find a good solution for specifying directories for data that we cannot distribute with CNTK ourselves.
+
+set BUILD=%1
+
+set  THIS=%~dp0
+
+::: ASR tests
+::: BUGBUG: We do not get to see stdout from CNTK, only from the BAT files.
+( %THIS%\ASR\config\runall.bat cpu %BUILD% ) 2>&1
+( %THIS%\ASR\config\runall.bat gpu %BUILD% ) 2>&1
+
+::: LM tests
+::: TODO: provide BAT file
+
+::: MNIST
+::: TODO: provide BAT file
+
+::: SLU
+::: TODO: update paths
+C:\dev\cntk3\CNTKSolution\x64\Release\cntk configFile=globals.config+rnnlu.config

MachineLearning/CNTK/CNTK.cpp

@@ -1114,10 +1114,11 @@ int wmain(int argc, wchar_t* argv[])
             fprintf(fp, "successfully finished at %s on %s\n", TimeDateStamp().c_str(), GetHostName().c_str());
             fcloseOrDie(fp);
         }
+        fprintf(stderr, "COMPLETED\n");
     }
     catch (const std::exception &err)
     {
-        fprintf(stderr, "EXCEPTION occurred: %s", err.what());
+        fprintf(stderr, "EXCEPTION occurred: %s\n", err.what());
 #ifdef _DEBUG
         DebugBreak();
 #endif