Revert some changes back (related to Gaussian and uniform initializer). Will create separate branch for that.

CNTK.sln

@@ -551,7 +551,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{83BFF5BF
 		Tools\generate_build_info = Tools\generate_build_info
 		Tools\msvc_collect_coverage.py = Tools\msvc_collect_coverage.py
 		Tools\run_boost_unit_tests.py = Tools\run_boost_unit_tests.py
-		Scripts\uci2ctf.py = Scripts\uci2ctf.py
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Extensibility", "Extensibility", "{60F87E25-BC87-4782-8E20-1621AAEBB113}"

Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Adv_Chapter.lyx

@@ -1704,7 +1704,7 @@ fromFile - No initialization is required, should only be used if the network
 \begin_layout Itemize
 uniform - Initializes the parameter matrix with uniform random numbers in
  the range of 
-\begin_inset Formula $\left[-1.0\times initValueScale,1.0\times initValueScale\right]$
+\begin_inset Formula $\left[-0.05\times initValueScale,0.05\times initValueScale\right]$
 \end_inset
 
 
@@ -1713,7 +1713,7 @@ uniform - Initializes the parameter matrix with uniform random numbers in
 \begin_layout Itemize
 gaussian - Initializes the parameter matrix with Gaussian random numbers
  with zero mean and standard deviation of 
-\begin_inset Formula $1.0\times initValueScale$
+\begin_inset Formula $0.2\times initValueScale/\sqrt{cols}$
 \end_inset
 
 

Documentation/CNTK-TechReport/lyx/CNTKBook_CNTK_Chapter.lyx

@@ -624,7 +624,7 @@ initValueScale
 .
  If the model parameters are initialized using the Gaussian distribution,
  the standard deviation will be adjusted to
-\begin_inset Formula $1.0\times initValueScale$
+\begin_inset Formula $0.2\times initValueScale/\sqrt{fanout}$
 \end_inset
 
 .

Source/ComputationNetworkLib/InputAndParamNodes.cpp

@@ -204,8 +204,8 @@ void LearnableParameter<ElemType>::PostInitParameters(const wstring& initString,
 }
 
 // understood options:
-//  uniform:       1.0
+//  uniform:       1/20
-//  gaussian:      1.0
+//  gaussian:      sqrt(0.04 / fanin)
 //  xavier:        sqrt(3 / fanin)
 //  glorotNormal:  sqrt(2 / (fanin+fanout))
 //  glorotUniform: sqrt(6 / (fanin+fanout))
@@ -214,8 +214,8 @@ void LearnableParameter<ElemType>::PostInitParameters(const wstring& initString,
 // returns (*,0) for unrecognized string
 static pair<bool/*uniform*/,double/*stddev or range*/> ParseRandomizationType(const wstring& type, size_t fanOut /* = 1*/, size_t fanIn /*= 1*/)
 {
-    if      (type == UniformInitializerTypeName)       return make_pair(true,  1.0f);
+    if      (type == UniformInitializerTypeName)       return make_pair( true, 0.05f);
-    else if (type == GaussianInitializerTypeName)      return make_pair(false, 1.0f);
+    else if (type == GaussianInitializerTypeName)      return make_pair(false, 0.2 / sqrt(fanIn));
     else if (type == XavierInitializerTypeName)        return make_pair(true,  sqrt(3.0 / fanIn));
     else if (type == GlorotUniformInitializerTypeName) return make_pair(true,  sqrt(6.0 / (fanIn + fanOut)));
     else if (type == GlorotNormalInitializerTypeName)  return make_pair(false, sqrt(2.0 / (fanIn + fanOut)));

bindings/python/cntk/initializer.py

@@ -19,7 +19,7 @@ def uniform(scale=DefaultParamInitScale, seed=None):
 
     Returns:
         initializer for :class:`cntk.variables.Parameter`
-        initialized to uniform distribution between `scale*[-1, 1]`
+        initialized to uniform distribution between `scale*[-0.05, 0.05]`
     '''
     if seed is None:
         seed = SentinelValueForAutoSelectRandomSeed
@@ -38,7 +38,7 @@ def gaussian(output_rank=SentinelValueForInferParamInitRank, filter_rank=Sentine
 
     Returns:
         initializer for :class:`cntk.variables.Parameter`
-        initialized to Gaussian distribution with mean `0` and standard deviation `scale`. 
+        initialized to Gaussian distribution with mean `0` and standard deviation `scale*0.2/sqrt(fanIn))`. 
     '''
     if seed is None:
         seed = SentinelValueForAutoSelectRandomSeed