Code revision based on review comments. (move scale to first parameter, remove unused initialized in C++ API)
This commit is contained in:
Cha Zhang
Родитель
ee4f3d1182
Коммит
cfd4c335ed
8
CNTK.sln
8
CNTK.sln
|
|
@ -1837,8 +1837,8 @@ Global
|
|||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Debug|x64.Build.0 = Debug|x64
|
||||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64
|
||||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64
|
||||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Release_NoOpt|x64.ActiveCfg = Release_NoOpt|x64
|
||||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64
|
||||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Release_NoOpt|x64.ActiveCfg = release_zmq|x64
|
||||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Release_NoOpt|x64.Build.0 = release_zmq|x64
|
||||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Release|x64.ActiveCfg = Release|x64
|
||||
{16F14058-B116-49D9-8BA0-209F3AFFE849}.Release|x64.Build.0 = Release|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64
|
||||
|
|
@ -1847,8 +1847,8 @@ Global
|
|||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Debug|x64.Build.0 = Debug|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Release_NoOpt|x64.ActiveCfg = Release_NoOpt|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Release_NoOpt|x64.ActiveCfg = Release_CpuOnly|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Release_NoOpt|x64.Build.0 = Release_CpuOnly|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Release|x64.ActiveCfg = Release|x64
|
||||
{EC7157E9-A51F-4702-A5FD-8DAF88C7029F}.Release|x64.Build.0 = Release|x64
|
||||
{5D29C76D-648A-456F-920D-48230F2FB3C8}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64
|
||||
|
|
|
|||
|
|
@ -1998,15 +1998,13 @@ private:
|
|||
static const int DefaultParamInitFilterRank = 0;
|
||||
|
||||
CNTK_API ParameterInitializer ConstantInitializer(double value = 0.0);
|
||||
CNTK_API ParameterInitializer UniformInitializer(double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer Uniform1Initializer(double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer GaussianInitializer(int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer NormalInitializer(int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer XavierInitializer(int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer GlorotUniformInitializer(int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer GlorotNormalInitializer(int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer HeUniformInitializer(int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer HeNormalInitializer(int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, double scale = DefaultParamInitScale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer UniformInitializer(double scale, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer NormalInitializer(double scale, int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer XavierInitializer(double scale = DefaultParamInitScale, int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer GlorotUniformInitializer(double scale = DefaultParamInitScale, int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer GlorotNormalInitializer(double scale = DefaultParamInitScale, int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer HeUniformInitializer(double scale = DefaultParamInitScale, int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer HeNormalInitializer(double scale = DefaultParamInitScale, int outputRank = SentinelValueForInferParamInitRank, int filterRank = SentinelValueForInferParamInitRank, unsigned long seed = SentinelValueForAutoSelectRandomSeed);
|
||||
CNTK_API ParameterInitializer BilinearInitializer(size_t kernelWidth, size_t kernelHeight);
|
||||
CNTK_API ParameterInitializer RandomInitializerWithRank(const ParameterInitializer& initializer, int outputRank, int filterRank);
|
||||
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ namespace CNTK
|
|||
|
||||
static std::atomic<unsigned long> s_currentRandomSeed(1);
|
||||
|
||||
static ParameterInitializer CreateInitializer(const std::wstring& initializerTypeName, int outputRank, int filterRank, double scale, unsigned long seed)
|
||||
static ParameterInitializer CreateInitializer(const std::wstring& initializerTypeName, double scale, int outputRank, int filterRank, unsigned long seed)
|
||||
{
|
||||
Dictionary initConfig;
|
||||
initConfig[InitializerTypeAttributeName] = initializerTypeName;
|
||||
|
|
@ -171,49 +171,49 @@ namespace CNTK
|
|||
return initConfig;
|
||||
}
|
||||
|
||||
ParameterInitializer Uniform1Initializer(double scale, unsigned long seed)
|
||||
ParameterInitializer UniformBSInitializer(double scale, unsigned long seed)
|
||||
{
|
||||
Dictionary initConfig;
|
||||
initConfig[InitializerTypeAttributeName] = Microsoft::MSR::CNTK::Uniform1InitializerTypeName;
|
||||
initConfig[InitializerTypeAttributeName] = Microsoft::MSR::CNTK::UniformBSInitializerTypeName;
|
||||
initConfig[ScaleAttributeName] = scale;
|
||||
initConfig[RandomSeedAttributeName] = (size_t)seed;
|
||||
|
||||
return initConfig;
|
||||
}
|
||||
|
||||
ParameterInitializer GaussianInitializer(int outputRank, int filterRank, double scale, unsigned long seed)
|
||||
ParameterInitializer GaussianInitializer(double scale, int outputRank, int filterRank, unsigned long seed)
|
||||
{
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::GaussianInitializerTypeName, outputRank, filterRank, scale, seed);
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::GaussianInitializerTypeName, scale, outputRank, filterRank, seed);
|
||||
}
|
||||
|
||||
ParameterInitializer NormalInitializer(int outputRank, int filterRank, double scale, unsigned long seed)
|
||||
ParameterInitializer NormalInitializer(double scale, int outputRank, int filterRank, unsigned long seed)
|
||||
{
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::NormalInitializerTypeName, outputRank, filterRank, scale, seed);
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::NormalInitializerTypeName, scale, outputRank, filterRank, seed);
|
||||
}
|
||||
|
||||
ParameterInitializer XavierInitializer(int outputRank, int filterRank, double scale, unsigned long seed)
|
||||
ParameterInitializer XavierInitializer(double scale, int outputRank, int filterRank, unsigned long seed)
|
||||
{
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::XavierInitializerTypeName, outputRank, filterRank, scale, seed);
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::XavierInitializerTypeName, scale, outputRank, filterRank, seed);
|
||||
}
|
||||
|
||||
ParameterInitializer GlorotUniformInitializer(int outputRank, int filterRank, double scale, unsigned long seed)
|
||||
ParameterInitializer GlorotUniformInitializer(double scale, int outputRank, int filterRank, unsigned long seed)
|
||||
{
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::GlorotUniformInitializerTypeName, outputRank, filterRank, scale, seed);
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::GlorotUniformInitializerTypeName, scale, outputRank, filterRank, seed);
|
||||
}
|
||||
|
||||
ParameterInitializer GlorotNormalInitializer(int outputRank, int filterRank, double scale, unsigned long seed)
|
||||
ParameterInitializer GlorotNormalInitializer(double scale, int outputRank, int filterRank, unsigned long seed)
|
||||
{
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::GlorotNormalInitializerTypeName, outputRank, filterRank, scale, seed);
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::GlorotNormalInitializerTypeName, scale, outputRank, filterRank, seed);
|
||||
}
|
||||
|
||||
ParameterInitializer HeUniformInitializer(int outputRank, int filterRank, double scale, unsigned long seed)
|
||||
ParameterInitializer HeUniformInitializer(double scale, int outputRank, int filterRank, unsigned long seed)
|
||||
{
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::HeUniformInitializerTypeName, outputRank, filterRank, scale, seed);
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::HeUniformInitializerTypeName, scale, outputRank, filterRank, seed);
|
||||
}
|
||||
|
||||
ParameterInitializer HeNormalInitializer(int outputRank, int filterRank, double scale, unsigned long seed)
|
||||
ParameterInitializer HeNormalInitializer(double scale, int outputRank, int filterRank, unsigned long seed)
|
||||
{
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::HeNormalInitializerTypeName, outputRank, filterRank, scale, seed);
|
||||
return CreateInitializer(Microsoft::MSR::CNTK::HeNormalInitializerTypeName, scale, outputRank, filterRank, seed);
|
||||
}
|
||||
|
||||
ParameterInitializer BilinearInitializer(size_t kernelWidth, size_t kernelHeight)
|
||||
|
|
|
|||
|
|
@ -204,8 +204,8 @@ void LearnableParameter<ElemType>::PostInitParameters(const wstring& initString,
|
|||
}
|
||||
|
||||
// understood options:
|
||||
// uniform: 1/20
|
||||
// uniform1: 1.0
|
||||
// uniformBS: 1/20
|
||||
// uniform: 1.0
|
||||
// gaussian: sqrt(0.04 / fanin)
|
||||
// normal: 1.0
|
||||
// xavier: sqrt(3 / fanin)
|
||||
|
|
@ -216,8 +216,8 @@ void LearnableParameter<ElemType>::PostInitParameters(const wstring& initString,
|
|||
// returns (*,0) for unrecognized string
|
||||
static pair<bool/*uniform*/,double/*stddev or range*/> ParseRandomizationType(const wstring& type, size_t fanOut /* = 1*/, size_t fanIn /*= 1*/)
|
||||
{
|
||||
if (type == UniformInitializerTypeName) return make_pair(true, 0.05f);
|
||||
if (type == Uniform1InitializerTypeName) return make_pair(true, 1.0f);
|
||||
if (type == UniformBSInitializerTypeName) return make_pair(true, 0.05f);
|
||||
if (type == UniformInitializerTypeName) return make_pair(true, 1.0f);
|
||||
else if (type == GaussianInitializerTypeName) return make_pair(false, 0.2 / sqrt(fanIn));
|
||||
else if (type == NormalInitializerTypeName) return make_pair(false, 1.0f);
|
||||
else if (type == XavierInitializerTypeName) return make_pair(true, sqrt(3.0 / fanIn));
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@
|
|||
namespace Microsoft { namespace MSR { namespace CNTK {
|
||||
|
||||
static const wchar_t* ConstantInitializerTypeName = L"constant";
|
||||
static const wchar_t* UniformInitializerTypeName = L"uniform";
|
||||
static const wchar_t* Uniform1InitializerTypeName = L"uniform1";
|
||||
static const wchar_t* UniformBSInitializerTypeName = L"uniform";
|
||||
static const wchar_t* UniformInitializerTypeName = L"uniform1";
|
||||
static const wchar_t* GaussianInitializerTypeName = L"gaussian";
|
||||
static const wchar_t* NormalInitializerTypeName = L"normal";
|
||||
static const wchar_t* XavierInitializerTypeName = L"xavier";
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit a1170ce7e7fd5dbf44b7552fe1570843877fc4b8
|
||||
Subproject commit 86d29abf747edbca73ec9d85618e16279492d72a
|
||||
|
|
@ -21,7 +21,6 @@
|
|||
%rename(times_transpose) CNTK::TransposeTimes;
|
||||
%rename(sequence_slice) CNTK::Sequence::Slice;
|
||||
%rename(sequence_reduce_sum) CNTK::Sequence::ReduceSum;
|
||||
%rename(uniform1_initializer) CNTK::Uniform1Initializer;
|
||||
|
||||
%rename(momentum_as_time_constant_schedule) CNTK::MomentumAsTimeConstantSchedule;
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from .cntk_py import default_param_init_scale as DefaultParamInitScale,\
|
|||
sentinel_value_for_auto_select_random_seed as SentinelValueForAutoSelectRandomSeed
|
||||
|
||||
|
||||
def uniform(scale=DefaultParamInitScale, seed=None):
|
||||
def uniform(scale, seed=None):
|
||||
'''
|
||||
Uniform initializer
|
||||
|
||||
|
|
@ -27,14 +27,14 @@ def uniform(scale=DefaultParamInitScale, seed=None):
|
|||
|
||||
return cntk_py.uniform1_initializer(scale, seed)
|
||||
|
||||
def normal(output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, scale=DefaultParamInitScale, seed=None):
|
||||
def normal(scale, output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, seed=None):
|
||||
'''
|
||||
Normal initializer
|
||||
|
||||
Args:
|
||||
scale (float): scale
|
||||
output_rank (int): output rank
|
||||
filter_rank (int): filter rank
|
||||
scale (float): scale
|
||||
seed (int): random seed
|
||||
|
||||
Returns:
|
||||
|
|
@ -44,16 +44,16 @@ def normal(output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelV
|
|||
if seed is None:
|
||||
seed = SentinelValueForAutoSelectRandomSeed
|
||||
|
||||
return cntk_py.normal_initializer(output_rank, filter_rank, scale, seed)
|
||||
return cntk_py.normal_initializer(scale, output_rank, filter_rank, seed)
|
||||
|
||||
def xavier(output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, scale=DefaultParamInitScale, seed=None):
|
||||
def xavier(scale=DefaultParamInitScale, output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, seed=None):
|
||||
'''
|
||||
Xavier initializer
|
||||
|
||||
Args:
|
||||
scale (float): scale
|
||||
output_rank (int): output rank
|
||||
filter_rank (int): filter rank
|
||||
scale (float): scale
|
||||
seed (int): random seed
|
||||
|
||||
Returns:
|
||||
|
|
@ -64,16 +64,16 @@ def xavier(output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelV
|
|||
if seed is None:
|
||||
seed = SentinelValueForAutoSelectRandomSeed
|
||||
|
||||
return cntk_py.xavier_initializer(output_rank, filter_rank, scale, seed)
|
||||
return cntk_py.xavier_initializer(scale, output_rank, filter_rank, seed)
|
||||
|
||||
def glorot_uniform(output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, scale=DefaultParamInitScale, seed=None):
|
||||
def glorot_uniform(scale=DefaultParamInitScale, output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, seed=None):
|
||||
'''
|
||||
Glorot initializer
|
||||
|
||||
Args:
|
||||
scale (float): scale
|
||||
output_rank (int): output rank
|
||||
filter_rank (int): filter rank
|
||||
scale (float): scale
|
||||
seed (int): random seed
|
||||
|
||||
Returns:
|
||||
|
|
@ -84,16 +84,16 @@ def glorot_uniform(output_rank=SentinelValueForInferParamInitRank, filter_rank=S
|
|||
if seed is None:
|
||||
seed = SentinelValueForAutoSelectRandomSeed
|
||||
|
||||
return cntk_py.glorot_uniform_initializer(output_rank, filter_rank, scale, seed)
|
||||
return cntk_py.glorot_uniform_initializer(scale, output_rank, filter_rank, seed)
|
||||
|
||||
def glorot_normal(output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, scale=DefaultParamInitScale, seed=None):
|
||||
def glorot_normal(scale=DefaultParamInitScale, output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, seed=None):
|
||||
'''
|
||||
initializer
|
||||
|
||||
Args:
|
||||
scale (float): scale
|
||||
output_rank (int): output rank
|
||||
filter_rank (int): filter rank
|
||||
scale (float): scale
|
||||
seed (int): random seed
|
||||
|
||||
Returns:
|
||||
|
|
@ -104,16 +104,16 @@ def glorot_normal(output_rank=SentinelValueForInferParamInitRank, filter_rank=Se
|
|||
if seed is None:
|
||||
seed = SentinelValueForAutoSelectRandomSeed
|
||||
|
||||
return cntk_py.glorot_normal_initializer(output_rank, filter_rank, scale, seed)
|
||||
return cntk_py.glorot_normal_initializer(scale, output_rank, filter_rank, seed)
|
||||
|
||||
def he_uniform(output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, scale=DefaultParamInitScale, seed=None):
|
||||
def he_uniform(scale=DefaultParamInitScale, output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, seed=None):
|
||||
'''
|
||||
initializer
|
||||
|
||||
Args:
|
||||
scale (float): scale
|
||||
output_rank (int): output rank
|
||||
filter_rank (int): filter rank
|
||||
scale (float): scale
|
||||
seed (int): random seed
|
||||
|
||||
Returns:
|
||||
|
|
@ -124,16 +124,16 @@ def he_uniform(output_rank=SentinelValueForInferParamInitRank, filter_rank=Senti
|
|||
if seed is None:
|
||||
seed = SentinelValueForAutoSelectRandomSeed
|
||||
|
||||
return cntk_py.he_uniform_initializer(output_rank, filter_rank, scale, seed)
|
||||
return cntk_py.he_uniform_initializer(scale, output_rank, filter_rank, seed)
|
||||
|
||||
def he_normal(output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, scale=DefaultParamInitScale, seed=None):
|
||||
def he_normal(scale=DefaultParamInitScale, output_rank=SentinelValueForInferParamInitRank, filter_rank=SentinelValueForInferParamInitRank, seed=None):
|
||||
'''
|
||||
initializer
|
||||
|
||||
Args:
|
||||
scale (float): scale
|
||||
output_rank (int): output rank
|
||||
filter_rank (int): filter rank
|
||||
scale (float): scale
|
||||
seed (int): random seed
|
||||
|
||||
Returns:
|
||||
|
|
@ -144,7 +144,7 @@ def he_normal(output_rank=SentinelValueForInferParamInitRank, filter_rank=Sentin
|
|||
if seed is None:
|
||||
seed = SentinelValueForAutoSelectRandomSeed
|
||||
|
||||
return cntk_py.he_normal_initializer(output_rank, filter_rank, scale, seed)
|
||||
return cntk_py.he_normal_initializer(scale, output_rank, filter_rank, seed)
|
||||
|
||||
def bilinear(kernel_width, kernel_height):
|
||||
'''
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче