some tidying-up of ComputationNetworkAnalysis.cpp
This commit is contained in:
Frank Seide
Родитель
3f9e8a7b53
Коммит
04c9bb48f9
|
|
@ -1,80 +0,0 @@
|
|||
#
|
||||
# test this with this command line:
|
||||
# configFile=$(SolutionDir)BrainScript/test.config RunDir=$(SolutionDir)\Tests\Speech\RunDir DataDir=$(SolutionDir)\Tests\Speech\Data DeviceId=Auto
|
||||
|
||||
precision=float
|
||||
command=speechTrain
|
||||
deviceId=$DeviceId$
|
||||
|
||||
parallelTrain=false
|
||||
|
||||
speechTrain=[
|
||||
action=train
|
||||
modelPath=$RunDir$/models/cntkSpeech.dnn
|
||||
deviceId=$DeviceId$
|
||||
traceLevel=1
|
||||
# inside here is the new stuff
|
||||
ExperimentalNetworkBuilder=[
|
||||
//deviceId = -21 ; precision = 'floax' // for now
|
||||
layerSizes=363:512:512:132
|
||||
trainingCriterion=CE
|
||||
evalCriterion=Err
|
||||
//layerTypes=Sigmoid
|
||||
//initValueScale=1.0
|
||||
//applyMeanVarNorm=true
|
||||
//uniformInit=true
|
||||
//needPrior=true
|
||||
|
||||
numHiddenLayers = 3
|
||||
myFeatures = Input(layerSizes[0]) ; myLabels = Input(layerSizes[Length(layerSizes)-1])
|
||||
featNorm = MeanVarNorm(myFeatures)
|
||||
layers = array[1..numHiddenLayers] (layer => if layer > 1 then SBFF(layers[layer-1].Eh, layerSizes[layer], layerSizes[layer-1]) else SBFF(featNorm, layerSizes[layer], layerSizes[layer-1]))
|
||||
outLayer = BFF(layers[numHiddenLayers].Eh, labelDim, hiddenDim)
|
||||
outZ = outLayer.z
|
||||
CE = CrossEntropyWithSoftmax(myLabels, outZ)
|
||||
Err = ErrorPrediction(myLabels, outZ)
|
||||
logPrior = LogPrior(myLabels)
|
||||
ScaledLogLikelihood = outZ - logPrior
|
||||
]
|
||||
|
||||
SGD=[
|
||||
epochSize=20480
|
||||
minibatchSize=64:256:1024:
|
||||
learningRatesPerMB=1.0:0.5:0.1
|
||||
numMBsToShowResult=10
|
||||
momentumPerMB=0.9:0.656119
|
||||
dropoutRate=0.0
|
||||
maxEpochs=3
|
||||
keepCheckPointFiles=true
|
||||
|
||||
AutoAdjust=[
|
||||
reduceLearnRateIfImproveLessThan=0
|
||||
loadBestModel=true
|
||||
increaseLearnRateIfImproveMoreThan=1000000000
|
||||
learnRateDecreaseFactor=0.5
|
||||
learnRateIncreaseFactor=1.382
|
||||
autoAdjustLR=AdjustAfterEpoch
|
||||
]
|
||||
clippingThresholdPerSample=1#INF
|
||||
]
|
||||
reader=[
|
||||
readerType=HTKMLFReader
|
||||
readMethod=blockRandomize
|
||||
miniBatchMode=Partial
|
||||
randomize=Auto
|
||||
verbosity=0
|
||||
features=[
|
||||
dim=363
|
||||
type=Real
|
||||
scpFile=glob_0000.scp
|
||||
]
|
||||
|
||||
labels=[
|
||||
mlfFile=$DataDir$/glob_0000.mlf
|
||||
labelMappingFile=$DataDir$/state.list
|
||||
|
||||
labelDim=132
|
||||
labelType=Category
|
||||
]
|
||||
]
|
||||
]
|
||||
|
|
@ -550,6 +550,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
|
|||
|
||||
// this is called from ClearCache() only, which in turn is called by model editing operations, such as DeleteNode(), and by RebuildNetwork()
|
||||
// Basically, it invalidates all post-processing, reducing the network to the graph.
|
||||
// TODO: This will go away once we validate/build/process the network in a non-lazy fashion.
|
||||
void ComputationNetwork::ClearCalcOrderCaches()
|
||||
{
|
||||
for (auto & it : m_cacheEvalOrders)
|
||||
|
|
|
|||
|
|
@ -813,15 +813,15 @@ protected:
|
|||
// -----------------------------------------------------------------------
|
||||
|
||||
// The methods below determine evaluation order, which is tricky in presence of recurrent loops.
|
||||
// TODO: Can this be moved to a separate class, or at least a separate CPP?
|
||||
// TODO: Can this be moved to a separate class?
|
||||
private:
|
||||
|
||||
void ClearCalcOrderCaches();
|
||||
|
||||
// This is part of the FormRecurrentLoops() process, and only called from there.
|
||||
void FormRecurrentLoops(const ComputationNodeBasePtr& rootNode);
|
||||
void DetermineSCCs(const ComputationNodeBasePtr& rootNode);
|
||||
void DetermineSCCsR(ComputationNodeBasePtr cur, std::list<ComputationNodeBasePtr>& sccStack, size_t& index, size_t& loopId);
|
||||
void UniqRecurrentLoops();
|
||||
void DetermineLoopForwardOrder(std::unordered_set<ComputationNodeBasePtr>& visited, std::unordered_set<ComputationNodeBasePtr>& recStack, std::list<ComputationNodeBasePtr>& nodesStack, ComputationNodeBasePtr cur);
|
||||
void GatherLoopNodesR(const ComputationNodeBasePtr& rootNode, std::unordered_set<ComputationNodeBasePtr>& visited, std::map<int, std::list<ComputationNodeBasePtr>>& recurrentResult, std::list<ComputationNodeBasePtr>& noRecurrentResult);
|
||||
void ReorderLoops(std::list<ComputationNodeBasePtr>& nodes, const std::map<int, std::list<ComputationNodeBasePtr>>& /*recurrentNodes*/, const std::list<ComputationNodeBasePtr> & /*noRecurrentNodes*/);
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
|
|||
// -----------------------------------------------------------------------
|
||||
|
||||
// The methods below determine evaluation order, which is tricky in presence of recurrent loops.
|
||||
// TODO: Can this be moved to a separate class, or at least a separate CPP?
|
||||
// TODO: Can this be moved to a separate class?
|
||||
|
||||
// MAIN ENTRY POINT for network recurrent-loop analysis. All other functions below are called from this one.
|
||||
|
||||
|
|
@ -41,6 +41,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
|
|||
{
|
||||
// determine the strongly connected cliques -> m_recurrentInfo[]
|
||||
DetermineSCCs(rootNode);
|
||||
// now we have formed all loops, with all nodes assigned to a loop or none
|
||||
|
||||
list<ComputationNodeBasePtr>& nodes = GetEvalOrder(rootNode, true/*skipPairNetwork*/);
|
||||
// recover m_visitedOrder
|
||||
|
|
@ -48,12 +49,6 @@ namespace Microsoft { namespace MSR { namespace CNTK {
|
|||
for (auto & node : nodes)
|
||||
node->m_visitedOrder = i++;
|
||||
|
||||
// purge identical loops (i.e. loops that have the same source node)
|
||||
// TODO: Is this for the case that we call this function multiple times, or do the nodes of a loop generate multiple entries? Comment this.
|
||||
UniqRecurrentLoops();
|
||||
|
||||
// now we have formed all loops, with all nodes assigned to a loop or none
|
||||
|
||||
// update m_visitedOrder of all nodes
|
||||
// This was originally set by EnumerateNodes(), which gets called from GetEvalOrder().
|
||||
// All nodes that participate in a loop get the same m_visitedOrder value.
|
||||
|
|
@ -71,19 +66,6 @@ namespace Microsoft { namespace MSR { namespace CNTK {
|
|||
// implant m_loopId in all nodes in all loops
|
||||
for (auto & iter : m_recurrentInfo)
|
||||
{
|
||||
#if 1 // instead of the redundant sort() below, we just verify
|
||||
for (auto & node : iter->m_nestedNodes)
|
||||
if (node->m_visitedOrder != iter->m_nestedNodes.front()->m_visitedOrder)
|
||||
LogicError("FormRecurrentLoops: m_visitedOrder was set to a constant, but actually... wasn't?");
|
||||
#else
|
||||
// sort the recurrent nodes in their ascending name, which is the same as visiting nodes in G^R
|
||||
// it is done in the mergerecurrentloops function, but just keep the code --TODO: why?? Why not rather verify the order?
|
||||
// BUGBUG: This sort() seems to do nothing, since the above loop sets all m_visitedOrder to the same value??
|
||||
sort(iter->m_nestedNodes.begin(),
|
||||
iter->m_nestedNodes.end(),
|
||||
iter->m_nestedNodes[0]->ByVisitedOrder);
|
||||
#endif
|
||||
|
||||
for (auto & node : iter->m_nestedNodes)
|
||||
{
|
||||
node->m_isPartOfLoop = true; // this is the only flag in ComputationNode that escapes FormRecurrentLoops()!
|
||||
|
|
@ -262,33 +244,6 @@ namespace Microsoft { namespace MSR { namespace CNTK {
|
|||
}
|
||||
}
|
||||
|
||||
// purge identical loops (i.e. loops that have the same source node)
|
||||
// TODO: Delete this function once we find it never triggers.
|
||||
void ComputationNetwork::UniqRecurrentLoops()
|
||||
{
|
||||
if (m_recurrentInfo.size() <= 1)
|
||||
return;
|
||||
|
||||
// uniq the m_recurrentInfo array w.r.t. m_sourceNode
|
||||
vector<shared_ptr<SEQTraversalFlowControlNode>> m_recurrentInfoTmp;
|
||||
for (const auto & iter : m_recurrentInfo) // enumerate all loops
|
||||
{
|
||||
bool bFound = false; // find a dup --TODO: check whether there is an STL algorithm for this
|
||||
for (const auto & iter2 : m_recurrentInfoTmp)
|
||||
{
|
||||
if ((*iter2).m_sourceNode == iter->m_sourceNode)
|
||||
{
|
||||
bFound = true;
|
||||
LogicError("UniqRecurrentLoops: Duplicate loops should no longer occur."); // ...since tested when creating in the first place.
|
||||
//break;
|
||||
}
|
||||
}
|
||||
if (!bFound)
|
||||
m_recurrentInfoTmp.push_back(iter);
|
||||
}
|
||||
m_recurrentInfo = move(m_recurrentInfoTmp);
|
||||
}
|
||||
|
||||
// recovers the processing order within a recurrent loop
|
||||
void ComputationNetwork::DetermineLoopForwardOrder(unordered_set<ComputationNodeBasePtr>& visited,
|
||||
unordered_set<ComputationNodeBasePtr>& recStack,
|
||||
|
|
@ -330,13 +285,6 @@ namespace Microsoft { namespace MSR { namespace CNTK {
|
|||
for (int i = 0; i < node->ChildrenSize(); i++)
|
||||
GatherLoopNodesR(node->Inputs(i), visited, recurrentResult, noRecurrentResult);
|
||||
|
||||
#if 0
|
||||
//children first for function evaluation
|
||||
// TODO: This seems not necessary here. Why does this get set here?
|
||||
if (!IsLeaf())
|
||||
m_needsGradient = ChildrenNeedGradient(); //only nodes that require gradient calculation is included in gradient calculation
|
||||
#endif
|
||||
|
||||
if (node->m_loopId >= 0)
|
||||
recurrentResult[node->m_loopId].push_back(node);
|
||||
else
|
||||
|
|
@ -348,7 +296,6 @@ namespace Microsoft { namespace MSR { namespace CNTK {
|
|||
// - that order is preserved for all nodes outside loops
|
||||
// - each node that belongs to a loop is replaced by all nodes of that loop in loop order
|
||||
// Called only from FormRecurrentLoops().
|
||||
// TODO: This could be a good place to insert sentinel nodes for nesting?
|
||||
void ComputationNetwork::ReorderLoops(list<ComputationNodeBasePtr>& nodes,
|
||||
const map<int, list<ComputationNodeBasePtr>>& /*recurrentNodes*/,
|
||||
const list<ComputationNodeBasePtr> & /*noRecurrentNodes*/)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче