more cleanup on when Resize() ops are done;

renamed SimpleEvaluator::PreCompute() to EvaluateBatchModeNodes() because the original name was misleading;
Linux Makefile now defines _DEBUG in debug builds

MachineLearning/CNTKComputationNetworkLib/ComputationNetwork.h

@@ -616,12 +616,12 @@ public:
 
                 // for every time step run through all nodes in this particular loop (treat the loop like a little ComputationNetwork)
 #if 1
+                (*nodeIter)->UpdateFunctionAndGradientMBSize(); // TODO: for sequence-to-sequence models we will need to be able to grow this step by step since size is unknown upfront
                 FrameRangeIteration range(pMBLayout, recInfo->m_isForwardLoop ? -1 : +1);
                 for (auto t = range.begin(); t != range.end(); t++)
                 {
                     for (auto nodeIter = recurrentNodes.begin(); nodeIter != recurrentNodes.end(); nodeIter++)
                     {
-                        (*nodeIter)->UpdateFunctionAndGradientMBSize();
                         (*nodeIter)->EvaluateThisNode(t);
                         if (IsNodeReqMultiSeqHandling(*nodeIter))
                             (*nodeIter)->MaskMissingValuesColumnsToZero(t.t());  // TODO: This should take a FrameRange as well

MachineLearning/CNTKComputationNetworkLib/ComputationNode.h

@@ -876,8 +876,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
         {
             FunctionValues().Resize(rows, cols);
 #ifdef _DEBUG
-            fprintf(stderr, "Resize: Invalidating %ls %ls operation.\n", NodeName().c_str(), OperationName().c_str());
-            FunctionValues().Invalidate();
+            fprintf(stderr, "Resize: Destructive resize to (%d x %d) in %ls %ls operation.\n", (int)rows, (int)cols, NodeName().c_str(), OperationName().c_str());
 #endif
         }
         /*implement*/double Get00Element() const { return FunctionValues().Get00Element(); }
@@ -946,6 +945,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
         //  - InputValue, which verifies instead of resizing (since Resize() is specified to be destructive, it should not call it).
         //  - LearnableParameters
         //  - GMMLogLikelihoodNode (which allocates some internal temp memory).
+        // Important: Unless overridden, this function is destructive. Nodes cannot carry over minibatch-size dependent state across minibatches through m_functionValues because of this.
         virtual size_t UpdateFunctionAndGradientMBSize(size_t numCols)
         {
             if (!m_pMBLayout)               // if no layout, this node contains parameters independent of MB size, don't resize
@@ -1342,7 +1342,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
                 s_constOnes[rows].find(cols) == s_constOnes[rows].end()) //not found
             {
                 Matrix<ElemType>* matrix = new Matrix<ElemType>(rows, cols, (DEVICEID_TYPE)deviceId);
-                matrix->SetValue(ElemType(1.000));
+                matrix->SetValue(1);
                 s_constOnes[rows][cols] = matrix;
             }
 

MachineLearning/CNTKComputationNetworkLib/RecurrentNodes.h

@@ -240,7 +240,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
 
             size_t t = frameRange.t();
 
-            Resize(Inputs(0));
+            VerifySize(Inputs(0));
 
             size_t T = GetNumTimeSteps();
             size_t T_delayedActivation = m_delayedActivationMBLayout ? m_delayedActivationMBLayout->GetNumTimeSteps() : 0;  // (note: should never happen in full-sequence mode)

MachineLearning/CNTKSGDLib/SGD.cpp

@@ -1153,7 +1153,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             return false;
         }
 
-        fprintf(stderr, "Found %lu PreCompute nodes\n", nodes.size());
+        fprintf(stderr, "\nPrecomputing --> %lu PreCompute nodes found.\n\n", nodes.size());
         for (auto nodeIter = nodes.begin(); nodeIter != nodes.end(); nodeIter++)
         {
             auto node = static_pointer_cast<PreComputedNode<ElemType>>(*nodeIter);
@@ -1192,6 +1192,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             auto node = static_pointer_cast<PreComputedNode<ElemType>>(*nodeIter);
             node->MarkComputed(true/*done accumulating*/);
         }
+        fprintf(stderr, "\nPrecomputing --> Completed.\n\n");
 
         return true;
     }

MachineLearning/CNTKSGDLib/SimpleEvaluator.h

@@ -759,7 +759,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
         }
 
         //return true if precomputation is executed.
-        bool PreCompute(ComputationNetwork& net,
+        bool EvaluateBatchModeNodes(ComputationNetwork& net,
                         const std::vector<ComputationNodeBasePtr>& featureNodes)
         {
             batchComputeNodes = net.GetNodesRequiringBatchMode();
@@ -923,7 +923,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             size_t maxSize = min(maxMbSize, mbSize);
 
             ResetPreCompute();
-            PreCompute(*evalnet, featureNodes);
+            EvaluateBatchModeNodes(*evalnet, featureNodes);
 
             /// need to set the minibatch size to 1, and initialize evalnet's sentence start information to let it know that this
             /// is the begining of sentence
@@ -1094,7 +1094,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             size_t itdx = 0;
 
             ResetPreCompute();
-            PreCompute(*evalnet, featureNodes);
+            EvaluateBatchModeNodes(*evalnet, featureNodes);
 
             /// need to set the minibatch size to 1, and initialize evalnet's sentence start information to let it know that this
             /// is the begining of sentence

Makefile

@@ -135,6 +135,7 @@ ifeq ("$(BUILDTYPE)","debug")
   endif
 
   CXXFLAGS += -g
+  CPPFLAGS += -D_DEBUG
   CUFLAGS += -O0 -G -lineinfo  $(GENCODE_FLAGS)
 endif
 

Math/Math/Matrix.cpp

@@ -1367,6 +1367,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
                 NOT_IMPLEMENTED
                 );
         }
+#ifdef _DEBUG
+        //Invalidate();
+#endif
     }
 
     // Note: Resize() will leave the matrix content undefined.