Merge branch 'fseide/reshaping' of https://git01.codeplex.com/cntk into fseide/reshaping

DataReader/HTKMLFReader/HTKMLFReader.cpp

@@ -1010,6 +1010,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
                             {
                                 m_pMBLayout->SetAsNoInput(i, m_numValidFrames[i], m_mbNumTimeSteps);
                             }
+
+                            // TODO: Also blast the gaps in the features and labels matrices with NaNs to prevent them from being read
                         }
 
                         typename std::map<std::wstring, Matrix<ElemType>*>::iterator iter;
@@ -1192,54 +1194,68 @@ namespace Microsoft { namespace MSR { namespace CNTK {
                             }
                             m_processedFrame[i] += (endFr-startFr);
                             m_switchFrame[i] = actualmbsize[i];
-                            if (actualmbsize[i] < m_mbNumTimeSteps)
-                                m_pMBLayout->Set(i, actualmbsize[i], MinibatchPackingFlags::SequenceStart); // NOTE: this ORs, while original code overwrote in matrix but ORed into vector
-                            if (actualmbsize[i] == m_mbNumTimeSteps)
+                            if (actualmbsize[i] != 0)
                                 m_pMBLayout->Set(i, actualmbsize[i] - 1, MinibatchPackingFlags::SequenceEnd); // NOTE: this ORs, while original code overwrote in matrix but ORed into vector
-                            startFr = m_switchFrame[i];
-                            endFr = m_mbNumTimeSteps;
-                            bool reNewSucc = ReNewBufferForMultiIO(i);
-                            for (iter = matrices.begin();iter!=matrices.end(); iter++)
-                            {
-                                // dereference matrix that corresponds to key (input/output name) and 
-                                // populate based on whether its a feature or a label
-                                //Matrix<ElemType>& data = *matrices[iter->first]; // can be features or labels
 
-                                if (m_nameToTypeMap[iter->first] == InputOutputTypes::real)
+                            // TODO: We should fill in a loop until we fill the minibatch for the case where just one ReNew is not sufficient
+                            // to fill up the remaining slots in the minibatch
+                            bool reNewSucc = ReNewBufferForMultiIO(i);
+                            if (actualmbsize[i] < m_mbNumTimeSteps)
+                            {
+                                if (reNewSucc)
                                 {
-                                    id = m_featureNameToIdMap[iter->first];
-                                    dim = m_featureNameToDimMap[iter->first];
-                                    if (sizeof(ElemType) == sizeof(float))
+                                    m_pMBLayout->Set(i, actualmbsize[i], MinibatchPackingFlags::SequenceStart); // NOTE: this ORs, while original code overwrote in matrix but ORed into vector
+                                    startFr = m_switchFrame[i];
+                                    endFr = m_mbNumTimeSteps;
+                                    for (iter = matrices.begin(); iter != matrices.end(); iter++)
                                     {
-                                        for (size_t j = startFr,k = 0; j < endFr; j++,k++) // column major, so iterate columns
+                                        // dereference matrix that corresponds to key (input/output name) and 
+                                        // populate based on whether its a feature or a label
+                                        //Matrix<ElemType>& data = *matrices[iter->first]; // can be features or labels
+
+                                        if (m_nameToTypeMap[iter->first] == InputOutputTypes::real)
                                         {
-                                            // copy over the entire column at once, need to do this because SSEMatrix may have gaps at the end of the columns
-                                            memcpy_s(&m_featuresBufferMultiIO[id].get()[(j * m_numSeqsPerMB + i) * dim], sizeof(ElemType) * dim, &m_featuresBufferMultiUtt[i].get()[k * dim + m_featuresStartIndexMultiUtt[id + i * numOfFea]], sizeof(ElemType) * dim);
-                                        }
-                                    }
-                                    else
-                                    {
-                                        for (size_t j=startFr,k=0; j < endFr; j++,k++) // column major, so iterate columns in outside loop
-                                        {
-                                            for (int d = 0; d < dim; d++)
-                                                m_featuresBufferMultiIO[id].get()[(j * m_numSeqsPerMB + i) * dim + d] = m_featuresBufferMultiUtt[i].get()[k * dim + d + m_featuresStartIndexMultiUtt[id + i * numOfFea]];
+                                            id = m_featureNameToIdMap[iter->first];
+                                            dim = m_featureNameToDimMap[iter->first];
+                                            if (sizeof(ElemType) == sizeof(float))
+                                            {
+                                                for (size_t j = startFr, k = 0; j < endFr; j++, k++) // column major, so iterate columns
+                                                {
+                                                    // copy over the entire column at once, need to do this because SSEMatrix may have gaps at the end of the columns
+                                                    memcpy_s(&m_featuresBufferMultiIO[id].get()[(j * m_numSeqsPerMB + i) * dim], sizeof(ElemType) * dim, &m_featuresBufferMultiUtt[i].get()[k * dim + m_featuresStartIndexMultiUtt[id + i * numOfFea]], sizeof(ElemType) * dim);
+                                                }
+                                            }
+                                            else
+                                            {
+                                                for (size_t j = startFr, k = 0; j < endFr; j++, k++) // column major, so iterate columns in outside loop
+                                                {
+                                                    for (int d = 0; d < dim; d++)
+                                                        m_featuresBufferMultiIO[id].get()[(j * m_numSeqsPerMB + i) * dim + d] = m_featuresBufferMultiUtt[i].get()[k * dim + d + m_featuresStartIndexMultiUtt[id + i * numOfFea]];
+                                                }
+                                            }
+                                        }
+                                        else if (m_nameToTypeMap[iter->first] == InputOutputTypes::category)
+                                        {
+                                            id = m_labelNameToIdMap[iter->first];
+                                            dim = m_labelNameToDimMap[iter->first];
+                                            for (size_t j = startFr, k = 0; j < endFr; j++, k++)
+                                            {
+                                                for (int d = 0; d < dim; d++)
+                                                    m_labelsBufferMultiIO[id].get()[(j * m_numSeqsPerMB + i) * dim + d] = m_labelsBufferMultiUtt[i].get()[k * dim + d + m_labelsStartIndexMultiUtt[id + i * numOfLabel]];
+                                            }
                                         }
                                     }
+
+                                    m_processedFrame[i] += (endFr - startFr);
                                 }
-                                else if (m_nameToTypeMap[iter->first] == InputOutputTypes::category)
+                                else
                                 {
-                                    id = m_labelNameToIdMap[iter->first];
-                                    dim = m_labelNameToDimMap[iter->first];
-                                    for (size_t j = startFr,k=0; j < endFr; j++,k++)
-                                    {
-                                        for (int d = 0; d < dim; d++)
-                                            m_labelsBufferMultiIO[id].get()[(j * m_numSeqsPerMB + i) * dim + d] = m_labelsBufferMultiUtt[i].get()[k * dim + d + m_labelsStartIndexMultiUtt[id + i * numOfLabel]];
-                                    }
+                                    // Mark gaps with NoInput
+                                    m_pMBLayout->SetAsNoInput(i, actualmbsize[i], m_mbNumTimeSteps);
+
+                                    // TODO: Also blast the gaps in the features and labels matrices with NaNs to prevent them from being read
                                 }
                             }
-
-                            if (reNewSucc) m_processedFrame[i] += (endFr-startFr);
-
                         }
                     }
                     for (auto iter = matrices.begin();iter!=matrices.end(); iter++)

MachineLearning/CNTKSGDLib/SGD.cpp

@@ -1824,11 +1824,11 @@ namespace Microsoft { namespace MSR { namespace CNTK {
         if (useDistributedMBReading)
         {
             trainSetDataReader->StartDistributedMinibatchLoop(tunedMBSize, epochNumber, g_mpi->CurrentNodeRank(),
-                                                              g_mpi->NumNodesInUse(), m_epochSize);
+                                                              g_mpi->NumNodesInUse(), epochSize);
         }
         else
         {
-            trainSetDataReader->StartMinibatchLoop(tunedMBSize, epochNumber, m_epochSize);
+            trainSetDataReader->StartMinibatchLoop(tunedMBSize, epochNumber, epochSize);
         }
 
         net.StartEvaluateMinibatchLoop(evaluationNodes);
@@ -2160,9 +2160,6 @@ namespace Microsoft { namespace MSR { namespace CNTK {
             totalEpochSamples += aggregateNumSamplesWithLabel;
             totalSamplesSeen += aggregateNumSamplesWithLabel;
 
-            if (totalEpochSamples >= epochSize)
-                break;
-
             // call DataEnd function
             // This signals something from SGD to the reader.
             // DataEnd does reader specific process if sentence ending is reached