Merge branch 'master' of https://github.com/Microsoft/CNTK into amitaga/cntkv2Library
This commit is contained in:
Amit Agarwal
Коммит
85e0d9e015
|
|
@ -1 +1 @@
|
|||
Subproject commit 87767425a4ec3b93aa574295f5332460155d0d74
|
||||
Subproject commit ed5d57e5f352c27d6739b41c0083951cb7067247
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
//
|
||||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "stdafx.h"
|
||||
#include "CNTKLibrary.h"
|
||||
|
||||
namespace CNTK
|
||||
{
|
||||
class CNTKValue final : public Value
|
||||
{
|
||||
public:
|
||||
|
||||
private:
|
||||
|
||||
};
|
||||
}
|
||||
|
|
@ -11,15 +11,13 @@ class IDistGradAggregator
|
|||
public:
|
||||
IDistGradAggregator(const MPIWrapperPtr& mpi)
|
||||
: m_mpi(mpi)
|
||||
{
|
||||
}
|
||||
{}
|
||||
|
||||
virtual ~IDistGradAggregator()
|
||||
{
|
||||
}
|
||||
{}
|
||||
|
||||
// Returns a boolean indicating if any samples were processed
|
||||
virtual bool AggregateGradients(const std::vector<Matrix<ElemType>*>& gradients, DistGradHeader* headerCPU, int epochNumber) = 0;
|
||||
virtual bool AggregateGradients(const std::vector<Matrix<ElemType>*>& gradients, DistGradHeader* headerCPU, bool resetState) = 0;
|
||||
|
||||
size_t NumProc()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -920,6 +920,7 @@ size_t SGD<ElemType>::TrainOneEpoch(ComputationNetworkPtr net,
|
|||
}
|
||||
|
||||
bool noMoreSamplesToProcess = false;
|
||||
bool isFirstMinibatch = true;
|
||||
for (;;)
|
||||
{
|
||||
// Per-minibatch performance measurements; only enabled when perfTraceLevel > 0
|
||||
|
|
@ -1105,7 +1106,7 @@ size_t SGD<ElemType>::TrainOneEpoch(ComputationNetworkPtr net,
|
|||
|
||||
// aggregate
|
||||
m_gradHeader->numEvalNode = evaluationNodes.size(); // TODO: rename numEvalNode (plural)
|
||||
bool samplesProcessed = m_distGradAgg->AggregateGradients(learnParamsGradients, m_gradHeader.get(), epochNumber);
|
||||
bool samplesProcessed = m_distGradAgg->AggregateGradients(learnParamsGradients, m_gradHeader.get(), isFirstMinibatch);
|
||||
noMoreSamplesToProcess = !samplesProcessed;
|
||||
|
||||
// read out the header--now everything is aggregated
|
||||
|
|
@ -1290,6 +1291,7 @@ size_t SGD<ElemType>::TrainOneEpoch(ComputationNetworkPtr net,
|
|||
AttemptUtteranceDerivativeFeatures(net, trainSetDataReader, featureNodes, inputMatrices);
|
||||
|
||||
profiler.NextSample();
|
||||
isFirstMinibatch = false;
|
||||
}
|
||||
|
||||
// --- END MAIN MINIBATCH LOOP
|
||||
|
|
|
|||
|
|
@ -16,27 +16,22 @@ class SimpleDistGradAggregator : public IDistGradAggregator<ElemType>
|
|||
|
||||
public:
|
||||
SimpleDistGradAggregator(const MPIWrapperPtr& mpi, bool useAsyncAggregation, int syncStatsTrace)
|
||||
: IDistGradAggregator<ElemType>(mpi), m_useAsyncAggregation(useAsyncAggregation), m_currentEpochNumber(-1), m_bufferedGradHeader(nullptr), m_syncStatsTrace(syncStatsTrace), m_iterationCount(0)
|
||||
{
|
||||
}
|
||||
: IDistGradAggregator<ElemType>(mpi), m_useAsyncAggregation(useAsyncAggregation), m_initialized(false), m_bufferedGradHeader(nullptr), m_syncStatsTrace(syncStatsTrace), m_iterationCount(0)
|
||||
{}
|
||||
|
||||
~SimpleDistGradAggregator()
|
||||
{
|
||||
for (size_t i = 0; i < m_recvHeaders.size(); ++i)
|
||||
{
|
||||
DistGradHeader::Destroy(m_recvHeaders[i]);
|
||||
}
|
||||
|
||||
if (m_bufferedGradHeader != nullptr)
|
||||
{
|
||||
DistGradHeader::Destroy(m_bufferedGradHeader);
|
||||
}
|
||||
}
|
||||
|
||||
// Aggregate the gradient matrices across all nodes
|
||||
bool AggregateGradients(const std::vector<Matrix<ElemType>*>& gradients, DistGradHeader* headerCPU, int epochNumber) override
|
||||
bool AggregateGradients(const std::vector<Matrix<ElemType>*>& gradients, DistGradHeader* headerCPU, bool resetState) override
|
||||
{
|
||||
bool isNewEpoch = ResetCurrentEpoch(gradients, headerCPU->numEvalNode, epochNumber);
|
||||
ResetState(gradients, headerCPU->numEvalNode, resetState);
|
||||
bool showSyncPerfStats = (m_syncStatsTrace > 0) && ((m_iterationCount % m_syncStatsTrace) == 0);
|
||||
m_iterationCount++;
|
||||
|
||||
|
|
@ -56,8 +51,8 @@ public:
|
|||
if (showSyncPerfStats)
|
||||
{
|
||||
aggregationTimer.Stop();
|
||||
double epochTime = aggregationTimer.ElapsedSeconds();
|
||||
fprintf(stderr, "Async gradient aggregation wait time: %.6g\n", epochTime);
|
||||
double gradientAggregationTime = aggregationTimer.ElapsedSeconds();
|
||||
fprintf(stderr, "Async gradient aggregation wait time: %.6g\n", gradientAggregationTime);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -84,7 +79,7 @@ public:
|
|||
swap(*headerCPU, *m_bufferedGradHeader);
|
||||
|
||||
// Initiate aggregation only if any samples were processed in previous iteration
|
||||
if (isNewEpoch || (headerCPU->numSamples != 0))
|
||||
if (resetState || (headerCPU->numSamples != 0))
|
||||
{
|
||||
int deviceId = gradients[0]->GetDeviceId();
|
||||
DistGradHeader* newGradHeader = m_bufferedGradHeader;
|
||||
|
|
@ -96,19 +91,18 @@ public:
|
|||
// the gradient aggregation asynchronously on a separate stream
|
||||
MatrixComputeStreamEvent* mainStreamSyncEvent = MatrixComputeStreamEvent::Create(deviceId);
|
||||
|
||||
m_pendingAsyncAggregation = std::async(std::launch::async, [=]
|
||||
{
|
||||
// We are starting on a new thread. Make sure the new thread is
|
||||
// setup to use the right device
|
||||
Matrix<ElemType>::SetDevice(deviceId);
|
||||
m_pendingAsyncAggregation = std::async(std::launch::async, [=] {
|
||||
// We are starting on a new thread. Make sure the new thread is
|
||||
// setup to use the right device
|
||||
Matrix<ElemType>::SetDevice(deviceId);
|
||||
|
||||
// Synchronize the Quantization compute stream with the completion of
|
||||
// compute of the gradient matrices on the main compute stream
|
||||
mainStreamSyncEvent->SynchronizeDataTransferFetchStreamWithEvent<ElemType>();
|
||||
delete mainStreamSyncEvent;
|
||||
// Synchronize the Quantization compute stream with the completion of
|
||||
// compute of the gradient matrices on the main compute stream
|
||||
mainStreamSyncEvent->SynchronizeDataTransferFetchStreamWithEvent<ElemType>();
|
||||
delete mainStreamSyncEvent;
|
||||
|
||||
AggregateGradientsImpl(newGradients, newGradHeader, showSyncPerfStats);
|
||||
});
|
||||
AggregateGradientsImpl(newGradients, newGradHeader, showSyncPerfStats);
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -135,18 +129,15 @@ private:
|
|||
});
|
||||
}
|
||||
|
||||
bool ResetCurrentEpoch(const std::vector<Matrix<ElemType>*>& gradients, int numEvalNode, int epochNumber)
|
||||
void ResetState(const std::vector<Matrix<ElemType>*>& gradients, int numEvalNodes, bool resetState)
|
||||
{
|
||||
bool isNewEpoch = (m_currentEpochNumber != epochNumber);
|
||||
|
||||
// When called the first time let's setup the intermediateCPU buffers for gradient aggregation if needed
|
||||
if (m_currentEpochNumber == -1)
|
||||
if (!m_initialized)
|
||||
{
|
||||
m_initialized = true;
|
||||
int deviceId = gradients[0]->GetDeviceId();
|
||||
if (deviceId != CPUDEVICE)
|
||||
{
|
||||
m_allocator.reset(new CUDAPageLockedMemAllocator(deviceId));
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < gradients.size(); i++)
|
||||
{
|
||||
|
|
@ -161,49 +152,36 @@ private:
|
|||
}
|
||||
|
||||
if (m_useAsyncAggregation)
|
||||
{
|
||||
m_bufferedGradients[gradients[i]].reset(new Matrix<ElemType>(gradients[i]->GetNumRows(), gradients[i]->GetNumCols(), deviceId));
|
||||
}
|
||||
}
|
||||
|
||||
if (m_useAsyncAggregation)
|
||||
{
|
||||
m_bufferedGradHeader = DistGradHeader::Create(numEvalNode);
|
||||
m_bufferedGradHeader = DistGradHeader::Create(numEvalNodes);
|
||||
m_bufferedGradHeader->Clear();
|
||||
}
|
||||
|
||||
if (m_mpi->IsMainNode())
|
||||
{
|
||||
for (size_t i = 0; i < NumProc() - 1; ++i)
|
||||
{
|
||||
m_recvHeaders.push_back(DistGradHeader::Create(numEvalNode));
|
||||
}
|
||||
m_recvHeaders.push_back(DistGradHeader::Create(numEvalNodes));
|
||||
}
|
||||
}
|
||||
else
|
||||
else if (resetState)
|
||||
{
|
||||
if (epochNumber != m_currentEpochNumber)
|
||||
// Make sure there is no pending async aggregation
|
||||
if (m_useAsyncAggregation && m_pendingAsyncAggregation.valid())
|
||||
LogicError("Unexpected pending async gradient aggregation found when resetting aggregator state!");
|
||||
|
||||
// Zero out the buffered gradients if resetting state
|
||||
if (m_useAsyncAggregation)
|
||||
{
|
||||
// Make sure there is no pending async aggregation
|
||||
if (m_useAsyncAggregation && m_pendingAsyncAggregation.valid())
|
||||
LogicError("Unexpected pending async gradient aggregation at the beginning of new epoch!");
|
||||
for (size_t i = 0; i < gradients.size(); i++)
|
||||
m_bufferedGradients[gradients[i]]->SetValue(0);
|
||||
|
||||
// Zero out the buffered gradients at the beginning of a new epoch
|
||||
if (m_useAsyncAggregation)
|
||||
{
|
||||
for (size_t i = 0; i < gradients.size(); i++)
|
||||
{
|
||||
m_bufferedGradients[gradients[i]]->SetValue(0);
|
||||
}
|
||||
|
||||
m_bufferedGradHeader->Clear();
|
||||
}
|
||||
m_bufferedGradHeader->Clear();
|
||||
}
|
||||
}
|
||||
|
||||
m_currentEpochNumber = epochNumber;
|
||||
|
||||
return isNewEpoch;
|
||||
}
|
||||
|
||||
void AggregateGradientsImpl(const std::vector<Matrix<ElemType>*>& gradients, DistGradHeader* headerCPU, bool showSyncPerfStats)
|
||||
|
|
@ -228,9 +206,7 @@ private:
|
|||
|
||||
// If the current node did not process any samples, the gradients should be zero'd
|
||||
for (size_t i = 0; i < numGradMatrices; ++i)
|
||||
{
|
||||
gradients[i]->SetValue(0);
|
||||
}
|
||||
|
||||
if (m_useAsyncAggregation)
|
||||
{
|
||||
|
|
@ -243,9 +219,7 @@ private:
|
|||
if (deviceId >= 0)
|
||||
{
|
||||
for (size_t i = 0; i < numGradMatrices; ++i)
|
||||
{
|
||||
m_gpuDataTransferers[i]->CopyGPUToCPUAsync(gradients[i]->Data(), gradients[i]->GetNumElements(), m_intermediateCPUBuffers[i].get());
|
||||
}
|
||||
}
|
||||
|
||||
// Initiate receive of the header on the main node
|
||||
|
|
@ -263,9 +237,7 @@ private:
|
|||
// Send the headers from all nodes but the main node
|
||||
MPI_Request sendHeaderRequest;
|
||||
if (!m_mpi->IsMainNode())
|
||||
{
|
||||
MPI_Isend(headerCPU, headerCPU->Size(), MPI_CHAR, m_mpi->MainNodeRank(), numGradMatrices, m_mpi->Communicator(), &sendHeaderRequest) || MpiFail("MPI_Isend");
|
||||
}
|
||||
|
||||
// Perform MPI async allreduce on the gradient data
|
||||
std::vector<MPI_Request> allReduceRequests(numGradMatrices);
|
||||
|
|
@ -303,12 +275,10 @@ private:
|
|||
assert(numNodesHeadersReceivedFrom == (NumProc() - 1));
|
||||
}
|
||||
|
||||
MPI_Request recvAggHeaderRequest;
|
||||
// Initiate receive of the aggregate header
|
||||
MPI_Request recvAggHeaderRequest;
|
||||
if (!m_mpi->IsMainNode())
|
||||
{
|
||||
MPI_Irecv(headerCPU, headerCPU->Size(), MPI_CHAR, m_mpi->MainNodeRank(), numGradMatrices + 1 + numGradMatrices, m_mpi->Communicator(), &recvAggHeaderRequest) || MpiFail("MPI_Irecv");
|
||||
}
|
||||
|
||||
// Intiate send of the aggregate header from main node
|
||||
std::vector<MPI_Request> sendAggHeaderRequests(NumProc() - 1);
|
||||
|
|
@ -327,41 +297,31 @@ private:
|
|||
{
|
||||
MPI_Wait(&allReduceRequests[i], MPI_STATUSES_IGNORE) || MpiFail("MPI_Wait");
|
||||
if (deviceId >= 0)
|
||||
{
|
||||
m_gpuDataTransferers[i]->CopyCPUToGPUAsync(m_intermediateCPUBuffers[i].get(), gradients[i]->GetNumElements(), gradients[i]->Data());
|
||||
}
|
||||
}
|
||||
|
||||
// Wait to receive aggregate header
|
||||
if (!m_mpi->IsMainNode())
|
||||
{
|
||||
MPI_Wait(&recvAggHeaderRequest, MPI_STATUSES_IGNORE) || MpiFail("MPI_Wait");
|
||||
}
|
||||
|
||||
// Wait for all the transfers to finish
|
||||
if (deviceId >= 0)
|
||||
{
|
||||
for (size_t i = 0; i < numGradMatrices; ++i)
|
||||
{
|
||||
m_gpuDataTransferers[i]->WaitForCopyCPUToGPUAsync();
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for completion of the async send requests
|
||||
if (!m_mpi->IsMainNode())
|
||||
{
|
||||
MPI_Wait(&sendHeaderRequest, MPI_STATUSES_IGNORE) || MpiFail("MPI_Wait");
|
||||
}
|
||||
else
|
||||
{
|
||||
MPI_Waitall(sendAggHeaderRequests.size(), sendAggHeaderRequests.data(), MPI_STATUSES_IGNORE) || MpiFail("MPI_Waitall");
|
||||
}
|
||||
|
||||
if (showSyncPerfStats)
|
||||
{
|
||||
aggregationTimer.Stop();
|
||||
double epochTime = aggregationTimer.ElapsedSeconds();
|
||||
fprintf(stderr, "Actual gradient aggregation time: %.6g\n", epochTime);
|
||||
double gradientAggregationTime = aggregationTimer.ElapsedSeconds();
|
||||
fprintf(stderr, "Actual gradient aggregation time: %.6g\n", gradientAggregationTime);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -388,6 +348,6 @@ private:
|
|||
// Only used for controlling frequency of measuring/showing gradient aggregation perf stats
|
||||
size_t m_iterationCount;
|
||||
|
||||
int m_currentEpochNumber;
|
||||
bool m_initialized;
|
||||
};
|
||||
} } }
|
||||
|
|
|
|||
|
|
@ -201,7 +201,7 @@ public:
|
|||
|
||||
// Using SimpleDistAggregator for eval results only. At some point we should rename the class to be just
|
||||
// IDistAggregator and SimpleDistAggregator.
|
||||
bool samplesProcessed = m_distGradAgg->AggregateGradients(learnParamsGradients, m_gradHeader.get(), 0);
|
||||
bool samplesProcessed = m_distGradAgg->AggregateGradients(learnParamsGradients, m_gradHeader.get(), /*resetState =*/ false);
|
||||
noMoreSamplesToProcess = !samplesProcessed;
|
||||
|
||||
aggregateNumSamplesWithLabel = m_gradHeader->numSamplesWithLabel;
|
||||
|
|
@ -389,7 +389,7 @@ public:
|
|||
learnParamsValues[1] = &(runStdNode->Value());
|
||||
|
||||
m_gradHeader->numSamples = actualMBSize ? 1 : actualMBSize;
|
||||
distGradAgg.AggregateGradients(learnParamsValues, m_gradHeader.get(), 0);
|
||||
distGradAgg.AggregateGradients(learnParamsValues, m_gradHeader.get(), /*resetState =*/ false);
|
||||
|
||||
for (auto& parameter : learnParamsValues)
|
||||
{
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,997 @@
|
|||
CPU info:
|
||||
CPU Model Name: Intel(R) Xeon(R) CPU W3530 @ 2.80GHz
|
||||
Hardware threads: 4
|
||||
Total Memory: 12580404 kB
|
||||
-------------------------------------------------------------------
|
||||
=== Running C:\Program Files\Microsoft MPI\Bin\/mpiexec.exe -n 2 C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\debug\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=2 stderr=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
CNTK 1.7+ (HEAD 28bddd, Sep 17 2016 17:51:24) on cntk-muc00 at 2016/09/17 18:04:54
|
||||
|
||||
C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\debug\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=2 stderr=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
Changed current directory to C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data
|
||||
MPIWrapper: initializing MPI
|
||||
CNTK 1.7+ (HEAD 28bddd, Sep 17 2016 17:51:24) on cntk-muc00 at 2016/09/17 18:04:54
|
||||
|
||||
C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\debug\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=2 stderr=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
Changed current directory to C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data
|
||||
MPIWrapper: initializing MPI
|
||||
ping [requestnodes (before change)]: 2 nodes pinging each other
|
||||
ping [requestnodes (before change)]: 2 nodes pinging each other
|
||||
ping [requestnodes (before change)]: all 2 nodes responded
|
||||
requestnodes [MPIWrapper]: using 2 out of 2 MPI nodes (2 requested); we (0) are in (participating)
|
||||
ping [requestnodes (before change)]: all 2 nodes responded
|
||||
ping [requestnodes (after change)]: 2 nodes pinging each other
|
||||
requestnodes [MPIWrapper]: using 2 out of 2 MPI nodes (2 requested); we (1) are in (participating)
|
||||
ping [requestnodes (after change)]: 2 nodes pinging each other
|
||||
ping [requestnodes (after change)]: all 2 nodes responded
|
||||
mpihelper: we are cog 1 in a gearbox of 2
|
||||
ping [requestnodes (after change)]: all 2 nodes responded
|
||||
ping [mpihelper]: 2 nodes pinging each other
|
||||
mpihelper: we are cog 0 in a gearbox of 2
|
||||
ping [mpihelper]: 2 nodes pinging each other
|
||||
ping [mpihelper]: all 2 nodes responded
|
||||
ping [mpihelper]: all 2 nodes responded
|
||||
MPI Rank 0: 09/17/2016 18:04:55: Redirecting stderr to file C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr_speechTrain.logrank0
|
||||
MPI Rank 0: CNTK 1.7+ (HEAD 28bddd, Sep 17 2016 17:51:24) on cntk-muc00 at 2016/09/17 18:04:54
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\debug\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=2 stderr=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
MPI Rank 0: 09/17/2016 18:04:55: Using 2 CPU threads.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:04:55: ##############################################################################
|
||||
MPI Rank 0: 09/17/2016 18:04:55: # #
|
||||
MPI Rank 0: 09/17/2016 18:04:55: # speechTrain command (train action) #
|
||||
MPI Rank 0: 09/17/2016 18:04:55: # #
|
||||
MPI Rank 0: 09/17/2016 18:04:55: ##############################################################################
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:04:55:
|
||||
MPI Rank 0: Creating virgin network.
|
||||
MPI Rank 0: SimpleNetworkBuilder Using GPU 0
|
||||
MPI Rank 0: Microsoft::MSR::CNTK::GPUMatrix<ElemType>::SetUniformRandomValue (GPU): creating curand object with seed 1, sizeof(ElemType)==8
|
||||
MPI Rank 0: reading script file glob_0000.scp ... 948 entries
|
||||
MPI Rank 0: total 132 state names in state list C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data/state.list
|
||||
MPI Rank 0: htkmlfreader: reading MLF file C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data/glob_0000.mlf ... total 948 entries
|
||||
MPI Rank 0: ...............................................................................................feature set 0: 252734 frames in 948 out of 948 utterances
|
||||
MPI Rank 0: label set 0: 129 classes
|
||||
MPI Rank 0: minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: 316.0 utterances, 84244.7 frames
|
||||
MPI Rank 0: 09/17/2016 18:04:57:
|
||||
MPI Rank 0: Model has 25 nodes. Using GPU 0.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Training criterion: CrossEntropyWithSoftmax = CrossEntropyWithSoftmax
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Evaluation criterion: EvalClassificationError = ClassificationError
|
||||
MPI Rank 0:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: Allocating matrices for forward and/or backward propagation.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: Memory Sharing: Out of 40 matrices, 19 are shared as 8, and 21 are not shared.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: { W0 : [512 x 363] (gradient)
|
||||
MPI Rank 0: W0*features+B0 : [512 x 1 x *] }
|
||||
MPI Rank 0: { H1 : [512 x 1 x *]
|
||||
MPI Rank 0: W0*features : [512 x *] (gradient) }
|
||||
MPI Rank 0: { W0*features+B0 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 0: W1*H1 : [512 x 1 x *] }
|
||||
MPI Rank 0: { B1 : [512 x 1] (gradient)
|
||||
MPI Rank 0: H2 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 0: HLast : [132 x 1 x *] (gradient) }
|
||||
MPI Rank 0: { H2 : [512 x 1 x *]
|
||||
MPI Rank 0: W1*H1 : [512 x 1 x *] (gradient) }
|
||||
MPI Rank 0: { B0 : [512 x 1] (gradient)
|
||||
MPI Rank 0: H1 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 0: W1*H1+B1 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 0: W2*H1 : [132 x 1 x *] }
|
||||
MPI Rank 0: { W1 : [512 x 512] (gradient)
|
||||
MPI Rank 0: W1*H1+B1 : [512 x 1 x *] }
|
||||
MPI Rank 0: { HLast : [132 x 1 x *]
|
||||
MPI Rank 0: W2 : [132 x 512] (gradient) }
|
||||
MPI Rank 0:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Training 516740 parameters in 6 out of 6 parameter tensors and 15 nodes with gradient:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Node 'B0' (LearnableParameter operation) : [512 x 1]
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Node 'B1' (LearnableParameter operation) : [512 x 1]
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Node 'B2' (LearnableParameter operation) : [132 x 1]
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Node 'W0' (LearnableParameter operation) : [512 x 363]
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Node 'W1' (LearnableParameter operation) : [512 x 512]
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Node 'W2' (LearnableParameter operation) : [132 x 512]
|
||||
MPI Rank 0:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Precomputing --> 3 PreCompute nodes found.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:04:57: MeanOfFeatures = Mean()
|
||||
MPI Rank 0: 09/17/2016 18:04:57: InvStdOfFeatures = InvStdDev()
|
||||
MPI Rank 0: 09/17/2016 18:04:57: Prior = Mean()
|
||||
MPI Rank 0: minibatchiterator: epoch 0: frames [0..252734] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses
|
||||
MPI Rank 0: requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:05:27: Precomputing --> Completed.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:05:28: Starting Epoch 1: learning rate per sample = 0.001953 effective momentum = 0.939413 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 0: minibatchiterator: epoch 0: frames [0..81920] (first utterance at frame 0), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:05:28: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:05:28: Epoch[ 1 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 4.73732304 * 640; EvalClassificationError = 0.96718750 * 640; time = 0.2001s; samplesPerSecond = 3198.9
|
||||
MPI Rank 0: 09/17/2016 18:05:28: Epoch[ 1 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 4.37245658 * 640; EvalClassificationError = 0.92187500 * 640; time = 0.1852s; samplesPerSecond = 3456.1
|
||||
MPI Rank 0: 09/17/2016 18:05:28: Epoch[ 1 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 4.17410438 * 640; EvalClassificationError = 0.95937500 * 640; time = 0.1837s; samplesPerSecond = 3484.5
|
||||
MPI Rank 0: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 3.88445939 * 640; EvalClassificationError = 0.86093750 * 640; time = 0.1869s; samplesPerSecond = 3424.0
|
||||
MPI Rank 0: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 4.05583056 * 640; EvalClassificationError = 0.89375000 * 640; time = 0.1836s; samplesPerSecond = 3486.3
|
||||
MPI Rank 0: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 3.94301179 * 640; EvalClassificationError = 0.86250000 * 640; time = 0.1862s; samplesPerSecond = 3437.2
|
||||
MPI Rank 0: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 3.80690392 * 640; EvalClassificationError = 0.85937500 * 640; time = 0.1982s; samplesPerSecond = 3229.0
|
||||
MPI Rank 0: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 3.92014544 * 640; EvalClassificationError = 0.87812500 * 640; time = 0.1855s; samplesPerSecond = 3449.6
|
||||
MPI Rank 0: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 3.97161496 * 640; EvalClassificationError = 0.87968750 * 640; time = 0.1847s; samplesPerSecond = 3465.1
|
||||
MPI Rank 0: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 3.93678341 * 640; EvalClassificationError = 0.90937500 * 640; time = 0.1846s; samplesPerSecond = 3466.7
|
||||
MPI Rank 0: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 4.05170333 * 640; EvalClassificationError = 0.92187500 * 640; time = 0.1851s; samplesPerSecond = 3457.4
|
||||
MPI Rank 0: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 3.89920006 * 640; EvalClassificationError = 0.86406250 * 640; time = 0.1863s; samplesPerSecond = 3435.3
|
||||
MPI Rank 0: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 3.87251780 * 640; EvalClassificationError = 0.89218750 * 640; time = 0.1838s; samplesPerSecond = 3481.9
|
||||
MPI Rank 0: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 3.81997328 * 640; EvalClassificationError = 0.88281250 * 640; time = 0.1849s; samplesPerSecond = 3460.4
|
||||
MPI Rank 0: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 3.80473823 * 640; EvalClassificationError = 0.88125000 * 640; time = 0.1853s; samplesPerSecond = 3454.5
|
||||
MPI Rank 0: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 3.94101364 * 640; EvalClassificationError = 0.88750000 * 640; time = 0.1852s; samplesPerSecond = 3455.2
|
||||
MPI Rank 0: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 161- 170, 13.28%]: CrossEntropyWithSoftmax = 3.82130368 * 640; EvalClassificationError = 0.87656250 * 640; time = 0.1853s; samplesPerSecond = 3453.2
|
||||
MPI Rank 0: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 171- 180, 14.06%]: CrossEntropyWithSoftmax = 3.75236005 * 640; EvalClassificationError = 0.85312500 * 640; time = 0.1847s; samplesPerSecond = 3465.3
|
||||
MPI Rank 0: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 181- 190, 14.84%]: CrossEntropyWithSoftmax = 3.79661132 * 640; EvalClassificationError = 0.86562500 * 640; time = 0.1864s; samplesPerSecond = 3434.3
|
||||
MPI Rank 0: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 191- 200, 15.63%]: CrossEntropyWithSoftmax = 3.73922363 * 640; EvalClassificationError = 0.87343750 * 640; time = 0.1846s; samplesPerSecond = 3467.0
|
||||
MPI Rank 0: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 201- 210, 16.41%]: CrossEntropyWithSoftmax = 3.73517402 * 640; EvalClassificationError = 0.86406250 * 640; time = 0.1855s; samplesPerSecond = 3450.0
|
||||
MPI Rank 0: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 211- 220, 17.19%]: CrossEntropyWithSoftmax = 3.72398663 * 640; EvalClassificationError = 0.85625000 * 640; time = 0.1840s; samplesPerSecond = 3477.3
|
||||
MPI Rank 0: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 221- 230, 17.97%]: CrossEntropyWithSoftmax = 3.71595086 * 640; EvalClassificationError = 0.84062500 * 640; time = 0.1852s; samplesPerSecond = 3455.3
|
||||
MPI Rank 0: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 231- 240, 18.75%]: CrossEntropyWithSoftmax = 3.75562845 * 640; EvalClassificationError = 0.83281250 * 640; time = 0.1861s; samplesPerSecond = 3438.2
|
||||
MPI Rank 0: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 241- 250, 19.53%]: CrossEntropyWithSoftmax = 3.72052883 * 640; EvalClassificationError = 0.85468750 * 640; time = 0.1859s; samplesPerSecond = 3443.0
|
||||
MPI Rank 0: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 251- 260, 20.31%]: CrossEntropyWithSoftmax = 3.66417431 * 640; EvalClassificationError = 0.83906250 * 640; time = 0.1839s; samplesPerSecond = 3480.2
|
||||
MPI Rank 0: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 261- 270, 21.09%]: CrossEntropyWithSoftmax = 3.58440175 * 640; EvalClassificationError = 0.77812500 * 640; time = 0.1844s; samplesPerSecond = 3469.9
|
||||
MPI Rank 0: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 271- 280, 21.88%]: CrossEntropyWithSoftmax = 3.69427679 * 640; EvalClassificationError = 0.80781250 * 640; time = 0.1855s; samplesPerSecond = 3450.0
|
||||
MPI Rank 0: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 281- 290, 22.66%]: CrossEntropyWithSoftmax = 3.57769881 * 640; EvalClassificationError = 0.79062500 * 640; time = 0.1839s; samplesPerSecond = 3479.8
|
||||
MPI Rank 0: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 291- 300, 23.44%]: CrossEntropyWithSoftmax = 3.52797012 * 640; EvalClassificationError = 0.82031250 * 640; time = 0.1849s; samplesPerSecond = 3460.7
|
||||
MPI Rank 0: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 301- 310, 24.22%]: CrossEntropyWithSoftmax = 3.53498040 * 640; EvalClassificationError = 0.81718750 * 640; time = 0.1846s; samplesPerSecond = 3467.3
|
||||
MPI Rank 0: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 311- 320, 25.00%]: CrossEntropyWithSoftmax = 3.50005340 * 640; EvalClassificationError = 0.79375000 * 640; time = 0.1860s; samplesPerSecond = 3440.2
|
||||
MPI Rank 0: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 321- 330, 25.78%]: CrossEntropyWithSoftmax = 3.37848051 * 640; EvalClassificationError = 0.79687500 * 640; time = 0.1850s; samplesPerSecond = 3460.0
|
||||
MPI Rank 0: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 331- 340, 26.56%]: CrossEntropyWithSoftmax = 3.44203869 * 640; EvalClassificationError = 0.78906250 * 640; time = 0.1846s; samplesPerSecond = 3467.0
|
||||
MPI Rank 0: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 341- 350, 27.34%]: CrossEntropyWithSoftmax = 3.49473674 * 640; EvalClassificationError = 0.80156250 * 640; time = 0.1865s; samplesPerSecond = 3432.4
|
||||
MPI Rank 0: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 351- 360, 28.13%]: CrossEntropyWithSoftmax = 3.42678273 * 640; EvalClassificationError = 0.80468750 * 640; time = 0.1835s; samplesPerSecond = 3488.0
|
||||
MPI Rank 0: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 361- 370, 28.91%]: CrossEntropyWithSoftmax = 3.49825811 * 640; EvalClassificationError = 0.82656250 * 640; time = 0.1850s; samplesPerSecond = 3458.6
|
||||
MPI Rank 0: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 371- 380, 29.69%]: CrossEntropyWithSoftmax = 3.18570771 * 640; EvalClassificationError = 0.75156250 * 640; time = 0.1849s; samplesPerSecond = 3460.9
|
||||
MPI Rank 0: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 381- 390, 30.47%]: CrossEntropyWithSoftmax = 3.31483570 * 640; EvalClassificationError = 0.79218750 * 640; time = 0.1841s; samplesPerSecond = 3475.8
|
||||
MPI Rank 0: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 391- 400, 31.25%]: CrossEntropyWithSoftmax = 3.27794269 * 640; EvalClassificationError = 0.77656250 * 640; time = 0.1848s; samplesPerSecond = 3463.5
|
||||
MPI Rank 0: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 401- 410, 32.03%]: CrossEntropyWithSoftmax = 3.42283181 * 640; EvalClassificationError = 0.80312500 * 640; time = 0.1858s; samplesPerSecond = 3445.3
|
||||
MPI Rank 0: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 411- 420, 32.81%]: CrossEntropyWithSoftmax = 3.28013444 * 640; EvalClassificationError = 0.75937500 * 640; time = 0.1843s; samplesPerSecond = 3471.7
|
||||
MPI Rank 0: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 421- 430, 33.59%]: CrossEntropyWithSoftmax = 3.23863079 * 640; EvalClassificationError = 0.78281250 * 640; time = 0.1879s; samplesPerSecond = 3406.8
|
||||
MPI Rank 0: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 431- 440, 34.38%]: CrossEntropyWithSoftmax = 3.13381567 * 640; EvalClassificationError = 0.75937500 * 640; time = 0.1904s; samplesPerSecond = 3360.7
|
||||
MPI Rank 0: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 441- 450, 35.16%]: CrossEntropyWithSoftmax = 3.18522093 * 640; EvalClassificationError = 0.74062500 * 640; time = 0.1909s; samplesPerSecond = 3351.8
|
||||
MPI Rank 0: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 451- 460, 35.94%]: CrossEntropyWithSoftmax = 3.23487402 * 640; EvalClassificationError = 0.77187500 * 640; time = 0.1971s; samplesPerSecond = 3247.5
|
||||
MPI Rank 0: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 461- 470, 36.72%]: CrossEntropyWithSoftmax = 3.19165708 * 640; EvalClassificationError = 0.77812500 * 640; time = 0.1842s; samplesPerSecond = 3474.4
|
||||
MPI Rank 0: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 471- 480, 37.50%]: CrossEntropyWithSoftmax = 3.16584445 * 640; EvalClassificationError = 0.73906250 * 640; time = 0.2009s; samplesPerSecond = 3186.3
|
||||
MPI Rank 0: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 481- 490, 38.28%]: CrossEntropyWithSoftmax = 3.24961355 * 640; EvalClassificationError = 0.77187500 * 640; time = 0.2048s; samplesPerSecond = 3124.4
|
||||
MPI Rank 0: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 491- 500, 39.06%]: CrossEntropyWithSoftmax = 3.18529030 * 640; EvalClassificationError = 0.76406250 * 640; time = 0.2747s; samplesPerSecond = 2329.6
|
||||
MPI Rank 0: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 501- 510, 39.84%]: CrossEntropyWithSoftmax = 2.98383964 * 640; EvalClassificationError = 0.72187500 * 640; time = 0.1890s; samplesPerSecond = 3385.7
|
||||
MPI Rank 0: 09/17/2016 18:05:38: Epoch[ 1 of 3]-Minibatch[ 511- 520, 40.63%]: CrossEntropyWithSoftmax = 3.14703955 * 640; EvalClassificationError = 0.73281250 * 640; time = 0.2083s; samplesPerSecond = 3072.1
|
||||
MPI Rank 0: 09/17/2016 18:05:38: Epoch[ 1 of 3]-Minibatch[ 521- 530, 41.41%]: CrossEntropyWithSoftmax = 3.08217828 * 640; EvalClassificationError = 0.72656250 * 640; time = 0.2300s; samplesPerSecond = 2783.2
|
||||
MPI Rank 0: 09/17/2016 18:05:38: Epoch[ 1 of 3]-Minibatch[ 531- 540, 42.19%]: CrossEntropyWithSoftmax = 2.93980472 * 640; EvalClassificationError = 0.70625000 * 640; time = 0.2435s; samplesPerSecond = 2628.8
|
||||
MPI Rank 0: 09/17/2016 18:05:38: Epoch[ 1 of 3]-Minibatch[ 541- 550, 42.97%]: CrossEntropyWithSoftmax = 3.02220354 * 640; EvalClassificationError = 0.72968750 * 640; time = 0.2409s; samplesPerSecond = 2657.1
|
||||
MPI Rank 0: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 551- 560, 43.75%]: CrossEntropyWithSoftmax = 2.95530592 * 640; EvalClassificationError = 0.71250000 * 640; time = 0.2355s; samplesPerSecond = 2718.0
|
||||
MPI Rank 0: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 561- 570, 44.53%]: CrossEntropyWithSoftmax = 2.95810228 * 640; EvalClassificationError = 0.72812500 * 640; time = 0.1847s; samplesPerSecond = 3464.8
|
||||
MPI Rank 0: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 571- 580, 45.31%]: CrossEntropyWithSoftmax = 3.00786863 * 640; EvalClassificationError = 0.72656250 * 640; time = 0.1850s; samplesPerSecond = 3459.7
|
||||
MPI Rank 0: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 581- 590, 46.09%]: CrossEntropyWithSoftmax = 3.00106811 * 640; EvalClassificationError = 0.72968750 * 640; time = 0.1874s; samplesPerSecond = 3415.4
|
||||
MPI Rank 0: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 591- 600, 46.88%]: CrossEntropyWithSoftmax = 2.91931245 * 640; EvalClassificationError = 0.70468750 * 640; time = 0.1851s; samplesPerSecond = 3457.7
|
||||
MPI Rank 0: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 601- 610, 47.66%]: CrossEntropyWithSoftmax = 3.00601604 * 640; EvalClassificationError = 0.73750000 * 640; time = 0.1848s; samplesPerSecond = 3462.9
|
||||
MPI Rank 0: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 611- 620, 48.44%]: CrossEntropyWithSoftmax = 2.93808431 * 640; EvalClassificationError = 0.70625000 * 640; time = 0.1858s; samplesPerSecond = 3444.0
|
||||
MPI Rank 0: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 621- 630, 49.22%]: CrossEntropyWithSoftmax = 2.97093532 * 640; EvalClassificationError = 0.74375000 * 640; time = 0.1853s; samplesPerSecond = 3453.2
|
||||
MPI Rank 0: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 631- 640, 50.00%]: CrossEntropyWithSoftmax = 2.86102307 * 640; EvalClassificationError = 0.67812500 * 640; time = 0.1834s; samplesPerSecond = 3489.4
|
||||
MPI Rank 0: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 641- 650, 50.78%]: CrossEntropyWithSoftmax = 2.88070307 * 640; EvalClassificationError = 0.71406250 * 640; time = 0.1845s; samplesPerSecond = 3469.4
|
||||
MPI Rank 0: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 651- 660, 51.56%]: CrossEntropyWithSoftmax = 2.90422279 * 640; EvalClassificationError = 0.71875000 * 640; time = 0.1872s; samplesPerSecond = 3418.3
|
||||
MPI Rank 0: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 661- 670, 52.34%]: CrossEntropyWithSoftmax = 2.82634561 * 640; EvalClassificationError = 0.70937500 * 640; time = 0.1864s; samplesPerSecond = 3433.8
|
||||
MPI Rank 0: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 671- 680, 53.13%]: CrossEntropyWithSoftmax = 2.86572702 * 640; EvalClassificationError = 0.67343750 * 640; time = 0.1852s; samplesPerSecond = 3456.5
|
||||
MPI Rank 0: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 681- 690, 53.91%]: CrossEntropyWithSoftmax = 2.82698660 * 640; EvalClassificationError = 0.70468750 * 640; time = 0.1850s; samplesPerSecond = 3459.3
|
||||
MPI Rank 0: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 691- 700, 54.69%]: CrossEntropyWithSoftmax = 2.69768998 * 640; EvalClassificationError = 0.68437500 * 640; time = 0.1865s; samplesPerSecond = 3431.2
|
||||
MPI Rank 0: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 701- 710, 55.47%]: CrossEntropyWithSoftmax = 2.74280097 * 640; EvalClassificationError = 0.66406250 * 640; time = 0.1858s; samplesPerSecond = 3444.4
|
||||
MPI Rank 0: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 711- 720, 56.25%]: CrossEntropyWithSoftmax = 2.79750038 * 640; EvalClassificationError = 0.71093750 * 640; time = 0.1838s; samplesPerSecond = 3481.3
|
||||
MPI Rank 0: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 721- 730, 57.03%]: CrossEntropyWithSoftmax = 2.68839718 * 640; EvalClassificationError = 0.67968750 * 640; time = 0.1856s; samplesPerSecond = 3447.7
|
||||
MPI Rank 0: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 731- 740, 57.81%]: CrossEntropyWithSoftmax = 2.76485288 * 640; EvalClassificationError = 0.68281250 * 640; time = 0.1853s; samplesPerSecond = 3454.7
|
||||
MPI Rank 0: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 741- 750, 58.59%]: CrossEntropyWithSoftmax = 2.77321739 * 640; EvalClassificationError = 0.67968750 * 640; time = 0.1853s; samplesPerSecond = 3454.0
|
||||
MPI Rank 0: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 751- 760, 59.38%]: CrossEntropyWithSoftmax = 2.71065612 * 640; EvalClassificationError = 0.68593750 * 640; time = 0.1846s; samplesPerSecond = 3467.8
|
||||
MPI Rank 0: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 761- 770, 60.16%]: CrossEntropyWithSoftmax = 2.78848250 * 640; EvalClassificationError = 0.65937500 * 640; time = 0.1836s; samplesPerSecond = 3486.0
|
||||
MPI Rank 0: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 771- 780, 60.94%]: CrossEntropyWithSoftmax = 2.78443193 * 640; EvalClassificationError = 0.68281250 * 640; time = 0.1857s; samplesPerSecond = 3445.9
|
||||
MPI Rank 0: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 781- 790, 61.72%]: CrossEntropyWithSoftmax = 2.72094929 * 640; EvalClassificationError = 0.67031250 * 640; time = 0.1852s; samplesPerSecond = 3456.4
|
||||
MPI Rank 0: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 791- 800, 62.50%]: CrossEntropyWithSoftmax = 2.70404088 * 640; EvalClassificationError = 0.64062500 * 640; time = 0.1853s; samplesPerSecond = 3454.4
|
||||
MPI Rank 0: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 801- 810, 63.28%]: CrossEntropyWithSoftmax = 2.70887221 * 640; EvalClassificationError = 0.66875000 * 640; time = 0.1843s; samplesPerSecond = 3472.9
|
||||
MPI Rank 0: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 811- 820, 64.06%]: CrossEntropyWithSoftmax = 2.67633326 * 640; EvalClassificationError = 0.66562500 * 640; time = 0.1848s; samplesPerSecond = 3463.3
|
||||
MPI Rank 0: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 821- 830, 64.84%]: CrossEntropyWithSoftmax = 2.53198524 * 640; EvalClassificationError = 0.62968750 * 640; time = 0.1852s; samplesPerSecond = 3455.0
|
||||
MPI Rank 0: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 831- 840, 65.63%]: CrossEntropyWithSoftmax = 2.63317481 * 640; EvalClassificationError = 0.66093750 * 640; time = 0.1846s; samplesPerSecond = 3466.1
|
||||
MPI Rank 0: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 841- 850, 66.41%]: CrossEntropyWithSoftmax = 2.65923035 * 640; EvalClassificationError = 0.65468750 * 640; time = 0.1845s; samplesPerSecond = 3469.7
|
||||
MPI Rank 0: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 851- 860, 67.19%]: CrossEntropyWithSoftmax = 2.58961930 * 640; EvalClassificationError = 0.66718750 * 640; time = 0.1863s; samplesPerSecond = 3435.3
|
||||
MPI Rank 0: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 861- 870, 67.97%]: CrossEntropyWithSoftmax = 2.72924811 * 640; EvalClassificationError = 0.67812500 * 640; time = 0.1852s; samplesPerSecond = 3455.1
|
||||
MPI Rank 0: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 871- 880, 68.75%]: CrossEntropyWithSoftmax = 2.66252872 * 640; EvalClassificationError = 0.66250000 * 640; time = 0.1844s; samplesPerSecond = 3470.9
|
||||
MPI Rank 0: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 881- 890, 69.53%]: CrossEntropyWithSoftmax = 2.52883427 * 640; EvalClassificationError = 0.65937500 * 640; time = 0.1849s; samplesPerSecond = 3461.0
|
||||
MPI Rank 0: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 891- 900, 70.31%]: CrossEntropyWithSoftmax = 2.62228341 * 640; EvalClassificationError = 0.65625000 * 640; time = 0.1852s; samplesPerSecond = 3454.8
|
||||
MPI Rank 0: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 901- 910, 71.09%]: CrossEntropyWithSoftmax = 2.55550779 * 640; EvalClassificationError = 0.66093750 * 640; time = 0.1856s; samplesPerSecond = 3449.0
|
||||
MPI Rank 0: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 911- 920, 71.88%]: CrossEntropyWithSoftmax = 2.55049429 * 640; EvalClassificationError = 0.64531250 * 640; time = 0.1856s; samplesPerSecond = 3448.6
|
||||
MPI Rank 0: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 921- 930, 72.66%]: CrossEntropyWithSoftmax = 2.59920014 * 640; EvalClassificationError = 0.65625000 * 640; time = 0.1854s; samplesPerSecond = 3451.9
|
||||
MPI Rank 0: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 931- 940, 73.44%]: CrossEntropyWithSoftmax = 2.54341577 * 640; EvalClassificationError = 0.62343750 * 640; time = 0.1843s; samplesPerSecond = 3472.8
|
||||
MPI Rank 0: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 941- 950, 74.22%]: CrossEntropyWithSoftmax = 2.48476222 * 640; EvalClassificationError = 0.63750000 * 640; time = 0.1850s; samplesPerSecond = 3459.4
|
||||
MPI Rank 0: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 951- 960, 75.00%]: CrossEntropyWithSoftmax = 2.53015221 * 640; EvalClassificationError = 0.63906250 * 640; time = 0.1869s; samplesPerSecond = 3424.4
|
||||
MPI Rank 0: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 961- 970, 75.78%]: CrossEntropyWithSoftmax = 2.35319566 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1855s; samplesPerSecond = 3450.6
|
||||
MPI Rank 0: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 971- 980, 76.56%]: CrossEntropyWithSoftmax = 2.54683738 * 640; EvalClassificationError = 0.64375000 * 640; time = 0.1856s; samplesPerSecond = 3448.0
|
||||
MPI Rank 0: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[ 981- 990, 77.34%]: CrossEntropyWithSoftmax = 2.45404859 * 640; EvalClassificationError = 0.63750000 * 640; time = 0.1852s; samplesPerSecond = 3456.6
|
||||
MPI Rank 0: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[ 991-1000, 78.13%]: CrossEntropyWithSoftmax = 2.42597335 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1860s; samplesPerSecond = 3440.7
|
||||
MPI Rank 0: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[1001-1010, 78.91%]: CrossEntropyWithSoftmax = 2.37566713 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1854s; samplesPerSecond = 3452.5
|
||||
MPI Rank 0: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[1011-1020, 79.69%]: CrossEntropyWithSoftmax = 2.35902642 * 640; EvalClassificationError = 0.59218750 * 640; time = 0.1852s; samplesPerSecond = 3456.5
|
||||
MPI Rank 0: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[1021-1030, 80.47%]: CrossEntropyWithSoftmax = 2.36171107 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1867s; samplesPerSecond = 3427.8
|
||||
MPI Rank 0: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[1031-1040, 81.25%]: CrossEntropyWithSoftmax = 2.33345715 * 640; EvalClassificationError = 0.57343750 * 640; time = 0.1851s; samplesPerSecond = 3456.7
|
||||
MPI Rank 0: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1041-1050, 82.03%]: CrossEntropyWithSoftmax = 2.44952411 * 640; EvalClassificationError = 0.61875000 * 640; time = 0.1860s; samplesPerSecond = 3441.7
|
||||
MPI Rank 0: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1051-1060, 82.81%]: CrossEntropyWithSoftmax = 2.31665914 * 640; EvalClassificationError = 0.58437500 * 640; time = 0.1854s; samplesPerSecond = 3452.2
|
||||
MPI Rank 0: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1061-1070, 83.59%]: CrossEntropyWithSoftmax = 2.32162968 * 640; EvalClassificationError = 0.61093750 * 640; time = 0.1849s; samplesPerSecond = 3461.5
|
||||
MPI Rank 0: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1071-1080, 84.38%]: CrossEntropyWithSoftmax = 2.48543345 * 640; EvalClassificationError = 0.63593750 * 640; time = 0.1845s; samplesPerSecond = 3469.6
|
||||
MPI Rank 0: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1081-1090, 85.16%]: CrossEntropyWithSoftmax = 2.35574244 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1858s; samplesPerSecond = 3444.5
|
||||
MPI Rank 0: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1091-1100, 85.94%]: CrossEntropyWithSoftmax = 2.38117646 * 640; EvalClassificationError = 0.62656250 * 640; time = 0.1849s; samplesPerSecond = 3462.0
|
||||
MPI Rank 0: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1101-1110, 86.72%]: CrossEntropyWithSoftmax = 2.38366197 * 640; EvalClassificationError = 0.62187500 * 640; time = 0.1860s; samplesPerSecond = 3440.1
|
||||
MPI Rank 0: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1111-1120, 87.50%]: CrossEntropyWithSoftmax = 2.50061273 * 640; EvalClassificationError = 0.63437500 * 640; time = 0.1850s; samplesPerSecond = 3459.6
|
||||
MPI Rank 0: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1121-1130, 88.28%]: CrossEntropyWithSoftmax = 2.19965354 * 640; EvalClassificationError = 0.63125000 * 640; time = 0.1855s; samplesPerSecond = 3450.1
|
||||
MPI Rank 0: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1131-1140, 89.06%]: CrossEntropyWithSoftmax = 2.27232693 * 640; EvalClassificationError = 0.60156250 * 640; time = 0.1850s; samplesPerSecond = 3459.7
|
||||
MPI Rank 0: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1141-1150, 89.84%]: CrossEntropyWithSoftmax = 2.20566415 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1861s; samplesPerSecond = 3439.5
|
||||
MPI Rank 0: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1151-1160, 90.63%]: CrossEntropyWithSoftmax = 2.33120303 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1853s; samplesPerSecond = 3453.0
|
||||
MPI Rank 0: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1161-1170, 91.41%]: CrossEntropyWithSoftmax = 2.29726772 * 640; EvalClassificationError = 0.57812500 * 640; time = 0.1854s; samplesPerSecond = 3452.3
|
||||
MPI Rank 0: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1171-1180, 92.19%]: CrossEntropyWithSoftmax = 2.29349025 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1857s; samplesPerSecond = 3445.9
|
||||
MPI Rank 0: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1181-1190, 92.97%]: CrossEntropyWithSoftmax = 2.30009060 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1850s; samplesPerSecond = 3458.9
|
||||
MPI Rank 0: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1191-1200, 93.75%]: CrossEntropyWithSoftmax = 2.21902385 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1862s; samplesPerSecond = 3436.5
|
||||
MPI Rank 0: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1201-1210, 94.53%]: CrossEntropyWithSoftmax = 2.23739674 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1854s; samplesPerSecond = 3452.9
|
||||
MPI Rank 0: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1211-1220, 95.31%]: CrossEntropyWithSoftmax = 2.25626016 * 640; EvalClassificationError = 0.60625000 * 640; time = 0.1844s; samplesPerSecond = 3470.4
|
||||
MPI Rank 0: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1221-1230, 96.09%]: CrossEntropyWithSoftmax = 2.39720147 * 640; EvalClassificationError = 0.62343750 * 640; time = 0.1859s; samplesPerSecond = 3443.6
|
||||
MPI Rank 0: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1231-1240, 96.88%]: CrossEntropyWithSoftmax = 2.29898501 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1856s; samplesPerSecond = 3449.2
|
||||
MPI Rank 0: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1241-1250, 97.66%]: CrossEntropyWithSoftmax = 2.24887214 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1851s; samplesPerSecond = 3457.7
|
||||
MPI Rank 0: 09/17/2016 18:05:52: Epoch[ 1 of 3]-Minibatch[1251-1260, 98.44%]: CrossEntropyWithSoftmax = 2.27546233 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1856s; samplesPerSecond = 3449.0
|
||||
MPI Rank 0: 09/17/2016 18:05:52: Epoch[ 1 of 3]-Minibatch[1261-1270, 99.22%]: CrossEntropyWithSoftmax = 2.22047744 * 640; EvalClassificationError = 0.60625000 * 640; time = 0.1875s; samplesPerSecond = 3413.2
|
||||
MPI Rank 0: 09/17/2016 18:05:52: Epoch[ 1 of 3]-Minibatch[1271-1280, 100.00%]: CrossEntropyWithSoftmax = 2.30530274 * 640; EvalClassificationError = 0.60000000 * 640; time = 0.1822s; samplesPerSecond = 3512.4
|
||||
MPI Rank 0: 09/17/2016 18:05:52: Finished Epoch[ 1 of 3]: [Training] CrossEntropyWithSoftmax = 3.00766611 * 81920; EvalClassificationError = 0.72410889 * 81920; totalSamplesSeen = 81920; learningRatePerSample = 0.001953125; epochTime=24.1916s
|
||||
MPI Rank 0: 09/17/2016 18:05:53: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/models/cntkSpeech.dnn.1'
|
||||
MPI Rank 0: 09/17/2016 18:05:53: AdaptiveMinibatchSearch Epoch[2]: Evaluating minibatchSizes 64..8192
|
||||
MPI Rank 0: 09/17/2016 18:05:53: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=64 (search range: 64..8192)...
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:05:53: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:05:53: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 2.02060591 * 640; EvalClassificationError = 0.52343750 * 640; time = 0.1986s; samplesPerSecond = 3222.4
|
||||
MPI Rank 0: 09/17/2016 18:05:53: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 2.04232000 * 640; EvalClassificationError = 0.53437500 * 640; time = 0.1851s; samplesPerSecond = 3457.0
|
||||
MPI Rank 0: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 2.14042914 * 640; EvalClassificationError = 0.59843750 * 640; time = 0.1850s; samplesPerSecond = 3459.6
|
||||
MPI Rank 0: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 2.12163859 * 640; EvalClassificationError = 0.56718750 * 640; time = 0.1843s; samplesPerSecond = 3471.9
|
||||
MPI Rank 0: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 2.05390998 * 640; EvalClassificationError = 0.55625000 * 640; time = 0.1852s; samplesPerSecond = 3456.3
|
||||
MPI Rank 0: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 2.09859542 * 640; EvalClassificationError = 0.55156250 * 640; time = 0.1849s; samplesPerSecond = 3460.6
|
||||
MPI Rank 0: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 2.11796400 * 640; EvalClassificationError = 0.56406250 * 640; time = 0.1892s; samplesPerSecond = 3381.9
|
||||
MPI Rank 0: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 2.11659597 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1861s; samplesPerSecond = 3438.2
|
||||
MPI Rank 0: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 2.14159740 * 640; EvalClassificationError = 0.58125000 * 640; time = 0.1851s; samplesPerSecond = 3457.0
|
||||
MPI Rank 0: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 2.13426067 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1845s; samplesPerSecond = 3469.8
|
||||
MPI Rank 0: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.98173765 * 640; EvalClassificationError = 0.54687500 * 640; time = 0.1848s; samplesPerSecond = 3463.3
|
||||
MPI Rank 0: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 2.18123085 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1854s; samplesPerSecond = 3452.1
|
||||
MPI Rank 0: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 2.01620029 * 640; EvalClassificationError = 0.53281250 * 640; time = 0.1844s; samplesPerSecond = 3471.1
|
||||
MPI Rank 0: 09/17/2016 18:05:56: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 2.02410759 * 640; EvalClassificationError = 0.55312500 * 640; time = 0.1847s; samplesPerSecond = 3465.4
|
||||
MPI Rank 0: 09/17/2016 18:05:56: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 2.03003578 * 640; EvalClassificationError = 0.51250000 * 640; time = 0.1845s; samplesPerSecond = 3468.3
|
||||
MPI Rank 0: 09/17/2016 18:05:56: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 2.07643576 * 640; EvalClassificationError = 0.58281250 * 640; time = 0.1820s; samplesPerSecond = 3517.2
|
||||
MPI Rank 0: 09/17/2016 18:05:56: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.08110406 * 10240; EvalClassificationError = 0.56123047 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:05:56: AdaptiveMinibatchSearch Epoch[2]: Computed baseCriterion 2.08110406 for minibatchSize=64
|
||||
MPI Rank 0: 09/17/2016 18:05:56: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=64 (search range: 64..8192)...
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:05:56: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:05:56: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 2.02060591 * 640; EvalClassificationError = 0.52343750 * 640; time = 0.1834s; samplesPerSecond = 3488.7
|
||||
MPI Rank 0: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 2.04232000 * 640; EvalClassificationError = 0.53437500 * 640; time = 0.1856s; samplesPerSecond = 3447.8
|
||||
MPI Rank 0: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 2.14042914 * 640; EvalClassificationError = 0.59843750 * 640; time = 0.1848s; samplesPerSecond = 3463.0
|
||||
MPI Rank 0: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 2.12163859 * 640; EvalClassificationError = 0.56718750 * 640; time = 0.1849s; samplesPerSecond = 3460.5
|
||||
MPI Rank 0: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 2.05390998 * 640; EvalClassificationError = 0.55625000 * 640; time = 0.1854s; samplesPerSecond = 3452.1
|
||||
MPI Rank 0: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 2.09859542 * 640; EvalClassificationError = 0.55156250 * 640; time = 0.1844s; samplesPerSecond = 3471.3
|
||||
MPI Rank 0: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 2.11796400 * 640; EvalClassificationError = 0.56406250 * 640; time = 0.1854s; samplesPerSecond = 3451.6
|
||||
MPI Rank 0: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 2.11659597 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1857s; samplesPerSecond = 3447.2
|
||||
MPI Rank 0: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 2.14159740 * 640; EvalClassificationError = 0.58125000 * 640; time = 0.1855s; samplesPerSecond = 3449.5
|
||||
MPI Rank 0: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 2.13426067 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1845s; samplesPerSecond = 3469.4
|
||||
MPI Rank 0: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.98173765 * 640; EvalClassificationError = 0.54687500 * 640; time = 0.1846s; samplesPerSecond = 3466.7
|
||||
MPI Rank 0: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 2.18123085 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1853s; samplesPerSecond = 3454.6
|
||||
MPI Rank 0: 09/17/2016 18:05:59: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 2.01620029 * 640; EvalClassificationError = 0.53281250 * 640; time = 0.1846s; samplesPerSecond = 3467.8
|
||||
MPI Rank 0: 09/17/2016 18:05:59: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 2.02410759 * 640; EvalClassificationError = 0.55312500 * 640; time = 0.1849s; samplesPerSecond = 3461.6
|
||||
MPI Rank 0: 09/17/2016 18:05:59: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 2.03003578 * 640; EvalClassificationError = 0.51250000 * 640; time = 0.1849s; samplesPerSecond = 3461.6
|
||||
MPI Rank 0: 09/17/2016 18:05:59: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 2.07643576 * 640; EvalClassificationError = 0.58281250 * 640; time = 0.1830s; samplesPerSecond = 3497.4
|
||||
MPI Rank 0: 09/17/2016 18:05:59: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.08110406 * 10240; EvalClassificationError = 0.56123047 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:05:59: AdaptiveMinibatchSearch Epoch[2]: Keep searching... epochCriterion = 2.08110406 vs. baseCriterion = 2.08110406
|
||||
MPI Rank 0: 09/17/2016 18:05:59: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=128 (search range: 64..8192)...
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:05:59: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:00: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 2.02947755 * 1280; EvalClassificationError = 0.52656250 * 1280; time = 0.2498s; samplesPerSecond = 5124.1
|
||||
MPI Rank 0: 09/17/2016 18:06:00: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 3.13%]: CrossEntropyWithSoftmax = 2.13334208 * 1280; EvalClassificationError = 0.58359375 * 1280; time = 0.2424s; samplesPerSecond = 5281.1
|
||||
MPI Rank 0: 09/17/2016 18:06:00: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 2.08344475 * 1280; EvalClassificationError = 0.55781250 * 1280; time = 0.2433s; samplesPerSecond = 5260.2
|
||||
MPI Rank 0: 09/17/2016 18:06:00: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 2.13081897 * 1280; EvalClassificationError = 0.56875000 * 1280; time = 0.2460s; samplesPerSecond = 5202.7
|
||||
MPI Rank 0: 09/17/2016 18:06:01: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 2.15740742 * 1280; EvalClassificationError = 0.58984375 * 1280; time = 0.2425s; samplesPerSecond = 5277.3
|
||||
MPI Rank 0: 09/17/2016 18:06:01: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 2.11081282 * 1280; EvalClassificationError = 0.58125000 * 1280; time = 0.2429s; samplesPerSecond = 5269.4
|
||||
MPI Rank 0: 09/17/2016 18:06:01: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 2.03640631 * 1280; EvalClassificationError = 0.55546875 * 1280; time = 0.2427s; samplesPerSecond = 5273.4
|
||||
MPI Rank 0: 09/17/2016 18:06:01: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 2.06206717 * 1280; EvalClassificationError = 0.55156250 * 1280; time = 0.2379s; samplesPerSecond = 5379.5
|
||||
MPI Rank 0: 09/17/2016 18:06:01: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.09297213 * 10240; EvalClassificationError = 0.56435547 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 128
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:02: AdaptiveMinibatchSearch Epoch[2]: Keep searching... epochCriterion = 2.09297213 vs. baseCriterion = 2.08110406
|
||||
MPI Rank 0: 09/17/2016 18:06:02: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=192 (search range: 64..8192)...
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:02: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:02: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 2.07040051 * 1920; EvalClassificationError = 0.54947917 * 1920; time = 0.3196s; samplesPerSecond = 6007.0
|
||||
MPI Rank 0: 09/17/2016 18:06:02: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 2.12004909 * 1920; EvalClassificationError = 0.56770833 * 1920; time = 0.3045s; samplesPerSecond = 6304.6
|
||||
MPI Rank 0: 09/17/2016 18:06:03: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 2.19270926 * 1920; EvalClassificationError = 0.58750000 * 1920; time = 0.3111s; samplesPerSecond = 6172.1
|
||||
MPI Rank 0: 09/17/2016 18:06:03: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 2.25179093 * 1920; EvalClassificationError = 0.60781250 * 1920; time = 0.3060s; samplesPerSecond = 6274.9
|
||||
MPI Rank 0: 09/17/2016 18:06:03: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 2.45413223 * 1920; EvalClassificationError = 0.59531250 * 1920; time = 0.3095s; samplesPerSecond = 6204.1
|
||||
MPI Rank 0: 09/17/2016 18:06:03: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.23761477 * 10240; EvalClassificationError = 0.58447266 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 192
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:03: AdaptiveMinibatchSearch Epoch[2]: Search successful. New minibatchSize is 128. epochCriterion = 2.09297213 vs baseCriterion = 2.08110406
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:03: Starting Epoch 2: learning rate per sample = 0.001953 effective momentum = 0.882497 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [81920..163840] (first utterance at frame 81920), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:03: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:04: Epoch[ 2 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 2.18100617 * 1280; EvalClassificationError = 0.55546875 * 1280; time = 0.2440s; samplesPerSecond = 5244.9
|
||||
MPI Rank 0: 09/17/2016 18:06:04: Epoch[ 2 of 3]-Minibatch[ 11- 20, 3.13%]: CrossEntropyWithSoftmax = 2.16631393 * 1280; EvalClassificationError = 0.59062500 * 1280; time = 0.2467s; samplesPerSecond = 5187.8
|
||||
MPI Rank 0: 09/17/2016 18:06:04: Epoch[ 2 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 2.23799285 * 1280; EvalClassificationError = 0.60468750 * 1280; time = 0.2471s; samplesPerSecond = 5180.4
|
||||
MPI Rank 0: 09/17/2016 18:06:04: Epoch[ 2 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 2.25188312 * 1280; EvalClassificationError = 0.60703125 * 1280; time = 0.2494s; samplesPerSecond = 5131.9
|
||||
MPI Rank 0: 09/17/2016 18:06:05: Epoch[ 2 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 2.12738463 * 1280; EvalClassificationError = 0.56328125 * 1280; time = 0.2495s; samplesPerSecond = 5129.6
|
||||
MPI Rank 0: 09/17/2016 18:06:05: Epoch[ 2 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 2.09543741 * 1280; EvalClassificationError = 0.58359375 * 1280; time = 0.2473s; samplesPerSecond = 5175.1
|
||||
MPI Rank 0: 09/17/2016 18:06:05: Epoch[ 2 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 2.12457852 * 1280; EvalClassificationError = 0.58906250 * 1280; time = 0.2477s; samplesPerSecond = 5167.1
|
||||
MPI Rank 0: 09/17/2016 18:06:05: Epoch[ 2 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 2.15260337 * 1280; EvalClassificationError = 0.57343750 * 1280; time = 0.2497s; samplesPerSecond = 5126.1
|
||||
MPI Rank 0: 09/17/2016 18:06:06: Epoch[ 2 of 3]-Minibatch[ 81- 90, 14.06%]: CrossEntropyWithSoftmax = 2.07975382 * 1280; EvalClassificationError = 0.55312500 * 1280; time = 0.2464s; samplesPerSecond = 5195.4
|
||||
MPI Rank 0: 09/17/2016 18:06:06: Epoch[ 2 of 3]-Minibatch[ 91- 100, 15.63%]: CrossEntropyWithSoftmax = 2.09557893 * 1280; EvalClassificationError = 0.56328125 * 1280; time = 0.2431s; samplesPerSecond = 5265.6
|
||||
MPI Rank 0: 09/17/2016 18:06:06: Epoch[ 2 of 3]-Minibatch[ 101- 110, 17.19%]: CrossEntropyWithSoftmax = 1.99564992 * 1280; EvalClassificationError = 0.54218750 * 1280; time = 0.2452s; samplesPerSecond = 5221.0
|
||||
MPI Rank 0: 09/17/2016 18:06:06: Epoch[ 2 of 3]-Minibatch[ 111- 120, 18.75%]: CrossEntropyWithSoftmax = 2.01696230 * 1280; EvalClassificationError = 0.53437500 * 1280; time = 0.2441s; samplesPerSecond = 5243.9
|
||||
MPI Rank 0: 09/17/2016 18:06:07: Epoch[ 2 of 3]-Minibatch[ 121- 130, 20.31%]: CrossEntropyWithSoftmax = 2.08247499 * 1280; EvalClassificationError = 0.55625000 * 1280; time = 0.2480s; samplesPerSecond = 5160.7
|
||||
MPI Rank 0: 09/17/2016 18:06:07: Epoch[ 2 of 3]-Minibatch[ 131- 140, 21.88%]: CrossEntropyWithSoftmax = 2.02583127 * 1280; EvalClassificationError = 0.55781250 * 1280; time = 0.2461s; samplesPerSecond = 5201.9
|
||||
MPI Rank 0: 09/17/2016 18:06:07: Epoch[ 2 of 3]-Minibatch[ 141- 150, 23.44%]: CrossEntropyWithSoftmax = 2.12427634 * 1280; EvalClassificationError = 0.57031250 * 1280; time = 0.2456s; samplesPerSecond = 5212.3
|
||||
MPI Rank 0: 09/17/2016 18:06:07: Epoch[ 2 of 3]-Minibatch[ 151- 160, 25.00%]: CrossEntropyWithSoftmax = 1.95297386 * 1280; EvalClassificationError = 0.55234375 * 1280; time = 0.2465s; samplesPerSecond = 5192.2
|
||||
MPI Rank 0: 09/17/2016 18:06:08: Epoch[ 2 of 3]-Minibatch[ 161- 170, 26.56%]: CrossEntropyWithSoftmax = 2.06940792 * 1280; EvalClassificationError = 0.57968750 * 1280; time = 0.2474s; samplesPerSecond = 5173.1
|
||||
MPI Rank 0: 09/17/2016 18:06:08: Epoch[ 2 of 3]-Minibatch[ 171- 180, 28.13%]: CrossEntropyWithSoftmax = 1.97910584 * 1280; EvalClassificationError = 0.53281250 * 1280; time = 0.2455s; samplesPerSecond = 5214.8
|
||||
MPI Rank 0: 09/17/2016 18:06:08: Epoch[ 2 of 3]-Minibatch[ 181- 190, 29.69%]: CrossEntropyWithSoftmax = 1.97550728 * 1280; EvalClassificationError = 0.56484375 * 1280; time = 0.2442s; samplesPerSecond = 5241.1
|
||||
MPI Rank 0: 09/17/2016 18:06:08: Epoch[ 2 of 3]-Minibatch[ 191- 200, 31.25%]: CrossEntropyWithSoftmax = 2.07046879 * 1280; EvalClassificationError = 0.58671875 * 1280; time = 0.2433s; samplesPerSecond = 5260.8
|
||||
MPI Rank 0: 09/17/2016 18:06:09: Epoch[ 2 of 3]-Minibatch[ 201- 210, 32.81%]: CrossEntropyWithSoftmax = 1.94086640 * 1280; EvalClassificationError = 0.54609375 * 1280; time = 0.2469s; samplesPerSecond = 5183.7
|
||||
MPI Rank 0: 09/17/2016 18:06:09: Epoch[ 2 of 3]-Minibatch[ 211- 220, 34.38%]: CrossEntropyWithSoftmax = 1.88656971 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.2440s; samplesPerSecond = 5246.0
|
||||
MPI Rank 0: 09/17/2016 18:06:09: Epoch[ 2 of 3]-Minibatch[ 221- 230, 35.94%]: CrossEntropyWithSoftmax = 1.90888794 * 1280; EvalClassificationError = 0.51640625 * 1280; time = 0.2453s; samplesPerSecond = 5219.2
|
||||
MPI Rank 0: 09/17/2016 18:06:09: Epoch[ 2 of 3]-Minibatch[ 231- 240, 37.50%]: CrossEntropyWithSoftmax = 1.91336087 * 1280; EvalClassificationError = 0.52265625 * 1280; time = 0.2475s; samplesPerSecond = 5171.4
|
||||
MPI Rank 0: 09/17/2016 18:06:10: Epoch[ 2 of 3]-Minibatch[ 241- 250, 39.06%]: CrossEntropyWithSoftmax = 1.89548772 * 1280; EvalClassificationError = 0.52343750 * 1280; time = 0.2480s; samplesPerSecond = 5162.0
|
||||
MPI Rank 0: 09/17/2016 18:06:10: Epoch[ 2 of 3]-Minibatch[ 251- 260, 40.63%]: CrossEntropyWithSoftmax = 1.87667719 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.2459s; samplesPerSecond = 5205.1
|
||||
MPI Rank 0: 09/17/2016 18:06:10: Epoch[ 2 of 3]-Minibatch[ 261- 270, 42.19%]: CrossEntropyWithSoftmax = 1.81964097 * 1280; EvalClassificationError = 0.52421875 * 1280; time = 0.2439s; samplesPerSecond = 5247.0
|
||||
MPI Rank 0: 09/17/2016 18:06:10: Epoch[ 2 of 3]-Minibatch[ 271- 280, 43.75%]: CrossEntropyWithSoftmax = 1.88226903 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.2440s; samplesPerSecond = 5245.7
|
||||
MPI Rank 0: 09/17/2016 18:06:11: Epoch[ 2 of 3]-Minibatch[ 281- 290, 45.31%]: CrossEntropyWithSoftmax = 1.85004922 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.2462s; samplesPerSecond = 5199.8
|
||||
MPI Rank 0: 09/17/2016 18:06:11: Epoch[ 2 of 3]-Minibatch[ 291- 300, 46.88%]: CrossEntropyWithSoftmax = 1.78267871 * 1280; EvalClassificationError = 0.50156250 * 1280; time = 0.2473s; samplesPerSecond = 5176.7
|
||||
MPI Rank 0: 09/17/2016 18:06:11: Epoch[ 2 of 3]-Minibatch[ 301- 310, 48.44%]: CrossEntropyWithSoftmax = 1.81674744 * 1280; EvalClassificationError = 0.50468750 * 1280; time = 0.2445s; samplesPerSecond = 5234.8
|
||||
MPI Rank 0: 09/17/2016 18:06:11: Epoch[ 2 of 3]-Minibatch[ 311- 320, 50.00%]: CrossEntropyWithSoftmax = 1.76854193 * 1280; EvalClassificationError = 0.49062500 * 1280; time = 0.2446s; samplesPerSecond = 5233.4
|
||||
MPI Rank 0: 09/17/2016 18:06:12: Epoch[ 2 of 3]-Minibatch[ 321- 330, 51.56%]: CrossEntropyWithSoftmax = 1.81330268 * 1280; EvalClassificationError = 0.51171875 * 1280; time = 0.2445s; samplesPerSecond = 5235.6
|
||||
MPI Rank 0: 09/17/2016 18:06:12: Epoch[ 2 of 3]-Minibatch[ 331- 340, 53.13%]: CrossEntropyWithSoftmax = 1.81775001 * 1280; EvalClassificationError = 0.51640625 * 1280; time = 0.2430s; samplesPerSecond = 5266.9
|
||||
MPI Rank 0: 09/17/2016 18:06:12: Epoch[ 2 of 3]-Minibatch[ 341- 350, 54.69%]: CrossEntropyWithSoftmax = 1.79768261 * 1280; EvalClassificationError = 0.50546875 * 1280; time = 0.2464s; samplesPerSecond = 5194.5
|
||||
MPI Rank 0: 09/17/2016 18:06:12: Epoch[ 2 of 3]-Minibatch[ 351- 360, 56.25%]: CrossEntropyWithSoftmax = 1.78876937 * 1280; EvalClassificationError = 0.49765625 * 1280; time = 0.2437s; samplesPerSecond = 5253.1
|
||||
MPI Rank 0: 09/17/2016 18:06:13: Epoch[ 2 of 3]-Minibatch[ 361- 370, 57.81%]: CrossEntropyWithSoftmax = 1.78753263 * 1280; EvalClassificationError = 0.51015625 * 1280; time = 0.2444s; samplesPerSecond = 5236.9
|
||||
MPI Rank 0: 09/17/2016 18:06:13: Epoch[ 2 of 3]-Minibatch[ 371- 380, 59.38%]: CrossEntropyWithSoftmax = 1.74071233 * 1280; EvalClassificationError = 0.49140625 * 1280; time = 0.2435s; samplesPerSecond = 5256.0
|
||||
MPI Rank 0: 09/17/2016 18:06:13: Epoch[ 2 of 3]-Minibatch[ 381- 390, 60.94%]: CrossEntropyWithSoftmax = 1.71575901 * 1280; EvalClassificationError = 0.46875000 * 1280; time = 0.2437s; samplesPerSecond = 5253.0
|
||||
MPI Rank 0: 09/17/2016 18:06:13: Epoch[ 2 of 3]-Minibatch[ 391- 400, 62.50%]: CrossEntropyWithSoftmax = 1.76465781 * 1280; EvalClassificationError = 0.49765625 * 1280; time = 0.2450s; samplesPerSecond = 5225.4
|
||||
MPI Rank 0: 09/17/2016 18:06:14: Epoch[ 2 of 3]-Minibatch[ 401- 410, 64.06%]: CrossEntropyWithSoftmax = 1.76532949 * 1280; EvalClassificationError = 0.51406250 * 1280; time = 0.2468s; samplesPerSecond = 5186.6
|
||||
MPI Rank 0: 09/17/2016 18:06:14: Epoch[ 2 of 3]-Minibatch[ 411- 420, 65.63%]: CrossEntropyWithSoftmax = 1.79718711 * 1280; EvalClassificationError = 0.50390625 * 1280; time = 0.2419s; samplesPerSecond = 5291.8
|
||||
MPI Rank 0: 09/17/2016 18:06:14: Epoch[ 2 of 3]-Minibatch[ 421- 430, 67.19%]: CrossEntropyWithSoftmax = 1.74168655 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.2422s; samplesPerSecond = 5284.9
|
||||
MPI Rank 0: 09/17/2016 18:06:14: Epoch[ 2 of 3]-Minibatch[ 431- 440, 68.75%]: CrossEntropyWithSoftmax = 1.73594884 * 1280; EvalClassificationError = 0.49609375 * 1280; time = 0.2532s; samplesPerSecond = 5056.2
|
||||
MPI Rank 0: 09/17/2016 18:06:15: Epoch[ 2 of 3]-Minibatch[ 441- 450, 70.31%]: CrossEntropyWithSoftmax = 1.75234022 * 1280; EvalClassificationError = 0.50859375 * 1280; time = 0.2429s; samplesPerSecond = 5269.4
|
||||
MPI Rank 0: 09/17/2016 18:06:15: Epoch[ 2 of 3]-Minibatch[ 451- 460, 71.88%]: CrossEntropyWithSoftmax = 1.64950906 * 1280; EvalClassificationError = 0.48281250 * 1280; time = 0.2427s; samplesPerSecond = 5274.9
|
||||
MPI Rank 0: 09/17/2016 18:06:15: Epoch[ 2 of 3]-Minibatch[ 461- 470, 73.44%]: CrossEntropyWithSoftmax = 1.72111861 * 1280; EvalClassificationError = 0.49921875 * 1280; time = 0.2444s; samplesPerSecond = 5238.0
|
||||
MPI Rank 0: 09/17/2016 18:06:15: Epoch[ 2 of 3]-Minibatch[ 471- 480, 75.00%]: CrossEntropyWithSoftmax = 1.75491334 * 1280; EvalClassificationError = 0.50312500 * 1280; time = 0.2454s; samplesPerSecond = 5216.0
|
||||
MPI Rank 0: 09/17/2016 18:06:16: Epoch[ 2 of 3]-Minibatch[ 481- 490, 76.56%]: CrossEntropyWithSoftmax = 1.68324400 * 1280; EvalClassificationError = 0.48281250 * 1280; time = 0.2438s; samplesPerSecond = 5249.3
|
||||
MPI Rank 0: 09/17/2016 18:06:16: Epoch[ 2 of 3]-Minibatch[ 491- 500, 78.13%]: CrossEntropyWithSoftmax = 1.71507576 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.2439s; samplesPerSecond = 5248.0
|
||||
MPI Rank 0: 09/17/2016 18:06:16: Epoch[ 2 of 3]-Minibatch[ 501- 510, 79.69%]: CrossEntropyWithSoftmax = 1.65489209 * 1280; EvalClassificationError = 0.45937500 * 1280; time = 0.2467s; samplesPerSecond = 5187.5
|
||||
MPI Rank 0: 09/17/2016 18:06:16: Epoch[ 2 of 3]-Minibatch[ 511- 520, 81.25%]: CrossEntropyWithSoftmax = 1.70993974 * 1280; EvalClassificationError = 0.48984375 * 1280; time = 0.2428s; samplesPerSecond = 5271.7
|
||||
MPI Rank 0: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 521- 530, 82.81%]: CrossEntropyWithSoftmax = 1.68373330 * 1280; EvalClassificationError = 0.48046875 * 1280; time = 0.2442s; samplesPerSecond = 5241.0
|
||||
MPI Rank 0: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 531- 540, 84.38%]: CrossEntropyWithSoftmax = 1.68961559 * 1280; EvalClassificationError = 0.48671875 * 1280; time = 0.2448s; samplesPerSecond = 5228.6
|
||||
MPI Rank 0: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 541- 550, 85.94%]: CrossEntropyWithSoftmax = 1.70437375 * 1280; EvalClassificationError = 0.48984375 * 1280; time = 0.2437s; samplesPerSecond = 5253.2
|
||||
MPI Rank 0: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 551- 560, 87.50%]: CrossEntropyWithSoftmax = 1.69558061 * 1280; EvalClassificationError = 0.48906250 * 1280; time = 0.2422s; samplesPerSecond = 5285.9
|
||||
MPI Rank 0: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 561- 570, 89.06%]: CrossEntropyWithSoftmax = 1.69535392 * 1280; EvalClassificationError = 0.48359375 * 1280; time = 0.2427s; samplesPerSecond = 5273.6
|
||||
MPI Rank 0: 09/17/2016 18:06:18: Epoch[ 2 of 3]-Minibatch[ 571- 580, 90.63%]: CrossEntropyWithSoftmax = 1.65016334 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.2431s; samplesPerSecond = 5265.7
|
||||
MPI Rank 0: 09/17/2016 18:06:18: Epoch[ 2 of 3]-Minibatch[ 581- 590, 92.19%]: CrossEntropyWithSoftmax = 1.64953906 * 1280; EvalClassificationError = 0.48515625 * 1280; time = 0.2406s; samplesPerSecond = 5320.0
|
||||
MPI Rank 0: 09/17/2016 18:06:18: Epoch[ 2 of 3]-Minibatch[ 591- 600, 93.75%]: CrossEntropyWithSoftmax = 1.64390878 * 1280; EvalClassificationError = 0.48125000 * 1280; time = 0.2437s; samplesPerSecond = 5251.9
|
||||
MPI Rank 0: 09/17/2016 18:06:18: Epoch[ 2 of 3]-Minibatch[ 601- 610, 95.31%]: CrossEntropyWithSoftmax = 1.71970142 * 1280; EvalClassificationError = 0.51093750 * 1280; time = 0.2409s; samplesPerSecond = 5313.8
|
||||
MPI Rank 0: 09/17/2016 18:06:19: Epoch[ 2 of 3]-Minibatch[ 611- 620, 96.88%]: CrossEntropyWithSoftmax = 1.62454036 * 1280; EvalClassificationError = 0.46562500 * 1280; time = 0.2426s; samplesPerSecond = 5275.8
|
||||
MPI Rank 0: 09/17/2016 18:06:19: Epoch[ 2 of 3]-Minibatch[ 621- 630, 98.44%]: CrossEntropyWithSoftmax = 1.59147885 * 1280; EvalClassificationError = 0.46875000 * 1280; time = 0.2424s; samplesPerSecond = 5281.2
|
||||
MPI Rank 0: 09/17/2016 18:06:19: Epoch[ 2 of 3]-Minibatch[ 631- 640, 100.00%]: CrossEntropyWithSoftmax = 1.66373476 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.2351s; samplesPerSecond = 5444.5
|
||||
MPI Rank 0: 09/17/2016 18:06:19: Finished Epoch[ 2 of 3]: [Training] CrossEntropyWithSoftmax = 1.86040693 * 81920; EvalClassificationError = 0.51970215 * 81920; totalSamplesSeen = 163840; learningRatePerSample = 0.001953125; epochTime=26.1883s
|
||||
MPI Rank 0: 09/17/2016 18:06:19: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/models/cntkSpeech.dnn.2'
|
||||
MPI Rank 0: 09/17/2016 18:06:19: AdaptiveMinibatchSearch Epoch[3]: Evaluating minibatchSizes 64..256
|
||||
MPI Rank 0: 09/17/2016 18:06:19: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=64 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:19: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 1.60900971 * 640; EvalClassificationError = 0.46875000 * 640; time = 0.1842s; samplesPerSecond = 3475.3
|
||||
MPI Rank 0: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 1.70269498 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1847s; samplesPerSecond = 3465.5
|
||||
MPI Rank 0: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 1.66433597 * 640; EvalClassificationError = 0.48281250 * 640; time = 0.1850s; samplesPerSecond = 3460.2
|
||||
MPI Rank 0: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 1.56997722 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1839s; samplesPerSecond = 3479.7
|
||||
MPI Rank 0: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 1.67908876 * 640; EvalClassificationError = 0.47656250 * 640; time = 0.1851s; samplesPerSecond = 3457.4
|
||||
MPI Rank 0: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 1.45189750 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1857s; samplesPerSecond = 3447.1
|
||||
MPI Rank 0: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 1.60053473 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1846s; samplesPerSecond = 3467.2
|
||||
MPI Rank 0: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 1.58536943 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1852s; samplesPerSecond = 3455.8
|
||||
MPI Rank 0: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 1.63246860 * 640; EvalClassificationError = 0.45781250 * 640; time = 0.1839s; samplesPerSecond = 3480.9
|
||||
MPI Rank 0: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 1.64770593 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1851s; samplesPerSecond = 3457.1
|
||||
MPI Rank 0: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.55685563 * 640; EvalClassificationError = 0.43750000 * 640; time = 0.1854s; samplesPerSecond = 3451.2
|
||||
MPI Rank 0: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 1.54493706 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1857s; samplesPerSecond = 3446.3
|
||||
MPI Rank 0: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 1.59221509 * 640; EvalClassificationError = 0.47031250 * 640; time = 0.1845s; samplesPerSecond = 3468.9
|
||||
MPI Rank 0: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 1.59923263 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1848s; samplesPerSecond = 3462.6
|
||||
MPI Rank 0: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 1.60345937 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1842s; samplesPerSecond = 3474.8
|
||||
MPI Rank 0: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 1.59599131 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1812s; samplesPerSecond = 3531.7
|
||||
MPI Rank 0: 09/17/2016 18:06:22: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60223587 * 10240; EvalClassificationError = 0.45957031 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:23: AdaptiveMinibatchSearch Epoch[3]: Computed baseCriterion 1.60223587 for minibatchSize=64
|
||||
MPI Rank 0: 09/17/2016 18:06:23: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=64 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:23: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 1.60900971 * 640; EvalClassificationError = 0.46875000 * 640; time = 0.1838s; samplesPerSecond = 3482.4
|
||||
MPI Rank 0: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 1.70269498 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1843s; samplesPerSecond = 3472.0
|
||||
MPI Rank 0: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 1.66433597 * 640; EvalClassificationError = 0.48281250 * 640; time = 0.1852s; samplesPerSecond = 3455.9
|
||||
MPI Rank 0: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 1.56997722 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1838s; samplesPerSecond = 3481.1
|
||||
MPI Rank 0: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 1.67908876 * 640; EvalClassificationError = 0.47656250 * 640; time = 0.1846s; samplesPerSecond = 3466.2
|
||||
MPI Rank 0: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 1.45189750 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1854s; samplesPerSecond = 3451.6
|
||||
MPI Rank 0: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 1.60053473 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1838s; samplesPerSecond = 3482.1
|
||||
MPI Rank 0: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 1.58536943 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1856s; samplesPerSecond = 3448.7
|
||||
MPI Rank 0: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 1.63246860 * 640; EvalClassificationError = 0.45781250 * 640; time = 0.1841s; samplesPerSecond = 3477.0
|
||||
MPI Rank 0: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 1.64770593 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1851s; samplesPerSecond = 3457.2
|
||||
MPI Rank 0: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.55685563 * 640; EvalClassificationError = 0.43750000 * 640; time = 0.1864s; samplesPerSecond = 3432.9
|
||||
MPI Rank 0: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 1.54493706 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1847s; samplesPerSecond = 3465.4
|
||||
MPI Rank 0: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 1.59221509 * 640; EvalClassificationError = 0.47031250 * 640; time = 0.1846s; samplesPerSecond = 3467.3
|
||||
MPI Rank 0: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 1.59923263 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1856s; samplesPerSecond = 3447.8
|
||||
MPI Rank 0: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 1.60345937 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1846s; samplesPerSecond = 3467.0
|
||||
MPI Rank 0: 09/17/2016 18:06:26: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 1.59599131 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1827s; samplesPerSecond = 3502.5
|
||||
MPI Rank 0: 09/17/2016 18:06:26: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60223587 * 10240; EvalClassificationError = 0.45957031 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:26: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.60223587 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 0: 09/17/2016 18:06:26: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=128 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:26: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:26: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 1.59906005 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.2435s; samplesPerSecond = 5256.7
|
||||
MPI Rank 0: 09/17/2016 18:06:26: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 3.13%]: CrossEntropyWithSoftmax = 1.59088280 * 1280; EvalClassificationError = 0.47031250 * 1280; time = 0.2427s; samplesPerSecond = 5273.2
|
||||
MPI Rank 0: 09/17/2016 18:06:26: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 1.55611991 * 1280; EvalClassificationError = 0.45468750 * 1280; time = 0.2462s; samplesPerSecond = 5198.5
|
||||
MPI Rank 0: 09/17/2016 18:06:27: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 1.59494809 * 1280; EvalClassificationError = 0.45937500 * 1280; time = 0.2441s; samplesPerSecond = 5244.1
|
||||
MPI Rank 0: 09/17/2016 18:06:27: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 1.64834247 * 1280; EvalClassificationError = 0.45703125 * 1280; time = 0.2435s; samplesPerSecond = 5257.4
|
||||
MPI Rank 0: 09/17/2016 18:06:27: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 1.56066565 * 1280; EvalClassificationError = 0.43593750 * 1280; time = 0.2437s; samplesPerSecond = 5253.0
|
||||
MPI Rank 0: 09/17/2016 18:06:27: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 1.62413186 * 1280; EvalClassificationError = 0.47265625 * 1280; time = 0.2458s; samplesPerSecond = 5206.7
|
||||
MPI Rank 0: 09/17/2016 18:06:28: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 1.67799703 * 1280; EvalClassificationError = 0.47031250 * 1280; time = 0.2385s; samplesPerSecond = 5366.0
|
||||
MPI Rank 0: 09/17/2016 18:06:28: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60651848 * 10240; EvalClassificationError = 0.46044922 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 128
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:28: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.60651848 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 0: 09/17/2016 18:06:28: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=192 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:28: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:28: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 1.59151349 * 1920; EvalClassificationError = 0.46302083 * 1920; time = 0.3095s; samplesPerSecond = 6203.0
|
||||
MPI Rank 0: 09/17/2016 18:06:29: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 1.55305006 * 1920; EvalClassificationError = 0.45000000 * 1920; time = 0.3119s; samplesPerSecond = 6156.5
|
||||
MPI Rank 0: 09/17/2016 18:06:29: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 1.60654814 * 1920; EvalClassificationError = 0.45781250 * 1920; time = 0.3121s; samplesPerSecond = 6151.5
|
||||
MPI Rank 0: 09/17/2016 18:06:29: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 1.60526168 * 1920; EvalClassificationError = 0.44947917 * 1920; time = 0.3090s; samplesPerSecond = 6214.1
|
||||
MPI Rank 0: 09/17/2016 18:06:29: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 1.68437187 * 1920; EvalClassificationError = 0.49531250 * 1920; time = 0.3134s; samplesPerSecond = 6126.3
|
||||
MPI Rank 0: 09/17/2016 18:06:30: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.61407118 * 10240; EvalClassificationError = 0.46357422 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 192
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:30: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.61407118 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 0: 09/17/2016 18:06:30: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=256 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:30: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:30: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 3.13%]: CrossEntropyWithSoftmax = 1.59241097 * 2560; EvalClassificationError = 0.46093750 * 2560; time = 0.3828s; samplesPerSecond = 6687.9
|
||||
MPI Rank 0: 09/17/2016 18:06:31: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 6.25%]: CrossEntropyWithSoftmax = 1.59578299 * 2560; EvalClassificationError = 0.46093750 * 2560; time = 0.3757s; samplesPerSecond = 6814.8
|
||||
MPI Rank 0: 09/17/2016 18:06:31: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 9.38%]: CrossEntropyWithSoftmax = 1.68033748 * 2560; EvalClassificationError = 0.47343750 * 2560; time = 0.3717s; samplesPerSecond = 6887.4
|
||||
MPI Rank 0: 09/17/2016 18:06:31: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 12.50%]: CrossEntropyWithSoftmax = 1.73124308 * 2560; EvalClassificationError = 0.48164062 * 2560; time = 0.3624s; samplesPerSecond = 7064.2
|
||||
MPI Rank 0: 09/17/2016 18:06:31: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.64994363 * 10240; EvalClassificationError = 0.46923828 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 256
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:32: AdaptiveMinibatchSearch Epoch[3]: Search successful. New minibatchSize is 192. epochCriterion = 1.61407118 vs baseCriterion = 1.60223587
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:32: Starting Epoch 3: learning rate per sample = 0.001953 effective momentum = 0.829029 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [163840..245760] (first utterance at frame 163840), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:32: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 18:06:32: Epoch[ 3 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 1.54988471 * 1920; EvalClassificationError = 0.46510417 * 1920; time = 0.3126s; samplesPerSecond = 6141.7
|
||||
MPI Rank 0: 09/17/2016 18:06:32: Epoch[ 3 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 1.62737285 * 1920; EvalClassificationError = 0.47395833 * 1920; time = 0.3143s; samplesPerSecond = 6109.6
|
||||
MPI Rank 0: 09/17/2016 18:06:32: Epoch[ 3 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 1.65385199 * 1920; EvalClassificationError = 0.46302083 * 1920; time = 0.3125s; samplesPerSecond = 6143.1
|
||||
MPI Rank 0: 09/17/2016 18:06:33: Epoch[ 3 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 1.87023223 * 1920; EvalClassificationError = 0.51666667 * 1920; time = 0.3177s; samplesPerSecond = 6043.0
|
||||
MPI Rank 0: 09/17/2016 18:06:33: Epoch[ 3 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 1.75815123 * 1920; EvalClassificationError = 0.49270833 * 1920; time = 0.3196s; samplesPerSecond = 6007.4
|
||||
MPI Rank 0: 09/17/2016 18:06:33: Epoch[ 3 of 3]-Minibatch[ 51- 60, 14.06%]: CrossEntropyWithSoftmax = 2.03099563 * 1920; EvalClassificationError = 0.50312500 * 1920; time = 0.3099s; samplesPerSecond = 6195.1
|
||||
MPI Rank 0: 09/17/2016 18:06:34: Epoch[ 3 of 3]-Minibatch[ 61- 70, 16.41%]: CrossEntropyWithSoftmax = 1.86165460 * 1920; EvalClassificationError = 0.53489583 * 1920; time = 0.3146s; samplesPerSecond = 6103.6
|
||||
MPI Rank 0: 09/17/2016 18:06:34: Epoch[ 3 of 3]-Minibatch[ 71- 80, 18.75%]: CrossEntropyWithSoftmax = 2.15179657 * 1920; EvalClassificationError = 0.49010417 * 1920; time = 0.3123s; samplesPerSecond = 6147.0
|
||||
MPI Rank 0: 09/17/2016 18:06:34: Epoch[ 3 of 3]-Minibatch[ 81- 90, 21.09%]: CrossEntropyWithSoftmax = 1.79956105 * 1920; EvalClassificationError = 0.50625000 * 1920; time = 0.3129s; samplesPerSecond = 6135.6
|
||||
MPI Rank 0: 09/17/2016 18:06:35: Epoch[ 3 of 3]-Minibatch[ 91- 100, 23.44%]: CrossEntropyWithSoftmax = 1.80123726 * 1920; EvalClassificationError = 0.48385417 * 1920; time = 0.3165s; samplesPerSecond = 6067.1
|
||||
MPI Rank 0: 09/17/2016 18:06:35: Epoch[ 3 of 3]-Minibatch[ 101- 110, 25.78%]: CrossEntropyWithSoftmax = 1.65520548 * 1920; EvalClassificationError = 0.47135417 * 1920; time = 0.3132s; samplesPerSecond = 6130.6
|
||||
MPI Rank 0: 09/17/2016 18:06:35: Epoch[ 3 of 3]-Minibatch[ 111- 120, 28.13%]: CrossEntropyWithSoftmax = 1.65536374 * 1920; EvalClassificationError = 0.46875000 * 1920; time = 0.3199s; samplesPerSecond = 6001.9
|
||||
MPI Rank 0: 09/17/2016 18:06:36: Epoch[ 3 of 3]-Minibatch[ 121- 130, 30.47%]: CrossEntropyWithSoftmax = 1.62640983 * 1920; EvalClassificationError = 0.47656250 * 1920; time = 0.3133s; samplesPerSecond = 6129.0
|
||||
MPI Rank 0: 09/17/2016 18:06:36: Epoch[ 3 of 3]-Minibatch[ 131- 140, 32.81%]: CrossEntropyWithSoftmax = 1.58072809 * 1920; EvalClassificationError = 0.44895833 * 1920; time = 0.3138s; samplesPerSecond = 6118.4
|
||||
MPI Rank 0: 09/17/2016 18:06:36: Epoch[ 3 of 3]-Minibatch[ 141- 150, 35.16%]: CrossEntropyWithSoftmax = 1.56697558 * 1920; EvalClassificationError = 0.44687500 * 1920; time = 0.3130s; samplesPerSecond = 6133.4
|
||||
MPI Rank 0: 09/17/2016 18:06:37: Epoch[ 3 of 3]-Minibatch[ 151- 160, 37.50%]: CrossEntropyWithSoftmax = 1.58252868 * 1920; EvalClassificationError = 0.46250000 * 1920; time = 0.3271s; samplesPerSecond = 5870.6
|
||||
MPI Rank 0: 09/17/2016 18:06:37: Epoch[ 3 of 3]-Minibatch[ 161- 170, 39.84%]: CrossEntropyWithSoftmax = 1.53206179 * 1920; EvalClassificationError = 0.45104167 * 1920; time = 0.3249s; samplesPerSecond = 5908.7
|
||||
MPI Rank 0: 09/17/2016 18:06:37: Epoch[ 3 of 3]-Minibatch[ 171- 180, 42.19%]: CrossEntropyWithSoftmax = 1.47925397 * 1920; EvalClassificationError = 0.44531250 * 1920; time = 0.3153s; samplesPerSecond = 6088.6
|
||||
MPI Rank 0: 09/17/2016 18:06:38: Epoch[ 3 of 3]-Minibatch[ 181- 190, 44.53%]: CrossEntropyWithSoftmax = 1.46468817 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.3127s; samplesPerSecond = 6140.0
|
||||
MPI Rank 0: 09/17/2016 18:06:38: Epoch[ 3 of 3]-Minibatch[ 191- 200, 46.88%]: CrossEntropyWithSoftmax = 1.51778272 * 1920; EvalClassificationError = 0.45468750 * 1920; time = 0.3098s; samplesPerSecond = 6198.1
|
||||
MPI Rank 0: 09/17/2016 18:06:38: Epoch[ 3 of 3]-Minibatch[ 201- 210, 49.22%]: CrossEntropyWithSoftmax = 1.43521243 * 1920; EvalClassificationError = 0.41979167 * 1920; time = 0.3106s; samplesPerSecond = 6182.4
|
||||
MPI Rank 0: 09/17/2016 18:06:38: Epoch[ 3 of 3]-Minibatch[ 211- 220, 51.56%]: CrossEntropyWithSoftmax = 1.55602002 * 1920; EvalClassificationError = 0.45468750 * 1920; time = 0.3132s; samplesPerSecond = 6130.1
|
||||
MPI Rank 0: 09/17/2016 18:06:39: Epoch[ 3 of 3]-Minibatch[ 221- 230, 53.91%]: CrossEntropyWithSoftmax = 1.48434301 * 1920; EvalClassificationError = 0.44479167 * 1920; time = 0.3084s; samplesPerSecond = 6225.2
|
||||
MPI Rank 0: 09/17/2016 18:06:39: Epoch[ 3 of 3]-Minibatch[ 231- 240, 56.25%]: CrossEntropyWithSoftmax = 1.47880634 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.3094s; samplesPerSecond = 6205.7
|
||||
MPI Rank 0: 09/17/2016 18:06:39: Epoch[ 3 of 3]-Minibatch[ 241- 250, 58.59%]: CrossEntropyWithSoftmax = 1.44452798 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.3088s; samplesPerSecond = 6218.6
|
||||
MPI Rank 0: 09/17/2016 18:06:40: Epoch[ 3 of 3]-Minibatch[ 251- 260, 60.94%]: CrossEntropyWithSoftmax = 1.42272624 * 1920; EvalClassificationError = 0.42395833 * 1920; time = 0.3173s; samplesPerSecond = 6051.3
|
||||
MPI Rank 0: 09/17/2016 18:06:40: Epoch[ 3 of 3]-Minibatch[ 261- 270, 63.28%]: CrossEntropyWithSoftmax = 1.39175020 * 1920; EvalClassificationError = 0.41354167 * 1920; time = 0.3084s; samplesPerSecond = 6225.3
|
||||
MPI Rank 0: 09/17/2016 18:06:40: Epoch[ 3 of 3]-Minibatch[ 271- 280, 65.63%]: CrossEntropyWithSoftmax = 1.47323044 * 1920; EvalClassificationError = 0.43333333 * 1920; time = 0.3119s; samplesPerSecond = 6156.7
|
||||
MPI Rank 0: 09/17/2016 18:06:41: Epoch[ 3 of 3]-Minibatch[ 281- 290, 67.97%]: CrossEntropyWithSoftmax = 1.46072580 * 1920; EvalClassificationError = 0.41666667 * 1920; time = 0.3142s; samplesPerSecond = 6111.6
|
||||
MPI Rank 0: 09/17/2016 18:06:41: Epoch[ 3 of 3]-Minibatch[ 291- 300, 70.31%]: CrossEntropyWithSoftmax = 1.47105355 * 1920; EvalClassificationError = 0.43177083 * 1920; time = 0.3102s; samplesPerSecond = 6189.4
|
||||
MPI Rank 0: 09/17/2016 18:06:41: Epoch[ 3 of 3]-Minibatch[ 301- 310, 72.66%]: CrossEntropyWithSoftmax = 1.40973818 * 1920; EvalClassificationError = 0.41927083 * 1920; time = 0.3078s; samplesPerSecond = 6237.3
|
||||
MPI Rank 0: 09/17/2016 18:06:42: Epoch[ 3 of 3]-Minibatch[ 311- 320, 75.00%]: CrossEntropyWithSoftmax = 1.44253894 * 1920; EvalClassificationError = 0.42604167 * 1920; time = 0.3102s; samplesPerSecond = 6188.6
|
||||
MPI Rank 0: 09/17/2016 18:06:42: Epoch[ 3 of 3]-Minibatch[ 321- 330, 77.34%]: CrossEntropyWithSoftmax = 1.39941047 * 1920; EvalClassificationError = 0.42239583 * 1920; time = 0.3107s; samplesPerSecond = 6180.2
|
||||
MPI Rank 0: 09/17/2016 18:06:42: Epoch[ 3 of 3]-Minibatch[ 331- 340, 79.69%]: CrossEntropyWithSoftmax = 1.42524482 * 1920; EvalClassificationError = 0.41927083 * 1920; time = 0.3069s; samplesPerSecond = 6256.7
|
||||
MPI Rank 0: 09/17/2016 18:06:43: Epoch[ 3 of 3]-Minibatch[ 341- 350, 82.03%]: CrossEntropyWithSoftmax = 1.46079356 * 1920; EvalClassificationError = 0.42916667 * 1920; time = 0.3105s; samplesPerSecond = 6182.9
|
||||
MPI Rank 0: 09/17/2016 18:06:43: Epoch[ 3 of 3]-Minibatch[ 351- 360, 84.38%]: CrossEntropyWithSoftmax = 1.39608704 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.3103s; samplesPerSecond = 6187.6
|
||||
MPI Rank 0: 09/17/2016 18:06:43: Epoch[ 3 of 3]-Minibatch[ 361- 370, 86.72%]: CrossEntropyWithSoftmax = 1.46624909 * 1920; EvalClassificationError = 0.43177083 * 1920; time = 0.3073s; samplesPerSecond = 6247.0
|
||||
MPI Rank 0: 09/17/2016 18:06:43: Epoch[ 3 of 3]-Minibatch[ 371- 380, 89.06%]: CrossEntropyWithSoftmax = 1.42432290 * 1920; EvalClassificationError = 0.42031250 * 1920; time = 0.3098s; samplesPerSecond = 6197.0
|
||||
MPI Rank 0: 09/17/2016 18:06:44: Epoch[ 3 of 3]-Minibatch[ 381- 390, 91.41%]: CrossEntropyWithSoftmax = 1.36453678 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.3092s; samplesPerSecond = 6209.6
|
||||
MPI Rank 0: 09/17/2016 18:06:44: Epoch[ 3 of 3]-Minibatch[ 391- 400, 93.75%]: CrossEntropyWithSoftmax = 1.38669452 * 1920; EvalClassificationError = 0.41041667 * 1920; time = 0.3097s; samplesPerSecond = 6200.4
|
||||
MPI Rank 0: 09/17/2016 18:06:44: Epoch[ 3 of 3]-Minibatch[ 401- 410, 96.09%]: CrossEntropyWithSoftmax = 1.37926773 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.3137s; samplesPerSecond = 6121.2
|
||||
MPI Rank 0: 09/17/2016 18:06:45: Epoch[ 3 of 3]-Minibatch[ 411- 420, 98.44%]: CrossEntropyWithSoftmax = 1.39890438 * 1920; EvalClassificationError = 0.41875000 * 1920; time = 0.3243s; samplesPerSecond = 5919.5
|
||||
MPI Rank 0: 09/17/2016 18:06:45: Finished Epoch[ 3 of 3]: [Training] CrossEntropyWithSoftmax = 1.55322819 * 81920; EvalClassificationError = 0.44776611 * 81920; totalSamplesSeen = 245760; learningRatePerSample = 0.001953125; epochTime=25.5708s
|
||||
MPI Rank 0: 09/17/2016 18:06:45: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/models/cntkSpeech.dnn'
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:45: Action "train" complete.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 18:06:45: __COMPLETED__
|
||||
MPI Rank 0: ~MPIWrapper
|
||||
MPI Rank 1: 09/17/2016 18:04:55: Redirecting stderr to file C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr_speechTrain.logrank1
|
||||
MPI Rank 1: CNTK 1.7+ (HEAD 28bddd, Sep 17 2016 17:51:24) on cntk-muc00 at 2016/09/17 18:04:54
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\debug\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\DNN\Parallel1BitQuantizationWithAutoMBScaling OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=2 stderr=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160917180323.310755\Speech\DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
MPI Rank 1: 09/17/2016 18:04:55: Using 2 CPU threads.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:04:55: ##############################################################################
|
||||
MPI Rank 1: 09/17/2016 18:04:55: # #
|
||||
MPI Rank 1: 09/17/2016 18:04:55: # speechTrain command (train action) #
|
||||
MPI Rank 1: 09/17/2016 18:04:55: # #
|
||||
MPI Rank 1: 09/17/2016 18:04:55: ##############################################################################
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:04:55:
|
||||
MPI Rank 1: Creating virgin network.
|
||||
MPI Rank 1: SimpleNetworkBuilder Using GPU 0
|
||||
MPI Rank 1: Microsoft::MSR::CNTK::GPUMatrix<ElemType>::SetUniformRandomValue (GPU): creating curand object with seed 1, sizeof(ElemType)==8
|
||||
MPI Rank 1: reading script file glob_0000.scp ... 948 entries
|
||||
MPI Rank 1: total 132 state names in state list C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data/state.list
|
||||
MPI Rank 1: htkmlfreader: reading MLF file C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\Speech\Data/glob_0000.mlf ... total 948 entries
|
||||
MPI Rank 1: ...............................................................................................feature set 0: 252734 frames in 948 out of 948 utterances
|
||||
MPI Rank 1: label set 0: 129 classes
|
||||
MPI Rank 1: minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: 316.0 utterances, 84244.7 frames
|
||||
MPI Rank 1: 09/17/2016 18:04:58:
|
||||
MPI Rank 1: Model has 25 nodes. Using GPU 0.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Training criterion: CrossEntropyWithSoftmax = CrossEntropyWithSoftmax
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Evaluation criterion: EvalClassificationError = ClassificationError
|
||||
MPI Rank 1:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: Allocating matrices for forward and/or backward propagation.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: Memory Sharing: Out of 40 matrices, 19 are shared as 8, and 21 are not shared.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: { H1 : [512 x 1 x *]
|
||||
MPI Rank 1: W0*features : [512 x *] (gradient) }
|
||||
MPI Rank 1: { W0*features+B0 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 1: W1*H1 : [512 x 1 x *] }
|
||||
MPI Rank 1: { W0 : [512 x 363] (gradient)
|
||||
MPI Rank 1: W0*features+B0 : [512 x 1 x *] }
|
||||
MPI Rank 1: { H2 : [512 x 1 x *]
|
||||
MPI Rank 1: W1*H1 : [512 x 1 x *] (gradient) }
|
||||
MPI Rank 1: { HLast : [132 x 1 x *]
|
||||
MPI Rank 1: W2 : [132 x 512] (gradient) }
|
||||
MPI Rank 1: { B1 : [512 x 1] (gradient)
|
||||
MPI Rank 1: H2 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 1: HLast : [132 x 1 x *] (gradient) }
|
||||
MPI Rank 1: { W1 : [512 x 512] (gradient)
|
||||
MPI Rank 1: W1*H1+B1 : [512 x 1 x *] }
|
||||
MPI Rank 1: { B0 : [512 x 1] (gradient)
|
||||
MPI Rank 1: H1 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 1: W1*H1+B1 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 1: W2*H1 : [132 x 1 x *] }
|
||||
MPI Rank 1:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Training 516740 parameters in 6 out of 6 parameter tensors and 15 nodes with gradient:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Node 'B0' (LearnableParameter operation) : [512 x 1]
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Node 'B1' (LearnableParameter operation) : [512 x 1]
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Node 'B2' (LearnableParameter operation) : [132 x 1]
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Node 'W0' (LearnableParameter operation) : [512 x 363]
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Node 'W1' (LearnableParameter operation) : [512 x 512]
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Node 'W2' (LearnableParameter operation) : [132 x 512]
|
||||
MPI Rank 1:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Precomputing --> 3 PreCompute nodes found.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:04:58: MeanOfFeatures = Mean()
|
||||
MPI Rank 1: 09/17/2016 18:04:58: InvStdOfFeatures = InvStdDev()
|
||||
MPI Rank 1: 09/17/2016 18:04:58: Prior = Mean()
|
||||
MPI Rank 1: minibatchiterator: epoch 0: frames [0..252734] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses
|
||||
MPI Rank 1: requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:05:28: Precomputing --> Completed.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:05:28: Starting Epoch 1: learning rate per sample = 0.001953 effective momentum = 0.939413 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 1: minibatchiterator: epoch 0: frames [0..81920] (first utterance at frame 0), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:05:28: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:05:28: Epoch[ 1 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 4.73732304 * 640; EvalClassificationError = 0.96718750 * 640; time = 0.2017s; samplesPerSecond = 3172.7
|
||||
MPI Rank 1: 09/17/2016 18:05:28: Epoch[ 1 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 4.37245658 * 640; EvalClassificationError = 0.92187500 * 640; time = 0.1852s; samplesPerSecond = 3455.5
|
||||
MPI Rank 1: 09/17/2016 18:05:28: Epoch[ 1 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 4.17410438 * 640; EvalClassificationError = 0.95937500 * 640; time = 0.1837s; samplesPerSecond = 3484.1
|
||||
MPI Rank 1: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 3.88445939 * 640; EvalClassificationError = 0.86093750 * 640; time = 0.1869s; samplesPerSecond = 3423.6
|
||||
MPI Rank 1: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 4.05583056 * 640; EvalClassificationError = 0.89375000 * 640; time = 0.1836s; samplesPerSecond = 3486.3
|
||||
MPI Rank 1: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 3.94301179 * 640; EvalClassificationError = 0.86250000 * 640; time = 0.1862s; samplesPerSecond = 3437.0
|
||||
MPI Rank 1: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 3.80690392 * 640; EvalClassificationError = 0.85937500 * 640; time = 0.1982s; samplesPerSecond = 3229.0
|
||||
MPI Rank 1: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 3.92014544 * 640; EvalClassificationError = 0.87812500 * 640; time = 0.1856s; samplesPerSecond = 3448.9
|
||||
MPI Rank 1: 09/17/2016 18:05:29: Epoch[ 1 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 3.97161496 * 640; EvalClassificationError = 0.87968750 * 640; time = 0.1847s; samplesPerSecond = 3465.1
|
||||
MPI Rank 1: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 3.93678341 * 640; EvalClassificationError = 0.90937500 * 640; time = 0.1846s; samplesPerSecond = 3466.3
|
||||
MPI Rank 1: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 4.05170333 * 640; EvalClassificationError = 0.92187500 * 640; time = 0.1851s; samplesPerSecond = 3457.3
|
||||
MPI Rank 1: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 3.89920006 * 640; EvalClassificationError = 0.86406250 * 640; time = 0.1863s; samplesPerSecond = 3435.2
|
||||
MPI Rank 1: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 3.87251780 * 640; EvalClassificationError = 0.89218750 * 640; time = 0.1838s; samplesPerSecond = 3481.9
|
||||
MPI Rank 1: 09/17/2016 18:05:30: Epoch[ 1 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 3.81997328 * 640; EvalClassificationError = 0.88281250 * 640; time = 0.1850s; samplesPerSecond = 3460.0
|
||||
MPI Rank 1: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 3.80473823 * 640; EvalClassificationError = 0.88125000 * 640; time = 0.1853s; samplesPerSecond = 3454.1
|
||||
MPI Rank 1: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 3.94101364 * 640; EvalClassificationError = 0.88750000 * 640; time = 0.1852s; samplesPerSecond = 3454.8
|
||||
MPI Rank 1: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 161- 170, 13.28%]: CrossEntropyWithSoftmax = 3.82130368 * 640; EvalClassificationError = 0.87656250 * 640; time = 0.1854s; samplesPerSecond = 3452.4
|
||||
MPI Rank 1: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 171- 180, 14.06%]: CrossEntropyWithSoftmax = 3.75236005 * 640; EvalClassificationError = 0.85312500 * 640; time = 0.1847s; samplesPerSecond = 3464.9
|
||||
MPI Rank 1: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 181- 190, 14.84%]: CrossEntropyWithSoftmax = 3.79661132 * 640; EvalClassificationError = 0.86562500 * 640; time = 0.1864s; samplesPerSecond = 3434.1
|
||||
MPI Rank 1: 09/17/2016 18:05:31: Epoch[ 1 of 3]-Minibatch[ 191- 200, 15.63%]: CrossEntropyWithSoftmax = 3.73922363 * 640; EvalClassificationError = 0.87343750 * 640; time = 0.1846s; samplesPerSecond = 3466.6
|
||||
MPI Rank 1: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 201- 210, 16.41%]: CrossEntropyWithSoftmax = 3.73517402 * 640; EvalClassificationError = 0.86406250 * 640; time = 0.1855s; samplesPerSecond = 3449.5
|
||||
MPI Rank 1: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 211- 220, 17.19%]: CrossEntropyWithSoftmax = 3.72398663 * 640; EvalClassificationError = 0.85625000 * 640; time = 0.1841s; samplesPerSecond = 3476.9
|
||||
MPI Rank 1: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 221- 230, 17.97%]: CrossEntropyWithSoftmax = 3.71595086 * 640; EvalClassificationError = 0.84062500 * 640; time = 0.1852s; samplesPerSecond = 3455.0
|
||||
MPI Rank 1: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 231- 240, 18.75%]: CrossEntropyWithSoftmax = 3.75562845 * 640; EvalClassificationError = 0.83281250 * 640; time = 0.1862s; samplesPerSecond = 3438.0
|
||||
MPI Rank 1: 09/17/2016 18:05:32: Epoch[ 1 of 3]-Minibatch[ 241- 250, 19.53%]: CrossEntropyWithSoftmax = 3.72052883 * 640; EvalClassificationError = 0.85468750 * 640; time = 0.1859s; samplesPerSecond = 3442.7
|
||||
MPI Rank 1: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 251- 260, 20.31%]: CrossEntropyWithSoftmax = 3.66417431 * 640; EvalClassificationError = 0.83906250 * 640; time = 0.1839s; samplesPerSecond = 3479.6
|
||||
MPI Rank 1: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 261- 270, 21.09%]: CrossEntropyWithSoftmax = 3.58440175 * 640; EvalClassificationError = 0.77812500 * 640; time = 0.1845s; samplesPerSecond = 3469.7
|
||||
MPI Rank 1: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 271- 280, 21.88%]: CrossEntropyWithSoftmax = 3.69427679 * 640; EvalClassificationError = 0.80781250 * 640; time = 0.1855s; samplesPerSecond = 3449.6
|
||||
MPI Rank 1: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 281- 290, 22.66%]: CrossEntropyWithSoftmax = 3.57769881 * 640; EvalClassificationError = 0.79062500 * 640; time = 0.1839s; samplesPerSecond = 3479.4
|
||||
MPI Rank 1: 09/17/2016 18:05:33: Epoch[ 1 of 3]-Minibatch[ 291- 300, 23.44%]: CrossEntropyWithSoftmax = 3.52797012 * 640; EvalClassificationError = 0.82031250 * 640; time = 0.1849s; samplesPerSecond = 3460.5
|
||||
MPI Rank 1: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 301- 310, 24.22%]: CrossEntropyWithSoftmax = 3.53498040 * 640; EvalClassificationError = 0.81718750 * 640; time = 0.1846s; samplesPerSecond = 3467.0
|
||||
MPI Rank 1: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 311- 320, 25.00%]: CrossEntropyWithSoftmax = 3.50005340 * 640; EvalClassificationError = 0.79375000 * 640; time = 0.1861s; samplesPerSecond = 3439.9
|
||||
MPI Rank 1: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 321- 330, 25.78%]: CrossEntropyWithSoftmax = 3.37848051 * 640; EvalClassificationError = 0.79687500 * 640; time = 0.1850s; samplesPerSecond = 3459.8
|
||||
MPI Rank 1: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 331- 340, 26.56%]: CrossEntropyWithSoftmax = 3.44203869 * 640; EvalClassificationError = 0.78906250 * 640; time = 0.1846s; samplesPerSecond = 3467.1
|
||||
MPI Rank 1: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 341- 350, 27.34%]: CrossEntropyWithSoftmax = 3.49473674 * 640; EvalClassificationError = 0.80156250 * 640; time = 0.1865s; samplesPerSecond = 3431.9
|
||||
MPI Rank 1: 09/17/2016 18:05:34: Epoch[ 1 of 3]-Minibatch[ 351- 360, 28.13%]: CrossEntropyWithSoftmax = 3.42678273 * 640; EvalClassificationError = 0.80468750 * 640; time = 0.1835s; samplesPerSecond = 3487.8
|
||||
MPI Rank 1: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 361- 370, 28.91%]: CrossEntropyWithSoftmax = 3.49825811 * 640; EvalClassificationError = 0.82656250 * 640; time = 0.1851s; samplesPerSecond = 3458.2
|
||||
MPI Rank 1: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 371- 380, 29.69%]: CrossEntropyWithSoftmax = 3.18570771 * 640; EvalClassificationError = 0.75156250 * 640; time = 0.1850s; samplesPerSecond = 3460.3
|
||||
MPI Rank 1: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 381- 390, 30.47%]: CrossEntropyWithSoftmax = 3.31483570 * 640; EvalClassificationError = 0.79218750 * 640; time = 0.1841s; samplesPerSecond = 3475.8
|
||||
MPI Rank 1: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 391- 400, 31.25%]: CrossEntropyWithSoftmax = 3.27794269 * 640; EvalClassificationError = 0.77656250 * 640; time = 0.1848s; samplesPerSecond = 3463.0
|
||||
MPI Rank 1: 09/17/2016 18:05:35: Epoch[ 1 of 3]-Minibatch[ 401- 410, 32.03%]: CrossEntropyWithSoftmax = 3.42283181 * 640; EvalClassificationError = 0.80312500 * 640; time = 0.1858s; samplesPerSecond = 3444.8
|
||||
MPI Rank 1: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 411- 420, 32.81%]: CrossEntropyWithSoftmax = 3.28013444 * 640; EvalClassificationError = 0.75937500 * 640; time = 0.1844s; samplesPerSecond = 3471.4
|
||||
MPI Rank 1: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 421- 430, 33.59%]: CrossEntropyWithSoftmax = 3.23863079 * 640; EvalClassificationError = 0.78281250 * 640; time = 0.1879s; samplesPerSecond = 3406.4
|
||||
MPI Rank 1: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 431- 440, 34.38%]: CrossEntropyWithSoftmax = 3.13381567 * 640; EvalClassificationError = 0.75937500 * 640; time = 0.1904s; samplesPerSecond = 3360.5
|
||||
MPI Rank 1: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 441- 450, 35.16%]: CrossEntropyWithSoftmax = 3.18522093 * 640; EvalClassificationError = 0.74062500 * 640; time = 0.1906s; samplesPerSecond = 3357.4
|
||||
MPI Rank 1: 09/17/2016 18:05:36: Epoch[ 1 of 3]-Minibatch[ 451- 460, 35.94%]: CrossEntropyWithSoftmax = 3.23487402 * 640; EvalClassificationError = 0.77187500 * 640; time = 0.1974s; samplesPerSecond = 3242.5
|
||||
MPI Rank 1: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 461- 470, 36.72%]: CrossEntropyWithSoftmax = 3.19165708 * 640; EvalClassificationError = 0.77812500 * 640; time = 0.1842s; samplesPerSecond = 3473.9
|
||||
MPI Rank 1: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 471- 480, 37.50%]: CrossEntropyWithSoftmax = 3.16584445 * 640; EvalClassificationError = 0.73906250 * 640; time = 0.2009s; samplesPerSecond = 3185.8
|
||||
MPI Rank 1: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 481- 490, 38.28%]: CrossEntropyWithSoftmax = 3.24961355 * 640; EvalClassificationError = 0.77187500 * 640; time = 0.2049s; samplesPerSecond = 3124.2
|
||||
MPI Rank 1: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 491- 500, 39.06%]: CrossEntropyWithSoftmax = 3.18529030 * 640; EvalClassificationError = 0.76406250 * 640; time = 0.2747s; samplesPerSecond = 2329.4
|
||||
MPI Rank 1: 09/17/2016 18:05:37: Epoch[ 1 of 3]-Minibatch[ 501- 510, 39.84%]: CrossEntropyWithSoftmax = 2.98383964 * 640; EvalClassificationError = 0.72187500 * 640; time = 0.1891s; samplesPerSecond = 3385.3
|
||||
MPI Rank 1: 09/17/2016 18:05:38: Epoch[ 1 of 3]-Minibatch[ 511- 520, 40.63%]: CrossEntropyWithSoftmax = 3.14703955 * 640; EvalClassificationError = 0.73281250 * 640; time = 0.2084s; samplesPerSecond = 3071.5
|
||||
MPI Rank 1: 09/17/2016 18:05:38: Epoch[ 1 of 3]-Minibatch[ 521- 530, 41.41%]: CrossEntropyWithSoftmax = 3.08217828 * 640; EvalClassificationError = 0.72656250 * 640; time = 0.2300s; samplesPerSecond = 2782.8
|
||||
MPI Rank 1: 09/17/2016 18:05:38: Epoch[ 1 of 3]-Minibatch[ 531- 540, 42.19%]: CrossEntropyWithSoftmax = 2.93980472 * 640; EvalClassificationError = 0.70625000 * 640; time = 0.2435s; samplesPerSecond = 2628.6
|
||||
MPI Rank 1: 09/17/2016 18:05:38: Epoch[ 1 of 3]-Minibatch[ 541- 550, 42.97%]: CrossEntropyWithSoftmax = 3.02220354 * 640; EvalClassificationError = 0.72968750 * 640; time = 0.2409s; samplesPerSecond = 2657.0
|
||||
MPI Rank 1: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 551- 560, 43.75%]: CrossEntropyWithSoftmax = 2.95530592 * 640; EvalClassificationError = 0.71250000 * 640; time = 0.2355s; samplesPerSecond = 2717.7
|
||||
MPI Rank 1: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 561- 570, 44.53%]: CrossEntropyWithSoftmax = 2.95810228 * 640; EvalClassificationError = 0.72812500 * 640; time = 0.1847s; samplesPerSecond = 3464.4
|
||||
MPI Rank 1: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 571- 580, 45.31%]: CrossEntropyWithSoftmax = 3.00786863 * 640; EvalClassificationError = 0.72656250 * 640; time = 0.1850s; samplesPerSecond = 3459.0
|
||||
MPI Rank 1: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 581- 590, 46.09%]: CrossEntropyWithSoftmax = 3.00106811 * 640; EvalClassificationError = 0.72968750 * 640; time = 0.1874s; samplesPerSecond = 3415.1
|
||||
MPI Rank 1: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 591- 600, 46.88%]: CrossEntropyWithSoftmax = 2.91931245 * 640; EvalClassificationError = 0.70468750 * 640; time = 0.1851s; samplesPerSecond = 3457.5
|
||||
MPI Rank 1: 09/17/2016 18:05:39: Epoch[ 1 of 3]-Minibatch[ 601- 610, 47.66%]: CrossEntropyWithSoftmax = 3.00601604 * 640; EvalClassificationError = 0.73750000 * 640; time = 0.1848s; samplesPerSecond = 3462.8
|
||||
MPI Rank 1: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 611- 620, 48.44%]: CrossEntropyWithSoftmax = 2.93808431 * 640; EvalClassificationError = 0.70625000 * 640; time = 0.1858s; samplesPerSecond = 3443.9
|
||||
MPI Rank 1: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 621- 630, 49.22%]: CrossEntropyWithSoftmax = 2.97093532 * 640; EvalClassificationError = 0.74375000 * 640; time = 0.1854s; samplesPerSecond = 3452.9
|
||||
MPI Rank 1: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 631- 640, 50.00%]: CrossEntropyWithSoftmax = 2.86102307 * 640; EvalClassificationError = 0.67812500 * 640; time = 0.1834s; samplesPerSecond = 3489.2
|
||||
MPI Rank 1: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 641- 650, 50.78%]: CrossEntropyWithSoftmax = 2.88070307 * 640; EvalClassificationError = 0.71406250 * 640; time = 0.1845s; samplesPerSecond = 3469.4
|
||||
MPI Rank 1: 09/17/2016 18:05:40: Epoch[ 1 of 3]-Minibatch[ 651- 660, 51.56%]: CrossEntropyWithSoftmax = 2.90422279 * 640; EvalClassificationError = 0.71875000 * 640; time = 0.1872s; samplesPerSecond = 3418.0
|
||||
MPI Rank 1: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 661- 670, 52.34%]: CrossEntropyWithSoftmax = 2.82634561 * 640; EvalClassificationError = 0.70937500 * 640; time = 0.1864s; samplesPerSecond = 3433.6
|
||||
MPI Rank 1: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 671- 680, 53.13%]: CrossEntropyWithSoftmax = 2.86572702 * 640; EvalClassificationError = 0.67343750 * 640; time = 0.1852s; samplesPerSecond = 3456.2
|
||||
MPI Rank 1: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 681- 690, 53.91%]: CrossEntropyWithSoftmax = 2.82698660 * 640; EvalClassificationError = 0.70468750 * 640; time = 0.1850s; samplesPerSecond = 3459.3
|
||||
MPI Rank 1: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 691- 700, 54.69%]: CrossEntropyWithSoftmax = 2.69768998 * 640; EvalClassificationError = 0.68437500 * 640; time = 0.1865s; samplesPerSecond = 3430.9
|
||||
MPI Rank 1: 09/17/2016 18:05:41: Epoch[ 1 of 3]-Minibatch[ 701- 710, 55.47%]: CrossEntropyWithSoftmax = 2.74280097 * 640; EvalClassificationError = 0.66406250 * 640; time = 0.1858s; samplesPerSecond = 3444.3
|
||||
MPI Rank 1: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 711- 720, 56.25%]: CrossEntropyWithSoftmax = 2.79750038 * 640; EvalClassificationError = 0.71093750 * 640; time = 0.1838s; samplesPerSecond = 3481.1
|
||||
MPI Rank 1: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 721- 730, 57.03%]: CrossEntropyWithSoftmax = 2.68839718 * 640; EvalClassificationError = 0.67968750 * 640; time = 0.1857s; samplesPerSecond = 3447.3
|
||||
MPI Rank 1: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 731- 740, 57.81%]: CrossEntropyWithSoftmax = 2.76485288 * 640; EvalClassificationError = 0.68281250 * 640; time = 0.1853s; samplesPerSecond = 3454.6
|
||||
MPI Rank 1: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 741- 750, 58.59%]: CrossEntropyWithSoftmax = 2.77321739 * 640; EvalClassificationError = 0.67968750 * 640; time = 0.1853s; samplesPerSecond = 3453.9
|
||||
MPI Rank 1: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 751- 760, 59.38%]: CrossEntropyWithSoftmax = 2.71065612 * 640; EvalClassificationError = 0.68593750 * 640; time = 0.1846s; samplesPerSecond = 3467.7
|
||||
MPI Rank 1: 09/17/2016 18:05:42: Epoch[ 1 of 3]-Minibatch[ 761- 770, 60.16%]: CrossEntropyWithSoftmax = 2.78848250 * 640; EvalClassificationError = 0.65937500 * 640; time = 0.1836s; samplesPerSecond = 3485.9
|
||||
MPI Rank 1: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 771- 780, 60.94%]: CrossEntropyWithSoftmax = 2.78443193 * 640; EvalClassificationError = 0.68281250 * 640; time = 0.1857s; samplesPerSecond = 3445.7
|
||||
MPI Rank 1: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 781- 790, 61.72%]: CrossEntropyWithSoftmax = 2.72094929 * 640; EvalClassificationError = 0.67031250 * 640; time = 0.1852s; samplesPerSecond = 3456.3
|
||||
MPI Rank 1: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 791- 800, 62.50%]: CrossEntropyWithSoftmax = 2.70404088 * 640; EvalClassificationError = 0.64062500 * 640; time = 0.1853s; samplesPerSecond = 3454.3
|
||||
MPI Rank 1: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 801- 810, 63.28%]: CrossEntropyWithSoftmax = 2.70887221 * 640; EvalClassificationError = 0.66875000 * 640; time = 0.1843s; samplesPerSecond = 3472.7
|
||||
MPI Rank 1: 09/17/2016 18:05:43: Epoch[ 1 of 3]-Minibatch[ 811- 820, 64.06%]: CrossEntropyWithSoftmax = 2.67633326 * 640; EvalClassificationError = 0.66562500 * 640; time = 0.1848s; samplesPerSecond = 3463.3
|
||||
MPI Rank 1: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 821- 830, 64.84%]: CrossEntropyWithSoftmax = 2.53198524 * 640; EvalClassificationError = 0.62968750 * 640; time = 0.1853s; samplesPerSecond = 3454.6
|
||||
MPI Rank 1: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 831- 840, 65.63%]: CrossEntropyWithSoftmax = 2.63317481 * 640; EvalClassificationError = 0.66093750 * 640; time = 0.1847s; samplesPerSecond = 3465.9
|
||||
MPI Rank 1: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 841- 850, 66.41%]: CrossEntropyWithSoftmax = 2.65923035 * 640; EvalClassificationError = 0.65468750 * 640; time = 0.1845s; samplesPerSecond = 3469.6
|
||||
MPI Rank 1: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 851- 860, 67.19%]: CrossEntropyWithSoftmax = 2.58961930 * 640; EvalClassificationError = 0.66718750 * 640; time = 0.1863s; samplesPerSecond = 3435.1
|
||||
MPI Rank 1: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 861- 870, 67.97%]: CrossEntropyWithSoftmax = 2.72924811 * 640; EvalClassificationError = 0.67812500 * 640; time = 0.1850s; samplesPerSecond = 3459.8
|
||||
MPI Rank 1: 09/17/2016 18:05:44: Epoch[ 1 of 3]-Minibatch[ 871- 880, 68.75%]: CrossEntropyWithSoftmax = 2.66252872 * 640; EvalClassificationError = 0.66250000 * 640; time = 0.1846s; samplesPerSecond = 3466.6
|
||||
MPI Rank 1: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 881- 890, 69.53%]: CrossEntropyWithSoftmax = 2.52883427 * 640; EvalClassificationError = 0.65937500 * 640; time = 0.1849s; samplesPerSecond = 3460.8
|
||||
MPI Rank 1: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 891- 900, 70.31%]: CrossEntropyWithSoftmax = 2.62228341 * 640; EvalClassificationError = 0.65625000 * 640; time = 0.1853s; samplesPerSecond = 3454.5
|
||||
MPI Rank 1: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 901- 910, 71.09%]: CrossEntropyWithSoftmax = 2.55550779 * 640; EvalClassificationError = 0.66093750 * 640; time = 0.1856s; samplesPerSecond = 3448.5
|
||||
MPI Rank 1: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 911- 920, 71.88%]: CrossEntropyWithSoftmax = 2.55049429 * 640; EvalClassificationError = 0.64531250 * 640; time = 0.1856s; samplesPerSecond = 3448.1
|
||||
MPI Rank 1: 09/17/2016 18:05:45: Epoch[ 1 of 3]-Minibatch[ 921- 930, 72.66%]: CrossEntropyWithSoftmax = 2.59920014 * 640; EvalClassificationError = 0.65625000 * 640; time = 0.1854s; samplesPerSecond = 3451.5
|
||||
MPI Rank 1: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 931- 940, 73.44%]: CrossEntropyWithSoftmax = 2.54341577 * 640; EvalClassificationError = 0.62343750 * 640; time = 0.1843s; samplesPerSecond = 3472.4
|
||||
MPI Rank 1: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 941- 950, 74.22%]: CrossEntropyWithSoftmax = 2.48476222 * 640; EvalClassificationError = 0.63750000 * 640; time = 0.1850s; samplesPerSecond = 3459.0
|
||||
MPI Rank 1: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 951- 960, 75.00%]: CrossEntropyWithSoftmax = 2.53015221 * 640; EvalClassificationError = 0.63906250 * 640; time = 0.1869s; samplesPerSecond = 3424.3
|
||||
MPI Rank 1: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 961- 970, 75.78%]: CrossEntropyWithSoftmax = 2.35319566 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1855s; samplesPerSecond = 3450.2
|
||||
MPI Rank 1: 09/17/2016 18:05:46: Epoch[ 1 of 3]-Minibatch[ 971- 980, 76.56%]: CrossEntropyWithSoftmax = 2.54683738 * 640; EvalClassificationError = 0.64375000 * 640; time = 0.1856s; samplesPerSecond = 3447.6
|
||||
MPI Rank 1: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[ 981- 990, 77.34%]: CrossEntropyWithSoftmax = 2.45404859 * 640; EvalClassificationError = 0.63750000 * 640; time = 0.1852s; samplesPerSecond = 3456.0
|
||||
MPI Rank 1: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[ 991-1000, 78.13%]: CrossEntropyWithSoftmax = 2.42597335 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1860s; samplesPerSecond = 3440.6
|
||||
MPI Rank 1: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[1001-1010, 78.91%]: CrossEntropyWithSoftmax = 2.37566713 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1854s; samplesPerSecond = 3451.9
|
||||
MPI Rank 1: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[1011-1020, 79.69%]: CrossEntropyWithSoftmax = 2.35902642 * 640; EvalClassificationError = 0.59218750 * 640; time = 0.1852s; samplesPerSecond = 3456.2
|
||||
MPI Rank 1: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[1021-1030, 80.47%]: CrossEntropyWithSoftmax = 2.36171107 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1867s; samplesPerSecond = 3427.5
|
||||
MPI Rank 1: 09/17/2016 18:05:47: Epoch[ 1 of 3]-Minibatch[1031-1040, 81.25%]: CrossEntropyWithSoftmax = 2.33345715 * 640; EvalClassificationError = 0.57343750 * 640; time = 0.1852s; samplesPerSecond = 3456.3
|
||||
MPI Rank 1: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1041-1050, 82.03%]: CrossEntropyWithSoftmax = 2.44952411 * 640; EvalClassificationError = 0.61875000 * 640; time = 0.1860s; samplesPerSecond = 3441.5
|
||||
MPI Rank 1: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1051-1060, 82.81%]: CrossEntropyWithSoftmax = 2.31665914 * 640; EvalClassificationError = 0.58437500 * 640; time = 0.1854s; samplesPerSecond = 3451.9
|
||||
MPI Rank 1: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1061-1070, 83.59%]: CrossEntropyWithSoftmax = 2.32162968 * 640; EvalClassificationError = 0.61093750 * 640; time = 0.1849s; samplesPerSecond = 3461.4
|
||||
MPI Rank 1: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1071-1080, 84.38%]: CrossEntropyWithSoftmax = 2.48543345 * 640; EvalClassificationError = 0.63593750 * 640; time = 0.1845s; samplesPerSecond = 3469.1
|
||||
MPI Rank 1: 09/17/2016 18:05:48: Epoch[ 1 of 3]-Minibatch[1081-1090, 85.16%]: CrossEntropyWithSoftmax = 2.35574244 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1858s; samplesPerSecond = 3444.2
|
||||
MPI Rank 1: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1091-1100, 85.94%]: CrossEntropyWithSoftmax = 2.38117646 * 640; EvalClassificationError = 0.62656250 * 640; time = 0.1849s; samplesPerSecond = 3461.6
|
||||
MPI Rank 1: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1101-1110, 86.72%]: CrossEntropyWithSoftmax = 2.38366197 * 640; EvalClassificationError = 0.62187500 * 640; time = 0.1860s; samplesPerSecond = 3440.0
|
||||
MPI Rank 1: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1111-1120, 87.50%]: CrossEntropyWithSoftmax = 2.50061273 * 640; EvalClassificationError = 0.63437500 * 640; time = 0.1850s; samplesPerSecond = 3459.2
|
||||
MPI Rank 1: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1121-1130, 88.28%]: CrossEntropyWithSoftmax = 2.19965354 * 640; EvalClassificationError = 0.63125000 * 640; time = 0.1852s; samplesPerSecond = 3456.0
|
||||
MPI Rank 1: 09/17/2016 18:05:49: Epoch[ 1 of 3]-Minibatch[1131-1140, 89.06%]: CrossEntropyWithSoftmax = 2.27232693 * 640; EvalClassificationError = 0.60156250 * 640; time = 0.1853s; samplesPerSecond = 3454.5
|
||||
MPI Rank 1: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1141-1150, 89.84%]: CrossEntropyWithSoftmax = 2.20566415 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1861s; samplesPerSecond = 3438.9
|
||||
MPI Rank 1: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1151-1160, 90.63%]: CrossEntropyWithSoftmax = 2.33120303 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1854s; samplesPerSecond = 3452.8
|
||||
MPI Rank 1: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1161-1170, 91.41%]: CrossEntropyWithSoftmax = 2.29726772 * 640; EvalClassificationError = 0.57812500 * 640; time = 0.1854s; samplesPerSecond = 3452.1
|
||||
MPI Rank 1: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1171-1180, 92.19%]: CrossEntropyWithSoftmax = 2.29349025 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1857s; samplesPerSecond = 3446.0
|
||||
MPI Rank 1: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1181-1190, 92.97%]: CrossEntropyWithSoftmax = 2.30009060 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1850s; samplesPerSecond = 3458.7
|
||||
MPI Rank 1: 09/17/2016 18:05:50: Epoch[ 1 of 3]-Minibatch[1191-1200, 93.75%]: CrossEntropyWithSoftmax = 2.21902385 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1862s; samplesPerSecond = 3436.5
|
||||
MPI Rank 1: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1201-1210, 94.53%]: CrossEntropyWithSoftmax = 2.23739674 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1854s; samplesPerSecond = 3452.5
|
||||
MPI Rank 1: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1211-1220, 95.31%]: CrossEntropyWithSoftmax = 2.25626016 * 640; EvalClassificationError = 0.60625000 * 640; time = 0.1844s; samplesPerSecond = 3470.4
|
||||
MPI Rank 1: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1221-1230, 96.09%]: CrossEntropyWithSoftmax = 2.39720147 * 640; EvalClassificationError = 0.62343750 * 640; time = 0.1859s; samplesPerSecond = 3443.3
|
||||
MPI Rank 1: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1231-1240, 96.88%]: CrossEntropyWithSoftmax = 2.29898501 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1855s; samplesPerSecond = 3449.2
|
||||
MPI Rank 1: 09/17/2016 18:05:51: Epoch[ 1 of 3]-Minibatch[1241-1250, 97.66%]: CrossEntropyWithSoftmax = 2.24887214 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1851s; samplesPerSecond = 3457.3
|
||||
MPI Rank 1: 09/17/2016 18:05:52: Epoch[ 1 of 3]-Minibatch[1251-1260, 98.44%]: CrossEntropyWithSoftmax = 2.27546233 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1856s; samplesPerSecond = 3448.9
|
||||
MPI Rank 1: 09/17/2016 18:05:52: Epoch[ 1 of 3]-Minibatch[1261-1270, 99.22%]: CrossEntropyWithSoftmax = 2.22047744 * 640; EvalClassificationError = 0.60625000 * 640; time = 0.1875s; samplesPerSecond = 3412.9
|
||||
MPI Rank 1: 09/17/2016 18:05:52: Epoch[ 1 of 3]-Minibatch[1271-1280, 100.00%]: CrossEntropyWithSoftmax = 2.30530274 * 640; EvalClassificationError = 0.60000000 * 640; time = 0.1822s; samplesPerSecond = 3512.2
|
||||
MPI Rank 1: 09/17/2016 18:05:52: Finished Epoch[ 1 of 3]: [Training] CrossEntropyWithSoftmax = 3.00766611 * 81920; EvalClassificationError = 0.72410889 * 81920; totalSamplesSeen = 81920; learningRatePerSample = 0.001953125; epochTime=24.1916s
|
||||
MPI Rank 1: 09/17/2016 18:05:53: AdaptiveMinibatchSearch Epoch[2]: Evaluating minibatchSizes 64..8192
|
||||
MPI Rank 1: 09/17/2016 18:05:53: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=64 (search range: 64..8192)...
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:05:53: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:05:53: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 2.02060591 * 640; EvalClassificationError = 0.52343750 * 640; time = 0.2004s; samplesPerSecond = 3193.3
|
||||
MPI Rank 1: 09/17/2016 18:05:53: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 2.04232000 * 640; EvalClassificationError = 0.53437500 * 640; time = 0.1851s; samplesPerSecond = 3456.7
|
||||
MPI Rank 1: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 2.14042914 * 640; EvalClassificationError = 0.59843750 * 640; time = 0.1850s; samplesPerSecond = 3459.5
|
||||
MPI Rank 1: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 2.12163859 * 640; EvalClassificationError = 0.56718750 * 640; time = 0.1844s; samplesPerSecond = 3471.5
|
||||
MPI Rank 1: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 2.05390998 * 640; EvalClassificationError = 0.55625000 * 640; time = 0.1852s; samplesPerSecond = 3456.1
|
||||
MPI Rank 1: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 2.09859542 * 640; EvalClassificationError = 0.55156250 * 640; time = 0.1849s; samplesPerSecond = 3460.5
|
||||
MPI Rank 1: 09/17/2016 18:05:54: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 2.11796400 * 640; EvalClassificationError = 0.56406250 * 640; time = 0.1893s; samplesPerSecond = 3381.7
|
||||
MPI Rank 1: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 2.11659597 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1862s; samplesPerSecond = 3437.1
|
||||
MPI Rank 1: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 2.14159740 * 640; EvalClassificationError = 0.58125000 * 640; time = 0.1852s; samplesPerSecond = 3456.5
|
||||
MPI Rank 1: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 2.13426067 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1845s; samplesPerSecond = 3469.3
|
||||
MPI Rank 1: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.98173765 * 640; EvalClassificationError = 0.54687500 * 640; time = 0.1848s; samplesPerSecond = 3462.8
|
||||
MPI Rank 1: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 2.18123085 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1854s; samplesPerSecond = 3451.5
|
||||
MPI Rank 1: 09/17/2016 18:05:55: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 2.01620029 * 640; EvalClassificationError = 0.53281250 * 640; time = 0.1844s; samplesPerSecond = 3470.7
|
||||
MPI Rank 1: 09/17/2016 18:05:56: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 2.02410759 * 640; EvalClassificationError = 0.55312500 * 640; time = 0.1847s; samplesPerSecond = 3464.6
|
||||
MPI Rank 1: 09/17/2016 18:05:56: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 2.03003578 * 640; EvalClassificationError = 0.51250000 * 640; time = 0.1846s; samplesPerSecond = 3467.9
|
||||
MPI Rank 1: 09/17/2016 18:05:56: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 2.07643576 * 640; EvalClassificationError = 0.58281250 * 640; time = 0.1820s; samplesPerSecond = 3516.5
|
||||
MPI Rank 1: 09/17/2016 18:05:56: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.08110406 * 10240; EvalClassificationError = 0.56123047 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:05:56: AdaptiveMinibatchSearch Epoch[2]: Computed baseCriterion 2.08110406 for minibatchSize=64
|
||||
MPI Rank 1: 09/17/2016 18:05:56: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=64 (search range: 64..8192)...
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:05:56: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:05:56: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 2.02060591 * 640; EvalClassificationError = 0.52343750 * 640; time = 0.1852s; samplesPerSecond = 3454.9
|
||||
MPI Rank 1: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 2.04232000 * 640; EvalClassificationError = 0.53437500 * 640; time = 0.1857s; samplesPerSecond = 3447.2
|
||||
MPI Rank 1: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 2.14042914 * 640; EvalClassificationError = 0.59843750 * 640; time = 0.1848s; samplesPerSecond = 3462.4
|
||||
MPI Rank 1: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 2.12163859 * 640; EvalClassificationError = 0.56718750 * 640; time = 0.1850s; samplesPerSecond = 3460.3
|
||||
MPI Rank 1: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 2.05390998 * 640; EvalClassificationError = 0.55625000 * 640; time = 0.1854s; samplesPerSecond = 3451.8
|
||||
MPI Rank 1: 09/17/2016 18:05:57: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 2.09859542 * 640; EvalClassificationError = 0.55156250 * 640; time = 0.1844s; samplesPerSecond = 3471.2
|
||||
MPI Rank 1: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 2.11796400 * 640; EvalClassificationError = 0.56406250 * 640; time = 0.1854s; samplesPerSecond = 3451.3
|
||||
MPI Rank 1: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 2.11659597 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1857s; samplesPerSecond = 3447.0
|
||||
MPI Rank 1: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 2.14159740 * 640; EvalClassificationError = 0.58125000 * 640; time = 0.1851s; samplesPerSecond = 3457.1
|
||||
MPI Rank 1: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 2.13426067 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1845s; samplesPerSecond = 3469.0
|
||||
MPI Rank 1: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.98173765 * 640; EvalClassificationError = 0.54687500 * 640; time = 0.1846s; samplesPerSecond = 3466.8
|
||||
MPI Rank 1: 09/17/2016 18:05:58: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 2.18123085 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1853s; samplesPerSecond = 3454.2
|
||||
MPI Rank 1: 09/17/2016 18:05:59: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 2.01620029 * 640; EvalClassificationError = 0.53281250 * 640; time = 0.1846s; samplesPerSecond = 3467.7
|
||||
MPI Rank 1: 09/17/2016 18:05:59: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 2.02410759 * 640; EvalClassificationError = 0.55312500 * 640; time = 0.1849s; samplesPerSecond = 3461.3
|
||||
MPI Rank 1: 09/17/2016 18:05:59: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 2.03003578 * 640; EvalClassificationError = 0.51250000 * 640; time = 0.1849s; samplesPerSecond = 3461.5
|
||||
MPI Rank 1: 09/17/2016 18:05:59: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 2.07643576 * 640; EvalClassificationError = 0.58281250 * 640; time = 0.1830s; samplesPerSecond = 3497.5
|
||||
MPI Rank 1: 09/17/2016 18:05:59: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.08110406 * 10240; EvalClassificationError = 0.56123047 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:05:59: AdaptiveMinibatchSearch Epoch[2]: Keep searching... epochCriterion = 2.08110406 vs. baseCriterion = 2.08110406
|
||||
MPI Rank 1: 09/17/2016 18:05:59: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=128 (search range: 64..8192)...
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:05:59: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:00: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 2.02947755 * 1280; EvalClassificationError = 0.52656250 * 1280; time = 0.2634s; samplesPerSecond = 4860.0
|
||||
MPI Rank 1: 09/17/2016 18:06:00: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 3.13%]: CrossEntropyWithSoftmax = 2.13334208 * 1280; EvalClassificationError = 0.58359375 * 1280; time = 0.2424s; samplesPerSecond = 5280.7
|
||||
MPI Rank 1: 09/17/2016 18:06:00: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 2.08344475 * 1280; EvalClassificationError = 0.55781250 * 1280; time = 0.2434s; samplesPerSecond = 5259.6
|
||||
MPI Rank 1: 09/17/2016 18:06:00: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 2.13081897 * 1280; EvalClassificationError = 0.56875000 * 1280; time = 0.2461s; samplesPerSecond = 5202.2
|
||||
MPI Rank 1: 09/17/2016 18:06:01: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 2.15740742 * 1280; EvalClassificationError = 0.58984375 * 1280; time = 0.2426s; samplesPerSecond = 5276.7
|
||||
MPI Rank 1: 09/17/2016 18:06:01: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 2.11081282 * 1280; EvalClassificationError = 0.58125000 * 1280; time = 0.2429s; samplesPerSecond = 5269.1
|
||||
MPI Rank 1: 09/17/2016 18:06:01: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 2.03640631 * 1280; EvalClassificationError = 0.55546875 * 1280; time = 0.2427s; samplesPerSecond = 5273.0
|
||||
MPI Rank 1: 09/17/2016 18:06:01: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 2.06206717 * 1280; EvalClassificationError = 0.55156250 * 1280; time = 0.2380s; samplesPerSecond = 5378.6
|
||||
MPI Rank 1: 09/17/2016 18:06:01: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.09297213 * 10240; EvalClassificationError = 0.56435547 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 128
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:02: AdaptiveMinibatchSearch Epoch[2]: Keep searching... epochCriterion = 2.09297213 vs. baseCriterion = 2.08110406
|
||||
MPI Rank 1: 09/17/2016 18:06:02: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=192 (search range: 64..8192)...
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:02: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:02: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 2.07040051 * 1920; EvalClassificationError = 0.54947917 * 1920; time = 0.3271s; samplesPerSecond = 5869.8
|
||||
MPI Rank 1: 09/17/2016 18:06:02: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 2.12004909 * 1920; EvalClassificationError = 0.56770833 * 1920; time = 0.3046s; samplesPerSecond = 6303.5
|
||||
MPI Rank 1: 09/17/2016 18:06:03: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 2.19270926 * 1920; EvalClassificationError = 0.58750000 * 1920; time = 0.3111s; samplesPerSecond = 6171.5
|
||||
MPI Rank 1: 09/17/2016 18:06:03: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 2.25179093 * 1920; EvalClassificationError = 0.60781250 * 1920; time = 0.3060s; samplesPerSecond = 6274.1
|
||||
MPI Rank 1: 09/17/2016 18:06:03: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 2.45413223 * 1920; EvalClassificationError = 0.59531250 * 1920; time = 0.3095s; samplesPerSecond = 6203.3
|
||||
MPI Rank 1: 09/17/2016 18:06:03: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.23761477 * 10240; EvalClassificationError = 0.58447266 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 192
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:03: AdaptiveMinibatchSearch Epoch[2]: Search successful. New minibatchSize is 128. epochCriterion = 2.09297213 vs baseCriterion = 2.08110406
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:03: Starting Epoch 2: learning rate per sample = 0.001953 effective momentum = 0.882497 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [81920..163840] (first utterance at frame 81920), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:03: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:04: Epoch[ 2 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 2.18100617 * 1280; EvalClassificationError = 0.55546875 * 1280; time = 0.2464s; samplesPerSecond = 5194.5
|
||||
MPI Rank 1: 09/17/2016 18:06:04: Epoch[ 2 of 3]-Minibatch[ 11- 20, 3.13%]: CrossEntropyWithSoftmax = 2.16631393 * 1280; EvalClassificationError = 0.59062500 * 1280; time = 0.2468s; samplesPerSecond = 5187.1
|
||||
MPI Rank 1: 09/17/2016 18:06:04: Epoch[ 2 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 2.23799285 * 1280; EvalClassificationError = 0.60468750 * 1280; time = 0.2471s; samplesPerSecond = 5180.2
|
||||
MPI Rank 1: 09/17/2016 18:06:04: Epoch[ 2 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 2.25188312 * 1280; EvalClassificationError = 0.60703125 * 1280; time = 0.2494s; samplesPerSecond = 5131.7
|
||||
MPI Rank 1: 09/17/2016 18:06:05: Epoch[ 2 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 2.12738463 * 1280; EvalClassificationError = 0.56328125 * 1280; time = 0.2496s; samplesPerSecond = 5128.9
|
||||
MPI Rank 1: 09/17/2016 18:06:05: Epoch[ 2 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 2.09543741 * 1280; EvalClassificationError = 0.58359375 * 1280; time = 0.2474s; samplesPerSecond = 5174.6
|
||||
MPI Rank 1: 09/17/2016 18:06:05: Epoch[ 2 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 2.12457852 * 1280; EvalClassificationError = 0.58906250 * 1280; time = 0.2477s; samplesPerSecond = 5166.5
|
||||
MPI Rank 1: 09/17/2016 18:06:05: Epoch[ 2 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 2.15260337 * 1280; EvalClassificationError = 0.57343750 * 1280; time = 0.2497s; samplesPerSecond = 5125.7
|
||||
MPI Rank 1: 09/17/2016 18:06:06: Epoch[ 2 of 3]-Minibatch[ 81- 90, 14.06%]: CrossEntropyWithSoftmax = 2.07975382 * 1280; EvalClassificationError = 0.55312500 * 1280; time = 0.2464s; samplesPerSecond = 5194.7
|
||||
MPI Rank 1: 09/17/2016 18:06:06: Epoch[ 2 of 3]-Minibatch[ 91- 100, 15.63%]: CrossEntropyWithSoftmax = 2.09557893 * 1280; EvalClassificationError = 0.56328125 * 1280; time = 0.2431s; samplesPerSecond = 5265.3
|
||||
MPI Rank 1: 09/17/2016 18:06:06: Epoch[ 2 of 3]-Minibatch[ 101- 110, 17.19%]: CrossEntropyWithSoftmax = 1.99564992 * 1280; EvalClassificationError = 0.54218750 * 1280; time = 0.2452s; samplesPerSecond = 5220.3
|
||||
MPI Rank 1: 09/17/2016 18:06:06: Epoch[ 2 of 3]-Minibatch[ 111- 120, 18.75%]: CrossEntropyWithSoftmax = 2.01696230 * 1280; EvalClassificationError = 0.53437500 * 1280; time = 0.2441s; samplesPerSecond = 5243.5
|
||||
MPI Rank 1: 09/17/2016 18:06:07: Epoch[ 2 of 3]-Minibatch[ 121- 130, 20.31%]: CrossEntropyWithSoftmax = 2.08247499 * 1280; EvalClassificationError = 0.55625000 * 1280; time = 0.2481s; samplesPerSecond = 5160.0
|
||||
MPI Rank 1: 09/17/2016 18:06:07: Epoch[ 2 of 3]-Minibatch[ 131- 140, 21.88%]: CrossEntropyWithSoftmax = 2.02583127 * 1280; EvalClassificationError = 0.55781250 * 1280; time = 0.2461s; samplesPerSecond = 5201.2
|
||||
MPI Rank 1: 09/17/2016 18:06:07: Epoch[ 2 of 3]-Minibatch[ 141- 150, 23.44%]: CrossEntropyWithSoftmax = 2.12427634 * 1280; EvalClassificationError = 0.57031250 * 1280; time = 0.2456s; samplesPerSecond = 5211.7
|
||||
MPI Rank 1: 09/17/2016 18:06:07: Epoch[ 2 of 3]-Minibatch[ 151- 160, 25.00%]: CrossEntropyWithSoftmax = 1.95297386 * 1280; EvalClassificationError = 0.55234375 * 1280; time = 0.2465s; samplesPerSecond = 5191.8
|
||||
MPI Rank 1: 09/17/2016 18:06:08: Epoch[ 2 of 3]-Minibatch[ 161- 170, 26.56%]: CrossEntropyWithSoftmax = 2.06940792 * 1280; EvalClassificationError = 0.57968750 * 1280; time = 0.2475s; samplesPerSecond = 5172.3
|
||||
MPI Rank 1: 09/17/2016 18:06:08: Epoch[ 2 of 3]-Minibatch[ 171- 180, 28.13%]: CrossEntropyWithSoftmax = 1.97910584 * 1280; EvalClassificationError = 0.53281250 * 1280; time = 0.2455s; samplesPerSecond = 5214.2
|
||||
MPI Rank 1: 09/17/2016 18:06:08: Epoch[ 2 of 3]-Minibatch[ 181- 190, 29.69%]: CrossEntropyWithSoftmax = 1.97550728 * 1280; EvalClassificationError = 0.56484375 * 1280; time = 0.2442s; samplesPerSecond = 5240.7
|
||||
MPI Rank 1: 09/17/2016 18:06:08: Epoch[ 2 of 3]-Minibatch[ 191- 200, 31.25%]: CrossEntropyWithSoftmax = 2.07046879 * 1280; EvalClassificationError = 0.58671875 * 1280; time = 0.2434s; samplesPerSecond = 5259.8
|
||||
MPI Rank 1: 09/17/2016 18:06:09: Epoch[ 2 of 3]-Minibatch[ 201- 210, 32.81%]: CrossEntropyWithSoftmax = 1.94086640 * 1280; EvalClassificationError = 0.54609375 * 1280; time = 0.2469s; samplesPerSecond = 5183.6
|
||||
MPI Rank 1: 09/17/2016 18:06:09: Epoch[ 2 of 3]-Minibatch[ 211- 220, 34.38%]: CrossEntropyWithSoftmax = 1.88656971 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.2440s; samplesPerSecond = 5245.5
|
||||
MPI Rank 1: 09/17/2016 18:06:09: Epoch[ 2 of 3]-Minibatch[ 221- 230, 35.94%]: CrossEntropyWithSoftmax = 1.90888794 * 1280; EvalClassificationError = 0.51640625 * 1280; time = 0.2453s; samplesPerSecond = 5218.8
|
||||
MPI Rank 1: 09/17/2016 18:06:09: Epoch[ 2 of 3]-Minibatch[ 231- 240, 37.50%]: CrossEntropyWithSoftmax = 1.91336087 * 1280; EvalClassificationError = 0.52265625 * 1280; time = 0.2475s; samplesPerSecond = 5171.1
|
||||
MPI Rank 1: 09/17/2016 18:06:10: Epoch[ 2 of 3]-Minibatch[ 241- 250, 39.06%]: CrossEntropyWithSoftmax = 1.89548772 * 1280; EvalClassificationError = 0.52343750 * 1280; time = 0.2480s; samplesPerSecond = 5161.4
|
||||
MPI Rank 1: 09/17/2016 18:06:10: Epoch[ 2 of 3]-Minibatch[ 251- 260, 40.63%]: CrossEntropyWithSoftmax = 1.87667719 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.2459s; samplesPerSecond = 5204.9
|
||||
MPI Rank 1: 09/17/2016 18:06:10: Epoch[ 2 of 3]-Minibatch[ 261- 270, 42.19%]: CrossEntropyWithSoftmax = 1.81964097 * 1280; EvalClassificationError = 0.52421875 * 1280; time = 0.2440s; samplesPerSecond = 5246.6
|
||||
MPI Rank 1: 09/17/2016 18:06:10: Epoch[ 2 of 3]-Minibatch[ 271- 280, 43.75%]: CrossEntropyWithSoftmax = 1.88226903 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.2440s; samplesPerSecond = 5245.5
|
||||
MPI Rank 1: 09/17/2016 18:06:11: Epoch[ 2 of 3]-Minibatch[ 281- 290, 45.31%]: CrossEntropyWithSoftmax = 1.85004922 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.2462s; samplesPerSecond = 5199.3
|
||||
MPI Rank 1: 09/17/2016 18:06:11: Epoch[ 2 of 3]-Minibatch[ 291- 300, 46.88%]: CrossEntropyWithSoftmax = 1.78267871 * 1280; EvalClassificationError = 0.50156250 * 1280; time = 0.2473s; samplesPerSecond = 5176.1
|
||||
MPI Rank 1: 09/17/2016 18:06:11: Epoch[ 2 of 3]-Minibatch[ 301- 310, 48.44%]: CrossEntropyWithSoftmax = 1.81674744 * 1280; EvalClassificationError = 0.50468750 * 1280; time = 0.2445s; samplesPerSecond = 5234.3
|
||||
MPI Rank 1: 09/17/2016 18:06:11: Epoch[ 2 of 3]-Minibatch[ 311- 320, 50.00%]: CrossEntropyWithSoftmax = 1.76854193 * 1280; EvalClassificationError = 0.49062500 * 1280; time = 0.2446s; samplesPerSecond = 5233.3
|
||||
MPI Rank 1: 09/17/2016 18:06:12: Epoch[ 2 of 3]-Minibatch[ 321- 330, 51.56%]: CrossEntropyWithSoftmax = 1.81330268 * 1280; EvalClassificationError = 0.51171875 * 1280; time = 0.2445s; samplesPerSecond = 5235.1
|
||||
MPI Rank 1: 09/17/2016 18:06:12: Epoch[ 2 of 3]-Minibatch[ 331- 340, 53.13%]: CrossEntropyWithSoftmax = 1.81775001 * 1280; EvalClassificationError = 0.51640625 * 1280; time = 0.2430s; samplesPerSecond = 5266.8
|
||||
MPI Rank 1: 09/17/2016 18:06:12: Epoch[ 2 of 3]-Minibatch[ 341- 350, 54.69%]: CrossEntropyWithSoftmax = 1.79768261 * 1280; EvalClassificationError = 0.50546875 * 1280; time = 0.2464s; samplesPerSecond = 5194.2
|
||||
MPI Rank 1: 09/17/2016 18:06:12: Epoch[ 2 of 3]-Minibatch[ 351- 360, 56.25%]: CrossEntropyWithSoftmax = 1.78876937 * 1280; EvalClassificationError = 0.49765625 * 1280; time = 0.2437s; samplesPerSecond = 5252.5
|
||||
MPI Rank 1: 09/17/2016 18:06:13: Epoch[ 2 of 3]-Minibatch[ 361- 370, 57.81%]: CrossEntropyWithSoftmax = 1.78753263 * 1280; EvalClassificationError = 0.51015625 * 1280; time = 0.2445s; samplesPerSecond = 5236.1
|
||||
MPI Rank 1: 09/17/2016 18:06:13: Epoch[ 2 of 3]-Minibatch[ 371- 380, 59.38%]: CrossEntropyWithSoftmax = 1.74071233 * 1280; EvalClassificationError = 0.49140625 * 1280; time = 0.2436s; samplesPerSecond = 5254.8
|
||||
MPI Rank 1: 09/17/2016 18:06:13: Epoch[ 2 of 3]-Minibatch[ 381- 390, 60.94%]: CrossEntropyWithSoftmax = 1.71575901 * 1280; EvalClassificationError = 0.46875000 * 1280; time = 0.2437s; samplesPerSecond = 5252.5
|
||||
MPI Rank 1: 09/17/2016 18:06:13: Epoch[ 2 of 3]-Minibatch[ 391- 400, 62.50%]: CrossEntropyWithSoftmax = 1.76465781 * 1280; EvalClassificationError = 0.49765625 * 1280; time = 0.2450s; samplesPerSecond = 5224.9
|
||||
MPI Rank 1: 09/17/2016 18:06:14: Epoch[ 2 of 3]-Minibatch[ 401- 410, 64.06%]: CrossEntropyWithSoftmax = 1.76532949 * 1280; EvalClassificationError = 0.51406250 * 1280; time = 0.2468s; samplesPerSecond = 5185.9
|
||||
MPI Rank 1: 09/17/2016 18:06:14: Epoch[ 2 of 3]-Minibatch[ 411- 420, 65.63%]: CrossEntropyWithSoftmax = 1.79718711 * 1280; EvalClassificationError = 0.50390625 * 1280; time = 0.2419s; samplesPerSecond = 5291.7
|
||||
MPI Rank 1: 09/17/2016 18:06:14: Epoch[ 2 of 3]-Minibatch[ 421- 430, 67.19%]: CrossEntropyWithSoftmax = 1.74168655 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.2422s; samplesPerSecond = 5284.5
|
||||
MPI Rank 1: 09/17/2016 18:06:14: Epoch[ 2 of 3]-Minibatch[ 431- 440, 68.75%]: CrossEntropyWithSoftmax = 1.73594884 * 1280; EvalClassificationError = 0.49609375 * 1280; time = 0.2532s; samplesPerSecond = 5055.9
|
||||
MPI Rank 1: 09/17/2016 18:06:15: Epoch[ 2 of 3]-Minibatch[ 441- 450, 70.31%]: CrossEntropyWithSoftmax = 1.75234022 * 1280; EvalClassificationError = 0.50859375 * 1280; time = 0.2429s; samplesPerSecond = 5268.7
|
||||
MPI Rank 1: 09/17/2016 18:06:15: Epoch[ 2 of 3]-Minibatch[ 451- 460, 71.88%]: CrossEntropyWithSoftmax = 1.64950906 * 1280; EvalClassificationError = 0.48281250 * 1280; time = 0.2427s; samplesPerSecond = 5274.4
|
||||
MPI Rank 1: 09/17/2016 18:06:15: Epoch[ 2 of 3]-Minibatch[ 461- 470, 73.44%]: CrossEntropyWithSoftmax = 1.72111861 * 1280; EvalClassificationError = 0.49921875 * 1280; time = 0.2444s; samplesPerSecond = 5237.4
|
||||
MPI Rank 1: 09/17/2016 18:06:15: Epoch[ 2 of 3]-Minibatch[ 471- 480, 75.00%]: CrossEntropyWithSoftmax = 1.75491334 * 1280; EvalClassificationError = 0.50312500 * 1280; time = 0.2454s; samplesPerSecond = 5215.8
|
||||
MPI Rank 1: 09/17/2016 18:06:16: Epoch[ 2 of 3]-Minibatch[ 481- 490, 76.56%]: CrossEntropyWithSoftmax = 1.68324400 * 1280; EvalClassificationError = 0.48281250 * 1280; time = 0.2439s; samplesPerSecond = 5248.7
|
||||
MPI Rank 1: 09/17/2016 18:06:16: Epoch[ 2 of 3]-Minibatch[ 491- 500, 78.13%]: CrossEntropyWithSoftmax = 1.71507576 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.2439s; samplesPerSecond = 5247.5
|
||||
MPI Rank 1: 09/17/2016 18:06:16: Epoch[ 2 of 3]-Minibatch[ 501- 510, 79.69%]: CrossEntropyWithSoftmax = 1.65489209 * 1280; EvalClassificationError = 0.45937500 * 1280; time = 0.2468s; samplesPerSecond = 5186.9
|
||||
MPI Rank 1: 09/17/2016 18:06:16: Epoch[ 2 of 3]-Minibatch[ 511- 520, 81.25%]: CrossEntropyWithSoftmax = 1.70993974 * 1280; EvalClassificationError = 0.48984375 * 1280; time = 0.2428s; samplesPerSecond = 5271.2
|
||||
MPI Rank 1: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 521- 530, 82.81%]: CrossEntropyWithSoftmax = 1.68373330 * 1280; EvalClassificationError = 0.48046875 * 1280; time = 0.2443s; samplesPerSecond = 5240.5
|
||||
MPI Rank 1: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 531- 540, 84.38%]: CrossEntropyWithSoftmax = 1.68961559 * 1280; EvalClassificationError = 0.48671875 * 1280; time = 0.2448s; samplesPerSecond = 5228.0
|
||||
MPI Rank 1: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 541- 550, 85.94%]: CrossEntropyWithSoftmax = 1.70437375 * 1280; EvalClassificationError = 0.48984375 * 1280; time = 0.2437s; samplesPerSecond = 5252.4
|
||||
MPI Rank 1: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 551- 560, 87.50%]: CrossEntropyWithSoftmax = 1.69558061 * 1280; EvalClassificationError = 0.48906250 * 1280; time = 0.2422s; samplesPerSecond = 5285.2
|
||||
MPI Rank 1: 09/17/2016 18:06:17: Epoch[ 2 of 3]-Minibatch[ 561- 570, 89.06%]: CrossEntropyWithSoftmax = 1.69535392 * 1280; EvalClassificationError = 0.48359375 * 1280; time = 0.2428s; samplesPerSecond = 5272.9
|
||||
MPI Rank 1: 09/17/2016 18:06:18: Epoch[ 2 of 3]-Minibatch[ 571- 580, 90.63%]: CrossEntropyWithSoftmax = 1.65016334 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.2431s; samplesPerSecond = 5264.9
|
||||
MPI Rank 1: 09/17/2016 18:06:18: Epoch[ 2 of 3]-Minibatch[ 581- 590, 92.19%]: CrossEntropyWithSoftmax = 1.64953906 * 1280; EvalClassificationError = 0.48515625 * 1280; time = 0.2407s; samplesPerSecond = 5318.9
|
||||
MPI Rank 1: 09/17/2016 18:06:18: Epoch[ 2 of 3]-Minibatch[ 591- 600, 93.75%]: CrossEntropyWithSoftmax = 1.64390878 * 1280; EvalClassificationError = 0.48125000 * 1280; time = 0.2438s; samplesPerSecond = 5251.2
|
||||
MPI Rank 1: 09/17/2016 18:06:18: Epoch[ 2 of 3]-Minibatch[ 601- 610, 95.31%]: CrossEntropyWithSoftmax = 1.71970142 * 1280; EvalClassificationError = 0.51093750 * 1280; time = 0.2409s; samplesPerSecond = 5312.7
|
||||
MPI Rank 1: 09/17/2016 18:06:19: Epoch[ 2 of 3]-Minibatch[ 611- 620, 96.88%]: CrossEntropyWithSoftmax = 1.62454036 * 1280; EvalClassificationError = 0.46562500 * 1280; time = 0.2426s; samplesPerSecond = 5275.7
|
||||
MPI Rank 1: 09/17/2016 18:06:19: Epoch[ 2 of 3]-Minibatch[ 621- 630, 98.44%]: CrossEntropyWithSoftmax = 1.59147885 * 1280; EvalClassificationError = 0.46875000 * 1280; time = 0.2424s; samplesPerSecond = 5280.5
|
||||
MPI Rank 1: 09/17/2016 18:06:19: Epoch[ 2 of 3]-Minibatch[ 631- 640, 100.00%]: CrossEntropyWithSoftmax = 1.66373476 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.2351s; samplesPerSecond = 5443.9
|
||||
MPI Rank 1: 09/17/2016 18:06:19: Finished Epoch[ 2 of 3]: [Training] CrossEntropyWithSoftmax = 1.86040693 * 81920; EvalClassificationError = 0.51970215 * 81920; totalSamplesSeen = 163840; learningRatePerSample = 0.001953125; epochTime=26.1885s
|
||||
MPI Rank 1: 09/17/2016 18:06:19: AdaptiveMinibatchSearch Epoch[3]: Evaluating minibatchSizes 64..256
|
||||
MPI Rank 1: 09/17/2016 18:06:19: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=64 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:19: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 1.60900971 * 640; EvalClassificationError = 0.46875000 * 640; time = 0.1868s; samplesPerSecond = 3425.9
|
||||
MPI Rank 1: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 1.70269498 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1847s; samplesPerSecond = 3465.4
|
||||
MPI Rank 1: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 1.66433597 * 640; EvalClassificationError = 0.48281250 * 640; time = 0.1850s; samplesPerSecond = 3459.8
|
||||
MPI Rank 1: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 1.56997722 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1839s; samplesPerSecond = 3479.3
|
||||
MPI Rank 1: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 1.67908876 * 640; EvalClassificationError = 0.47656250 * 640; time = 0.1851s; samplesPerSecond = 3457.3
|
||||
MPI Rank 1: 09/17/2016 18:06:20: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 1.45189750 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1857s; samplesPerSecond = 3446.5
|
||||
MPI Rank 1: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 1.60053473 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1846s; samplesPerSecond = 3467.0
|
||||
MPI Rank 1: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 1.58536943 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1852s; samplesPerSecond = 3455.6
|
||||
MPI Rank 1: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 1.63246860 * 640; EvalClassificationError = 0.45781250 * 640; time = 0.1839s; samplesPerSecond = 3480.2
|
||||
MPI Rank 1: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 1.64770593 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1851s; samplesPerSecond = 3457.0
|
||||
MPI Rank 1: 09/17/2016 18:06:21: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.55685563 * 640; EvalClassificationError = 0.43750000 * 640; time = 0.1855s; samplesPerSecond = 3450.8
|
||||
MPI Rank 1: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 1.54493706 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1857s; samplesPerSecond = 3446.0
|
||||
MPI Rank 1: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 1.59221509 * 640; EvalClassificationError = 0.47031250 * 640; time = 0.1845s; samplesPerSecond = 3468.9
|
||||
MPI Rank 1: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 1.59923263 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1848s; samplesPerSecond = 3462.4
|
||||
MPI Rank 1: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 1.60345937 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1842s; samplesPerSecond = 3474.5
|
||||
MPI Rank 1: 09/17/2016 18:06:22: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 1.59599131 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1812s; samplesPerSecond = 3531.7
|
||||
MPI Rank 1: 09/17/2016 18:06:22: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60223587 * 10240; EvalClassificationError = 0.45957031 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:23: AdaptiveMinibatchSearch Epoch[3]: Computed baseCriterion 1.60223587 for minibatchSize=64
|
||||
MPI Rank 1: 09/17/2016 18:06:23: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=64 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:23: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 1.60900971 * 640; EvalClassificationError = 0.46875000 * 640; time = 0.1896s; samplesPerSecond = 3376.3
|
||||
MPI Rank 1: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 1.70269498 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1843s; samplesPerSecond = 3471.7
|
||||
MPI Rank 1: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 1.66433597 * 640; EvalClassificationError = 0.48281250 * 640; time = 0.1852s; samplesPerSecond = 3455.8
|
||||
MPI Rank 1: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 3.13%]: CrossEntropyWithSoftmax = 1.56997722 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1839s; samplesPerSecond = 3480.9
|
||||
MPI Rank 1: 09/17/2016 18:06:23: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 1.67908876 * 640; EvalClassificationError = 0.47656250 * 640; time = 0.1846s; samplesPerSecond = 3466.1
|
||||
MPI Rank 1: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 1.45189750 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1854s; samplesPerSecond = 3451.3
|
||||
MPI Rank 1: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 1.60053473 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1838s; samplesPerSecond = 3481.9
|
||||
MPI Rank 1: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 1.58536943 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1856s; samplesPerSecond = 3448.6
|
||||
MPI Rank 1: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 1.63246860 * 640; EvalClassificationError = 0.45781250 * 640; time = 0.1841s; samplesPerSecond = 3475.8
|
||||
MPI Rank 1: 09/17/2016 18:06:24: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 1.64770593 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1851s; samplesPerSecond = 3457.0
|
||||
MPI Rank 1: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.55685563 * 640; EvalClassificationError = 0.43750000 * 640; time = 0.1864s; samplesPerSecond = 3433.6
|
||||
MPI Rank 1: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 1.54493706 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1847s; samplesPerSecond = 3465.4
|
||||
MPI Rank 1: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 1.59221509 * 640; EvalClassificationError = 0.47031250 * 640; time = 0.1846s; samplesPerSecond = 3467.0
|
||||
MPI Rank 1: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 1.59923263 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1856s; samplesPerSecond = 3447.8
|
||||
MPI Rank 1: 09/17/2016 18:06:25: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 1.60345937 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1846s; samplesPerSecond = 3466.6
|
||||
MPI Rank 1: 09/17/2016 18:06:26: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 1.59599131 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1827s; samplesPerSecond = 3502.4
|
||||
MPI Rank 1: 09/17/2016 18:06:26: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60223587 * 10240; EvalClassificationError = 0.45957031 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:26: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.60223587 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 1: 09/17/2016 18:06:26: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=128 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:26: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:26: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 1.59906005 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.2461s; samplesPerSecond = 5202.1
|
||||
MPI Rank 1: 09/17/2016 18:06:26: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 3.13%]: CrossEntropyWithSoftmax = 1.59088280 * 1280; EvalClassificationError = 0.47031250 * 1280; time = 0.2428s; samplesPerSecond = 5272.3
|
||||
MPI Rank 1: 09/17/2016 18:06:26: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 1.55611991 * 1280; EvalClassificationError = 0.45468750 * 1280; time = 0.2463s; samplesPerSecond = 5197.5
|
||||
MPI Rank 1: 09/17/2016 18:06:27: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 1.59494809 * 1280; EvalClassificationError = 0.45937500 * 1280; time = 0.2441s; samplesPerSecond = 5243.4
|
||||
MPI Rank 1: 09/17/2016 18:06:27: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 1.64834247 * 1280; EvalClassificationError = 0.45703125 * 1280; time = 0.2435s; samplesPerSecond = 5256.7
|
||||
MPI Rank 1: 09/17/2016 18:06:27: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 1.56066565 * 1280; EvalClassificationError = 0.43593750 * 1280; time = 0.2437s; samplesPerSecond = 5252.2
|
||||
MPI Rank 1: 09/17/2016 18:06:27: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 1.62413186 * 1280; EvalClassificationError = 0.47265625 * 1280; time = 0.2459s; samplesPerSecond = 5205.5
|
||||
MPI Rank 1: 09/17/2016 18:06:28: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 1.67799703 * 1280; EvalClassificationError = 0.47031250 * 1280; time = 0.2386s; samplesPerSecond = 5365.1
|
||||
MPI Rank 1: 09/17/2016 18:06:28: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60651848 * 10240; EvalClassificationError = 0.46044922 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 128
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:28: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.60651848 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 1: 09/17/2016 18:06:28: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=192 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:28: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:28: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 1.59151349 * 1920; EvalClassificationError = 0.46302083 * 1920; time = 0.3187s; samplesPerSecond = 6024.9
|
||||
MPI Rank 1: 09/17/2016 18:06:29: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 1.55305006 * 1920; EvalClassificationError = 0.45000000 * 1920; time = 0.3119s; samplesPerSecond = 6155.8
|
||||
MPI Rank 1: 09/17/2016 18:06:29: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 1.60654814 * 1920; EvalClassificationError = 0.45781250 * 1920; time = 0.3122s; samplesPerSecond = 6150.8
|
||||
MPI Rank 1: 09/17/2016 18:06:29: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 1.60526168 * 1920; EvalClassificationError = 0.44947917 * 1920; time = 0.3090s; samplesPerSecond = 6213.6
|
||||
MPI Rank 1: 09/17/2016 18:06:29: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 1.68437187 * 1920; EvalClassificationError = 0.49531250 * 1920; time = 0.3134s; samplesPerSecond = 6125.4
|
||||
MPI Rank 1: 09/17/2016 18:06:30: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.61407118 * 10240; EvalClassificationError = 0.46357422 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 192
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:30: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.61407118 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 1: 09/17/2016 18:06:30: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=256 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:30: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:30: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 3.13%]: CrossEntropyWithSoftmax = 1.59241097 * 2560; EvalClassificationError = 0.46093750 * 2560; time = 0.3891s; samplesPerSecond = 6579.3
|
||||
MPI Rank 1: 09/17/2016 18:06:31: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 6.25%]: CrossEntropyWithSoftmax = 1.59578299 * 2560; EvalClassificationError = 0.46093750 * 2560; time = 0.3757s; samplesPerSecond = 6814.3
|
||||
MPI Rank 1: 09/17/2016 18:06:31: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 9.38%]: CrossEntropyWithSoftmax = 1.68033748 * 2560; EvalClassificationError = 0.47343750 * 2560; time = 0.3717s; samplesPerSecond = 6886.8
|
||||
MPI Rank 1: 09/17/2016 18:06:31: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 12.50%]: CrossEntropyWithSoftmax = 1.73124308 * 2560; EvalClassificationError = 0.48164062 * 2560; time = 0.3624s; samplesPerSecond = 7063.6
|
||||
MPI Rank 1: 09/17/2016 18:06:31: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.64994363 * 10240; EvalClassificationError = 0.46923828 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 256
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 18:06:32: AdaptiveMinibatchSearch Epoch[3]: Search successful. New minibatchSize is 192. epochCriterion = 1.61407118 vs baseCriterion = 1.60223587
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:32: Starting Epoch 3: learning rate per sample = 0.001953 effective momentum = 0.829029 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [163840..245760] (first utterance at frame 163840), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:32: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 18:06:32: Epoch[ 3 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 1.54988471 * 1920; EvalClassificationError = 0.46510417 * 1920; time = 0.3189s; samplesPerSecond = 6020.3
|
||||
MPI Rank 1: 09/17/2016 18:06:32: Epoch[ 3 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 1.62737285 * 1920; EvalClassificationError = 0.47395833 * 1920; time = 0.3143s; samplesPerSecond = 6109.1
|
||||
MPI Rank 1: 09/17/2016 18:06:32: Epoch[ 3 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 1.65385199 * 1920; EvalClassificationError = 0.46302083 * 1920; time = 0.3126s; samplesPerSecond = 6142.6
|
||||
MPI Rank 1: 09/17/2016 18:06:33: Epoch[ 3 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 1.87023223 * 1920; EvalClassificationError = 0.51666667 * 1920; time = 0.3178s; samplesPerSecond = 6042.4
|
||||
MPI Rank 1: 09/17/2016 18:06:33: Epoch[ 3 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 1.75815123 * 1920; EvalClassificationError = 0.49270833 * 1920; time = 0.3196s; samplesPerSecond = 6007.0
|
||||
MPI Rank 1: 09/17/2016 18:06:33: Epoch[ 3 of 3]-Minibatch[ 51- 60, 14.06%]: CrossEntropyWithSoftmax = 2.03099563 * 1920; EvalClassificationError = 0.50312500 * 1920; time = 0.3100s; samplesPerSecond = 6194.4
|
||||
MPI Rank 1: 09/17/2016 18:06:34: Epoch[ 3 of 3]-Minibatch[ 61- 70, 16.41%]: CrossEntropyWithSoftmax = 1.86165460 * 1920; EvalClassificationError = 0.53489583 * 1920; time = 0.3146s; samplesPerSecond = 6102.9
|
||||
MPI Rank 1: 09/17/2016 18:06:34: Epoch[ 3 of 3]-Minibatch[ 71- 80, 18.75%]: CrossEntropyWithSoftmax = 2.15179657 * 1920; EvalClassificationError = 0.49010417 * 1920; time = 0.3124s; samplesPerSecond = 6146.4
|
||||
MPI Rank 1: 09/17/2016 18:06:34: Epoch[ 3 of 3]-Minibatch[ 81- 90, 21.09%]: CrossEntropyWithSoftmax = 1.79956105 * 1920; EvalClassificationError = 0.50625000 * 1920; time = 0.3130s; samplesPerSecond = 6135.1
|
||||
MPI Rank 1: 09/17/2016 18:06:35: Epoch[ 3 of 3]-Minibatch[ 91- 100, 23.44%]: CrossEntropyWithSoftmax = 1.80123726 * 1920; EvalClassificationError = 0.48385417 * 1920; time = 0.3165s; samplesPerSecond = 6066.3
|
||||
MPI Rank 1: 09/17/2016 18:06:35: Epoch[ 3 of 3]-Minibatch[ 101- 110, 25.78%]: CrossEntropyWithSoftmax = 1.65520548 * 1920; EvalClassificationError = 0.47135417 * 1920; time = 0.3132s; samplesPerSecond = 6130.0
|
||||
MPI Rank 1: 09/17/2016 18:06:35: Epoch[ 3 of 3]-Minibatch[ 111- 120, 28.13%]: CrossEntropyWithSoftmax = 1.65536374 * 1920; EvalClassificationError = 0.46875000 * 1920; time = 0.3199s; samplesPerSecond = 6001.3
|
||||
MPI Rank 1: 09/17/2016 18:06:36: Epoch[ 3 of 3]-Minibatch[ 121- 130, 30.47%]: CrossEntropyWithSoftmax = 1.62640983 * 1920; EvalClassificationError = 0.47656250 * 1920; time = 0.3133s; samplesPerSecond = 6128.2
|
||||
MPI Rank 1: 09/17/2016 18:06:36: Epoch[ 3 of 3]-Minibatch[ 131- 140, 32.81%]: CrossEntropyWithSoftmax = 1.58072809 * 1920; EvalClassificationError = 0.44895833 * 1920; time = 0.3138s; samplesPerSecond = 6117.8
|
||||
MPI Rank 1: 09/17/2016 18:06:36: Epoch[ 3 of 3]-Minibatch[ 141- 150, 35.16%]: CrossEntropyWithSoftmax = 1.56697558 * 1920; EvalClassificationError = 0.44687500 * 1920; time = 0.3131s; samplesPerSecond = 6132.6
|
||||
MPI Rank 1: 09/17/2016 18:06:37: Epoch[ 3 of 3]-Minibatch[ 151- 160, 37.50%]: CrossEntropyWithSoftmax = 1.58252868 * 1920; EvalClassificationError = 0.46250000 * 1920; time = 0.3271s; samplesPerSecond = 5870.1
|
||||
MPI Rank 1: 09/17/2016 18:06:37: Epoch[ 3 of 3]-Minibatch[ 161- 170, 39.84%]: CrossEntropyWithSoftmax = 1.53206179 * 1920; EvalClassificationError = 0.45104167 * 1920; time = 0.3250s; samplesPerSecond = 5908.0
|
||||
MPI Rank 1: 09/17/2016 18:06:37: Epoch[ 3 of 3]-Minibatch[ 171- 180, 42.19%]: CrossEntropyWithSoftmax = 1.47925397 * 1920; EvalClassificationError = 0.44531250 * 1920; time = 0.3154s; samplesPerSecond = 6088.2
|
||||
MPI Rank 1: 09/17/2016 18:06:38: Epoch[ 3 of 3]-Minibatch[ 181- 190, 44.53%]: CrossEntropyWithSoftmax = 1.46468817 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.3127s; samplesPerSecond = 6139.2
|
||||
MPI Rank 1: 09/17/2016 18:06:38: Epoch[ 3 of 3]-Minibatch[ 191- 200, 46.88%]: CrossEntropyWithSoftmax = 1.51778272 * 1920; EvalClassificationError = 0.45468750 * 1920; time = 0.3098s; samplesPerSecond = 6197.7
|
||||
MPI Rank 1: 09/17/2016 18:06:38: Epoch[ 3 of 3]-Minibatch[ 201- 210, 49.22%]: CrossEntropyWithSoftmax = 1.43521243 * 1920; EvalClassificationError = 0.41979167 * 1920; time = 0.3106s; samplesPerSecond = 6181.5
|
||||
MPI Rank 1: 09/17/2016 18:06:38: Epoch[ 3 of 3]-Minibatch[ 211- 220, 51.56%]: CrossEntropyWithSoftmax = 1.55602002 * 1920; EvalClassificationError = 0.45468750 * 1920; time = 0.3132s; samplesPerSecond = 6129.5
|
||||
MPI Rank 1: 09/17/2016 18:06:39: Epoch[ 3 of 3]-Minibatch[ 221- 230, 53.91%]: CrossEntropyWithSoftmax = 1.48434301 * 1920; EvalClassificationError = 0.44479167 * 1920; time = 0.3085s; samplesPerSecond = 6224.5
|
||||
MPI Rank 1: 09/17/2016 18:06:39: Epoch[ 3 of 3]-Minibatch[ 231- 240, 56.25%]: CrossEntropyWithSoftmax = 1.47880634 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.3094s; samplesPerSecond = 6205.0
|
||||
MPI Rank 1: 09/17/2016 18:06:39: Epoch[ 3 of 3]-Minibatch[ 241- 250, 58.59%]: CrossEntropyWithSoftmax = 1.44452798 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.3088s; samplesPerSecond = 6218.1
|
||||
MPI Rank 1: 09/17/2016 18:06:40: Epoch[ 3 of 3]-Minibatch[ 251- 260, 60.94%]: CrossEntropyWithSoftmax = 1.42272624 * 1920; EvalClassificationError = 0.42395833 * 1920; time = 0.3173s; samplesPerSecond = 6050.7
|
||||
MPI Rank 1: 09/17/2016 18:06:40: Epoch[ 3 of 3]-Minibatch[ 261- 270, 63.28%]: CrossEntropyWithSoftmax = 1.39175020 * 1920; EvalClassificationError = 0.41354167 * 1920; time = 0.3085s; samplesPerSecond = 6224.7
|
||||
MPI Rank 1: 09/17/2016 18:06:40: Epoch[ 3 of 3]-Minibatch[ 271- 280, 65.63%]: CrossEntropyWithSoftmax = 1.47323044 * 1920; EvalClassificationError = 0.43333333 * 1920; time = 0.3119s; samplesPerSecond = 6156.3
|
||||
MPI Rank 1: 09/17/2016 18:06:41: Epoch[ 3 of 3]-Minibatch[ 281- 290, 67.97%]: CrossEntropyWithSoftmax = 1.46072580 * 1920; EvalClassificationError = 0.41666667 * 1920; time = 0.3142s; samplesPerSecond = 6110.8
|
||||
MPI Rank 1: 09/17/2016 18:06:41: Epoch[ 3 of 3]-Minibatch[ 291- 300, 70.31%]: CrossEntropyWithSoftmax = 1.47105355 * 1920; EvalClassificationError = 0.43177083 * 1920; time = 0.3102s; samplesPerSecond = 6189.1
|
||||
MPI Rank 1: 09/17/2016 18:06:41: Epoch[ 3 of 3]-Minibatch[ 301- 310, 72.66%]: CrossEntropyWithSoftmax = 1.40973818 * 1920; EvalClassificationError = 0.41927083 * 1920; time = 0.3079s; samplesPerSecond = 6236.6
|
||||
MPI Rank 1: 09/17/2016 18:06:42: Epoch[ 3 of 3]-Minibatch[ 311- 320, 75.00%]: CrossEntropyWithSoftmax = 1.44253894 * 1920; EvalClassificationError = 0.42604167 * 1920; time = 0.3103s; samplesPerSecond = 6188.2
|
||||
MPI Rank 1: 09/17/2016 18:06:42: Epoch[ 3 of 3]-Minibatch[ 321- 330, 77.34%]: CrossEntropyWithSoftmax = 1.39941047 * 1920; EvalClassificationError = 0.42239583 * 1920; time = 0.3107s; samplesPerSecond = 6179.6
|
||||
MPI Rank 1: 09/17/2016 18:06:42: Epoch[ 3 of 3]-Minibatch[ 331- 340, 79.69%]: CrossEntropyWithSoftmax = 1.42524482 * 1920; EvalClassificationError = 0.41927083 * 1920; time = 0.3069s; samplesPerSecond = 6256.1
|
||||
MPI Rank 1: 09/17/2016 18:06:43: Epoch[ 3 of 3]-Minibatch[ 341- 350, 82.03%]: CrossEntropyWithSoftmax = 1.46079356 * 1920; EvalClassificationError = 0.42916667 * 1920; time = 0.3106s; samplesPerSecond = 6182.2
|
||||
MPI Rank 1: 09/17/2016 18:06:43: Epoch[ 3 of 3]-Minibatch[ 351- 360, 84.38%]: CrossEntropyWithSoftmax = 1.39608704 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.3103s; samplesPerSecond = 6186.9
|
||||
MPI Rank 1: 09/17/2016 18:06:43: Epoch[ 3 of 3]-Minibatch[ 361- 370, 86.72%]: CrossEntropyWithSoftmax = 1.46624909 * 1920; EvalClassificationError = 0.43177083 * 1920; time = 0.3074s; samplesPerSecond = 6246.2
|
||||
MPI Rank 1: 09/17/2016 18:06:43: Epoch[ 3 of 3]-Minibatch[ 371- 380, 89.06%]: CrossEntropyWithSoftmax = 1.42432290 * 1920; EvalClassificationError = 0.42031250 * 1920; time = 0.3098s; samplesPerSecond = 6196.7
|
||||
MPI Rank 1: 09/17/2016 18:06:44: Epoch[ 3 of 3]-Minibatch[ 381- 390, 91.41%]: CrossEntropyWithSoftmax = 1.36453678 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.3092s; samplesPerSecond = 6209.1
|
||||
MPI Rank 1: 09/17/2016 18:06:44: Epoch[ 3 of 3]-Minibatch[ 391- 400, 93.75%]: CrossEntropyWithSoftmax = 1.38669452 * 1920; EvalClassificationError = 0.41041667 * 1920; time = 0.3097s; samplesPerSecond = 6199.9
|
||||
MPI Rank 1: 09/17/2016 18:06:44: Epoch[ 3 of 3]-Minibatch[ 401- 410, 96.09%]: CrossEntropyWithSoftmax = 1.37926773 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.3137s; samplesPerSecond = 6120.7
|
||||
MPI Rank 1: 09/17/2016 18:06:45: Epoch[ 3 of 3]-Minibatch[ 411- 420, 98.44%]: CrossEntropyWithSoftmax = 1.39890438 * 1920; EvalClassificationError = 0.41875000 * 1920; time = 0.3244s; samplesPerSecond = 5919.1
|
||||
MPI Rank 1: 09/17/2016 18:06:45: Finished Epoch[ 3 of 3]: [Training] CrossEntropyWithSoftmax = 1.55322819 * 81920; EvalClassificationError = 0.44776611 * 81920; totalSamplesSeen = 245760; learningRatePerSample = 0.001953125; epochTime=25.571s
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:45: Action "train" complete.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 18:06:45: __COMPLETED__
|
||||
MPI Rank 1: ~MPIWrapper
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,997 @@
|
|||
CPU info:
|
||||
CPU Model Name: Intel(R) Xeon(R) CPU E5-2630 v2 @ 2.60GHz
|
||||
Hardware threads: 24
|
||||
Total Memory: 264172964 kB
|
||||
-------------------------------------------------------------------
|
||||
=== Running mpiexec -n 2 /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/1bitsgd/debug/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data RunDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling OutputDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=12 stderr=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
CNTK 1.7+ (HEAD b8eb6d, Sep 17 2016 22:48:35) on localhost at 2016/09/17 22:50:40
|
||||
|
||||
/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/1bitsgd/debug/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data RunDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling OutputDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=trueCNTK 1.7+ (HEAD b8eb6d, Sep 17 2016 22:48:35) on localhost at 2016/09/17 22:50:40
|
||||
|
||||
/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/1bitsgd/debug/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data RunDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling OutputDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=12 stderr=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
numCPUThreads=12 stderr=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
Changed current directory to /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data
|
||||
Changed current directory to /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data
|
||||
MPIWrapper: initializing MPI
|
||||
MPIWrapper: initializing MPI
|
||||
ping [requestnodes (before change)]: 2 nodes pinging each other
|
||||
ping [requestnodes (before change)]: 2 nodes pinging each other
|
||||
ping [requestnodes (before change)]: all 2 nodes responded
|
||||
requestnodes [MPIWrapper]: using 2 out of 2 MPI nodes (2 requested); we (1) are in (participating)
|
||||
ping [requestnodes (after change)]: 2 nodes pinging each other
|
||||
ping [requestnodes (before change)]: all 2 nodes responded
|
||||
requestnodes [MPIWrapper]: using 2 out of 2 MPI nodes (2 requested); we (0) are in (participating)
|
||||
ping [requestnodes (after change)]: 2 nodes pinging each other
|
||||
ping [requestnodes (after change)]: all 2 nodes responded
|
||||
mpihelper: we are cog 0 in a gearbox of 2
|
||||
ping [mpihelper]: 2 nodes pinging each other
|
||||
ping [mpihelper]: all 2 nodes responded
|
||||
ping [requestnodes (after change)]: all 2 nodes responded
|
||||
mpihelper: we are cog 1 in a gearbox of 2
|
||||
ping [mpihelper]: 2 nodes pinging each other
|
||||
ping [mpihelper]: all 2 nodes responded
|
||||
09/17/2016 22:50:42: Redirecting stderr to file /tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr_speechTrain.logrank0
|
||||
09/17/2016 22:50:42: Redirecting stderr to file /tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr_speechTrain.logrank1
|
||||
MPI Rank 0: CNTK 1.7+ (HEAD b8eb6d, Sep 17 2016 22:48:35) on localhost at 2016/09/17 22:50:40
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/1bitsgd/debug/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data RunDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling OutputDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=12 stderr=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Using 12 CPU threads.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:42: ##############################################################################
|
||||
MPI Rank 0: 09/17/2016 22:50:42: # #
|
||||
MPI Rank 0: 09/17/2016 22:50:42: # speechTrain command (train action) #
|
||||
MPI Rank 0: 09/17/2016 22:50:42: # #
|
||||
MPI Rank 0: 09/17/2016 22:50:42: ##############################################################################
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:42:
|
||||
MPI Rank 0: Creating virgin network.
|
||||
MPI Rank 0: SimpleNetworkBuilder Using GPU 0
|
||||
MPI Rank 0: SetUniformRandomValue (GPU): creating curand object with seed 1, sizeof(ElemType)==8
|
||||
MPI Rank 0: reading script file glob_0000.scp ... 948 entries
|
||||
MPI Rank 0: total 132 state names in state list /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data/state.list
|
||||
MPI Rank 0: htkmlfreader: reading MLF file /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data/glob_0000.mlf ... total 948 entries
|
||||
MPI Rank 0: ...............................................................................................feature set 0: 252734 frames in 948 out of 948 utterances
|
||||
MPI Rank 0: label set 0: 129 classes
|
||||
MPI Rank 0: minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: 316.0 utterances, 84244.7 frames
|
||||
MPI Rank 0: 09/17/2016 22:50:42:
|
||||
MPI Rank 0: Model has 25 nodes. Using GPU 0.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Training criterion: CrossEntropyWithSoftmax = CrossEntropyWithSoftmax
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Evaluation criterion: EvalClassificationError = ClassificationError
|
||||
MPI Rank 0:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: Allocating matrices for forward and/or backward propagation.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: Memory Sharing: Out of 40 matrices, 19 are shared as 8, and 21 are not shared.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: { W0 : [512 x 363] (gradient)
|
||||
MPI Rank 0: W0*features+B0 : [512 x 1 x *] }
|
||||
MPI Rank 0: { H1 : [512 x 1 x *]
|
||||
MPI Rank 0: W0*features : [512 x *] (gradient) }
|
||||
MPI Rank 0: { B1 : [512 x 1] (gradient)
|
||||
MPI Rank 0: H2 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 0: HLast : [132 x 1 x *] (gradient) }
|
||||
MPI Rank 0: { W0*features+B0 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 0: W1*H1 : [512 x 1 x *] }
|
||||
MPI Rank 0: { W1 : [512 x 512] (gradient)
|
||||
MPI Rank 0: W1*H1+B1 : [512 x 1 x *] }
|
||||
MPI Rank 0: { H2 : [512 x 1 x *]
|
||||
MPI Rank 0: W1*H1 : [512 x 1 x *] (gradient) }
|
||||
MPI Rank 0: { B0 : [512 x 1] (gradient)
|
||||
MPI Rank 0: H1 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 0: W1*H1+B1 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 0: W2*H1 : [132 x 1 x *] }
|
||||
MPI Rank 0: { HLast : [132 x 1 x *]
|
||||
MPI Rank 0: W2 : [132 x 512] (gradient) }
|
||||
MPI Rank 0:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Training 516740 parameters in 6 out of 6 parameter tensors and 15 nodes with gradient:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Node 'B0' (LearnableParameter operation) : [512 x 1]
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Node 'B1' (LearnableParameter operation) : [512 x 1]
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Node 'B2' (LearnableParameter operation) : [132 x 1]
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Node 'W0' (LearnableParameter operation) : [512 x 363]
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Node 'W1' (LearnableParameter operation) : [512 x 512]
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Node 'W2' (LearnableParameter operation) : [132 x 512]
|
||||
MPI Rank 0:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Precomputing --> 3 PreCompute nodes found.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:42: MeanOfFeatures = Mean()
|
||||
MPI Rank 0: 09/17/2016 22:50:42: InvStdOfFeatures = InvStdDev()
|
||||
MPI Rank 0: 09/17/2016 22:50:42: Prior = Mean()
|
||||
MPI Rank 0: minibatchiterator: epoch 0: frames [0..252734] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses
|
||||
MPI Rank 0: requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:55: Precomputing --> Completed.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:55: Starting Epoch 1: learning rate per sample = 0.001953 effective momentum = 0.939413 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 0: minibatchiterator: epoch 0: frames [0..81920] (first utterance at frame 0), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:50:55: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 4.73732304 * 640; EvalClassificationError = 0.96718750 * 640; time = 0.1806s; samplesPerSecond = 3544.4
|
||||
MPI Rank 0: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 4.37245658 * 640; EvalClassificationError = 0.92187500 * 640; time = 0.1618s; samplesPerSecond = 3955.1
|
||||
MPI Rank 0: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 4.17410438 * 640; EvalClassificationError = 0.95937500 * 640; time = 0.1567s; samplesPerSecond = 4084.1
|
||||
MPI Rank 0: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 3.88445939 * 640; EvalClassificationError = 0.86093750 * 640; time = 0.1581s; samplesPerSecond = 4048.3
|
||||
MPI Rank 0: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 4.05583056 * 640; EvalClassificationError = 0.89375000 * 640; time = 0.1558s; samplesPerSecond = 4108.0
|
||||
MPI Rank 0: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 3.94301179 * 640; EvalClassificationError = 0.86250000 * 640; time = 0.1556s; samplesPerSecond = 4112.4
|
||||
MPI Rank 0: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 3.80690392 * 640; EvalClassificationError = 0.85937500 * 640; time = 0.1554s; samplesPerSecond = 4119.6
|
||||
MPI Rank 0: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 3.92014544 * 640; EvalClassificationError = 0.87812500 * 640; time = 0.1553s; samplesPerSecond = 4120.0
|
||||
MPI Rank 0: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 3.97161496 * 640; EvalClassificationError = 0.87968750 * 640; time = 0.1558s; samplesPerSecond = 4108.5
|
||||
MPI Rank 0: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 3.93678341 * 640; EvalClassificationError = 0.90937500 * 640; time = 0.1550s; samplesPerSecond = 4128.9
|
||||
MPI Rank 0: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 4.05170333 * 640; EvalClassificationError = 0.92187500 * 640; time = 0.1560s; samplesPerSecond = 4101.4
|
||||
MPI Rank 0: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 3.89920006 * 640; EvalClassificationError = 0.86406250 * 640; time = 0.1569s; samplesPerSecond = 4079.7
|
||||
MPI Rank 0: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 3.87251780 * 640; EvalClassificationError = 0.89218750 * 640; time = 0.1552s; samplesPerSecond = 4122.4
|
||||
MPI Rank 0: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 3.81997328 * 640; EvalClassificationError = 0.88281250 * 640; time = 0.1556s; samplesPerSecond = 4114.3
|
||||
MPI Rank 0: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 3.80473823 * 640; EvalClassificationError = 0.88125000 * 640; time = 0.1552s; samplesPerSecond = 4125.0
|
||||
MPI Rank 0: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 3.94101364 * 640; EvalClassificationError = 0.88750000 * 640; time = 0.1550s; samplesPerSecond = 4130.1
|
||||
MPI Rank 0: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 161- 170, 13.28%]: CrossEntropyWithSoftmax = 3.82130368 * 640; EvalClassificationError = 0.87656250 * 640; time = 0.1569s; samplesPerSecond = 4079.8
|
||||
MPI Rank 0: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 171- 180, 14.06%]: CrossEntropyWithSoftmax = 3.75236005 * 640; EvalClassificationError = 0.85312500 * 640; time = 0.1551s; samplesPerSecond = 4125.5
|
||||
MPI Rank 0: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 181- 190, 14.84%]: CrossEntropyWithSoftmax = 3.79661132 * 640; EvalClassificationError = 0.86562500 * 640; time = 0.1550s; samplesPerSecond = 4129.6
|
||||
MPI Rank 0: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 191- 200, 15.62%]: CrossEntropyWithSoftmax = 3.73922363 * 640; EvalClassificationError = 0.87343750 * 640; time = 0.1546s; samplesPerSecond = 4140.5
|
||||
MPI Rank 0: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 201- 210, 16.41%]: CrossEntropyWithSoftmax = 3.73517402 * 640; EvalClassificationError = 0.86406250 * 640; time = 0.1553s; samplesPerSecond = 4119.9
|
||||
MPI Rank 0: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 211- 220, 17.19%]: CrossEntropyWithSoftmax = 3.72398663 * 640; EvalClassificationError = 0.85625000 * 640; time = 0.1556s; samplesPerSecond = 4113.7
|
||||
MPI Rank 0: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 221- 230, 17.97%]: CrossEntropyWithSoftmax = 3.71595086 * 640; EvalClassificationError = 0.84062500 * 640; time = 0.1556s; samplesPerSecond = 4112.1
|
||||
MPI Rank 0: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 231- 240, 18.75%]: CrossEntropyWithSoftmax = 3.75562845 * 640; EvalClassificationError = 0.83281250 * 640; time = 0.1557s; samplesPerSecond = 4109.8
|
||||
MPI Rank 0: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 241- 250, 19.53%]: CrossEntropyWithSoftmax = 3.72052883 * 640; EvalClassificationError = 0.85468750 * 640; time = 0.1554s; samplesPerSecond = 4119.7
|
||||
MPI Rank 0: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 251- 260, 20.31%]: CrossEntropyWithSoftmax = 3.66417431 * 640; EvalClassificationError = 0.83906250 * 640; time = 0.1551s; samplesPerSecond = 4126.6
|
||||
MPI Rank 0: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 261- 270, 21.09%]: CrossEntropyWithSoftmax = 3.58440175 * 640; EvalClassificationError = 0.77812500 * 640; time = 0.1553s; samplesPerSecond = 4121.3
|
||||
MPI Rank 0: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 271- 280, 21.88%]: CrossEntropyWithSoftmax = 3.69427679 * 640; EvalClassificationError = 0.80781250 * 640; time = 0.1557s; samplesPerSecond = 4109.9
|
||||
MPI Rank 0: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 281- 290, 22.66%]: CrossEntropyWithSoftmax = 3.57769881 * 640; EvalClassificationError = 0.79062500 * 640; time = 0.1556s; samplesPerSecond = 4113.9
|
||||
MPI Rank 0: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 291- 300, 23.44%]: CrossEntropyWithSoftmax = 3.52797012 * 640; EvalClassificationError = 0.82031250 * 640; time = 0.1547s; samplesPerSecond = 4135.8
|
||||
MPI Rank 0: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 301- 310, 24.22%]: CrossEntropyWithSoftmax = 3.53498040 * 640; EvalClassificationError = 0.81718750 * 640; time = 0.1566s; samplesPerSecond = 4086.2
|
||||
MPI Rank 0: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 311- 320, 25.00%]: CrossEntropyWithSoftmax = 3.50005340 * 640; EvalClassificationError = 0.79375000 * 640; time = 0.1562s; samplesPerSecond = 4097.3
|
||||
MPI Rank 0: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 321- 330, 25.78%]: CrossEntropyWithSoftmax = 3.37848051 * 640; EvalClassificationError = 0.79687500 * 640; time = 0.1561s; samplesPerSecond = 4100.8
|
||||
MPI Rank 0: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 331- 340, 26.56%]: CrossEntropyWithSoftmax = 3.44203869 * 640; EvalClassificationError = 0.78906250 * 640; time = 0.1551s; samplesPerSecond = 4126.0
|
||||
MPI Rank 0: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 341- 350, 27.34%]: CrossEntropyWithSoftmax = 3.49473674 * 640; EvalClassificationError = 0.80156250 * 640; time = 0.1556s; samplesPerSecond = 4114.1
|
||||
MPI Rank 0: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 351- 360, 28.12%]: CrossEntropyWithSoftmax = 3.42678273 * 640; EvalClassificationError = 0.80468750 * 640; time = 0.1552s; samplesPerSecond = 4124.1
|
||||
MPI Rank 0: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 361- 370, 28.91%]: CrossEntropyWithSoftmax = 3.49825811 * 640; EvalClassificationError = 0.82656250 * 640; time = 0.1559s; samplesPerSecond = 4106.5
|
||||
MPI Rank 0: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 371- 380, 29.69%]: CrossEntropyWithSoftmax = 3.18570771 * 640; EvalClassificationError = 0.75156250 * 640; time = 0.1552s; samplesPerSecond = 4122.5
|
||||
MPI Rank 0: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 381- 390, 30.47%]: CrossEntropyWithSoftmax = 3.31483570 * 640; EvalClassificationError = 0.79218750 * 640; time = 0.1553s; samplesPerSecond = 4120.0
|
||||
MPI Rank 0: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 391- 400, 31.25%]: CrossEntropyWithSoftmax = 3.27794269 * 640; EvalClassificationError = 0.77656250 * 640; time = 0.1558s; samplesPerSecond = 4108.7
|
||||
MPI Rank 0: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 401- 410, 32.03%]: CrossEntropyWithSoftmax = 3.42283181 * 640; EvalClassificationError = 0.80312500 * 640; time = 0.1558s; samplesPerSecond = 4107.3
|
||||
MPI Rank 0: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 411- 420, 32.81%]: CrossEntropyWithSoftmax = 3.28013444 * 640; EvalClassificationError = 0.75937500 * 640; time = 0.1552s; samplesPerSecond = 4124.8
|
||||
MPI Rank 0: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 421- 430, 33.59%]: CrossEntropyWithSoftmax = 3.23863079 * 640; EvalClassificationError = 0.78281250 * 640; time = 0.1563s; samplesPerSecond = 4095.4
|
||||
MPI Rank 0: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 431- 440, 34.38%]: CrossEntropyWithSoftmax = 3.13381567 * 640; EvalClassificationError = 0.75937500 * 640; time = 0.1552s; samplesPerSecond = 4124.2
|
||||
MPI Rank 0: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 441- 450, 35.16%]: CrossEntropyWithSoftmax = 3.18522093 * 640; EvalClassificationError = 0.74062500 * 640; time = 0.1551s; samplesPerSecond = 4127.5
|
||||
MPI Rank 0: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 451- 460, 35.94%]: CrossEntropyWithSoftmax = 3.23487402 * 640; EvalClassificationError = 0.77187500 * 640; time = 0.1561s; samplesPerSecond = 4099.7
|
||||
MPI Rank 0: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 461- 470, 36.72%]: CrossEntropyWithSoftmax = 3.19165708 * 640; EvalClassificationError = 0.77812500 * 640; time = 0.1555s; samplesPerSecond = 4116.7
|
||||
MPI Rank 0: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 471- 480, 37.50%]: CrossEntropyWithSoftmax = 3.16584445 * 640; EvalClassificationError = 0.73906250 * 640; time = 0.1548s; samplesPerSecond = 4133.8
|
||||
MPI Rank 0: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 481- 490, 38.28%]: CrossEntropyWithSoftmax = 3.24961355 * 640; EvalClassificationError = 0.77187500 * 640; time = 0.1551s; samplesPerSecond = 4126.0
|
||||
MPI Rank 0: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 491- 500, 39.06%]: CrossEntropyWithSoftmax = 3.18529030 * 640; EvalClassificationError = 0.76406250 * 640; time = 0.1559s; samplesPerSecond = 4105.6
|
||||
MPI Rank 0: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 501- 510, 39.84%]: CrossEntropyWithSoftmax = 2.98383964 * 640; EvalClassificationError = 0.72187500 * 640; time = 0.1555s; samplesPerSecond = 4116.8
|
||||
MPI Rank 0: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 511- 520, 40.62%]: CrossEntropyWithSoftmax = 3.14703955 * 640; EvalClassificationError = 0.73281250 * 640; time = 0.1553s; samplesPerSecond = 4120.7
|
||||
MPI Rank 0: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 521- 530, 41.41%]: CrossEntropyWithSoftmax = 3.08217828 * 640; EvalClassificationError = 0.72656250 * 640; time = 0.1554s; samplesPerSecond = 4118.8
|
||||
MPI Rank 0: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 531- 540, 42.19%]: CrossEntropyWithSoftmax = 2.93980472 * 640; EvalClassificationError = 0.70625000 * 640; time = 0.1553s; samplesPerSecond = 4121.1
|
||||
MPI Rank 0: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 541- 550, 42.97%]: CrossEntropyWithSoftmax = 3.02220354 * 640; EvalClassificationError = 0.72968750 * 640; time = 0.1557s; samplesPerSecond = 4111.4
|
||||
MPI Rank 0: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 551- 560, 43.75%]: CrossEntropyWithSoftmax = 2.95530592 * 640; EvalClassificationError = 0.71250000 * 640; time = 0.1559s; samplesPerSecond = 4106.0
|
||||
MPI Rank 0: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 561- 570, 44.53%]: CrossEntropyWithSoftmax = 2.95810228 * 640; EvalClassificationError = 0.72812500 * 640; time = 0.1558s; samplesPerSecond = 4108.6
|
||||
MPI Rank 0: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 571- 580, 45.31%]: CrossEntropyWithSoftmax = 3.00786863 * 640; EvalClassificationError = 0.72656250 * 640; time = 0.1559s; samplesPerSecond = 4104.3
|
||||
MPI Rank 0: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 581- 590, 46.09%]: CrossEntropyWithSoftmax = 3.00106811 * 640; EvalClassificationError = 0.72968750 * 640; time = 0.1546s; samplesPerSecond = 4140.1
|
||||
MPI Rank 0: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 591- 600, 46.88%]: CrossEntropyWithSoftmax = 2.91931245 * 640; EvalClassificationError = 0.70468750 * 640; time = 0.1558s; samplesPerSecond = 4107.3
|
||||
MPI Rank 0: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 601- 610, 47.66%]: CrossEntropyWithSoftmax = 3.00601604 * 640; EvalClassificationError = 0.73750000 * 640; time = 0.1566s; samplesPerSecond = 4086.1
|
||||
MPI Rank 0: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 611- 620, 48.44%]: CrossEntropyWithSoftmax = 2.93808431 * 640; EvalClassificationError = 0.70625000 * 640; time = 0.1549s; samplesPerSecond = 4132.3
|
||||
MPI Rank 0: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 621- 630, 49.22%]: CrossEntropyWithSoftmax = 2.97093532 * 640; EvalClassificationError = 0.74375000 * 640; time = 0.1549s; samplesPerSecond = 4131.3
|
||||
MPI Rank 0: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 631- 640, 50.00%]: CrossEntropyWithSoftmax = 2.86102307 * 640; EvalClassificationError = 0.67812500 * 640; time = 0.1549s; samplesPerSecond = 4132.1
|
||||
MPI Rank 0: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 641- 650, 50.78%]: CrossEntropyWithSoftmax = 2.88070307 * 640; EvalClassificationError = 0.71406250 * 640; time = 0.1553s; samplesPerSecond = 4120.4
|
||||
MPI Rank 0: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 651- 660, 51.56%]: CrossEntropyWithSoftmax = 2.90422279 * 640; EvalClassificationError = 0.71875000 * 640; time = 0.1573s; samplesPerSecond = 4069.5
|
||||
MPI Rank 0: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 661- 670, 52.34%]: CrossEntropyWithSoftmax = 2.82634561 * 640; EvalClassificationError = 0.70937500 * 640; time = 0.1562s; samplesPerSecond = 4097.8
|
||||
MPI Rank 0: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 671- 680, 53.12%]: CrossEntropyWithSoftmax = 2.86572702 * 640; EvalClassificationError = 0.67343750 * 640; time = 0.1557s; samplesPerSecond = 4111.2
|
||||
MPI Rank 0: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 681- 690, 53.91%]: CrossEntropyWithSoftmax = 2.82698660 * 640; EvalClassificationError = 0.70468750 * 640; time = 0.1561s; samplesPerSecond = 4098.9
|
||||
MPI Rank 0: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 691- 700, 54.69%]: CrossEntropyWithSoftmax = 2.69768998 * 640; EvalClassificationError = 0.68437500 * 640; time = 0.1568s; samplesPerSecond = 4081.6
|
||||
MPI Rank 0: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 701- 710, 55.47%]: CrossEntropyWithSoftmax = 2.74280097 * 640; EvalClassificationError = 0.66406250 * 640; time = 0.1557s; samplesPerSecond = 4109.4
|
||||
MPI Rank 0: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 711- 720, 56.25%]: CrossEntropyWithSoftmax = 2.79750038 * 640; EvalClassificationError = 0.71093750 * 640; time = 0.1559s; samplesPerSecond = 4104.9
|
||||
MPI Rank 0: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 721- 730, 57.03%]: CrossEntropyWithSoftmax = 2.68839718 * 640; EvalClassificationError = 0.67968750 * 640; time = 0.1570s; samplesPerSecond = 4076.3
|
||||
MPI Rank 0: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 731- 740, 57.81%]: CrossEntropyWithSoftmax = 2.76485288 * 640; EvalClassificationError = 0.68281250 * 640; time = 0.1554s; samplesPerSecond = 4119.4
|
||||
MPI Rank 0: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 741- 750, 58.59%]: CrossEntropyWithSoftmax = 2.77321739 * 640; EvalClassificationError = 0.67968750 * 640; time = 0.1558s; samplesPerSecond = 4109.0
|
||||
MPI Rank 0: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 751- 760, 59.38%]: CrossEntropyWithSoftmax = 2.71065612 * 640; EvalClassificationError = 0.68593750 * 640; time = 0.1548s; samplesPerSecond = 4133.7
|
||||
MPI Rank 0: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 761- 770, 60.16%]: CrossEntropyWithSoftmax = 2.78848250 * 640; EvalClassificationError = 0.65937500 * 640; time = 0.1545s; samplesPerSecond = 4142.0
|
||||
MPI Rank 0: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 771- 780, 60.94%]: CrossEntropyWithSoftmax = 2.78443193 * 640; EvalClassificationError = 0.68281250 * 640; time = 0.1560s; samplesPerSecond = 4101.5
|
||||
MPI Rank 0: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 781- 790, 61.72%]: CrossEntropyWithSoftmax = 2.72094929 * 640; EvalClassificationError = 0.67031250 * 640; time = 0.1549s; samplesPerSecond = 4131.5
|
||||
MPI Rank 0: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 791- 800, 62.50%]: CrossEntropyWithSoftmax = 2.70404088 * 640; EvalClassificationError = 0.64062500 * 640; time = 0.1548s; samplesPerSecond = 4135.1
|
||||
MPI Rank 0: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 801- 810, 63.28%]: CrossEntropyWithSoftmax = 2.70887221 * 640; EvalClassificationError = 0.66875000 * 640; time = 0.1550s; samplesPerSecond = 4128.9
|
||||
MPI Rank 0: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 811- 820, 64.06%]: CrossEntropyWithSoftmax = 2.67633326 * 640; EvalClassificationError = 0.66562500 * 640; time = 0.1552s; samplesPerSecond = 4123.2
|
||||
MPI Rank 0: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 821- 830, 64.84%]: CrossEntropyWithSoftmax = 2.53198524 * 640; EvalClassificationError = 0.62968750 * 640; time = 0.1567s; samplesPerSecond = 4084.0
|
||||
MPI Rank 0: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 831- 840, 65.62%]: CrossEntropyWithSoftmax = 2.63317481 * 640; EvalClassificationError = 0.66093750 * 640; time = 0.1557s; samplesPerSecond = 4109.4
|
||||
MPI Rank 0: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 841- 850, 66.41%]: CrossEntropyWithSoftmax = 2.65923035 * 640; EvalClassificationError = 0.65468750 * 640; time = 0.1559s; samplesPerSecond = 4104.8
|
||||
MPI Rank 0: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 851- 860, 67.19%]: CrossEntropyWithSoftmax = 2.58961930 * 640; EvalClassificationError = 0.66718750 * 640; time = 0.1553s; samplesPerSecond = 4121.6
|
||||
MPI Rank 0: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 861- 870, 67.97%]: CrossEntropyWithSoftmax = 2.72924811 * 640; EvalClassificationError = 0.67812500 * 640; time = 0.1563s; samplesPerSecond = 4095.6
|
||||
MPI Rank 0: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 871- 880, 68.75%]: CrossEntropyWithSoftmax = 2.66252872 * 640; EvalClassificationError = 0.66250000 * 640; time = 0.1549s; samplesPerSecond = 4132.4
|
||||
MPI Rank 0: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 881- 890, 69.53%]: CrossEntropyWithSoftmax = 2.52883427 * 640; EvalClassificationError = 0.65937500 * 640; time = 0.1555s; samplesPerSecond = 4116.4
|
||||
MPI Rank 0: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 891- 900, 70.31%]: CrossEntropyWithSoftmax = 2.62228341 * 640; EvalClassificationError = 0.65625000 * 640; time = 0.1551s; samplesPerSecond = 4127.3
|
||||
MPI Rank 0: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 901- 910, 71.09%]: CrossEntropyWithSoftmax = 2.55550779 * 640; EvalClassificationError = 0.66093750 * 640; time = 0.1550s; samplesPerSecond = 4127.9
|
||||
MPI Rank 0: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 911- 920, 71.88%]: CrossEntropyWithSoftmax = 2.55049429 * 640; EvalClassificationError = 0.64531250 * 640; time = 0.1556s; samplesPerSecond = 4113.4
|
||||
MPI Rank 0: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 921- 930, 72.66%]: CrossEntropyWithSoftmax = 2.59920014 * 640; EvalClassificationError = 0.65625000 * 640; time = 0.1556s; samplesPerSecond = 4113.5
|
||||
MPI Rank 0: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 931- 940, 73.44%]: CrossEntropyWithSoftmax = 2.54341577 * 640; EvalClassificationError = 0.62343750 * 640; time = 0.1551s; samplesPerSecond = 4126.4
|
||||
MPI Rank 0: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 941- 950, 74.22%]: CrossEntropyWithSoftmax = 2.48476222 * 640; EvalClassificationError = 0.63750000 * 640; time = 0.1551s; samplesPerSecond = 4126.9
|
||||
MPI Rank 0: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 951- 960, 75.00%]: CrossEntropyWithSoftmax = 2.53015221 * 640; EvalClassificationError = 0.63906250 * 640; time = 0.1570s; samplesPerSecond = 4076.5
|
||||
MPI Rank 0: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[ 961- 970, 75.78%]: CrossEntropyWithSoftmax = 2.35319566 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1555s; samplesPerSecond = 4115.5
|
||||
MPI Rank 0: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[ 971- 980, 76.56%]: CrossEntropyWithSoftmax = 2.54683738 * 640; EvalClassificationError = 0.64375000 * 640; time = 0.1560s; samplesPerSecond = 4103.7
|
||||
MPI Rank 0: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[ 981- 990, 77.34%]: CrossEntropyWithSoftmax = 2.45404859 * 640; EvalClassificationError = 0.63750000 * 640; time = 0.1555s; samplesPerSecond = 4115.0
|
||||
MPI Rank 0: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[ 991-1000, 78.12%]: CrossEntropyWithSoftmax = 2.42597335 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1549s; samplesPerSecond = 4131.4
|
||||
MPI Rank 0: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[1001-1010, 78.91%]: CrossEntropyWithSoftmax = 2.37566713 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1552s; samplesPerSecond = 4122.5
|
||||
MPI Rank 0: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[1011-1020, 79.69%]: CrossEntropyWithSoftmax = 2.35902642 * 640; EvalClassificationError = 0.59218750 * 640; time = 0.1554s; samplesPerSecond = 4117.8
|
||||
MPI Rank 0: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1021-1030, 80.47%]: CrossEntropyWithSoftmax = 2.36171107 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1562s; samplesPerSecond = 4098.2
|
||||
MPI Rank 0: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1031-1040, 81.25%]: CrossEntropyWithSoftmax = 2.33345715 * 640; EvalClassificationError = 0.57343750 * 640; time = 0.1546s; samplesPerSecond = 4138.5
|
||||
MPI Rank 0: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1041-1050, 82.03%]: CrossEntropyWithSoftmax = 2.44952411 * 640; EvalClassificationError = 0.61875000 * 640; time = 0.1550s; samplesPerSecond = 4128.3
|
||||
MPI Rank 0: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1051-1060, 82.81%]: CrossEntropyWithSoftmax = 2.31665914 * 640; EvalClassificationError = 0.58437500 * 640; time = 0.1553s; samplesPerSecond = 4119.9
|
||||
MPI Rank 0: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1061-1070, 83.59%]: CrossEntropyWithSoftmax = 2.32162968 * 640; EvalClassificationError = 0.61093750 * 640; time = 0.1553s; samplesPerSecond = 4121.8
|
||||
MPI Rank 0: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1071-1080, 84.38%]: CrossEntropyWithSoftmax = 2.48543345 * 640; EvalClassificationError = 0.63593750 * 640; time = 0.1546s; samplesPerSecond = 4139.1
|
||||
MPI Rank 0: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1081-1090, 85.16%]: CrossEntropyWithSoftmax = 2.35574244 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1556s; samplesPerSecond = 4113.2
|
||||
MPI Rank 0: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1091-1100, 85.94%]: CrossEntropyWithSoftmax = 2.38117646 * 640; EvalClassificationError = 0.62656250 * 640; time = 0.1553s; samplesPerSecond = 4121.6
|
||||
MPI Rank 0: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1101-1110, 86.72%]: CrossEntropyWithSoftmax = 2.38366197 * 640; EvalClassificationError = 0.62187500 * 640; time = 0.1559s; samplesPerSecond = 4105.8
|
||||
MPI Rank 0: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1111-1120, 87.50%]: CrossEntropyWithSoftmax = 2.50061273 * 640; EvalClassificationError = 0.63437500 * 640; time = 0.1565s; samplesPerSecond = 4090.6
|
||||
MPI Rank 0: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1121-1130, 88.28%]: CrossEntropyWithSoftmax = 2.19965354 * 640; EvalClassificationError = 0.63125000 * 640; time = 0.1551s; samplesPerSecond = 4125.8
|
||||
MPI Rank 0: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1131-1140, 89.06%]: CrossEntropyWithSoftmax = 2.27232693 * 640; EvalClassificationError = 0.60156250 * 640; time = 0.1551s; samplesPerSecond = 4126.6
|
||||
MPI Rank 0: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1141-1150, 89.84%]: CrossEntropyWithSoftmax = 2.20566415 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1553s; samplesPerSecond = 4120.9
|
||||
MPI Rank 0: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1151-1160, 90.62%]: CrossEntropyWithSoftmax = 2.33120303 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1556s; samplesPerSecond = 4112.4
|
||||
MPI Rank 0: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1161-1170, 91.41%]: CrossEntropyWithSoftmax = 2.29726772 * 640; EvalClassificationError = 0.57812500 * 640; time = 0.1558s; samplesPerSecond = 4107.8
|
||||
MPI Rank 0: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1171-1180, 92.19%]: CrossEntropyWithSoftmax = 2.29349025 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1558s; samplesPerSecond = 4106.7
|
||||
MPI Rank 0: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1181-1190, 92.97%]: CrossEntropyWithSoftmax = 2.30009060 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1542s; samplesPerSecond = 4149.3
|
||||
MPI Rank 0: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1191-1200, 93.75%]: CrossEntropyWithSoftmax = 2.21902385 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1566s; samplesPerSecond = 4086.3
|
||||
MPI Rank 0: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1201-1210, 94.53%]: CrossEntropyWithSoftmax = 2.23739674 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1565s; samplesPerSecond = 4089.2
|
||||
MPI Rank 0: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1211-1220, 95.31%]: CrossEntropyWithSoftmax = 2.25626016 * 640; EvalClassificationError = 0.60625000 * 640; time = 0.1547s; samplesPerSecond = 4137.7
|
||||
MPI Rank 0: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1221-1230, 96.09%]: CrossEntropyWithSoftmax = 2.39720147 * 640; EvalClassificationError = 0.62343750 * 640; time = 0.1563s; samplesPerSecond = 4094.0
|
||||
MPI Rank 0: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1231-1240, 96.88%]: CrossEntropyWithSoftmax = 2.29898501 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1556s; samplesPerSecond = 4112.8
|
||||
MPI Rank 0: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1241-1250, 97.66%]: CrossEntropyWithSoftmax = 2.24887214 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1563s; samplesPerSecond = 4094.2
|
||||
MPI Rank 0: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1251-1260, 98.44%]: CrossEntropyWithSoftmax = 2.27546233 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1565s; samplesPerSecond = 4090.7
|
||||
MPI Rank 0: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1261-1270, 99.22%]: CrossEntropyWithSoftmax = 2.22047744 * 640; EvalClassificationError = 0.60625000 * 640; time = 0.1572s; samplesPerSecond = 4070.8
|
||||
MPI Rank 0: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1271-1280, 100.00%]: CrossEntropyWithSoftmax = 2.30530274 * 640; EvalClassificationError = 0.60000000 * 640; time = 0.1537s; samplesPerSecond = 4164.8
|
||||
MPI Rank 0: 09/17/2016 22:51:15: Finished Epoch[ 1 of 3]: [Training] CrossEntropyWithSoftmax = 3.00766611 * 81920; EvalClassificationError = 0.72410889 * 81920; totalSamplesSeen = 81920; learningRatePerSample = 0.001953125; epochTime=19.9651s
|
||||
MPI Rank 0: 09/17/2016 22:51:15: SGD: Saving checkpoint model '/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/models/cntkSpeech.dnn.1'
|
||||
MPI Rank 0: 09/17/2016 22:51:16: AdaptiveMinibatchSearch Epoch[2]: Evaluating minibatchSizes 64..8192
|
||||
MPI Rank 0: 09/17/2016 22:51:16: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=64 (search range: 64..8192)...
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:16: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 2.02060591 * 640; EvalClassificationError = 0.52343750 * 640; time = 0.1554s; samplesPerSecond = 4117.7
|
||||
MPI Rank 0: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 2.04232000 * 640; EvalClassificationError = 0.53437500 * 640; time = 0.1556s; samplesPerSecond = 4113.1
|
||||
MPI Rank 0: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 2.14042914 * 640; EvalClassificationError = 0.59843750 * 640; time = 0.1552s; samplesPerSecond = 4123.3
|
||||
MPI Rank 0: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 2.12163859 * 640; EvalClassificationError = 0.56718750 * 640; time = 0.1550s; samplesPerSecond = 4128.3
|
||||
MPI Rank 0: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 2.05390998 * 640; EvalClassificationError = 0.55625000 * 640; time = 0.1552s; samplesPerSecond = 4123.0
|
||||
MPI Rank 0: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 2.09859542 * 640; EvalClassificationError = 0.55156250 * 640; time = 0.1556s; samplesPerSecond = 4111.8
|
||||
MPI Rank 0: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 2.11796400 * 640; EvalClassificationError = 0.56406250 * 640; time = 0.1556s; samplesPerSecond = 4113.9
|
||||
MPI Rank 0: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 2.11659597 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1559s; samplesPerSecond = 4105.8
|
||||
MPI Rank 0: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 2.14159740 * 640; EvalClassificationError = 0.58125000 * 640; time = 0.1556s; samplesPerSecond = 4112.8
|
||||
MPI Rank 0: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 2.13426067 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1558s; samplesPerSecond = 4106.9
|
||||
MPI Rank 0: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.98173765 * 640; EvalClassificationError = 0.54687500 * 640; time = 0.1549s; samplesPerSecond = 4130.7
|
||||
MPI Rank 0: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 2.18123085 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1557s; samplesPerSecond = 4110.3
|
||||
MPI Rank 0: 09/17/2016 22:51:18: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 2.01620029 * 640; EvalClassificationError = 0.53281250 * 640; time = 0.1553s; samplesPerSecond = 4120.1
|
||||
MPI Rank 0: 09/17/2016 22:51:18: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 2.02410759 * 640; EvalClassificationError = 0.55312500 * 640; time = 0.1550s; samplesPerSecond = 4130.3
|
||||
MPI Rank 0: 09/17/2016 22:51:18: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 2.03003578 * 640; EvalClassificationError = 0.51250000 * 640; time = 0.1556s; samplesPerSecond = 4112.0
|
||||
MPI Rank 0: 09/17/2016 22:51:18: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 2.07643576 * 640; EvalClassificationError = 0.58281250 * 640; time = 0.1528s; samplesPerSecond = 4188.3
|
||||
MPI Rank 0: 09/17/2016 22:51:18: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.08110406 * 10240; EvalClassificationError = 0.56123047 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:18: AdaptiveMinibatchSearch Epoch[2]: Computed baseCriterion 2.08110406 for minibatchSize=64
|
||||
MPI Rank 0: 09/17/2016 22:51:18: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=64 (search range: 64..8192)...
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:18: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:18: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 2.02060591 * 640; EvalClassificationError = 0.52343750 * 640; time = 0.1552s; samplesPerSecond = 4124.3
|
||||
MPI Rank 0: 09/17/2016 22:51:18: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 2.04232000 * 640; EvalClassificationError = 0.53437500 * 640; time = 0.1549s; samplesPerSecond = 4130.9
|
||||
MPI Rank 0: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 2.14042914 * 640; EvalClassificationError = 0.59843750 * 640; time = 0.1548s; samplesPerSecond = 4134.2
|
||||
MPI Rank 0: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 2.12163859 * 640; EvalClassificationError = 0.56718750 * 640; time = 0.1565s; samplesPerSecond = 4088.8
|
||||
MPI Rank 0: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 2.05390998 * 640; EvalClassificationError = 0.55625000 * 640; time = 0.1555s; samplesPerSecond = 4115.1
|
||||
MPI Rank 0: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 2.09859542 * 640; EvalClassificationError = 0.55156250 * 640; time = 0.1554s; samplesPerSecond = 4118.3
|
||||
MPI Rank 0: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 2.11796400 * 640; EvalClassificationError = 0.56406250 * 640; time = 0.1551s; samplesPerSecond = 4126.0
|
||||
MPI Rank 0: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 2.11659597 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1554s; samplesPerSecond = 4119.3
|
||||
MPI Rank 0: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 2.14159740 * 640; EvalClassificationError = 0.58125000 * 640; time = 0.1558s; samplesPerSecond = 4107.2
|
||||
MPI Rank 0: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 2.13426067 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1552s; samplesPerSecond = 4123.7
|
||||
MPI Rank 0: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.98173765 * 640; EvalClassificationError = 0.54687500 * 640; time = 0.1563s; samplesPerSecond = 4094.5
|
||||
MPI Rank 0: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 2.18123085 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1559s; samplesPerSecond = 4105.6
|
||||
MPI Rank 0: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 2.01620029 * 640; EvalClassificationError = 0.53281250 * 640; time = 0.1560s; samplesPerSecond = 4103.7
|
||||
MPI Rank 0: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 2.02410759 * 640; EvalClassificationError = 0.55312500 * 640; time = 0.1550s; samplesPerSecond = 4129.0
|
||||
MPI Rank 0: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 2.03003578 * 640; EvalClassificationError = 0.51250000 * 640; time = 0.1561s; samplesPerSecond = 4099.8
|
||||
MPI Rank 0: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 2.07643576 * 640; EvalClassificationError = 0.58281250 * 640; time = 0.1530s; samplesPerSecond = 4183.7
|
||||
MPI Rank 0: 09/17/2016 22:51:21: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.08110406 * 10240; EvalClassificationError = 0.56123047 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:21: AdaptiveMinibatchSearch Epoch[2]: Keep searching... epochCriterion = 2.08110406 vs. baseCriterion = 2.08110406
|
||||
MPI Rank 0: 09/17/2016 22:51:21: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=128 (search range: 64..8192)...
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:21: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 2.02947755 * 1280; EvalClassificationError = 0.52656250 * 1280; time = 0.1797s; samplesPerSecond = 7123.1
|
||||
MPI Rank 0: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 3.12%]: CrossEntropyWithSoftmax = 2.13334208 * 1280; EvalClassificationError = 0.58359375 * 1280; time = 0.1733s; samplesPerSecond = 7384.4
|
||||
MPI Rank 0: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 2.08344475 * 1280; EvalClassificationError = 0.55781250 * 1280; time = 0.1740s; samplesPerSecond = 7354.5
|
||||
MPI Rank 0: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 2.13081897 * 1280; EvalClassificationError = 0.56875000 * 1280; time = 0.1765s; samplesPerSecond = 7253.6
|
||||
MPI Rank 0: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 2.15740742 * 1280; EvalClassificationError = 0.58984375 * 1280; time = 0.1747s; samplesPerSecond = 7326.3
|
||||
MPI Rank 0: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 2.11081282 * 1280; EvalClassificationError = 0.58125000 * 1280; time = 0.1742s; samplesPerSecond = 7349.6
|
||||
MPI Rank 0: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 2.03640631 * 1280; EvalClassificationError = 0.55546875 * 1280; time = 0.1736s; samplesPerSecond = 7373.8
|
||||
MPI Rank 0: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 2.06206717 * 1280; EvalClassificationError = 0.55156250 * 1280; time = 0.1722s; samplesPerSecond = 7434.0
|
||||
MPI Rank 0: 09/17/2016 22:51:22: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.09297213 * 10240; EvalClassificationError = 0.56435547 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 128
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:22: AdaptiveMinibatchSearch Epoch[2]: Keep searching... epochCriterion = 2.09297213 vs. baseCriterion = 2.08110406
|
||||
MPI Rank 0: 09/17/2016 22:51:22: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=192 (search range: 64..8192)...
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:22: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 2.07040051 * 1920; EvalClassificationError = 0.54947917 * 1920; time = 0.2026s; samplesPerSecond = 9474.6
|
||||
MPI Rank 0: 09/17/2016 22:51:23: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 2.12004909 * 1920; EvalClassificationError = 0.56770833 * 1920; time = 0.1920s; samplesPerSecond = 9998.6
|
||||
MPI Rank 0: 09/17/2016 22:51:23: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 2.19270926 * 1920; EvalClassificationError = 0.58750000 * 1920; time = 0.1957s; samplesPerSecond = 9809.8
|
||||
MPI Rank 0: 09/17/2016 22:51:23: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 2.25179093 * 1920; EvalClassificationError = 0.60781250 * 1920; time = 0.1925s; samplesPerSecond = 9971.6
|
||||
MPI Rank 0: 09/17/2016 22:51:23: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 2.45413223 * 1920; EvalClassificationError = 0.59531250 * 1920; time = 0.1954s; samplesPerSecond = 9826.6
|
||||
MPI Rank 0: 09/17/2016 22:51:23: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.23761477 * 10240; EvalClassificationError = 0.58447266 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 192
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:23: AdaptiveMinibatchSearch Epoch[2]: Search successful. New minibatchSize is 128. epochCriterion = 2.09297213 vs baseCriterion = 2.08110406
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:23: Starting Epoch 2: learning rate per sample = 0.001953 effective momentum = 0.882497 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 0: minibatchiterator: epoch 1: frames [81920..163840] (first utterance at frame 81920), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:23: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:23: Epoch[ 2 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 2.18100617 * 1280; EvalClassificationError = 0.55546875 * 1280; time = 0.1756s; samplesPerSecond = 7288.7
|
||||
MPI Rank 0: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 11- 20, 3.12%]: CrossEntropyWithSoftmax = 2.16631393 * 1280; EvalClassificationError = 0.59062500 * 1280; time = 0.1755s; samplesPerSecond = 7292.5
|
||||
MPI Rank 0: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 2.23799285 * 1280; EvalClassificationError = 0.60468750 * 1280; time = 0.1764s; samplesPerSecond = 7258.0
|
||||
MPI Rank 0: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 2.25188312 * 1280; EvalClassificationError = 0.60703125 * 1280; time = 0.1768s; samplesPerSecond = 7240.3
|
||||
MPI Rank 0: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 2.12738463 * 1280; EvalClassificationError = 0.56328125 * 1280; time = 0.1767s; samplesPerSecond = 7243.1
|
||||
MPI Rank 0: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 2.09543741 * 1280; EvalClassificationError = 0.58359375 * 1280; time = 0.1757s; samplesPerSecond = 7283.9
|
||||
MPI Rank 0: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 2.12457852 * 1280; EvalClassificationError = 0.58906250 * 1280; time = 0.1760s; samplesPerSecond = 7273.3
|
||||
MPI Rank 0: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 2.15260337 * 1280; EvalClassificationError = 0.57343750 * 1280; time = 0.1761s; samplesPerSecond = 7268.5
|
||||
MPI Rank 0: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 81- 90, 14.06%]: CrossEntropyWithSoftmax = 2.07975382 * 1280; EvalClassificationError = 0.55312500 * 1280; time = 0.1750s; samplesPerSecond = 7315.9
|
||||
MPI Rank 0: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 91- 100, 15.62%]: CrossEntropyWithSoftmax = 2.09557893 * 1280; EvalClassificationError = 0.56328125 * 1280; time = 0.1749s; samplesPerSecond = 7317.7
|
||||
MPI Rank 0: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 101- 110, 17.19%]: CrossEntropyWithSoftmax = 1.99564992 * 1280; EvalClassificationError = 0.54218750 * 1280; time = 0.1756s; samplesPerSecond = 7289.0
|
||||
MPI Rank 0: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 111- 120, 18.75%]: CrossEntropyWithSoftmax = 2.01696230 * 1280; EvalClassificationError = 0.53437500 * 1280; time = 0.1750s; samplesPerSecond = 7313.2
|
||||
MPI Rank 0: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 121- 130, 20.31%]: CrossEntropyWithSoftmax = 2.08247499 * 1280; EvalClassificationError = 0.55625000 * 1280; time = 0.1758s; samplesPerSecond = 7283.1
|
||||
MPI Rank 0: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 131- 140, 21.88%]: CrossEntropyWithSoftmax = 2.02583127 * 1280; EvalClassificationError = 0.55781250 * 1280; time = 0.1752s; samplesPerSecond = 7306.8
|
||||
MPI Rank 0: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 141- 150, 23.44%]: CrossEntropyWithSoftmax = 2.12427634 * 1280; EvalClassificationError = 0.57031250 * 1280; time = 0.1759s; samplesPerSecond = 7277.7
|
||||
MPI Rank 0: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 151- 160, 25.00%]: CrossEntropyWithSoftmax = 1.95297386 * 1280; EvalClassificationError = 0.55234375 * 1280; time = 0.1747s; samplesPerSecond = 7328.1
|
||||
MPI Rank 0: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 161- 170, 26.56%]: CrossEntropyWithSoftmax = 2.06940792 * 1280; EvalClassificationError = 0.57968750 * 1280; time = 0.1758s; samplesPerSecond = 7281.1
|
||||
MPI Rank 0: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 171- 180, 28.12%]: CrossEntropyWithSoftmax = 1.97910584 * 1280; EvalClassificationError = 0.53281250 * 1280; time = 0.1762s; samplesPerSecond = 7263.4
|
||||
MPI Rank 0: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 181- 190, 29.69%]: CrossEntropyWithSoftmax = 1.97550728 * 1280; EvalClassificationError = 0.56484375 * 1280; time = 0.1748s; samplesPerSecond = 7321.6
|
||||
MPI Rank 0: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 191- 200, 31.25%]: CrossEntropyWithSoftmax = 2.07046879 * 1280; EvalClassificationError = 0.58671875 * 1280; time = 0.1750s; samplesPerSecond = 7314.9
|
||||
MPI Rank 0: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 201- 210, 32.81%]: CrossEntropyWithSoftmax = 1.94086640 * 1280; EvalClassificationError = 0.54609375 * 1280; time = 0.1756s; samplesPerSecond = 7290.1
|
||||
MPI Rank 0: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 211- 220, 34.38%]: CrossEntropyWithSoftmax = 1.88656971 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.1750s; samplesPerSecond = 7315.7
|
||||
MPI Rank 0: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 221- 230, 35.94%]: CrossEntropyWithSoftmax = 1.90888794 * 1280; EvalClassificationError = 0.51640625 * 1280; time = 0.1745s; samplesPerSecond = 7335.6
|
||||
MPI Rank 0: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 231- 240, 37.50%]: CrossEntropyWithSoftmax = 1.91336087 * 1280; EvalClassificationError = 0.52265625 * 1280; time = 0.1749s; samplesPerSecond = 7317.2
|
||||
MPI Rank 0: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 241- 250, 39.06%]: CrossEntropyWithSoftmax = 1.89548772 * 1280; EvalClassificationError = 0.52343750 * 1280; time = 0.1758s; samplesPerSecond = 7282.1
|
||||
MPI Rank 0: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 251- 260, 40.62%]: CrossEntropyWithSoftmax = 1.87667719 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.1744s; samplesPerSecond = 7338.5
|
||||
MPI Rank 0: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 261- 270, 42.19%]: CrossEntropyWithSoftmax = 1.81964097 * 1280; EvalClassificationError = 0.52421875 * 1280; time = 0.1748s; samplesPerSecond = 7320.8
|
||||
MPI Rank 0: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 271- 280, 43.75%]: CrossEntropyWithSoftmax = 1.88226903 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.1754s; samplesPerSecond = 7296.0
|
||||
MPI Rank 0: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 281- 290, 45.31%]: CrossEntropyWithSoftmax = 1.85004922 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.1755s; samplesPerSecond = 7292.7
|
||||
MPI Rank 0: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 291- 300, 46.88%]: CrossEntropyWithSoftmax = 1.78267871 * 1280; EvalClassificationError = 0.50156250 * 1280; time = 0.1757s; samplesPerSecond = 7286.6
|
||||
MPI Rank 0: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 301- 310, 48.44%]: CrossEntropyWithSoftmax = 1.81674744 * 1280; EvalClassificationError = 0.50468750 * 1280; time = 0.1750s; samplesPerSecond = 7313.5
|
||||
MPI Rank 0: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 311- 320, 50.00%]: CrossEntropyWithSoftmax = 1.76854193 * 1280; EvalClassificationError = 0.49062500 * 1280; time = 0.1756s; samplesPerSecond = 7289.8
|
||||
MPI Rank 0: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 321- 330, 51.56%]: CrossEntropyWithSoftmax = 1.81330268 * 1280; EvalClassificationError = 0.51171875 * 1280; time = 0.1758s; samplesPerSecond = 7279.7
|
||||
MPI Rank 0: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 331- 340, 53.12%]: CrossEntropyWithSoftmax = 1.81775001 * 1280; EvalClassificationError = 0.51640625 * 1280; time = 0.1744s; samplesPerSecond = 7337.7
|
||||
MPI Rank 0: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 341- 350, 54.69%]: CrossEntropyWithSoftmax = 1.79768261 * 1280; EvalClassificationError = 0.50546875 * 1280; time = 0.1748s; samplesPerSecond = 7322.0
|
||||
MPI Rank 0: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 351- 360, 56.25%]: CrossEntropyWithSoftmax = 1.78876937 * 1280; EvalClassificationError = 0.49765625 * 1280; time = 0.1746s; samplesPerSecond = 7330.2
|
||||
MPI Rank 0: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 361- 370, 57.81%]: CrossEntropyWithSoftmax = 1.78753263 * 1280; EvalClassificationError = 0.51015625 * 1280; time = 0.1745s; samplesPerSecond = 7336.5
|
||||
MPI Rank 0: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 371- 380, 59.38%]: CrossEntropyWithSoftmax = 1.74071233 * 1280; EvalClassificationError = 0.49140625 * 1280; time = 0.1740s; samplesPerSecond = 7356.1
|
||||
MPI Rank 0: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 381- 390, 60.94%]: CrossEntropyWithSoftmax = 1.71575901 * 1280; EvalClassificationError = 0.46875000 * 1280; time = 0.1850s; samplesPerSecond = 6920.2
|
||||
MPI Rank 0: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 391- 400, 62.50%]: CrossEntropyWithSoftmax = 1.76465781 * 1280; EvalClassificationError = 0.49765625 * 1280; time = 0.1741s; samplesPerSecond = 7352.7
|
||||
MPI Rank 0: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 401- 410, 64.06%]: CrossEntropyWithSoftmax = 1.76532949 * 1280; EvalClassificationError = 0.51406250 * 1280; time = 0.1759s; samplesPerSecond = 7277.1
|
||||
MPI Rank 0: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 411- 420, 65.62%]: CrossEntropyWithSoftmax = 1.79718711 * 1280; EvalClassificationError = 0.50390625 * 1280; time = 0.1732s; samplesPerSecond = 7388.6
|
||||
MPI Rank 0: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 421- 430, 67.19%]: CrossEntropyWithSoftmax = 1.74168655 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.1736s; samplesPerSecond = 7374.9
|
||||
MPI Rank 0: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 431- 440, 68.75%]: CrossEntropyWithSoftmax = 1.73594884 * 1280; EvalClassificationError = 0.49609375 * 1280; time = 0.1747s; samplesPerSecond = 7328.8
|
||||
MPI Rank 0: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 441- 450, 70.31%]: CrossEntropyWithSoftmax = 1.75234022 * 1280; EvalClassificationError = 0.50859375 * 1280; time = 0.1753s; samplesPerSecond = 7302.2
|
||||
MPI Rank 0: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 451- 460, 71.88%]: CrossEntropyWithSoftmax = 1.64950906 * 1280; EvalClassificationError = 0.48281250 * 1280; time = 0.1730s; samplesPerSecond = 7397.7
|
||||
MPI Rank 0: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 461- 470, 73.44%]: CrossEntropyWithSoftmax = 1.72111861 * 1280; EvalClassificationError = 0.49921875 * 1280; time = 0.1737s; samplesPerSecond = 7371.0
|
||||
MPI Rank 0: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 471- 480, 75.00%]: CrossEntropyWithSoftmax = 1.75491334 * 1280; EvalClassificationError = 0.50312500 * 1280; time = 0.1750s; samplesPerSecond = 7313.1
|
||||
MPI Rank 0: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 481- 490, 76.56%]: CrossEntropyWithSoftmax = 1.68324400 * 1280; EvalClassificationError = 0.48281250 * 1280; time = 0.1739s; samplesPerSecond = 7359.1
|
||||
MPI Rank 0: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 491- 500, 78.12%]: CrossEntropyWithSoftmax = 1.71507576 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.1735s; samplesPerSecond = 7376.2
|
||||
MPI Rank 0: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 501- 510, 79.69%]: CrossEntropyWithSoftmax = 1.65489209 * 1280; EvalClassificationError = 0.45937500 * 1280; time = 0.1760s; samplesPerSecond = 7272.2
|
||||
MPI Rank 0: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 511- 520, 81.25%]: CrossEntropyWithSoftmax = 1.70993974 * 1280; EvalClassificationError = 0.48984375 * 1280; time = 0.1742s; samplesPerSecond = 7346.7
|
||||
MPI Rank 0: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 521- 530, 82.81%]: CrossEntropyWithSoftmax = 1.68373330 * 1280; EvalClassificationError = 0.48046875 * 1280; time = 0.1741s; samplesPerSecond = 7352.6
|
||||
MPI Rank 0: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 531- 540, 84.38%]: CrossEntropyWithSoftmax = 1.68961559 * 1280; EvalClassificationError = 0.48671875 * 1280; time = 0.1753s; samplesPerSecond = 7302.6
|
||||
MPI Rank 0: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 541- 550, 85.94%]: CrossEntropyWithSoftmax = 1.70437375 * 1280; EvalClassificationError = 0.48984375 * 1280; time = 0.1747s; samplesPerSecond = 7325.6
|
||||
MPI Rank 0: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 551- 560, 87.50%]: CrossEntropyWithSoftmax = 1.69558061 * 1280; EvalClassificationError = 0.48906250 * 1280; time = 0.1744s; samplesPerSecond = 7338.4
|
||||
MPI Rank 0: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 561- 570, 89.06%]: CrossEntropyWithSoftmax = 1.69535392 * 1280; EvalClassificationError = 0.48359375 * 1280; time = 0.1746s; samplesPerSecond = 7333.0
|
||||
MPI Rank 0: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 571- 580, 90.62%]: CrossEntropyWithSoftmax = 1.65016334 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.1748s; samplesPerSecond = 7321.3
|
||||
MPI Rank 0: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 581- 590, 92.19%]: CrossEntropyWithSoftmax = 1.64953906 * 1280; EvalClassificationError = 0.48515625 * 1280; time = 0.1731s; samplesPerSecond = 7395.8
|
||||
MPI Rank 0: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 591- 600, 93.75%]: CrossEntropyWithSoftmax = 1.64390878 * 1280; EvalClassificationError = 0.48125000 * 1280; time = 0.1733s; samplesPerSecond = 7386.1
|
||||
MPI Rank 0: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 601- 610, 95.31%]: CrossEntropyWithSoftmax = 1.71970142 * 1280; EvalClassificationError = 0.51093750 * 1280; time = 0.1725s; samplesPerSecond = 7421.1
|
||||
MPI Rank 0: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 611- 620, 96.88%]: CrossEntropyWithSoftmax = 1.62454036 * 1280; EvalClassificationError = 0.46562500 * 1280; time = 0.1736s; samplesPerSecond = 7371.4
|
||||
MPI Rank 0: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 621- 630, 98.44%]: CrossEntropyWithSoftmax = 1.59147885 * 1280; EvalClassificationError = 0.46875000 * 1280; time = 0.1728s; samplesPerSecond = 7409.4
|
||||
MPI Rank 0: 09/17/2016 22:51:35: Epoch[ 2 of 3]-Minibatch[ 631- 640, 100.00%]: CrossEntropyWithSoftmax = 1.66373476 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.1699s; samplesPerSecond = 7535.0
|
||||
MPI Rank 0: 09/17/2016 22:51:35: Finished Epoch[ 2 of 3]: [Training] CrossEntropyWithSoftmax = 1.86040693 * 81920; EvalClassificationError = 0.51970215 * 81920; totalSamplesSeen = 163840; learningRatePerSample = 0.001953125; epochTime=18.9953s
|
||||
MPI Rank 0: 09/17/2016 22:51:35: SGD: Saving checkpoint model '/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/models/cntkSpeech.dnn.2'
|
||||
MPI Rank 0: 09/17/2016 22:51:35: AdaptiveMinibatchSearch Epoch[3]: Evaluating minibatchSizes 64..256
|
||||
MPI Rank 0: 09/17/2016 22:51:35: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=64 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:35: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 1.60900971 * 640; EvalClassificationError = 0.46875000 * 640; time = 0.1546s; samplesPerSecond = 4140.7
|
||||
MPI Rank 0: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 1.70269498 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1555s; samplesPerSecond = 4116.6
|
||||
MPI Rank 0: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 1.66433597 * 640; EvalClassificationError = 0.48281250 * 640; time = 0.1553s; samplesPerSecond = 4121.5
|
||||
MPI Rank 0: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 1.56997722 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1550s; samplesPerSecond = 4127.9
|
||||
MPI Rank 0: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 1.67908876 * 640; EvalClassificationError = 0.47656250 * 640; time = 0.1552s; samplesPerSecond = 4124.7
|
||||
MPI Rank 0: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 1.45189750 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1551s; samplesPerSecond = 4127.5
|
||||
MPI Rank 0: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 1.60053473 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1553s; samplesPerSecond = 4122.0
|
||||
MPI Rank 0: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 1.58536943 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1556s; samplesPerSecond = 4111.9
|
||||
MPI Rank 0: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 1.63246860 * 640; EvalClassificationError = 0.45781250 * 640; time = 0.1548s; samplesPerSecond = 4134.3
|
||||
MPI Rank 0: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 1.64770593 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1547s; samplesPerSecond = 4137.3
|
||||
MPI Rank 0: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.55685563 * 640; EvalClassificationError = 0.43750000 * 640; time = 0.1545s; samplesPerSecond = 4141.3
|
||||
MPI Rank 0: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 1.54493706 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1549s; samplesPerSecond = 4130.7
|
||||
MPI Rank 0: 09/17/2016 22:51:37: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 1.59221509 * 640; EvalClassificationError = 0.47031250 * 640; time = 0.1549s; samplesPerSecond = 4131.7
|
||||
MPI Rank 0: 09/17/2016 22:51:37: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 1.59923263 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1549s; samplesPerSecond = 4130.8
|
||||
MPI Rank 0: 09/17/2016 22:51:37: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 1.60345937 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1540s; samplesPerSecond = 4156.6
|
||||
MPI Rank 0: 09/17/2016 22:51:37: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 1.59599131 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1531s; samplesPerSecond = 4180.7
|
||||
MPI Rank 0: 09/17/2016 22:51:37: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60223587 * 10240; EvalClassificationError = 0.45957031 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:37: AdaptiveMinibatchSearch Epoch[3]: Computed baseCriterion 1.60223587 for minibatchSize=64
|
||||
MPI Rank 0: 09/17/2016 22:51:37: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=64 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:37: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:37: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 1.60900971 * 640; EvalClassificationError = 0.46875000 * 640; time = 0.1550s; samplesPerSecond = 4129.4
|
||||
MPI Rank 0: 09/17/2016 22:51:37: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 1.70269498 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1551s; samplesPerSecond = 4127.6
|
||||
MPI Rank 0: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 1.66433597 * 640; EvalClassificationError = 0.48281250 * 640; time = 0.1551s; samplesPerSecond = 4127.6
|
||||
MPI Rank 0: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 1.56997722 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1548s; samplesPerSecond = 4134.1
|
||||
MPI Rank 0: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 1.67908876 * 640; EvalClassificationError = 0.47656250 * 640; time = 0.1549s; samplesPerSecond = 4131.8
|
||||
MPI Rank 0: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 1.45189750 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1540s; samplesPerSecond = 4155.4
|
||||
MPI Rank 0: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 1.60053473 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1549s; samplesPerSecond = 4130.6
|
||||
MPI Rank 0: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 1.58536943 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1557s; samplesPerSecond = 4111.1
|
||||
MPI Rank 0: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 1.63246860 * 640; EvalClassificationError = 0.45781250 * 640; time = 0.1550s; samplesPerSecond = 4128.7
|
||||
MPI Rank 0: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 1.64770593 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1550s; samplesPerSecond = 4128.0
|
||||
MPI Rank 0: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.55685563 * 640; EvalClassificationError = 0.43750000 * 640; time = 0.1546s; samplesPerSecond = 4139.7
|
||||
MPI Rank 0: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 1.54493706 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1554s; samplesPerSecond = 4118.7
|
||||
MPI Rank 0: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 1.59221509 * 640; EvalClassificationError = 0.47031250 * 640; time = 0.1551s; samplesPerSecond = 4125.6
|
||||
MPI Rank 0: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 1.59923263 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1555s; samplesPerSecond = 4117.0
|
||||
MPI Rank 0: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 1.60345937 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1547s; samplesPerSecond = 4136.6
|
||||
MPI Rank 0: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 1.59599131 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1536s; samplesPerSecond = 4168.0
|
||||
MPI Rank 0: 09/17/2016 22:51:40: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60223587 * 10240; EvalClassificationError = 0.45957031 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:40: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.60223587 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 0: 09/17/2016 22:51:40: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=128 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:40: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 1.59906005 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.1749s; samplesPerSecond = 7319.6
|
||||
MPI Rank 0: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 3.12%]: CrossEntropyWithSoftmax = 1.59088280 * 1280; EvalClassificationError = 0.47031250 * 1280; time = 0.1751s; samplesPerSecond = 7311.2
|
||||
MPI Rank 0: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 1.55611991 * 1280; EvalClassificationError = 0.45468750 * 1280; time = 0.1739s; samplesPerSecond = 7361.1
|
||||
MPI Rank 0: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 1.59494809 * 1280; EvalClassificationError = 0.45937500 * 1280; time = 0.1734s; samplesPerSecond = 7383.4
|
||||
MPI Rank 0: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 1.64834247 * 1280; EvalClassificationError = 0.45703125 * 1280; time = 0.1747s; samplesPerSecond = 7327.0
|
||||
MPI Rank 0: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 1.56066565 * 1280; EvalClassificationError = 0.43593750 * 1280; time = 0.1729s; samplesPerSecond = 7402.3
|
||||
MPI Rank 0: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 1.62413186 * 1280; EvalClassificationError = 0.47265625 * 1280; time = 0.1743s; samplesPerSecond = 7342.1
|
||||
MPI Rank 0: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 1.67799703 * 1280; EvalClassificationError = 0.47031250 * 1280; time = 0.1708s; samplesPerSecond = 7495.2
|
||||
MPI Rank 0: 09/17/2016 22:51:41: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60651848 * 10240; EvalClassificationError = 0.46044922 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 128
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:41: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.60651848 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 0: 09/17/2016 22:51:41: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=192 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:41: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 1.59151349 * 1920; EvalClassificationError = 0.46302083 * 1920; time = 0.1956s; samplesPerSecond = 9815.0
|
||||
MPI Rank 0: 09/17/2016 22:51:42: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 1.55305006 * 1920; EvalClassificationError = 0.45000000 * 1920; time = 0.1957s; samplesPerSecond = 9808.8
|
||||
MPI Rank 0: 09/17/2016 22:51:42: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 1.60654814 * 1920; EvalClassificationError = 0.45781250 * 1920; time = 0.1948s; samplesPerSecond = 9856.1
|
||||
MPI Rank 0: 09/17/2016 22:51:42: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 1.60526168 * 1920; EvalClassificationError = 0.44947917 * 1920; time = 0.1941s; samplesPerSecond = 9892.9
|
||||
MPI Rank 0: 09/17/2016 22:51:42: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 1.68437187 * 1920; EvalClassificationError = 0.49531250 * 1920; time = 0.1971s; samplesPerSecond = 9738.9
|
||||
MPI Rank 0: 09/17/2016 22:51:42: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.61407118 * 10240; EvalClassificationError = 0.46357422 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 192
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:42: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.61407118 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 0: 09/17/2016 22:51:42: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=256 (search range: 64..256)...
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:42: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:43: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 3.12%]: CrossEntropyWithSoftmax = 1.59241097 * 2560; EvalClassificationError = 0.46093750 * 2560; time = 0.2274s; samplesPerSecond = 11256.8
|
||||
MPI Rank 0: 09/17/2016 22:51:43: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 6.25%]: CrossEntropyWithSoftmax = 1.59578299 * 2560; EvalClassificationError = 0.46093750 * 2560; time = 0.2209s; samplesPerSecond = 11590.1
|
||||
MPI Rank 0: 09/17/2016 22:51:43: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 9.38%]: CrossEntropyWithSoftmax = 1.68033748 * 2560; EvalClassificationError = 0.47343750 * 2560; time = 0.2164s; samplesPerSecond = 11830.8
|
||||
MPI Rank 0: 09/17/2016 22:51:43: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 12.50%]: CrossEntropyWithSoftmax = 1.73124308 * 2560; EvalClassificationError = 0.48164062 * 2560; time = 0.2136s; samplesPerSecond = 11983.8
|
||||
MPI Rank 0: 09/17/2016 22:51:43: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.64994363 * 10240; EvalClassificationError = 0.46923828 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 256
|
||||
MPI Rank 0: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:43: AdaptiveMinibatchSearch Epoch[3]: Search successful. New minibatchSize is 192. epochCriterion = 1.61407118 vs baseCriterion = 1.60223587
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:43: Starting Epoch 3: learning rate per sample = 0.001953 effective momentum = 0.829029 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 0: minibatchiterator: epoch 2: frames [163840..245760] (first utterance at frame 163840), data subset 0 of 2, with 1 datapasses
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:43: Starting minibatch loop, DataParallelSGD training (MyRank = 0, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 0: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 1.54988471 * 1920; EvalClassificationError = 0.46510417 * 1920; time = 0.1969s; samplesPerSecond = 9751.8
|
||||
MPI Rank 0: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 1.62737285 * 1920; EvalClassificationError = 0.47395833 * 1920; time = 0.1967s; samplesPerSecond = 9762.7
|
||||
MPI Rank 0: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 1.65385199 * 1920; EvalClassificationError = 0.46302083 * 1920; time = 0.1938s; samplesPerSecond = 9906.5
|
||||
MPI Rank 0: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 1.87023223 * 1920; EvalClassificationError = 0.51666667 * 1920; time = 0.1971s; samplesPerSecond = 9742.8
|
||||
MPI Rank 0: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 1.75815123 * 1920; EvalClassificationError = 0.49270833 * 1920; time = 0.1967s; samplesPerSecond = 9759.9
|
||||
MPI Rank 0: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 51- 60, 14.06%]: CrossEntropyWithSoftmax = 2.03099563 * 1920; EvalClassificationError = 0.50312500 * 1920; time = 0.1945s; samplesPerSecond = 9870.7
|
||||
MPI Rank 0: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 61- 70, 16.41%]: CrossEntropyWithSoftmax = 1.86165460 * 1920; EvalClassificationError = 0.53489583 * 1920; time = 0.1954s; samplesPerSecond = 9826.6
|
||||
MPI Rank 0: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 71- 80, 18.75%]: CrossEntropyWithSoftmax = 2.15179657 * 1920; EvalClassificationError = 0.49010417 * 1920; time = 0.1944s; samplesPerSecond = 9878.2
|
||||
MPI Rank 0: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 81- 90, 21.09%]: CrossEntropyWithSoftmax = 1.79956105 * 1920; EvalClassificationError = 0.50625000 * 1920; time = 0.1959s; samplesPerSecond = 9799.9
|
||||
MPI Rank 0: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 91- 100, 23.44%]: CrossEntropyWithSoftmax = 1.80123726 * 1920; EvalClassificationError = 0.48385417 * 1920; time = 0.1949s; samplesPerSecond = 9850.3
|
||||
MPI Rank 0: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 101- 110, 25.78%]: CrossEntropyWithSoftmax = 1.65520548 * 1920; EvalClassificationError = 0.47135417 * 1920; time = 0.1949s; samplesPerSecond = 9850.0
|
||||
MPI Rank 0: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 111- 120, 28.12%]: CrossEntropyWithSoftmax = 1.65536374 * 1920; EvalClassificationError = 0.46875000 * 1920; time = 0.1954s; samplesPerSecond = 9827.9
|
||||
MPI Rank 0: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 121- 130, 30.47%]: CrossEntropyWithSoftmax = 1.62640983 * 1920; EvalClassificationError = 0.47656250 * 1920; time = 0.1946s; samplesPerSecond = 9867.6
|
||||
MPI Rank 0: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 131- 140, 32.81%]: CrossEntropyWithSoftmax = 1.58072809 * 1920; EvalClassificationError = 0.44895833 * 1920; time = 0.1944s; samplesPerSecond = 9878.5
|
||||
MPI Rank 0: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 141- 150, 35.16%]: CrossEntropyWithSoftmax = 1.56697558 * 1920; EvalClassificationError = 0.44687500 * 1920; time = 0.1948s; samplesPerSecond = 9858.8
|
||||
MPI Rank 0: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 151- 160, 37.50%]: CrossEntropyWithSoftmax = 1.58252868 * 1920; EvalClassificationError = 0.46250000 * 1920; time = 0.1954s; samplesPerSecond = 9826.1
|
||||
MPI Rank 0: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 161- 170, 39.84%]: CrossEntropyWithSoftmax = 1.53206179 * 1920; EvalClassificationError = 0.45104167 * 1920; time = 0.1949s; samplesPerSecond = 9850.5
|
||||
MPI Rank 0: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 171- 180, 42.19%]: CrossEntropyWithSoftmax = 1.47925397 * 1920; EvalClassificationError = 0.44531250 * 1920; time = 0.1941s; samplesPerSecond = 9891.3
|
||||
MPI Rank 0: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 181- 190, 44.53%]: CrossEntropyWithSoftmax = 1.46468817 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.1945s; samplesPerSecond = 9871.6
|
||||
MPI Rank 0: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 191- 200, 46.88%]: CrossEntropyWithSoftmax = 1.51778272 * 1920; EvalClassificationError = 0.45468750 * 1920; time = 0.1956s; samplesPerSecond = 9818.2
|
||||
MPI Rank 0: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 201- 210, 49.22%]: CrossEntropyWithSoftmax = 1.43521243 * 1920; EvalClassificationError = 0.41979167 * 1920; time = 0.1930s; samplesPerSecond = 9946.7
|
||||
MPI Rank 0: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 211- 220, 51.56%]: CrossEntropyWithSoftmax = 1.55602002 * 1920; EvalClassificationError = 0.45468750 * 1920; time = 0.1946s; samplesPerSecond = 9868.1
|
||||
MPI Rank 0: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 221- 230, 53.91%]: CrossEntropyWithSoftmax = 1.48434301 * 1920; EvalClassificationError = 0.44479167 * 1920; time = 0.1926s; samplesPerSecond = 9968.8
|
||||
MPI Rank 0: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 231- 240, 56.25%]: CrossEntropyWithSoftmax = 1.47880634 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.1941s; samplesPerSecond = 9892.3
|
||||
MPI Rank 0: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 241- 250, 58.59%]: CrossEntropyWithSoftmax = 1.44452798 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.1942s; samplesPerSecond = 9887.7
|
||||
MPI Rank 0: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 251- 260, 60.94%]: CrossEntropyWithSoftmax = 1.42272624 * 1920; EvalClassificationError = 0.42395833 * 1920; time = 0.1942s; samplesPerSecond = 9886.1
|
||||
MPI Rank 0: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 261- 270, 63.28%]: CrossEntropyWithSoftmax = 1.39175020 * 1920; EvalClassificationError = 0.41354167 * 1920; time = 0.1950s; samplesPerSecond = 9845.6
|
||||
MPI Rank 0: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 271- 280, 65.62%]: CrossEntropyWithSoftmax = 1.47323044 * 1920; EvalClassificationError = 0.43333333 * 1920; time = 0.1948s; samplesPerSecond = 9858.6
|
||||
MPI Rank 0: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 281- 290, 67.97%]: CrossEntropyWithSoftmax = 1.46072580 * 1920; EvalClassificationError = 0.41666667 * 1920; time = 0.1949s; samplesPerSecond = 9849.0
|
||||
MPI Rank 0: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 291- 300, 70.31%]: CrossEntropyWithSoftmax = 1.47105355 * 1920; EvalClassificationError = 0.43177083 * 1920; time = 0.1947s; samplesPerSecond = 9860.4
|
||||
MPI Rank 0: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 301- 310, 72.66%]: CrossEntropyWithSoftmax = 1.40973818 * 1920; EvalClassificationError = 0.41927083 * 1920; time = 0.1929s; samplesPerSecond = 9952.4
|
||||
MPI Rank 0: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 311- 320, 75.00%]: CrossEntropyWithSoftmax = 1.44253894 * 1920; EvalClassificationError = 0.42604167 * 1920; time = 0.1941s; samplesPerSecond = 9893.7
|
||||
MPI Rank 0: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 321- 330, 77.34%]: CrossEntropyWithSoftmax = 1.39941047 * 1920; EvalClassificationError = 0.42239583 * 1920; time = 0.1945s; samplesPerSecond = 9871.3
|
||||
MPI Rank 0: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 331- 340, 79.69%]: CrossEntropyWithSoftmax = 1.42524482 * 1920; EvalClassificationError = 0.41927083 * 1920; time = 0.1938s; samplesPerSecond = 9909.3
|
||||
MPI Rank 0: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 341- 350, 82.03%]: CrossEntropyWithSoftmax = 1.46079356 * 1920; EvalClassificationError = 0.42916667 * 1920; time = 0.1959s; samplesPerSecond = 9802.5
|
||||
MPI Rank 0: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 351- 360, 84.38%]: CrossEntropyWithSoftmax = 1.39608704 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.1944s; samplesPerSecond = 9878.6
|
||||
MPI Rank 0: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 361- 370, 86.72%]: CrossEntropyWithSoftmax = 1.46624909 * 1920; EvalClassificationError = 0.43177083 * 1920; time = 0.1934s; samplesPerSecond = 9929.3
|
||||
MPI Rank 0: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 371- 380, 89.06%]: CrossEntropyWithSoftmax = 1.42432290 * 1920; EvalClassificationError = 0.42031250 * 1920; time = 0.1941s; samplesPerSecond = 9894.2
|
||||
MPI Rank 0: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 381- 390, 91.41%]: CrossEntropyWithSoftmax = 1.36453678 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.1931s; samplesPerSecond = 9945.5
|
||||
MPI Rank 0: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 391- 400, 93.75%]: CrossEntropyWithSoftmax = 1.38669452 * 1920; EvalClassificationError = 0.41041667 * 1920; time = 0.1949s; samplesPerSecond = 9853.6
|
||||
MPI Rank 0: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 401- 410, 96.09%]: CrossEntropyWithSoftmax = 1.37926773 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.1959s; samplesPerSecond = 9803.0
|
||||
MPI Rank 0: 09/17/2016 22:51:52: Epoch[ 3 of 3]-Minibatch[ 411- 420, 98.44%]: CrossEntropyWithSoftmax = 1.39890438 * 1920; EvalClassificationError = 0.41875000 * 1920; time = 0.1940s; samplesPerSecond = 9896.3
|
||||
MPI Rank 0: 09/17/2016 22:51:52: Finished Epoch[ 3 of 3]: [Training] CrossEntropyWithSoftmax = 1.55322819 * 81920; EvalClassificationError = 0.44776611 * 81920; totalSamplesSeen = 245760; learningRatePerSample = 0.001953125; epochTime=17.0574s
|
||||
MPI Rank 0: 09/17/2016 22:51:52: SGD: Saving checkpoint model '/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/models/cntkSpeech.dnn'
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:52: Action "train" complete.
|
||||
MPI Rank 0:
|
||||
MPI Rank 0: 09/17/2016 22:51:52: __COMPLETED__
|
||||
MPI Rank 0: ~MPIWrapper
|
||||
MPI Rank 1: CNTK 1.7+ (HEAD b8eb6d, Sep 17 2016 22:48:35) on localhost at 2016/09/17 22:50:40
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/1bitsgd/debug/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling/cntk.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data RunDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantizationWithAutoMBScaling OutputDir=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu DeviceId=0 timestamping=true numCPUThreads=12 stderr=/tmp/cntk-test-20160917225018.461279/Speech/DNN_Parallel1BitQuantizationWithAutoMBScaling@debug_gpu/stderr
|
||||
MPI Rank 1: 09/17/2016 22:50:42: Using 12 CPU threads.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:42: ##############################################################################
|
||||
MPI Rank 1: 09/17/2016 22:50:42: # #
|
||||
MPI Rank 1: 09/17/2016 22:50:42: # speechTrain command (train action) #
|
||||
MPI Rank 1: 09/17/2016 22:50:42: # #
|
||||
MPI Rank 1: 09/17/2016 22:50:42: ##############################################################################
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:42:
|
||||
MPI Rank 1: Creating virgin network.
|
||||
MPI Rank 1: SimpleNetworkBuilder Using GPU 0
|
||||
MPI Rank 1: SetUniformRandomValue (GPU): creating curand object with seed 1, sizeof(ElemType)==8
|
||||
MPI Rank 1: reading script file glob_0000.scp ... 948 entries
|
||||
MPI Rank 1: total 132 state names in state list /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data/state.list
|
||||
MPI Rank 1: htkmlfreader: reading MLF file /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Speech/Data/glob_0000.mlf ... total 948 entries
|
||||
MPI Rank 1: ...............................................................................................feature set 0: 252734 frames in 948 out of 948 utterances
|
||||
MPI Rank 1: label set 0: 129 classes
|
||||
MPI Rank 1: minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: 316.0 utterances, 84244.7 frames
|
||||
MPI Rank 1: 09/17/2016 22:50:43:
|
||||
MPI Rank 1: Model has 25 nodes. Using GPU 0.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Training criterion: CrossEntropyWithSoftmax = CrossEntropyWithSoftmax
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Evaluation criterion: EvalClassificationError = ClassificationError
|
||||
MPI Rank 1:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: Allocating matrices for forward and/or backward propagation.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: Memory Sharing: Out of 40 matrices, 19 are shared as 8, and 21 are not shared.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: { HLast : [132 x 1 x *]
|
||||
MPI Rank 1: W2 : [132 x 512] (gradient) }
|
||||
MPI Rank 1: { H1 : [512 x 1 x *]
|
||||
MPI Rank 1: W0*features : [512 x *] (gradient) }
|
||||
MPI Rank 1: { W0*features+B0 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 1: W1*H1 : [512 x 1 x *] }
|
||||
MPI Rank 1: { W0 : [512 x 363] (gradient)
|
||||
MPI Rank 1: W0*features+B0 : [512 x 1 x *] }
|
||||
MPI Rank 1: { B1 : [512 x 1] (gradient)
|
||||
MPI Rank 1: H2 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 1: HLast : [132 x 1 x *] (gradient) }
|
||||
MPI Rank 1: { W1 : [512 x 512] (gradient)
|
||||
MPI Rank 1: W1*H1+B1 : [512 x 1 x *] }
|
||||
MPI Rank 1: { H2 : [512 x 1 x *]
|
||||
MPI Rank 1: W1*H1 : [512 x 1 x *] (gradient) }
|
||||
MPI Rank 1: { B0 : [512 x 1] (gradient)
|
||||
MPI Rank 1: H1 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 1: W1*H1+B1 : [512 x 1 x *] (gradient)
|
||||
MPI Rank 1: W2*H1 : [132 x 1 x *] }
|
||||
MPI Rank 1:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Training 516740 parameters in 6 out of 6 parameter tensors and 15 nodes with gradient:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Node 'B0' (LearnableParameter operation) : [512 x 1]
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Node 'B1' (LearnableParameter operation) : [512 x 1]
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Node 'B2' (LearnableParameter operation) : [132 x 1]
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Node 'W0' (LearnableParameter operation) : [512 x 363]
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Node 'W1' (LearnableParameter operation) : [512 x 512]
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Node 'W2' (LearnableParameter operation) : [132 x 512]
|
||||
MPI Rank 1:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Precomputing --> 3 PreCompute nodes found.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:43: MeanOfFeatures = Mean()
|
||||
MPI Rank 1: 09/17/2016 22:50:43: InvStdOfFeatures = InvStdDev()
|
||||
MPI Rank 1: 09/17/2016 22:50:43: Prior = Mean()
|
||||
MPI Rank 1: minibatchiterator: epoch 0: frames [0..252734] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses
|
||||
MPI Rank 1: requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:55: Precomputing --> Completed.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:55: Starting Epoch 1: learning rate per sample = 0.001953 effective momentum = 0.939413 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 1: minibatchiterator: epoch 0: frames [0..81920] (first utterance at frame 0), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:50:55: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 4.73732304 * 640; EvalClassificationError = 0.96718750 * 640; time = 0.1804s; samplesPerSecond = 3548.1
|
||||
MPI Rank 1: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 4.37245658 * 640; EvalClassificationError = 0.92187500 * 640; time = 0.1618s; samplesPerSecond = 3955.5
|
||||
MPI Rank 1: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 4.17410438 * 640; EvalClassificationError = 0.95937500 * 640; time = 0.1571s; samplesPerSecond = 4074.7
|
||||
MPI Rank 1: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 3.88445939 * 640; EvalClassificationError = 0.86093750 * 640; time = 0.1577s; samplesPerSecond = 4057.3
|
||||
MPI Rank 1: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 4.05583056 * 640; EvalClassificationError = 0.89375000 * 640; time = 0.1558s; samplesPerSecond = 4109.1
|
||||
MPI Rank 1: 09/17/2016 22:50:56: Epoch[ 1 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 3.94301179 * 640; EvalClassificationError = 0.86250000 * 640; time = 0.1556s; samplesPerSecond = 4112.4
|
||||
MPI Rank 1: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 3.80690392 * 640; EvalClassificationError = 0.85937500 * 640; time = 0.1557s; samplesPerSecond = 4110.5
|
||||
MPI Rank 1: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 3.92014544 * 640; EvalClassificationError = 0.87812500 * 640; time = 0.1550s; samplesPerSecond = 4129.6
|
||||
MPI Rank 1: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 3.97161496 * 640; EvalClassificationError = 0.87968750 * 640; time = 0.1558s; samplesPerSecond = 4108.5
|
||||
MPI Rank 1: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 3.93678341 * 640; EvalClassificationError = 0.90937500 * 640; time = 0.1554s; samplesPerSecond = 4119.0
|
||||
MPI Rank 1: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 4.05170333 * 640; EvalClassificationError = 0.92187500 * 640; time = 0.1561s; samplesPerSecond = 4100.9
|
||||
MPI Rank 1: 09/17/2016 22:50:57: Epoch[ 1 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 3.89920006 * 640; EvalClassificationError = 0.86406250 * 640; time = 0.1565s; samplesPerSecond = 4088.9
|
||||
MPI Rank 1: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 3.87251780 * 640; EvalClassificationError = 0.89218750 * 640; time = 0.1556s; samplesPerSecond = 4112.9
|
||||
MPI Rank 1: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 3.81997328 * 640; EvalClassificationError = 0.88281250 * 640; time = 0.1552s; samplesPerSecond = 4123.0
|
||||
MPI Rank 1: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 3.80473823 * 640; EvalClassificationError = 0.88125000 * 640; time = 0.1555s; samplesPerSecond = 4115.6
|
||||
MPI Rank 1: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 3.94101364 * 640; EvalClassificationError = 0.88750000 * 640; time = 0.1550s; samplesPerSecond = 4130.3
|
||||
MPI Rank 1: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 161- 170, 13.28%]: CrossEntropyWithSoftmax = 3.82130368 * 640; EvalClassificationError = 0.87656250 * 640; time = 0.1565s; samplesPerSecond = 4088.6
|
||||
MPI Rank 1: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 171- 180, 14.06%]: CrossEntropyWithSoftmax = 3.75236005 * 640; EvalClassificationError = 0.85312500 * 640; time = 0.1555s; samplesPerSecond = 4116.7
|
||||
MPI Rank 1: 09/17/2016 22:50:58: Epoch[ 1 of 3]-Minibatch[ 181- 190, 14.84%]: CrossEntropyWithSoftmax = 3.79661132 * 640; EvalClassificationError = 0.86562500 * 640; time = 0.1546s; samplesPerSecond = 4139.0
|
||||
MPI Rank 1: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 191- 200, 15.62%]: CrossEntropyWithSoftmax = 3.73922363 * 640; EvalClassificationError = 0.87343750 * 640; time = 0.1546s; samplesPerSecond = 4140.5
|
||||
MPI Rank 1: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 201- 210, 16.41%]: CrossEntropyWithSoftmax = 3.73517402 * 640; EvalClassificationError = 0.86406250 * 640; time = 0.1557s; samplesPerSecond = 4110.5
|
||||
MPI Rank 1: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 211- 220, 17.19%]: CrossEntropyWithSoftmax = 3.72398663 * 640; EvalClassificationError = 0.85625000 * 640; time = 0.1552s; samplesPerSecond = 4122.8
|
||||
MPI Rank 1: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 221- 230, 17.97%]: CrossEntropyWithSoftmax = 3.71595086 * 640; EvalClassificationError = 0.84062500 * 640; time = 0.1556s; samplesPerSecond = 4112.3
|
||||
MPI Rank 1: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 231- 240, 18.75%]: CrossEntropyWithSoftmax = 3.75562845 * 640; EvalClassificationError = 0.83281250 * 640; time = 0.1557s; samplesPerSecond = 4110.3
|
||||
MPI Rank 1: 09/17/2016 22:50:59: Epoch[ 1 of 3]-Minibatch[ 241- 250, 19.53%]: CrossEntropyWithSoftmax = 3.72052883 * 640; EvalClassificationError = 0.85468750 * 640; time = 0.1553s; samplesPerSecond = 4119.9
|
||||
MPI Rank 1: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 251- 260, 20.31%]: CrossEntropyWithSoftmax = 3.66417431 * 640; EvalClassificationError = 0.83906250 * 640; time = 0.1554s; samplesPerSecond = 4117.1
|
||||
MPI Rank 1: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 261- 270, 21.09%]: CrossEntropyWithSoftmax = 3.58440175 * 640; EvalClassificationError = 0.77812500 * 640; time = 0.1549s; samplesPerSecond = 4130.9
|
||||
MPI Rank 1: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 271- 280, 21.88%]: CrossEntropyWithSoftmax = 3.69427679 * 640; EvalClassificationError = 0.80781250 * 640; time = 0.1557s; samplesPerSecond = 4110.4
|
||||
MPI Rank 1: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 281- 290, 22.66%]: CrossEntropyWithSoftmax = 3.57769881 * 640; EvalClassificationError = 0.79062500 * 640; time = 0.1559s; samplesPerSecond = 4104.0
|
||||
MPI Rank 1: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 291- 300, 23.44%]: CrossEntropyWithSoftmax = 3.52797012 * 640; EvalClassificationError = 0.82031250 * 640; time = 0.1548s; samplesPerSecond = 4134.7
|
||||
MPI Rank 1: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 301- 310, 24.22%]: CrossEntropyWithSoftmax = 3.53498040 * 640; EvalClassificationError = 0.81718750 * 640; time = 0.1560s; samplesPerSecond = 4102.4
|
||||
MPI Rank 1: 09/17/2016 22:51:00: Epoch[ 1 of 3]-Minibatch[ 311- 320, 25.00%]: CrossEntropyWithSoftmax = 3.50005340 * 640; EvalClassificationError = 0.79375000 * 640; time = 0.1568s; samplesPerSecond = 4080.6
|
||||
MPI Rank 1: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 321- 330, 25.78%]: CrossEntropyWithSoftmax = 3.37848051 * 640; EvalClassificationError = 0.79687500 * 640; time = 0.1557s; samplesPerSecond = 4110.3
|
||||
MPI Rank 1: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 331- 340, 26.56%]: CrossEntropyWithSoftmax = 3.44203869 * 640; EvalClassificationError = 0.78906250 * 640; time = 0.1551s; samplesPerSecond = 4125.9
|
||||
MPI Rank 1: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 341- 350, 27.34%]: CrossEntropyWithSoftmax = 3.49473674 * 640; EvalClassificationError = 0.80156250 * 640; time = 0.1559s; samplesPerSecond = 4104.3
|
||||
MPI Rank 1: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 351- 360, 28.12%]: CrossEntropyWithSoftmax = 3.42678273 * 640; EvalClassificationError = 0.80468750 * 640; time = 0.1548s; samplesPerSecond = 4133.7
|
||||
MPI Rank 1: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 361- 370, 28.91%]: CrossEntropyWithSoftmax = 3.49825811 * 640; EvalClassificationError = 0.82656250 * 640; time = 0.1559s; samplesPerSecond = 4106.5
|
||||
MPI Rank 1: 09/17/2016 22:51:01: Epoch[ 1 of 3]-Minibatch[ 371- 380, 29.69%]: CrossEntropyWithSoftmax = 3.18570771 * 640; EvalClassificationError = 0.75156250 * 640; time = 0.1556s; samplesPerSecond = 4112.3
|
||||
MPI Rank 1: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 381- 390, 30.47%]: CrossEntropyWithSoftmax = 3.31483570 * 640; EvalClassificationError = 0.79218750 * 640; time = 0.1554s; samplesPerSecond = 4119.5
|
||||
MPI Rank 1: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 391- 400, 31.25%]: CrossEntropyWithSoftmax = 3.27794269 * 640; EvalClassificationError = 0.77656250 * 640; time = 0.1558s; samplesPerSecond = 4109.0
|
||||
MPI Rank 1: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 401- 410, 32.03%]: CrossEntropyWithSoftmax = 3.42283181 * 640; EvalClassificationError = 0.80312500 * 640; time = 0.1555s; samplesPerSecond = 4116.6
|
||||
MPI Rank 1: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 411- 420, 32.81%]: CrossEntropyWithSoftmax = 3.28013444 * 640; EvalClassificationError = 0.75937500 * 640; time = 0.1555s; samplesPerSecond = 4115.2
|
||||
MPI Rank 1: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 421- 430, 33.59%]: CrossEntropyWithSoftmax = 3.23863079 * 640; EvalClassificationError = 0.78281250 * 640; time = 0.1559s; samplesPerSecond = 4105.2
|
||||
MPI Rank 1: 09/17/2016 22:51:02: Epoch[ 1 of 3]-Minibatch[ 431- 440, 34.38%]: CrossEntropyWithSoftmax = 3.13381567 * 640; EvalClassificationError = 0.75937500 * 640; time = 0.1551s; samplesPerSecond = 4125.0
|
||||
MPI Rank 1: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 441- 450, 35.16%]: CrossEntropyWithSoftmax = 3.18522093 * 640; EvalClassificationError = 0.74062500 * 640; time = 0.1554s; samplesPerSecond = 4118.4
|
||||
MPI Rank 1: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 451- 460, 35.94%]: CrossEntropyWithSoftmax = 3.23487402 * 640; EvalClassificationError = 0.77187500 * 640; time = 0.1561s; samplesPerSecond = 4099.7
|
||||
MPI Rank 1: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 461- 470, 36.72%]: CrossEntropyWithSoftmax = 3.19165708 * 640; EvalClassificationError = 0.77812500 * 640; time = 0.1551s; samplesPerSecond = 4126.6
|
||||
MPI Rank 1: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 471- 480, 37.50%]: CrossEntropyWithSoftmax = 3.16584445 * 640; EvalClassificationError = 0.73906250 * 640; time = 0.1548s; samplesPerSecond = 4133.9
|
||||
MPI Rank 1: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 481- 490, 38.28%]: CrossEntropyWithSoftmax = 3.24961355 * 640; EvalClassificationError = 0.77187500 * 640; time = 0.1551s; samplesPerSecond = 4125.7
|
||||
MPI Rank 1: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 491- 500, 39.06%]: CrossEntropyWithSoftmax = 3.18529030 * 640; EvalClassificationError = 0.76406250 * 640; time = 0.1562s; samplesPerSecond = 4096.8
|
||||
MPI Rank 1: 09/17/2016 22:51:03: Epoch[ 1 of 3]-Minibatch[ 501- 510, 39.84%]: CrossEntropyWithSoftmax = 2.98383964 * 640; EvalClassificationError = 0.72187500 * 640; time = 0.1551s; samplesPerSecond = 4126.3
|
||||
MPI Rank 1: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 511- 520, 40.62%]: CrossEntropyWithSoftmax = 3.14703955 * 640; EvalClassificationError = 0.73281250 * 640; time = 0.1553s; samplesPerSecond = 4120.4
|
||||
MPI Rank 1: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 521- 530, 41.41%]: CrossEntropyWithSoftmax = 3.08217828 * 640; EvalClassificationError = 0.72656250 * 640; time = 0.1554s; samplesPerSecond = 4119.3
|
||||
MPI Rank 1: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 531- 540, 42.19%]: CrossEntropyWithSoftmax = 2.93980472 * 640; EvalClassificationError = 0.70625000 * 640; time = 0.1553s; samplesPerSecond = 4121.1
|
||||
MPI Rank 1: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 541- 550, 42.97%]: CrossEntropyWithSoftmax = 3.02220354 * 640; EvalClassificationError = 0.72968750 * 640; time = 0.1557s; samplesPerSecond = 4111.3
|
||||
MPI Rank 1: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 551- 560, 43.75%]: CrossEntropyWithSoftmax = 2.95530592 * 640; EvalClassificationError = 0.71250000 * 640; time = 0.1559s; samplesPerSecond = 4106.4
|
||||
MPI Rank 1: 09/17/2016 22:51:04: Epoch[ 1 of 3]-Minibatch[ 561- 570, 44.53%]: CrossEntropyWithSoftmax = 2.95810228 * 640; EvalClassificationError = 0.72812500 * 640; time = 0.1561s; samplesPerSecond = 4100.3
|
||||
MPI Rank 1: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 571- 580, 45.31%]: CrossEntropyWithSoftmax = 3.00786863 * 640; EvalClassificationError = 0.72656250 * 640; time = 0.1556s; samplesPerSecond = 4113.3
|
||||
MPI Rank 1: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 581- 590, 46.09%]: CrossEntropyWithSoftmax = 3.00106811 * 640; EvalClassificationError = 0.72968750 * 640; time = 0.1550s; samplesPerSecond = 4129.7
|
||||
MPI Rank 1: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 591- 600, 46.88%]: CrossEntropyWithSoftmax = 2.91931245 * 640; EvalClassificationError = 0.70468750 * 640; time = 0.1555s; samplesPerSecond = 4116.7
|
||||
MPI Rank 1: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 601- 610, 47.66%]: CrossEntropyWithSoftmax = 3.00601604 * 640; EvalClassificationError = 0.73750000 * 640; time = 0.1570s; samplesPerSecond = 4077.2
|
||||
MPI Rank 1: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 611- 620, 48.44%]: CrossEntropyWithSoftmax = 2.93808431 * 640; EvalClassificationError = 0.70625000 * 640; time = 0.1546s; samplesPerSecond = 4141.0
|
||||
MPI Rank 1: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 621- 630, 49.22%]: CrossEntropyWithSoftmax = 2.97093532 * 640; EvalClassificationError = 0.74375000 * 640; time = 0.1549s; samplesPerSecond = 4131.9
|
||||
MPI Rank 1: 09/17/2016 22:51:05: Epoch[ 1 of 3]-Minibatch[ 631- 640, 50.00%]: CrossEntropyWithSoftmax = 2.86102307 * 640; EvalClassificationError = 0.67812500 * 640; time = 0.1549s; samplesPerSecond = 4132.7
|
||||
MPI Rank 1: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 641- 650, 50.78%]: CrossEntropyWithSoftmax = 2.88070307 * 640; EvalClassificationError = 0.71406250 * 640; time = 0.1553s; samplesPerSecond = 4120.2
|
||||
MPI Rank 1: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 651- 660, 51.56%]: CrossEntropyWithSoftmax = 2.90422279 * 640; EvalClassificationError = 0.71875000 * 640; time = 0.1573s; samplesPerSecond = 4069.5
|
||||
MPI Rank 1: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 661- 670, 52.34%]: CrossEntropyWithSoftmax = 2.82634561 * 640; EvalClassificationError = 0.70937500 * 640; time = 0.1562s; samplesPerSecond = 4098.3
|
||||
MPI Rank 1: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 671- 680, 53.12%]: CrossEntropyWithSoftmax = 2.86572702 * 640; EvalClassificationError = 0.67343750 * 640; time = 0.1560s; samplesPerSecond = 4101.6
|
||||
MPI Rank 1: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 681- 690, 53.91%]: CrossEntropyWithSoftmax = 2.82698660 * 640; EvalClassificationError = 0.70468750 * 640; time = 0.1558s; samplesPerSecond = 4108.0
|
||||
MPI Rank 1: 09/17/2016 22:51:06: Epoch[ 1 of 3]-Minibatch[ 691- 700, 54.69%]: CrossEntropyWithSoftmax = 2.69768998 * 640; EvalClassificationError = 0.68437500 * 640; time = 0.1568s; samplesPerSecond = 4081.8
|
||||
MPI Rank 1: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 701- 710, 55.47%]: CrossEntropyWithSoftmax = 2.74280097 * 640; EvalClassificationError = 0.66406250 * 640; time = 0.1561s; samplesPerSecond = 4100.1
|
||||
MPI Rank 1: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 711- 720, 56.25%]: CrossEntropyWithSoftmax = 2.79750038 * 640; EvalClassificationError = 0.71093750 * 640; time = 0.1559s; samplesPerSecond = 4104.7
|
||||
MPI Rank 1: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 721- 730, 57.03%]: CrossEntropyWithSoftmax = 2.68839718 * 640; EvalClassificationError = 0.67968750 * 640; time = 0.1567s; samplesPerSecond = 4085.2
|
||||
MPI Rank 1: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 731- 740, 57.81%]: CrossEntropyWithSoftmax = 2.76485288 * 640; EvalClassificationError = 0.68281250 * 640; time = 0.1557s; samplesPerSecond = 4109.5
|
||||
MPI Rank 1: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 741- 750, 58.59%]: CrossEntropyWithSoftmax = 2.77321739 * 640; EvalClassificationError = 0.67968750 * 640; time = 0.1557s; samplesPerSecond = 4109.6
|
||||
MPI Rank 1: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 751- 760, 59.38%]: CrossEntropyWithSoftmax = 2.71065612 * 640; EvalClassificationError = 0.68593750 * 640; time = 0.1549s; samplesPerSecond = 4132.7
|
||||
MPI Rank 1: 09/17/2016 22:51:07: Epoch[ 1 of 3]-Minibatch[ 761- 770, 60.16%]: CrossEntropyWithSoftmax = 2.78848250 * 640; EvalClassificationError = 0.65937500 * 640; time = 0.1542s; samplesPerSecond = 4151.7
|
||||
MPI Rank 1: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 771- 780, 60.94%]: CrossEntropyWithSoftmax = 2.78443193 * 640; EvalClassificationError = 0.68281250 * 640; time = 0.1560s; samplesPerSecond = 4101.7
|
||||
MPI Rank 1: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 781- 790, 61.72%]: CrossEntropyWithSoftmax = 2.72094929 * 640; EvalClassificationError = 0.67031250 * 640; time = 0.1549s; samplesPerSecond = 4131.5
|
||||
MPI Rank 1: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 791- 800, 62.50%]: CrossEntropyWithSoftmax = 2.70404088 * 640; EvalClassificationError = 0.64062500 * 640; time = 0.1548s; samplesPerSecond = 4135.4
|
||||
MPI Rank 1: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 801- 810, 63.28%]: CrossEntropyWithSoftmax = 2.70887221 * 640; EvalClassificationError = 0.66875000 * 640; time = 0.1550s; samplesPerSecond = 4128.9
|
||||
MPI Rank 1: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 811- 820, 64.06%]: CrossEntropyWithSoftmax = 2.67633326 * 640; EvalClassificationError = 0.66562500 * 640; time = 0.1556s; samplesPerSecond = 4114.0
|
||||
MPI Rank 1: 09/17/2016 22:51:08: Epoch[ 1 of 3]-Minibatch[ 821- 830, 64.84%]: CrossEntropyWithSoftmax = 2.53198524 * 640; EvalClassificationError = 0.62968750 * 640; time = 0.1564s; samplesPerSecond = 4093.3
|
||||
MPI Rank 1: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 831- 840, 65.62%]: CrossEntropyWithSoftmax = 2.63317481 * 640; EvalClassificationError = 0.66093750 * 640; time = 0.1557s; samplesPerSecond = 4109.7
|
||||
MPI Rank 1: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 841- 850, 66.41%]: CrossEntropyWithSoftmax = 2.65923035 * 640; EvalClassificationError = 0.65468750 * 640; time = 0.1559s; samplesPerSecond = 4105.1
|
||||
MPI Rank 1: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 851- 860, 67.19%]: CrossEntropyWithSoftmax = 2.58961930 * 640; EvalClassificationError = 0.66718750 * 640; time = 0.1556s; samplesPerSecond = 4111.9
|
||||
MPI Rank 1: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 861- 870, 67.97%]: CrossEntropyWithSoftmax = 2.72924811 * 640; EvalClassificationError = 0.67812500 * 640; time = 0.1559s; samplesPerSecond = 4104.8
|
||||
MPI Rank 1: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 871- 880, 68.75%]: CrossEntropyWithSoftmax = 2.66252872 * 640; EvalClassificationError = 0.66250000 * 640; time = 0.1549s; samplesPerSecond = 4132.4
|
||||
MPI Rank 1: 09/17/2016 22:51:09: Epoch[ 1 of 3]-Minibatch[ 881- 890, 69.53%]: CrossEntropyWithSoftmax = 2.52883427 * 640; EvalClassificationError = 0.65937500 * 640; time = 0.1558s; samplesPerSecond = 4107.1
|
||||
MPI Rank 1: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 891- 900, 70.31%]: CrossEntropyWithSoftmax = 2.62228341 * 640; EvalClassificationError = 0.65625000 * 640; time = 0.1551s; samplesPerSecond = 4127.2
|
||||
MPI Rank 1: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 901- 910, 71.09%]: CrossEntropyWithSoftmax = 2.55550779 * 640; EvalClassificationError = 0.66093750 * 640; time = 0.1550s; samplesPerSecond = 4128.4
|
||||
MPI Rank 1: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 911- 920, 71.88%]: CrossEntropyWithSoftmax = 2.55049429 * 640; EvalClassificationError = 0.64531250 * 640; time = 0.1556s; samplesPerSecond = 4113.2
|
||||
MPI Rank 1: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 921- 930, 72.66%]: CrossEntropyWithSoftmax = 2.59920014 * 640; EvalClassificationError = 0.65625000 * 640; time = 0.1552s; samplesPerSecond = 4122.8
|
||||
MPI Rank 1: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 931- 940, 73.44%]: CrossEntropyWithSoftmax = 2.54341577 * 640; EvalClassificationError = 0.62343750 * 640; time = 0.1554s; samplesPerSecond = 4117.6
|
||||
MPI Rank 1: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 941- 950, 74.22%]: CrossEntropyWithSoftmax = 2.48476222 * 640; EvalClassificationError = 0.63750000 * 640; time = 0.1551s; samplesPerSecond = 4126.4
|
||||
MPI Rank 1: 09/17/2016 22:51:10: Epoch[ 1 of 3]-Minibatch[ 951- 960, 75.00%]: CrossEntropyWithSoftmax = 2.53015221 * 640; EvalClassificationError = 0.63906250 * 640; time = 0.1570s; samplesPerSecond = 4076.4
|
||||
MPI Rank 1: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[ 961- 970, 75.78%]: CrossEntropyWithSoftmax = 2.35319566 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1551s; samplesPerSecond = 4125.5
|
||||
MPI Rank 1: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[ 971- 980, 76.56%]: CrossEntropyWithSoftmax = 2.54683738 * 640; EvalClassificationError = 0.64375000 * 640; time = 0.1560s; samplesPerSecond = 4103.7
|
||||
MPI Rank 1: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[ 981- 990, 77.34%]: CrossEntropyWithSoftmax = 2.45404859 * 640; EvalClassificationError = 0.63750000 * 640; time = 0.1555s; samplesPerSecond = 4115.4
|
||||
MPI Rank 1: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[ 991-1000, 78.12%]: CrossEntropyWithSoftmax = 2.42597335 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1549s; samplesPerSecond = 4131.3
|
||||
MPI Rank 1: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[1001-1010, 78.91%]: CrossEntropyWithSoftmax = 2.37566713 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1556s; samplesPerSecond = 4113.1
|
||||
MPI Rank 1: 09/17/2016 22:51:11: Epoch[ 1 of 3]-Minibatch[1011-1020, 79.69%]: CrossEntropyWithSoftmax = 2.35902642 * 640; EvalClassificationError = 0.59218750 * 640; time = 0.1551s; samplesPerSecond = 4126.7
|
||||
MPI Rank 1: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1021-1030, 80.47%]: CrossEntropyWithSoftmax = 2.36171107 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1565s; samplesPerSecond = 4088.5
|
||||
MPI Rank 1: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1031-1040, 81.25%]: CrossEntropyWithSoftmax = 2.33345715 * 640; EvalClassificationError = 0.57343750 * 640; time = 0.1543s; samplesPerSecond = 4148.0
|
||||
MPI Rank 1: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1041-1050, 82.03%]: CrossEntropyWithSoftmax = 2.44952411 * 640; EvalClassificationError = 0.61875000 * 640; time = 0.1550s; samplesPerSecond = 4127.9
|
||||
MPI Rank 1: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1051-1060, 82.81%]: CrossEntropyWithSoftmax = 2.31665914 * 640; EvalClassificationError = 0.58437500 * 640; time = 0.1557s; samplesPerSecond = 4110.7
|
||||
MPI Rank 1: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1061-1070, 83.59%]: CrossEntropyWithSoftmax = 2.32162968 * 640; EvalClassificationError = 0.61093750 * 640; time = 0.1549s; samplesPerSecond = 4131.5
|
||||
MPI Rank 1: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1071-1080, 84.38%]: CrossEntropyWithSoftmax = 2.48543345 * 640; EvalClassificationError = 0.63593750 * 640; time = 0.1550s; samplesPerSecond = 4129.2
|
||||
MPI Rank 1: 09/17/2016 22:51:12: Epoch[ 1 of 3]-Minibatch[1081-1090, 85.16%]: CrossEntropyWithSoftmax = 2.35574244 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1552s; samplesPerSecond = 4122.7
|
||||
MPI Rank 1: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1091-1100, 85.94%]: CrossEntropyWithSoftmax = 2.38117646 * 640; EvalClassificationError = 0.62656250 * 640; time = 0.1556s; samplesPerSecond = 4112.4
|
||||
MPI Rank 1: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1101-1110, 86.72%]: CrossEntropyWithSoftmax = 2.38366197 * 640; EvalClassificationError = 0.62187500 * 640; time = 0.1559s; samplesPerSecond = 4106.5
|
||||
MPI Rank 1: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1111-1120, 87.50%]: CrossEntropyWithSoftmax = 2.50061273 * 640; EvalClassificationError = 0.63437500 * 640; time = 0.1561s; samplesPerSecond = 4100.3
|
||||
MPI Rank 1: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1121-1130, 88.28%]: CrossEntropyWithSoftmax = 2.19965354 * 640; EvalClassificationError = 0.63125000 * 640; time = 0.1551s; samplesPerSecond = 4126.3
|
||||
MPI Rank 1: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1131-1140, 89.06%]: CrossEntropyWithSoftmax = 2.27232693 * 640; EvalClassificationError = 0.60156250 * 640; time = 0.1554s; samplesPerSecond = 4117.8
|
||||
MPI Rank 1: 09/17/2016 22:51:13: Epoch[ 1 of 3]-Minibatch[1141-1150, 89.84%]: CrossEntropyWithSoftmax = 2.20566415 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1553s; samplesPerSecond = 4120.5
|
||||
MPI Rank 1: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1151-1160, 90.62%]: CrossEntropyWithSoftmax = 2.33120303 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1556s; samplesPerSecond = 4111.8
|
||||
MPI Rank 1: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1161-1170, 91.41%]: CrossEntropyWithSoftmax = 2.29726772 * 640; EvalClassificationError = 0.57812500 * 640; time = 0.1558s; samplesPerSecond = 4107.8
|
||||
MPI Rank 1: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1171-1180, 92.19%]: CrossEntropyWithSoftmax = 2.29349025 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1555s; samplesPerSecond = 4117.0
|
||||
MPI Rank 1: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1181-1190, 92.97%]: CrossEntropyWithSoftmax = 2.30009060 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1542s; samplesPerSecond = 4149.6
|
||||
MPI Rank 1: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1191-1200, 93.75%]: CrossEntropyWithSoftmax = 2.21902385 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1570s; samplesPerSecond = 4076.7
|
||||
MPI Rank 1: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1201-1210, 94.53%]: CrossEntropyWithSoftmax = 2.23739674 * 640; EvalClassificationError = 0.60312500 * 640; time = 0.1562s; samplesPerSecond = 4098.5
|
||||
MPI Rank 1: 09/17/2016 22:51:14: Epoch[ 1 of 3]-Minibatch[1211-1220, 95.31%]: CrossEntropyWithSoftmax = 2.25626016 * 640; EvalClassificationError = 0.60625000 * 640; time = 0.1547s; samplesPerSecond = 4137.5
|
||||
MPI Rank 1: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1221-1230, 96.09%]: CrossEntropyWithSoftmax = 2.39720147 * 640; EvalClassificationError = 0.62343750 * 640; time = 0.1567s; samplesPerSecond = 4085.0
|
||||
MPI Rank 1: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1231-1240, 96.88%]: CrossEntropyWithSoftmax = 2.29898501 * 640; EvalClassificationError = 0.60468750 * 640; time = 0.1553s; samplesPerSecond = 4122.1
|
||||
MPI Rank 1: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1241-1250, 97.66%]: CrossEntropyWithSoftmax = 2.24887214 * 640; EvalClassificationError = 0.61250000 * 640; time = 0.1567s; samplesPerSecond = 4085.1
|
||||
MPI Rank 1: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1251-1260, 98.44%]: CrossEntropyWithSoftmax = 2.27546233 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1561s; samplesPerSecond = 4099.4
|
||||
MPI Rank 1: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1261-1270, 99.22%]: CrossEntropyWithSoftmax = 2.22047744 * 640; EvalClassificationError = 0.60625000 * 640; time = 0.1572s; samplesPerSecond = 4071.0
|
||||
MPI Rank 1: 09/17/2016 22:51:15: Epoch[ 1 of 3]-Minibatch[1271-1280, 100.00%]: CrossEntropyWithSoftmax = 2.30530274 * 640; EvalClassificationError = 0.60000000 * 640; time = 0.1540s; samplesPerSecond = 4155.6
|
||||
MPI Rank 1: 09/17/2016 22:51:15: Finished Epoch[ 1 of 3]: [Training] CrossEntropyWithSoftmax = 3.00766611 * 81920; EvalClassificationError = 0.72410889 * 81920; totalSamplesSeen = 81920; learningRatePerSample = 0.001953125; epochTime=19.9651s
|
||||
MPI Rank 1: 09/17/2016 22:51:16: AdaptiveMinibatchSearch Epoch[2]: Evaluating minibatchSizes 64..8192
|
||||
MPI Rank 1: 09/17/2016 22:51:16: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=64 (search range: 64..8192)...
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:16: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 2.02060591 * 640; EvalClassificationError = 0.52343750 * 640; time = 0.1565s; samplesPerSecond = 4089.3
|
||||
MPI Rank 1: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 2.04232000 * 640; EvalClassificationError = 0.53437500 * 640; time = 0.1556s; samplesPerSecond = 4113.0
|
||||
MPI Rank 1: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 2.14042914 * 640; EvalClassificationError = 0.59843750 * 640; time = 0.1552s; samplesPerSecond = 4123.1
|
||||
MPI Rank 1: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 2.12163859 * 640; EvalClassificationError = 0.56718750 * 640; time = 0.1546s; samplesPerSecond = 4138.5
|
||||
MPI Rank 1: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 2.05390998 * 640; EvalClassificationError = 0.55625000 * 640; time = 0.1552s; samplesPerSecond = 4123.1
|
||||
MPI Rank 1: 09/17/2016 22:51:16: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 2.09859542 * 640; EvalClassificationError = 0.55156250 * 640; time = 0.1556s; samplesPerSecond = 4112.3
|
||||
MPI Rank 1: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 2.11796400 * 640; EvalClassificationError = 0.56406250 * 640; time = 0.1559s; samplesPerSecond = 4105.1
|
||||
MPI Rank 1: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 2.11659597 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1559s; samplesPerSecond = 4105.9
|
||||
MPI Rank 1: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 2.14159740 * 640; EvalClassificationError = 0.58125000 * 640; time = 0.1556s; samplesPerSecond = 4112.9
|
||||
MPI Rank 1: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 2.13426067 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1555s; samplesPerSecond = 4116.2
|
||||
MPI Rank 1: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.98173765 * 640; EvalClassificationError = 0.54687500 * 640; time = 0.1553s; samplesPerSecond = 4121.5
|
||||
MPI Rank 1: 09/17/2016 22:51:17: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 2.18123085 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1557s; samplesPerSecond = 4109.9
|
||||
MPI Rank 1: 09/17/2016 22:51:18: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 2.01620029 * 640; EvalClassificationError = 0.53281250 * 640; time = 0.1554s; samplesPerSecond = 4119.7
|
||||
MPI Rank 1: 09/17/2016 22:51:18: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 2.02410759 * 640; EvalClassificationError = 0.55312500 * 640; time = 0.1546s; samplesPerSecond = 4139.9
|
||||
MPI Rank 1: 09/17/2016 22:51:18: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 2.03003578 * 640; EvalClassificationError = 0.51250000 * 640; time = 0.1556s; samplesPerSecond = 4112.0
|
||||
MPI Rank 1: 09/17/2016 22:51:18: BaseAdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 2.07643576 * 640; EvalClassificationError = 0.58281250 * 640; time = 0.1528s; samplesPerSecond = 4188.7
|
||||
MPI Rank 1: 09/17/2016 22:51:18: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.08110406 * 10240; EvalClassificationError = 0.56123047 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:18: AdaptiveMinibatchSearch Epoch[2]: Computed baseCriterion 2.08110406 for minibatchSize=64
|
||||
MPI Rank 1: 09/17/2016 22:51:18: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=64 (search range: 64..8192)...
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:18: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:18: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 2.02060591 * 640; EvalClassificationError = 0.52343750 * 640; time = 0.1559s; samplesPerSecond = 4105.8
|
||||
MPI Rank 1: 09/17/2016 22:51:18: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 2.04232000 * 640; EvalClassificationError = 0.53437500 * 640; time = 0.1549s; samplesPerSecond = 4131.0
|
||||
MPI Rank 1: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 2.14042914 * 640; EvalClassificationError = 0.59843750 * 640; time = 0.1562s; samplesPerSecond = 4097.8
|
||||
MPI Rank 1: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 2.12163859 * 640; EvalClassificationError = 0.56718750 * 640; time = 0.1555s; samplesPerSecond = 4115.5
|
||||
MPI Rank 1: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 2.05390998 * 640; EvalClassificationError = 0.55625000 * 640; time = 0.1552s; samplesPerSecond = 4124.7
|
||||
MPI Rank 1: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 2.09859542 * 640; EvalClassificationError = 0.55156250 * 640; time = 0.1554s; samplesPerSecond = 4118.4
|
||||
MPI Rank 1: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 2.11796400 * 640; EvalClassificationError = 0.56406250 * 640; time = 0.1551s; samplesPerSecond = 4125.4
|
||||
MPI Rank 1: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 2.11659597 * 640; EvalClassificationError = 0.57031250 * 640; time = 0.1554s; samplesPerSecond = 4119.2
|
||||
MPI Rank 1: 09/17/2016 22:51:19: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 2.14159740 * 640; EvalClassificationError = 0.58125000 * 640; time = 0.1558s; samplesPerSecond = 4107.6
|
||||
MPI Rank 1: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 2.13426067 * 640; EvalClassificationError = 0.59687500 * 640; time = 0.1556s; samplesPerSecond = 4114.2
|
||||
MPI Rank 1: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.98173765 * 640; EvalClassificationError = 0.54687500 * 640; time = 0.1560s; samplesPerSecond = 4103.8
|
||||
MPI Rank 1: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 2.18123085 * 640; EvalClassificationError = 0.60781250 * 640; time = 0.1563s; samplesPerSecond = 4095.7
|
||||
MPI Rank 1: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 2.01620029 * 640; EvalClassificationError = 0.53281250 * 640; time = 0.1560s; samplesPerSecond = 4102.7
|
||||
MPI Rank 1: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 2.02410759 * 640; EvalClassificationError = 0.55312500 * 640; time = 0.1550s; samplesPerSecond = 4128.6
|
||||
MPI Rank 1: 09/17/2016 22:51:20: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 2.03003578 * 640; EvalClassificationError = 0.51250000 * 640; time = 0.1561s; samplesPerSecond = 4099.7
|
||||
MPI Rank 1: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 2.07643576 * 640; EvalClassificationError = 0.58281250 * 640; time = 0.1530s; samplesPerSecond = 4183.7
|
||||
MPI Rank 1: 09/17/2016 22:51:21: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.08110406 * 10240; EvalClassificationError = 0.56123047 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:21: AdaptiveMinibatchSearch Epoch[2]: Keep searching... epochCriterion = 2.08110406 vs. baseCriterion = 2.08110406
|
||||
MPI Rank 1: 09/17/2016 22:51:21: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=128 (search range: 64..8192)...
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:21: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 2.02947755 * 1280; EvalClassificationError = 0.52656250 * 1280; time = 0.1821s; samplesPerSecond = 7028.6
|
||||
MPI Rank 1: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 3.12%]: CrossEntropyWithSoftmax = 2.13334208 * 1280; EvalClassificationError = 0.58359375 * 1280; time = 0.1730s; samplesPerSecond = 7398.3
|
||||
MPI Rank 1: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 2.08344475 * 1280; EvalClassificationError = 0.55781250 * 1280; time = 0.1744s; samplesPerSecond = 7339.1
|
||||
MPI Rank 1: 09/17/2016 22:51:21: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 2.13081897 * 1280; EvalClassificationError = 0.56875000 * 1280; time = 0.1765s; samplesPerSecond = 7253.3
|
||||
MPI Rank 1: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 2.15740742 * 1280; EvalClassificationError = 0.58984375 * 1280; time = 0.1747s; samplesPerSecond = 7326.5
|
||||
MPI Rank 1: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 2.11081282 * 1280; EvalClassificationError = 0.58125000 * 1280; time = 0.1738s; samplesPerSecond = 7362.8
|
||||
MPI Rank 1: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 2.03640631 * 1280; EvalClassificationError = 0.55546875 * 1280; time = 0.1739s; samplesPerSecond = 7359.5
|
||||
MPI Rank 1: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 2.06206717 * 1280; EvalClassificationError = 0.55156250 * 1280; time = 0.1718s; samplesPerSecond = 7449.3
|
||||
MPI Rank 1: 09/17/2016 22:51:22: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.09297213 * 10240; EvalClassificationError = 0.56435547 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 128
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:22: AdaptiveMinibatchSearch Epoch[2]: Keep searching... epochCriterion = 2.09297213 vs. baseCriterion = 2.08110406
|
||||
MPI Rank 1: 09/17/2016 22:51:22: AdaptiveMinibatchSearch Epoch[2]: Evaluating trial minibatchSize=192 (search range: 64..8192)...
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [10240..20480] (first utterance at frame 10240), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:22: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:22: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 2.07040051 * 1920; EvalClassificationError = 0.54947917 * 1920; time = 0.2071s; samplesPerSecond = 9272.3
|
||||
MPI Rank 1: 09/17/2016 22:51:23: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 2.12004909 * 1920; EvalClassificationError = 0.56770833 * 1920; time = 0.1920s; samplesPerSecond = 9999.0
|
||||
MPI Rank 1: 09/17/2016 22:51:23: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 2.19270926 * 1920; EvalClassificationError = 0.58750000 * 1920; time = 0.1957s; samplesPerSecond = 9810.6
|
||||
MPI Rank 1: 09/17/2016 22:51:23: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 2.25179093 * 1920; EvalClassificationError = 0.60781250 * 1920; time = 0.1925s; samplesPerSecond = 9972.1
|
||||
MPI Rank 1: 09/17/2016 22:51:23: AdaptiveMinibatchSearch: Epoch[ 2 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 2.45413223 * 1920; EvalClassificationError = 0.59531250 * 1920; time = 0.1954s; samplesPerSecond = 9826.1
|
||||
MPI Rank 1: 09/17/2016 22:51:23: Finished Mini-Epoch[2]: CrossEntropyWithSoftmax = 2.23761477 * 10240; EvalClassificationError = 0.58447266 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 192
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:23: AdaptiveMinibatchSearch Epoch[2]: Search successful. New minibatchSize is 128. epochCriterion = 2.09297213 vs baseCriterion = 2.08110406
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:23: Starting Epoch 2: learning rate per sample = 0.001953 effective momentum = 0.882497 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 1: minibatchiterator: epoch 1: frames [81920..163840] (first utterance at frame 81920), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:23: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:23: Epoch[ 2 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 2.18100617 * 1280; EvalClassificationError = 0.55546875 * 1280; time = 0.1774s; samplesPerSecond = 7217.0
|
||||
MPI Rank 1: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 11- 20, 3.12%]: CrossEntropyWithSoftmax = 2.16631393 * 1280; EvalClassificationError = 0.59062500 * 1280; time = 0.1752s; samplesPerSecond = 7307.0
|
||||
MPI Rank 1: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 2.23799285 * 1280; EvalClassificationError = 0.60468750 * 1280; time = 0.1767s; samplesPerSecond = 7243.2
|
||||
MPI Rank 1: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 2.25188312 * 1280; EvalClassificationError = 0.60703125 * 1280; time = 0.1768s; samplesPerSecond = 7240.6
|
||||
MPI Rank 1: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 2.12738463 * 1280; EvalClassificationError = 0.56328125 * 1280; time = 0.1764s; samplesPerSecond = 7256.9
|
||||
MPI Rank 1: 09/17/2016 22:51:24: Epoch[ 2 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 2.09543741 * 1280; EvalClassificationError = 0.58359375 * 1280; time = 0.1760s; samplesPerSecond = 7271.3
|
||||
MPI Rank 1: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 2.12457852 * 1280; EvalClassificationError = 0.58906250 * 1280; time = 0.1756s; samplesPerSecond = 7287.5
|
||||
MPI Rank 1: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 2.15260337 * 1280; EvalClassificationError = 0.57343750 * 1280; time = 0.1764s; samplesPerSecond = 7254.3
|
||||
MPI Rank 1: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 81- 90, 14.06%]: CrossEntropyWithSoftmax = 2.07975382 * 1280; EvalClassificationError = 0.55312500 * 1280; time = 0.1750s; samplesPerSecond = 7315.8
|
||||
MPI Rank 1: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 91- 100, 15.62%]: CrossEntropyWithSoftmax = 2.09557893 * 1280; EvalClassificationError = 0.56328125 * 1280; time = 0.1749s; samplesPerSecond = 7316.8
|
||||
MPI Rank 1: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 101- 110, 17.19%]: CrossEntropyWithSoftmax = 1.99564992 * 1280; EvalClassificationError = 0.54218750 * 1280; time = 0.1753s; samplesPerSecond = 7302.9
|
||||
MPI Rank 1: 09/17/2016 22:51:25: Epoch[ 2 of 3]-Minibatch[ 111- 120, 18.75%]: CrossEntropyWithSoftmax = 2.01696230 * 1280; EvalClassificationError = 0.53437500 * 1280; time = 0.1754s; samplesPerSecond = 7299.1
|
||||
MPI Rank 1: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 121- 130, 20.31%]: CrossEntropyWithSoftmax = 2.08247499 * 1280; EvalClassificationError = 0.55625000 * 1280; time = 0.1754s; samplesPerSecond = 7297.8
|
||||
MPI Rank 1: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 131- 140, 21.88%]: CrossEntropyWithSoftmax = 2.02583127 * 1280; EvalClassificationError = 0.55781250 * 1280; time = 0.1755s; samplesPerSecond = 7292.4
|
||||
MPI Rank 1: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 141- 150, 23.44%]: CrossEntropyWithSoftmax = 2.12427634 * 1280; EvalClassificationError = 0.57031250 * 1280; time = 0.1759s; samplesPerSecond = 7277.6
|
||||
MPI Rank 1: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 151- 160, 25.00%]: CrossEntropyWithSoftmax = 1.95297386 * 1280; EvalClassificationError = 0.55234375 * 1280; time = 0.1747s; samplesPerSecond = 7327.2
|
||||
MPI Rank 1: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 161- 170, 26.56%]: CrossEntropyWithSoftmax = 2.06940792 * 1280; EvalClassificationError = 0.57968750 * 1280; time = 0.1758s; samplesPerSecond = 7280.3
|
||||
MPI Rank 1: 09/17/2016 22:51:26: Epoch[ 2 of 3]-Minibatch[ 171- 180, 28.12%]: CrossEntropyWithSoftmax = 1.97910584 * 1280; EvalClassificationError = 0.53281250 * 1280; time = 0.1762s; samplesPerSecond = 7263.1
|
||||
MPI Rank 1: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 181- 190, 29.69%]: CrossEntropyWithSoftmax = 1.97550728 * 1280; EvalClassificationError = 0.56484375 * 1280; time = 0.1748s; samplesPerSecond = 7322.0
|
||||
MPI Rank 1: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 191- 200, 31.25%]: CrossEntropyWithSoftmax = 2.07046879 * 1280; EvalClassificationError = 0.58671875 * 1280; time = 0.1750s; samplesPerSecond = 7314.7
|
||||
MPI Rank 1: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 201- 210, 32.81%]: CrossEntropyWithSoftmax = 1.94086640 * 1280; EvalClassificationError = 0.54609375 * 1280; time = 0.1752s; samplesPerSecond = 7305.5
|
||||
MPI Rank 1: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 211- 220, 34.38%]: CrossEntropyWithSoftmax = 1.88656971 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.1749s; samplesPerSecond = 7316.6
|
||||
MPI Rank 1: 09/17/2016 22:51:27: Epoch[ 2 of 3]-Minibatch[ 221- 230, 35.94%]: CrossEntropyWithSoftmax = 1.90888794 * 1280; EvalClassificationError = 0.51640625 * 1280; time = 0.1745s; samplesPerSecond = 7336.8
|
||||
MPI Rank 1: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 231- 240, 37.50%]: CrossEntropyWithSoftmax = 1.91336087 * 1280; EvalClassificationError = 0.52265625 * 1280; time = 0.1753s; samplesPerSecond = 7302.7
|
||||
MPI Rank 1: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 241- 250, 39.06%]: CrossEntropyWithSoftmax = 1.89548772 * 1280; EvalClassificationError = 0.52343750 * 1280; time = 0.1754s; samplesPerSecond = 7296.9
|
||||
MPI Rank 1: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 251- 260, 40.62%]: CrossEntropyWithSoftmax = 1.87667719 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.1748s; samplesPerSecond = 7324.1
|
||||
MPI Rank 1: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 261- 270, 42.19%]: CrossEntropyWithSoftmax = 1.81964097 * 1280; EvalClassificationError = 0.52421875 * 1280; time = 0.1745s; samplesPerSecond = 7335.5
|
||||
MPI Rank 1: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 271- 280, 43.75%]: CrossEntropyWithSoftmax = 1.88226903 * 1280; EvalClassificationError = 0.51875000 * 1280; time = 0.1758s; samplesPerSecond = 7281.4
|
||||
MPI Rank 1: 09/17/2016 22:51:28: Epoch[ 2 of 3]-Minibatch[ 281- 290, 45.31%]: CrossEntropyWithSoftmax = 1.85004922 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.1756s; samplesPerSecond = 7291.0
|
||||
MPI Rank 1: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 291- 300, 46.88%]: CrossEntropyWithSoftmax = 1.78267871 * 1280; EvalClassificationError = 0.50156250 * 1280; time = 0.1757s; samplesPerSecond = 7286.1
|
||||
MPI Rank 1: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 301- 310, 48.44%]: CrossEntropyWithSoftmax = 1.81674744 * 1280; EvalClassificationError = 0.50468750 * 1280; time = 0.1750s; samplesPerSecond = 7313.3
|
||||
MPI Rank 1: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 311- 320, 50.00%]: CrossEntropyWithSoftmax = 1.76854193 * 1280; EvalClassificationError = 0.49062500 * 1280; time = 0.1752s; samplesPerSecond = 7304.3
|
||||
MPI Rank 1: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 321- 330, 51.56%]: CrossEntropyWithSoftmax = 1.81330268 * 1280; EvalClassificationError = 0.51171875 * 1280; time = 0.1758s; samplesPerSecond = 7279.6
|
||||
MPI Rank 1: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 331- 340, 53.12%]: CrossEntropyWithSoftmax = 1.81775001 * 1280; EvalClassificationError = 0.51640625 * 1280; time = 0.1748s; samplesPerSecond = 7323.6
|
||||
MPI Rank 1: 09/17/2016 22:51:29: Epoch[ 2 of 3]-Minibatch[ 341- 350, 54.69%]: CrossEntropyWithSoftmax = 1.79768261 * 1280; EvalClassificationError = 0.50546875 * 1280; time = 0.1748s; samplesPerSecond = 7322.4
|
||||
MPI Rank 1: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 351- 360, 56.25%]: CrossEntropyWithSoftmax = 1.78876937 * 1280; EvalClassificationError = 0.49765625 * 1280; time = 0.1746s; samplesPerSecond = 7330.0
|
||||
MPI Rank 1: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 361- 370, 57.81%]: CrossEntropyWithSoftmax = 1.78753263 * 1280; EvalClassificationError = 0.51015625 * 1280; time = 0.1741s; samplesPerSecond = 7351.8
|
||||
MPI Rank 1: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 371- 380, 59.38%]: CrossEntropyWithSoftmax = 1.74071233 * 1280; EvalClassificationError = 0.49140625 * 1280; time = 0.1744s; samplesPerSecond = 7340.1
|
||||
MPI Rank 1: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 381- 390, 60.94%]: CrossEntropyWithSoftmax = 1.71575901 * 1280; EvalClassificationError = 0.46875000 * 1280; time = 0.1846s; samplesPerSecond = 6932.3
|
||||
MPI Rank 1: 09/17/2016 22:51:30: Epoch[ 2 of 3]-Minibatch[ 391- 400, 62.50%]: CrossEntropyWithSoftmax = 1.76465781 * 1280; EvalClassificationError = 0.49765625 * 1280; time = 0.1741s; samplesPerSecond = 7353.2
|
||||
MPI Rank 1: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 401- 410, 64.06%]: CrossEntropyWithSoftmax = 1.76532949 * 1280; EvalClassificationError = 0.51406250 * 1280; time = 0.1759s; samplesPerSecond = 7278.1
|
||||
MPI Rank 1: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 411- 420, 65.62%]: CrossEntropyWithSoftmax = 1.79718711 * 1280; EvalClassificationError = 0.50390625 * 1280; time = 0.1732s; samplesPerSecond = 7389.1
|
||||
MPI Rank 1: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 421- 430, 67.19%]: CrossEntropyWithSoftmax = 1.74168655 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.1736s; samplesPerSecond = 7375.0
|
||||
MPI Rank 1: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 431- 440, 68.75%]: CrossEntropyWithSoftmax = 1.73594884 * 1280; EvalClassificationError = 0.49609375 * 1280; time = 0.1750s; samplesPerSecond = 7314.5
|
||||
MPI Rank 1: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 441- 450, 70.31%]: CrossEntropyWithSoftmax = 1.75234022 * 1280; EvalClassificationError = 0.50859375 * 1280; time = 0.1753s; samplesPerSecond = 7301.6
|
||||
MPI Rank 1: 09/17/2016 22:51:31: Epoch[ 2 of 3]-Minibatch[ 451- 460, 71.88%]: CrossEntropyWithSoftmax = 1.64950906 * 1280; EvalClassificationError = 0.48281250 * 1280; time = 0.1731s; samplesPerSecond = 7395.9
|
||||
MPI Rank 1: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 461- 470, 73.44%]: CrossEntropyWithSoftmax = 1.72111861 * 1280; EvalClassificationError = 0.49921875 * 1280; time = 0.1733s; samplesPerSecond = 7386.9
|
||||
MPI Rank 1: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 471- 480, 75.00%]: CrossEntropyWithSoftmax = 1.75491334 * 1280; EvalClassificationError = 0.50312500 * 1280; time = 0.1754s; samplesPerSecond = 7298.3
|
||||
MPI Rank 1: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 481- 490, 76.56%]: CrossEntropyWithSoftmax = 1.68324400 * 1280; EvalClassificationError = 0.48281250 * 1280; time = 0.1736s; samplesPerSecond = 7374.2
|
||||
MPI Rank 1: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 491- 500, 78.12%]: CrossEntropyWithSoftmax = 1.71507576 * 1280; EvalClassificationError = 0.50234375 * 1280; time = 0.1735s; samplesPerSecond = 7376.4
|
||||
MPI Rank 1: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 501- 510, 79.69%]: CrossEntropyWithSoftmax = 1.65489209 * 1280; EvalClassificationError = 0.45937500 * 1280; time = 0.1763s; samplesPerSecond = 7258.3
|
||||
MPI Rank 1: 09/17/2016 22:51:32: Epoch[ 2 of 3]-Minibatch[ 511- 520, 81.25%]: CrossEntropyWithSoftmax = 1.70993974 * 1280; EvalClassificationError = 0.48984375 * 1280; time = 0.1742s; samplesPerSecond = 7345.8
|
||||
MPI Rank 1: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 521- 530, 82.81%]: CrossEntropyWithSoftmax = 1.68373330 * 1280; EvalClassificationError = 0.48046875 * 1280; time = 0.1741s; samplesPerSecond = 7353.2
|
||||
MPI Rank 1: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 531- 540, 84.38%]: CrossEntropyWithSoftmax = 1.68961559 * 1280; EvalClassificationError = 0.48671875 * 1280; time = 0.1749s; samplesPerSecond = 7317.4
|
||||
MPI Rank 1: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 541- 550, 85.94%]: CrossEntropyWithSoftmax = 1.70437375 * 1280; EvalClassificationError = 0.48984375 * 1280; time = 0.1751s; samplesPerSecond = 7310.7
|
||||
MPI Rank 1: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 551- 560, 87.50%]: CrossEntropyWithSoftmax = 1.69558061 * 1280; EvalClassificationError = 0.48906250 * 1280; time = 0.1744s; samplesPerSecond = 7338.1
|
||||
MPI Rank 1: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 561- 570, 89.06%]: CrossEntropyWithSoftmax = 1.69535392 * 1280; EvalClassificationError = 0.48359375 * 1280; time = 0.1742s; samplesPerSecond = 7346.9
|
||||
MPI Rank 1: 09/17/2016 22:51:33: Epoch[ 2 of 3]-Minibatch[ 571- 580, 90.62%]: CrossEntropyWithSoftmax = 1.65016334 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.1748s; samplesPerSecond = 7322.1
|
||||
MPI Rank 1: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 581- 590, 92.19%]: CrossEntropyWithSoftmax = 1.64953906 * 1280; EvalClassificationError = 0.48515625 * 1280; time = 0.1734s; samplesPerSecond = 7380.3
|
||||
MPI Rank 1: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 591- 600, 93.75%]: CrossEntropyWithSoftmax = 1.64390878 * 1280; EvalClassificationError = 0.48125000 * 1280; time = 0.1733s; samplesPerSecond = 7385.8
|
||||
MPI Rank 1: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 601- 610, 95.31%]: CrossEntropyWithSoftmax = 1.71970142 * 1280; EvalClassificationError = 0.51093750 * 1280; time = 0.1721s; samplesPerSecond = 7436.5
|
||||
MPI Rank 1: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 611- 620, 96.88%]: CrossEntropyWithSoftmax = 1.62454036 * 1280; EvalClassificationError = 0.46562500 * 1280; time = 0.1736s; samplesPerSecond = 7371.7
|
||||
MPI Rank 1: 09/17/2016 22:51:34: Epoch[ 2 of 3]-Minibatch[ 621- 630, 98.44%]: CrossEntropyWithSoftmax = 1.59147885 * 1280; EvalClassificationError = 0.46875000 * 1280; time = 0.1727s; samplesPerSecond = 7411.1
|
||||
MPI Rank 1: 09/17/2016 22:51:35: Epoch[ 2 of 3]-Minibatch[ 631- 640, 100.00%]: CrossEntropyWithSoftmax = 1.66373476 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.1699s; samplesPerSecond = 7535.2
|
||||
MPI Rank 1: 09/17/2016 22:51:35: Finished Epoch[ 2 of 3]: [Training] CrossEntropyWithSoftmax = 1.86040693 * 81920; EvalClassificationError = 0.51970215 * 81920; totalSamplesSeen = 163840; learningRatePerSample = 0.001953125; epochTime=18.9953s
|
||||
MPI Rank 1: 09/17/2016 22:51:35: AdaptiveMinibatchSearch Epoch[3]: Evaluating minibatchSizes 64..256
|
||||
MPI Rank 1: 09/17/2016 22:51:35: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=64 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:35: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 1.60900971 * 640; EvalClassificationError = 0.46875000 * 640; time = 0.1566s; samplesPerSecond = 4088.0
|
||||
MPI Rank 1: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 1.70269498 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1555s; samplesPerSecond = 4116.9
|
||||
MPI Rank 1: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 1.66433597 * 640; EvalClassificationError = 0.48281250 * 640; time = 0.1553s; samplesPerSecond = 4120.7
|
||||
MPI Rank 1: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 1.56997722 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1551s; samplesPerSecond = 4127.4
|
||||
MPI Rank 1: 09/17/2016 22:51:35: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 1.67908876 * 640; EvalClassificationError = 0.47656250 * 640; time = 0.1548s; samplesPerSecond = 4133.4
|
||||
MPI Rank 1: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 1.45189750 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1550s; samplesPerSecond = 4127.9
|
||||
MPI Rank 1: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 1.60053473 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1556s; samplesPerSecond = 4112.2
|
||||
MPI Rank 1: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 1.58536943 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1553s; samplesPerSecond = 4121.2
|
||||
MPI Rank 1: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 1.63246860 * 640; EvalClassificationError = 0.45781250 * 640; time = 0.1548s; samplesPerSecond = 4134.8
|
||||
MPI Rank 1: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 1.64770593 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1547s; samplesPerSecond = 4137.5
|
||||
MPI Rank 1: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.55685563 * 640; EvalClassificationError = 0.43750000 * 640; time = 0.1549s; samplesPerSecond = 4132.8
|
||||
MPI Rank 1: 09/17/2016 22:51:36: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 1.54493706 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1546s; samplesPerSecond = 4140.7
|
||||
MPI Rank 1: 09/17/2016 22:51:37: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 1.59221509 * 640; EvalClassificationError = 0.47031250 * 640; time = 0.1549s; samplesPerSecond = 4131.8
|
||||
MPI Rank 1: 09/17/2016 22:51:37: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 1.59923263 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1549s; samplesPerSecond = 4130.7
|
||||
MPI Rank 1: 09/17/2016 22:51:37: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 1.60345937 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1544s; samplesPerSecond = 4146.4
|
||||
MPI Rank 1: 09/17/2016 22:51:37: BaseAdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 1.59599131 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1531s; samplesPerSecond = 4180.6
|
||||
MPI Rank 1: 09/17/2016 22:51:37: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60223587 * 10240; EvalClassificationError = 0.45957031 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:37: AdaptiveMinibatchSearch Epoch[3]: Computed baseCriterion 1.60223587 for minibatchSize=64
|
||||
MPI Rank 1: 09/17/2016 22:51:37: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=64 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:37: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:37: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 0.78%]: CrossEntropyWithSoftmax = 1.60900971 * 640; EvalClassificationError = 0.46875000 * 640; time = 0.1562s; samplesPerSecond = 4097.7
|
||||
MPI Rank 1: 09/17/2016 22:51:37: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 1.56%]: CrossEntropyWithSoftmax = 1.70269498 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1550s; samplesPerSecond = 4128.3
|
||||
MPI Rank 1: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 2.34%]: CrossEntropyWithSoftmax = 1.66433597 * 640; EvalClassificationError = 0.48281250 * 640; time = 0.1554s; samplesPerSecond = 4118.0
|
||||
MPI Rank 1: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 3.12%]: CrossEntropyWithSoftmax = 1.56997722 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1545s; samplesPerSecond = 4143.4
|
||||
MPI Rank 1: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 3.91%]: CrossEntropyWithSoftmax = 1.67908876 * 640; EvalClassificationError = 0.47656250 * 640; time = 0.1553s; samplesPerSecond = 4122.3
|
||||
MPI Rank 1: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 4.69%]: CrossEntropyWithSoftmax = 1.45189750 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1537s; samplesPerSecond = 4165.2
|
||||
MPI Rank 1: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 5.47%]: CrossEntropyWithSoftmax = 1.60053473 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1553s; samplesPerSecond = 4121.8
|
||||
MPI Rank 1: 09/17/2016 22:51:38: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 6.25%]: CrossEntropyWithSoftmax = 1.58536943 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1557s; samplesPerSecond = 4111.2
|
||||
MPI Rank 1: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 81- 90, 7.03%]: CrossEntropyWithSoftmax = 1.63246860 * 640; EvalClassificationError = 0.45781250 * 640; time = 0.1552s; samplesPerSecond = 4124.6
|
||||
MPI Rank 1: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 91- 100, 7.81%]: CrossEntropyWithSoftmax = 1.64770593 * 640; EvalClassificationError = 0.45156250 * 640; time = 0.1550s; samplesPerSecond = 4128.4
|
||||
MPI Rank 1: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 101- 110, 8.59%]: CrossEntropyWithSoftmax = 1.55685563 * 640; EvalClassificationError = 0.43750000 * 640; time = 0.1546s; samplesPerSecond = 4139.2
|
||||
MPI Rank 1: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 111- 120, 9.38%]: CrossEntropyWithSoftmax = 1.54493706 * 640; EvalClassificationError = 0.42500000 * 640; time = 0.1550s; samplesPerSecond = 4128.2
|
||||
MPI Rank 1: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 121- 130, 10.16%]: CrossEntropyWithSoftmax = 1.59221509 * 640; EvalClassificationError = 0.47031250 * 640; time = 0.1551s; samplesPerSecond = 4126.0
|
||||
MPI Rank 1: 09/17/2016 22:51:39: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 131- 140, 10.94%]: CrossEntropyWithSoftmax = 1.59923263 * 640; EvalClassificationError = 0.48125000 * 640; time = 0.1558s; samplesPerSecond = 4108.1
|
||||
MPI Rank 1: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 141- 150, 11.72%]: CrossEntropyWithSoftmax = 1.60345937 * 640; EvalClassificationError = 0.45625000 * 640; time = 0.1547s; samplesPerSecond = 4136.4
|
||||
MPI Rank 1: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 151- 160, 12.50%]: CrossEntropyWithSoftmax = 1.59599131 * 640; EvalClassificationError = 0.46562500 * 640; time = 0.1535s; samplesPerSecond = 4168.2
|
||||
MPI Rank 1: 09/17/2016 22:51:40: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60223587 * 10240; EvalClassificationError = 0.45957031 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 64
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:40: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.60223587 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 1: 09/17/2016 22:51:40: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=128 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:40: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 1.56%]: CrossEntropyWithSoftmax = 1.59906005 * 1280; EvalClassificationError = 0.46328125 * 1280; time = 0.1775s; samplesPerSecond = 7210.7
|
||||
MPI Rank 1: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 3.12%]: CrossEntropyWithSoftmax = 1.59088280 * 1280; EvalClassificationError = 0.47031250 * 1280; time = 0.1751s; samplesPerSecond = 7309.9
|
||||
MPI Rank 1: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 4.69%]: CrossEntropyWithSoftmax = 1.55611991 * 1280; EvalClassificationError = 0.45468750 * 1280; time = 0.1739s; samplesPerSecond = 7360.4
|
||||
MPI Rank 1: 09/17/2016 22:51:40: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 6.25%]: CrossEntropyWithSoftmax = 1.59494809 * 1280; EvalClassificationError = 0.45937500 * 1280; time = 0.1734s; samplesPerSecond = 7382.6
|
||||
MPI Rank 1: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 7.81%]: CrossEntropyWithSoftmax = 1.64834247 * 1280; EvalClassificationError = 0.45703125 * 1280; time = 0.1743s; samplesPerSecond = 7342.5
|
||||
MPI Rank 1: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 51- 60, 9.38%]: CrossEntropyWithSoftmax = 1.56066565 * 1280; EvalClassificationError = 0.43593750 * 1280; time = 0.1733s; samplesPerSecond = 7387.2
|
||||
MPI Rank 1: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 61- 70, 10.94%]: CrossEntropyWithSoftmax = 1.62413186 * 1280; EvalClassificationError = 0.47265625 * 1280; time = 0.1740s; samplesPerSecond = 7357.5
|
||||
MPI Rank 1: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 71- 80, 12.50%]: CrossEntropyWithSoftmax = 1.67799703 * 1280; EvalClassificationError = 0.47031250 * 1280; time = 0.1712s; samplesPerSecond = 7478.8
|
||||
MPI Rank 1: 09/17/2016 22:51:41: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.60651848 * 10240; EvalClassificationError = 0.46044922 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 128
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:41: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.60651848 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 1: 09/17/2016 22:51:41: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=192 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:41: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:41: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 1.59151349 * 1920; EvalClassificationError = 0.46302083 * 1920; time = 0.1994s; samplesPerSecond = 9627.7
|
||||
MPI Rank 1: 09/17/2016 22:51:42: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 1.55305006 * 1920; EvalClassificationError = 0.45000000 * 1920; time = 0.1961s; samplesPerSecond = 9791.9
|
||||
MPI Rank 1: 09/17/2016 22:51:42: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 1.60654814 * 1920; EvalClassificationError = 0.45781250 * 1920; time = 0.1945s; samplesPerSecond = 9873.0
|
||||
MPI Rank 1: 09/17/2016 22:51:42: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 1.60526168 * 1920; EvalClassificationError = 0.44947917 * 1920; time = 0.1944s; samplesPerSecond = 9874.7
|
||||
MPI Rank 1: 09/17/2016 22:51:42: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 1.68437187 * 1920; EvalClassificationError = 0.49531250 * 1920; time = 0.1972s; samplesPerSecond = 9737.7
|
||||
MPI Rank 1: 09/17/2016 22:51:42: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.61407118 * 10240; EvalClassificationError = 0.46357422 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 192
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:42: AdaptiveMinibatchSearch Epoch[3]: Keep searching... epochCriterion = 1.61407118 vs. baseCriterion = 1.60223587
|
||||
MPI Rank 1: 09/17/2016 22:51:42: AdaptiveMinibatchSearch Epoch[3]: Evaluating trial minibatchSize=256 (search range: 64..256)...
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [20480..30720] (first utterance at frame 20480), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:42: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:43: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 1- 10, 3.12%]: CrossEntropyWithSoftmax = 1.59241097 * 2560; EvalClassificationError = 0.46093750 * 2560; time = 0.2327s; samplesPerSecond = 11003.6
|
||||
MPI Rank 1: 09/17/2016 22:51:43: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 11- 20, 6.25%]: CrossEntropyWithSoftmax = 1.59578299 * 2560; EvalClassificationError = 0.46093750 * 2560; time = 0.2209s; samplesPerSecond = 11589.6
|
||||
MPI Rank 1: 09/17/2016 22:51:43: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 21- 30, 9.38%]: CrossEntropyWithSoftmax = 1.68033748 * 2560; EvalClassificationError = 0.47343750 * 2560; time = 0.2160s; samplesPerSecond = 11849.5
|
||||
MPI Rank 1: 09/17/2016 22:51:43: AdaptiveMinibatchSearch: Epoch[ 3 of 3]-Minibatch[ 31- 40, 12.50%]: CrossEntropyWithSoftmax = 1.73124308 * 2560; EvalClassificationError = 0.48164062 * 2560; time = 0.2136s; samplesPerSecond = 11985.6
|
||||
MPI Rank 1: 09/17/2016 22:51:43: Finished Mini-Epoch[3]: CrossEntropyWithSoftmax = 1.64994363 * 10240; EvalClassificationError = 0.46923828 * 10240; learningRatePerSample = 0.001953125; minibatchSize = 256
|
||||
MPI Rank 1: ValidateSubNetwork: InvStdOfFeatures InvStdDev operation changed, from [363 x 1] to [363].ValidateSubNetwork: MeanOfFeatures Mean operation changed, from [363 x 1] to [363].ValidateSubNetwork: Prior Mean operation changed, from [132 x 1] to [132].09/17/2016 22:51:43: AdaptiveMinibatchSearch Epoch[3]: Search successful. New minibatchSize is 192. epochCriterion = 1.61407118 vs baseCriterion = 1.60223587
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:43: Starting Epoch 3: learning rate per sample = 0.001953 effective momentum = 0.829029 momentum as time constant = 1024.0 samples
|
||||
MPI Rank 1: minibatchiterator: epoch 2: frames [163840..245760] (first utterance at frame 163840), data subset 1 of 2, with 1 datapasses
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:43: Starting minibatch loop, DataParallelSGD training (MyRank = 1, NumNodes = 2, NumGradientBits = 2), distributed reading is ENABLED.
|
||||
MPI Rank 1: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 1- 10, 2.34%]: CrossEntropyWithSoftmax = 1.54988471 * 1920; EvalClassificationError = 0.46510417 * 1920; time = 0.2004s; samplesPerSecond = 9583.0
|
||||
MPI Rank 1: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 11- 20, 4.69%]: CrossEntropyWithSoftmax = 1.62737285 * 1920; EvalClassificationError = 0.47395833 * 1920; time = 0.1970s; samplesPerSecond = 9744.7
|
||||
MPI Rank 1: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 21- 30, 7.03%]: CrossEntropyWithSoftmax = 1.65385199 * 1920; EvalClassificationError = 0.46302083 * 1920; time = 0.1938s; samplesPerSecond = 9907.1
|
||||
MPI Rank 1: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 31- 40, 9.38%]: CrossEntropyWithSoftmax = 1.87023223 * 1920; EvalClassificationError = 0.51666667 * 1920; time = 0.1967s; samplesPerSecond = 9760.2
|
||||
MPI Rank 1: 09/17/2016 22:51:44: Epoch[ 3 of 3]-Minibatch[ 41- 50, 11.72%]: CrossEntropyWithSoftmax = 1.75815123 * 1920; EvalClassificationError = 0.49270833 * 1920; time = 0.1971s; samplesPerSecond = 9743.6
|
||||
MPI Rank 1: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 51- 60, 14.06%]: CrossEntropyWithSoftmax = 2.03099563 * 1920; EvalClassificationError = 0.50312500 * 1920; time = 0.1945s; samplesPerSecond = 9870.3
|
||||
MPI Rank 1: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 61- 70, 16.41%]: CrossEntropyWithSoftmax = 1.86165460 * 1920; EvalClassificationError = 0.53489583 * 1920; time = 0.1951s; samplesPerSecond = 9843.3
|
||||
MPI Rank 1: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 71- 80, 18.75%]: CrossEntropyWithSoftmax = 2.15179657 * 1920; EvalClassificationError = 0.49010417 * 1920; time = 0.1947s; samplesPerSecond = 9860.9
|
||||
MPI Rank 1: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 81- 90, 21.09%]: CrossEntropyWithSoftmax = 1.79956105 * 1920; EvalClassificationError = 0.50625000 * 1920; time = 0.1956s; samplesPerSecond = 9816.5
|
||||
MPI Rank 1: 09/17/2016 22:51:45: Epoch[ 3 of 3]-Minibatch[ 91- 100, 23.44%]: CrossEntropyWithSoftmax = 1.80123726 * 1920; EvalClassificationError = 0.48385417 * 1920; time = 0.1949s; samplesPerSecond = 9852.0
|
||||
MPI Rank 1: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 101- 110, 25.78%]: CrossEntropyWithSoftmax = 1.65520548 * 1920; EvalClassificationError = 0.47135417 * 1920; time = 0.1949s; samplesPerSecond = 9850.2
|
||||
MPI Rank 1: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 111- 120, 28.12%]: CrossEntropyWithSoftmax = 1.65536374 * 1920; EvalClassificationError = 0.46875000 * 1920; time = 0.1957s; samplesPerSecond = 9810.9
|
||||
MPI Rank 1: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 121- 130, 30.47%]: CrossEntropyWithSoftmax = 1.62640983 * 1920; EvalClassificationError = 0.47656250 * 1920; time = 0.1946s; samplesPerSecond = 9867.6
|
||||
MPI Rank 1: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 131- 140, 32.81%]: CrossEntropyWithSoftmax = 1.58072809 * 1920; EvalClassificationError = 0.44895833 * 1920; time = 0.1944s; samplesPerSecond = 9878.2
|
||||
MPI Rank 1: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 141- 150, 35.16%]: CrossEntropyWithSoftmax = 1.56697558 * 1920; EvalClassificationError = 0.44687500 * 1920; time = 0.1948s; samplesPerSecond = 9856.5
|
||||
MPI Rank 1: 09/17/2016 22:51:46: Epoch[ 3 of 3]-Minibatch[ 151- 160, 37.50%]: CrossEntropyWithSoftmax = 1.58252868 * 1920; EvalClassificationError = 0.46250000 * 1920; time = 0.1951s; samplesPerSecond = 9842.6
|
||||
MPI Rank 1: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 161- 170, 39.84%]: CrossEntropyWithSoftmax = 1.53206179 * 1920; EvalClassificationError = 0.45104167 * 1920; time = 0.1949s; samplesPerSecond = 9852.3
|
||||
MPI Rank 1: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 171- 180, 42.19%]: CrossEntropyWithSoftmax = 1.47925397 * 1920; EvalClassificationError = 0.44531250 * 1920; time = 0.1945s; samplesPerSecond = 9873.2
|
||||
MPI Rank 1: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 181- 190, 44.53%]: CrossEntropyWithSoftmax = 1.46468817 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.1942s; samplesPerSecond = 9889.0
|
||||
MPI Rank 1: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 191- 200, 46.88%]: CrossEntropyWithSoftmax = 1.51778272 * 1920; EvalClassificationError = 0.45468750 * 1920; time = 0.1956s; samplesPerSecond = 9818.3
|
||||
MPI Rank 1: 09/17/2016 22:51:47: Epoch[ 3 of 3]-Minibatch[ 201- 210, 49.22%]: CrossEntropyWithSoftmax = 1.43521243 * 1920; EvalClassificationError = 0.41979167 * 1920; time = 0.1930s; samplesPerSecond = 9948.2
|
||||
MPI Rank 1: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 211- 220, 51.56%]: CrossEntropyWithSoftmax = 1.55602002 * 1920; EvalClassificationError = 0.45468750 * 1920; time = 0.1949s; samplesPerSecond = 9850.9
|
||||
MPI Rank 1: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 221- 230, 53.91%]: CrossEntropyWithSoftmax = 1.48434301 * 1920; EvalClassificationError = 0.44479167 * 1920; time = 0.1926s; samplesPerSecond = 9968.3
|
||||
MPI Rank 1: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 231- 240, 56.25%]: CrossEntropyWithSoftmax = 1.47880634 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.1941s; samplesPerSecond = 9892.2
|
||||
MPI Rank 1: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 241- 250, 58.59%]: CrossEntropyWithSoftmax = 1.44452798 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.1942s; samplesPerSecond = 9886.9
|
||||
MPI Rank 1: 09/17/2016 22:51:48: Epoch[ 3 of 3]-Minibatch[ 251- 260, 60.94%]: CrossEntropyWithSoftmax = 1.42272624 * 1920; EvalClassificationError = 0.42395833 * 1920; time = 0.1939s; samplesPerSecond = 9902.6
|
||||
MPI Rank 1: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 261- 270, 63.28%]: CrossEntropyWithSoftmax = 1.39175020 * 1920; EvalClassificationError = 0.41354167 * 1920; time = 0.1954s; samplesPerSecond = 9828.0
|
||||
MPI Rank 1: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 271- 280, 65.62%]: CrossEntropyWithSoftmax = 1.47323044 * 1920; EvalClassificationError = 0.43333333 * 1920; time = 0.1949s; samplesPerSecond = 9852.5
|
||||
MPI Rank 1: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 281- 290, 67.97%]: CrossEntropyWithSoftmax = 1.46072580 * 1920; EvalClassificationError = 0.41666667 * 1920; time = 0.1950s; samplesPerSecond = 9848.2
|
||||
MPI Rank 1: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 291- 300, 70.31%]: CrossEntropyWithSoftmax = 1.47105355 * 1920; EvalClassificationError = 0.43177083 * 1920; time = 0.1947s; samplesPerSecond = 9859.2
|
||||
MPI Rank 1: 09/17/2016 22:51:49: Epoch[ 3 of 3]-Minibatch[ 301- 310, 72.66%]: CrossEntropyWithSoftmax = 1.40973818 * 1920; EvalClassificationError = 0.41927083 * 1920; time = 0.1926s; samplesPerSecond = 9970.9
|
||||
MPI Rank 1: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 311- 320, 75.00%]: CrossEntropyWithSoftmax = 1.44253894 * 1920; EvalClassificationError = 0.42604167 * 1920; time = 0.1941s; samplesPerSecond = 9894.1
|
||||
MPI Rank 1: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 321- 330, 77.34%]: CrossEntropyWithSoftmax = 1.39941047 * 1920; EvalClassificationError = 0.42239583 * 1920; time = 0.1949s; samplesPerSecond = 9853.2
|
||||
MPI Rank 1: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 331- 340, 79.69%]: CrossEntropyWithSoftmax = 1.42524482 * 1920; EvalClassificationError = 0.41927083 * 1920; time = 0.1938s; samplesPerSecond = 9909.0
|
||||
MPI Rank 1: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 341- 350, 82.03%]: CrossEntropyWithSoftmax = 1.46079356 * 1920; EvalClassificationError = 0.42916667 * 1920; time = 0.1959s; samplesPerSecond = 9800.5
|
||||
MPI Rank 1: 09/17/2016 22:51:50: Epoch[ 3 of 3]-Minibatch[ 351- 360, 84.38%]: CrossEntropyWithSoftmax = 1.39608704 * 1920; EvalClassificationError = 0.42552083 * 1920; time = 0.1940s; samplesPerSecond = 9895.2
|
||||
MPI Rank 1: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 361- 370, 86.72%]: CrossEntropyWithSoftmax = 1.46624909 * 1920; EvalClassificationError = 0.43177083 * 1920; time = 0.1933s; samplesPerSecond = 9930.4
|
||||
MPI Rank 1: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 371- 380, 89.06%]: CrossEntropyWithSoftmax = 1.42432290 * 1920; EvalClassificationError = 0.42031250 * 1920; time = 0.1940s; samplesPerSecond = 9894.6
|
||||
MPI Rank 1: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 381- 390, 91.41%]: CrossEntropyWithSoftmax = 1.36453678 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.1934s; samplesPerSecond = 9927.9
|
||||
MPI Rank 1: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 391- 400, 93.75%]: CrossEntropyWithSoftmax = 1.38669452 * 1920; EvalClassificationError = 0.41041667 * 1920; time = 0.1949s; samplesPerSecond = 9852.6
|
||||
MPI Rank 1: 09/17/2016 22:51:51: Epoch[ 3 of 3]-Minibatch[ 401- 410, 96.09%]: CrossEntropyWithSoftmax = 1.37926773 * 1920; EvalClassificationError = 0.41614583 * 1920; time = 0.1955s; samplesPerSecond = 9820.6
|
||||
MPI Rank 1: 09/17/2016 22:51:52: Epoch[ 3 of 3]-Minibatch[ 411- 420, 98.44%]: CrossEntropyWithSoftmax = 1.39890438 * 1920; EvalClassificationError = 0.41875000 * 1920; time = 0.1944s; samplesPerSecond = 9878.7
|
||||
MPI Rank 1: 09/17/2016 22:51:52: Finished Epoch[ 3 of 3]: [Training] CrossEntropyWithSoftmax = 1.55322819 * 81920; EvalClassificationError = 0.44776611 * 81920; totalSamplesSeen = 245760; learningRatePerSample = 0.001953125; epochTime=17.0573s
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:52: Action "train" complete.
|
||||
MPI Rank 1:
|
||||
MPI Rank 1: 09/17/2016 22:51:52: __COMPLETED__
|
||||
MPI Rank 1: ~MPIWrapper
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
precision = "double"
|
||||
command = speechTrain
|
||||
deviceId = $DeviceId$
|
||||
|
||||
parallelTrain = true
|
||||
|
||||
speechTrain = [
|
||||
action = "train"
|
||||
modelPath = "$RunDir$/models/cntkSpeech.dnn"
|
||||
deviceId = $DeviceId$
|
||||
traceLevel = 1
|
||||
|
||||
SimpleNetworkBuilder = [
|
||||
layerSizes = 363:512:512:132
|
||||
trainingCriterion = "CrossEntropyWithSoftmax"
|
||||
evalCriterion = "ClassificationError"
|
||||
layerTypes = "Sigmoid"
|
||||
initValueScale = 1.0
|
||||
applyMeanVarNorm = true
|
||||
uniformInit = true
|
||||
needPrior = true
|
||||
]
|
||||
|
||||
SGD = [
|
||||
epochSize = 81920
|
||||
minibatchSize = 64
|
||||
learningRatesPerSample = 0.001953125
|
||||
numMBsToShowResult = 10
|
||||
momentumAsTimeConstant = 1024
|
||||
dropoutRate = 0.0
|
||||
maxEpochs = 3
|
||||
keepCheckPointFiles = true
|
||||
clippingThresholdPerSample = 1#INF
|
||||
|
||||
ParallelTrain = [
|
||||
parallelizationMethod = "DataParallelSGD"
|
||||
distributedMBReading = true
|
||||
DataParallelSGD = [
|
||||
gradientBits = 2
|
||||
]
|
||||
]
|
||||
|
||||
AutoAdjust = [
|
||||
autoAdjustMinibatch = true
|
||||
numMiniBatch4LRSearch = 160
|
||||
minibatchSizeTuningFrequency = 1
|
||||
minibatchSizeTuningMax = 8192
|
||||
minibatchSearchCriterionErrorMargin = 1
|
||||
]
|
||||
]
|
||||
reader = [
|
||||
readerType = "HTKMLFReader"
|
||||
readMethod = "blockRandomize"
|
||||
miniBatchMode = "partial"
|
||||
randomize = "auto"
|
||||
verbosity = 0
|
||||
useMersenneTwisterRand=true
|
||||
|
||||
features = [
|
||||
dim = 363
|
||||
type = "real"
|
||||
scpFile = "glob_0000.scp"
|
||||
]
|
||||
|
||||
labels = [
|
||||
mlfFile = "$DataDir$/glob_0000.mlf"
|
||||
labelMappingFile = "$DataDir$/state.list"
|
||||
labelDim = 132
|
||||
labelType = "category"
|
||||
]
|
||||
]
|
||||
]
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
. $TEST_ROOT_DIR/run-test-common
|
||||
|
||||
ConfigDir=$TEST_DIR
|
||||
LogFileName=stderr
|
||||
Instances=2
|
||||
NumCPUThreads=$(threadsPerInstance $Instances)
|
||||
|
||||
# cntkmpirun <MPI args> <CNTK config file name> <additional CNTK args>
|
||||
cntkmpirun "-n $Instances" cntk.cntk "numCPUThreads=$NumCPUThreads"
|
||||
ExitCode=$?
|
||||
sed 's/^/MPI Rank 0: /' $TEST_RUN_DIR/"$LogFileName"_speechTrain.logrank0
|
||||
sed 's/^/MPI Rank 1: /' $TEST_RUN_DIR/"$LogFileName"_speechTrain.logrank1
|
||||
exit $ExitCode
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
dataDir: ../../Data
|
||||
tags:
|
||||
# running for 1bitsgd build SKU on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
|
||||
- bvt-s (build_sku == '1bitsgd') and ((flavor=='debug') ^ (device=='cpu'))
|
||||
# running for 1bitsgd build SKU on every Nightly job in 'S' leg
|
||||
- nightly-s (build_sku == '1bitsgd')
|
||||
|
||||
testCases:
|
||||
Must train epochs in exactly same order and parameters for each MPI Rank:
|
||||
patterns:
|
||||
- ^MPI Rank {{integer}}
|
||||
- Starting Epoch {{integer}}
|
||||
- learning rate per sample = {{float}}
|
||||
- momentum = {{float}}
|
||||
|
||||
Epochs must be finished with expected results for each MPI Rank:
|
||||
patterns:
|
||||
- ^MPI Rank {{integer}}
|
||||
- Finished Epoch[{{integer}} of {{integer}}]
|
||||
- CrossEntropyWithSoftmax = {{float,tolerance=0%}}
|
||||
- EvalClassificationError = {{float,tolerance=0%}}
|
||||
- learningRatePerSample = {{float,tolerance=0.001%}}
|
||||
|
||||
Per-minibatch training results must match for each MPI Rank:
|
||||
patterns:
|
||||
- ^MPI Rank {{integer}}
|
||||
- Epoch[{{integer}} of {{integer}}]-Minibatch[{{integer}}-{{integer}}
|
||||
- " * {{integer}}; "
|
||||
- CrossEntropyWithSoftmax = {{float,tolerance=0%}}
|
||||
- EvalClassificationError = {{float,tolerance=0%}}
|
||||
|
||||
DataParallelSGD training parameters must match for each MPI Rank:
|
||||
patterns:
|
||||
- ^MPI Rank {{integer}}
|
||||
- Starting minibatch loop
|
||||
- DataParallelSGD training
|
||||
- MyRank = {{integer}}
|
||||
- NumNodes = 2
|
||||
- NumGradientBits = 2
|
||||
- distributed reading is ENABLED
|
||||
Загрузка…
Ссылка в новой задаче