microsoft
/
CNTK
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

split ComputationNode.h off into 5 additional headers (InputAndParam, LinearAlgebra, Nonlinearity, Recurrent, Convolutional), grouped by nature of their function (but no code change other than moving classes in this check-in); 

sorted #includes in many places (system #includes first), and removed some unnecessary #includes;
TWO_PI moved into Basics.h;
fixed an unused-argument warning in GPUMatrix.cu
This commit is contained in:
Frank Seide 2015-05-18 23:40:48 -07:00
Родитель 87a504dd6c
Коммит ec4de72b7e
22 изменённых файлов: 5929 добавлений и 5768 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

5
Common/Include/Basics.h
Просмотреть файл

@ -8,8 +8,9 @@
#define _BASICS_H_
#include "basetypes.h" // TODO: gradually move over here all that's needed of basetypes.h, then remove basetypes.h.
#define TWO_PI 6.283185307f // TODO: find the official standards-confirming definition of this and use it instead
// ===========================================================================
// emulation of some MSVC proprietary CRT
// ===========================================================================

5
MachineLearning/CNTK/CNTK.vcxproj
Просмотреть файл

@ -167,13 +167,18 @@
<ClInclude Include="ComputationNetwork.h" />
<ClInclude Include="ComputationNetworkHelper.h" />
<ClInclude Include="ComputationNode.h" />
<ClInclude Include="ConvolutionalNodes.h" />
<ClInclude Include="EvaluationCriterionNodes.h" />
<ClInclude Include="IComputationNetBuilder.h" />
<ClInclude Include="IExecutionEngine.h" />
<ClInclude Include="InputAndParamNodes.h" />
<ClInclude Include="LinearAlgebraNodes.h" />
<ClInclude Include="ModelEditLanguage.h" />
<ClInclude Include="NDLNetworkBuilder.h" />
<ClInclude Include="NDLUtil.h" />
<ClInclude Include="NetworkDescriptionLanguage.h" />
<ClInclude Include="NonlinearityNodes.h" />
<ClInclude Include="RecurrentNodes.h" />
<ClInclude Include="SimpleEvaluator.h" />
<ClInclude Include="SimpleOutputWriter.h" />
<ClInclude Include="SGD.h" />

15
MachineLearning/CNTK/CNTK.vcxproj.filters
Просмотреть файл

@ -136,6 +136,21 @@
<ClInclude Include="TrainingCriterionNodes.h">
<Filter>Nodes</Filter>
</ClInclude>
<ClInclude Include="NonlinearityNodes.h">
<Filter>Nodes</Filter>
</ClInclude>
<ClInclude Include="LinearAlgebraNodes.h">
<Filter>Nodes</Filter>
</ClInclude>
<ClInclude Include="ConvolutionalNodes.h">
<Filter>Nodes</Filter>
</ClInclude>
<ClInclude Include="RecurrentNodes.h">
<Filter>Nodes</Filter>
</ClInclude>
<ClInclude Include="InputAndParamNodes.h">
<Filter>Nodes</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Text Include="modelEditor.txt">

1391
MachineLearning/CNTK/CompositeComputationNodes.h
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

21
MachineLearning/CNTK/ComputationNetwork.h
Просмотреть файл

@ -17,17 +17,24 @@
#include <fstream>
#include <sstream>
#include <stdlib.h>
#include "ComputationNode.h"
#include "TrainingCriterionNodes.h"
#include "CompositeComputationNodes.h"
#include "EvaluationCriterionNodes.h"
#include "File.h"
#include "Matrix.h"
#include "commandArgUtil.h"
#include <iostream>
#include <regex>
#include <chrono>
#include "File.h"
#include "Matrix.h"
#include "commandArgUtil.h" // for nocase_compare
#include "ComputationNode.h"
#include "InputAndParamNodes.h"
#include "LinearAlgebraNodes.h"
#include "NonlinearityNodes.h"
#include "ConvolutionalNodes.h"
#include "RecurrentNodes.h"
#include "TrainingCriterionNodes.h"
#include "CompositeComputationNodes.h"
#include "EvaluationCriterionNodes.h"
namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
class ComputationNetwork

11
MachineLearning/CNTK/ComputationNetworkHelper.h
Просмотреть файл

@ -5,17 +5,16 @@
//
#pragma once
#include "ComputationNetwork.h"
#include "DataReader.h"
#include <vector>
#include <string>
#include <stdexcept>
#include <fstream>
#include "Basics.h"
#include "fileutil.h"
#include "commandArgUtil.h"
//#include <Windows.h>
//#include <WinBase.h>
#include <fstream>
#include "ComputationNetwork.h"
#include "DataReader.h"
using namespace std;

4
MachineLearning/CNTK/ComputationNode.cpp
Просмотреть файл

@ -7,9 +7,7 @@
#define _CRT_SECURE_NO_WARNINGS // "secure" CRT not available on all platforms --add this at the top of all CPP files that give "function or variable may be unsafe" warnings
#include "ComputationNode.h"
#include "SimpleEvaluator.h"
#include "IComputationNetBuilder.h"
#include "SGD.h"
#include "InputAndParamNodes.h"
namespace Microsoft { namespace MSR { namespace CNTK {

4378
MachineLearning/CNTK/ComputationNode.h
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

945
MachineLearning/CNTK/ConvolutionalNodes.h Normal file
Просмотреть файл

@ -0,0 +1,945 @@
//
// <copyright file="ConvolutionalNodes.h" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//
#pragma once
#include <unordered_set>
#include <map>
#include <string>
#include <vector>
#include <stdexcept>
#include <list>
#include <memory>
#include <algorithm>
#include <assert.h>
#include <atomic>
#include <sstream>
#include <iostream>
#include "Basics.h"
#include "Matrix.h"
#include "ComputationNode.h"
namespace Microsoft { namespace MSR { namespace CNTK {
// convolution parameters structure, to make it easier to pass these around all these parameters
struct ConvolutionParams
{
size_t inputWidth, inputHeight, inputChannels;
size_t kernelWidth, kernelHeight;
size_t horizontalSubsample, verticalSubsample;
size_t outputWidth, outputHeight, outputChannels;
size_t maxTempMemSizeInSamples;
bool zeroPadding;
};
//convolutional network
//follow "high performance convolutional neural networks for document processing" by Kumar chellapilla, Sidde Puri, and Patrice Simard
//assume each column is an input sample. Each sample is stored in [channel, row, col] (r00, g00, b00, r01, g01, b01, r10, g10, b10, r11, g11, b11)
template<class ElemType>
class ConvolutionNode : public ComputationNode<ElemType>
{
UsingComputationNodeMembers;
public:
ConvolutionNode(const size_t kernelWidth, const size_t kernelHeight, const size_t outputChannels,
const size_t horizontalSubsample, const size_t verticalSubsample,
const bool zeroPadding = false,
const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"",
const size_t maxTempMemSizeInSamples = 0)
: ComputationNode<ElemType>(deviceId), m_tempMatrix(deviceId),
m_kernelWidth(kernelWidth), m_kernelHeight(kernelHeight),
m_horizontalSubsample(horizontalSubsample), m_verticalSubsample(verticalSubsample),
m_zeroPadding(zeroPadding), m_maxTempMemSizeInSamples(maxTempMemSizeInSamples)
{
m_outputChannels = outputChannels;
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
InitRecurrentNode();
}
ConvolutionNode(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"")
: ComputationNode<ElemType>(deviceId), m_tempMatrix(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
LoadFromFile(fstream, modelVersion, deviceId);
}
virtual void SaveToFile(File& fstream) const
{
ComputationNode<ElemType>::SaveToFile(fstream);
fstream << m_kernelWidth << m_kernelHeight << m_horizontalSubsample << m_verticalSubsample;
fstream << m_outputChannels << m_zeroPadding << m_maxTempMemSizeInSamples;
}
virtual void LoadFromFile(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX)
{
ComputationNode<ElemType>::LoadFromFile(fstream, modelVersion, deviceId);
fstream >> m_kernelWidth >> m_kernelHeight >> m_horizontalSubsample >> m_verticalSubsample;
fstream >> m_outputChannels >> m_zeroPadding >> m_maxTempMemSizeInSamples;
}
virtual void CopyTo(const ComputationNodePtr nodeP, const std::wstring& newName, const CopyNodeFlags flags) const
{
ComputationNode<ElemType>::CopyTo(nodeP, newName, flags);
ConvolutionNode<ElemType>* node = (ConvolutionNode<ElemType>*) nodeP;
if (flags & CopyNodeFlags::copyNodeValue)
{
node->m_kernelWidth = m_kernelWidth;
node->m_kernelHeight = m_kernelHeight;
node->m_horizontalSubsample = m_horizontalSubsample;
node->m_verticalSubsample = m_verticalSubsample;
node->m_zeroPadding = m_zeroPadding;
node->m_maxTempMemSizeInSamples = m_maxTempMemSizeInSamples;
node->m_tempMatrix = m_tempMatrix;
}
}
// copy constructor
ConvolutionNode(const ConvolutionNode<ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags)
: ComputationNode<ElemType>(node->m_deviceId), m_tempMatrix(node->m_deviceId)
{
node->CopyTo(this, newName, flags);
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new ConvolutionNode<ElemType>(this, name, flags);
return node;
}
virtual const std::wstring OperationName() const {return TypeName();}
static const std::wstring TypeName() {return L"Convolution";}
ConvolutionParams GetConvolutionParams() const
{
ConvolutionParams convParam;
convParam.inputWidth = m_inputWidth;
convParam.inputHeight = m_inputHeight;
convParam.inputChannels = m_inputChannels;
convParam.kernelWidth = m_kernelWidth;
convParam.kernelHeight = m_kernelHeight;
convParam.horizontalSubsample = m_horizontalSubsample;
convParam.verticalSubsample = m_verticalSubsample;
convParam.outputWidth = m_outputWidth;
convParam.outputHeight = m_outputHeight;
convParam.outputChannels = m_outputChannels;
convParam.zeroPadding = m_zeroPadding;
convParam.maxTempMemSizeInSamples = m_maxTempMemSizeInSamples;
return convParam;
}
virtual void ComputeInputPartial(const size_t inputIndex)
{
if (inputIndex > 1)
throw std::invalid_argument("Convolution operation only takes two inputs.");
if (inputIndex == 0) //derivative with regard to the weight matrix
{
ComputeInputPartialOverWeight(this, GradientValues(), Inputs(0)->GradientValues(), Inputs(0)->FunctionValues(), Inputs(1)->FunctionValues(), m_tempMatrix, true);
}
else // derivative with regard to the input feature
{
ComputeInputPartialOverInputFeature(this, GradientValues(), Inputs(1)->GradientValues(), Inputs(0)->FunctionValues(), Inputs(1)->FunctionValues(), m_tempMatrix);
}
}
virtual void ComputeInputPartial(const size_t inputIndex, const size_t timeIdxInSeq)
{
if (inputIndex > 1)
throw std::invalid_argument("Convolution operation only takes two inputs.");
Matrix<ElemType> sliceOutputGrad = GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceInput1Value = Inputs(1)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
if (inputIndex == 0) //derivative with regard to the weight matrix
{
ComputeInputPartialOverWeight(this, sliceOutputGrad, Inputs(0)->GradientValues(), Inputs(0)->FunctionValues(), sliceInput1Value, m_tempMatrix);
}
else // derivative with regard to the input feature
{
Matrix<ElemType> sliceInput1Grad = Inputs(1)->GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
ComputeInputPartialOverInputFeature(this, sliceOutputGrad, sliceInput1Grad, Inputs(0)->FunctionValues(), sliceInput1Value, m_tempMatrix);
}
}
virtual void EvaluateThisNode()
{
EvaluateThisNodeS(this, FunctionValues(), Inputs(0)->FunctionValues(), Inputs(1)->FunctionValues(), m_tempMatrix);
}
virtual void EvaluateThisNode(const size_t timeIdxInSeq)
{
Matrix<ElemType> sliceInput1Value = Inputs(1)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceOutputValue = m_functionValues.ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
EvaluateThisNodeS(this, sliceOutputValue, Inputs(0)->FunctionValues(), sliceInput1Value, m_tempMatrix);
}
static void WINAPI EvaluateThisNodeS(const ConvolutionNode<ElemType>* pConv, Matrix<ElemType> &functionValues, const Matrix<ElemType> &input0,
const Matrix<ElemType> &input1, Matrix<ElemType> &tempMatrix)
{
#if NANCHECK
input0.HasNan("Convolution-input0");
input1.HasNan("Convolution-input1");
#endif
ConvolutionParams convParam = pConv->GetConvolutionParams();
size_t packedInputRows = convParam.kernelWidth * convParam.kernelHeight * convParam.inputChannels;
size_t packedInputColsPerSample = convParam.outputWidth * convParam.outputHeight;
size_t outputSizePerChannel = packedInputColsPerSample;
//size_t packedInputDim = packedInputRows * packedInputColsPerSample; // size of each packed input sample
//size_t inputDim = convParam.inputWidth * convParam.inputHeight * convParam.inputChannels; //size of each input sample
long batchSize = (long)input1.GetNumCols(); //right child is the input sample
long maxTempMemSizeInSamples = (long)(convParam.maxTempMemSizeInSamples == 0? batchSize : convParam.maxTempMemSizeInSamples);
const Matrix<ElemType> & weightMatrix = input0;
assert(weightMatrix.GetNumCols() == packedInputRows && weightMatrix.GetNumRows() == convParam.outputChannels);
functionValues.Resize(convParam.outputChannels, outputSizePerChannel * batchSize);
long subBatchSize = (long)min(batchSize, maxTempMemSizeInSamples);
long numSubBatches = (batchSize+subBatchSize-1)/subBatchSize;
for (long i=0; i<numSubBatches; i++)
{
long startSampleID = i*subBatchSize;
long endSampleID = min(batchSize, startSampleID + subBatchSize);
long smallBatchSize = endSampleID-startSampleID;
tempMatrix.Resize(packedInputRows, packedInputColsPerSample * smallBatchSize);
Matrix<ElemType> inputSubBatch = input1.ColumnSlice(startSampleID, smallBatchSize);
tempMatrix.AssignPackedConvolutionInput(inputSubBatch,
convParam.inputWidth, convParam.inputHeight, convParam.inputChannels,
convParam.outputWidth, convParam.outputHeight, convParam.outputChannels,
convParam.kernelWidth, convParam.kernelHeight, convParam.horizontalSubsample, convParam.verticalSubsample,
convParam.zeroPadding);
Matrix<ElemType> outputSubBatch = functionValues.ColumnSlice(outputSizePerChannel * startSampleID, outputSizePerChannel * smallBatchSize);
Matrix<ElemType>::Multiply(weightMatrix, false, tempMatrix, false, outputSubBatch);
}
functionValues.Reshape(convParam.outputChannels * outputSizePerChannel, batchSize); //each sample becomes a column
#if NANCHECK
functionValues.HasNan("Convolution");
#endif
}
virtual void Validate()
{
PrintSelfBeforeValidation();
if (m_children.size() != 2)
throw std::logic_error("ConvolutionNode requires two inputs.");
//we may want to remove this check in the future if we want to support the case that the weight itself is result of some computation
//if (Inputs(0)->OperationName() != LearnableParameter<ElemType>::TypeName())
// throw std::logic_error("ConvolutionNode requires the first input to be LearnableParameter type.");
if (m_horizontalSubsample > m_kernelWidth || m_verticalSubsample > m_kernelHeight)
throw std::invalid_argument("In ConvolutionNode horizontalSubsample must <= kernelWidth and verticalSubsample must <= kernelHeight.");
CopyImageSizeFromInputs();
size_t weightCols = m_kernelWidth * m_kernelHeight * m_inputChannels;
if (Inputs(0)->OperationName() == LearnableParameter<ElemType>::TypeName() && Inputs(0)->FunctionValues().GetNumElements() == 0)
{
Inputs(0)->FunctionValues().Resize(m_outputChannels, weightCols);
}
if (m_children[0]->FunctionValues().GetNumCols() != weightCols || m_children[0]->FunctionValues().GetNumRows() != m_outputChannels)
{
msra::strfun::strprintf msg("convolutionWeight matrix %ls should have dimension [%d, %d] which is [outputChannels, kernelWidth * kernelHeight * inputChannels]",
m_children[0]->NodeName().c_str(), m_outputChannels, weightCols);
throw std::logic_error(msg.c_str());
}
size_t inputDim = m_inputWidth * m_inputHeight * m_inputChannels;
if (Inputs(1)->OperationName() == LearnableParameter<ElemType>::TypeName() && Inputs(1)->FunctionValues().GetNumRows() == 0)
{
Inputs(1)->FunctionValues().Resize(inputDim, Inputs(1)->FunctionValues().GetNumCols());
}
if (m_children[1]->FunctionValues().GetNumRows() != inputDim)
{
msra::strfun::strprintf msg("each column of input to the convolution node %ls is a sample and should have dimension %d, which is inputWidth * inputHeight * inputChannels",
NodeName().c_str(), inputDim);
throw std::logic_error(msg.c_str());
}
if (Inputs(0)->FunctionValues().GetNumElements() == 0 || Inputs(1)->FunctionValues().GetNumElements() == 0 )
throw std::logic_error("Convolution operation: one of the operants has 0 element.");
size_t outputDim = m_outputWidth * m_outputHeight * m_outputChannels;
FunctionValues().Resize(outputDim, m_children[1]->FunctionValues().GetNumCols());
}
virtual void CopyImageSizeFromInputs()
{
CopyImageSizeFromInput(1, false);
if (m_inputWidth < m_kernelWidth || m_inputHeight < m_kernelHeight)
throw std::invalid_argument("inputWidth must >= kernelWidth and inputHeight must >= kernelHeight.");
if (m_zeroPadding)
{
const int kernelWidthCenter = m_kernelWidth % 2;
const int kernelHeightCenter = m_kernelHeight % 2;
m_outputWidth = (m_inputWidth-kernelWidthCenter)/m_horizontalSubsample + 1;
m_outputHeight = (m_inputHeight-kernelHeightCenter)/m_verticalSubsample + 1;
}
else
{
m_outputWidth = (m_inputWidth-m_kernelWidth)/m_horizontalSubsample + 1;
m_outputHeight = (m_inputHeight-m_kernelHeight)/m_verticalSubsample + 1;
}
}
virtual void AttachInputs(const ComputationNodePtr convolutionWeight, const ComputationNodePtr inputFeature)
{
m_children.resize(2);
m_children[0] = convolutionWeight;
m_children[1] = inputFeature;
}
virtual void MoveMatricesToDevice(const DEVICEID_TYPE deviceId)
{
ComputationNode<ElemType>::MoveMatricesToDevice(deviceId);
if (deviceId != AUTOPLACEMATRIX)
{
if (m_tempMatrix.GetDeviceId() != deviceId)
m_tempMatrix.TransferFromDeviceToDevice(m_tempMatrix.GetDeviceId(), deviceId);
}
}
virtual void DumpNodeInfo(const bool printValues, File& fstream) const
{
ComputationNode<ElemType>::DumpNodeInfo(printValues, fstream);
char str[4096];
sprintf(str, "Input[Width:%lu, Height:%lu, Channels:%lu] \n", m_inputWidth, m_inputHeight, m_inputChannels);
fstream << string(str);
sprintf(str, "Kernel[Width:%lu, Height:%lu] SubSample[Horizontal:%lu, Vertical:%lu]\n", m_kernelWidth, m_kernelHeight, m_horizontalSubsample, m_verticalSubsample);
fstream << string(str);
sprintf(str, "Output[Width:%lu, Height:%lu, Channels:%lu] \n", m_outputWidth, m_outputHeight, m_outputChannels);
fstream << string(str);
sprintf(str, "ZeroPadding=%ls maxTempMemSizeInSamples=%lu\n", m_zeroPadding? L"true" : L"false", m_maxTempMemSizeInSamples);
fstream << string(str);
}
void SetmMaxTempMemSizeInSamples(const size_t maxTempMemSizeInSamples)
{
m_maxTempMemSizeInSamples = maxTempMemSizeInSamples;
}
private:
static void WINAPI ComputeInputPartialOverWeight(const ConvolutionNode<ElemType>* pConv, Matrix<ElemType> &gradientValues,
Matrix<ElemType> &inputGradientValues, const Matrix<ElemType> &/*input0*/, const Matrix<ElemType> &input1, Matrix<ElemType> &tempMatrix, const bool inLoop=false)
{
ConvolutionParams convParam = pConv->GetConvolutionParams();
size_t packedInputRows = convParam.kernelWidth * convParam.kernelHeight * convParam.inputChannels;
size_t packedInputColsPerSample = convParam.outputWidth * convParam.outputHeight;
size_t outputSizePerChannel = packedInputColsPerSample;
//size_t packedInputDim = packedInputRows * packedInputColsPerSample; // size of each packed input sample
//size_t inputDim = convParam.inputWidth * convParam.inputHeight * convParam.inputChannels; //size of each input sample
long batchSize = (long) input1.GetNumCols(); //right child is the input sample
long maxTempMemSizeInSamples = (long) (convParam.maxTempMemSizeInSamples == 0? batchSize : convParam.maxTempMemSizeInSamples);
//const Matrix<ElemType> & weightMatrix = input0;
//inputGradientValues.Resize(weightMatrix.GetNumRows(), weightMatrix.GetNumCols()); //should have been resized when preparing gradient computation
gradientValues.Reshape(convParam.outputChannels, outputSizePerChannel * batchSize); //reshape to match the longernal operation
long subBatchSize = min(batchSize, maxTempMemSizeInSamples);
long numSubBatches = (batchSize+subBatchSize-1)/subBatchSize;
if (numSubBatches == 1 && !inLoop) //reuse packed input from evaluation step if it's not changed by either subbatch or recurrent steps.
{
Matrix<ElemType>::MultiplyAndAdd(gradientValues, false, tempMatrix, true, inputGradientValues);
}
else
{
for (long i=0; i<numSubBatches; i++)
{
long startSampleID = i*subBatchSize;
long endSampleID = min(batchSize, startSampleID + subBatchSize);
long smallBatchSize = endSampleID-startSampleID;
tempMatrix.Resize(packedInputRows, packedInputColsPerSample * smallBatchSize);
Matrix<ElemType> inputSubBatch = input1.ColumnSlice(startSampleID, smallBatchSize);
tempMatrix.AssignPackedConvolutionInput(inputSubBatch,
convParam.inputWidth, convParam.inputHeight, convParam.inputChannels,
convParam.outputWidth, convParam.outputHeight, convParam.outputChannels,
convParam.kernelWidth, convParam.kernelHeight, convParam.horizontalSubsample, convParam.verticalSubsample,
convParam.zeroPadding);
Matrix<ElemType> outputGradientSubBatch = gradientValues.ColumnSlice(startSampleID * outputSizePerChannel, smallBatchSize * outputSizePerChannel);
Matrix<ElemType>::MultiplyAndAdd(outputGradientSubBatch, false, tempMatrix, true, inputGradientValues);
}
}
gradientValues.Reshape(convParam.outputChannels * outputSizePerChannel, batchSize); //change back
}
//compute gradient over the packed input and then convert the result to the original input
static void WINAPI ComputeInputPartialOverInputFeature(const ConvolutionNode<ElemType>* pConv, Matrix<ElemType> &gradientValues, const Matrix<ElemType> &inputGradientValues, const Matrix<ElemType> &input0, const Matrix<ElemType> &input1, Matrix<ElemType> &tempMatrix)
{
ConvolutionParams convParam = pConv->GetConvolutionParams();
size_t packedInputRows = convParam.kernelWidth * convParam.kernelHeight * convParam.inputChannels;
size_t packedInputColsPerSample = convParam.outputWidth * convParam.outputHeight;
size_t outputSizePerChannel = packedInputColsPerSample;
//size_t packedInputDim = packedInputRows * packedInputColsPerSample; // size of each packed input sample
//size_t inputDim = convParam.inputWidth * convParam.inputHeight * convParam.inputChannels; //size of each input sample
long batchSize = (long) input1.GetNumCols(); //right child is the input sample
long maxTempMemSizeInSamples = (long) (convParam.maxTempMemSizeInSamples == 0? batchSize : convParam.maxTempMemSizeInSamples);
const Matrix<ElemType> & weightMatrix = input0;
gradientValues.Reshape(convParam.outputChannels, outputSizePerChannel * batchSize); //reshape to match the longernal operation
long subBatchSize = min(batchSize, maxTempMemSizeInSamples);
long numSubBatches = (batchSize+subBatchSize-1)/subBatchSize;
for (long i=0; i<numSubBatches; i++)
{
long startSampleID = i*subBatchSize;
long endSampleID = min(batchSize, startSampleID + subBatchSize);
long smallBatchSize = endSampleID-startSampleID;
tempMatrix.Resize(packedInputRows, packedInputColsPerSample * smallBatchSize);
Matrix<ElemType> outputGradientSubBatch = gradientValues.ColumnSlice(startSampleID * outputSizePerChannel, smallBatchSize * outputSizePerChannel);
Matrix<ElemType>::Multiply(weightMatrix, true, outputGradientSubBatch, false, tempMatrix);
Matrix<ElemType> inputGradientSubBatch = inputGradientValues.ColumnSlice(startSampleID, smallBatchSize);
tempMatrix.UnpackConvolutionInput(inputGradientSubBatch,
convParam.inputWidth, convParam.inputHeight, convParam.inputChannels,
convParam.outputWidth, convParam.outputHeight, convParam.outputChannels,
convParam.kernelWidth, convParam.kernelHeight, convParam.horizontalSubsample, convParam.verticalSubsample,
convParam.zeroPadding);
}
gradientValues.Reshape(convParam.outputChannels * outputSizePerChannel, batchSize); //change back
}
private:
size_t m_kernelWidth, m_kernelHeight;
size_t m_horizontalSubsample, m_verticalSubsample;
bool m_zeroPadding;
Matrix<ElemType> m_tempMatrix;
size_t m_maxTempMemSizeInSamples; // can change during runtime
};
template class ConvolutionNode<float>;
template class ConvolutionNode<double>;
struct PoolParams
{
size_t inputWidth, inputHeight, inputChannels;
size_t windowWidth, windowHeight;
size_t horizontalSubsample, verticalSubsample;
size_t outputWidth, outputHeight, outputChannels;
size_t inputSizePerSample, outputSizePerSample;
};
//Max Pooling: support multi channel
//assume each column is an input sample. Each sample is stored in (r00, g00, b00, r01, g01, b01, r10, g10, b10, r11, g11, b11)
template<class ElemType>
class MaxPoolingNode : public ComputationNode<ElemType>
{
UsingComputationNodeMembers;
public:
MaxPoolingNode( const size_t windowWidth, const size_t windowHeight,
const size_t horizontalSubsample, const size_t verticalSubsample,
const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId),
m_windowWidth(windowWidth), m_windowHeight(windowHeight),
m_horizontalSubsample(horizontalSubsample), m_verticalSubsample(verticalSubsample)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
InitRecurrentNode();
}
MaxPoolingNode(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
LoadFromFile(fstream, modelVersion, deviceId);
}
virtual void SaveToFile(File& fstream) const
{
ComputationNode<ElemType>::SaveToFile(fstream);
fstream << m_windowWidth << m_windowHeight << m_horizontalSubsample << m_verticalSubsample;
}
virtual void LoadFromFile(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX)
{
ComputationNode<ElemType>::LoadFromFile(fstream, modelVersion, deviceId);
fstream >> m_windowWidth >> m_windowHeight >> m_horizontalSubsample >> m_verticalSubsample;
}
virtual void CopyTo(const ComputationNodePtr nodeP, const std::wstring& newName, const CopyNodeFlags flags) const
{
ComputationNode<ElemType>::CopyTo(nodeP, newName, flags);
MaxPoolingNode<ElemType>* node = (MaxPoolingNode<ElemType>*) nodeP;
if (flags & CopyNodeFlags::copyNodeValue)
{
node->m_inputWidth = m_inputWidth;
node->m_inputHeight = m_inputHeight;
node->m_inputChannels = m_inputChannels;
node->m_windowWidth = m_windowWidth;
node->m_windowHeight = m_windowHeight;
node->m_horizontalSubsample = m_horizontalSubsample;
node->m_verticalSubsample = m_verticalSubsample;
node->m_outputWidth = m_outputWidth;
node->m_outputHeight = m_outputHeight;
node->m_outputChannels = m_outputChannels;
node->m_inputSizePerSample = m_inputSizePerSample;
node->m_outputSizePerSample = m_outputSizePerSample;
}
}
// copy constructor
MaxPoolingNode(const MaxPoolingNode<ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags) : ComputationNode<ElemType>(node->m_deviceId)
{
node->CopyTo(this, newName, flags);
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new MaxPoolingNode<ElemType>(this, name, flags);
return node;
}
virtual const std::wstring OperationName() const {return TypeName();}
static const std::wstring TypeName() {return L"MaxPooling";}
PoolParams GetPoolParams() const
{
PoolParams poolParams;
poolParams.inputWidth = m_inputWidth;
poolParams.inputHeight = m_inputHeight;
poolParams.inputChannels = m_inputChannels;
poolParams.windowWidth = m_windowWidth;
poolParams.windowHeight = m_windowHeight;
poolParams.horizontalSubsample = m_horizontalSubsample;
poolParams.verticalSubsample = m_verticalSubsample;
poolParams.outputWidth = m_outputWidth;
poolParams.outputHeight = m_outputHeight;
poolParams.outputChannels = m_outputChannels;
poolParams.inputSizePerSample = m_inputSizePerSample;
poolParams.outputSizePerSample = m_outputSizePerSample;
return poolParams;
}
virtual void ComputeInputPartial(const size_t inputIndex)
{
if (inputIndex > 0)
throw std::invalid_argument("MaxPooling operation only takes one inputs.");
ComputeInputPartialS(this, GradientValues(), Inputs(0)->GradientValues(), Inputs(0)->FunctionValues(), FunctionValues());
}
virtual void ComputeInputPartial(const size_t inputIndex, const size_t timeIdxInSeq)
{
if (inputIndex > 0)
throw std::invalid_argument("MaxPooling operation only takes one inputs.");
Matrix<ElemType> sliceInput0Grad = Inputs(0)->GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceOutputGrad = GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceInput0Value = Inputs(0)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceOutputValue = m_functionValues.ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
ComputeInputPartialS(this, sliceOutputGrad, sliceInput0Grad, sliceInput0Value, sliceOutputValue);
}
static void WINAPI ComputeInputPartialS(const MaxPoolingNode<ElemType>* ppool, const Matrix<ElemType> &gradientValues, Matrix<ElemType> &inputGradientValues, const Matrix<ElemType> &input0, const Matrix<ElemType> &functionValues)
{
PoolParams poolParams = ppool->GetPoolParams();
inputGradientValues.AddMaxPoolingGradient(gradientValues, input0, functionValues, poolParams.inputChannels,
poolParams.inputWidth, poolParams.inputHeight, poolParams.inputSizePerSample,
poolParams.outputWidth, poolParams.outputHeight, poolParams.outputSizePerSample,
poolParams.windowWidth, poolParams.windowHeight, poolParams.horizontalSubsample, poolParams.verticalSubsample);
}
virtual void EvaluateThisNode()
{
#if NANCHECK
Inputs(0)->FunctionValues().HasNan("MaxPooling-input0");
#endif
EvaluateThisNodeS(this, FunctionValues(), Inputs(0)->FunctionValues());
#if NANCHECK
m_functionValues.HasNan("MaxPooling");
#endif
}
virtual void EvaluateThisNode(const size_t timeIdxInSeq)
{
Matrix<ElemType> sliceInput0Value = Inputs(0)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceOutputValue = m_functionValues.ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
EvaluateThisNodeS(this, sliceOutputValue, sliceInput0Value);
}
static void WINAPI EvaluateThisNodeS(const MaxPoolingNode<ElemType>* ppool, Matrix<ElemType> &functionValues, const Matrix<ElemType> &input0)
{
PoolParams poolParams = ppool->GetPoolParams();
functionValues.AssignMaxPoolingResult(input0, poolParams.inputChannels,
poolParams.inputWidth, poolParams.inputHeight, poolParams.inputSizePerSample,
poolParams.outputWidth, poolParams.outputHeight, poolParams.outputSizePerSample,
poolParams.windowWidth, poolParams.windowHeight, poolParams.horizontalSubsample, poolParams.verticalSubsample);
}
virtual void Validate()
{
PrintSelfBeforeValidation();
if (m_children.size() != 1)
throw std::logic_error("MaxPoolingNode requires one input.");
if (m_horizontalSubsample > m_windowWidth || m_verticalSubsample > m_windowHeight)
throw std::invalid_argument("MaxPoolingNode: horizontalSubsample must <= windowWidth and verticalSubsample must <= windowHeight.");
CopyImageSizeFromInputs();
m_inputSizePerSample = m_inputWidth * m_inputHeight * m_inputChannels;
m_outputSizePerSample = m_outputWidth * m_outputHeight * m_outputChannels;
if (Inputs(0)->OperationName() == LearnableParameter<ElemType>::TypeName() && Inputs(0)->FunctionValues().GetNumRows() == 0)
{
Inputs(0)->FunctionValues().Resize(m_inputSizePerSample, Inputs(0)->FunctionValues().GetNumCols());
}
if (m_children[0]->FunctionValues().GetNumRows() != m_inputSizePerSample)
{
msra::strfun::strprintf msg("each column of input to the MaxPooling node %ls is a sample and should have dimension %d, which is inputWidth * inputHeight * inputChannels",
NodeName().c_str(), m_inputSizePerSample);
throw std::logic_error(msg.c_str());
}
if (Inputs(0)->FunctionValues().GetNumElements() == 0)
throw std::logic_error("MaxPoolingNode operation: the input node has 0 element.");
m_functionValues.Resize(m_outputSizePerSample, m_children[0]->FunctionValues().GetNumCols());
}
virtual void CopyImageSizeFromInputs()
{
CopyImageSizeFromInput(0, false);
if (m_inputWidth < m_windowWidth || m_inputHeight < m_windowHeight)
throw std::invalid_argument("MaxPoolingNode: inputWidth must >= windowWidth and inputHeight must >= windowHeight.");
m_outputWidth = (m_inputWidth-m_windowWidth)/m_horizontalSubsample + 1;
m_outputHeight = (m_inputHeight-m_windowHeight)/m_verticalSubsample + 1;
m_outputChannels = m_inputChannels;
}
virtual void AttachInputs(const ComputationNodePtr inputFeature)
{
m_children.resize(1);
m_children[0] = inputFeature;
}
virtual void DumpNodeInfo(const bool printValues, File& fstream) const
{
ComputationNode<ElemType>::DumpNodeInfo(printValues, fstream);
char str[4096];
sprintf(str, "Input[Width:%lu, Height:%lu, Channels:%lu] \n", m_inputWidth, m_inputHeight, m_inputChannels);
fstream << string(str);
sprintf(str, "PoolingWindow[Width:%lu, Height:%lu] SubSampling[Horizontal:%lu, Vertical:%lu]\n", m_windowWidth, m_windowHeight, m_horizontalSubsample, m_verticalSubsample);
fstream << string(str);
sprintf(str, "Output[Width:%lu, Height:%lu, Channels:%lu] \n", m_outputWidth, m_outputHeight, m_outputChannels);
fstream << string(str);
sprintf(str, "TotalSizePerSample[Input:%lu, Output:%lu] \n", m_inputSizePerSample, m_outputSizePerSample);
fstream << string(str);
}
private:
size_t m_windowWidth, m_windowHeight;
size_t m_horizontalSubsample, m_verticalSubsample;
size_t m_inputSizePerSample, m_outputSizePerSample;
};
template class MaxPoolingNode<float>;
template class MaxPoolingNode<double>;
//Average Pooling: support multi channel
//assume each column is an input sample. Each sample is stored in (r00, g00, b00, r01, g01, b01, r10, g10, b10, r11, g11, b11)
template<class ElemType>
class AveragePoolingNode : public ComputationNode<ElemType>
{
UsingComputationNodeMembers;
public:
AveragePoolingNode(const size_t windowWidth, const size_t windowHeight,
const size_t horizontalSubsample, const size_t verticalSubsample,
const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId),
m_windowWidth(windowWidth), m_windowHeight(windowHeight),
m_horizontalSubsample(horizontalSubsample), m_verticalSubsample(verticalSubsample)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
InitRecurrentNode();
}
AveragePoolingNode(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
LoadFromFile(fstream, modelVersion, deviceId);
}
virtual void SaveToFile(File& fstream) const
{
ComputationNode<ElemType>::SaveToFile(fstream);
fstream << m_windowWidth << m_windowHeight << m_horizontalSubsample << m_verticalSubsample;
}
virtual void LoadFromFile(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX)
{
ComputationNode<ElemType>::LoadFromFile(fstream, modelVersion, deviceId);
fstream >> m_windowWidth >> m_windowHeight >> m_horizontalSubsample >> m_verticalSubsample;
}
virtual void CopyTo(const ComputationNodePtr nodeP, const std::wstring& newName, const CopyNodeFlags flags) const
{
ComputationNode<ElemType>::CopyTo(nodeP, newName, flags);
AveragePoolingNode<ElemType>* node = (AveragePoolingNode<ElemType>*) nodeP;
if (flags & CopyNodeFlags::copyNodeValue)
{
node->m_inputWidth = m_inputWidth;
node->m_inputHeight = m_inputHeight;
node->m_inputChannels = m_inputChannels;
node->m_windowWidth = m_windowWidth;
node->m_windowHeight = m_windowHeight;
node->m_horizontalSubsample = m_horizontalSubsample;
node->m_verticalSubsample = m_verticalSubsample;
node->m_outputWidth = m_outputWidth;
node->m_outputHeight = m_outputHeight;
node->m_outputChannels = m_outputChannels;
node->m_inputSizePerSample = m_inputSizePerSample;
node->m_outputSizePerSample = m_outputSizePerSample;
}
}
// copy constructor
AveragePoolingNode(const AveragePoolingNode<ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags) : ComputationNode<ElemType>(node->m_deviceId)
{
node->CopyTo(this, newName, flags);
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new AveragePoolingNode<ElemType>(this, name, flags);
return node;
}
virtual const std::wstring OperationName() const {return TypeName();}
static const std::wstring TypeName() {return L"AveragePooling";}
PoolParams GetPoolParams() const
{
PoolParams poolParams;
poolParams.inputWidth = m_inputWidth;
poolParams.inputHeight = m_inputHeight;
poolParams.inputChannels = m_inputChannels;
poolParams.windowWidth = m_windowWidth;
poolParams.windowHeight = m_windowHeight;
poolParams.horizontalSubsample = m_horizontalSubsample;
poolParams.verticalSubsample = m_verticalSubsample;
poolParams.outputWidth = m_outputWidth;
poolParams.outputHeight = m_outputHeight;
poolParams.outputChannels = m_outputChannels;
poolParams.inputSizePerSample = m_inputSizePerSample;
poolParams.outputSizePerSample = m_outputSizePerSample;
return poolParams;
}
virtual void ComputeInputPartial(const size_t inputIndex)
{
if (inputIndex > 0)
throw std::invalid_argument("AveragePooling operation only takes one inputs.");
ComputeInputPartialS(this, GradientValues(), Inputs(0)->GradientValues());
}
virtual void ComputeInputPartial(const size_t inputIndex, const size_t timeIdxInSeq)
{
if (inputIndex > 0)
throw std::invalid_argument("AveragePooling operation only takes one inputs.");
Matrix<ElemType> sliceInput0Grad = Inputs(0)->GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceOutputGrad = GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
ComputeInputPartialS(this, sliceOutputGrad, sliceInput0Grad);
}
static void WINAPI ComputeInputPartialS(const AveragePoolingNode<ElemType>* ppool, const Matrix<ElemType> &gradientValues, Matrix<ElemType> &inputGradientValues)
{
PoolParams poolParams = ppool->GetPoolParams();
inputGradientValues.AddAveragePoolingGradient(gradientValues, poolParams.inputChannels,
poolParams.inputWidth, poolParams.inputHeight, poolParams.inputSizePerSample,
poolParams.outputWidth, poolParams.outputHeight, poolParams.outputSizePerSample,
poolParams.windowWidth, poolParams.windowHeight, poolParams.horizontalSubsample, poolParams.verticalSubsample);
}
virtual void EvaluateThisNode()
{
#if NANCHECK
Inputs(0)->FunctionValues().HasNan("AveragePooling-input0");
#endif
EvaluateThisNodeS(this, FunctionValues(), Inputs(0)->FunctionValues());
#if NANCHECK
m_functionValues.HasNan("AveragePooling");
#endif
}
virtual void EvaluateThisNode(const size_t timeIdxInSeq)
{
Matrix<ElemType> sliceInput0Value = Inputs(0)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceOutputValue = m_functionValues.ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
EvaluateThisNodeS(this, sliceOutputValue, sliceInput0Value);
}
static void WINAPI EvaluateThisNodeS(const AveragePoolingNode<ElemType>* ppool, Matrix<ElemType> &functionValues, const Matrix<ElemType> &input0)
{
PoolParams poolParams = ppool->GetPoolParams();
functionValues.AssignAveragePoolingResult(input0, poolParams.inputChannels,
poolParams.inputWidth, poolParams.inputHeight, poolParams.inputSizePerSample,
poolParams.outputWidth, poolParams.outputHeight, poolParams.outputSizePerSample,
poolParams.windowWidth, poolParams.windowHeight, poolParams.horizontalSubsample, poolParams.verticalSubsample);
}
virtual void Validate()
{
PrintSelfBeforeValidation();
if (m_children.size() != 1)
throw std::logic_error("AveragePoolingNode requires one input.");
if (m_horizontalSubsample > m_windowWidth || m_verticalSubsample > m_windowHeight)
throw std::invalid_argument("AveragePoolingNode: horizontalSubsample must <= windowWidth and verticalSubsample must <= windowHeight.");
CopyImageSizeFromInputs();
m_inputSizePerSample = m_inputWidth * m_inputHeight * m_inputChannels;
m_outputSizePerSample = m_outputWidth * m_outputHeight * m_outputChannels;
if (Inputs(0)->OperationName() == LearnableParameter<ElemType>::TypeName() && Inputs(0)->FunctionValues().GetNumRows() == 0)
{
Inputs(0)->FunctionValues().Resize(m_inputSizePerSample, Inputs(0)->FunctionValues().GetNumCols());
}
if (m_children[0]->FunctionValues().GetNumRows() != m_inputSizePerSample)
{
msra::strfun::strprintf msg("each column of input to the AveragePooling node %ls is a sample and should have dimension %d, which is inputWidth * inputHeight * inputChannels",
NodeName().c_str(), m_inputSizePerSample);
throw std::logic_error(msg.c_str());
}
if (Inputs(0)->FunctionValues().GetNumElements() == 0)
throw std::logic_error("AveragePoolingNode operation: the input node has 0 element.");
FunctionValues().Resize(m_outputSizePerSample, m_children[0]->FunctionValues().GetNumCols());
}
virtual void CopyImageSizeFromInputs()
{
CopyImageSizeFromInput(0, false);
if (m_inputWidth < m_windowWidth || m_inputHeight < m_windowHeight)
throw std::invalid_argument("AveragePoolingNode: inputWidth must >= windowWidth and inputHeight must >= windowHeight.");
m_outputWidth = (m_inputWidth-m_windowWidth)/m_horizontalSubsample + 1;
m_outputHeight = (m_inputHeight-m_windowHeight)/m_verticalSubsample + 1;
m_outputChannels = m_inputChannels;
}
virtual void AttachInputs(const ComputationNodePtr inputFeature)
{
m_children.resize(1);
m_children[0] = inputFeature;
}
virtual void DumpNodeInfo(const bool printValues, File& fstream) const
{
ComputationNode<ElemType>::DumpNodeInfo(printValues, fstream);
char str[4096];
sprintf(str, "Input[Width:%lu, Height:%lu, Channels:%lu] \n", m_inputWidth, m_inputHeight, m_inputChannels);
fstream << string(str);
sprintf(str, "PoolingWindow[Width:%lu, Height:%lu] SubSample[Horizontal:%lu, Vertical:%lu]\n", m_windowWidth, m_windowHeight, m_horizontalSubsample, m_verticalSubsample);
fstream << string(str);
sprintf(str, "Output[Width:%lu, Height:%lu, Channels:%lu] \n", m_outputWidth, m_outputHeight, m_outputChannels);
fstream << string(str);
sprintf(str, "TotalSizePerSample[Input:%lu, Output:%lu]\n", m_inputSizePerSample, m_outputSizePerSample);
fstream << string(str);
}
private:
size_t m_windowWidth, m_windowHeight;
size_t m_horizontalSubsample, m_verticalSubsample;
size_t m_inputSizePerSample, m_outputSizePerSample;
};
template class AveragePoolingNode<float>;
template class AveragePoolingNode<double>;
}}}

504
MachineLearning/CNTK/InputAndParamNodes.h Normal file
Просмотреть файл

@ -0,0 +1,504 @@
//
// <copyright file="InputAndParamNodes.h" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//
#pragma once
#include <unordered_set>
#include <map>
#include <string>
#include <vector>
#include <stdexcept>
#include <list>
#include <memory>
#include <algorithm>
#include <assert.h>
#include <atomic>
#include <sstream>
#include <iostream>
#include "Basics.h"
#include "Matrix.h"
#include "ComputationNode.h"
namespace Microsoft { namespace MSR { namespace CNTK {
//used to represent weight Matrix<ElemType> and biases
template<class ElemType>
class LearnableParameter : public ComputationNode<ElemType>
{
UsingComputationNodeMembers;
public:
LearnableParameter(size_t rows, size_t cols, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
//intentionally comment out so that we may support automatic dimention inference
//if (rows * cols == 0)
// throw std::logic_error("This LearnableParameter dimension is 0.");
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_needGradient = true;
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
m_functionValues.Resize(rows, cols);
m_outputWidth = 1;
m_outputHeight = rows;
m_outputChannels = 1;
InitRecurrentNode();
}
LearnableParameter(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
LoadFromFile(fstream, modelVersion, deviceId);
}
virtual void SaveToFile(File& fstream) const
{
ComputationNode<ElemType>::SaveToFile(fstream);
fstream << NeedGradient();
fstream << FunctionValues().GetNumRows() << FunctionValues().GetNumCols();
fstream << FunctionValues();
}
virtual void LoadFromFile(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX)
{
ComputationNode<ElemType>::LoadFromFile(fstream, modelVersion, deviceId);
size_t rows, cols;
fstream >> m_needGradient;
fstream >> rows >> cols;
//intentionally comment out to support automatic dimention inference
//if (rows * cols == 0)
// throw std::logic_error("This LearnableParameter dimension is 0.");
m_functionValues.Resize(rows, cols);
fstream >> m_functionValues;
m_outputWidth = 1;
m_outputHeight = rows;
m_outputChannels = 1;
}
virtual const std::wstring OperationName() const {return TypeName();}
virtual void ComputeInputPartial(const size_t /*inputIndex*/) {}
virtual void ComputeInputPartial(const size_t /*inputIndex*/, const size_t /*timeIdxInSeq*/) {}
virtual void EvaluateThisNode() {}
virtual void EvaluateThisNode(const size_t /*timeIdxInSeq*/) {}
virtual void Validate()
{
PrintSelfBeforeValidation();
}
static const std::wstring TypeName() {return L"LearnableParameter";}
virtual void DumpNodeInfo(const bool printValues, File& fstream) const
{
ComputationNode<ElemType>::DumpNodeInfo(printValues, fstream);
char str[4096];
sprintf(str, "[%lu,%lu] ", FunctionValues().GetNumRows(), FunctionValues().GetNumCols());
fstream << string(str);
sprintf(str, "NeedGradient=%s", NeedGradient()? "true" : "false");
fstream << string(str);
PrintNodeValuesToFile(printValues, fstream);
}
// copy constructor
LearnableParameter(const LearnableParameter<ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags) : ComputationNode<ElemType>(node->m_deviceId)
{
node->CopyTo(this, newName, flags);
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new LearnableParameter<ElemType>(this, name, flags);
return node;
}
};
//WARNING: Don't use SparseLearnableParameter yet since the current version assumes the parameter is dense instead of sparse
//WARNING: After the right implementation is put here we need to turn it on in NetworkDescriptionLangauge.cpp
template<class ElemType>
class SparseLearnableParameter : public LearnableParameter<ElemType>
{
UsingComputationNodeMembers;
public:
SparseLearnableParameter(size_t rows, size_t cols, const size_t size, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX, const std::wstring name = L"")
: LearnableParameter<ElemType>(rows, cols, deviceId, name)
{
m_gradientValues.SwitchToMatrixType(MatrixType::SPARSE, matrixFormatSparseBlockCol, false);
m_gradientValues.Resize(rows, cols, size);
}
SparseLearnableParameter (File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX, const std::wstring name = L"")
: LearnableParameter<ElemType>(fstream, modelVersion, deviceId, name)
{
m_gradientValues.SwitchToMatrixType(MatrixType::SPARSE, matrixFormatSparseBlockCol, false);
m_gradientValues.Resize(FunctionValues().GetNumRows(), FunctionValues().GetNumCols());
}
virtual void LoadFromFile(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX)
{
LearnableParameter<ElemType>::LoadFromFile(fstream, modelVersion, deviceId);
m_gradientValues.SwitchToMatrixType(MatrixType::SPARSE, matrixFormatSparseBlockCol, false);
m_gradientValues.Resize(FunctionValues().GetNumRows(), FunctionValues().GetNumCols());
}
virtual const std::wstring OperationName() const {return TypeName();}
static const std::wstring TypeName() {return L"SparseLearnableParameter";}
// copy constructor
SparseLearnableParameter (const SparseLearnableParameter <ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags)
: LearnableParameter<ElemType>( node, newName, flags)
{
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new SparseLearnableParameter<ElemType>(this, name, flags);
return node;
}
};
template class SparseLearnableParameter<float>;
template class SparseLearnableParameter<double>;
template<class ElemType>
class InputValue : public ComputationNode<ElemType>
{
UsingComputationNodeMembers;
public:
InputValue(size_t rows, size_t cols, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
if (rows * cols == 0)
throw std::logic_error("This InputValue dimension is 0.");
m_outputWidth = 1;
m_outputHeight = rows;
m_outputChannels = 1;
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
m_functionValues.Resize(rows, cols);
m_needGradient = false;
InitRecurrentNode();
}
InputValue(size_t imageWidth, size_t imageHeight, size_t imageChannels, size_t numImages, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
size_t rows = imageWidth * imageHeight * imageChannels;
size_t cols = numImages;
if (rows * cols == 0)
throw std::logic_error("This InputValue dimension is 0.");
m_outputWidth = imageWidth;
m_outputHeight = imageHeight;
m_outputChannels = imageChannels;
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
m_functionValues.Resize(rows, cols);
m_needGradient = false;
InitRecurrentNode();
}
InputValue(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
LoadFromFile(fstream, modelVersion, deviceId);
}
virtual void SaveToFile(File& fstream) const
{
ComputationNode<ElemType>::SaveToFile(fstream);
fstream << FunctionValues().GetNumRows() << FunctionValues().GetNumCols();
fstream << m_outputWidth << m_outputHeight << m_outputChannels;
}
virtual void LoadFromFile(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX)
{
ComputationNode<ElemType>::LoadFromFile(fstream, modelVersion, deviceId);
size_t rows, cols;
fstream >> rows >> cols;
if (rows * cols == 0)
throw std::logic_error("This InputValue dimension is 0.");
fstream >> m_outputWidth >> m_outputHeight >> m_outputChannels;
m_functionValues.Resize(rows, cols);
m_needGradient = false;
}
virtual const std::wstring OperationName() const {return TypeName();}
static const std::wstring TypeName() {return L"InputValue";}
virtual void EvaluateThisNode() {}
virtual void EvaluateThisNode(const size_t /*timeIdxInSeq*/) {}
virtual void ComputeInputPartial(const size_t /*inputIndex*/) {}
virtual void ComputeInputPartial(const size_t /*inputIndex*/, const size_t /*timeIdxInSeq*/) {}
virtual void Validate()
{
PrintSelfBeforeValidation();
//CopyImageSizeFromInputs(); //not necessary since InputValue are leafs. put it here for consistent
}
virtual void DumpNodeInfo(const bool printValues, File& fstream) const
{
ComputationNode<ElemType>::DumpNodeInfo(printValues, fstream);
char str[4096];
sprintf(str, "[%lu,%lu]", FunctionValues().GetNumRows(), FunctionValues().GetNumCols());
fstream << string(str);
}
// copy constructor
InputValue(const InputValue<ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags) : ComputationNode<ElemType>(node->m_deviceId)
{
node->CopyTo(this, newName, flags);
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new InputValue<ElemType>(this, name, flags);
return node;
}
};
template class InputValue<float>;
template class InputValue<double>;
template<class ElemType>
class SparseInputValue : public InputValue<ElemType>
{
UsingComputationNodeMembers;
public:
SparseInputValue (size_t rows, size_t cols, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : InputValue<ElemType>(rows, cols, deviceId, name)
{
ConvertToSparseMatrix();
}
SparseInputValue (size_t imageWidth, size_t imageHeight, size_t imageChannels, size_t numImages, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"")
: InputValue<ElemType>(imageWidth, imageHeight, imageChannels, numImages, deviceId, name)
{
ConvertToSparseMatrix();
}
SparseInputValue (File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : InputValue<ElemType>(fstream, modelVersion, deviceId, name)
{
ConvertToSparseMatrix();
}
virtual void LoadFromFile(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX)
{
InputValue<ElemType>::LoadFromFile(fstream, modelVersion, deviceId);
ConvertToSparseMatrix();
}
virtual const std::wstring OperationName() const {return TypeName();}
static const std::wstring TypeName() {return L"SparseInputValue";}
// copy constructor
SparseInputValue (const SparseInputValue <ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags) : InputValue<ElemType>(node, newName, flags)
{
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new SparseInputValue<ElemType>(this, name, flags);
return node;
}
private:
void ConvertToSparseMatrix()
{
size_t rows = m_functionValues.GetNumRows();
size_t cols = m_functionValues.GetNumCols();
m_functionValues.SwitchToMatrixType(MatrixType::SPARSE, matrixFormatSparseCSC, false);
m_functionValues.Resize(rows, cols); //SwitchToMatrixType does not reserve information right now.
}
};
template class SparseInputValue<float>;
template class SparseInputValue<double>;
//originally designed to extract word embedding representation from bag-of-word.
//takes two inputs, input0 is weight matrix and input1 is the bag-of-word representation of the inputs
template<class ElemType>
class LookupTableNode : public ComputationNode<ElemType>
{
UsingComputationNodeMembers;
public:
LookupTableNode(const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
InitRecurrentNode();
}
LookupTableNode(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
LoadFromFile(fstream, modelVersion, deviceId);
}
virtual const std::wstring OperationName() const {return TypeName();}
static const std::wstring TypeName() {return L"LookupTable";}
virtual void ComputeInputPartial(const size_t inputIndex)
{
if (inputIndex > 1)
throw std::invalid_argument("LookupTable operation only takes two inputs.");
if (inputIndex == 0) //left derivative
{
ComputeInputPartialLeft(Inputs(1)->FunctionValues(), Inputs(0)->GradientValues(), GradientValues());
}
else //right derivative
{
ComputeInputPartialRight(Inputs(0)->FunctionValues(), Inputs(1)->GradientValues(), GradientValues());
}
}
virtual void ComputeInputPartial(const size_t inputIndex, const size_t timeIdxInSeq)
{
if (inputIndex > 1)
throw std::invalid_argument("LookupTable operation only takes two inputs.");
if (inputIndex == 0) //left derivative
{
Matrix<ElemType> sliceOutputGrad = GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceInput1Value = Inputs(1)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
ComputeInputPartialLeft(sliceInput1Value, Inputs(0)->GradientValues(), sliceOutputGrad);
}
else //right derivative
{
Matrix<ElemType> sliceInput1Grad = Inputs(1)->GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceOutputGrad = GradientValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
ComputeInputPartialRight(Inputs(0)->FunctionValues(), sliceInput1Grad, sliceOutputGrad);
}
}
static void WINAPI ComputeInputPartialLeft(Matrix<ElemType>& inputFunctionValues, Matrix<ElemType>& inputGradientValues, Matrix<ElemType>& gradientValues)
{
size_t rows1 =inputFunctionValues.GetNumRows(), cols1 = inputFunctionValues.GetNumCols();
size_t rowsp = gradientValues.GetNumRows(), colsp = gradientValues.GetNumCols();
int wordsInEachSample = rows1 / inputGradientValues.GetNumCols();
inputFunctionValues.Reshape(rows1 / wordsInEachSample, cols1 * wordsInEachSample);
gradientValues.Reshape(rowsp / wordsInEachSample, colsp * wordsInEachSample);
Matrix<ElemType>::MultiplyAndAdd(gradientValues, false, inputFunctionValues, true, inputGradientValues);
inputFunctionValues.Reshape(rows1, cols1);
gradientValues.Reshape(rowsp, colsp);
}
static void WINAPI ComputeInputPartialRight(Matrix<ElemType>& inputFunctionValues, Matrix<ElemType>& inputGradientValues, Matrix<ElemType>& gradientValues)
{
size_t rows1 =inputGradientValues.GetNumRows(), cols1 = inputGradientValues.GetNumCols();
size_t rowsp = gradientValues.GetNumRows(), colsp = gradientValues.GetNumCols();
int wordsInEachSample = rows1 / inputFunctionValues.GetNumCols();
inputGradientValues.Reshape(rows1 / wordsInEachSample, cols1 * wordsInEachSample);
gradientValues.Reshape(rowsp / wordsInEachSample, colsp * wordsInEachSample);
Matrix<ElemType>::MultiplyAndAdd(inputFunctionValues, true, gradientValues, false, inputGradientValues);
inputGradientValues.Reshape(rows1, cols1);
gradientValues.Reshape(rowsp, colsp);
}
virtual void EvaluateThisNode()
{
EvaluateThisNodeS(FunctionValues(), Inputs(0)->FunctionValues(), Inputs(1)->FunctionValues());
}
virtual void EvaluateThisNode(const size_t timeIdxInSeq)
{
Matrix<ElemType> sliceInput1Value = Inputs(1)->FunctionValues().ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
Matrix<ElemType> sliceOutputValue = m_functionValues.ColumnSlice(timeIdxInSeq * m_samplesInRecurrentStep, m_samplesInRecurrentStep);
EvaluateThisNodeS(sliceOutputValue, Inputs(0)->FunctionValues(), sliceInput1Value);
}
//input0 is the weight (each column is an embedding of one word), input 1 contains m_bnrLooked words in each column (sample)
static void WINAPI EvaluateThisNodeS(Matrix<ElemType>& functionValues, const Matrix<ElemType>& input0, Matrix<ElemType>& input1)
{
size_t rows1 =input1.GetNumRows(), cols1 = input1.GetNumCols();
int wordsInEachSample = rows1 / input0.GetNumCols();
input1.Reshape(rows1 / wordsInEachSample, cols1 * wordsInEachSample);
functionValues.AssignProductOf(input0, false, input1, false);
input1.Reshape(rows1, cols1);
size_t rows = functionValues.GetNumRows();
functionValues.Reshape(rows * wordsInEachSample, cols1);
}
virtual void Validate()
{
PrintSelfBeforeValidation();
if (Inputs(1)->FunctionValues().GetNumRows() % Inputs(0)->FunctionValues().GetNumCols() != 0)
throw invalid_argument("Mismatched dimention. rows in input1 must be multiples of cols in input0.");
int wordsInEachSample = Inputs(1)->FunctionValues().GetNumRows() / Inputs(0)->FunctionValues().GetNumCols();
FunctionValues().Resize(Inputs(0)->FunctionValues().GetNumRows() * wordsInEachSample, Inputs(1)->FunctionValues().GetNumCols());
CopyImageSizeFromInputs();
}
virtual void AttachInputs(const ComputationNodePtr leftNode, const ComputationNodePtr rightNode)
{
m_children.resize(2);
m_children[0] = leftNode;
m_children[1] = rightNode;
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new LookupTableNode<ElemType>(this, name, flags);
return node;
}
LookupTableNode(const LookupTableNode<ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags) : ComputationNode<ElemType>(node->m_deviceId)
{
node->CopyTo(this, newName, flags);
}
};
template class LookupTableNode<float>;
template class LookupTableNode<double>;
}}}

2360
MachineLearning/CNTK/LinearAlgebraNodes.h Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

6
MachineLearning/CNTK/NetworkDescriptionLanguage.cpp
Просмотреть файл

@ -122,9 +122,9 @@ bool EqualInsensitive(std::wstring& string1, const std::wstring& string2, const
// ++ operator for this enum, so loops work
NDLPass &operator++(NDLPass &ndlPass) {
assert(ndlPass != ndlPassMax);
ndlPass = static_cast<NDLPass>(ndlPass + 1);
return ndlPass;
assert(ndlPass != ndlPassMax);
ndlPass = static_cast<NDLPass>(ndlPass + 1);
return ndlPass;
}
// CheckFunction - check to see if we match a function name

7
MachineLearning/CNTK/NetworkDescriptionLanguage.h
Просмотреть файл

@ -4,13 +4,8 @@
// </copyright>
//
#pragma once
#include "commandArgUtil.h"
#include "ComputationNode.h"
#include "TrainingCriterionNodes.h"
#include "CompositeComputationNodes.h"
#include "EvaluationCriterionNodes.h"
#include "ComputationNetwork.h"
#include <stdarg.h>
namespace Microsoft { namespace MSR { namespace CNTK {

1617
MachineLearning/CNTK/NonlinearityNodes.h Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

378
MachineLearning/CNTK/RecurrentNodes.h Normal file
Просмотреть файл

@ -0,0 +1,378 @@
//
// <copyright file="RecurrentNodes.h" company="Microsoft">
// Copyright (c) Microsoft Corporation. All rights reserved.
// </copyright>
//
#pragma once
#include <unordered_set>
#include <map>
#include <string>
#include <vector>
#include <stdexcept>
#include <list>
#include <memory>
#include <algorithm>
#include <assert.h>
#include <atomic>
#include <sstream>
#include <iostream>
#include "Basics.h"
#include "Matrix.h"
#include "ComputationNode.h"
namespace Microsoft { namespace MSR { namespace CNTK {
template<class ElemType>
class DelayNode : public ComputationNode<ElemType>
{
ElemType m_default_activity;
UsingComputationNodeMembers;
public:
DelayNode(const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"")
: ComputationNode<ElemType>(deviceId), m_pastActivity(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
m_default_activity = (ElemType)DEFAULT_HIDDEN_ACTIVITY;
m_delay = 1;
m_functionValues.Resize(1,1);
m_pastActivity.Resize(1,1);
Reset();
InitRecurrentNode();
}
DelayNode(File& fstream, const size_t modelVersion, const DEVICEID_TYPE deviceId=AUTOPLACEMATRIX, const std::wstring name = L"")
: ComputationNode<ElemType>(deviceId), m_pastActivity(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_default_activity = (ElemType)DEFAULT_HIDDEN_ACTIVITY;
m_delay = 1;
m_functionValues.Resize(1,1);
m_pastActivity.Resize(1,1);
Reset();
LoadFromFile(fstream, modelVersion, deviceId);
}
void SaveToFile(File& fstream) const
{
fstream << OperationName() << NodeName();
fstream << m_delay;
fstream << FunctionValues().GetNumRows() << FunctionValues().GetNumCols();
}
void LoadFromFile(File& fstream, const size_t /*modelVersion*/, const DEVICEID_TYPE deviceId = AUTOPLACEMATRIX)
{
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
InitRecurrentNode();
fstream >> m_delay;
size_t iRow, timeIdxInSeq;
fstream >> iRow >> timeIdxInSeq;
FunctionValues().Resize(iRow,timeIdxInSeq);
m_pastActivity.Resize(iRow, timeIdxInSeq);
}
DelayNode(const DEVICEID_TYPE deviceId, ElemType initHiddenActivity, size_t row_size, size_t col_size, const std::wstring name = L"") : ComputationNode<ElemType>(deviceId)
{
m_nodeName = (name == L""? CreateUniqNodeName() : name);
m_deviceId = deviceId;
MoveMatricesToDevice(deviceId);
m_default_activity = initHiddenActivity;
m_delay = 1;
m_functionValues.Resize(row_size, col_size);
m_functionValues.SetValue(m_default_activity);
m_pastActivity.Resize(row_size, col_size);
m_pastActivity.SetValue(m_default_activity);
m_gradientValues.Resize(row_size, col_size);
m_gradientValues.SetValue(0.0f);
Reset();
InitRecurrentNode();
}
virtual const std::wstring OperationName() const {return TypeName();}
virtual void ComputeInputPartial(const size_t inputIndex)
{
if (inputIndex > 0)
throw std::invalid_argument("Delay operation only takes one input.");
size_t nbrSamples = GradientValues().GetNumCols() / m_samplesInRecurrentStep;
for (int i = nbrSamples - 1; i >= 0; i--)
{
ComputeInputPartialSR(i, m_delay, Inputs(0)->GradientValues(), GradientValues(), m_samplesInRecurrentStep);
}
}
virtual void ComputeInputPartial(const size_t inputIndex, const size_t timeIdxInSeq)
{
if (inputIndex > 0)
throw std::invalid_argument("Delay operation only takes one input.");
assert(m_functionValues.GetNumRows() == GradientValues().GetNumRows()); // original used m_functionValues.GetNumRows() for loop dimension
if (m_samplesInRecurrentStep == 1)
{
ComputeInputPartialSR(timeIdxInSeq, m_delay, Inputs(0)->GradientValues(), GradientValues(), m_samplesInRecurrentStep);
}else
{
for (size_t i = 0 ; i < m_samplesInRecurrentStep; i++)
{
bool reset = false;
if ((((int)m_sentenceEnd[i] +(int)m_delay - 1) >= (int)timeIdxInSeq) && (m_sentenceEnd[i] <= timeIdxInSeq))
{
reset = true;
}
ComputeInputPartialSRP(timeIdxInSeq, m_delay, reset, Inputs(0)->GradientValues(), GradientValues(), i, m_samplesInRecurrentStep);
}
}
}
static void WINAPI ComputeInputPartialSR(int timeIdxInSeq, int delay,
Matrix<ElemType>& inputGradientValues, const Matrix<ElemType>& gradientValues, const size_t mNbr)
{
assert(timeIdxInSeq >= 0);
if ((timeIdxInSeq - delay) >= 0 && (timeIdxInSeq - delay) * mNbr <= inputGradientValues.GetNumCols())
{
Matrix<ElemType> to = inputGradientValues.ColumnSlice((timeIdxInSeq - delay)*mNbr, mNbr);
Matrix<ElemType> frm= gradientValues.ColumnSlice(timeIdxInSeq * mNbr, mNbr);
to += frm;
}
}
static void WINAPI ComputeInputPartialSRP(int timeIdxInSeq, int delay, bool reset,
Matrix<ElemType>& inputGradientValues, const Matrix<ElemType>& gradientValues, const size_t indexInBatch, const size_t mNbr)
{
assert(timeIdxInSeq >= 0);
if ((timeIdxInSeq - delay) >= 0 && (timeIdxInSeq - delay) * mNbr <= inputGradientValues.GetNumCols() && !reset)
{
Matrix<ElemType> to = inputGradientValues.ColumnSlice((timeIdxInSeq - delay)*mNbr + indexInBatch, 1);
Matrix<ElemType> frm= gradientValues.ColumnSlice(timeIdxInSeq * mNbr + indexInBatch, 1);
to += frm;
}
}
virtual void EvaluateThisNode()
{
ASSERT(m_delay > 0);
size_t blogSize = Inputs(0)->FunctionValues().GetNumCols();
for (size_t i = 0; i < blogSize / m_samplesInRecurrentStep; i++)
EvaluateThisNodeSR(i, m_delay, m_Reset, m_default_activity, m_functionValues, m_pastActivity, Inputs(0)->FunctionValues(), m_samplesInRecurrentStep);
/// reset past activity
m_pastActivity = Inputs(0)->FunctionValues();
}
void Reset()
{
m_Reset = true;
}
void NotReset()
{
m_Reset = false;
}
virtual void EvaluateThisNode(const size_t timeIdxInSeq)
{
/// reset past activity as it reached to the begining of a minibatch
/// the node pointed hasn't yet updated, so it is the past activity
if (timeIdxInSeq == 0)
m_pastActivity = Inputs(0)->FunctionValues();
if (m_samplesInRecurrentStep == 1)
{
//bool reset = (m_sentenceEnd[0] == timeIdxInSeq);
bool reset = false;
if ((((int)m_sentenceEnd[0] +(int)m_delay - 1) >= (int)timeIdxInSeq) && (m_sentenceEnd[0] <= timeIdxInSeq))
{
reset = true;
}
EvaluateThisNodeSR(timeIdxInSeq, m_delay, reset, m_default_activity, m_functionValues, m_pastActivity, Inputs(0)->FunctionValues(), m_samplesInRecurrentStep);
} else
{
for (size_t i = 0 ; i < m_samplesInRecurrentStep; i++)
{
// bool reset = (m_sentenceEnd[i] == timeIdxInSeq);
bool reset = false;
if ((((int)m_sentenceEnd[i] +(int)m_delay - 1) >= (int)timeIdxInSeq) && (m_sentenceEnd[i] <= timeIdxInSeq))
{
reset = true;
}
EvaluateThisNodeSRP(timeIdxInSeq, m_delay, reset, m_default_activity, m_functionValues, m_pastActivity, Inputs(0)->FunctionValues(), i, m_samplesInRecurrentStep);
}
}
}
static void WINAPI EvaluateThisNodeSR(const size_t timeIdxInSeq, const int delay, const bool reset, const ElemType default_activity, Matrix<ElemType>& functionValues, const Matrix<ElemType>& pastActivity, const Matrix<ElemType>& inputFunctionValues, const size_t mNbr)
{
ASSERT(delay > 0);
if (functionValues.GetNumRows() != inputFunctionValues.GetNumRows() ||
functionValues.GetNumCols() != inputFunctionValues.GetNumCols())
functionValues.Resize(inputFunctionValues.GetNumRows(),
inputFunctionValues.GetNumCols());
int iPastIndex = (int) (timeIdxInSeq - delay) * mNbr;
int d = iPastIndex;
if (d < 0)
d = (int)functionValues.Mod((float)iPastIndex, (float)pastActivity.GetNumCols());
/// this can point to the past activity of the previous mninibatch
Matrix<ElemType> out = functionValues.ColumnSlice(timeIdxInSeq * mNbr, mNbr);
Matrix<ElemType> inp((DEVICEID_TYPE)functionValues.GetDeviceId()) ;
if (reset)
out.SetValue(default_activity);
else
{
if (iPastIndex < 0)
inp = pastActivity.ColumnSlice(d, mNbr);
else
inp = inputFunctionValues.ColumnSlice(d, mNbr);
out.SetValue(inp);
}
}
static void WINAPI EvaluateThisNodeSRP(const size_t timeIdxInSeq, const int delay, const bool reset, const ElemType default_activity, Matrix<ElemType>& functionValues, const Matrix<ElemType>& pastActivity, const Matrix<ElemType>& inputFunctionValues, const size_t indexInBatch, const size_t mNbr)
{
ASSERT(delay > 0);
if (functionValues.GetNumRows() != inputFunctionValues.GetNumRows() ||
functionValues.GetNumCols() != inputFunctionValues.GetNumCols())
functionValues.Resize(inputFunctionValues.GetNumRows(),
inputFunctionValues.GetNumCols());
int iPastIndex = (int) (timeIdxInSeq - delay) * mNbr;
int d = iPastIndex;
if (d < 0)
d = (int)functionValues.Mod((float)iPastIndex, (float)pastActivity.GetNumCols());
/// this can point to the past activity of the previous mninibatch
Matrix<ElemType> out = functionValues.ColumnSlice(timeIdxInSeq * mNbr+indexInBatch, 1);
Matrix<ElemType> inp((DEVICEID_TYPE)functionValues.GetDeviceId()) ;
if (reset)
out.SetValue(default_activity);
else
{
if (iPastIndex < 0)
inp = pastActivity.ColumnSlice(d+indexInBatch, 1);
else
inp = inputFunctionValues.ColumnSlice(d+indexInBatch, 1);
out.SetValue(inp);
}
}
virtual const Matrix<ElemType>& FunctionValues() const
{
return m_functionValues;
}
virtual Matrix<ElemType>& FunctionValues()
{
return m_functionValues;
}
virtual void Validate()
{
PrintSelfBeforeValidation(true/*allowNulls*/);
if (m_children.size() != 1)
throw std::logic_error("Delay operation should have one input.");
if (!(Inputs(0) == nullptr))
{
size_t rows0 = Inputs(0)->FunctionValues().GetNumRows(), cols0 = Inputs(0)->FunctionValues().GetNumCols();
if (rows0 > 0 && cols0 > 0) FunctionValues().Resize(rows0, cols0);
}
CopyImageSizeFromInputs();
}
virtual void AttachInputs(const ComputationNodePtr inputNode)
{
m_children.resize(1);
m_children[0] = inputNode;
}
void SetDelay(const int val)
{
if (val <= 0)
throw std::logic_error("Delay must be > 0.");
m_delay = val;
}
virtual void MoveMatricesToDevice(const DEVICEID_TYPE deviceId)
{
ComputationNode<ElemType>::MoveMatricesToDevice(deviceId);
if (deviceId != AUTOPLACEMATRIX)
{
if (m_functionValues.GetDeviceId() != deviceId)
m_functionValues.TransferFromDeviceToDevice(m_functionValues.GetDeviceId(), deviceId);
if (m_pastActivity.GetDeviceId() != deviceId)
m_pastActivity.TransferFromDeviceToDevice(m_pastActivity.GetDeviceId(), deviceId, true);
}
}
static const std::wstring TypeName() {return L"Delay";}
virtual void CopyTo(const ComputationNodePtr nodeP, const std::wstring& newName, const CopyNodeFlags flags) const
{
ComputationNode<ElemType>::CopyTo(nodeP, newName, flags);
DelayNode<ElemType>* node = (DelayNode<ElemType>*) nodeP;
if (flags & CopyNodeFlags::copyNodeValue)
{
node->m_delay = m_delay;
node->m_Reset = m_Reset;
node->m_default_activity = m_default_activity;
node->m_pastActivity = m_pastActivity;
}
}
// copy constructor
DelayNode(const DelayNode<ElemType>* node, const std::wstring& newName, const CopyNodeFlags flags)
: ComputationNode<ElemType>(node->m_deviceId), m_pastActivity(node->m_deviceId)
{
node->CopyTo(this, newName, flags);
}
virtual ComputationNodePtr Duplicate(const std::wstring& newName, const CopyNodeFlags flags) const
{
const std::wstring& name = (newName == L"")?NodeName():newName;
ComputationNodePtr node = new DelayNode<ElemType>(this, name, flags);
return node;
}
private:
Matrix<ElemType> m_pastActivity; /// saves the past activity this delay node points to
int m_delay; /// steps for delay
bool m_Reset;
};
template class DelayNode<float>;
template class DelayNode<double>;
}}}

10
MachineLearning/CNTK/SimpleEvaluator.h
Просмотреть файл

@ -5,16 +5,16 @@
//
#pragma once
#include "ComputationNetwork.h"
#include "ComputationNetworkHelper.h"
#include "DataReader.h"
#include <vector>
#include <string>
#include <stdexcept>
#include <fstream>
#include "Basics.h"
#include "fileutil.h"
#include "commandArgUtil.h"
#include <fstream>
#include "ComputationNetwork.h"
#include "ComputationNetworkHelper.h"
using namespace std;

6
MachineLearning/CNTK/SimpleNetworkBuilder.cpp
Просмотреть файл

@ -7,6 +7,12 @@
#define _CRT_SECURE_NO_WARNINGS // "secure" CRT not available on all platforms --add this at the top of all CPP files that give "function or variable may be unsafe" warnings
#include "ComputationNode.h"
#include "InputAndParamNodes.h"
#include "LinearAlgebraNodes.h"
#include "NonlinearityNodes.h"
#include "ConvolutionalNodes.h"
#include "RecurrentNodes.h"
#include "SimpleEvaluator.h"
#include "IComputationNetBuilder.h"
#include "SGD.h"

14
MachineLearning/CNTK/SimpleNetworkBuilder.h
Просмотреть файл

@ -5,16 +5,18 @@
//
#pragma once
#include "Basics.h"
#include "ComputationNetwork.h"
#include "IComputationNetBuilder.h"
#include <string>
#include "commandArgUtil.h"
#include "Matrix.h"
#include <stdexcept>
#include <regex>
#include <string>
#include "Basics.h"
#include "Matrix.h"
#include "BestGpu.h"
#include "ComputationNetwork.h"
#include "IComputationNetBuilder.h"
#include "commandArgUtil.h"
#pragma warning (disable: 4661)
using namespace std;

10
MachineLearning/CNTK/tests.cpp
Просмотреть файл

@ -6,19 +6,20 @@
// cn.cpp : Defines the entry point for the console application.
//
#define _CRT_SECURE_NO_WARNINGS // "secure" CRT not available on all platforms --add this at the top of all CPP files that give "function or variable may be unsafe" warnings
#include "stdafx.h"
#include <string>
#include <chrono>
#include "Basics.h"
#include "ComputationNetwork.h"
#include "ComputationNode.h"
#include "DataReader.h"
#include "SimpleNetworkBuilder.h"
#include "SGD.h"
#include <string>
#include "commandArgUtil.h"
#include "SimpleEvaluator.h"
#include "NetworkDescriptionLanguage.h"
#include "SynchronousExecutionEngine.h"
#include <chrono>
using namespace std;
using namespace Microsoft::MSR::CNTK;
@ -558,4 +559,3 @@ void GenerateTemplates()
TestCn<float>(config);
TestCn<double>(config);
}

3
MachineLearning/CNTKEval/CNTKEval.cpp
Просмотреть файл

@ -10,7 +10,6 @@
#define EVAL_EXPORTS // creating the exports here
#include "Eval.h"
#include "CNTKEval.h"
#include "commandArgUtil.h"
#include "SimpleOutputWriter.h"
#ifdef LEAKDETECT
#include <vld.h> // leak detection
@ -171,4 +170,4 @@ void CNTKEval<ElemType>::ResetState()
// instantiate all the combinations we expect to be used
template class CNTKEval<double>;
template class CNTKEval<float>;
}}}
}}}

5
MachineLearning/CNTKEval/CNTKEval.h
Просмотреть файл

@ -5,12 +5,15 @@
//
// CNTKEval.h - Include file for the CNTK Evaluation DLL
#pragma once
#include "Eval.h"
#include <string>
#include <map>
#include <vector>
#include "Eval.h"
#include "EvalReader.h"
#include "EvalWriter.h"
#include "ComputationNetwork.h"
namespace Microsoft { namespace MSR { namespace CNTK {

2
Math/Math/GPUMatrix.cu
Просмотреть файл

@ -1837,7 +1837,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
assert(GetNumRows() == a.GetNumRows());
assert(GetNumCols() == b.GetNumRows());
assert(a.GetNumCols() == b.GetNumRows());
c; // TODO: this function seems like a stub
a; b; c; // TODO: this function seems like a stub
/*
EnsureAuxMemory();
int p = 512;