microsoft
/
CNTK
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge branch 'master' of https://github.com/Microsoft/CNTK into sayanpa/cntk206

This commit is contained in:
REDMOND\sayanpa 2017-01-19 08:42:06 -08:00
Родитель cf6f50cc26 9b5eb6c299
Коммит f51a85050c
80 изменённых файлов: 2518 добавлений и 985 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

15
CNTK.sln
Просмотреть файл

@ -1453,6 +1453,19 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerformanceProfilerDll", "S
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryCSEvalExamplesTest", "Tests\EndToEndTests\EvalClientTests\CNTKLibraryCSEvalExamplesTest\CNTKLibraryCSEvalExamplesTest.csproj", "{3500A847-E024-4E7D-92DD-CC587C17460B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "GoogLeNet", "GoogLeNet", "{789B4AB8-40F1-4A37-823A-BC20D80C8BF1}"
ProjectSection(SolutionItems) = preProject
Examples\Image\Classification\GoogLeNet\README.md = Examples\Image\Classification\GoogLeNet\README.md
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BrainScript", "BrainScript", "{95AE4F8A-7618-4B6F-8411-31576062AA63}"
ProjectSection(SolutionItems) = preProject
Examples\Image\Classification\GoogLeNet\BrainScript\InceptionBlocks.bs = Examples\Image\Classification\GoogLeNet\BrainScript\InceptionBlocks.bs
Examples\Image\Classification\GoogLeNet\BrainScript\InceptionV3.bs = Examples\Image\Classification\GoogLeNet\BrainScript\InceptionV3.bs
Examples\Image\Classification\GoogLeNet\BrainScript\InceptionV3.cntk = Examples\Image\Classification\GoogLeNet\BrainScript\InceptionV3.cntk
Examples\Image\Classification\GoogLeNet\BrainScript\README.md = Examples\Image\Classification\GoogLeNet\BrainScript\README.md
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug_CpuOnly|x64 = Debug_CpuOnly|x64
@ -2107,5 +2120,7 @@ Global
{CB4566F1-6C8F-4270-83EE-F6AED84EBB2B} = {39C3C8CA-9A8A-4733-ADBB-3E19D0F52528}
{4B442D34-641A-4B37-9A4B-D18DBE28A979} = {DD043083-71A4-409A-AA91-F9C548DCF7EC}
{3500A847-E024-4E7D-92DD-CC587C17460B} = {05E45AF7-C069-4057-BC16-0A532D068CE4}
{789B4AB8-40F1-4A37-823A-BC20D80C8BF1} = {151202CF-C2E4-47A6-A31C-CE039D698519}
{95AE4F8A-7618-4B6F-8411-31576062AA63} = {789B4AB8-40F1-4A37-823A-BC20D80C8BF1}
EndGlobalSection
EndGlobal

5
Examples/Image/Classification/GoogLeNet/BrainScript/InceptionBlocks.bs
Просмотреть файл

@ -1,13 +1,16 @@
#
# This file contains the basic build block of Inception Network as defined in:
#
# https://arxiv.org/pdf/1512.00567.pdf
#
# and in Tensorflow implementation
#
#
# Convolution layer with Batch Normalization and Rectifier Linear activation.
#
ConvBNReLULayer {numOutputChannels, filterShape, stride, pad = true, bnTimeConst = 4096} = Sequential(
ConvolutionalLayer {numOutputChannels, filterShape, init = "heNormal", stride = stride, pad = pad, bias = false} :
ConvolutionalLayer {numOutputChannels, filterShape, init = "glorotUniform", stride = stride, pad = pad, bias = false} :
BatchNormalizationLayer {spatialRank = 2, normalizationTimeConstant = bnTimeConst, useCntkEngine = false} :
ReLU
)

51
Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.bs
Просмотреть файл

@ -1,8 +1,11 @@
#
# Inception V3 model from:
#
# https://arxiv.org/pdf/1512.00567.pdf
#
# and in Tensorflow implementation
#
InceptionV3(input, labelDim, bnTimeConst) =
{
# 299 x 299 x 3
@ -21,14 +24,15 @@ InceptionV3(input, labelDim, bnTimeConst) =
pool_2 = MaxPoolingLayer{(3:3), stride = (2:2), pad = false}(conv_5)
# 35 x 35 x 192
#
# Inception Blocks
# 35 x 35 x 256
#
mixed_1 = InceptionBlock1{64, (48:64), (64:96:96), 32, bnTimeConst}(pool_2)
# 35 x 35 x 288
# 35 x 35 x 256
mixed_2 = InceptionBlock1{64, (48:64), (64:96:96), 64, bnTimeConst}(mixed_1)
# 35 x 35 x 288
mixed_3 = InceptionBlock1{64, (48:64), (64:96:96), 64, bnTimeConst}(mixed_2)
# 17 x 17 x 768
# 35 x 35 x 288
mixed_4 = InceptionBlock2{384, (64:96:96), bnTimeConst}(mixed_3)
# 17 x 17 x 768
mixed_5 = InceptionBlock3{192, (128:128:192), (128:128:128:128:192), 192, bnTimeConst}(mixed_4)
@ -38,28 +42,47 @@ InceptionV3(input, labelDim, bnTimeConst) =
mixed_7 = InceptionBlock3{192, (160:160:192), (160:160:160:160:192), 192, bnTimeConst}(mixed_6)
# 17 x 17 x 768
mixed_8 = InceptionBlock3{192, (192:192:192), (192:192:192:192:192), 192, bnTimeConst}(mixed_7)
# 8 x 8 x 1280
# 17 x 17 x 768
mixed_9 = InceptionBlock4{(192:320), (192:192:192:192), bnTimeConst}(mixed_8)
# 8 x 8 x 2048
# 17 x 17 x 1280
mixed_10 = InceptionBlock5{320, (384:384:384), (448:384:384:384), 192, bnTimeConst}(mixed_9)
# 8 x 8 x 2048
mixed_11 = InceptionBlock5{320, (384:384:384), (448:384:384:384), 192, bnTimeConst}(mixed_10)
# 8 x 8 x 2048
# Global average
#
# Prediction
#
pool_3 = AveragePoolingLayer{(8:8), pad = false}(mixed_11)
# 1 x 1 x 2048
drop = Dropout(pool_3)
# 1 x 1 x 2048
z = DenseLayer{labelDim}(drop)
z = LinearLayer{labelDim}(drop)
#
# Auxiliary
# 8 x 8 x 1280
aux_pool_1 = AveragePoolingLayer{(5:5), pad = false}(mixed_8)
# 3 x 3 x 1280
#
# 17 x 17 x 768
aux_pool_1 = AveragePoolingLayer{(5:5), stride = (3:3), pad = false}(mixed_8)
# 5 x 5 x 768
aux_conv_1 = ConvBNReLULayer{128, (1:1), (1:1), pad=true, bnTimeConst = bnTimeConst}(aux_pool_1)
# 3 x 3 x 128
aux_conv_2 = ConvBNReLULayer{768, (3:3), (1:1), pad=false, bnTimeConst = bnTimeConst}(aux_conv_1)
aux = DenseLayer{labelDim}(aux_conv_2)
# 5 x 5 x 128
aux_conv_2 = ConvBNReLULayer{768, (5:5), (1:1), pad=false, bnTimeConst = bnTimeConst}(aux_conv_1)
# 1 x 1 x 768
aux = LinearLayer{labelDim}(aux_conv_2)
}
#
# Inception V3 model with normalized input, to use the below function
# remove "ImageNet1K_mean.xml" from each reader.
#
InceptionV3Norm(input, labelDim, bnTimeConst) =
{
# Normalize inputs to -1 and 1.
featMean = 128
featScale = 1/128
Normalize{m,f} = x => f .* (x - m)
inputNorm = Normalize{featMean, featScale}(input)
model = InceptionV3(inputNorm, labelDim, bnTimeConst)
}.model

70
Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk
Просмотреть файл

@ -7,19 +7,19 @@ command = Train:Eval
deviceId = "Auto"
precision = "float"
traceLevel = 1
#traceLevel = 1
#perfTraceLevel = 1
parallelTrain = true
RootDir = "."
ConfigDir = "$RootDir$"
ImageNetDir = "$ConfigDir$"
DataDir = "$RootDir$"
OutputDir = "$RootDir$/Output"
ModelDir = "$OutputDir$/Models"
modelPath = "$ModelDir$/InceptionV3"
#stderr = "$OutputDir$/InceptionV3.log"
ModelDir = "$OutputDir$/Model"
stderr = "$OutputDir$/InceptionV3.log"
modelPath = "$ModelDir$/InceptionV3.model"
ImageH = 299
ImageW = 299
ImageC = 3
@ -27,7 +27,7 @@ NumLabels = 1000
Train = {
action = "train"
traceLevel = 1
BrainScriptNetworkBuilder = {
include "$ConfigDir$/InceptionBlocks.bs"
include "$ConfigDir$/InceptionV3.bs"
@ -35,16 +35,16 @@ Train = {
imageShape = $ImageH$:$ImageW$:$ImageC$
labelDim = $NumLabels$
bnTimeConst = 4096
auxWeight = Constant(0.4)
auxWeight = Constant(0.3)
# inputs
features = Input {imageShape}
labels = Input {labelDim}
# apply model to features
model = InceptionV3(features, labelDim, bnTimeConst)
z = model.z
aux = model.aux
model = InceptionV3Norm(features, labelDim, bnTimeConst)
z = model.z
aux = model.aux
# connect to system
ceAux = CrossEntropyWithSoftmax (labels, aux)
@ -61,52 +61,59 @@ Train = {
}
SGD = {
epochSize = 256000
maxEpochs = 1
minibatchSize = 128 # 16 GPU
epochSize = 0
maxEpochs = 160
minibatchSize = 512 # 16 GPUs, 32 per GPU.
dropoutRate = 0.2
learningRatesPerMB = 1
momentumAsTimeConstant = 4096
#momentumPerMB = 0.9
gradUpdateType = "rmsProp"
normWithAveMultiplier = true
rms_wgt_inc = 1.2
rms_wgt_dec = 0.75
rms_wgt_max = 10.0
rms_wgt_min = 0.1
rms_gamma = 0.9
learningRatesPerMB = 3.2*10: 1.6*10: 0.8*10: 0.4*10: 0.2*10: 0.1*10: 0.05*10: 0.025*10: 0.0125*10: 0.00625*10: 0.003125*10: 0.0015625*10: 0.00078125*10: 0.000390625*10: 0.0001953125
momentumPerMB = 0.9
disableRegInBatchNormalization = true
numMBsToShowResult = 20
parallelTrain = {
parallelizationMethod = "dataParallelSGD"
parallelizationStartEpoch = 1
distributedMBReading = true
dataParallelSGD = {
gradientBits = 32
gradientBits = 32
}
}
firstMBsToShowResult = 10 ; numMBsToShowResult = 500
}
reader = {
verbosity = 0 ; randomize = true
deserializers = ({
type = "ImageDeserializer" ; module = "ImageReader"
file = "$DataDir$/val_map.txt"
file = "$DataDir$/train_map.txt"
input = {
features = { transforms = (
{ type = "Crop" ; cropType = "RandomSide" ; sideRatio = 0.8 ; jitterType = "UniRatio" } :
{ type = "Crop" ; cropType = "randomArea" ; areaRatio = 0.08:1.0 ; jitterType = "uniRatio" ; aspectRatio = 0.75:1.0 } :
{ type = "Color" ; brightnessRadius = 0.2 ; contrastRadius = 0.2 ; saturationRadius = 0.4 } :
{ type = "Scale" ; width = $ImageW$ ; height = $ImageH$ ; channels = $ImageC$ ; interpolations = "linear" } :
{ type = "Mean" ; meanFile = "$ConfigDir$/ImageNet1K_mean.xml" } :
{ type = "Transpose" }
)}
labels = { labelDim = $NumLabels$ }
}
})
}
cvreader = {
verbosity = 0 ; randomize = false
deserializers = ({
type = "ImageDeserializer" ; module = "ImageReader"
file = "$DataDir$/val_map.txt"
input = {
features = { transforms = (
{ type = "Scale" ; width = $ImageW$ ; height = $ImageH$ ; channels = $ImageC$ ; interpolations = "linear" } :
{ type = "Transpose" }
)}
labels = { labelDim = $NumLabels$ }
}
})
}
}
# Eval action
@ -124,7 +131,6 @@ Eval = {
input = {
features = { transforms = (
{ type = "Scale" ; width = $ImageW$ ; height = $ImageH$ ; channels = $ImageC$ ; interpolations = "linear" } :
{ type = "Mean"; meanFile = "$ConfigDir$/ImageNet1K_mean.xml" } :
{ type = "Transpose" }
)}
labels = { labelDim = $NumLabels$ }

4
Scripts/install/windows/_operations.ps1
Просмотреть файл

@ -11,8 +11,8 @@ $operations = @(
Verification = @( @{Function = "VerifyInstallationContent"; Path = "$cntkRootDir" } )
},
@{Name = "Installation VS2015 Runtime"; ShortName = "VS2015"; Info = "Install VS2015 Runtime";
Verification = @( @{Function = "VerifyWin32ProductExists"; Match = "^Microsoft Visual C\+\+ 2015 x64 Additional Runtime" },
@{Function = "VerifyWin32ProductExists"; Match = "^Microsoft Visual C\+\+ 2015 x64 Minimum Runtime" } );
Verification = @( @{Function = "VerifyWin32ProductExists"; Match = "^Microsoft Visual C\+\+ 201(5|7) x64 Additional Runtime" },
@{Function = "VerifyWin32ProductExists"; Match = "^Microsoft Visual C\+\+ 201(5|7) x64 Minimum Runtime" } );
Action = @( @{Function = "InstallExe"; Command = "$cntkRootDir\prerequisites\VS2015\vc_redist.x64.exe"; Param = "/install /passive /norestart"; Message="Installing VS2015 Runtime...." } )
},
@{Name = "MSMPI Installation"; ShortName = "CNTK"; Info = "Install MSMPI";

22
Scripts/install/windows/install.ps1
Просмотреть файл

@ -7,8 +7,13 @@
.SYNOPSIS
Use this cmdlet to install CNTK from a precompiled binary drop (see https://github.com/Microsoft/CNTK/releases)
By default the script will:
- Create or reuse Anaconda3 in the folder `C:\local\Anaconda3-4.1.1-Windows-x86_64`
- Create or update a CNTK Python 3.5 environment in `C:\local\Anaconda3-4.1.1-Windows-x86_64\envs\cntk-py35`
.DESCRIPTION
The script will download and install the CNTK prerequisites and Anaconda environment
The script will download and install the CNTK prerequisites and Anaconda environment.
It will analyse your machine and will determine which components are required.
The required components will be downloaded and cached.
@ -21,15 +26,18 @@
- CNTK will be installed or updated in the CNTK-PY<version> environment
.PARAMETER Execute
This is an optional parameter. Without setting this switch, no changes to the machine setup/installation will be performed
You need to supply this optional parameter to have the install script perform any changes to your machine.
Without this parameter NO CHANGES will be done to your machine.
.PARAMETER AnacondaBasePath
This is an optional parameter and can be used to specify an already installed Anaconda3 installation.
This optional parameter allows you to specify the location of an Anaconda installation to be used or created on your
machine. If the directory exists on your machine, the script will continue under the assumption that this is a working
Anaconda 3 (4.1.1) (or compatible) installation, and will create the CNTK Python environment in that location.
By default a version of Anaconda3 will be installed into [C:\local\Anaconda3-4.1.1-Windows-x86_64]
.PARAMETER PyVersion
This is an optional parameter and can be used to specify the Python version to be used for the CNTK Python environment.
Allowed values for this parameter are 27 34 or 35. The default values is 34 (for a CNTK Python 34 environment)
This is an optional parameter and can be used to specify the Python version used in the CNTK Python environment.
Supported values for this parameter are 27, 34, or 35. The default values is 35 (for a CNTK Python 35 environment).
.EXAMPLE
.\install.ps1
@ -43,14 +51,12 @@
.\install.ps1 -Execute -AnacondaBasePath d:\cntkBeta
This will install Anaconda in the [d:\cntkBeta] directory.
#>
[CmdletBinding()]
Param(
[parameter(Mandatory=$false)] [string] $AnacondaBasePath = "C:\local\Anaconda3-4.1.1-Windows-x86_64",
[parameter(Mandatory=$false)] [ValidateSet("27", "34", "35")] [string] $PyVersion = "34",
[parameter(Mandatory=$false)] [ValidateSet("27", "34", "35")] [string] $PyVersion = "35",
[parameter(Mandatory=$false)] [switch] $Execute)
$MyDir = Split-Path $MyInvocation.MyCommand.Definition

2
Source/1BitSGD

@ -1 +1 @@
Subproject commit 8114febb0f491cd0fec4b60e389672cda8ababfb
Subproject commit 4b2396f36b8129d035a0166cd2d1a1e457404249

104
Source/CNTKv2LibraryDll/API/CNTKLibrary.h
Просмотреть файл

@ -121,6 +121,7 @@ namespace CNTK
struct MinibatchInfo
{
bool atEndOfData;
bool atEndOfSweep;
size_t numberOfSamples;
NDArrayViewPtr trainingLossValue;
NDArrayViewPtr evalCriterionValue;
@ -611,6 +612,11 @@ namespace CNTK
///
CNTK_API NDArrayViewPtr Alias(bool readOnly = false) const;
///
/// Creates a new NDArrayView which is an alias of 'this' view but with a new shape.
///
CNTK_API NDArrayViewPtr AsShape(const NDShape& newShape) const;
///
/// Copies the contents of the 'source' NDArrayView to 'this' view.
/// The shapes of the 'source' view and 'this' view must be identical.
@ -2379,6 +2385,7 @@ namespace CNTK
friend class Trainer;
public:
///
/// Computes and stores the values of specified variables in the 'outputs' map, using provided 'inputs' values corresponding
/// to each leaf variable of the Function of VariableKind 'Input'.
@ -2410,11 +2417,15 @@ namespace CNTK
CNTK_API virtual void Backward(const BackPropStatePtr& state,
const std::unordered_map<Variable, ValuePtr>& rootGradientValues,
std::unordered_map<Variable, ValuePtr>& backPropagatedGradientValuesForInputs);
///
/// Returns the name of the operation that this Function denotes
///
virtual const std::wstring& OpName() const = 0;
virtual const std::wstring& OpName() const
#ifdef SWIG
{ NOT_IMPLEMENTED; }
#else
= 0;
#endif
protected:
///
@ -2471,6 +2482,11 @@ namespace CNTK
///
CNTK_API static FunctionPtr Deserialize(const Dictionary& dictionary, const ::CNTK::DeviceDescriptor& device = DeviceDescriptor::UseDefaultDevice());
///
/// This method needs to be explicitly overriden in subclasses.
///
size_t CurrentVersion() const override { NOT_IMPLEMENTED; }
public:
///
/// Returns the name of 'this' Function.
@ -2694,8 +2710,10 @@ namespace CNTK
// Disallow copy and move construction and assignment
Function(const Function&) = delete; Function(Function&&) = delete; Function& operator=(const Function&) = delete; Function& operator=(Function&&) = delete;
private:
public:
CNTK_API Function(const std::vector<Variable>& inputs, const std::vector<Variable>& outputs, const std::wstring& name = L"", const std::wstring& uid = Internal::GenerateUid(L"UserDefinedFunction"));
private:
CNTK_API Function(const std::vector<Variable>& inputs, const std::vector<Variable>& outputs, Dictionary&& functionConfig, const FunctionPtr& rootFunction, const std::wstring& name, const std::wstring& uid);
std::vector<Variable> m_inputs;
@ -3258,7 +3276,7 @@ namespace CNTK
///
/// A special value that can be used for the epochSize to indicate that the schedule is sweep-based.
///
static const size_t EntireSweep = 0;
static const size_t FullDataSweep = 0;
///
/// Create a schedule with a constant parameter value.
@ -3270,7 +3288,7 @@ namespace CNTK
/// schedule[0] is used for the first 'epochSize' samples, schedule[1] -- for the second,
/// and so on. The last value is then used repeatedly until the end of training.
///
CNTK_API TrainingParameterSchedule(const std::vector<T>& schedule, UnitType unit, size_t epochSize = 1);
CNTK_API TrainingParameterSchedule(const std::vector<T>& schedule, UnitType unit, size_t epochSize = FullDataSweep);
///
/// Create a schedule using the list of key-value pairs, where the key specifies
@ -3281,7 +3299,7 @@ namespace CNTK
/// the first 100 samples, then '0.1' is used for the second 200 samples,
/// after which the values is switched to '0.005'.
///
CNTK_API TrainingParameterSchedule(const std::vector<std::pair<size_t, T>>& schedule, UnitType unit, size_t epochSize = 1);
CNTK_API TrainingParameterSchedule(const std::vector<std::pair<size_t, T>>& schedule, UnitType unit, size_t epochSize = FullDataSweep);
///
/// Returns a value corresponding to the absolute sample (or sweep)
@ -3296,7 +3314,7 @@ namespace CNTK
///
UnitType Unit() const { return m_unit; }
bool IsSweepBased() const { return m_epochSize == EntireSweep; }
bool IsSweepBased() const { return m_epochSize == FullDataSweep; }
CNTK_API virtual ~TrainingParameterSchedule();
@ -3331,12 +3349,15 @@ namespace CNTK
TrainingParameterPerUnitSchedule(T value)
: TrainingParameterSchedule<T>::TrainingParameterSchedule(value, U)
{ }
TrainingParameterPerUnitSchedule(const std::vector<T>& schedule, size_t epochSize = 1)
TrainingParameterPerUnitSchedule(const std::vector<T>& schedule,
size_t epochSize = TrainingParameterSchedule<T>::FullDataSweep)
: TrainingParameterSchedule<T>::TrainingParameterSchedule(schedule, U, epochSize)
{ }
TrainingParameterPerUnitSchedule(const std::vector<std::pair<size_t, T>>& schedule, size_t epochSize = 1)
TrainingParameterPerUnitSchedule(const std::vector<std::pair<size_t, T>>& schedule,
size_t epochSize = TrainingParameterSchedule<T>::FullDataSweep)
: TrainingParameterSchedule<T>::TrainingParameterSchedule(schedule, U, epochSize)
{ }
@ -3384,13 +3405,13 @@ namespace CNTK
ConvertToPerSampleValues();
}
MomentumAsTimeConstantSchedule(const std::vector<double>& schedule, size_t epochSize = 1)
MomentumAsTimeConstantSchedule(const std::vector<double>& schedule, size_t epochSize = FullDataSweep)
: TrainingParameterSchedule<double>::TrainingParameterSchedule(schedule, UnitType::Sample, epochSize)
{
ConvertToPerSampleValues();
}
MomentumAsTimeConstantSchedule(const std::vector<std::pair<size_t, double>>& schedule, size_t epochSize = 1)
MomentumAsTimeConstantSchedule(const std::vector<std::pair<size_t, double>>& schedule, size_t epochSize = FullDataSweep)
: TrainingParameterSchedule<double>::TrainingParameterSchedule(schedule, UnitType::Sample, epochSize)
{
ConvertToPerSampleValues();
@ -3407,9 +3428,10 @@ namespace CNTK
CNTK_API void ConvertToPerSampleValues();
};
///
/// A collection of additional options that affect parameter updates and
/// are applicable for all standard learners
///
struct AdditionalLearningOptions
{
double l1RegularizationWeight = 0.0;
@ -3435,7 +3457,7 @@ namespace CNTK
// Method to update the parameters associated with this learner. By returning false, this method indicates that
// learning has stopped for all of the parameters associated with this learner
//
virtual bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t trainingSampleCount) = 0;
virtual bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t trainingSampleCount, bool sweepEnd = false) = 0;
///
/// Returns the set of parameters associated with this learner.
@ -3593,9 +3615,9 @@ namespace CNTK
return m_communicator;
}
bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t minibatchSampleCount) override
bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t minibatchSampleCount, bool sweepEnd = false) override
{
MinibatchInfo info{ false, minibatchSampleCount };
MinibatchInfo info{ false, sweepEnd, minibatchSampleCount };
return Update(gradientValues, info);
}
@ -3709,6 +3731,11 @@ namespace CNTK
/// Optimize model parameters using the specified 'arguments' minibatch of training samples.
/// Returns false if all parameter learners indicate end of learning (through their Update method's return value).
///
CNTK_API bool TrainMinibatch(const std::unordered_map<Variable, MinibatchData>& arguments, const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice());
///
/// An overload of the TrainMinibatch above that takes a map of variables and their values (as its first argument).
///
CNTK_API bool TrainMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice());
///
@ -3718,12 +3745,22 @@ namespace CNTK
/// for the 'outputs' for which the ValuePtr mapping was left null by the caller.
/// Returns false if all parameter learners indicate end of learning (through their Update method's return value).
///
CNTK_API bool TrainMinibatch(const std::unordered_map<Variable, MinibatchData>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice());
///
/// An overload of the TrainMinibatch above that takes a map of variables and their values (as its first argument).
///
CNTK_API bool TrainMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice());
///
/// Test the model on the specified batch of samples using the evaluation Function specified during construction of the Trainer
/// Returns the average evaluation criterion value per sample for the tested minibatch of samples
///
CNTK_API double TestMinibatch(const std::unordered_map<Variable, MinibatchData>& arguments, const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice());
///
/// An overload of the TestMinibatch above that takes a map of variables and their values (as its first argument).
///
CNTK_API double TestMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice());
///
@ -3789,8 +3826,8 @@ namespace CNTK
const DeviceDescriptor& computeDevice,
std::unordered_map<Variable, ValuePtr>& parameterGradients);
bool TrainLocalMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice);
bool TrainDistributedMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice);
bool TrainLocalMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, bool sweepEnd, const DeviceDescriptor& computeDevice);
bool TrainDistributedMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, bool sweepEnd, const DeviceDescriptor& computeDevice);
void Save(const std::wstring& modelFilePath, const std::vector<DictionaryValue>& learnerState, const Dictionary& externalState);
@ -3837,11 +3874,34 @@ namespace std {
namespace CNTK
{
///
/// A struct that combines the minibatch meta-data with the actual minibatch data.
/// The former includes the number of sequences and samples in the minibatch,
/// as well as the sweep-end flag, which is set to true to indicate that the minibatch
/// concludes a data sweep (i.e, it's the last minibatch at the end of the sweep).
///
struct MinibatchData
{
size_t m_numSequences;
size_t m_numSamples;
ValuePtr m_data;
MinibatchData() : MinibatchData(nullptr)
{}
// a convenience constructor to allow passing ValuePtr arguments in place
// of MinibatchData parameter (e.g., in Trainer::TrainMinibatch)
MinibatchData(ValuePtr value) : MinibatchData(value, 0)
{}
MinibatchData(ValuePtr value, size_t numSamples, bool sweepEnd = false)
: MinibatchData(value, numSamples, numSamples, sweepEnd)
{}
MinibatchData(ValuePtr value, size_t numSequences, size_t numSamples, bool sweepEnd)
: data(value), numberOfSequences(numSequences), numberOfSamples(numSamples), sweepEnd(sweepEnd)
{}
ValuePtr data;
size_t numberOfSequences;
size_t numberOfSamples;
bool sweepEnd;
};
///

3
Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h
Просмотреть файл

@ -160,6 +160,9 @@ namespace CNTK
enum class PrimitiveOpType : unsigned int;
enum class DataType : unsigned int;
struct MinibatchInfo;
struct MinibatchData;
class Serializer;
// Similar to make_shared except that it associates a custom deleter with the shared_ptr to ensure

2
Source/CNTKv2LibraryDll/CompositeFunction.cpp
Просмотреть файл

@ -1344,7 +1344,7 @@ namespace CNTK
PopulateNetworkInputs(arguments);
// Dropout nodes have an implicit input in the form of the random mask that is applied to its explicit input
// This mask is regerated every minibatch and hence dropout nodes with a non-zero dropout rate must me marked outdated
// This mask is regenerated every minibatch and hence dropout nodes with a non-zero dropout rate must me marked outdated
// w.r.t. inputs to force evaluation in each minibatch
list<ComputationNodeBasePtr> dropoutNodes = m_computationNetwork->GetNodesWithType(OperationNameOf(DropoutNode));
for (auto& nodeIter : dropoutNodes)

2
Source/CNTKv2LibraryDll/ComputeInputStatistics.cpp
Просмотреть файл

@ -84,7 +84,7 @@ namespace CNTK
break;
for (auto& currentStreamKV : computedMeanAndInvStdDevs)
CompositeFunction::PopulateComputationNodeValue<float>({ streamToDummyInputVariableMap[currentStreamKV.first], minibatchData[currentStreamKV.first].m_data }, streamToInputNodeMap[currentStreamKV.first], layoutsPopulated);
CompositeFunction::PopulateComputationNodeValue<float>({ streamToDummyInputVariableMap[currentStreamKV.first], minibatchData[currentStreamKV.first].data }, streamToInputNodeMap[currentStreamKV.first], layoutsPopulated);
ComputationNetwork::BumpEvalTimeStamp(allInputNodes);

2
Source/CNTKv2LibraryDll/DataParallelDistributedLearner.cpp
Просмотреть файл

@ -147,6 +147,6 @@ namespace CNTK
if (info.IsEmpty())
return false;
return m_learner->Update(gradientValues, info.numberOfSamples);
return m_learner->Update(gradientValues, info.numberOfSamples, info.atEndOfSweep);
}
}

3
Source/CNTKv2LibraryDll/Function.cpp
Просмотреть файл

@ -29,6 +29,9 @@ namespace CNTK
: Function(inputs, outputs, std::move(functionConfig), nullptr, name, uid)
{}
Function::Function(const std::vector<Variable>& inputs, const std::vector<Variable>& outputs, const std::wstring& name, const std::wstring& uid) :
Function(inputs, outputs, Dictionary(), name, uid) {}
Function::Function(const std::vector<Variable>& inputs, const std::vector<Variable>& outputs, Dictionary&& functionConfig, const FunctionPtr& rootFunction, const std::wstring& name, const std::wstring& uid)
: m_rootFunction(rootFunction), m_name(name != L"" ? name : uid), m_uid(uid), m_attributes(std::move(functionConfig))
{

13
Source/CNTKv2LibraryDll/Learner.cpp
Просмотреть файл

@ -225,7 +225,7 @@ namespace CNTK
}
}
/*virtual*/ bool LearnerBase::Update(unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t trainingSampleCount) /*override*/
/*virtual*/ bool LearnerBase::Update(unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t trainingSampleCount, bool sweepEnd) /*override*/
{
if (LearningRate(trainingSampleCount) == 0.0)
{
@ -233,7 +233,10 @@ namespace CNTK
}
// make sure trainingSampleCount is a valid value
assert(trainingSampleCount > 0);
if (trainingSampleCount == 0)
{
InvalidArgument("Learner::Update(): cannot perform an update with an empty minibatch.");
}
for (const auto& parameter : Parameters())
{
@ -273,7 +276,11 @@ namespace CNTK
}
m_sampleCount += trainingSampleCount;
m_minibatchCount++;
// TODO: sweep count also needs to be updated.
if (sweepEnd)
{
m_sweepCount++;
}
return true;
}

2
Source/CNTKv2LibraryDll/Learner.h
Просмотреть файл

@ -17,7 +17,7 @@ namespace CNTK
class LearnerBase : public Learner
{
public:
virtual bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t trainingSampleCount) override final;
virtual bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t trainingSampleCount, bool sweepEnd = false) override final;
virtual Dictionary CreateCheckpoint() override final;

33
Source/CNTKv2LibraryDll/MinibatchSource.cpp
Просмотреть файл

@ -77,7 +77,7 @@ namespace CNTK
CompositeMinibatchSource::CompositeMinibatchSource(const Dictionary& configuration)
: m_epochEndReached(false),
m_prevMinibatchSize(0),
m_epochSize(MinibatchSource::InfinitelyRepeat),
m_maxNumSamplesToRead(MinibatchSource::InfinitelyRepeat),
m_randomizedWindow(MinibatchSource::DefaultRandomizationWindow),
m_truncationLength(0),
m_numWorkers(1),
@ -136,13 +136,7 @@ namespace CNTK
const wchar_t* epochSizeConfigurationKey = L"epochSize";
if (augmentedConfiguration.Contains(epochSizeConfigurationKey))
m_epochSize = augmentedConfiguration[epochSizeConfigurationKey].Value<size_t>();
if (m_epochSize == MinibatchSource::FullDataSweep)
m_epochSize = Microsoft::MSR::CNTK::requestDataSize;
// Setting big value, but not the max in order to aviod bit overflow.
else if (m_epochSize == MinibatchSource::InfinitelyRepeat)
m_epochSize = std::numeric_limits<size_t>::max() / 2;
m_maxNumSamplesToRead = augmentedConfiguration[epochSizeConfigurationKey].Value<size_t>();
const wchar_t* randomizedWindowConfigurationKey = L"randomizationWindow";
if (augmentedConfiguration.Contains(randomizedWindowConfigurationKey))
@ -212,9 +206,24 @@ namespace CNTK
epochConfig.m_workerRank = workerRank;
epochConfig.m_minibatchSizeInSamples = minibatchSizeInSamples;
epochConfig.m_truncationSize = m_truncationLength;
epochConfig.m_allowMinibatchesToCrossSweepBoundaries = true;
if (m_maxNumSamplesToRead == MinibatchSource::FullDataSweep)
{
epochConfig.m_totalEpochSizeInSamples = Microsoft::MSR::CNTK::requestDataSize;
}
else if (m_maxNumSamplesToRead == MinibatchSource::InfinitelyRepeat)
{
// Setting big value, but not the max in order to aviod bit overflow.
epochConfig.m_totalEpochSizeInSamples = std::numeric_limits<size_t>::max() / 2;
}
else
{
epochConfig.m_totalEpochSizeInSamples = m_maxNumSamplesToRead;
}
epochConfig.m_totalEpochSizeInSamples = m_epochSize;
epochConfig.m_epochIndex = 0;
m_matrices.clear();
std::unordered_set<InputStreamDescription> inputs;
@ -257,6 +266,7 @@ namespace CNTK
newConfig.m_workerRank = workerRank;
newConfig.m_minibatchSizeInSamples = minibatchSizeInSamples;
newConfig.m_truncationSize = m_truncationLength;
newConfig.m_allowMinibatchesToCrossSweepBoundaries = true;
m_shim->SetConfiguration(newConfig, inputDescriptions);
@ -267,9 +277,12 @@ namespace CNTK
auto hasData = m_shim->GetMinibatch(m_matrices);
m_epochEndReached = m_shim->IsEndOfEpoch();
if (m_epochEndReached && !hasData)
return m_minibatchData;
bool hasReachedSweepEnd = m_shim->IsEndOfSweep();
for (const auto& s: m_streamInfos)
{
auto input = m_matrices.GetInput(s.m_name);
@ -293,7 +306,7 @@ namespace CNTK
size_t numSamples = input.pMBLayout->GetActualNumSamples();
size_t numSequences = input.pMBLayout->GetNumSequences();
m_minibatchData[currentStreamInfo] = { numSequences, numSamples, minibatchValuePtr };
m_minibatchData[currentStreamInfo] = { minibatchValuePtr, numSequences, numSamples, hasReachedSweepEnd };
}
else
LogicError("Input data of type other than DataType::Float is currently unsupported by the CNTK built-in composite MinibatchSource!");

2
Source/CNTKv2LibraryDll/MinibatchSource.h
Просмотреть файл

@ -50,7 +50,7 @@ namespace CNTK
size_t m_numWorkers;
size_t m_workerRank;
size_t m_prevMinibatchSize;
size_t m_epochSize;
size_t m_maxNumSamplesToRead;
size_t m_randomizedWindow;
size_t m_truncationLength;
std::unordered_map<StreamInformation, MinibatchData> m_minibatchData;

27
Source/CNTKv2LibraryDll/NDArrayView.cpp
Просмотреть файл

@ -289,6 +289,33 @@ namespace CNTK
return MakeSharedObject<NDArrayView>(GetDataType(), Device(), GetStorageFormat(), Shape(), IsReadOnly() || readOnly, tensorView);
}
NDArrayViewPtr NDArrayView::AsShape(const NDShape& newShape) const
{
if (newShape.TotalSize() != Shape().TotalSize())
{
InvalidArgument("NDArrayView::AsShape: The size (%d) of 'source' view shape's (%S) must be same as the size (%d) of the newShape (%S)!",
(int)Shape().TotalSize(), AsStringForErrorReporting(Shape()).c_str(),
(int)newShape.TotalSize(), AsStringForErrorReporting(newShape).c_str());
}
auto newTensorShape = AsTensorShape(newShape);
void* tensorView = nullptr;
switch (m_dataType)
{
case DataType::Float:
tensorView = new TensorView<float>(*(GetTensorView<float>()), newTensorShape);
break;
case DataType::Double:
tensorView = new TensorView<double>(*(GetTensorView<double>()), newTensorShape);
break;
default:
LogicError("Unsupported DataType %s", DataTypeName(m_dataType));
break;
}
return MakeSharedObject<NDArrayView>(GetDataType(), Device(), GetStorageFormat(), newShape, IsReadOnly(), tensorView);
}
// TODO: This could actually be strided?
template <typename ElementType>
ElementType* NDArrayView::WritableDataBuffer()

49
Source/CNTKv2LibraryDll/Trainer.cpp
Просмотреть файл

@ -116,6 +116,29 @@ namespace CNTK
return (numSamplesInDataArrayView - numMaskedSamples);
}
static std::unordered_map<Variable, ValuePtr> GetInputs(const std::unordered_map<Variable, MinibatchData>& arguments)
{
std::unordered_map<Variable, ValuePtr> inputs(arguments.size());
for (const auto& kv : arguments)
{
inputs[kv.first] = kv.second.data;
}
return inputs;
}
static bool IsAtSweepEnd(const std::unordered_map<Variable, MinibatchData>& arguments)
{
return std::any_of(arguments.begin(), arguments.end(), [](const std::pair<const Variable, MinibatchData>& kv)
{
return kv.second.sweepEnd;
});
}
double Trainer::TestMinibatch(const std::unordered_map<Variable, MinibatchData>& arguments, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
return TestMinibatch(GetInputs(arguments), computeDevice);
}
double Trainer::TestMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
if (!m_aggregatedEvaluationFunction)
@ -123,12 +146,26 @@ namespace CNTK
// TODO: Should we refactor this code that is somewhat similar to the prologue of the TrainMinibatch function
std::unordered_map<Variable, ValuePtr> outputs = { { m_aggregatedEvaluationFunction, nullptr }, { m_testSampleCountVar, nullptr } };
m_combinedTrainingFunction->Forward(arguments, outputs, computeDevice);
auto sampleCount = GetSampleCount(m_testSampleCountVar, outputs[m_testSampleCountVar]);
return (GetScalarValue(outputs[m_aggregatedEvaluationFunction]) / sampleCount);
}
bool Trainer::TrainMinibatch(const std::unordered_map<Variable, MinibatchData>& arguments, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
std::unordered_map<Variable, ValuePtr> outputsToFetch = {};
return TrainMinibatch(arguments, outputsToFetch, computeDevice);
}
bool Trainer::TrainMinibatch(const std::unordered_map<Variable, MinibatchData>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
if (!m_distributed)
return TrainLocalMinibatch(GetInputs(arguments), outputsToFetch, IsAtSweepEnd(arguments), computeDevice);
return TrainDistributedMinibatch(GetInputs(arguments), outputsToFetch, IsAtSweepEnd(arguments), computeDevice);
}
bool Trainer::TrainMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
std::unordered_map<Variable, ValuePtr> outputsToFetch = {};
@ -138,11 +175,11 @@ namespace CNTK
bool Trainer::TrainMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
if (!m_distributed)
return TrainLocalMinibatch(arguments, outputsToFetch, computeDevice);
return TrainDistributedMinibatch(arguments, outputsToFetch, computeDevice);
return TrainLocalMinibatch(arguments, outputsToFetch, false, computeDevice);
return TrainDistributedMinibatch(arguments, outputsToFetch, false, computeDevice);
}
bool Trainer::TrainLocalMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
bool Trainer::TrainLocalMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, bool sweepEnd, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
bool emptyMinibatch = arguments.empty() || (arguments.begin()->second == nullptr);
if (emptyMinibatch) // Nothing to train with.
@ -154,10 +191,10 @@ namespace CNTK
std::unordered_map<Parameter, NDArrayViewPtr> gradients;
for (const auto& parameter : m_combinedTrainingFunction->Parameters())
gradients[parameter] = parameterGradients[parameter]->Data();
return m_parameterLearners->Update(gradients, m_prevMinibatchNumSamples);
return m_parameterLearners->Update(gradients, m_prevMinibatchNumSamples, sweepEnd);
}
bool Trainer::TrainDistributedMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
bool Trainer::TrainDistributedMinibatch(const std::unordered_map<Variable, ValuePtr>& arguments, std::unordered_map<Variable, ValuePtr>& outputsToFetch, bool sweepEnd, const DeviceDescriptor& computeDevice /*= DeviceDescriptor::UseDefaultDevice()*/)
{
std::unordered_map<Parameter, NDArrayViewPtr> gradients;
auto modelParameters = m_combinedTrainingFunction->Parameters();
@ -184,7 +221,7 @@ namespace CNTK
evalCriterion = m_prevMinibatchAggregateEvalCriterionValue->Data();
}
MinibatchInfo info { arguments.empty(), m_prevMinibatchNumSamples, trainingLoss, evalCriterion };
MinibatchInfo info{ arguments.empty(), sweepEnd, m_prevMinibatchNumSamples, trainingLoss, evalCriterion };
bool updated = m_parameterLearners->Update(gradients, info);
m_prevMinibatchNumSamples = info.numberOfSamples;

2
Source/CNTKv2LibraryDll/TrainingSession.cpp
Просмотреть файл

@ -93,7 +93,7 @@ namespace CNTK
if (!minibatchData.empty())
{
for (auto v : m_modelInputToMinibatchSourceStream)
minibatch.insert({ v.first, minibatchData[v.second].m_data });
minibatch.insert({ v.first, minibatchData[v.second].data });
}
OnMinibatchStart();

18
Source/CNTKv2LibraryDll/Utils.cpp
Просмотреть файл

@ -241,7 +241,7 @@ namespace CNTK
template <typename T>
TrainingParameterSchedule<T>::TrainingParameterSchedule(T value, UnitType unit)
: m_schedule({ make_pair(0, value) }), m_unit(unit), m_epochSize(EntireSweep)
: m_schedule({ make_pair(0, value) }), m_unit(unit), m_epochSize(FullDataSweep)
{
}
@ -268,13 +268,9 @@ namespace CNTK
template <typename T>
void TrainingParameterSchedule<T>::ConstructSchedule(const std::vector<std::pair<size_t, T>>& schedule)
{
if (m_epochSize == EntireSweep)
{
//Sweep based schedules are currently not functional (learners don't have sweep info).
NOT_IMPLEMENTED;
}
const auto epochSize = (m_epochSize == EntireSweep) ? 1 : m_epochSize;
// In case of the FullDataSweep, the scheduling unit is just 1 sweep,
// otherwise, it's the epoch size in samples.
const auto unitSize = (m_epochSize == FullDataSweep) ? 1 : m_epochSize;
if (schedule.size() == 0)
RuntimeError("TrainingParameterSchedule::ConstructSchedule : schedule is empty.");
@ -288,7 +284,7 @@ namespace CNTK
RuntimeError("TrainingParameterSchedule::ConstructSchedule : unit count in the 'schedule' argument cannot be 0.");
unitCount += (pair.first != 0) ? pair.first : 1;
m_schedule[epochSize * unitCount] = pair.second;
m_schedule[unitSize * unitCount] = pair.second;
}
}
@ -880,14 +876,14 @@ namespace CNTK
}
}
bool Learners::Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t sampleInMinibatch)
bool Learners::Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t sampleInMinibatch, bool sweepEnd)
{
bool anyUpdatesPerformed = false;
for (auto learner : m_learners)
{
std::unordered_map<Parameter, NDArrayViewPtr> learnerGradients;
GetLearnerGradients(learner, gradientValues, learnerGradients);
anyUpdatesPerformed |= learner->Update(learnerGradients, sampleInMinibatch);
anyUpdatesPerformed |= learner->Update(learnerGradients, sampleInMinibatch, sweepEnd);
}
return anyUpdatesPerformed;
}

2
Source/CNTKv2LibraryDll/Utils.h
Просмотреть файл

@ -501,7 +501,7 @@ namespace CNTK
public:
explicit Learners(const std::vector<LearnerPtr>& learners);
bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t trainingSampleCount);
bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, size_t trainingSampleCount, bool sweepEnd);
bool Update(std::unordered_map<Parameter, NDArrayViewPtr>& gradientValues, MinibatchInfo& minibatchInfo);
std::vector<DictionaryValue> CreateCheckpoint();

5
Source/CNTKv2LibraryDll/Value.cpp
Просмотреть файл

@ -194,16 +194,19 @@ namespace CNTK
NDMaskPtr deviceValueMask = CreateMask(sequenceLengths, sequenceStartFlags, DeviceDescriptor::CPUDevice());
NDArrayViewPtr valueData;
NDShape valueDataShape = sampleShape.AppendShape({ maxSequenceLength, numSequences });
if (numSequences == 1)
{
if (createNewCopy)
valueData = sequences[0]->DeepClone();
else
valueData = sequences[0];
// We can use the original buffer directly but need to reshape to the valueDataShape
valueData = valueData->AsShape(valueDataShape);
}
else
{
NDShape valueDataShape = sampleShape.AppendShape({ maxSequenceLength, numSequences });
if (isDataSparse)
{
if (storageFormat != StorageFormat::SparseCSC)

20
Source/Common/BestGpu.cpp
Просмотреть файл

@ -61,8 +61,6 @@ struct ProcessorData
nvmlMemory_t memory;
nvmlUtilization_t utilization;
cudaDeviceProp deviceProp;
size_t cudaFreeMem;
size_t cudaTotalMem;
bool cntkFound;
int deviceId; // the deviceId (cuda side) for this processor
};
@ -270,29 +268,16 @@ void BestGpu::GetCudaProperties()
if (m_cudaData)
return;
int currentDevice, rc;
rc = cudaGetDevice(&currentDevice);
int dev = 0;
for (ProcessorData* pd : m_procData)
{
cudaSetDevice(dev);
pd->deviceId = dev;
cudaGetDeviceProperties(&pd->deviceProp, dev);
size_t free;
size_t total;
cudaMemGetInfo(&free, &total);
pd->cores = _ConvertSMVer2Cores(pd->deviceProp.major, pd->deviceProp.minor) * pd->deviceProp.multiProcessorCount;
pd->cudaFreeMem = free;
pd->cudaTotalMem = total;
dev++;
cudaDeviceReset();
}
m_cudaData = m_procData.size() > 0;
if (rc == CUDA_SUCCESS)
{
cudaSetDevice(currentDevice);
}
}
void BestGpu::Init()
@ -486,10 +471,7 @@ std::vector<int> BestGpu::GetDevices(int number, BestGpuFlags p_bestFlags)
score = (1.0 - pd->utilization.gpu / 75.0f) * utilGpuW;
score += (1.0 - pd->utilization.memory / 60.0f) * utilMemW;
score += pd->cores / 1000.0f * speedW;
double mem = pd->memory.total > 0 ? pd->memory.free / (double) pd->memory.total : 1000000; // I saw this to be 0 when remoted in
// if it's not a tcc driver, then it's WDDM driver and values will be off because windows allocates all the memory from the nvml point of view
if (!pd->deviceProp.tccDriver || pd->memory.total == 0)
mem = pd->cudaFreeMem / (double) pd->cudaTotalMem;
double mem = pd->memory.total > 0 ? pd->memory.free / (double) pd->memory.total : 1; // I saw this to be 0 when remoted in
score += mem * freeMemW;
score += (pd->cntkFound ? 0 : 1) * mlAppRunningW;
for (int i = 0; i < best.size(); i++)

2
Source/Readers/ImageReader/ImageTransformers.cpp
Просмотреть файл

@ -211,7 +211,7 @@ cv::Rect CropTransformer::GetCropRectCenter(int crow, int ccol, std::mt19937 &rn
else if (m_useAreaRatio)
{
double areaRatio = ApplyRatioJitter(m_areaRatioMin, m_areaRatioMax, rng);
assert(areaRatio >= m_areaRatioMin && areaRatio <= m_sideRatioMax);
assert(areaRatio >= m_areaRatioMin && areaRatio <= m_areaRatioMax);
cropSizeX = cropSizeY = (int)std::round(std::sqrt(crow * ccol * areaRatio)); // we always crop square shape unless aspectRatio is not 1.0
}

131
Source/Readers/ReaderLib/BlockRandomizer.cpp
Просмотреть файл

@ -29,7 +29,7 @@ BlockRandomizer::BlockRandomizer(
m_epochSize(SIZE_MAX),
m_globalSamplePosition(SIZE_MAX),
m_epochStartPosition(0),
m_sweepTotalNumberOfSamples(0),
m_sweepSizeInSamples(0),
m_chunkRandomizer(std::make_shared<ChunkRandomizer>(deserializer, randomizationRangeInSamples)),
m_multithreadedGetNextSequences(multithreadedGetNextSequence),
m_prefetchedChunk(CHUNKID_MAX),
@ -43,10 +43,10 @@ BlockRandomizer::BlockRandomizer(
m_sequenceRandomizer = std::make_shared<SequenceRandomizer>(verbosity, m_deserializer, m_chunkRandomizer);
// Calculate total number of samples.
m_sweepTotalNumberOfSamples = 0;
m_sweepSizeInSamples = 0;
for (auto const & chunk : m_deserializer->GetChunkDescriptions())
{
m_sweepTotalNumberOfSamples += chunk->m_numberOfSamples;
m_sweepSizeInSamples += chunk->m_numberOfSamples;
}
}
@ -63,7 +63,7 @@ void BlockRandomizer::StartEpoch(const EpochConfiguration& config)
m_config = config;
if (config.m_totalEpochSizeInSamples == requestDataSize)
{
m_epochSize = m_sweepTotalNumberOfSamples;
m_epochSize = m_sweepSizeInSamples;
}
else
{
@ -92,7 +92,7 @@ void BlockRandomizer::StartEpoch(const EpochConfiguration& config)
// Prepares a new sweep if needed.
void BlockRandomizer::PrepareNewSweepIfNeeded(size_t samplePosition)
{
size_t sweep = samplePosition / m_sweepTotalNumberOfSamples;
size_t sweep = samplePosition / m_sweepSizeInSamples;
if (m_sweep != sweep)
{
if (m_verbosity >= Notification)
@ -115,32 +115,94 @@ Sequences BlockRandomizer::GetNextSequences(size_t globalSampleCount, size_t loc
{
// Get next sequence descriptions.
Sequences result;
ClosedOpenChunkInterval windowRange;
m_sequenceBuffer.clear();
result.m_endOfEpoch = GetNextSequenceDescriptions(globalSampleCount, localSampleCount, m_sequenceBuffer, windowRange);
if (m_sequenceBuffer.size() == 0)
size_t numGlobalSamplesLoaded = 0, numLocalSamplesLoaded = 0;
do
{
return result;
assert(globalSampleCount > numGlobalSamplesLoaded && localSampleCount > numLocalSamplesLoaded);
bool atTheSweepBoundary = result.m_endOfSweep;
// in case when we continue filling up a minibatch that crosses a sweep boundary,
// make sure that it does not exceed the required number of samples. Set the atLeastOnceSequenceNeeded
// flag to false.
size_t numGlobalSamples = 0, numLocalSamples = 0;
std::tie(numGlobalSamples, numLocalSamples) =
LoadSequenceData(globalSampleCount - numGlobalSamplesLoaded,
localSampleCount - numLocalSamplesLoaded,
result, !atTheSweepBoundary);
if (atTheSweepBoundary && numGlobalSamples == 0)
{
break;
}
numGlobalSamplesLoaded += numGlobalSamples;
numLocalSamplesLoaded += numLocalSamples;
} while (m_config.m_allowMinibatchesToCrossSweepBoundaries &&
!result.m_endOfEpoch &&
result.m_endOfSweep &&
globalSampleCount > numGlobalSamplesLoaded &&
localSampleCount > numLocalSamplesLoaded);
m_cleaner.Clean(result);
return result;
}
std::pair<size_t, size_t> BlockRandomizer::LoadSequenceData(size_t globalSampleCount, size_t localSampleCount, Sequences& sequences, bool atLeastOneSequenceNeeded)
{
ClosedOpenChunkInterval windowRange;
m_sequenceBuffer.clear();
size_t numGlobalSamples = 0, numLocalSamples = 0; // actual number of samples to load (filled in from the sequence descriptions)
bool endOfSweep, endOfEpoch;
std::tie(endOfSweep, endOfEpoch, numGlobalSamples, numLocalSamples) = GetNextSequenceDescriptions(globalSampleCount, localSampleCount, m_sequenceBuffer, windowRange, atLeastOneSequenceNeeded);
sequences.m_endOfSweep |= endOfSweep;
sequences.m_endOfEpoch |= endOfEpoch;
assert(atLeastOneSequenceNeeded || (numGlobalSamples <= globalSampleCount && numLocalSamples <= localSampleCount));
if (numGlobalSamples == 0)
{
assert(!atLeastOneSequenceNeeded || sequences.m_endOfEpoch);
return {0, 0};
}
// Retrieve new data chunks if required.
LoadDataChunks(windowRange);
result.m_data.resize(m_streams.size(), std::vector<SequenceDataPtr>(m_sequenceBuffer.size()));
auto& data = sequences.m_data;
size_t offset = 0;
if (data.empty())
{
data.resize(m_streams.size(), std::vector<SequenceDataPtr>(m_sequenceBuffer.size()));
}
else
{
// sequence data is not empty, we're appending new items to exiting
// sequence data vectors.
offset = data.front().size();
for (auto& sequenceDataVector : data)
{
// make sure that all streams contain the same number of sequences
assert(sequenceDataVector.size() == offset);
sequenceDataVector.resize(offset + m_sequenceBuffer.size());
}
}
auto process = [&](int i) -> void {
const auto& description = m_sequenceBuffer[i];
std::vector<SequenceDataPtr> sequence;
std::vector<SequenceDataPtr> sequenceData;
auto it = m_chunks.find(description.m_chunk->m_original->m_id);
if (it == m_chunks.end())
{
LogicError("Invalid chunk requested.");
}
it->second->GetSequence(description.m_id, sequence);
it->second->GetSequence(description.m_id, sequenceData);
for (int j = 0; j < m_streams.size(); ++j)
{
result.m_data[j][i] = sequence[j];
assert(offset + i < data[j].size());
data[j][offset + i] = sequenceData[j];
}
};
@ -158,18 +220,16 @@ Sequences BlockRandomizer::GetNextSequences(size_t globalSampleCount, size_t loc
process(i);
}
m_cleaner.Clean(result);
// Now it is safe to start the new chunk prefetch.
ChunkIdType chunkToPrefetchNext = GetChunkToPrefetch(windowRange);
Prefetch(chunkToPrefetchNext);
return result;
return { numGlobalSamples, numLocalSamples };
}
// Get next sequence descriptions for that worker that do not exceed global and local sample count.
// Returns true if epoch end is reached.
bool BlockRandomizer::GetNextSequenceDescriptions(size_t globalSampleCount, size_t localSampleCount, std::vector<RandomizedSequenceDescription>& result, ClosedOpenChunkInterval& windowRange)
std::tuple<bool, bool, size_t, size_t> BlockRandomizer::GetNextSequenceDescriptions(size_t globalSampleCount, size_t localSampleCount, std::vector<RandomizedSequenceDescription>& result, ClosedOpenChunkInterval& windowRange, bool atLeastOneSequenceNeeded)
{
if (globalSampleCount == 0)
LogicError("Global sample count must not be zero.");
@ -179,17 +239,22 @@ bool BlockRandomizer::GetNextSequenceDescriptions(size_t globalSampleCount, size
PrepareNewSweepIfNeeded(m_globalSamplePosition);
auto sweepPosition = m_globalSamplePosition % m_sweepSizeInSamples;
auto epochEndPosition = m_epochSize + m_epochStartPosition;
// Check epoch end.
if (m_globalSamplePosition >= m_epochSize + m_epochStartPosition)
if (m_globalSamplePosition >= epochEndPosition)
{
return true;
auto reachedEndOfEpoch = true;
auto reachedEndOfSweep = (m_globalSamplePosition >= m_sweepSizeInSamples) && (sweepPosition == 0);
return std::make_tuple(reachedEndOfSweep, reachedEndOfEpoch, 0, 0);
}
// Global sample count should not exceed the epoch.
globalSampleCount = std::min(globalSampleCount, m_epochSize + m_epochStartPosition - m_globalSamplePosition);
globalSampleCount = std::min(globalSampleCount, epochEndPosition - m_globalSamplePosition);
// Global sample count should also not exceed the sweep.
globalSampleCount = std::min(globalSampleCount, (long)m_sweepTotalNumberOfSamples - m_globalSamplePosition % m_sweepTotalNumberOfSamples);
globalSampleCount = std::min(globalSampleCount, m_sweepSizeInSamples - sweepPosition);
if (globalSampleCount == 0)
LogicError("Global sample count must not result in zero.");
@ -197,12 +262,17 @@ bool BlockRandomizer::GetNextSequenceDescriptions(size_t globalSampleCount, size
std::function<bool(const RandomizedSequenceDescription*)> isLocalSequence =
[this](const RandomizedSequenceDescription* s) { return s->m_chunk->m_chunkId % m_config.m_numberOfWorkers == m_config.m_workerRank; };
size_t actualNumberOfGlobalSamples = m_sequenceRandomizer->GetNextSequenceDescriptions(
size_t actualNumberOfGlobalSamples = 0, actualNumberOfLocalSamples = 0;
std::tie(actualNumberOfGlobalSamples, actualNumberOfLocalSamples) = m_sequenceRandomizer->GetNextSequenceDescriptions(
globalSampleCount,
localSampleCount,
isLocalSequence,
windowRange,
result);
result,
atLeastOneSequenceNeeded);
if (actualNumberOfLocalSamples > actualNumberOfGlobalSamples)
LogicError("Local sample count cannot be greater than the global sample count.");
if (m_verbosity >= Debug)
fprintf(stderr, "BlockRandomizer::GetNextSequenceDescriptions(): getting %" PRIu64 " sequences for %" PRIu64 "/%" PRIu64 " requested local/global samples in sweep %" PRIu64 "\n",
@ -211,10 +281,15 @@ bool BlockRandomizer::GetNextSequenceDescriptions(size_t globalSampleCount, size
globalSampleCount,
m_sweep);
// set "reachedEndOfSweep" to true if the minibatch is last in a sweep
auto reachedEndOfSweep = (sweepPosition + actualNumberOfGlobalSamples >= m_sweepSizeInSamples);
// set "reachedEndOfEpoch" to true if the current batch is last in an epoch.
auto reachedEndOfEpoch = (m_globalSamplePosition + actualNumberOfGlobalSamples >= epochEndPosition);
// Update the global sample position.
m_globalSamplePosition += actualNumberOfGlobalSamples;
// return true if the current batch is last in an epoch.
return m_globalSamplePosition >= m_epochSize + m_epochStartPosition;
return std::make_tuple(reachedEndOfSweep, reachedEndOfEpoch, actualNumberOfGlobalSamples, actualNumberOfLocalSamples);
}
// Retrieves chunk data based on the window information provided by SequenceRandomizer
@ -350,9 +425,9 @@ void BlockRandomizer::SetCurrentSamplePosition(size_t currentSamplePosition)
// Sets sequence cursor to the sequence that corresponds to the epoch start position.
// If last epoch ended in the middle of a sequence, the cursor is moved to the next sequence in the sweep.
size_t offsetInSweep = currentSamplePosition % m_sweepTotalNumberOfSamples;
size_t offsetInSweep = currentSamplePosition % m_sweepSizeInSamples;
size_t newOffset = m_sequenceRandomizer->Seek(offsetInSweep, m_sweep);
m_globalSamplePosition = m_sweep * m_sweepTotalNumberOfSamples + newOffset;
m_globalSamplePosition = m_sweep * m_sweepSizeInSamples + newOffset;
// Check if we have some data, if not set to the end of epoch.
if (m_config.m_workerRank >= m_chunkRandomizer->GetRandomizedChunks().size())

19
Source/Readers/ReaderLib/BlockRandomizer.h
Просмотреть файл

@ -77,8 +77,21 @@ private:
// Load data for chunks if needed.
void LoadDataChunks(const ClosedOpenChunkInterval& windowRange);
// Get next sequence descriptions that do not exceed global and local sample count.
bool GetNextSequenceDescriptions(size_t globalSampleCount, size_t localSampleCount, std::vector<RandomizedSequenceDescription>& result, ClosedOpenChunkInterval& windowRange);
// Load actual sequence data up to the specified global/local sample count
// (or at least one sequence when atLeastOneSequenceNeeded is true),
// Returns the total number of global and local samples loaded.
std::pair<size_t, size_t> LoadSequenceData(size_t globalSampleCount, size_t localSampleCount, Sequences& sequence, bool atLeastOneSequenceNeeded);
// Gets the next sequence descriptions with the total number of samples not exceeding
// the sample count, when atLeastOneSequenceNeeded is false. Otherwise (when atLeastOneSequenceNeeded is true),
// returns at least one sequence description even when its length is greater than the required sample count.
// Returns a tuple containing "end of sweep", "end of epoch" flags and
// the total numbers of global and local samples to be processed.
std::tuple<bool, bool, size_t, size_t> GetNextSequenceDescriptions(size_t globalSampleCount,
size_t localSampleCount,
std::vector<RandomizedSequenceDescription>& result,
ClosedOpenChunkInterval& windowRange,
bool atLeastOneSequenceNeeded);
// Prepares a new sweep if needed.
void PrepareNewSweepIfNeeded(size_t samplePosition);
@ -105,7 +118,7 @@ private:
size_t m_sweep;
// Total number of samples in a sweep.
size_t m_sweepTotalNumberOfSamples;
size_t m_sweepSizeInSamples;
IDataDeserializerPtr m_deserializer;

39
Source/Readers/ReaderLib/NoRandomizer.cpp
Просмотреть файл

@ -17,7 +17,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_currentChunkPosition(CHUNKID_MAX),
m_globalSamplePosition(0),
m_globalSequencePosition(0),
m_totalNumberOfSamples(0),
m_sweepSizeInSamples(0),
m_currentSequencePositionInChunk(0),
m_multithreadedGetNextSequences(multithreadedGetNextSequences),
m_cleaner(maxNumberOfInvalidSequences)
@ -41,7 +41,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
RuntimeError("NoRandomizer: Expected input to contain samples, but the number of successfully read samples was 0.");
}
m_totalNumberOfSamples = sampleCount;
m_sweepSizeInSamples = sampleCount;
}
ChunkIdType NoRandomizer::GetChunkIndexOf(size_t samplePosition)
@ -55,7 +55,7 @@ void NoRandomizer::StartEpoch(const EpochConfiguration& config)
m_config = config;
if (m_config.m_totalEpochSizeInSamples == requestDataSize)
m_config.m_totalEpochSizeInSamples = m_totalNumberOfSamples;
m_config.m_totalEpochSizeInSamples = m_sweepSizeInSamples;
SetCurrentSamplePosition(m_config.m_totalEpochSizeInSamples * config.m_epochIndex);
}
@ -83,39 +83,38 @@ void NoRandomizer::GetNextSequenceDescriptions(size_t globalSampleCount, size_t
assert(globalSampleCount != 0);
assert(localSampleCount != 0);
if (globalSampleCount > std::numeric_limits<int>::max() ||
if (globalSampleCount > std::numeric_limits<int>::max() &&
localSampleCount > std::numeric_limits<int>::max())
RuntimeError("Global and local size of the minibatch cannot exceed max int.");
assert(m_sequenceWindow.size() != 0);
assert(m_chunkDescriptions[m_currentChunkPosition]->m_numberOfSequences > m_currentSequencePositionInChunk);
int localSamplesLeft = (int)localSampleCount;
int globalSamplesLeft = (int)globalSampleCount;
size_t numGlobalSamplesLoaded = 0, numLocalSamplesLoaded = 0;
result.reserve(localSampleCount);
result.clear();
while (globalSamplesLeft > 0 && localSamplesLeft > 0)
while (globalSampleCount > numGlobalSamplesLoaded && localSampleCount > numLocalSamplesLoaded)
{
const SequenceDescription& sequence = m_sequenceWindow[m_currentSequencePositionInChunk];
int sequenceLength = (int)sequence.m_numberOfSamples;
auto sequenceLength = sequence.m_numberOfSamples;
// Let's check whether we need to return this sequence or skip it.
bool isLocal = m_globalSequencePosition % m_config.m_numberOfWorkers == m_config.m_workerRank;
if (result.empty() ||
((localSamplesLeft >= sequenceLength) && (globalSamplesLeft >= sequenceLength)))
((localSampleCount - numLocalSamplesLoaded >= sequenceLength) && (globalSampleCount - numGlobalSamplesLoaded >= sequenceLength)))
{
if (isLocal) // Ok good to add it to the result.
{
result.push_back(sequence);
localSamplesLeft -= sequence.m_numberOfSamples;
numLocalSamplesLoaded += sequence.m_numberOfSamples;
}
}
else // otherwise there is no room, return what we have.
break;
globalSamplesLeft -= sequence.m_numberOfSamples;
numGlobalSamplesLoaded += sequence.m_numberOfSamples;
m_globalSamplePosition += sequence.m_numberOfSamples;
m_globalSequencePosition++;
@ -141,25 +140,35 @@ Sequences NoRandomizer::GetNextSequences(size_t globalSampleCount, size_t localS
if (m_globalSamplePosition >= endOfEpochPosition)
{
result.m_endOfEpoch = true;
result.m_endOfSweep = (m_globalSamplePosition >= m_sweepSizeInSamples) &&
(m_globalSamplePosition % m_sweepSizeInSamples == 0);
return result;
}
// Check we do not go over epoch.
globalSampleCount = std::min(globalSampleCount, endOfEpochPosition - m_globalSamplePosition);
// Check that we do not go over the sweep.
size_t sweepPosition = m_globalSamplePosition % m_totalNumberOfSamples;
globalSampleCount = std::min(globalSampleCount, m_totalNumberOfSamples - sweepPosition);
if (!m_config.m_allowMinibatchesToCrossSweepBoundaries)
{
// Cut down the required sample count if we're not allowed to go over the
// sweep boundary
size_t sweepPosition = m_globalSamplePosition % m_sweepSizeInSamples;
globalSampleCount = std::min(globalSampleCount, m_sweepSizeInSamples - sweepPosition);
}
if (globalSampleCount == 0)
LogicError("Global sample count must not result in zero.");
auto sweepIndex = m_globalSamplePosition / m_sweepSizeInSamples;
m_sequenceBuffer.clear();
GetNextSequenceDescriptions(globalSampleCount, localSampleCount, m_sequenceBuffer);
// m_globalSamplePosition is already shifted in GetNextSequenceDescriptions() by the current minibatch size.
// Set the end-of-epoch flag (true when the current batch is last in an epoch).
result.m_endOfEpoch = (m_globalSamplePosition >= endOfEpochPosition);
result.m_endOfSweep = sweepIndex != m_globalSamplePosition / m_sweepSizeInSamples;
if (m_sequenceBuffer.size() == 0)
{
return result;
@ -229,7 +238,7 @@ void NoRandomizer::SetCurrentSamplePosition(size_t samplePosition)
{
m_currentSequencePositionInChunk = 0;
m_globalSamplePosition = samplePosition;
size_t sweepSamplePosition = m_globalSamplePosition % m_totalNumberOfSamples;
size_t sweepSamplePosition = m_globalSamplePosition % m_sweepSizeInSamples;
ChunkIdType chunkIndex = GetChunkIndexOf(sweepSamplePosition);
if (chunkIndex != m_currentChunkPosition)

2
Source/Readers/ReaderLib/NoRandomizer.h
Просмотреть файл

@ -88,7 +88,7 @@ private:
size_t m_globalSequencePosition;
// Total number of samples in the sweep.
size_t m_totalNumberOfSamples;
size_t m_sweepSizeInSamples;
// Temp buffer to avoid allocations.
std::vector<SequenceDescription> m_sequenceBuffer;

16
Source/Readers/ReaderLib/Reader.h
Просмотреть файл

@ -32,6 +32,11 @@ struct ReaderConfiguration
size_t m_workerRank; // Rank of the Open MPI worker, worker rank has to be less than the number of workers
size_t m_minibatchSizeInSamples; // Maximum minibatch size for the epoch in samples
size_t m_truncationSize; // Truncation size in samples for truncated BPTT mode.
// This flag indicates whether the minibatches are allowed to overlap the boundary
// between sweeps (in which case, they can contain data from different sweeps) or
// if they need to be trimmed at the sweep end.
bool m_allowMinibatchesToCrossSweepBoundaries{ false };
};
// TODO: Should be deprecated.
@ -87,6 +92,10 @@ typedef std::shared_ptr<StreamMinibatch> StreamMinibatchPtr;
// Represents a single minibatch, that contains information about all streams.
struct Minibatch
{
// Indicates that this minibatch is either adjacent to the data sweep boundary
// (-----<minibatch>|---) or crosses the boundary (-----<mini|batch>---).
bool m_endOfSweep;
// Indicates that the end of epoch has been reached.
// It is set to true for the last minibatch, there still
// can be data in m_data field even if this flag is set.
@ -95,11 +104,8 @@ struct Minibatch
// Minibatch data
std::vector<StreamMinibatchPtr> m_data;
Minibatch() : m_endOfEpoch(false)
{
}
Minibatch(bool endOfEpoch) : m_endOfEpoch(endOfEpoch)
Minibatch(bool endOfSweep = false, bool endOfEpoch = false)
: m_endOfSweep(endOfSweep), m_endOfEpoch(endOfEpoch)
{
}
};

6
Source/Readers/ReaderLib/ReaderShim.cpp
Просмотреть файл

@ -28,6 +28,7 @@ ReaderShim<ElemType>::ReaderShim() :
m_dataTransferers(2, DataTransfererPtr()),
m_currentDataTransferIndex(0),
m_endOfEpoch(false),
m_endOfSweep(false),
m_currentSamplePosition(0),
m_reader(nullptr),
m_factory(nullptr)
@ -274,6 +275,7 @@ bool ReaderShim<ElemType>::GetMinibatch(StreamMinibatchInputs& matrices)
m_currentSamplePosition = m_reader->GetCurrentSamplePosition();
m_endOfEpoch = result.m_isEndOfEpoch;
m_endOfSweep = result.m_isEndOfSweep;
if (m_endOfEpoch && !result.m_isDataAvailable)
{
// No data and end of epoch, simply return.
@ -357,7 +359,7 @@ typename ReaderShim<ElemType>::PrefetchResult ReaderShim<ElemType>::PrefetchMini
// If there is no data we can simply return.
if (minibatch.m_data.empty())
return PrefetchResult{ minibatch.m_endOfEpoch, false };
return PrefetchResult{ minibatch.m_endOfSweep, minibatch.m_endOfEpoch, false };
// Ok we have some data. Let's load it to GPU.
// But before we need to make sure that corresponding compute has already finished from the last iteration.
@ -380,7 +382,7 @@ typename ReaderShim<ElemType>::PrefetchResult ReaderShim<ElemType>::PrefetchMini
if (m_dataTransferers[currentDataTransferIndex])
m_dataTransferers[currentDataTransferIndex]->RecordCPUToGPUCopy();
return PrefetchResult{ minibatch.m_endOfEpoch, true };
return PrefetchResult{ minibatch.m_endOfSweep, minibatch.m_endOfEpoch, true };
}

7
Source/Readers/ReaderLib/ReaderShim.h
Просмотреть файл

@ -100,9 +100,15 @@ public:
return m_endOfEpoch;
}
bool IsEndOfSweep() const
{
return m_endOfSweep;
}
private:
struct PrefetchResult
{
bool m_isEndOfSweep;
bool m_isEndOfEpoch;
bool m_isDataAvailable;
};
@ -113,6 +119,7 @@ private:
ReaderPtr m_reader;
ReaderFactory m_factory;
bool m_endOfEpoch;
bool m_endOfSweep;
size_t m_numParallelSequences;

7
Source/Readers/ReaderLib/SequenceEnumerator.h
Просмотреть файл

@ -20,8 +20,13 @@ struct Sequences
// Indices in the outer vector have to correspond to the stream ids returned from the GetStreamDescriptions().
std::vector<std::vector<SequenceDataPtr>> m_data;
// Indicates whether the returned data comes from a sweep end or
// crosses a sweep boundary (and as a result includes sequences
// from different sweeps).
bool m_endOfSweep{ false };
// Indicates whether the epoch ends with the data returned.
bool m_endOfEpoch = false;
bool m_endOfEpoch{ false };
};
class SequenceEnumerator;

2
Source/Readers/ReaderLib/SequencePacker.cpp
Просмотреть файл

@ -41,7 +41,7 @@ Minibatch SequencePacker::ReadMinibatch()
auto sequences = m_sequenceEnumerator->GetNextSequences(m_globalMinibatchSizeInSamples, m_localMinibatchSizeInSamples);
const auto& batch = sequences.m_data;
Minibatch minibatch(sequences.m_endOfEpoch);
Minibatch minibatch(sequences.m_endOfSweep, sequences.m_endOfEpoch);
if (batch.empty())
return minibatch;

38
Source/Readers/ReaderLib/SequenceRandomizer.cpp
Просмотреть файл

@ -63,26 +63,28 @@ namespace Microsoft { namespace MSR { namespace CNTK {
RandomizeNextChunkIfNeeded();
}
// Gets the next randomized sequence descriptions not exceeding the global and local sample count.
// Whether the sequence is considered local is defined by the isLocalSequence predicate.
// Returns how many global samples have been read.
size_t SequenceRandomizer::GetNextSequenceDescriptions(
// Gets the next randomized sequence descriptions not exceeding the global and local sample count,
// when atLeastOneSequenceNeeded is false. Otherwise (when atLeastOneSequenceNeeded is true),
// returns at least one sequence description even when its length is greater than the required sample counts.
// Whether a sequence is considered local is defined by the isLocalSequence predicate.
// Returns a pair whose first element indicates the number of global samples read,
// and second -- the number of local samples read (== sum of number of sample over all elements in the
// 'sequences' vector).
std::pair<size_t, size_t> SequenceRandomizer::GetNextSequenceDescriptions(
size_t globalSampleCount,
size_t localSampleCount,
const std::function<bool(const RandomizedSequenceDescription*)>& isLocalSequence,
ClosedOpenChunkInterval& requiredChunks,
std::vector<RandomizedSequenceDescription>& sequences)
std::vector<RandomizedSequenceDescription>& sequences,
bool atLeastOneSequenceNeeded)
{
assert(globalSampleCount != 0);
assert(localSampleCount != 0);
if (globalSampleCount > std::numeric_limits<int>::max() ||
if (globalSampleCount > std::numeric_limits<int>::max() &&
localSampleCount > std::numeric_limits<int>::max())
RuntimeError("Global and local size of the minibatch cannot exceed max int.");
int localSamplesLeft = (int)localSampleCount;
int globalSamplesLeft = (int)globalSampleCount;
// Initialize the range to the current chunk.
requiredChunks.m_begin = (ChunkIdType)std::min(m_currentChunkCursor, m_randomizedChunks.size() - 1);
requiredChunks.m_end = requiredChunks.m_begin + 1;
@ -90,9 +92,9 @@ namespace Microsoft { namespace MSR { namespace CNTK {
sequences.reserve(localSampleCount);
sequences.clear();
size_t totalSamplesRead = 0;
size_t globalSamplesRead = 0, localSamplesRead = 0;
while (m_currentChunkCursor < m_randomizedChunks.size() &&
globalSamplesLeft > 0 && localSamplesLeft > 0)
(localSamplesRead < localSampleCount && globalSamplesRead < globalSampleCount))
{
size_t sequenceOffsetInsideChunk = m_currentSequenceCursor - m_randomizedChunks[m_currentChunkCursor].m_sequencePositionStart;
const RandomizedSequenceDescription* sequence = &m_sequenceWindow[m_currentChunkCursor - m_chunkWindowBegin][sequenceOffsetInsideChunk];
@ -100,20 +102,22 @@ namespace Microsoft { namespace MSR { namespace CNTK {
bool isLocal = isLocalSequence(sequence);
// Let's check whether we need to return this sequence or skip it.
if (sequences.empty() ||
((localSamplesLeft >= sequenceLength) && (globalSamplesLeft >= sequenceLength)))
if ((sequences.empty() && atLeastOneSequenceNeeded) ||
((localSamplesRead + sequenceLength <= localSampleCount) && (globalSamplesRead + sequenceLength <= globalSampleCount)))
{
if (isLocal) // Ok good to add it to the result.
{
sequences.push_back(*sequence);
localSamplesLeft -= sequenceLength;
localSamplesRead += sequenceLength;
}
// even when the next sequence is not local, somebody else would return it, so
// we need to ivalidate the 'atLeastOneSequenceNeeded' flag.
atLeastOneSequenceNeeded = false;
}
else // otherwise there is no room, return what we have.
break;
totalSamplesRead += sequenceLength;
globalSamplesLeft -= sequenceLength;
globalSamplesRead += sequenceLength;
// Update the required chunk window.
requiredChunks.m_begin = std::min(m_randomizedChunks[m_currentChunkCursor].m_randomizationWindow.m_begin, requiredChunks.m_begin);
@ -130,7 +134,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
}
}
return totalSamplesRead;
return { globalSamplesRead, localSamplesRead };
}
// Move the chunk cursor to the next chunk, randomizing more sequences if necessary.

13
Source/Readers/ReaderLib/SequenceRandomizer.h
Просмотреть файл

@ -45,15 +45,20 @@ public:
// If the offset points in the middle of last sequence, the end of the sweep is returned.
size_t Seek(size_t sweepSampleOffset, size_t sweep);
// Gets the next randomized sequence descriptions not exceeding the global and local sample count.
// Gets the next randomized sequence descriptions not exceeding the global and local sample count,
// when atLeastOneSequenceNeeded is false. Otherwise (when atLeastOneSequenceNeeded is true),
// returns at least one sequence description even when its length is greater than the required sample count.
// Whether a sequence is considered local is defined by the isLocalSequence predicate.
// The return value is how many global samples have been read.
size_t GetNextSequenceDescriptions(
// Returns a pair whose first element indicates the number of global samples read,
// and second -- the number of local samples read (== sum of number of sample over all elements in the
// 'sequences' vector).
std::pair<size_t, size_t> GetNextSequenceDescriptions(
size_t globalSampleCount,
size_t localSampleCount,
const std::function<bool(const RandomizedSequenceDescription*)>& isLocalSequence,
ClosedOpenChunkInterval& requiredChunks,
std::vector<RandomizedSequenceDescription>& sequences);
std::vector<RandomizedSequenceDescription>& sequences,
bool atLeastOneSequenceNeeded = true);
private:
DISABLE_COPY_AND_MOVE(SequenceRandomizer);

89
Source/Readers/ReaderLib/TruncatedBpttPacker.cpp
Просмотреть файл

@ -7,7 +7,6 @@
#define _SCL_SECURE_NO_WARNINGS
#include <cmath>
#include <deque>
#include "TruncatedBpttPacker.h"
#include "ReaderUtil.h"
@ -38,9 +37,10 @@ public:
}
// Adds a new sequence to the end of the slot.
void PushSequence(SequenceDataPtr s)
void PushSequence(SequenceDataPtr s, bool endOfSweep)
{
m_sequences.push_back(s);
m_endOfSweepFlags.push_back(endOfSweep);
m_length += s->m_numberOfSamples;
}
@ -51,13 +51,16 @@ public:
}
// Pops the front sequence at the beginning of the slot.
void PopSequence()
bool PopSequence()
{
assert(!m_sequences.empty());
m_sampleCursor = 0;
m_sampleOffset = 0;
m_length -= m_sequences.front()->m_numberOfSamples;
m_sequences.pop_front();
bool endOfSweepFlag = m_endOfSweepFlags.front();
m_endOfSweepFlags.pop_front();
return endOfSweepFlag;
}
// Contains the current sample cursor in the first sequence(m_sequences.front()) of the slot.
@ -72,6 +75,10 @@ public:
private:
// Prepared sequences.
deque<SequenceDataPtr> m_sequences;
// For each 'in-flight' sequence we keep a flag that indicate whether
// the sequence data comes from an the end of a sweep.
std::deque<bool> m_endOfSweepFlags;
// Contains the size of the slot in samples (accumulated over all m_sequences).
size_t m_length;
@ -155,7 +162,7 @@ void TruncatedBPTTPacker::SetConfiguration(const ReaderConfiguration& config, co
m_sequenceBufferPerStream.clear();
// Preparing the buffers.
// Preparing the buffers.
for (int j = 0; j < m_streamBuffers.size(); ++j)
for (int i = 0; i < m_outputStreamDescriptions.size(); ++i)
{
@ -166,25 +173,22 @@ void TruncatedBPTTPacker::SetConfiguration(const ReaderConfiguration& config, co
}
}
// Filling in the initial set of sequences
for (size_t slotIndex = 0; slotIndex < m_numParallelSequences; ++slotIndex)
{
ReadSequencesToSlot(slotIndex);
}
FillOutAvailableSlots();
}
Minibatch TruncatedBPTTPacker::ReadMinibatch()
{
Minibatch result;
FillOutAvailableSlots();
// Currently all we expect sequences of identical length between different streams,
// so it is sufficient to check a single stream only.
if (m_sequenceBufferPerStream.front()->NothingToPack())
{
result.m_endOfEpoch = true;
return result;
{
return Minibatch(/*endOfSweep = */false,/*endOfEpoch = */ true);
}
Minibatch result;
// Iterating over the streams/slots and packing them into the minibatch.
for (size_t streamIndex = 0; streamIndex < m_outputStreamDescriptions.size(); ++streamIndex)
{
@ -192,7 +196,7 @@ Minibatch TruncatedBPTTPacker::ReadMinibatch()
size_t sequenceId = 0;
for (size_t slotIndex = 0; slotIndex < m_numParallelSequences; ++slotIndex)
{
PackSlot(streamIndex, slotIndex, sequenceId);
result.m_endOfSweep |= PackSlot(streamIndex, slotIndex, sequenceId);
}
StreamMinibatchPtr m = make_shared<StreamMinibatch>();
@ -203,17 +207,18 @@ Minibatch TruncatedBPTTPacker::ReadMinibatch()
m_currentBufferIndex = (m_currentBufferIndex + 1) % m_numberOfBuffers;
// Eagerly set the end of epoch flag if all the data have been packed.
result.m_endOfEpoch = m_sequenceBufferPerStream.front()->NothingToPack();
return result;
}
// Packs a slot of sequences into the minibatch.
void TruncatedBPTTPacker::PackSlot(size_t streamIndex, size_t slotIndex, size_t& sequenceId)
bool TruncatedBPTTPacker::PackSlot(size_t streamIndex, size_t slotIndex, size_t& sequenceId)
{
bool containsEndOfSweepSequence = false;
auto& slot = m_sequenceBufferPerStream[streamIndex]->m_slots[slotIndex];
// Fill free space in the slot.
ReadSequencesToSlot(slotIndex);
// Let's see how much samples we need to read.
size_t numberOfSamples = min(m_config.m_truncationSize, slot.AvailableNumberOfSamples());
if (numberOfSamples == 0)
@ -223,7 +228,7 @@ void TruncatedBPTTPacker::PackSlot(size_t streamIndex, size_t slotIndex, size_t&
// Check that nothing is in the slot any more.
assert(slot.IsEmpty());
return;
return false;
}
size_t sampleSize = GetSampleSize(m_inputStreamDescriptions[streamIndex]);
@ -247,7 +252,7 @@ void TruncatedBPTTPacker::PackSlot(size_t streamIndex, size_t slotIndex, size_t&
if (slot.m_sampleCursor >= slot.FrontSequence()->m_numberOfSamples)
{
// Starting a new sequence. Have to reset current pointers and add it to the minibatch layout.
slot.PopSequence();
containsEndOfSweepSequence |= slot.PopSequence();
//Adding next sequence to the minibatch.
m_currentLayouts[streamIndex]->AddSequence(
@ -290,7 +295,7 @@ void TruncatedBPTTPacker::PackSlot(size_t streamIndex, size_t slotIndex, size_t&
// Cleaning up the last sequence we have just read if needed.
if (slot.m_sampleCursor >= slot.FrontSequence()->m_numberOfSamples)
{
slot.PopSequence();
containsEndOfSweepSequence |= slot.PopSequence();
}
// Adding the last gap if there is one.
@ -302,35 +307,59 @@ void TruncatedBPTTPacker::PackSlot(size_t streamIndex, size_t slotIndex, size_t&
numberOfSamples,
m_config.m_truncationSize);
}
return containsEndOfSweepSequence;
}
void TruncatedBPTTPacker::FillOutAvailableSlots()
{
// Filling out any available spaces
for (size_t slotIndex = 0; slotIndex < m_numParallelSequences; ++slotIndex)
{
ReadSequencesToSlot(slotIndex);
}
}
void TruncatedBPTTPacker::ReadSequencesToSlot(size_t slotIndex)
{
const auto& slot = m_sequenceBufferPerStream.front()->m_slots[slotIndex];
while (m_config.m_truncationSize >= slot.AvailableNumberOfSamples())
const auto& firstStreamSlot = m_sequenceBufferPerStream.front()->m_slots[slotIndex];
while (m_config.m_truncationSize >= firstStreamSlot.AvailableNumberOfSamples())
{
// We need a single sequence, potentially we can request (m_truncationSize - slot.AvailableNumberOfSamples())
// to be more efficient. In reality the truncation size usually is less the sequence size.
// Bptt always operates on a local timeline, so we do not limit the global minibatch count.
auto s = m_sequenceEnumerator->GetNextSequences(SIZE_MAX, 1);
const auto& sequences = m_sequenceEnumerator->GetNextSequences(SIZE_MAX, 1);
// assert that number of input streams == number of output streams --
// this does not have to be the case in general, but the current
// implementation makes this implicit assumption, so let's make it
// explicit instead until we can get rid of it altogether.
assert(sequences.m_endOfEpoch || sequences.m_data.size() == m_outputStreamDescriptions.size());
const auto& data = sequences.m_data;
// Adding sequence to the slot for all streams.
for (size_t i = 0; i < s.m_data.size(); ++i)
for (size_t streamIndex = 0; streamIndex < data.size(); ++streamIndex)
{
assert(s.m_data[i].size() == 1);
assert(data[streamIndex].size() == 1);
const auto& streamSequenceDataVector = data[streamIndex];
auto& slot = m_sequenceBufferPerStream[streamIndex]->m_slots[slotIndex];
// Check that all sequences are of the same length.
if (s.m_data.front().front()->m_numberOfSamples != s.m_data[i].front()->m_numberOfSamples)
if (data.front().front()->m_numberOfSamples != streamSequenceDataVector.front()->m_numberOfSamples)
{
RuntimeError("For BPTT sequences between different input stream should have the same length.");
}
slot.PushSequence(streamSequenceDataVector.front(), sequences.m_endOfSweep);
m_sequenceBufferPerStream[i]->m_slots[slotIndex].PushSequence(s.m_data[i].front());
assert(firstStreamSlot.AvailableNumberOfSamples() == slot.AvailableNumberOfSamples());
}
if (s.m_endOfEpoch)
if (sequences.m_endOfEpoch)
{
break;
return;
}
}
}

11
Source/Readers/ReaderLib/TruncatedBpttPacker.h
Просмотреть файл

@ -30,8 +30,13 @@ public:
virtual void SetConfiguration(const ReaderConfiguration& config, const std::vector<MemoryProviderPtr>& memoryProviders) override;
private:
// Iterates over all (m_parallelNumberOfSequences) slots,
// pulling in and filling out those slots with new sequence data,
// for which AvailableNumberOfSamples (= current size in samples) < m_truncationSize.
void FillOutAvailableSlots();
// Reads sequences to slot with the specified index.
// Number of slots = m_parallelNumberOfSequences
// Number of slots = m_parallelNumberOfSequences.
void ReadSequencesToSlot(size_t slotIndex);
// Packs a slot into the data buffer.
@ -39,7 +44,9 @@ private:
// For each new input, sequence id is reset to 0, and incremented each time
// a sequence is added to the layout. This allows layouts corresponding to different
// inputs to have consistent sequence ids.
void PackSlot(size_t streamIndex, size_t slotIndex, size_t& sequenceId);
// Returns a boolean indicating if a packed data contains a sequence
// (i.e., sequence tail) that was read last in a data sweep.
bool PackSlot(size_t streamIndex, size_t slotIndex, size_t& sequenceId);
virtual MBLayoutPtr CreateMBLayout(const StreamBatch& batch)
{

2
Tests/EndToEndTests/CNTKv2Library/UnitTests/run-test
Просмотреть файл

@ -22,7 +22,7 @@ TestDataDir=$TEST_RUN_DIR/TestData
mkdir $TestDataDir
cp -R $DataSourceDir/MNIST/v0/Train-28x28_cntk_text.txt $TestDataDir || exit $?
cp -R $DataSourceDir/CIFAR/v0/cifar-10-batches-py $TestDataDir || exit $?
cp -R $TEST_DIR/../../Simple2d/Data/SimpleDataTrain_cntk_text.txt $TestDataDir || exit $?
cp -R $TEST_DIR/../../Simple2d/Data/SimpleDataT*_cntk_text.txt $TestDataDir || exit $?
cp -R $TEST_DIR/../../Text/SequenceClassification/Data/Train.ctf $TestDataDir || exit $?
cp -R $TEST_DIR/../../../../Examples/SequenceToSequence/CMUDict/Data/cmudict-0.7b.train-dev-20-21.ctf $TestDataDir || exit $?
cp -R $TEST_DIR/../../../../Examples/Speech/AN4/Data/* $TestDataDir || exit $?

2
Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py
Просмотреть файл

@ -16,7 +16,7 @@ def test_cntk_103_mnist_feedforwardnetwork_noErrors(nb):
for output in cell['outputs'] if output.output_type == "error"]
assert errors == []
expectedEvalErrorByDeviceId = { -1: 1.90, 0: 1.85 }
expectedEvalErrorByDeviceId = { -1: 1.67, 0: 1.71 }
def test_cntk_103_mnist_feedforwardnetwork_evalCorrect(nb, device_id):
testCell = [cell for cell in nb.cells

2
Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_202_Language_Understanding_test.py
Просмотреть файл

@ -31,6 +31,6 @@ def test_cntk_202_language_understanding_trainerror(nb):
pass
except KeyError:
pass
expectedMetrics = [2.8, 1.9, 2.2, 2.3]
expectedMetrics = [2.8, 1.9, 2.2, 2.0]
# TODO tighten tolerances
assert numpy.allclose(expectedMetrics, metrics, atol=0.2)

1
Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py
Просмотреть файл

@ -9,6 +9,7 @@ import re
abs_path = os.path.dirname(os.path.abspath(__file__))
notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_203_Reinforcement_Learning_Basics.ipynb")
notebook_timeoutSeconds = 450
def test_cntk_203_reinforcement_learning_basics_noErrors(nb):
errors = [output for cell in nb.cells if 'outputs' in cell

6
Tests/EndToEndTests/TestDriver.py
Просмотреть файл

@ -398,10 +398,10 @@ class Test:
'Quadro M2000M': 5,
'Quadro M4000': 5,
}
cc = sys.maxint
cc = sys.maxsize
try:
gpuList = subprocess.check_output([nvidiaSmiPath, '-L'])
for line in gpuList.split('\n'):
for line in gpuList.decode('utf-8').split('\n'):
m = re.match(r"GPU (?P<id>\d+): (?P<type>[^(]*) \(UUID: (?P<guid>GPU-.*)\)\r?$", line)
if m:
try:
@ -412,7 +412,7 @@ class Test:
pass
except OSError:
pass
if cc != sys.maxint:
if cc != sys.maxsize:
self.gpuBaselinePatternList.insert(0, ".gpu.cc" + str(cc))
return self.gpuBaselinePatternList

1
Tests/EndToEndTests/TestDriverTest/EmptyTest/baseline.txt Normal file
Просмотреть файл

@ -0,0 +1 @@
__COMPLETED__

1
Tests/EndToEndTests/TestDriverTest/EmptyTest/run-test Normal file
Просмотреть файл

@ -0,0 +1 @@
echo __COMPLETED__

10
Tests/EndToEndTests/TestDriverTest/EmptyTest/testcases.yml Normal file
Просмотреть файл

@ -0,0 +1,10 @@
dataDir: .
# Note: no tags, so this test is not being picked up by normal test run,
# instead only by the ./TestDriver test
tags:
testCases:
Must output __COMPLETED__:
patterns:
- __COMPLETED__

13
Tests/EndToEndTests/TestDriverTest/baseline.txt Normal file
Просмотреть файл

@ -0,0 +1,13 @@
+ ./TestDriver.py run -s gpu -d gpu -f release TestDriverTest/EmptyTest
CNTK Test Driver is started
Running tests: TestDriverTest/EmptyTest
Build location: /cygdrive/c/Users/mahilleb/Repos/CNTK2/x64
Build SKU: gpu
Run location: /tmp/cntk-test-20170118204425.248953
Flavors: release
Devices: gpu
Running test TestDriverTest/EmptyTest (release gpu) - [OK] 0.10 sec
1/1 tests passed, 0 failed
+ echo __COMPLETED__
__COMPLETED__

8
Tests/EndToEndTests/TestDriverTest/run-test Normal file
Просмотреть файл

@ -0,0 +1,8 @@
#!/bin/bash
set -x -e -o pipefail
# Note: must match current test, cf. testcases.yml
./TestDriver.py run -s gpu -d gpu -f release TestDriverTest/EmptyTest
echo __COMPLETED__

12
Tests/EndToEndTests/TestDriverTest/testcases.yml Normal file
Просмотреть файл

@ -0,0 +1,12 @@
isPythonTest: True
dataDir: ..
tags:
- bvt-e (build_sku == 'gpu') and (device == 'gpu') and (flavor == 'release')
- nightly-e (build_sku == 'gpu') and (device == 'gpu') and (flavor == 'release')
testCases:
PyTest run must finish with error code 0 (outputs __COMPLETED__ in that case):
patterns:
- __COMPLETED__

37
Tests/EndToEndTests/UnitTests/ReaderTests/baseline.txt
Просмотреть файл

@ -1266,8 +1266,8 @@ Legacy configuration is used for truncated BPTT mode, please adapt the config to
Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests
Test module "ReaderTests" has passed with:
122 test cases out of 122 passed
22235579 assertions out of 22235579 passed
127 test cases out of 127 passed
22629927 assertions out of 22629927 passed
Test suite "ReaderTestSuite" has passed with:
92 test cases out of 92 passed
@ -1566,11 +1566,8 @@ Test module "ReaderTests" has passed with:
2 assertions out of 2 passed
Test suite "ReaderLibTests" has passed with:
18 test cases out of 18 passed
2265 assertions out of 2265 passed
Test case "ReaderLibTests/CheckGetCurrentCursorForRandomizers" has passed with:
16 assertions out of 16 passed
23 test cases out of 23 passed
396613 assertions out of 396613 passed
Test case "ReaderLibTests/CheckSetCurrentCursorForRandomizers" has passed with:
12 assertions out of 12 passed
@ -1589,8 +1586,23 @@ Test module "ReaderTests" has passed with:
Test case "ReaderLibTests/BlockRandomizerInstantiate" has passed
Test case "ReaderLibTests/BlockRandomizerOneEpoch" has passed with:
68 assertions out of 68 passed
Test case "ReaderLibTests/TestRandomization_FirstEpoch" has passed with:
1476 assertions out of 1476 passed
Test case "ReaderLibTests/TestRandomization_SecondEpoch" has passed with:
1476 assertions out of 1476 passed
Test case "ReaderLibTests/TestRandomization_TwoSweeps" has passed with:
3372 assertions out of 3372 passed
Test case "ReaderLibTests/TestRandomization_TwoSweeps_WithSequences" has passed with:
196668 assertions out of 196668 passed
Test case "ReaderLibTests/TestRandomization_TwoSweeps_AllowToCrossSweepBoundary" has passed with:
3204 assertions out of 3204 passed
Test case "ReaderLibTests/TestRandomization_TwoSweeps_AllowToCrossSweepBoundary_WithSequences" has passed with:
189756 assertions out of 189756 passed
Test case "ReaderLibTests/BlockRandomizerOneEpochWithChunks1" has passed with:
68 assertions out of 68 passed
@ -1598,8 +1610,8 @@ Test module "ReaderTests" has passed with:
Test case "ReaderLibTests/BlockRandomizerOneEpochWithChunks2" has passed with:
128 assertions out of 128 passed
Test case "ReaderLibTests/BlockRandomizerChaosMonkey" has passed with:
1836 assertions out of 1836 passed
Test case "ReaderLibTests/RandomizerChaosMonkey" has passed with:
300 assertions out of 300 passed
Test case "ReaderLibTests/BlockRandomizerOneEpochLegacyRandomization" has passed with:
68 assertions out of 68 passed
@ -1607,6 +1619,9 @@ Test module "ReaderTests" has passed with:
Test case "ReaderLibTests/NoRandomizerOneEpoch" has passed with:
35 assertions out of 35 passed
Test case "ReaderLibTests/CheckGetCurrentCursorForRandomizers" has passed with:
16 assertions out of 16 passed
Test case "ReaderLibTests/DefaultCorpusDescriptor" has passed with:
2 assertions out of 2 passed

425
Tests/UnitTests/ReaderTests/ReaderLibTests.cpp
Просмотреть файл

@ -2,10 +2,10 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
//
#include "stdafx.h"
#include <numeric>
#include <random>
#include "NoRandomizer.h"
#include "DataDeserializer.h"
#include "BlockRandomizer.h"
@ -79,7 +79,7 @@ private:
vector<vector<float>> m_sequenceData;
public:
MockDeserializer(size_t numChunks, size_t numSequencesPerChunks, vector<float>& data, uint32_t sequenceLength = 1)
MockDeserializer(size_t numChunks, size_t numSequencesPerChunks, const vector<float>& data, uint32_t sequenceLength = 1)
: m_numChunks(numChunks),
m_numSequencesPerChunk(numSequencesPerChunks),
m_sampleLayout(make_shared<TensorShape>(1)),
@ -170,49 +170,7 @@ void BlockRandomizerInstantiateTest(bool prefetch)
{
vector<float> data;
auto mockDeserializer = make_shared<MockDeserializer>(0, 0, data);
auto randomizer = make_shared<BlockRandomizer>(0, SIZE_MAX, mockDeserializer, prefetch);
}
BOOST_AUTO_TEST_CASE(CheckGetCurrentCursorForRandomizers)
{
size_t chunkSizeInSamples = 10000;
size_t sweepNumberOfSamples = 500000;
uint32_t maxSequenceLength = 300;
size_t randomizationWindow = chunkSizeInSamples * 5;
auto deserializer = make_shared<SequentialDeserializer>(0, chunkSizeInSamples, sweepNumberOfSamples, maxSequenceLength);
auto blockRandomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true);
auto noRandomizer = make_shared<NoRandomizer>(deserializer, false);
auto test = [](SequenceEnumeratorPtr r, size_t epochSize)
{
auto firstEpoch = ReadFullEpoch(r, epochSize, 0);
auto firstCursor = r->GetCurrentSamplePosition();
BOOST_CHECK_EQUAL(firstCursor, firstEpoch.size());
auto secondEpoch = ReadFullEpoch(r, epochSize, 1);
auto secondCursor = r->GetCurrentSamplePosition();
BOOST_CHECK_EQUAL(secondCursor - firstCursor, secondEpoch.size());
auto thirdEpoch = ReadFullEpoch(r, epochSize, 2);
auto thirdCursor = r->GetCurrentSamplePosition();
BOOST_CHECK_EQUAL(thirdCursor - secondCursor, thirdEpoch.size());
auto anotherSecondEpoch = ReadFullEpoch(r, epochSize, 1);
auto anotherSecondCursor = r->GetCurrentSamplePosition();
BOOST_CHECK_EQUAL(anotherSecondCursor, secondCursor);
};
// Inside sweep
size_t epochSize = 50000;
test(blockRandomizer, epochSize);
test(noRandomizer, epochSize);
// Between sweeps
epochSize = (size_t)(sweepNumberOfSamples / 1.5);
test(blockRandomizer, epochSize);
test(noRandomizer, epochSize);
auto randomizer = make_shared<BlockRandomizer>(0, SIZE_MAX, mockDeserializer, prefetch, false);
}
BOOST_AUTO_TEST_CASE(CheckSetCurrentCursorForRandomizers)
@ -223,11 +181,11 @@ BOOST_AUTO_TEST_CASE(CheckSetCurrentCursorForRandomizers)
size_t randomizationWindow = chunkSizeInSamples * 5;
auto deserializer = make_shared<SequentialDeserializer>(0, chunkSizeInSamples, sweepNumberOfSamples, maxSequenceLength);
auto expectedBlock = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true);
auto expectedNo = make_shared<NoRandomizer>(deserializer);
auto expectedBlock = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true, false);
auto expectedNo = make_shared<NoRandomizer>(deserializer, false);
auto underTestBlock = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true);
auto unterTestNo = make_shared<NoRandomizer>(deserializer);
auto underTestBlock = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true, false);
auto unterTestNo = make_shared<NoRandomizer>(deserializer, false);
auto test = [](SequenceEnumeratorPtr expected, SequenceEnumeratorPtr underTest, size_t epochSize)
{
@ -292,7 +250,7 @@ BOOST_AUTO_TEST_CASE(RandRollbackToEarlierEpochBetweenSweeps)
auto deserializer = make_shared<SequentialDeserializer>(0, chunkSizeInSamples, sweepNumberOfSamples, maxSequenceLength);
// Let's randomize complete sweep, so that we have a baseline.
auto randomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true);
auto randomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true, false);
// Let's read all sequences from the first three sweeps in the randomized order.
auto firstSweep = ReadFullSweep(randomizer, 0, sweepNumberOfSamples);
@ -330,7 +288,7 @@ BOOST_AUTO_TEST_CASE(RandRollbackToEarlierEpochInTheSweep)
auto deserializer = make_shared<SequentialDeserializer>(0, chunkSizeInSamples, sweepNumberOfSamples, maxSequenceLength);
// Let's randomize complete sweep, so that we have a baseline.
auto randomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true);
auto randomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true, false);
// Let's read all sequences from the first three sweeps in the randomized order.
auto firstSweep = ReadFullSweep(randomizer, 0, sweepNumberOfSamples);
@ -361,7 +319,7 @@ BOOST_AUTO_TEST_CASE(RandRollbackToSameEpochInTheSweep)
auto deserializer = make_shared<SequentialDeserializer>(0, chunkSizeInSamples, sweepNumberOfSamples, maxSequenceLength);
// Let's randomize complete sweep, so that we have a baseline.
auto randomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true);
auto randomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true, false);
// Let's read all sequences from the first three sweeps in the randomized order.
auto firstSweep = ReadFullSweep(randomizer, 0, sweepNumberOfSamples);
@ -388,7 +346,7 @@ BOOST_AUTO_TEST_CASE(RandRollbackToSameEpochInBigRandomizationWindow)
auto deserializer = make_shared<SequentialDeserializer>(0, chunkSizeInSamples, sweepNumberOfSamples, maxSequenceLength);
// Let's randomize complete sweep, so that we have a baseline.
auto randomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true);
auto randomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true, false);
// Let's read all sequences from the first three sweeps in the randomized order.
auto firstSweep = ReadFullSweep(randomizer, 0, sweepNumberOfSamples);
@ -421,45 +379,221 @@ BOOST_AUTO_TEST_CASE(BlockRandomizerInstantiate)
BlockRandomizerInstantiateTest(true);
}
void BlockRandomizerOneEpochTest(bool prefetch)
void OneEpochRandomizationTest(SequenceEnumerator& randomizer, size_t sweepSize, const EpochConfiguration& epochConfig, const vector<float>& expectedOutput, size_t sequenceLength = 1)
{
vector<float> data(10);
iota(data.begin(), data.end(), 0.0f);
auto mockDeserializer = make_shared<MockDeserializer>(5, 2, data);
auto epochSize = epochConfig.m_totalEpochSizeInSamples;
auto mbSize = epochConfig.m_minibatchSizeInSamples;
auto randomizer = make_shared<BlockRandomizer>(0, SIZE_MAX, mockDeserializer, prefetch);
BOOST_ASSERT(epochSize == expectedOutput.size());
EpochConfiguration epochConfiguration;
epochConfiguration.m_numberOfWorkers = 1;
epochConfiguration.m_workerRank = 0;
epochConfiguration.m_minibatchSizeInSamples = 0;
epochConfiguration.m_totalEpochSizeInSamples = data.size();
epochConfiguration.m_epochIndex = 0;
randomizer->StartEpoch(epochConfiguration);
randomizer.StartEpoch(epochConfig);
vector<float> expected { 6, 3, 1, 5, 9, 0, 4, 2, 7, 8 };
BOOST_CHECK_EQUAL(data.size(), expected.size());
vector<float> actual;
for (int i = 0; i < data.size() + 1; i++)
for (int totalSamplesRead = 0; totalSamplesRead < epochSize;)
{
Sequences sequences = randomizer->GetNextSequences(1, 1);
BOOST_CHECK_EQUAL(sequences.m_data.size(), 1 - (i / data.size()));
if (i < data.size())
Sequences sequences = randomizer.GetNextSequences(mbSize, mbSize);
BOOST_ASSERT(sequences.m_data.size() == 1); // only one input stream
auto& stream = sequences.m_data[0];
auto numSampleRead = 0;
for (auto& sequence : stream)
{
auto& data2 = reinterpret_cast<DenseSequenceData&>(*sequences.m_data[0][0]);
BOOST_CHECK_EQUAL(data2.m_numberOfSamples, 1u);
actual.push_back(*((float*)data2.GetDataBuffer()));
auto numSamples = sequence->m_numberOfSamples;
numSampleRead += numSamples;
auto& data = reinterpret_cast<DenseSequenceData&>(*sequence);
actual.reserve(actual.size() + numSamples);
std::copy_n(((float*)data.GetDataBuffer()), numSamples, std::back_inserter(actual));
}
BOOST_CHECK_EQUAL(sequences.m_endOfEpoch, (data.size() <= i + 1));
auto expectedSize = std::min(epochSize - totalSamplesRead, mbSize);
if (!epochConfig.m_allowMinibatchesToCrossSweepBoundaries)
{
expectedSize = std::min(sweepSize - totalSamplesRead % sweepSize, expectedSize);
}
// at least one sequence is returned in case when mbSize < sequenceLength
expectedSize = std::max(expectedSize, sequenceLength);
BOOST_REQUIRE(numSampleRead <= std::max(mbSize, sequenceLength));
if (sequenceLength == 1)
BOOST_REQUIRE(numSampleRead == expectedSize);
else
BOOST_REQUIRE(expectedSize - numSampleRead < sequenceLength);
BOOST_REQUIRE(sequences.m_endOfEpoch == (totalSamplesRead + numSampleRead == epochSize));
BOOST_REQUIRE(sequences.m_endOfSweep == (totalSamplesRead / sweepSize != (totalSamplesRead + numSampleRead) / sweepSize));
totalSamplesRead += numSampleRead;
}
BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(),
for (int i = 0; i < 3; i++)
{
auto numSamples = i + 1;
Sequences sequences = randomizer.GetNextSequences(numSamples, numSamples);
BOOST_REQUIRE(sequences.m_data.size() == 0);
BOOST_REQUIRE(sequences.m_endOfEpoch == true);
BOOST_REQUIRE(sequences.m_endOfSweep == (epochSize % sweepSize == 0));
}
BOOST_REQUIRE_EQUAL_COLLECTIONS(expectedOutput.begin(), expectedOutput.end(),
actual.begin(), actual.end());
}
BOOST_AUTO_TEST_CASE(BlockRandomizerOneEpoch)
void TestRandomization(EpochConfiguration& epochConfiguration, IDataDeserializerPtr deserializer, size_t sweepSize, const vector<float>& expectedRandomized, const vector<float>& expectedNotRandomized, size_t sequenceLength = 1)
{
BlockRandomizerOneEpochTest(false);
BlockRandomizerOneEpochTest(true);
BlockRandomizer randomizer1(0, SIZE_MAX, deserializer, /*prefetch =*/ false);
BlockRandomizer randomizer2(0, SIZE_MAX, deserializer, /*prefetch =*/ true);
NoRandomizer randomizer3(deserializer);
BlockRandomizer randomizer4(0, SIZE_MAX, deserializer, /*prefetch =*/ false, false, /*multithreadedGetNextSequences =*/ true);
BlockRandomizer randomizer5(0, SIZE_MAX, deserializer, /*prefetch =*/ true, false, /*multithreadedGetNextSequences =*/ true);
NoRandomizer randomizer6(deserializer, /*multithreadedGetNextSequences =*/ true);
epochConfiguration.m_numberOfWorkers = 1;
epochConfiguration.m_workerRank = 0;
epochConfiguration.m_totalEpochSizeInSamples = expectedRandomized.size();
for (int i = 1; i <= epochConfiguration.m_totalEpochSizeInSamples + 1; i++)
{
epochConfiguration.m_minibatchSizeInSamples = i;
OneEpochRandomizationTest(randomizer1, sweepSize, epochConfiguration, expectedRandomized, sequenceLength);
OneEpochRandomizationTest(randomizer2, sweepSize, epochConfiguration, expectedRandomized, sequenceLength);
OneEpochRandomizationTest(randomizer3, sweepSize, epochConfiguration, expectedNotRandomized, sequenceLength);
OneEpochRandomizationTest(randomizer4, sweepSize, epochConfiguration, expectedRandomized, sequenceLength);
OneEpochRandomizationTest(randomizer5, sweepSize, epochConfiguration, expectedRandomized, sequenceLength);
OneEpochRandomizationTest(randomizer6, sweepSize, epochConfiguration, expectedNotRandomized, sequenceLength);
}
}
BOOST_AUTO_TEST_CASE(TestRandomization_FirstEpoch)
{
vector<float> data(10);
iota(data.begin(), data.end(), 0.0f);
vector<float> expected{ 6, 3, 1, 5, 9, 0, 4, 2, 7, 8 };
auto mockDeserializer = make_shared<MockDeserializer>(5, 2, data);
EpochConfiguration epochConfiguration;
epochConfiguration.m_epochIndex = 0;
TestRandomization(epochConfiguration, mockDeserializer, data.size(), expected, data);
}
BOOST_AUTO_TEST_CASE(TestRandomization_SecondEpoch)
{
vector<float> data(10);
iota(data.begin(), data.end(), 0.0f);
vector<float> expected{ 3, 0, 8, 4, 7, 5, 2, 9, 1, 6 };
auto mockDeserializer = make_shared<MockDeserializer>(5, 2, data);
EpochConfiguration epochConfiguration;
epochConfiguration.m_epochIndex = 1;
TestRandomization(epochConfiguration, mockDeserializer, data.size(), expected, data);
}
BOOST_AUTO_TEST_CASE(TestRandomization_TwoSweeps)
{
vector<float> data(10);
iota(data.begin(), data.end(), 0.0f);
vector<float> expected{ 6, 3, 1, 5, 9, 0, 4, 2, 7, 8, 3, 0, 8, 4, 7, 5, 2, 9, 1, 6 };
auto mockDeserializer = make_shared<MockDeserializer>(5, 2, data);
auto sweepSize = data.size();
data.reserve(2 * sweepSize);
std::copy_n(data.begin(), sweepSize, std::back_inserter(data));
EpochConfiguration epochConfiguration;
epochConfiguration.m_epochIndex = 0;
TestRandomization(epochConfiguration, mockDeserializer, sweepSize, expected, data);
}
BOOST_AUTO_TEST_CASE(TestRandomization_TwoSweeps_WithSequences)
{
vector<float> data(10);
iota(data.begin(), data.end(), 0.0f);
vector<float> expected{ 6, 3, 1, 5, 9, 0, 4, 2, 7, 8, 3, 0, 8, 4, 7, 5, 2, 9, 1, 6 };
for (int seqLength = 2; seqLength <= 10; seqLength++)
{
vector<float> expectedRandomized;
vector<float> expectedNotRandomized;
for (auto f : expected) {
std::fill_n(back_inserter(expectedRandomized), seqLength, f);
}
for (int i = 0; i < 2 * data.size(); i++) {
std::fill_n(back_inserter(expectedNotRandomized), seqLength, data[i % data.size()]);
}
auto mockDeserializer = make_shared<MockDeserializer>(5, 2, data, seqLength);
auto sweepSize = data.size() * seqLength;
EpochConfiguration epochConfiguration;
epochConfiguration.m_epochIndex = 0;
TestRandomization(epochConfiguration, mockDeserializer, sweepSize, expectedRandomized, expectedNotRandomized, seqLength);
}
}
BOOST_AUTO_TEST_CASE(TestRandomization_TwoSweeps_AllowToCrossSweepBoundary)
{
vector<float> data(10);
iota(data.begin(), data.end(), 0.0f);
vector<float> expected{ 6, 3, 1, 5, 9, 0, 4, 2, 7, 8, 3, 0, 8, 4, 7, 5, 2, 9, 1, 6 };
auto mockDeserializer = make_shared<MockDeserializer>(5, 2, data);
auto sweepSize = data.size();
data.reserve(2 * sweepSize);
std::copy_n(data.begin(), sweepSize, std::back_inserter(data));
EpochConfiguration epochConfiguration;
epochConfiguration.m_epochIndex = 0;
epochConfiguration.m_allowMinibatchesToCrossSweepBoundaries = true;
TestRandomization(epochConfiguration, mockDeserializer, sweepSize, expected, data);
}
BOOST_AUTO_TEST_CASE(TestRandomization_TwoSweeps_AllowToCrossSweepBoundary_WithSequences)
{
vector<float> data(10);
iota(data.begin(), data.end(), 0.0f);
vector<float> expected{ 6, 3, 1, 5, 9, 0, 4, 2, 7, 8, 3, 0, 8, 4, 7, 5, 2, 9, 1, 6 };
for (int seqLength = 2; seqLength <= 10; seqLength++)
{
vector<float> expectedRandomized;
vector<float> expectedNotRandomized;
for (auto f : expected) {
std::fill_n(back_inserter(expectedRandomized), seqLength, f);
}
for (int i = 0; i < 2 * data.size(); i++) {
std::fill_n(back_inserter(expectedNotRandomized), seqLength, data[i % data.size()]);
}
auto mockDeserializer = make_shared<MockDeserializer>(5, 2, data, seqLength);
auto sweepSize = data.size() * seqLength;
EpochConfiguration epochConfiguration;
epochConfiguration.m_epochIndex = 0;
epochConfiguration.m_allowMinibatchesToCrossSweepBoundaries = true;
TestRandomization(epochConfiguration, mockDeserializer, sweepSize, expectedRandomized, expectedNotRandomized, seqLength);
}
}
void BlockRandomizerOneEpochWithChunks1Test(bool prefetch)
@ -468,7 +602,7 @@ void BlockRandomizerOneEpochWithChunks1Test(bool prefetch)
iota(data.begin(), data.end(), 0.0f);
auto mockDeserializer = make_shared<MockDeserializer>(5, 2, data);
auto randomizer = make_shared<BlockRandomizer>(0, 4, mockDeserializer, prefetch);
auto randomizer = make_shared<BlockRandomizer>(0, 4, mockDeserializer, prefetch, false);
EpochConfiguration epochConfiguration;
epochConfiguration.m_numberOfWorkers = 1;
@ -510,7 +644,7 @@ void BlockRandomizerOneEpochWithChunks2Test(bool prefetch)
auto mockDeserializer = make_shared<MockDeserializer>(10, 2, data);
auto randomizer = make_shared<BlockRandomizer>(0, 18, mockDeserializer, prefetch);
auto randomizer = make_shared<BlockRandomizer>(0, 18, mockDeserializer, prefetch, false);
EpochConfiguration epochConfiguration;
epochConfiguration.m_numberOfWorkers = 1;
@ -548,65 +682,75 @@ BOOST_AUTO_TEST_CASE(BlockRandomizerOneEpochWithChunks2)
BlockRandomizerOneEpochWithChunks2Test(true);
}
void BlockRandomizerChaosMonkeyTest(bool prefetch)
void RandomizerChaosMonkeyTest(SequenceEnumerator& randomizer, size_t sweepSize, int seed)
{
const int sequenceLength = 3;
const int seed = 42;
const int numChunks = 100;
const int numSequencesPerChunk = 10;
const int windowSize = 18;
vector<float> data(numChunks * numSequencesPerChunk);
iota(data.begin(), data.end(), 0.0f);
std::mt19937 rng(seed);
boost::random::uniform_int_distribution<int> distr(1, 10);
auto mockDeserializer = make_shared<MockDeserializer>(numChunks, numSequencesPerChunk, data, sequenceLength);
auto randomizer = make_shared<BlockRandomizer>(0, windowSize, mockDeserializer, prefetch);
boost::random::uniform_int_distribution<int> distr(1, 100);
for (int t = 0; t < 100; t++)
{
EpochConfiguration epochConfiguration;
epochConfiguration.m_numberOfWorkers = distr(rng);
do
{
epochConfiguration.m_workerRank = distr(rng) - 1;
}
while (epochConfiguration.m_numberOfWorkers <= epochConfiguration.m_workerRank);
epochConfiguration.m_workerRank = distr(rng) % epochConfiguration.m_numberOfWorkers;
epochConfiguration.m_minibatchSizeInSamples = 0; // don't care
epochConfiguration.m_totalEpochSizeInSamples = data.size() / distr(rng);
epochConfiguration.m_totalEpochSizeInSamples = sweepSize * distr(rng) / distr(rng);
epochConfiguration.m_epochIndex = distr(rng);
randomizer->StartEpoch(epochConfiguration);
epochConfiguration.m_allowMinibatchesToCrossSweepBoundaries = (distr(rng) % 2 == 0);
randomizer.StartEpoch(epochConfiguration);
auto epochStart = epochConfiguration.m_epochIndex * epochConfiguration.m_totalEpochSizeInSamples;
auto epochEnd = epochStart + epochConfiguration.m_totalEpochSizeInSamples;
auto numSweeps = epochEnd / sweepSize - epochStart / sweepSize;
auto sweepCount = 0;
int samplesToGet = 0;
for (int i = 0; i < epochConfiguration.m_totalEpochSizeInSamples + 1; i += samplesToGet)
for (;;)
{
samplesToGet = distr(rng);
Sequences sequences = randomizer->GetNextSequences(samplesToGet, samplesToGet);
Sequences sequences = randomizer.GetNextSequences(samplesToGet, samplesToGet);
if (sequences.m_endOfSweep)
sweepCount++;
// In case end of epoch/decimation/single sequence -> skip the mbSize check.
if (sequences.m_endOfEpoch || sequences.m_data.empty() || sequences.m_data.front().size() < 2)
if (!(sequences.m_data.empty() || sequences.m_data.size() == 1))
{
continue;
// Check that we do not exceed the minibatch size.
size_t count = 0;
for (const auto& sequence : sequences.m_data.front())
{
count += sequence->m_numberOfSamples;
}
BOOST_REQUIRE_LE(count, samplesToGet);
}
// Check that we do not exceed the minibatch size.
size_t count = 0;
for (const auto& sequence : sequences.m_data.front())
{
count += sequence->m_numberOfSamples;
}
BOOST_CHECK_LE(count, samplesToGet);
if (sequences.m_endOfEpoch)
break;
}
BOOST_REQUIRE(sweepCount == numSweeps);
}
}
BOOST_AUTO_TEST_CASE(BlockRandomizerChaosMonkey)
BOOST_AUTO_TEST_CASE(RandomizerChaosMonkey)
{
BlockRandomizerChaosMonkeyTest(false);
BlockRandomizerChaosMonkeyTest(true);
const int sequenceLength = 3;
const int numChunks = 100;
const int numSequencesPerChunk = 10;
const int windowSize = 18;
vector<float> data(numChunks * numSequencesPerChunk);
iota(data.begin(), data.end(), 0.0f);
auto mockDeserializer = make_shared<MockDeserializer>(numChunks, numSequencesPerChunk, data, sequenceLength);
BlockRandomizer blockRandomizerNoPrefetch(0, windowSize, mockDeserializer, false, false);
BlockRandomizer blockRandomizerWithPrefetch(0, windowSize, mockDeserializer, true, false);
NoRandomizer norandomizer(mockDeserializer);
auto sweepSize = data.size() * sequenceLength;
RandomizerChaosMonkeyTest(blockRandomizerNoPrefetch, sweepSize, 42);
RandomizerChaosMonkeyTest(blockRandomizerWithPrefetch, sweepSize, 43);
RandomizerChaosMonkeyTest(norandomizer, sweepSize, 44);
}
void BlockRandomizerOneEpochLegacyRandomizationTest(bool prefetch)
@ -618,7 +762,8 @@ void BlockRandomizerOneEpochLegacyRandomizationTest(bool prefetch)
auto randomizer = make_shared<BlockRandomizer>(0,
SIZE_MAX,
mockDeserializer,
prefetch);
prefetch,
true);
EpochConfiguration epochConfiguration;
epochConfiguration.m_numberOfWorkers = 1;
@ -691,6 +836,48 @@ BOOST_AUTO_TEST_CASE(NoRandomizerOneEpoch)
actual.begin(), actual.end());
}
BOOST_AUTO_TEST_CASE(CheckGetCurrentCursorForRandomizers)
{
size_t chunkSizeInSamples = 10000;
size_t sweepNumberOfSamples = 500000;
uint32_t maxSequenceLength = 300;
size_t randomizationWindow = chunkSizeInSamples * 5;
auto deserializer = make_shared<SequentialDeserializer>(0, chunkSizeInSamples, sweepNumberOfSamples, maxSequenceLength);
auto blockRandomizer = make_shared<BlockRandomizer>(0, randomizationWindow, deserializer, true, false);
auto noRandomizer = make_shared<NoRandomizer>(deserializer, false);
auto test = [](SequenceEnumeratorPtr r, size_t epochSize)
{
auto firstEpoch = ReadFullEpoch(r, epochSize, 0);
auto firstCursor = r->GetCurrentSamplePosition();
BOOST_CHECK_EQUAL(firstCursor, firstEpoch.size());
auto secondEpoch = ReadFullEpoch(r, epochSize, 1);
auto secondCursor = r->GetCurrentSamplePosition();
BOOST_CHECK_EQUAL(secondCursor - firstCursor, secondEpoch.size());
auto thirdEpoch = ReadFullEpoch(r, epochSize, 2);
auto thirdCursor = r->GetCurrentSamplePosition();
BOOST_CHECK_EQUAL(thirdCursor - secondCursor, thirdEpoch.size());
auto anotherSecondEpoch = ReadFullEpoch(r, epochSize, 1);
auto anotherSecondCursor = r->GetCurrentSamplePosition();
BOOST_CHECK_EQUAL(anotherSecondCursor, secondCursor);
};
// Inside sweep
size_t epochSize = 50000;
test(blockRandomizer, epochSize);
test(noRandomizer, epochSize);
// Between sweeps
epochSize = (size_t)(sweepNumberOfSamples / 1.5);
test(blockRandomizer, epochSize);
test(noRandomizer, epochSize);
}
BOOST_AUTO_TEST_CASE(DefaultCorpusDescriptor)
{
const int seed = 13;
@ -713,8 +900,8 @@ BOOST_AUTO_TEST_CASE(NumericCorpusDescriptor)
CorpusDescriptor corpus(true);
for (int i = 0; i < 10; ++i)
{
auto value = distr(rng);
BOOST_CHECK_EQUAL(value, corpus.KeyToId(std::to_string(value)));
auto value = distr(rng);
BOOST_CHECK_EQUAL(value, corpus.KeyToId(std::to_string(value)));
}
BOOST_CHECK_EXCEPTION(
corpus.KeyToId("not a number"),

2
Tests/UnitTests/V2LibraryTests/CifarResNet.cpp
Просмотреть файл

@ -166,7 +166,7 @@ void TrainResNetCifarClassifer(const DeviceDescriptor& device, bool testSaveAndR
for (size_t i = 0; i < numMinibatchesToTrain; ++i)
{
auto minibatchData = minibatchSource->GetNextMinibatch(minibatchSize, device);
trainer->TrainMinibatch({ { imageInput, minibatchData[imageStreamInfo].m_data }, { labelsVar, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { imageInput, minibatchData[imageStreamInfo] }, { labelsVar, minibatchData[labelStreamInfo] } }, device);
PrintTrainingProgress(trainer, i, outputFrequencyInMinibatches);
}
}

69
Tests/UnitTests/V2LibraryTests/LearnerTests.cpp
Просмотреть файл

@ -106,10 +106,6 @@ void TestRMSPropLearner(size_t numParameters, size_t numMinibatches, const Devic
void TestTrainingParametersSchedule()
{
VerifyException([]() {
LearningRatePerMinibatchSchedule({ 3.0, 2.0, 1.0 }, LearningRateSchedule::EntireSweep);
}, "Was able to create not-yet-implemented sweep-based schedule.");
LearningRatePerSampleSchedule schedule1 = 0.5;
assert(schedule1.Unit() == LearningRateSchedule::UnitType::Sample);
assert(schedule1[0] == 0.5);
@ -229,11 +225,76 @@ void TestTrainingParametersSchedule()
}
void TestSweepBasedSchedule()
{
DeviceDescriptor device = DeviceDescriptor::CPUDevice();
auto schedule = LearningRatePerSampleSchedule({ 1, 2, 3, 4, 5 }, LearningRateSchedule::FullDataSweep);
auto learner1 = SGDLearner({}, schedule);
assert(1 == learner1->LearningRate());
for (auto i : {2, 3, 4, 5 })
{
std::unordered_map<Parameter, NDArrayViewPtr> gradients {};
learner1->Update(gradients, 1, true);
assert(i == learner1->LearningRate());
}
const size_t inputDim = 2;
const size_t numOutputClasses = 2;
auto minibatchSource = TextFormatMinibatchSource(L"SimpleDataTest_cntk_text.txt", { { L"features", inputDim }, { L"labels", numOutputClasses } });
auto sweepSize = 603; // == wc -l SimpleDataTest_cntk_text.txt
auto minibatchSize = 400;
auto featureStreamInfo = minibatchSource->StreamInfo(L"features");
auto labelStreamInfo = minibatchSource->StreamInfo(L"labels");
auto input = InputVariable({ inputDim }, DataType::Float, L"features");
auto labels = InputVariable({ numOutputClasses }, DataType::Float, L"labels");
auto classifierOutput = FullyConnectedLinearLayer(input, numOutputClasses, device);
auto trainingLoss = CNTK::CrossEntropyWithSoftmax(classifierOutput, labels, L"lossFunction");
auto prediction = CNTK::ClassificationError(classifierOutput, labels, L"classificationError");
auto learner2 = SGDLearner(classifierOutput->Parameters(), schedule);
auto trainer = CreateTrainer(classifierOutput, trainingLoss, prediction, { learner2 });
for (auto i = 0; i <= 4000; i+= minibatchSize)
{
auto sweepIndex1 = i / sweepSize;
auto minibatchData = minibatchSource->GetNextMinibatch(minibatchSize, device);
if (minibatchData[featureStreamInfo].sweepEnd != minibatchData[labelStreamInfo].sweepEnd) {
ReportFailure("TestSweepBasedSchedule failed: "
"different streams have different end of sweep flag values.");
}
auto sweepIndex2 = (i + minibatchSize) / sweepSize;
if ((sweepIndex1 != sweepIndex2) != minibatchData[labelStreamInfo].sweepEnd) {
ReportFailure("TestSweepBasedSchedule failed: "
"end of sweep flag value is different from expected.");
}
trainer->TrainMinibatch({ { input, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
auto expectedLR = std::min((sweepIndex2 + 1), 5);
if (expectedLR != learner2->LearningRate()) {
ReportFailure("TestSweepBasedSchedule failed: "
"learning rate value is different from expected.");
}
}
}
void LearnerTests()
{
fprintf(stderr, "\nLearnerTests..\n");
TestTrainingParametersSchedule();
TestSweepBasedSchedule();
vector<DeviceDescriptor> devices{DeviceDescriptor::CPUDevice()};

89
Tests/UnitTests/V2LibraryTests/MinibatchSourceTest.cpp
Просмотреть файл

@ -159,11 +159,11 @@ void TestMinibatchSourceWarmStart(size_t minibatchSize, size_t warmStartSamples,
auto minibatchData = minibatchSource->GetNextMinibatch(0, minibatchSize, 1, 0);
auto minibatchData2 = minibatchSource2->GetNextMinibatch(0, minibatchSize, 1, 0);
if (minibatchData[featureStreamInfo].m_numSamples != minibatchData2[featureStreamInfo].m_numSamples)
if (minibatchData[featureStreamInfo].numberOfSamples != minibatchData2[featureStreamInfo].numberOfSamples)
ReportFailure("Data does not match, reads are not deterministic!!!");
// Because they are supposed to read the same data - adding it only once.
totalSamples += minibatchData[featureStreamInfo].m_numSamples;
totalSamples += minibatchData[featureStreamInfo].numberOfSamples;
}
else
{
@ -179,27 +179,27 @@ void TestMinibatchSourceWarmStart(size_t minibatchSize, size_t warmStartSamples,
// Update the counter
size_t accumulative = 0;
if (!minibatchData.empty())
accumulative += minibatchData[featureStreamInfo].m_numSamples;
accumulative += minibatchData[featureStreamInfo].numberOfSamples;
if (!minibatchData2.empty())
accumulative += minibatchData2[featureStreamInfo].m_numSamples;
accumulative += minibatchData2[featureStreamInfo].numberOfSamples;
totalSamples += accumulative;
if (expectNoData) // second worker does not have any data.
{
if (minibatchData[featureStreamInfo].m_numSamples != minibatchSize/2 && totalSamples != numberOfSamplesInSweep)
if (minibatchData[featureStreamInfo].numberOfSamples != minibatchSize/2 && totalSamples != numberOfSamplesInSweep)
ReportFailure("TestMinibatchSourceWarmStart failed because data did not match."
"Expected minibatch size '%d', acutal '%d'. Total number of sample '%d', sweep '%d'.",
(int)minibatchSize,
(int)minibatchData[featureStreamInfo].m_numSamples,
(int)minibatchData[featureStreamInfo].numberOfSamples,
(int)totalSamples,
(int)numberOfSamplesInSweep);
}
else
{
if (accumulative != minibatchSize &&
minibatchData[featureStreamInfo].m_numSamples != minibatchSize / 2 &&
minibatchData2[featureStreamInfo].m_numSamples != minibatchSize / 2 &&
minibatchData[featureStreamInfo].numberOfSamples != minibatchSize / 2 &&
minibatchData2[featureStreamInfo].numberOfSamples != minibatchSize / 2 &&
totalSamples != numberOfSamplesInSweep)
ReportFailure("TestMinibatchSourceWarmStart failed because data did not match."
"Expected minibatch size '%d', acutal '%d'. Total number of sample '%d', sweep '%d'.",
@ -217,8 +217,81 @@ void TestMinibatchSourceWarmStart(size_t minibatchSize, size_t warmStartSamples,
(int)totalSamples);
}
void TestEndOfSweepFlag(size_t maxSamples, size_t mbSize, bool randomize)
{
const size_t sweepSize = 603;
auto ctfInput = L"SimpleDataTest_cntk_text.txt";
std::vector<StreamConfiguration> streamConfig { { L"features", 2 } };
auto cpuDevice = DeviceDescriptor::CPUDevice();
auto src = TextFormatMinibatchSource(ctfInput, streamConfig, maxSamples, randomize);
maxSamples = (maxSamples == MinibatchSource::FullDataSweep) ? sweepSize : maxSamples;
bool reachedEndOfEpoch = false;
size_t sampleCount = 0;
while (sampleCount < maxSamples)
{
auto& dataMap = src->GetNextMinibatch(mbSize, cpuDevice);
if (dataMap.size() != streamConfig.size())
{
ReportFailure("TestThatEndOfSweepFlagIsSetCorrectly failed: "
"unexpected number of streams in the minibatch (%zu).", dataMap.size());
}
for (auto& streamData : dataMap)
{
auto numSamplesInMinibatch = streamData.second.numberOfSamples;
bool expectedEndOfSweep = ((sampleCount + numSamplesInMinibatch) % sweepSize) == 0;
expectedEndOfSweep |= ((sampleCount) / sweepSize) < ((sampleCount + numSamplesInMinibatch) / sweepSize);
reachedEndOfEpoch = (sampleCount + mbSize >= maxSamples);
size_t expectedNumSamples = reachedEndOfEpoch ? (maxSamples - sampleCount) : mbSize;
if (streamData.second.sweepEnd != expectedEndOfSweep)
{
ReportFailure("TestThatEndOfSweepFlagIsSetCorrectly failed: end of sweep flag is not set.");
}
if (streamData.second.numberOfSamples != expectedNumSamples)
{
ReportFailure("TestThatEndOfSweepFlagIsSetCorrectly failed: "
"unexpected number of samples in the minibatch (%zu).", streamData.second.numberOfSamples);
}
if (streamData.second.numberOfSequences != expectedNumSamples)
{
ReportFailure("TestThatEndOfSweepFlagIsSetCorrectly failed: "
"unexpected number of sequences in the minibatch (%zu).", streamData.second.numberOfSequences);
}
}
sampleCount += mbSize;
}
auto& emptyDataMap = src->GetNextMinibatch(mbSize, cpuDevice);
assert(emptyDataMap.empty());
}
void TestThatEndOfSweepFlagIsSetCorrectly()
{
for (auto randomize : { false, true })
{
TestEndOfSweepFlag(MinibatchSource::FullDataSweep, 603, randomize);
TestEndOfSweepFlag(MinibatchSource::FullDataSweep, 1000, randomize);
TestEndOfSweepFlag(MinibatchSource::FullDataSweep, 100, randomize);
TestEndOfSweepFlag(100, 30, randomize);
TestEndOfSweepFlag(2000, 500, randomize);
TestEndOfSweepFlag(2412, 301, randomize);
}
}
void MinibatchSourceTests()
{
TestThatEndOfSweepFlagIsSetCorrectly();
// Test no-randomize minibatch source with small data chunks
TestMinibatchSourceWarmStart(64, 128, false, 1024);
TestMinibatchSourceWarmStart(64, 0, false, 1024);

4
Tests/UnitTests/V2LibraryTests/Seq2Seq.cpp
Просмотреть файл

@ -209,7 +209,7 @@ void TrainSequenceToSequenceTranslator(const DeviceDescriptor& device, bool useS
if (minibatchData.empty())
break;
trainer->TrainMinibatch({ { rawInput, minibatchData[rawInputStreamInfo].m_data }, { rawLabels, minibatchData[rawLabelsStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { rawInput, minibatchData[rawInputStreamInfo] }, { rawLabels, minibatchData[rawLabelsStreamInfo] } }, device);
PrintTrainingProgress(trainer, i, outputFrequencyInMinibatches);
if ((i + 1) == numMinibatchesToCheckpointAfter)
@ -222,7 +222,7 @@ void TrainSequenceToSequenceTranslator(const DeviceDescriptor& device, bool useS
if ((i % decodingFrequency) == 0)
{
std::unordered_map<Variable, ValuePtr> outputs = { { decodingFunction, nullptr }};
decodingFunction->Forward({ { decodingFunction->Arguments()[0], minibatchData[rawInputStreamInfo].m_data }, { decodingFunction->Arguments()[1], minibatchData[rawLabelsStreamInfo].m_data } },
decodingFunction->Forward({ { decodingFunction->Arguments()[0], minibatchData[rawInputStreamInfo].data }, { decodingFunction->Arguments()[1], minibatchData[rawLabelsStreamInfo].data } },
outputs,
device);
}

22
Tests/UnitTests/V2LibraryTests/SequenceClassification.cpp
Просмотреть файл

@ -59,7 +59,7 @@ void TrainLSTMSequenceClassifer(const DeviceDescriptor& device, bool useSparseLa
if (minibatchData.empty())
break;
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
PrintTrainingProgress(trainer, i, outputFrequencyInMinibatches);
}
}
@ -87,37 +87,37 @@ void TestLearningRateControl(const DeviceDescriptor& device)
const size_t minibatchSize = 200;
auto minibatchData = minibatchSource->GetNextMinibatch(minibatchSize, device);
auto actualMBSize = minibatchData[labelStreamInfo].m_numSamples;
auto actualMBSize = minibatchData[labelStreamInfo].numberOfSamples;
LearningRatePerSampleSchedule learningRateSchedule({ { 2, 0.0005 }, { 2, 0.00025 } }, actualMBSize);
auto learner = SGDLearner(classifierOutput->Parameters(), learningRateSchedule);
auto trainer = CreateTrainer(classifierOutput, trainingLoss, prediction, { learner });
FloatingPointCompare(learner->LearningRate(), 0.0005, "Learner::LearningRate does not match expectation");
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
FloatingPointCompare(learner->LearningRate(), 0.0005, "Learner::LearningRate does not match expectation");
const wchar_t* modelFile = L"seq2seq.model";
trainer->SaveCheckpoint(modelFile);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
auto MB2Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(learner->LearningRate(), 0.00025, "Learner::LearningRate does not match expectation");
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
auto MB3Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(learner->LearningRate(), 0.00025, "Learner::LearningRate does not match expectation");
trainer->RestoreFromCheckpoint(modelFile);
FloatingPointCompare(learner->LearningRate(), 0.0005, "Learner::LearningRate does not match expectation");
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
auto postRestoreMB2Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(postRestoreMB2Loss, MB2Loss, "Post checkpoint restoration training loss does not match expectation");
FloatingPointCompare(learner->LearningRate(), 0.00025, "Learner::LearningRate does not match expectation");
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
auto postRestoreMB3Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(postRestoreMB3Loss, MB3Loss, "Post checkpoint restoration training loss does not match expectation");
@ -128,13 +128,13 @@ void TestLearningRateControl(const DeviceDescriptor& device)
FloatingPointCompare(learner->LearningRate(), 0.0004, "Learner::LearningRate does not match expectation");
trainer->SaveCheckpoint(modelFile);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
postRestoreMB2Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(postRestoreMB2Loss, MB2Loss, "Post checkpoint restoration training loss does not match expectation");
FloatingPointCompare(learner->LearningRate(), 0.0004, "Learner::LearningRate does not match expectation");
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
postRestoreMB3Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(postRestoreMB3Loss, MB3Loss, "Post checkpoint restoration training loss does not match expectation");
@ -143,13 +143,13 @@ void TestLearningRateControl(const DeviceDescriptor& device)
trainer->RestoreFromCheckpoint(modelFile);
FloatingPointCompare(learner->LearningRate(), 0.0004, "Learner::LearningRate does not match expectation");
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
postRestoreMB2Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(postRestoreMB2Loss, MB2Loss, "Post checkpoint restoration training loss does not match expectation");
FloatingPointCompare(learner->LearningRate(), 0.0004, "Learner::LearningRate does not match expectation");
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
postRestoreMB3Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(postRestoreMB3Loss, MB3Loss, "Post checkpoint restoration training loss does not match expectation");

42
Tests/UnitTests/V2LibraryTests/SerializationTests.cpp
Просмотреть файл

@ -434,7 +434,7 @@ void TestFunctionSerializationDuringTraining(const FunctionPtr& function, const
Dictionary model = classifierOutput1->Serialize();
trainer1->TrainMinibatch({ { classifierOutput1->Arguments()[0], minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer1->TrainMinibatch({ { classifierOutput1->Arguments()[0], minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
auto classifierOutput2 = Function::Deserialize(model, device);
@ -458,8 +458,8 @@ void TestFunctionSerializationDuringTraining(const FunctionPtr& function, const
for (int j = 0; j < 3; ++j)
{
trainer1->TrainMinibatch({ { classifierOutput1->Arguments()[0], minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer2->TrainMinibatch({ { classifierOutput3->Arguments()[0], minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer1->TrainMinibatch({ { classifierOutput1->Arguments()[0], minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
trainer2->TrainMinibatch({ { classifierOutput3->Arguments()[0], minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
double mbLoss1 = trainer1->PreviousMinibatchLossAverage();
double mbLoss2 = trainer2->PreviousMinibatchLossAverage();
@ -503,7 +503,7 @@ void TestTrainingWithCheckpointing(const FunctionPtr& function1, const FunctionP
const size_t minibatchSize = 50;
auto minibatchData = minibatchSource->GetNextMinibatch(minibatchSize, device);
auto actualMBSize = minibatchData[labelStreamInfo].m_numSamples;
auto actualMBSize = minibatchData[labelStreamInfo].numberOfSamples;
LearningRatePerSampleSchedule learningRateSchedule({ { 2, 0.005 }, { 2, 0.0025 }, { 2, 0.0005 }, { 2, 0.00025 } }, actualMBSize);
MomentumAsTimeConstantSchedule momentumValues({ { 2, 100 }, { 2, 200 }, { 2, 400 }, { 2, 800 } }, actualMBSize);
@ -522,14 +522,14 @@ void TestTrainingWithCheckpointing(const FunctionPtr& function1, const FunctionP
throw std::runtime_error("TestModelSerialization: reloaded function is not identical to the original.");
}
trainer1->TrainMinibatch({ { function1->Arguments()[0], minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer1->TrainMinibatch({ { function1->Arguments()[0], minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
if (AreEqual(function1, function2))
{
throw std::runtime_error("TestModelSerialization: reloaded function is still identical to the original after it was trained.");
}
trainer2->TrainMinibatch({ { function2->Arguments()[0], minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer2->TrainMinibatch({ { function2->Arguments()[0], minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
if (!AreEqual(function1, function2))
{
@ -548,8 +548,8 @@ void TestTrainingWithCheckpointing(const FunctionPtr& function1, const FunctionP
for (int j = 0; j < 3; ++j)
{
trainer1->TrainMinibatch({ { function1->Arguments()[0], minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer2->TrainMinibatch({ { function2->Arguments()[0], minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer1->TrainMinibatch({ { function1->Arguments()[0], minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
trainer2->TrainMinibatch({ { function2->Arguments()[0], minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
double mbLoss1 = trainer1->PreviousMinibatchLossAverage();
double mbLoss2 = trainer2->PreviousMinibatchLossAverage();
@ -638,36 +638,36 @@ void TestLegacyModelSaving(const DeviceDescriptor& device)
const size_t minibatchSize = 50;
auto minibatchData = minibatchSource->GetNextMinibatch(minibatchSize, device);
auto actualMBSize = minibatchData[labelStreamInfo].m_numSamples;
auto actualMBSize = minibatchData[labelStreamInfo].numberOfSamples;
LearningRatePerSampleSchedule learningRateSchedule({ { 2, 0.0005 }, { 2, 0.00025 } }, actualMBSize);
auto learner = SGDLearner(classifierOutput->Parameters(), learningRateSchedule);
auto trainer = CreateTrainer(classifierOutput, trainingLoss, prediction, { learner });
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
const wchar_t* modelFile = L"seq2seq.legacy.model";
Internal::SaveAsLegacyModel(classifierOutput, modelFile);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
auto MB2Loss = trainer->PreviousMinibatchLossAverage();
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
classifierOutput->RestoreModel(modelFile);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
auto postRestoreMB2Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(postRestoreMB2Loss, MB2Loss, "Post checkpoint restoration training loss does not match expectation");
classifierOutput->RestoreModel(modelFile);
Internal::SaveAsLegacyModel(classifierOutput, modelFile);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
classifierOutput->RestoreModel(modelFile);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
postRestoreMB2Loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(postRestoreMB2Loss, MB2Loss, "Post checkpoint restoration training loss does not match expectation");
@ -684,7 +684,7 @@ void TestLegacyModelSaving(const DeviceDescriptor& device)
{
trainer->SaveCheckpoint(L"trainer.checkpoint" + std::to_wstring(i));
Internal::SaveAsLegacyModel(classifierOutput, modelFile + std::to_wstring(i));
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
expectedLoss.push_back(trainer->PreviousMinibatchLossAverage());
}
@ -692,7 +692,7 @@ void TestLegacyModelSaving(const DeviceDescriptor& device)
{
trainer->RestoreFromCheckpoint(L"trainer.checkpoint" + std::to_wstring(i));
classifierOutput->RestoreModel(modelFile + std::to_wstring(i));
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
double loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(loss, expectedLoss[i], "Post checkpoint restoration training loss does not match expectation");
}
@ -767,20 +767,20 @@ void TestCheckpointingWithStatefulNodes(const DeviceDescriptor& device)
auto featureStreamInfo = minibatchSource->StreamInfo(features);
auto labelStreamInfo = minibatchSource->StreamInfo(labels);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
vector<double> expectedLoss;
for (int i = 0; i < epochSize / minibatchSize; i++)
{
trainer->SaveCheckpoint(L"stateful_nodes.model" + std::to_wstring(i));
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
expectedLoss.push_back(trainer->PreviousMinibatchLossAverage());
}
for (int i = 0; i < epochSize / minibatchSize; i++)
{
trainer->RestoreFromCheckpoint(L"stateful_nodes.model" + std::to_wstring(i));
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
double loss = trainer->PreviousMinibatchLossAverage();
FloatingPointCompare(loss, expectedLoss[i], "Post checkpoint restoration training loss does not match expectation");
}

4
Tests/UnitTests/V2LibraryTests/TrainerTests.cpp
Просмотреть файл

@ -67,7 +67,7 @@ void TrainSimpleFeedForwardClassifer(const DeviceDescriptor& device)
for (size_t i = 0; i < numMinibatchesToTrain; ++i)
{
auto minibatchData = minibatchSource->GetNextMinibatch(minibatchSize, device);
trainer->TrainMinibatch({ { input, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { input, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
PrintTrainingProgress(trainer, i, outputFrequencyInMinibatches);
if ((i % trainingCheckpointFrequency) == (trainingCheckpointFrequency - 1))
@ -128,7 +128,7 @@ void TrainMNISTClassifier(const DeviceDescriptor& device)
for (size_t i = 0; i < numMinibatchesToTrain; ++i)
{
auto minibatchData = minibatchSource->GetNextMinibatch(minibatchSize, device);
trainer->TrainMinibatch({ { input, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { input, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
PrintTrainingProgress(trainer, i, outputFrequencyInMinibatches);
}
}

4
Tests/UnitTests/V2LibraryTests/TruncatedLSTMAcousticModel.cpp
Просмотреть файл

@ -125,11 +125,11 @@ void TrainTruncatedLSTMAcousticModelClassifer(const DeviceDescriptor& device, bo
break;
// Make sure our truncation length setting was honored
auto actualMaxSequenceLength = minibatchData[featureStreamInfo].m_data->Shape()[featureStreamInfo.m_sampleLayout.Rank()];
auto actualMaxSequenceLength = minibatchData[featureStreamInfo].data->Shape()[featureStreamInfo.m_sampleLayout.Rank()];
if (actualMaxSequenceLength != truncationLength)
ReportFailure("Actual max sequence length (%d) in minibatch data does not equal specified truncation length (%d)", (int)actualMaxSequenceLength, (int)truncationLength);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo].m_data }, { labels, minibatchData[labelStreamInfo].m_data } }, device);
trainer->TrainMinibatch({ { features, minibatchData[featureStreamInfo] }, { labels, minibatchData[labelStreamInfo] } }, device);
PrintTrainingProgress(trainer, i, outputFrequencyInMinibatches);
}
}

9
Tests/UnitTests/V2LibraryTests/ValueTests.cpp
Просмотреть файл

@ -187,7 +187,6 @@ void ValueCreationWithNDMaskTest(const DeviceDescriptor device, bool readOnly)
NDShape sampleShape(dims);
size_t numberOfSequences;
size_t maxAllowedSeqLen = 128;
size_t maxSeqLen;
std::vector<std::vector<ElementType>> data;
std::vector<size_t> seqLenList;
@ -200,12 +199,18 @@ void ValueCreationWithNDMaskTest(const DeviceDescriptor device, bool readOnly)
{
numberOfSequences = distribution(generator);
seqLenList = GenerateSequenceLengths(numberOfSequences, maxAllowedSeqLen);
maxSeqLen = *std::max_element(seqLenList.begin(), seqLenList.end());
data = GenerateSequences<ElementType>(seqLenList, sampleShape);
ValuePtr testValue = Value::Create(sampleShape, data, device, readOnly);
CheckValue(testValue, sampleShape, data, seqLenList);
}
// Test with only 1 sequence with a sequenceStartFlag=false to ensure a mask
seqLenList = GenerateSequenceLengths(1, maxAllowedSeqLen);
data = GenerateSequences<ElementType>(seqLenList, sampleShape);
ValuePtr testValue = Value::Create(sampleShape, data, {false}, device, readOnly);
CheckValue(testValue, sampleShape, data, seqLenList);
}
template <typename ElementType>

180
Tutorials/CNTK_102_FeedForward.ipynb
Просмотреть файл

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

188
Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb
Просмотреть файл

@ -53,11 +53,15 @@
"import sys\n",
"import os\n",
"\n",
"import cntk as C\n",
"from cntk import Trainer, learning_rate_schedule, UnitType\n",
"from cntk.blocks import default_options, Input\n",
"from cntk.io import CTFDeserializer, MinibatchSource, StreamDef, StreamDefs\n",
"from cntk.io import INFINITELY_REPEAT, FULL_DATA_SWEEP\n",
"from cntk.initializer import glorot_uniform\n",
"from cntk.layers import Dense\n",
"from cntk.learner import sgd\n",
"from cntk.ops import *\n",
"\n",
"# Select the right target device when this notebook is being tested:\n",
"if 'TEST_DEVICE' in os.environ:\n",
" import cntk\n",
@ -98,9 +102,32 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### Input and Labels\n",
"## Data reading\n",
"\n",
"In this tutorial we are using the MNIST data you have downloaded using CNTK_103A_MNIST_DataLoader notebook. The dataset has 60,000 training images and 10,000 test images with each image being 28 x 28 pixels. Thus the number of features is equal to 784 (= 28 x 28 pixels), 1 per pixel. The variable `num_output_classes` is set to 10 corresponding to the number of digits (0-9) in the dataset."
"In this tutorial we are using the MNIST data you have downloaded using CNTK_103A_MNIST_DataLoader notebook. The dataset has 60,000 training images and 10,000 test images with each image being 28 x 28 pixels. Thus the number of features is equal to 784 (= 28 x 28 pixels), 1 per pixel. The variable `num_output_classes` is set to 10 corresponding to the number of digits (0-9) in the dataset.\n",
"\n",
"The data is in the following format:\n",
"\n",
" |labels 0 0 0 0 0 0 0 1 0 0 |features 0 0 0 0 ... \n",
" (784 integers each representing a pixel)\n",
" \n",
"In this tutorial we are going to use the image pixels corresponding the integer stream named \"features\". We define a `create_reader` function to read the training and test data using the [CTF deserializer](https://cntk.ai/pythondocs/cntk.io.html?highlight=ctfdeserializer#cntk.io.CTFDeserializer). The labels are [1-hot encoded](https://en.wikipedia.org/wiki/One-hot). \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Read a CTF formatted text (as mentioned above) using the CTF deserializer from a file\n",
"def create_reader(path, is_training, input_dim, num_label_classes):\n",
" return MinibatchSource(CTFDeserializer(path, StreamDefs(\n",
" labels = StreamDef(field='labels', shape=num_label_classes, is_sparse=False),\n",
" features = StreamDef(field='features', shape=input_dim, is_sparse=False)\n",
" )), randomize = is_training, epoch_size = INFINITELY_REPEAT if is_training else FULL_DATA_SWEEP)"
]
},
{
@ -123,21 +150,7 @@
" break\n",
"if not data_found:\n",
" raise ValueError(\"Please generate the data by completing CNTK 103 Part A\")\n",
"print(\"Data directory is {0}\".format(data_dir))\n",
"\n",
"# Set up data reading for the training data\n",
"feature_stream_name = 'features'\n",
"labels_stream_name = 'labels'\n",
"\n",
"mb_source = MinibatchSource(CTFDeserializer(train_file, StreamDefs(\n",
" features = StreamDef(field='features', shape=input_dim, is_sparse=False),\n",
" labels = StreamDef(field='labels', shape=num_output_classes, is_sparse=False)\n",
" )))\n",
"\n",
"features_si = mb_source[feature_stream_name]\n",
"labels_si = mb_source[labels_stream_name]\n",
"\n",
"print(\"Training data successfully read from file {0}.\".format(train_file))"
"print(\"Data directory is {0}\".format(data_dir))"
]
},
{
@ -192,8 +205,8 @@
},
"outputs": [],
"source": [
"input = input_variable((input_dim), np.float32)\n",
"label = input_variable((num_output_classes), np.float32)"
"input = Input(input_dim)\n",
"label = Input(num_output_classes)"
]
},
{
@ -213,30 +226,15 @@
},
"outputs": [],
"source": [
"# Define a fully connected feedforward network\n",
"\n",
"def linear_layer(input_var, output_dim):\n",
"\n",
" input_dim = input_var.shape[0]\n",
" times_param = parameter(shape=(input_dim, output_dim), init=glorot_uniform())\n",
" bias_param = parameter(shape=(output_dim))\n",
"\n",
" t = times(input_var, times_param)\n",
" return bias_param + t\n",
"\n",
"def dense_layer(input, output_dim, nonlinearity):\n",
" r = linear_layer(input, output_dim)\n",
" r = nonlinearity(r)\n",
" return r;\n",
"\n",
"def fully_connected_classifier_net(input, num_output_classes, hidden_layer_dim, \n",
" num_hidden_layers, nonlinearity):\n",
" \n",
" h = dense_layer(input, hidden_layer_dim, nonlinearity)\n",
" for i in range(1, num_hidden_layers):\n",
" h = dense_layer(h, hidden_layer_dim, nonlinearity)\n",
" r = linear_layer(h, num_output_classes)\n",
" return r"
"def create_model(features):\n",
" with default_options(init = glorot_uniform(), activation = C.ops.relu):\n",
" h = features\n",
" for _ in range(num_hidden_layers):\n",
" h = Dense(hidden_layers_dim)(h)\n",
" r = Dense(num_output_classes, activation = None)(h)\n",
" return r\n",
" \n",
"z = create_model(input)"
]
},
{
@ -266,9 +264,7 @@
"outputs": [],
"source": [
"# Scale the input to 0-1 range by dividing each pixel by 256.\n",
"scaled_input = element_times(constant(1.0 / 256.0), input)\n",
"# Create the fully connected classifier.\n",
"z = fully_connected_classifier_net(scaled_input, num_output_classes, hidden_layers_dim, num_hidden_layers, relu)"
"z = create_model(input/256.0)"
]
},
{
@ -301,7 +297,7 @@
},
"outputs": [],
"source": [
"loss = cross_entropy_with_softmax(z, label)"
"loss = C.ops.cross_entropy_with_softmax(z, label)"
]
},
{
@ -321,7 +317,7 @@
},
"outputs": [],
"source": [
"label_error = classification_error(z, label)"
"label_error = C.ops.classification_error(z, label)"
]
},
{
@ -433,24 +429,32 @@
},
"outputs": [],
"source": [
"# Create the reader to training data set\n",
"reader_train = create_reader(train_file, True, input_dim, num_output_classes)\n",
"\n",
"# Map the data streams to the input and labels.\n",
"input_map = {\n",
" label : reader_train.streams.labels,\n",
" input : reader_train.streams.features\n",
"} \n",
"\n",
"# Run the trainer on and perform model training\n",
"training_progress_output_freq = 500\n",
"\n",
"plotdata = {\"batchsize\":[], \"loss\":[], \"error\":[]}\n",
"\n",
"for i in range(0, int(num_minibatches_to_train)):\n",
" mb = mb_source.next_minibatch(minibatch_size)\n",
" \n",
" # Specify the input variables mapping in the model to actual minibatch data to be trained\n",
" arguments = {input: mb[features_si],\n",
" label: mb[labels_si]}\n",
" trainer.train_minibatch(arguments)\n",
" # Read a mini batch from the training data file\n",
" data = reader_train.next_minibatch(minibatch_size, input_map = input_map)\n",
" \n",
" trainer.train_minibatch(data)\n",
" batchsize, loss, error = print_training_progress(trainer, i, training_progress_output_freq, verbose=1)\n",
" \n",
" if not (loss == \"NA\" or error ==\"NA\"):\n",
" plotdata[\"batchsize\"].append(batchsize)\n",
" plotdata[\"loss\"].append(loss)\n",
" plotdata[\"error\"].append(error)\n"
" plotdata[\"error\"].append(error)"
]
},
{
@ -511,41 +515,30 @@
},
"outputs": [],
"source": [
"feature_stream_name = 'features'\n",
"labels_stream_name = 'labels'\n",
"# Read the training data\n",
"reader_test = create_reader(test_file, False, input_dim, num_output_classes)\n",
"\n",
"test_mb_source = MinibatchSource(CTFDeserializer(test_file, StreamDefs(\n",
" features = StreamDef(field='features', shape=input_dim, is_sparse=False),\n",
" labels = StreamDef(field='labels', shape=num_output_classes, is_sparse=False)\n",
" )))\n",
"test_input_map = {\n",
" label : reader_test.streams.labels,\n",
" input : reader_test.streams.features,\n",
"}\n",
"\n",
"features_si = test_mb_source[feature_stream_name]\n",
"labels_si = test_mb_source[labels_stream_name]\n",
"\n",
"print(\"Test data successfully read from file {0}\".format(test_file))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"# Test data for trained model\n",
"test_minibatch_size = 512\n",
"num_samples = 10000\n",
"num_minibatches_to_test = num_samples / test_minibatch_size\n",
"num_minibatches_to_test = num_samples // test_minibatch_size\n",
"test_result = 0.0\n",
"for i in range(0, int(num_minibatches_to_test)):\n",
" mb = test_mb_source.next_minibatch(test_minibatch_size)\n",
"\n",
" # Specify the mapping of input variables in the model to actual\n",
" # minibatch data to be tested with\n",
" arguments = {input: mb[features_si],\n",
" label: mb[labels_si]}\n",
" eval_error = trainer.test_minibatch(arguments)\n",
"for i in range(num_minibatches_to_test):\n",
" \n",
" # We are loading test data in batches specified by test_minibatch_size\n",
" # Each data point in the minibatch is a MNIST digit image of 784 dimensions \n",
" # with one pixel per dimension that we will encode / decode with the \n",
" # trained model.\n",
" data = reader_test.next_minibatch(test_minibatch_size,\n",
" input_map = test_input_map)\n",
"\n",
" eval_error = trainer.test_minibatch(data)\n",
" test_result = test_result + eval_error\n",
"\n",
"# Average of evaluation errors of all test minibatches\n",
@ -574,7 +567,7 @@
},
"outputs": [],
"source": [
"out = softmax(z)"
"out = C.ops.softmax(z)"
]
},
{
@ -592,12 +585,17 @@
},
"outputs": [],
"source": [
"mb = test_mb_source.next_minibatch(test_minibatch_size)\n",
"# Read the data for evaluation\n",
"reader_eval = create_reader(test_file, False, input_dim, num_output_classes)\n",
"\n",
"predicted_label_prob = out.eval({input : mb[features_si]})\n",
"eval_minibatch_size = 25\n",
"eval_input_map = { input : reader_eval.streams.features } \n",
"\n",
"#orig_label=np.array(mb[labels_si].m_data.data().to_numpy())\n",
"orig_label = np.asarray(mb[labels_si].m_data)"
"data = reader_test.next_minibatch(eval_minibatch_size, input_map = test_input_map)\n",
"\n",
"img_label = data[label].value\n",
"img_data = data[input].value\n",
"predicted_label_prob = [out.eval(img_data[i,:,:]) for i in range(img_data.shape[0])]"
]
},
{
@ -609,8 +607,8 @@
"outputs": [],
"source": [
"# Find the index with the maximum value for both predicted as well as the ground truth\n",
"pred = [np.argmax(predicted_label_prob[i,:,:]) for i in range(0,predicted_label_prob.shape[0])]\n",
"gtlabel = [np.argmax(orig_label[i,:,:]) for i in range(0, orig_label.shape[0])]"
"pred = [np.argmax(predicted_label_prob[i]) for i in range(len(predicted_label_prob))]\n",
"gtlabel = [np.argmax(img_label[i,:,:]) for i in range(img_label.shape[0])]"
]
},
{
@ -622,7 +620,7 @@
"outputs": [],
"source": [
"print(\"Label :\", gtlabel[:25])\n",
"print(\"Predicted:\", pred[:25])"
"print(\"Predicted:\", pred)"
]
},
{
@ -642,9 +640,7 @@
"source": [
"# Plot a random image\n",
"sample_number = 5\n",
"#img_data = mb[features_si].m_data.data().to_numpy()\n",
"img_data = mb[features_si].value\n",
"plt.imshow(img_data[sample_number,:,:].reshape(28,28), cmap=\"gray_r\")\n",
"plt.imshow(img_data[sample_number].reshape(28,28), cmap=\"gray_r\")\n",
"plt.axis('off')\n",
"\n",
"img_gt, img_pred = gtlabel[sample_number], pred[sample_number]\n",
@ -687,9 +683,9 @@
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [default]",
"display_name": "Python [conda env:cntk-py34]",
"language": "python",
"name": "python3"
"name": "conda-env-cntk-py34-py"
},
"language_info": {
"codemirror_mode": {

4
Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb
Просмотреть файл

@ -489,7 +489,7 @@
},
"outputs": [],
"source": [
"TOTAL_EPISODES = 1500 if isFast else 3000\n",
"TOTAL_EPISODES = 2000 if isFast else 3000\n",
"\n",
"def run(agent):\n",
" s = env.reset()\n",
@ -716,7 +716,7 @@
"source": [
"import cntk as C\n",
"\n",
"TOTAL_EPISODES = 1500 if isFast else 10000\n",
"TOTAL_EPISODES = 2000 if isFast else 10000\n",
"\n",
"D = 4 # input dimensionality\n",
"H = 10 # number of hidden layer neurons\n",

1
bindings/python/cntk/__init__.py
Просмотреть файл

@ -8,6 +8,7 @@ __version__ = '2.0'
import os
import numpy as np
from .core import *
from . import ops
from . import cntk_py

282
bindings/python/cntk/cntk_py.i
Просмотреть файл

@ -24,6 +24,15 @@
%rename(momentum_as_time_constant_schedule) CNTK::MomentumAsTimeConstantSchedule;
// renaming overloads for TrainMinibatch and TestMinibatch that take a map
// of Variables and MinibatchData as their first parameter. If this is not done,
// the overloads that are legal in C++ will be shadowed and ignored by SWIG.
// The naming here is somewhat cumbersome, but it's only intended for internal
// consumption in proxy objects.
%rename(train_minibatch_overload_for_minibatchdata) CNTK::Trainer::TrainMinibatch(const std::unordered_map<Variable, MinibatchData>&, const DeviceDescriptor& = DeviceDescriptor::UseDefaultDevice());
%rename(train_minibatch_overload_for_minibatchdata) CNTK::Trainer::TrainMinibatch(const std::unordered_map<Variable, MinibatchData>&, std::unordered_map<Variable, ValuePtr>&, const DeviceDescriptor& = DeviceDescriptor::UseDefaultDevice());
%rename(test_minibatch_overload_for_minibatchdata) CNTK::Trainer::TestMinibatch(const std::unordered_map<Variable, MinibatchData>&, const DeviceDescriptor& = DeviceDescriptor::UseDefaultDevice());
%rename(l1_regularization_weight) CNTK::AdditionalLearningOptions::l1RegularizationWeight;
%rename(l2_regularization_weight) CNTK::AdditionalLearningOptions::l2RegularizationWeight;
%rename(ndcg_at_1) CNTK::NDCGAt1;
@ -41,12 +50,14 @@
%template() std::vector<std::vector<double>>;
%template() std::vector<CNTK::Variable>;
%template() std::vector<CNTK::OutputVariable>;
%template() std::vector<CNTK::Parameter>;
%template() std::vector<CNTK::Constant>;
%template() std::vector<CNTK::Axis>;
%template() std::vector<CNTK::DeviceDescriptor>;
%template() std::vector<CNTK::StreamConfiguration>;
%template() std::vector<std::shared_ptr<CNTK::NDArrayView>>;
%template() std::vector<std::shared_ptr<CNTK::Value>>;
%template() std::vector<std::shared_ptr<CNTK::Function>>;
%template() std::vector<std::shared_ptr<CNTK::Learner>>;
%template() std::vector<std::shared_ptr<CNTK::DistributedLearner>>;
@ -72,6 +83,8 @@
%ignore CNTK::Internal::IsAutomaticUnpackingOfPackedValuesDisabled;
%ignore CNTK::Internal::GetComputationNetworkTraceLevel;
%ignore CNTK::Function::Function(const std::vector<Variable>& inputs, const std::vector<Variable>& outputs, Dictionary&& functionConfig, const std::wstring& name = L"", const std::wstring& uid = Internal::GenerateUid(L"UserDefinedFunction"));
%{
#define SWIG_FILE_WITH_INIT
%}
@ -325,7 +338,7 @@ fail:
PyObject* container = PyDict_New();
if (container == NULL)
{
SWIG_exception(SWIG_RuntimeError, "error passing dictionary to Python");
SWIG_exception(SWIG_RuntimeError, "error passing a dictionary to Python");
}
for (auto it = $1->begin(); it != $1->end(); ++it)
@ -398,6 +411,9 @@ public:
}
}
// Callback support
%feature("director") CNTK::Function;
%{
#include "CNTKLibrary.h"
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
@ -406,9 +422,6 @@ public:
using namespace CNTK;
%}
// Callback support
%feature("director") Callback;
//
// Exception handling
//
@ -574,13 +587,13 @@ public:
std::unordered_map<CNTK::Variable, CNTK::ValuePtr>& outputsToFetch,
std::unordered_map<CNTK::Variable, CNTK::ValuePtr>& outputs,
std::unordered_map<CNTK::Variable, CNTK::ValuePtr>& backPropagatedGradientValuesForInputs
{
if (!PyDict_Check($input)) {
SWIG_exception(SWIG_TypeError, "dictionary expected");
}
{
if (!PyDict_Check($input)) {
SWIG_exception(SWIG_TypeError, "dictionary expected");
}
for (auto it: *$1)
{
for (auto it: *$1)
{
// Push the ValuePtr onto the heap so that it survives
std::shared_ptr<CNTK::Value> *smartresult = it.second ? new std::shared_ptr<CNTK::Value>(it.second) : 0;
PyObject *returned_val = SWIG_NewPointerObj(SWIG_as_voidptr(smartresult), SWIGTYPE_p_std__shared_ptrT_CNTK__Value_t, SWIG_POINTER_OWN);
@ -601,27 +614,96 @@ public:
PyObject *py_key, *py_value;
Py_ssize_t pos = 0;
bool found = false;
while (PyDict_Next($input, &pos, &py_key, &py_value)) {
void *cntk_key = 0 ;
int res = SWIG_ConvertPtr(py_key, &cntk_key, SWIGTYPE_p_CNTK__Variable, 0);
void *cpp_key = 0 ;
int res = SWIG_ConvertPtr(py_key, &cpp_key, SWIGTYPE_p_CNTK__Variable, 0);
if (!SWIG_IsOK(res)) {
SWIG_exception_fail(SWIG_ArgError(res), "cannot convert key of dictionary to CNTK::Variable");
}
if (!cntk_key) {
if (!cpp_key) {
SWIG_exception_fail(SWIG_ValueError, "invalid null reference when converting key of dictionary to CNTK::Variable");
}
CNTK::Variable* cntk_var = reinterpret_cast<CNTK::Variable*>(cntk_key);
if (*cntk_var == *&it.first)
CNTK::Variable* cntk_var = reinterpret_cast<CNTK::Variable*>(cpp_key);
if (*cntk_var == it.first)
{
found = true;
PyDict_SetItem($input, py_key, returned_val);
break;
}
}
if (!found)
{
RuntimeError("could not convert dictionary");
}
Py_DECREF(returned_val);
}
}
// For the output dict (the non-const unordered_map) we need to get the
// modified values and put them back into the dictionary. This is used, when
// e.g. the user puts a variable into the dictionary, hoping that it will
// afterwards point to the proper value.
%typemap(directorargout)
// Swig would create this conversion for the 'const' variants as well, which
// we do not want. Therefor, we have to explicitly tell it for which ones it should do it.
std::unordered_map<CNTK::Variable, CNTK::ValuePtr>& outputs,
std::unordered_map<CNTK::Variable, CNTK::ValuePtr>& backPropagatedGradientValuesForInputs
{
// $1 is the C++ input that needs to be filled with the data from the PyDict
for (auto it: $1)
{
// Find the corresponding Variable instance in the Python dictionary
// and set its value to the new ValuePtr
/* FIXME We would love to do the following, but the hashing does not
* correctly work here, which is why we never find the keys. Instead,
* we will for now loop over the dictionary and use C++ comparison.
* Although not beautiful, there should not be a lot of overhead since
* the dictionary usually contains only a handful of variables as keys.
if (PyDict_Contains($input, returned_var))
{
SWIG_exception_fail(SWIG_ValueError, "returned output map contains unknown key");
}
*/
PyObject *py_key, *py_value;
Py_ssize_t pos = 0;
bool found = false;
while (PyDict_Next($input, &pos, &py_key, &py_value)) {
void *cpp_key = 0;
int key_res = SWIG_ConvertPtr(py_key, &cpp_key, SWIGTYPE_p_CNTK__Variable, 0);
if (!SWIG_IsOK(key_res)) {
RuntimeError("cannot convert key of dictionary");
}
CNTK::Variable* cntk_var = reinterpret_cast<CNTK::Variable*>(cpp_key);
if (*cntk_var == it.first)
{
found = true;
void *cpp_val = 0;
int val_res = SWIG_ConvertPtr(py_value, &cpp_val, SWIGTYPE_p_std__shared_ptrT_CNTK__Value_t, 0);
if (!SWIG_IsOK(val_res)) {
RuntimeError("cannot convert value of dictionary");
}
CNTK::ValuePtr* cpp_value = reinterpret_cast<CNTK::ValuePtr*>(cpp_val);
$1[it.first] = *cpp_value;
break;
}
}
if (!found)
{
RuntimeError("could not convert dictionary");
}
}
}
//
// Converting Python dictionary {StreamInformation: (mbsize, Value)} to std::unordered_map<CNTK::StreamInformation, std::pair<size_t, size_t>>&
@ -663,7 +745,6 @@ public:
} else {
SWIG_exception(SWIG_TypeError, "tuple expected");
}
}
$1 = &args_map;
@ -772,23 +853,30 @@ public:
PyObject *py_key, *py_value;
Py_ssize_t pos = 0;
bool found = false;
while (PyDict_Next($input, &pos, &py_key, &py_value)) {
void *cntk_key = 0 ;
int res = SWIG_ConvertPtr(py_key, &cntk_key, SWIGTYPE_p_CNTK__StreamInformation, 0);
void *cpp_key = 0 ;
int res = SWIG_ConvertPtr(py_key, &cpp_key, SWIGTYPE_p_CNTK__StreamInformation, 0);
if (!SWIG_IsOK(res)) {
SWIG_exception_fail(SWIG_ArgError(res), "cannot convert key of dictionary to CNTK::StreamInformation");
}
if (!cntk_key) {
if (!cpp_key) {
SWIG_exception_fail(SWIG_ValueError, "invalid null reference when converting key of dictionary to CNTK::StreamInformation");
}
CNTK::StreamInformation* cntk_var = reinterpret_cast<CNTK::StreamInformation*>(cntk_key);
if (*cntk_var == *&it.first)
CNTK::StreamInformation* cntk_var = reinterpret_cast<CNTK::StreamInformation*>(cpp_key);
if (*cntk_var == it.first)
{
found = true;
PyDict_SetItem($input, py_key, PyTuple_Pack(2, returned_val1, returned_val2));
break;
}
}
if (!found)
{
RuntimeError("could not convert dictionary");
}
Py_DECREF(returned_val1);
Py_DECREF(returned_val2);
}
@ -801,19 +889,34 @@ public:
// we need to define a hash function on SwigPyObject.
//
%define %unordered_set_ref_conversion_director(DATA_TYPE, _SWIG_TYPE)
%typemap(directorin) std::unordered_set<DATA_TYPE>& {
PyObject* container = PyList_New(0);
for (auto var : $1)
{
PyObject *item = SWIG_NewPointerObj(new DATA_TYPE(var), _SWIG_TYPE, SWIG_POINTER_OWN );
// No error handling here, because the error will be passed directly to Python
PyList_Append(container, item);
Py_DECREF(item);
}
$input = container;
}
%enddef
%define %unordered_set_conversion(DATA_TYPE, _SWIG_TYPE)
%typemap(out) std::unordered_set<CNTK::DATA_TYPE> {
PyObject* container = PyList_New(0);
if (container == NULL)
{
SWIG_exception(SWIG_RuntimeError, "error passing set to Python");
SWIG_exception(SWIG_RuntimeError, "error passing a set to Python");
}
// *&$1 -> $1 is the returned result being converted (unordered_set<...>*),
// wrapped by SwigValueWrapper. So we need to unwrap it using '&',
// then access its value using '*'.
for (auto var : *&$1)
for (auto var : $1)
{
PyObject *item = SWIG_NewPointerObj(new CNTK::DATA_TYPE(var), _SWIG_TYPE, SWIG_POINTER_OWN );
// No error handling here, because the error will be passed directly to Python
@ -879,7 +982,7 @@ public:
PyObject* container = PyList_New(0);
if (container == NULL)
{
SWIG_exception(SWIG_RuntimeError, "error passing set to Python");
SWIG_exception(SWIG_RuntimeError, "error passing a set to Python");
}
for (auto var : *$1)
@ -894,29 +997,6 @@ public:
}
%enddef
// Trainer initializers.
// Because SWIG cannot properly handle smart pointers to derived classes (causes memory leak during the check),
// we need custom constructors.
%extend Trainer
{
Trainer(const FunctionPtr& model, const FunctionPtr& lossFunction, const FunctionPtr& evaluationFunction, const std::vector<DistributedLearnerPtr>& parameterLearners)
{
std::vector<LearnerPtr> learners;
learners.reserve(parameterLearners.size());
for(const auto& l : parameterLearners)
learners.push_back(l);
return CreateTrainer(model, lossFunction, evaluationFunction, learners);
}
Trainer(const FunctionPtr& model, const FunctionPtr& lossFunction, const FunctionPtr& evaluationFunction, const std::vector<LearnerPtr>& parameterLearners)
{
return CreateTrainer(model, lossFunction, evaluationFunction, parameterLearners);
}
}
%ignore CNTK::Trainer::Trainer;
%unordered_set_conversion(CNTK::Variable, SWIGTYPE_p_CNTK__Variable)
%unordered_set_conversion(CNTK::Constant, SWIGTYPE_p_CNTK__Constant)
%unordered_set_conversion(CNTK::Parameter, SWIGTYPE_p_CNTK__Parameter)
@ -930,6 +1010,38 @@ public:
%unordered_set_ref_conversion(CNTK::DistributedWorkerDescriptor, SWIGTYPE_p_CNTK__DistributedWorkerDescriptor)
// Unordered map conversion
%define %unordered_map_ref_conversion_director(DATA_TYPE1, _SWIG_TYPE1, DATA_TYPE2, _SWIG_TYPE2)
%typemap(directorin) std::unordered_map<DATA_TYPE1, DATA_TYPE2>& {
PyObject* container = PyDict_New();
for (auto it : $1)
{
PyObject *returned_var = SWIG_NewPointerObj(SWIG_as_voidptr(new DATA_TYPE1(it.first)), _SWIG_TYPE1, SWIG_POINTER_OWN);
PyObject *returned_val;
if (it.second == nullptr)
{
returned_val = Py_None;
Py_INCREF(Py_None);
}
else {
returned_val = SWIG_NewPointerObj(SWIG_as_voidptr(new DATA_TYPE2(it.second)), _SWIG_TYPE2, SWIG_POINTER_OWN);
}
PyDict_SetItem(container, returned_var, returned_val);
Py_DECREF(returned_var);
Py_DECREF(returned_val);
}
$input = container;
}
%enddef
%unordered_set_ref_conversion_director(CNTK::Variable, SWIGTYPE_p_CNTK__Variable)
%unordered_set_ref_conversion_director(CNTK::ValuePtr, SWIGTYPE_p_std__shared_ptrT_CNTK__Value_t)
%unordered_map_ref_conversion_director(CNTK::Variable, SWIGTYPE_p_CNTK__Variable, CNTK::ValuePtr, SWIGTYPE_p_std__shared_ptrT_CNTK__Value_t)
%define %unordered_map_ref_conversion(DATA_TYPE1, _SWIG_TYPE1, DATA_TYPE2, _SWIG_TYPE2)
@ -937,12 +1049,9 @@ public:
PyObject* container = PyDict_New();
if (container == NULL)
{
SWIG_exception(SWIG_RuntimeError, "error passing dictionary to Python");
SWIG_exception(SWIG_RuntimeError, "error passing a dictionary to Python");
}
// *&$1 -> $1 is the returned result being converted (unordered_map<...>*),
// wrapped by SwigValueWrapper. So we need to unwrap it using '&',
// then access its value using '*'.
for (auto it : *$1)
{
PyObject *returned_var = SWIG_NewPointerObj(SWIG_as_voidptr(new DATA_TYPE1(it.first)), _SWIG_TYPE1, SWIG_POINTER_OWN);
@ -964,6 +1073,7 @@ public:
%unordered_map_conversion(CNTK::Parameter, const CNTK::NDArrayViewPtr, SWIGTYPE_p_CNTK__Parameter, SWIGTYPE_p_std__shared_ptrT_CNTK__NDArrayView_t)
%unordered_map_conversion(CNTK::Parameter, CNTK::NDArrayViewPtr, SWIGTYPE_p_CNTK__Parameter, SWIGTYPE_p_std__shared_ptrT_CNTK__NDArrayView_t)
%unordered_map_conversion(CNTK::Variable, CNTK::StreamInformation, SWIGTYPE_p_CNTK__Variable, SWIGTYPE_p_CNTK__StreamInformation)
%unordered_map_conversion(CNTK::Variable, CNTK::MinibatchData, SWIGTYPE_p_CNTK__Variable, SWIGTYPE_p_CNTK__MinibatchData)
%unordered_map_ref_conversion(CNTK::StreamInformation, SWIGTYPE_p_CNTK__StreamInformation, CNTK::MinibatchData, SWIGTYPE_p_CNTK__MinibatchData);
%unordered_map_ref_conversion(CNTK::Parameter, SWIGTYPE_p_CNTK__Parameter, CNTK::NDArrayViewPtr, SWIGTYPE_p_std__shared_ptrT_CNTK__NDArrayView);
@ -986,7 +1096,24 @@ public:
%include "CNTKLibraryInternals.h"
%include "CNTKLibrary.h"
%inline {
%inline %{
// Trainer initializers.
// Because SWIG cannot properly handle smart pointers to derived classes (causes memory leak during the check for distributed learners),
// we need to redefine CreateTrainer.
CNTK::TrainerPtr TrainerImpl(const ::CNTK::FunctionPtr& model, const ::CNTK::FunctionPtr& lossFunction, const ::CNTK::FunctionPtr& evaluationFunction, const std::vector<CNTK::DistributedLearnerPtr>& parameterLearners)
{
std::vector<LearnerPtr> learners;
learners.reserve(parameterLearners.size());
for(const auto& l : parameterLearners)
learners.push_back(l);
return CreateTrainer(model, lossFunction, evaluationFunction, learners);
}
CNTK::TrainerPtr TrainerImpl(const ::CNTK::FunctionPtr& model, const ::CNTK::FunctionPtr& lossFunction, const ::CNTK::FunctionPtr& evaluationFunction, const std::vector<CNTK::LearnerPtr>& parameterLearners)
{
return CreateTrainer(model, lossFunction, evaluationFunction, parameterLearners);
}
// Global rank of current worker
size_t WorkerGlobalRank()
{
@ -998,7 +1125,7 @@ public:
{
return CNTK::MPICommunicator()->Workers().size();
}
}
%}
//
@ -1163,6 +1290,8 @@ public:
}
}
// end of NDArrayView
%template(NDArrayViewFloat) CNTK::NDArrayView::NDArrayView<float>;
%template(NDArrayViewDouble) CNTK::NDArrayView::NDArrayView<double>;
%template(ConstantFloat) CNTK::Constant::Constant<float>;
@ -1173,7 +1302,6 @@ public:
%template(random_uniform_double) CNTK::NDArrayView::RandomUniform<double>;
%template(DictionaryValueFromDict) CNTK::DictionaryValue::DictionaryValue<CNTK::Dictionary>;
// end of NDArrayView
%template(training_parameter_per_sample_schedule) CNTK::TrainingParameterPerUnitSchedule<double, CNTK::TrainingParameterSchedule<double>::UnitType::Sample>;
%template(training_parameter_per_minibatch_schedule) CNTK::TrainingParameterPerUnitSchedule<double, CNTK::TrainingParameterSchedule<double>::UnitType::Minibatch>;
@ -1185,16 +1313,44 @@ typedef CNTK::TrainingParameterPerUnitSchedule<size_t, CNTK::TrainingParameterSc
// The following callback code is only for testing. Will have to be merged with
// the operator classes.
//
%shared_ptr(CNTK::UserBackPropState)
%inline %{
class Callback {
public:
virtual ~Callback() { std::cout << "Callback::~Callback()" << std:: endl; }
virtual void forward() { std::cout << "Callback::forward()" << std::endl; }
virtual void backward() { std::cout << "Callback::backward()" << std::endl; }
};
namespace CNTK {
class UserBackPropState;
typedef std::shared_ptr<UserBackPropState> UserBackPropStatePtr;
class UserBackPropState : public BackPropState {
public:
UserBackPropState(const FunctionPtr& function, const DeviceDescriptor& computeDevice, PyObject* userData)
: BackPropState(function, computeDevice), m_userData(userData)
{ }
const PyObject* Data()
{
return m_userData;
}
static const PyObject* Data(BackPropStatePtr state)
{
CNTK::UserBackPropStatePtr user_state = std::dynamic_pointer_cast<CNTK::UserBackPropState>(state);
if (user_state == nullptr)
InvalidArgument("Invalid backprop state specified");
return user_state->Data();
}
private:
const PyObject* m_userData;
};
}
%}
//
// Release the GIL before calling into C++
//

291
bindings/python/cntk/core.py Normal file
Просмотреть файл

@ -0,0 +1,291 @@
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE.md file in the project root
# for full license information.
# ==============================================================================
import warnings
import numpy as np
from scipy import sparse
from . import cntk_py
from .device import use_default_device, cpu
from .utils.swig_helper import typemap
def _is_c_contiguous(data):
while isinstance(data, list):
data = data[0]
return data.flags.c_contiguous
class NDArrayView(cntk_py.NDArrayView):
'''
Creates an empty dense internal data representation of a :class:`Value` object.
To create an NDArrayView from a NumPy array, use :meth:`from_dense`.
To create an NDArrayView from a sparse array, use :meth:`from_csr`.
Args:
shape (tuple): shape of the data
data_type (np.float32, np.float64): data type of the data
device (:class:`~cntk.device.DeviceDescriptor`): device this value should be put
on
'''
def __init__(self, shape, data_type, device=None):
from .utils import sanitize_shape, sanitize_dtype_cntk
shape = sanitize_shape(shape)
data_type = sanitize_dtype_cntk(data_type)
if device is None:
device = use_default_device()
super(NDArrayView, self).__init__(data_type, cntk_py.StorageFormat_Dense, shape,
device)
@staticmethod
@typemap
def from_dense(np_array, device=None, read_only=False):
'''
Create a :class:`NDArrayView` instance from a NumPy array.
Args:
np_array (numpy.ndarray): NumPy array
device (:class:`~cntk.device.DeviceDescriptor`): device this value should be put
on
read_only (bool): whether the data can be modified or not
Returns:
:class:`NDArrayView` instance
'''
if not isinstance(np_array, np.ndarray):
raise TypeError('data must be of type numpy.ndarray'
' and not %s'%type(np_array))
if not _is_c_contiguous(np_array):
warnings.warn('data is not C contiguous; rearrange your data/computation to avoid this', RuntimeWarning)
if device is None:
device = use_default_device()
return cntk_py.NDArrayView(np_array, device, read_only)
@staticmethod
@typemap
def from_csr(csr_array, device=None, read_only=False):
'''
Create a :class:`NDArrayView` instance from a SciPy sparse array in CSR
format.
Args:
csr_array (scipy.sparse.csr.csr_matrix): SciPy sparse matrix in CSR
format
device (:class:`~cntk.device.DeviceDescriptor`): device this value should be put
on
read_only (bool): whether the data can be modified or not
Returns:
:class:`NDArrayView` instance
'''
if not sparse.isspmatrix_csr(csr_array):
raise TypeError("only CSR is supported as of now. Please "
"convert your data using 'tocsr()'")
if device is None:
device = use_default_device()
return cntk_py.NDArrayView(csr_array.shape, csr_array.data,
csr_array.indptr, csr_array.indices, device, read_only)
@staticmethod
@typemap
def from_data(data, device=None, read_only=False):
'''
Create a :class:`NDArrayView` instance from a NumPy or SciPy sparse array in CSR
format.
Args:
data (numpy.ndarray or scipy.sparse.csr.csr_matrix): data
device (:class:`~cntk.device.DeviceDescriptor`): device this value should be put
on
read_only (bool): whether the data can be modified or not
Returns:
:class:`NDArrayView` instance
'''
if isinstance(data, cntk_py.NDArrayView):
return data
if isinstance(data, np.ndarray):
ndav = NDArrayView.from_dense(data, device)
elif sparse.issparse(data):
ndav = NDArrayView.from_csr(data, device)
else:
raise TypeError('data type "%s" is not supported. Please '
'provide the data as a Python list of NumPy arrays '
'or Scipy CSR matrices.'%type(data))
return ndav
class Value(cntk_py.Value):
'''
Internal representation of minibatch data.
Args:
shape (tuple): shape of the value
value (None or value that can be cast to NumPy array): the value to
be converted
dtype: data type (np.float32 or np.float64)
batch: batch input for `var`.
It can be:
* a pure Python structure (list of lists, ...),
* a list of NumPy arrays or SciPy sparse CSR matrices
* a :class:`Value` object (e.g. returned by :func:`one_hot`)
seq_starts (list of `bool`s or None): if None, every sequence is
treated as a new sequence. Otherwise, it is interpreted as a list of
Booleans that tell whether a sequence is a new sequence (`True`) or a
continuation of the sequence in the same slot of the previous
minibatch (`False`)
device (:class:`~cntk.device.DeviceDescriptor`): device this value should be put
on
'''
def __init__(self, shape=None, dtype=None, batch=None, seq_starts=None, device=None):
if device is None:
device = use_default_device()
if shape and dtype:
# FIXME is this needed?
ndav = NDArrayView(shape, dtype, device)
elif batch:
if isinstance(batch, np.ndarray):
ndav = NDArrayView.from_dense(batch, device)
else:
ndav = batch
if seq_starts:
super(Value, self).__init__(ndav, seq_starts)
else:
super(Value, self).__init__(ndav)
@staticmethod
def _as_best_data_type(var, sample):
if isinstance(sample, list):
sample = np.asarray(sample, dtype=var.dtype)
if sample.dtype != var.dtype:
raise ValueError('could not convert sample data to '
'NumPy array')
if np.issubdtype(sample.dtype, int):
sample = sample.astype(var.dtype)
elif sample.dtype not in (np.float32, np.float64):
raise ValueError('only integer, float32 and float64 are supported, '
'you gave %s'%sample.dtype)
else:
sample = sample.astype(var.dtype)
return sample
@staticmethod
@typemap
def create(var, data, seq_starts=None, device=None, read_only=False):
'''
Creates a :class:`Value` object.
Args:
var (:class:`~cntk.ops.variables.Variable`): variable into which
``data`` is passed
data: data for `var`
It can be:
* a single NumPy array denoting the full minibatch
* a list of NumPy arrays or SciPy sparse CSR matrices
* a single NumPy array denoting one parameter or constant
seq_starts (list of `bool`s or None): if None, every sequence is
treated as a new sequence. Otherwise, it is interpreted as a list of
Booleans that tell whether a sequence is a new sequence (`True`) or a
continuation of the sequence in the same slot of the previous
minibatch (`False`)
device (:class:`~cntk.device.DeviceDescriptor`, default None): device
this value should be put on
read_only (bool, default False): whether the data is read only
Returns:
:class:`Value` object.
'''
if not isinstance(var, cntk_py.Variable):
raise TypeError('Variable expected, but got "%s"'%type(var))
cpu_dev = cpu()
if not var.dynamic_axes:
# No dynamic axes -> no batch
data = Value._as_best_data_type(var, data)
ndav = NDArrayView.from_data(data, cpu_dev)
return cntk_py.Value(ndav)
if isinstance(data, np.ndarray):
# The outermost axis has to be Python list. If the user passes a
# full minibatch as one NumPy array, we have to convert it.
if data.dtype == object:
raise ValueError('dtype object is not supported. If this is a batch '
'of sequences, you need to pass them as a pure-Python list '
'of NumPy arrays')
# FIXME if not seq_starts: directly pass it to Value constructor
data = list(data)
if not isinstance(data, list):
raise ValueError('batch has to be a list of NumPy arrays or '
'SciPy CSR matrices')
list_of_ndavs = []
# NDArrayViews are all created on CPU. The Value object later then will
# move it to the requested device.
for sample in data:
sample = Value._as_best_data_type(var, sample)
ndav = NDArrayView.from_data(sample, cpu_dev)
list_of_ndavs.append(ndav)
from .utils import sanitize_shape
return cntk_py.Value_create(
sanitize_shape(var.shape), list_of_ndavs,
seq_starts or [],
device or use_default_device(),
read_only)
@property
def shape(self):
'''
The rectangular shape of this value. I.e., if this value has sequences
of varying lengths, the shape will have the max sequence length in the
sequence dimension.
'''
return super(Value, self).shape().dimensions()
@property
def mask(self):
'''
The mask matrix of this value. Each row denotes a sequence with its
elements describing the mask of the element:
* 2: beginning of sequence (e.g. an LSTM would be reset)
* 1: valid element
* 0: invalid element
Example:
A mask of ``[[2, 1, 1], [1, 1, 0]]`` describes a batch of two
sequences. The first has three elements, of which the first element
(2) signals the beginning of a sequence. The second sequence has two
elements (last element marked 'invalid' by '0'). As it starts with
(1), it is a continuation of the 2nd sequence in the previous
minibatch.
'''
return np.asarray(super(Value, self).mask())
def __len__(self):
'''
Number of samples in this value object.
'''
return self.shape[0]

21
bindings/python/cntk/io/__init__.py
Просмотреть файл

@ -27,28 +27,28 @@ class MinibatchData(cntk_py.MinibatchData, ArrayMixin):
'''
The number of sequences in this minibatch
'''
return self.m_num_sequences
return self.number_of_sequences
@property
def num_samples(self):
'''
The number of samples in this minibatch
'''
return self.m_num_samples
return self.number_of_samples
@property
def value(self):
'''
The value of the minibatch as a NumPy array.
'''
return value_to_seq(self.m_data)
return value_to_seq(self.data)
@property
def shape(self):
'''
The shape of the data in this minibatch as tuple.
'''
return self.m_data.shape().dimensions()
return self.data.shape().dimensions()
@property
def mask(self):
@ -57,14 +57,23 @@ class MinibatchData(cntk_py.MinibatchData, ArrayMixin):
sequence, `1` marks a sequence element as valid, and `0` marks it as
invalid.
'''
return self.m_data.mask().to_ndarray()
return self.data.mask().to_ndarray()
@property
def end_of_sweep(self):
'''
Indicates whether the data in this minibatch is comes from a sweep end
or crosses a sweep boundary (and as a result includes data from
different sweeps).
'''
return self.sweep_end
@property
def is_sparse(self):
'''
Whether the data in this minibatch is sparse.
'''
return self.m_data.is_sparse()
return self.data.is_sparse()
def __len__(self):
return self.num_sequences

69
bindings/python/cntk/io/tests/io_tests.py
Просмотреть файл

@ -9,15 +9,13 @@ import os
import numpy as np
import pytest
from cntk.io import _is_tensor, sequence_to_cntk_text_format
from cntk.io import *
abs_path = os.path.dirname(os.path.abspath(__file__))
AA = np.asarray
def test_text_format(tmpdir):
from cntk.io import CTFDeserializer, MinibatchSource, StreamDef, StreamDefs
mbdata = r'''0 |x 560:1 |y 1 0 0 0 0
0 |x 0:1
0 |x 0:1
@ -48,6 +46,9 @@ def test_text_format(tmpdir):
features = mb[features_si]
# 2 samples, max seq len 4, 1000 dim
assert features.shape == (2, 4, input_dim)
assert features.end_of_sweep
assert features.num_sequences == 2
assert features.num_samples == 7
assert features.is_sparse
# TODO features is sparse and cannot be accessed right now:
# *** RuntimeError: DataBuffer/WritableDataBuffer methods can only be called for NDArrayiew objects with dense storage format
@ -58,6 +59,9 @@ def test_text_format(tmpdir):
labels = mb[labels_si]
# 2 samples, max seq len 1, 5 dim
assert labels.shape == (2, 1, num_output_classes)
assert labels.end_of_sweep
assert labels.num_sequences == 2
assert labels.num_samples == 2
assert not labels.is_sparse
label_data = np.asarray(labels)
@ -67,8 +71,16 @@ def test_text_format(tmpdir):
[[ 0., 1., 0., 0., 0.]]
]))
mb = mb_source.next_minibatch(1)
features = mb[features_si]
labels = mb[labels_si]
assert not features.end_of_sweep
assert not labels.end_of_sweep
assert features.num_samples < 7
assert labels.num_samples == 1
def test_image():
from cntk.io import ReaderConfig, ImageDeserializer
map_file = "input.txt"
mean_file = "mean.txt"
epoch_size = 150
@ -153,7 +165,7 @@ def test_image():
assert set(sis.keys()) == { feature_name, label_name }
'''
def test_minibatch(tmpdir):
def test_full_sweep_minibatch(tmpdir):
mbdata = r'''0 |S0 0 |S1 0
0 |S0 1 |S1 1
@ -168,10 +180,10 @@ def test_minibatch(tmpdir):
with open(tmpfile, 'w') as f:
f.write(mbdata)
from cntk.io import CTFDeserializer, MinibatchSource, StreamDef, StreamDefs
mb_source = MinibatchSource(CTFDeserializer(tmpfile, StreamDefs(
features = StreamDef(field='S0', shape=1),
labels = StreamDef(field='S1', shape=1))))
labels = StreamDef(field='S1', shape=1))),
randomize=False, epoch_size=FULL_DATA_SWEEP)
features_si = mb_source.stream_info('features')
labels_si = mb_source.stream_info('labels')
@ -181,6 +193,7 @@ def test_minibatch(tmpdir):
assert mb[labels_si].num_sequences == 2
features = mb[features_si]
assert features.end_of_sweep
assert len(features.value) == 2
expected_features = \
[
@ -196,6 +209,7 @@ def test_minibatch(tmpdir):
[2, 1, 1, 0]])
labels = mb[labels_si]
assert labels.end_of_sweep
assert len(labels.value) == 2
expected_labels = \
[
@ -209,6 +223,46 @@ def test_minibatch(tmpdir):
[[2, 1, 1],
[2, 1, 0]])
def test_large_minibatch(tmpdir):
mbdata = r'''0 |S0 0 |S1 0
0 |S0 1 |S1 1
0 |S0 2
0 |S0 3 |S1 3
0 |S0 4
0 |S0 5 |S1 1
0 |S0 6 |S1 2
'''
tmpfile = str(tmpdir/'mbtest.txt')
with open(tmpfile, 'w') as f:
f.write(mbdata)
mb_source = MinibatchSource(CTFDeserializer(tmpfile, StreamDefs(
features = StreamDef(field='S0', shape=1),
labels = StreamDef(field='S1', shape=1))),
randomize=False)
features_si = mb_source.stream_info('features')
labels_si = mb_source.stream_info('labels')
mb = mb_source.next_minibatch(1000)
features = mb[features_si]
labels = mb[labels_si]
# Actually, the minibatch spans over multiple sweeps,
# not sure if this is an artificial situation, but
# maybe instead of a boolean flag we should indicate
# the largest sweep index the data was taken from.
assert features.end_of_sweep
assert labels.end_of_sweep
assert features.num_samples == 1000 - 1000 % 7
assert labels.num_samples == 5 * (1000 // 7)
assert mb[features_si].num_sequences == (1000 // 7)
assert mb[labels_si].num_sequences == (1000 // 7)
@pytest.mark.parametrize("idx, alias_tensor_map, expected", [
(0, {'A': [object()]}, ValueError),
@ -250,4 +304,5 @@ def test_sequence_conversion_dense(idx, alias_tensor_map, expected):
([AA([1, 2]), AA([])], False),
])
def test_is_tensor(data, expected):
from cntk.io import _is_tensor
assert _is_tensor(data) == expected

37
bindings/python/cntk/learner.py
Просмотреть файл

@ -4,7 +4,7 @@
# ==============================================================================
import math
from . import cntk_py
from . import cntk_py, NDArrayView
from .utils import typemap
from enum import Enum, unique
import numpy as np
@ -99,8 +99,7 @@ class Learner(cntk_py.Learner):
Returns:
`False` to indicate that learning has stopped for all of the parameters associated with this learner
'''
from .utils import _create_NDArrayView_from_NumPy
var_nd_map = { var: _create_NDArrayView_from_NumPy(val) for var, val in
var_nd_map = { var: NDArrayView.from_data(val) for var, val in
gradient_values.items() }
return super(Learner, self).update(var_nd_map, training_sample_count)
@ -131,7 +130,7 @@ class Learner(cntk_py.Learner):
return super(Learner, self).learning_rate()
@typemap
def training_parameter_schedule(schedule, unit, epoch_size=1):
def training_parameter_schedule(schedule, unit, epoch_size=None):
'''
Create a training parameter schedule containing either per-sample (default)
or per-minibatch values.
@ -161,8 +160,13 @@ def training_parameter_schedule(schedule, unit, epoch_size=1):
unit (:class:`UnitType`): one of two
* ``sample``: the returned schedule contains per-sample values
* ``minibatch``: the returned schedule contains per-minibatch values.
epoch_size (int): number of samples as a scheduling unit. Parameters in
the schedule change their values every ``epoch_size`` samples.
epoch_size (optional, int): number of samples as a scheduling unit.
Parameters in the schedule change their values every ``epoch_size``
samples. If no ``epoch_size`` is provided, this parameter is substituted
by the size of the full data sweep, in which case the scheduling unit is
the entire data sweep (as indicated by the MinibatchSource) and parameters
change their values on the sweep-by-sweep basis specified by the
``schedule``.
Returns:
training parameter schedule
@ -178,7 +182,7 @@ def training_parameter_schedule(schedule, unit, epoch_size=1):
return schedule
if isinstance(schedule, (int, float)):
if epoch_size != 1:
if epoch_size is not None:
raise ValueError('when providing the schedule as a number,'
' epoch_size is ignored')
if UnitType(unit) is UnitType.sample:
@ -186,16 +190,18 @@ def training_parameter_schedule(schedule, unit, epoch_size=1):
else:
return cntk_py.training_parameter_per_minibatch_schedule(schedule)
args = [schedule] if epoch_size is None else [schedule, epoch_size]
if isinstance(schedule, list):
if UnitType(unit) is UnitType.sample:
return cntk_py.training_parameter_per_sample_schedule(schedule, epoch_size)
return cntk_py.training_parameter_per_sample_schedule(*args)
else:
return cntk_py.training_parameter_per_minibatch_schedule(schedule, epoch_size)
return cntk_py.training_parameter_per_minibatch_schedule(*args)
raise ValueError('schedule must be either a float or a list, not %s'%type(schedule))
@typemap
def learning_rate_schedule(lr, unit, epoch_size=1):
def learning_rate_schedule(lr, unit, epoch_size=None):
'''
Create a learning rate schedule (using the same semantics as
:func:`training_parameter_schedule`).
@ -217,7 +223,7 @@ def learning_rate_schedule(lr, unit, epoch_size=1):
return training_parameter_schedule(lr, unit, epoch_size)
@typemap
def momentum_schedule(momentum, epoch_size=1):
def momentum_schedule(momentum, epoch_size=None):
'''
Create a per-minibatch momentum schedule (using the same semantics as
:func:`training_parameter_schedule` with the `unit=UnitType.minibatch`).
@ -254,7 +260,7 @@ def momentum_schedule(momentum, epoch_size=1):
return training_parameter_schedule(momentum, UnitType.minibatch, epoch_size)
@typemap
def momentum_as_time_constant_schedule(momentum, epoch_size=1):
def momentum_as_time_constant_schedule(momentum, epoch_size=None):
'''
Create a momentum schedule in a minibatch-size agnostic way
(using the same semantics as :func:`training_parameter_schedule`
@ -289,9 +295,14 @@ def momentum_as_time_constant_schedule(momentum, epoch_size=1):
return momentum
if isinstance(momentum, (int, float)):
if epoch_size is not None:
raise ValueError('when providing the schedule as a number,'
' epoch_size is ignored')
return cntk_py.momentum_as_time_constant_schedule(momentum)
if isinstance(momentum, list):
return cntk_py.momentum_as_time_constant_schedule(momentum, epoch_size)
args = [momentum] if epoch_size is None else [momentum, epoch_size]
return cntk_py.momentum_as_time_constant_schedule(*args)
raise ValueError('momentum must be either a float or a list, not %s'%type(momentum))

59
bindings/python/cntk/ops/__init__.py
Просмотреть файл

@ -1722,7 +1722,7 @@ def reshape(x, shape, begin_axis=None, end_axis=None, name=''):
The output tensor has the shape specified by 'shape'.
Examples:
Example:
>>> i1 = C.input_variable(shape=(3,2))
>>> C.reshape(i1, (2,3)).eval({i1:np.asarray([[[[0., 1.],[2., 3.],[4., 5.]]]], dtype=np.float32)})
array([[[[ 0., 1., 2.],
@ -1773,7 +1773,7 @@ def transpose(x, axis1=0, axis2=1, name=''):
Swaps two axes of the tensor. The output tensor has the same data but with
``axis1`` and ``axis2`` swapped.
Examples:
Example:
>>> C.transpose([[[0,1],[2,3],[4,5]]], 1, 2).eval()
array([[[ 0., 2., 4.],
[ 1., 3., 5.]]], dtype=float32)
@ -1798,7 +1798,7 @@ def slice(x, axis, begin_index, end_index, name=''):
'''
Slice the input along an axis.
Examples:
Example:
>>> # Slice using input variable
>>> # create 2x3 matrix
>>> x1 = C.input_variable((2,3))
@ -1824,8 +1824,8 @@ def slice(x, axis, begin_index, end_index, name=''):
NumPy's way of slicing works, too:
Examples:
TODO: Make following lines work. Uncomment when done
Example:
#TODO: Make following lines work. Uncomment when done
#>>> x1[1].eval()
#array([[ 4., 5., 6.]], dtype=float32)
#>>> x1[:,:2,:].eval()
@ -1859,7 +1859,7 @@ def splice(inputs, axis=-1, name=''):
'''
Concatenate the input tensors along an axis.
Examples:
Example:
>>> # create 2x2 matrix in a sequence of length 1 in a batch of one sample
>>> data1 = np.asarray([[[1, 2],
... [4, 5]]], dtype=np.float32)
@ -1908,7 +1908,7 @@ def reduce_sum(x, axis=None, name=''):
is not specified then the sum will be computed over all axes, that is, the output is a scalar,
which is the sum of tensor's elements.
Examples:
Example:
>>> # create 3x2 matrix in a sequence of length 1 in a batch of one sample
>>> data = [[10, 20],[30, 40],[50, 60]]
@ -1957,7 +1957,7 @@ def reduce_log_sum(x, axis=None, name=''):
Computes the log of the sum of the exponentiations of the input tensor's
elements across the specified axis.
Examples:
Example:
>>> x = C.input_variable(shape=(3,2))
>>> val = np.reshape(np.arange(6.0, dtype=np.float32), (3,2))
>>> lse = C.reduce_log_sum(x)
@ -1985,7 +1985,7 @@ def reduce_mean(x, axis=None, name=''):
'''
Computes the mean of the input tensor's elements across the specified axis.
Examples:
Example:
>>> # create 3x2 matrix in a sequence of length 1 in a batch of one sample
>>> data = [[5, 20],[30, 40],[55, 60]]
@ -2016,7 +2016,7 @@ def reduce_max(x, axis=None, name=''):
'''
Computes the max of the input tensor's elements across the specified axis.
Examples:
Example:
>>> # create 3x2 matrix in a sequence of length 1 in a batch of one sample
>>> data = [[10, 20],[30, 40],[50, 60]]
@ -2047,7 +2047,7 @@ def reduce_min(x, axis=None, name=''):
'''
Computes the min of the input tensor's elements across the specified axis.
Examples:
Example:
>>> # create 3x2 matrix in a sequence of length 1 in a batch of one sample
>>> data = [[10, 20],[30, 40],[50, 60]]
@ -2133,7 +2133,7 @@ def random_sample_inclusion_frequency(
allow_duplicates (bool): If sampling is done
with replacement (`True`) or without (`False`).
Examples:
Example:
>>> import numpy as np
>>> from cntk import *
>>> # weight vector with 100 '1000'-values followed
@ -2179,7 +2179,7 @@ def dropout(x, dropout_rate=0.0, name=''):
In CNTK's implementation, because the values that are not set to 0 are multiplied
with (1 / (1 - ``dropout_rate``)), this is not necessary.
Examples:
Example:
>>> data = [[10, 20],[30, 40],[50, 60]]
>>> C.dropout(data, 0.5).eval() # doctest: +SKIP
array([[ 0., 40.],
@ -2252,6 +2252,35 @@ def input_variable(shape, dtype=np.float32, needs_gradient=False, is_sparse=Fals
return input_variable(shape, is_sparse, dtype, needs_gradient, name, dynamic_axes)
@typemap
def output_variable(shape, dtype, dynamic_axes, name=''):
'''
It creates an output node that is used to define a user defined function.
Args:
shape (tuple or int): the shape of the input tensor
dtype (type): np.float32 or np.float64
dynamic_axes (list or tuple): a list of dynamic axis (e.g., batch axis, time axis)
name (str, optional): the name of the Function instance in the network
Returns:
:class:`~cntk.ops.variables.Variable` that is of output type
'''
from cntk.cntk_py import output_variable
from ..utils import sanitize_shape, sanitize_dtype_cntk
shape = sanitize_shape(shape)
dtype = sanitize_dtype_cntk(dtype)
for a in dynamic_axes:
if not a.is_dynamic_axis:
raise ValueError('axis in dynamic_axes attribute is not dynamic')
dynamic_axes = list(reversed(dynamic_axes))
return output_variable(shape, dtype, dynamic_axes, name)
@typemap
def placeholder_variable(shape=None, dynamic_axes=None, name=''):
'''
@ -2284,7 +2313,7 @@ def parameter(shape=None, init=None, dtype=None, device=None, name=''):
'''
It creates a parameter tensor.
Examples:
Example:
>>> init_parameter = C.parameter(shape=(3,4), init=2)
>>> np.asarray(init_parameter) # doctest: +SKIP
array([[ 2., 2., 2., 2.],
@ -2328,7 +2357,7 @@ def constant(value=None, shape=None, device=None, name=''):
'''
It creates a constant tensor initialized from a numpy array
Examples
Example:
>>> constant_data = C.constant([[1., 2.], [3., 4.], [5., 6.]])
>>> constant_data.value
array([[ 1., 2.],

142
bindings/python/cntk/ops/functions.py
Просмотреть файл

@ -1,6 +1,8 @@
from cntk import cntk_py
from cntk.device import DeviceDescriptor
from cntk.utils import typemap, sanitize_var_map, value_to_seq
from cntk.utils import typemap, sanitize_var_map, sanitize_batch, \
sanitize_dtype_cntk, value_to_seq
from cntk.utils.swig_helper import map_if_possible
from enum import Enum, unique
import numpy as np
@ -29,7 +31,6 @@ class CloneMethod(Enum):
(e.g. for use as a fixed feature extractor)
'''
class Function(cntk_py.Function):
'''
Base class of all primitive tensor operators.
@ -38,7 +39,6 @@ class Function(cntk_py.Function):
will relay to its only output.
'''
# define input shapes, in-place
# e.g.
# model.declare_args(42)
@ -96,11 +96,22 @@ class Function(cntk_py.Function):
try:
return self.__dict__[name]
except KeyError:
if len(self.outputs) == 1:
return getattr(self.output, name)
# If name is a member of self's single output, then we relay to
# that.
if name in ['outputs', 'output', 'this']:
# 'outputs' and 'output' are required to fetch the attribute for
# in the Variable.
# 'this' is required for Swig and needs to be thrown if the
# object is created the first time.
# All others we try to find in self.output.
raise
if len(self.outputs) == 1 and hasattr(self.output, name):
return getattr(self.output, name)
else:
raise AttributeError("'%s' object has no attribute '%s'" %
(type(self), name))
raise AttributeError("'%s' object has no attribute '%s'" %
(type(self), name))
@property
@typemap
@ -263,8 +274,8 @@ class Function(cntk_py.Function):
computation is. If `None`, the default device is used.
Returns:
A tuple (BackpropState, map of outputs to NumPy arrays). The
BackpropState is a handle taken by :func:`backward`.
A tuple (BackPropState, map of outputs to NumPy arrays). The
BackPropState is a handle taken by :func:`backward`.
'''
if device is None:
device = DeviceDescriptor.use_default_device()
@ -366,9 +377,9 @@ class Function(cntk_py.Function):
unique_wrt = set(wrt)
output = [self.output]
df, f = self.forward(at, output, set(output), device)
ones = {self.output: np.ones_like(v) for v in f.values()}
grad_dict = self.backward(df, ones, unique_wrt)
state, results = self.forward(at, output, set(output), device)
ones = {self.output: np.ones_like(v) for v in results.values()}
grad_dict = self.backward(state, ones, unique_wrt)
return [grad_dict[v] for v in wrt]
@property
@ -534,7 +545,7 @@ class Function(cntk_py.Function):
def find_all_with_name(self, name):
'''
Returns a list of primitive function with ``name`` in the graph
starting from this node. Throws an exceptoin if ``name`` occurs
starting from this node. Throws an exception if ``name`` occurs
multiple times. If you expect only one function to be returned, use
:func:`find_by_name`.
@ -564,7 +575,7 @@ class Function(cntk_py.Function):
def find_by_name(self, name):
'''
Returns a primitive function with ``name`` in the graph starting from
this node. Throws an exceptoin if ``name`` occurs multiple times. If
this node. Throws an exception if ``name`` occurs multiple times. If
you expect multiple functions to be returned, use
:func:`find_all_with_name`.
@ -599,7 +610,8 @@ class Function(cntk_py.Function):
@typemap
def save_model(self, filename):
'''
Save this function graph into a model file using protobuf-based serialization.
Save this function graph into a model file using protobuf-based
serialization.
Args:
filename (str): model path
@ -619,15 +631,111 @@ class Function(cntk_py.Function):
'''
return super(Function, self).restore_model(filename)
class UserFunction(Function):
'''
Base class of all user extension functions.
If it has only one output, one can invoke Variable methods on it, which it
will relay to its only output.
'''
def __init__(self, inputs, outputs, name=''):
var_inputs = []
# TODO: this should be done in Swig
for i in inputs:
if isinstance(i, cntk_py.Variable):
var_inputs.append(i)
elif isinstance(i, cntk_py.Function):
var_inputs.append(i.output)
else:
raise ValueError('expected Variable, but got "%s"'%type(i))
super(Function, self).__init__(var_inputs, outputs, name)
def _forward(self, arguments, outputs, device=None, outputs_to_retain=None):
'''
Computes the values of speficied variables in ``outputs``, using values
provided in ``arguments`` that correspond to each input `Variable` of
the function whose ``is_input`` is `True`.
This function calls :func:`forward`, which is to be implemented by the
user.
Args:
arguments (tuple): Value objects of the Function's input
outputs (iterable): outputs to fetch values for.
device (:class:`~cntk.device.DeviceDescriptor`, default `None`): the device
descriptor that contains the type and id of the device on which the
computation is. If `None`, the default device is used.
Returns:
A BackPropState instance, which is used by :func:`backward`.
'''
arguments = tuple(value_to_seq(v) for v in arguments)
map_if_possible(outputs)
map_if_possible(outputs_to_retain)
state, results = self.forward(arguments, outputs, device, outputs_to_retain)
if not isinstance(state, cntk_py.BackPropState):
state = cntk_py.UserBackPropState(self, device, state)
for k,v in outputs.items():
if v is None:
raise ValueError('not all outputs have been provided')
# FIXME: seq_starts
outputs[k] = sanitize_batch(k, v, None, device)
return state, results
def _backward(self, state, root_gradients, variables):
'''
Backpropagates supplied ``root_gradients`` for one or more of the output
variables of the Function, to calculate gradients with respect to
``variables``. Formally, multiplies the values of ``root_gradients`` by
the Jacobian of the Function and returns the subset of the output that
corresponds to ``variables``.
This function calls :func:`backward`, which is to be implemented by the
user.
Example:
TBD
Args:
state (BackPropState): state obtained from a previous call to the
func:`cntk.ops.Function.forward` method on this Function for the
computation that this gradient backpropagation corresponds to.
root_gradients (dict): the gradients that will be backpropagated
variables (set): a list of input variables with respect to which
the gradients have to be computed.
Returns:
dict: mapping of ``variables`` to NumPy arrays
'''
for v in root_gradients:
root_gradients[v] = value_to_seq(root_gradients[v])
map_if_possible(variables)
self.backward(cntk_py.UserBackPropState.data(state), root_gradients, variables)
for k,v in variables.items():
if v is None:
raise ValueError('gradients were not provided for all variables')
variables[k] = sanitize_batch(k, v, None, state.device())
@typemap
def load_model(filename, device=None):
'''
Load the model in ``filename``, that has been saved using
`:func:save_model`.
:func:`~cntk.ops.functions.Function.save_model`.
Args:
filename (str): filename to load the model from
device (:class:`~cntk.DeviceDescriptor`, default is the default device):
device (:class:`~cntk.device.DeviceDescriptor`, default is the default device):
instance of DeviceDescriptor
Returns:

1
bindings/python/cntk/ops/tests/function_tests.py
Просмотреть файл

@ -180,4 +180,3 @@ def test_sequence_data_mismatch():
with pytest.raises(ValueError):
y_broadcast_first_result = y_broadcast_first.eval({x:[x0], ones:[o0]})

256
bindings/python/cntk/ops/tests/userfunction_test.py Normal file
Просмотреть файл

@ -0,0 +1,256 @@
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE.md file in the project root
# for full license information.
# ==============================================================================
"""
Unit tests for function extension
"""
from __future__ import division, print_function
import numpy as np
import pytest
from cntk import *
from cntk.trainer import *
from cntk.learner import *
from cntk.ops.functions import UserFunction
class Plus3Func(UserFunction):
def __init__(self, arg, name='f1'):
outputs = [output_variable(arg.shape, arg.dtype, arg.dynamic_axes)]
super(Plus3Func, self).__init__([arg], outputs,
name=name)
self.forward_calls = 0
self.backward_calls = 0
def forward(self, arguments, outputs, device=None, outputs_to_retain=None):
assert len(self.inputs)==1
assert len(outputs)==1
for k in outputs:
outputs[k] = arguments[0] + 3
break
self.forward_calls += 1
return None, outputs
def backward(self, state, root_gradients, variables):
assert len(root_gradients) == 1
assert len(variables) == 1
for rk, rv in root_gradients.items():
break
for var_key in variables:
break
self.backward_calls += 1
variables[var_key] = rv
def test_ext_eval_1():
dim = 4
p = parameter(shape=(dim,), init=10, name='p')
i = input_variable(dim, needs_gradient=True, name='i_var')
m = Plus3Func(i)
z = m+p
input_data = np.random.rand(dim)
result = z.eval([input_data])
assert np.allclose(result[0][0], input_data+3+10)
def test_ext_eval_2_only_param():
dim = 4
p = parameter(shape=(dim,), init=10, name='p')
i = input_variable(dim, needs_gradient=True, name='i_var')
m = Plus3Func(p)
# combine does not work
# z = combine([m.output])
z = m+i
input_data = np.random.rand(dim)
result = z.eval([input_data])
assert np.allclose(result[0][0], input_data+3+10)
def test_ext_eval_3_no_input():
dim = 4
p = parameter(shape=(dim,), init=10, name='p')
m = Plus3Func(p)
z = m+0
result = z.eval()
# No batch dimension since we have no input
assert np.allclose(result, np.zeros_like(p)+10+3)
def test_ext_eval_4_a_inside_graph():
dim = 4
p_init = 10
p = parameter(shape=(dim,), init=p_init, name='p')
m = Plus3Func(p)
z = p * m
result = z.eval()
# No batch dimension since we have no input
assert np.allclose(result, ((p_init*np.ones_like(result))+3)*p_init)
def _test_ext_eval_4_b_inside_graph():
dim = 4
p_init = 10
p = parameter(shape=(dim,), init=p_init, name='p')
z = p * Plus3Func(p)
result = z.eval()
# No batch dimension since we have no input
assert np.allclose(result, ((p_init*np.ones_like(result))+3)*p_init)
def test_ext_eval_5_times():
dim = 2
p_init = 10
p = parameter(shape=(dim,), init=p_init, name='p')
m = Plus3Func(p)
z = times(m, parameter(shape=(2,50), init=2))
result = z.eval()
# No batch dimension since we have no input
assert np.allclose(result, ((p_init*np.ones_like(result))+3)*2*2)
# TODO change to real training example
def test_ext_train():
dim = 4
p = parameter(shape=(dim,), init=10)
i = input_variable(dim, needs_gradient=True, name='i_var')
m = Plus3Func(i)
z = m+p
momentum_time_constant = momentum_as_time_constant_schedule(1100)
lr_per_sample = learning_rate_schedule(0.007, UnitType.sample)
trainer = Trainer(z, z+0, z+0, \
[momentum_sgd(z.parameters, lr_per_sample, momentum_time_constant,
True)])
i = 0
while i<100:
i+=1
input_data = np.random.rand(dim)
trainer.train_minibatch([input_data])
assert m.forward_calls == m.backward_calls == 100
@pytest.mark.parametrize("payload", [
(77,),
("a", 2),
(),
(None)
])
def test_ext_backpropstate(payload):
class TestBackPropState(UserFunction):
def __init__(self, arg, payload, name='f1'):
outputs = [output_variable(arg.shape, arg.dtype, arg.dynamic_axes)]
self.payload = payload
super(TestBackPropState, self).__init__([arg], outputs)
def forward(self, arguments, outputs, device=None, outputs_to_retain=None):
for k in outputs:
outputs[k] = arguments[0]
break
return self.payload, outputs
def backward(self, state, root_gradients, variables):
assert state == self.payload
for rk, rv in root_gradients.items():
break
for var_key in variables:
break
variables[var_key] = rv
dim = 4
p = parameter(shape=(dim,), init=10)
i = input_variable(dim, needs_gradient=True, name='i_var')
m = TestBackPropState(i, payload)
z = m+p
momentum_time_constant = momentum_as_time_constant_schedule(1100)
lr_per_sample = learning_rate_schedule(0.007, UnitType.sample)
trainer = Trainer(z, z+0, z+0, \
[momentum_sgd(z.parameters, lr_per_sample, momentum_time_constant,
True)])
i = 0
input_data = np.random.rand(dim)
trainer.train_minibatch([input_data])
class LambdaFunc(UserFunction):
def __init__(self,
arg,
when=lambda arg: True,
execute=lambda arg: print(arg),
name=''):
self.when = when
self.execute = execute
outputs = [output_variable(arg.shape, arg.dtype, arg.dynamic_axes)]
super(LambdaFunc, self).__init__([arg], outputs,
name=name)
def forward(self, arguments, outputs, device=None, outputs_to_retain=None):
if len(arguments)!=1:
raise ValueError('LambdaFunc expects exactly one input')
if self.when(arguments):
self.execute(arguments)
for k in outputs:
outputs[k] = arguments[0]
break
return None, outputs
def backward(self, state, root_gradients, variables):
for rk, rv in root_gradients.items():
break
for var_key in variables:
break
variables[var_key] = rv
def test_ext_lambdafunc():
dim = 4
class CallbackCounter(object):
def __init__(self):
self.count = 0
def inc(self, arg):
self.count += 1
cb = CallbackCounter()
p = parameter(shape=(dim,), init=1)
i = input_variable(dim, needs_gradient=True, name='i_var')
k = i*p
m = LambdaFunc(k,
when=lambda arg: np.sum(arg)>1,
execute=cb.inc)
z = m+0
momentum_time_constant = momentum_as_time_constant_schedule(1100)
lr_per_sample = learning_rate_schedule(0.007, UnitType.sample)
trainer = Trainer(z, z+0, z+0, \
[momentum_sgd(z.parameters, lr_per_sample, momentum_time_constant,
True)])
i = 0
input_data = 0.1 * np.ones(dim)
trainer.train_minibatch([input_data])
assert cb.count == 0
input_data = 0.3 * np.ones(dim)
trainer.train_minibatch([input_data])
assert cb.count == 1

17
bindings/python/cntk/ops/variables.py
Просмотреть файл

@ -1,7 +1,8 @@
import numpy as np
from cntk import cntk_py, utils
from cntk import cntk_py, NDArrayView
from ..tensor import TensorOpsMixin
from ..utils import typemap, sanitize_precision, sanitize_value, sanitize_dtype_cntk, _create_NDArrayView_from_NumPy
from ..utils import typemap, sanitize_precision, sanitize_value, \
sanitize_shape, sanitize_dtype_cntk
class VariableMixin(object):
'''
@ -121,11 +122,11 @@ class Variable(VariableMixin, TensorOpsMixin, cntk_py.Variable):
'''
def __init__(self, shape=None, dtype=None, needs_gradient=False, is_sparse=False,
dynamic_axes=[cntk_py.Axis.default_dynamic_axis(), cntk_py.Axis.default_batch_axis()], name=''):
shape = utils.sanitize_shape(shape)
shape = sanitize_shape(shape)
if dtype is None:
dtype = np.float32
dtype = utils.sanitize_dtype_cntk(dtype)
dtype = sanitize_dtype_cntk(dtype)
super(Variable, self).__init__(shape, is_sparse, dtype, needs_gradient, name,
dynamic_axes)
@ -166,8 +167,8 @@ class Parameter(VariableMixin, TensorOpsMixin, cntk_py.Parameter):
ndav = sanitize_value(shape, init, dtype, device)
super(Parameter, self).__init__(ndav, name)
else:
shape = utils.sanitize_shape(shape)
cntk_dtype = utils.sanitize_dtype_cntk(dtype)
shape = sanitize_shape(shape)
cntk_dtype = sanitize_dtype_cntk(dtype)
super(Parameter, self).__init__(shape, cntk_dtype, init,
device, name)
@ -181,7 +182,7 @@ class Parameter(VariableMixin, TensorOpsMixin, cntk_py.Parameter):
@value.setter
def value(self, val):
if isinstance(val, np.ndarray):
ndarray = _create_NDArrayView_from_NumPy(val.astype(self.dtype))
ndarray = NDArrayView.from_dense(val.astype(self.dtype))
super(Parameter, self).set_value(ndarray)
elif isinstance(val, cntk_py.NDArrayView):
super(Parameter, self).set_value(val)
@ -212,7 +213,7 @@ class Constant(VariableMixin, TensorOpsMixin, cntk_py.Constant):
dtype = np.float32
if np.isscalar(value):
super(Constant, self).__init__(utils.sanitize_shape(shape), sanitize_dtype_cntk(dtype), value)
super(Constant, self).__init__(sanitize_shape(shape), sanitize_dtype_cntk(dtype), value)
else:
ndav = sanitize_value(shape, value, dtype, device)
super(Constant, self).__init__(ndav, name)

11
bindings/python/cntk/tensor.py
Просмотреть файл

@ -166,15 +166,16 @@ class ArrayMixin(object):
@property
def __array_interface__(self):
try:
# This first check is for a Value object. Trying with self.to_ndarray first would lead to
# a infinite recursion, since Value has a to_ndarray method
np_array = self.data().to_ndarray()
# This checks for a MinibatchData object.
np_array = self.value
except AttributeError:
try:
np_array = self.to_ndarray()
# This checks for a Value object. Trying with self.to_ndarray first would lead to
# a infinite recursion, since Value has a to_ndarray method
np_array = self.data().to_ndarray()
except AttributeError:
try:
np_array = self.value
np_array = self.to_ndarray()
except AttributeError:
# Ideally an exception would be raised here, but getattr would swallow it
# so we return None

65
bindings/python/cntk/tests/learner_test.py
Просмотреть файл

@ -96,7 +96,7 @@ def test_learner_update():
w = parameter(shape=(1,), init=w_init)
res = i * w
learner = sgd(res.parameters, lr=learning_rate_schedule([0.1]*50 + [0.2]*50, UnitType.sample))
learner = sgd(res.parameters, lr=learning_rate_schedule([0.1]*50 + [0.2]*50, UnitType.sample, 1))
assert learner.learning_rate() == 0.1
x = learner.update({w: np.asarray([[2.]], dtype=np.float32)}, 100)
assert learner.learning_rate() == 0.2
@ -110,3 +110,66 @@ def test_training_parameter_schedule():
training_parameter_schedule(0.01, unit='not_supported')
with pytest.raises(ValueError):
training_parameter_schedule(0.01, unit=5)
def test_sweep_based_schedule(tmpdir, device_id):
from cntk.io import MinibatchSource, CTFDeserializer, StreamDef, StreamDefs
from .. import cross_entropy_with_softmax, classification_error, plus, reduce_sum
from ..trainer import Trainer
input_dim = 69
ctf_data = '''\
0 |S0 3:1 |S1 3:1 |# <s>
0 |S0 4:1 |# A |S1 32:1 |# ~AH
0 |S0 5:1 |# B |S1 36:1 |# ~B
0 |S0 4:1 |# A |S1 31:1 |# ~AE
0 |S0 7:1 |# D |S1 38:1 |# ~D
0 |S0 12:1 |# I |S1 47:1 |# ~IY
0 |S0 1:1 |# </s> |S1 1:1 |# </s>
2 |S0 60:1 |# <s> |S1 3:1 |# <s>
2 |S0 61:1 |# A |S1 32:1 |# ~AH
'''
ctf_file = str(tmpdir/'2seqtest.txt')
with open(ctf_file, 'w') as f:
f.write(ctf_data)
mbs = MinibatchSource(CTFDeserializer(ctf_file, StreamDefs(
features = StreamDef(field='S0', shape=input_dim, is_sparse=True),
labels = StreamDef(field='S1', shape=input_dim, is_sparse=True)
)), randomize=False)
in1 = input_variable(shape=(input_dim,))
labels = input_variable(shape=(input_dim,))
p = parameter(shape=(input_dim,), init=10)
z = plus(in1, reduce_sum(p), name='z')
ce = cross_entropy_with_softmax(z, labels)
errs = classification_error(z, labels)
lr_per_sample = learning_rate_schedule([0.3, 0.2, 0.1, 0.0], UnitType.sample)
learner = sgd(z.parameters, lr_per_sample)
trainer = Trainer(z, ce, errs, [learner])
input_map = {
in1 : mbs.streams.features,
labels : mbs.streams.labels
}
# fetch minibatch (first sequence)
data = mbs.next_minibatch(1, input_map=input_map)
trainer.train_minibatch(data)
assert learner.learning_rate() == 0.3
# fetch minibatch (second sequence, sweep ends at this point)
data = mbs.next_minibatch(1, input_map=input_map)
trainer.train_minibatch(data)
assert learner.learning_rate() == 0.2
# fetch minibatch (both sequences -- entire sweep in one go)
data = mbs.next_minibatch(9, input_map=input_map)
trainer.train_minibatch(data)
assert learner.learning_rate() == 0.1
# fetch minibatch (multiple sweeps)
data = mbs.next_minibatch(30, input_map=input_map)
trainer.train_minibatch(data, [z.output])
assert learner.learning_rate() == 0.0

28
bindings/python/cntk/trainer.py
Просмотреть файл

@ -7,7 +7,7 @@
from . import cntk_py
from .device import use_default_device
from .utils import sanitize_var_map, sanitize_function, typemap, value_to_seq
from .io import _py_dict_to_cntk_dict
from .io import _py_dict_to_cntk_dict, MinibatchData
__doc__= '''\
A trainer encapsulates the overall training process and employs one or more
@ -37,7 +37,7 @@ class Trainer(cntk_py.Trainer):
if not isinstance(parameter_learners, list):
parameter_learners = [parameter_learners]
trainer = cntk_py.create_trainer(model, loss_function, eval_function, parameter_learners)
trainer = cntk_py.trainer_impl(model, loss_function, eval_function, parameter_learners)
# transplant into this class instance
self.__dict__ = trainer.__dict__
@ -78,18 +78,36 @@ class Trainer(cntk_py.Trainer):
device = use_default_device()
if arguments:
arguments = sanitize_var_map(self.model.arguments, arguments)
arguments = sanitize_var_map(self.model.arguments, arguments,
extract_values_from_minibatch_data = False)
contains_minibatch_data = False
if (len(arguments) > 0):
value = next(iter(arguments.values()))
contains_minibatch_data = isinstance(value, MinibatchData)
if outputs:
output_map = {v: None for v in outputs}
updated = super(Trainer, self).train_minibatch(arguments,
if contains_minibatch_data:
updated = super(Trainer, self).train_minibatch_overload_for_minibatchdata(
arguments, output_map, device)
else:
updated = super(Trainer, self).train_minibatch(arguments,
output_map, device)
for k,v in output_map.items():
output_map[k] = value_to_seq(v)
return updated, output_map
else:
updated = super(Trainer, self).train_minibatch(arguments, device)
if contains_minibatch_data:
updated = super(Trainer, self).train_minibatch_overload_for_minibatchdata(
arguments, device)
else:
updated = super(Trainer, self).train_minibatch(arguments,
device)
return updated

230
bindings/python/cntk/utils/__init__.py
Просмотреть файл

@ -16,8 +16,8 @@ from cntk.device import use_default_device, cpu
from .swig_helper import typemap
from ..axis import Axis
from .progress_print import *
import warnings
_VARIABLE_OR_FUNCTION = (cntk_py.Variable, cntk_py.Function)
def sanitize_precision(precision):
'''
@ -43,8 +43,7 @@ def one_hot(batch, num_classes, dtype=None, device=None):
'''
Converts ``batch`` into a :class:`Value` object of ``dtype``
such that the integer data in ``batch`` is interpreted as the indices
representing one-hot vectors. Additionally, a SciPy CSR matrix can be obtained
by calling :meth:`~cntk.utils.Value.to_csr`.
representing one-hot vectors.
Example:
>>> num_classes = 6
@ -81,6 +80,7 @@ def one_hot(batch, num_classes, dtype=None, device=None):
if data_type != int:
raise ValueError('supplied data to one_hot() must be of type integer'
' and not "%s" since it is index data.'%data_type)
if dtype in [np.float32, None]:
value = cntk_py.Value.create_one_hot_float(num_classes, batch, device, False)
elif dtype == np.float64:
@ -161,7 +161,7 @@ def get_data_type(*args):
cntk_dtypes = set()
numpy_dtypes = set()
if len(args) == 1 and isinstance(args, cntk_py.Function):
if len(args) == 1 and isinstance(args, _VARIABLE_OR_FUNCTION):
args = [args]
for arg in args:
@ -178,7 +178,7 @@ def get_data_type(*args):
raise ValueError(
'NumPy type "%s" is not supported' % arg.dtype)
numpy_dtypes.add(arg.dtype.type)
elif isinstance(arg, cntk_py.Function):
elif isinstance(arg, _VARIABLE_OR_FUNCTION):
var_outputs = arg.outputs
if len(var_outputs) > 1:
raise ValueError(
@ -223,12 +223,6 @@ def _is_dense(batch):
return True
def _is_c_contiguous(data):
while isinstance(data, list):
data = data[0]
return data.flags.c_contiguous
@typemap
def sanitize_batch(var, batch, seq_starts=None, device=None):
'''
@ -265,12 +259,13 @@ def sanitize_batch(var, batch, seq_starts=None, device=None):
if device is None:
device = use_default_device()
from .. import Value
return Value.create(var, batch, seq_starts, device)
def sanitize_value(shape, value, dtype, device):
'''
Converts a given ``value`` to an :class:`NDArrayView` object that can be passed to
Converts a given ``value`` to an :class:`~cntk.NDArrayView` object that can be passed to
the CNTK core.
Args:
@ -282,13 +277,14 @@ def sanitize_value(shape, value, dtype, device):
on
Returns:
:class:`~cntk.cntk_py.NDArrayView` object representing ``value``
:class:`~cntk.NDArrayView` object representing ``value``
'''
from .. import NDArrayView
if value is None:
if shape is None:
raise ValueError('you need to specify at least shape or value')
cntk_dtype = sanitize_dtype_cntk(dtype)
ndav = _create_NDArrayView(shape, cntk_dtype, device)
ndav = NDArrayView(shape, cntk_dtype, device)
else:
np_dtype = sanitize_dtype_numpy(dtype)
if not isinstance(value, np.ndarray) or value.dtype != np_dtype:
@ -297,7 +293,7 @@ def sanitize_value(shape, value, dtype, device):
else:
value = np.asarray(value, dtype=np_dtype)
ndav = _create_NDArrayView_from_NumPy(value, device)
ndav = NDArrayView.from_dense(value, device)
return ndav
@ -312,14 +308,14 @@ def sanitize_function(arg):
arg = arg.owner
if not isinstance(arg, cntk_py.Function):
raise TypeError("Object of type '%s' cannot be cast to Variable" %
raise TypeError("Object of type %s cannot be cast to Variable" %
str(type(arg)))
return arg
def sanitize_var_map(op_arguments, arguments, precision=None,
device=None):
device=None, extract_values_from_minibatch_data=True):
'''
Sanitizes a dictionary of `Variable` s to input data such that it can be
handed off to the evaluation methods
@ -365,12 +361,21 @@ def sanitize_var_map(op_arguments, arguments, precision=None,
one of 'float' 'float32, 'double', 'float64', or None
device (:class:`~cntk.device.DeviceDescriptor`, default None): device
this value should be put on
extract_values_from_minibatch_data (`bool`, defaults to `True`): specifies
if :class:`~cntk.io.MinibatchData` instances in the arguments map are
converted to the underlying value (:class:`Value`) instances (default),
or if they should remain intact, as they contain additional meta
information required by the Trainer (specifically, by the
:meth:`~cntk.Trainer.train_minibatch` method).
Returns:
`dict` that maps variables to sanitized batches
'''
from ..io import MinibatchData
if not op_arguments:
return {}
if isinstance(arguments, tuple):
arguments, seq_starts = arguments
else:
@ -437,9 +442,10 @@ def sanitize_var_map(op_arguments, arguments, precision=None,
'sequence begin markers' % (sample_sizes, len(seq_starts)))
if isinstance(batch, MinibatchData):
batch = batch.m_data
elif not isinstance(batch, cntk_py.Value):
if isinstance(batch, MinibatchData) and extract_values_from_minibatch_data:
batch = batch.data
if not (isinstance(batch, MinibatchData) or isinstance(batch, cntk_py.Value)):
batch = sanitize_batch(var, batch, seq_starts, device)
var_map[var] = batch
@ -457,190 +463,6 @@ def _ones_like(batch, precision):
'''
return [np.ones_like(sample, dtype=sanitize_precision(precision)) for sample in batch]
def _create_NDArrayView(shape, data_type=cntk_py.DataType_Float, device=None):
shape = sanitize_shape(shape)
if device is None:
device = use_default_device()
# FIXME only dense supported so far
view = cntk_py.NDArrayView(data_type, cntk_py.StorageFormat_Dense, shape,
device)
return view
def _create_NDArrayView_from_NumPy(nd, device=None):
if device is None:
device = use_default_device()
return cntk_py.NDArrayView(nd, device, False)
class Value(cntk_py.Value):
'''
Internal representation of minibatch data.
Args:
shape (tuple): shape of the value
value (None or value that can be cast to NumPy array): the value to
be converted
dtype: data type (np.float32 or np.float64)
batch: batch input for `var`.
It can be:
* a pure Python structure (list of lists, ...),
* a list of NumPy arrays or SciPy sparse CSR matrices
* a :class:`Value` object (e.g. returned by :func:`one_hot`)
seq_starts (list of `bool`s or None): if None, every sequence is
treated as a new sequence. Otherwise, it is interpreted as a list of
Booleans that tell whether a sequence is a new sequence (`True`) or a
continuation of the sequence in the same slot of the previous
minibatch (`False`)
device (:class:`~cntk.device.DeviceDescriptor`): device this value should be put
on
'''
def __init__(self, shape=None, dtype=None, batch=None, seq_starts=None, device=None):
if device is None:
device = use_default_device()
if shape and dtype:
# FIXME is this needed?
ndav = _create_NDArrayView(shape, dtype, device)
elif batch:
if isinstance(batch, np.ndarray):
ndav = _create_NDArrayView_from_NumPy(batch, device)
else:
ndav = batch
if seq_starts:
super(Value, self).__init__(ndav, seq_starts)
else:
super(Value, self).__init__(ndav)
@staticmethod
@typemap
def create(var, batch, seq_starts=None, device=None, read_only=False):
'''
Creates a :class:`Value` object.
Args:
var (:class:`~cntk.ops.variables.Variable`): input variable into which
``batch`` is passed
batch: batch input.
It can be:
* a single NumPy array denoting the full minibatch
* a list of NumPy arrays or SciPy sparse CSR matrices
seq_starts (list of `bool`s or None): if None, every sequence is
treated as a new sequence. Otherwise, it is interpreted as a list of
Booleans that tell whether a sequence is a new sequence (`True`) or a
continuation of the sequence in the same slot of the previous
minibatch (`False`)
device (:class:`~cntk.device.DeviceDescriptor`, default None): device
this value should be put on
read_only (bool, default False): whether the data is read only
Returns:
:class:`Value` object.
'''
if isinstance(batch, np.ndarray):
# The outermost axis has to be Python list. If the user passes a
# full minibatch as one NumPy array, we have to convert it.
if batch.dtype == object:
raise ValueError('dtype object is not supported. If this is a batch '
'of sequences, you need to pass them as a pure-Python list '
'of NumPy arrays')
# FIXME if not seq_starts: directly pass it to Value constructor
batch = list(np.atleast_1d(batch))
if not isinstance(batch, list):
raise ValueError('batch has to be a list of NumPy arrays or '
'SciPy CSR matrices')
list_of_ndavs = []
# NDArrayViews are all created on CPU. The Value object later then will
# move it to the requested device.
cpu_dev = cpu()
for sample in batch:
if isinstance(sample, list):
sample = np.asarray(sample, dtype=var.dtype)
if sample.dtype != var.dtype:
raise ValueError('could not convert sample data to '
'NumPy array')
if isinstance(sample, np.number):
sample = np.asarray(sample)
if not (isinstance(sample, np.ndarray) or sparse.issparse(sample)):
raise ValueError('sample type "%s" is not supported. Please '
'provide the data as a Python list of NumPy arrays '
'or Scipy CSR matrices.'%type(sample))
if np.issubdtype(sample.dtype, int):
sample = sample.astype(var.dtype)
elif sample.dtype not in (np.float32, np.float64):
raise ValueError('only integer, float32 and float64 are supported, '
'you gave %s'%sample.dtype)
else:
sample = sample.astype(var.dtype)
if isinstance(sample, np.ndarray):
if not _is_c_contiguous(sample):
warnings.warn('supplied data is not C contiguous; rearrange your data/computation to avoid this', RuntimeWarning)
sample = np.ascontiguousarray(sample)
ndav = _create_NDArrayView_from_NumPy(sample, cpu_dev)
elif sparse.issparse(sample):
if not sparse.isspmatrix_csr(sample):
raise ValueError("only CSR is supported as of now. Please "
"convert your data using 'tocsr()'")
ndav = cntk_py.NDArrayView(sample.shape, sample.data,
sample.indptr, sample.indices, cpu_dev, False)
list_of_ndavs.append(ndav)
return cntk_py.Value_create(
_as_tuple(var.shape), list_of_ndavs,
seq_starts or [],
device or use_default_device(),
read_only)
@property
def shape(self):
'''
The rectangular shape of this value. I.e., if this value has sequences
of varying lengths, the shape will have the max sequence length in the
sequence dimension.
'''
return super(Value, self).shape().dimensions()
@property
def mask(self):
'''
The mask matrix of this value. Each row denotes a sequence with its
elements describing the mask of the element:
* 2: beginning of sequence (e.g. an LSTM would be reset)
* 1: valid element
* 0: invalid element
Example:
A mask of ``[[2, 1, 1], [1, 1, 0]]`` describes a batch of two
sequences. The first has three elements, of which the first element
(2) signals the beginning of a sequence. The second sequence has two
elements (last element marked 'invalid' by '0'). As it starts with
(1), it is a continuation of the 2nd sequence in the previous
minibatch.
'''
return np.asarray(super(Value, self).mask())
def __len__(self):
'''
Number of samples in this value object.
'''
return self.shape[0]
def sanitize_dtype_numpy(dtype):
is_type = isinstance(dtype, type) or isinstance(dtype, np.dtype)
is_str = isinstance(dtype, str)

14
bindings/python/cntk/utils/swig_helper.py
Просмотреть файл

@ -4,10 +4,10 @@
# ==============================================================================
from .. import cntk_py
__typemap = None
_typemap = None
def map_if_possible(obj):
global __typemap
if __typemap is None:
global _typemap
if _typemap is None:
# We can do this only if cntk_py and the cntk classes are already
# known, which is the case, when map_if_possible is called.
from cntk.ops.variables import Variable, Parameter, Constant
@ -18,8 +18,8 @@ def map_if_possible(obj):
from cntk.io import MinibatchSource, MinibatchData, StreamConfiguration
from cntk.axis import Axis
from cntk.distributed import WorkerDescriptor, Communicator, DistributedLearner
from cntk.utils import Value
__typemap = {
from cntk import Value
_typemap = {
cntk_py.Variable: Variable,
cntk_py.Parameter: Parameter,
cntk_py.Constant: Constant,
@ -38,8 +38,8 @@ def map_if_possible(obj):
}
# Some types like NumPy arrays don't let to set the __class__
if obj.__class__ in __typemap:
obj.__class__ = __typemap[obj.__class__]
if obj.__class__ in _typemap:
obj.__class__ = _typemap[obj.__class__]
else:
if isinstance(obj, (tuple, list, set)):
for o in obj:

2
bindings/python/doc/test/simplernn_test.py
Просмотреть файл

@ -19,7 +19,7 @@ def test_rnn_error(device_id):
error, loss = train_sequence_classifier()
expected_error = 0.333333
expected_loss = 1.060453
expected_loss = 1.12
assert np.allclose(error, expected_error, atol=TOLERANCE_ABSOLUTE)
assert np.allclose(loss, expected_loss, atol=TOLERANCE_ABSOLUTE)