microsoft
/
CNTK
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

merge with master

This commit is contained in:
Thilo Will 2016-07-08 13:57:01 +02:00
Родитель 3e7ae08d97 e88bd46090
Коммит 94bd96eaba
63 изменённых файлов: 85509 добавлений и 5752 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

33
CNTK.Cpp.props
Просмотреть файл

@ -27,6 +27,11 @@
<CudaVersion Condition="Exists('$(CUDA_PATH_V7_5)') And '$(CudaVersion)' == ''">7.5</CudaVersion>
<CudaVersion Condition="Exists('$(CUDA_PATH_V7_0)') And '$(CudaVersion)' == ''">7.0</CudaVersion>
<HasOpenCv>false</HasOpenCv>
<HasOpenCv Condition="Exists('$(OPENCV_PATH)') Or Exists('$(OPENCV_PATH_V31)')">true</HasOpenCv>
<UseZip>false</UseZip>
<UseZip Condition="Exists('$(ZLIB_PATH)')">true</UseZip>
</PropertyGroup>
<Choose>
@ -70,7 +75,33 @@
<UnitTestDlls>$(OutDir)mkl_cntk_s.dll;</UnitTestDlls>
</PropertyGroup>
</When>
</Choose>
</Choose>
<PropertyGroup Condition="$(UseZip)">
<ZipInclude>$(ZLIB_PATH)\include;$(ZLIB_PATH)\lib\libzip\include;</ZipInclude>
<ZipDefine>USE_ZIP</ZipDefine>
<ZipLibPath>$(ZLIB_PATH)\lib;</ZipLibPath>
<ZipLibs>zlib.lib;zip.lib;</ZipLibs>
</PropertyGroup>
<PropertyGroup Condition="Exists('$(OPENCV_PATH)')">
<OpenCvPath>$(OPENCV_PATH)</OpenCvPath>
<OpenCvVersion>300</OpenCvVersion>
</PropertyGroup>
<PropertyGroup Condition="Exists('$(OPENCV_PATH_V31)')">
<OpenCvPath>$(OPENCV_PATH_V31)</OpenCvPath>
<OpenCvVersion>310</OpenCvVersion>
</PropertyGroup>
<PropertyGroup Condition="$(HasOpenCv)">
<OpenCvInclude>$(OpenCvPath)\include;</OpenCvInclude>
<OpenCvWorld Condition="$(ReleaseBuild)">opencv_world$(OpenCvVersion)</OpenCvWorld>
<OpenCvWorld Condition="$(DebugBuild)">opencv_world$(OpenCvVersion)d</OpenCvWorld>
<OpenCvLib>$(OpenCvWorld).lib</OpenCvLib>
<OpenCvLibPath>$(OpenCvPath)\x64\vc12\lib</OpenCvLibPath>
<OpenCvBinPath>$(OpenCvPath)\x64\vc12\bin</OpenCvBinPath>
</PropertyGroup>
<PropertyGroup Condition="'$(CudaVersion)' == '7.5'">
<CudaPath>$(CUDA_PATH_V7_5)</CudaPath>

1
CNTK.sln
Просмотреть файл

@ -1133,6 +1133,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{6826
ProjectSection(SolutionItems) = preProject
Scripts\pytest.ini = Scripts\pytest.ini
Scripts\txt2ctf.py = Scripts\txt2ctf.py
Scripts\uci2ctf.py = Scripts\uci2ctf.py
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManagedEvalTests", "Tests\UnitTests\ManagedEvalTests\ManagedEvalTests.csproj", "{CC8DDDCB-D53A-4B30-8596-AEF1C493DB31}"

2
Examples/Evaluation/CSEvalClient/CSEvalClient.csproj
Просмотреть файл

@ -56,8 +56,10 @@
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Drawing" />
</ItemGroup>
<ItemGroup>
<Compile Include="CntkBitmapExtensions.cs" />
<Compile Include="ModelEvaluator.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />

212
Examples/Evaluation/CSEvalClient/CntkBitmapExtensions.cs Normal file
Просмотреть файл

@ -0,0 +1,212 @@
//
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
//
// CntkBitmapExtensions.cs -- extension methods for transforming images used in CNTK.
//
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
{
public static class CntkBitmapExtensions
{
/// <summary>
/// Resizes an image
/// </summary>
/// <param name="image">The image to resize</param>
/// <param name="width">New width in pixels</param>
/// <param name="height">New height in pixesl</param>
/// <param name="useHighQuality">Resize quality</param>
/// <returns>The resized image</returns>
public static Bitmap Resize(this Bitmap image, int width, int height, bool useHighQuality)
{
var newImg = new Bitmap(width, height);
newImg.SetResolution(image.HorizontalResolution, image.VerticalResolution);
using (var g = Graphics.FromImage(newImg))
{
g.CompositingMode = System.Drawing.Drawing2D.CompositingMode.SourceCopy;
if (useHighQuality)
{
g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
g.CompositingQuality = System.Drawing.Drawing2D.CompositingQuality.HighQuality;
g.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.HighQuality;
g.PixelOffsetMode = System.Drawing.Drawing2D.PixelOffsetMode.HighQuality;
}
else
{
g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.Default;
g.CompositingQuality = System.Drawing.Drawing2D.CompositingQuality.Default;
g.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.Default;
g.PixelOffsetMode = System.Drawing.Drawing2D.PixelOffsetMode.Default;
}
var attributes = new ImageAttributes();
attributes.SetWrapMode(System.Drawing.Drawing2D.WrapMode.TileFlipXY);
g.DrawImage(image, new Rectangle(0, 0, width, height), 0, 0, image.Width, image.Height, GraphicsUnit.Pixel, attributes);
}
return newImg;
}
/// <summary>
/// Extracts image pixels in CHW
/// </summary>
/// <param name="image">The bitmap image to extract features from</param>
/// <returns>A list of pixels in HWC order</returns>
public static List<float> ExtractCHW(this Bitmap image)
{
var features = new List<float>(image.Width * image.Height * 3);
for (int c = 0; c < 3; c++)
{
for (int h = 0; h < image.Height; h++)
{
for (int w = 0; w < image.Width; w++)
{
var pixel = image.GetPixel(w, h);
float v = c == 0 ? pixel.B : c == 1 ? pixel.G : pixel.R;
features.Add(v);
}
}
}
return features;
}
/// <summary>
/// Extracts image pixels in CHW using parallelization
/// </summary>
/// <param name="image">The bitmap image to extract features from</param>
/// <returns>A list of pixels in CHW order</returns>
public static List<float> ParallelExtractCHW(this Bitmap image)
{
// We use local variables to avoid contention on the image object through the multiple threads.
int channelStride = image.Width * image.Height;
int imageWidth = image.Width;
int imageHeight = image.Height;
var features = new byte[imageWidth * imageHeight * 3];
var bitmapData = image.LockBits(new Rectangle(0, 0, imageWidth, imageHeight), ImageLockMode.ReadOnly, image.PixelFormat);
IntPtr ptr = bitmapData.Scan0;
int bytes = Math.Abs(bitmapData.Stride) * bitmapData.Height;
byte[] rgbValues = new byte[bytes];
int stride = bitmapData.Stride;
// Copy the RGB values into the array.
System.Runtime.InteropServices.Marshal.Copy(ptr, rgbValues, 0, bytes);
// The mapping depends on the pixel format
// The mapPixel lambda will return the right color channel for the desired pixel
Func<int, int, int, int> mapPixel = GetPixelMapper(image.PixelFormat, stride);
Parallel.For(0, imageHeight, (int h) =>
{
Parallel.For(0, imageWidth, (int w) =>
{
Parallel.For(0, 3, (int c) =>
{
features[channelStride * c + imageWidth * h + w] = rgbValues[mapPixel(h, w, c)];
});
});
});
image.UnlockBits(bitmapData);
return features.Select(b => (float)b).ToList();
}
/// <summary>
/// Extracts image pixels in HWC
/// </summary>
/// <param name="image">The bitmap image to extract features from</param>
/// <returns>A list of pixels in HWC order</returns>
public static List<float> ExtractHWC(this Bitmap image)
{
var features = new List<float>(image.Width * image.Height * 3);
for (int w = 0; w < image.Width; w++)
{
for (int h = 0; h < image.Height; h++)
{
for (int c = 0; c < 3; c++)
{
var pixel = image.GetPixel(w, h);
float v = c == 0 ? pixel.B : c == 1 ? pixel.G : pixel.R;
features.Add(v);
}
}
}
return features;
}
/// <summary>
/// Extracts image pixels in HWC using multiple threads
/// </summary>
/// <param name="image">The bitmap image to extract features from</param>
/// <returns>A list of pixels in HWC order</returns>
public static List<float> ParallelExtractHWC(this Bitmap image)
{
int heightStride = image.Width * 3;
int widthStride = image.Height * 3;
int imageWidth = image.Width;
int imageHeight = image.Height;
var features = new byte[image.Width * image.Height * 3];
var bitmapData = image.LockBits(new Rectangle(0, 0, image.Width, image.Height), ImageLockMode.ReadOnly, image.PixelFormat);
IntPtr ptr = bitmapData.Scan0;
int bytes = Math.Abs(bitmapData.Stride) * bitmapData.Height;
byte[] rgbValues = new byte[bytes];
int stride = bitmapData.Stride;
// Copy the RGB values into the array.
System.Runtime.InteropServices.Marshal.Copy(ptr, rgbValues, 0, bytes);
// The mapping depends on the pixel format
// The mapPixel lambda will return the right color channel for the desired pixel
Func<int, int, int, int> mapPixel = GetPixelMapper(image.PixelFormat, stride);
Parallel.For(0, 3, (int c) =>
{
Parallel.For(0, imageHeight, (int h) =>
{
Parallel.For(0, imageWidth, (int w) =>
{
features[w * widthStride + h * 3 + c] = rgbValues[mapPixel(h, w, c)];
});
});
});
image.UnlockBits(bitmapData);
return features.Select(b => (float)b).ToList();
}
/// <summary>
/// Returns a function for extracting the R-G-B values properly from an image based on its pixel format
/// </summary>
/// <param name="pixelFormat">The image's pixel format</param>
/// <param name="heightStride">The stride (row byte count)</param>
/// <returns>A function with signature (height, width, channel) returning the corresponding color value</returns>
private static Func<int, int, int, int> GetPixelMapper(PixelFormat pixelFormat, int heightStride)
{
switch (pixelFormat)
{
case PixelFormat.Format32bppArgb:
return (h, w, c) => h * heightStride + w * 4 + c; // bytes are B-G-R-A
case PixelFormat.Format24bppRgb:
default:
return (h, w, c) => h * heightStride + w * 3 + c; // bytes are B-G-R
}
}
}
}

95
Examples/Evaluation/CSEvalClient/Program.cs
Просмотреть файл

@ -8,6 +8,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
@ -40,9 +41,16 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
///
/// EvaluateMultipleModels
/// ----------------------
/// This case requires the 02_Convolution model and the Test-28x28.txt test file which are part of the <CNTK>/Examples/Image/MNIST example.
/// This case requires the 02_Convolution model and the Test-28x28_cntk_text.txt test file which are part of the <CNTK>/Examples/Image/MNIST example.
/// Refer to <see cref="https://github.com/Microsoft/CNTK/blob/master/Examples/Image/MNIST/README.md"/> for how to train
/// the model used in this example.
///
/// EvaluateImageClassificationModel
/// -----------------------
/// This case requires the ResNet_18 trained model which can be downloaded from <see cref="https://www.cntk.ai/resnet/ResNet_18.model"/>.
/// This case shows how to evaluate a model that was trained with the ImageReader.
/// The input for evaluation needs to be transformed in a similar manner as the ImageReader did during training.
///
/// </description>
class Program
{
@ -55,7 +63,7 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
private static void Main(string[] args)
{
initialDirectory = Environment.CurrentDirectory;
Console.WriteLine("====== EvaluateModelSingleLayer ========");
EvaluateModelSingleLayer();
@ -74,6 +82,9 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
Console.WriteLine("\n====== EvaluateMultipleModels ========");
EvaluateMultipleModels();
Console.WriteLine("\n====== EvaluateModelImageInput ========");
EvaluateImageClassificationModel();
Console.WriteLine("Press <Enter> to terminate.");
Console.ReadLine();
}
@ -348,7 +359,7 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
// Initializes the model instances
ModelEvaluator.Initialize(numConcurrentModels, modelFilePath);
string testfile = Path.Combine(Environment.CurrentDirectory, @"Test-28x28.txt");
string testfile = Path.Combine(Environment.CurrentDirectory, @"Test-28x28_cntk_text.txt");
Stopwatch sw = new Stopwatch();
sw.Start();
@ -361,16 +372,24 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
{
Interlocked.Increment(ref count);
// The first value in the line is the expected label index for the record's outcome
int expected = int.Parse(line.Substring(0, line.IndexOf('\t')));
var inputs = line.Substring(line.IndexOf('\t') + 1).Split('\t').Select(float.Parse).ToList();
// The file format correspond to the CNTK Text Format Reader format (https://github.com/Microsoft/CNTK/wiki/CNTKTextFormat-Reader)
var sets = line.Split('|');
var labels = sets[1].Trim().Split(' ').Skip(1);
var features = sets[2].Trim().Split(' ').Skip(1);
// Retrieve the 1-hot vector with the label index
var expected = labels.Select(float.Parse).Select((v, index) => new { Value = v, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
// Retrieve the features
var inputs = features.Select(float.Parse).ToList();
// We can call the evaluate method and get back the results (single layer)...
var outputs = ModelEvaluator.Evaluate(inputs);
// Retrieve the outcome index (so we can compare it with the expected index)
int index = 0;
var max = outputs.Select(v => new { Value = v, Index = index++ })
var max = outputs.Select((v, index) => new { Value = v, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
@ -394,10 +413,68 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
sw.Stop();
ModelEvaluator.DisposeAll();
Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28.txt",
Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28_cntk_text.txt",
numConcurrentModels, (float)errorCount / count, errorCount, count, (count + errorCount) * 1000.0 / sw.ElapsedMilliseconds);
}
/// <summary>
/// This method shows how to evaluate a trained image classification model
/// </summary>
public static void EvaluateImageClassificationModel()
{
try
{
// This example requires the RestNet_18 model.
// The model can be downloaded from <see cref="https://www.cntk.ai/resnet/ResNet_18.model"/>
// The model is assumed to be located at: <CNTK>\Examples\Image\Miscellaneous\ImageNet\ResNet
// along with a sample image file named "zebra.jpg".
string workingDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\Miscellaneous\ImageNet\ResNet");
Environment.CurrentDirectory = initialDirectory;
List<float> outputs;
using (var model = new IEvaluateModelManagedF())
{
string modelFilePath = Path.Combine(workingDirectory, "ResNet_18.model");
model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId: -1);
// Prepare input value in the appropriate structure and size
var inDims = model.GetNodeDimensions(NodeGroup.Input);
if (inDims.First().Value != 224 * 224 * 3)
{
throw new CNTKRuntimeException(string.Format("The input dimension for {0} is {1} which is not the expected size of {2}.", inDims.First(), inDims.First().Value, 224 * 224 * 3), string.Empty);
}
// Transform the image
string imageFileName = Path.Combine(workingDirectory, "zebra.jpg");
Bitmap bmp = new Bitmap(Bitmap.FromFile(imageFileName));
var resized = bmp.Resize(224, 224, true);
var resizedCHW = resized.ParallelExtractCHW();
var inputs = new Dictionary<string, List<float>>() { {inDims.First().Key, resizedCHW } };
// We can call the evaluate method and get back the results (single layer output)...
var outDims = model.GetNodeDimensions(NodeGroup.Output);
outputs = model.Evaluate(inputs, outDims.First().Key);
}
// Retrieve the outcome index (so we can compare it with the expected index)
var max = outputs.Select((value, index) => new { Value = value, Index = index })
.Aggregate((a, b) => (a.Value > b.Value) ? a : b)
.Index;
Console.WriteLine("Outcome: {0}", max);
}
catch (CNTKException ex)
{
Console.WriteLine("Error: {0}\nNative CallStack: {1}\n Inner Exception: {2}", ex.Message, ex.NativeCallStack, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
}
catch (Exception ex)
{
Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
}
}
/// <summary>
/// Dumps the output to the console
/// </summary>

54
Examples/Image/MNIST/Config/04_DeConv.cntk
Просмотреть файл

@ -49,25 +49,21 @@ train = [
maxEpochs = 10
]
# Note: this reader crashes if randomization is turned on.
reader = [
readerType = "UCIFastReader"
# To get the data (Train-28x28.txt) please run `python mnist_convert.py`
# from the 'AdditionalFiles' folder. See REAMDE.md for details.
file = "$DataDir$/Train-28x28.txt"
features = [
dim = 784
start = 1
readerType = "CNTKTextFormatReader"
# See ../README.md for details on getting the data (Train-28x28_cntk_text.txt).
file = "$DataDir$/Train-28x28_cntk_text.txt"
input = [
features = [
dim = 784
format = "dense"
]
labels = [
dim = 10
format = "dense"
]
]
labels = [
dim = 1
start = 0
labelDim = 10
labelMappingFile = "$DataDir$/labelsmap.txt"
]
]
]
]
#######################################
@ -83,19 +79,17 @@ test = [
]
reader = [
readerType = "UCIFastReader"
file = "$DataDir$/Test-28x28.txt"
features = [
dim = 784
start = 1
]
labels = [
dim = 1
start = 0
labelDim = 10
labelMappingFile = "$DataDir$/labelsmap.txt"
readerType = "CNTKTextFormatReader"
file = "$DataDir$/Test-28x28_cntk_text.txt"
input = [
features = [
dim = 784
format = "dense"
]
labels = [
dim = 10
format = "dense"
]
]
]
]

10
Examples/Image/MNIST/README.md
Просмотреть файл

@ -7,7 +7,7 @@
|Purpose |This example demonstrates usage of the NDL (Network Description Language) to define networks.
|Network |NDLNetworkBuilder, simple feed forward and convolutional networks, cross entropy with softmax.
|Training |Stochastic gradient descent both with and without momentum.
|Comments |There are two config files, details are provided below.
|Comments |There are four config files, details are provided below.
## Running the example
@ -57,7 +57,7 @@ The output folder will be created inside Image/MNIST/.
### Config files
There are three config files and corresponding network description files in the 'Config' folder:
There are four config files and the corresponding network description files in the 'Config' folder:
1. 01_OneHidden.ndl is a simple, one hidden layer network that produces 2.3% of error.
To run the sample, navigate to the Data folder and run the following command:
@ -74,7 +74,11 @@ As a result, it achieves around 0.8% of error after training for just 2 epochs (
To run the sample, navigate to the Data folder and run the following command:
`cntk configFile=../Config/03_ConvBatchNorm.cntk`
For more details, refer to .ndl and corresponding .cntk files.
4. 04_DeConv.ndl illustrates the usage of Deconvolution and Unpooling. It is a network with one Convolution, one Pooling, one Unpooling and one Deconvolution layer. In fact it is an auto-encoder network where Rectified Linear Unit (ReLU) or Sigmoid layer is now replaced with Convolutional ReLU (for encoding) and Deconvolutional ReLU (for decoding) layers. The network goal is to reconstruct the original signal, with Mean Squared Error (MSE) used to minimize the reconstruction error. Generally such networks are used in semantic segmentation.
To run the sample, navigate to the Data folder and run the following command:
`cntk configFile=../Config/04_DeConv.cntk`
For more details, refer to .ndl and the corresponding .cntk files.
### Additional files

Двоичные данные
Examples/Image/Miscellaneous/ImageNet/ResNet/zebra.jpg Normal file
Просмотреть файл

Двоичный файл не отображается.
Режим "рядом"

После

Ширина:  |  Высота:  |  Размер: 92 KiB

183
Examples/Text/ATIS/ATIS.cntk Normal file
Просмотреть файл

@ -0,0 +1,183 @@
# The configuration file to build language understanding model with ATIS corpus.
# LSTM model is built to tage each word in sentences with its semantic label.
WorkDir = work
DataDir = data
modelPath = $WorkDir$/ATIS.slot.lstm
parallelTrain = true
#stderr = $WorkDir$/log
command = Train:Test
precision = "float"
deviceId = "auto"
wordCount = 944 # number of words
labelCount = 127 # number of labels
Train = [
action = train
traceLevel = 1
BrainScriptNetworkBuilder = [
inputDim = $wordCount$
labelDim = $labelCount$
featDim = inputDim*3 # contextual words are used as features: previous word, current word, next word.
embDim = 150
hiddenDim = 300
maxLayer = 0
initScale = 6
initBias = -1.0
featuresPW = Input(inputDim) # the previous word
featuresCW = Input(inputDim) # the current word
featuresNW = Input(inputDim) # the next word
features = RowStack(featuresPW : featuresCW : featuresNW)
labels = Input(labelDim, tag = "label")
# embedding layer
emb = LearnableParameter(embDim, featDim)
featEmbedded = Times(emb, features)
# build the LSTM stack
lstmDims[i:0..maxLayer] = hiddenDim
NoAuxInputHook (input, lstmState) = BS.Constants.None
lstmStack = BS.RNNs.RecurrentLSTMPStack (lstmDims,
cellDims=lstmDims,
featEmbedded,
inputDim=embDim,
previousHook=BS.RNNs.PreviousHC,
augmentInputHook=BS.RNNs.NoAuxInputHook,
augmentInputDim=0,
enableSelfStabilization=false)
lstmOutputLayer = Length (lstmStack)-1
LSTMoutput = lstmStack[lstmOutputLayer].h
W = Parameter(labelDim, hiddenDim, init = "uniform", initValueScale=initScale)
b = Parameter(labelDim, 1, init = "fixedValue", value=0)
outputs = W * LSTMoutput + b
cr = CrossEntropyWithSoftmax(labels, outputs)
criterionNodes = (cr)
evaluationNodes = (cr)
outputNodes = (outputs)
]
SGD = [
# maximum number of epochs
maxEpochs = 1 # set to 1 so this can be added to regression test. Increase to 20 get a good accuracy
# for each epoch, maximum number of input samples(words) is set below
epochSize = 36000
# minibatchSize should be larger than the maximum sentence length
minibatchSize = 70
learningRatesPerSample = 0.01*2:0.005*12:0.001
gradUpdateType = "FSAdaGrad"
gradientClippingWithTruncation = true
clippingThresholdPerSample = 15.0
# number of minibatches to report progress
numMBsToShowResult = 100
firstMBsToShowResult = 10
# if validation shows that the model has no improvement, then do back-up to the previously
# estimated model and reduce learning rate
loadBestModel = true
parallelTrain = [
parallelizationMethod = "DataParallelSGD"
parallelizationStartEpoch = 2
distributedMBReading = true
dataParallelSGD = [
gradientBits = 1
]
]
]
reader = [
readerType = "CNTKTextFormatReader"
file = "$DataDir$/ATIS.train.cntk.sparse"
miniBatchMode = "partial"
randomize = true
input = [
featuresPW = [
alias = "PW" # previous word
dim = $wordCount$
format = "sparse"
]
featuresCW = [
alias = "CW" # current word
dim = $wordCount$
format = "sparse"
]
featuresNW = [
alias = "NW" # next word
dim = $wordCount$
format = "sparse"
]
labels = [
alias = "L" # label
dim = $labelCount$
format = "sparse"
]
]
]
]
# Test the model built from Train
Test = [
action = write
traceLevel = 1
epochSize = 0
defaultHiddenActivity = 0.1
BrainScriptNetworkBuilder = [
modelAsTrained = BS.Network.Load ("$modelPath$")
final = Softmax(modelAsTrained.outputs)
]
outputPath = $WorkDir$/model.writeaction
outputNodeNames = final
reader = [
readerType = "CNTKTextFormatReader"
file = "$DataDir$/ATIS.test.cntk.sparse"
miniBatchMode = "partial"
randomize = false
input = [
featuresPW = [
alias = "PW" # previous word
dim = $wordCount$
format = "sparse"
]
featuresCW = [
alias = "CW" # current word
dim = $wordCount$
format = "sparse"
]
featuresNW = [
alias = "NW" # next word
dim = $wordCount$
format = "sparse"
]
labels = [
alias = "L" # label
dim = $labelCount$
format = "sparse"
]
]
]
]

163
Examples/Text/ATIS/README.md Normal file
Просмотреть файл

@ -0,0 +1,163 @@
# Build Language Understanding Models with CNTK
This example demonstrates how to use build language understanding model with CNTK using ATIS data set. This example is similar to
[SLU example](https://github.com/Microsoft/CNTK/tree/master/Examples/Text/Miscellaneous/SLU). They are different in that
- CNTKTextFormatReader is used here, instead of LUSequenceReader
- With CNTKTextFormatReader, the input format is much more flexible. In the example setting, sparse contextual feature vectors are explored
- Sparse label input is used.
The Air travel information system (ATIS) corpus is used for training and testing.
## Download the example
The data and configuration is checked in to github. You can get it by command:
`git clone https://github.com/Microsoft/cntk`
The example is under folder:
`<cntk_root>\Examples\Text\ATIS`
## Data File Format
There are four files under `data` sub-folder
|Files |Content |
|:----------------------|:--------|
|ATIS.train.cntk.sparse |featurized training data set
|ATIS.test.cntk.sparse |featurized test data set
|ATIS.vocab |all words extracted from training data. Vocab size: 944
|ATIS.labels |all semantic labels extracted from training data. Total labels: 127
We preprocess ATIS data by converting words into word indexes, and labels into label IDs in order to use
[CNTKTextFormatReader](https://github.com/Microsoft/CNTK/wiki/CNTKTextFormat-Reader). You can use any
script/tool to preprocess your text data files. In this example, data is already preprocessed.
The last two files ATIS.vocab and ATIS.labels are not really required to run the example. They are included for evaluation and debugging purpose.
E.g. they can be used to convert .sparse files back to original text files.
To understand the data format (two .sparse files), let's start with a sample sentence:
```
BOS i would like to find a flight from charlotte to Las Vegas that makes a stop in St. Louis EOS
```
it is converted into the following text:
```
1 |PW 1:1 |CW 1:1 |NW 12:1 |L 126:1
1 |PW 1:1 |CW 12:1 |NW 39:1 |L 126:1
1 |PW 12:1 |CW 39:1 |NW 28:1 |L 126:1
1 |PW 39:1 |CW 28:1 |NW 3:1 |L 126:1
1 |PW 28:1 |CW 3:1 |NW 86:1 |L 126:1
1 |PW 3:1 |CW 86:1 |NW 15:1 |L 126:1
1 |PW 86:1 |CW 15:1 |NW 10:1 |L 126:1
1 |PW 15:1 |CW 10:1 |NW 4:1 |L 126:1
1 |PW 10:1 |CW 4:1 |NW 101:1 |L 126:1
1 |PW 4:1 |CW 101:1 |NW 3:1 |L 48:1
1 |PW 101:1 |CW 3:1 |NW 92:1 |L 126:1
1 |PW 3:1 |CW 92:1 |NW 90:1 |L 78:1
1 |PW 92:1 |CW 90:1 |NW 33:1 |L 123:1
1 |PW 90:1 |CW 33:1 |NW 338:1 |L 126:1
1 |PW 33:1 |CW 338:1 |NW 15:1 |L 126:1
1 |PW 338:1 |CW 15:1 |NW 132:1 |L 126:1
1 |PW 15:1 |CW 132:1 |NW 17:1 |L 126:1
1 |PW 132:1 |CW 17:1 |NW 72:1 |L 126:1
1 |PW 17:1 |CW 72:1 |NW 144:1 |L 71:1
1 |PW 72:1 |CW 144:1 |NW 2:1 |L 119:1
1 |PW 144:1 |CW 2:1 |NW 2:1 |L 126:1
```
where the first column identifies the sequence (sentence) ID, which is the same for all words of the same sentence. There are four input streams: PW, CW, NW, L.
The input "PW" represents the previous word ID, "CW" for current word, and "NW" for next word. Input name "L" is for labels. The input names can be anything you
like and you can add more input as needed, e.g. words in a bigger window.
Words "BOS" and "EOS" denote beginning of sentence and end of sentences respectively.
Each line above represents one sample (word). E.g. the meaning of this line: `1 |PW 4:1 |CW 101:1 |NW 3:1 |L 48:1`:
* the sequence ID is 1
* the current word is "charlotte" whose word ID is 101
* the previous word is "from" whose ID is 4
* the next word is "to" whose ID is 3
* the semantic label is "B-fromloc.city_name" whose label Id is 48.
All word IDs, label IDs and corresponding words and labels are stored in ATIS.vocab and ATIS.labels.
## CNTK Configuration
In this example, we use BrainScript to create one-layer LSTM with embedding for slot tagging. The consolidated config file is ATIS.cntk. One can check the file (with some comments)
for details, especially how the reader is configured in ATIS.cntk.
reader=[
readerType = "CNTKTextFormatReader"
file = "$DataDir$/ATIS.train.cntk.sparse"
miniBatchMode = "partial"
randomize = true
input = [
featuresPW = [
alias = "PW" # previous word
dim = $wordCount$
format = "sparse"
]
featuresCW = [
alias = "CW" # current word
dim = $wordCount$
format = "sparse"
]
featuresNW = [
alias = "NW" # next word
dim = $wordCount$
format = "sparse"
]
labels = [
alias = "L" # label
dim = $labelCount$
format = "sparse"
]
]
]
The above section tell CNTK to use CNTKTextFormatReader to read data from the file "$DataDir/ATIS.train.cntk.sparse". The same input names (PW, CW, NW, L) are used to refer inputs (features and labels) provided in data files. The input is read into different
feature vectors: featuresPW, featuresCW, featuresNW and labels. These vectors are later used to build LSTM node with BrainScript as follows.
```
featuresPW = Input(inputDim)
featuresCW = Input(inputDim)
featuresNW = Input(inputDim)
features = RowStack(featuresPW : featuresCW : featuresNW)
labels=Input(labelDim, tag="label")
# embedding layer
emb = LearnableParameter(embDim, featDim)
featEmbedded = Times(emb, features)
# build the LSTM stack
lstmDims[i:0..maxLayer] = hiddenDim
NoAuxInputHook (input, lstmState) = BS.Constants.None
lstmStack = BS.RNNs.RecurrentLSTMPStack (lstmDims,
cellDims=lstmDims,
featEmbedded,
inputDim=embDim,
previousHook=BS.RNNs.PreviousHC,
augmentInputHook=BS.RNNs.NoAuxInputHook,
augmentInputDim=0,
enableSelfStabilization=false)
lstmOutputLayer = Length (lstmStack)-1
LSTMoutput = lstmStack[lstmOutputLayer].h
```
A few other notes about the config:
- it is important to specify the format is "sparse".
- the gradUpdateType is set FSAdaGrad. This setting reports better model accuracy comparing any other update methods.
- multiple LSTM layers can be used by changing the value of maxLayer.
## Run the example
One can run the example locally or on Philly (for Microsoft internal users).
To run locally,
```sh
> mkdir work # the default work_dir
> cntk.exe configFile=ATIS.cntk
```
By default, the maxEpochs is set to 1 to save training time. One can change it to larger value such as 20 in order to get a good model accuracy. Depends on GPU, it normally takes about 20 minutes to run 20 epochs on single GPU, and slot F1 score is about 93.
**For Microsoft users only**, to run the job on Philly:
- first upload data folder to philly cloud. e.g. `\\storage.gcr.philly.selfhost.corp.microsoft.com\pnrsy\<your_alias>\ATIS `
- update the config file to philly cloud, e.g. `\\storage.gcr.philly.selfhost.corp.microsoft.com\pnrsy_scratch\<your_alias>\ATIS`
- go to http://philly/ to create a new job by specifying data folder and config file, and start the job.
More details about Philly, including how to upload data to Philly and start jobs, can be found [here](https://microsoft.sharepoint.com/teams/ATISG/SitePages/Philly%20Users%20Guide.aspx)

127
Examples/Text/ATIS/data/ATIS.label Normal file
Просмотреть файл

@ -0,0 +1,127 @@
B-aircraft_code
B-airline_code
B-airline_name
B-airport_code
B-airport_name
B-arrive_date.date_relative
B-arrive_date.day_name
B-arrive_date.day_number
B-arrive_date.month_name
B-arrive_date.today_relative
B-arrive_time.end_time
B-arrive_time.period_mod
B-arrive_time.period_of_day
B-arrive_time.start_time
B-arrive_time.time
B-arrive_time.time_relative
B-booking_class
B-city_name
B-class_type
B-compartment
B-connect
B-cost_relative
B-day_name
B-day_number
B-days_code
B-depart_date.date_relative
B-depart_date.day_name
B-depart_date.day_number
B-depart_date.month_name
B-depart_date.today_relative
B-depart_date.year
B-depart_time.end_time
B-depart_time.period_mod
B-depart_time.period_of_day
B-depart_time.start_time
B-depart_time.time
B-depart_time.time_relative
B-economy
B-fare_amount
B-fare_basis_code
B-flight
B-flight_days
B-flight_mod
B-flight_number
B-flight_stop
B-flight_time
B-fromloc.airport_code
B-fromloc.airport_name
B-fromloc.city_name
B-fromloc.state_code
B-fromloc.state_name
B-meal
B-meal_code
B-meal_description
B-mod
B-month_name
B-or
B-period_of_day
B-restriction_code
B-return_date.date_relative
B-return_date.day_name
B-return_date.day_number
B-return_date.month_name
B-return_date.today_relative
B-return_time.period_mod
B-return_time.period_of_day
B-round_trip
B-state_code
B-state_name
B-stoploc.airport_code
B-stoploc.airport_name
B-stoploc.city_name
B-stoploc.state_code
B-time
B-time_relative
B-today_relative
B-toloc.airport_code
B-toloc.airport_name
B-toloc.city_name
B-toloc.country_name
B-toloc.state_code
B-toloc.state_name
B-transport_type
I-airline_name
I-airport_name
I-arrive_date.day_number
I-arrive_time.end_time
I-arrive_time.period_of_day
I-arrive_time.start_time
I-arrive_time.time
I-arrive_time.time_relative
I-city_name
I-class_type
I-cost_relative
I-depart_date.day_number
I-depart_date.today_relative
I-depart_time.end_time
I-depart_time.period_of_day
I-depart_time.start_time
I-depart_time.time
I-depart_time.time_relative
I-economy
I-fare_amount
I-fare_basis_code
I-flight_mod
I-flight_number
I-flight_stop
I-flight_time
I-fromloc.airport_name
I-fromloc.city_name
I-fromloc.state_name
I-meal_code
I-meal_description
I-restriction_code
I-return_date.date_relative
I-return_date.day_number
I-return_date.today_relative
I-round_trip
I-state_name
I-stoploc.city_name
I-time
I-today_relative
I-toloc.airport_name
I-toloc.city_name
I-toloc.state_name
I-transport_type
O

10984
Examples/Text/ATIS/data/ATIS.test.cntk.sparse Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

66546
Examples/Text/ATIS/data/ATIS.train.cntk.sparse Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

944
Examples/Text/ATIS/data/ATIS.vocab Normal file
Просмотреть файл

@ -0,0 +1,944 @@
</s>
BOS
EOS
to
from
flights
the
on
what
me
flight
show
i
boston
san
a
denver
in
and
francisco
atlanta
is
pittsburgh
dallas
all
baltimore
list
philadelphia
like
are
airlines
of
between
that
washington
pm
leaving
please
morning
would
fly
for
city
fare
wednesday
first
need
after
trip
oakland
there
ground
round
does
transportation
'd
which
cheapest
you
arriving
class
before
available
american
new
fares
milwaukee
with
give
have
afternoon
york
st.
one
dc
at
way
monday
leave
arrive
airport
thursday
how
want
tuesday
nonstop
find
am
earliest
go
vegas
miami
las
united
information
orlando
phoenix
chicago
sunday
saturday
evening
charlotte
twenty
newark
can
delta
toronto
seattle
diego
kansas
indianapolis
houston
airline
noon
any
friday
lake
salt
's
next
us
o'clock
cleveland
continental
air
angeles
los
august
worth
do
fort
july
stop
code
5
seventh
early
memphis
tell
aircraft
downtown
or
june
6
louis
montreal
cincinnati
around
tomorrow
cost
going
latest
petersburg
tampa
many
minneapolis
nashville
8
get
mean
jose
detroit
10
an
departing
stopover
tacoma
by
about
twa
much
7
leaves
may
long
type
burbank
see
expensive
ticket
international
12
travel
could
dollars
than
daily
columbus
service
beach
'm
california
9
night
least
know
economy
time
4
depart
into
meal
paul
coach
book
april
airports
northwest
la
lowest
now
december
less
westchester
day
serves
it
serve
november
okay
arrives
used
field
love
last
ontario
second
county
return
kind
september
mitchell
general
as
stops
flying
2
third
be
direct
fifth
eighth
stopping
times
breakfast
out
make
capacity
car
take
schedule
seating
sixth
1000
number
goes
cities
dinner
connecting
3
dl
fourth
airfare
possible
this
has
served
meals
ninth
looking
also
restriction
week
late
eastern
returning
back
today
interested
price
business
most
prices
1991
two
types
flies
twentieth
will
through
limousine
ua
bwi
via
tenth
using
stand
plane
ap
fifteenth
guardia
same
1
should
other
arrangements
f
only
rental
then
display
your
shortest
wednesdays
listing
canadian
classes
again
numbers
thirtieth
florida
express
midwest
tickets
where
twelfth
sixteenth
h
north
eleventh
carolina
seventeenth
under
smallest
mco
distance
lunch
either
makes
if
qx
transport
far
hp
57
october
no
my
m80
thank
arizona
jfk
colorado
jersey
q
weekday
airplane
y
planes
some
departure
use
ewr
their
ohio
thirty
nineteenth
when
fourteenth
explain
layover
alaska
march
stopovers
live
people
traveling
serving
rent
hi
offer
later
yes
january
area
logan
right
booking
sfo
midnight
yn
but
during
landings
february
dfw
abbreviation
630
both
're
230
qw
boeing
coming
passengers
arrange
hours
qo
codes
trying
tower
466
canada
each
530
over
uses
arrivals
11
southwest
281
trips
838
days
those
takeoffs
lufthansa
west
1100
arrival
757
minnesota
anywhere
america
430
thrift
let
mornings
nationair
'll
kinds
cheap
close
seats
pennsylvania
name
quebec
indiana
michigan
saturdays
different
taxi
provided
rates
utah
these
starting
sometime
costs
making
bh
eighteenth
following
another
ff
near
747
ea
1992
connect
help
choices
sa
maximum
wish
1115
six
weekdays
more
total
s
dc10
d9s
2100
snack
1245
georgia
72s
73s
f28
heading
departures
amount
825
737
813
ap57
sixteen
m
sorry
serviced
three
miles
departs
1700
requesting
718
land
nevada
100
so
tennessee
tuesdays
hello
destination
reservation
texas
rentals
co
meaning
ap80
1500
270
thursdays
philly
thirteenth
services
sundays
turboprop
stands
415
provide
cars
we
great
mondays
include
sure
't
well
2134
fn
555
ord
934
connection
296
abbreviations
755
highest
hold
720
fit
80
soon
four
ten
noontime
too
offers
options
within
difference
c
restrictions
plan
originating
describe
nw
1110
connections
dulles
21
733
say
approximately
define
852
1291
rate
who
proper
beginning
being
329
352
don
1024
such
wanted
615
mealtime
provides
prefer
1288
257
across
continent
overnight
local
route
746
off
j31
closest
19
lax
l10
be1
1994
red
eye
not
aa
dca
determine
1200
1205
dtw
airfares
capacities
200
town
lga
300
1993
database
1765
eight
up
originate
look
cp
carries
here
201
located
dinnertime
1039
lastest
1222
they
just
d
limo
3724
210
stapleton
343
1145
schedules
932
nonstops
without
landing
b
midway
217
bound
727
takeoff
324
train
along
friends
transcontinental
missouri
reservations
lives
767
269
ac
atl
month
taking
repeat
845
airplanes
buy
still
itinerary
actually
earlier
various
reaching
very
names
505
grounds
ap68
must
kennedy
operation
4400
1201
297
question
combination
basis
laying
1133
650
tonight
43
ls
sam
ap58
once
nighttime
yx
kw
212
1600
tpa
prior
good
1800
819
inform
k
dc9
305
anything
771
459
calling
designate
417
spend
hou
1220
directly
jet
reverse
staying
l1011
belong
445
515
travels
order
mci
150
110
connects
charges
minimum
intercontinental
497766
sounds
811
seat
final
phl
20
start
823
1059
271
382
able
put
locate
hartfield
scheduled
run
225
1158
equipment
begins
lands
reaches
carried
wn
bn
try
included
130
continuing
india
lester
pearson
listings
1209
everywhere
sd
whether
offered
486
1300
950
usa
1045
al
currently
enroute
visit
them
takes
55
thing
705
fridays
catch
straight
advertises
having
planning
listed
1055
405
468
equal
working
sb
hopefully
dh8
symbols
sort
cover
810
operating
320
639
seventeen
1207
608
besides
companies
've
got
somebody
else
wants
level
vicinity
1940
311
mia
instead
priced
eleven
comes
greatest
summer
economic
bay
402
gets
date
1020
730
400
doesn
toward
home
1850
1505
runs
673
723
thanks
bring
zone
yyz
afternoons
non
largest
500
come
428
98
qualify
279
137338
d10
539
fine
while
665
concerning
iah
1230
oak
preferably
twelve
3357
323
nights
229
regarding
seven
inexpensive
420
416
repeating
scenario
139
82
kindly
limousines
345
afterwards
734
place
includes
106
1026
124
fifteen
bna
supper
oh
71
thereafter
2153
year
discount
1130
1030
world
trans
including
represented
o
'hare
exceeding
815
928
163
bur
419
cvg
1017
315
842
1083
0900
longest
called
snacks
645
ever
single

24
Makefile
Просмотреть файл

@ -110,7 +110,7 @@ ifdef CUDA_PATH
# This is a suggested/default location for NVML
INCLUDEPATH+=$(GDK_PATH)/include/nvidia/gdk
INCLUDEPATH+=$(CUB_PATH)
NVMLPATH=$(GDK_PATH)/src/gdk/nvml/lib
NVMLLIBPATH=$(GDK_PATH)/src/gdk/nvml/lib
# Set up CUDA includes and libraries
INCLUDEPATH += $(CUDA_PATH)/include
@ -232,6 +232,7 @@ ORIGINDIR:='$$ORIGIN'
CNTKMATH:=cntkmath
RPATH=-Wl,-rpath,
########################################
# Build info
@ -246,7 +247,6 @@ ifneq ("$(BUILDINFO_OUTPUT)","Success")
$(error Could not generate $(BUILDINFO))
endif
########################################
# Math library
########################################
@ -324,13 +324,11 @@ CNTKMATH_LIB:= $(LIBDIR)/lib$(CNTKMATH).so
ALL += $(CNTKMATH_LIB)
SRC+=$(MATH_SRC)
RPATH=-Wl,-rpath,
$(CNTKMATH_LIB): $(MATH_OBJ)
@echo $(SEPARATOR)
@echo creating $@ for $(ARCH) with build type $(BUILDTYPE)
@mkdir -p $(dir $@)
$(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBPATH) $(NVMLPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ $(LIBS) -fopenmp
$(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBPATH) $(NVMLLIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ $(LIBS) -fopenmp
########################################
# CNTKLibrary
@ -391,13 +389,11 @@ CNTKLIBRARY_LIB:=$(LIBDIR)/lib$(CNTKLIBRARY).so
ALL+=$(CNTKLIBRARY_LIB)
SRC+=$(CNTKLIBRARY_SRC)
RPATH=-Wl,-rpath,
$(CNTKLIBRARY_LIB): $(CNTKLIBRARY_OBJ) | $(CNTKMATH_LIB)
@echo $(SEPARATOR)
@mkdir -p $(dir $@)
@echo building output for $(ARCH) with build type $(BUILDTYPE)
$(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ $(LIBS) -l$(CNTKMATH)
$(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLLIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ $(LIBS) -l$(CNTKMATH)
########################################
# CNTKLibrary tests
@ -416,13 +412,11 @@ CNTKLIBRARY_TESTS_OBJ := $(patsubst %.cu, $(OBJDIR)/%.o, $(patsubst %.cpp, $(OBJ
ALL+=$(CNTKLIBRARY_TESTS)
SRC+=$(CNTKLIBRARY_TESTS_SRC)
RPATH=-Wl,-rpath,
$(CNTKLIBRARY_TESTS): $(CNTKLIBRARY_TESTS_OBJ) | $(CNTKLIBRARY_LIB)
@echo $(SEPARATOR)
@mkdir -p $(dir $@)
@echo building output for $(ARCH) with build type $(BUILDTYPE)
$(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLPATH)) $(patsubst %,$(RPATH)%, $(ORIGINLIBDIR) $(LIBPATH)) -o $@ $^ $(LIBS) -l$(CNTKLIBRARY) -l$(CNTKMATH)
$(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLLIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINLIBDIR) $(LIBPATH)) -o $@ $^ $(LIBS) -l$(CNTKLIBRARY) -l$(CNTKMATH)
########################################
# LibEval
@ -456,13 +450,11 @@ EVAL_LIB:=$(LIBDIR)/lib$(EVAL).so
ALL+=$(EVAL_LIB)
SRC+=$(EVAL_SRC)
RPATH=-Wl,-rpath,
$(EVAL_LIB): $(EVAL_OBJ)
@echo $(SEPARATOR)
@mkdir -p $(dir $@)
@echo Building $(EVAL_LIB) for $(ARCH) with build type $(BUILDTYPE)
$(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBDIR) $(LIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ $(LIBS)
$(CXX) $(LDFLAGS) -shared $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLLIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINDIR) $(LIBPATH)) -o $@ $^ $(LIBS)
########################################
# Eval Sample client
@ -477,8 +469,6 @@ EVAL_SAMPLE_CLIENT_OBJ:=$(patsubst %.cpp, $(OBJDIR)/%.o, $(EVAL_SAMPLE_CLIENT_SR
ALL+=$(EVAL_SAMPLE_CLIENT)
SRC+=$(EVAL_SAMPLE_CLIENT_SRC)
RPATH=-Wl,-rpath,
$(EVAL_SAMPLE_CLIENT): $(EVAL_SAMPLE_CLIENT_OBJ) | $(EVAL_LIB) $(CNTKMATH_LIB)
@echo $(SEPARATOR)
@mkdir -p $(dir $@)
@ -797,7 +787,7 @@ $(CNTK): $(CNTK_OBJ) | $(CNTKMATH_LIB)
@echo $(SEPARATOR)
@mkdir -p $(dir $@)
@echo building output for $(ARCH) with build type $(BUILDTYPE)
$(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLPATH)) $(patsubst %,$(RPATH)%, $(ORIGINLIBDIR) $(LIBPATH)) -o $@ $^ $(LIBS) -l$(CNTKMATH) -fopenmp
$(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(NVMLLIBPATH)) $(patsubst %,$(RPATH)%, $(ORIGINLIBDIR) $(LIBPATH)) -o $@ $^ $(LIBS) -l$(CNTKMATH) -fopenmp
# deployable resources: standard library of BS
CNTK_CORE_BS:=$(BINDIR)/cntk.core.bs

4
README.md
Просмотреть файл

@ -1,6 +1,8 @@
# CNTK
## Latest news
*2016-07-05.* CNTK now supports *Deconvolution* and *Unpooling*. See the usage example in the Network number 4 in [MNIST Sample](https://github.com/Microsoft/CNTK/blob/master/Examples/Image/MNIST/README.md).
*2016-06-23.* New License Terms for CNTK 1bit-SGD and related components.
Effective immediately the License Terms for CNTK 1bit-SGD and related components have changed. The new Terms provide more flexibility and enable new usage scenarios, especially in commercial environments. Read the new Terms at the [standard location](https://cntk1bitsgd.codeplex.com/license). Please note, that while the new Terms are significantly more flexible comparing to the previous ones, they are still **more restrictive** than the main CNTK License. Consequently everything described in [Enabling 1bit-SGD](https://github.com/Microsoft/CNTK/wiki/Enabling-1bit-SGD) section of the Wiki remains valid.
@ -12,8 +14,6 @@ NuGet Package is added to CNTK v.1.5 binaries. See [CNTK Releases page](https://
*2016-06-15.* CNTK now supports building against a custom Intel® Math Kernel Library (MKL).
See [setup instructions](https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-your-machine) on how to set this up for your platform.
*2016-06-10.* See CNTK v.1.5 binary release announcement in the official [Microsoft Research Blog](https://blogs.msdn.microsoft.com/msr_er/2016/06/10/microsoft-improves-programming-flexibility-of-its-ai-toolkit/)
See [all news](https://github.com/Microsoft/CNTK/wiki/News).
## What is CNTK

0
Source/Readers/CNTKTextFormatReader/uci_to_cntk_text_format_converter.py → Scripts/uci2ctf.py
Просмотреть файл

73
Source/CNTK/CNTK.cpp
Просмотреть файл

@ -117,6 +117,23 @@ size_t GetMaxEpochs(const ConfigParameters& configParams)
return maxEpochs;
}
#ifndef CPUONLY
// abort execution is GPU is not supported (e.g. compute capability not supported)
void CheckSupportForGpu(DEVICEID_TYPE deviceId)
{
auto gpuData = GetGpuData(deviceId);
if (gpuData.validity == GpuValidity::ComputeCapabilityNotSupported)
{
InvalidArgument("CNTK: The GPU (%s) has compute capability %d.%d. CNTK is only supported on GPUs with compute capability 3.0 or greater",
gpuData.name.c_str(), gpuData.versionMajor, gpuData.versionMinor);
}
else if (gpuData.validity == GpuValidity::UnknownDevice)
{
InvalidArgument("CNTK: Unknown GPU with Device ID %d.", gpuData.deviceId);
}
}
#endif
// special temporary function to guard against a now invalid usage of "truncated" which exists in some IPG production setups
static void DisableLegacyTruncationSettings(const ConfigParameters& TopLevelConfig, const ConfigParameters& commandConfig)
{
@ -370,6 +387,30 @@ void PrintUsageInfo()
LOGPRINTF(stderr, "-------------------------------------------------------------------\n");
}
// print gpu info for current gpu devices (e.g. Device[0]: cores = 2496; computeCapability = 5.2; type = "Quadro M4000"; memory = 8192 MB)
void PrintGpuInfo()
{
#ifndef CPUONLY
std::vector<GpuData> gpusData = GetAllGpusData();
if (gpusData.empty())
{
LOGPRINTF(stderr, "No GPUs found\n");
return;
}
LOGPRINTF(stderr, "-------------------------------------------------------------------\n");
LOGPRINTF(stderr, "GPU info:\n\n");
for (GpuData& data : gpusData)
{
LOGPRINTF(stderr, "\t\tDevice[%d]: cores = %d; computeCapability = %d.%d; type = \"%s\"; memory = %lu MB\n",
data.deviceId, data.cudaCores, data.versionMajor, data.versionMinor, data.name.c_str(), data.totalMemory);
}
LOGPRINTF(stderr, "-------------------------------------------------------------------\n");
#endif
}
// ---------------------------------------------------------------------------
// main() for use with BrainScript
// ---------------------------------------------------------------------------
@ -461,6 +502,21 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wrapp
let valp = BS::Evaluate(expr); // evaluate parse into a dictionary
let& config = valp.AsRef<ScriptableObjects::IConfigRecord>(); // this is the dictionary
#ifndef CPUONLY
auto valpp = config.Find(L"deviceId");
if (valpp)
{
auto valp = *valpp;
if (!valp.Is<ScriptableObjects::String>()) // if it's not string 'auto' or 'cpu', then it's a gpu
{
if (static_cast<int>(valp) >= 0) // gpu (id >= 0)
{
CheckSupportForGpu(valp); // throws if gpu is not supported
}
}
}
#endif
// legacy parameters that have changed spelling
if (config.Find(L"DoneFile")) // variables follow camel case (start with lower-case letters)
InvalidArgument("Legacy spelling of 'DoneFile' no longer allowed. Use 'doneFile'.");
@ -499,6 +555,9 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wrapp
// echo config info to log
PrintBuiltInfo();
// echo gpu info to log
PrintGpuInfo();
// execute the actions
// std::string type = config(L"precision", "float");
int numCPUThreads = config(L"numCPUThreads", 0);
@ -556,6 +615,18 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[])
{
ConfigParameters config;
std::string rawConfigString = ConfigParameters::ParseCommandLine(argc, argv, config); // get the command param set they want
#ifndef CPUONLY
ConfigValue val = config("deviceId", "auto");
if (!EqualCI(val, "cpu") && !EqualCI(val, "auto"))
{
if (static_cast<int>(val) >= 0) // gpu (id >= 0)
{
CheckSupportForGpu(static_cast<int>(val)); // throws if gpu is not supported
}
}
#endif
bool timestamping = config(L"timestamping", false);
if (timestamping)
{
@ -599,6 +670,8 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[])
}
PrintBuiltInfo(); // this one goes to log file
PrintGpuInfo();
std::string timestamp = TimeDateStamp();
// dump config info

512
Source/CNTKv2LibraryDll/API/CNTKLibrary.h
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

232
Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h
Просмотреть файл

@ -53,6 +53,10 @@ namespace Microsoft { namespace MSR { namespace CNTK {
#ifndef _MSC_VER
#define _countof(_Array) (sizeof(_Array) / sizeof(_Array[0]))
static inline wchar_t* _wcsdup(const wchar_t *s)
{
return ::wcsdup(s);
}
#endif
namespace CNTK
@ -132,102 +136,76 @@ namespace CNTK
class CompositeFunction;
class Function;
namespace _Internal
namespace Internal
{
// A reference counter to be used as the base class for all reference counted types.
class _ReferenceCounter
// A reference count to be used as the base class for all reference counted types.
class CNTK_API ReferenceCount
{
public:
// Constructor.
_ReferenceCounter() : m_rc(0) {}
ReferenceCount();
virtual ~ReferenceCount();
// Destructor.
virtual ~_ReferenceCounter() {}
// Add a reference.
// Thread-safe.
size_t AddReference()
{
return ++m_rc;
}
// Remove a reference.
// Thread-safe.
size_t RemoveReference()
{
assert(m_rc.load() > 0);
return --m_rc;
}
// Return the reference count value
size_t GetReferenceCount()
{
return m_rc.load();
}
size_t AddReference();
size_t RemoveReference();
size_t GetReferenceCount();
private:
std::atomic<size_t> m_rc;
std::atomic<size_t>* m_rc;
};
// A smart pointer to a reference counted object
// T must be a type derived from _Reference_counter
// T must be a type derived from ReferenceCount
template <class T>
class _ReferenceCounterSharedPtr final
class CNTK_API ReferenceCountedPtr final
{
typedef void(*_ReferenceCounterDeleter)(_ReferenceCounter* obj);
typedef void(*ReferenceCountedObjectDeleter)(ReferenceCount* obj);
public:
// Constructor
_ReferenceCounterSharedPtr(T* ptr = nullptr, _ReferenceCounterDeleter deleter = nullptr) : m_objPtr(ptr), m_deleter(deleter)
ReferenceCountedPtr(T* ptr = nullptr, ReferenceCountedObjectDeleter deleter = nullptr) : m_objPtr(ptr), m_deleter(deleter)
{
Init();
AddReferenceIfNeeded();
}
// Copy constructor
_ReferenceCounterSharedPtr(const _ReferenceCounterSharedPtr& other) : m_objPtr(nullptr), m_deleter(nullptr)
ReferenceCountedPtr(const ReferenceCountedPtr& other) : m_objPtr(nullptr), m_deleter(nullptr)
{
*this = other;
}
// Move constructor
_ReferenceCounterSharedPtr(_ReferenceCounterSharedPtr&& other) : m_objPtr(nullptr), m_deleter(nullptr)
ReferenceCountedPtr(ReferenceCountedPtr&& other) : m_objPtr(nullptr), m_deleter(nullptr)
{
*this = std::move(other);
}
// Destructor
~_ReferenceCounterSharedPtr()
~ReferenceCountedPtr()
{
UnInitialize(m_objPtr, m_deleter);
DeleteReferenceIfNeeded(m_objPtr, m_deleter);
}
// Assignment operator
_ReferenceCounterSharedPtr& operator=(const _ReferenceCounterSharedPtr& other)
ReferenceCountedPtr& operator=(const ReferenceCountedPtr& other)
{
if (this != &other)
{
T* oldPtr = m_objPtr;
_ReferenceCounterDeleter oldDeleter = m_deleter;
ReferenceCountedObjectDeleter oldDeleter = m_deleter;
m_objPtr = other.m_objPtr;
m_deleter = other.m_deleter;
Init();
AddReferenceIfNeeded();
UnInitialize(oldPtr, oldDeleter);
DeleteReferenceIfNeeded(oldPtr, oldDeleter);
}
return *this;
}
// Move-assignment operator
_ReferenceCounterSharedPtr& operator=(_ReferenceCounterSharedPtr&& other)
ReferenceCountedPtr& operator=(ReferenceCountedPtr&& other)
{
assert(this != &other);
T* oldPtr = m_objPtr;
_ReferenceCounterDeleter oldDeleter = m_deleter;
ReferenceCountedObjectDeleter oldDeleter = m_deleter;
m_objPtr = other.m_objPtr;
m_deleter = other.m_deleter;
@ -236,16 +214,16 @@ namespace CNTK
other.m_objPtr = nullptr;
other.m_deleter = nullptr;
UnInitialize(oldPtr, oldDeleter);
DeleteReferenceIfNeeded(oldPtr, oldDeleter);
return *this;
}
// Conversion to a ReferenceCountedSharedPtr instance of a base type
template <typename Base, typename std::enable_if<std::is_base_of<Base, T>::value>::type* = nullptr>
operator _ReferenceCounterSharedPtr<Base>()
operator ReferenceCountedPtr<Base>()
{
return _ReferenceCounterSharedPtr<Base>(m_objPtr, m_deleter);
return ReferenceCountedPtr<Base>(m_objPtr, m_deleter);
}
T* operator->() const
@ -269,25 +247,25 @@ namespace CNTK
}
private:
void Init()
void AddReferenceIfNeeded()
{
static_assert(std::is_base_of<_ReferenceCounter, T>::value, "_ReferenceCounterSharedPtr<T> can only be used when _ReferenceCounter is a base type of T!");
static_assert(std::is_base_of<ReferenceCount, T>::value, "ReferenceCountedPtr<T> can only be used when ReferenceCount is a base type of T!");
if (m_objPtr != nullptr)
reinterpret_cast<_ReferenceCounter*>(m_objPtr)->AddReference();
reinterpret_cast<ReferenceCount*>(m_objPtr)->AddReference();
}
static void UnInitialize(T* objPtr, _ReferenceCounterDeleter deleter)
static void DeleteReferenceIfNeeded(T* objPtr, ReferenceCountedObjectDeleter deleter)
{
static_assert(std::is_base_of<_ReferenceCounter, T>::value, "_ReferenceCounterSharedPtr<T> can only be used when _ReferenceCounter is a base type of T!");
static_assert(std::is_base_of<ReferenceCount, T>::value, "ReferenceCountedPtr<T> can only be used when ReferenceCount is a base type of T!");
if (objPtr != nullptr)
{
size_t refCountRemaining = reinterpret_cast<_ReferenceCounter*>(objPtr)->RemoveReference();
size_t refCountRemaining = reinterpret_cast<ReferenceCount*>(objPtr)->RemoveReference();
if (refCountRemaining == 0)
{
if (deleter != nullptr)
deleter(reinterpret_cast<_ReferenceCounter*>(objPtr));
deleter(reinterpret_cast<ReferenceCount*>(objPtr));
else
delete objPtr;
}
@ -296,35 +274,45 @@ namespace CNTK
private:
T* m_objPtr;
_ReferenceCounterDeleter m_deleter;
ReferenceCountedObjectDeleter m_deleter;
};
template <typename T>
bool operator==(const _ReferenceCounterSharedPtr<T>& first, const _ReferenceCounterSharedPtr<T>& second)
bool operator==(const ReferenceCountedPtr<T>& first, const ReferenceCountedPtr<T>& second)
{
return first.GetPtr() == second.GetPtr();
}
// A simple vector implementation with a C ABI to allow usage across the library DLL boundary
// A wrapper around the STL vector implementation with a safe ABI to allow usage across the library DLL boundary
// as STL vectors cannot be used across the DLL boundary
template <typename T>
class CNTK_API _SimpleVector final
class CNTK_API SimpleVector final
{
template <typename ValueType>
friend CNTK_API bool operator==(const _SimpleVector<ValueType>& first, const _SimpleVector<ValueType>& second);
friend CNTK_API bool operator==(const SimpleVector<ValueType>& first, const SimpleVector<ValueType>& second);
friend class CNTK::Function;
public:
_SimpleVector();
_SimpleVector(size_t numElements, const T& initVal = T());
~_SimpleVector();
SimpleVector();
_SimpleVector(const _SimpleVector& other);
_SimpleVector& operator=(const _SimpleVector& other);
template <typename ContainerType, typename std::enable_if<std::is_same<ContainerType, std::vector<T>>::value ||
std::is_same<ContainerType, std::initializer_list<T>>::value ||
std::is_same<ContainerType, std::array<T, sizeof(ContainerType) / sizeof(T)>>::value>::type* = nullptr>
SimpleVector(const ContainerType& initList)
: SimpleVector(initList.size())
{
std::copy(initList.begin(), initList.end(), Data());
}
_SimpleVector(_SimpleVector&& other);
_SimpleVector& operator=(_SimpleVector&& other);
SimpleVector(size_t numElements, const T& initVal = T());
~SimpleVector();
SimpleVector(const SimpleVector& other);
SimpleVector& operator=(const SimpleVector& other);
SimpleVector(SimpleVector&& other);
SimpleVector& operator=(SimpleVector&& other);
T& operator[](size_t idx);
const T& operator[](size_t idx) const;
@ -353,73 +341,62 @@ namespace CNTK
{
auto insertRet = retSet.insert(this->operator[](i));
if (ensureUnique && !insertRet.second)
RuntimeError("A _SimpleVector with duplicate elements cannot be converted to an unordered_set");
RuntimeError("A SimpleVector with duplicate elements cannot be converted to an unordered_set");
}
return retSet;
}
template <typename ContainerType, typename std::enable_if<std::is_same<ContainerType, std::vector<T>>::value ||
std::is_same<ContainerType, std::initializer_list<T>>::value ||
std::is_same<ContainerType, std::array<T, sizeof(ContainerType) / sizeof(T)>>::value>::type* = nullptr>
static _SimpleVector<T> CreateSimpleVector(const ContainerType& initList)
{
_SimpleVector<T> simpleVector(initList.size());
std::copy(initList.begin(), initList.end(), simpleVector.Data());
return simpleVector;
}
private:
std::vector<T>* m_vector;
};
template <typename ValueType>
CNTK_API bool operator==(const _SimpleVector<ValueType>& first, const _SimpleVector<ValueType>& second);
CNTK_API bool operator==(const SimpleVector<ValueType>& first, const SimpleVector<ValueType>& second);
template <typename ValueType>
bool operator!=(const _SimpleVector<ValueType>& first, const _SimpleVector<ValueType>& second)
bool operator!=(const SimpleVector<ValueType>& first, const SimpleVector<ValueType>& second)
{
return !(first == second);
}
// A simple set implementation with a C ABI to allow usage across the library DLL boundary
// A wrapper around the STL set implementation with a safe ABI to allow usage across the library DLL boundary
// as STL sets cannot be used across the DLL boundary
template <typename KeyType>
class CNTK_API _SimpleSet final
class CNTK_API SimpleSet final
{
friend class CNTK::CompositeFunction;
template <typename T>
friend CNTK_API bool operator==(const _SimpleSet<T>& first, const _SimpleSet<T>& second);
friend CNTK_API bool operator==(const SimpleSet<T>& first, const SimpleSet<T>& second);
public:
_SimpleSet();
~_SimpleSet();
SimpleSet();
~SimpleSet();
_SimpleSet(const _SimpleSet& other);
_SimpleSet& operator=(const _SimpleSet& other);
SimpleSet(const SimpleSet& other);
SimpleSet& operator=(const SimpleSet& other);
_SimpleSet(_SimpleSet&& other);
_SimpleSet& operator=(_SimpleSet&& other);
SimpleSet(SimpleSet&& other);
SimpleSet& operator=(SimpleSet&& other);
bool Insert(const KeyType& key);
bool Contains(const KeyType& key) const;
size_t Size() const;
operator _SimpleVector<KeyType>() const;
operator SimpleVector<KeyType>() const;
operator std::unordered_set<KeyType>() const
{
return ((_SimpleVector<KeyType>)(*this)).GetAsUnorderedSet();
return ((SimpleVector<KeyType>)(*this)).GetAsUnorderedSet();
}
static _SimpleSet<KeyType> CreateSimpleSet(const std::unordered_set<KeyType>& initSet)
static SimpleSet<KeyType> CreateSimpleSet(const std::unordered_set<KeyType>& initSet)
{
_SimpleSet<KeyType> simpleSet;
for (auto iter = initSet.begin(); iter != initSet.end(); ++iter)
simpleSet.Insert(*iter);
SimpleSet<KeyType> simpleSet;
for (auto key : initSet)
simpleSet.Insert(key);
return simpleSet;
}
@ -429,31 +406,31 @@ namespace CNTK
};
template <typename KeyType>
CNTK_API bool operator==(const _SimpleSet<KeyType>& first, const _SimpleSet<KeyType>& second);
CNTK_API bool operator==(const SimpleSet<KeyType>& first, const SimpleSet<KeyType>& second);
template <typename KeyType>
bool operator!=(const _SimpleSet<KeyType>& first, const _SimpleSet<KeyType>& second)
bool operator!=(const SimpleSet<KeyType>& first, const SimpleSet<KeyType>& second)
{
return !(first == second);
}
// A simple map implementation with a C ABI to allow usage across the library DLL boundary
// A wrapper aroound the STL map implementation with a safe ABI to allow usage across the library DLL boundary
// as STL maps cannot be used across the DLL boundary
template <typename KeyType, typename ValueType>
class CNTK_API _SimpleMap final
class CNTK_API SimpleMap final
{
friend class CNTK::CompositeFunction;
friend class CNTK::Function;
public:
_SimpleMap();
~_SimpleMap();
SimpleMap();
~SimpleMap();
_SimpleMap(const _SimpleMap& other);
_SimpleMap& operator=(const _SimpleMap& other);
SimpleMap(const SimpleMap& other);
SimpleMap& operator=(const SimpleMap& other);
_SimpleMap(_SimpleMap&& other);
_SimpleMap& operator=(_SimpleMap&& other);
SimpleMap(SimpleMap&& other);
SimpleMap& operator=(SimpleMap&& other);
ValueType& operator[](const KeyType& key);
const ValueType& operator[](const KeyType& key) const;
@ -462,13 +439,13 @@ namespace CNTK
bool Contains(const KeyType& key) const;
size_t Size() const;
_SimpleSet<KeyType> Keys() const;
SimpleSet<KeyType> Keys() const;
static _SimpleMap<KeyType, ValueType> CreateSimpleMap(const std::unordered_map<KeyType, ValueType>& initMap)
static SimpleMap<KeyType, ValueType> CreateSimpleMap(const std::unordered_map<KeyType, ValueType>& initMap)
{
_SimpleMap<KeyType, ValueType> simpleMap;
for (auto iter = initMap.begin(); iter != initMap.end(); ++iter)
simpleMap.Insert(iter->first, iter->second);
SimpleMap<KeyType, ValueType> simpleMap;
for (auto keyValuePair : initMap)
simpleMap.Insert(keyValuePair.first, keyValuePair.second);
return simpleMap;
}
@ -480,35 +457,28 @@ namespace CNTK
// Forward declarations
class NDArrayView;
typedef _Internal::_ReferenceCounterSharedPtr<NDArrayView> NDArrayViewPtr;
typedef Internal::ReferenceCountedPtr<NDArrayView> NDArrayViewPtr;
class NDMask;
typedef _Internal::_ReferenceCounterSharedPtr<NDMask> NDMaskPtr;
typedef Internal::ReferenceCountedPtr<NDMask> NDMaskPtr;
class Value;
typedef _Internal::_ReferenceCounterSharedPtr<Value> ValuePtr;
typedef Internal::ReferenceCountedPtr<Value> ValuePtr;
class Function;
typedef _Internal::_ReferenceCounterSharedPtr<Function> FunctionPtr;
typedef Internal::ReferenceCountedPtr<Function> FunctionPtr;
inline wchar_t* CopyString(const wchar_t* source)
namespace Internal
{
size_t len = wcslen(source) + 1;
wchar_t* copy = new wchar_t[len];
#ifdef _WIN32
wcscpy_s(copy, len, source);
#else
wcscpy(copy, source);
#endif
return copy;
CNTK_API FunctionPtr Combine(const Internal::SimpleVector<FunctionPtr>& operands, const std::wstring& name = L"");
}
}
namespace std {
template <typename T>
struct hash<CNTK::_Internal::_ReferenceCounterSharedPtr<T>>
struct hash<CNTK::Internal::ReferenceCountedPtr<T>>
{
size_t operator()(const CNTK::_Internal::_ReferenceCounterSharedPtr<T>& x) const
size_t operator()(const CNTK::Internal::ReferenceCountedPtr<T>& x) const
{
return std::hash<const void*>()(x.GetPtr());
}

291
Source/CNTKv2LibraryDll/Function.cpp
Просмотреть файл

@ -17,40 +17,50 @@ bool g_shareNodeValueMatrices = true;
namespace CNTK
{
_Internal::_SimpleVector<Variable> Function::_Inputs() const
Internal::SimpleVector<Variable> Function::InputsImpl() const
{
const CompositeFunction* compositeFunction = dynamic_cast<const CompositeFunction*>(this);
if (compositeFunction == nullptr)
return m_inputs;
else
return _Internal::_SimpleVector<Variable>::CreateSimpleVector(compositeFunction->DetermineInputs());
return Internal::SimpleVector<Variable>(compositeFunction->DetermineInputs());
}
/*virtual*/ void Function::_ReplacePlaceholders(const _Internal::_SimpleMap<Placeholder, Variable>& placeholderReplacements, _Internal::_SimpleSet<const Function*>& visitedFunctions, _Internal::_SimpleSet<Placeholder>& replacedPlaceholders)
/*virtual*/ void Function::_ReplacePlaceholders(const Internal::SimpleMap<Placeholder, Variable>& placeholderReplacements,
Internal::SimpleSet<const Function*>& visitedFunctions,
Internal::SimpleSet<Placeholder>& replacedPlaceholders)
{
visitedFunctions.Insert(this);
for (auto iter = m_inputs.m_vector->begin(); iter != m_inputs.m_vector->end(); ++iter)
for (auto& inputVar : *(m_inputs.m_vector))
{
if (iter->IsPlaceholder())
if (inputVar.IsPlaceholder())
{
Placeholder placeholder(*iter);
Placeholder placeholder(inputVar);
if (placeholderReplacements.Contains(placeholder))
{
*iter = placeholderReplacements[placeholder];
inputVar = placeholderReplacements[placeholder];
replacedPlaceholders.Insert(placeholder);
}
}
else if ((iter->Kind() == VariableKind::Output) && !visitedFunctions.Contains(iter->Owner()))
iter->Owner()->_ReplacePlaceholders(placeholderReplacements, visitedFunctions, replacedPlaceholders);
else if (inputVar.IsOutput() && !visitedFunctions.Contains(inputVar.Owner()))
inputVar.Owner()->_ReplacePlaceholders(placeholderReplacements, visitedFunctions, replacedPlaceholders);
}
}
// Recursively create a sub-network of ComputationNode instances corresponding to the graph of Functions
// underlying the specified 'variable' and return the ComputationNode instance that corresponds to the
// top level 'variable'
template <typename ElementType>
/*static*/ ComputationNodeBasePtr CompositeFunction::GetNode(const Variable& variable, Microsoft::MSR::CNTK::ComputationNetworkPtr& network, ComputationNetworkBuilder<ElementType>& builder, std::unordered_map<Variable, ComputationNodeBasePtr>& variableToNodeMap, std::unordered_map<Variable, bool>& isVariableRootMap)
/*static*/ ComputationNodeBasePtr CompositeFunction::GetNode(const Variable& variable,
Microsoft::MSR::CNTK::ComputationNetworkPtr& network,
ComputationNetworkBuilder<ElementType>& builder,
std::unordered_map<Variable, ComputationNodeBasePtr>& variableToNodeMap,
std::unordered_map<Variable, bool>& isVariableRootMap)
{
if (variableToNodeMap.find(variable) != variableToNodeMap.end())
return variableToNodeMap[variable];
auto iter = variableToNodeMap.find(variable);
if (iter != variableToNodeMap.end())
return iter->second;
// Lets add a null entry in the map for this variable, to break infinite recursion when processing recurrent graphs
variableToNodeMap[variable] = nullptr;
@ -66,7 +76,7 @@ namespace CNTK
auto matrix = variable.IsConstant() ? value->GetMatrix<ElementType>()->AsReference() : value->GetWritableMatrix<ElementType>()->AsReference();
computationNodePtr->Value() = std::move(matrix);
}
else if (variable.Kind() == VariableKind::Input)
else if (variable.IsInput())
{
// TODO: Specify dynamic axis
if (variable.IsSparseInput())
@ -83,23 +93,27 @@ namespace CNTK
}
else
{
assert(variable.Kind() == VariableKind::Output);
assert(variable.IsOutput());
computationNodePtr = GetOutputVariableNode(variable, network, builder, variableToNodeMap, isVariableRootMap)->template As<ComputationNode<ElementType>>()->shared_from_this();
}
variableToNodeMap[variable] = computationNodePtr;
isVariableRootMap[variable] = (variable.Kind() == VariableKind::Output);
isVariableRootMap[variable] = variable.IsOutput();
return computationNodePtr;
}
template <typename ElementType>
/*static*/ ComputationNodeBasePtr CompositeFunction::GetOutputVariableNode(const Variable& variable, Microsoft::MSR::CNTK::ComputationNetworkPtr& network, ComputationNetworkBuilder<ElementType>& builder, std::unordered_map<Variable, ComputationNodeBasePtr>& variableToNodeMap, std::unordered_map<Variable, bool>& isVariableRootMap)
/*static*/ ComputationNodeBasePtr CompositeFunction::GetOutputVariableNode(const Variable& variable,
Microsoft::MSR::CNTK::ComputationNetworkPtr& network,
ComputationNetworkBuilder<ElementType>& builder,
std::unordered_map<Variable, ComputationNodeBasePtr>& variableToNodeMap,
std::unordered_map<Variable, bool>& isVariableRootMap)
{
assert(variable.Kind() == VariableKind::Output);
assert(variable.IsOutput());
Function* function = variable.Owner();
ComputationNodeBasePtr computationNodePtr;
if (dynamic_cast<PrimitiveFunction*>(function) != nullptr)
if (dynamic_cast<PrimitiveFunction*>(function))
{
PrimitiveFunction* primitiveFunction = dynamic_cast<PrimitiveFunction*>(function);
@ -134,7 +148,7 @@ namespace CNTK
case PrimitiveOpType::CrossEntropyWithSoftmax:
computationNodePtr = builder.CrossEntropyWithSoftmax(input1Node, input0Node, function->Name());
break;
case PrimitiveOpType::PredictionError:
case PrimitiveOpType::ClassificationError:
computationNodePtr = builder.ErrorPrediction(input1Node, input0Node, function->Name());
break;
case PrimitiveOpType::Exp:
@ -180,8 +194,10 @@ namespace CNTK
break;
}
case PrimitiveOpType::Combine:
for (size_t i = 0; i < functionInputs.size(); ++i)
GetNode(functionInputs[i], network, builder, variableToNodeMap, isVariableRootMap);
// This operation is just a no-op and is a means to combine multiple functions to create a single Function
// whose outputs are a union of tyhe outputs of the Functions being combined.
for (auto inputVar : functionInputs)
GetNode(inputVar, network, builder, variableToNodeMap, isVariableRootMap);
computationNodePtr = variableToNodeMap[variable];
@ -193,8 +209,8 @@ namespace CNTK
if (op != PrimitiveOpType::Combine)
{
for (size_t i = 0; i < functionInputs.size(); ++i)
isVariableRootMap[functionInputs[i]] = false;
for (auto inputVar : functionInputs)
isVariableRootMap[inputVar] = false;
}
}
else
@ -206,7 +222,7 @@ namespace CNTK
}
template <typename ElementType>
ComputationNetworkPtr CompositeFunction::GetComputationNetwork(const DeviceDescriptor& device, const _Internal::_SimpleSet<Variable>& backpropRoots)
ComputationNetworkPtr CompositeFunction::GetComputationNetwork(const DeviceDescriptor& device, const Internal::SimpleSet<Variable>& backpropRoots)
{
if (m_computationNetwork != nullptr)
{
@ -237,52 +253,52 @@ namespace CNTK
auto rootFunction = RootFunction();
auto rootFunctionOutputs = rootFunction->Outputs();
std::vector<ComputationNodeBasePtr> forwardRootNodes;
for (size_t i = 0; i < rootFunctionOutputs.size(); ++i)
for (auto rootOutput : rootFunctionOutputs)
{
auto currentRootNode = GetNode(rootFunctionOutputs[i], m_computationNetwork, builder, m_variableToNodeMap, m_isVariableRootMap);
auto currentRootNode = GetNode(rootOutput, m_computationNetwork, builder, m_variableToNodeMap, m_isVariableRootMap);
forwardRootNodes.push_back(currentRootNode);
if (backpropRoots.Contains(rootFunctionOutputs[i]))
backpropRootNode = m_variableToNodeMap[rootFunctionOutputs[i]];
if (backpropRoots.Contains(rootOutput))
backpropRootNode = m_variableToNodeMap[rootOutput];
}
// If any of the function outputs is not a root node, we need to explicitly add it to the 'output' group of the ComputationNetwork
for (size_t i = 0; i < rootFunctionOutputs.size(); ++i)
for (auto rootOutput : rootFunctionOutputs)
{
if (!m_isVariableRootMap[rootFunctionOutputs[i]])
m_computationNetwork->AddToNodeGroup(L"output", m_variableToNodeMap[rootFunctionOutputs[i]]);
if (!m_isVariableRootMap[rootOutput])
m_computationNetwork->AddToNodeGroup(L"output", m_variableToNodeMap[rootOutput]);
}
m_currentBackpropRoots = backpropRoots;
// In case of recurrence, the inputs of some of the ComputationNodes are not attached due to cycles.
// Now attach those after we have created all ComputationNodes in the network
for (auto iter = m_variableToNodeMap.begin(); iter != m_variableToNodeMap.end(); ++iter)
for (auto varNodePair : m_variableToNodeMap)
{
auto currentComputationNodeInputs = iter->second->GetInputs();
auto currentComputationNodeInputs = varNodePair.second->GetInputs();
// TODO: Can any node other than a non PastValue/FutureValue Function have a null input attached after the first pass is finished?
if (std::find(currentComputationNodeInputs.begin(), currentComputationNodeInputs.end(), nullptr) != currentComputationNodeInputs.end())
{
// We found a null input; this variable must correspond to a PastValue or FutureValue function
const PrimitiveFunction* primitiveFunc = dynamic_cast<const PrimitiveFunction*>(iter->first.Owner().GetPtr());
const PrimitiveFunction* primitiveFunc = dynamic_cast<const PrimitiveFunction*>(varNodePair.first.Owner().GetPtr());
if ((primitiveFunc == nullptr) || ((primitiveFunc->OpType() != PrimitiveOpType::PastValue) && (primitiveFunc->OpType() != PrimitiveOpType::FutureValue)))
InvalidArgument("Invalid Function graph detected; recurrence found at a Function that is not a PastValue/FutureValue function");
// The 2nd input of the PastValue/FutureValue function denotes the recurrent input
auto actualInput = m_variableToNodeMap[primitiveFunc->Inputs()[1]];
iter->second->AttachInputs({ actualInput });
varNodePair.second->AttachInputs({ actualInput });
}
}
m_computationNetwork->CompileNetwork();
// Verify that the shapes of the output Variables that we computed match the corresponding nodes in the ComputationNetwork
for (auto iter = m_variableToNodeMap.begin(); iter != m_variableToNodeMap.end(); ++iter)
for (auto varNodePair : m_variableToNodeMap)
{
if (iter->first.Kind() == VariableKind::Output)
if (varNodePair.first.IsOutput())
{
auto outputVar = iter->first;
auto outputVar = varNodePair.first;
auto computationNodePtr = m_variableToNodeMap[outputVar];
auto outputShape = outputVar.Shape();
auto computationNodeSampleLayout = computationNodePtr->GetSampleLayout();
@ -442,8 +458,8 @@ namespace CNTK
{
// Just create a view over the existing matrix itself
auto tensorView = new TensorView<ElementType>(std::make_shared<Matrix<ElementType>>(matrix.AsReference()), AsTensorShape(valueDataShape));
auto data = NDArrayViewPtr(new NDArrayView(AsDataType<ElementType>(), AsDeviceDescriptor(matrix.GetDeviceId()), AsStorageFormat(matrix.GetFormat()), valueDataShape, true, tensorView), [](_ReferenceCounter* ptr) { delete ptr; });
return ValuePtr(new Value(data), [](_ReferenceCounter* ptr) { delete ptr; });
auto data = NDArrayViewPtr(new NDArrayView(AsDataType<ElementType>(), AsDeviceDescriptor(matrix.GetDeviceId()), AsStorageFormat(matrix.GetFormat()), valueDataShape, true, tensorView), [](ReferenceCount* ptr) { delete ptr; });
return ValuePtr(new Value(data), [](ReferenceCount* ptr) { delete ptr; });
}
if (layout->GetNumCols() != matrix.GetNumCols())
@ -454,10 +470,10 @@ namespace CNTK
std::vector<size_t> sequenceLengths;
auto& layoutSequences = layout->GetAllSequences();
for (auto iter = layoutSequences.begin(); iter != layoutSequences.end(); ++iter)
for (auto sequenceInfo : layoutSequences)
{
if (iter->seqId != GAP_SEQUENCE_ID)
sequenceLengths.push_back(iter->GetNumTimeSteps());
if (sequenceInfo.seqId != GAP_SEQUENCE_ID)
sequenceLengths.push_back(sequenceInfo.GetNumTimeSteps());
}
// Reshuffle to data to unpack and uninterleave the CNTK form data
@ -473,13 +489,13 @@ namespace CNTK
size_t targetColIdxForInvalidColumns = sequencesShorterThanLongestSequence.empty() ? 0 : (((sequencesShorterThanLongestSequence[0] + 1) * maxNumTimeSteps) - 1);
std::vector<ElementType> scatterIndicesVector(layout->GetNumCols(), (ElementType)targetColIdxForInvalidColumns);
size_t i = 0;
for (auto iter = layoutSequences.begin(); iter != layoutSequences.end(); ++iter)
for (auto sequenceInfo : layoutSequences)
{
if (iter->seqId != GAP_SEQUENCE_ID)
if (sequenceInfo.seqId != GAP_SEQUENCE_ID)
{
size_t targetParallelStreamIdx = iter->s;
size_t targetStartIdxInParallelStream = iter->tBegin;
for (size_t j = 0; j < iter->GetNumTimeSteps(); ++j)
size_t targetParallelStreamIdx = sequenceInfo.s;
size_t targetStartIdxInParallelStream = sequenceInfo.tBegin;
for (size_t j = 0; j < sequenceInfo.GetNumTimeSteps(); ++j)
scatterIndicesVector[((targetStartIdxInParallelStream + j) * layout->GetNumParallelSequences()) + targetParallelStreamIdx] = (ElementType)((i * maxNumTimeSteps) + j);
i++;
@ -493,40 +509,39 @@ namespace CNTK
NDMaskPtr mask;
if (!sequencesShorterThanLongestSequence.empty())
{
mask = NDMaskPtr(new NDMask({ maxNumTimeSteps, numSequences }, AsDeviceDescriptor(matrix.GetDeviceId())), [](_ReferenceCounter* ptr) { delete ptr; });
for (size_t i = 0; i < sequencesShorterThanLongestSequence.size(); ++i)
mask = NDMaskPtr(new NDMask({ maxNumTimeSteps, numSequences }, AsDeviceDescriptor(matrix.GetDeviceId())), [](ReferenceCount* ptr) { delete ptr; });
for (auto shortSequenceIdx : sequencesShorterThanLongestSequence)
{
size_t shorterSequenceIdx = sequencesShorterThanLongestSequence[i];
mask->MaskSection({ sequenceLengths[shorterSequenceIdx], shorterSequenceIdx }, { NDShape::InferredDimension, 1 });
mask->MaskSection({ sequenceLengths[shortSequenceIdx], shortSequenceIdx }, { NDShape::InferredDimension, 1 });
}
}
auto tensorView = new TensorView<ElementType>(shuffledMatrixData, AsTensorShape(valueDataShape));
auto data = NDArrayViewPtr(new NDArrayView(AsDataType<ElementType>(), AsDeviceDescriptor(matrix.GetDeviceId()), StorageFormat::Dense, valueDataShape, true, tensorView), [](_ReferenceCounter* ptr) { delete ptr; });
return ValuePtr(new Value(data, mask), [](_ReferenceCounter* ptr) { delete ptr; });
auto data = NDArrayViewPtr(new NDArrayView(AsDataType<ElementType>(), AsDeviceDescriptor(matrix.GetDeviceId()), StorageFormat::Dense, valueDataShape, true, tensorView), [](ReferenceCount* ptr) { delete ptr; });
return ValuePtr(new Value(data, mask), [](ReferenceCount* ptr) { delete ptr; });
}
void CompositeFunction::PopulateNetworkInputs(const _Internal::_SimpleMap<Variable, const ValuePtr>& arguments)
void CompositeFunction::PopulateNetworkInputs(const Internal::SimpleMap<Variable, const ValuePtr>& arguments)
{
auto functionArguments = this->Arguments();
std::vector<ComputationNodeBasePtr> inputNodes;
for (auto iter = functionArguments.begin(); iter != functionArguments.end(); ++iter)
for (auto argument : functionArguments)
{
// Ensure we have values for all arguments of the function
if (!arguments.Contains(*iter))
if (!arguments.Contains(argument))
InvalidArgument("Value not specified for required Function Argument");
auto argumentComputationNode = m_variableToNodeMap[*iter];
auto argumentComputationNode = m_variableToNodeMap[argument];
inputNodes.push_back(argumentComputationNode);
ValuePtr argumentValue = arguments[*iter];
ValuePtr argumentValue = arguments[argument];
MBLayoutPtr layout;
switch (argumentValue->Data()->GetDataType())
{
case DataType::Float:
{
auto CNTKMatrixAndMBLayout = GetCNTKImplMatrixAndMBLayoutFromValueObject<float>(*iter, argumentValue);
auto CNTKMatrixAndMBLayout = GetCNTKImplMatrixAndMBLayoutFromValueObject<float>(argument, argumentValue);
layout = CNTKMatrixAndMBLayout.second;
auto& nodeData = argumentComputationNode->As<ComputationNode<float>>()->Value();
@ -537,7 +552,7 @@ namespace CNTK
}
case DataType::Double:
{
auto CNTKMatrixAndMBLayout = GetCNTKImplMatrixAndMBLayoutFromValueObject<double>(*iter, argumentValue);
auto CNTKMatrixAndMBLayout = GetCNTKImplMatrixAndMBLayoutFromValueObject<double>(argument, argumentValue);
layout = CNTKMatrixAndMBLayout.second;
auto& nodeData = argumentComputationNode->As<ComputationNode<double>>()->Value();
@ -557,40 +572,40 @@ namespace CNTK
m_computationNetwork->BumpEvalTimeStamp(inputNodes);
}
void CompositeFunction::PopulateNetworkGradients(const _Internal::_SimpleMap<Variable, const ValuePtr>& gradients)
void CompositeFunction::PopulateNetworkGradients(const Internal::SimpleMap<Variable, const ValuePtr>& gradients)
{
auto functionOutputs = this->Outputs();
std::unordered_map<Variable, const ValuePtr>& gradientsValueMap = *gradients.m_map;
for (auto iter = gradientsValueMap.begin(); iter != gradientsValueMap.end(); ++iter)
for (auto gradientVarValuePair : gradientsValueMap)
{
// Only gradients for roots of the function can be specified
if (std::find(functionOutputs.begin(), functionOutputs.end(), iter->first) == functionOutputs.end())
if (std::find(functionOutputs.begin(), functionOutputs.end(), gradientVarValuePair.first) == functionOutputs.end())
InvalidArgument("Gradients cannot be specified for a Variable that is not an Output of the Function");
auto outputComputationNode = m_variableToNodeMap[iter->first];
auto outputComputationNode = m_variableToNodeMap[gradientVarValuePair.first];
auto nodeLayout = outputComputationNode->GetMBLayout();
ValuePtr gradientValue = iter->second;
ValuePtr gradientValue = gradientVarValuePair.second;
MBLayoutPtr layout;
switch (gradientValue->Data()->GetDataType())
{
case DataType::Float:
{
auto CNTKMatrixAndMBLayout = GetCNTKImplMatrixAndMBLayoutFromValueObject<float>(iter->first, gradientValue);
auto CNTKMatrixAndMBLayout = GetCNTKImplMatrixAndMBLayoutFromValueObject<float>(gradientVarValuePair.first, gradientValue);
layout = CNTKMatrixAndMBLayout.second;
if (((layout == nullptr) != (nodeLayout == nullptr)) || ((layout != nullptr) && (*layout != *nodeLayout)))
InvalidArgument("The layout of the specified gradient Value in incompatible with the layout of the corresponding Variable computed during Forward call");
outputComputationNode->As<ComputationNode<float>>()->ResetGradient(*CNTKMatrixAndMBLayout.first);
outputComputationNode->As<ComputationNode<float>>()->AssignGradient(*CNTKMatrixAndMBLayout.first);
break;
}
case DataType::Double:
{
auto CNTKMatrixAndMBLayout = GetCNTKImplMatrixAndMBLayoutFromValueObject<double>(iter->first, gradientValue);
auto CNTKMatrixAndMBLayout = GetCNTKImplMatrixAndMBLayoutFromValueObject<double>(gradientVarValuePair.first, gradientValue);
layout = CNTKMatrixAndMBLayout.second;
if (((layout == nullptr) != (nodeLayout == nullptr)) || ((layout != nullptr) && (*layout != *nodeLayout)))
InvalidArgument("The layout of the specified gradient Value in incompatible with the layout of the corresponding Variable computed during Forward call");
outputComputationNode->As<ComputationNode<double>>()->ResetGradient(*CNTKMatrixAndMBLayout.first);
outputComputationNode->As<ComputationNode<double>>()->AssignGradient(*CNTKMatrixAndMBLayout.first);
break;
}
default:
@ -622,12 +637,12 @@ namespace CNTK
void CompositeFunction::GetNetworkOutputs(std::unordered_map<Variable, ValuePtr>& outputs)
{
// Now copy the Forward values of output nodes from the network to outputs' Value objects
for (auto iter = outputs.begin(); iter != outputs.end(); ++iter)
for (auto outputVarValuePair : outputs)
{
auto computationNodePtr = m_variableToNodeMap[iter->first];
auto outputValuePtr = iter->second;
auto computationNodePtr = m_variableToNodeMap[outputVarValuePair.first];
auto outputValuePtr = outputVarValuePair.second;
auto outputShape = GetValueShape(iter->first, computationNodePtr);
auto outputShape = GetValueShape(outputVarValuePair.first, computationNodePtr);
if (outputValuePtr != nullptr)
{
// TODO: The shape of the specified output Value object must match the actual output shape
@ -635,38 +650,38 @@ namespace CNTK
InvalidArgument("The shape %s of the specified Value object for output does not match the actual output shape %s", AsString(outputValuePtr->Data()->Shape()).c_str(), AsString(outputShape).c_str());
}
switch (iter->first.GetDataType())
switch (outputVarValuePair.first.GetDataType())
{
case DataType::Float:
{
auto nodeValue = GetValueObjectFromCNTKImplMatrixAndMBLayout<float>(iter->first, computationNodePtr->As<ComputationNode<float>>()->Value(), computationNodePtr->GetMBLayout());
auto nodeValue = GetValueObjectFromCNTKImplMatrixAndMBLayout<float>(outputVarValuePair.first, computationNodePtr->As<ComputationNode<float>>()->Value(), computationNodePtr->GetMBLayout());
if (outputValuePtr == nullptr)
{
auto data = NDArrayViewPtr(new NDArrayView(iter->first.GetDataType(), outputShape, AsDeviceDescriptor(computationNodePtr->ValuePtr()->GetDeviceId())), [](_ReferenceCounter* ptr) { delete ptr; });
auto mask = (nodeValue->Mask() != nullptr) ? NDMaskPtr(new NDMask(nodeValue->Mask()->Shape(), nodeValue->Mask()->Device()), [](_ReferenceCounter* ptr) { delete ptr; }) : nullptr;
outputValuePtr = ValuePtr(new Value(data, mask), [](_ReferenceCounter* ptr) { delete ptr; });
auto data = NDArrayViewPtr(new NDArrayView(outputVarValuePair.first.GetDataType(), outputShape, AsDeviceDescriptor(computationNodePtr->ValuePtr()->GetDeviceId())), [](ReferenceCount* ptr) { delete ptr; });
auto mask = (nodeValue->Mask() != nullptr) ? NDMaskPtr(new NDMask(nodeValue->Mask()->Shape(), nodeValue->Mask()->Device()), [](ReferenceCount* ptr) { delete ptr; }) : nullptr;
outputValuePtr = ValuePtr(new Value(data, mask), [](ReferenceCount* ptr) { delete ptr; });
}
outputValuePtr->CopyFrom(*nodeValue);
break;
}
case DataType::Double:
{
auto nodeValue = GetValueObjectFromCNTKImplMatrixAndMBLayout<double>(iter->first, computationNodePtr->As<ComputationNode<double>>()->Value(), computationNodePtr->GetMBLayout());
auto nodeValue = GetValueObjectFromCNTKImplMatrixAndMBLayout<double>(outputVarValuePair.first, computationNodePtr->As<ComputationNode<double>>()->Value(), computationNodePtr->GetMBLayout());
if (outputValuePtr == nullptr)
{
auto data = NDArrayViewPtr(new NDArrayView(iter->first.GetDataType(), outputShape, AsDeviceDescriptor(computationNodePtr->ValuePtr()->GetDeviceId())), [](_ReferenceCounter* ptr) { delete ptr; });
auto mask = (nodeValue->Mask() != nullptr) ? NDMaskPtr(new NDMask(nodeValue->Mask()->Shape(), nodeValue->Mask()->Device()), [](_ReferenceCounter* ptr) { delete ptr; }) : nullptr;
outputValuePtr = ValuePtr(new Value(data, mask), [](_ReferenceCounter* ptr) { delete ptr; });
auto data = NDArrayViewPtr(new NDArrayView(outputVarValuePair.first.GetDataType(), outputShape, AsDeviceDescriptor(computationNodePtr->ValuePtr()->GetDeviceId())), [](ReferenceCount* ptr) { delete ptr; });
auto mask = (nodeValue->Mask() != nullptr) ? NDMaskPtr(new NDMask(nodeValue->Mask()->Shape(), nodeValue->Mask()->Device()), [](ReferenceCount* ptr) { delete ptr; }) : nullptr;
outputValuePtr = ValuePtr(new Value(data, mask), [](ReferenceCount* ptr) { delete ptr; });
}
outputValuePtr->CopyFrom(*nodeValue);
break;
}
default:
LogicError("Unsupported DataType %s", DataTypeName(iter->first.GetDataType()));
LogicError("Unsupported DataType %s", DataTypeName(outputVarValuePair.first.GetDataType()));
break;
}
outputs[iter->first] = outputValuePtr;
outputs[outputVarValuePair.first] = outputValuePtr;
}
}
@ -674,20 +689,20 @@ namespace CNTK
{
auto networkInputs = this->Inputs();
// Now copy the gradient values of input nodes of the network to gradients' Value objects
for (auto iter = gradients.begin(); iter != gradients.end(); ++iter)
for (auto gradientVarValuePair : gradients)
{
// Only gradients corresponding to inputs of the network can be obtained
if (std::find(networkInputs.begin(), networkInputs.end(), iter->first) == networkInputs.end())
if (std::find(networkInputs.begin(), networkInputs.end(), gradientVarValuePair.first) == networkInputs.end())
InvalidArgument("Backpropagated gradient values can only be obtained for inputs of a Function");
// Gradients can only be obtained for parameter variables or input variables that NeedsGradient
if (!iter->first.NeedsGradient())
if (!gradientVarValuePair.first.NeedsGradient())
InvalidArgument("Gradient value incorrectly requested for an Output or Constant Variable, or an Input Variable with NeedsGradient setting of false");
auto computationNodePtr = m_variableToNodeMap[iter->first];
auto gradientValuePtr = iter->second;
auto computationNodePtr = m_variableToNodeMap[gradientVarValuePair.first];
auto gradientValuePtr = gradientVarValuePair.second;
auto gradientShape = GetValueShape(iter->first, computationNodePtr);
auto gradientShape = GetValueShape(gradientVarValuePair.first, computationNodePtr);
if (gradientValuePtr != nullptr)
{
// TODO: The shape of the specified output Value object must match the actual output shape
@ -698,45 +713,45 @@ namespace CNTK
if (!computationNodePtr->NeedsGradient())
LogicError("Backpropagated gradient value cannot be read from a ComputationNode that has NeedsGradient set to false");
switch (iter->first.GetDataType())
switch (gradientVarValuePair.first.GetDataType())
{
case DataType::Float:
{
auto nodeValue = GetValueObjectFromCNTKImplMatrixAndMBLayout<float>(iter->first, computationNodePtr->As<ComputationNode<float>>()->Gradient(), computationNodePtr->GetMBLayout());
auto nodeValue = GetValueObjectFromCNTKImplMatrixAndMBLayout<float>(gradientVarValuePair.first, computationNodePtr->As<ComputationNode<float>>()->Gradient(), computationNodePtr->GetMBLayout());
if (gradientValuePtr == nullptr)
{
auto data = NDArrayViewPtr(new NDArrayView(iter->first.GetDataType(), gradientShape, AsDeviceDescriptor(computationNodePtr->ValuePtr()->GetDeviceId())), [](_ReferenceCounter* ptr) { delete ptr; });
auto mask = NDMaskPtr((nodeValue->Mask() != nullptr) ? new NDMask(nodeValue->Mask()->Shape(), nodeValue->Mask()->Device()) : nullptr, [](_ReferenceCounter* ptr) { delete ptr; });
gradientValuePtr = ValuePtr(new Value(data, mask), [](_ReferenceCounter* ptr) { delete ptr; });
auto data = NDArrayViewPtr(new NDArrayView(gradientVarValuePair.first.GetDataType(), gradientShape, AsDeviceDescriptor(computationNodePtr->ValuePtr()->GetDeviceId())), [](ReferenceCount* ptr) { delete ptr; });
auto mask = NDMaskPtr((nodeValue->Mask() != nullptr) ? new NDMask(nodeValue->Mask()->Shape(), nodeValue->Mask()->Device()) : nullptr, [](ReferenceCount* ptr) { delete ptr; });
gradientValuePtr = ValuePtr(new Value(data, mask), [](ReferenceCount* ptr) { delete ptr; });
}
gradientValuePtr->CopyFrom(*nodeValue);
break;
}
case DataType::Double:
{
auto nodeValue = GetValueObjectFromCNTKImplMatrixAndMBLayout<double>(iter->first, computationNodePtr->As<ComputationNode<double>>()->Gradient(), computationNodePtr->GetMBLayout());
auto nodeValue = GetValueObjectFromCNTKImplMatrixAndMBLayout<double>(gradientVarValuePair.first, computationNodePtr->As<ComputationNode<double>>()->Gradient(), computationNodePtr->GetMBLayout());
if (gradientValuePtr == nullptr)
{
auto data = NDArrayViewPtr(new NDArrayView(iter->first.GetDataType(), gradientShape, AsDeviceDescriptor(computationNodePtr->ValuePtr()->GetDeviceId())), [](_ReferenceCounter* ptr) { delete ptr; });
auto mask = NDMaskPtr((nodeValue->Mask() != nullptr) ? new NDMask(nodeValue->Mask()->Shape(), nodeValue->Mask()->Device()) : nullptr, [](_ReferenceCounter* ptr) { delete ptr; });
gradientValuePtr = ValuePtr(new Value(data, mask), [](_ReferenceCounter* ptr) { delete ptr; });
auto data = NDArrayViewPtr(new NDArrayView(gradientVarValuePair.first.GetDataType(), gradientShape, AsDeviceDescriptor(computationNodePtr->ValuePtr()->GetDeviceId())), [](ReferenceCount* ptr) { delete ptr; });
auto mask = NDMaskPtr((nodeValue->Mask() != nullptr) ? new NDMask(nodeValue->Mask()->Shape(), nodeValue->Mask()->Device()) : nullptr, [](ReferenceCount* ptr) { delete ptr; });
gradientValuePtr = ValuePtr(new Value(data, mask), [](ReferenceCount* ptr) { delete ptr; });
}
gradientValuePtr->CopyFrom(*nodeValue);
break;
}
default:
LogicError("Unsupported DataType %s", DataTypeName(iter->first.GetDataType()));
LogicError("Unsupported DataType %s", DataTypeName(gradientVarValuePair.first.GetDataType()));
break;
}
gradients[iter->first] = gradientValuePtr;
gradients[gradientVarValuePair.first] = gradientValuePtr;
}
}
/*virtual*/ BackPropStatePtr CompositeFunction::Forward(const _Internal::_SimpleMap<Variable, const ValuePtr>& arguments,
_Internal::_SimpleMap<Variable, ValuePtr>& outputs,
const _Internal::_SimpleSet<Variable>& outputsToRetainBackwardStateFor,
/*virtual*/ BackPropStatePtr CompositeFunction::Forward(const Internal::SimpleMap<Variable, const ValuePtr>& arguments,
Internal::SimpleMap<Variable, ValuePtr>& outputs,
const Internal::SimpleSet<Variable>& outputsToRetainBackwardStateFor,
const DeviceDescriptor& computeDevice)
{
// TODO: How about zero argument functions?
@ -752,24 +767,24 @@ namespace CNTK
// Feed data into the arguments of the network
PopulateNetworkInputs(arguments);
std::unordered_set<Variable> functionOutputs = _Internal::_SimpleVector<Variable>::CreateSimpleVector(this->Outputs()).GetAsUnorderedSet();
std::unordered_set<Variable> functionOutputs = Internal::SimpleVector<Variable>(this->Outputs()).GetAsUnorderedSet();
std::vector<ComputationNodeBasePtr> outputsToEvaluate;
for (auto iter = outputs.m_map->begin(); iter != outputs.m_map->end(); ++iter)
for (auto outputVarValuePair : *outputs.m_map)
{
// Ensure that only a subset of this function's outputs are being asked to be evaluated
if (functionOutputs.find(iter->first) == functionOutputs.end())
if (functionOutputs.find(outputVarValuePair.first) == functionOutputs.end())
InvalidArgument("Requested output is not an Ouptut of the Function");
auto outputComputationNode = m_variableToNodeMap[iter->first];
auto outputComputationNode = m_variableToNodeMap[outputVarValuePair.first];
outputsToEvaluate.push_back(outputComputationNode);
}
// The 'outputsToRetainBackwardStateFor' nodes also need to be evaluated if not already specified in 'outputs'
for (auto iter = outputsToRetainBackwardStateFor.m_set->begin(); iter != outputsToRetainBackwardStateFor.m_set->end(); ++iter)
for (auto rootVarForBackprop : *outputsToRetainBackwardStateFor.m_set)
{
if (outputs.m_map->find(*iter) == outputs.m_map->end())
outputsToEvaluate.push_back(m_variableToNodeMap[*iter]);
if (outputs.m_map->find(rootVarForBackprop) == outputs.m_map->end())
outputsToEvaluate.push_back(m_variableToNodeMap[rootVarForBackprop]);
}
m_computationNetwork->ForwardProp(outputsToEvaluate);
@ -778,12 +793,12 @@ namespace CNTK
// TODO: How to deal with the specified 'computeDevice'
return (outputsToRetainBackwardStateFor.Size() > 0) ? BackPropStatePtr(new CNTKBackPropState(this, { arguments.m_map->begin()->first, m_variableToNodeMap[arguments.m_map->begin()->first]->GetEvalTimeStamp() }), [](_ReferenceCounter* ptr) { delete ptr; }) : nullptr;
return (outputsToRetainBackwardStateFor.Size() > 0) ? BackPropStatePtr(new CNTKBackPropState(this, { arguments.m_map->begin()->first, m_variableToNodeMap[arguments.m_map->begin()->first]->GetEvalTimeStamp() }), [](ReferenceCount* ptr) { delete ptr; }) : nullptr;
}
/*virtual*/ void CompositeFunction::Backward(const BackPropStatePtr& state,
const _Internal::_SimpleMap<Variable, const ValuePtr>& rootGradientValues,
_Internal::_SimpleMap<Variable, ValuePtr>& backPropagatedGradientValuesForInputs)
const Internal::SimpleMap<Variable, const ValuePtr>& rootGradientValues,
Internal::SimpleMap<Variable, ValuePtr>& backPropagatedGradientValuesForInputs)
{
if ((state == nullptr) || (dynamic_cast<const CNTKBackPropState*>(state.GetPtr()) == nullptr))
InvalidArgument("Invalid backprop state specified");
@ -800,8 +815,8 @@ namespace CNTK
// TODO: Avoid copying the data when possible
// Zero all gradients of nodes below the root nodes
for (auto iter = rootGradientValues.m_map->begin(); iter != rootGradientValues.m_map->end(); ++iter)
m_computationNetwork->ZeroInputGradients(m_variableToNodeMap[iter->first]);
for (auto rootGradientVarValuePair : *rootGradientValues.m_map)
m_computationNetwork->ZeroInputGradients(m_variableToNodeMap[rootGradientVarValuePair.first]);
// Feed data into the arguments of the network
PopulateNetworkGradients(rootGradientValues);
@ -815,19 +830,19 @@ namespace CNTK
// TODO: How to deal with the specified 'computeDevice'
}
/*virtual*/ void CompositeFunction::_ReplacePlaceholders(const _Internal::_SimpleMap<Placeholder, Variable>& placeholderReplacements, _Internal::_SimpleSet<const Function*>& visitedFunctions, _Internal::_SimpleSet<Placeholder>& replacedPlaceholders)
/*virtual*/ void CompositeFunction::_ReplacePlaceholders(const Internal::SimpleMap<Placeholder, Variable>& placeholderReplacements, Internal::SimpleSet<const Function*>& visitedFunctions, Internal::SimpleSet<Placeholder>& replacedPlaceholders)
{
RootFunction()->_ReplacePlaceholders(placeholderReplacements, visitedFunctions, replacedPlaceholders);
// If any of the placeholders were replaced with Output variables, let's add the graph of function underneath each of those to 'm_allPrimitiveFunctions' set
for (auto iter = replacedPlaceholders.m_set->begin(); iter != replacedPlaceholders.m_set->end(); ++iter)
for (auto replacedPlaceholder : *replacedPlaceholders.m_set)
{
auto replacingVariable = placeholderReplacements[*iter];
if (replacingVariable.Kind() == VariableKind::Output)
auto replacingVariable = placeholderReplacements[replacedPlaceholder];
if (replacingVariable.IsOutput())
{
auto ownerFunc = replacingVariable.Owner();
_Internal::_SimpleSet<FunctionPtr> visitedFunctions;
_DetermineInputs(ownerFunc, visitedFunctions);
Internal::SimpleSet<FunctionPtr> visitedFunctions;
DetermineInputs(ownerFunc, visitedFunctions);
// Add the newly visited functions to 'm_allPrimitiveFunctions' set
m_allPrimitiveFunctions.m_set->insert(visitedFunctions.m_set->begin(), visitedFunctions.m_set->end());
@ -855,22 +870,24 @@ namespace CNTK
return CompositeFunction::Create(new PrimitiveFunction(PrimitiveOpType::Tanh, { operand }, Dictionary(), name), name);
}
FunctionPtr _Combine(const _Internal::_SimpleVector<FunctionPtr>& operands, const std::wstring& name/* = L""*/)
namespace Internal
{
_Internal::_SimpleSet<FunctionPtr> uniqueOperands;
std::vector<Variable> inputs;
for (size_t i = 0; i < operands.Size(); ++i)
FunctionPtr Combine(const Internal::SimpleVector<FunctionPtr>& operands, const std::wstring& name/* = L""*/)
{
if (uniqueOperands.Contains(operands[i]))
LogicError("All function operands specified to Combine must be unique");
Internal::SimpleSet<FunctionPtr> uniqueOperands;
std::vector<Variable> inputs;
for (size_t i = 0; i < operands.Size(); ++i)
{
if (uniqueOperands.Contains(operands[i]))
LogicError("All function operands specified to Combine must be unique");
uniqueOperands.Insert(operands[i]);
auto currentFunctionOutputs = operands[i]->Outputs();
std::copy(currentFunctionOutputs.begin(), currentFunctionOutputs.end(), std::back_inserter(inputs));
uniqueOperands.Insert(operands[i]);
auto currentFunctionOutputs = operands[i]->Outputs();
std::copy(currentFunctionOutputs.begin(), currentFunctionOutputs.end(), std::back_inserter(inputs));
}
return CompositeFunction::Create(new PrimitiveFunction(PrimitiveOpType::Combine, inputs, Dictionary(), name), name);
}
return CompositeFunction::Create(new PrimitiveFunction(PrimitiveOpType::Combine, inputs, Dictionary(), name), name);
}
FunctionPtr CrossEntropyWithSoftmax(const Variable& output, const Variable& labels, const std::wstring& name/* = L""*/)
@ -878,9 +895,9 @@ namespace CNTK
return CompositeFunction::Create(new PrimitiveFunction(PrimitiveOpType::CrossEntropyWithSoftmax, { output, labels }, Dictionary(), name), name);
}
FunctionPtr PredictionError(const Variable& prediction, const Variable& labels, const std::wstring& name/* = L""*/)
FunctionPtr ClassificationError(const Variable& prediction, const Variable& labels, const std::wstring& name/* = L""*/)
{
return CompositeFunction::Create(new PrimitiveFunction(PrimitiveOpType::PredictionError, { prediction, labels }, Dictionary(), name), name);
return CompositeFunction::Create(new PrimitiveFunction(PrimitiveOpType::ClassificationError, { prediction, labels }, Dictionary(), name), name);
}
FunctionPtr Exp(const Variable& operand, const std::wstring& name/* = L""*/)

105
Source/CNTKv2LibraryDll/Function.h
Просмотреть файл

@ -3,6 +3,8 @@
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
//
#pragma once
#include "stdafx.h"
#include "CNTKLibrary.h"
#include <iterator>
@ -19,7 +21,7 @@ namespace CNTK
Tanh,
Combine,
CrossEntropyWithSoftmax,
PredictionError,
ClassificationError,
Exp,
PastValue,
FutureValue,
@ -29,6 +31,7 @@ namespace CNTK
inline const char* PrimitiveOpTypeName(PrimitiveOpType opType)
{
// TODO: Put these in table form
if (opType == PrimitiveOpType::Plus)
return "Plus";
else if (opType == PrimitiveOpType::Times)
@ -41,8 +44,8 @@ namespace CNTK
return "Combine";
else if (opType == PrimitiveOpType::CrossEntropyWithSoftmax)
return "CrossEntropyWithSoftmax";
else if (opType == PrimitiveOpType::PredictionError)
return "PredictionError";
else if (opType == PrimitiveOpType::ClassificationError)
return "ClassificationError";
else if (opType == PrimitiveOpType::Exp)
return "Exp";
else if (opType == PrimitiveOpType::PastValue)
@ -65,17 +68,17 @@ namespace CNTK
{
}
virtual BackPropStatePtr Forward(const _Internal::_SimpleMap<Variable, const ValuePtr>& /*arguments*/,
_Internal::_SimpleMap<Variable, ValuePtr>& /*outputs*/,
const _Internal::_SimpleSet<Variable>& /*outputsToRetainBackwardStateFor*/,
virtual BackPropStatePtr Forward(const Internal::SimpleMap<Variable, const ValuePtr>& /*arguments*/,
Internal::SimpleMap<Variable, ValuePtr>& /*outputs*/,
const Internal::SimpleSet<Variable>& /*outputsToRetainBackwardStateFor*/,
const DeviceDescriptor& /*computeDevice*/) override
{
NOT_IMPLEMENTED;
}
virtual void Backward(const BackPropStatePtr& /*state*/,
const _Internal::_SimpleMap<Variable, const ValuePtr>& /*rootGradientValues*/,
_Internal::_SimpleMap<Variable, ValuePtr>& /*backPropagatedGradientValuesForInputs*/) override
const Internal::SimpleMap<Variable, const ValuePtr>& /*rootGradientValues*/,
Internal::SimpleMap<Variable, ValuePtr>& /*backPropagatedGradientValuesForInputs*/) override
{
NOT_IMPLEMENTED;
}
@ -91,6 +94,8 @@ namespace CNTK
}
private:
// The following helper functions are used to determine the output shape for different
// types of primitive operations accounting for broadcasting and reductions where applicable.
static NDShape UnaryElementwiseOpOutputShape(const NDShape& operandShape)
{
return operandShape;
@ -98,17 +103,17 @@ namespace CNTK
static NDShape BinaryElementwiseOpOutputShape(PrimitiveOpType op, const NDShape& leftOperandShape, const NDShape& rightOperandShape, bool broadcastAllowed = true)
{
auto& shapeWithSmallerNumAxes = (leftOperandShape.NumAxes() > rightOperandShape.NumAxes()) ? rightOperandShape : leftOperandShape;
auto& shapeWithLargerNumAxes = (leftOperandShape.NumAxes() > rightOperandShape.NumAxes()) ? leftOperandShape : rightOperandShape;
const auto& shapeWithSmallerNumAxes = (leftOperandShape.NumAxes() > rightOperandShape.NumAxes()) ? rightOperandShape : leftOperandShape;
const auto& shapeWithLargerNumAxes = (leftOperandShape.NumAxes() > rightOperandShape.NumAxes()) ? leftOperandShape : rightOperandShape;
size_t numOutputAxes = shapeWithLargerNumAxes.NumAxes();
std::vector<size_t> outputDims(numOutputAxes);
for (size_t i = 0; i < shapeWithSmallerNumAxes.NumAxes(); ++i)
{
if ((leftOperandShape[i] == NDShape::InferredDimension) && (rightOperandShape[i] == NDShape::InferredDimension))
outputDims[i] = NDShape::InferredDimension;
else if ((leftOperandShape[i] == NDShape::InferredDimension) && (rightOperandShape[i] != NDShape::InferredDimension))
else if (leftOperandShape[i] == NDShape::InferredDimension)
outputDims[i] = rightOperandShape[i];
else if ((leftOperandShape[i] != NDShape::InferredDimension) && (rightOperandShape[i] == NDShape::InferredDimension))
else if (rightOperandShape[i] == NDShape::InferredDimension)
outputDims[i] = leftOperandShape[i];
else
{
@ -126,7 +131,7 @@ namespace CNTK
return NDShape(std::move(outputDims));
}
static NDShape TimesOpOutputShape(const NDShape& leftOperandShape, const NDShape& rightOperandShape, bool broadcastAllowed = true)
static NDShape TimesOpOutputShape(const NDShape& leftOperandShape, const NDShape& rightOperandShape)
{
if (rightOperandShape.NumAxes() > 2)
RuntimeError("The right operand of a times operation can have at most 2 axes");
@ -166,6 +171,7 @@ namespace CNTK
return NDShape(std::move(outputDims));
}
// TODO: Reconcile this with the ComputationNode::Validate functionality in core CNTK to avoid duplication of inference logic
static std::vector<Variable> GetOutputVariables(PrimitiveOpType op, const std::vector<Variable>& inputs, Function* owner)
{
std::vector<Variable> outputs;
@ -175,9 +181,9 @@ namespace CNTK
// We currently require that the inputs' dynamic axes if any match
std::vector<Axis> outputDynamicAxes = inputs[0].DynamicAxes();
for (size_t i = 1; i < inputs.size(); ++i)
for (auto inputVar : inputs)
{
auto currentInputDynamicAxes = inputs[i].DynamicAxes();
auto currentInputDynamicAxes = inputVar.DynamicAxes();
if (outputDynamicAxes.empty())
outputDynamicAxes = currentInputDynamicAxes;
else
@ -210,7 +216,7 @@ namespace CNTK
outputs.push_back(Variable(TimesOpOutputShape(inputs[0].Shape(), inputs[1].Shape()), outputDataType, owner, outputDynamicAxes));
break;
case PrimitiveOpType::CrossEntropyWithSoftmax:
case PrimitiveOpType::PredictionError:
case PrimitiveOpType::ClassificationError:
{
assert(inputs.size() == 2);
@ -274,10 +280,10 @@ namespace CNTK
private:
std::pair<Variable, int64_t> m_evalTimeStamp;
};
typedef _Internal::_ReferenceCounterSharedPtr<CNTKBackPropState> CNTKBackPropStatePtr;
typedef Internal::ReferenceCountedPtr<CNTKBackPropState> CNTKBackPropStatePtr;
class CompositeFunction;
typedef _Internal::_ReferenceCounterSharedPtr<CompositeFunction> CompositeFunctionPtr;
typedef Internal::ReferenceCountedPtr<CompositeFunction> CompositeFunctionPtr;
class CompositeFunction final : public Function
{
@ -286,53 +292,53 @@ namespace CNTK
public:
static CompositeFunctionPtr Create(const FunctionPtr& rootFunction, const std::wstring& name = L"")
{
_Internal::_SimpleSet<FunctionPtr> visitedFunctions;
Internal::SimpleSet<FunctionPtr> visitedFunctions;
// Call _DetermineInputs to get the set of all functions in the graph
_DetermineInputs(rootFunction, visitedFunctions);
// Call DetermineInputs to get the set of all functions in the graph
DetermineInputs(rootFunction, visitedFunctions);
auto func = new CompositeFunction(rootFunction, std::move(visitedFunctions), name);
return CompositeFunctionPtr(func, [](_ReferenceCounter* ptr) { delete ptr; });
return CompositeFunctionPtr(func, [](ReferenceCount* ptr) { delete ptr; });
}
virtual BackPropStatePtr Forward(const _Internal::_SimpleMap<Variable, const ValuePtr>& arguments,
_Internal::_SimpleMap<Variable, ValuePtr>& outputs,
const _Internal::_SimpleSet<Variable>& outputsToRetainBackwardStateFor,
virtual BackPropStatePtr Forward(const Internal::SimpleMap<Variable, const ValuePtr>& arguments,
Internal::SimpleMap<Variable, ValuePtr>& outputs,
const Internal::SimpleSet<Variable>& outputsToRetainBackwardStateFor,
const DeviceDescriptor& computeDevice) override;
virtual void Backward(const BackPropStatePtr& state,
const _Internal::_SimpleMap<Variable, const ValuePtr>& rootGradientValues,
_Internal::_SimpleMap<Variable, ValuePtr>& backPropagatedGradientValuesForInputs) override;
const Internal::SimpleMap<Variable, const ValuePtr>& rootGradientValues,
Internal::SimpleMap<Variable, ValuePtr>& backPropagatedGradientValuesForInputs) override;
private:
virtual void _ReplacePlaceholders(const _Internal::_SimpleMap<Placeholder, Variable>& placeholderReplacements, _Internal::_SimpleSet<const Function*>& visitedFunctions, _Internal::_SimpleSet<Placeholder>& replacedPlaceholders) override;
virtual void _ReplacePlaceholders(const Internal::SimpleMap<Placeholder, Variable>& placeholderReplacements, Internal::SimpleSet<const Function*>& visitedFunctions, Internal::SimpleSet<Placeholder>& replacedPlaceholders) override;
CompositeFunction(const FunctionPtr& rootFunction, _Internal::_SimpleSet<FunctionPtr>&& allPrimitiveFunctions, const std::wstring& name)
CompositeFunction(const FunctionPtr& rootFunction, Internal::SimpleSet<FunctionPtr>&& allPrimitiveFunctions, const std::wstring& name)
: Function({}, rootFunction->Outputs(), rootFunction, name), m_allPrimitiveFunctions(std::move(allPrimitiveFunctions))
{
}
std::vector<Variable> DetermineInputs() const
{
_Internal::_SimpleSet<FunctionPtr> visitedFunctions;
return _DetermineInputs(RootFunction(), visitedFunctions);
Internal::SimpleSet<FunctionPtr> visitedFunctions;
return DetermineInputs(RootFunction(), visitedFunctions);
}
static std::vector<Variable> _DetermineInputs(const FunctionPtr& rootFunction, _Internal::_SimpleSet<FunctionPtr>& visitedFunctions)
// Recursively traverses the Function graph underlying the 'rootFunction' to determine all the leaves (aka inputs) of the graph
static std::vector<Variable> DetermineInputs(const FunctionPtr& rootFunction, Internal::SimpleSet<FunctionPtr>& visitedFunctions)
{
visitedFunctions.Insert(rootFunction);
std::vector<Variable> inputs;
std::vector<Variable> rootFunctionInputs = rootFunction->Inputs();
for (size_t i = 0; i < rootFunctionInputs.size(); ++i)
for (auto rootInput : rootFunctionInputs)
{
Variable currentInput = rootFunctionInputs[i];
if (currentInput.Kind() != VariableKind::Output)
inputs.push_back(currentInput);
else if (!visitedFunctions.Contains(currentInput.Owner()))
if (!rootInput.IsOutput())
inputs.push_back(rootInput);
else if (!visitedFunctions.Contains(rootInput.Owner()))
{
FunctionPtr function = currentInput.Owner();
std::vector<Variable> functionInputs = _DetermineInputs(function, visitedFunctions);
FunctionPtr function = rootInput.Owner();
std::vector<Variable> functionInputs = DetermineInputs(function, visitedFunctions);
std::copy(functionInputs.begin(), functionInputs.end(), std::back_inserter(inputs));
}
}
@ -341,7 +347,7 @@ namespace CNTK
}
template <typename ElementType>
Microsoft::MSR::CNTK::ComputationNetworkPtr GetComputationNetwork(const DeviceDescriptor& device, const _Internal::_SimpleSet<Variable>& backpropRoots);
Microsoft::MSR::CNTK::ComputationNetworkPtr GetComputationNetwork(const DeviceDescriptor& device, const Internal::SimpleSet<Variable>& backpropRoots);
template <typename ElementType>
static Microsoft::MSR::CNTK::ComputationNodeBasePtr GetOutputVariableNode(const Variable& variable, Microsoft::MSR::CNTK::ComputationNetworkPtr& network, Microsoft::MSR::CNTK::ComputationNetworkBuilder<ElementType>& builder, std::unordered_map<Variable, Microsoft::MSR::CNTK::ComputationNodeBasePtr>& variableToNodeMap, std::unordered_map<Variable, bool>& isVariableRootMap);
@ -349,8 +355,8 @@ namespace CNTK
template <typename ElementType>
static Microsoft::MSR::CNTK::ComputationNodeBasePtr GetNode(const Variable& variable, Microsoft::MSR::CNTK::ComputationNetworkPtr& network, Microsoft::MSR::CNTK::ComputationNetworkBuilder<ElementType>& builder, std::unordered_map<Variable, Microsoft::MSR::CNTK::ComputationNodeBasePtr>& variableToNodeMap, std::unordered_map<Variable, bool>& isVariableRootMap);
void PopulateNetworkInputs(const _Internal::_SimpleMap<Variable, const ValuePtr>& arguments);
void PopulateNetworkGradients(const _Internal::_SimpleMap<Variable, const ValuePtr>& gradients);
void PopulateNetworkInputs(const Internal::SimpleMap<Variable, const ValuePtr>& arguments);
void PopulateNetworkGradients(const Internal::SimpleMap<Variable, const ValuePtr>& gradients);
void GetNetworkOutputs(std::unordered_map<Variable, ValuePtr>& outputs);
void GetNetworkGradients(std::unordered_map<Variable, ValuePtr>& gradients);
@ -362,10 +368,23 @@ namespace CNTK
static ValuePtr GetValueObjectFromCNTKImplMatrixAndMBLayout(Variable var, const Microsoft::MSR::CNTK::Matrix<ElementType>& matrix, const Microsoft::MSR::CNTK::MBLayoutPtr& layout);
private:
_Internal::_SimpleSet<FunctionPtr> m_allPrimitiveFunctions;
// Set of all primitive functions in the graph underlying 'this' Function. Also keeps the primitive Function objects alive
// by holding strong references to them
Internal::SimpleSet<FunctionPtr> m_allPrimitiveFunctions;
// A map from Variable objects to ComputationNode objects in the ComputationNetwork instance that implements 'this' Composite Function
std::unordered_map<Variable, Microsoft::MSR::CNTK::ComputationNodeBasePtr> m_variableToNodeMap;
// A map that tells whether a Variable in the graph underlying 'this' Function is a root of the graph
std::unordered_map<Variable, bool> m_isVariableRootMap;
Microsoft::MSR::CNTK::ComputationNetworkPtr m_computationNetwork;
// The backpropRoots sepecified in the most recent 'Forward' call on 'this' Function.
// This indicates for which of it's roots has 'this' Function retained required intermediate
// states from the previos Forward call to be able to backpropagate gradients backwards from in
// the next 'Backward' call.
std::unordered_set<Variable> m_currentBackpropRoots;
};
}

32
Source/CNTKv2LibraryDll/NDArrayView.cpp
Просмотреть файл

@ -17,9 +17,9 @@ namespace CNTK
{
template <typename ElementType>
static TensorView<ElementType>* AllocateTensorView(const NDShape& viewShape,
const DeviceDescriptor& device,
void* dataBuffer,
size_t bufferSizeInBytes)
const DeviceDescriptor& device,
void* dataBuffer,
size_t bufferSizeInBytes)
{
if (dataBuffer == nullptr)
InvalidArgument("Cannot create a NDArrayView over a null data buffer");
@ -33,10 +33,10 @@ namespace CNTK
}
static void* AllocateTensorView(CNTK::DataType dataType,
const NDShape& viewShape,
const DeviceDescriptor& device,
void* dataBuffer,
size_t bufferSizeInBytes)
const NDShape& viewShape,
const DeviceDescriptor& device,
void* dataBuffer,
size_t bufferSizeInBytes)
{
switch (dataType)
{
@ -195,7 +195,7 @@ namespace CNTK
const TensorView<ElementType>* NDArrayView::GetTensorView() const
{
if (AsDataType<ElementType>() != m_dataType)
LogicError("NDArrayView::GetWritableTensorView: The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(m_dataType));
LogicError("NDArrayView::GetTensorView: The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(m_dataType));
return (const TensorView<ElementType>*)(m_tensorView);
}
@ -211,7 +211,7 @@ namespace CNTK
NDArrayViewPtr NDArrayView::DeepClone(bool readOnly/* = false*/) const
{
NDArrayViewPtr newView(new NDArrayView(this->GetDataType(), this->GetStorageFormat(), this->Shape(), this->Device()), [](_ReferenceCounter* ptr) { delete ptr; });
NDArrayViewPtr newView(new NDArrayView(this->GetDataType(), this->GetStorageFormat(), this->Shape(), this->Device()), [](ReferenceCount* ptr) { delete ptr; });
switch (m_dataType)
{
case DataType::Float:
@ -234,7 +234,7 @@ namespace CNTK
}
newView->m_isReadOnly = readOnly;
return NDArrayViewPtr(newView, [](_ReferenceCounter* ptr) {
return NDArrayViewPtr(newView, [](ReferenceCount* ptr) {
delete ptr;
});
}
@ -286,7 +286,7 @@ namespace CNTK
}
auto aliasView = new NDArrayView(GetDataType(), Device(), GetStorageFormat(), Shape(), IsReadOnly() || readOnly, tensorView);;
return NDArrayViewPtr(aliasView, [](_ReferenceCounter* ptr) { delete ptr; });
return NDArrayViewPtr(aliasView, [](ReferenceCount* ptr) { delete ptr; });
}
// TODO: This could actually be strided?
@ -316,19 +316,19 @@ namespace CNTK
}
template <typename ElementType>
NDArrayViewPtr NDArrayView::RandomUniform(const NDShape& shape, double rangeStart, double rangeEnd, unsigned long seed, const DeviceDescriptor& device/* = DeviceDescriptor::DefaultDevice()*/)
NDArrayViewPtr NDArrayView::RandomUniform(const NDShape& shape, double rangeBegin, double rangeEnd, unsigned long seed, const DeviceDescriptor& device/* = DeviceDescriptor::DefaultDevice()*/)
{
auto matrixDims = GetMatrixDimensions(shape);
auto randomUniformMatrix = std::make_shared<Matrix<ElementType>>(Matrix<ElementType>::RandomUniform(matrixDims.first, matrixDims.second, AsCNTKImplDeviceId(device), (ElementType)rangeStart, (ElementType)rangeEnd, seed));
auto randomUniformMatrix = std::make_shared<Matrix<ElementType>>(Matrix<ElementType>::RandomUniform(matrixDims.first, matrixDims.second, AsCNTKImplDeviceId(device), (ElementType)rangeBegin, (ElementType)rangeEnd, seed));
auto tensorView = new TensorView<ElementType>(randomUniformMatrix, AsTensorShape(shape));
auto view = new NDArrayView(AsDataType<ElementType>(), device, StorageFormat::Dense, shape, false, tensorView);
return NDArrayViewPtr(view, [](_ReferenceCounter* ptr) { delete ptr; });
return NDArrayViewPtr(view, [](ReferenceCount* ptr) { delete ptr; });
}
// Explicit template instantiations
template CNTK_API NDArrayViewPtr NDArrayView::RandomUniform<float>(const NDShape& shape, double rangeStart, double rangeEnd, unsigned long seed, const DeviceDescriptor& device/* = DeviceDescriptor::DefaultDevice()*/);
template CNTK_API NDArrayViewPtr NDArrayView::RandomUniform<double>(const NDShape& shape, double rangeStart, double rangeEnd, unsigned long seed, const DeviceDescriptor& device/* = DeviceDescriptor::DefaultDevice()*/);
template CNTK_API NDArrayViewPtr NDArrayView::RandomUniform<float>(const NDShape& shape, double rangeBegin, double rangeEnd, unsigned long seed, const DeviceDescriptor& device/* = DeviceDescriptor::DefaultDevice()*/);
template CNTK_API NDArrayViewPtr NDArrayView::RandomUniform<double>(const NDShape& shape, double rangeBegin, double rangeEnd, unsigned long seed, const DeviceDescriptor& device/* = DeviceDescriptor::DefaultDevice()*/);
template CNTK_API const float* NDArrayView::DataBuffer<float>() const;
template CNTK_API const double* NDArrayView::DataBuffer<double>() const;

4
Source/CNTKv2LibraryDll/NDMask.cpp
Просмотреть файл

@ -99,11 +99,11 @@ namespace CNTK
NDMaskPtr newMask = new NDMask(this->Shape(), this->Device());
newMask->CopyFrom(*this);
return NDMaskPtr(newMask, [](_ReferenceCounter* ptr) { delete ptr; });
return NDMaskPtr(newMask, [](ReferenceCount* ptr) { delete ptr; });
}
NDMaskPtr NDMask::Alias() const
{
return NDMaskPtr(new NDMask(this->Shape(), new Matrix<char>(GetMatrix()->AsReference())), [](_ReferenceCounter* ptr) { delete ptr; });
return NDMaskPtr(new NDMask(this->Shape(), new Matrix<char>(GetMatrix()->AsReference())), [](ReferenceCount* ptr) { delete ptr; });
}
}

155
Source/CNTKv2LibraryDll/Utils.cpp
Просмотреть файл

@ -9,36 +9,61 @@
namespace CNTK
{
namespace _Internal
namespace Internal
{
#pragma region _SimpleVector
ReferenceCount::ReferenceCount()
: m_rc(new std::atomic<size_t>(0))
{}
/*virtual*/ ReferenceCount::~ReferenceCount()
{
delete m_rc;
}
size_t ReferenceCount::AddReference()
{
return ++(*m_rc);
}
size_t ReferenceCount::RemoveReference()
{
assert(m_rc->load() > 0);
return --(*m_rc);
}
size_t ReferenceCount::GetReferenceCount()
{
return m_rc->load();
}
#pragma region SimpleVector
template <typename T>
_SimpleVector<T>::_SimpleVector()
SimpleVector<T>::SimpleVector()
: m_vector(new std::vector<T>())
{
}
template <typename T>
_SimpleVector<T>::_SimpleVector(size_t numElements, const T& initVal/* = T()*/)
SimpleVector<T>::SimpleVector(size_t numElements, const T& initVal/* = T()*/)
: m_vector(new std::vector<T>(numElements, initVal))
{
}
template <typename T>
_SimpleVector<T>::~_SimpleVector()
SimpleVector<T>::~SimpleVector()
{
delete m_vector;
}
template <typename T>
_SimpleVector<T>::_SimpleVector(const _SimpleVector<T>& other)
SimpleVector<T>::SimpleVector(const SimpleVector<T>& other)
: m_vector(new std::vector<T>(*other.m_vector))
{
}
template <typename T>
_SimpleVector<T>& _SimpleVector<T>::operator=(const _SimpleVector<T>& other)
SimpleVector<T>& SimpleVector<T>::operator=(const SimpleVector<T>& other)
{
if (this != &other)
{
@ -50,14 +75,14 @@ namespace CNTK
}
template <typename T>
_SimpleVector<T>::_SimpleVector(_SimpleVector<T>&& other)
SimpleVector<T>::SimpleVector(SimpleVector<T>&& other)
: m_vector(nullptr)
{
*this = std::move(other);
}
template <typename T>
_SimpleVector<T>& _SimpleVector<T>::operator=(_SimpleVector<T>&& other)
SimpleVector<T>& SimpleVector<T>::operator=(SimpleVector<T>&& other)
{
assert(this != &other);
@ -70,88 +95,88 @@ namespace CNTK
}
template <typename T>
T& _SimpleVector<T>::operator[](size_t idx)
T& SimpleVector<T>::operator[](size_t idx)
{
assert(idx < Size());
return (*m_vector)[idx];
}
template <typename T>
const T& _SimpleVector<T>::operator[](size_t idx) const
const T& SimpleVector<T>::operator[](size_t idx) const
{
assert(idx < Size());
return (*m_vector)[idx];
}
template <typename T>
size_t _SimpleVector<T>::Size() const
size_t SimpleVector<T>::Size() const
{
return m_vector->size();
}
template <typename T>
T* _SimpleVector<T>::Data()
T* SimpleVector<T>::Data()
{
return m_vector->data();
}
template <typename T>
const T* _SimpleVector<T>::Data() const
const T* SimpleVector<T>::Data() const
{
return m_vector->data();
}
template <typename T>
void _SimpleVector<T>::PushBack(const T& value)
void SimpleVector<T>::PushBack(const T& value)
{
m_vector->push_back(value);
}
template <typename T>
void _SimpleVector<T>::PushBack(T&& value)
void SimpleVector<T>::PushBack(T&& value)
{
m_vector->push_back(std::move(value));
}
template <typename ValueType>
bool operator==(const _SimpleVector<ValueType>& first, const _SimpleVector<ValueType>& second)
bool operator==(const SimpleVector<ValueType>& first, const SimpleVector<ValueType>& second)
{
return *first.m_vector == *second.m_vector;
}
// Explicit template instantiations
template class _SimpleVector<Variable>;
template class _SimpleVector<size_t>;
template class _SimpleVector<Axis>;
template class _SimpleVector<FunctionPtr>;
template class SimpleVector<Variable>;
template class SimpleVector<size_t>;
template class SimpleVector<Axis>;
template class SimpleVector<FunctionPtr>;
template bool operator==(const _SimpleVector<size_t>& first, const _SimpleVector<size_t>& second);
template bool operator==(const SimpleVector<size_t>& first, const SimpleVector<size_t>& second);
#pragma endregion _SimpleVector
#pragma endregion SimpleVector
#pragma region _SimpleSet
#pragma region SimpleSet
template <typename KeyType>
_SimpleSet<KeyType>::_SimpleSet()
SimpleSet<KeyType>::SimpleSet()
: m_set(new std::unordered_set<KeyType>())
{
}
template <typename KeyType>
_SimpleSet<KeyType>::~_SimpleSet()
SimpleSet<KeyType>::~SimpleSet()
{
delete m_set;
}
template <typename KeyType>
_SimpleSet<KeyType>::_SimpleSet(const _SimpleSet& other)
SimpleSet<KeyType>::SimpleSet(const SimpleSet& other)
: m_set(nullptr)
{
*this = other;
}
template <typename KeyType>
_SimpleSet<KeyType>& _SimpleSet<KeyType>::operator=(const _SimpleSet& other)
SimpleSet<KeyType>& SimpleSet<KeyType>::operator=(const SimpleSet& other)
{
if (this != &other)
{
@ -163,14 +188,14 @@ namespace CNTK
}
template <typename KeyType>
_SimpleSet<KeyType>::_SimpleSet(_SimpleSet&& other)
SimpleSet<KeyType>::SimpleSet(SimpleSet&& other)
: m_set(nullptr)
{
*this = std::move(other);
}
template <typename KeyType>
_SimpleSet<KeyType>& _SimpleSet<KeyType>::operator=(_SimpleSet&& other)
SimpleSet<KeyType>& SimpleSet<KeyType>::operator=(SimpleSet&& other)
{
assert(this != &other);
@ -182,73 +207,73 @@ namespace CNTK
}
template <typename KeyType>
bool _SimpleSet<KeyType>::Insert(const KeyType& key)
bool SimpleSet<KeyType>::Insert(const KeyType& key)
{
return m_set->insert(key).second;
}
template <typename KeyType>
bool _SimpleSet<KeyType>::Contains(const KeyType& key) const
bool SimpleSet<KeyType>::Contains(const KeyType& key) const
{
return (m_set->find(key) != m_set->end());
}
template <typename KeyType>
size_t _SimpleSet<KeyType>::Size() const
size_t SimpleSet<KeyType>::Size() const
{
return m_set->size();
}
template <typename KeyType>
_SimpleSet<KeyType>::operator _SimpleVector<KeyType>() const
SimpleSet<KeyType>::operator SimpleVector<KeyType>() const
{
_SimpleVector<KeyType> retVector;
for (auto iter = m_set->begin(); iter != m_set->end(); ++iter)
retVector.PushBack(*iter);
SimpleVector<KeyType> retVector;
for (auto key : *m_set)
retVector.PushBack(key);
return retVector;
}
template <typename KeyType>
bool operator==(const _SimpleSet<KeyType>& first, const _SimpleSet<KeyType>& second)
bool operator==(const SimpleSet<KeyType>& first, const SimpleSet<KeyType>& second)
{
return *first.m_set == *second.m_set;
}
// Explicit template instantiations
template class _SimpleSet<FunctionPtr>;
template class _SimpleSet<Variable>;
template class _SimpleSet<Placeholder>;
template class _SimpleSet<const Function*>;
template class SimpleSet<FunctionPtr>;
template class SimpleSet<Variable>;
template class SimpleSet<Placeholder>;
template class SimpleSet<const Function*>;
template bool operator==(const _SimpleSet<Variable>& first, const _SimpleSet<Variable>& second);
template bool operator==(const _SimpleSet<Placeholder>& first, const _SimpleSet<Placeholder>& second);
template bool operator==(const SimpleSet<Variable>& first, const SimpleSet<Variable>& second);
template bool operator==(const SimpleSet<Placeholder>& first, const SimpleSet<Placeholder>& second);
#pragma endregion _SimpleSet
#pragma endregion SimpleSet
#pragma region _SimpleMap
#pragma region SimpleMap
template <typename KeyType, typename ValueType>
_SimpleMap<KeyType, ValueType>::_SimpleMap()
SimpleMap<KeyType, ValueType>::SimpleMap()
: m_map(new std::unordered_map<KeyType, ValueType>())
{
}
template <typename KeyType, typename ValueType>
_SimpleMap<KeyType, ValueType>::~_SimpleMap()
SimpleMap<KeyType, ValueType>::~SimpleMap()
{
delete m_map;
}
template <typename KeyType, typename ValueType>
_SimpleMap<KeyType, ValueType>::_SimpleMap(const _SimpleMap& other)
SimpleMap<KeyType, ValueType>::SimpleMap(const SimpleMap& other)
: m_map(nullptr)
{
*this = other;
}
template <typename KeyType, typename ValueType>
_SimpleMap<KeyType, ValueType>& _SimpleMap<KeyType, ValueType>::operator=(const _SimpleMap& other)
SimpleMap<KeyType, ValueType>& SimpleMap<KeyType, ValueType>::operator=(const SimpleMap& other)
{
if (this != &other)
{
@ -260,14 +285,14 @@ namespace CNTK
}
template <typename KeyType, typename ValueType>
_SimpleMap<KeyType, ValueType>::_SimpleMap(_SimpleMap&& other)
SimpleMap<KeyType, ValueType>::SimpleMap(SimpleMap&& other)
: m_map(nullptr)
{
*this = std::move(other);
}
template <typename KeyType, typename ValueType>
_SimpleMap<KeyType, ValueType>& _SimpleMap<KeyType, ValueType>::operator=(_SimpleMap&& other)
SimpleMap<KeyType, ValueType>& SimpleMap<KeyType, ValueType>::operator=(SimpleMap&& other)
{
assert(this != &other);
@ -279,51 +304,51 @@ namespace CNTK
}
template <typename KeyType, typename ValueType>
ValueType& _SimpleMap<KeyType, ValueType>::operator[](const KeyType& key)
ValueType& SimpleMap<KeyType, ValueType>::operator[](const KeyType& key)
{
return (*m_map)[key];
}
template <typename KeyType, typename ValueType>
const ValueType& _SimpleMap<KeyType, ValueType>::operator[](const KeyType& key) const
const ValueType& SimpleMap<KeyType, ValueType>::operator[](const KeyType& key) const
{
return (*m_map)[key];
}
template <typename KeyType, typename ValueType>
bool _SimpleMap<KeyType, ValueType>::Insert(const KeyType& key, const ValueType& value)
bool SimpleMap<KeyType, ValueType>::Insert(const KeyType& key, const ValueType& value)
{
return m_map->insert({ key, value }).second;
}
template <typename KeyType, typename ValueType>
bool _SimpleMap<KeyType, ValueType>::Contains(const KeyType& key) const
bool SimpleMap<KeyType, ValueType>::Contains(const KeyType& key) const
{
return (m_map->find(key) != m_map->end());
}
template <typename KeyType, typename ValueType>
size_t _SimpleMap<KeyType, ValueType>::Size() const
size_t SimpleMap<KeyType, ValueType>::Size() const
{
return m_map->size();
}
template <typename KeyType, typename ValueType>
_SimpleSet<KeyType> _SimpleMap<KeyType, ValueType>::Keys() const
SimpleSet<KeyType> SimpleMap<KeyType, ValueType>::Keys() const
{
_SimpleSet<KeyType> keys;
for (auto iter = m_map->begin(); iter != m_map->end(); ++iter)
keys.Insert(iter->first);
SimpleSet<KeyType> keys;
for (auto keyValuePair : *m_map)
keys.Insert(keyValuePair.first);
return keys;
}
// Explicit template instantiations
template class _SimpleMap<Variable, ValuePtr>;
template class _SimpleMap<Variable, const ValuePtr>;
template class _SimpleMap<Placeholder, Variable>;
template class SimpleMap<Variable, ValuePtr>;
template class SimpleMap<Variable, const ValuePtr>;
template class SimpleMap<Placeholder, Variable>;
#pragma endregion _SimpleMap
#pragma endregion SimpleMap
}

23
Source/CNTKv2LibraryDll/Utils.h
Просмотреть файл

@ -75,7 +75,7 @@ namespace CNTK
: m_valueType(GetValueType<T>())
{
static_assert(std::is_same<T, NDShape>::value ||
std::is_same<T, _Internal::_SimpleVector<DictionaryValue>>::value,
std::is_same<T, Internal::SimpleVector<DictionaryValue>>::value,
"Unsupported ValueType");
AllocateDataPtr(value);
@ -84,6 +84,8 @@ namespace CNTK
DictionaryValue(const DictionaryValue& other)
: m_valueType(Type::Bool)
{
// The m_valueType must hvae been set to a non-ptr type to prevent an attempt to interpret
// the underlying underlying uninitialized value as a ptr and free it.
*this = other;
}
@ -99,7 +101,7 @@ namespace CNTK
if (other.m_valueType == Type::NDShape)
AllocateDataPtr(other.GetValue<NDShape>());
else if (other.m_valueType == Type::Vector)
AllocateDataPtr(other.GetValue<_Internal::_SimpleVector<DictionaryValue>>());
AllocateDataPtr(other.GetValue<Internal::SimpleVector<DictionaryValue>>());
}
return *this;
@ -131,7 +133,7 @@ namespace CNTK
return m_data.m_double;
}
template <typename T, typename std::enable_if<std::is_same<T, NDShape>::value || std::is_same<T, _Internal::_SimpleVector<DictionaryValue>>::value>::type* = nullptr>
template <typename T, typename std::enable_if<std::is_same<T, NDShape>::value || std::is_same<T, Internal::SimpleVector<DictionaryValue>>::value>::type* = nullptr>
const T& GetValue() const
{
VerifyType<T>();
@ -156,7 +158,7 @@ namespace CNTK
std::is_same<T, size_t>::value ||
std::is_same<T, double>::value ||
std::is_same<T, NDShape>::value ||
std::is_same<T, _Internal::_SimpleVector<DictionaryValue>>::value ||
std::is_same<T, Internal::SimpleVector<DictionaryValue>>::value ||
std::is_same<T, CNTK::Dictionary>::value,
"Unsupported ValueType");
@ -168,7 +170,7 @@ namespace CNTK
return Type::Double;
else if (std::is_same<T, NDShape>::value)
return Type::NDShape;
else if (std::is_same<T, _Internal::_SimpleVector<DictionaryValue>>::value)
else if (std::is_same<T, Internal::SimpleVector<DictionaryValue>>::value)
return Type::Vector;
}
@ -182,7 +184,7 @@ namespace CNTK
template <typename T>
void AllocateDataPtr(const T& value)
{
static_assert(std::is_same<T, NDShape>::value || std::is_same<T, _Internal::_SimpleVector<DictionaryValue>>::value, "AllocateDataPtr called with invalid type");
static_assert(std::is_same<T, NDShape>::value || std::is_same<T, Internal::SimpleVector<DictionaryValue>>::value, "AllocateDataPtr called with invalid type");
m_data.m_ptr = new T(value);
}
@ -200,7 +202,7 @@ namespace CNTK
if (m_valueType == Type::NDShape)
FreePtrAsType<NDShape>();
else if (m_valueType == Type::Vector)
FreePtrAsType<_Internal::_SimpleVector<DictionaryValue>>();
FreePtrAsType<Internal::SimpleVector<DictionaryValue>>();
}
private:
@ -222,8 +224,7 @@ namespace CNTK
~Dictionary();
// Disallow copy contruction and assignment
Dictionary(const Dictionary&) = delete;
Dictionary& operator=(const Dictionary&) = delete;
Dictionary(const Dictionary&) = delete; Dictionary& operator=(const Dictionary&) = delete;
Dictionary(Dictionary&& other);
Dictionary& operator=(Dictionary&& other);
@ -266,9 +267,9 @@ namespace CNTK
inline DEVICEID_TYPE AsCNTKImplDeviceId(const DeviceDescriptor& device)
{
if (device.Type() == DeviceType::CPU)
if (device.Type() == DeviceKind::CPU)
return -1;
else if (device.Type() == DeviceType::GPU)
else if (device.Type() == DeviceKind::GPU)
return device.Id();
else
NOT_IMPLEMENTED;

23
Source/CNTKv2LibraryDll/Value.cpp
Просмотреть файл

@ -15,13 +15,14 @@ namespace CNTK
Value::Value(const NDArrayViewPtr& data, const NDMaskPtr& mask)
: m_data(data), m_mask(mask)
{
if ((mask != nullptr) && (mask->Shape().NumAxes() > data->Shape().NumAxes()))
InvalidArgument("The number of axes of the mask of a Value object cannot exceed the number of axes of the data NDArrayView object");
if (mask != nullptr)
{
auto dataShape = data->Shape();
auto maskShape = mask->Shape();
if (maskShape.NumAxes() > dataShape.NumAxes())
InvalidArgument("The number of axes of the mask of a Value object cannot exceed the number of axes of the data NDArrayView object");
if (dataShape.SubShape(dataShape.NumAxes() - maskShape.NumAxes()) != maskShape)
InvalidArgument("Invalid Value object; the data and mask are incompatible. The trailing dimensions of the data do not match the dimensions of the mask");
}
@ -49,7 +50,7 @@ namespace CNTK
if (needsMask)
{
NDShape valueMaskShape = { maxSequenceLength, numSequences };
deviceValueMask = NDMaskPtr(new NDMask(valueMaskShape, device), [](_Internal::_ReferenceCounter* ptr) {delete ptr; });
deviceValueMask = NDMaskPtr(new NDMask(valueMaskShape, device), [](Internal::ReferenceCount* ptr) {delete ptr; });
for (size_t i = 0; i < numSequences; ++i)
deviceValueMask->MaskSection({ sequenceLengths[i], i }, { NDShape::InferredDimension, 1 });
}
@ -86,8 +87,8 @@ namespace CNTK
}
colStarts[numSequences * maxSequenceLength] = (SparseIndexType)(nonZeroValues.size());
NDArrayViewPtr deviceValueData(new NDArrayView(valueDataShape, colStarts.data(), rowIndices.data(), nonZeroValues.data(), nonZeroValues.size(), device, readOnly), [](_ReferenceCounter* ptr) { delete ptr; });
return ValuePtr(new Value(deviceValueData, deviceValueMask), [](_ReferenceCounter* ptr) { delete ptr; });
NDArrayViewPtr deviceValueData(new NDArrayView(valueDataShape, colStarts.data(), rowIndices.data(), nonZeroValues.data(), nonZeroValues.size(), device, readOnly), [](ReferenceCount* ptr) { delete ptr; });
return ValuePtr(new Value(deviceValueData, deviceValueMask), [](ReferenceCount* ptr) { delete ptr; });
}
template <typename ElementType>
@ -99,7 +100,7 @@ namespace CNTK
size_t numSequences = sequences.size();
NDShape valueDataShape = sampleShape.AppendShape({ maxSequenceLength, numSequences });
NDArrayViewPtr valueData(new NDArrayView(AsDataType<ElementType>(), valueDataShape, DeviceDescriptor::CPUDevice()), [](_ReferenceCounter* ptr) { delete ptr; });
NDArrayViewPtr valueData(new NDArrayView(AsDataType<ElementType>(), valueDataShape, DeviceDescriptor::CPUDevice()), [](ReferenceCount* ptr) { delete ptr; });
ElementType* dataBuffer = valueData->WritableDataBuffer<ElementType>();
for (size_t i = 0; i < numSequences; ++i)
std::copy(sequences[i].data(), sequences[i].data() + sequences[i].size(), dataBuffer + (maxSequenceLength * i * sampleSize));
@ -114,13 +115,13 @@ namespace CNTK
}
else
{
deviceValueData = NDArrayViewPtr(new NDArrayView(AsDataType<ElementType>(), valueDataShape, device), [](_ReferenceCounter* ptr) { delete ptr; });
deviceValueData = NDArrayViewPtr(new NDArrayView(AsDataType<ElementType>(), valueDataShape, device), [](ReferenceCount* ptr) { delete ptr; });
deviceValueData->CopyFrom(*valueData);
if (readOnly)
deviceValueData = deviceValueData->Alias(true);
}
return ValuePtr(new Value(deviceValueData, deviceValueMask), [](_ReferenceCounter* ptr) { delete ptr; });
return ValuePtr(new Value(deviceValueData, deviceValueMask), [](ReferenceCount* ptr) { delete ptr; });
}
/*virtual*/ Value::~Value()
@ -142,13 +143,13 @@ namespace CNTK
/*virtual*/ ValuePtr Value::DeepClone(bool readOnly/* = false*/) const
{
// TODO: Check if this is a derived type and throw an exception in that case
return ValuePtr(new Value(Data()->DeepClone(readOnly), (Mask() != nullptr) ? Mask()->DeepClone() : nullptr), [](_ReferenceCounter* ptr) { delete ptr; });
return ValuePtr(new Value(Data()->DeepClone(readOnly), (Mask() != nullptr) ? Mask()->DeepClone() : nullptr), [](ReferenceCount* ptr) { delete ptr; });
}
/*virtual*/ ValuePtr Value::Alias(bool readOnly/* = false*/) const
{
// TODO: Check if this is a derived type and throw an exception in that case
return ValuePtr(new Value(Data()->Alias(readOnly), (Mask() != nullptr) ? Mask()->Alias() : nullptr), [](_ReferenceCounter* ptr) { delete ptr; });
return ValuePtr(new Value(Data()->Alias(readOnly), (Mask() != nullptr) ? Mask()->Alias() : nullptr), [](ReferenceCount* ptr) { delete ptr; });
}
/*virtual*/ void Value::CopyFrom(const Value& source)

102
Source/Common/BestGpu.cpp
Просмотреть файл

@ -106,22 +106,18 @@ public:
~BestGpu();
void Init();
void SetAllowedDevices(const std::vector<int>& devices); // only allow certain GPUs
bool DeviceAllowed(int device);
void DisallowDevice(int device)
{
assert((device >= -1) && (device <= 31));
if (device < 0)
m_disallowCPUDevice = true;
else
m_allowedDevices &= ~(1 << device);
}
bool DeviceAllowed(int deviceId);
void DisallowUnsupportedDevices();
void DisallowDevice(int deviceId);
void AllowAll(); // reset to allow all GPUs (no allowed list)
bool UseMultiple(); // using multiple GPUs?
int GetDevice(BestGpuFlags flags = bestGpuNormal); // get a single device
static const int AllDevices = -1; // can be used to specify all GPUs in GetDevices() call
static const int RequeryDevices = -2; // Requery refreshing statistics and picking the same number as last query
static const int MininumCCMajorForGpu = 3; // cntk supports GPUs with Compute Capability > 3.0
std::vector<int> GetDevices(int number = AllDevices, BestGpuFlags flags = bestGpuNormal); // get multiple devices
std::vector<ProcessorData *> GetProcessorData();
private:
bool LockDevice(int deviceId, bool trial = true);
};
@ -156,6 +152,8 @@ static DEVICEID_TYPE SelectDevice(DEVICEID_TYPE deviceId, bool bLockGPU, const i
{
g_bestGpu->DisallowDevice(excludedDevices[i]);
}
g_bestGpu->DisallowUnsupportedDevices();
}
bestDeviceId = (DEVICEID_TYPE)g_bestGpu->GetDevice(BestGpuFlags(bLockGPU ? (bestGpuAvoidSharing | bestGpuExclusiveLock) : bestGpuAvoidSharing));
@ -345,22 +343,32 @@ int BestGpu::GetDevice(BestGpuFlags bestFlags)
void BestGpu::SetAllowedDevices(const std::vector<int>& devices)
{
m_allowedDevices = 0;
for (int device : devices)
for (int deviceId : devices)
{
m_allowedDevices |= (1 << device);
m_allowedDevices |= (1 << deviceId);
}
}
// DeviceAllowed - is a particular device allowed?
// returns: true if the device is allowed, otherwise false
bool BestGpu::DeviceAllowed(int device)
bool BestGpu::DeviceAllowed(int deviceId)
{
assert((device >= -1) && (device <= 31));
assert((deviceId >= -1) && (deviceId <= 31));
if (device < 0)
if (deviceId < 0)
return !m_disallowCPUDevice;
else
return !!(m_allowedDevices & (1 << device));
return !!(m_allowedDevices & (1 << deviceId));
}
void BestGpu::DisallowDevice(int deviceId)
{
assert((deviceId >= -1) && (deviceId <= 31));
if (deviceId < 0)
m_disallowCPUDevice = true;
else
m_allowedDevices &= ~(1 << deviceId);
}
// AllowAll - Reset the allowed filter to allow all GPUs
@ -527,6 +535,68 @@ std::vector<int> BestGpu::GetDevices(int number, BestGpuFlags p_bestFlags)
return best; // return the array of the best GPUs
}
// disallow devices wich don't comply with compute capability restriction when cntk runs with deviceId = 'auto'
void BestGpu::DisallowUnsupportedDevices()
{
for (auto pd : m_procData)
{
if (pd->deviceProp.major < BestGpu::MininumCCMajorForGpu)
{
DisallowDevice(pd->deviceId);
}
}
}
GpuData GetGpuData(DEVICEID_TYPE deviceId)
{
std::vector<GpuData> gpusData = GetAllGpusData();
auto it = std::find_if(gpusData.begin(), gpusData.end(), [&deviceId](const GpuData& gpu){return gpu.deviceId == deviceId;});
if (it != gpusData.end())
{
return *it;
}
return GpuData(0, 0, deviceId, 0, GpuValidity::UnknownDevice, "", 0);
}
// populate a vector with data (id, major/minor version, cuda cores, name and memory) for each gpu device in the machine
std::vector<GpuData> GetAllGpusData()
{
std::vector<GpuData> data;
auto bestGpu = make_unique<BestGpu>();
std::vector<ProcessorData*> processorData = bestGpu->GetProcessorData();
for (ProcessorData* pd : processorData)
{
GpuValidity validity = GpuValidity::UnknownDevice;
if (pd->deviceProp.major < BestGpu::MininumCCMajorForGpu)
{
validity = GpuValidity::ComputeCapabilityNotSupported;
}
else
{
validity = GpuValidity::Valid;
}
size_t totalMemory = pd->deviceProp.totalGlobalMem/(1024*1024); //From bytes to MBytes
GpuData gpuData = GpuData(pd->deviceProp.major, pd->deviceProp.minor, pd->deviceId, pd->cores, validity, string(pd->deviceProp.name), totalMemory);
data.push_back(gpuData);
}
return data;
}
std::vector<ProcessorData*> BestGpu::GetProcessorData()
{
return m_procData;
}
// QueryNvmlData - Query data from the Nvidia Management Library, and accumulate counters,
// In case failure, this function simply backs out without filling in the data structure and without setting m_nvmlData.
void BestGpu::QueryNvmlData()

31
Source/Common/Include/BestGpu.h
Просмотреть файл

@ -8,15 +8,46 @@
// #define CPUONLY // #define this to build without GPU support nor needing the SDK installed
#include "CommonMatrix.h"
#include <vector>
// define IConfigRecord and ConfigParameters as incomplete types, in order to avoid having to include "ScriptableObjects.h" and "Config.h", as that confuses some .CU code
namespace Microsoft { namespace MSR { namespace ScriptableObjects { struct IConfigRecord; }}}
namespace Microsoft { namespace MSR { namespace CNTK {
using namespace std;
#ifndef CPUONLY
enum class GpuValidity
{
Valid,
UnknownDevice,
ComputeCapabilityNotSupported
};
struct GpuData
{
int versionMajor;
int versionMinor;
int deviceId;
int cudaCores;
GpuValidity validity;
string name;
size_t totalMemory;
GpuData(int versionMajor, int versionMinor, int deviceId, int cudaCores, GpuValidity validity, const string& name, size_t totalMemory)
:versionMajor(versionMajor), versionMinor(versionMinor), deviceId(deviceId), cudaCores(cudaCores), validity(validity), name(name), totalMemory(totalMemory)
{
}
};
std::vector<GpuData> GetAllGpusData();
GpuData GetGpuData(DEVICEID_TYPE deviceId);
class ConfigParameters;
DEVICEID_TYPE DeviceFromConfig(const ConfigParameters& config);
DEVICEID_TYPE DeviceFromConfig(const ScriptableObjects::IConfigRecord& config);
#else
template <class ConfigRecordType>
static inline DEVICEID_TYPE DeviceFromConfig(const ConfigRecordType& /*config*/)

3
Source/Common/Include/TensorShape.h
Просмотреть файл

@ -665,7 +665,8 @@ public:
std::swap(m_strides[i], m_strides[j]);
}
// Flatten the shape in place to a 2D tensor.
// Flatten a tensor shape into a 2D tensor, where splitPoint is the first index to go into the second dimension
// The tensor shape must be flattenable this way, i.e. each of the two index ranges must be dense.
void FlattenTo2DInPlace(size_t splitPoint, const char* errorPrefix/* = nullptr*/)
{
// check & print meaningful error message

4
Source/ComputationNetworkLib/ComputationNode.h
Просмотреть файл

@ -1423,8 +1423,8 @@ public:
m_gradientInitialized = true;
}
// resize and reset this node's gradient to a given matrix's value
void ResetGradient(const Matrix<ElemType>& val)
// Assign the given matrix's value to this node's gradient. The matrix sizes must match.
void AssignGradient(const Matrix<ElemType>& val)
{
UpdateDataSize(Gradient());

141
Source/Math/BlockMultiplier.h
Просмотреть файл

@ -180,7 +180,7 @@ template<typename BlockHandlerT> class BlockMultiplier
// get rid of the allocation but I haven't had time to do this yet.
static void BlockHandler128x4Thread(HandlerArgs<BlockHandlerT> ha)
{
//Accumulate full row results locally b/f writing to C
// Accumulate full row results locally b/f writing to C
VectorT* resultStorage = (VectorT*)ALIGNED_ALLOC(sizeof(VectorT) * ha.rowsPerBlock * ha.n, 16);
memset(resultStorage, 0, sizeof(VectorT) * ha.rowsPerBlock * ha.n);
const int blocksAtOnce = 2;
@ -195,7 +195,7 @@ template<typename BlockHandlerT> class BlockMultiplier
int n = ha.n;
{
//This takes about the same amount of time as the memcpy version below.
// This takes about the same amount of time as the memcpy version below.
for (int c = 0; c < n; ++c)
{
//_mm_prefetch((char*)&(transC[RowColToOffset(c, startRow, m)]), _MM_HINT_T1);
@ -297,7 +297,6 @@ template<typename BlockHandlerT> class BlockMultiplier
int n = ha.n;
{
//This takes about the same amount of time as the memcpy version below.
for (int c = 0; c < n; ++c)
{
VectorT result1 = resultStorage[RowColToOffset(0, c, n)];
@ -475,14 +474,14 @@ template<typename BlockHandlerT> class BlockMultiplier
public:
//Borrowed from [https://software.intel.com/en-us/forums/intel-isa-extensions/topic/285219]
//Results in a saturated add, i.e. in the overflow case we return int_max, and in the
//underflow we return int_min.
//The code is clever - it uses the _mm_blendv_ps instruction to avoid branching.
//It's based on the fact that you can only have overflow if the signs of the operands
//are identical and different from the sign of the result.
//The resulting code results in matrix multiplies that are a few percent slower than the non-saturating code,
//but we avoid the wild inaccuracies resulting from over/underflow.
// Borrowed from [https://software.intel.com/en-us/forums/intel-isa-extensions/topic/285219]
// Results in a saturated add, i.e. in the overflow case we return int_max, and in the
// underflow we return int_min.
// The code is clever - it uses the _mm_blendv_ps instruction to avoid branching.
// It's based on the fact that you can only have overflow if the signs of the operands
// are identical and different from the sign of the result.
// The resulting code results in matrix multiplies that are a few percent slower than the non-saturating code,
// but we avoid the wild inaccuracies resulting from over/underflow.
FORCEINLINE static __m128i my_adds_epi32(__m128i a, __m128i b)
{
__m128i int_min = _mm_set1_epi32(0x80000000);
@ -499,15 +498,15 @@ template<typename BlockHandlerT> class BlockMultiplier
//Saturated horizontal add
FORCEINLINE static int32_t my_hadd(__m128i hAddMe)
{
//We have ABCD, we want A + B + C +D.
//Shuffle to get BADC
// We have ABCD, we want A + B + C +D.
// Shuffle to get BADC
__m128i shuff1 = _mm_shuffle_epi32(hAddMe, _MM_SHUFFLE(1, 0, 3, 2));
__m128i res1 = my_adds_epi32(hAddMe, shuff1);
//Now we have
//A+B B+A, C+D D+C, so shuffle and add once more
// Now we have
// A+B B+A, C+D D+C, so shuffle and add once more
__m128i shuff2 = _mm_shuffle_epi32(res1, _MM_SHUFFLE(0, 1, 2, 3));
//This gives us
//D+C A+B B+A C+D
// This gives us
// D+C A+B B+A C+D
__m128i res2 = my_adds_epi32(res1, shuff2);
return _mm_extract_epi32(res2, 0);
}
@ -560,6 +559,7 @@ template<typename BlockHandlerT> class BlockMultiplier
m_pPool.reset(new StdThreadPool<HandlerArgs<BlockHandlerT>>(threads));
#else
#ifdef OPENMPTHREAD
m_oldNumThreads = omp_get_num_threads();
omp_set_num_threads(threads);
#endif
#endif
@ -568,16 +568,18 @@ template<typename BlockHandlerT> class BlockMultiplier
~BlockMultiplier()
{
BlockHandlerT::FreePreparedB(m_pBlockHandlerBInfo);
omp_set_num_threads(m_oldNumThreads);
}
static ScalarAT* CreateMatrixA(int m, int n, ScalarAT initVal = 0);
static ScalarBT* CreateMatrixB(int m, int n, ScalarBT initVal = 0);
static int32_t* CreateMatrixC(int m, int n, int32_t initVal = 0);
ScalarBT* PrepareB(ScalarBT* oldB, int k, int n);
template<typename ScalarT> static void FreeMatrix(ScalarT* freeMe) { FreeAlignedMatrix<ScalarT>(freeMe); }
//We assume B has been rewritten in block order.
//For now we assume m, k and n are all multiples of kernelsize.
// We assume B has been rewritten in block order.
// For now we assume m, k and n are all multiples of kernelsize.
void MultiplyMatrices(ScalarAT* A, int m, int k, ScalarBT* B, int n, int32_t* C, ScalarAT alpha = 1, ScalarBT beta = 0);
static const int MAXRANGE = 1 << 13;
int m_oldNumThreads;
};
// Instantiate block multipliers
@ -606,8 +608,8 @@ template<typename BlockHandlerT> int32_t* BlockMultiplier<BlockHandlerT>::Create
template<typename BlockHandlerT> FORCEINLINE int BlockMultiplier<BlockHandlerT>::referenceKernel(ScalarAT* A,
ScalarBT* B, int blockSize)
{
//For now just use regular instructions
//until we have the breakdown figured out.
// For now just use regular instructions
// until we have the breakdown figured out.
int accum = 0;
for (int i = 0; i < blockSize; ++i)
{
@ -616,8 +618,8 @@ template<typename BlockHandlerT> FORCEINLINE int BlockMultiplier<BlockHandlerT>:
return accum;
}
//Rewrites B in Block order so that memory accesses to B will be sequential.
//See comments on RewriteBInBlockOrder for details.
// Rewrites B in Block order so that memory accesses to B will be sequential.
// See comments on RewriteBInBlockOrder for details.
template<typename BlockHandlerT> typename BlockMultiplier<BlockHandlerT>::ScalarBT* BlockMultiplier<BlockHandlerT>::PrepareB(ScalarBT* oldB, int k, int n)
{
ScalarBT* newB = CreateMatrixB(k, n);
@ -651,37 +653,37 @@ template<typename BlockHandlerT> typename BlockMultiplier<BlockHandlerT>::Scalar
return newB;
}
//A version of RewriteBInBlockOrder that allows for multiple decreasing block sizes
//(Which is necessary for letting us handle arbitrarily-sized matrices). On the first call, set kOffset to zero, and
//newB to the beginning of the newly allocated B array.
//We return a pointer to the next writable element in newB, and kOffset gets the new offset into the shared dimension.
// A version of RewriteBInBlockOrder that allows for multiple decreasing block sizes
// (Which is necessary for letting us handle arbitrarily-sized matrices). On the first call, set kOffset to zero, and
// newB to the beginning of the newly allocated B array.
// We return a pointer to the next writable element in newB, and kOffset gets the new offset into the shared dimension.
//So if you have blocks of size 128 and 64, do something like this:
//short* newB = alloc_b();
//int kOffset=0;
//int retBlockSize = firstBlockSize;
//(Write out the blocks up to k=128)
//short* nextB = RewriteBInBlockOrder(oldB, newB, k, n, 128, *kOffset);
//short* lastB = RewriteBInBlockOrder(oldB, nextB, k, n, 64, &kOffset);
// So if you have blocks of size 128 and 64, do something like this:
// short* newB = alloc_b();
// int kOffset=0;
// int retBlockSize = firstBlockSize;
// (Write out the blocks up to k=128)
// short* nextB = RewriteBInBlockOrder(oldB, newB, k, n, 128, *kOffset);
// short* lastB = RewriteBInBlockOrder(oldB, nextB, k, n, 64, &kOffset);
//For each block, the logic is as follows:
//Given a block size of 2 and a matrix that looks like this:
// For each block, the logic is as follows:
// Given a block size of 2 and a matrix that looks like this:
// B00 B01 B02 B03
// B10 B11 B12 B13
// B20 B21 B22 B23
// B30 B31 B32 B33
//.. we will rewrite it like this
// .. we will rewrite it like this
// B00 B10 B01 B11 B02 B12 B03 B13 //The first block of columns 0,1,2, and 3.
//B20 B30 B21 B31 B22 B32 B23 B33 //The second block of columns, 0, 1, 2, and 3.
// B20 B30 B21 B31 B22 B32 B23 B33 //The second block of columns, 0, 1, 2, and 3.
//So each "row" of the new matrix (block size * num_original_cols) contains all of the values that
//will be multiplied with block zero of a given row.
//Column offsets within the row can be computed by (block_size * col_offset).
// So each "row" of the new matrix (block size * num_original_cols) contains all of the values that
// will be multiplied with block zero of a given row.
// Column offsets within the row can be computed by (block_size * col_offset).
//Given the original B matrix, we restructure it so that we will access all of its elements in block order.
//That way we can load in blocksize elements from A, then read in the corresponding elements from each
//column of B in sequential order.
//Old, replaced with fn below, keeping around for now for reference.
// Given the original B matrix, we restructure it so that we will access all of its elements in block order.
// That way we can load in blocksize elements from A, then read in the corresponding elements from each
// column of B in sequential order.
// Old, replaced with fn below, keeping around for now for reference.
template<typename BlockHandlerT> typename BlockMultiplier<BlockHandlerT>::ScalarBT* BlockMultiplier<BlockHandlerT>::RewriteBInBlockOrder(ScalarBT* oldB, ScalarBT* newB, int k,
int n, int blockSize, int* kOffset)
{
@ -689,7 +691,7 @@ template<typename BlockHandlerT> typename BlockMultiplier<BlockHandlerT>::Scalar
int numBlocks = (k - *kOffset) / blockSize;
for (int b = 0; b < numBlocks; ++b)
{
//row offset to beginning of block
// Row offset to beginning of block
int blockOffset = *kOffset + (b * blockSize);
for (int c = 0; c < n; ++c)
{
@ -700,28 +702,28 @@ template<typename BlockHandlerT> typename BlockMultiplier<BlockHandlerT>::Scalar
}
}
}
//Bump offset for next level down
// Bump offset for next level down
*kOffset += (numBlocks * blockSize);
//Return ptr to next value to be written
// Return ptr to next value to be written
return curr;
}
//We can also reorder A, pulling in multiple rows at once.
//So while the input looks like this in memory:
// We can also reorder A, pulling in multiple rows at once.
// So while the input looks like this in memory:
// Row1Block1 Row1Block2 Row1Block3
// Row2Block1 Row2Block2 Row2Block3
// Row3Block1 Row3Block2 Row3Block3
// Row4Block1 Row4Block2 Row4Block3
//
// we want the rewritten array to look like this (assuming 2 rows per block):
//Row1Block1 Row2Block1 Row1Block2 Row2Block2 Row1Block3 Row2Block3
//Row3Block1 Row4Block1 Row3Block2 Row4Block2 Row3Block3 Row4Block3
// Row1Block1 Row2Block1 Row1Block2 Row2Block2 Row1Block3 Row2Block3
// Row3Block1 Row4Block1 Row3Block2 Row4Block2 Row3Block3 Row4Block3
//length = 2 (rows per block) * 3 (blocks per row)
// length = 2 (rows per block) * 3 (blocks per row)
//So the first "row" contains the first rowsPerBlock rows in block order.
//We can index into the start of block b in a row via (b * blockSize * rowsPerBlock)
//blockSize is the size of our dot product kernel.
// So the first "row" contains the first rowsPerBlock rows in block order.
// We can index into the start of block b in a row via (b * blockSize * rowsPerBlock)
// blockSize is the size of our dot product kernel.
template<typename BlockHandlerT> void BlockMultiplier<BlockHandlerT>::RewriteAInBlockOrder(ScalarAT* A, ScalarAT* newA, int m,
int k, int /*blockSize*/, int rowsPerBlock)
{
@ -774,7 +776,7 @@ template<typename BlockHandlerT> typename BlockMultiplier<BlockHandlerT>::Scalar
}
//I tried parallelizing this and got no speedup - that doesn't really make sense though.
// I tried parallelizing this and got no speedup - that doesn't really make sense though.
template<typename BlockHandlerT> typename BlockMultiplier<BlockHandlerT>::ScalarAT* BlockMultiplier<BlockHandlerT>::RewriteAInBlockOrder2(ScalarAT* A, ScalarAT* newA,
int m, int k, int blockSize, int rowsPerBlock, int* pKOffset)
{
@ -820,8 +822,9 @@ template<typename BlockHandlerT>struct BlockInfo
std::function<void (HandlerArgs<BlockHandlerT>)> oneFn;
};
//We assume B has been rewritten in block order.
//For now we assume m, k and n are all multiples of kernelsize.
// We assume B has been rewritten in block order.
// We assume C has been zeroed out.
template<typename BlockHandlerT> void BlockMultiplier<BlockHandlerT>::MultiplyMatrices(ScalarAT* A, int m, int k, ScalarBT* B, int n,
int32_t* C, ScalarAT alpha, ScalarBT beta)
{
@ -832,15 +835,24 @@ template<typename BlockHandlerT> void BlockMultiplier<BlockHandlerT>::MultiplyMa
// in the openmp case, but I have not tested this scenario so I'm leaving it in for now.
std::lock_guard<std::mutex> lock(m_MultiplyMut);
{
// We want to multithread to the extent possible. When batch size is small this
// means doing a row at a time so we can take advantage of multiple procs.
// But only row sizes 1 and 4 are supported so far (should be fixed by codegen).
int rowsPerBlock = m / m_numThreads;
if (rowsPerBlock < 4)
rowsPerBlock = 1;
if (rowsPerBlock > 4)
rowsPerBlock = 4;
// Fall back to row at a time if we end up with an invalid # of rows at a time
// TODO: We should always do 4 rows at a time if it makes sense from a threading standpoint
// since it is significantly more efficient. So if we have e.g. 7 rows, we should do
// one set of four rows at a time and then three single rows. This however will require
// changes to this function, RewriteAInBlockOrder and RowToColOffsetRewrittenA, so for now
// we are silently backing off to row at a time.
if (m % rowsPerBlock != 0)
{
throw new std::runtime_error("Error: m must be < 4 or a multiple of 4.");
rowsPerBlock = 1;
}
if (alpha != 1 || beta != 0)
@ -890,10 +902,6 @@ template<typename BlockHandlerT> void BlockMultiplier<BlockHandlerT>::MultiplyMa
std::vector<BlockInfo<BlockHandlerT>*> blockInfos(5);
//std::function<void __cdecl(HandlerArgs<BlockHandlerT>)> fourFn = BlockHandler128x4Thread;
//std::function<void __cdecl(HandlerArgs<BlockHandlerT>)> oneFn = BlockHandler128x1Thread;
BlockInfo<BlockHandlerT> bi128(blocks128, k128, 0, 0, fn128x4, fn128x1);
BlockInfo<BlockHandlerT> bi64(blocks64, k64, offsetA64, offsetB64, fn64x4, fn64x1);
BlockInfo<BlockHandlerT> bi32(blocks32, k32, offsetA32, offsetB32, fn32x4, fn32x1);
@ -911,8 +919,6 @@ template<typename BlockHandlerT> void BlockMultiplier<BlockHandlerT>::MultiplyMa
if ( currBlockInfo.blockCnt > 0)
{
HandlerArgs<BlockHandlerT> ha;
ha.blocks = currBlockInfo.blockCnt;
ha.m = m;
ha.k = currBlockInfo.k;
@ -920,7 +926,6 @@ template<typename BlockHandlerT> void BlockMultiplier<BlockHandlerT>::MultiplyMa
ha.newA = newA + currBlockInfo.offsetA;
ha.B = B + currBlockInfo.offsetB;
ha.transC = C;
ha.rowsPerThread = m / m_numThreads;
ha.rowsPerBlock = rowsPerBlock;
ha.pBlockPreparedB = m_pBlockHandlerBInfo;
@ -999,4 +1004,4 @@ template<typename BlockHandlerT> void BlockMultiplier<BlockHandlerT>::MultiplyMa
}}}//end namespace
}}}// end namespace

3
Source/Readers/CNTKTextFormatReader/CNTKTextFormatReader.vcxproj
Просмотреть файл

@ -110,9 +110,6 @@
<PrecompiledHeader>Create</PrecompiledHeader>
</ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="uci_to_cntk_text_format_converter.py" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets" />
</Project>

10
Source/Readers/CNTKTextFormatReader/CNTKTextFormatReader.vcxproj.filters
Просмотреть файл

@ -47,13 +47,5 @@
<Filter Include="Common\Include">
<UniqueIdentifier>{C6F55578-121A-4D7C-8F57-4172BC5C463B}</UniqueIdentifier>
</Filter>
<Filter Include="Scripts">
<UniqueIdentifier>{cd70d891-88aa-40a4-8e47-0e31e4cac48e}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<None Include="uci_to_cntk_text_format_converter.py">
<Filter>Scripts</Filter>
</None>
</ItemGroup>
</Project>
</Project>

95
Source/Readers/CNTKTextFormatReader/TextParser.cpp
Просмотреть файл

@ -16,6 +16,11 @@
namespace Microsoft { namespace MSR { namespace CNTK {
inline bool IsDigit(char c)
{
return '0' <= c && c <= '9';
}
enum State
{
Init = 0,
@ -38,7 +43,7 @@ public:
void GetSequence(size_t sequenceId, std::vector<SequenceDataPtr>& result) override;
// A map from sequence ids to the sequence data.
std::map<size_t, SequenceBuffer> m_sequenceMap;
std::vector<SequenceBuffer> m_sequenceMap;
// chunk id (copied from the descriptor)
ChunkIdType m_id;
@ -234,40 +239,11 @@ TextParser<ElemType>::TextDataChunk::TextDataChunk(const ChunkDescriptor& descri
template <class ElemType>
void TextParser<ElemType>::TextDataChunk::GetSequence(size_t sequenceId, std::vector<SequenceDataPtr>& result)
{
auto it = m_sequenceMap.find(sequenceId);
assert(it != m_sequenceMap.end());
assert(sequenceId < m_sequenceMap.size());
result.reserve(m_parser->m_streamInfos.size());
const auto& sequenceData = it->second;
for (size_t j = 0; j < m_parser->m_streamInfos.size(); ++j)
{
InputStreamBuffer* input = sequenceData[j].get();
const StreamInfo& stream = m_parser->m_streamInfos[j];
SequenceDataPtr data;
if (stream.m_type == StorageType::dense)
{
auto denseData = make_shared<DenseSequenceData>();
denseData->m_sampleLayout = m_parser->m_streams[j]->m_sampleLayout;
data = denseData;
}
else
{
auto sparseData = make_shared<SparseSequenceData>();
SparseInputStreamBuffer* sparseInput = static_cast<SparseInputStreamBuffer*>(input);
sparseData->m_indices = sparseInput->m_indices.data();
sparseData->m_nnzCounts.reserve(sparseInput->m_nnzCounts.size());
copy(sparseInput->m_nnzCounts.begin(), sparseInput->m_nnzCounts.end(),
back_inserter(sparseData->m_nnzCounts));
sparseData->m_totalNnzCount = sparseInput->m_totalNnzCount;
assert(input->m_numberOfSamples == sparseInput->m_nnzCounts.size());
data = sparseData;
}
data->m_data = input->m_buffer.data();
data->m_numberOfSamples = input->m_numberOfSamples;
data->m_chunk = shared_from_this();
data->m_id = sequenceId;
result.push_back(data);
}
const auto& sequenceData = m_sequenceMap[sequenceId];
result.insert(result.end(), sequenceData.begin(), sequenceData.end());
}
template <class ElemType>
@ -292,11 +268,10 @@ ChunkPtr TextParser<ElemType>::GetChunk(ChunkIdType chunkId)
template <class ElemType>
void TextParser<ElemType>::LoadChunk(TextChunkPtr& chunk, const ChunkDescriptor& descriptor)
{
chunk->m_sequenceMap.resize(descriptor.m_sequences.size());
for (const auto& sequenceDescriptor : descriptor.m_sequences)
{
chunk->m_sequenceMap.insert(make_pair(
sequenceDescriptor.m_id,
LoadSequence(sequenceDescriptor)));
chunk->m_sequenceMap[sequenceDescriptor.m_id] = LoadSequence(sequenceDescriptor);
}
}
@ -480,13 +455,39 @@ typename TextParser<ElemType>::SequenceBuffer TextParser<ElemType>::LoadSequence
GetSequenceKey(sequenceDsc).c_str(), GetFileInfo().c_str(), numRowsRead, expectedRowCount);
}
FillSequenceMetadata(sequence, sequenceDsc.m_id);
return sequence;
}
template<class ElemType>
void TextParser<ElemType>::FillSequenceMetadata(SequenceBuffer& sequenceData, size_t sequenceId)
{
for (size_t j = 0; j < m_streamInfos.size(); ++j)
{
const StreamInfo& stream = m_streamInfos[j];
SequenceDataBase* data = sequenceData[j].get();
if (stream.m_type == StorageType::dense)
{
auto denseData = static_cast<DenseInputStreamBuffer*>(data);
denseData->m_sampleLayout = m_streams[j]->m_sampleLayout;
data->m_data = denseData->m_buffer.data();
}
else
{
auto sparseData = static_cast<SparseInputStreamBuffer*>(data);
sparseData->m_indices = sparseData->m_indicesBuffer.data();
assert(data->m_numberOfSamples == sparseData->m_nnzCounts.size());
data->m_data = sparseData->m_buffer.data();
}
data->m_id = sequenceId;
}
}
template <class ElemType>
bool TextParser<ElemType>::TryReadRow(SequenceBuffer& sequence, size_t& bytesToRead)
{
while (bytesToRead && CanRead() && isdigit(*m_pos))
while (bytesToRead && CanRead() && IsDigit(*m_pos))
{
// skip sequence ids
++m_pos;
@ -616,7 +617,7 @@ bool TextParser<ElemType>::TryReadSample(SequenceBuffer& sequence, size_t& bytes
{
SparseInputStreamBuffer* data = reinterpret_cast<SparseInputStreamBuffer*>(sequence[id].get());
vector<ElemType>& values = data->m_buffer;
vector<IndexType>& indices = data->m_indices;
vector<IndexType>& indices = data->m_indicesBuffer;
assert(values.size() == indices.size());
size_t size = values.size();
if (!TryReadSparseSample(values, indices, stream.m_sampleDimension, bytesToRead))
@ -919,7 +920,7 @@ bool TextParser<ElemType>::TryReadUint64(size_t& value, size_t& bytesToRead)
{
char c = *m_pos;
if (!isdigit(c))
if (!IsDigit(c))
{
return found;
}
@ -977,7 +978,7 @@ bool TextParser<ElemType>::TryReadRealNumber(ElemType& value, size_t& bytesToRea
{
case State::Init:
// the number must either start with a number or a sign
if (isdigit(c))
if (IsDigit(c))
{
state = IntegralPart;
number = (c - '0');
@ -1001,7 +1002,7 @@ bool TextParser<ElemType>::TryReadRealNumber(ElemType& value, size_t& bytesToRea
break;
case Sign:
// the sign must be followed by a number
if (isdigit(c))
if (IsDigit(c))
{
state = IntegralPart;
number = (c - '0');
@ -1019,7 +1020,7 @@ bool TextParser<ElemType>::TryReadRealNumber(ElemType& value, size_t& bytesToRea
}
break;
case IntegralPart:
if (isdigit(c))
if (IsDigit(c))
{
number = number * 10 + (c - '0');
}
@ -1040,7 +1041,7 @@ bool TextParser<ElemType>::TryReadRealNumber(ElemType& value, size_t& bytesToRea
}
break;
case Period:
if (isdigit(c))
if (IsDigit(c))
{
state = FractionalPart;
coefficient = number;
@ -1054,7 +1055,7 @@ bool TextParser<ElemType>::TryReadRealNumber(ElemType& value, size_t& bytesToRea
}
break;
case FractionalPart:
if (isdigit(c))
if (IsDigit(c))
{
// TODO: ignore if number of precision digits > FLT_[MANT_]DIG/DBL_[MANT_]DIG
// no state change
@ -1079,7 +1080,7 @@ bool TextParser<ElemType>::TryReadRealNumber(ElemType& value, size_t& bytesToRea
break;
case TheLetterE:
// followed with optional minus or plus sign and nonempty sequence of decimal digits
if (isdigit(c))
if (IsDigit(c))
{
state = Exponent;
negative = false;
@ -1104,7 +1105,7 @@ bool TextParser<ElemType>::TryReadRealNumber(ElemType& value, size_t& bytesToRea
break;
case ExponentSign:
// exponent sign must be followed by a number
if (isdigit(c))
if (IsDigit(c))
{
state = Exponent;
number = (c - '0');
@ -1122,7 +1123,7 @@ bool TextParser<ElemType>::TryReadRealNumber(ElemType& value, size_t& bytesToRea
}
break;
case Exponent:
if (isdigit(c))
if (IsDigit(c))
{
// no state change
number = number * 10 + (c - '0');

35
Source/Readers/CNTKTextFormatReader/TextParser.h
Просмотреть файл

@ -42,37 +42,33 @@ private:
// Builds an index of the input data.
void Initialize();
// A buffer to keep data for all samples in a (variable length) sequence
// from a single input stream.
struct InputStreamBuffer
{
virtual ~InputStreamBuffer() { };
uint32_t m_numberOfSamples = 0;
std::vector<ElemType> m_buffer;
};
struct DenseInputStreamBuffer : InputStreamBuffer
struct DenseInputStreamBuffer : DenseSequenceData
{
// capacity = expected number of samples * sample size
DenseInputStreamBuffer(size_t capacity)
{
InputStreamBuffer::m_buffer.reserve(capacity);
m_buffer.reserve(capacity);
}
std::vector<ElemType> m_buffer;
};
// In case of sparse input, we also need a vector of
// indices (one index for each input value) and a vector
// of NNZ counts (one for each sample).
struct SparseInputStreamBuffer : InputStreamBuffer
struct SparseInputStreamBuffer : SparseSequenceData
{
IndexType m_totalNnzCount = 0;
std::vector<IndexType> m_indices;
std::vector<IndexType> m_nnzCounts;
SparseInputStreamBuffer()
{
m_totalNnzCount = 0;
}
std::vector<IndexType> m_indicesBuffer;
std::vector<ElemType> m_buffer;
};
// A sequence buffer is a vector that contains an input buffer for each input stream.
typedef std::vector<std::unique_ptr<InputStreamBuffer>> SequenceBuffer;
// A sequence buffer is a vector that contains sequence data for each input stream.
typedef std::vector<SequenceDataPtr> SequenceBuffer;
// A chunk of input data in the text format.
class TextDataChunk;
@ -176,6 +172,9 @@ private:
TextParser(CorpusDescriptorPtr corpus, const std::wstring& filename, const vector<StreamDescriptor>& streams);
// Fills some metadata members to be conformant to the exposed SequenceData interface.
void FillSequenceMetadata(SequenceBuffer& sequenceBuffer, size_t sequenceId);
void SetTraceLevel(unsigned int traceLevel);
void SetMaxAllowedErrors(unsigned int maxErrors);

2
Source/Readers/CNTKTextFormatReader/TextReaderConstants.h
Просмотреть файл

@ -18,7 +18,7 @@ namespace Microsoft { namespace MSR { namespace CNTK {
const char ESCAPE_SYMBOL = '#';
const auto BUFFER_SIZE = 256 * 1024;
const auto BUFFER_SIZE = 2 * 1024 * 1024;
inline bool isPrintable(char c)
{

36
Source/Readers/ImageReader/ImageReader.vcxproj
Просмотреть файл

@ -30,18 +30,6 @@
<PlatformToolset>v120</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup>
<HasOpenCV>false</HasOpenCV>
<HasOpenCV Condition="Exists('$(OPENCV_PATH)')">true</HasOpenCV>
<UseZip>false</UseZip>
<UseZip Condition="Exists('$(ZLIB_PATH)')">true</UseZip>
</PropertyGroup>
<PropertyGroup Condition="$(UseZip)">
<ZipInclude>$(ZLIB_PATH)\include;$(ZLIB_PATH)\lib\libzip\include;</ZipInclude>
<ZipDefine>USE_ZIP</ZipDefine>
<ZipLibPath>$(ZLIB_PATH)\lib;</ZipLibPath>
<ZipLibs>zlib.lib;zip.lib;</ZipLibs>
</PropertyGroup>
<PropertyGroup Condition="$(DebugBuild)" Label="Configuration">
<UseDebugLibraries>true</UseDebugLibraries>
</PropertyGroup>
@ -54,9 +42,6 @@
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup>
<OpenCVLib Condition="$(HasOpenCV)">opencv_world300.lib</OpenCVLib>
</PropertyGroup>
<PropertyGroup Condition="$(DebugBuild)">
<LinkIncremental>true</LinkIncremental>
</PropertyGroup>
@ -76,22 +61,23 @@
<Link>
<SubSystem>Windows</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>ReaderLib.lib;Common.lib;Math.lib;$(OpenCVLib);$(ZipLibs);%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>ReaderLib.lib;Common.lib;Math.lib;$(OpenCvLib);$(ZipLibs);%(AdditionalDependencies)</AdditionalDependencies>
</Link>
<PostBuildEvent>
<Command>if "$(HasOpenCv)" == "true" xcopy /I /D /Y "$(OPENCV_PATH)\x64\vc12\bin\opencv_world300.dll" "$(TargetDir)"
if "$(UseZip)" == "true" xcopy /I /D /Y "$(ZLIB_PATH)\bin\zip.dll" "$(TargetDir)"
if "$(UseZip)" == "true" if exist "$(ZLIB_PATH)\bin\zlib1.dll" (xcopy /I /D /Y "$(ZLIB_PATH)\bin\zlib1.dll" "$(TargetDir)") else (copy /Y "$(ZLIB_PATH)\bin\zlib.dll" "$(TargetDir)\zlib1.dll")
</Command>
<Command>
if "$(HasOpenCv)" == "true" xcopy /I /D /Y "$(OpenCvBinPath)\$(OpenCvWorld).dll" "$(TargetDir)"
if "$(UseZip)" == "true" xcopy /I /D /Y "$(ZLIB_PATH)\bin\zip.dll" "$(TargetDir)"
if "$(UseZip)" == "true" if exist "$(ZLIB_PATH)\bin\zlib1.dll" (xcopy /I /D /Y "$(ZLIB_PATH)\bin\zlib1.dll" "$(TargetDir)") else (copy /Y "$(ZLIB_PATH)\bin\zlib.dll" "$(TargetDir)\zlib1.dll")
</Command>
<Message>Copying dependencies</Message>
</PostBuildEvent>
</ItemDefinitionGroup>
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>$(SolutionDir)Source\Common\Include;$(SolutionDir)Source\Math;$(OPENCV_PATH)\include;$(ZipInclude);$(SolutionDir)Source\Readers\ReaderLib</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(SolutionDir)Source\Common\Include;$(SolutionDir)Source\Math;$(OpenCvInclude);$(ZipInclude);$(SolutionDir)Source\Readers\ReaderLib</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<AdditionalLibraryDirectories>$(OutDir);$(OPENCV_PATH)\x64\vc12\lib;$(ZipLibPath)</AdditionalLibraryDirectories>
<AdditionalLibraryDirectories>$(OutDir);$(OpenCvLibPath);$(ZipLibPath)</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="$(DebugBuild)">
@ -131,7 +117,7 @@ if "$(UseZip)" == "true" if exist "$(ZLIB_PATH)\bin\zlib1.dll" (xcopy /I /D /Y "
<ClCompile Include="ImageDataDeserializer.cpp" />
<ClCompile Include="dllmain.cpp" />
<ClCompile Include="Exports.cpp">
<ExcludedFromBuild Condition="!$(HasOpenCV)">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="!$(HasOpenCv)">true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="ImageReader.cpp" />
<ClCompile Include="ImageTransformers.cpp" />
@ -141,10 +127,10 @@ if "$(UseZip)" == "true" if exist "$(ZLIB_PATH)\bin\zlib1.dll" (xcopy /I /D /Y "
<ClCompile Include="ZipByteReader.cpp" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<Target Name="Build" Condition="$(HasOpenCV)" Outputs="$(TargetPath)" DependsOnTargets="$(BuildDependsOn)" />
<Target Name="Build" Condition="$(HasOpenCv)" Outputs="$(TargetPath)" DependsOnTargets="$(BuildDependsOn)" />
<ImportGroup Label="ExtensionTargets" />
<Target Name="CheckDependencies">
<Warning Condition="!$(HasOpenCV)" Text="ImageReader requires the OpenCV library to build. Please see https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows#opencv for installation instructions." />
<Warning Condition="!$(HasOpenCv)" Text="ImageReader requires the OpenCV library to build. Please see https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows#opencv for installation instructions." />
<Warning Condition="!$(UseZip)" Text="zlib and libzip libraries were not found, ImageReader will be built without zip container support. Please see https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows#libzip for installation instructions." />
</Target>
</Project>

39
Source/Readers/ReaderLib/BlockRandomizer.cpp
Просмотреть файл

@ -102,9 +102,6 @@ void BlockRandomizer::PrepareNewSweepIfNeeded(size_t samplePosition)
// Resetting sequence randomizer.
m_sequenceRandomizer->Reset(m_sweep + 1);
// Unloading all chunk data from memory.
m_chunks.clear();
m_lastSeenChunkId = CHUNKID_MAX;
}
}
@ -142,7 +139,7 @@ Sequences BlockRandomizer::GetNextSequences(size_t sampleCount)
auto process = [&](int i) -> void {
const auto& description = decimated[i];
std::vector<SequenceDataPtr> sequence;
auto it = m_chunks.find(description.m_chunk->m_chunkId);
auto it = m_chunks.find(description.m_chunk->m_original->m_id);
if (it == m_chunks.end())
{
LogicError("Invalid chunk requested.");
@ -253,6 +250,11 @@ void BlockRandomizer::RetrieveDataChunks()
// There could be some chunks in the m_chunks that are not required anymore, by swapping the chunks with m_chunks, we are removing those.
std::map<size_t, ChunkPtr> chunks;
size_t numLoadedChunks = m_chunks.size();
std::vector<bool> needed;
needed.resize(randomizedEnd, false);
// Firstly, make sure we unload all not needed chunks:
for (size_t i = 0; i < randomizedEnd; ++i)
{
auto const& chunk = window[i];
@ -261,27 +263,36 @@ void BlockRandomizer::RetrieveDataChunks()
continue;
}
auto it = m_chunks.find(chunk.m_chunkId);
auto it = m_chunks.find(chunk.m_original->m_id);
if (it != m_chunks.end())
{
chunks[chunk.m_chunkId] = it->second;
chunks[chunk.m_original->m_id] = it->second;
}
else
{
chunks[chunk.m_chunkId] = m_deserializer->GetChunk(chunk.m_original->m_id);
if (m_verbosity >= Information)
fprintf(stderr, "BlockRandomizer::RetrieveDataChunks: paged in randomized chunk %u (original chunk: %u), now %" PRIu64 " chunks in memory\n",
chunk.m_chunkId,
chunk.m_original->m_id,
++numLoadedChunks);
needed[i] = true;
}
}
// Swapping current chunks in the m_chunks, by that removing all stale and remembering newly loaded.
// Swapping current chunks in the m_chunks, by that removing all stale.
// TODO diagnostics for paged out chunks?
m_chunks.swap(chunks);
// Adding new ones.
for (size_t i = 0; i < randomizedEnd; ++i)
{
if (needed[i])
{
auto const& chunk = window[i];
m_chunks[chunk.m_original->m_id] = m_deserializer->GetChunk(chunk.m_original->m_id);
if (m_verbosity >= Information)
fprintf(stderr, "BlockRandomizer::RetrieveDataChunks: paged in randomized chunk %u (original chunk: %u), now %" PRIu64 " chunks in memory\n",
chunk.m_chunkId,
chunk.m_original->m_id,
++numLoadedChunks);
}
}
if (m_verbosity >= Notification)
fprintf(stderr, "BlockRandomizer::RetrieveDataChunks: %" PRIu64 " chunks paged-in from chunk window [%u..%u]\n",
m_chunks.size(),

2
Source/Readers/ReaderLib/BlockRandomizer.h
Просмотреть файл

@ -107,7 +107,7 @@ private:
// Exposed streams.
std::vector<StreamDescriptionPtr> m_streams;
// A map of data chunks.
// A map of data chunks from original chunk id into chunk.
std::map<size_t, ChunkPtr> m_chunks;
// Last seen data chunk id.

31
Source/Readers/ReaderLib/SequenceRandomizer.cpp
Просмотреть файл

@ -191,21 +191,28 @@ namespace Microsoft { namespace MSR { namespace CNTK {
size_t posBegin = m_randomizedChunks[chunkWindowBegin].m_sequencePositionStart;
size_t posEnd = m_randomizedChunks[chunkWindowEnd - 1].SequenceEndPosition();
ChunkIdType tChunkIndex = GetChunkIndexForSequencePosition(t);
auto& tSequence = GetRandomizedSequenceDescriptionByPosition(tChunkIndex, t);
for (;;)
{
// Pick a sequence position from [posBegin, posEnd)
const size_t j = rand(posBegin, posEnd);
// Pick up j sequence.
ChunkIdType jChunkIndex = GetChunkIndexForSequencePosition(j);
auto& jSequence = GetRandomizedSequenceDescriptionByPosition(jChunkIndex, j);
// Try again if the sequence currently at j cannot be placed at position i.
if (!IsValidForPosition(t, GetRandomizedSequenceDescriptionBySequenceId(j)))
if (!IsValidForPosition(tChunkIndex, jSequence))
continue;
// Try again if the sequence currently at i cannot be placed at position j.
if (!IsValidForPosition(j, GetRandomizedSequenceDescriptionBySequenceId(t)))
if (!IsValidForPosition(jChunkIndex, tSequence))
continue;
// Swap and break out.
std::swap(GetRandomizedSequenceDescriptionBySequenceId(t), GetRandomizedSequenceDescriptionBySequenceId(j)); // TODO old swap was perhaps more efficient
std::swap(tSequence, jSequence);
break;
}
}
@ -214,7 +221,8 @@ namespace Microsoft { namespace MSR { namespace CNTK {
for (size_t t = firstSequencePositionToRandomize; t < endSequencePosToRandomize; ++t)
{
// TODO assert only
if (!IsValidForPosition(t, GetRandomizedSequenceDescriptionBySequenceId(t)))
ChunkIdType tChunkIndex = GetChunkIndexForSequencePosition(t);
if (!IsValidForPosition(tChunkIndex, GetRandomizedSequenceDescriptionByPosition(tChunkIndex, t)))
{
LogicError("SequenceRandomizer::RandomizeNextSequenceDescriptions: randomization logic mangled!");
}
@ -322,10 +330,10 @@ namespace Microsoft { namespace MSR { namespace CNTK {
return m_currentSampleCursor;
}
// Checks if the randomized sequence is valid for a target position using its chunk randomization window.
bool SequenceRandomizer::IsValidForPosition(size_t targetPosition, const RandomizedSequenceDescription& seqDesc) const
// Checks if the randomized sequence is valid for a target chunk.
bool SequenceRandomizer::IsValidForPosition(ChunkIdType chunkIndex, const RandomizedSequenceDescription& seqDesc) const
{
const auto& chunk = m_randomizedChunks[GetChunkIndexForSequencePosition(targetPosition)];
const auto& chunk = m_randomizedChunks[chunkIndex];
return chunk.m_randomizationWindow.m_begin <= seqDesc.m_chunk->m_chunkId && seqDesc.m_chunk->m_chunkId < chunk.m_randomizationWindow.m_end;
}
@ -365,11 +373,10 @@ namespace Microsoft { namespace MSR { namespace CNTK {
m_chunkWindowEnd++;
}
// Gets randomized sequence by the sequence id.
RandomizedSequenceDescription& SequenceRandomizer::GetRandomizedSequenceDescriptionBySequenceId(size_t sequenceId)
// Gets randomized sequence by the sequence position in the sweep and randomized chunk index.
RandomizedSequenceDescription& SequenceRandomizer::GetRandomizedSequenceDescriptionByPosition(ChunkIdType chunkIndex, size_t sequenceSweepPosition)
{
ChunkIdType globalChunkIdx = GetChunkIndexForSequencePosition(sequenceId);
size_t sequenceOffsetInsideChunk = sequenceId - m_randomizedChunks[globalChunkIdx].m_sequencePositionStart;
return m_sequenceWindow[globalChunkIdx - m_chunkWindowBegin][sequenceOffsetInsideChunk];
size_t sequenceOffsetInsideChunk = sequenceSweepPosition - m_randomizedChunks[chunkIndex].m_sequencePositionStart;
return m_sequenceWindow[chunkIndex - m_chunkWindowBegin][sequenceOffsetInsideChunk];
}
}}}

14
Source/Readers/ReaderLib/SequenceRandomizer.h
Просмотреть файл

@ -19,10 +19,10 @@ struct RandomizedSequenceDescription
{
// Sequence id.
size_t m_id;
// Number of samples in sequence.
uint32_t m_numberOfSamples;
// Randomized chunk this sequence belongs to.
const RandomizedChunk* m_chunk;
// Number of samples in sequence.
uint32_t m_numberOfSamples;
};
// Class that given randomized chunks, randomizes sequence descriptions in a window of chunks.
@ -64,14 +64,14 @@ private:
// Randomize one more chunk if needed after the chunk cursor has been incremented.
void RandomizeNextChunkIfNeeded();
// Checks if the randomized sequence is valid for a target position using its chunk randomization window.
bool IsValidForPosition(size_t targetPosition, const RandomizedSequenceDescription& seqDesc) const;
// Checks if the randomized sequence is valid for a target chunk.
bool IsValidForPosition(ChunkIdType chunkIndex, const RandomizedSequenceDescription& seqDesc) const;
// Gets randomized chunk index using a sequence position in the sweep.
ChunkIdType GetChunkIndexForSequencePosition(size_t sequencePosition) const;
ChunkIdType GetChunkIndexForSequencePosition(size_t sequenceSweepPosition) const;
// Gets randomized sequence by the sequence id.
RandomizedSequenceDescription& GetRandomizedSequenceDescriptionBySequenceId(size_t sequenceId);
// Gets randomized sequence by sequence position in sweep and its randomized chunk index.
RandomizedSequenceDescription& GetRandomizedSequenceDescriptionByPosition(ChunkIdType chunkIndex, size_t sequenceSweepPosition);
// Add randomizes sequences for the chunk with a given index.
void AddRandomizedSequencesForChunk(ChunkIdType chunkIndex);

657
Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/baseline.linux.cpu.txt
Просмотреть файл

@ -1,48 +1,48 @@
=== Running /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/gpu/debug/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config/rnn.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data RunDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config OutputDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu DeviceId=-1 timestamping=true initOnCPUOnly=true command=writeWordAndClassInfo:train:test train=[SGD=[maxEpochs=3]] train=[epochSize=2048]] test=[SGD=[maxEpochs=3]] train=[epochSize=2048]]
=== Running /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/gpu/release/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config/rnn.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data RunDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config OutputDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu DeviceId=-1 timestamping=true initOnCPUOnly=true command=writeWordAndClassInfo:train:test train=[SGD=[maxEpochs=3]] train=[epochSize=2048]] test=[SGD=[maxEpochs=3]] train=[epochSize=2048]]
-------------------------------------------------------------------
Build info:
Built time: May 5 2016 21:53:50
Last modified date: Thu May 5 19:06:09 2016
Build type: debug
Built time: Jun 15 2016 08:10:22
Last modified date: Wed Jun 15 08:04:52 2016
Build type: release
Build target: GPU
With 1bit-SGD: no
Math lib: acml
Math lib: mkl
CUDA_PATH: /usr/local/cuda-7.5
CUB_PATH: /usr/local/cub-1.4.1
CUDNN_PATH: /usr/local/cudnn-4.0
Build Branch: HEAD
Build SHA1: 163a49b07f300a0a2c74b074373431af0fc6959a
Built by philly on cded08e6b560
Build SHA1: 35cb5738e7ef794177a2fff06892a39700722dee
Built by philly on d7142b6d43fb
Build Path: /home/philly/jenkins/workspace/CNTK-Build-Linux
-------------------------------------------------------------------
Changed current directory to /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data
05/05/2016 22:20:12: -------------------------------------------------------------------
05/05/2016 22:20:12: Build info:
06/16/2016 10:51:17: -------------------------------------------------------------------
06/16/2016 10:51:17: Build info:
05/05/2016 22:20:12: Built time: May 5 2016 21:53:50
05/05/2016 22:20:12: Last modified date: Thu May 5 19:06:09 2016
05/05/2016 22:20:12: Build type: debug
05/05/2016 22:20:12: Build target: GPU
05/05/2016 22:20:12: With 1bit-SGD: no
05/05/2016 22:20:12: Math lib: acml
05/05/2016 22:20:12: CUDA_PATH: /usr/local/cuda-7.5
05/05/2016 22:20:12: CUB_PATH: /usr/local/cub-1.4.1
05/05/2016 22:20:12: CUDNN_PATH: /usr/local/cudnn-4.0
05/05/2016 22:20:12: Build Branch: HEAD
05/05/2016 22:20:12: Build SHA1: 163a49b07f300a0a2c74b074373431af0fc6959a
05/05/2016 22:20:12: Built by philly on cded08e6b560
05/05/2016 22:20:12: Build Path: /home/philly/jenkins/workspace/CNTK-Build-Linux
05/05/2016 22:20:12: -------------------------------------------------------------------
06/16/2016 10:51:17: Built time: Jun 15 2016 08:10:22
06/16/2016 10:51:17: Last modified date: Wed Jun 15 08:04:52 2016
06/16/2016 10:51:17: Build type: release
06/16/2016 10:51:17: Build target: GPU
06/16/2016 10:51:17: With 1bit-SGD: no
06/16/2016 10:51:17: Math lib: mkl
06/16/2016 10:51:17: CUDA_PATH: /usr/local/cuda-7.5
06/16/2016 10:51:17: CUB_PATH: /usr/local/cub-1.4.1
06/16/2016 10:51:17: CUDNN_PATH: /usr/local/cudnn-4.0
06/16/2016 10:51:17: Build Branch: HEAD
06/16/2016 10:51:17: Build SHA1: 35cb5738e7ef794177a2fff06892a39700722dee
06/16/2016 10:51:17: Built by philly on d7142b6d43fb
06/16/2016 10:51:17: Build Path: /home/philly/jenkins/workspace/CNTK-Build-Linux
06/16/2016 10:51:17: -------------------------------------------------------------------
05/05/2016 22:20:12: Running on localhost at 2016/05/05 22:20:12
05/05/2016 22:20:12: Command line:
/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/gpu/debug/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config/rnn.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data RunDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config OutputDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu DeviceId=-1 timestamping=true initOnCPUOnly=true command=writeWordAndClassInfo:train:test train=[SGD=[maxEpochs=3]] train=[epochSize=2048]] test=[SGD=[maxEpochs=3]] train=[epochSize=2048]]
06/16/2016 10:51:17: Running on localhost at 2016/06/16 10:51:17
06/16/2016 10:51:17: Command line:
/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/build/gpu/release/bin/cntk configFile=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config/rnn.cntk currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data RunDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config OutputDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu DeviceId=-1 timestamping=true initOnCPUOnly=true command=writeWordAndClassInfo:train:test train=[SGD=[maxEpochs=3]] train=[epochSize=2048]] test=[SGD=[maxEpochs=3]] train=[epochSize=2048]]
05/05/2016 22:20:12: >>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>>
05/05/2016 22:20:12: RootDir = ".."
06/16/2016 10:51:17: >>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>>
06/16/2016 10:51:17: RootDir = ".."
ConfigDir = "$RootDir$/Config"
DataDir = "$RootDir$/Data"
OutputDir = "$RootDir$/Output"
@ -350,10 +350,10 @@ cacheBlockSize = 1
]
]
currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data
RunDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu
RunDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu
DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data
ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config
OutputDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu
OutputDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu
DeviceId=-1
timestamping=true
initOnCPUOnly=true
@ -363,19 +363,19 @@ train=[epochSize=2048]]
test=[SGD=[maxEpochs=3]]
train=[epochSize=2048]]
05/05/2016 22:20:12: <<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<<
06/16/2016 10:51:17: <<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<<
05/05/2016 22:20:12: >>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>
05/05/2016 22:20:12: RootDir = ".."
06/16/2016 10:51:17: >>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>
06/16/2016 10:51:17: RootDir = ".."
ConfigDir = "../Config"
DataDir = "../Data"
OutputDir = "../Output"
ModelDir = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models"
ModelDir = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models"
deviceId = "auto"
command = writeWordAndClassInfo:train:test:write
precision = "float"
traceLevel = 1
modelPath = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/rnn.dnn"
modelPath = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/rnn.dnn"
numCPUThreads = 1
confVocabSize = 10000
confClassSize = 50
@ -387,9 +387,9 @@ writeWordAndClassInfo = [
inputFile = "/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data/ptb.train.txt"
beginSequence = "</s>"
endSequence = "</s>"
outputVocabFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
outputWord2Cls = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/word2cls.txt"
outputCls2Index = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/cls2idx.txt"
outputVocabFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
outputWord2Cls = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/word2cls.txt"
outputCls2Index = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/cls2idx.txt"
vocabSize = "10000"
nbrClass = "50"
cutoff = 0
@ -443,8 +443,8 @@ lookupTableOrder = 1
randomize = "none"
nbruttsineachrecurrentiter = 0
cacheBlockSize = 2000000
wordclass = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sequenceSentence.bin"
wordclass = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sequenceSentence.bin"
wsize = 256
wrecords = 1000
windowSize = "10000"
@ -459,7 +459,7 @@ cacheBlockSize = 2000000
beginSequence = "</s>"
endSequence = "</s>"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -478,7 +478,7 @@ cacheBlockSize = 2000000
beginSequence = "O"
endSequence = "O"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -497,8 +497,8 @@ cacheBlockSize = 2000000
randomize = "none"
nbruttsineachrecurrentiter = 0
cacheBlockSize = 2000000
wordclass = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sequenceSentence.valid.bin"
wordclass = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sequenceSentence.valid.bin"
wsize = 256
wrecords = 1000
windowSize = "10000"
@ -510,7 +510,7 @@ cacheBlockSize = 2000000
labelIn = [
dim = 1
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
labelType = "Category"
beginSequence = "</s>"
endSequence = "</s>"
@ -532,7 +532,7 @@ cacheBlockSize = 2000000
beginSequence = "O"
endSequence = "O"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -557,8 +557,8 @@ minibatchSize = 8192
randomize = "none"
nbruttsineachrecurrentiter = 0
cacheBlockSize = 2000000
wordclass = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sequenceSentence.bin"
wordclass = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sequenceSentence.bin"
wsize = 256
wrecords = 1000
windowSize = "10000"
@ -570,7 +570,7 @@ cacheBlockSize = 2000000
labelIn = [
dim = 1
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.txt"
labelType = "Category"
beginSequence = "</s>"
endSequence = "</s>"
@ -592,7 +592,7 @@ cacheBlockSize = 2000000
beginSequence = "O"
endSequence = "O"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -609,7 +609,7 @@ cacheBlockSize = 2000000
]
write = [
action = "write"
outputPath = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Write"
outputPath = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Write"
outputNodeNames = TrainNodeClassBasedCrossEntropy
format = [
sequencePrologue = "log P(W)="
@ -623,8 +623,8 @@ minibatchSize = 8192
randomize = "none"
nbruttsineachrecurrentiter = 1
cacheBlockSize = 1
wordclass = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sequenceSentence.bin"
wordclass = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sequenceSentence.bin"
wsize = 256
wrecords = 1000
windowSize = "10000"
@ -636,7 +636,7 @@ cacheBlockSize = 1
labelIn = [
dim = 1
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.txt"
labelType = "Category"
beginSequence = "</s>"
endSequence = "</s>"
@ -658,7 +658,7 @@ cacheBlockSize = 1
beginSequence = "O"
endSequence = "O"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -674,10 +674,10 @@ cacheBlockSize = 1
]
]
currentDirectory=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data
RunDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu
RunDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu
DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data
ConfigDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/../../../../../../Examples/Text/PennTreebank/Config
OutputDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu
OutputDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu
DeviceId=-1
timestamping=true
initOnCPUOnly=true
@ -687,9 +687,9 @@ train=[epochSize=2048]]
test=[SGD=[maxEpochs=3]]
train=[epochSize=2048]]
05/05/2016 22:20:12: <<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<
06/16/2016 10:51:17: <<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<
05/05/2016 22:20:12: >>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>
06/16/2016 10:51:17: >>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>>
configparameters: rnn.cntk:]=true
configparameters: rnn.cntk:command=writeWordAndClassInfo:train:test
configparameters: rnn.cntk:confClassSize=50
@ -699,13 +699,13 @@ configparameters: rnn.cntk:currentDirectory=/home/philly/jenkins/workspace/CNTK-
configparameters: rnn.cntk:DataDir=/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data
configparameters: rnn.cntk:deviceId=-1
configparameters: rnn.cntk:initOnCPUOnly=true
configparameters: rnn.cntk:ModelDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models
configparameters: rnn.cntk:modelPath=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/rnn.dnn
configparameters: rnn.cntk:ModelDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models
configparameters: rnn.cntk:modelPath=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/rnn.dnn
configparameters: rnn.cntk:numCPUThreads=1
configparameters: rnn.cntk:OutputDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu
configparameters: rnn.cntk:OutputDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu
configparameters: rnn.cntk:precision=float
configparameters: rnn.cntk:RootDir=..
configparameters: rnn.cntk:RunDir=/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu
configparameters: rnn.cntk:RunDir=/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu
configparameters: rnn.cntk:test=[
action = "eval"
minibatchSize = 8192
@ -716,8 +716,8 @@ minibatchSize = 8192
randomize = "none"
nbruttsineachrecurrentiter = 0
cacheBlockSize = 2000000
wordclass = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sequenceSentence.bin"
wordclass = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sequenceSentence.bin"
wsize = 256
wrecords = 1000
windowSize = "10000"
@ -729,7 +729,7 @@ cacheBlockSize = 2000000
labelIn = [
dim = 1
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.txt"
labelType = "Category"
beginSequence = "</s>"
endSequence = "</s>"
@ -751,7 +751,7 @@ cacheBlockSize = 2000000
beginSequence = "O"
endSequence = "O"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -818,8 +818,8 @@ lookupTableOrder = 1
randomize = "none"
nbruttsineachrecurrentiter = 0
cacheBlockSize = 2000000
wordclass = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sequenceSentence.bin"
wordclass = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sequenceSentence.bin"
wsize = 256
wrecords = 1000
windowSize = "10000"
@ -834,7 +834,7 @@ cacheBlockSize = 2000000
beginSequence = "</s>"
endSequence = "</s>"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -853,7 +853,7 @@ cacheBlockSize = 2000000
beginSequence = "O"
endSequence = "O"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -872,8 +872,8 @@ cacheBlockSize = 2000000
randomize = "none"
nbruttsineachrecurrentiter = 0
cacheBlockSize = 2000000
wordclass = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sequenceSentence.valid.bin"
wordclass = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sequenceSentence.valid.bin"
wsize = 256
wrecords = 1000
windowSize = "10000"
@ -885,7 +885,7 @@ cacheBlockSize = 2000000
labelIn = [
dim = 1
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
labelType = "Category"
beginSequence = "</s>"
endSequence = "</s>"
@ -907,7 +907,7 @@ cacheBlockSize = 2000000
beginSequence = "O"
endSequence = "O"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -927,7 +927,7 @@ configparameters: rnn.cntk:trainFile=ptb.train.txt
configparameters: rnn.cntk:validFile=ptb.valid.txt
configparameters: rnn.cntk:write=[
action = "write"
outputPath = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Write"
outputPath = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Write"
outputNodeNames = TrainNodeClassBasedCrossEntropy
format = [
sequencePrologue = "log P(W)="
@ -941,8 +941,8 @@ minibatchSize = 8192
randomize = "none"
nbruttsineachrecurrentiter = 1
cacheBlockSize = 1
wordclass = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sequenceSentence.bin"
wordclass = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
wfile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sequenceSentence.bin"
wsize = 256
wrecords = 1000
windowSize = "10000"
@ -954,7 +954,7 @@ cacheBlockSize = 1
labelIn = [
dim = 1
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.txt"
labelType = "Category"
beginSequence = "</s>"
endSequence = "</s>"
@ -976,7 +976,7 @@ cacheBlockSize = 1
beginSequence = "O"
endSequence = "O"
labelDim = "10000"
labelMappingFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt"
labelMappingFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt"
elementSize = 4
sectionType = "labels"
mapping = [
@ -997,32 +997,32 @@ configparameters: rnn.cntk:writeWordAndClassInfo=[
inputFile = "/home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data/ptb.train.txt"
beginSequence = "</s>"
endSequence = "</s>"
outputVocabFile = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt"
outputWord2Cls = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/word2cls.txt"
outputCls2Index = "/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/cls2idx.txt"
outputVocabFile = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt"
outputWord2Cls = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/word2cls.txt"
outputCls2Index = "/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/cls2idx.txt"
vocabSize = "10000"
nbrClass = "50"
cutoff = 0
printValues = true
]
05/05/2016 22:20:12: <<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<
05/05/2016 22:20:12: Commands: writeWordAndClassInfo train test
05/05/2016 22:20:12: Precision = "float"
05/05/2016 22:20:12: Using 1 CPU threads.
05/05/2016 22:20:12: CNTKModelPath: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/rnn.dnn
05/05/2016 22:20:12: CNTKCommandTrainInfo: train : 3
05/05/2016 22:20:12: CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : 3
06/16/2016 10:51:17: <<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<<
06/16/2016 10:51:17: Commands: writeWordAndClassInfo train test
06/16/2016 10:51:17: Precision = "float"
06/16/2016 10:51:17: Using 1 CPU threads.
06/16/2016 10:51:17: CNTKModelPath: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/rnn.dnn
06/16/2016 10:51:17: CNTKCommandTrainInfo: train : 3
06/16/2016 10:51:17: CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : 3
05/05/2016 22:20:12: ##############################################################################
05/05/2016 22:20:12: # #
05/05/2016 22:20:12: # Action "writeWordAndClass" #
05/05/2016 22:20:12: # #
05/05/2016 22:20:12: ##############################################################################
06/16/2016 10:51:17: ##############################################################################
06/16/2016 10:51:17: # #
06/16/2016 10:51:17: # Action "writeWordAndClass" #
06/16/2016 10:51:17: # #
06/16/2016 10:51:17: ##############################################################################
Vocabulary file --> /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/vocab.txt
Word-to-class map --> /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/word2cls.txt
Class-to-index map --> /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/cls2idx.txt
Vocabulary file --> /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/vocab.txt
Word-to-class map --> /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/word2cls.txt
Class-to-index map --> /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/cls2idx.txt
Reading input file inputFile: /home/philly/jenkins/workspace/CNTK-Test-Linux-W1/Examples/Text/PennTreebank/Data/ptb.train.txt
Vocabulary size 10000.
@ -1030,23 +1030,23 @@ Created vocabulary file with 10000 entries.
Created word-to-class map with 10000 entries.
Created class-to-index map with 50 entries.
05/05/2016 22:20:12: Action "writeWordAndClass" complete.
06/16/2016 10:51:17: Action "writeWordAndClass" complete.
05/05/2016 22:20:12: ##############################################################################
05/05/2016 22:20:12: # #
05/05/2016 22:20:12: # Action "train" #
05/05/2016 22:20:12: # #
05/05/2016 22:20:12: ##############################################################################
06/16/2016 10:51:17: ##############################################################################
06/16/2016 10:51:17: # #
06/16/2016 10:51:17: # Action "train" #
06/16/2016 10:51:17: # #
06/16/2016 10:51:17: ##############################################################################
05/05/2016 22:20:12: CNTKCommandTrainBegin: train
06/16/2016 10:51:17: CNTKCommandTrainBegin: train
SimpleNetworkBuilder Using CPU
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt
05/05/2016 22:20:13: Creating virgin network.
06/16/2016 10:51:17: Creating virgin network.
Post-processing network...
@ -1156,10 +1156,10 @@ Validating network, final pass.
Post-processing network complete.
05/05/2016 22:20:13: Created model with 63 nodes on CPU.
06/16/2016 10:51:17: Created model with 63 nodes on CPU.
05/05/2016 22:20:13: Training criterion node(s):
05/05/2016 22:20:13: TrainNodeClassBasedCrossEntropy = ClassBasedCrossEntropyWithSoftmax
06/16/2016 10:51:17: Training criterion node(s):
06/16/2016 10:51:17: TrainNodeClassBasedCrossEntropy = ClassBasedCrossEntropyWithSoftmax
Allocating matrices for forward and/or backward propagation.
@ -1167,187 +1167,187 @@ Allocating matrices for forward and/or backward propagation.
Memory Sharing Structure:
(nil): {[PosteriorProb Gradient[10000 x 1 x *]] [PosteriorProb Value[10000 x 1 x *]] [features Gradient[10000 x *]] [labels Gradient[4 x *]] [outputs Gradient[10000 x 1 x *]] }
0x2debf88: {[WXC0 Value[200 x 150]] }
0x2dec998: {[E0 Value[150 x 10000]] }
0x2dee658: {[features Value[10000 x *]] }
0x2def538: {[bf0 Value[200 x 1]] }
0x2defac8: {[WXF0 Value[200 x 150]] }
0x2df40f8: {[WCI0 Value[200 x 1]] }
0x2e08d38: {[WHI0 Value[200 x 200]] }
0x2e09858: {[WXI0 Value[200 x 150]] }
0x2e09bb8: {[WXO0 Value[200 x 150]] }
0x341fbe8: {[bo0 Value[200 x 1]] }
0x341ffd8: {[bi0 Value[200 x 1]] }
0x3420708: {[bc0 Value[200 x 1]] }
0x3420888: {[WHF0 Value[200 x 200]] }
0x3a2b938: {[WCF0 Value[200 x 1]] }
0x3a2c108: {[WHO0 Value[200 x 200]] }
0x3a2c5a8: {[WCO0 Value[200 x 1]] }
0x3a2cd78: {[WHC0 Value[200 x 200]] }
0x3a2d418: {[AutoName0 Value[200 x 1 x *]] }
0x3a2db78: {[AutoName1 Value[200 x 1 x *]] }
0x3a2e1c8: {[AutoName2 Value[200 x 1 x *]] [WXC0 Gradient[200 x 150]] }
0x3a2e818: {[AutoName3 Value[200 x 1 x *]] }
0x3a2ef78: {[AutoName4 Value[200 x 1 x *]] }
0x3a2f5c8: {[AutoName5 Value[200 x 1 x *]] }
0x3a2fc18: {[AutoName6 Value[200 x 1 x *]] }
0x3ac85b8: {[W2 Value[200 x 10000]] }
0x3ac8a38: {[labels Value[4 x *]] }
0x3ac8f18: {[WeightForClassPostProb Value[50 x 200]] }
0x3ad5c78: {[outputs Value[10000 x 1 x *]] }
0x3ad7398: {[TrainNodeClassBasedCrossEntropy Value[1]] }
0x3aded18: {[LookupTable Value[150 x *]] }
0x3adf0b8: {[AutoName31 Value[200 x *]] [E0 Gradient[150 x 10000]] }
0x3adf928: {[AutoName22 Value[200 x *]] [AutoName31 Gradient[200 x *]] }
0x3adfae8: {[AutoName23 Value[200 x 1 x *]] [WXF0 Gradient[200 x 150]] }
0x3adfbe8: {[AutoName22 Gradient[200 x *]] [AutoName9 Value[200 x *]] [bo0 Gradient[200 x 1]] }
0x3ae01c8: {[AutoName32 Value[200 x 1 x *]] [WXO0 Gradient[200 x 150]] }
0x3ae09c8: {[AutoName10 Value[200 x 1 x *]] [WXI0 Gradient[200 x 150]] }
0x3ae0b88: {[AutoName16 Value[200 x *]] [AutoName9 Gradient[200 x *]] [bf0 Gradient[200 x 1]] }
0x3ae0d48: {[AutoName30 Value[200 x 1 x *]] }
0x3ae0f08: {[AutoName33 Value[200 x 1 x *]] }
0x3ae10c8: {[AutoName21 Value[200 x 1 x *]] }
0x3ae1288: {[AutoName24 Value[200 x 1 x *]] }
0x3ae1448: {[AutoName20 Value[200 x 1 x *]] }
0x3ae1608: {[AutoName25 Value[200 x 1 x *]] }
0x3ae17c8: {[AutoName26 Value[200 x 1 x *]] }
0x3ae1988: {[AutoName27 Value[200 x 1 x *]] }
0x3ae1b48: {[AutoName8 Value[200 x 1 x *]] }
0x3ae1d08: {[AutoName11 Value[200 x 1 x *]] }
0x3ae1ec8: {[AutoName7 Value[200 x 1 x *]] }
0x3ae2088: {[AutoName12 Value[200 x 1 x *]] }
0x3ae2248: {[AutoName13 Value[200 x 1 x *]] }
0x3ae2408: {[AutoName14 Value[200 x 1 x *]] }
0x3ae25c8: {[AutoName15 Value[200 x 1 x *]] }
0x3ae2788: {[AutoName17 Value[200 x 1 x *]] }
0x3ae2948: {[AutoName18 Value[200 x 1 x *]] }
0x3ae2b08: {[AutoName19 Value[200 x 1 x *]] }
0x3ae2cc8: {[AutoName28 Value[200 x 1 x *]] }
0x3ae2e88: {[AutoName29 Value[200 x 1 x *]] }
0x3ae3048: {[AutoName34 Value[200 x 1 x *]] }
0x3ae3208: {[AutoName35 Value[200 x 1 x *]] }
0x3ae33c8: {[AutoName36 Value[200 x 1 x *]] }
0x3ae3588: {[AutoName37 Value[200 x 1 x *]] }
0x3ae3748: {[AutoName35 Gradient[200 x 1 x *]] [ClassPostProb Value[50 x 1 x *]] }
0x3ae3908: {[TrainNodeClassBasedCrossEntropy Gradient[1]] }
0x3ae3ac8: {[AutoName37 Gradient[200 x 1 x *]] }
0x3ae3c88: {[W2 Gradient[200 x 10000]] }
0x3ae3e48: {[AutoName36 Gradient[200 x 1 x *]] [ClassPostProb Gradient[50 x 1 x *]] }
0x3ae4008: {[WeightForClassPostProb Gradient[50 x 200]] }
0x3ae41c8: {[AutoName28 Gradient[200 x 1 x *]] }
0x3ae4388: {[AutoName34 Gradient[200 x 1 x *]] }
0x3ae4548: {[AutoName33 Gradient[200 x 1 x *]] }
0x3ae4708: {[AutoName29 Gradient[200 x 1 x *]] }
0x3ae4c48: {[WCO0 Gradient[200 x 1]] }
0x3ae4e08: {[AutoName27 Gradient[200 x 1 x *]] }
0x3ae4fc8: {[AutoName19 Gradient[200 x 1 x *]] }
0x3ae5188: {[AutoName13 Gradient[200 x 1 x *]] }
0x3ae5348: {[AutoName18 Gradient[200 x 1 x *]] }
0x3ae5508: {[AutoName17 Gradient[200 x 1 x *]] }
0x3ae56c8: {[AutoName16 Gradient[200 x *]] [bi0 Gradient[200 x 1]] }
0x3ae5888: {[AutoName15 Gradient[200 x 1 x *]] }
0x3ae5a48: {[AutoName14 Gradient[200 x 1 x *]] }
0x3ae5c08: {[bc0 Gradient[200 x 1]] }
0x3ae5dc8: {[WHC0 Gradient[200 x 200]] }
0x3ae5f88: {[AutoName3 Gradient[200 x 1 x *]] }
0x3ae6148: {[AutoName12 Gradient[200 x 1 x *]] }
0x3ae6308: {[AutoName11 Gradient[200 x 1 x *]] }
0x3ae64c8: {[AutoName7 Gradient[200 x 1 x *]] }
0x3ae6a08: {[WCI0 Gradient[200 x 1]] }
0x3ae6bc8: {[AutoName4 Gradient[200 x 1 x *]] }
0x3ae6d88: {[AutoName10 Gradient[200 x 1 x *]] }
0x3ae6f48: {[AutoName8 Gradient[200 x 1 x *]] }
0x3ae7108: {[WHI0 Gradient[200 x 200]] }
0x3ae72c8: {[AutoName0 Gradient[200 x 1 x *]] }
0x3ae7488: {[AutoName26 Gradient[200 x 1 x *]] }
0x3ae7648: {[AutoName6 Gradient[200 x 1 x *]] }
0x3ae7808: {[AutoName25 Gradient[200 x 1 x *]] }
0x3ae79c8: {[AutoName24 Gradient[200 x 1 x *]] }
0x3ae7b88: {[AutoName20 Gradient[200 x 1 x *]] }
0x3ae80c8: {[WCF0 Gradient[200 x 1]] }
0x3ae8288: {[AutoName5 Gradient[200 x 1 x *]] }
0x3ae8448: {[AutoName23 Gradient[200 x 1 x *]] }
0x3ae8608: {[AutoName21 Gradient[200 x 1 x *]] }
0x3ae87c8: {[WHF0 Gradient[200 x 200]] }
0x3ae8988: {[AutoName1 Gradient[200 x 1 x *]] }
0x3ae8b48: {[AutoName32 Gradient[200 x 1 x *]] }
0x3ae8d08: {[AutoName30 Gradient[200 x 1 x *]] }
0x3ae8ec8: {[WHO0 Gradient[200 x 200]] }
0x3ae9088: {[AutoName2 Gradient[200 x 1 x *]] [LookupTable Gradient[150 x *]] }
0x2198e68: {[E0 Value[150 x 10000]] }
0x219a888: {[WHI0 Value[200 x 200]] }
0x219c018: {[WXC0 Value[200 x 150]] }
0x219dd48: {[bc0 Value[200 x 1]] }
0x219e0c8: {[features Value[10000 x *]] }
0x21b14a8: {[WXO0 Value[200 x 150]] }
0x21b1fb8: {[bo0 Value[200 x 1]] }
0x27c7b88: {[bi0 Value[200 x 1]] }
0x27c8188: {[WXF0 Value[200 x 150]] }
0x27c8688: {[WXI0 Value[200 x 150]] }
0x27c8798: {[bf0 Value[200 x 1]] }
0x2dd2d88: {[WCI0 Value[200 x 1]] }
0x2dd34d8: {[WHF0 Value[200 x 200]] }
0x2dd3978: {[WCF0 Value[200 x 1]] }
0x2dd4148: {[WHO0 Value[200 x 200]] }
0x2dd45e8: {[WCO0 Value[200 x 1]] }
0x2dd4db8: {[WHC0 Value[200 x 200]] }
0x2dd5458: {[AutoName0 Value[200 x 1 x *]] }
0x2dd5bb8: {[AutoName1 Value[200 x 1 x *]] }
0x2dd6208: {[AutoName2 Value[200 x 1 x *]] [WXC0 Gradient[200 x 150]] }
0x2dd6858: {[AutoName3 Value[200 x 1 x *]] }
0x2dd6fb8: {[AutoName4 Value[200 x 1 x *]] }
0x2dd7608: {[AutoName5 Value[200 x 1 x *]] }
0x2dd7c58: {[AutoName6 Value[200 x 1 x *]] }
0x2e70588: {[W2 Value[200 x 10000]] }
0x2e70a08: {[labels Value[4 x *]] }
0x2e70ee8: {[WeightForClassPostProb Value[50 x 200]] }
0x2e7da38: {[outputs Value[10000 x 1 x *]] }
0x2e7f988: {[TrainNodeClassBasedCrossEntropy Value[1]] }
0x2e86e38: {[LookupTable Value[150 x *]] }
0x2e871d8: {[AutoName31 Value[200 x *]] [E0 Gradient[150 x 10000]] }
0x2e87a48: {[AutoName22 Value[200 x *]] [AutoName31 Gradient[200 x *]] }
0x2e87c08: {[AutoName23 Value[200 x 1 x *]] [WXF0 Gradient[200 x 150]] }
0x2e87d08: {[AutoName22 Gradient[200 x *]] [AutoName9 Value[200 x *]] [bo0 Gradient[200 x 1]] }
0x2e882e8: {[AutoName32 Value[200 x 1 x *]] [WXO0 Gradient[200 x 150]] }
0x2e88a88: {[AutoName10 Value[200 x 1 x *]] [WXI0 Gradient[200 x 150]] }
0x2e88c48: {[AutoName16 Value[200 x *]] [AutoName9 Gradient[200 x *]] [bf0 Gradient[200 x 1]] }
0x2e88e08: {[AutoName30 Value[200 x 1 x *]] }
0x2e88fc8: {[AutoName33 Value[200 x 1 x *]] }
0x2e89188: {[AutoName21 Value[200 x 1 x *]] }
0x2e89348: {[AutoName24 Value[200 x 1 x *]] }
0x2e89508: {[AutoName20 Value[200 x 1 x *]] }
0x2e896c8: {[AutoName25 Value[200 x 1 x *]] }
0x2e89888: {[AutoName26 Value[200 x 1 x *]] }
0x2e89a48: {[AutoName27 Value[200 x 1 x *]] }
0x2e89c08: {[AutoName8 Value[200 x 1 x *]] }
0x2e89dc8: {[AutoName11 Value[200 x 1 x *]] }
0x2e89f88: {[AutoName7 Value[200 x 1 x *]] }
0x2e8a148: {[AutoName12 Value[200 x 1 x *]] }
0x2e8a308: {[AutoName13 Value[200 x 1 x *]] }
0x2e8a4c8: {[AutoName14 Value[200 x 1 x *]] }
0x2e8a688: {[AutoName15 Value[200 x 1 x *]] }
0x2e8a848: {[AutoName17 Value[200 x 1 x *]] }
0x2e8aa08: {[AutoName18 Value[200 x 1 x *]] }
0x2e8abc8: {[AutoName19 Value[200 x 1 x *]] }
0x2e8ad88: {[AutoName28 Value[200 x 1 x *]] }
0x2e8af48: {[AutoName29 Value[200 x 1 x *]] }
0x2e8b108: {[AutoName34 Value[200 x 1 x *]] }
0x2e8b2c8: {[AutoName35 Value[200 x 1 x *]] }
0x2e8b488: {[AutoName36 Value[200 x 1 x *]] }
0x2e8b648: {[AutoName37 Value[200 x 1 x *]] }
0x2e8b808: {[AutoName35 Gradient[200 x 1 x *]] [ClassPostProb Value[50 x 1 x *]] }
0x2e8b9c8: {[TrainNodeClassBasedCrossEntropy Gradient[1]] }
0x2e8bb88: {[AutoName37 Gradient[200 x 1 x *]] }
0x2e8bd48: {[W2 Gradient[200 x 10000]] }
0x2e8bf08: {[AutoName36 Gradient[200 x 1 x *]] [ClassPostProb Gradient[50 x 1 x *]] }
0x2e8c0c8: {[WeightForClassPostProb Gradient[50 x 200]] }
0x2e8c288: {[AutoName28 Gradient[200 x 1 x *]] }
0x2e8c448: {[AutoName34 Gradient[200 x 1 x *]] }
0x2e8c608: {[AutoName33 Gradient[200 x 1 x *]] }
0x2e8c7c8: {[AutoName29 Gradient[200 x 1 x *]] }
0x2e8cd08: {[WCO0 Gradient[200 x 1]] }
0x2e8cec8: {[AutoName27 Gradient[200 x 1 x *]] }
0x2e8d088: {[AutoName19 Gradient[200 x 1 x *]] }
0x2e8d248: {[AutoName13 Gradient[200 x 1 x *]] }
0x2e8d408: {[AutoName18 Gradient[200 x 1 x *]] }
0x2e8d5c8: {[AutoName17 Gradient[200 x 1 x *]] }
0x2e8d788: {[AutoName16 Gradient[200 x *]] [bi0 Gradient[200 x 1]] }
0x2e8d948: {[AutoName15 Gradient[200 x 1 x *]] }
0x2e8db08: {[AutoName14 Gradient[200 x 1 x *]] }
0x2e8dcc8: {[bc0 Gradient[200 x 1]] }
0x2e8de88: {[WHC0 Gradient[200 x 200]] }
0x2e8e048: {[AutoName3 Gradient[200 x 1 x *]] }
0x2e8e208: {[AutoName12 Gradient[200 x 1 x *]] }
0x2e8e3c8: {[AutoName11 Gradient[200 x 1 x *]] }
0x2e8e588: {[AutoName7 Gradient[200 x 1 x *]] }
0x2e8eac8: {[WCI0 Gradient[200 x 1]] }
0x2e8ec88: {[AutoName4 Gradient[200 x 1 x *]] }
0x2e8ee48: {[AutoName10 Gradient[200 x 1 x *]] }
0x2e8f008: {[AutoName8 Gradient[200 x 1 x *]] }
0x2e8f1c8: {[WHI0 Gradient[200 x 200]] }
0x2e8f388: {[AutoName0 Gradient[200 x 1 x *]] }
0x2e8f548: {[AutoName26 Gradient[200 x 1 x *]] }
0x2e8f708: {[AutoName6 Gradient[200 x 1 x *]] }
0x2e8f8c8: {[AutoName25 Gradient[200 x 1 x *]] }
0x2e8fa88: {[AutoName24 Gradient[200 x 1 x *]] }
0x2e8fc48: {[AutoName20 Gradient[200 x 1 x *]] }
0x2e90188: {[WCF0 Gradient[200 x 1]] }
0x2e90348: {[AutoName5 Gradient[200 x 1 x *]] }
0x2e90508: {[AutoName23 Gradient[200 x 1 x *]] }
0x2e906c8: {[AutoName21 Gradient[200 x 1 x *]] }
0x2e90888: {[WHF0 Gradient[200 x 200]] }
0x2e90a48: {[AutoName1 Gradient[200 x 1 x *]] }
0x2e90c08: {[AutoName32 Gradient[200 x 1 x *]] }
0x2e90dc8: {[AutoName30 Gradient[200 x 1 x *]] }
0x2e90f88: {[WHO0 Gradient[200 x 200]] }
0x2e91148: {[AutoName2 Gradient[200 x 1 x *]] [LookupTable Gradient[150 x *]] }
05/05/2016 22:20:13: No PreCompute nodes found, skipping PreCompute step.
06/16/2016 10:51:17: No PreCompute nodes found, skipping PreCompute step.
05/05/2016 22:20:14: Starting Epoch 1: learning rate per sample = 0.100000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
06/16/2016 10:51:17: Starting Epoch 1: learning rate per sample = 0.100000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
05/05/2016 22:20:14: Starting minibatch loop.
06/16/2016 10:51:17: Starting minibatch loop.
LMSequenceReader: Reading epoch data... 42068 sequences read.
05/05/2016 22:20:27: Finished Epoch[ 1 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 6.83413224 * 2087; totalSamplesSeen = 2087; learningRatePerSample = 0.1; epochTime=13.6927s
06/16/2016 10:51:18: Finished Epoch[ 1 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 6.83413224 * 2087; totalSamplesSeen = 2087; learningRatePerSample = 0.1; epochTime=0.806587s
LMSequenceReader: Reading epoch data... 3370 sequences read.
LMSequenceReader: Reading epoch data... 0 sequences read.
05/05/2016 22:21:20: Final Results: Minibatch[1-704]: TrainNodeClassBasedCrossEntropy = 6.52944688 * 73760; perplexity = 685.01920624
05/05/2016 22:21:20: Finished Epoch[ 1 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 6.52944688 * 73760
05/05/2016 22:21:20: SGD: Saving checkpoint model '/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/rnn.dnn.1'
06/16/2016 10:51:25: Final Results: Minibatch[1-704]: TrainNodeClassBasedCrossEntropy = 6.52944686 * 73760; perplexity = 685.01919179
06/16/2016 10:51:25: Finished Epoch[ 1 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 6.52944686 * 73760
06/16/2016 10:51:25: SGD: Saving checkpoint model '/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/rnn.dnn.1'
05/05/2016 22:21:21: Starting Epoch 2: learning rate per sample = 0.100000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
06/16/2016 10:51:25: Starting Epoch 2: learning rate per sample = 0.100000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
05/05/2016 22:21:21: Starting minibatch loop.
06/16/2016 10:51:25: Starting minibatch loop.
LMSequenceReader: Reading epoch data... 42068 sequences read.
05/05/2016 22:21:29: Finished Epoch[ 2 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 6.57536885 * 2099; totalSamplesSeen = 4186; learningRatePerSample = 0.1; epochTime=7.99991s
06/16/2016 10:51:26: Finished Epoch[ 2 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 6.57536839 * 2099; totalSamplesSeen = 4186; learningRatePerSample = 0.1; epochTime=0.669527s
LMSequenceReader: Reading epoch data... 3370 sequences read.
LMSequenceReader: Reading epoch data... 0 sequences read.
05/05/2016 22:22:14: Final Results: Minibatch[1-353]: TrainNodeClassBasedCrossEntropy = 6.45979633 * 73760; perplexity = 638.93091153
05/05/2016 22:22:14: Finished Epoch[ 2 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 6.45979633 * 73760
05/05/2016 22:22:14: SGD: Saving checkpoint model '/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/rnn.dnn.2'
06/16/2016 10:51:32: Final Results: Minibatch[1-353]: TrainNodeClassBasedCrossEntropy = 6.45979661 * 73760; perplexity = 638.93109334
06/16/2016 10:51:32: Finished Epoch[ 2 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 6.45979661 * 73760
06/16/2016 10:51:32: SGD: Saving checkpoint model '/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/rnn.dnn.2'
05/05/2016 22:22:14: Starting Epoch 3: learning rate per sample = 0.100000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
06/16/2016 10:51:32: Starting Epoch 3: learning rate per sample = 0.100000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
05/05/2016 22:22:14: Starting minibatch loop.
06/16/2016 10:51:32: Starting minibatch loop.
LMSequenceReader: Reading epoch data... 42068 sequences read.
05/05/2016 22:22:20: Finished Epoch[ 3 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 7.26376970 * 2276; totalSamplesSeen = 6462; learningRatePerSample = 0.1; epochTime=5.82946s
06/16/2016 10:51:33: Finished Epoch[ 3 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 7.26366844 * 2276; totalSamplesSeen = 6462; learningRatePerSample = 0.1; epochTime=0.596137s
LMSequenceReader: Reading epoch data... 3370 sequences read.
LMSequenceReader: Reading epoch data... 0 sequences read.
05/05/2016 22:23:02: Final Results: Minibatch[1-193]: TrainNodeClassBasedCrossEntropy = 9.55545905 * 73760; perplexity = 14121.57503880
05/05/2016 22:23:02: Finished Epoch[ 3 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 9.55545905 * 73760
05/05/2016 22:23:02: Loading (rolling back to) previous model with best training-criterion value: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/rnn.dnn.2.
05/05/2016 22:23:02: learnRatePerSample reduced to 0.050000001
05/05/2016 22:23:02: SGD: revoke back to and update checkpoint file for epoch 2
06/16/2016 10:51:39: Final Results: Minibatch[1-193]: TrainNodeClassBasedCrossEntropy = 9.55520160 * 73760; perplexity = 14117.93994423
06/16/2016 10:51:39: Finished Epoch[ 3 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 9.55520160 * 73760
06/16/2016 10:51:39: Loading (rolling back to) previous model with best training-criterion value: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/rnn.dnn.2.
06/16/2016 10:51:39: learnRatePerSample reduced to 0.050000001
06/16/2016 10:51:39: SGD: revoke back to and update checkpoint file for epoch 2
05/05/2016 22:23:03: Starting Epoch 3: learning rate per sample = 0.050000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
06/16/2016 10:51:39: Starting Epoch 3: learning rate per sample = 0.050000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
05/05/2016 22:23:03: Starting minibatch loop.
06/16/2016 10:51:39: Starting minibatch loop.
LMSequenceReader: Reading epoch data... 42068 sequences read.
05/05/2016 22:23:08: Finished Epoch[ 3 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 6.43178308 * 2276; totalSamplesSeen = 6462; learningRatePerSample = 0.050000001; epochTime=5.80466s
06/16/2016 10:51:40: Finished Epoch[ 3 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 6.43178908 * 2276; totalSamplesSeen = 6462; learningRatePerSample = 0.050000001; epochTime=0.585515s
LMSequenceReader: Reading epoch data... 3370 sequences read.
LMSequenceReader: Reading epoch data... 0 sequences read.
05/05/2016 22:23:50: Final Results: Minibatch[1-193]: TrainNodeClassBasedCrossEntropy = 6.67105423 * 73760; perplexity = 789.22719648
05/05/2016 22:23:50: Finished Epoch[ 3 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 6.67105423 * 73760
05/05/2016 22:23:50: Loading (rolling back to) previous model with best training-criterion value: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/rnn.dnn.2.
05/05/2016 22:23:50: learnRatePerSample reduced to 0.025
05/05/2016 22:23:50: SGD: revoke back to and update checkpoint file for epoch 2
06/16/2016 10:51:46: Final Results: Minibatch[1-193]: TrainNodeClassBasedCrossEntropy = 6.67106400 * 73760; perplexity = 789.23490530
06/16/2016 10:51:46: Finished Epoch[ 3 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 6.67106400 * 73760
06/16/2016 10:51:46: Loading (rolling back to) previous model with best training-criterion value: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/rnn.dnn.2.
06/16/2016 10:51:46: learnRatePerSample reduced to 0.025
06/16/2016 10:51:46: SGD: revoke back to and update checkpoint file for epoch 2
05/05/2016 22:23:51: Starting Epoch 3: learning rate per sample = 0.025000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
06/16/2016 10:51:46: Starting Epoch 3: learning rate per sample = 0.025000 effective momentum = 0.000000 momentum as time constant = 0.0 samples
05/05/2016 22:23:51: Starting minibatch loop.
06/16/2016 10:51:46: Starting minibatch loop.
LMSequenceReader: Reading epoch data... 42068 sequences read.
05/05/2016 22:23:56: Finished Epoch[ 3 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 6.24460230 * 2276; totalSamplesSeen = 6462; learningRatePerSample = 0.025; epochTime=5.80792s
06/16/2016 10:51:47: Finished Epoch[ 3 of 3]: [Training] TrainNodeClassBasedCrossEntropy = 6.24460187 * 2276; totalSamplesSeen = 6462; learningRatePerSample = 0.025; epochTime=0.572276s
LMSequenceReader: Reading epoch data... 3370 sequences read.
LMSequenceReader: Reading epoch data... 0 sequences read.
05/05/2016 22:24:38: Final Results: Minibatch[1-193]: TrainNodeClassBasedCrossEntropy = 6.42579765 * 73760; perplexity = 617.57322733
05/05/2016 22:24:38: Finished Epoch[ 3 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 6.42579765 * 73760
05/05/2016 22:24:38: SGD: Saving checkpoint model '/tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/Models/rnn.dnn'
05/05/2016 22:24:38: CNTKCommandTrainEnd: train
06/16/2016 10:51:53: Final Results: Minibatch[1-193]: TrainNodeClassBasedCrossEntropy = 6.42579772 * 73760; perplexity = 617.57327345
06/16/2016 10:51:53: Finished Epoch[ 3 of 3]: [Validate] TrainNodeClassBasedCrossEntropy = 6.42579772 * 73760
06/16/2016 10:51:53: SGD: Saving checkpoint model '/tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/Models/rnn.dnn'
06/16/2016 10:51:53: CNTKCommandTrainEnd: train
05/05/2016 22:24:38: Action "train" complete.
06/16/2016 10:51:53: Action "train" complete.
05/05/2016 22:24:38: ##############################################################################
05/05/2016 22:24:38: # #
05/05/2016 22:24:38: # Action "eval" #
05/05/2016 22:24:38: # #
05/05/2016 22:24:38: ##############################################################################
06/16/2016 10:51:53: ##############################################################################
06/16/2016 10:51:53: # #
06/16/2016 10:51:53: # Action "eval" #
06/16/2016 10:51:53: # #
06/16/2016 10:51:53: ##############################################################################
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160505220136.870944/Examples/Text/PennTreebank_RNN@debug_cpu/sentenceLabels.out.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.txt
LMSequenceReader: Label mapping will be created internally on the fly because the labelMappingFile was not found: /tmp/cntk-test-20160616105116.484664/Examples/Text/PennTreebank_RNN@release_cpu/sentenceLabels.out.txt
Post-processing network...
@ -1465,72 +1465,73 @@ Allocating matrices for forward and/or backward propagation.
Memory Sharing Structure:
(nil): {[AutoName0 Gradient[200 x 1 x *1]] [AutoName1 Gradient[200 x 1 x *1]] [AutoName10 Gradient[200 x 1 x *1]] [AutoName11 Gradient[200 x 1 x *1]] [AutoName12 Gradient[200 x 1 x *1]] [AutoName13 Gradient[200 x 1 x *1]] [AutoName14 Gradient[200 x 1 x *1]] [AutoName15 Gradient[200 x 1 x *1]] [AutoName16 Gradient[200 x *1]] [AutoName17 Gradient[200 x 1 x *1]] [AutoName18 Gradient[200 x 1 x *1]] [AutoName19 Gradient[200 x 1 x *1]] [AutoName2 Gradient[200 x 1 x *1]] [AutoName20 Gradient[200 x 1 x *1]] [AutoName21 Gradient[200 x 1 x *1]] [AutoName22 Gradient[200 x *1]] [AutoName23 Gradient[200 x 1 x *1]] [AutoName24 Gradient[200 x 1 x *1]] [AutoName25 Gradient[200 x 1 x *1]] [AutoName26 Gradient[200 x 1 x *1]] [AutoName27 Gradient[200 x 1 x *1]] [AutoName28 Gradient[200 x 1 x *1]] [AutoName29 Gradient[200 x 1 x *1]] [AutoName3 Gradient[200 x 1 x *1]] [AutoName30 Gradient[200 x 1 x *1]] [AutoName31 Gradient[200 x *1]] [AutoName32 Gradient[200 x 1 x *1]] [AutoName33 Gradient[200 x 1 x *1]] [AutoName34 Gradient[200 x 1 x *1]] [AutoName35 Gradient[200 x 1 x *1]] [AutoName36 Gradient[200 x 1 x *1]] [AutoName37 Gradient[200 x 1 x *1]] [AutoName4 Gradient[200 x 1 x *1]] [AutoName5 Gradient[200 x 1 x *1]] [AutoName6 Gradient[200 x 1 x *1]] [AutoName7 Gradient[200 x 1 x *1]] [AutoName8 Gradient[200 x 1 x *1]] [AutoName9 Gradient[200 x *1]] [ClassPostProb Gradient[50 x 1 x *1]] [E0 Gradient[150 x 10000]] [LookupTable Gradient[150 x *1]] [PosteriorProb Gradient[10000 x 1 x *1]] [PosteriorProb Value[10000 x 1 x *1]] [TrainNodeClassBasedCrossEntropy Gradient[1]] [W2 Gradient[200 x 10000]] [WCF0 Gradient[200 x 1]] [WCI0 Gradient[200 x 1]] [WCO0 Gradient[200 x 1]] [WHC0 Gradient[200 x 200]] [WHF0 Gradient[200 x 200]] [WHI0 Gradient[200 x 200]] [WHO0 Gradient[200 x 200]] [WXC0 Gradient[200 x 150]] [WXF0 Gradient[200 x 150]] [WXI0 Gradient[200 x 150]] [WXO0 Gradient[200 x 150]] [WeightForClassPostProb Gradient[50 x 200]] [bc0 Gradient[200 x 1]] [bf0 Gradient[200 x 1]] [bi0 Gradient[200 x 1]] [bo0 Gradient[200 x 1]] [features Gradient[10000 x *1]] [labels Gradient[4 x *1]] [outputs Gradient[10000 x 1 x *1]] [outputs Value[10000 x 1 x *1]] }
0x2df51a8: {[AutoName1 Value[200 x 1 x *1]] }
0x47c30d8: {[features Value[10000 x *1]] }
0x47d6e78: {[labels Value[4 x *1]] }
0x47d88e8: {[W2 Value[200 x 10000]] }
0x47d8d08: {[WCF0 Value[200 x 1]] }
0x47d9528: {[WCI0 Value[200 x 1]] }
0x47d96e8: {[WeightForClassPostProb Value[50 x 200]] }
0x47d9d48: {[WCO0 Value[200 x 1]] }
0x47d9f08: {[WHC0 Value[200 x 200]] }
0x47db238: {[TrainNodeClassBasedCrossEntropy Value[1]] }
0x47ede88: {[LookupTable Value[150 x *1]] }
0x47ee228: {[AutoName31 Value[200 x *1]] }
0x47eea98: {[AutoName22 Value[200 x *1]] }
0x47eec58: {[AutoName23 Value[200 x 1 x *1]] }
0x47ef2e8: {[AutoName32 Value[200 x 1 x *1]] }
0x47ef9c8: {[AutoName9 Value[200 x *1]] }
0x47efb88: {[AutoName10 Value[200 x 1 x *1]] }
0x47efd48: {[AutoName16 Value[200 x *1]] }
0x47eff08: {[AutoName30 Value[200 x 1 x *1]] }
0x47f00c8: {[AutoName33 Value[200 x 1 x *1]] }
0x47f0288: {[AutoName21 Value[200 x 1 x *1]] }
0x47f0448: {[AutoName24 Value[200 x 1 x *1]] }
0x47f0608: {[AutoName20 Value[200 x 1 x *1]] }
0x47f07c8: {[AutoName25 Value[200 x 1 x *1]] }
0x47f0988: {[AutoName26 Value[200 x 1 x *1]] }
0x47f0b48: {[AutoName27 Value[200 x 1 x *1]] }
0x47f0d08: {[AutoName8 Value[200 x 1 x *1]] }
0x47f0ec8: {[AutoName11 Value[200 x 1 x *1]] }
0x47f1088: {[AutoName7 Value[200 x 1 x *1]] }
0x47f1248: {[AutoName12 Value[200 x 1 x *1]] }
0x47f1408: {[AutoName13 Value[200 x 1 x *1]] }
0x47f15c8: {[AutoName14 Value[200 x 1 x *1]] }
0x47f1788: {[AutoName15 Value[200 x 1 x *1]] }
0x47f1948: {[AutoName17 Value[200 x 1 x *1]] }
0x47f1b08: {[AutoName18 Value[200 x 1 x *1]] }
0x47f1cc8: {[AutoName19 Value[200 x 1 x *1]] }
0x47f1e88: {[AutoName28 Value[200 x 1 x *1]] }
0x47f2048: {[AutoName29 Value[200 x 1 x *1]] }
0x47f2208: {[AutoName34 Value[200 x 1 x *1]] }
0x47f23c8: {[AutoName35 Value[200 x 1 x *1]] }
0x47f2588: {[AutoName36 Value[200 x 1 x *1]] }
0x47f2748: {[AutoName37 Value[200 x 1 x *1]] }
0x47f2908: {[ClassPostProb Value[50 x 1 x *1]] }
0x484c2a8: {[WHF0 Value[200 x 200]] }
0x484c778: {[WHI0 Value[200 x 200]] }
0x484cc48: {[WHO0 Value[200 x 200]] }
0x484d118: {[WXC0 Value[200 x 150]] }
0x484d5e8: {[WXF0 Value[200 x 150]] }
0x484dab8: {[WXI0 Value[200 x 150]] }
0x484df88: {[WXO0 Value[200 x 150]] }
0x48fef08: {[AutoName2 Value[200 x 1 x *1]] }
0x49015a8: {[AutoName3 Value[200 x 1 x *1]] }
0x4903578: {[AutoName4 Value[200 x 1 x *1]] }
0x4903c48: {[AutoName5 Value[200 x 1 x *1]] }
0x4904318: {[AutoName6 Value[200 x 1 x *1]] }
0x4905238: {[bc0 Value[200 x 1]] }
0x4905698: {[bf0 Value[200 x 1]] }
0x4905e88: {[bi0 Value[200 x 1]] }
0x4906678: {[bo0 Value[200 x 1]] }
0x4906bc8: {[E0 Value[150 x 10000]] }
0x6d663d8: {[AutoName0 Value[200 x 1 x *1]] }
0x4d01f78: {[AutoName1 Value[200 x 1 x *1]] }
0x4d04a98: {[AutoName2 Value[200 x 1 x *1]] }
0x4d07138: {[AutoName3 Value[200 x 1 x *1]] }
0x4d09108: {[AutoName4 Value[200 x 1 x *1]] }
0x4d097d8: {[AutoName5 Value[200 x 1 x *1]] }
0x4d09ea8: {[AutoName6 Value[200 x 1 x *1]] }
0x4d0adc8: {[bc0 Value[200 x 1]] }
0x4d0b228: {[bf0 Value[200 x 1]] }
0x4d0ba18: {[bi0 Value[200 x 1]] }
0x4d0c208: {[bo0 Value[200 x 1]] }
0x4d0c758: {[E0 Value[150 x 10000]] }
0x4da9778: {[AutoName7 Value[200 x 1 x *1]] }
0x4da9938: {[AutoName12 Value[200 x 1 x *1]] }
0x4da9af8: {[AutoName13 Value[200 x 1 x *1]] }
0x4da9cb8: {[AutoName14 Value[200 x 1 x *1]] }
0x4da9e78: {[AutoName15 Value[200 x 1 x *1]] }
0x4daa038: {[AutoName17 Value[200 x 1 x *1]] }
0x4daa1f8: {[AutoName18 Value[200 x 1 x *1]] }
0x4daa3b8: {[AutoName19 Value[200 x 1 x *1]] }
0x4daa578: {[AutoName28 Value[200 x 1 x *1]] }
0x4daa738: {[AutoName29 Value[200 x 1 x *1]] }
0x4daa8f8: {[AutoName34 Value[200 x 1 x *1]] }
0x4daaab8: {[AutoName35 Value[200 x 1 x *1]] }
0x4daac78: {[AutoName36 Value[200 x 1 x *1]] }
0x4daae38: {[AutoName37 Value[200 x 1 x *1]] }
0x4daaff8: {[ClassPostProb Value[50 x 1 x *1]] }
0x55a6348: {[features Value[10000 x *1]] }
0x55ba0e8: {[labels Value[4 x *1]] }
0x55bbb58: {[W2 Value[200 x 10000]] }
0x55bbf78: {[WCF0 Value[200 x 1]] }
0x55bc798: {[WCI0 Value[200 x 1]] }
0x55bc958: {[WeightForClassPostProb Value[50 x 200]] }
0x55bcfb8: {[WCO0 Value[200 x 1]] }
0x55bd178: {[WHC0 Value[200 x 200]] }
0x55be1b8: {[WHF0 Value[200 x 200]] }
0x55be688: {[WHI0 Value[200 x 200]] }
0x55beb58: {[WHO0 Value[200 x 200]] }
0x55bf028: {[WXC0 Value[200 x 150]] }
0x55bf4f8: {[WXF0 Value[200 x 150]] }
0x55bf9c8: {[WXI0 Value[200 x 150]] }
0x55bfe98: {[WXO0 Value[200 x 150]] }
0x55c04c8: {[TrainNodeClassBasedCrossEntropy Value[1]] }
0x55d4278: {[LookupTable Value[150 x *1]] }
0x55d4618: {[AutoName31 Value[200 x *1]] }
0x55d4e88: {[AutoName22 Value[200 x *1]] }
0x55d5048: {[AutoName23 Value[200 x 1 x *1]] }
0x55d56d8: {[AutoName32 Value[200 x 1 x *1]] }
0x55d5db8: {[AutoName9 Value[200 x *1]] }
0x55d5f78: {[AutoName10 Value[200 x 1 x *1]] }
0x55d6138: {[AutoName16 Value[200 x *1]] }
0x55d62f8: {[AutoName30 Value[200 x 1 x *1]] }
0x55d64b8: {[AutoName33 Value[200 x 1 x *1]] }
0x55d6678: {[AutoName21 Value[200 x 1 x *1]] }
0x55d6838: {[AutoName24 Value[200 x 1 x *1]] }
0x55d69f8: {[AutoName20 Value[200 x 1 x *1]] }
0x55d6bb8: {[AutoName25 Value[200 x 1 x *1]] }
0x55d6d78: {[AutoName26 Value[200 x 1 x *1]] }
0x55d6f38: {[AutoName27 Value[200 x 1 x *1]] }
0x55d70f8: {[AutoName8 Value[200 x 1 x *1]] }
0x55d72b8: {[AutoName11 Value[200 x 1 x *1]] }
0x611f698: {[AutoName0 Value[200 x 1 x *1]] }
LMSequenceReader: Reading epoch data... 3760 sequences read.
LMSequenceReader: Reading epoch data... 0 sequences read.
05/05/2016 22:25:22: Final Results: Minibatch[1-60]: TrainNodeClassBasedCrossEntropy = 6.37864559 * 82402; perplexity = 589.12924332
06/16/2016 10:51:59: Minibatch[1-60]: TrainNodeClassBasedCrossEntropy = 6.37864556 * 82402
06/16/2016 10:51:59: Final Results: Minibatch[1-60]: TrainNodeClassBasedCrossEntropy = 6.37864556 * 82402; perplexity = 589.12922778
05/05/2016 22:25:22: Action "eval" complete.
06/16/2016 10:51:59: Action "eval" complete.
05/05/2016 22:25:22: __COMPLETED__
06/16/2016 10:51:59: __COMPLETED__

2
Tests/EndToEndTests/Examples/Text/PennTreebank/RNN/testcases.yml
Просмотреть файл

@ -31,4 +31,4 @@ testCases:
- Minibatch[{{integer}}-{{integer}}
- " * {{integer}}; "
- TrainNodeClassBasedCrossEntropy = {{float,tolerance=0.05}}
- perplexity = {{float,tolerance=0.05}}
- perplexity = {{float,tolerance=0.1%}}

189
Tests/EndToEndTests/MetricsDriver.py Normal file
Просмотреть файл

@ -0,0 +1,189 @@
#!/usr/bin/env python
# ----------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
# ---------------------------------------------------------
# This script extracts information (hardware used, final results) contained in the baselines files
# and generates a markdown file (wiki page)
import sys, os, re
import TestDriver as td
try:
import six
except ImportError:
print("Python package 'six' not installed. Please run 'pip install six'.")
sys.exit(1)
thisDir = os.path.dirname(os.path.realpath(__file__))
windows = os.getenv("OS")=="Windows_NT"
class Baseline:
def __init__(self, fullPath, testResult = "", trainResult = ""):
self.fullPath = fullPath
self.cpuInfo = ""
self.gpuInfo = ""
self.testResult = testResult
self.trainResult = trainResult
# extracts results info. e.g.
# Finished Epoch[ 5 of 5]: [Training] ce = 2.32253198 * 1000 err = 0.90000000 * 1000 totalSamplesSeen = 5000 learningRatePerSample = 2e-06 epochTime=0.175781
# Final Results: Minibatch[1-1]: err = 0.90000000 * 100 ce = 2.32170486 * 100 perplexity = 10.1930372
def extractResultsInfo(self, baselineContent):
trainResults = re.findall('.*(Finished Epoch\[ *\d+ of \d+\]\: \[Training\]) (.*)', baselineContent)
if trainResults:
self.trainResult = Baseline.formatLastTrainResult(trainResults[-1])[0:-2]
testResults = re.findall('.*(Final Results: Minibatch\[1-\d+\]:)(\s+\* \d+)?\s+(.*)', baselineContent)
if testResults:
self.testResult = Baseline.formatLastTestResult(testResults[-1])[0:-2]
# extracts cpu and gpu info from baseline content. e.g.:
#CPU info:
# CPU Model Name: Intel(R) Xeon(R) CPU E5-2620 v3 @ 2.40GHz
# Hardware threads: 12
#GPU info:
#
#Device[0]: cores = 2496; computeCapability = 5.2; type = "Quadro M4000"; memory = 8192 MB
#Device[1]: cores = 96; computeCapability = 2.1; type = "Quadro 600"; memory = 1024 MB
# Total Memory: 33474872 kB
def extractHardwareInfo(self, baselineContent):
startCpuInfoIndex = baselineContent.find("CPU info:")
endCpuInfoIndex = baselineContent.find("----------", startCpuInfoIndex)
cpuInfo = re.search("^CPU info:\s+"
"CPU Model (Name:\s*.*)\s+"
"(Hardware threads: \d+)\s+"
"Total (Memory:\s*.*)\s+", baselineContent[startCpuInfoIndex:endCpuInfoIndex], re.MULTILINE)
if cpuInfo is None:
return
self.cpuInfo = "\n".join(cpuInfo.groups())
startGpuInfoIndex = baselineContent.find("GPU info:")
endGpuInfoIndex = baselineContent.find("----------", startGpuInfoIndex)
gpuInfoSnippet = baselineContent[startGpuInfoIndex:endGpuInfoIndex]
gpuDevices = re.findall("\t\t(Device\[\d+\]: cores = \d+; computeCapability = \d\.\d; type = .*; memory = \d+ MB)[\r\n]?", gpuInfoSnippet)
if not gpuDevices:
return
gpuInfo = [ device for device in gpuDevices ]
self.gpuInfo = "\n".join(gpuInfo)
@staticmethod
def formatLastTestResult(line):
return line[0] + line[1] + "\n" + line[2].replace('; ', '\n').replace(' ','\n')
@staticmethod
def formatLastTrainResult(line):
epochsInfo, parameters = line[0], line[1]
return epochsInfo + '\n' + parameters.replace('; ', '\n')
class Example:
allExamplesIndexedByFullName = {}
def __init__(self, suite, name, testDir):
self.suite = suite
self.name = name
self.fullName = suite + "/" + name
self.testDir = testDir
self.baselineList = []
self.gitHash = ""
@staticmethod
def discoverAllExamples():
testsDir = thisDir
for dirName, subdirList, fileList in os.walk(testsDir):
if 'testcases.yml' in fileList:
testDir = dirName
exampleName = os.path.basename(dirName)
suiteDir = os.path.dirname(dirName)
# suite name will be derived from the path components
suiteName = os.path.relpath(suiteDir, testsDir).replace('\\', '/')
example = Example(suiteName, exampleName, testDir)
Example.allExamplesIndexedByFullName[example.fullName.lower()] = example
# it returns a list with all baseline files for current example
def findBaselineFilesList(self):
baselineFilesList = []
oses = [".windows", ".linux", ""]
devices = [".cpu", ".gpu", ""]
flavors = [".debug", ".release", ""]
for o in oses:
for device in devices:
for flavor in flavors:
candidateName = "baseline" + o + flavor + device + ".txt"
fullPath = td.cygpath(os.path.join(self.testDir, candidateName), relative=True)
if os.path.isfile(fullPath):
baseline = Baseline(fullPath);
baselineFilesList.append(baseline)
return baselineFilesList
# extracts information for every example and stores it in Example.allExamplesIndexedByFullName
def getExamplesMetrics():
Example.allExamplesIndexedByFullName = list(sorted(Example.allExamplesIndexedByFullName.values(), key=lambda test: test.fullName))
allExamples = Example.allExamplesIndexedByFullName
print ("CNTK - Metrics collector")
for example in allExamples:
baselineListForExample = example.findBaselineFilesList()
six.print_("Example: " + example.fullName)
for baseline in baselineListForExample:
with open(baseline.fullPath, "r") as f:
baselineContent = f.read()
gitHash = re.search('.*Build SHA1:\s([a-z0-9]{40})[\r\n]+', baselineContent, re.MULTILINE)
if gitHash is None:
continue
example.gitHash = gitHash.group(1)
baseline.extractHardwareInfo(baselineContent)
baseline.extractResultsInfo(baselineContent)
example.baselineList.append(baseline)
# creates a list with links to each example result
def createAsciidocExampleList(file):
for example in Example.allExamplesIndexedByFullName:
if not example.baselineList:
continue
file.write("".join(["<<", example.fullName.replace("/","").lower(),",", example.fullName, ">> +\n"]))
file.write("\n")
def writeMetricsToAsciidoc():
metricsFile = open("metrics.adoc",'wb')
createAsciidocExampleList(metricsFile)
for example in Example.allExamplesIndexedByFullName:
if not example.baselineList:
continue
metricsFile.write("".join(["===== ", example.fullName, "\n"]))
metricsFile.write("".join(["**Git Hash: **", example.gitHash, "\n\n"]))
metricsFile.write("[cols=3, options=\"header\"]\n")
metricsFile.write("|====\n")
metricsFile.write("|Log file / Configuration | Train Result | Test Result\n")
for baseline in example.baselineList:
pathInDir=baseline.fullPath.split(thisDir)[1][1:]
metricsFile.write("".join(["|link:../blob/", example.gitHash[:7],"/Tests/EndToEndTests/", pathInDir, "[",
baseline.fullPath.split("/")[-1], "] .2+|", baseline.trainResult.replace("\n", " "), " .2+|",
baseline.testResult.replace("\n", " "), "|\n"]))
cpuInfo = "".join(["CPU: ", re.sub("[\r]?\n", ' ', baseline.cpuInfo)])
gpuInfo = re.sub("[\r]?\n", ' ', baseline.gpuInfo)
if gpuInfo:
metricsFile.write("".join([cpuInfo, " GPU: ", gpuInfo]))
else:
metricsFile.write(cpuInfo)
metricsFile.write("\n|====\n\n")
# ======================= Entry point =======================
six.print_("==============================================================================")
Example.discoverAllExamples()
getExamplesMetrics()
writeMetricsToAsciidoc()

6500
Tests/EndToEndTests/Speech/DNN/WriteCommand/Output.ScaledLogLikelihood.windows.cpu
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

2188
Tests/EndToEndTests/Speech/Simple/baseline.windows.cpu.txt
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

155
Tests/EndToEndTests/TestDriver.py
Просмотреть файл

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python
# ----------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# ---------------------------------------------------------
@ -687,89 +687,90 @@ def runCommand(args):
sys.exit(10)
# ======================= Entry point =======================
parser = argparse.ArgumentParser(description="TestDriver - CNTK Test Driver")
subparsers = parser.add_subparsers(help="command to execute. Run TestDriver.py <command> --help for command-specific help")
runSubparser = subparsers.add_parser("run", help="run test(s)")
runSubparser.add_argument("test", nargs="*",
help="optional test name(s) to run, specified as Suite/TestName. "
"Use list command to list available tests. "
"If not specified then all tests will be run.")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="TestDriver - CNTK Test Driver")
subparsers = parser.add_subparsers(help="command to execute. Run TestDriver.py <command> --help for command-specific help")
runSubparser = subparsers.add_parser("run", help="run test(s)")
runSubparser.add_argument("test", nargs="*",
help="optional test name(s) to run, specified as Suite/TestName. "
"Use list command to list available tests. "
"If not specified then all tests will be run.")
defaultBuildSKU = "gpu"
defaultBuildSKU = "gpu"
runSubparser.add_argument("-b", "--build-location", help="location of the CNTK build to run")
runSubparser.add_argument("-t", "--tag", help="runs tests which match the specified tag")
runSubparser.add_argument("-d", "--device", help="cpu|gpu - run on a specified device")
runSubparser.add_argument("-f", "--flavor", help="release|debug - run only a specified flavor")
runSubparser.add_argument("-s", "--build-sku", default=defaultBuildSKU, help="cpu|gpu|1bitsgd - run tests only for a specified build SKU")
tmpDir = os.getenv("TEMP") if windows else "/tmp"
defaultRunDir=os.path.join(tmpDir, "cntk-test-{0}.{1}".format(time.strftime("%Y%m%d%H%M%S"), random.randint(0,1000000)))
runSubparser.add_argument("-r", "--run-dir", default=defaultRunDir, help="directory where to store test output, default: a random dir within /tmp")
runSubparser.add_argument("--update-baseline", action='store_true', help="update baseline file(s) instead of matching them")
runSubparser.add_argument("--create-baseline", action='store_true', help="create new baseline file(s) (named as baseline.<os>.<device>.txt) for tests that do not currently have baselines")
runSubparser.add_argument("-v", "--verbose", action='store_true', help="verbose output - dump all output of test script")
runSubparser.add_argument("-n", "--dry-run", action='store_true', help="do not run the tests, only print test names and configurations to be run along with full command lines")
runSubparser.add_argument("-b", "--build-location", help="location of the CNTK build to run")
runSubparser.add_argument("-t", "--tag", help="runs tests which match the specified tag")
runSubparser.add_argument("-d", "--device", help="cpu|gpu - run on a specified device")
runSubparser.add_argument("-f", "--flavor", help="release|debug - run only a specified flavor")
runSubparser.add_argument("-s", "--build-sku", default=defaultBuildSKU, help="cpu|gpu|1bitsgd - run tests only for a specified build SKU")
tmpDir = os.getenv("TEMP") if windows else "/tmp"
defaultRunDir=os.path.join(tmpDir, "cntk-test-{0}.{1}".format(time.strftime("%Y%m%d%H%M%S"), random.randint(0,1000000)))
runSubparser.add_argument("-r", "--run-dir", default=defaultRunDir, help="directory where to store test output, default: a random dir within /tmp")
runSubparser.add_argument("--update-baseline", action='store_true', help="update baseline file(s) instead of matching them")
runSubparser.add_argument("--create-baseline", action='store_true', help="create new baseline file(s) (named as baseline.<os>.<device>.txt) for tests that do not currently have baselines")
runSubparser.add_argument("-v", "--verbose", action='store_true', help="verbose output - dump all output of test script")
runSubparser.add_argument("-n", "--dry-run", action='store_true', help="do not run the tests, only print test names and configurations to be run along with full command lines")
runSubparser.set_defaults(func=runCommand)
runSubparser.set_defaults(func=runCommand)
listSubparser = subparsers.add_parser("list", help="list available tests")
listSubparser.add_argument("-t", "--tag", help="limits a resulting list to tests matching the specified tag")
listSubparser.add_argument("-d", "--device", help="cpu|gpu - tests for a specified device")
listSubparser.add_argument("-f", "--flavor", help="release|debug - tests for specified flavor")
listSubparser.add_argument("-s", "--build-sku", default=defaultBuildSKU, help="cpu|gpu|1bitsgd - list tests only for a specified build SKU")
listSubparser.add_argument("--os", help="windows|linux - tests for a specified operating system")
listSubparser = subparsers.add_parser("list", help="list available tests")
listSubparser.add_argument("-t", "--tag", help="limits a resulting list to tests matching the specified tag")
listSubparser.add_argument("-d", "--device", help="cpu|gpu - tests for a specified device")
listSubparser.add_argument("-f", "--flavor", help="release|debug - tests for specified flavor")
listSubparser.add_argument("-s", "--build-sku", default=defaultBuildSKU, help="cpu|gpu|1bitsgd - list tests only for a specified build SKU")
listSubparser.add_argument("--os", help="windows|linux - tests for a specified operating system")
listSubparser.set_defaults(func=listCommand)
listSubparser.set_defaults(func=listCommand)
if len(sys.argv)==1:
parser.print_help()
sys.exit(1)
args = parser.parse_args(sys.argv[1:])
# parsing a --device, --flavor and --os options:
args.devices = ["cpu", "gpu"]
if (args.device):
args.device = args.device.lower()
if not args.device in args.devices:
six.print_("--device must be one of", args.devices, file=sys.stderr)
sys.exit(1)
args.devices = [args.device]
args.flavors = ["debug", "release"]
if (args.flavor):
args.flavor = args.flavor.lower()
if not args.flavor in args.flavors:
six.print_("--flavor must be one of", args.flavors, file=sys.stderr)
sys.exit(1)
args.flavors = [args.flavor]
args.buildSKUs = ["cpu", "gpu", "1bitsgd"]
if (args.build_sku):
args.build_sku = args.build_sku.lower()
if not args.build_sku in args.buildSKUs:
six.print_("--build-sku must be one of", args.buildSKUs, file=sys.stderr)
sys.exit(1)
args.buildSKUs = [args.build_sku]
if args.build_sku == "cpu" and args.devices == ["gpu"]:
print >>sys.stderr, "Invalid combination: --build-sku cpu and --device gpu"
sys.exit(1)
if args.func == runCommand and not args.build_location:
args.build_location = os.path.realpath(os.path.join(thisDir, "../..", "x64" if windows else "build/"))
if args.func == listCommand:
args.oses = ["windows", "linux"]
if (args.os):
args.os = args.os.lower()
if not args.os in args.oses:
six.print_("--os must be one of", args.oses, file=sys.stderr)
if len(sys.argv)==1:
parser.print_help()
sys.exit(1)
args.oses = [args.os]
# discover all the tests
Test.discoverAllTests()
args = parser.parse_args(sys.argv[1:])
# execute the command
args.func(args)
# parsing a --device, --flavor and --os options:
args.devices = ["cpu", "gpu"]
if (args.device):
args.device = args.device.lower()
if not args.device in args.devices:
six.print_("--device must be one of", args.devices, file=sys.stderr)
sys.exit(1)
args.devices = [args.device]
args.flavors = ["debug", "release"]
if (args.flavor):
args.flavor = args.flavor.lower()
if not args.flavor in args.flavors:
six.print_("--flavor must be one of", args.flavors, file=sys.stderr)
sys.exit(1)
args.flavors = [args.flavor]
args.buildSKUs = ["cpu", "gpu", "1bitsgd"]
if (args.build_sku):
args.build_sku = args.build_sku.lower()
if not args.build_sku in args.buildSKUs:
six.print_("--build-sku must be one of", args.buildSKUs, file=sys.stderr)
sys.exit(1)
args.buildSKUs = [args.build_sku]
if args.build_sku == "cpu" and args.devices == ["gpu"]:
print >>sys.stderr, "Invalid combination: --build-sku cpu and --device gpu"
sys.exit(1)
if args.func == runCommand and not args.build_location:
args.build_location = os.path.realpath(os.path.join(thisDir, "../..", "x64" if windows else "build/"))
if args.func == listCommand:
args.oses = ["windows", "linux"]
if (args.os):
args.os = args.os.lower()
if not args.os in args.oses:
six.print_("--os must be one of", args.oses, file=sys.stderr)
sys.exit(1)
args.oses = [args.os]
# discover all the tests
Test.discoverAllTests()
# execute the command
args.func(args)

23
Tests/EndToEndTests/run-test-common
Просмотреть файл

@ -1,5 +1,8 @@
#!/bin/bash
# ----------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
# ----------------------------------------------------------
# Helper script containing common code used by run-test scripts of E2E tests
BinaryPath=$TEST_CNTK_BINARY
@ -26,6 +29,21 @@ MPIArgs=
DeleteExistingModels=1
DeleteModelsAfterTest=1
# Print info needed by MetricsDriver.py to extract tests metrics
printHardwareInfo()
{
cpuName=$(cat /proc/cpuinfo 2> /dev/null | grep -m 1 'model name' | cut -d : -f 2- | tr -s " " | cut -c 2-)
totalMemory=$(cat /proc/meminfo 2> /dev/null | grep 'MemTotal' | cut -d : -f 2- | tr -s " " | cut -c 2-)
nproc=$(nproc)
# Note that MetricsDriver.py depends on this format
echo "CPU info:"
echo " CPU Model Name: $cpuName"
echo " Hardware threads: $nproc"
echo " Total Memory: $totalMemory"
echo "-------------------------------------------------------------------"
}
# Helper function to print and run a command
run()
{
@ -119,3 +137,6 @@ cntkmpirun()
cntkrun "$2" "$3"
return $?
}
# place printHardwareInfo here, so that all tests print it
printHardwareInfo

47
Tests/UnitTests/MathTests/BlockMultiplierTests.cpp
Просмотреть файл

@ -145,13 +145,50 @@ template<typename ScalarCT> void CompareMatricesAndDump(const ScalarCT* ref, con
BOOST_AUTO_TEST_SUITE(BlockMultiplierSuite)
BOOST_AUTO_TEST_CASE(BlockMultiplyTest)
BOOST_AUTO_TEST_CASE(BlockMultiplyTest8x128x8SingleThread)
{
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(8, 128, 8, 1);
}
int m = 8;
int k = 128;
int n = 8;
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(m, k, n);
// Test with numblocks > 4 && numblocks % 4 != 0
BOOST_AUTO_TEST_CASE(BlockMultiplyTest7x128x8SingleThread)
{
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(7, 128, 8, 1);
}
// Test that hits all the kernel functions in BlockMultiplier (single row)
BOOST_AUTO_TEST_CASE(BlockMultiplyTestAllKSingleRowSingleThread)
{
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(1, 128 + 64 + 32 + 16 + 8 + 1, 1, 1);
}
// Test that hits all the kernel functions in BlockMultiplier (four rows)
BOOST_AUTO_TEST_CASE(BlockMultiplyTestAllKFourRowsSingleThread)
{
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(4, 128 + 64 + 32 + 16 + 8 + 1, 1, 1);
}
BOOST_AUTO_TEST_CASE(BlockMultiplyTest8x128x8MultiThread)
{
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(8, 128, 8, 2);
}
// Test with numblocks > 4 && numblocks % 4 != 0
BOOST_AUTO_TEST_CASE(BlockMultiplyTest7x128x8MultiThread)
{
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(7, 128, 8, 2);
}
// Test that hits all the kernel functions in BlockMultiplier (single row)
BOOST_AUTO_TEST_CASE(BlockMultiplyTestAllKSingleRowMultiThread)
{
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(1, 128 + 64 + 32 + 16 + 8 + 1, 1, 2);
}
// Test that hits all the kernel functions in BlockMultiplier (four rows)
BOOST_AUTO_TEST_CASE(BlockMultiplyTestAllKFourRowsMultiThread)
{
TestMultiplierSub<int16_t, int16_t, int32_t, BlockMultiplier<BlockHandlerSSE>>(4, 128 + 64 + 32 + 16 + 8 + 1, 1, 2);
}
BOOST_AUTO_TEST_SUITE_END()

11
Tests/UnitTests/ReaderTests/ReaderTests.vcxproj
Просмотреть файл

@ -29,12 +29,7 @@
<PropertyGroup>
<HasBoost>false</HasBoost>
<HasBoost Condition="Exists('$(BOOST_INCLUDE_PATH)') And Exists('$(BOOST_LIB_PATH)')">true</HasBoost>
<HasOpenCV>false</HasOpenCV>
<HasOpenCV Condition="Exists('$(OPENCV_PATH)')">true</HasOpenCV>
<ImageReaderDefine Condition="$(HasOpenCV)">ENABLE_IMAGEREADER_TESTS</ImageReaderDefine>
<UseZip>false</UseZip>
<UseZip Condition="Exists('$(ZLIB_PATH)')">true</UseZip>
<ZipDefine Condition="$(HasOpenCV) And $(UseZip)">USE_ZIP</ZipDefine>
<ImageReaderDefine Condition="$(HasOpenCv)">ENABLE_IMAGEREADER_TESTS</ImageReaderDefine>
</PropertyGroup>
<PropertyGroup Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
@ -257,8 +252,8 @@
<Target Name="CheckDependencies">
<Warning Condition="!$(HasBoost)" Text="ReaderTests requires the Boost library to build. Please see https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows#boost for installation instructions." />
</Target>
<PropertyGroup Condition="$(HasOpenCV)">
<ImageReaderDependencies>$(OutDir)..\ImageReader.dll;$(OPENCV_PATH)\x64\vc12\bin\opencv_world300.dll</ImageReaderDependencies>
<PropertyGroup Condition="$(HasOpenCv)">
<ImageReaderDependencies>$(OutDir)..\ImageReader.dll;$(OpenCvBinPath)\$(OpenCvWorld).dll</ImageReaderDependencies>
<ImageReaderDependencies Condition="$(UseZip)">$(ImageReaderDependencies);$(OutDir)..\zip.dll;$(OutDir)..\zlib1.dll</ImageReaderDependencies>
</PropertyGroup>
<Target Name="CopyUnitTestDependencies" AfterTargets="Build">

14
Tests/UnitTests/V2LibraryTests/FeedForwardTests.cpp
Просмотреть файл

@ -44,7 +44,7 @@ void TestFeedForwardNetworkCreation(const DeviceDescriptor& device)
Variable labelsVar({ numOutputClasses }, DataType::Float, L"Labels");
auto trainingLossFunction = CNTK::CrossEntropyWithSoftmax(classifierOutputFunction, labelsVar, L"LossFunction");
auto predictionFunction = CNTK::PredictionError(classifierOutputFunction, labelsVar, L"PredictionError");
auto predictionFunction = CNTK::ClassificationError(classifierOutputFunction, labelsVar, L"ClassificationError");
auto ffNet = CNTK::Combine({ trainingLossFunction, predictionFunction, classifierOutputFunction }, L"ClassifierModel");
@ -128,7 +128,7 @@ void TestTimesAndPlus(size_t inputDim,
if (usePreAllocatedOutputs)
{
auto outputAllocationDevice = outputOnSpecifiedDevice ? device : DeviceDescriptor::CPUDevice();
if (outputAllocationDevice.Type() == DeviceType::CPU)
if (outputAllocationDevice.Type() == DeviceKind::CPU)
outputValue = new Value(new NDArrayView(outputShape, outputData.data(), outputData.size(), outputAllocationDevice, false));
else
outputValue = new Value(new NDArrayView(AsDataType<ElementType>(), outputShape, outputAllocationDevice));
@ -143,7 +143,7 @@ void TestTimesAndPlus(size_t inputDim,
// Perform backprop
std::vector<ElementType> rootGradientsData(outputShape.TotalSize(), 1);
ValuePtr rootGradientValue;
if (device.Type() == DeviceType::CPU)
if (device.Type() == DeviceKind::CPU)
rootGradientValue = new Value(new NDArrayView(outputShape, rootGradientsData.data(), rootGradientsData.size(), device, true));
else
{
@ -159,7 +159,7 @@ void TestTimesAndPlus(size_t inputDim,
if (usePreAllocatedOutputs)
{
auto outputAllocationDevice = outputOnSpecifiedDevice ? device : DeviceDescriptor::CPUDevice();
if (outputAllocationDevice.Type() == DeviceType::CPU)
if (outputAllocationDevice.Type() == DeviceKind::CPU)
{
plusParameterGradientValue = new Value(new NDArrayView(plusParam.Shape(), plusParameterGradientData.data(), plusParameterGradientData.size(), outputAllocationDevice, false));
timesParameterGradientValue = new Value(new NDArrayView(timesParam.Shape(), timesParameterGradientData.data(), timesParameterGradientData.size(), outputAllocationDevice, false));
@ -181,7 +181,7 @@ void TestTimesAndPlus(size_t inputDim,
}
// Verify forward prop results
if (!usePreAllocatedOutputs || (outputOnSpecifiedDevice && (device.Type() != DeviceType::CPU)))
if (!usePreAllocatedOutputs || (outputOnSpecifiedDevice && (device.Type() != DeviceKind::CPU)))
{
NDArrayViewPtr cpuArrayView = new NDArrayView(outputShape, outputData.data(), outputData.size(), DeviceDescriptor::CPUDevice(), false);
cpuArrayView->CopyFrom(*outputValue->Data());
@ -201,7 +201,7 @@ void TestTimesAndPlus(size_t inputDim,
FloatingPointVectorCompare(outputData, expectedOutputValues, "TestTimesAndPlus: Forward prop results do not match expected results");
// Verify backward prop results
if (device.Type() != DeviceType::CPU)
if (device.Type() != DeviceKind::CPU)
{
NDArrayViewPtr cpuArrayView = new NDArrayView(AsDataType<ElementType>(), plusParam.Shape(), DeviceDescriptor::CPUDevice());
cpuArrayView->CopyFrom(*plusParameterGradientValue->Data());
@ -236,9 +236,11 @@ void TestTimesAndPlus(size_t inputDim,
void FeedForwardTests()
{
TestTimesAndPlus<double>(4, 2, 5, DeviceDescriptor::CPUDevice(), 3, true, true);
#ifndef CPUONLY
TestTimesAndPlus<float>(145, 32, 2, DeviceDescriptor::GPUDevice(0), 10, true, false);
TestTimesAndPlus<double>(145, 15, 200, DeviceDescriptor::GPUDevice(0), 21, false);
TestFeedForwardNetworkCreation(DeviceDescriptor::GPUDevice(0));
#endif
TestFeedForwardNetworkCreation(DeviceDescriptor::CPUDevice());
}

10
Tests/UnitTests/V2LibraryTests/NDArrayViewTests.cpp
Просмотреть файл

@ -31,7 +31,7 @@ void TestNDArrayView(size_t numAxes, const DeviceDescriptor& device)
throw std::runtime_error("The DataBuffer of the NDArrayView does not match the original buffer it was created over");
NDArrayViewPtr dataView;
if ((device.Type() == DeviceType::CPU))
if ((device.Type() == DeviceKind::CPU))
dataView = cpuDataView;
else
{
@ -47,7 +47,7 @@ void TestNDArrayView(size_t numAxes, const DeviceDescriptor& device)
ElementType* first = nullptr;
const ElementType* second = cpuDataView->DataBuffer<ElementType>();
NDArrayViewPtr temp1CpuDataView, temp2CpuDataView;
if ((device.Type() == DeviceType::CPU))
if ((device.Type() == DeviceKind::CPU))
{
if (dataView->DataBuffer<ElementType>() != data.data())
throw std::runtime_error("The DataBuffer of the NDArrayView does not match the original buffer it was created over");
@ -69,10 +69,10 @@ void TestNDArrayView(size_t numAxes, const DeviceDescriptor& device)
}
first[0] += 1;
if ((device.Type() != DeviceType::CPU))
if ((device.Type() != DeviceKind::CPU))
clonedView->CopyFrom(*temp1CpuDataView);
if ((device.Type() == DeviceType::CPU))
if ((device.Type() == DeviceKind::CPU))
{
first = clonedView->WritableDataBuffer<ElementType>();
second = dataView->DataBuffer<ElementType>();
@ -197,10 +197,12 @@ void TestSparseCSCArrayView(size_t numAxes, const DeviceDescriptor& device)
void NDArrayViewTests()
{
TestNDArrayView<float>(2, DeviceDescriptor::CPUDevice());
#ifndef CPUONLY
TestNDArrayView<float>(0, DeviceDescriptor::GPUDevice(0));
TestNDArrayView<double>(4, DeviceDescriptor::GPUDevice(0));
TestSparseCSCArrayView<float>(1, DeviceDescriptor::GPUDevice(0));
TestSparseCSCArrayView<double>(4, DeviceDescriptor::GPUDevice(0));
#endif
TestSparseCSCArrayView<float>(2, DeviceDescriptor::CPUDevice());
}

8
Tests/UnitTests/V2LibraryTests/RecurrentFunctionTests.cpp
Просмотреть файл

@ -150,7 +150,7 @@ void TestRecurrentNetworkCreation(const DeviceDescriptor& device)
Variable labelsVar = Variable({ numOutputClasses }, AsDataType<ElementType>(), L"Labels");
auto trainingLossFunction = CrossEntropyWithSoftmax(classifierOutputFunction, labelsVar, L"lossFunction");
auto predictionFunction = PredictionError(classifierOutputFunction, labelsVar, L"predictionError");
auto predictionFunction = ClassificationError(classifierOutputFunction, labelsVar, L"predictionError");
auto LSTMClassifier = Combine({ trainingLossFunction, predictionFunction, classifierOutputFunction }, L"LSTMClassifier");
@ -474,12 +474,18 @@ void TestSimpleRecurrence(size_t inputDim,
void RecurrentFunctionTests()
{
TestSimpleRecurrence<float>(2, 1, 4, 1, DeviceDescriptor::CPUDevice(), 3, false, false);
#ifndef CPUONLY
TestSimpleRecurrence<double>(11, 9, 16, 7, DeviceDescriptor::GPUDevice(0), 5, true, false);
#endif
TestSimpleRecurrence<double>(1000, 9, 16, 3, DeviceDescriptor::CPUDevice(), 2, true, true);
#ifndef CPUONLY
TestSimpleRecurrence<float>(5000, 200, 19, 6, DeviceDescriptor::GPUDevice(0), 3, false, true);
TestSimpleRecurrence<double>(1000, 9, 16, 3, DeviceDescriptor::GPUDevice(0), 3, true, true, true);
#endif
TestSimpleRecurrence<float>(5000, 200, 19, 6, DeviceDescriptor::CPUDevice(), 2, false, true, true);
#ifndef CPUONLY
TestRecurrentNetworkCreation<float>(DeviceDescriptor::GPUDevice(0));
#endif
TestRecurrentNetworkCreation<double>(DeviceDescriptor::CPUDevice());
}

2
Tests/UnitTests/V2LibraryTests/TensorTests.cpp
Просмотреть файл

@ -81,8 +81,10 @@ void TestTensorPlus(size_t numAxesLeftOperand, size_t numAxesRightOperand, const
void TensorTests()
{
TestTensorPlus<float>(0, 3, DeviceDescriptor::CPUDevice());
#ifndef CPUONLY
TestTensorPlus<double>(4, 1, DeviceDescriptor::GPUDevice(0));
TestTensorPlus<float>(1, 3, DeviceDescriptor::GPUDevice(0));
TestTensorPlus<double>(2, 0, DeviceDescriptor::GPUDevice(0));
TestTensorPlus<float>(0, 0, DeviceDescriptor::GPUDevice(0));
#endif
}

5
Tests/UnitTests/V2LibraryTests/V2LibraryTests.vcxproj
Просмотреть файл

@ -99,6 +99,11 @@
<AdditionalDependencies>CNTKLibrary-2.0.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="$(CpuOnlyBuild)">
<ClCompile>
<PreprocessorDefinitions>CPUONLY;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="FeedForwardTests.cpp" />
<ClCompile Include="Main.cpp" />

2
Tools/cntk-binary-drop/linux/1bitsgd/Install-Linux.txt
Просмотреть файл

@ -6,7 +6,7 @@ Installation on Linux.
- If not installed, install Open MPI version 1.10 or later as described at https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Linux#open-mpi
- Set the following environment variables (the example below assumes, that CNTK is extracted to /home/username/cntk)
export PATH=/home/username/cntk/cntk/bin:$PATH
export LD_LIBRARY_PATH=/home/username/cntk/cntk/lib: /home/username/cntk/cntk/dependencies/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/home/username/cntk/cntk/lib:/home/username/cntk/cntk/dependencies/lib:$LD_LIBRARY_PATH
export ACML_FMA=0
(ACML_FMA is set to ensure the correct work of ACML library)

2
Tools/cntk-binary-drop/linux/cpu/Install-Linux.txt
Просмотреть файл

@ -6,7 +6,7 @@ Installation on Linux.
- If not installed, install Open MPI version 1.10 or later as described at https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Linux#open-mpi
- Set the following environment variables (the example below assumes, that CNTK is extracted to /home/username/cntk)
export PATH=/home/username/cntk/cntk/bin:$PATH
export LD_LIBRARY_PATH=/home/username/cntk/cntk/lib: /home/username/cntk/cntk/dependencies/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/home/username/cntk/cntk/lib:/home/username/cntk/cntk/dependencies/lib:$LD_LIBRARY_PATH
export ACML_FMA=0
(ACML_FMA is set to ensure the correct work of ACML library)

2
Tools/cntk-binary-drop/linux/gpu/Install-Linux.txt
Просмотреть файл

@ -6,7 +6,7 @@ Installation on Linux.
- If not installed, install Open MPI version 1.10 or later as described at https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Linux#open-mpi
- Set the following environment variables (the example below assumes, that CNTK is extracted to /home/username/cntk)
export PATH=/home/username/cntk/cntk/bin:$PATH
export LD_LIBRARY_PATH=/home/username/cntk/cntk/lib: /home/username/cntk/cntk/dependencies/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/home/username/cntk/cntk/lib:/home/username/cntk/cntk/dependencies/lib:$LD_LIBRARY_PATH
export ACML_FMA=0
(ACML_FMA is set to ensure the correct work of ACML library)

26
Tools/make_binary_drop_linux
Просмотреть файл

@ -106,11 +106,11 @@ fi
# Dependency files
# ACML
declare -a acmlFiles=("libacml_mp.so" "libiomp5.so")
# MKL
declare -a mklFiles=("libmkl_cntk_p.so" "libiomp5.so")
# Open CV
declare -a opencvFiles=("libopencv_core.so.3.0" "libopencv_imgproc.so.3.0" "libopencv_imgproc.so.3.0" "libopencv_imgcodecs.so.3.0")
declare -a opencvFiles=("libopencv_core.so.3.1" "libopencv_imgproc.so.3.1" "libopencv_imgproc.so.3.1" "libopencv_imgcodecs.so.3.1")
# libzip
declare -a libzipFiles=("libzip.so")
@ -122,8 +122,8 @@ declare -a cudaFiles=("libcudart.so.7.5" "libcublas.so.7.5" "libcurand.so.7.5" "
declare -a cudnnFiles=("libcudnn.so.4")
# Set dependency sources paths
acmlPath="/usr/local/acml5.3.1/ifort64_mp/lib"
opencvPath="/usr/local/opencv-3.0.0/lib"
mklPath="/usr/local/CNTKCustomMKL/1/x64/parallel"
opencvPath="/usr/local/opencv-3.1.0/lib"
libzipPath="/usr/local/lib"
cudaPath="/usr/local/cuda/lib64"
cudnnPath="/usr/local/cudnn-4.0/cuda/lib64"
@ -143,11 +143,21 @@ mkdir -p $baseBinariesPath
# Copy build binaries
echo "Copying build binaries..." >&3
cp -r $buildPath/* $baseBinariesPath
# Remove unnessesary file(s) if exist(s)
rm -f $baseBinariesPath/bin/v2librarytests
rm -f $baseBinariesPath/bin/cppevalclient
rm -f $baseBinariesPath/lib/libcntklibrary-2.0.so
# Copy Examples
echo "Copying Examples..." >&3
cp -r Examples $baseDropPath
# Copy Scripts (Scripts folder from the root of the Repo)
echo "Copying Scripts..." >&3
cp -r Scripts $baseDropPath
# Remove test related file(s) if exist(s)
rm -f $baseDropPath/Scripts/pytest.ini
# Copy Extras
echo "Copying Extras..." >&3
cp -r $extrasPath/* $baseDropPath
@ -158,9 +168,9 @@ echo "Copying Dependencies..." >&3
# Make dependencies directory
mkdir -p $baseDependenciesPath
# Copy ACML
echo "Copying ACML..." >&3
CopyFilesFromList $acmlPath acmlFiles[@] $baseDependenciesPath
# Copy MKL
echo "Copying MKL" >&3
CopyFilesFromList $mklPath mklFiles[@] $baseDependenciesPath
# Copy Open CV
echo "Copying Open CV..." >&3

32
Tools/make_binary_drop_windows.ps1
Просмотреть файл

@ -74,15 +74,45 @@ Remove-Item $baseDropPath\cntk\*.pdb
Remove-Item $baseDropPath\cntk\*.lib
Remove-Item $baseDropPath\cntk\*.exp
Remove-Item $baseDropPath\cntk\*.metagen
# Remove specific items
If (Test-Path $baseDropPath\cntk\CNTKLibrary-2.0.dll)
{
Remove-Item $baseDropPath\cntk\CNTKLibrary-2.0.dll
}
If (Test-Path $baseDropPath\cntk\CPPEvalClient.exe)
{
Remove-Item $baseDropPath\cntk\CPPEvalClient.exe
}
If (Test-Path $baseDropPath\cntk\CSEvalClient.exe)
{
Remove-Item $baseDropPath\cntk\CSEvalClient.exe
}
If (Test-Path $baseDropPath\cntk\CSEvalClient.exe.config)
{
Remove-Item $baseDropPath\cntk\CSEvalClient.exe.config
}
If (Test-Path $baseDropPath\cntk\CommandEval.exe)
{
Remove-Item $baseDropPath\cntk\CommandEval.exe
}
# Copy Examples
Write-Verbose "Copying Examples ..."
Copy-Item Examples -Recurse -Destination $baseDropPath\Examples
# Copy Scripts
Write-Verbose "Copying Scripts ..."
Copy-Item Scripts -Recurse -Destination $baseDropPath\Scripts
# Remove test related file(s) if exist(s)
If (Test-Path $baseDropPath\Scripts\pytest.ini)
{
Remove-Item $baseDropPath\Scripts\pytest.ini
}
# Copy all items from the share
# For whatever reason Copy-Item in the line below does not work
# Copy-Item $sharePath"\*" -Recurse -Destination $baseDropPath
# Copying with Robocopy. Maximum 2 retries, 30 sec waiting times in between
# Copying with Robocopy. Maximum 2 retries, 30 sec waiting time in between
Write-Verbose "Copying dependencies and other files from Remote Share ..."
robocopy $sharePath $baseDropPath /s /e /r:2 /w:30
# Check that Robocopy finished OK.

4
contrib/Python/cntk/examples/MNIST/mnist_one_layer.py
Просмотреть файл

@ -8,8 +8,8 @@
MNIST Example, one hidden layer neural network using training and testing data
through files. To generate the data first run fetch_mnist_data.py to fetch the data.
Train and Test files obtained need to be converted to CNTKTextFormatReader format using
`uci_to_cntk_text_format_converter.py
<https://github.com/Microsoft/CNTK/blob/master/Source/Readers/CNTKTextFormatReader/uci_to_cntk_text_format_converter.py>`_
`uci2ctf.py
<https://github.com/Microsoft/CNTK/blob/master/Scripts/uci2ctf.py>`_
Rename train data to Train-28x28_text.txt and test data to Test-28x28_text.txt
"""

2
contrib/Python/cntk/reader.py
Просмотреть файл

@ -102,7 +102,7 @@ class CNTKTextFormatReader(AbstractReader):
with a label.
If your data is in matrix format (one column per feature), you can use
`uci_to_cntk_text_format_converter.py <https://github.com/Microsoft/CNTK/blob/master/Source/Readers/CNTKTextFormatReader/uci_to_cntk_text_format_converter.py>`_
`uci2ctf.py <https://github.com/Microsoft/CNTK/blob/master/Scripts/uci2ctf.py>`_
to convert it to the CNTKTextFormatReader format.
Args: