diff --git a/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj b/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj
index 5085006d6..1f5ef8755 100644
--- a/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj
+++ b/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj
@@ -58,6 +58,7 @@
+
diff --git a/Examples/Evaluation/CSEvalClient/ModelEvaluator.cs b/Examples/Evaluation/CSEvalClient/ModelEvaluator.cs
new file mode 100644
index 000000000..f753d8fbd
--- /dev/null
+++ b/Examples/Evaluation/CSEvalClient/ModelEvaluator.cs
@@ -0,0 +1,202 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+//
+// ModelEvaluator.cs -- wrapper for a network so it can be evaluated one call at a time.
+//
+// THIS CODE IS FOR ILLUSTRATION PURPOSES ONLY. NOT FOR PRODUCTION.
+//
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+
+namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
+{
+ ///
+ /// This class provides an Eval model wrapper to restrict model evaluation calls to one at a time.
+ ///
+ ///
+ /// This class is not thread-safe except through the static methods.
+ /// Each ModelEvaluator instance wraps an Eval model, and exposes the Evaluate method for either
+ /// a vector of inputs or a record string.
+ /// The static interface provides the management of the concurrency of the models and restricts
+ /// the evaluations to a single thread.
+ ///
+ public sealed class ModelEvaluator
+ {
+ ///
+ /// The cntk model evaluation instance
+ ///
+ private readonly IEvaluateModelManagedF m_model;
+
+ ///
+ /// The input layer key
+ ///
+ private readonly string m_inKey;
+
+ ///
+ /// The output layer key
+ ///
+ private readonly string m_outKey;
+
+ ///
+ /// The model instance number
+ ///
+ private readonly int m_modelInstance;
+
+ ///
+ /// The input buffer
+ ///
+ private Dictionary> m_inputs;
+
+ ///
+ /// Indicates if the object is diposed
+ ///
+ private static bool Disposed
+ {
+ get;
+ set;
+ }
+
+ ///
+ /// The ModelEvaluator's models to manage
+ ///
+ private static readonly BlockingCollection Models = new BlockingCollection();
+
+ ///
+ /// Initializes the Model Evaluator to process multiple models concurrently
+ ///
+ /// The number of concurrent models
+ /// The model file path to load the model from
+ ///
+ public static void Initialize(int numConcurrentModels, string modelFilePath, int numThreads = 1)
+ {
+ if (Disposed)
+ {
+ throw new CNTKRuntimeException("Model Evaluator has been disposed", string.Empty);
+ }
+
+ for (int i = 0; i < numConcurrentModels; i++)
+ {
+ Models.Add(new ModelEvaluator(modelFilePath, numThreads, i));
+ }
+
+ Disposed = false;
+ }
+
+ ///
+ /// Disposes of all models
+ ///
+ public static void DisposeAll()
+ {
+ Disposed = true;
+
+ foreach (var model in Models)
+ {
+ model.Dispose();
+ }
+
+ Models.Dispose();
+ }
+
+ ///
+ /// Evaluates a record containing the input data and the expected outcome value
+ ///
+ /// A tab-delimited string with the first entry being the expected value.
+ /// true if the outcome is as expected, false otherwise
+ public static bool Evaluate(string record)
+ {
+ var model = Models.Take();
+ var outcome = model.EvaluateRecord(record);
+ Models.Add(model);
+ return outcome;
+ }
+
+ ///
+ /// Evaluated a vector and returns the output vector
+ ///
+ /// The input vector
+ /// The output vector
+ public static List Evaluate(List inputs)
+ {
+ var model = Models.Take();
+ var outcome = model.EvaluateInput(inputs);
+ Models.Add(model);
+ return outcome;
+ }
+
+ ///
+ /// Creates an instance of the class.
+ ///
+ /// The model file path
+ /// The number of concurrent threads for the model
+ /// A unique id for the model
+ /// The id is used only for debugging purposes
+ private ModelEvaluator(string modelFilePath, int numThreads, int id)
+ {
+ m_modelInstance = id;
+
+ m_model = new IEvaluateModelManagedF();
+
+ // Configure the model to run with a specific number of threads
+ m_model.Init(string.Format("numCPUThreads={0}", numThreads));
+
+ // Load model
+ m_model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId: -1);
+
+ // Generate random input values in the appropriate structure and size
+ var inDims = m_model.GetNodeDimensions(NodeGroup.Input);
+ m_inKey = inDims.First().Key;
+ m_inputs = new Dictionary>() { { m_inKey, null } };
+
+ // We request the output layer names(s) and dimension, we'll use the first one.
+ var outDims = m_model.GetNodeDimensions(NodeGroup.Output);
+ m_outKey = outDims.First().Key;
+ }
+
+ ///
+ /// Evaluates a test record
+ ///
+ /// A tab-delimited string containing as the first entry the expected outcome, values after that are the input data
+ /// true if the record's expected outcome value matches the computed value
+ private bool EvaluateRecord(string record)
+ {
+ // The first value in the line is the expected label index for the record's outcome
+ int expected = int.Parse(record.Substring(0, record.IndexOf('\t')));
+ m_inputs[m_inKey] =
+ record.Substring(record.IndexOf('\t') + 1).Split('\t').Select(float.Parse).ToList();
+
+ // We can call the evaluate method and get back the results (single layer)...
+ var outputs = m_model.Evaluate(m_inputs, m_outKey);
+
+ // Retrieve the outcome index (so we can compare it with the expected index)
+ int index = 0;
+ var max = outputs.Select(v => new { Value = v, Index = index++ })
+ .Aggregate((a, b) => (a.Value > b.Value) ? a : b)
+ .Index;
+
+ return (expected == max);
+ }
+
+ ///
+ /// Evaluates an input vector against the model as the first defined input layer, and returns the first defined output layer
+ ///
+ /// Input vector
+ /// The output vector
+ private List EvaluateInput(List inputs)
+ {
+ return m_model.Evaluate(new Dictionary>() { { m_inKey, inputs } }, m_outKey);
+ }
+
+ ///
+ /// Disposes of the resources
+ ///
+ private void Dispose()
+ {
+ m_model.Dispose();
+ }
+ }
+}
diff --git a/Examples/Evaluation/CSEvalClient/Program.cs b/Examples/Evaluation/CSEvalClient/Program.cs
index 1ac65953b..c98bcf9cf 100644
--- a/Examples/Evaluation/CSEvalClient/Program.cs
+++ b/Examples/Evaluation/CSEvalClient/Program.cs
@@ -7,9 +7,12 @@
using System;
using System.Collections.Generic;
+using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
+using System.Threading;
+using System.Threading.Tasks;
using Microsoft.MSR.CNTK.Extensibility.Managed;
namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
@@ -21,8 +24,8 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
/// This program is a managed client using the CLIWrapper to run the model evaluator in CNTK.
/// There are four cases shown in this program related to model loading, network creation and evaluation.
///
- /// To run this program from the CNTK binary drop, you must add the Evaluation NuGet package first.
- /// Refer to for information regarding the Evalution NuGet package.
+ /// To run this program from the CNTK binary drop, you must add the NuGet package for model evaluation first.
+ /// Refer to for information regarding the NuGet package for model evaluation.
///
/// EvaluateModelSingleLayer and EvaluateModelMultipleLayers
/// --------------------------------------------------------
@@ -34,6 +37,12 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
/// ----------------------------------------------------------------
/// These two cases do not required a trained model (just the network description). These cases show how to extract values from a single forward-pass
/// without any input to the model.
+ ///
+ /// EvaluateMultipleModels
+ /// ----------------------
+ /// This case requires the 02_Convolution model and the Test-28x28.txt test file which are part of the /Examples/Image/MNIST example.
+ /// Refer to for how to train
+ /// the model used in this example.
///
class Program
{
@@ -46,7 +55,7 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
private static void Main(string[] args)
{
initialDirectory = Environment.CurrentDirectory;
-
+
Console.WriteLine("====== EvaluateModelSingleLayer ========");
EvaluateModelSingleLayer();
@@ -58,10 +67,13 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
Console.WriteLine("\n====== EvaluateNetworkSingleLayerNoInput ========");
EvaluateNetworkSingleLayerNoInput();
-
+
Console.WriteLine("\n====== EvaluateExtendedNetworkSingleLayerNoInput ========");
EvaluateExtendedNetworkSingleLayerNoInput();
+ Console.WriteLine("\n====== EvaluateMultipleModels ========");
+ EvaluateMultipleModels();
+
Console.WriteLine("Press to terminate.");
Console.ReadLine();
}
@@ -290,7 +302,7 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
model.ForwardPass(inputBuffer, outputBuffer);
// We expect two outputs: the v2 constant, and the ol Plus result
- var expected = new float[][]{new float[]{2}, new float[]{3}};
+ var expected = new float[][] { new float[] { 2 }, new float[] { 3 } };
Console.WriteLine("Expected values: {0}", string.Join(" - ", expected.Select(b => string.Join(", ", b)).ToList()));
Console.WriteLine("Actual Values : {0}", string.Join(" - ", outputBuffer.Select(b => string.Join(", ", b.Buffer)).ToList()));
@@ -306,6 +318,86 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
}
}
+ ///
+ /// Evaluates multiple instances of a model in the same process.
+ ///
+ ///
+ /// Although all models execute concurrently (multiple tasks), each model is evaluated with a single task at a time.
+ ///
+ private static void EvaluateMultipleModels()
+ {
+ // Specifies the number of models in memory as well as the number of parallel tasks feeding these models (1 to 1)
+ int numConcurrentModels = 4;
+
+ // Specifies the number of times to iterate through the test file (epochs)
+ int numRounds = 1;
+
+ // Counts the number of evaluations accross all models
+ int count = 0;
+
+ // Counts the number of failed evaluations (output != expected) accross all models
+ int errorCount = 0;
+
+ // The examples assume the executable is running from the data folder
+ // We switch the current directory to the data folder (assuming the executable is in the /x64/Debug|Release folder
+ Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\MNIST\Data\");
+
+ // Load model
+ string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\02_Convolution");
+
+ // Initializes the model instances
+ ModelEvaluator.Initialize(numConcurrentModels, modelFilePath);
+
+ string testfile = Path.Combine(Environment.CurrentDirectory, @"Test-28x28.txt");
+ Stopwatch sw = new Stopwatch();
+ sw.Start();
+
+ try
+ {
+ for (int i = 0; i < numRounds; i++)
+ {
+ // Feed each line to a single model in parallel
+ Parallel.ForEach(File.ReadLines(testfile), new ParallelOptions() { MaxDegreeOfParallelism = numConcurrentModels }, (line) =>
+ {
+ Interlocked.Increment(ref count);
+
+ // The first value in the line is the expected label index for the record's outcome
+ int expected = int.Parse(line.Substring(0, line.IndexOf('\t')));
+ var inputs = line.Substring(line.IndexOf('\t') + 1).Split('\t').Select(float.Parse).ToList();
+
+ // We can call the evaluate method and get back the results (single layer)...
+ var outputs = ModelEvaluator.Evaluate(inputs);
+
+ // Retrieve the outcome index (so we can compare it with the expected index)
+ int index = 0;
+ var max = outputs.Select(v => new { Value = v, Index = index++ })
+ .Aggregate((a, b) => (a.Value > b.Value) ? a : b)
+ .Index;
+
+ // Count the errors
+ if (expected != max)
+ {
+ Interlocked.Increment(ref errorCount);
+ }
+ });
+ }
+ }
+ catch (CNTKException ex)
+ {
+ Console.WriteLine("Error: {0}\nNative CallStack: {1}\n Inner Exception: {2}", ex.Message, ex.NativeCallStack, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
+ }
+
+ sw.Stop();
+ ModelEvaluator.DisposeAll();
+
+ Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28.txt",
+ numConcurrentModels, (float)errorCount / count, errorCount, count, (count + errorCount) * 1000.0 / sw.ElapsedMilliseconds);
+ }
+
///
/// Dumps the output to the console
///