Added MultiModel sample for CSEvalClient

Examples/Evaluation/CSEvalClient/CSEvalClient.csproj

@@ -58,6 +58,7 @@
     <Reference Include="Microsoft.CSharp" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="ModelEvaluator.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>

Examples/Evaluation/CSEvalClient/ModelEvaluator.cs

@@ -0,0 +1,202 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+//
+// ModelEvaluator.cs -- wrapper for a network so it can be evaluated one call at a time.
+// 
+// THIS CODE IS FOR ILLUSTRATION PURPOSES ONLY. NOT FOR PRODUCTION.
+//
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+
+namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
+{
+    /// <summary>
+    /// This class provides an Eval model wrapper to restrict model evaluation calls to one at a time.
+    /// </summary>
+    /// <remarks>
+    /// This class is not thread-safe except through the static methods.
+    /// Each ModelEvaluator instance wraps an Eval model, and exposes the Evaluate method for either
+    /// a vector of inputs or a record string.
+    /// The static interface provides the management of the concurrency of the models and restricts
+    /// the evaluations to a single thread.
+    /// </remarks>
+    public sealed class ModelEvaluator
+    {
+        /// <summary>
+        /// The cntk model evaluation instance
+        /// </summary>
+        private readonly IEvaluateModelManagedF m_model;
+
+        /// <summary>
+        /// The input layer key
+        /// </summary>
+        private readonly string m_inKey;
+
+        /// <summary>
+        /// The output layer key
+        /// </summary>
+        private readonly string m_outKey;
+
+        /// <summary>
+        /// The model instance number
+        /// </summary>
+        private readonly int m_modelInstance;
+
+        /// <summary>
+        /// The input buffer
+        /// </summary>
+        private Dictionary<string, List<float>> m_inputs;
+
+        /// <summary>
+        /// Indicates if the object is diposed
+        /// </summary>
+        private static bool Disposed
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// The ModelEvaluator's models to manage
+        /// </summary>
+        private static readonly BlockingCollection<ModelEvaluator> Models = new BlockingCollection<ModelEvaluator>();
+
+        /// <summary>
+        /// Initializes the Model Evaluator to process multiple models concurrently
+        /// </summary>
+        /// <param name="numConcurrentModels">The number of concurrent models</param>
+        /// <param name="modelFilePath">The model file path to load the model from</param>
+        /// <param name="numThreads"></param>
+        public static void Initialize(int numConcurrentModels, string modelFilePath, int numThreads = 1)
+        {
+            if (Disposed)
+            {
+                throw new CNTKRuntimeException("Model Evaluator has been disposed", string.Empty);
+            }
+
+            for (int i = 0; i < numConcurrentModels; i++)
+            {
+                Models.Add(new ModelEvaluator(modelFilePath, numThreads, i));
+            }
+            
+            Disposed = false;
+        }
+
+        /// <summary>
+        /// Disposes of all models
+        /// </summary>
+        public static void DisposeAll()
+        {
+            Disposed = true;
+
+            foreach (var model in Models)
+            {
+                model.Dispose();
+            }
+            
+            Models.Dispose();
+        }
+
+        /// <summary>
+        /// Evaluates a record containing the input data and the expected outcome value
+        /// </summary>
+        /// <param name="record">A tab-delimited string with the first entry being the expected value.</param>
+        /// <returns>true if the outcome is as expected, false otherwise</returns>
+        public static bool Evaluate(string record)
+        {
+            var model = Models.Take();
+            var outcome = model.EvaluateRecord(record);
+            Models.Add(model);
+            return outcome;
+        }
+
+        /// <summary>
+        /// Evaluated a vector and returns the output vector
+        /// </summary>
+        /// <param name="inputs">The input vector</param>
+        /// <returns>The output vector</returns>
+        public static List<float> Evaluate(List<float> inputs)
+        {
+            var model = Models.Take();
+            var outcome = model.EvaluateInput(inputs);
+            Models.Add(model);
+            return outcome;
+        }
+
+        /// <summary>
+        /// Creates an instance of the <see cref="ModelEvaluator"/> class.
+        /// </summary>
+        /// <param name="modelFilePath">The model file path</param>
+        /// <param name="numThreads">The number of concurrent threads for the model</param>
+        /// <param name="id">A unique id for the model</param>
+        /// <remarks>The id is used only for debugging purposes</remarks>
+        private ModelEvaluator(string modelFilePath, int numThreads, int id)
+        {
+            m_modelInstance = id;
+
+            m_model = new IEvaluateModelManagedF();
+
+            // Configure the model to run with a specific number of threads
+            m_model.Init(string.Format("numCPUThreads={0}", numThreads));
+
+            // Load model
+            m_model.CreateNetwork(string.Format("modelPath=\"{0}\"", modelFilePath), deviceId: -1);
+
+            // Generate random input values in the appropriate structure and size
+            var inDims = m_model.GetNodeDimensions(NodeGroup.Input);
+            m_inKey = inDims.First().Key;
+            m_inputs = new Dictionary<string, List<float>>() { { m_inKey, null } };
+
+            // We request the output layer names(s) and dimension, we'll use the first one.
+            var outDims = m_model.GetNodeDimensions(NodeGroup.Output);
+            m_outKey = outDims.First().Key;
+        }
+
+        /// <summary>
+        /// Evaluates a test record
+        /// </summary>
+        /// <param name="record">A tab-delimited string containing as the first entry the expected outcome, values after that are the input data</param>
+        /// <returns>true if the record's expected outcome value matches the computed value</returns>
+        private bool EvaluateRecord(string record)
+        {
+            // The first value in the line is the expected label index for the record's outcome
+            int expected = int.Parse(record.Substring(0, record.IndexOf('\t')));
+            m_inputs[m_inKey] =
+                record.Substring(record.IndexOf('\t') + 1).Split('\t').Select(float.Parse).ToList();
+
+            // We can call the evaluate method and get back the results (single layer)...
+            var outputs = m_model.Evaluate(m_inputs, m_outKey);
+
+            // Retrieve the outcome index (so we can compare it with the expected index)
+            int index = 0;
+            var max = outputs.Select(v => new { Value = v, Index = index++ })
+                .Aggregate((a, b) => (a.Value > b.Value) ? a : b)
+                .Index;
+
+            return (expected == max);
+        }
+
+        /// <summary>
+        /// Evaluates an input vector against the model as the first defined input layer, and returns the first defined output layer
+        /// </summary>
+        /// <param name="inputs">Input vector</param>
+        /// <returns>The output vector</returns>
+        private List<float> EvaluateInput(List<float> inputs)
+        {
+            return m_model.Evaluate(new Dictionary<string, List<float>>() { { m_inKey, inputs } }, m_outKey);
+        }
+
+        /// <summary>
+        /// Disposes of the resources
+        /// </summary>
+        private void Dispose()
+        {
+            m_model.Dispose();
+        }
+    }
+}

Examples/Evaluation/CSEvalClient/Program.cs

@@ -7,9 +7,12 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Linq.Expressions;
+using System.Threading;
+using System.Threading.Tasks;
 using Microsoft.MSR.CNTK.Extensibility.Managed;
 
 namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
@@ -21,8 +24,8 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
     /// This program is a managed client using the CLIWrapper to run the model evaluator in CNTK.
     /// There are four cases shown in this program related to model loading, network creation and evaluation.
     /// 
-    /// To run this program from the CNTK binary drop, you must add the Evaluation NuGet package first.
+    /// To run this program from the CNTK binary drop, you must add the NuGet package for model evaluation first.
-    /// Refer to <see cref="https://github.com/Microsoft/CNTK/wiki/Eval-Nuget"/> for information regarding the Evalution NuGet package.
+    /// Refer to <see cref="https://github.com/Microsoft/CNTK/wiki/NuGet-Package"/> for information regarding the NuGet package for model evaluation.
     /// 
     /// EvaluateModelSingleLayer and EvaluateModelMultipleLayers
     /// --------------------------------------------------------
@@ -34,6 +37,12 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
     /// ----------------------------------------------------------------
     /// These two cases do not required a trained model (just the network description). These cases show how to extract values from a single forward-pass
     /// without any input to the model.
+    /// 
+    /// EvaluateMultipleModels
+    /// ----------------------
+    /// This case requires the 02_Convolution model and the Test-28x28.txt test file which are part of the <CNTK>/Examples/Image/MNIST example.
+    /// Refer to <see cref="https://github.com/Microsoft/CNTK/blob/master/Examples/Image/MNIST/README.md"/> for how to train
+    /// the model used in this example.
     /// </description>
     class Program
     {
@@ -62,6 +71,9 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
             Console.WriteLine("\n====== EvaluateExtendedNetworkSingleLayerNoInput ========");
             EvaluateExtendedNetworkSingleLayerNoInput();
 
+            Console.WriteLine("\n====== EvaluateMultipleModels ========");
+            EvaluateMultipleModels();
+
             Console.WriteLine("Press <Enter> to terminate.");
             Console.ReadLine();
         }
@@ -290,7 +302,7 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
                     model.ForwardPass(inputBuffer, outputBuffer);
 
                     // We expect two outputs: the v2 constant, and the ol Plus result
-                    var expected = new float[][]{new float[]{2}, new float[]{3}};
+                    var expected = new float[][] { new float[] { 2 }, new float[] { 3 } };
 
                     Console.WriteLine("Expected values: {0}", string.Join(" - ", expected.Select(b => string.Join(", ", b)).ToList<string>()));
                     Console.WriteLine("Actual Values  : {0}", string.Join(" - ", outputBuffer.Select(b => string.Join(", ", b.Buffer)).ToList<string>()));
@@ -306,6 +318,86 @@ namespace Microsoft.MSR.CNTK.Extensibility.Managed.CSEvalClient
             }
         }
 
+        /// <summary>
+        /// Evaluates multiple instances of a model in the same process.
+        /// </summary>
+        /// <remarks>
+        /// Although all models execute concurrently (multiple tasks), each model is evaluated with a single task at a time.
+        /// </remarks>
+        private static void EvaluateMultipleModels()
+        {
+            // Specifies the number of models in memory as well as the number of parallel tasks feeding these models (1 to 1)
+            int numConcurrentModels = 4;
+
+            // Specifies the number of times to iterate through the test file (epochs)
+            int numRounds = 1;
+
+            // Counts the number of evaluations accross all models
+            int count = 0;
+
+            // Counts the number of failed evaluations (output != expected) accross all models
+            int errorCount = 0;
+
+            // The examples assume the executable is running from the data folder
+            // We switch the current directory to the data folder (assuming the executable is in the <CNTK>/x64/Debug|Release folder
+            Environment.CurrentDirectory = Path.Combine(initialDirectory, @"..\..\Examples\Image\MNIST\Data\");
+
+            // Load model
+            string modelFilePath = Path.Combine(Environment.CurrentDirectory, @"..\Output\Models\02_Convolution");
+
+            // Initializes the model instances
+            ModelEvaluator.Initialize(numConcurrentModels, modelFilePath);
+
+            string testfile = Path.Combine(Environment.CurrentDirectory, @"Test-28x28.txt");
+            Stopwatch sw = new Stopwatch();
+            sw.Start();
+
+            try
+            {
+                for (int i = 0; i < numRounds; i++)
+                {
+                    // Feed each line to a single model in parallel
+                    Parallel.ForEach(File.ReadLines(testfile), new ParallelOptions() { MaxDegreeOfParallelism = numConcurrentModels }, (line) =>
+                    {
+                        Interlocked.Increment(ref count);
+
+                        // The first value in the line is the expected label index for the record's outcome
+                        int expected = int.Parse(line.Substring(0, line.IndexOf('\t')));
+                        var inputs = line.Substring(line.IndexOf('\t') + 1).Split('\t').Select(float.Parse).ToList();
+
+                        // We can call the evaluate method and get back the results (single layer)...
+                        var outputs = ModelEvaluator.Evaluate(inputs);
+
+                        // Retrieve the outcome index (so we can compare it with the expected index)
+                        int index = 0;
+                        var max = outputs.Select(v => new { Value = v, Index = index++ })
+                            .Aggregate((a, b) => (a.Value > b.Value) ? a : b)
+                            .Index;
+
+                        // Count the errors
+                        if (expected != max)
+                        {
+                            Interlocked.Increment(ref errorCount);
+                        }
+                    });
+                }
+            }
+            catch (CNTKException ex)
+            {
+                Console.WriteLine("Error: {0}\nNative CallStack: {1}\n Inner Exception: {2}", ex.Message, ex.NativeCallStack, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Error: {0}\nCallStack: {1}\n Inner Exception: {2}", ex.Message, ex.StackTrace, ex.InnerException != null ? ex.InnerException.Message : "No Inner Exception");
+            }
+
+            sw.Stop();
+            ModelEvaluator.DisposeAll();
+            
+            Console.WriteLine("The file {0} was processed using {1} concurrent model(s) with an error rate of: {2:P2} ({3} error(s) out of {4} record(s)), and a throughput of {5:N2} records/sec", @"Test-28x28.txt", 
+                numConcurrentModels, (float)errorCount / count, errorCount, count, (count + errorCount) * 1000.0 / sw.ElapsedMilliseconds);
+        }
+
         /// <summary>
         /// Dumps the output to the console
         /// </summary>