add ONNX support (yolov3tiny and yolov3)

Config.bat

@@ -7,4 +7,8 @@ powershell -Command "(New-Object Net.WebClient).DownloadFile('https://aka.ms/Mic
 
 powershell -Command "(New-Object Net.WebClient).DownloadFile('https://pjreddie.com/media/files/yolov3.weights', './src/VAP/YoloWrapper/Yolo.Config/YoloV3Coco/yolov3.weights')"
 
-powershell -Command "(New-Object Net.WebClient).DownloadFile('https://pjreddie.com/media/files/yolov3-tiny.weights', './src/VAP/YoloWrapper/Yolo.Config/YoloV3TinyCoco/yolov3-tiny.weights')"
+powershell -Command "(New-Object Net.WebClient).DownloadFile('https://pjreddie.com/media/files/yolov3-tiny.weights', './src/VAP/YoloWrapper/Yolo.Config/YoloV3TinyCoco/yolov3-tiny.weights')"
+
+powershell -Command "(New-Object Net.WebClient).DownloadFile('https://aka.ms/Microsoft-Rocket-Video-Analytics-Platform-yolov3ort.onnx', './modelOnnx/yolov3ort.onnx')"
+
+powershell -Command "(New-Object Net.WebClient).DownloadFile('https://aka.ms/Microsoft-Rocket-Video-Analytics-Platform-yolov3tinyort.onnx', './modelOnnx/yolov3tinyort.onnx')"

modelOnnx/coco.names

@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorbike
+aeroplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+sofa
+pottedplant
+bed
+diningtable
+toilet
+tvmonitor
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush

src/VAP/DNNDetector/CascadedDNNORTYolo.cs

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using DNNDetector.Config;
+using DNNDetector.Model;
+using OpenCvSharp;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Utils.Config;
+using Wrapper.ORT;
+
+namespace DNNDetector
+{
+    public class CascadedDNNORTYolo
+    {
+        FrameDNNOnnxYolo frameDNNOnnxYolo;
+
+        public CascadedDNNORTYolo(List<Tuple<string, int[]>> lines, string modelName)
+        {
+            frameDNNOnnxYolo = new FrameDNNOnnxYolo(lines, modelName, DNNMode.CC);
+
+            Utils.Utils.cleanFolder(@OutputFolder.OutputFolderCcDNN);
+        }
+
+        public List<Item> Run(int frameIndex, List<Item> ltDNNItemList, List<Tuple<string, int[]>> lines, Dictionary<string, int> category, ref long teleCountsHeavyDNN, bool savePictures = false)
+        {
+            if (ltDNNItemList == null)
+            {
+                return null;
+            }
+
+            List<Item> ccDNNItem = new List<Item>();
+
+            foreach (Item ltDNNItem in ltDNNItemList)
+            {
+                if (ltDNNItem.Confidence >= DNNConfig.CONFIDENCE_THRESHOLD)
+                {
+                    ccDNNItem.Add(ltDNNItem);
+                    continue;
+                }
+                else
+                {
+                    List<Item> analyzedTrackingItems = null;
+
+                    Console.WriteLine("** Calling Heavy DNN **");
+                    analyzedTrackingItems = frameDNNOnnxYolo.Run(Cv2.ImDecode(ltDNNItem.RawImageData, ImreadModes.Color), frameIndex, category, System.Drawing.Brushes.Yellow, ltDNNItem.TriggerLineID, DNNConfig.MIN_SCORE_FOR_LINEBBOX_OVERLAP_LARGE);
+                    teleCountsHeavyDNN++;
+
+                    // object detected by heavy YOLO
+                    if (analyzedTrackingItems != null)
+                    {
+                        foreach (Item item in analyzedTrackingItems)
+                        {
+                            item.RawImageData = ltDNNItem.RawImageData;
+                            item.TriggerLine = ltDNNItem.TriggerLine;
+                            item.TriggerLineID = ltDNNItem.TriggerLineID;
+                            item.Model = "Heavy";
+                            ccDNNItem.Add(item);
+
+                            // output heavy YOLO results
+                            if (savePictures)
+                            {
+                                string blobName_Heavy = $@"frame-{frameIndex}-Heavy-{item.Confidence}.jpg";
+                                string fileName_Heavy = @OutputFolder.OutputFolderCcDNN + blobName_Heavy;
+                                File.WriteAllBytes(fileName_Heavy, item.TaggedImageData);
+                                File.WriteAllBytes(@OutputFolder.OutputFolderAll + blobName_Heavy, item.TaggedImageData);
+                            }
+
+                            return ccDNNItem; // if we only return the closest object detected by heavy model
+                        }
+                    }
+                    else
+                    {
+                        Console.WriteLine("** Not detected by Heavy DNN **");
+                    }
+                }
+            }
+
+            return ccDNNItem;
+        }
+    }
+}

src/VAP/DNNDetector/DNNDetector.csproj

@@ -4,4 +4,13 @@
     <TargetFramework>netcoreapp2.2</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="OpenCvSharp4" Version="$(OpenCvSharp4Version)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ORTWrapper\ORTWrapper.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
 </Project>

src/VAP/DNNDetector/FrameBuffer.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+
+using OpenCvSharp;
+
+namespace DNNDetector
+{
+    class FrameBuffer
+    {
+        Queue<Mat> frameBuffer;
+        int bSize;
+
+        public FrameBuffer(int size)
+        {
+            bSize = size;
+            frameBuffer = new Queue<Mat>(bSize);
+        }
+
+        public void Buffer(Mat frame)
+        {
+            frameBuffer.Enqueue(frame);
+            if (frameBuffer.Count > bSize)
+                frameBuffer.Dequeue();
+        }
+
+        public Mat[] ToArray()
+        {
+            return frameBuffer.ToArray();
+        }
+    }
+}

src/VAP/DNNDetector/FrameDNNORTYolo.cs

@@ -0,0 +1,130 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using DNNDetector.Model;
+using OpenCvSharp;
+using System;
+using System.Collections.Generic;
+using System.Configuration;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.IO;
+using System.Linq;
+using Utils.Config;
+using Wrapper.ORT;
+
+namespace DNNDetector
+{
+    public class FrameDNNOnnxYolo
+    {
+        private static int _imageWidth, _imageHeight, _index;
+        private static List<Tuple<string, int[]>> _lines;
+        private static Dictionary<string, int> _category;
+
+        ORTWrapper onnxWrapper;
+        byte[] imageByteArray;
+
+        public FrameDNNOnnxYolo(List<Tuple<string, int[]>> lines, string modelName, DNNMode mode)
+        {
+            _lines = lines;
+            onnxWrapper = new ORTWrapper($@"..\..\..\..\..\..\modelOnnx\{modelName}ort.onnx", mode);
+        }
+
+        public List<Item> Run(Mat frameOnnx, int frameIndex, Dictionary<string, int> category, Brush bboxColor, int lineID, double min_score_for_linebbox_overlap, bool savePictures = false)
+        {
+            _imageWidth = frameOnnx.Width;
+            _imageHeight = frameOnnx.Height;
+            _category = category;
+            imageByteArray = Utils.Utils.ImageToByteJpeg(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameOnnx)); // Todo: feed in bmp
+
+            List<ORTItem> boundingBoxes = onnxWrapper.UseApi(
+                    OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameOnnx),
+                    _imageHeight,
+                    _imageWidth);
+
+            List<Item> preValidItems = new List<Item>();
+            foreach (ORTItem bbox in boundingBoxes)
+            {
+                preValidItems.Add(new Item(bbox));
+            }
+            List<Item> validObjects = new List<Item>();
+
+            //run _category and overlap ratio-based validation
+            if (_lines != null)
+            {
+                var overlapItems = preValidItems.Select(o => new { Overlap = Utils.Utils.checkLineBboxOverlapRatio(_lines[lineID].Item2, o.X, o.Y, o.Width, o.Height), Bbox_x = o.X + o.Width, Bbox_y = o.Y + o.Height, Distance = this.Distance(_lines[lineID].Item2, o.Center()), Item = o })
+                    .Where(o => o.Bbox_x <= _imageWidth && o.Bbox_y <= _imageHeight && o.Overlap >= min_score_for_linebbox_overlap && _category.ContainsKey(o.Item.ObjName)).OrderBy(o => o.Distance);
+                foreach (var item in overlapItems)
+                {
+                    item.Item.TaggedImageData = Utils.Utils.DrawImage(imageByteArray, item.Item.X, item.Item.Y, item.Item.Width, item.Item.Height, bboxColor);
+                    item.Item.CroppedImageData = Utils.Utils.CropImage(imageByteArray, item.Item.X, item.Item.Y, item.Item.Width, item.Item.Height);
+                    item.Item.Index = _index;
+                    item.Item.TriggerLine = _lines[lineID].Item1;
+                    item.Item.TriggerLineID = lineID;
+                    item.Item.Model = "FrameDNN";
+                    validObjects.Add(item.Item);
+                    _index++;
+                }
+            }
+            else if (min_score_for_linebbox_overlap == 0.0) //frameDNN object detection
+            {
+                var overlapItems = preValidItems.Select(o => new { Bbox_x = o.X + o.Width, Bbox_y = o.Y + o.Height, Item = o })
+                    .Where(o => o.Bbox_x <= _imageWidth && o.Bbox_y <= _imageHeight && _category.ContainsKey(o.Item.ObjName));
+                foreach (var item in overlapItems)
+                {
+                    item.Item.TaggedImageData = Utils.Utils.DrawImage(imageByteArray, item.Item.X, item.Item.Y, item.Item.Width, item.Item.Height, bboxColor);
+                    item.Item.CroppedImageData = Utils.Utils.CropImage(imageByteArray, item.Item.X, item.Item.Y, item.Item.Width, item.Item.Height);
+                    item.Item.Index = _index;
+                    item.Item.TriggerLine = "";
+                    item.Item.TriggerLineID = -1;
+                    item.Item.Model = "FrameDNN";
+                    validObjects.Add(item.Item);
+                    _index++;
+                }
+            }
+
+            //output onnxyolo results
+            if (savePictures)
+            {
+                foreach (Item it in validObjects)
+                {
+                    using (Image image = Image.FromStream(new MemoryStream(it.TaggedImageData)))
+                    {
+
+                        image.Save(@OutputFolder.OutputFolderFrameDNNONNX + $"frame-{frameIndex}-ONNX-{it.Confidence}.jpg", ImageFormat.Jpeg);
+                        image.Save(@OutputFolder.OutputFolderAll + $"frame-{frameIndex}-ONNX-{it.Confidence}.jpg", ImageFormat.Jpeg);
+                    }
+                }
+                //byte[] imgBboxes = DrawAllBb(frameIndex, Utils.Utils.ImageToByteBmp(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameOnnx)),
+                //        validObjects, Brushes.Pink);
+            }
+
+            return (validObjects.Count == 0 ? null : validObjects);
+        }
+
+        private double Distance(int[] line, System.Drawing.Point bboxCenter)
+        {
+            System.Drawing.Point p1 = new System.Drawing.Point((int)((line[0] + line[2]) / 2), (int)((line[1] + line[3]) / 2));
+            return Math.Sqrt(this.Pow2(bboxCenter.X - p1.X) + Pow2(bboxCenter.Y - p1.Y));
+        }
+
+        private double Pow2(double x)
+        {
+            return x * x;
+        }
+
+        public static byte[] DrawAllBb(int frameIndex, byte[] imgByte, List<Item> items, Brush bboxColor)
+        {
+            byte[] canvas = new byte[imgByte.Length];
+            canvas = imgByte;
+            foreach (var item in items)
+            {
+                canvas = Utils.Utils.DrawImage(canvas, item.X, item.Y, item.Width, item.Height, bboxColor);
+            }
+            string frameIndexString = frameIndex.ToString("000000.##");
+            File.WriteAllBytes(@OutputFolder.OutputFolderFrameDNNONNX + $@"frame{frameIndexString}-Raw.jpg", canvas);
+
+            return canvas;
+        }
+    }
+}

src/VAP/DNNDetector/LineTriggeredDNNORTYolo.cs

@@ -0,0 +1,104 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using DNNDetector.Config;
+using DNNDetector.Model;
+using OpenCvSharp;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Utils.Config;
+using Wrapper.ORT;
+
+namespace DNNDetector
+{
+    //Todo: merge it with LineTriggeredDNNYolo
+    public class LineTriggeredDNNORTYolo
+    {
+        Dictionary<string, int> counts_prev = new Dictionary<string, int>();
+
+        FrameDNNOnnxYolo frameDNNOnnxYolo;
+        FrameBuffer frameBufferLtDNNOnnxYolo;
+
+        public LineTriggeredDNNORTYolo(List<Tuple<string, int[]>> lines, string modelName)
+        {
+            frameBufferLtDNNOnnxYolo = new FrameBuffer(DNNConfig.FRAME_SEARCH_RANGE);
+
+            frameDNNOnnxYolo = new FrameDNNOnnxYolo(lines, modelName, DNNMode.LT);
+
+            Utils.Utils.cleanFolder(@OutputFolder.OutputFolderLtDNN);
+            Utils.Utils.cleanFolder(@OutputFolder.OutputFolderFrameDNNONNX);
+        }
+
+        public List<Item> Run(Mat frame, int frameIndex, Dictionary<string, int> counts, List<Tuple<string, int[]>> lines, Dictionary<string, int> category, ref long teleCountsCheapDNN, bool savePictures = false)
+        {
+            // buffer frame
+            frameBufferLtDNNOnnxYolo.Buffer(frame);
+
+            if (counts_prev.Count != 0)
+            {
+                foreach (string lane in counts.Keys)
+                {
+                    int diff = Math.Abs(counts[lane] - counts_prev[lane]);
+                    if (diff > 0) //object detected by BGS
+                    {
+                        if (frameIndex >= DNNConfig.FRAME_SEARCH_RANGE)
+                        {
+                            // call onnx cheap model for crosscheck
+                            int lineID = Array.IndexOf(counts.Keys.ToArray(), lane);
+                            Mat[] frameBufferArray = frameBufferLtDNNOnnxYolo.ToArray();
+                            int frameIndexOnnxYolo = frameIndex - 1;
+                            List<Item> analyzedTrackingItems = null;
+
+                            while (frameIndex - frameIndexOnnxYolo < DNNConfig.FRAME_SEARCH_RANGE)
+                            {
+                                Console.WriteLine("** Calling DNN on " + (DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexOnnxYolo)));
+                                Mat frameOnnx = frameBufferArray[DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexOnnxYolo)];
+
+                                analyzedTrackingItems = frameDNNOnnxYolo.Run(frameOnnx, (DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexOnnxYolo)), category, System.Drawing.Brushes.Pink, lineID, DNNConfig.MIN_SCORE_FOR_LINEBBOX_OVERLAP_LARGE);
+                                teleCountsCheapDNN++;
+
+                                // object detected by cheap model
+                                if (analyzedTrackingItems != null)
+                                {
+                                    List<Item> ltDNNItem = new List<Item>();
+                                    foreach (Item item in analyzedTrackingItems)
+                                    {
+                                        item.RawImageData = Utils.Utils.ImageToByteBmp(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameOnnx));
+                                        item.TriggerLine = lane;
+                                        item.TriggerLineID = lineID;
+                                        item.Model = "Cheap";
+                                        ltDNNItem.Add(item);
+
+                                        // output cheap onnx results
+                                        if (savePictures)
+                                        {
+                                            string blobName_Cheap = $@"frame-{frameIndex}-DNN-{item.Confidence}.jpg";
+                                            string fileName_Cheap = @OutputFolder.OutputFolderLtDNN + blobName_Cheap;
+                                            File.WriteAllBytes(fileName_Cheap, item.TaggedImageData);
+                                            File.WriteAllBytes(@OutputFolder.OutputFolderAll + blobName_Cheap, item.TaggedImageData);
+                                        }
+                                    }
+                                    updateCount(counts);
+                                    return ltDNNItem;
+                                }
+                                frameIndexOnnxYolo--;
+                            }
+                        }
+                    }
+                }
+            }
+            updateCount(counts);
+            return null;
+        }
+
+        void updateCount(Dictionary<string, int> counts)
+        {
+            foreach (string dir in counts.Keys)
+            {
+                counts_prev[dir] = counts[dir];
+            }
+        }
+    }
+}

src/VAP/DNNDetector/Model/Item.cs

@@ -37,6 +37,19 @@ namespace DNNDetector.Model
             this.TriggerLine = lineName;
         }
 
+        public Item(Wrapper.ORT.ORTItem onnxYoloItem)
+        {
+            this.X = onnxYoloItem.X;
+            this.Y = onnxYoloItem.Y;
+            this.Width = onnxYoloItem.Width;
+            this.Height = onnxYoloItem.Height;
+            this.ObjId = onnxYoloItem.ObjId;
+            this.ObjName = onnxYoloItem.ObjName;
+            this.Confidence = onnxYoloItem.Confidence;
+            this.TriggerLineID = onnxYoloItem.TriggerLineID;
+            this.TriggerLine = onnxYoloItem.TriggerLine;
+        }
+
         public Point Center()
         {
             return new Point(this.X + this.Width / 2, this.Y + this.Height / 2);

src/VAP/ORTWrapper/DNNMode.cs

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+namespace Wrapper.ORT
+{
+    public enum DNNMode
+    {
+        Unknown,
+        Frame, //FrameDNN
+        LT, //LineTriggered
+        CC //Cascaded
+    }
+}

src/VAP/ORTWrapper/IYoloConfiguration.cs

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+
+namespace Wrapper.ORT
+{
+    public interface IYoloConfiguration
+    {
+        uint ImageHeight { get; }
+        uint ImageWidth { get; }
+        string[] Labels { get; }
+    }
+}

src/VAP/ORTWrapper/ORTItem.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Wrapper.ORT
+{
+    public class ORTItem
+    {
+        public string ObjName { get; set; }
+        public double Confidence { get; set; }
+        public int X { get; set; }
+        public int Y { get; set; }
+        public int Width { get; set; }
+        public int Height { get; set; }
+        public int ObjId { get; set; }
+        public string TriggerLine { get; set; }
+        public int TriggerLineID { get; set; }
+
+        public ORTItem(int x, int y, int width, int height, int catId, string catName, double confidence, int lineID, string lineName)
+        {
+            this.X = Math.Max(0, x);
+            this.Y = Math.Max(0, y);
+            this.Width = width;
+            this.Height = height;
+            this.ObjId = catId;
+            this.ObjName = catName;
+            this.Confidence = confidence;
+            this.TriggerLineID = lineID;
+            this.TriggerLine = lineName;
+        }
+    }
+}

src/VAP/ORTWrapper/ORTWrapper.cs

@@ -0,0 +1,266 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using Microsoft.ML.OnnxRuntime;
+using Microsoft.ML.OnnxRuntime.Tensors;
+using NumSharp;
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Drawing.Drawing2D;
+using System.Drawing.Imaging;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+namespace Wrapper.ORT
+{
+    public class ORTWrapper
+    {
+        static IYoloConfiguration cfg;
+        static InferenceSession session1, session2;
+        DNNMode mode = DNNMode.Unknown;
+
+        public ORTWrapper(string modelPath, DNNMode mode)
+        {
+            // Optional : Create session options and set the graph optimization level for the session
+            SessionOptions options = new SessionOptions();
+            //options.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_EXTENDED;
+            cfg = new Yolov3BaseConfig();
+
+            this.mode = mode;
+            switch (mode)
+            {
+                case DNNMode.LT:
+                case DNNMode.Frame:
+                    session1 = new InferenceSession(modelPath, SessionOptions.MakeSessionOptionWithCudaProvider(0));
+                    break;
+                case DNNMode.CC:
+                    session2 = new InferenceSession(modelPath, SessionOptions.MakeSessionOptionWithCudaProvider(0));
+                    break;
+            }
+        }
+
+        public List<ORTItem> UseApi(Bitmap bitmap, int h, int w)
+        {
+            float[] imgData = LoadTensorFromImageFile(bitmap);
+
+            var container = new List<NamedOnnxValue>();
+            //yolov3 customization
+            var tensor1 = new DenseTensor<float>(imgData, new int[] { 1, 3, 416, 416 });
+            container.Add(NamedOnnxValue.CreateFromTensor<float>("input_1", tensor1));
+            var tensor2 = new DenseTensor<float>(new float[] { h, w }, new int[] { 1, 2 });
+            container.Add(NamedOnnxValue.CreateFromTensor<float>("image_shape", tensor2));
+
+            // Run the inference
+            switch (mode)
+            {
+                case DNNMode.LT:
+                case DNNMode.Frame:
+                    using (var results = session1.Run(container))  // results is an IDisposableReadOnlyCollection<DisposableNamedOnnxValue> container
+                    {
+                        List<ORTItem> itemList = PostProcessing(results);
+                        return itemList;
+                    }
+                case DNNMode.CC:
+                    using (var results = session2.Run(container))  // results is an IDisposableReadOnlyCollection<DisposableNamedOnnxValue> container
+                    {
+                        List<ORTItem> itemList = PostProcessing(results);
+                        return itemList;
+                    }
+            }
+
+            return null;
+        }
+
+        public static float[] LoadTensorFromPreProcessedFile(string filename)
+        {
+            var tensorData = new List<float>();
+
+            // read data from file
+            using (var inputFile = new StreamReader(filename))
+            {
+                List<string> dataStrList = new List<string>();
+                string line;
+                while ((line = inputFile.ReadLine()) != null)
+                {
+                    dataStrList.AddRange(line.Split(new char[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries));
+                }
+
+                string[] dataStr = dataStrList.ToArray();
+                for (int i = 0; i < dataStr.Length; i++)
+                {
+                    tensorData.Add(Single.Parse(dataStr[i]));
+                }
+            }
+
+            return tensorData.ToArray();
+        }
+
+        static float[] LoadTensorFromImageFile(Bitmap bitmap)
+        {
+            RGBtoBGR(bitmap);
+            int iw = bitmap.Width, ih = bitmap.Height, w = 416, h = 416, nw, nh;
+
+            float scale = Math.Min((float)w/iw, (float)h/ih);
+            nw = (int)(iw * scale);
+            nh = (int)(ih * scale);
+
+            //resize
+            Bitmap rsImg = ResizeImage(bitmap, nw, nh);
+            Bitmap boxedImg = new Bitmap(w, h, PixelFormat.Format24bppRgb);
+            using (Graphics gr = Graphics.FromImage(boxedImg))
+            {
+                gr.FillRectangle(new SolidBrush(Color.FromArgb(255, 128, 128, 128)), 0, 0, boxedImg.Width, boxedImg.Height);
+                gr.DrawImage(rsImg, new Point((int)((w - nw) / 2), (int)((h - nh) / 2)));
+            }
+            var imgData = boxedImg.ToNDArray(flat: false, copy: true);
+            imgData /= 255.0;
+            imgData = np.transpose(imgData, new int[] { 0, 3, 1, 2 });
+            imgData = imgData.reshape(1, 3 * w * h);
+
+            double[] doubleArray = imgData[0].ToArray<double>();
+            float[] floatArray = new float[doubleArray.Length];
+            for (int i = 0; i < doubleArray.Length; i++)
+            {
+                floatArray[i] = (float)doubleArray[i];
+            }
+
+            return floatArray;
+        }
+
+        private static void RGBtoBGR(Bitmap bmp)
+        {
+            BitmapData data = bmp.LockBits(new Rectangle(0, 0, bmp.Width, bmp.Height),
+                                           ImageLockMode.ReadWrite, bmp.PixelFormat);
+
+            int length = Math.Abs(data.Stride) * bmp.Height;
+
+            byte[] imageBytes = new byte[length];
+            IntPtr scan0 = data.Scan0;
+            Marshal.Copy(scan0, imageBytes, 0, imageBytes.Length);
+
+            byte[] rgbValues = new byte[length];
+            for (int i = 0; i < length; i += 3)
+            {
+                rgbValues[i] = imageBytes[i + 2];
+                rgbValues[i + 1] = imageBytes[i + 1];
+                rgbValues[i + 2] = imageBytes[i];
+            }
+            Marshal.Copy(rgbValues, 0, scan0, length);
+
+            bmp.UnlockBits(data);
+        }
+
+        static List<ORTItem> PostProcessing(IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results)
+        {
+            List<ORTItem> itemList = new List<ORTItem>();
+
+            List<float[]> out_boxes = new List<float[]>();
+            List<float[]> out_scores = new List<float[]>();
+            List<int> out_classes = new List<int>();
+
+            var boxes = results.AsEnumerable().ElementAt(0).AsTensor<float>();
+            var scores = results.AsEnumerable().ElementAt(1).AsTensor<float>();
+            var indices = results.AsEnumerable().ElementAt(2).AsTensor<int>();
+            int nbox = indices.Count() / 3;
+
+            for (int ibox = 0; ibox < nbox; ibox++)
+            {
+                out_classes.Add(indices[0, 0, ibox * 3 + 1]);
+
+                float[] score = new float[80];
+                for (int j = 0; j < 80; j++)
+                {
+                    score[j] = scores[indices[0, 0, ibox * 3 + 0], j, indices[0, 0, ibox * 3 + 2]];
+                }
+                out_scores.Add(score);
+
+                float[] box = new float[]
+                {
+                    boxes[indices[0, 0, ibox * 3 + 0], indices[0, 0, ibox * 3 + 2], 0],
+                    boxes[indices[0, 0, ibox * 3 + 0], indices[0, 0, ibox * 3 + 2], 1],
+                    boxes[indices[0, 0, ibox * 3 + 0], indices[0, 0, ibox * 3 + 2], 2],
+                    boxes[indices[0, 0, ibox * 3 + 0], indices[0, 0, ibox * 3 + 2], 3]
+                };
+                out_boxes.Add(box);
+
+                //output
+                ORTItem item = new ORTItem((int)box[1], (int)box[0], (int)(box[3] - box[1]), (int)(box[2] - box[0]), out_classes[ibox], cfg.Labels[out_classes[ibox]], out_scores[ibox][out_classes[ibox]], 0, "lineName");
+                itemList.Add(item);
+            }
+
+            return itemList;
+        }
+
+        public void DrawBoundingBox(Image imageOri,
+            string outputImageLocation,
+            string imageName,
+            List<ORTItem> itemList)
+        {
+            Image image = (Image)imageOri.Clone();
+
+            foreach (var item in itemList)
+            {
+                var x = Math.Max(item.X, 0);
+                var y = Math.Max(item.Y, 0);
+                var width = item.Width;
+                var height = item.Height;
+                string text = $"{item.ObjName} ({(item.Confidence * 100).ToString("0")}%)";
+                using (Graphics thumbnailGraphic = Graphics.FromImage(image))
+                {
+                    thumbnailGraphic.CompositingQuality = CompositingQuality.HighQuality;
+                    thumbnailGraphic.SmoothingMode = SmoothingMode.HighQuality;
+                    thumbnailGraphic.InterpolationMode = InterpolationMode.HighQualityBicubic;
+
+                    Font drawFont = new Font("Arial", 12, FontStyle.Bold);
+                    SizeF size = thumbnailGraphic.MeasureString(text, drawFont);
+                    SolidBrush fontBrush = new SolidBrush(Color.Black);
+                    Point atPoint = new Point((int)(x + width / 2), (int)(y + height / 2) - (int)size.Height - 1);
+
+                    // Define BoundingBox options
+                    Pen pen = new Pen(Color.Pink, 3.2f);
+                    SolidBrush colorBrush = new SolidBrush(Color.Pink);
+
+                    thumbnailGraphic.FillRectangle(colorBrush, (int)(x + width / 2), (int)(y + height / 2 - size.Height - 1), size.Width, (int)size.Height);
+                    thumbnailGraphic.DrawString(text, drawFont, fontBrush, atPoint);
+
+                    // Draw bounding box on image
+                    thumbnailGraphic.DrawRectangle(pen, x, y, width, height);
+                    if (!Directory.Exists(outputImageLocation))
+                    {
+                        Directory.CreateDirectory(outputImageLocation);
+                    }
+
+                    image.Save(Path.Combine(outputImageLocation, imageName));
+                }
+            }
+        }
+
+        private static Bitmap ResizeImage(Bitmap image, int width, int height)
+        {
+            var destRect = new Rectangle(0, 0, width, height);
+            var destImage = new Bitmap(width, height);
+
+            destImage.SetResolution(96, 96); //@todo: image.HorizontalResolution throw exceptions during docker build on linux
+            //destImage.SetResolution(image.HorizontalResolution, image.VerticalResolution);
+
+            using (var graphics = Graphics.FromImage(destImage))
+            {
+                graphics.CompositingMode = CompositingMode.SourceCopy;
+                graphics.CompositingQuality = CompositingQuality.HighQuality;
+                graphics.InterpolationMode = InterpolationMode.HighQualityBicubic;
+                graphics.SmoothingMode = SmoothingMode.HighQuality;
+                graphics.PixelOffsetMode = PixelOffsetMode.HighQuality;
+
+                using (var wrapMode = new ImageAttributes())
+                {
+                    wrapMode.SetWrapMode(WrapMode.TileFlipXY);
+                    graphics.DrawImage(image, destRect, 0, 0, image.Width, image.Height, GraphicsUnit.Pixel, wrapMode);
+                }
+            }
+
+            return destImage;
+        }
+    }
+}

src/VAP/ORTWrapper/ORTWrapper.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <AllowUnsafeBlocks>false</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" Version="1.1.2" />
+    <PackageReference Include="NumSharp" Version="0.20.5" />
+    <PackageReference Include="NumSharp.Bitmap" Version="0.20.4" />
+    <PackageReference Include="System.Drawing.Common" Version="4.7.0" />
+  </ItemGroup>
+
+</Project>

src/VAP/ORTWrapper/Yolov3BaseConfig.cs

@@ -0,0 +1,94 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+namespace Wrapper.ORT
+{
+    public class Yolov3BaseConfig : IYoloConfiguration
+    {
+        public string[] Labels => new string[] {
+        "person",
+        "bicycle",
+        "car",
+        "motorbike",
+        "aeroplane",
+        "bus",
+        "train",
+        "truck",
+        "boat",
+        "traffic light",
+        "fire hydrant",
+        "stop sign",
+        "parking meter",
+        "bench",
+        "bird",
+        "cat",
+        "dog",
+        "horse",
+        "sheep",
+        "cow",
+        "elephant",
+        "bear",
+        "zebra",
+        "giraffe",
+        "backpack",
+        "umbrella",
+        "handbag",
+        "tie",
+        "suitcase",
+        "frisbee",
+        "skis",
+        "snowboard",
+        "sports ball",
+        "kite",
+        "baseball bat",
+        "baseball glove",
+        "skateboard",
+        "surfboard",
+        "tennis racket",
+        "bottle",
+        "wine glass",
+        "cup",
+        "fork",
+        "knife",
+        "spoon",
+        "bowl",
+        "banana",
+        "apple",
+        "sandwich",
+        "orange",
+        "broccoli",
+        "carrot",
+        "hot dog",
+        "pizza",
+        "donut",
+        "cake",
+        "chair",
+        "sofa",
+        "pottedplant",
+        "bed",
+        "diningtable",
+        "toilet",
+        "tvmonitor",
+        "laptop",
+        "mouse",
+        "remote",
+        "keyboard",
+        "cell phone",
+        "microwave",
+        "oven",
+        "toaster",
+        "sink",
+        "refrigerator",
+        "book",
+        "clock",
+        "vase",
+        "scissors",
+        "teddy bear",
+        "hair drier",
+        "toothbrush"
+        };
+
+        public uint ImageWidth => 416;
+        public uint ImageHeight => 416;
+    }
+}

src/VAP/Utils/Config/OutputFolder.cs

@@ -18,5 +18,7 @@
         public static string OutputFolderFrameDNNDarknet { get; set; } = "../../output_framednndarknet/";
 
         public static string OutputFolderFrameDNNTF { get; set; } = "../../output_framednntf/";
+
+        public static string OutputFolderFrameDNNONNX { get; set; } = "../../output_framednnonnx/";
     }
 }

src/VAP/Utils/Utils.cs

@@ -12,7 +12,7 @@ namespace Utils
 {
     public class Utils
     {
-        static void cleanFolder(string folder)
+        public static void cleanFolder(string folder)
         {
             Directory.CreateDirectory(folder);
             DirectoryInfo di = new DirectoryInfo(folder);
@@ -28,6 +28,7 @@ namespace Utils
             cleanFolder(Config.OutputFolder.OutputFolderAML);
             cleanFolder(Config.OutputFolder.OutputFolderFrameDNNDarknet);
             cleanFolder(Config.OutputFolder.OutputFolderFrameDNNTF);
+            cleanFolder(Config.OutputFolder.OutputFolderFrameDNNONNX);
         }
 
         public static byte[] ImageToByteBmp(Image imageIn)
@@ -48,6 +49,12 @@ namespace Utils
             }
         }
 
+        public static float checkLineBboxOverlapRatio(int[] line, int bbox_x, int bbox_y, int bbox_w, int bbox_h)
+        {
+            (Point p1, Point p2) newLine = (new Point(line[0], line[1]), new Point(line[2], line[3]));
+            return checkLineBboxOverlapRatio(newLine, bbox_x, bbox_y, bbox_w, bbox_h);
+        }
+
         public static float checkLineBboxOverlapRatio((Point p1, Point p2) line, int bbox_x, int bbox_y, int bbox_w, int bbox_h)
         {
             float overlapRatio = 0.0F;
@@ -130,5 +137,27 @@ namespace Utils
                 }
             }
         }
+
+        public static List<Tuple<string, int[]>> ConvertLines(List<(string key, (Point p1, Point p2) coordinates)> lines)
+        {
+            List<Tuple<string, int[]>> newLines = new List<Tuple<string, int[]>>();
+            foreach ((string key, (Point p1, Point p2) coordinates) line in lines)
+            {
+                int[] coor = new int[] { line.coordinates.p1.X, line.coordinates.p1.Y, line.coordinates.p2.X, line.coordinates.p2.Y };
+                Tuple<string, int[]> newLine = new Tuple<string, int[]>(line.key, coor);
+                newLines.Add(newLine);
+            }
+            return newLines;
+        }
+
+        public static Dictionary<string, int> CatHashSet2Dict(HashSet<string> cat)
+        {
+            Dictionary<string, int> catDict = new Dictionary<string, int>();
+            foreach (string c in cat)
+            {
+                catDict.Add(c, 0);
+            }
+            return catDict;
+        }
     }
 }

src/VAP/VAP.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.28307.271
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.30309.148
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VideoPipelineCore", "VideoPipelineCore\VideoPipelineCore.csproj", "{FCC65D32-909C-4C0F-AFB2-A5CCA1819E99}"
 EndProject
@@ -11,7 +11,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BGSObjectDetector", "BGSObj
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FramePreProcessor", "FramePreProcessor\FramePreProcessor.csproj", "{F991D5B2-CDF3-419A-9D20-7FBDCAAD61F6}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YoloWrapper", "YoloWrapper\YoloWrapper.csproj", "{992D9B38-FCC1-406A-92A2-6C1B5A310CCF}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YoloWrapper", "YoloWrapper\YoloWrapper.csproj", "{992D9B38-FCC1-406A-92A2-6C1B5A310CCF}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AML.Client", "AML.Client\AML.Client.csproj", "{399AB73B-DE90-4FAE-9429-7ACB68FFC442}"
 EndProject
@@ -31,6 +31,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DNNDetector", "DNNDetector\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LineDetector", "LineDetector\LineDetector.csproj", "{068A7FE9-F45E-4F36-BA6A-6E64ED63DA78}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ORTWrapper", "ORTWrapper\ORTWrapper.csproj", "{4B1EBDD7-928E-4B7E-A468-C258FF4BC325}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -93,6 +95,10 @@ Global
 		{068A7FE9-F45E-4F36-BA6A-6E64ED63DA78}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{068A7FE9-F45E-4F36-BA6A-6E64ED63DA78}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{068A7FE9-F45E-4F36-BA6A-6E64ED63DA78}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4B1EBDD7-928E-4B7E-A468-C258FF4BC325}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4B1EBDD7-928E-4B7E-A468-C258FF4BC325}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4B1EBDD7-928E-4B7E-A468-C258FF4BC325}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4B1EBDD7-928E-4B7E-A468-C258FF4BC325}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/VAP/VideoPipelineCore/App.config

@@ -9,6 +9,8 @@
     3: BGS early filtering -> Darknet Tiny Yolo -> Darknet Yolo v3
     4: BGS early filtering -> Darknet Tiny Yolo -> Database (ArangoDB and blob storage on Azure)
     5: BGS early filtering -> TensorFlow Fast R-CNN -> Azure Machine Learning (cloud)
+    6: BGS early filtering -> ONNX Tiny Yolo v3 -> ONNX Yolo v3
+    7: ONNX Yolo v3 on every frame
     -->
     <add key="PplConfig" value="1" />
     

src/VAP/VideoPipelineCore/Program.cs

@@ -4,6 +4,8 @@
 using AML.Client;
 using BGSObjectDetector;
 using DarknetDetector;
+using DNNDetector;
+using DNNDetector.Config;
 using DNNDetector.Model;
 using LineDetector;
 using OpenCvSharp;
@@ -88,13 +90,31 @@ namespace VideoPipelineCore
                 ltDNNItemListTF = new List<Item>();
             }
 
+            //-----LineTriggeredDNN (ONNX)-----
+            LineTriggeredDNNORTYolo ltDNNOnnx = null;
+            List<Item> ltDNNItemListOnnx = null;
+            if (new int[] { 6 }.Contains(pplConfig))
+            {
+                ltDNNOnnx = new LineTriggeredDNNORTYolo(Utils.Utils.ConvertLines(lines), "yolov3tiny");
+                ltDNNItemListOnnx = new List<Item>();
+            }
+
             //-----CascadedDNN (Darknet)-----
             CascadedDNNDarknet ccDNNDarknet = null;
-            List<Item> ccDNNItemList = null;
+            List<Item> ccDNNItemListDarknet = null;
             if (new int[] { 3 }.Contains(pplConfig))
             {
                 ccDNNDarknet = new CascadedDNNDarknet(lines);
-                ccDNNItemList = new List<Item>();
+                ccDNNItemListDarknet = new List<Item>();
+            }
+
+            //-----CascadedDNN (ONNX)-----
+            CascadedDNNORTYolo ccDNNOnnx = null;
+            List<Item> ccDNNItemListOnnx = null;
+            if (new int[] { 6 }.Contains(pplConfig))
+            {
+                ccDNNOnnx = new CascadedDNNORTYolo(Utils.Utils.ConvertLines(lines), "yolov3");
+                ccDNNItemListOnnx = new List<Item>();
             }
 
             //-----DNN on every frame (Darknet)-----
@@ -115,6 +135,15 @@ namespace VideoPipelineCore
                 frameDNNTFItemList = new List<Item>();
             }
 
+            //-----DNN on every frame (ONNX)-----
+            FrameDNNOnnxYolo frameDNNOnnxYolo = null;
+            List<Item> frameDNNONNXItemList = null;
+            if (new int[] { 7 }.Contains(pplConfig))
+            {
+                frameDNNOnnxYolo = new FrameDNNOnnxYolo(Utils.Utils.ConvertLines(lines), "yolov3", Wrapper.ORT.DNNMode.Frame);
+                frameDNNONNXItemList = new List<Item>();
+            }
+
             //-----Call ML models deployed on Azure Machine Learning Workspace-----
             AMLCaller amlCaller = null;
             List<bool> amlConfirmed;
@@ -134,6 +163,7 @@ namespace VideoPipelineCore
             if (!isVideoStream)
                 videoTotalFrame = decoder.getTotalFrameNum() - 1; //skip the last frame which could be wrongly encoded from vlc capture
 
+            long teleCountsBGS = 0, teleCountsCheapDNN = 0, teleCountsHeavyDNN = 0;
 
             //RUN PIPELINE 
             DateTime startTime = DateTime.Now;
@@ -165,7 +195,7 @@ namespace VideoPipelineCore
                 
 
                 //line detector
-                if (new int[] { 0, 3, 4, 5 }.Contains(pplConfig))
+                if (new int[] { 0, 3, 4, 5, 6 }.Contains(pplConfig))
                 {
                     (counts, occupancy) = lineDetector.updateLineResults(frame, frameIndex, fgmask, foregroundBoxes);
                 }
@@ -182,13 +212,23 @@ namespace VideoPipelineCore
                     ltDNNItemListTF = ltDNNTF.Run(frame, frameIndex, occupancy, lines, category);
                     ItemList = ltDNNItemListTF;
                 }
+                else if (new int[] { 6 }.Contains(pplConfig))
+                {
+                    ltDNNItemListOnnx = ltDNNOnnx.Run(frame, frameIndex, counts, Utils.Utils.ConvertLines(lines), Utils.Utils.CatHashSet2Dict(category), ref teleCountsCheapDNN, true);
+                    ItemList = ltDNNItemListOnnx;
+                }
 
 
                 //heavy DNN
                 if (new int[] { 3 }.Contains(pplConfig))
                 {
-                    ccDNNItemList = ccDNNDarknet.Run(frame, frameIndex, ltDNNItemListDarknet, lines, category);
-                    ItemList = ccDNNItemList;
+                    ccDNNItemListDarknet = ccDNNDarknet.Run(frame, frameIndex, ltDNNItemListDarknet, lines, category);
+                    ItemList = ccDNNItemListDarknet;
+                }
+                else if (new int[] { 6 }.Contains(pplConfig))
+                {
+                    ccDNNItemListOnnx = ccDNNOnnx.Run(frameIndex, ItemList, Utils.Utils.ConvertLines(lines), Utils.Utils.CatHashSet2Dict(category), ref teleCountsHeavyDNN, true);
+                    ItemList = ccDNNItemListOnnx;
                 }
 
 
@@ -208,6 +248,14 @@ namespace VideoPipelineCore
                 }
 
 
+                //frame DNN ONNX Yolo
+                if (new int[] { 7 }.Contains(pplConfig))
+                {
+                    frameDNNONNXItemList = frameDNNOnnxYolo.Run(frame, frameIndex, Utils.Utils.CatHashSet2Dict(category), System.Drawing.Brushes.Pink, 0, DNNConfig.MIN_SCORE_FOR_LINEBBOX_OVERLAP_SMALL, true);
+                    ItemList = frameDNNONNXItemList;
+                }
+
+
                 //Azure Machine Learning
                 if (new int[] { 5 }.Contains(pplConfig))
                 {