Changes to test project to include emotion recognition

EmotionRecognizer/EmotionRecognizer.cs

@@ -128,6 +128,19 @@ namespace CommunityToolkit.Labs.Intelligent.EmotionRecognition
             return await instance.EvaluateFrame(bitmap);
         }
 
+        /// <summary>
+        /// Detect emotion on a face in an input image
+        /// </summary>
+        /// <param name="bitmap"></param>
+        /// <returns>
+        /// Returns detected emotion in DetectedEmotion object, null if no face was detected.
+        /// </returns>
+        public async static Task<DetectedEmotion> DetectEmotion(StorageFile storageFile)
+        {
+            SoftwareBitmap bitmap = await GenerateSoftwareBitmapFromStorageFile(storageFile);
+            return await DetectEmotion(bitmap);
+        }
+
         /// <summary>
         /// Evaluate frame
         /// </summary>
@@ -190,6 +203,41 @@ namespace CommunityToolkit.Labs.Intelligent.EmotionRecognition
             return detectedFace;
         }
 
+        /// <summary>
+        /// Converts object of type StorageFile to SoftwareBitmap
+        /// </summary>
+        /// <param name="selectedStorageFile"></param>
+        /// <returns></returns>
+        private static async Task<SoftwareBitmap> GenerateSoftwareBitmapFromStorageFile(StorageFile selectedStorageFile)
+        {
+            SoftwareBitmap softwareBitmap;
+            using (IRandomAccessStream stream = await selectedStorageFile.OpenAsync(FileAccessMode.Read))
+            {
+                // Create the decoder from the stream 
+                BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
+
+                // Get the SoftwareBitmap representation of the file in BGRA8 format
+                softwareBitmap = await decoder.GetSoftwareBitmapAsync();
+                softwareBitmap = GetSoftwareBitmap(softwareBitmap);
+            }
+
+            return softwareBitmap;
+        }
+
+        /// <summary>
+        /// Get Software Bitmap
+        /// </summary>
+        /// <param name="softwareBitmap"></param>
+        /// <returns></returns>
+        private static SoftwareBitmap GetSoftwareBitmap(SoftwareBitmap softwareBitmap)
+        {
+            if (softwareBitmap.BitmapPixelFormat != BitmapPixelFormat.Bgra8 || (softwareBitmap.BitmapAlphaMode != BitmapAlphaMode.Ignore && softwareBitmap.BitmapAlphaMode != BitmapAlphaMode.Premultiplied))
+            {
+                softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
+            }
+            return softwareBitmap;
+        }
+
         private static async Task<SoftwareBitmap> Crop(SoftwareBitmap softwareBitmap, Rect bounds)
         {
             VideoFrame vid = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);

ImageClassifier/SqueezeNetImageClassifier.cs

@@ -70,7 +70,7 @@ namespace CommunityToolkit.Labs.Intelligent.ImageClassification
         {
             CreateInstanceIfNone();
             SoftwareBitmap softwareBitmap = await GenerateSoftwareBitmapFromStorageFile(selectedStorageFile);
-            VideoFrame videoFrame = await GenerateVideoFrameFromBitmap(softwareBitmap);
+            VideoFrame videoFrame = GenerateVideoFrameFromBitmap(softwareBitmap);
             return await instance.EvaluateModel(videoFrame, top);
         }
 
@@ -84,7 +84,7 @@ namespace CommunityToolkit.Labs.Intelligent.ImageClassification
         {
             CreateInstanceIfNone();
             softwareBitmap = GetSoftwareBitmap(softwareBitmap);
-            VideoFrame videoFrame = await GenerateVideoFrameFromBitmap(softwareBitmap);
+            VideoFrame videoFrame = GenerateVideoFrameFromBitmap(softwareBitmap);
             return await instance.EvaluateModel(videoFrame, top);
         }
 
@@ -129,11 +129,8 @@ namespace CommunityToolkit.Labs.Intelligent.ImageClassification
         /// </summary>
         /// <param name="softwareBitmap"></param>
         /// <returns></returns>
-        private static async Task<VideoFrame> GenerateVideoFrameFromBitmap(SoftwareBitmap softwareBitmap)
+        private static VideoFrame GenerateVideoFrameFromBitmap(SoftwareBitmap softwareBitmap)
         {
-            SoftwareBitmapSource imageSource = new SoftwareBitmapSource();
-            await imageSource.SetBitmapAsync(softwareBitmap);
-
             // Encapsulate the image within a VideoFrame to be bound and evaluated
             VideoFrame videoFrame = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
             return videoFrame;
@@ -170,7 +167,6 @@ namespace CommunityToolkit.Labs.Intelligent.ImageClassification
         /// <returns></returns>
         private static SoftwareBitmap GetSoftwareBitmap(SoftwareBitmap softwareBitmap)
         {
-            SoftwareBitmapSource imageSource = new SoftwareBitmapSource();
             if (softwareBitmap.BitmapPixelFormat != BitmapPixelFormat.Bgra8 || (softwareBitmap.BitmapAlphaMode != BitmapAlphaMode.Ignore && softwareBitmap.BitmapAlphaMode != BitmapAlphaMode.Premultiplied))
             {
                 softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);

IntelligentAPIsTester/IntelligentLabsTest.csproj

@@ -166,6 +166,10 @@
     </PackageReference>
   </ItemGroup>
   <ItemGroup>
+    <ProjectReference Include="..\EmotionRecognizer\IntelligentAPI_EmotionRecognizer.csproj">
+      <Project>{784a3c01-bd0b-4db9-b1ff-2a16e2b03a65}</Project>
+      <Name>IntelligentAPI_EmotionRecognizer</Name>
+    </ProjectReference>
     <ProjectReference Include="..\ImageClassifier\IntelligentAPI_ImageClassifier.csproj">
       <Project>{f705b499-d0b5-408c-b1d1-d3f379d0fcd6}</Project>
       <Name>IntelligentAPI_ImageClassifier</Name>

IntelligentAPIsTester/MainPage.xaml

@@ -14,7 +14,7 @@
             <ImageBrush Stretch="Fill" ImageSource="./Assets/background.jpg" Opacity="0.7"></ImageBrush>
         </Grid.Background>
         <StackPanel HorizontalAlignment="Center" VerticalAlignment="Center" Spacing="20" Background="White" Padding="20" CornerRadius="20">
-            <TextBlock>Welcome to IntelligentAPIs Test Project! Pick an image to continue.</TextBlock>
+            <TextBlock>Welcome to IntelligentAPIs Test Project! Pick an image to try Image Classification and Object Detection.</TextBlock>
             <Button Name="FilePickerButton"
                         ToolTipService.ToolTip="Pick and evaluate and image"
                         Width="100"
@@ -26,6 +26,18 @@
                     Pick image
                 </Button.Content>
             </Button>
+            <TextBlock HorizontalAlignment="Center">Capture an image on your webcam to try Emotion Detection.</TextBlock>
+            <Button Name="CaptureImageButton"
+                        ToolTipService.ToolTip="Capture image using webcam"
+                        Width="130"
+                        Height="40"
+                        Click="CaptureImageButton_Click" 
+                        IsEnabled="True" 
+                        HorizontalAlignment="Center">
+                <Button.Content>
+                    Webcam capture
+                </Button.Content>
+            </Button>
             <ProgressRing Name="FilePickerProgressRing"></ProgressRing>
         </StackPanel>
     </Grid>

IntelligentAPIsTester/MainPage.xaml.cs

@@ -13,8 +13,8 @@ using CommunityToolkit.Labs.Intelligent.ImageClassification;
 using CommunityToolkit.Labs.Intelligent.ObjectDetection;
 using Windows.UI.Xaml.Shapes;
 using Windows.UI;
-
-
+using Windows.Media.Capture;
+using Windows.Foundation;
 namespace IntelligentLabsTest
 {
     /// <summary>
@@ -46,7 +46,8 @@ namespace IntelligentLabsTest
             if (selectedStorageFile != null)
             {
                 Frame rootFrame = Window.Current.Content as Frame;
-                rootFrame.Navigate(typeof(ResultsPage), selectedStorageFile);
+                Input input = new Input() { file = selectedStorageFile, typeOfInput = TypeOfInput.File };
+                rootFrame.Navigate(typeof(ResultsPage), input);
             }
             else
             {
@@ -54,5 +55,33 @@ namespace IntelligentLabsTest
                 FilePickerProgressRing.IsActive = false;
             }
         }
+
+        private async void CaptureImageButton_Click(object sender, RoutedEventArgs e)
+        {
+            CameraCaptureUI dialog = new CameraCaptureUI();
+            Size aspectRatio = new Size(16, 9);
+            dialog.PhotoSettings.CroppedAspectRatio = aspectRatio;
+
+            StorageFile file = await dialog.CaptureFileAsync(CameraCaptureUIMode.Photo);
+
+            if(file != null)
+            {
+                Frame rootFrame = Window.Current.Content as Frame;
+                Input input = new Input() { file = file, typeOfInput = TypeOfInput.Camera };
+                rootFrame.Navigate(typeof(ResultsPage), input);
+            }
+        }
+
+        public enum TypeOfInput
+        {
+            Camera,
+            File
+        }
+
+        public class Input
+        {
+            public StorageFile file;
+            public TypeOfInput typeOfInput;
+        }
     }
 }

IntelligentAPIsTester/ResultsPage.xaml.cs

@@ -1,6 +1,7 @@
 
 using CommunityToolkit.Labs.Intelligent.ImageClassification;
 using CommunityToolkit.Labs.Intelligent.ObjectDetection;
+using CommunityToolkit.Labs.Intelligent.EmotionRecognition;
 using System;
 using System.Collections.Generic;
 using System.IO;
@@ -10,6 +11,7 @@ using System.Threading.Tasks;
 using Windows.Foundation;
 using Windows.Foundation.Collections;
 using Windows.Graphics.Imaging;
+using Windows.Media.Capture;
 using Windows.Storage;
 using Windows.Storage.Streams;
 using Windows.System.Profile;
@@ -60,37 +62,47 @@ namespace IntelligentLabsTest
         protected async override void OnNavigatedTo(NavigationEventArgs e)
         {
             Frame rootFrame = Window.Current.Content as Frame;
-            if (!(e.Parameter is StorageFile selectedStorageFile))
+            if (!(e.Parameter is MainPage.Input input))
             {
                 rootFrame.GoBack();
                 return;
             }
 
-            await DisplayImage(selectedStorageFile);
+            await DisplayImage(input.file);
 
-            try
+            if (input.typeOfInput.Equals(MainPage.TypeOfInput.File))
             {
-                //Use Squeezenet model to classify image
-                List<ClassificationResult> imageClasses = await SqueezeNetImageClassifier.ClassifyImage(selectedStorageFile, 3 );
-                UpdateTextBox(imageClasses);
-
-              
-            }
-            catch(Exception exc)
-            {
-               
-                if(exc is ArgumentOutOfRangeException)
+                try
                 {
-                    ResultsBlock.Text = exc.Message;
-                }
-            }
+                    //Use Squeezenet model to classify image
+                    List<ClassificationResult> imageClasses = await SqueezeNetImageClassifier.ClassifyImage(input.file, 3);
+                    UpdateTextBoxForImageClassification(imageClasses);
 
-            
-            //Use YOLOv4 to detect objects. WORKS ONLY IF YOU ARE RUNNING WINDOWS 11!!
-            if (CheckWindowsBuildNumber())
+
+                }
+                catch (Exception exc)
+                {
+
+                    if (exc is ArgumentOutOfRangeException)
+                    {
+                        ResultsBlock.Text = exc.Message;
+                    }
+                }
+
+
+                //Use YOLOv4 to detect objects. WORKS ONLY IF YOU ARE RUNNING WINDOWS 11!!
+                if (CheckWindowsBuildNumber())
+                {
+                    List<DetectionResult> listOfObjects = await YOLOObjectDetector.DetectObjects(input.file);
+                    DrawBoxes(listOfObjects);
+                }
+
+            }
+            else
             {
-                List<DetectionResult> listOfObjects = await YOLOObjectDetector.DetectObjects(selectedStorageFile);
-                DrawBoxes(listOfObjects);
+                DetectedEmotion detectedEmotion = await EmotionRecognizer.DetectEmotion(input.file);
+                UpdateTextBoxForEmotionRecognition(detectedEmotion);
+
             }
 
             ProgressRing.IsActive = false;
@@ -139,7 +151,7 @@ namespace IntelligentLabsTest
         /// <param name="imageClasses"></param>
         /// <param name="listOfObjects"></param>
         /// <returns></returns>
-        private void UpdateTextBox(List<ClassificationResult> imageClasses)
+        private void UpdateTextBoxForImageClassification(List<ClassificationResult> imageClasses)
         {
             ResultsBlock.Text = "";
 
@@ -162,6 +174,27 @@ namespace IntelligentLabsTest
 
         }
 
+        /// <summary>
+        /// Updates the text box with results from image classification and object detection
+        /// </summary>
+        /// <param name="imageClasses"></param>
+        /// <param name="listOfObjects"></param>
+        /// <returns></returns>
+        private void UpdateTextBoxForEmotionRecognition(DetectedEmotion detectedEmotion)
+        {
+            ResultsBlock.Text = "";
+
+            if (detectedEmotion == null)
+            {
+                ResultsBlock.Text = "No face detected";
+                return;
+            }
+
+
+            ResultsBlock.Text += "Detected Emotion: " + detectedEmotion.emotion;
+
+        }
+
         /// <summary>
         /// Draws bounding boxes on the output frame based on evaluation result
         /// </summary>