Added emotion pacakge (#14)
* added new project for emotion recognition * emotion package Co-authored-by: Amrutha Srinivasan <amsrin@microsoft.com>
This commit is contained in:
Родитель
1d2f655392
Коммит
88acd74a0b
|
@ -0,0 +1,164 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Windows.AI.MachineLearning;
|
||||||
|
using Windows.Foundation;
|
||||||
|
using Windows.Graphics.Imaging;
|
||||||
|
using Windows.Media;
|
||||||
|
using Windows.Media.FaceAnalysis;
|
||||||
|
using Windows.Storage;
|
||||||
|
using Windows.Storage.Streams;
|
||||||
|
|
||||||
|
namespace CommunityToolkit.Labs.Intelligent.EmotionRecognition
|
||||||
|
{
|
||||||
|
public class DetectedEmotion
|
||||||
|
{
|
||||||
|
public int emotionIndex;
|
||||||
|
public string emotion;
|
||||||
|
}
|
||||||
|
public class EmotionRecognizer
|
||||||
|
{
|
||||||
|
private LearningModel _model = null;
|
||||||
|
private LearningModelSession _session = null;
|
||||||
|
private LearningModelBinding _binding = null;
|
||||||
|
private static EmotionRecognizer instance = null;
|
||||||
|
|
||||||
|
private static List<string> labels;
|
||||||
|
|
||||||
|
|
||||||
|
private async void InitModelAsync()
|
||||||
|
{
|
||||||
|
// load model file
|
||||||
|
var file = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///IntelligentAPI_EmotionRecognizer/Assets/model_emotion.onnx"));
|
||||||
|
|
||||||
|
//Loads the mdoel from the file
|
||||||
|
_model = await LearningModel.LoadFromStorageFileAsync(file);
|
||||||
|
|
||||||
|
//Creating a session that binds the model to the device running the model
|
||||||
|
_session = new LearningModelSession(_model, new LearningModelDevice(GetDeviceKind()));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LoadLabels()
|
||||||
|
{
|
||||||
|
labels = new List<string>()
|
||||||
|
{
|
||||||
|
"Neutral",
|
||||||
|
"Happiness",
|
||||||
|
"Surprise",
|
||||||
|
"Sadness",
|
||||||
|
"Anger",
|
||||||
|
"Disgust",
|
||||||
|
"Fear",
|
||||||
|
"Contempt"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
LearningModelDeviceKind GetDeviceKind()
|
||||||
|
{
|
||||||
|
return LearningModelDeviceKind.Cpu;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async static Task<IList<DetectedFace>> DetectFacesInImageAsync(SoftwareBitmap bitmap)
|
||||||
|
{
|
||||||
|
FaceDetector faceDetector = await FaceDetector.CreateAsync();
|
||||||
|
var convertedBitmap = SoftwareBitmap.Convert(bitmap, BitmapPixelFormat.Gray8);
|
||||||
|
return await faceDetector.DetectFacesAsync(convertedBitmap);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public async static Task<DetectedEmotion> DetectEmotion(SoftwareBitmap bitmap)
|
||||||
|
{
|
||||||
|
if (instance == null)
|
||||||
|
{
|
||||||
|
instance = new EmotionRecognizer();
|
||||||
|
}
|
||||||
|
|
||||||
|
return await instance.EvaluateFrame(bitmap);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<DetectedEmotion> EvaluateFrame(SoftwareBitmap softwareBitmap)
|
||||||
|
{
|
||||||
|
InitModelAsync();
|
||||||
|
LoadLabels();
|
||||||
|
DetectedFace detectedFace = await DetectFace(softwareBitmap);
|
||||||
|
if (detectedFace != null)
|
||||||
|
{
|
||||||
|
return await EvaluateEmotionInFace(detectedFace, softwareBitmap);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<DetectedEmotion> EvaluateEmotionInFace(DetectedFace detectedFace, SoftwareBitmap softwareBitmap)
|
||||||
|
{
|
||||||
|
|
||||||
|
var boundingBox = new Rect(detectedFace.FaceBox.X,
|
||||||
|
detectedFace.FaceBox.Y,
|
||||||
|
detectedFace.FaceBox.Width,
|
||||||
|
detectedFace.FaceBox.Height);
|
||||||
|
|
||||||
|
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8);
|
||||||
|
|
||||||
|
var croppedFace = await Crop(softwareBitmap, boundingBox);
|
||||||
|
LearningModelEvaluationResult emotionResults = await BindAndEvaluateModelAsync(croppedFace);
|
||||||
|
|
||||||
|
// to get percentages, you'd need to run the output through a softmax function
|
||||||
|
// we don't need percentages, we just need max value
|
||||||
|
TensorFloat emotionIndexTensor = emotionResults.Outputs["Plus692_Output_0"] as TensorFloat;
|
||||||
|
|
||||||
|
var emotionList = emotionIndexTensor.GetAsVectorView().ToList();
|
||||||
|
var emotionIndex = emotionList.IndexOf(emotionList.Max());
|
||||||
|
|
||||||
|
return new DetectedEmotion() { emotionIndex = emotionIndex, emotion = labels[emotionIndex] };
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<DetectedFace> DetectFace(SoftwareBitmap softwareBitmap)
|
||||||
|
{
|
||||||
|
var faces = await DetectFacesInImageAsync(softwareBitmap);
|
||||||
|
|
||||||
|
// if there is a face in the frame, evaluate the emotion
|
||||||
|
var detectedFace = faces.FirstOrDefault();
|
||||||
|
return detectedFace;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async Task<SoftwareBitmap> Crop(SoftwareBitmap softwareBitmap, Rect bounds)
|
||||||
|
{
|
||||||
|
VideoFrame vid = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
|
||||||
|
vid = await Crop(vid, bounds);
|
||||||
|
return vid.SoftwareBitmap;
|
||||||
|
|
||||||
|
}
|
||||||
|
public static async Task<VideoFrame> Crop(VideoFrame videoFrame, Rect bounds)
|
||||||
|
{
|
||||||
|
BitmapBounds cropBounds = new BitmapBounds()
|
||||||
|
{
|
||||||
|
Width = (uint)bounds.Width,
|
||||||
|
Height = (uint)bounds.Height,
|
||||||
|
X = (uint)bounds.X,
|
||||||
|
Y = (uint)bounds.Y
|
||||||
|
};
|
||||||
|
VideoFrame result = new VideoFrame(BitmapPixelFormat.Bgra8,
|
||||||
|
(int)cropBounds.Width,
|
||||||
|
(int)cropBounds.Height,
|
||||||
|
BitmapAlphaMode.Premultiplied);
|
||||||
|
|
||||||
|
await videoFrame.CopyToAsync(result, cropBounds, null);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private async Task<LearningModelEvaluationResult> BindAndEvaluateModelAsync(SoftwareBitmap croppedFace)
|
||||||
|
{
|
||||||
|
//Create Learning model binding which binds
|
||||||
|
_binding = new LearningModelBinding(_session);
|
||||||
|
_binding.Bind("Input3", VideoFrame.CreateWithSoftwareBitmap(croppedFace));
|
||||||
|
var emotionResults = await _session.EvaluateAsync(_binding, "id");
|
||||||
|
return emotionResults;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,24 @@
|
||||||
|
<Project Sdk="MSBuild.Sdk.Extras">
|
||||||
|
<PropertyGroup>
|
||||||
|
<RootNamespace>IntelligentAPI.EmotionRecognition</RootNamespace>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup>
|
||||||
|
<PackageId>CommunityToolkit.Labs.Intelligent.EmotionRecognition</PackageId>
|
||||||
|
<Description>
|
||||||
|
This package performs Emotion Recognition on an input image by using the Emotion FERPlus model.
|
||||||
|
</Description>
|
||||||
|
<Version>0.0.1</Version>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<EmbeddedResource Include="Properties\IntelligentAPI_EmotionRecognizer.rd.xml" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Content Include="Assets\model_emotion.onnx" Pack="True" PackagePath="lib/uap10.0.17763/Assets"/>
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<None Include="ImageClassifier.licenseheader" />
|
||||||
|
</ItemGroup>
|
||||||
|
<Target Name="CustomBeforeBuild" BeforeTargets="BeforeBuild">
|
||||||
|
<Exec Command="powershell.exe –NonInteractive –ExecutionPolicy Unrestricted -command "& {.\Scripts\script.ps1 }"" />
|
||||||
|
</Target>
|
||||||
|
</Project>
|
|
@ -0,0 +1,21 @@
|
||||||
|
using System.Reflection;
|
||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
|
// General Information about an assembly is controlled through the following
|
||||||
|
// set of attributes. Change these attribute values to modify the information
|
||||||
|
// associated with an assembly.
|
||||||
|
[assembly: AssemblyTrademark("")]
|
||||||
|
[assembly: AssemblyCulture("")]
|
||||||
|
|
||||||
|
// Version information for an assembly consists of the following four values:
|
||||||
|
//
|
||||||
|
// Major Version
|
||||||
|
// Minor Version
|
||||||
|
// Build Number
|
||||||
|
// Revision
|
||||||
|
//
|
||||||
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
|
// by using the '*' as shown below:
|
||||||
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
|
[assembly: ComVisible(false)]
|
|
@ -0,0 +1,33 @@
|
||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<!--
|
||||||
|
This file contains Runtime Directives, specifications about types your application accesses
|
||||||
|
through reflection and other dynamic code patterns. Runtime Directives are used to control the
|
||||||
|
.NET Native optimizer and ensure that it does not remove code accessed by your library. If your
|
||||||
|
library does not do any reflection, then you generally do not need to edit this file. However,
|
||||||
|
if your library reflects over types, especially types passed to it or derived from its types,
|
||||||
|
then you should write Runtime Directives.
|
||||||
|
|
||||||
|
The most common use of reflection in libraries is to discover information about types passed
|
||||||
|
to the library. Runtime Directives have three ways to express requirements on types passed to
|
||||||
|
your library.
|
||||||
|
|
||||||
|
1. Parameter, GenericParameter, TypeParameter, TypeEnumerableParameter
|
||||||
|
Use these directives to reflect over types passed as a parameter.
|
||||||
|
|
||||||
|
2. SubTypes
|
||||||
|
Use a SubTypes directive to reflect over types derived from another type.
|
||||||
|
|
||||||
|
3. AttributeImplies
|
||||||
|
Use an AttributeImplies directive to indicate that your library needs to reflect over
|
||||||
|
types or methods decorated with an attribute.
|
||||||
|
|
||||||
|
For more information on writing Runtime Directives for libraries, please visit
|
||||||
|
https://go.microsoft.com/fwlink/?LinkID=391919
|
||||||
|
-->
|
||||||
|
<Directives xmlns="http://schemas.microsoft.com/netfx/2013/01/metadata">
|
||||||
|
<Library Name="IntelligentAPI_EmotionRecognizer">
|
||||||
|
|
||||||
|
<!-- add directives for your library here -->
|
||||||
|
|
||||||
|
</Library>
|
||||||
|
</Directives>
|
|
@ -0,0 +1,15 @@
|
||||||
|
$ProgressPreference = 'SilentlyContinue'
|
||||||
|
|
||||||
|
$emotionferplusfile = "./Assets/model_emotion.onnx"
|
||||||
|
if (-not(Test-Path -Path $emotionferplusfile -PathType Leaf)) {
|
||||||
|
try {
|
||||||
|
Invoke-WebRequest -URI "https://github.com/onnx/models/raw/master/vision/body_analysis/emotion_ferplus/model/emotion-ferplus-8.onnx" -OutFile $emotionferplusfile
|
||||||
|
Write-Host "The file [$emotionferplusfile] has been created."
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
throw $_.Exception.Message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Write-Host "The file [$emotionferplusfile] exists."
|
||||||
|
}
|
|
@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IntelligentAPI_ImageClassif
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntelligentLabsTest", "IntelligentAPIsTester\IntelligentLabsTest.csproj", "{47D87733-B357-4706-88BD-211FB7D8679D}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntelligentLabsTest", "IntelligentAPIsTester\IntelligentLabsTest.csproj", "{47D87733-B357-4706-88BD-211FB7D8679D}"
|
||||||
EndProject
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntelligentAPI_EmotionRecognizer", "IntelligentAPI_EmotionRecognizer\IntelligentAPI_EmotionRecognizer.csproj", "{2AAF76B9-2B93-4932-909C-09BBF54731FC}"
|
||||||
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
|
@ -123,6 +125,36 @@ Global
|
||||||
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.ActiveCfg = Release|x86
|
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.ActiveCfg = Release|x86
|
||||||
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.Build.0 = Release|x86
|
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.Build.0 = Release|x86
|
||||||
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.Deploy.0 = Release|x86
|
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.Deploy.0 = Release|x86
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|ARM.ActiveCfg = Debug|ARM
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|ARM.Build.0 = Debug|ARM
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|ARM64.ActiveCfg = Debug|ARM64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|ARM64.Build.0 = Debug|ARM64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|x64.Build.0 = Debug|x64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|x86.ActiveCfg = Debug|x86
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|x86.Build.0 = Debug|x86
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|ARM.ActiveCfg = Debug|ARM
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|ARM.Build.0 = Debug|ARM
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|ARM64.ActiveCfg = Debug|ARM64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|ARM64.Build.0 = Debug|ARM64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|x64.ActiveCfg = Debug|x64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|x64.Build.0 = Debug|x64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|x86.ActiveCfg = Debug|x86
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|x86.Build.0 = Debug|x86
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|ARM.ActiveCfg = Release|ARM
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|ARM.Build.0 = Release|ARM
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|ARM64.ActiveCfg = Release|ARM64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|ARM64.Build.0 = Release|ARM64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|x64.ActiveCfg = Release|x64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|x64.Build.0 = Release|x64
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|x86.ActiveCfg = Release|x86
|
||||||
|
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|x86.Build.0 = Release|x86
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
|
|
@ -170,6 +170,10 @@
|
||||||
<Project>{f705b499-d0b5-408c-b1d1-d3f379d0fcd6}</Project>
|
<Project>{f705b499-d0b5-408c-b1d1-d3f379d0fcd6}</Project>
|
||||||
<Name>IntelligentAPI_ImageClassifier</Name>
|
<Name>IntelligentAPI_ImageClassifier</Name>
|
||||||
</ProjectReference>
|
</ProjectReference>
|
||||||
|
<ProjectReference Include="..\IntelligentAPI_EmotionRecognizer\IntelligentAPI_EmotionRecognizer.csproj">
|
||||||
|
<Project>{2aaf76b9-2b93-4932-909c-09bbf54731fc}</Project>
|
||||||
|
<Name>IntelligentAPI_EmotionRecognizer</Name>
|
||||||
|
</ProjectReference>
|
||||||
<ProjectReference Include="..\ObjectDetector\IntelligentAPI_ObjectDetector.csproj">
|
<ProjectReference Include="..\ObjectDetector\IntelligentAPI_ObjectDetector.csproj">
|
||||||
<Project>{6f985dc2-db68-49e5-add8-6d11f35e552e}</Project>
|
<Project>{6f985dc2-db68-49e5-add8-6d11f35e552e}</Project>
|
||||||
<Name>IntelligentAPI_ObjectDetector</Name>
|
<Name>IntelligentAPI_ObjectDetector</Name>
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
|
|
||||||
using CommunityToolkit.Labs.Intelligent.ImageClassification;
|
using CommunityToolkit.Labs.Intelligent.ImageClassification;
|
||||||
using CommunityToolkit.Labs.Intelligent.ObjectDetection;
|
using CommunityToolkit.Labs.Intelligent.ObjectDetection;
|
||||||
|
using CommunityToolkit.Labs.Intelligent.EmotionRecognition;
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
@ -73,6 +74,8 @@ namespace IntelligentLabsTest
|
||||||
//Use Squeezenet model to classify image
|
//Use Squeezenet model to classify image
|
||||||
List<ClassificationResult> imageClasses = await SqueezeNetImageClassifier.ClassifyImage(selectedStorageFile, 3 );
|
List<ClassificationResult> imageClasses = await SqueezeNetImageClassifier.ClassifyImage(selectedStorageFile, 3 );
|
||||||
UpdateTextBox(imageClasses);
|
UpdateTextBox(imageClasses);
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
catch(Exception exc)
|
catch(Exception exc)
|
||||||
{
|
{
|
||||||
|
@ -124,6 +127,7 @@ namespace IntelligentLabsTest
|
||||||
// Get the SoftwareBitmap representation of the file in BGRA8 format
|
// Get the SoftwareBitmap representation of the file in BGRA8 format
|
||||||
softwareBitmap = await decoder.GetSoftwareBitmapAsync();
|
softwareBitmap = await decoder.GetSoftwareBitmapAsync();
|
||||||
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
|
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
|
||||||
|
var stuff = await EmotionRecognizer.DetectEmotion(softwareBitmap);
|
||||||
}
|
}
|
||||||
// Display the image
|
// Display the image
|
||||||
SoftwareBitmapSource imageSource = new SoftwareBitmapSource();
|
SoftwareBitmapSource imageSource = new SoftwareBitmapSource();
|
||||||
|
|
Загрузка…
Ссылка в новой задаче