Added emotion pacakge (#14)
* added new project for emotion recognition * emotion package Co-authored-by: Amrutha Srinivasan <amsrin@microsoft.com>
This commit is contained in:
amrutha95
GitHub
Родитель
1d2f655392
Коммит
88acd74a0b
|
|
@ -0,0 +1,164 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Windows.AI.MachineLearning;
|
||||
using Windows.Foundation;
|
||||
using Windows.Graphics.Imaging;
|
||||
using Windows.Media;
|
||||
using Windows.Media.FaceAnalysis;
|
||||
using Windows.Storage;
|
||||
using Windows.Storage.Streams;
|
||||
|
||||
namespace CommunityToolkit.Labs.Intelligent.EmotionRecognition
|
||||
{
|
||||
public class DetectedEmotion
|
||||
{
|
||||
public int emotionIndex;
|
||||
public string emotion;
|
||||
}
|
||||
public class EmotionRecognizer
|
||||
{
|
||||
private LearningModel _model = null;
|
||||
private LearningModelSession _session = null;
|
||||
private LearningModelBinding _binding = null;
|
||||
private static EmotionRecognizer instance = null;
|
||||
|
||||
private static List<string> labels;
|
||||
|
||||
|
||||
private async void InitModelAsync()
|
||||
{
|
||||
// load model file
|
||||
var file = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///IntelligentAPI_EmotionRecognizer/Assets/model_emotion.onnx"));
|
||||
|
||||
//Loads the mdoel from the file
|
||||
_model = await LearningModel.LoadFromStorageFileAsync(file);
|
||||
|
||||
//Creating a session that binds the model to the device running the model
|
||||
_session = new LearningModelSession(_model, new LearningModelDevice(GetDeviceKind()));
|
||||
}
|
||||
|
||||
private void LoadLabels()
|
||||
{
|
||||
labels = new List<string>()
|
||||
{
|
||||
"Neutral",
|
||||
"Happiness",
|
||||
"Surprise",
|
||||
"Sadness",
|
||||
"Anger",
|
||||
"Disgust",
|
||||
"Fear",
|
||||
"Contempt"
|
||||
};
|
||||
}
|
||||
|
||||
LearningModelDeviceKind GetDeviceKind()
|
||||
{
|
||||
return LearningModelDeviceKind.Cpu;
|
||||
}
|
||||
|
||||
private async static Task<IList<DetectedFace>> DetectFacesInImageAsync(SoftwareBitmap bitmap)
|
||||
{
|
||||
FaceDetector faceDetector = await FaceDetector.CreateAsync();
|
||||
var convertedBitmap = SoftwareBitmap.Convert(bitmap, BitmapPixelFormat.Gray8);
|
||||
return await faceDetector.DetectFacesAsync(convertedBitmap);
|
||||
|
||||
}
|
||||
|
||||
public async static Task<DetectedEmotion> DetectEmotion(SoftwareBitmap bitmap)
|
||||
{
|
||||
if (instance == null)
|
||||
{
|
||||
instance = new EmotionRecognizer();
|
||||
}
|
||||
|
||||
return await instance.EvaluateFrame(bitmap);
|
||||
}
|
||||
|
||||
public async Task<DetectedEmotion> EvaluateFrame(SoftwareBitmap softwareBitmap)
|
||||
{
|
||||
InitModelAsync();
|
||||
LoadLabels();
|
||||
DetectedFace detectedFace = await DetectFace(softwareBitmap);
|
||||
if (detectedFace != null)
|
||||
{
|
||||
return await EvaluateEmotionInFace(detectedFace, softwareBitmap);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public async Task<DetectedEmotion> EvaluateEmotionInFace(DetectedFace detectedFace, SoftwareBitmap softwareBitmap)
|
||||
{
|
||||
|
||||
var boundingBox = new Rect(detectedFace.FaceBox.X,
|
||||
detectedFace.FaceBox.Y,
|
||||
detectedFace.FaceBox.Width,
|
||||
detectedFace.FaceBox.Height);
|
||||
|
||||
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8);
|
||||
|
||||
var croppedFace = await Crop(softwareBitmap, boundingBox);
|
||||
LearningModelEvaluationResult emotionResults = await BindAndEvaluateModelAsync(croppedFace);
|
||||
|
||||
// to get percentages, you'd need to run the output through a softmax function
|
||||
// we don't need percentages, we just need max value
|
||||
TensorFloat emotionIndexTensor = emotionResults.Outputs["Plus692_Output_0"] as TensorFloat;
|
||||
|
||||
var emotionList = emotionIndexTensor.GetAsVectorView().ToList();
|
||||
var emotionIndex = emotionList.IndexOf(emotionList.Max());
|
||||
|
||||
return new DetectedEmotion() { emotionIndex = emotionIndex, emotion = labels[emotionIndex] };
|
||||
|
||||
|
||||
}
|
||||
|
||||
private static async Task<DetectedFace> DetectFace(SoftwareBitmap softwareBitmap)
|
||||
{
|
||||
var faces = await DetectFacesInImageAsync(softwareBitmap);
|
||||
|
||||
// if there is a face in the frame, evaluate the emotion
|
||||
var detectedFace = faces.FirstOrDefault();
|
||||
return detectedFace;
|
||||
}
|
||||
|
||||
public static async Task<SoftwareBitmap> Crop(SoftwareBitmap softwareBitmap, Rect bounds)
|
||||
{
|
||||
VideoFrame vid = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
|
||||
vid = await Crop(vid, bounds);
|
||||
return vid.SoftwareBitmap;
|
||||
|
||||
}
|
||||
public static async Task<VideoFrame> Crop(VideoFrame videoFrame, Rect bounds)
|
||||
{
|
||||
BitmapBounds cropBounds = new BitmapBounds()
|
||||
{
|
||||
Width = (uint)bounds.Width,
|
||||
Height = (uint)bounds.Height,
|
||||
X = (uint)bounds.X,
|
||||
Y = (uint)bounds.Y
|
||||
};
|
||||
VideoFrame result = new VideoFrame(BitmapPixelFormat.Bgra8,
|
||||
(int)cropBounds.Width,
|
||||
(int)cropBounds.Height,
|
||||
BitmapAlphaMode.Premultiplied);
|
||||
|
||||
await videoFrame.CopyToAsync(result, cropBounds, null);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
private async Task<LearningModelEvaluationResult> BindAndEvaluateModelAsync(SoftwareBitmap croppedFace)
|
||||
{
|
||||
//Create Learning model binding which binds
|
||||
_binding = new LearningModelBinding(_session);
|
||||
_binding.Bind("Input3", VideoFrame.CreateWithSoftwareBitmap(croppedFace));
|
||||
var emotionResults = await _session.EvaluateAsync(_binding, "id");
|
||||
return emotionResults;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
<Project Sdk="MSBuild.Sdk.Extras">
|
||||
<PropertyGroup>
|
||||
<RootNamespace>IntelligentAPI.EmotionRecognition</RootNamespace>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup>
|
||||
<PackageId>CommunityToolkit.Labs.Intelligent.EmotionRecognition</PackageId>
|
||||
<Description>
|
||||
This package performs Emotion Recognition on an input image by using the Emotion FERPlus model.
|
||||
</Description>
|
||||
<Version>0.0.1</Version>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Properties\IntelligentAPI_EmotionRecognizer.rd.xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Content Include="Assets\model_emotion.onnx" Pack="True" PackagePath="lib/uap10.0.17763/Assets"/>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="ImageClassifier.licenseheader" />
|
||||
</ItemGroup>
|
||||
<Target Name="CustomBeforeBuild" BeforeTargets="BeforeBuild">
|
||||
<Exec Command="powershell.exe –NonInteractive –ExecutionPolicy Unrestricted -command "& {.\Scripts\script.ps1 }"" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: ComVisible(false)]
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
This file contains Runtime Directives, specifications about types your application accesses
|
||||
through reflection and other dynamic code patterns. Runtime Directives are used to control the
|
||||
.NET Native optimizer and ensure that it does not remove code accessed by your library. If your
|
||||
library does not do any reflection, then you generally do not need to edit this file. However,
|
||||
if your library reflects over types, especially types passed to it or derived from its types,
|
||||
then you should write Runtime Directives.
|
||||
|
||||
The most common use of reflection in libraries is to discover information about types passed
|
||||
to the library. Runtime Directives have three ways to express requirements on types passed to
|
||||
your library.
|
||||
|
||||
1. Parameter, GenericParameter, TypeParameter, TypeEnumerableParameter
|
||||
Use these directives to reflect over types passed as a parameter.
|
||||
|
||||
2. SubTypes
|
||||
Use a SubTypes directive to reflect over types derived from another type.
|
||||
|
||||
3. AttributeImplies
|
||||
Use an AttributeImplies directive to indicate that your library needs to reflect over
|
||||
types or methods decorated with an attribute.
|
||||
|
||||
For more information on writing Runtime Directives for libraries, please visit
|
||||
https://go.microsoft.com/fwlink/?LinkID=391919
|
||||
-->
|
||||
<Directives xmlns="http://schemas.microsoft.com/netfx/2013/01/metadata">
|
||||
<Library Name="IntelligentAPI_EmotionRecognizer">
|
||||
|
||||
<!-- add directives for your library here -->
|
||||
|
||||
</Library>
|
||||
</Directives>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
$emotionferplusfile = "./Assets/model_emotion.onnx"
|
||||
if (-not(Test-Path -Path $emotionferplusfile -PathType Leaf)) {
|
||||
try {
|
||||
Invoke-WebRequest -URI "https://github.com/onnx/models/raw/master/vision/body_analysis/emotion_ferplus/model/emotion-ferplus-8.onnx" -OutFile $emotionferplusfile
|
||||
Write-Host "The file [$emotionferplusfile] has been created."
|
||||
}
|
||||
catch {
|
||||
throw $_.Exception.Message
|
||||
}
|
||||
}
|
||||
else {
|
||||
Write-Host "The file [$emotionferplusfile] exists."
|
||||
}
|
||||
|
|
@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IntelligentAPI_ImageClassif
|
|||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntelligentLabsTest", "IntelligentAPIsTester\IntelligentLabsTest.csproj", "{47D87733-B357-4706-88BD-211FB7D8679D}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntelligentAPI_EmotionRecognizer", "IntelligentAPI_EmotionRecognizer\IntelligentAPI_EmotionRecognizer.csproj", "{2AAF76B9-2B93-4932-909C-09BBF54731FC}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
|
|
@ -123,6 +125,36 @@ Global
|
|||
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.ActiveCfg = Release|x86
|
||||
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.Build.0 = Release|x86
|
||||
{47D87733-B357-4706-88BD-211FB7D8679D}.Release|x86.Deploy.0 = Release|x86
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|ARM.ActiveCfg = Debug|ARM
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|ARM.Build.0 = Debug|ARM
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|ARM64.ActiveCfg = Debug|ARM64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|ARM64.Build.0 = Debug|ARM64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|x64.Build.0 = Debug|x64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|x86.ActiveCfg = Debug|x86
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Debug|x86.Build.0 = Debug|x86
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|ARM.ActiveCfg = Debug|ARM
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|ARM.Build.0 = Debug|ARM
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|ARM64.ActiveCfg = Debug|ARM64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|ARM64.Build.0 = Debug|ARM64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|x64.ActiveCfg = Debug|x64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|x64.Build.0 = Debug|x64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|x86.ActiveCfg = Debug|x86
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Packages|x86.Build.0 = Debug|x86
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|ARM.ActiveCfg = Release|ARM
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|ARM.Build.0 = Release|ARM
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|ARM64.ActiveCfg = Release|ARM64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|ARM64.Build.0 = Release|ARM64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|x64.ActiveCfg = Release|x64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|x64.Build.0 = Release|x64
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|x86.ActiveCfg = Release|x86
|
||||
{2AAF76B9-2B93-4932-909C-09BBF54731FC}.Release|x86.Build.0 = Release|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
|
|||
|
|
@ -170,6 +170,10 @@
|
|||
<Project>{f705b499-d0b5-408c-b1d1-d3f379d0fcd6}</Project>
|
||||
<Name>IntelligentAPI_ImageClassifier</Name>
|
||||
</ProjectReference>
|
||||
<ProjectReference Include="..\IntelligentAPI_EmotionRecognizer\IntelligentAPI_EmotionRecognizer.csproj">
|
||||
<Project>{2aaf76b9-2b93-4932-909c-09bbf54731fc}</Project>
|
||||
<Name>IntelligentAPI_EmotionRecognizer</Name>
|
||||
</ProjectReference>
|
||||
<ProjectReference Include="..\ObjectDetector\IntelligentAPI_ObjectDetector.csproj">
|
||||
<Project>{6f985dc2-db68-49e5-add8-6d11f35e552e}</Project>
|
||||
<Name>IntelligentAPI_ObjectDetector</Name>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
|
||||
using CommunityToolkit.Labs.Intelligent.ImageClassification;
|
||||
using CommunityToolkit.Labs.Intelligent.ObjectDetection;
|
||||
using CommunityToolkit.Labs.Intelligent.EmotionRecognition;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
|
@ -73,6 +74,8 @@ namespace IntelligentLabsTest
|
|||
//Use Squeezenet model to classify image
|
||||
List<ClassificationResult> imageClasses = await SqueezeNetImageClassifier.ClassifyImage(selectedStorageFile, 3 );
|
||||
UpdateTextBox(imageClasses);
|
||||
|
||||
|
||||
}
|
||||
catch(Exception exc)
|
||||
{
|
||||
|
|
@ -83,7 +86,7 @@ namespace IntelligentLabsTest
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
//Use YOLOv4 to detect objects. WORKS ONLY IF YOU ARE RUNNING WINDOWS 11!!
|
||||
if (CheckWindowsBuildNumber())
|
||||
{
|
||||
|
|
@ -124,6 +127,7 @@ namespace IntelligentLabsTest
|
|||
// Get the SoftwareBitmap representation of the file in BGRA8 format
|
||||
softwareBitmap = await decoder.GetSoftwareBitmapAsync();
|
||||
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
|
||||
var stuff = await EmotionRecognizer.DetectEmotion(softwareBitmap);
|
||||
}
|
||||
// Display the image
|
||||
SoftwareBitmapSource imageSource = new SoftwareBitmapSource();
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче