microsoft
/
Windows-Machine-Learning
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Start to add launch window code but won't show up in front of samples gallery window

This commit is contained in:
Linnea May 2022-01-10 13:37:56 -08:00
Родитель 6b3602e36b
Коммит 3bf371c039
10 изменённых файлов: 397 добавлений и 613 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
Samples/WinMLSamplesGallery/WinMLSamplesGallery/Pages/SampleBasePage.xaml.cs
Просмотреть файл

@ -37,6 +37,9 @@ namespace WinMLSamplesGallery
case "EncryptedModel":
SampleFrame.Navigate(typeof(Samples.EncryptedModel));
break;
case "StreamEffect":
SampleFrame.Navigate(typeof(Samples.StreamEffect));
break;
}
if (sampleMetadata.Docs.Count > 0)
DocsHeader.Visibility = Visibility.Visible;

13
Samples/WinMLSamplesGallery/WinMLSamplesGallery/SampleMetadata/SampleMetadata.json
Просмотреть файл

@ -71,16 +71,17 @@
"CSharpGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/master/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/EncryptedModel/EncryptedModel.xaml.cs",
"Docs": [],
"IsRecentlyAdded": true
"Docs": []
},
{
"Title": "Stream Effect",
"Description": "Pick an input media source to run real-time background image blur powered by Windows AI MachineLearning.",
"Title": "Real-Time Inference",
"DescriptionShort": "The sample shows how to use Windows ML to apply a model to a camera stream in realtime.",
"Description": "The sample shows how to use Windows ML to apply a model to a camera stream in realtime. Select a style transfer model and it will be applied to the camera stream.",
"Icon": "\uE155",
"Tag": "StreamEffect",
"XAMLGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/user/numform/winml-samples-gallery/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/ImageEffects/ImageEffects.xaml",
"CSharpGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/user/numform/winml-samples-gallery/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/ImageEffects/ImageEffects.xaml.cs",
"Docs": []
"XAMLGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/master/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/EncryptedModel/EncryptedModel.xaml",
"CSharpGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/master/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/EncryptedModel/EncryptedModel.xaml.cs",
"Docs": [],
"IsRecentlyAdded": true
}
]
}

484
Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/Batching/Batching.xaml.cs
Просмотреть файл

@ -1,263 +1,263 @@
using Microsoft.AI.MachineLearning;
using Microsoft.UI.Xaml;
using Microsoft.UI.Xaml.Controls;
using System;
using System.Collections.Generic;
using Microsoft.AI.MachineLearning;
using Microsoft.UI.Xaml;
using Microsoft.UI.Xaml.Controls;
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using Windows.Graphics.Imaging;
using Windows.Media;
using Windows.Storage;
namespace WinMLSamplesGallery.Samples
{
public sealed class EvalResult
{
public string nonBatchedAvgTime { get; set; }
public string batchedAvgTime { get; set; }
public string timeRatio { get; set; }
}
public sealed partial class Batching : Page
{
const int NumInputImages = 50;
const int NumEvalIterations = 100;
private LearningModel _model = null;
private LearningModelSession _nonBatchingSession = null;
using System.Threading.Tasks;
using Windows.Graphics.Imaging;
using Windows.Media;
using Windows.Storage;
namespace WinMLSamplesGallery.Samples
{
public sealed class EvalResult
{
public string nonBatchedAvgTime { get; set; }
public string batchedAvgTime { get; set; }
public string timeRatio { get; set; }
}
public sealed partial class Batching : Page
{
const int NumInputImages = 50;
const int NumEvalIterations = 100;
private LearningModel _model = null;
private LearningModelSession _nonBatchingSession = null;
private LearningModelSession _batchingSession = null;
float _avgNonBatchedDuration = 0;
float _avgBatchDuration = 0;
// Marked volatile since it's updated across threads
static volatile bool navigatingAwayFromPage = false;
public Batching()
{
this.InitializeComponent();
// Ensure static variable is always false on page initialization
float _avgNonBatchedDuration = 0;
float _avgBatchDuration = 0;
// Marked volatile since it's updated across threads
static volatile bool navigatingAwayFromPage = false;
public Batching()
{
this.InitializeComponent();
// Ensure static variable is always false on page initialization
navigatingAwayFromPage = false;
// Load the model
var modelName = "squeezenet1.1-7-batched.onnx";
var modelPath = Path.Join(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, "Models", modelName);
_model = LearningModel.LoadFromFilePath(modelPath);
}
async private void StartInference(object sender, RoutedEventArgs e)
{
ShowStatus();
ResetMetrics();
var inputImages = await GetInputImages();
int batchSize = GetBatchSizeFromBatchSizeSlider();
_nonBatchingSession = await CreateLearningModelSession(_model);
_batchingSession = await CreateLearningModelSession(_model, batchSize);
UpdateStatus(false);
await Classify(inputImages);
UpdateStatus(true);
await ClassifyBatched(inputImages, batchSize);
ShowUI();
}
private void ShowStatus()
{
StartInferenceBtn.IsEnabled = false;
BatchSizeSlider.IsEnabled = false;
DeviceComboBox.IsEnabled = false;
EvalResults.Visibility = Visibility.Collapsed;
LoadingContainer.Visibility = Visibility.Visible;
}
private void ResetMetrics()
{
_avgNonBatchedDuration = 0;
_avgBatchDuration = 0;
}
// Test input consists of 50 images (25 bird and 25 cat)
private async Task<List<VideoFrame>> GetInputImages()
{
var birdFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///InputData/hummingbird.jpg"));
var catFile = await StorageFile .GetFileFromApplicationUriAsync(new Uri("ms-appx:///InputData/kitten.png"));
var birdImage = await CreateSoftwareBitmapFromStorageFile(birdFile);
var catImage = await CreateSoftwareBitmapFromStorageFile(catFile);
var inputImages = new List<VideoFrame>();
for (int i = 0; i < NumInputImages / 2; i++)
{
inputImages.Add(VideoFrame.CreateWithSoftwareBitmap(birdImage));
inputImages.Add(VideoFrame.CreateWithSoftwareBitmap(catImage));
}
return inputImages;
}
private async Task<SoftwareBitmap> CreateSoftwareBitmapFromStorageFile(StorageFile file)
{
var stream = await file.OpenAsync(FileAccessMode.Read);
var decoder = await BitmapDecoder.CreateAsync(stream);
var bitmap = await decoder.GetSoftwareBitmapAsync();
return bitmap;
}
private void UpdateStatus(bool isBatchingEval)
{
var modelName = "squeezenet1.1-7-batched.onnx";
var modelPath = Path.Join(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, "Models", modelName);
_model = LearningModel.LoadFromFilePath(modelPath);
}
async private void StartInference(object sender, RoutedEventArgs e)
{
ShowStatus();
ResetMetrics();
var inputImages = await GetInputImages();
int batchSize = GetBatchSizeFromBatchSizeSlider();
_nonBatchingSession = await CreateLearningModelSession(_model);
_batchingSession = await CreateLearningModelSession(_model, batchSize);
UpdateStatus(false);
await Classify(inputImages);
UpdateStatus(true);
await ClassifyBatched(inputImages, batchSize);
ShowUI();
}
private void ShowStatus()
{
StartInferenceBtn.IsEnabled = false;
BatchSizeSlider.IsEnabled = false;
DeviceComboBox.IsEnabled = false;
EvalResults.Visibility = Visibility.Collapsed;
LoadingContainer.Visibility = Visibility.Visible;
}
private void ResetMetrics()
{
_avgNonBatchedDuration = 0;
_avgBatchDuration = 0;
}
// Test input consists of 50 images (25 bird and 25 cat)
private async Task<List<VideoFrame>> GetInputImages()
{
var birdFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///InputData/hummingbird.jpg"));
var catFile = await StorageFile .GetFileFromApplicationUriAsync(new Uri("ms-appx:///InputData/kitten.png"));
var birdImage = await CreateSoftwareBitmapFromStorageFile(birdFile);
var catImage = await CreateSoftwareBitmapFromStorageFile(catFile);
var inputImages = new List<VideoFrame>();
for (int i = 0; i < NumInputImages / 2; i++)
{
inputImages.Add(VideoFrame.CreateWithSoftwareBitmap(birdImage));
inputImages.Add(VideoFrame.CreateWithSoftwareBitmap(catImage));
}
return inputImages;
}
private async Task<SoftwareBitmap> CreateSoftwareBitmapFromStorageFile(StorageFile file)
{
var stream = await file.OpenAsync(FileAccessMode.Read);
var decoder = await BitmapDecoder.CreateAsync(stream);
var bitmap = await decoder.GetSoftwareBitmapAsync();
return bitmap;
}
private void UpdateStatus(bool isBatchingEval)
{
if (isBatchingEval)
{
EvalText.Text = "Inferencing Batched Inputs:";
}
}
else
{
EvalText.Text = "Inferencing Non-Batched Inputs:";
}
}
private async Task<LearningModelSession> CreateLearningModelSession(LearningModel model, int batchSizeOverride=-1)
{
var deviceKind = DeviceComboBox.GetDeviceKind();
var device = new LearningModelDevice(deviceKind);
var options = new LearningModelSessionOptions();
if (batchSizeOverride > 0)
{
options.BatchSizeOverride = (uint)batchSizeOverride;
}
var session = new LearningModelSession(model, device, options);
return session;
}
async private Task Classify(List<VideoFrame> inputImages)
{
float totalEvalDurations = 0;
for (int i = 0; i < NumEvalIterations; i++)
{
}
}
private async Task<LearningModelSession> CreateLearningModelSession(LearningModel model, int batchSizeOverride=-1)
{
var deviceKind = DeviceComboBox.GetDeviceKind();
var device = new LearningModelDevice(deviceKind);
var options = new LearningModelSessionOptions();
if (batchSizeOverride > 0)
{
options.BatchSizeOverride = (uint)batchSizeOverride;
}
var session = new LearningModelSession(model, device, options);
return session;
}
async private Task Classify(List<VideoFrame> inputImages)
{
float totalEvalDurations = 0;
for (int i = 0; i < NumEvalIterations; i++)
{
if (navigatingAwayFromPage)
{
break;
}
UpdateProgress(i);
float evalDuration = await Task.Run(() => Evaluate(_nonBatchingSession, inputImages));
totalEvalDurations += evalDuration;
}
_avgNonBatchedDuration = totalEvalDurations / NumEvalIterations;
}
private static float Evaluate(LearningModelSession session, List<VideoFrame> input)
{
string inputName = session.Model.InputFeatures[0].Name;
float totalDuration = 0;
var binding = new LearningModelBinding(session);
for (int j = 0; j < input.Count; j++)
{
}
UpdateProgress(i);
float evalDuration = await Task.Run(() => Evaluate(_nonBatchingSession, inputImages));
totalEvalDurations += evalDuration;
}
_avgNonBatchedDuration = totalEvalDurations / NumEvalIterations;
}
private static float Evaluate(LearningModelSession session, List<VideoFrame> input)
{
string inputName = session.Model.InputFeatures[0].Name;
float totalDuration = 0;
var binding = new LearningModelBinding(session);
for (int j = 0; j < input.Count; j++)
{
if (navigatingAwayFromPage)
{
break;
}
var start = HighResolutionClock.UtcNow();
binding.Bind(inputName, input[j]);
session.Evaluate(binding, "");
var stop = HighResolutionClock.UtcNow();
var duration = HighResolutionClock.DurationInMs(start, stop);
totalDuration += duration;
}
return totalDuration;
}
async private Task ClassifyBatched(List<VideoFrame> inputImages, int batchSize)
{
float totalEvalDurations = 0;
for (int i = 0; i < NumEvalIterations; i++)
{
if (navigatingAwayFromPage)
break;
UpdateProgress(i);
float evalDuration = await Task.Run(() => EvaluateBatched(_batchingSession, inputImages, batchSize));
totalEvalDurations += evalDuration;
}
_avgBatchDuration = totalEvalDurations / NumEvalIterations;
}
private static float EvaluateBatched(LearningModelSession session, List<VideoFrame> input, int batchSize)
{
int numBatches = (int) Math.Ceiling((Decimal) input.Count / batchSize);
string inputName = session.Model.InputFeatures[0].Name;
float totalDuration = 0;
var binding = new LearningModelBinding(session);
for (int i = 0; i < numBatches; i++)
{
}
var start = HighResolutionClock.UtcNow();
binding.Bind(inputName, input[j]);
session.Evaluate(binding, "");
var stop = HighResolutionClock.UtcNow();
var duration = HighResolutionClock.DurationInMs(start, stop);
totalDuration += duration;
}
return totalDuration;
}
async private Task ClassifyBatched(List<VideoFrame> inputImages, int batchSize)
{
float totalEvalDurations = 0;
for (int i = 0; i < NumEvalIterations; i++)
{
if (navigatingAwayFromPage)
break;
UpdateProgress(i);
float evalDuration = await Task.Run(() => EvaluateBatched(_batchingSession, inputImages, batchSize));
totalEvalDurations += evalDuration;
}
_avgBatchDuration = totalEvalDurations / NumEvalIterations;
}
private static float EvaluateBatched(LearningModelSession session, List<VideoFrame> input, int batchSize)
{
int numBatches = (int) Math.Ceiling((Decimal) input.Count / batchSize);
string inputName = session.Model.InputFeatures[0].Name;
float totalDuration = 0;
var binding = new LearningModelBinding(session);
for (int i = 0; i < numBatches; i++)
{
if (navigatingAwayFromPage)
{
break;
}
int rangeStart = batchSize * i;
List<VideoFrame> batch;
// Add padding to the last batch if necessary
if (rangeStart + batchSize > input.Count)
{
int numInputsRemaining = input.Count - rangeStart;
int paddingAmount = batchSize - numInputsRemaining;
batch = input.GetRange(rangeStart, numInputsRemaining);
batch.AddRange(input.GetRange(0, paddingAmount));
}
else
{
batch = input.GetRange(rangeStart, batchSize);
}
var start = HighResolutionClock.UtcNow();
binding.Bind(inputName, batch);
session.Evaluate(binding, "");
var stop = HighResolutionClock.UtcNow();
var duration = HighResolutionClock.DurationInMs(start, stop);
totalDuration += duration;
}
return totalDuration;
}
private int GetBatchSizeFromBatchSizeSlider()
{
return int.Parse(BatchSizeSlider.Value.ToString());
}
private void UpdateProgress(int attemptNumber)
{
EvalProgressText.Text = "Attempt " + attemptNumber.ToString() + "/" + NumEvalIterations.ToString();
EvalProgressBar.Value = attemptNumber + 1;
}
private void ShowUI()
{
float ratio = (1 - (_avgBatchDuration / _avgNonBatchedDuration)) * 100;
var evalResult = new EvalResult
{
nonBatchedAvgTime = _avgNonBatchedDuration.ToString("0.00"),
batchedAvgTime = _avgBatchDuration.ToString("0.00"),
timeRatio = ratio.ToString("0.0")
};
List<EvalResult> results = new List<EvalResult>();
results.Insert(0, evalResult);
LoadingContainer.Visibility = Visibility.Collapsed;
EvalResults.Visibility = Visibility.Visible;
StartInferenceBtn.IsEnabled = true;
BatchSizeSlider.IsEnabled = true;
DeviceComboBox.IsEnabled = true;
EvalResults.ItemsSource = results;
}
private void UpdateBatchSizeText(object sender, RoutedEventArgs e)
{
BatchSizeText.Text = "Batch Size: " + BatchSizeSlider.Value.ToString();
}
public void StopAllEvents()
{
navigatingAwayFromPage = true;
}
}
}
}
int rangeStart = batchSize * i;
List<VideoFrame> batch;
// Add padding to the last batch if necessary
if (rangeStart + batchSize > input.Count)
{
int numInputsRemaining = input.Count - rangeStart;
int paddingAmount = batchSize - numInputsRemaining;
batch = input.GetRange(rangeStart, numInputsRemaining);
batch.AddRange(input.GetRange(0, paddingAmount));
}
else
{
batch = input.GetRange(rangeStart, batchSize);
}
var start = HighResolutionClock.UtcNow();
binding.Bind(inputName, batch);
session.Evaluate(binding, "");
var stop = HighResolutionClock.UtcNow();
var duration = HighResolutionClock.DurationInMs(start, stop);
totalDuration += duration;
}
return totalDuration;
}
private int GetBatchSizeFromBatchSizeSlider()
{
return int.Parse(BatchSizeSlider.Value.ToString());
}
private void UpdateProgress(int attemptNumber)
{
EvalProgressText.Text = "Attempt " + attemptNumber.ToString() + "/" + NumEvalIterations.ToString();
EvalProgressBar.Value = attemptNumber + 1;
}
private void ShowUI()
{
float ratio = (1 - (_avgBatchDuration / _avgNonBatchedDuration)) * 100;
var evalResult = new EvalResult
{
nonBatchedAvgTime = _avgNonBatchedDuration.ToString("0.00"),
batchedAvgTime = _avgBatchDuration.ToString("0.00"),
timeRatio = ratio.ToString("0.0")
};
List<EvalResult> results = new List<EvalResult>();
results.Insert(0, evalResult);
LoadingContainer.Visibility = Visibility.Collapsed;
EvalResults.Visibility = Visibility.Visible;
StartInferenceBtn.IsEnabled = true;
BatchSizeSlider.IsEnabled = true;
DeviceComboBox.IsEnabled = true;
EvalResults.ItemsSource = results;
}
private void UpdateBatchSizeText(object sender, RoutedEventArgs e)
{
BatchSizeText.Text = "Batch Size: " + BatchSizeSlider.Value.ToString();
}
public void StopAllEvents()
{
navigatingAwayFromPage = true;
}
}
}

20
Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml
Просмотреть файл

@ -1,12 +1,11 @@
<Page
x:Class="WinMLSamplesGallery.Samples.StreamEffect"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="using:WinMLSamplesGallery"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:local_controls="using:WinMLSamplesGallery.Controls"
xmlns:local_samples="using:WinMLSamplesGallery.Samples"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
mc:Ignorable="d"
FontFamily="Arial">
@ -16,7 +15,10 @@
</Style>
</Page.Resources>
<StackPanel>
<TextBlock FontFamily="Segoe UI Light" FontSize="14" Margin="0,15,0,0" Text="TEST TEST" TextWrapping="Wrap" />
<!--<StackPanel>
<StackPanel
Name="UICameraSelectionControls"
Grid.Row="1"
@ -31,18 +33,18 @@
SelectedIndex="{x:Bind streamEffectViewModel.SelectedCameraIndex, Mode=TwoWay}"
Margin="0,0,10,0"
>
<!--<i:Interaction.Behaviors>
--><!--<i:Interaction.Behaviors>
<core:EventTriggerBehavior EventName="SelectionChanged">
<core:InvokeCommandAction Command="{Binding ChangeLiveStreamCommand}" />
</core:EventTriggerBehavior>
</i:Interaction.Behaviors>!-->
</i:Interaction.Behaviors>!--><!--
<ComboBox.Background>
<SolidColorBrush Opacity="0.3" Color="Black" />
</ComboBox.Background>
</ComboBox>
<local_controls:DeviceComboBox x:Name="DeviceCmbBox" />
<!--TODO: Change button to be start/restart stream as needed. ie. if changing sources. -->
<!--TODO: Disable until a media source is selected. -->
--><!--TODO: Change button to be start/restart stream as needed. ie. if changing sources. -->
<!--TODO: Disable until a media source is selected. --><!--
<Button x:Name="StartInferenceBtn"
ToolTipService.ToolTip="Start Stream"
Click="ChangeLiveStream">
@ -53,7 +55,7 @@
</StackPanel>
<Image x:Name="InputImage" VerticalAlignment="Top" HorizontalAlignment="Left" Source="ms-appx:///InputData/stream.png">
</Image>
</StackPanel>
</StackPanel>-->

361
Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml.cs
Просмотреть файл

@ -26,6 +26,9 @@ using Windows.Media.Editing;
using Windows.Media.Core;
using Windows.Media.Playback;
using WinMLSamplesGalleryNative;
namespace WinMLSamplesGallery.Samples
{
public class StreamEffectViewModel : INotifyPropertyChanged
@ -80,368 +83,14 @@ namespace WinMLSamplesGallery.Samples
public sealed partial class StreamEffect : Page
{
private LearningModel _learningModel = null;
private LearningModelBinding _binding;
private string _modelSource = "fcn-resnet50-11.onnx";
private bool _useGpu = true;
private string _imgSource = "stream.jpg";
uint _inWidth, _inHeight;
LearningModelSession tensorizationSession = null;
LearningModelSession normalizeSession = null;
LearningModelSession modelSession = null;
LearningModelSession labelsSession = null;
LearningModelSession backgroundSession = null;
LearningModelSession blurSession = null;
LearningModelSession foregroundSession = null;
LearningModelSession detensorizationSession = null;
//Media capture fields
StreamEffectViewModel streamEffectViewModel = new StreamEffectViewModel();
private MediaComposition composition;
public StreamEffect()
{
this.InitializeComponent();
//composition = new MediaComposition();
//streamEffectViewModel.GetDevices();
}
private async Task PickFileAndAddClip()
{
var picker = new Windows.Storage.Pickers.FileOpenPicker();
picker.SuggestedStartLocation = Windows.Storage.Pickers.PickerLocationId.VideosLibrary;
picker.FileTypeFilter.Add(".mp4");
Windows.Storage.StorageFile pickedFile = await picker.PickSingleFileAsync();
if (pickedFile == null)
{
Debug.WriteLine("File picking cancelled");
return;
}
// These files could be picked from a location that we won't have access to later
var storageItemAccessList = Windows.Storage.AccessCache.StorageApplicationPermissions.FutureAccessList;
storageItemAccessList.Add(pickedFile);
var clip = await MediaClip.CreateFromFileAsync(pickedFile);
composition.Clips.Add(clip);
WinMLSamplesGalleryNative.StreamEffect.LaunchNewWindow();
}
private async void ChangeLiveStream(object sender, RoutedEventArgs e)
{
CameraCaptureUI captureUI = new CameraCaptureUI();
captureUI.PhotoSettings.AllowCropping = false;
captureUI.PhotoSettings.Format = CameraCaptureUIPhotoFormat.Png;
StorageFile videoFile = await captureUI.CaptureFileAsync(CameraCaptureUIMode.Photo);
if (videoFile == null)
{
// User cancelled photo capture
return;
}
}
private async void CleanUpCameraAsync()
{
return;
}
private async void LoadModelAsync()
{
Debug.Write("LoadModelBegin | ");
Debug.Write("LoadModel Lock | ");
_binding?.Clear();
modelSession?.Dispose();
StorageFile modelFile = StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///LargeModels/{_modelSource}")).AsTask().GetAwaiter().GetResult();
_learningModel = LearningModel.LoadFromStorageFileAsync(modelFile).AsTask().GetAwaiter().GetResult();
modelSession = CreateLearningModelSession(_learningModel);
_binding = new LearningModelBinding(modelSession);
debugModelIO();
Debug.Write("LoadModel Unlock\n");
}
private LearningModelSession CreateLearningModelSession(LearningModel model, bool closeModel = true)
{
var device = _useGpu ? new LearningModelDevice(LearningModelDeviceKind.DirectX) : new LearningModelDevice(LearningModelDeviceKind.Cpu);
var options = new LearningModelSessionOptions()
{
CloseModelOnSessionCreation = closeModel, // Close the model to prevent extra memory usage
BatchSizeOverride = 0
};
var session = new LearningModelSession(model, device, options);
return session;
}
public void debugModelIO()
{
foreach (var inputF in _learningModel.InputFeatures)
{
TensorFeatureDescriptor tfDesc = inputF as TensorFeatureDescriptor;
Debug.WriteLine($"input | kind:{inputF.Kind}, name:{inputF.Name}" +
$" Shape: {string.Join(",", tfDesc.Shape.ToArray<long>())}");
}
foreach (var outputF in _learningModel.OutputFeatures)
{
TensorFeatureDescriptor tfDesc = outputF as TensorFeatureDescriptor;
Debug.WriteLine($"output | kind:{outputF.Kind}, name:{outputF.Name}" +
$" Shape: {string.Join(",", tfDesc.Shape.ToArray<long>())}");
}
}
public async void getImageAsync()
{
Debug.WriteLine("In GetImage");
StorageFile file = StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///InputData/{_imgSource}")).GetAwaiter().GetResult();
using (IRandomAccessStream stream = file.OpenAsync(FileAccessMode.Read).GetAwaiter().GetResult())
{
var start = HighResolutionClock.UtcNow();
Debug.WriteLine("Got stream");
BitmapDecoder decoder = BitmapDecoder.CreateAsync(stream).AsTask().GetAwaiter().GetResult();
Debug.WriteLine("Got decoder");
_inWidth = decoder.PixelWidth;
_inHeight = decoder.PixelHeight;
var pixelDataProvider = await decoder.GetPixelDataAsync();//.GetAwaiter().GetResult();
Debug.WriteLine("Got pixeldata");
// 1. Tensorize input image
var bytes = pixelDataProvider.DetachPixelData();
var buffer = bytes.AsBuffer(); // Does this do a copy??
var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
var nextOutputShape = new long[] { 1, 3, _inHeight, _inWidth };
var tensorizedImg = TensorFloat.Create(nextOutputShape); // Need to keep this intermediate for blur/bckground
// Reshape initial input
tensorizationSession = CreateLearningModelSession(
TensorizationModels.ReshapeFlatBufferToNCHW(1, 4, _inHeight, _inWidth));
var tensorizationBinding = Evaluate(tensorizationSession, inputRawTensor, tensorizedImg);
Debug.WriteLine($"Intermediate Shape: {string.Join(",", tensorizedImg.Shape.ToArray<long>())}");
// 2. Normalize input image
float[] mean = new float[] { 0.485f, 0.456f, 0.406f };
float[] std = new float[] { 0.229f, 0.224f, 0.225f };
// Already sliced out alpha, but normalize0_1 expects the input to still have it- could just write another version later
// Normalize model will tensorize the arrays (and that means that it will move over to the GPU?)
normalizeSession = CreateLearningModelSession(
TensorizationModels.Normalize0_1ThenZScore(_inHeight, _inWidth, 4, mean, std));
var intermediateTensor = TensorFloat.Create(tensorizedImg.Shape);
var normalizationBinding = Evaluate(normalizeSession, tensorizedImg, intermediateTensor);
// 3. Run through actual model
var modelOutputShape = new long[] { 1, 21, _inHeight, _inWidth };
var modelOutputTensor = TensorFloat.Create(modelOutputShape);
var modelBinding = Evaluate(modelSession, intermediateTensor, modelOutputTensor);
// 4. Get the class predictions for each pixel
var rawLabels = TensorFloat.Create(new long[] { 1, 1, _inHeight, _inWidth });
labelsSession = CreateLearningModelSession(ArgMax(1, _inHeight, _inWidth));
var labelsBinding = Evaluate(labelsSession, modelOutputTensor, rawLabels);
// 5. Extract just the blurred background based on mask
// Create a blurred version of the original picture
var outputBindProperties = new PropertySet();
outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true));
Trace.Assert(nextOutputShape != null);
var blurNoMask = TensorFloat.Create((IEnumerable<long>)(nextOutputShape));
Trace.WriteLine($"test {nextOutputShape.Length}");
blurSession = CreateLearningModelSession(TensorizationModels.AveragePool(20));
var blurBinding = Evaluate(blurSession, tensorizedImg, blurNoMask, true);
var blurredImg = TensorFloat.Create(nextOutputShape);
backgroundSession = CreateLearningModelSession(GetBackground(1, 3, _inHeight, _inWidth));
var binding = new LearningModelBinding(backgroundSession);
Console.WriteLine($"Intermediate Shape: {string.Join(",", blurNoMask.Shape.ToArray<long>())}");
binding.Bind(backgroundSession.Model.InputFeatures[0].Name, blurNoMask); // Intermediate tensor isn't on GPU?
binding.Bind(backgroundSession.Model.InputFeatures[1].Name, rawLabels);
binding.Bind(backgroundSession.Model.OutputFeatures[0].Name, blurredImg);
EvaluateInternal(backgroundSession, binding);
// 6. Extract just the foreground based on mask and combine with background
intermediateTensor = TensorFloat.Create(nextOutputShape);
foregroundSession = CreateLearningModelSession(GetOutputFrame(1, 3, _inHeight, _inWidth));
binding = new LearningModelBinding(foregroundSession);
binding.Bind(foregroundSession.Model.InputFeatures[0].Name, tensorizedImg);
binding.Bind(foregroundSession.Model.InputFeatures[1].Name, rawLabels);
binding.Bind(foregroundSession.Model.InputFeatures[2].Name, blurredImg);
binding.Bind(foregroundSession.Model.OutputFeatures[0].Name, intermediateTensor, outputBindProperties);
EvaluateInternal(foregroundSession, binding);
// 7. Detensorize and display output
var outputFrame = Detensorize(intermediateTensor);
RenderOutputFrame(outputFrame);
var stop = HighResolutionClock.UtcNow();
Debug.WriteLine($"*** DONE IN {HighResolutionClock.DurationInMs(start, stop)} Ms");
}
}
public static LearningModel ArgMax(long axis, long h, long w)
{
var builder = LearningModelBuilder.Create(12)
.Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Data", TensorKind.Float, new long[] { -1, -1, h, w })) // Different input type?
.Outputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Output", TensorKind.Float, new long[] { -1, -1, h, w })) // Output of int64?
.Operators.Add(new LearningModelOperator("ArgMax")
.SetInput("data", "Data")
.SetAttribute("keepdims", TensorInt64Bit.CreateFromArray(new List<long>(), new long[] { 1 }))
.SetAttribute("axis", TensorInt64Bit.CreateFromIterable(new long[] { }, new long[] { axis })) // Correct way of passing axis?
.SetOutput("reduced", "Reduced"))
.Operators.Add(new LearningModelOperator("Cast")
.SetInput("input", "Reduced")
.SetAttribute("to", TensorInt64Bit.CreateFromIterable(new long[] { }, new long[] { (long)TensorizationModels.OnnxDataType.FLOAT }))
.SetOutput("output", "Output"))
;
return builder.CreateModel();
}
public static LearningModel GetOutputFrame(long n, long c, long h, long w)
{
var builder = LearningModelBuilder.Create(12)
.Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputImage", TensorKind.Float, new long[] { n, c, h, w }))
.Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputMask", TensorKind.Float, new long[] { n, 1, h, w })) // Broadcast to each color channel
.Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputBackground", TensorKind.Float, new long[] { n, c, h, w }))
.Outputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Output", TensorKind.Float, new long[] { n, c, h, w }))
.Operators.Add(new LearningModelOperator("Clip")
.SetInput("input", "InputMask")
.SetConstant("max", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 1 }))
.SetOutput("output", "MaskBinary"))
.Operators.Add(new LearningModelOperator("Mul")
.SetInput("A", "InputImage")
.SetInput("B", "MaskBinary")
.SetOutput("C", "Foreground"))
.Operators.Add(new LearningModelOperator("Add")
.SetInput("A", "InputBackground")
.SetInput("B", "Foreground")
.SetOutput("C", "Output"))
;
return builder.CreateModel();
}
public static LearningModel GetBackground(long n, long c, long h, long w)
{
var builder = LearningModelBuilder.Create(11)
.Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputImage", TensorKind.Float, new long[] { n, c, h, w }))
.Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputMask", TensorKind.Float, new long[] { n, 1, h, w })) // Broadcast to each color channel
.Outputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Output", TensorKind.Float, new long[] { n, c, h, w }))
.Operators.Add(new LearningModelOperator("Clip") // Make mask binary
.SetInput("input", "InputMask")
.SetConstant("max", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 1 }))
.SetOutput("output", "ClipMask"))
.Operators.Add(new LearningModelOperator("Mul")
.SetInput("A", "ClipMask")
.SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { -1 }))
.SetOutput("C", "NegMask"))
.Operators.Add(new LearningModelOperator("Add") // BackgroundMask = (1- InputMask)
.SetConstant("A", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 1 }))
.SetInput("B", "NegMask")
.SetOutput("C", "BackgroundMask"))
.Operators.Add(new LearningModelOperator("Mul") // Extract the background
.SetInput("A", "InputImage")
.SetInput("B", "BackgroundMask")
.SetOutput("C", "Output"))
;
return builder.CreateModel();
}
public void RenderOutputFrame(VideoFrame outputFrame)
{
SoftwareBitmap displayBitmap = outputFrame.SoftwareBitmap;
//Image control only accepts BGRA8 encoding and Premultiplied/no alpha channel. This checks and converts
//the SoftwareBitmap we want to bind.
if (displayBitmap.BitmapPixelFormat != BitmapPixelFormat.Bgra8 ||
displayBitmap.BitmapAlphaMode != BitmapAlphaMode.Premultiplied)
{
displayBitmap = SoftwareBitmap.Convert(displayBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
}
// get software bitmap souce
var source = new SoftwareBitmapSource();
source.SetBitmapAsync(displayBitmap).GetAwaiter();
// draw the input image
InputImage.Source = source;
}
private VideoFrame Detensorize(TensorFloat intermediateTensor)
{
// Change from Int64 to float
//ITensor interm = intermediateTensor;
var shape = intermediateTensor.Shape;
var n = (int)shape[0];
var c = (int)shape[1];
var h = (int)shape[2];
var w = (int)shape[3];
// Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu)
// we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization.
var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Ignore);
var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage);
detensorizationSession = CreateLearningModelSession(TensorizationModels.IdentityNCHW(1, c, _inHeight, _inWidth));
var descriptor = detensorizationSession.Model.InputFeatures[0] as TensorFeatureDescriptor;
var detensorizerShape = descriptor.Shape;
/*if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3])
{
detensorizationSession = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w));
}*/
var detensorizationBinding = Evaluate(detensorizationSession, intermediateTensor, outputFrame, true);
return outputFrame;
}
private LearningModelBinding Evaluate(LearningModelSession session, object input, object output, bool wait = false)
{
// Create the binding
var binding = new LearningModelBinding(session);
// Bind inputs and outputs
string inputName = session.Model.InputFeatures[0].Name;
string outputName = session.Model.OutputFeatures[0].Name;
binding.Bind(inputName, input);
var outputBindProperties = new PropertySet();
outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true));
binding.Bind(outputName, output, outputBindProperties);
// Evaluate
EvaluateInternal(session, binding, wait);
return binding;
}
private void EvaluateInternal(LearningModelSession session, LearningModelBinding binding, bool wait = true)
{
if (!_useGpu) //originally isCpu
{
session.Evaluate(binding, "");
}
else
{
var results = session.Evaluate(binding, "");
/*if (wait)
{
results.GetAwaiter().GetResult();
}*/
}
}
}
}

78
Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/StreamEffect.cpp Normal file
Просмотреть файл

@ -0,0 +1,78 @@
#include "pch.h"
#include "StreamEffect.h"
#include "StreamEffect.g.cpp"
#include <commctrl.h>
#include <mfapi.h>
namespace winrt::WinMLSamplesGalleryNative::implementation
{
LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) {
switch (uMsg) {
default:
return 1;
}
};
void StreamEffect::LaunchNewWindow()
{
HWND hwnd;
HRESULT hr = S_OK;
BOOL bMFStartup = false;
// Initialize the common controls
const INITCOMMONCONTROLSEX icex = { sizeof(INITCOMMONCONTROLSEX), ICC_WIN95_CLASSES };
InitCommonControlsEx(&icex);
hr = MFStartup(MF_VERSION);
if (FAILED(hr))
{
goto done;
}
// Create window
const wchar_t CLASS_NAME[] = L"Capture Engine Window Class";
INT nCmdShow = 1;
WNDCLASS wc = { 0 };
wc.lpfnWndProc = WindowProc;
wc.hInstance = GetModuleHandle(NULL);
wc.lpszClassName = CLASS_NAME;
RegisterClass(&wc);
hwnd = CreateWindowEx(
0, CLASS_NAME, L"Capture Application", WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT,
CW_USEDEFAULT, CW_USEDEFAULT, NULL, NULL, GetModuleHandle(NULL), NULL
);
if (hwnd == 0)
{
throw_hresult(E_FAIL);
}
ShowWindow(hwnd, nCmdShow);
// Run the main message loop
MSG msg;
while (GetMessage(&msg, NULL, 0, 0))
{
TranslateMessage(&msg);
DispatchMessage(&msg);
}
done:
if (FAILED(hr))
{
// TODO: Add utils from BackgroundBlur sample utils.cpp
//ShowError(NULL, L"Failed to start application", hr);
}
if (bMFStartup)
{
MFShutdown();
}
return ;
}
}

19
Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/StreamEffect.h Normal file
Просмотреть файл

@ -0,0 +1,19 @@
#pragma once
#include "StreamEffect.g.h"
namespace winrt::WinMLSamplesGalleryNative::implementation
{
struct StreamEffect : StreamEffectT<StreamEffect>
{
StreamEffect() = default;
static void LaunchNewWindow();
};
}
namespace winrt::WinMLSamplesGalleryNative::factory_implementation
{
struct StreamEffect : StreamEffectT<StreamEffect, implementation::StreamEffect>
{
};
}

9
Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.idl
Просмотреть файл

@ -16,4 +16,13 @@ namespace WinMLSamplesGalleryNative
{
static Microsoft.AI.MachineLearning.LearningModel LoadEncryptedResource(String key);
}
[default_interface]
runtimeclass StreamEffect
{
static void LaunchNewWindow();
}
}

14
Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj
Просмотреть файл

@ -128,6 +128,15 @@
<ResourceCompile>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(MSBuildThisFileDirectory)../WinMLSamplesGallery/Models/;$(MSBuildThisFileDirectory)../../build/native/include/;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ResourceCompile>
<Link>
<AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">Comctl32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
<Link>
<AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Comctl32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
<Link>
<AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Comctl32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
<ClCompile>
@ -136,6 +145,9 @@
<Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">Comctl32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Comctl32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Comctl32.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
<ResourceCompile>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(MSBuildThisFileDirectory)../WinMLSamplesGallery/Models/;$(MSBuildThisFileDirectory)../../build/native/include/;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
@ -146,6 +158,7 @@
<ClInclude Include="OpenCVImage.h" />
<ClInclude Include="pch.h" />
<ClInclude Include="RandomAccessStream.h" />
<ClInclude Include="StreamEffect.h" />
<ClInclude Include="WeakBuffer.h" />
</ItemGroup>
<ItemGroup>
@ -155,6 +168,7 @@
<PrecompiledHeader>Create</PrecompiledHeader>
</ClCompile>
<ClCompile Include="$(GeneratedFilesDir)module.g.cpp" />
<ClCompile Include="StreamEffect.cpp" />
</ItemGroup>
<ItemGroup>
<Midl Include="WinMLSamplesGalleryNative.idl" />

9
Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj.filters
Просмотреть файл

@ -9,6 +9,9 @@
<ClCompile Include="OpenCVImage.cpp">
<Filter>OpenCVInterop</Filter>
</ClCompile>
<ClCompile Include="StreamEffect.cpp">
<Filter>StreamEffect</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="pch.h" />
@ -24,6 +27,9 @@
<ClInclude Include="WeakBuffer.h">
<Filter>AbiHelpers</Filter>
</ClInclude>
<ClInclude Include="StreamEffect.h">
<Filter>StreamEffect</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="Resource.rc" />
@ -51,5 +57,8 @@
<Filter Include="AbiHelpers">
<UniqueIdentifier>{ed4f3871-4514-4b93-9ace-1315cad5491e}</UniqueIdentifier>
</Filter>
<Filter Include="StreamEffect">
<UniqueIdentifier>{91675ea9-10e7-4490-af40-762311cf5ba9}</UniqueIdentifier>
</Filter>
</ItemGroup>
</Project>