diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Pages/SampleBasePage.xaml.cs b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Pages/SampleBasePage.xaml.cs index 21036f04..d8a13059 100644 --- a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Pages/SampleBasePage.xaml.cs +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Pages/SampleBasePage.xaml.cs @@ -37,6 +37,9 @@ namespace WinMLSamplesGallery case "EncryptedModel": SampleFrame.Navigate(typeof(Samples.EncryptedModel)); break; + case "StreamEffect": + SampleFrame.Navigate(typeof(Samples.StreamEffect)); + break; } if (sampleMetadata.Docs.Count > 0) DocsHeader.Visibility = Visibility.Visible; diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/SampleMetadata/SampleMetadata.json b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/SampleMetadata/SampleMetadata.json index 9679050d..70eb3db7 100644 --- a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/SampleMetadata/SampleMetadata.json +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/SampleMetadata/SampleMetadata.json @@ -71,16 +71,17 @@ "CSharpGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/master/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/EncryptedModel/EncryptedModel.xaml.cs", "Docs": [], "IsRecentlyAdded": true - "Docs": [] }, { - "Title": "Stream Effect", - "Description": "Pick an input media source to run real-time background image blur powered by Windows AI MachineLearning.", + "Title": "Real-Time Inference", + "DescriptionShort": "The sample shows how to use Windows ML to apply a model to a camera stream in realtime.", + "Description": "The sample shows how to use Windows ML to apply a model to a camera stream in realtime. Select a style transfer model and it will be applied to the camera stream.", "Icon": "\uE155", "Tag": "StreamEffect", - "XAMLGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/user/numform/winml-samples-gallery/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/ImageEffects/ImageEffects.xaml", - "CSharpGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/user/numform/winml-samples-gallery/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/ImageEffects/ImageEffects.xaml.cs", - "Docs": [] + "XAMLGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/master/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/EncryptedModel/EncryptedModel.xaml", + "CSharpGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/master/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/EncryptedModel/EncryptedModel.xaml.cs", + "Docs": [], + "IsRecentlyAdded": true } ] } \ No newline at end of file diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/Batching/Batching.xaml.cs b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/Batching/Batching.xaml.cs index 593d0b90..50f8a594 100644 --- a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/Batching/Batching.xaml.cs +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/Batching/Batching.xaml.cs @@ -1,263 +1,263 @@ -using Microsoft.AI.MachineLearning; -using Microsoft.UI.Xaml; -using Microsoft.UI.Xaml.Controls; -using System; -using System.Collections.Generic; +using Microsoft.AI.MachineLearning; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +using System; +using System.Collections.Generic; using System.IO; -using System.Threading.Tasks; -using Windows.Graphics.Imaging; -using Windows.Media; -using Windows.Storage; - -namespace WinMLSamplesGallery.Samples -{ - public sealed class EvalResult - { - public string nonBatchedAvgTime { get; set; } - public string batchedAvgTime { get; set; } - public string timeRatio { get; set; } - } - - public sealed partial class Batching : Page - { - const int NumInputImages = 50; - const int NumEvalIterations = 100; - - private LearningModel _model = null; - private LearningModelSession _nonBatchingSession = null; +using System.Threading.Tasks; +using Windows.Graphics.Imaging; +using Windows.Media; +using Windows.Storage; + +namespace WinMLSamplesGallery.Samples +{ + public sealed class EvalResult + { + public string nonBatchedAvgTime { get; set; } + public string batchedAvgTime { get; set; } + public string timeRatio { get; set; } + } + + public sealed partial class Batching : Page + { + const int NumInputImages = 50; + const int NumEvalIterations = 100; + + private LearningModel _model = null; + private LearningModelSession _nonBatchingSession = null; private LearningModelSession _batchingSession = null; - float _avgNonBatchedDuration = 0; - float _avgBatchDuration = 0; - - // Marked volatile since it's updated across threads - static volatile bool navigatingAwayFromPage = false; - - - public Batching() - { - this.InitializeComponent(); - // Ensure static variable is always false on page initialization + float _avgNonBatchedDuration = 0; + float _avgBatchDuration = 0; + + // Marked volatile since it's updated across threads + static volatile bool navigatingAwayFromPage = false; + + + public Batching() + { + this.InitializeComponent(); + // Ensure static variable is always false on page initialization navigatingAwayFromPage = false; // Load the model - var modelName = "squeezenet1.1-7-batched.onnx"; - var modelPath = Path.Join(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, "Models", modelName); - _model = LearningModel.LoadFromFilePath(modelPath); - } - - async private void StartInference(object sender, RoutedEventArgs e) - { - ShowStatus(); - ResetMetrics(); - - var inputImages = await GetInputImages(); - int batchSize = GetBatchSizeFromBatchSizeSlider(); - - _nonBatchingSession = await CreateLearningModelSession(_model); - _batchingSession = await CreateLearningModelSession(_model, batchSize); - - UpdateStatus(false); - await Classify(inputImages); - - UpdateStatus(true); - await ClassifyBatched(inputImages, batchSize); - - ShowUI(); - } - - private void ShowStatus() - { - StartInferenceBtn.IsEnabled = false; - BatchSizeSlider.IsEnabled = false; - DeviceComboBox.IsEnabled = false; - EvalResults.Visibility = Visibility.Collapsed; - LoadingContainer.Visibility = Visibility.Visible; - } - - private void ResetMetrics() - { - _avgNonBatchedDuration = 0; - _avgBatchDuration = 0; - } - - // Test input consists of 50 images (25 bird and 25 cat) - private async Task> GetInputImages() - { - var birdFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///InputData/hummingbird.jpg")); - var catFile = await StorageFile .GetFileFromApplicationUriAsync(new Uri("ms-appx:///InputData/kitten.png")); - var birdImage = await CreateSoftwareBitmapFromStorageFile(birdFile); - var catImage = await CreateSoftwareBitmapFromStorageFile(catFile); - var inputImages = new List(); - for (int i = 0; i < NumInputImages / 2; i++) - { - inputImages.Add(VideoFrame.CreateWithSoftwareBitmap(birdImage)); - inputImages.Add(VideoFrame.CreateWithSoftwareBitmap(catImage)); - } - return inputImages; - } - - private async Task CreateSoftwareBitmapFromStorageFile(StorageFile file) - { - var stream = await file.OpenAsync(FileAccessMode.Read); - var decoder = await BitmapDecoder.CreateAsync(stream); - var bitmap = await decoder.GetSoftwareBitmapAsync(); - return bitmap; - } - - private void UpdateStatus(bool isBatchingEval) - { + var modelName = "squeezenet1.1-7-batched.onnx"; + var modelPath = Path.Join(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, "Models", modelName); + _model = LearningModel.LoadFromFilePath(modelPath); + } + + async private void StartInference(object sender, RoutedEventArgs e) + { + ShowStatus(); + ResetMetrics(); + + var inputImages = await GetInputImages(); + int batchSize = GetBatchSizeFromBatchSizeSlider(); + + _nonBatchingSession = await CreateLearningModelSession(_model); + _batchingSession = await CreateLearningModelSession(_model, batchSize); + + UpdateStatus(false); + await Classify(inputImages); + + UpdateStatus(true); + await ClassifyBatched(inputImages, batchSize); + + ShowUI(); + } + + private void ShowStatus() + { + StartInferenceBtn.IsEnabled = false; + BatchSizeSlider.IsEnabled = false; + DeviceComboBox.IsEnabled = false; + EvalResults.Visibility = Visibility.Collapsed; + LoadingContainer.Visibility = Visibility.Visible; + } + + private void ResetMetrics() + { + _avgNonBatchedDuration = 0; + _avgBatchDuration = 0; + } + + // Test input consists of 50 images (25 bird and 25 cat) + private async Task> GetInputImages() + { + var birdFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///InputData/hummingbird.jpg")); + var catFile = await StorageFile .GetFileFromApplicationUriAsync(new Uri("ms-appx:///InputData/kitten.png")); + var birdImage = await CreateSoftwareBitmapFromStorageFile(birdFile); + var catImage = await CreateSoftwareBitmapFromStorageFile(catFile); + var inputImages = new List(); + for (int i = 0; i < NumInputImages / 2; i++) + { + inputImages.Add(VideoFrame.CreateWithSoftwareBitmap(birdImage)); + inputImages.Add(VideoFrame.CreateWithSoftwareBitmap(catImage)); + } + return inputImages; + } + + private async Task CreateSoftwareBitmapFromStorageFile(StorageFile file) + { + var stream = await file.OpenAsync(FileAccessMode.Read); + var decoder = await BitmapDecoder.CreateAsync(stream); + var bitmap = await decoder.GetSoftwareBitmapAsync(); + return bitmap; + } + + private void UpdateStatus(bool isBatchingEval) + { if (isBatchingEval) { EvalText.Text = "Inferencing Batched Inputs:"; - } + } else { EvalText.Text = "Inferencing Non-Batched Inputs:"; - } - } - - private async Task CreateLearningModelSession(LearningModel model, int batchSizeOverride=-1) - { - var deviceKind = DeviceComboBox.GetDeviceKind(); - var device = new LearningModelDevice(deviceKind); - var options = new LearningModelSessionOptions(); - if (batchSizeOverride > 0) - { - options.BatchSizeOverride = (uint)batchSizeOverride; - } - var session = new LearningModelSession(model, device, options); - return session; - } - - async private Task Classify(List inputImages) - { - float totalEvalDurations = 0; - for (int i = 0; i < NumEvalIterations; i++) - { + } + } + + private async Task CreateLearningModelSession(LearningModel model, int batchSizeOverride=-1) + { + var deviceKind = DeviceComboBox.GetDeviceKind(); + var device = new LearningModelDevice(deviceKind); + var options = new LearningModelSessionOptions(); + if (batchSizeOverride > 0) + { + options.BatchSizeOverride = (uint)batchSizeOverride; + } + var session = new LearningModelSession(model, device, options); + return session; + } + + async private Task Classify(List inputImages) + { + float totalEvalDurations = 0; + for (int i = 0; i < NumEvalIterations; i++) + { if (navigatingAwayFromPage) { break; - } - - UpdateProgress(i); - float evalDuration = await Task.Run(() => Evaluate(_nonBatchingSession, inputImages)); - totalEvalDurations += evalDuration; - } - _avgNonBatchedDuration = totalEvalDurations / NumEvalIterations; - } - - private static float Evaluate(LearningModelSession session, List input) - { - string inputName = session.Model.InputFeatures[0].Name; - float totalDuration = 0; - var binding = new LearningModelBinding(session); - for (int j = 0; j < input.Count; j++) - { + } + + UpdateProgress(i); + float evalDuration = await Task.Run(() => Evaluate(_nonBatchingSession, inputImages)); + totalEvalDurations += evalDuration; + } + _avgNonBatchedDuration = totalEvalDurations / NumEvalIterations; + } + + private static float Evaluate(LearningModelSession session, List input) + { + string inputName = session.Model.InputFeatures[0].Name; + float totalDuration = 0; + var binding = new LearningModelBinding(session); + for (int j = 0; j < input.Count; j++) + { if (navigatingAwayFromPage) { break; - } - - var start = HighResolutionClock.UtcNow(); - binding.Bind(inputName, input[j]); - session.Evaluate(binding, ""); - var stop = HighResolutionClock.UtcNow(); - var duration = HighResolutionClock.DurationInMs(start, stop); - totalDuration += duration; - } - return totalDuration; - } - - async private Task ClassifyBatched(List inputImages, int batchSize) - { - float totalEvalDurations = 0; - for (int i = 0; i < NumEvalIterations; i++) - { - if (navigatingAwayFromPage) - break; - UpdateProgress(i); - float evalDuration = await Task.Run(() => EvaluateBatched(_batchingSession, inputImages, batchSize)); - totalEvalDurations += evalDuration; - } - _avgBatchDuration = totalEvalDurations / NumEvalIterations; - } - - private static float EvaluateBatched(LearningModelSession session, List input, int batchSize) - { - int numBatches = (int) Math.Ceiling((Decimal) input.Count / batchSize); - string inputName = session.Model.InputFeatures[0].Name; - float totalDuration = 0; - var binding = new LearningModelBinding(session); - for (int i = 0; i < numBatches; i++) - { + } + + var start = HighResolutionClock.UtcNow(); + binding.Bind(inputName, input[j]); + session.Evaluate(binding, ""); + var stop = HighResolutionClock.UtcNow(); + var duration = HighResolutionClock.DurationInMs(start, stop); + totalDuration += duration; + } + return totalDuration; + } + + async private Task ClassifyBatched(List inputImages, int batchSize) + { + float totalEvalDurations = 0; + for (int i = 0; i < NumEvalIterations; i++) + { + if (navigatingAwayFromPage) + break; + UpdateProgress(i); + float evalDuration = await Task.Run(() => EvaluateBatched(_batchingSession, inputImages, batchSize)); + totalEvalDurations += evalDuration; + } + _avgBatchDuration = totalEvalDurations / NumEvalIterations; + } + + private static float EvaluateBatched(LearningModelSession session, List input, int batchSize) + { + int numBatches = (int) Math.Ceiling((Decimal) input.Count / batchSize); + string inputName = session.Model.InputFeatures[0].Name; + float totalDuration = 0; + var binding = new LearningModelBinding(session); + for (int i = 0; i < numBatches; i++) + { if (navigatingAwayFromPage) { break; - } - - int rangeStart = batchSize * i; - List batch; - // Add padding to the last batch if necessary - if (rangeStart + batchSize > input.Count) - { - int numInputsRemaining = input.Count - rangeStart; - int paddingAmount = batchSize - numInputsRemaining; - batch = input.GetRange(rangeStart, numInputsRemaining); - batch.AddRange(input.GetRange(0, paddingAmount)); - } - else - { - batch = input.GetRange(rangeStart, batchSize); - } - var start = HighResolutionClock.UtcNow(); - binding.Bind(inputName, batch); - session.Evaluate(binding, ""); - var stop = HighResolutionClock.UtcNow(); - var duration = HighResolutionClock.DurationInMs(start, stop); - totalDuration += duration; - } - return totalDuration; - } - - private int GetBatchSizeFromBatchSizeSlider() - { - return int.Parse(BatchSizeSlider.Value.ToString()); - } - - private void UpdateProgress(int attemptNumber) - { - EvalProgressText.Text = "Attempt " + attemptNumber.ToString() + "/" + NumEvalIterations.ToString(); - EvalProgressBar.Value = attemptNumber + 1; - } - - private void ShowUI() - { - float ratio = (1 - (_avgBatchDuration / _avgNonBatchedDuration)) * 100; - var evalResult = new EvalResult - { - nonBatchedAvgTime = _avgNonBatchedDuration.ToString("0.00"), - batchedAvgTime = _avgBatchDuration.ToString("0.00"), - timeRatio = ratio.ToString("0.0") - }; - List results = new List(); - results.Insert(0, evalResult); - LoadingContainer.Visibility = Visibility.Collapsed; - EvalResults.Visibility = Visibility.Visible; - StartInferenceBtn.IsEnabled = true; - BatchSizeSlider.IsEnabled = true; - DeviceComboBox.IsEnabled = true; - EvalResults.ItemsSource = results; - } - - private void UpdateBatchSizeText(object sender, RoutedEventArgs e) - { - BatchSizeText.Text = "Batch Size: " + BatchSizeSlider.Value.ToString(); - } - - public void StopAllEvents() - { - navigatingAwayFromPage = true; - } - } -} + } + + int rangeStart = batchSize * i; + List batch; + // Add padding to the last batch if necessary + if (rangeStart + batchSize > input.Count) + { + int numInputsRemaining = input.Count - rangeStart; + int paddingAmount = batchSize - numInputsRemaining; + batch = input.GetRange(rangeStart, numInputsRemaining); + batch.AddRange(input.GetRange(0, paddingAmount)); + } + else + { + batch = input.GetRange(rangeStart, batchSize); + } + var start = HighResolutionClock.UtcNow(); + binding.Bind(inputName, batch); + session.Evaluate(binding, ""); + var stop = HighResolutionClock.UtcNow(); + var duration = HighResolutionClock.DurationInMs(start, stop); + totalDuration += duration; + } + return totalDuration; + } + + private int GetBatchSizeFromBatchSizeSlider() + { + return int.Parse(BatchSizeSlider.Value.ToString()); + } + + private void UpdateProgress(int attemptNumber) + { + EvalProgressText.Text = "Attempt " + attemptNumber.ToString() + "/" + NumEvalIterations.ToString(); + EvalProgressBar.Value = attemptNumber + 1; + } + + private void ShowUI() + { + float ratio = (1 - (_avgBatchDuration / _avgNonBatchedDuration)) * 100; + var evalResult = new EvalResult + { + nonBatchedAvgTime = _avgNonBatchedDuration.ToString("0.00"), + batchedAvgTime = _avgBatchDuration.ToString("0.00"), + timeRatio = ratio.ToString("0.0") + }; + List results = new List(); + results.Insert(0, evalResult); + LoadingContainer.Visibility = Visibility.Collapsed; + EvalResults.Visibility = Visibility.Visible; + StartInferenceBtn.IsEnabled = true; + BatchSizeSlider.IsEnabled = true; + DeviceComboBox.IsEnabled = true; + EvalResults.ItemsSource = results; + } + + private void UpdateBatchSizeText(object sender, RoutedEventArgs e) + { + BatchSizeText.Text = "Batch Size: " + BatchSizeSlider.Value.ToString(); + } + + public void StopAllEvents() + { + navigatingAwayFromPage = true; + } + } +} diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml index 4b780b19..e5260dd0 100644 --- a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml @@ -1,12 +1,11 @@  @@ -16,7 +15,10 @@ - + + + + + !--> - + --> + diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml.cs b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml.cs index d1d9029f..bd32acf6 100644 --- a/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml.cs +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/StreamEffect/StreamEffect.xaml.cs @@ -26,6 +26,9 @@ using Windows.Media.Editing; using Windows.Media.Core; using Windows.Media.Playback; +using WinMLSamplesGalleryNative; + + namespace WinMLSamplesGallery.Samples { public class StreamEffectViewModel : INotifyPropertyChanged @@ -80,368 +83,14 @@ namespace WinMLSamplesGallery.Samples public sealed partial class StreamEffect : Page { - private LearningModel _learningModel = null; - private LearningModelBinding _binding; - private string _modelSource = "fcn-resnet50-11.onnx"; - private bool _useGpu = true; - private string _imgSource = "stream.jpg"; - uint _inWidth, _inHeight; - - LearningModelSession tensorizationSession = null; - LearningModelSession normalizeSession = null; - LearningModelSession modelSession = null; - LearningModelSession labelsSession = null; - LearningModelSession backgroundSession = null; - LearningModelSession blurSession = null; - LearningModelSession foregroundSession = null; - LearningModelSession detensorizationSession = null; - - //Media capture fields - StreamEffectViewModel streamEffectViewModel = new StreamEffectViewModel(); - private MediaComposition composition; - - - + public StreamEffect() { this.InitializeComponent(); - //composition = new MediaComposition(); - //streamEffectViewModel.GetDevices(); - } - - private async Task PickFileAndAddClip() - { - var picker = new Windows.Storage.Pickers.FileOpenPicker(); - picker.SuggestedStartLocation = Windows.Storage.Pickers.PickerLocationId.VideosLibrary; - picker.FileTypeFilter.Add(".mp4"); - Windows.Storage.StorageFile pickedFile = await picker.PickSingleFileAsync(); - if (pickedFile == null) - { - Debug.WriteLine("File picking cancelled"); - return; - } - - // These files could be picked from a location that we won't have access to later - var storageItemAccessList = Windows.Storage.AccessCache.StorageApplicationPermissions.FutureAccessList; - storageItemAccessList.Add(pickedFile); - - var clip = await MediaClip.CreateFromFileAsync(pickedFile); - composition.Clips.Add(clip); - + WinMLSamplesGalleryNative.StreamEffect.LaunchNewWindow(); } - private async void ChangeLiveStream(object sender, RoutedEventArgs e) - { - CameraCaptureUI captureUI = new CameraCaptureUI(); - captureUI.PhotoSettings.AllowCropping = false; - captureUI.PhotoSettings.Format = CameraCaptureUIPhotoFormat.Png; - - StorageFile videoFile = await captureUI.CaptureFileAsync(CameraCaptureUIMode.Photo); - if (videoFile == null) - { - // User cancelled photo capture - return; - } - } - - private async void CleanUpCameraAsync() - { - return; - } - - private async void LoadModelAsync() - { - Debug.Write("LoadModelBegin | "); - - Debug.Write("LoadModel Lock | "); - - _binding?.Clear(); - modelSession?.Dispose(); - - StorageFile modelFile = StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///LargeModels/{_modelSource}")).AsTask().GetAwaiter().GetResult(); - - _learningModel = LearningModel.LoadFromStorageFileAsync(modelFile).AsTask().GetAwaiter().GetResult(); - - modelSession = CreateLearningModelSession(_learningModel); - _binding = new LearningModelBinding(modelSession); - - debugModelIO(); - - Debug.Write("LoadModel Unlock\n"); - } - - private LearningModelSession CreateLearningModelSession(LearningModel model, bool closeModel = true) - { - var device = _useGpu ? new LearningModelDevice(LearningModelDeviceKind.DirectX) : new LearningModelDevice(LearningModelDeviceKind.Cpu); - var options = new LearningModelSessionOptions() - { - CloseModelOnSessionCreation = closeModel, // Close the model to prevent extra memory usage - BatchSizeOverride = 0 - }; - var session = new LearningModelSession(model, device, options); - return session; - } - - public void debugModelIO() - { - foreach (var inputF in _learningModel.InputFeatures) - { - TensorFeatureDescriptor tfDesc = inputF as TensorFeatureDescriptor; - Debug.WriteLine($"input | kind:{inputF.Kind}, name:{inputF.Name}" + - $" Shape: {string.Join(",", tfDesc.Shape.ToArray())}"); - } - foreach (var outputF in _learningModel.OutputFeatures) - { - TensorFeatureDescriptor tfDesc = outputF as TensorFeatureDescriptor; - Debug.WriteLine($"output | kind:{outputF.Kind}, name:{outputF.Name}" + - $" Shape: {string.Join(",", tfDesc.Shape.ToArray())}"); - } - } - - public async void getImageAsync() - { - - Debug.WriteLine("In GetImage"); - StorageFile file = StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///InputData/{_imgSource}")).GetAwaiter().GetResult(); - using (IRandomAccessStream stream = file.OpenAsync(FileAccessMode.Read).GetAwaiter().GetResult()) - { - var start = HighResolutionClock.UtcNow(); - - Debug.WriteLine("Got stream"); - BitmapDecoder decoder = BitmapDecoder.CreateAsync(stream).AsTask().GetAwaiter().GetResult(); - Debug.WriteLine("Got decoder"); - - _inWidth = decoder.PixelWidth; - _inHeight = decoder.PixelHeight; - - var pixelDataProvider = await decoder.GetPixelDataAsync();//.GetAwaiter().GetResult(); - Debug.WriteLine("Got pixeldata"); - - // 1. Tensorize input image - var bytes = pixelDataProvider.DetachPixelData(); - var buffer = bytes.AsBuffer(); // Does this do a copy?? - var inputRawTensor = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer); - var nextOutputShape = new long[] { 1, 3, _inHeight, _inWidth }; - var tensorizedImg = TensorFloat.Create(nextOutputShape); // Need to keep this intermediate for blur/bckground - - // Reshape initial input - tensorizationSession = CreateLearningModelSession( - TensorizationModels.ReshapeFlatBufferToNCHW(1, 4, _inHeight, _inWidth)); - var tensorizationBinding = Evaluate(tensorizationSession, inputRawTensor, tensorizedImg); - Debug.WriteLine($"Intermediate Shape: {string.Join(",", tensorizedImg.Shape.ToArray())}"); - - // 2. Normalize input image - float[] mean = new float[] { 0.485f, 0.456f, 0.406f }; - float[] std = new float[] { 0.229f, 0.224f, 0.225f }; - // Already sliced out alpha, but normalize0_1 expects the input to still have it- could just write another version later - // Normalize model will tensorize the arrays (and that means that it will move over to the GPU?) - normalizeSession = CreateLearningModelSession( - TensorizationModels.Normalize0_1ThenZScore(_inHeight, _inWidth, 4, mean, std)); - var intermediateTensor = TensorFloat.Create(tensorizedImg.Shape); - var normalizationBinding = Evaluate(normalizeSession, tensorizedImg, intermediateTensor); - - // 3. Run through actual model - var modelOutputShape = new long[] { 1, 21, _inHeight, _inWidth }; - var modelOutputTensor = TensorFloat.Create(modelOutputShape); - var modelBinding = Evaluate(modelSession, intermediateTensor, modelOutputTensor); - - // 4. Get the class predictions for each pixel - var rawLabels = TensorFloat.Create(new long[] { 1, 1, _inHeight, _inWidth }); - labelsSession = CreateLearningModelSession(ArgMax(1, _inHeight, _inWidth)); - var labelsBinding = Evaluate(labelsSession, modelOutputTensor, rawLabels); - - // 5. Extract just the blurred background based on mask - // Create a blurred version of the original picture - - var outputBindProperties = new PropertySet(); - outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true)); - - Trace.Assert(nextOutputShape != null); - var blurNoMask = TensorFloat.Create((IEnumerable)(nextOutputShape)); - Trace.WriteLine($"test {nextOutputShape.Length}"); - blurSession = CreateLearningModelSession(TensorizationModels.AveragePool(20)); - var blurBinding = Evaluate(blurSession, tensorizedImg, blurNoMask, true); - - var blurredImg = TensorFloat.Create(nextOutputShape); - backgroundSession = CreateLearningModelSession(GetBackground(1, 3, _inHeight, _inWidth)); - var binding = new LearningModelBinding(backgroundSession); - Console.WriteLine($"Intermediate Shape: {string.Join(",", blurNoMask.Shape.ToArray())}"); - binding.Bind(backgroundSession.Model.InputFeatures[0].Name, blurNoMask); // Intermediate tensor isn't on GPU? - binding.Bind(backgroundSession.Model.InputFeatures[1].Name, rawLabels); - - - binding.Bind(backgroundSession.Model.OutputFeatures[0].Name, blurredImg); - EvaluateInternal(backgroundSession, binding); - - // 6. Extract just the foreground based on mask and combine with background - intermediateTensor = TensorFloat.Create(nextOutputShape); - foregroundSession = CreateLearningModelSession(GetOutputFrame(1, 3, _inHeight, _inWidth)); - binding = new LearningModelBinding(foregroundSession); - binding.Bind(foregroundSession.Model.InputFeatures[0].Name, tensorizedImg); - binding.Bind(foregroundSession.Model.InputFeatures[1].Name, rawLabels); - binding.Bind(foregroundSession.Model.InputFeatures[2].Name, blurredImg); - binding.Bind(foregroundSession.Model.OutputFeatures[0].Name, intermediateTensor, outputBindProperties); - EvaluateInternal(foregroundSession, binding); - - // 7. Detensorize and display output - var outputFrame = Detensorize(intermediateTensor); - RenderOutputFrame(outputFrame); - var stop = HighResolutionClock.UtcNow(); - Debug.WriteLine($"*** DONE IN {HighResolutionClock.DurationInMs(start, stop)} Ms"); - } - - } - - public static LearningModel ArgMax(long axis, long h, long w) - { - var builder = LearningModelBuilder.Create(12) - .Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Data", TensorKind.Float, new long[] { -1, -1, h, w })) // Different input type? - .Outputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Output", TensorKind.Float, new long[] { -1, -1, h, w })) // Output of int64? - .Operators.Add(new LearningModelOperator("ArgMax") - .SetInput("data", "Data") - .SetAttribute("keepdims", TensorInt64Bit.CreateFromArray(new List(), new long[] { 1 })) - .SetAttribute("axis", TensorInt64Bit.CreateFromIterable(new long[] { }, new long[] { axis })) // Correct way of passing axis? - .SetOutput("reduced", "Reduced")) - .Operators.Add(new LearningModelOperator("Cast") - .SetInput("input", "Reduced") - .SetAttribute("to", TensorInt64Bit.CreateFromIterable(new long[] { }, new long[] { (long)TensorizationModels.OnnxDataType.FLOAT })) - .SetOutput("output", "Output")) - ; - - return builder.CreateModel(); - } - - public static LearningModel GetOutputFrame(long n, long c, long h, long w) - { - var builder = LearningModelBuilder.Create(12) - .Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputImage", TensorKind.Float, new long[] { n, c, h, w })) - .Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputMask", TensorKind.Float, new long[] { n, 1, h, w })) // Broadcast to each color channel - .Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputBackground", TensorKind.Float, new long[] { n, c, h, w })) - .Outputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Output", TensorKind.Float, new long[] { n, c, h, w })) - .Operators.Add(new LearningModelOperator("Clip") - .SetInput("input", "InputMask") - .SetConstant("max", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 1 })) - .SetOutput("output", "MaskBinary")) - .Operators.Add(new LearningModelOperator("Mul") - .SetInput("A", "InputImage") - .SetInput("B", "MaskBinary") - .SetOutput("C", "Foreground")) - .Operators.Add(new LearningModelOperator("Add") - .SetInput("A", "InputBackground") - .SetInput("B", "Foreground") - .SetOutput("C", "Output")) - ; - - return builder.CreateModel(); - } - - public static LearningModel GetBackground(long n, long c, long h, long w) - { - var builder = LearningModelBuilder.Create(11) - .Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputImage", TensorKind.Float, new long[] { n, c, h, w })) - .Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("InputMask", TensorKind.Float, new long[] { n, 1, h, w })) // Broadcast to each color channel - .Outputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Output", TensorKind.Float, new long[] { n, c, h, w })) - .Operators.Add(new LearningModelOperator("Clip") // Make mask binary - .SetInput("input", "InputMask") - .SetConstant("max", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 1 })) - .SetOutput("output", "ClipMask")) - .Operators.Add(new LearningModelOperator("Mul") - .SetInput("A", "ClipMask") - .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { -1 })) - .SetOutput("C", "NegMask")) - .Operators.Add(new LearningModelOperator("Add") // BackgroundMask = (1- InputMask) - .SetConstant("A", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 1 })) - .SetInput("B", "NegMask") - .SetOutput("C", "BackgroundMask")) - .Operators.Add(new LearningModelOperator("Mul") // Extract the background - .SetInput("A", "InputImage") - .SetInput("B", "BackgroundMask") - .SetOutput("C", "Output")) - ; - - return builder.CreateModel(); - } - - public void RenderOutputFrame(VideoFrame outputFrame) - { - SoftwareBitmap displayBitmap = outputFrame.SoftwareBitmap; - //Image control only accepts BGRA8 encoding and Premultiplied/no alpha channel. This checks and converts - //the SoftwareBitmap we want to bind. - if (displayBitmap.BitmapPixelFormat != BitmapPixelFormat.Bgra8 || - displayBitmap.BitmapAlphaMode != BitmapAlphaMode.Premultiplied) - { - displayBitmap = SoftwareBitmap.Convert(displayBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); - } - - // get software bitmap souce - var source = new SoftwareBitmapSource(); - source.SetBitmapAsync(displayBitmap).GetAwaiter(); - // draw the input image - InputImage.Source = source; - } - private VideoFrame Detensorize(TensorFloat intermediateTensor) - { - // Change from Int64 to float - //ITensor interm = intermediateTensor; - var shape = intermediateTensor.Shape; - var n = (int)shape[0]; - var c = (int)shape[1]; - var h = (int)shape[2]; - var w = (int)shape[3]; - - // Rather than writing the data into the software bitmap ourselves from a Tensor (which may be on the gpu) - // we call an indentity model to move the gpu memory back to the cpu via WinML de-tensorization. - var outputImage = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Ignore); - var outputFrame = VideoFrame.CreateWithSoftwareBitmap(outputImage); - - detensorizationSession = CreateLearningModelSession(TensorizationModels.IdentityNCHW(1, c, _inHeight, _inWidth)); - var descriptor = detensorizationSession.Model.InputFeatures[0] as TensorFeatureDescriptor; - var detensorizerShape = descriptor.Shape; - /*if (c != detensorizerShape[1] || h != detensorizerShape[2] || w != detensorizerShape[3]) - { - detensorizationSession = CreateLearningModelSession(TensorizationModels.IdentityNCHW(n, c, h, w)); - }*/ - var detensorizationBinding = Evaluate(detensorizationSession, intermediateTensor, outputFrame, true); - return outputFrame; - - } - - private LearningModelBinding Evaluate(LearningModelSession session, object input, object output, bool wait = false) - { - // Create the binding - var binding = new LearningModelBinding(session); - - // Bind inputs and outputs - string inputName = session.Model.InputFeatures[0].Name; - string outputName = session.Model.OutputFeatures[0].Name; - binding.Bind(inputName, input); - - var outputBindProperties = new PropertySet(); - outputBindProperties.Add("DisableTensorCpuSync", PropertyValue.CreateBoolean(true)); - binding.Bind(outputName, output, outputBindProperties); - - // Evaluate - EvaluateInternal(session, binding, wait); - - return binding; - } - - private void EvaluateInternal(LearningModelSession session, LearningModelBinding binding, bool wait = true) - { - if (!_useGpu) //originally isCpu - { - session.Evaluate(binding, ""); - } - else - { - var results = session.Evaluate(binding, ""); - /*if (wait) - { - results.GetAwaiter().GetResult(); - }*/ - } - } } } diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/StreamEffect.cpp b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/StreamEffect.cpp new file mode 100644 index 00000000..00bf0f8f --- /dev/null +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/StreamEffect.cpp @@ -0,0 +1,78 @@ +#include "pch.h" +#include "StreamEffect.h" +#include "StreamEffect.g.cpp" + +#include +#include + + +namespace winrt::WinMLSamplesGalleryNative::implementation +{ + LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { + switch (uMsg) { + default: + return 1; + } + }; + + void StreamEffect::LaunchNewWindow() + { + HWND hwnd; + HRESULT hr = S_OK; + BOOL bMFStartup = false; + + // Initialize the common controls + const INITCOMMONCONTROLSEX icex = { sizeof(INITCOMMONCONTROLSEX), ICC_WIN95_CLASSES }; + InitCommonControlsEx(&icex); + + hr = MFStartup(MF_VERSION); + if (FAILED(hr)) + { + goto done; + } + + // Create window + const wchar_t CLASS_NAME[] = L"Capture Engine Window Class"; + INT nCmdShow = 1; + + WNDCLASS wc = { 0 }; + + wc.lpfnWndProc = WindowProc; + wc.hInstance = GetModuleHandle(NULL); + wc.lpszClassName = CLASS_NAME; + + RegisterClass(&wc); + + hwnd = CreateWindowEx( + 0, CLASS_NAME, L"Capture Application", WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, + CW_USEDEFAULT, CW_USEDEFAULT, NULL, NULL, GetModuleHandle(NULL), NULL + ); + + if (hwnd == 0) + { + throw_hresult(E_FAIL); + } + + ShowWindow(hwnd, nCmdShow); + + // Run the main message loop + MSG msg; + while (GetMessage(&msg, NULL, 0, 0)) + { + TranslateMessage(&msg); + DispatchMessage(&msg); + } + + done: + if (FAILED(hr)) + { + // TODO: Add utils from BackgroundBlur sample utils.cpp + //ShowError(NULL, L"Failed to start application", hr); + } + if (bMFStartup) + { + MFShutdown(); + } + return ; + } +} diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/StreamEffect.h b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/StreamEffect.h new file mode 100644 index 00000000..6c2a2074 --- /dev/null +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/StreamEffect.h @@ -0,0 +1,19 @@ +#pragma once +#include "StreamEffect.g.h" + + +namespace winrt::WinMLSamplesGalleryNative::implementation +{ + struct StreamEffect : StreamEffectT + { + StreamEffect() = default; + + static void LaunchNewWindow(); + }; +} +namespace winrt::WinMLSamplesGalleryNative::factory_implementation +{ + struct StreamEffect : StreamEffectT + { + }; +} diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.idl b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.idl index c997de1d..b4b3bee6 100644 --- a/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.idl +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.idl @@ -16,4 +16,13 @@ namespace WinMLSamplesGalleryNative { static Microsoft.AI.MachineLearning.LearningModel LoadEncryptedResource(String key); } + + [default_interface] + runtimeclass StreamEffect + { + static void LaunchNewWindow(); + } + + + } diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj index 769e2331..f638acb9 100644 --- a/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj @@ -128,6 +128,15 @@ $(MSBuildThisFileDirectory)../WinMLSamplesGallery/Models/;$(MSBuildThisFileDirectory)../../build/native/include/;%(AdditionalIncludeDirectories) + + Comctl32.lib;%(AdditionalDependencies) + + + Comctl32.lib;%(AdditionalDependencies) + + + Comctl32.lib;%(AdditionalDependencies) + @@ -136,6 +145,9 @@ true true + Comctl32.lib;%(AdditionalDependencies) + Comctl32.lib;%(AdditionalDependencies) + Comctl32.lib;%(AdditionalDependencies) $(MSBuildThisFileDirectory)../WinMLSamplesGallery/Models/;$(MSBuildThisFileDirectory)../../build/native/include/;%(AdditionalIncludeDirectories) @@ -146,6 +158,7 @@ + @@ -155,6 +168,7 @@ Create + diff --git a/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj.filters b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj.filters index 5fcb1294..3822b608 100644 --- a/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj.filters +++ b/Samples/WinMLSamplesGallery/WinMLSamplesGalleryNative/WinMLSamplesGalleryNative.vcxproj.filters @@ -9,6 +9,9 @@ OpenCVInterop + + StreamEffect + @@ -24,6 +27,9 @@ AbiHelpers + + StreamEffect + @@ -51,5 +57,8 @@ {ed4f3871-4514-4b93-9ace-1315cad5491e} + + {91675ea9-10e7-4490-af40-762311cf5ba9} + \ No newline at end of file