Merge pull request #1 from Microsoft/user/Louis/StyleTransferSample

Adding Candy Style Transfer sample
This commit is contained in:
ngeisler11 2018-03-06 10:27:18 -08:00 коммит произвёл GitHub
Родитель f6ce549f13 105cfe192d
Коммит 1d81208f88
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
21 изменённых файлов: 1120 добавлений и 0 удалений

Просмотреть файл

@ -0,0 +1,42 @@
# Candy Style Transfer Sample
A simple UWP application that uses a trained machine learning model to transfer the Candy style onto an image selected by the user either from file or captured with a camera.
This sample demonstrates the use of generic Windows.AI.MachineLearning.Preview API to load a model, bind input and output images, and evaluate a binding. You can use Netron to determine the input and output requirements of your ONNX model which are presumed to be known in this particular sample. https://github.com/lutzroeder/Netron
## Build the sample
1. If you download the samples ZIP, be sure to unzip the entire archive, not just the folder with
the sample you want to build.
2. Start Microsoft Visual Studio 2017 and select **File** \> **Open** \> **Project/Solution**.
3. Starting in the folder where you unzipped the samples, go to the Samples subfolder, then the
subfolder for this specific sample. Double-click the Visual Studio project file (.csproj) file.
4. Press Ctrl+Shift+B, or select **Build** \> **Build Solution**.
## Run the sample
The next steps depend on whether you just want to deploy the sample or you want to both deploy and
run it.
### Deploying the sample
- Select Build > Deploy Solution.
### Deploying and running the sample
- To debug the sample and then run it, press F5 or select Debug > Start Debugging. To run the sample without debugging, press Ctrl+F5 or selectDebug > Start Without Debugging.

Просмотреть файл

@ -0,0 +1,8 @@
<Application
x:Class="StyleTransfer.App"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="using:StyleTransfer"
RequestedTheme="Light">
</Application>

Просмотреть файл

@ -0,0 +1,90 @@
using System;
using Windows.ApplicationModel;
using Windows.ApplicationModel.Activation;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Navigation;
namespace StyleTransfer
{
/// <summary>
/// Provides application-specific behavior to supplement the default Application class.
/// </summary>
sealed partial class App : Application
{
/// <summary>
/// Initializes the singleton application object. This is the first line of authored code
/// executed, and as such is the logical equivalent of main() or WinMain().
/// </summary>
public App()
{
this.InitializeComponent();
this.Suspending += OnSuspending;
}
/// <summary>
/// Invoked when the application is launched normally by the end user. Other entry points
/// will be used such as when the application is launched to open a specific file.
/// </summary>
/// <param name="e">Details about the launch request and process.</param>
protected override void OnLaunched(LaunchActivatedEventArgs e)
{
Frame rootFrame = Window.Current.Content as Frame;
// Do not repeat app initialization when the Window already has content,
// just ensure that the window is active
if (rootFrame == null)
{
// Create a Frame to act as the navigation context and navigate to the first page
rootFrame = new Frame();
rootFrame.NavigationFailed += OnNavigationFailed;
if (e.PreviousExecutionState == ApplicationExecutionState.Terminated)
{
//TODO: Load state from previously suspended application
}
// Place the frame in the current Window
Window.Current.Content = rootFrame;
}
if (e.PrelaunchActivated == false)
{
if (rootFrame.Content == null)
{
// When the navigation stack isn't restored navigate to the first page,
// configuring the new page by passing required information as a navigation
// parameter
rootFrame.Navigate(typeof(MainPage), e.Arguments);
}
// Ensure the current window is active
Window.Current.Activate();
}
}
/// <summary>
/// Invoked when Navigation to a certain page fails
/// </summary>
/// <param name="sender">The Frame which failed navigation</param>
/// <param name="e">Details about the navigation failure</param>
void OnNavigationFailed(object sender, NavigationFailedEventArgs e)
{
throw new Exception("Failed to load Page " + e.SourcePageType.FullName);
}
/// <summary>
/// Invoked when application execution is being suspended. Application state is saved
/// without knowing whether the application will be terminated or resumed with the contents
/// of memory still intact.
/// </summary>
/// <param name="sender">The source of the suspend request.</param>
/// <param name="e">Details about the suspend request.</param>
private void OnSuspending(object sender, SuspendingEventArgs e)
{
var deferral = e.SuspendingOperation.GetDeferral();
//TODO: Save application state and stop any background activity
deferral.Complete();
}
}
}

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/Candy.onnx Normal file

Двоичный файл не отображается.

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/DefaultImage.jpg Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 277 KiB

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/frame_camera2.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 2.6 KiB

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/frame_camera_150.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 12 KiB

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/frame_camera_24.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 895 B

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/frame_camera_310-150.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 14 KiB

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/frame_camera_310.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 43 KiB

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/frame_camera_44.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 1.8 KiB

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/frame_camera_50.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 2.1 KiB

Двоичные данные
Samples/UWP/CandyStyleTransfer/src/Assets/frame_camera_620-300.png Normal file

Двоичный файл не отображается.

После

Ширина:  |  Высота:  |  Размер: 38 KiB

Просмотреть файл

@ -0,0 +1,175 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
<ProjectGuid>{97D29DB4-FA18-4816-A9A5-FFBFCFC0104C}</ProjectGuid>
<OutputType>AppContainerExe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>CandyStyleTransfer</RootNamespace>
<AssemblyName>CandyStyleTransfer</AssemblyName>
<DefaultLanguage>en-US</DefaultLanguage>
<TargetPlatformIdentifier>UAP</TargetPlatformIdentifier>
<TargetPlatformVersion Condition=" '$(TargetPlatformVersion)' == '' ">10.0.17110.0</TargetPlatformVersion>
<TargetPlatformMinVersion>10.0.17110.0</TargetPlatformMinVersion>
<MinimumVisualStudioVersion>14</MinimumVisualStudioVersion>
<FileAlignment>512</FileAlignment>
<ProjectTypeGuids>{A5A43C5B-DE2A-4C0C-9213-0A381AF9435A};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
<WindowsXamlEnableOverview>true</WindowsXamlEnableOverview>
<PackageCertificateKeyFile>StyleTransfer_TemporaryKey.pfx</PackageCertificateKeyFile>
<AppxAutoIncrementPackageRevision>True</AppxAutoIncrementPackageRevision>
<AppxBundle>Always</AppxBundle>
<AppxBundlePlatforms>x64</AppxBundlePlatforms>
<PackageCertificateThumbprint>ED2E5FC66B1C377DF2340632FDD8F66F6BAF3DE7</PackageCertificateThumbprint>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x86\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP</DefineConstants>
<NoWarn>;2008</NoWarn>
<DebugType>full</DebugType>
<PlatformTarget>x86</PlatformTarget>
<UseVSHostingProcess>false</UseVSHostingProcess>
<ErrorReport>prompt</ErrorReport>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
<OutputPath>bin\x86\Release\</OutputPath>
<DefineConstants>TRACE;NETFX_CORE;WINDOWS_UWP</DefineConstants>
<Optimize>true</Optimize>
<NoWarn>;2008</NoWarn>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x86</PlatformTarget>
<UseVSHostingProcess>false</UseVSHostingProcess>
<ErrorReport>prompt</ErrorReport>
<Prefer32Bit>true</Prefer32Bit>
<UseDotNetNativeToolchain>true</UseDotNetNativeToolchain>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|ARM'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\ARM\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP</DefineConstants>
<NoWarn>;2008</NoWarn>
<DebugType>full</DebugType>
<PlatformTarget>ARM</PlatformTarget>
<UseVSHostingProcess>false</UseVSHostingProcess>
<ErrorReport>prompt</ErrorReport>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|ARM'">
<OutputPath>bin\ARM\Release\</OutputPath>
<DefineConstants>TRACE;NETFX_CORE;WINDOWS_UWP</DefineConstants>
<Optimize>true</Optimize>
<NoWarn>;2008</NoWarn>
<DebugType>pdbonly</DebugType>
<PlatformTarget>ARM</PlatformTarget>
<UseVSHostingProcess>false</UseVSHostingProcess>
<ErrorReport>prompt</ErrorReport>
<Prefer32Bit>true</Prefer32Bit>
<UseDotNetNativeToolchain>true</UseDotNetNativeToolchain>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP</DefineConstants>
<NoWarn>;2008</NoWarn>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<UseVSHostingProcess>false</UseVSHostingProcess>
<ErrorReport>prompt</ErrorReport>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>bin\x64\Release\</OutputPath>
<DefineConstants>TRACE;NETFX_CORE;WINDOWS_UWP</DefineConstants>
<Optimize>true</Optimize>
<NoWarn>;2008</NoWarn>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x64</PlatformTarget>
<UseVSHostingProcess>false</UseVSHostingProcess>
<ErrorReport>prompt</ErrorReport>
<Prefer32Bit>true</Prefer32Bit>
<UseDotNetNativeToolchain>true</UseDotNetNativeToolchain>
</PropertyGroup>
<PropertyGroup>
<RestoreProjectStyle>PackageReference</RestoreProjectStyle>
</PropertyGroup>
<ItemGroup>
<Compile Include="App.xaml.cs">
<DependentUpon>App.xaml</DependentUpon>
</Compile>
<Compile Include="FrameRenderer.cs" />
<Compile Include="HelperMethods.cs" />
<Compile Include="MainPage.xaml.cs">
<DependentUpon>MainPage.xaml</DependentUpon>
</Compile>
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<AppxManifest Include="Package.appxmanifest">
<SubType>Designer</SubType>
</AppxManifest>
</ItemGroup>
<ItemGroup>
<Content Include="Assets\DefaultImage.jpg">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Assets\frame_camera_150.png">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Assets\frame_camera_24.png">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Assets\frame_camera_310-150.png">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Assets\frame_camera_310.png">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Assets\frame_camera_44.png">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Assets\frame_camera_50.png">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Assets\frame_camera_620-300.png">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Properties\Default.rd.xml" />
</ItemGroup>
<ItemGroup>
<ApplicationDefinition Include="App.xaml">
<Generator>MSBuild:Compile</Generator>
<SubType>Designer</SubType>
</ApplicationDefinition>
<Page Include="MainPage.xaml">
<Generator>MSBuild:Compile</Generator>
<SubType>Designer</SubType>
</Page>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NETCore.UniversalWindowsPlatform">
<Version>6.0.4</Version>
</PackageReference>
</ItemGroup>
<ItemGroup>
<Content Include="Assets\Candy.onnx">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<None Include="StyleTransfer_TemporaryKey.pfx" />
</ItemGroup>
<PropertyGroup Condition=" '$(VisualStudioVersion)' == '' or '$(VisualStudioVersion)' &lt; '14.0' ">
<VisualStudioVersion>14.0</VisualStudioVersion>
</PropertyGroup>
<Import Project="$(MSBuildExtensionsPath)\Microsoft\WindowsXaml\v$(VisualStudioVersion)\Microsoft.Windows.UI.Xaml.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

Просмотреть файл

@ -0,0 +1,69 @@
using System;
using System.Diagnostics;
using System.Threading;
using Windows.Graphics.Imaging;
using Windows.UI.Core;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Media.Imaging;
namespace StyleTransfer
{
public sealed class FrameRenderer
{
private Image _imageElement;
private SoftwareBitmap _backBuffer;
private bool _taskRunning = false;
public FrameRenderer(Image imageElement)
{
_imageElement = imageElement;
_imageElement.Source = new SoftwareBitmapSource();
}
public void RenderFrame(SoftwareBitmap softwareBitmap)
{
if (softwareBitmap != null)
{
// Swap the processed frame to _backBuffer and trigger UI thread to render it
softwareBitmap = Interlocked.Exchange(ref _backBuffer, softwareBitmap);
// UI thread always reset _backBuffer before using it. Unused bitmap should be disposed.
softwareBitmap?.Dispose();
// Changes to xaml ImageElement must happen in UI thread through Dispatcher
var task = _imageElement.Dispatcher.RunAsync(CoreDispatcherPriority.Normal,
async () =>
{
// Don't let two copies of this task run at the same time.
if (_taskRunning)
{
return;
}
_taskRunning = true;
try
{
// Keep draining frames from the backbuffer until the backbuffer is empty.
SoftwareBitmap latestBitmap;
while ((latestBitmap = Interlocked.Exchange(ref _backBuffer, null)) != null)
{
if (_imageElement.MaxHeight != latestBitmap.PixelHeight)
_imageElement.MaxHeight = latestBitmap.PixelHeight;
if (_imageElement.MaxWidth != latestBitmap.PixelWidth)
_imageElement.MaxWidth = latestBitmap.PixelWidth;
var imageSource = (SoftwareBitmapSource)_imageElement.Source;
await imageSource.SetBitmapAsync(latestBitmap);
latestBitmap.Dispose();
}
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
_taskRunning = false;
});
}
}
}
}

Просмотреть файл

@ -0,0 +1,148 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.InteropServices.WindowsRuntime;
using Windows.Foundation;
using Windows.Graphics.Imaging;
using Windows.Media;
using Windows.Storage;
using Windows.Storage.Pickers;
using Windows.Storage.Streams;
namespace StyleTransfer
{
public sealed class ImageHelper
{
/// <summary>
/// Pass the nput frame to a frame renderer and ensure proper image format is used
/// </summary>
/// <param name="inputVideoFrame"></param>
/// <param name="useDX"></param>
/// <returns></returns>
public static IAsyncAction RenderFrameAsync(FrameRenderer frameRenderer, VideoFrame inputVideoFrame)
{
return AsyncInfo.Run(async (token) =>
{
bool useDX = inputVideoFrame.SoftwareBitmap == null;
if (frameRenderer == null)
{
throw (new InvalidOperationException("FrameRenderer is null"));
}
SoftwareBitmap softwareBitmap = null;
if (useDX)
{
softwareBitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(inputVideoFrame.Direct3DSurface);
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
}
else
{
softwareBitmap = new SoftwareBitmap(
inputVideoFrame.SoftwareBitmap.BitmapPixelFormat,
inputVideoFrame.SoftwareBitmap.PixelWidth,
inputVideoFrame.SoftwareBitmap.PixelHeight,
inputVideoFrame.SoftwareBitmap.BitmapAlphaMode);
inputVideoFrame.SoftwareBitmap.CopyTo(softwareBitmap);
}
frameRenderer.RenderFrame(softwareBitmap);
});
}
/// <summary>
/// Launch file picker for user to select a picture file and return a VideoFrame
/// </summary>
/// <returns>VideoFrame instanciated from the selected image file</returns>
public static IAsyncOperation<VideoFrame> LoadVideoFrameFromFilePickedAsync()
{
return AsyncInfo.Run(async (token) =>
{
// Trigger file picker to select an image file
FileOpenPicker fileOpenPicker = new FileOpenPicker();
fileOpenPicker.SuggestedStartLocation = PickerLocationId.PicturesLibrary;
fileOpenPicker.FileTypeFilter.Add(".jpg");
fileOpenPicker.FileTypeFilter.Add(".png");
fileOpenPicker.ViewMode = PickerViewMode.Thumbnail;
StorageFile selectedStorageFile = await fileOpenPicker.PickSingleFileAsync();
if (selectedStorageFile == null)
{
return null;
}
return await LoadVideoFrameFromStorageFileAsync(selectedStorageFile);
});
}
/// <summary>
/// Decode image from a StorageFile and return a VideoFrame
/// </summary>
/// <param name="file"></param>
/// <returns></returns>
public static IAsyncOperation<VideoFrame> LoadVideoFrameFromStorageFileAsync(StorageFile file)
{
return AsyncInfo.Run(async (token) =>
{
VideoFrame resultFrame = null;
SoftwareBitmap softwareBitmap;
using (IRandomAccessStream stream = await file.OpenAsync(FileAccessMode.Read))
{
// Create the decoder from the stream
BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream);
// Get the SoftwareBitmap representation of the file in BGRA8 format
softwareBitmap = await decoder.GetSoftwareBitmapAsync();
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
}
// Encapsulate the image in the WinML image type (VideoFrame) to be bound and evaluated
resultFrame = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
return resultFrame;
});
}
/// <summary>
/// Launch file picker for user to select a file and save a VideoFrame to it
/// </summary>
/// <param name="frame"></param>
/// <returns></returns>
public static IAsyncAction SaveVideoFrameToFilePickedAsync(VideoFrame frame)
{
return AsyncInfo.Run(async (token) =>
{
// Trigger file picker to select an image file
FileSavePicker fileSavePicker = new FileSavePicker();
fileSavePicker.SuggestedStartLocation = PickerLocationId.PicturesLibrary;
fileSavePicker.FileTypeChoices.Add("image file", new List<string>() { ".jpg" });
fileSavePicker.SuggestedFileName = "NewImage";
StorageFile selectedStorageFile = await fileSavePicker.PickSaveFileAsync();
if (selectedStorageFile == null)
{
return;
}
using (IRandomAccessStream stream = await selectedStorageFile.OpenAsync(FileAccessMode.ReadWrite))
{
VideoFrame frameToEncode = frame;
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, stream);
if (frameToEncode.SoftwareBitmap == null)
{
Debug.Assert(frame.Direct3DSurface != null);
frameToEncode = new VideoFrame(BitmapPixelFormat.Bgra8, frame.Direct3DSurface.Description.Width, frame.Direct3DSurface.Description.Height);
await frame.CopyToAsync(frameToEncode);
}
encoder.SetSoftwareBitmap(
frameToEncode.SoftwareBitmap.BitmapPixelFormat.Equals(BitmapPixelFormat.Bgra8) ?
frameToEncode.SoftwareBitmap
: SoftwareBitmap.Convert(frameToEncode.SoftwareBitmap, BitmapPixelFormat.Bgra8));
await encoder.FlushAsync();
}
});
}
}
}

Просмотреть файл

@ -0,0 +1,120 @@
<Page
x:Class="StyleTransfer.MainPage"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="using:StyleTransfer"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
mc:Ignorable="d">
<Page.Resources>
<SolidColorBrush x:Key="TranslucentBlackBrush" Color="Black" Opacity="0.3"/>
<Style x:Key="TextBlockStyling" TargetType="TextBlock">
<Setter Property="Foreground" Value="Black"/>
</Style>
</Page.Resources>
<Grid Background="{ThemeResource ApplicationPageBackgroundThemeBrush}">
<Grid>
<Grid.RowDefinitions>
<RowDefinition Height="80"/>
<RowDefinition Height="300*"/>
<RowDefinition Height="30"/>
</Grid.RowDefinitions>
<!--Status and result display-->
<StackPanel x:Name="UIStatusPanel" Background="#BFFFFFFF" VerticalAlignment="Top" Grid.Row="0">
<ContentControl Name="UIModelControls">
<StackPanel Orientation="Horizontal">
<ToggleSwitch Name="UIToggleInferenceDevice"
OnContent="GPU"
OffContent="CPU"
IsOn="True"
Toggled="UIToggleInferenceDevice_Toggled"
Margin="10,0,0,0"/>
</StackPanel>
</ContentControl>
<!--Image preview and acquisition control-->
<ContentControl Name="UIImageControls"
VerticalAlignment="Stretch"
HorizontalAlignment="Stretch"
IsEnabled="False">
<StackPanel Orientation="Horizontal">
<Button Name="UIButtonAcquireImage"
ToolTipService.ToolTip="Take a photo"
Click="UIButtonAcquireImage_Click">
<Button.Content>
<SymbolIcon Symbol="Camera"/>
</Button.Content>
</Button>
<Button Name="UIButtonFilePick"
ToolTipService.ToolTip="Select an image from a file"
Click="UIButtonFilePick_Click">
<Button.Content>
<SymbolIcon Symbol="OpenFile"/>
</Button.Content>
</Button>
<Button Name="UIButtonSaveImage"
ToolTipService.ToolTip="Save the image result to a file"
IsEnabled="false"
VerticalAlignment="Bottom"
Click="UIButtonSaveImage_Click"
Background="#FF939393" >
<Button.Content>
<SymbolIcon Symbol="Save"/>
</Button.Content>
</Button>
</StackPanel>
</ContentControl>
</StackPanel>
<Grid Grid.Row="1">
<Grid.ColumnDefinitions>
<ColumnDefinition Width="200*"/>
<ColumnDefinition Width="200*"/>
</Grid.ColumnDefinitions>
<Image Name="UIInputImage"
Grid.Column="0"
Stretch="Uniform"
MaxWidth="720"
MaxHeight="720"/>
<Grid Grid.Column="1" VerticalAlignment="Stretch">
<Image Name="UIResultImage"
Stretch="Uniform"
MaxWidth="720"
MaxHeight="720"/>
<ProgressRing Name="UIProcessingProgressRing"
MaxWidth="720"
MaxHeight="720"
IsActive="false"
Visibility="Collapsed"/>
</Grid>
</Grid>
<Border x:Name="UIStatusBorder" Grid.Row="2">
<ScrollViewer VerticalScrollMode="Auto" VerticalScrollBarVisibility="Auto" MaxHeight="200">
<TextBlock x:Name="StatusBlock"
Text="Select a style to begin"
FontWeight="Bold"
MaxWidth="{Binding ElementName=Splitter, Path=ActualWidth}"
Margin="10,10,10,20"
TextWrapping="Wrap" />
</ScrollViewer>
</Border>
</Grid>
</Grid>
</Page>

Просмотреть файл

@ -0,0 +1,367 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using Windows.Media.Capture;
using Windows.Storage;
using Windows.UI.Core;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Media;
using Windows.UI.Xaml.Navigation;
using Windows.Graphics.Imaging;
using Windows.AI.MachineLearning.Preview;
using Windows.Media;
using System.Threading;
namespace StyleTransfer
{
/// <summary>
/// An empty page that can be used on its own or navigated to within a Frame.
/// </summary>
public sealed partial class MainPage : Page
{
// States
private bool _isReadyForEval = true;
private SemaphoreSlim _evaluationLock = new SemaphoreSlim(1);
private bool _useGPU = true;
// Rendering related
private FrameRenderer _resultframeRenderer;
private FrameRenderer _inputFrameRenderer;
// WinML related
private const string _kModelFileName = "Candy";
private const string _kDefaultImageFileName = "DefaultImage.jpg";
private ImageVariableDescriptorPreview _inputImageDescription;
private ImageVariableDescriptorPreview _outputImageDescription;
private LearningModelPreview _model;
private LearningModelBindingPreview _binding = null;
VideoFrame _inputFrame = null;
VideoFrame _outputFrame = null;
public MainPage()
{
this.InitializeComponent();
}
protected override void OnNavigatedTo(NavigationEventArgs e)
{
_resultframeRenderer = new FrameRenderer(UIResultImage);
_inputFrameRenderer = new FrameRenderer(UIInputImage);
}
/// <summary>
/// Display a message to the user.
/// This method may be called from any thread.
/// </summary>
/// <param name="strMessage"></param>
/// <param name="type"></param>
public void NotifyUser(string strMessage, NotifyType type)
{
// If called from the UI thread, then update immediately.
// Otherwise, schedule a task on the UI thread to perform the update.
if (Dispatcher.HasThreadAccess)
{
UpdateStatus(strMessage, type);
}
else
{
var task = Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () => UpdateStatus(strMessage, type));
task.AsTask().Wait();
}
}
/// <summary>
/// Update the status message displayed on the UI
/// </summary>
/// <param name="strMessage"></param>
/// <param name="type"></param>
private void UpdateStatus(string strMessage, NotifyType type)
{
switch (type)
{
case NotifyType.StatusMessage:
UIStatusBorder.Background = new SolidColorBrush(Windows.UI.Colors.Green);
break;
case NotifyType.ErrorMessage:
UIStatusBorder.Background = new SolidColorBrush(Windows.UI.Colors.Red);
break;
}
StatusBlock.Text = strMessage;
// Collapse the StatusBlock if it has no text to conserve real estate.
UIStatusBorder.Visibility = (StatusBlock.Text != String.Empty) ? Visibility.Visible : Visibility.Collapsed;
if (StatusBlock.Text != String.Empty)
{
UIStatusBorder.Visibility = Visibility.Visible;
UIStatusPanel.Visibility = Visibility.Visible;
}
else
{
UIStatusBorder.Visibility = Visibility.Collapsed;
UIStatusPanel.Visibility = Visibility.Collapsed;
}
}
/// <summary>
/// Load the labels and model and initialize WinML
/// </summary>
/// <returns></returns>
private async Task LoadModelAsync()
{
_evaluationLock.Wait();
{
_binding = null;
_model = null;
_isReadyForEval = false;
try
{
// Load Model
StorageFile modelFile = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/{_kModelFileName}.onnx"));
_model = await LearningModelPreview.LoadModelFromStorageFileAsync(modelFile);
// Hardcoding to use GPU
InferencingOptionsPreview options = _model.InferencingOptions;
options.PreferredDeviceKind = _useGPU ? LearningModelDeviceKindPreview.LearningDeviceGpu : LearningModelDeviceKindPreview.LearningDeviceCpu;
_model.InferencingOptions = options;
// Debugging logic to see the input and output of ther model
List<ILearningModelVariableDescriptorPreview> inputFeatures = _model.Description.InputFeatures.ToList();
List<ILearningModelVariableDescriptorPreview> outputFeatures = _model.Description.OutputFeatures.ToList();
var metadata = _model.Description.Metadata;
foreach (var md in metadata)
{
Debug.WriteLine($"{md.Key} | {md.Value}");
}
_inputImageDescription =
inputFeatures.FirstOrDefault(feature => feature.ModelFeatureKind == LearningModelFeatureKindPreview.Image)
as ImageVariableDescriptorPreview;
_outputImageDescription =
outputFeatures.FirstOrDefault(feature => feature.ModelFeatureKind == LearningModelFeatureKindPreview.Image)
as ImageVariableDescriptorPreview;
_isReadyForEval = true;
}
catch (Exception ex)
{
NotifyUser($"error: {ex.Message}", NotifyType.ErrorMessage);
Debug.WriteLine($"error: {ex.Message}");
}
}
_evaluationLock.Release();
}
/// <summary>
/// Acquire manually an image from the camera preview stream
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private async void UIButtonAcquireImage_Click(object sender, RoutedEventArgs e)
{
UIInputImage.Visibility = Visibility.Visible;
UIProcessingProgressRing.IsActive = true;
UIProcessingProgressRing.Visibility = Visibility.Visible;
UIButtonSaveImage.IsEnabled = false;
CameraCaptureUI dialog = new CameraCaptureUI();
dialog.PhotoSettings.AllowCropping = false;
dialog.PhotoSettings.Format = CameraCaptureUIPhotoFormat.Png;
StorageFile file = await dialog.CaptureFileAsync(CameraCaptureUIMode.Photo);
if (file != null)
{
var vf = await ImageHelper.LoadVideoFrameFromStorageFileAsync(file);
await Task.Run(async () =>
{
await EvaluateVideoFrameAsync(vf);
});
}
}
/// <summary>
/// Select and evaluate a picture
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private async void UIButtonFilePick_Click(object sender, RoutedEventArgs e)
{
UIInputImage.Visibility = Visibility.Visible;
UIProcessingProgressRing.IsActive = true;
UIProcessingProgressRing.Visibility = Visibility.Visible;
UIButtonSaveImage.IsEnabled = false;
try
{
VideoFrame inputFrame = null;
// use a default image or a picture selected by the user
if (sender == null && e == null)
{
// use a default image..
if (_inputFrame == null)
{
var file = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/{_kDefaultImageFileName}"));
inputFrame = await ImageHelper.LoadVideoFrameFromStorageFileAsync(file);
}
// ..or use a picture already selected by the user..
else
{
inputFrame = _inputFrame;
}
}
// ..or use a new picture selected by the user
else
{
inputFrame = await ImageHelper.LoadVideoFrameFromFilePickedAsync();
}
// if the picture is valid, let's process it with the model
if (inputFrame == null)
{
NotifyUser("no valid image file selected", NotifyType.ErrorMessage);
}
else
{
await Task.Run(async () => await EvaluateVideoFrameAsync(inputFrame));
}
}
catch (Exception ex)
{
Debug.WriteLine($"error: {ex.Message}");
NotifyUser(ex.Message, NotifyType.ErrorMessage);
}
}
/// <summary>
/// 1) Bind input and output features
/// 2) Run evaluation of the model
/// 3) Retrieve the result
/// </summary>
/// <param name="inputVideoFrame"></param>
/// <returns></returns>
private async Task EvaluateVideoFrameAsync(VideoFrame inputVideoFrame)
{
LearningModelPreview model = null;
bool isReadyForEval = false;
_evaluationLock.Wait();
{
model = _model;
isReadyForEval = _isReadyForEval;
_isReadyForEval = false;
}
_evaluationLock.Release();
if (inputVideoFrame != null && isReadyForEval && model != null)
{
_inputFrame = inputVideoFrame;
try
{
NotifyUser("Processing...", NotifyType.StatusMessage);
if (_outputFrame == null)
{
_outputFrame = new VideoFrame(BitmapPixelFormat.Bgra8, (int)_outputImageDescription.Width, (int)_outputImageDescription.Height);
}
// Create bindings for the input and output buffers
if (_binding == null)
{
_binding = new LearningModelBindingPreview(model as LearningModelPreview);
// since we reuse the output at each evaluation, bind it only once
_binding.Bind(_outputImageDescription.Name, _outputFrame);
}
_binding.Bind(_inputImageDescription.Name, inputVideoFrame);
// Render the input frame
await ImageHelper.RenderFrameAsync(_inputFrameRenderer, inputVideoFrame);
// Process the frame with the model
var results = await _model.EvaluateAsync(_binding, "test");
// Parse result
IReadOnlyDictionary<string, object> outputs = results.Outputs;
foreach (var output in outputs)
{
Debug.WriteLine($"{output.Key} : {output.Value} -> {output.Value.GetType()}");
}
// Display result
VideoFrame vf = results.Outputs[_outputImageDescription.Name] as VideoFrame;
await ImageHelper.RenderFrameAsync(_resultframeRenderer, vf);
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
UIProcessingProgressRing.IsActive = false;
UIProcessingProgressRing.Visibility = Visibility.Collapsed;
UIButtonSaveImage.IsEnabled = true;
UIToggleInferenceDevice.IsEnabled = true;
});
NotifyUser("Done!", NotifyType.StatusMessage);
}
catch (Exception ex)
{
NotifyUser(ex.Message, NotifyType.ErrorMessage);
Debug.WriteLine(ex.ToString());
}
_evaluationLock.Wait();
{
_isReadyForEval = true;
}
_evaluationLock.Release();
}
}
/// <summary>
/// Toggle inference device (GPU or CPU) from UI
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void UIToggleInferenceDevice_Toggled(object sender, RoutedEventArgs e)
{
_useGPU = (bool)UIToggleInferenceDevice.IsOn;
UIToggleInferenceDevice.IsEnabled = false;
// Reload model
Task.Run(async () => await LoadModelAsync()).ContinueWith(async (antecedent) =>
{
if (antecedent.IsCompletedSuccessfully && _isReadyForEval)
{
await Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
{
NotifyUser($"Ready to stylize! ", NotifyType.StatusMessage);
UIImageControls.IsEnabled = true;
UIModelControls.IsEnabled = true;
UIButtonFilePick_Click(null, null);
});
}
});
}
/// <summary>
/// Save image result to file
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private async void UIButtonSaveImage_Click(object sender, RoutedEventArgs e)
{
await ImageHelper.SaveVideoFrameToFilePickedAsync(_outputFrame);
}
}
public enum NotifyType
{
StatusMessage,
ErrorMessage
};
}

Просмотреть файл

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="utf-8"?>
<Package xmlns="http://schemas.microsoft.com/appx/manifest/foundation/windows10" xmlns:mp="http://schemas.microsoft.com/appx/2014/phone/manifest" xmlns:uap="http://schemas.microsoft.com/appx/manifest/uap/windows10" IgnorableNamespaces="uap mp">
<Identity Name="029EAC61-59A4-450D-855B-77158271A282" Publisher="CN=Louis-Philippe Bourret" Version="1.0.0.0" />
<mp:PhoneIdentity PhoneProductId="3bd922b5-d779-4208-8c35-aec4dea61330" PhonePublisherId="00000000-0000-0000-0000-000000000000" />
<Properties>
<DisplayName>CandyStyleTransfer</DisplayName>
<PublisherDisplayName>Louis-Philippe Bourret</PublisherDisplayName>
<Logo>Assets\frame_camera_50.png</Logo>
</Properties>
<Dependencies>
<TargetDeviceFamily Name="Windows.Universal" MinVersion="10.0.0.0" MaxVersionTested="10.0.0.0" />
</Dependencies>
<Resources>
<Resource Language="x-generate" />
</Resources>
<Applications>
<Application Id="App" Executable="$targetnametoken$.exe" EntryPoint="StyleTransfer.App">
<uap:VisualElements DisplayName="CandyStyleTransfer" Square150x150Logo="Assets\frame_camera_150.png" Square44x44Logo="Assets\frame_camera_44.png" Description="Passing images taken from a camera or from an image file through style transfer" BackgroundColor="transparent">
<uap:LockScreen Notification="badge" BadgeLogo="Assets\frame_camera_24.png" />
<uap:DefaultTile Wide310x150Logo="Assets\frame_camera_310-150.png" Square310x310Logo="Assets\frame_camera_310.png">
<uap:ShowNameOnTiles>
<uap:ShowOn Tile="square150x150Logo" />
<uap:ShowOn Tile="wide310x150Logo" />
<uap:ShowOn Tile="square310x310Logo" />
</uap:ShowNameOnTiles>
</uap:DefaultTile>
<uap:SplashScreen Image="Assets\frame_camera_620-300.png" />
</uap:VisualElements>
</Application>
</Applications>
<Capabilities>
<uap:Capability Name="picturesLibrary" />
</Capabilities>
</Package>

Просмотреть файл

@ -0,0 +1,36 @@
//*@@@+++@@@@******************************************************************
//
// Microsoft Windows Media Foundation
// Copyright (C) Microsoft Corporation. All rights reserved.
//
//*@@@---@@@@******************************************************************
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("CandyStyleTransfer")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("CandyStyleTransfer")]
[assembly: AssemblyCopyright("Copyright © 2018")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
[assembly: ComVisible(false)]

Просмотреть файл

@ -0,0 +1,31 @@
<!--
This file contains Runtime Directives used by .NET Native. The defaults here are suitable for most
developers. However, you can modify these parameters to modify the behavior of the .NET Native
optimizer.
Runtime Directives are documented at https://go.microsoft.com/fwlink/?LinkID=391919
To fully enable reflection for App1.MyClass and all of its public/private members
<Type Name="App1.MyClass" Dynamic="Required All"/>
To enable dynamic creation of the specific instantiation of AppClass<T> over System.Int32
<TypeInstantiation Name="App1.AppClass" Arguments="System.Int32" Activate="Required Public" />
Using the Namespace directive to apply reflection policy to all the types in a particular namespace
<Namespace Name="DataClasses.ViewModels" Serialize="All" />
-->
<Directives xmlns="http://schemas.microsoft.com/netfx/2013/01/metadata">
<Application>
<!--
An Assembly element with Name="*Application*" applies to all assemblies in
the application package. The asterisks are not wildcards.
-->
<Assembly Name="*Application*" Dynamic="Required All" />
<!-- Add your application specific runtime directives here. -->
</Application>
</Directives>