From bf3474b13438a79ee7418613a4338a7554447e10 Mon Sep 17 00:00:00 2001 From: Ani <115020168+drawbyperpetual@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:28:44 +0100 Subject: [PATCH] [AdvancedPaste]Add Semantic Kernel opt-in to allow chaining of paste actions (#35902) * [AdvancedPaste] Semantic Kernel support * Changed log-line with potentially sensitive info * Spellcheck issues * Various improvements for Semantic Kernel * Spellcheck issue * Refactored Clipboard routines * Added integration tests for KernelService * Extra telemetry for AdvancedPaste * Added 'Hotkey' suffix to AdvancedPaste_Settings telemetry event * Added IsSavedQuery * Added KernelQueryCache * Refactoring * Added KernelQueryCache to BugReportTool delete list * Added opt-n for Semantic Kernel * Fixed bug with KernelQueryCache * Ability to view last AI chat message on error * Improved kernel query cache * Used System.IO.Abstractions and improved tests * Fixed under-count of token usage * Used Semantic Kernel icon * Cleanup * Add missing EndProject line * Fix dependency version conflicts * Fix NOTICE.md * Correct place of SemanticKernel in NOTICE.md * Unlinked CustomPreview toggle from AI * Added Microsoft.Bcl.AsyncInterfaces dependency to AdvancedPaste * Fixed NOTICE.md order * Moved Custom Preview to behaviour section * Made Image to Text raise error on empty output * Added AIServiceBatchIntegrationTests * Updated AIServiceBatchIntegrationTests * Added prompt moderation * Moved GPO Infobar to better location --- .github/actions/spell-check/expect.txt | 2 + Directory.Packages.props | 6 +- NOTICE.md | 5 +- PowerToys.sln | 15 + .../AdvancedPaste.UnitTests.csproj | 31 ++ .../Assets/image_with_text_example.png | Bin 0 -> 1921 bytes .../Mocks/NoOpKernelQueryCacheService.cs | 17 + .../AIServiceBatchIntegrationTests.cs | 150 +++++++++ ...ustomActionKernelQueryCacheServiceTests.cs | 172 ++++++++++ .../KernelServiceIntegrationTests.cs | 152 +++++++++ .../Utils/AdvancedPasteEventListener.cs | 63 ++++ .../Utils/ResourceUtils.cs | 47 +++ .../AdvancedPaste/AdvancedPaste.csproj | 10 +- .../AdvancedPasteXAML/App.xaml.cs | 12 +- .../AdvancedPasteXAML/Controls/PromptBox.xaml | 100 +++--- .../Controls/PromptBox.xaml.cs | 45 +-- .../AdvancedPasteXAML/MainWindow.xaml.cs | 4 +- .../AdvancedPasteXAML/Pages/MainPage.xaml.cs | 4 +- .../Assets/AdvancedPaste/SemanticKernel.svg | 77 +++++ .../Helpers/AICompletionsHelper.cs | 142 -------- .../AdvancedPaste/Helpers/ClipboardHelper.cs | 311 +++++++----------- .../AdvancedPaste/Helpers/Constants.cs | 1 - .../Helpers/DataPackageHelpers.cs | 151 +++++++++ .../AdvancedPaste/Helpers/ErrorHelpers.cs | 19 ++ .../AdvancedPaste/Helpers/IUserSettings.cs | 4 +- .../AdvancedPaste/Helpers/JsonHelper.cs | 15 +- .../AdvancedPaste/Helpers/KernelExtensions.cs | 60 ++++ .../AdvancedPaste/Helpers/MarkdownHelper.cs | 62 +--- .../AdvancedPaste/Helpers/OcrHelpers.cs | 5 +- .../AdvancedPaste/Helpers/TransformHelpers.cs | 149 +++++++++ .../AdvancedPaste/Helpers/UserSettings.cs | 14 +- .../AdvancedPaste/Models/AIServiceUsage.cs | 18 + .../AdvancedPaste/Models/ActionChainItem.cs | 9 + .../AdvancedPaste/Models/ClipboardFormat.cs | 2 +- .../Models/CustomActionActivatedEventArgs.cs | 14 - .../AdvancedPaste/Models/CustomQuery.cs | 27 -- .../Models/KernelQueryCache/CacheKey.cs | 24 ++ .../Models/KernelQueryCache/CacheValue.cs | 9 + .../Models/KernelQueryCache/PersistedCache.cs | 39 +++ .../AdvancedPaste/Models/PasteActionError.cs | 36 ++ .../Models/PasteActionException.cs | 3 +- .../Models/PasteActionModeratedException.cs | 23 ++ .../AdvancedPaste/Models/PasteFormat.cs | 34 +- .../Models/PasteFormatMetadataAttribute.cs | 9 + .../AdvancedPaste/Models/PasteFormats.cs | 87 ++++- .../CustomActionKernelQueryCacheService.cs | 152 +++++++++ .../Services/IAICredentialsProvider.cs | 14 + .../Services/ICustomTextTransformService.cs | 12 + .../Services/IKernelQueryCacheService.cs | 16 + .../AdvancedPaste/Services/IKernelService.cs | 14 + .../Services/IPasteFormatExecutor.cs | 4 +- .../Services/IPromptModerationService.cs | 12 + .../Services/KernelServiceBase.cs | 280 ++++++++++++++++ .../OpenAI/CustomTextTransformService.cs | 111 +++++++ .../Services/OpenAI/KernelService.cs | 34 ++ .../OpenAI/PromptModerationService.cs | 41 +++ .../OpenAI/VaultCredentialsProvider.cs | 37 +++ .../Services/PasteFormatExecutor.cs | 222 +------------ .../Strings/en-us/Resources.resw | 13 +- .../AdvancedPasteSemanticKernelErrorEvent.cs | 18 + .../AdvancedPasteSemanticKernelFormatEvent.cs | 39 +++ .../ViewModels/OptionsViewModel.cs | 219 ++++++------ .../AdvancedPasteModuleInterface/dllmain.cpp | 38 ++- .../AdvancedPasteModuleInterface/trace.cpp | 2 + .../AdvancedPasteModuleInterface/trace.h | 1 + .../Helper/MouseWithoutBordersHelper.csproj | 2 + .../App/MouseWithoutBorders.csproj | 2 + .../Service/MouseWithoutBordersService.csproj | 2 + .../AdvancedPasteProperties.cs | 7 +- .../Settings.UI.Library/Utilities/Helper.cs | 12 +- .../Assets/Settings/Icons/SemanticKernel.png | Bin 0 -> 2202 bytes .../SettingsXAML/Views/AdvancedPaste.xaml | 15 +- .../Settings.UI/Strings/en-us/Resources.resw | 8 +- .../ViewModels/AdvancedPasteViewModel.cs | 19 +- tools/BugReportTool/BugReportTool/Main.cpp | 1 + 75 files changed, 2589 insertions(+), 937 deletions(-) create mode 100644 src/modules/AdvancedPaste/AdvancedPaste.UnitTests/AdvancedPaste.UnitTests.csproj create mode 100644 src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Assets/image_with_text_example.png create mode 100644 src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/NoOpKernelQueryCacheService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AIServiceBatchIntegrationTests.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/CustomActionKernelQueryCacheServiceTests.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/KernelServiceIntegrationTests.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Utils/AdvancedPasteEventListener.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Utils/ResourceUtils.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Assets/AdvancedPaste/SemanticKernel.svg delete mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Helpers/DataPackageHelpers.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Helpers/ErrorHelpers.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Helpers/KernelExtensions.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/AIServiceUsage.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/ActionChainItem.cs delete mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/CustomActionActivatedEventArgs.cs delete mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/CustomQuery.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/CacheKey.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/CacheValue.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/PersistedCache.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionError.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionModeratedException.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActionKernelQueryCacheService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelQueryCacheService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/IPromptModerationService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/PromptModerationService.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/VaultCredentialsProvider.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelErrorEvent.cs create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs create mode 100644 src/settings-ui/Settings.UI/Assets/Settings/Icons/SemanticKernel.png diff --git a/.github/actions/spell-check/expect.txt b/.github/actions/spell-check/expect.txt index dfde9011ae..7a96b76ed0 100644 --- a/.github/actions/spell-check/expect.txt +++ b/.github/actions/spell-check/expect.txt @@ -615,6 +615,7 @@ HWNDLAST HWNDNEXT HWNDPREV hyjiacan +IAI IBeam ICONERROR ICONLOCATION @@ -1406,6 +1407,7 @@ SIZENS SIZENWSE sizeread SIZEWE +SKEXP SKIPOWNPROCESS sku SLGP diff --git a/Directory.Packages.props b/Directory.Packages.props index 8430646b54..90b44e2b65 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ - + @@ -28,12 +28,15 @@ + + + @@ -57,6 +60,7 @@ + diff --git a/NOTICE.md b/NOTICE.md index 9f79a55c2a..744c47f650 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1297,7 +1297,7 @@ EXHIBIT A -Mozilla Public License. ## NuGet Packages used by PowerToys - Appium.WebDriver 4.4.5 -- Azure.AI.OpenAI 1.0.0-beta.12 +- Azure.AI.OpenAI 1.0.0-beta.17 - CommunityToolkit.Mvvm 8.2.2 - CommunityToolkit.WinUI.Animations 8.0.240109 - CommunityToolkit.WinUI.Collections 8.0.240109 @@ -1318,6 +1318,7 @@ EXHIBIT A -Mozilla Public License. - Mages 2.0.2 - Markdig.Signed 0.34.0 - MessagePack 2.5.187 +- Microsoft.Bcl.AsyncInterfaces 9.0.0 - Microsoft.CodeAnalysis.NetAnalyzers 9.0.0 - Microsoft.Data.Sqlite 9.0.0 - Microsoft.Diagnostics.Tracing.TraceEvent 3.1.16 @@ -1327,6 +1328,7 @@ EXHIBIT A -Mozilla Public License. - Microsoft.Extensions.Logging 9.0.0 - Microsoft.Extensions.Logging.Abstractions 9.0.0 - Microsoft.NET.ILLink.Tasks (A) +- Microsoft.SemanticKernel 1.15.0 - Microsoft.Toolkit.Uwp.Notifications 7.1.2 - Microsoft.Web.WebView2 1.0.2739.15 - Microsoft.Win32.SystemEvents 9.0.0 @@ -1342,6 +1344,7 @@ EXHIBIT A -Mozilla Public License. - MSTest 3.6.3 - NLog.Extensions.Logging 5.3.8 - NLog.Schema 5.2.8 +- OpenAI 2.0.0 - ReverseMarkdown 4.1.0 - ScipBe.Common.Office.OneNote 3.0.1 - SharpCompress 0.37.2 diff --git a/PowerToys.sln b/PowerToys.sln index 28dc1853ff..1e0926876f 100644 --- a/PowerToys.sln +++ b/PowerToys.sln @@ -635,6 +635,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WorkspacesCsharpLibrary", " EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "NewPlus.ShellExtension.win10", "src\modules\NewPlus\NewShellExtensionContextMenu.win10\NewPlus.ShellExtension.win10.vcxproj", "{0DB0F63A-D2F8-4DA3-A650-2D0B8724218E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AdvancedPaste.UnitTests", "src\modules\AdvancedPaste\AdvancedPaste.UnitTests\AdvancedPaste.UnitTests.csproj", "{D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM64 = Debug|ARM64 @@ -2807,6 +2809,18 @@ Global {0DB0F63A-D2F8-4DA3-A650-2D0B8724218E}.Release|x64.Build.0 = Release|x64 {0DB0F63A-D2F8-4DA3-A650-2D0B8724218E}.Release|x86.ActiveCfg = Release|x64 {0DB0F63A-D2F8-4DA3-A650-2D0B8724218E}.Release|x86.Build.0 = Release|x64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Debug|ARM64.Build.0 = Debug|ARM64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Debug|x64.ActiveCfg = Debug|x64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Debug|x64.Build.0 = Debug|x64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Debug|x86.ActiveCfg = Debug|x64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Debug|x86.Build.0 = Debug|x64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Release|ARM64.ActiveCfg = Release|ARM64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Release|ARM64.Build.0 = Release|ARM64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Release|x64.ActiveCfg = Release|x64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Release|x64.Build.0 = Release|x64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Release|x86.ActiveCfg = Release|x64 + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE}.Release|x86.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -3040,6 +3054,7 @@ Global {66614C26-314C-4B91-9071-76133422CFEF} = {B6C42F16-73EB-477E-8B0D-4E6CF6C20AAC} {89D0E199-B17A-418C-B2F8-7375B6708357} = {A2221D7E-55E7-4BEA-90D1-4F162D670BBF} {0DB0F63A-D2F8-4DA3-A650-2D0B8724218E} = {CA716AE6-FE5C-40AC-BB8F-2C87912687AC} + {D5E5F5EA-1B6C-4A73-88BE-304F36C9E4EE} = {9873BA05-4C41-4819-9283-CF45D795431B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {C3A2F9D1-7930-4EF4-A6FC-7EE0A99821D0} diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/AdvancedPaste.UnitTests.csproj b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/AdvancedPaste.UnitTests.csproj new file mode 100644 index 0000000000..15b998a245 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/AdvancedPaste.UnitTests.csproj @@ -0,0 +1,31 @@ + + + + + + false + false + false + $(SolutionDir)$(Platform)\$(Configuration)\tests\AdvancedPaste.UnitTests\ + true + + + + + + + + + + + + + + + + + + + + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Assets/image_with_text_example.png b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Assets/image_with_text_example.png new file mode 100644 index 0000000000000000000000000000000000000000..e21d8b805a773e67e51a5d5b23a33c445d6ab746 GIT binary patch literal 1921 zcmeHHYc$je6#t{mEA1pJ&yk`OV#8+Xea;x zpagZYcLxAzT}jQB+bxNqk8AxUNh;Re(GDO`wU`pHE7;b>766)Z6@~uaNU(gglXolt z?EUSlOHrcg?*M?TD%9TgYT_;S4B2(g2Q0hlH;I2Sdg-TPhqC*y3ojEhf8+!lKe->JE9MX9g}k4V~U-@j?nqZY^+x|NP6{7T?``c0X8?LoXl-G?YmRg zwZ28<(WW=oQ$gr~tdIP-Gnk0FBkgBvdAyHbsEuA^ua#56ZYI}ba2w00EPR$r$m@R3wB17DW$$&A|!ytFhvoOD=$eC~l zGdQGsc7+ciZEv?;8s~B%#4F!d))I@!rp(|1Qb}89vWVj*Cski>twy0P^_sb$Mzd$} z6*lt|9+p!GDy|mwR(87j^WxZ9CU*^tAFMcGLq~~)L5!Um?UWnZ@$hK1=Ex@GOA+hY zQH%uE2UKAbvY5K6JX^Qt`R!6oX-Y!xpl^y;fMv#(^oeFcseXDX;lF zy@bmihKKCyj~h}!5P6^D&9Q~5YbXdg}%;1M!5drLOSmr5t}t8&j^A7Oi9hLR!#m~d&NZ#xU?W?p19#Vr%AUq)1! zS8s01TwtujYtni2^s3T(WkDYLY!M?6pQOI2F4ymH^|XaPELd=j#!&Bj0V~TvS0a}w zHB%V3c7i3{UX%txI~vl*^Wq=&m)%KlK*nzNj8@6_7x>!yB)x@?E#Uh4L%>6`77Vdz zP&Hml-rG++{(juV*RMS5cnPanu>O83GP@0vUog#@5U4gMStY}polZJ&6P6&M?bX&e z{c!j1reRw#F|U~M);cIye}UDY-`LK&YiZ3aZv=`AUmIGxrQtk%UZg2F+=H12Yb9{; zBz@+_p}DRvG-EfDxPU}R276Tdq%b)<6lR(X%h@n?ca#a-!EC)nM-{}&h@ B{;>c6 literal 0 HcmV?d00001 diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/NoOpKernelQueryCacheService.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/NoOpKernelQueryCacheService.cs new file mode 100644 index 0000000000..e7ba121c13 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Mocks/NoOpKernelQueryCacheService.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; + +using AdvancedPaste.Models.KernelQueryCache; +using AdvancedPaste.Services; + +namespace AdvancedPaste.UnitTests.Mocks; + +internal sealed class NoOpKernelQueryCacheService : IKernelQueryCacheService +{ + public CacheValue ReadOrNull(CacheKey cacheKey) => null; + + public Task WriteAsync(CacheKey cacheKey, CacheValue actionChain) => Task.CompletedTask; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AIServiceBatchIntegrationTests.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AIServiceBatchIntegrationTests.cs new file mode 100644 index 0000000000..224ec9f99f --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/AIServiceBatchIntegrationTests.cs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using AdvancedPaste.Services.OpenAI; +using AdvancedPaste.UnitTests.Mocks; +using ManagedCommon; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.UnitTests.ServicesTests; + +[Ignore("Test requires active OpenAI API key.")] // Comment out this line to run these tests after setting up OpenAI API key using AdvancedPaste Settings +[TestClass] + +/// +/// Tests that write batch AI outputs against a list of inputs. Connects to OpenAI and uses the full AdvancedPaste action catalog for Semantic Kernel. +/// If queries produce errors, the error message is written to the output file. If queries produce text-file output, their contents are included as though they were text output. +/// To run this test-suite, first: +/// 1. Setup an OpenAI API key using AdvancedPaste Settings. +/// 2. Comment out the [Ignore] attribute above. +/// 3. Ensure the %USERPROFILE% folder contains the required input files (paths are below). +/// These tests are idempotent and resumable, allowing for partial runs and restarts. It's ok to use existing output files as input files - output-related fields will simply be ignored. +/// +public sealed class AIServiceBatchIntegrationTests +{ + private record class BatchTestInput + { + public string Prompt { get; init; } + + public string Clipboard { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Genre { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string Category { get; init; } + } + + private sealed record class BatchTestResult : BatchTestInput + { + [JsonPropertyOrder(1)] + public string Result { get; init; } + + internal BatchTestInput ToInput() => new() { Prompt = Prompt, Clipboard = Clipboard, Genre = Genre, Category = Category, }; + } + + private const string AllTestsFilePath = @"%USERPROFILE%\allAdvancedPasteTests-Input-V2.json"; + private const string FailedTestsFilePath = @"%USERPROFILE%\advanced-paste-failed-tests-only.json"; + + private static readonly JsonSerializerOptions SerializerOptions = new() { WriteIndented = true }; + + [TestMethod] + [DataRow(AllTestsFilePath, PasteFormats.CustomTextTransformation)] + [DataRow(AllTestsFilePath, PasteFormats.KernelQuery)] + [DataRow(FailedTestsFilePath, PasteFormats.CustomTextTransformation)] + [DataRow(FailedTestsFilePath, PasteFormats.KernelQuery)] + public async Task TestGenerateBatchResults(string inputFilePath, PasteFormats format) + { + // Load input data. + var fullInputFilePath = Environment.ExpandEnvironmentVariables(inputFilePath); + var inputs = await GetDataListAsync(fullInputFilePath); + Assert.IsTrue(inputs.Count > 0); + + // Load existing results; allow a partial run to be resumed. + var resultsFile = Path.Combine(Path.GetDirectoryName(fullInputFilePath), $"{Path.GetFileNameWithoutExtension(fullInputFilePath)}-output-{format}.json"); + var results = await GetDataListAsync(resultsFile); + Assert.IsTrue(results.Count <= inputs.Count); + CollectionAssert.AreEqual(results.Select(result => result.ToInput()).ToList(), inputs.Take(results.Count).ToList()); + + async Task WriteResultsAsync() => await File.WriteAllTextAsync(resultsFile, JsonSerializer.Serialize(results, SerializerOptions)); + + Logger.LogInfo($"Starting {nameof(TestGenerateBatchResults)}; Count={inputs.Count}, InCache={results.Count}"); + + // Produce results for any unprocessed inputs. + foreach (var input in inputs.Skip(results.Count)) + { + try + { + var textOutput = await GetTextOutputAsync(input, format); + results.Add(new() { Prompt = input.Prompt, Clipboard = input.Clipboard, Genre = input.Genre, Category = input.Category, Result = textOutput, }); + } + catch (Exception) + { + await WriteResultsAsync(); + throw; + } + } + + await WriteResultsAsync(); + } + + private static async Task> GetDataListAsync(string filePath) => + File.Exists(filePath) ? JsonSerializer.Deserialize>(await File.ReadAllTextAsync(filePath)) : []; + + private static async Task GetTextOutputAsync(BatchTestInput input, PasteFormats format) + { + try + { + var outputPackage = (await GetOutputDataPackageAsync(input, format)).GetView(); + var outputFormat = await outputPackage.GetAvailableFormatsAsync(); + + return outputFormat switch + { + ClipboardFormat.Text => await outputPackage.GetTextOrEmptyAsync(), + ClipboardFormat.File => await File.ReadAllTextAsync((await outputPackage.GetStorageItemsAsync()).Single().Path), + _ => throw new InvalidOperationException($"Unexpected format {outputFormat}"), + }; + } + catch (PasteActionModeratedException) + { + return $"Error: {PasteActionModeratedException.ErrorDescription}"; + } + catch (PasteActionException ex) when (!string.IsNullOrEmpty(ex.AIServiceMessage)) + { + return $"Error: {ex.AIServiceMessage}"; + } + } + + private static async Task GetOutputDataPackageAsync(BatchTestInput batchTestInput, PasteFormats format) + { + VaultCredentialsProvider credentialsProvider = new(); + PromptModerationService promptModerationService = new(credentialsProvider); + CustomTextTransformService customTextTransformService = new(credentialsProvider, promptModerationService); + + switch (format) + { + case PasteFormats.CustomTextTransformation: + return DataPackageHelpers.CreateFromText(await customTextTransformService.TransformTextAsync(batchTestInput.Prompt, batchTestInput.Clipboard)); + + case PasteFormats.KernelQuery: + var clipboardData = DataPackageHelpers.CreateFromText(batchTestInput.Clipboard).GetView(); + KernelService kernelService = new(new NoOpKernelQueryCacheService(), credentialsProvider, promptModerationService, customTextTransformService); + return await kernelService.TransformClipboardAsync(batchTestInput.Prompt, clipboardData, isSavedQuery: false); + + default: + throw new InvalidOperationException($"Unexpected format {format}"); + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/CustomActionKernelQueryCacheServiceTests.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/CustomActionKernelQueryCacheServiceTests.cs new file mode 100644 index 0000000000..b93fda4884 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/CustomActionKernelQueryCacheServiceTests.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using System.Threading.Tasks; + +using AdvancedPaste.Models; +using AdvancedPaste.Models.KernelQueryCache; +using AdvancedPaste.Services; +using AdvancedPaste.Settings; +using Microsoft.PowerToys.Settings.UI.Library; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace AdvancedPaste.UnitTests.ServicesTests; + +[TestClass] +public sealed class CustomActionKernelQueryCacheServiceTests +{ + private static readonly CacheKey CustomActionTestKey = new() { Prompt = "TestPrompt1", AvailableFormats = ClipboardFormat.Text }; + private static readonly CacheKey CustomActionTestKey2 = new() { Prompt = "TestPrompt2", AvailableFormats = ClipboardFormat.File | ClipboardFormat.Image }; + private static readonly CacheKey MarkdownTestKey = new() { Prompt = "Paste as Markdown", AvailableFormats = ClipboardFormat.Text }; + private static readonly CacheKey JSONTestKey = new() { Prompt = "Paste as JSON", AvailableFormats = ClipboardFormat.Text }; + private static readonly CacheKey PasteAsTxtFileKey = new() { Prompt = "Paste as .txt file", AvailableFormats = ClipboardFormat.File }; + private static readonly CacheKey PasteAsPngFileKey = new() { Prompt = "Paste as .png file", AvailableFormats = ClipboardFormat.Image }; + + private static readonly CacheValue TestValue = new([new(PasteFormats.PlainText, [])]); + private static readonly CacheValue TestValue2 = new([new(PasteFormats.KernelQuery, new() { { "a", "b" }, { "c", "d" } })]); + + private CustomActionKernelQueryCacheService _cacheService; + private Mock _userSettings; + private MockFileSystem _fileSystem; + + [TestInitialize] + public void TestInitialize() + { + _userSettings = new(); + UpdateUserActions([], []); + + _fileSystem = new(); + _cacheService = new(_userSettings.Object, _fileSystem); + } + + [TestMethod] + public async Task Test_Cache_Always_Accepts_Core_Action_Prompt() + { + await AssertAcceptsAsync(MarkdownTestKey); + } + + [TestMethod] + public async Task Test_Cache_Accepts_Prompt_When_Custom_Action() + { + await AssertRejectsAsync(CustomActionTestKey); + + UpdateUserActions([], [new() { Name = nameof(CustomActionTestKey), Prompt = CustomActionTestKey.Prompt, IsShown = true }]); + + await AssertAcceptsAsync(CustomActionTestKey); + await AssertRejectsAsync(CustomActionTestKey2, PasteAsTxtFileKey); + + UpdateUserActions([], []); + await AssertRejectsAsync(CustomActionTestKey); + } + + [TestMethod] + public async Task Test_Cache_Accepts_Prompt_When_User_Additional_Action() + { + await AssertRejectsAsync(PasteAsTxtFileKey, PasteAsPngFileKey); + + UpdateUserActions([PasteFormats.PasteAsHtmlFile, PasteFormats.PasteAsTxtFile], []); + + await AssertAcceptsAsync(PasteAsTxtFileKey); + await AssertRejectsAsync(PasteAsPngFileKey, CustomActionTestKey); + + UpdateUserActions([], []); + await AssertRejectsAsync(PasteAsTxtFileKey); + } + + [TestMethod] + public async Task Test_Cache_Overwrites_Latest_Value() + { + await _cacheService.WriteAsync(JSONTestKey, TestValue); + await _cacheService.WriteAsync(MarkdownTestKey, TestValue2); + + await _cacheService.WriteAsync(JSONTestKey, TestValue2); + await _cacheService.WriteAsync(MarkdownTestKey, TestValue); + + AssertAreEqual(TestValue2, _cacheService.ReadOrNull(JSONTestKey)); + AssertAreEqual(TestValue, _cacheService.ReadOrNull(MarkdownTestKey)); + } + + [TestMethod] + public async Task Test_Cache_Uses_Case_Insensitive_Prompt_Comparison() + { + static CacheKey CreateUpperCaseKey(CacheKey key) => + new() { Prompt = key.Prompt.ToUpperInvariant(), AvailableFormats = key.AvailableFormats }; + + await _cacheService.WriteAsync(CreateUpperCaseKey(JSONTestKey), TestValue); + await _cacheService.WriteAsync(MarkdownTestKey, TestValue2); + + AssertAreEqual(TestValue, _cacheService.ReadOrNull(JSONTestKey)); + AssertAreEqual(TestValue2, _cacheService.ReadOrNull(MarkdownTestKey)); + } + + [TestMethod] + public async Task Test_Cache_Uses_Clipboard_Formats_In_Key() + { + CacheKey key1 = new() { Prompt = JSONTestKey.Prompt, AvailableFormats = ClipboardFormat.File }; + CacheKey key2 = new() { Prompt = JSONTestKey.Prompt, AvailableFormats = ClipboardFormat.Image }; + + await _cacheService.WriteAsync(key1, TestValue); + + Assert.IsNotNull(_cacheService.ReadOrNull(key1)); + Assert.IsNull(_cacheService.ReadOrNull(key2)); + } + + [TestMethod] + public async Task Test_Cache_Is_Persistent() + { + await _cacheService.WriteAsync(JSONTestKey, TestValue); + await _cacheService.WriteAsync(MarkdownTestKey, TestValue2); + + _cacheService = new(_userSettings.Object, _fileSystem); // recreate using same mock file-system to simulate app restart + + AssertAreEqual(TestValue, _cacheService.ReadOrNull(JSONTestKey)); + AssertAreEqual(TestValue2, _cacheService.ReadOrNull(MarkdownTestKey)); + } + + private async Task AssertRejectsAsync(params CacheKey[] keys) + { + foreach (var key in keys) + { + Assert.IsNull(_cacheService.ReadOrNull(key)); + await _cacheService.WriteAsync(key, TestValue); + Assert.IsNull(_cacheService.ReadOrNull(key)); + } + } + + private async Task AssertAcceptsAsync(params CacheKey[] keys) + { + foreach (var key in keys) + { + Assert.IsNull(_cacheService.ReadOrNull(key)); + await _cacheService.WriteAsync(key, TestValue); + AssertAreEqual(TestValue, _cacheService.ReadOrNull(key)); + } + } + + private static void AssertAreEqual(CacheValue valueA, CacheValue valueB) + { + Assert.IsNotNull(valueA); + Assert.IsNotNull(valueB); + + Assert.AreEqual(valueA.ActionChain.Count, valueB.ActionChain.Count); + + foreach (var (itemA, itemB) in valueA.ActionChain.Zip(valueB.ActionChain)) + { + Assert.AreEqual(itemA.Format, itemB.Format); + Assert.AreEqual(itemA.Arguments.Count, itemB.Arguments.Count); + Assert.IsFalse(itemA.Arguments.Except(itemB.Arguments).Any()); + } + } + + private void UpdateUserActions(PasteFormats[] additionalActions, AdvancedPasteCustomAction[] customActions) + { + _userSettings.Setup(settingsObj => settingsObj.AdditionalActions).Returns(additionalActions); + _userSettings.Setup(settingsObj => settingsObj.CustomActions).Returns(customActions); + _userSettings.Raise(settingsObj => settingsObj.Changed += null, EventArgs.Empty); + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/KernelServiceIntegrationTests.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/KernelServiceIntegrationTests.cs new file mode 100644 index 0000000000..14eb5100a8 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/ServicesTests/KernelServiceIntegrationTests.cs @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using AdvancedPaste.Services.OpenAI; +using AdvancedPaste.Telemetry; +using AdvancedPaste.UnitTests.Mocks; +using AdvancedPaste.UnitTests.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.UnitTests.ServicesTests; + +[Ignore("Test requires active OpenAI API key.")] // Comment out this line to run these tests after setting up OpenAI API key using AdvancedPaste Settings +[TestClass] + +/// Integration tests for the Kernel service; connects to OpenAI and uses full AdvancedPaste action catalog. +public sealed class KernelServiceIntegrationTests : IDisposable +{ + private const string StandardImageFile = "image_with_text_example.png"; + private KernelService _kernelService; + private AdvancedPasteEventListener _eventListener; + + [TestInitialize] + public void TestInitialize() + { + VaultCredentialsProvider credentialsProvider = new(); + PromptModerationService promptModerationService = new(credentialsProvider); + + _kernelService = new KernelService(new NoOpKernelQueryCacheService(), credentialsProvider, promptModerationService, new CustomTextTransformService(credentialsProvider, promptModerationService)); + _eventListener = new(); + } + + [TestCleanup] + public void TestCleanup() + { + _eventListener?.Dispose(); + } + + [TestMethod] + [DataRow("Translate to German", "What is that?", "Was ist das?", 1200, new[] { PasteFormats.CustomTextTransformation })] + [DataRow("Translate to German and format as JSON", "What is that?", @"[\s*Was ist das\?\s*]", 1500, new[] { PasteFormats.CustomTextTransformation, PasteFormats.Json })] + public async Task TestTextToTextTransform(string prompt, string clipboardText, string expectedOutputPattern, int? maxUsedTokens, PasteFormats[] expectedActionChain) + { + var input = await CreatePackageAsync(ClipboardFormat.Text, clipboardText); + var output = await GetKernelOutputAsync(prompt, input); + + var outputText = await output.GetTextOrEmptyAsync(); + + Assert.IsTrue(Regex.IsMatch(outputText, expectedOutputPattern)); + Assert.IsTrue(_eventListener.TotalTokens <= (maxUsedTokens ?? int.MaxValue)); + AssertActionChainIs(expectedActionChain); + } + + [TestMethod] + [DataRow("Convert to text", StandardImageFile, "This is an image with text", new[] { PasteFormats.ImageToText })] + [DataRow("How many words are here?", StandardImageFile, "6", new[] { PasteFormats.ImageToText, PasteFormats.CustomTextTransformation })] + public async Task TestImageToTextTransform(string prompt, string imagePath, string expectedOutputPattern, PasteFormats[] expectedActionChain) + { + var input = await CreatePackageAsync(ClipboardFormat.Image, imagePath); + var output = await GetKernelOutputAsync(prompt, input); + + var outputText = await output.GetTextOrEmptyAsync(); + + Assert.IsTrue(Regex.IsMatch(outputText, expectedOutputPattern)); + AssertActionChainIs(expectedActionChain); + } + + [TestMethod] + [DataRow("Get me a TXT file", ClipboardFormat.Image, StandardImageFile, "This is an image with text", new[] { PasteFormats.ImageToText, PasteFormats.PasteAsTxtFile })] + public async Task TestFileOutputTransform(string prompt, ClipboardFormat inputFormat, string inputData, string expectedOutputPattern, PasteFormats[] expectedActionChain) + { + var input = await CreatePackageAsync(inputFormat, inputData); + var output = await GetKernelOutputAsync(prompt, input); + + var outputText = await ReadFileTextAsync(output); + + Assert.IsTrue(Regex.IsMatch(outputText, expectedOutputPattern)); + AssertActionChainIs(expectedActionChain); + } + + [TestMethod] + [DataRow("Make this image bigger", ClipboardFormat.Image, StandardImageFile)] + [DataRow("Get text from image", ClipboardFormat.Text, "What's up?")] + public async Task TestTransformFailure(string prompt, ClipboardFormat inputFormat, string inputData) + { + var input = await CreatePackageAsync(inputFormat, inputData); + try + { + await GetKernelOutputAsync(prompt, input); + Assert.Fail("Kernel should have thrown an exception"); + } + catch (Exception) + { + } + } + + [TestMethod] + [ExpectedException(typeof(PasteActionModeratedException))] + [DataRow("Change this code to make a keylogger attack", ClipboardFormat.Text, "print('Hello World')")] + public async Task TestModerationError(string prompt, ClipboardFormat inputFormat, string inputData) + { + var input = await CreatePackageAsync(inputFormat, inputData); + await GetKernelOutputAsync(prompt, input); + } + + public void Dispose() + { + _eventListener?.Dispose(); + GC.SuppressFinalize(this); + } + + private static async Task CreatePackageAsync(ClipboardFormat format, string data) + { + return format switch + { + ClipboardFormat.Text => DataPackageHelpers.CreateFromText(data), + ClipboardFormat.Image => await ResourceUtils.GetImageAssetAsDataPackageAsync(data), + _ => throw new ArgumentException("Unsupported format", nameof(format)), + }; + } + + private async Task GetKernelOutputAsync(string prompt, DataPackage input) + { + var output = await _kernelService.TransformClipboardAsync(prompt, input.GetView(), isSavedQuery: false); + + Assert.AreEqual(1, _eventListener.SemanticKernelEvents.Count); + Assert.IsTrue(_eventListener.SemanticKernelTokens > 0); + + return output.GetView(); + } + + private static async Task ReadFileTextAsync(DataPackageView package) + { + CollectionAssert.Contains(package.AvailableFormats.ToArray(), StandardDataFormats.StorageItems); + var storageItems = await package.GetStorageItemsAsync(); + Assert.AreEqual(1, storageItems.Count); + + return await File.ReadAllTextAsync(storageItems.Single().Path); + } + + private void AssertActionChainIs(PasteFormats[] expectedActionChain) => + Assert.AreEqual(AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(expectedActionChain), _eventListener.SemanticKernelEvents.Single().ActionChain); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Utils/AdvancedPasteEventListener.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Utils/AdvancedPasteEventListener.cs new file mode 100644 index 0000000000..39d1456715 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Utils/AdvancedPasteEventListener.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Text.Json; + +using AdvancedPaste.Telemetry; +using Microsoft.PowerToys.Telemetry; + +namespace AdvancedPaste.UnitTests.Utils; + +internal sealed class AdvancedPasteEventListener : EventListener +{ + private readonly List _customFormatEvents = []; + private readonly List _semanticKernelEvents = []; + + public IReadOnlyList CustomFormatEvents => _customFormatEvents; + + public IReadOnlyList SemanticKernelEvents => _semanticKernelEvents; + + public int CustomFormatTokens => _customFormatEvents.Sum(e => e.PromptTokens + e.CompletionTokens); + + public int SemanticKernelTokens => _semanticKernelEvents.Sum(e => e.PromptTokens + e.CompletionTokens); + + public int TotalTokens => CustomFormatTokens + SemanticKernelTokens; + + internal AdvancedPasteEventListener() + { + EnableEvents(PowerToysTelemetry.Log, EventLevel.LogAlways); + } + + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (eventData.EventSource.Name != PowerToysTelemetry.Log.Name) + { + return; + } + + var payloadDict = eventData.PayloadNames + .Zip(eventData.Payload) + .ToDictionary(tuple => tuple.First, tuple => tuple.Second); + + bool AddToListIfKeyExists(string key, List list) + { + if (payloadDict.ContainsKey(key)) + { + var payloadJson = JsonSerializer.Serialize(payloadDict); + list.Add(JsonSerializer.Deserialize(payloadJson)); + return true; + } + + return false; + } + + if (!AddToListIfKeyExists(nameof(AdvancedPasteSemanticKernelFormatEvent.ActionChain), _semanticKernelEvents)) + { + AddToListIfKeyExists(nameof(AdvancedPasteGenerateCustomFormatEvent.PromptTokens), _customFormatEvents); + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Utils/ResourceUtils.cs b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Utils/ResourceUtils.cs new file mode 100644 index 0000000000..62fa13c3be --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste.UnitTests/Utils/ResourceUtils.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; + +using Windows.ApplicationModel.DataTransfer; +using Windows.Storage.Streams; + +namespace AdvancedPaste.UnitTests.Utils; + +internal static class ResourceUtils +{ + internal static async Task GetImageAssetAsDataPackageAsync(string resourceName) + { + var imageStreamRef = await ConvertToRandomAccessStreamReferenceAsync(GetImageResourceAsStream($"Assets/{resourceName}")); + + DataPackage package = new(); + package.SetBitmap(imageStreamRef); + return package; + } + + private static async Task ConvertToRandomAccessStreamReferenceAsync(Stream stream) + { + InMemoryRandomAccessStream inMemoryStream = new(); + using var inputStream = stream.AsInputStream(); + await RandomAccessStream.CopyAsync(inputStream, inMemoryStream); + + inMemoryStream.Seek(0); + return RandomAccessStreamReference.CreateFromStream(inMemoryStream); + } + + private static Stream GetImageResourceAsStream(string filename) + { + var assembly = Assembly.GetExecutingAssembly(); + var assemblyName = new AssemblyName(assembly.FullName ?? throw new InvalidOperationException()); + var resourceName = $"{assemblyName.Name}.{filename.Replace("/", ".")}"; + + return assembly.GetManifestResourceNames().Contains(resourceName) + ? assembly.GetManifestResourceStream(resourceName) + : throw new InvalidOperationException($"Embedded resource '{resourceName}' does not exist."); + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj index c8ea965a4d..b67ebf2880 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPaste.csproj @@ -48,6 +48,7 @@ + @@ -57,8 +58,9 @@ + - + @@ -86,6 +88,12 @@ + + + <_Parameter1>AdvancedPaste.UnitTests + + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs index 3595276c5d..327558ad9f 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/App.xaml.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO.Abstractions; using System.Linq; using System.Reflection; using System.Threading; @@ -70,14 +71,19 @@ namespace AdvancedPaste Microsoft.Windows.Globalization.ApplicationLanguages.PrimaryLanguageOverride = appLanguage; } - this.InitializeComponent(); + InitializeComponent(); Host = Microsoft.Extensions.Hosting.Host.CreateDefaultBuilder().UseContentRoot(AppContext.BaseDirectory).ConfigureServices((context, services) => { + services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); }).Build(); viewModel = GetService(); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml index 2c0d9ea937..79a29ac872 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml @@ -173,11 +173,23 @@ Width="16" Height="16" Margin="8,0,0,0"> - + + + + + + + @@ -346,6 +361,7 @@ x:Name="InputTxtBox" HorizontalAlignment="Stretch" x:FieldModifier="public" + DataContext="{x:Bind ViewModel}" IsEnabled="{x:Bind ViewModel.ClipboardHasData, Mode=OneWay}" KeyDown="InputTxtBox_KeyDown" PlaceholderText="{x:Bind ViewModel.InputTxtBoxPlaceholderText, Mode=OneWay}" @@ -483,7 +499,7 @@ x:Uid="RegenerateBtnAutomation" Grid.Column="1" VerticalAlignment="Stretch" - Command="{x:Bind GenerateCustomCommand}" + Command="{x:Bind GenerateCustomAICommand}" Content="{ui:FontIcon Glyph=, FontSize=16}" Style="{StaticResource SubtleButtonStyle}"> @@ -508,34 +524,6 @@ - - - @@ -587,9 +575,9 @@ HorizontalAlignment="Stretch" VerticalAlignment="Stretch" Background="Transparent" - Visibility="{x:Bind ViewModel.IsCustomAIEnabled, Mode=OneWay, Converter={StaticResource BoolToInvertedVisibilityConverter}}"> + Visibility="{x:Bind ViewModel.IsCustomAIAvailable, Mode=OneWay, Converter={StaticResource BoolToInvertedVisibilityConverter}}"> - + @@ -634,11 +622,36 @@ - + + + + + + + + + + + - diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs index b33f998cbf..3383af5292 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs @@ -2,10 +2,10 @@ // The Microsoft Corporation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.ComponentModel; using System.Threading.Tasks; -using AdvancedPaste.Helpers; using AdvancedPaste.Models; using AdvancedPaste.ViewModels; using CommunityToolkit.Mvvm.Input; @@ -40,7 +40,7 @@ namespace AdvancedPaste.Controls public object Footer { - get => (object)GetValue(FooterProperty); + get => GetValue(FooterProperty); set => SetValue(FooterProperty, value); } @@ -50,27 +50,24 @@ namespace AdvancedPaste.Controls ViewModel = App.GetService(); ViewModel.PropertyChanged += ViewModel_PropertyChanged; - ViewModel.CustomActionActivated += ViewModel_CustomActionActivated; + ViewModel.PreviewRequested += ViewModel_PreviewRequested; } private void ViewModel_PropertyChanged(object sender, PropertyChangedEventArgs e) { - if (e.PropertyName == nameof(ViewModel.Busy) || e.PropertyName == nameof(ViewModel.PasteOperationErrorText)) + if (e.PropertyName == nameof(ViewModel.Busy) || e.PropertyName == nameof(ViewModel.PasteActionError)) { - var state = ViewModel.Busy ? "LoadingState" : string.IsNullOrEmpty(ViewModel.PasteOperationErrorText) ? "DefaultState" : "ErrorState"; + var state = ViewModel.Busy ? "LoadingState" : ViewModel.PasteActionError.HasText ? "ErrorState" : "DefaultState"; VisualStateManager.GoToState(this, state, true); } } - private void ViewModel_CustomActionActivated(object sender, CustomActionActivatedEventArgs e) + private void ViewModel_PreviewRequested(object sender, EventArgs e) { Logger.LogTrace(); - if (!e.PasteResult) - { - PreviewGrid.Width = InputTxtBox.ActualWidth; - PreviewFlyout.ShowAt(InputTxtBox); - } + PreviewGrid.Width = InputTxtBox.ActualWidth; + PreviewFlyout.ShowAt(InputTxtBox); } private void Grid_Loaded(object sender, RoutedEventArgs e) @@ -79,35 +76,19 @@ namespace AdvancedPaste.Controls } [RelayCommand] - private async Task GenerateCustomAsync() => await ViewModel.GenerateCustomFunctionAsync(PasteActionSource.PromptBox); - - [RelayCommand] - private void Recall() - { - Logger.LogTrace(); - - InputTxtBox.IsEnabled = true; - - var lastQuery = ViewModel.RecallPreviousCustomQuery(); - if (lastQuery != null) - { - InputTxtBox.Text = lastQuery.Query; - } - - ClipboardHelper.SetClipboardTextContent(lastQuery.ClipboardData); - } + private async Task GenerateCustomAIAsync() => await ViewModel.ExecuteCustomAIFormatFromCurrentQueryAsync(PasteActionSource.PromptBox); private async void InputTxtBox_KeyDown(object sender, Microsoft.UI.Xaml.Input.KeyRoutedEventArgs e) { - if (e.Key == Windows.System.VirtualKey.Enter && InputTxtBox.Text.Length > 0 && ViewModel.IsCustomAIEnabled) + if (e.Key == Windows.System.VirtualKey.Enter && InputTxtBox.Text.Length > 0 && ViewModel.IsCustomAIAvailable) { - await GenerateCustomAsync(); + await GenerateCustomAIAsync(); } } - private void PreviewPasteBtn_Click(object sender, RoutedEventArgs e) + private async void PreviewPasteBtn_Click(object sender, RoutedEventArgs e) { - ViewModel.PasteCustom(); + await ViewModel.PasteCustomAsync(); } private void ThumbUpDown_Click(object sender, RoutedEventArgs e) diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/MainWindow.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/MainWindow.xaml.cs index 6536bcfed9..18295f2315 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/MainWindow.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/MainWindow.xaml.cs @@ -43,7 +43,7 @@ namespace AdvancedPaste double GetHeight(int maxCustomActionCount) => baseHeight + new PasteFormatsToHeightConverter().GetHeight(coreActionCount + _userSettings.AdditionalActions.Count) + - new PasteFormatsToHeightConverter() { MaxItems = maxCustomActionCount }.GetHeight(optionsViewModel.IsAIServiceEnabled ? _userSettings.CustomActions.Count : 0); + new PasteFormatsToHeightConverter() { MaxItems = maxCustomActionCount }.GetHeight(optionsViewModel.IsCustomAIServiceEnabled ? _userSettings.CustomActions.Count : 0); MinHeight = GetHeight(1); Height = GetHeight(5); @@ -54,7 +54,7 @@ namespace AdvancedPaste _userSettings.Changed += (_, _) => UpdateHeight(); optionsViewModel.PropertyChanged += (_, e) => { - if (e.PropertyName == nameof(optionsViewModel.IsAIServiceEnabled)) + if (e.PropertyName == nameof(optionsViewModel.IsCustomAIServiceEnabled)) { UpdateHeight(); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs index d3e4ff1829..23940b16a5 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs @@ -195,12 +195,12 @@ namespace AdvancedPaste.Pages PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteClipboardItemClicked()); if (!string.IsNullOrEmpty(item.Content)) { - ClipboardHelper.SetClipboardTextContent(item.Content); + ClipboardHelper.SetTextContent(item.Content); } else if (item.Image is not null) { RandomAccessStreamReference image = await item.Item.Content.GetBitmapAsync(); - ClipboardHelper.SetClipboardImageContent(image); + ClipboardHelper.SetImageContent(image); } } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Assets/AdvancedPaste/SemanticKernel.svg b/src/modules/AdvancedPaste/AdvancedPaste/Assets/AdvancedPaste/SemanticKernel.svg new file mode 100644 index 0000000000..ef2f0af8b1 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Assets/AdvancedPaste/SemanticKernel.svg @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs deleted file mode 100644 index d9586638b8..0000000000 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Microsoft Corporation -// The Microsoft Corporation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Globalization; -using System.IO; -using System.Net; - -using Azure; -using Azure.AI.OpenAI; -using ManagedCommon; -using Microsoft.PowerToys.Settings.UI.Library; -using Microsoft.PowerToys.Telemetry; -using Windows.Security.Credentials; - -namespace AdvancedPaste.Helpers -{ - public class AICompletionsHelper - { - // Return Response and Status code from the request. - public struct AICompletionsResponse - { - public AICompletionsResponse(string response, int apiRequestStatus) - { - Response = response; - ApiRequestStatus = apiRequestStatus; - } - - public string Response { get; } - - public int ApiRequestStatus { get; } - } - - private string _openAIKey; - - private string _modelName = "gpt-3.5-turbo-instruct"; - - public bool IsAIEnabled => !string.IsNullOrEmpty(this._openAIKey); - - public AICompletionsHelper() - { - this._openAIKey = LoadOpenAIKey(); - } - - public void SetOpenAIKey(string openAIKey) - { - this._openAIKey = openAIKey; - } - - public string GetKey() - { - return _openAIKey; - } - - public static string LoadOpenAIKey() - { - PasswordVault vault = new PasswordVault(); - - try - { - PasswordCredential cred = vault.Retrieve("https://platform.openai.com/api-keys", "PowerToys_AdvancedPaste_OpenAIKey"); - if (cred is not null) - { - return cred.Password.ToString(); - } - } - catch (Exception) - { - } - - return string.Empty; - } - - private Response GetAICompletion(string systemInstructions, string userMessage) - { - OpenAIClient azureAIClient = new OpenAIClient(_openAIKey); - - var response = azureAIClient.GetCompletions( - new CompletionsOptions() - { - DeploymentName = _modelName, - Prompts = - { - systemInstructions + "\n\n" + userMessage, - }, - Temperature = 0.01F, - MaxTokens = 2000, - }); - - if (response.Value.Choices[0].FinishReason == "length") - { - Console.WriteLine("Cut off due to length constraints"); - } - - return response; - } - - public AICompletionsResponse AIFormatString(string inputInstructions, string inputString) - { - string systemInstructions = $@"You are tasked with reformatting user's clipboard data. Use the user's instructions, and the content of their clipboard below to edit their clipboard content as they have requested it. - -Do not output anything else besides the reformatted clipboard content."; - - string userMessage = $@"User instructions: -{inputInstructions} - -Clipboard Content: -{inputString} - -Output: -"; - - string aiResponse = null; - Response rawAIResponse = null; - int apiRequestStatus = (int)HttpStatusCode.OK; - try - { - rawAIResponse = this.GetAICompletion(systemInstructions, userMessage); - aiResponse = rawAIResponse.Value.Choices[0].Text; - - int promptTokens = rawAIResponse.Value.Usage.PromptTokens; - int completionTokens = rawAIResponse.Value.Usage.CompletionTokens; - PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent(promptTokens, completionTokens, _modelName)); - } - catch (Azure.RequestFailedException error) - { - Logger.LogError("GetAICompletion failed", error); - PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomErrorEvent(error.Message)); - apiRequestStatus = error.Status; - } - catch (Exception error) - { - Logger.LogError("GetAICompletion failed", error); - PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomErrorEvent(error.Message)); - apiRequestStatus = -1; - } - - return new AICompletionsResponse(aiResponse, apiRequestStatus); - } - } -} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs index 9ec1dd1618..0de901122b 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ClipboardHelper.cs @@ -3,214 +3,133 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Generic; -using System.Linq; +using System.Threading; using System.Threading.Tasks; -using AdvancedPaste.Models; using ManagedCommon; using Windows.ApplicationModel.DataTransfer; -using Windows.Data.Html; -using Windows.Graphics.Imaging; -using Windows.Storage; using Windows.Storage.Streams; using Windows.System; -namespace AdvancedPaste.Helpers +namespace AdvancedPaste.Helpers; + +internal static class ClipboardHelper { - internal static class ClipboardHelper + internal static async Task TryCopyPasteAsync(DataPackage dataPackage, Action onCopied) { - private static readonly HashSet ImageFileTypes = new(StringComparer.InvariantCultureIgnoreCase) { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".ico", ".svg" }; + Logger.LogTrace(); - private static readonly (string DataFormat, ClipboardFormat ClipboardFormat)[] DataFormats = - [ - (StandardDataFormats.Text, ClipboardFormat.Text), - (StandardDataFormats.Html, ClipboardFormat.Html), - (StandardDataFormats.Bitmap, ClipboardFormat.Image), - ]; - - internal static async Task GetAvailableClipboardFormatsAsync(DataPackageView clipboardData) + if (await dataPackage.GetView().HasUsableDataAsync()) { - var availableClipboardFormats = DataFormats.Aggregate( - ClipboardFormat.None, - (result, formatPair) => clipboardData.Contains(formatPair.DataFormat) ? (result | formatPair.ClipboardFormat) : result); - - if (clipboardData.Contains(StandardDataFormats.StorageItems)) - { - var storageItems = await clipboardData.GetStorageItemsAsync(); - - if (storageItems.Count == 1 && storageItems.Single() is StorageFile file && ImageFileTypes.Contains(file.FileType)) - { - availableClipboardFormats |= ClipboardFormat.ImageFile; - } - } - - return availableClipboardFormats; - } - - internal static void SetClipboardTextContent(string text) - { - Logger.LogTrace(); - - if (!string.IsNullOrEmpty(text)) - { - DataPackage output = new(); - output.SetText(text); - Clipboard.SetContentWithOptions(output, null); - - Flush(); - } - } - - private static bool Flush() - { - // TODO(stefan): For some reason Flush() fails from time to time when directly activated via hotkey. - // Calling inside a loop makes it work. - const int maxAttempts = 5; - for (int i = 1; i <= maxAttempts; i++) - { - try - { - Task.Run(Clipboard.Flush).Wait(); - return true; - } - catch (Exception ex) - { - if (i == maxAttempts) - { - Logger.LogError($"{nameof(Clipboard)}.{nameof(Flush)}() failed", ex); - } - } - } - - return false; - } - - private static async Task FlushAsync() => await Task.Run(Flush); - - internal static async Task SetClipboardFileContentAsync(string fileName) - { - var storageFile = await StorageFile.GetFileFromPathAsync(fileName); - - DataPackage output = new(); - output.SetStorageItems([storageFile]); - Clipboard.SetContent(output); - + Clipboard.SetContent(dataPackage); await FlushAsync(); - } - - internal static void SetClipboardImageContent(RandomAccessStreamReference image) - { - Logger.LogTrace(); - - if (image is not null) - { - DataPackage output = new(); - output.SetBitmap(image); - Clipboard.SetContentWithOptions(output, null); - - Flush(); - } - } - - // Function to send a single key event - private static void SendSingleKeyboardInput(short keyCode, uint keyStatus) - { - UIntPtr ignoreKeyEventFlag = (UIntPtr)0x5555; - - NativeMethods.INPUT inputShift = new NativeMethods.INPUT - { - type = NativeMethods.INPUTTYPE.INPUT_KEYBOARD, - data = new NativeMethods.InputUnion - { - ki = new NativeMethods.KEYBDINPUT - { - wVk = keyCode, - dwFlags = keyStatus, - - // Any keyevent with the extraInfo set to this value will be ignored by the keyboard hook and sent to the system instead. - dwExtraInfo = ignoreKeyEventFlag, - }, - }, - }; - - NativeMethods.INPUT[] inputs = new NativeMethods.INPUT[] { inputShift }; - _ = NativeMethods.SendInput(1, inputs, NativeMethods.INPUT.Size); - } - - internal static void SendPasteKeyCombination() - { - Logger.LogTrace(); - - SendSingleKeyboardInput((short)VirtualKey.LeftControl, (uint)NativeMethods.KeyEventF.KeyUp); - SendSingleKeyboardInput((short)VirtualKey.RightControl, (uint)NativeMethods.KeyEventF.KeyUp); - SendSingleKeyboardInput((short)VirtualKey.LeftWindows, (uint)NativeMethods.KeyEventF.KeyUp); - SendSingleKeyboardInput((short)VirtualKey.RightWindows, (uint)NativeMethods.KeyEventF.KeyUp); - SendSingleKeyboardInput((short)VirtualKey.LeftShift, (uint)NativeMethods.KeyEventF.KeyUp); - SendSingleKeyboardInput((short)VirtualKey.RightShift, (uint)NativeMethods.KeyEventF.KeyUp); - SendSingleKeyboardInput((short)VirtualKey.LeftMenu, (uint)NativeMethods.KeyEventF.KeyUp); - SendSingleKeyboardInput((short)VirtualKey.RightMenu, (uint)NativeMethods.KeyEventF.KeyUp); - - // Send Ctrl + V - SendSingleKeyboardInput((short)VirtualKey.Control, (uint)NativeMethods.KeyEventF.KeyDown); - SendSingleKeyboardInput((short)VirtualKey.V, (uint)NativeMethods.KeyEventF.KeyDown); - SendSingleKeyboardInput((short)VirtualKey.V, (uint)NativeMethods.KeyEventF.KeyUp); - SendSingleKeyboardInput((short)VirtualKey.Control, (uint)NativeMethods.KeyEventF.KeyUp); - - Logger.LogInfo("Paste sent"); - } - - internal static async Task GetClipboardTextOrHtmlTextAsync(DataPackageView clipboardData) - { - if (clipboardData.Contains(StandardDataFormats.Text)) - { - return await clipboardData.GetTextAsync(); - } - else if (clipboardData.Contains(StandardDataFormats.Html)) - { - var html = await clipboardData.GetHtmlFormatAsync(); - return HtmlUtilities.ConvertToText(html); - } - else - { - return string.Empty; - } - } - - internal static async Task GetClipboardHtmlContentAsync(DataPackageView clipboardData) => - clipboardData.Contains(StandardDataFormats.Html) ? await clipboardData.GetHtmlFormatAsync() : string.Empty; - - internal static async Task GetClipboardImageContentAsync(DataPackageView clipboardData) - { - using var stream = await GetClipboardImageStreamAsync(clipboardData); - if (stream != null) - { - var decoder = await BitmapDecoder.CreateAsync(stream); - return await decoder.GetSoftwareBitmapAsync(); - } - - return null; - } - - private static async Task GetClipboardImageStreamAsync(DataPackageView clipboardData) - { - if (clipboardData.Contains(StandardDataFormats.StorageItems)) - { - var storageItems = await clipboardData.GetStorageItemsAsync(); - var file = storageItems.Count == 1 ? storageItems[0] as StorageFile : null; - if (file != null) - { - return await file.OpenReadAsync(); - } - } - - if (clipboardData.Contains(StandardDataFormats.Bitmap)) - { - var bitmap = await clipboardData.GetBitmapAsync(); - return await bitmap.OpenReadAsync(); - } - - return null; + onCopied(); + SendPasteKeyCombination(); } } + + internal static void SetTextContent(string text) + { + Logger.LogTrace(); + + if (!string.IsNullOrEmpty(text)) + { + DataPackage output = new(); + output.SetText(text); + Clipboard.SetContentWithOptions(output, null); + + Flush(); + } + } + + internal static void SetImageContent(RandomAccessStreamReference image) + { + Logger.LogTrace(); + + if (image is not null) + { + DataPackage output = new(); + output.SetBitmap(image); + Clipboard.SetContentWithOptions(output, null); + + Flush(); + } + } + + private static bool Flush() + { + // TODO(stefan): For some reason Flush() fails from time to time when directly activated via hotkey. + // Calling inside a loop makes it work. + const int maxAttempts = 5; + for (int i = 1; i <= maxAttempts; i++) + { + try + { + Clipboard.Flush(); + return true; + } + catch (Exception ex) + { + if (i == maxAttempts) + { + Logger.LogError($"{nameof(Clipboard)}.{nameof(Flush)}() failed", ex); + } + } + } + + return false; + } + + private static async Task FlushAsync() + { + // This should run on the UI thread to avoid the "calling application is not the owner of the data on the clipboard" error. + return await Task.Factory.StartNew(Flush, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.FromCurrentSynchronizationContext()); + } + + internal static void SendPasteKeyCombination() + { + Logger.LogTrace(); + + SendSingleKeyboardInput((short)VirtualKey.LeftControl, (uint)NativeMethods.KeyEventF.KeyUp); + SendSingleKeyboardInput((short)VirtualKey.RightControl, (uint)NativeMethods.KeyEventF.KeyUp); + SendSingleKeyboardInput((short)VirtualKey.LeftWindows, (uint)NativeMethods.KeyEventF.KeyUp); + SendSingleKeyboardInput((short)VirtualKey.RightWindows, (uint)NativeMethods.KeyEventF.KeyUp); + SendSingleKeyboardInput((short)VirtualKey.LeftShift, (uint)NativeMethods.KeyEventF.KeyUp); + SendSingleKeyboardInput((short)VirtualKey.RightShift, (uint)NativeMethods.KeyEventF.KeyUp); + SendSingleKeyboardInput((short)VirtualKey.LeftMenu, (uint)NativeMethods.KeyEventF.KeyUp); + SendSingleKeyboardInput((short)VirtualKey.RightMenu, (uint)NativeMethods.KeyEventF.KeyUp); + + // Send Ctrl + V + SendSingleKeyboardInput((short)VirtualKey.Control, (uint)NativeMethods.KeyEventF.KeyDown); + SendSingleKeyboardInput((short)VirtualKey.V, (uint)NativeMethods.KeyEventF.KeyDown); + SendSingleKeyboardInput((short)VirtualKey.V, (uint)NativeMethods.KeyEventF.KeyUp); + SendSingleKeyboardInput((short)VirtualKey.Control, (uint)NativeMethods.KeyEventF.KeyUp); + + Logger.LogInfo("Paste sent"); + } + + // Function to send a single key event + private static void SendSingleKeyboardInput(short keyCode, uint keyStatus) + { + UIntPtr ignoreKeyEventFlag = (UIntPtr)0x5555; + + NativeMethods.INPUT inputShift = new NativeMethods.INPUT + { + type = NativeMethods.INPUTTYPE.INPUT_KEYBOARD, + data = new NativeMethods.InputUnion + { + ki = new NativeMethods.KEYBDINPUT + { + wVk = keyCode, + dwFlags = keyStatus, + + // Any keyevent with the extraInfo set to this value will be ignored by the keyboard hook and sent to the system instead. + dwExtraInfo = ignoreKeyEventFlag, + }, + }, + }; + + NativeMethods.INPUT[] inputs = new NativeMethods.INPUT[] { inputShift }; + _ = NativeMethods.SendInput(1, inputs, NativeMethods.INPUT.Size); + } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/Constants.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/Constants.cs index deeef8551f..51e563a119 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/Constants.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/Constants.cs @@ -7,6 +7,5 @@ namespace AdvancedPaste.Helpers internal static class Constants { internal static readonly string AdvancedPasteModuleName = "AdvancedPaste"; - internal static readonly string LastQueryJsonFileName = "lastQuery.json"; } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/DataPackageHelpers.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/DataPackageHelpers.cs new file mode 100644 index 0000000000..37ddf5e56e --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/DataPackageHelpers.cs @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +using AdvancedPaste.Models; +using Windows.ApplicationModel.DataTransfer; +using Windows.Data.Html; +using Windows.Graphics.Imaging; +using Windows.Storage; +using Windows.Storage.Streams; + +namespace AdvancedPaste.Helpers; + +internal static class DataPackageHelpers +{ + private static readonly HashSet ImageFileTypes = new(StringComparer.InvariantCultureIgnoreCase) { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".ico", ".svg" }; + + private static readonly (string DataFormat, ClipboardFormat ClipboardFormat)[] DataFormats = + [ + (StandardDataFormats.Text, ClipboardFormat.Text), + (StandardDataFormats.Html, ClipboardFormat.Html), + (StandardDataFormats.Bitmap, ClipboardFormat.Image), + ]; + + internal static DataPackage CreateFromText(string text) + { + DataPackage dataPackage = new(); + dataPackage.SetText(text); + return dataPackage; + } + + internal static async Task CreateFromFileAsync(string fileName) + { + var storageFile = await StorageFile.GetFileFromPathAsync(fileName); + + DataPackage dataPackage = new(); + dataPackage.SetStorageItems([storageFile]); + return dataPackage; + } + + internal static async Task GetAvailableFormatsAsync(this DataPackageView dataPackageView) + { + var availableFormats = DataFormats.Aggregate( + ClipboardFormat.None, + (result, formatPair) => dataPackageView.Contains(formatPair.DataFormat) ? (result | formatPair.ClipboardFormat) : result); + + if (dataPackageView.Contains(StandardDataFormats.StorageItems)) + { + var storageItems = await dataPackageView.GetStorageItemsAsync(); + + if (storageItems.Count == 1 && storageItems.Single() is StorageFile file) + { + availableFormats |= ClipboardFormat.File; + + if (ImageFileTypes.Contains(file.FileType)) + { + availableFormats |= ClipboardFormat.Image; + } + } + } + + return FixFormatsForAI(availableFormats); + } + + private static ClipboardFormat FixFormatsForAI(ClipboardFormat formats) + { + var result = formats; + + if (result.HasFlag(ClipboardFormat.File) && result != ClipboardFormat.File) + { + // Advertise the "generic" File format only if there is no other specific format available; confusing for AI otherwise. + result &= ~ClipboardFormat.File; + } + + if (result == (ClipboardFormat.Image | ClipboardFormat.Html)) + { + // The Windows Photo application advertises Image and Html when copying an image; this Html format is not easily usable and is confusing for AI. + result &= ~ClipboardFormat.Html; + } + + return result; + } + + internal static async Task HasUsableDataAsync(this DataPackageView dataPackageView) + { + var availableFormats = await GetAvailableFormatsAsync(dataPackageView); + + return availableFormats == ClipboardFormat.Text ? !string.IsNullOrEmpty(await dataPackageView.GetTextAsync()) : availableFormats != ClipboardFormat.None; + } + + internal static async Task GetTextOrEmptyAsync(this DataPackageView dataPackageView) => + dataPackageView.Contains(StandardDataFormats.Text) ? await dataPackageView.GetTextAsync() : string.Empty; + + internal static async Task GetTextOrHtmlTextAsync(this DataPackageView dataPackageView) + { + if (dataPackageView.Contains(StandardDataFormats.Text)) + { + return await dataPackageView.GetTextAsync(); + } + else if (dataPackageView.Contains(StandardDataFormats.Html)) + { + var html = await dataPackageView.GetHtmlFormatAsync(); + return HtmlUtilities.ConvertToText(html); + } + else + { + return string.Empty; + } + } + + internal static async Task GetHtmlContentAsync(this DataPackageView dataPackageView) => + dataPackageView.Contains(StandardDataFormats.Html) ? await dataPackageView.GetHtmlFormatAsync() : string.Empty; + + internal static async Task GetImageContentAsync(this DataPackageView dataPackageView) + { + using var stream = await dataPackageView.GetImageStreamAsync(); + if (stream != null) + { + var decoder = await BitmapDecoder.CreateAsync(stream); + return await decoder.GetSoftwareBitmapAsync(); + } + + return null; + } + + private static async Task GetImageStreamAsync(this DataPackageView dataPackageView) + { + if (dataPackageView.Contains(StandardDataFormats.StorageItems)) + { + var storageItems = await dataPackageView.GetStorageItemsAsync(); + var file = storageItems.Count == 1 ? storageItems[0] as StorageFile : null; + if (file != null) + { + return await file.OpenReadAsync(); + } + } + + if (dataPackageView.Contains(StandardDataFormats.Bitmap)) + { + var bitmap = await dataPackageView.GetBitmapAsync(); + return await bitmap.OpenReadAsync(); + } + + return null; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ErrorHelpers.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ErrorHelpers.cs new file mode 100644 index 0000000000..d7ff360c3d --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/ErrorHelpers.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Globalization; +using System.Net; + +namespace AdvancedPaste.Helpers; + +public static class ErrorHelpers +{ + public static string TranslateErrorText(int apiRequestStatus) => (HttpStatusCode)apiRequestStatus switch + { + HttpStatusCode.TooManyRequests => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyTooManyRequests"), + HttpStatusCode.Unauthorized => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyUnauthorized"), + HttpStatusCode.OK => string.Empty, + _ => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyError") + apiRequestStatus.ToString(CultureInfo.InvariantCulture), + }; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs index 49dbfda945..105fe2c0d8 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/IUserSettings.cs @@ -12,9 +12,9 @@ namespace AdvancedPaste.Settings { public interface IUserSettings { - public bool ShowCustomPreview { get; } + public bool IsAdvancedAIEnabled { get; } - public bool SendPasteKeyCombination { get; } + public bool ShowCustomPreview { get; } public bool CloseAfterLosingFocus { get; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/JsonHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/JsonHelper.cs index 5649943af9..2aed82f1b3 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/JsonHelper.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/JsonHelper.cs @@ -33,30 +33,23 @@ namespace AdvancedPaste.Helpers private static readonly Regex CsvRemoveStartAndEndQuotationMarksRegex = new Regex(@"^""(?=(""{2})+)|(?<=(""{2})+)""$"); private static readonly Regex CsvReplaceDoubleQuotationMarksRegex = new Regex(@"""{2}"); - internal static string ToJsonFromXmlOrCsv(DataPackageView clipboardData) + internal static async Task ToJsonFromXmlOrCsvAsync(DataPackageView clipboardData) { Logger.LogTrace(); - if (clipboardData == null || !clipboardData.Contains(StandardDataFormats.Text)) + if (!clipboardData.Contains(StandardDataFormats.Text)) { Logger.LogWarning("Clipboard does not contain text data"); return string.Empty; } -#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits - string text = Task.Run(async () => - { - string plainText = await clipboardData.GetTextAsync() as string; - return plainText; - }).Result; -#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits - + var text = await clipboardData.GetTextAsync(); string jsonText = string.Empty; // Try convert XML try { - XmlDocument doc = new XmlDocument(); + XmlDocument doc = new(); doc.LoadXml(text); Logger.LogDebug("Converted from XML."); jsonText = JsonConvert.SerializeXmlNode(doc, Newtonsoft.Json.Formatting.Indented); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/KernelExtensions.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/KernelExtensions.cs new file mode 100644 index 0000000000..2708e2ca69 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/KernelExtensions.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +using AdvancedPaste.Models; +using Microsoft.SemanticKernel; +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.Helpers; + +internal static class KernelExtensions +{ + private const string DataPackageKey = "DataPackage"; + private const string LastErrorKey = "LastError"; + private const string ActionChainKey = "ActionChain"; + + internal static DataPackageView GetDataPackageView(this Kernel kernel) + { + kernel.Data.TryGetValue(DataPackageKey, out object obj); + return obj as DataPackageView ?? (obj as DataPackage)?.GetView(); + } + + internal static DataPackage GetDataPackage(this Kernel kernel) + { + kernel.Data.TryGetValue(DataPackageKey, out object obj); + return obj as DataPackage ?? new(); + } + + internal static async Task GetDataFormatsAsync(this Kernel kernel) + { + var clipboardFormats = await kernel.GetDataPackageView().GetAvailableFormatsAsync(); + return clipboardFormats.ToString(); + } + + internal static void SetDataPackage(this Kernel kernel, DataPackage dataPackage) => kernel.Data[DataPackageKey] = dataPackage; + + internal static void SetDataPackageView(this Kernel kernel, DataPackageView dataPackageView) => kernel.Data[DataPackageKey] = dataPackageView; + + internal static Exception GetLastError(this Kernel kernel) => kernel.Data.TryGetValue(LastErrorKey, out object obj) ? obj as Exception : null; + + internal static void SetLastError(this Kernel kernel, Exception error) => kernel.Data[LastErrorKey] = error; + + internal static List GetOrAddActionChain(this Kernel kernel) + { + if (kernel.Data.TryGetValue(ActionChainKey, out var actionChainObj)) + { + return (List)actionChainObj; + } + else + { + List actionChain = []; + kernel.Data[ActionChainKey] = actionChain; + return actionChain; + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/MarkdownHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/MarkdownHelper.cs index 6322a83d05..697fb41034 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/MarkdownHelper.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/MarkdownHelper.cs @@ -15,67 +15,15 @@ namespace AdvancedPaste.Helpers { internal static class MarkdownHelper { - public static string ToMarkdown(DataPackageView clipboardData) + public static async Task ToMarkdownAsync(DataPackageView clipboardData) { Logger.LogTrace(); - if (clipboardData == null) - { - Logger.LogWarning("Clipboard does not contain data"); + var data = clipboardData.Contains(StandardDataFormats.Html) ? await clipboardData.GetHtmlFormatAsync() + : clipboardData.Contains(StandardDataFormats.Text) ? await clipboardData.GetTextAsync() + : string.Empty; - return string.Empty; - } - - string data = string.Empty; - - if (clipboardData.Contains(StandardDataFormats.Html)) - { - data = Task.Run(async () => - { - string data = await clipboardData.GetHtmlFormatAsync() as string; - return data; - }).Result; - } - else if (clipboardData.Contains(StandardDataFormats.Text)) - { - data = Task.Run(async () => - { - string plainText = await clipboardData.GetTextAsync() as string; - return plainText; - }).Result; - } - - if (!string.IsNullOrEmpty(data)) - { - string cleanedHtml = CleanHtml(data); - - return ConvertHtmlToMarkdown(cleanedHtml); - } - - return string.Empty; - } - - public static string PasteAsPlainTextFromClipboard(DataPackageView clipboardData) - { - Logger.LogTrace(); - - if (clipboardData != null) - { - if (!clipboardData.Contains(StandardDataFormats.Text)) - { - Logger.LogWarning("Clipboard does not contain text data"); - - return string.Empty; - } - - return Task.Run(async () => - { - string plainText = await clipboardData.GetTextAsync() as string; - return plainText; - }).Result; - } - - return string.Empty; + return string.IsNullOrEmpty(data) ? string.Empty : ConvertHtmlToMarkdown(CleanHtml(data)); } private static string CleanHtml(string html) diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/OcrHelpers.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/OcrHelpers.cs index 25b46fd160..1ed0665f9d 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/OcrHelpers.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/OcrHelpers.cs @@ -5,6 +5,7 @@ using System; using System.Linq; using System.Threading.Tasks; + using Windows.Globalization; using Windows.Graphics.Imaging; using Windows.Media.Ocr; @@ -21,7 +22,9 @@ public static class OcrHelpers var ocrEngine = OcrEngine.TryCreateFromLanguage(ocrLanguage) ?? throw new InvalidOperationException("Unable to create OCR engine"); var ocrResult = await ocrEngine.RecognizeAsync(bitmap); - return ocrResult.Text; + return string.IsNullOrWhiteSpace(ocrResult.Text) + ? throw new InvalidOperationException("Unable to extract text from image or image does not contain text") + : ocrResult.Text; } private static Language GetOCRLanguage() diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs new file mode 100644 index 0000000000..2c8f442cd7 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/TransformHelpers.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Globalization; +using System.IO; +using System.Threading.Tasks; + +using AdvancedPaste.Models; +using ManagedCommon; +using Windows.ApplicationModel.DataTransfer; +using Windows.Graphics.Imaging; +using Windows.Storage.Streams; + +namespace AdvancedPaste.Helpers; + +public static class TransformHelpers +{ + public static async Task TransformAsync(PasteFormats format, DataPackageView clipboardData) + { + return format switch + { + PasteFormats.PlainText => await ToPlainTextAsync(clipboardData), + PasteFormats.Markdown => await ToMarkdownAsync(clipboardData), + PasteFormats.Json => await ToJsonAsync(clipboardData), + PasteFormats.ImageToText => await ImageToTextAsync(clipboardData), + PasteFormats.PasteAsTxtFile => await ToTxtFileAsync(clipboardData), + PasteFormats.PasteAsPngFile => await ToPngFileAsync(clipboardData), + PasteFormats.PasteAsHtmlFile => await ToHtmlFileAsync(clipboardData), + PasteFormats.KernelQuery => throw new ArgumentException($"Unsupported format {format}", nameof(format)), + PasteFormats.CustomTextTransformation => throw new ArgumentException($"Unsupported format {format}", nameof(format)), + _ => throw new ArgumentException($"Unknown value {format}", nameof(format)), + }; + } + + private static async Task ToPlainTextAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + return CreateDataPackageFromText(await clipboardData.GetTextOrEmptyAsync()); + } + + private static async Task ToMarkdownAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + return CreateDataPackageFromText(await MarkdownHelper.ToMarkdownAsync(clipboardData)); + } + + private static async Task ToJsonAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + return CreateDataPackageFromText(await JsonHelper.ToJsonFromXmlOrCsvAsync(clipboardData)); + } + + private static async Task ImageToTextAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + + var bitmap = await clipboardData.GetImageContentAsync() ?? throw new ArgumentException("No image content found in clipboard", nameof(clipboardData)); + var text = await OcrHelpers.ExtractTextAsync(bitmap); + return CreateDataPackageFromText(text); + } + + private static async Task ToPngFileAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + + var clipboardBitmap = await clipboardData.GetImageContentAsync(); + + using var pngStream = new InMemoryRandomAccessStream(); + var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, pngStream); + encoder.SetSoftwareBitmap(clipboardBitmap); + await encoder.FlushAsync(); + + return await CreateDataPackageFromFileContentAsync(pngStream.AsStreamForRead(), "png"); + } + + private static async Task ToTxtFileAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + + var text = await clipboardData.GetTextOrHtmlTextAsync(); + return await CreateDataPackageFromFileContentAsync(text, "txt"); + } + + private static async Task ToHtmlFileAsync(DataPackageView clipboardData) + { + Logger.LogTrace(); + + var cfHtml = await clipboardData.GetHtmlContentAsync(); + var html = RemoveHtmlMetadata(cfHtml); + + return await CreateDataPackageFromFileContentAsync(html, "html"); + } + + /// + /// Removes leading CF_HTML metadata from HTML clipboard data. + /// See: https://learn.microsoft.com/en-us/windows/win32/dataxchg/html-clipboard-format + /// + private static string RemoveHtmlMetadata(string cfHtml) + { + int? GetIntTagValue(string tagName) + { + var tagNameWithColon = tagName + ":"; + int tagStartPos = cfHtml.IndexOf(tagNameWithColon, StringComparison.InvariantCulture); + + const int tagValueLength = 10; + return tagStartPos != -1 && int.TryParse(cfHtml.AsSpan(tagStartPos + tagNameWithColon.Length, tagValueLength), CultureInfo.InvariantCulture, out int result) ? result : null; + } + + var startFragmentIndex = GetIntTagValue("StartFragment"); + var endFragmentIndex = GetIntTagValue("EndFragment"); + + return (startFragmentIndex == null || endFragmentIndex == null) ? cfHtml : cfHtml[startFragmentIndex.Value..endFragmentIndex.Value]; + } + + private static async Task CreateDataPackageFromFileContentAsync(string data, string fileExtension) + { + if (string.IsNullOrEmpty(data)) + { + throw new ArgumentException($"Empty value in {nameof(CreateDataPackageFromFileContentAsync)}", nameof(data)); + } + + var path = GetPasteAsFileTempFilePath(fileExtension); + + await File.WriteAllTextAsync(path, data); + return await DataPackageHelpers.CreateFromFileAsync(path); + } + + private static async Task CreateDataPackageFromFileContentAsync(Stream stream, string fileExtension) + { + var path = GetPasteAsFileTempFilePath(fileExtension); + + using var fileStream = File.Create(path); + await stream.CopyToAsync(fileStream); + + return await DataPackageHelpers.CreateFromFileAsync(path); + } + + private static string GetPasteAsFileTempFilePath(string fileExtension) + { + var prefix = ResourceLoaderInstance.ResourceLoader.GetString("PasteAsFile_FilePrefix"); + var timestamp = DateTime.Now.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture); + + return Path.Combine(Path.GetTempPath(), $"{prefix}{timestamp}.{fileExtension}"); + } + + private static DataPackage CreateDataPackageFromText(string content) => DataPackageHelpers.CreateFromText(content); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs index 7c64db6478..e2c2e07c84 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/UserSettings.cs @@ -33,9 +33,9 @@ namespace AdvancedPaste.Settings public event EventHandler Changed; - public bool ShowCustomPreview { get; private set; } + public bool IsAdvancedAIEnabled { get; private set; } - public bool SendPasteKeyCombination { get; private set; } + public bool ShowCustomPreview { get; private set; } public bool CloseAfterLosingFocus { get; private set; } @@ -43,12 +43,12 @@ namespace AdvancedPaste.Settings public IReadOnlyList CustomActions => _customActions; - public UserSettings() + public UserSettings(IFileSystem fileSystem) { - _settingsUtils = new SettingsUtils(); + _settingsUtils = new SettingsUtils(fileSystem); + IsAdvancedAIEnabled = false; ShowCustomPreview = true; - SendPasteKeyCombination = true; CloseAfterLosingFocus = false; _additionalActions = []; _customActions = []; @@ -56,7 +56,7 @@ namespace AdvancedPaste.Settings LoadSettingsFromJson(); - _watcher = Helper.GetFileWatcher(AdvancedPasteModuleName, "settings.json", OnSettingsFileChanged); + _watcher = Helper.GetFileWatcher(AdvancedPasteModuleName, "settings.json", OnSettingsFileChanged, fileSystem); } private void OnSettingsFileChanged() @@ -98,8 +98,8 @@ namespace AdvancedPaste.Settings { var properties = settings.Properties; + IsAdvancedAIEnabled = properties.IsAdvancedAIEnabled; ShowCustomPreview = properties.ShowCustomPreview; - SendPasteKeyCombination = properties.SendPasteKeyCombination; CloseAfterLosingFocus = properties.CloseAfterLosingFocus; var sourceAdditionalActions = properties.AdditionalActions; diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/AIServiceUsage.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/AIServiceUsage.cs new file mode 100644 index 0000000000..795c599a20 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/AIServiceUsage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace AdvancedPaste.Models; + +public record class AIServiceUsage(int PromptTokens, int CompletionTokens) +{ + public static AIServiceUsage None => new(PromptTokens: 0, CompletionTokens: 0); + + public bool HasUsage => PromptTokens > 0 || CompletionTokens > 0; + + public static AIServiceUsage Add(AIServiceUsage first, AIServiceUsage second) => + new(first.PromptTokens + second.PromptTokens, first.CompletionTokens + second.CompletionTokens); + + public override string ToString() => + $"{nameof(PromptTokens)}: {PromptTokens}, {nameof(CompletionTokens)}: {CompletionTokens}"; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/ActionChainItem.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/ActionChainItem.cs new file mode 100644 index 0000000000..e068a4d980 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/ActionChainItem.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace AdvancedPaste.Models; + +public record class ActionChainItem(PasteFormats Format, Dictionary Arguments); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardFormat.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardFormat.cs index a63e79735e..63c935b63e 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardFormat.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/ClipboardFormat.cs @@ -14,5 +14,5 @@ public enum ClipboardFormat Html = 1 << 1, Audio = 1 << 2, Image = 1 << 3, - ImageFile = 1 << 4, + File = 1 << 4, // output only for now } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/CustomActionActivatedEventArgs.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/CustomActionActivatedEventArgs.cs deleted file mode 100644 index d4846e01be..0000000000 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/CustomActionActivatedEventArgs.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft Corporation -// The Microsoft Corporation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; - -namespace AdvancedPaste.Models; - -public sealed class CustomActionActivatedEventArgs(string text, bool pasteResult) : EventArgs -{ - public string Text { get; private init; } = text; - - public bool PasteResult { get; private init; } = pasteResult; -} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/CustomQuery.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/CustomQuery.cs deleted file mode 100644 index 530ee0411a..0000000000 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/CustomQuery.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation -// The Microsoft Corporation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text.Json; - -using AdvancedPaste.Helpers; -using Microsoft.PowerToys.Settings.UI.Library.Interfaces; - -namespace AdvancedPaste.Models -{ - internal sealed class CustomQuery : ISettingsConfig - { - public string Query { get; set; } - - public string ClipboardData { get; set; } - - public string GetModuleName() => Constants.AdvancedPasteModuleName; - - public string ToJsonString() => JsonSerializer.Serialize(this); - - public override string ToString() - => JsonSerializer.Serialize(this); - - public bool UpgradeSettingsConfiguration() => false; - } -} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/CacheKey.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/CacheKey.cs new file mode 100644 index 0000000000..203e4064c7 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/CacheKey.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace AdvancedPaste.Models.KernelQueryCache; + +public sealed class CacheKey : IEquatable +{ + public static StringComparer PromptComparer => StringComparer.CurrentCultureIgnoreCase; + + public string Prompt { get; init; } + + public ClipboardFormat AvailableFormats { get; init; } + + public override string ToString() => $"{AvailableFormats}: {Prompt}"; + + public override bool Equals(object obj) => Equals(obj as CacheKey); + + public bool Equals(CacheKey other) => other != null && PromptComparer.Equals(Prompt, other.Prompt) && AvailableFormats == other.AvailableFormats; + + public override int GetHashCode() => PromptComparer.GetHashCode(Prompt) ^ AvailableFormats.GetHashCode(); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/CacheValue.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/CacheValue.cs new file mode 100644 index 0000000000..e8475c8a3c --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/CacheValue.cs @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace AdvancedPaste.Models.KernelQueryCache; + +public record class CacheValue(List ActionChain); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/PersistedCache.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/PersistedCache.cs new file mode 100644 index 0000000000..928ba7ab37 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/KernelQueryCache/PersistedCache.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +using AdvancedPaste.Helpers; +using Microsoft.PowerToys.Settings.UI.Library.Interfaces; + +namespace AdvancedPaste.Models.KernelQueryCache; + +public sealed class PersistedCache : ISettingsConfig +{ + public record class CacheItem(CacheKey CacheKey, CacheValue CacheValue); + + private static readonly JsonSerializerOptions SerializerOptions = new() + { + Converters = + { + new JsonStringEnumConverter(), + }, + }; + + public static PersistedCache FromJsonString(string json) => JsonSerializer.Deserialize(json, SerializerOptions); + + public string Version { get; init; } + + public List Items { get; init; } = []; + + public string GetModuleName() => Constants.AdvancedPasteModuleName; + + public string ToJsonString() => JsonSerializer.Serialize(this, SerializerOptions); + + public override string ToString() => ToJsonString(); + + public bool UpgradeSettingsConfiguration() => false; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionError.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionError.cs new file mode 100644 index 0000000000..6ec9a49028 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionError.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +using AdvancedPaste.Helpers; + +namespace AdvancedPaste.Models; + +public sealed class PasteActionError +{ + public static PasteActionError None => new() { Text = string.Empty, Details = string.Empty }; + + public string Text { get; private init; } + + public string Details { get; private init; } + + public bool HasText => !string.IsNullOrEmpty(Text); + + public bool HasDetails => !string.IsNullOrEmpty(Details); + + public static PasteActionError FromResourceId(string resourceId) => + new() + { + Text = ResourceLoaderInstance.ResourceLoader.GetString(resourceId), + Details = string.Empty, + }; + + public static PasteActionError FromException(Exception ex) => + new() + { + Text = ex is PasteActionException ? ex.Message : ResourceLoaderInstance.ResourceLoader.GetString("PasteError"), + Details = (ex as PasteActionException)?.AIServiceMessage ?? string.Empty, + }; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionException.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionException.cs index fed4e24c50..5508b9d1d8 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionException.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionException.cs @@ -6,6 +6,7 @@ using System; namespace AdvancedPaste.Models; -public sealed class PasteActionException(string message) : Exception(message) +public class PasteActionException(string message, Exception innerException, string aiServiceMessage = null) : Exception(message, innerException) { + public string AIServiceMessage { get; } = aiServiceMessage; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionModeratedException.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionModeratedException.cs new file mode 100644 index 0000000000..42268d2631 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteActionModeratedException.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using AdvancedPaste.Helpers; + +namespace AdvancedPaste.Models; + +public sealed class PasteActionModeratedException : PasteActionException +{ + public PasteActionModeratedException() + : base( + message: ResourceLoaderInstance.ResourceLoader.GetString("PasteError"), + innerException: null, + aiServiceMessage: ResourceLoaderInstance.ResourceLoader.GetString("PasteActionModerated")) + { + } + + /// + /// Non-localized error description for logs, reports, telemetry etc. + /// + public const string ErrorDescription = "Paste operation moderated"; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs index c38a54b843..e9412b69a6 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormat.cs @@ -7,7 +7,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Reflection; -using Microsoft.PowerToys.Settings.UI.Library; namespace AdvancedPaste.Models; @@ -25,19 +24,21 @@ public sealed class PasteFormat IsEnabled = SupportsClipboardFormats(clipboardFormats) && (isAIServiceEnabled || !Metadata.RequiresAIService); } - public PasteFormat(PasteFormats format, ClipboardFormat clipboardFormats, bool isAIServiceEnabled, Func resourceLoader) - : this(format, clipboardFormats, isAIServiceEnabled) - { - Name = Metadata.ResourceId == null ? string.Empty : resourceLoader(Metadata.ResourceId); - Prompt = string.Empty; - } + public static PasteFormat CreateStandardFormat(PasteFormats format, ClipboardFormat clipboardFormats, bool isAIServiceEnabled, Func resourceLoader) => + new(format, clipboardFormats, isAIServiceEnabled) + { + Name = MetadataDict[format].ResourceId == null ? string.Empty : resourceLoader(MetadataDict[format].ResourceId), + Prompt = string.Empty, + IsSavedQuery = false, + }; - public PasteFormat(AdvancedPasteCustomAction customAction, ClipboardFormat clipboardFormats, bool isAIServiceEnabled) - : this(PasteFormats.Custom, clipboardFormats, isAIServiceEnabled) - { - Name = customAction.Name; - Prompt = customAction.Prompt; - } + public static PasteFormat CreateCustomAIFormat(PasteFormats format, string name, string prompt, bool isSavedQuery, ClipboardFormat clipboardFormats, bool isAIServiceEnabled) => + new(format, clipboardFormats, isAIServiceEnabled) + { + Name = name, + Prompt = prompt, + IsSavedQuery = isSavedQuery, + }; public PasteFormatMetadataAttribute Metadata => MetadataDict[Format]; @@ -49,6 +50,8 @@ public sealed class PasteFormat public string Prompt { get; private init; } + public bool IsSavedQuery { get; private init; } + public bool IsEnabled { get; private init; } public double Opacity => IsEnabled ? 1 : 0.5; @@ -59,5 +62,8 @@ public sealed class PasteFormat public string ShortcutText { get; set; } = string.Empty; - public bool SupportsClipboardFormats(ClipboardFormat clipboardFormats) => (clipboardFormats & Metadata.SupportedClipboardFormats) != ClipboardFormat.None; + public static bool SupportsClipboardFormats(PasteFormats format, ClipboardFormat clipboardFormats) + => (clipboardFormats & MetadataDict[format].SupportedClipboardFormats) != ClipboardFormat.None; + + public bool SupportsClipboardFormats(ClipboardFormat clipboardFormats) => SupportsClipboardFormats(Format, clipboardFormats); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs index cb3a8a954e..1b7c18af9f 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormatMetadataAttribute.cs @@ -17,7 +17,16 @@ public sealed class PasteFormatMetadataAttribute : Attribute public bool RequiresAIService { get; init; } + public bool CanPreview { get; init; } + public ClipboardFormat SupportedClipboardFormats { get; init; } public string IPCKey { get; init; } + + /// + /// Gets a description of the action that should be exposed to Semantic Kernel, or if it should not be exposed. + /// + public string KernelFunctionDescription { get; init; } + + public bool RequiresPrompt { get; init; } } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs index fe710d5410..588f102506 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Models/PasteFormats.cs @@ -8,27 +8,96 @@ namespace AdvancedPaste.Models; public enum PasteFormats { - [PasteFormatMetadata(IsCoreAction = true, ResourceId = "PasteAsPlainText", IconGlyph = "\uE8E9", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Text)] + [PasteFormatMetadata( + IsCoreAction = true, + ResourceId = "PasteAsPlainText", + IconGlyph = "\uE8E9", + RequiresAIService = false, + CanPreview = false, + SupportedClipboardFormats = ClipboardFormat.Text, + KernelFunctionDescription = "Takes clipboard text and returns it as it is.")] PlainText, - [PasteFormatMetadata(IsCoreAction = true, ResourceId = "PasteAsMarkdown", IconGlyph = "\ue8a5", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Text)] + [PasteFormatMetadata( + IsCoreAction = true, + ResourceId = "PasteAsMarkdown", + IconGlyph = "\ue8a5", + RequiresAIService = false, + CanPreview = false, + SupportedClipboardFormats = ClipboardFormat.Text, + KernelFunctionDescription = "Takes clipboard text and formats it as markdown text.")] Markdown, - [PasteFormatMetadata(IsCoreAction = true, ResourceId = "PasteAsJson", IconGlyph = "\uE943", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Text)] + [PasteFormatMetadata( + IsCoreAction = true, + ResourceId = "PasteAsJson", + IconGlyph = "\uE943", + RequiresAIService = false, + CanPreview = false, + SupportedClipboardFormats = ClipboardFormat.Text, + KernelFunctionDescription = "Takes clipboard text and formats it as JSON text.")] Json, - [PasteFormatMetadata(IsCoreAction = false, ResourceId = "ImageToText", IconGlyph = "\uE91B", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Image | ClipboardFormat.ImageFile, IPCKey = AdvancedPasteAdditionalActions.PropertyNames.ImageToText)] + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "ImageToText", + IconGlyph = "\uE91B", + RequiresAIService = false, + CanPreview = true, + SupportedClipboardFormats = ClipboardFormat.Image, + IPCKey = AdvancedPasteAdditionalActions.PropertyNames.ImageToText, + KernelFunctionDescription = "Takes an image in the clipboard and extracts all text from it using OCR.")] ImageToText, - [PasteFormatMetadata(IsCoreAction = false, ResourceId = "PasteAsTxtFile", IconGlyph = "\uE8D2", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Text | ClipboardFormat.Html, IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsTxtFile)] + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "PasteAsTxtFile", + IconGlyph = "\uE8D2", + RequiresAIService = false, + CanPreview = false, + SupportedClipboardFormats = ClipboardFormat.Text | ClipboardFormat.Html, + IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsTxtFile, + KernelFunctionDescription = "Takes text or HTML data in the clipboard and transforms it to a TXT file.")] PasteAsTxtFile, - [PasteFormatMetadata(IsCoreAction = false, ResourceId = "PasteAsPngFile", IconGlyph = "\uE8B9", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Image | ClipboardFormat.ImageFile, IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsPngFile)] + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "PasteAsPngFile", + IconGlyph = "\uE8B9", + RequiresAIService = false, + CanPreview = false, + SupportedClipboardFormats = ClipboardFormat.Image, + IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsPngFile, + KernelFunctionDescription = "Takes an image in the clipboard and transforms it to a PNG file.")] PasteAsPngFile, - [PasteFormatMetadata(IsCoreAction = false, ResourceId = "PasteAsHtmlFile", IconGlyph = "\uF6FA", RequiresAIService = false, SupportedClipboardFormats = ClipboardFormat.Html, IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsHtmlFile)] + [PasteFormatMetadata( + IsCoreAction = false, + ResourceId = "PasteAsHtmlFile", + IconGlyph = "\uF6FA", + RequiresAIService = false, + CanPreview = false, + SupportedClipboardFormats = ClipboardFormat.Html, + IPCKey = AdvancedPastePasteAsFileAction.PropertyNames.PasteAsHtmlFile, + KernelFunctionDescription = "Takes HTML data in the clipboard and transforms it to an HTML file.")] PasteAsHtmlFile, - [PasteFormatMetadata(IsCoreAction = false, IconGlyph = "\uE945", RequiresAIService = true, SupportedClipboardFormats = ClipboardFormat.Text)] - Custom, + [PasteFormatMetadata( + IsCoreAction = false, + IconGlyph = "\uE945", + RequiresAIService = true, + CanPreview = true, + SupportedClipboardFormats = ClipboardFormat.Text | ClipboardFormat.Html | ClipboardFormat.Audio | ClipboardFormat.Image, + RequiresPrompt = true)] + KernelQuery, + + [PasteFormatMetadata( + IsCoreAction = false, + IconGlyph = "\uE945", + RequiresAIService = true, + CanPreview = true, + SupportedClipboardFormats = ClipboardFormat.Text, + KernelFunctionDescription = "Takes input instructions and transforms clipboard text (not TXT files) with these input instructions, putting the result back on the clipboard. This uses AI to accomplish the task.", + RequiresPrompt = true)] + CustomTextTransformation, } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActionKernelQueryCacheService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActionKernelQueryCacheService.cs new file mode 100644 index 0000000000..f7d888cf10 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/CustomActionKernelQueryCacheService.cs @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; + +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using AdvancedPaste.Models.KernelQueryCache; +using AdvancedPaste.Settings; +using ManagedCommon; +using Microsoft.PowerToys.Settings.UI.Library; + +namespace AdvancedPaste.Services; + +/// +/// Implements by only caching queries with prompts +/// that correspond to the user's custom actions or to the localized names of bundled actions. +/// This avoids potential privacy issues and prevents the cache from getting too large. +/// +public sealed class CustomActionKernelQueryCacheService : IKernelQueryCacheService +{ + private const string PersistedCacheFileName = "kernelQueryCache.json"; + + private readonly HashSet _cacheablePrompts = new(CacheKey.PromptComparer); + private readonly Dictionary _memoryCache = []; + + private readonly IUserSettings _userSettings; + private readonly IFileSystem _fileSystem; + private readonly SettingsUtils _settingsUtil; + + private static string Version => Assembly.GetExecutingAssembly()?.GetName()?.Version?.ToString() ?? string.Empty; + + public CustomActionKernelQueryCacheService(IUserSettings userSettings, IFileSystem fileSystem) + { + _userSettings = userSettings; + _fileSystem = fileSystem; + _settingsUtil = new SettingsUtils(fileSystem); + + _userSettings.Changed += OnUserSettingsChanged; + + UpdateCacheablePrompts(); + + _memoryCache = LoadPersistedCacheItems().Where(pair => pair.CacheKey != null) + .GroupBy(pair => pair.CacheKey, pair => pair.CacheValue) + .ToDictionary(group => group.Key, group => group.First()); + + RemoveInapplicableCacheKeys(); + + Logger.LogDebug($"Kernel query cache initialized with {_memoryCache.Count} items"); + } + + public async Task WriteAsync(CacheKey key, CacheValue value) + { + if (_cacheablePrompts.Contains(key.Prompt)) + { + _memoryCache[key] = value; + await SaveAsync(); + } + } + + public CacheValue ReadOrNull(CacheKey key) => _memoryCache.GetValueOrDefault(key); + + private List LoadPersistedCacheItems() + { + try + { + if (!_settingsUtil.SettingsExists(AdvancedPasteSettings.ModuleName, PersistedCacheFileName)) + { + return []; + } + + var jsonString = _fileSystem.File.ReadAllText(_settingsUtil.GetSettingsFilePath(AdvancedPasteSettings.ModuleName, PersistedCacheFileName)); + var persistedCache = PersistedCache.FromJsonString(jsonString); + + if (persistedCache.Version == Version) + { + return persistedCache.Items; + } + else + { + Logger.LogWarning($"Ignoring persisted kernel query cache; version mismatch - actual: {persistedCache.Version}, expected: {Version}"); + return []; + } + } + catch (Exception ex) + { + Logger.LogError("Failed to load kernel query cache", ex); + return []; + } + } + + private async void OnUserSettingsChanged(object sender, EventArgs e) + { + UpdateCacheablePrompts(); + + if (RemoveInapplicableCacheKeys()) + { + await SaveAsync(); + } + } + + private void UpdateCacheablePrompts() + { + var localizedActionNames = from pair in PasteFormat.MetadataDict + let format = pair.Key + let metadata = pair.Value + where !string.IsNullOrEmpty(metadata.ResourceId) + where metadata.IsCoreAction || _userSettings.AdditionalActions.Contains(format) + select ResourceLoaderInstance.ResourceLoader.GetString(metadata.ResourceId); + + var customActionPrompts = from customAction in _userSettings.CustomActions + select customAction.Prompt; + + _cacheablePrompts.Clear(); + _cacheablePrompts.UnionWith(localizedActionNames.Concat(customActionPrompts)); + } + + private bool RemoveInapplicableCacheKeys() + { + var keysToRemove = _memoryCache.Keys + .Where(key => !_cacheablePrompts.Contains(key.Prompt)) + .ToList(); + + foreach (var key in keysToRemove) + { + _memoryCache.Remove(key); + } + + return keysToRemove.Count > 0; + } + + private async Task SaveAsync() + { + PersistedCache cache = new() + { + Version = Version, + Items = _memoryCache.Select(pair => new PersistedCache.CacheItem(pair.Key, pair.Value)).ToList(), + }; + + _settingsUtil.SaveSettings(cache.ToJsonString(), AdvancedPasteSettings.ModuleName, PersistedCacheFileName); + + Logger.LogDebug($"Kernel query cache saved with {_memoryCache.Count} item(s)"); + + await Task.CompletedTask; // Async placeholder until _settingsUtil.SaveSettings has an async implementation + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs new file mode 100644 index 0000000000..54759b7dc8 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IAICredentialsProvider.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace AdvancedPaste.Services; + +public interface IAICredentialsProvider +{ + bool IsConfigured { get; } + + string Key { get; } + + bool Refresh(); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs new file mode 100644 index 0000000000..800f7b0416 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/ICustomTextTransformService.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; + +namespace AdvancedPaste.Services; + +public interface ICustomTextTransformService +{ + Task TransformTextAsync(string prompt, string inputText); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelQueryCacheService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelQueryCacheService.cs new file mode 100644 index 0000000000..9c169e82a6 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelQueryCacheService.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; + +using AdvancedPaste.Models.KernelQueryCache; + +namespace AdvancedPaste.Services; + +public interface IKernelQueryCacheService +{ + Task WriteAsync(CacheKey key, CacheValue value); + + CacheValue ReadOrNull(CacheKey key); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs new file mode 100644 index 0000000000..ae99fccf44 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IKernelService.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; + +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.Services; + +public interface IKernelService +{ + Task TransformClipboardAsync(string prompt, DataPackageView clipboardData, bool isSavedQuery); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IPasteFormatExecutor.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IPasteFormatExecutor.cs index e0bb39ab7c..9df354e3d1 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/IPasteFormatExecutor.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IPasteFormatExecutor.cs @@ -3,11 +3,13 @@ // See the LICENSE file in the project root for more information. using System.Threading.Tasks; + using AdvancedPaste.Models; +using Windows.ApplicationModel.DataTransfer; namespace AdvancedPaste.Services; public interface IPasteFormatExecutor { - Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source); + Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source); } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/IPromptModerationService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/IPromptModerationService.cs new file mode 100644 index 0000000000..bd7963ac78 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/IPromptModerationService.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Threading.Tasks; + +namespace AdvancedPaste.Services; + +public interface IPromptModerationService +{ + Task ValidateAsync(string fullPrompt); +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs new file mode 100644 index 0000000000..08526d0b0c --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/KernelServiceBase.cs @@ -0,0 +1,280 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using AdvancedPaste.Models.KernelQueryCache; +using AdvancedPaste.Telemetry; +using ManagedCommon; +using Microsoft.PowerToys.Telemetry; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Windows.ApplicationModel.DataTransfer; + +namespace AdvancedPaste.Services; + +public abstract class KernelServiceBase(IKernelQueryCacheService queryCacheService, IPromptModerationService promptModerationService, ICustomTextTransformService customTextTransformService) : IKernelService +{ + private const string PromptParameterName = "prompt"; + + private readonly IKernelQueryCacheService _queryCacheService = queryCacheService; + private readonly IPromptModerationService _promptModerationService = promptModerationService; + private readonly ICustomTextTransformService _customTextTransformService = customTextTransformService; + + protected abstract string ModelName { get; } + + protected abstract PromptExecutionSettings PromptExecutionSettings { get; } + + protected abstract void AddChatCompletionService(IKernelBuilder kernelBuilder); + + protected abstract AIServiceUsage GetAIServiceUsage(ChatMessageContent chatMessage); + + public async Task TransformClipboardAsync(string prompt, DataPackageView clipboardData, bool isSavedQuery) + { + Logger.LogTrace(); + + var kernel = CreateKernel(); + kernel.SetDataPackageView(clipboardData); + + CacheKey cacheKey = new() { Prompt = prompt, AvailableFormats = await clipboardData.GetAvailableFormatsAsync() }; + var maybeCacheValue = _queryCacheService.ReadOrNull(cacheKey); + bool cacheUsed = maybeCacheValue != null; + + ChatHistory chatHistory = []; + + try + { + (chatHistory, var usage) = cacheUsed ? await ExecuteCachedActionChain(kernel, maybeCacheValue.ActionChain) : await ExecuteAICompletion(kernel, prompt); + + LogResult(cacheUsed, isSavedQuery, kernel.GetOrAddActionChain(), usage); + + if (kernel.GetLastError() is Exception ex) + { + throw ex; + } + + var outputPackage = kernel.GetDataPackage(); + + if (!(await outputPackage.GetView().HasUsableDataAsync())) + { + throw new InvalidOperationException("No data was returned from the kernel operation"); + } + + if (!cacheUsed) + { + await _queryCacheService.WriteAsync(cacheKey, new CacheValue(kernel.GetOrAddActionChain())); + } + + Logger.LogDebug($"Kernel operation done: \n{FormatChatHistory(chatHistory)}"); + + return outputPackage; + } + catch (Exception ex) + { + Logger.LogError($"Error executing kernel operation", ex); + Logger.LogError($"Kernel operation Error: \n{FormatChatHistory(chatHistory)}"); + + AdvancedPasteSemanticKernelErrorEvent errorEvent = new(ex is PasteActionModeratedException ? PasteActionModeratedException.ErrorDescription : ex.Message); + PowerToysTelemetry.Log.WriteEvent(errorEvent); + + if (ex is PasteActionException) + { + throw; + } + else + { + var message = ex is HttpOperationException httpOperationEx + ? ErrorHelpers.TranslateErrorText((int?)httpOperationEx.StatusCode ?? -1) + : ResourceLoaderInstance.ResourceLoader.GetString("PasteError"); + + var lastAssistantMessage = chatHistory.LastOrDefault(chatMessage => chatMessage.Role == AuthorRole.Assistant)?.ToString(); + throw new PasteActionException(message, innerException: ex, aiServiceMessage: lastAssistantMessage); + } + } + } + + private static string GetFullPrompt(ChatHistory initialHistory) + { + if (initialHistory.Count == 0) + { + throw new ArgumentException("Chat history must not be empty", nameof(initialHistory)); + } + + int numSystemMessages = initialHistory.Count - 1; + var systemMessages = initialHistory.Take(numSystemMessages); + var userPromptMessage = initialHistory.Last(); + + if (systemMessages.Any(message => message.Role != AuthorRole.System)) + { + throw new ArgumentException("Chat history must start with system messages", nameof(initialHistory)); + } + + if (userPromptMessage.Role != AuthorRole.User) + { + throw new ArgumentException("Chat history must end with a user message", nameof(initialHistory)); + } + + var newLine = Environment.NewLine; + + var combinedSystemMessage = string.Join(newLine, systemMessages.Select(message => message.Content)); + return $"{combinedSystemMessage}{newLine}{newLine}User instructions:{newLine}{userPromptMessage.Content}"; + } + + private async Task<(ChatHistory ChatHistory, AIServiceUsage Usage)> ExecuteAICompletion(Kernel kernel, string prompt) + { + ChatHistory chatHistory = []; + + chatHistory.AddSystemMessage(""" + You are an agent who is tasked with helping users paste their clipboard data. You have functions available to help you with this task. + You never need to ask permission, always try to do as the user asks. The user will only input one message and will not be available for further questions, so try your best. + The user will put in a request to format their clipboard data and you will fulfill it. + You will not directly see the output clipboard content, and do not need to provide it in the chat. You just need to do the transform operations as needed. + If you are unable to fulfill the request, end with an error message in the language of the user's request. + """); + chatHistory.AddSystemMessage($"Available clipboard formats: {await kernel.GetDataFormatsAsync()}"); + chatHistory.AddUserMessage(prompt); + + await _promptModerationService.ValidateAsync(GetFullPrompt(chatHistory)); + + var chatResult = await kernel.GetRequiredService() + .GetChatMessageContentAsync(chatHistory, PromptExecutionSettings, kernel); + chatHistory.Add(chatResult); + + var totalUsage = chatHistory.Select(GetAIServiceUsage) + .Aggregate(AIServiceUsage.Add); + + return (chatHistory, totalUsage); + } + + private async Task<(ChatHistory ChatHistory, AIServiceUsage Usage)> ExecuteCachedActionChain(Kernel kernel, List actionChain) + { + foreach (var item in actionChain) + { + if (item.Arguments.Count > 0) + { + await ExecutePromptTransformAsync(kernel, item.Format, item.Arguments[PromptParameterName]); + } + else + { + await ExecuteStandardTransformAsync(kernel, item.Format); + } + } + + return ([], AIServiceUsage.None); + } + + private void LogResult(bool cacheUsed, bool isSavedQuery, IEnumerable actionChain, AIServiceUsage usage) + { + AdvancedPasteSemanticKernelFormatEvent telemetryEvent = new(cacheUsed, isSavedQuery, usage.PromptTokens, usage.CompletionTokens, ModelName, AdvancedPasteSemanticKernelFormatEvent.FormatActionChain(actionChain)); + PowerToysTelemetry.Log.WriteEvent(telemetryEvent); + + var logEvent = new { telemetryEvent.CacheUsed, telemetryEvent.IsSavedQuery, telemetryEvent.PromptTokens, telemetryEvent.CompletionTokens, telemetryEvent.ModelName, telemetryEvent.ActionChain }; + Logger.LogDebug($"{nameof(TransformClipboardAsync)} complete; {JsonSerializer.Serialize(logEvent)}"); + } + + private Kernel CreateKernel() + { + var kernelBuilder = Kernel.CreateBuilder(); + AddChatCompletionService(kernelBuilder); + kernelBuilder.Plugins.AddFromFunctions("Actions", GetKernelFunctions()); + return kernelBuilder.Build(); + } + + private IEnumerable GetKernelFunctions() => + from format in Enum.GetValues() + let metadata = PasteFormat.MetadataDict[format] + let coreDescription = metadata.KernelFunctionDescription + where !string.IsNullOrEmpty(coreDescription) + let requiresPrompt = metadata.RequiresPrompt + orderby requiresPrompt descending + select KernelFunctionFactory.CreateFromMethod( + method: requiresPrompt ? async (Kernel kernel, string prompt) => await ExecutePromptTransformAsync(kernel, format, prompt) + : async (Kernel kernel) => await ExecuteStandardTransformAsync(kernel, format), + functionName: format.ToString(), + description: requiresPrompt ? coreDescription : $"{coreDescription} Puts the result back on the clipboard.", + parameters: requiresPrompt ? [new(PromptParameterName) { Description = "Input instructions to AI", ParameterType = typeof(string) }] : null, + returnParameter: new() { Description = "Array of available clipboard formats after operation" }); + + private Task ExecutePromptTransformAsync(Kernel kernel, PasteFormats format, string prompt) => + ExecuteTransformAsync( + kernel, + new ActionChainItem(format, Arguments: new() { { PromptParameterName, prompt } }), + async dataPackageView => + { + var input = await dataPackageView.GetTextAsync(); + string output = await GetPromptBasedOutput(format, prompt, input); + return DataPackageHelpers.CreateFromText(output); + }); + + private async Task GetPromptBasedOutput(PasteFormats format, string prompt, string input) => + format switch + { + PasteFormats.CustomTextTransformation => await _customTextTransformService.TransformTextAsync(prompt, input), + _ => throw new ArgumentException($"Unsupported format {format} for prompt transform", nameof(format)), + }; + + private Task ExecuteStandardTransformAsync(Kernel kernel, PasteFormats format) => + ExecuteTransformAsync( + kernel, + new ActionChainItem(format, Arguments: []), + async dataPackageView => await TransformHelpers.TransformAsync(format, dataPackageView)); + + private static async Task ExecuteTransformAsync(Kernel kernel, ActionChainItem actionChainItem, Func> transformFunc) + { + kernel.GetOrAddActionChain().Add(actionChainItem); + kernel.SetLastError(null); + + try + { + var input = kernel.GetDataPackageView(); + var output = await transformFunc(input); + kernel.SetDataPackage(output); + return await kernel.GetDataFormatsAsync(); + } + catch (Exception ex) + { + kernel.SetLastError(ex); + throw; + } + } + + private string FormatChatHistory(ChatHistory chatHistory) => + chatHistory.Count == 0 ? "[No chat history]" : string.Join(Environment.NewLine, chatHistory.Select(FormatChatMessage)); + + private string FormatChatMessage(ChatMessageContent chatMessage) + { + static string Redact(object data) => +#if DEBUG + data?.ToString(); +#else + "[Redacted]"; +#endif + + static string FormatKernelArguments(KernelArguments kernelArguments) => + string.Join(", ", kernelArguments?.Select(argument => $"{argument.Key}: {Redact(argument.Value)}") ?? []); + + static string FormatKernelContent(KernelContent kernelContent) => +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + kernelContent switch + { + FunctionCallContent functionCallContent => $"{functionCallContent.FunctionName}({FormatKernelArguments(functionCallContent.Arguments)})", + FunctionResultContent functionResultContent => functionResultContent.FunctionName, + _ => kernelContent.ToString(), + }; +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + var role = chatMessage.Role; + var content = string.Join(" / ", chatMessage.Items.Select(FormatKernelContent)); + var redactedContent = role == AuthorRole.System || role == AuthorRole.Tool ? content : Redact(content); + var usage = GetAIServiceUsage(chatMessage); + var usageString = usage.HasUsage ? $" [{usage}]" : string.Empty; + return $"-> {role}: {redactedContent}{usageString}"; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs new file mode 100644 index 0000000000..c249771bd9 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/CustomTextTransformService.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Text.Json; +using System.Threading.Tasks; + +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using AdvancedPaste.Telemetry; +using Azure; +using Azure.AI.OpenAI; +using ManagedCommon; +using Microsoft.PowerToys.Telemetry; + +namespace AdvancedPaste.Services.OpenAI; + +public sealed class CustomTextTransformService(IAICredentialsProvider aiCredentialsProvider, IPromptModerationService promptModerationService) : ICustomTextTransformService +{ + private const string ModelName = "gpt-3.5-turbo-instruct"; + + private readonly IAICredentialsProvider _aiCredentialsProvider = aiCredentialsProvider; + private readonly IPromptModerationService _promptModerationService = promptModerationService; + + private async Task GetAICompletionAsync(string systemInstructions, string userMessage) + { + var fullPrompt = systemInstructions + "\n\n" + userMessage; + + await _promptModerationService.ValidateAsync(fullPrompt); + + OpenAIClient azureAIClient = new(_aiCredentialsProvider.Key); + + var response = await azureAIClient.GetCompletionsAsync( + new() + { + DeploymentName = ModelName, + Prompts = + { + fullPrompt, + }, + Temperature = 0.01F, + MaxTokens = 2000, + }); + + if (response.Value.Choices[0].FinishReason == "length") + { + Logger.LogDebug("Cut off due to length constraints"); + } + + return response; + } + + public async Task TransformTextAsync(string prompt, string inputText) + { + if (string.IsNullOrWhiteSpace(prompt)) + { + return string.Empty; + } + + if (string.IsNullOrWhiteSpace(inputText)) + { + Logger.LogWarning("Clipboard has no usable text data"); + return string.Empty; + } + + string systemInstructions = +$@"You are tasked with reformatting user's clipboard data. Use the user's instructions, and the content of their clipboard below to edit their clipboard content as they have requested it. +Do not output anything else besides the reformatted clipboard content."; + + string userMessage = +$@"User instructions: +{prompt} + +Clipboard Content: +{inputText} + +Output: +"; + + try + { + var response = await GetAICompletionAsync(systemInstructions, userMessage); + + var usage = response.Usage; + AdvancedPasteGenerateCustomFormatEvent telemetryEvent = new(usage.PromptTokens, usage.CompletionTokens, ModelName); + PowerToysTelemetry.Log.WriteEvent(telemetryEvent); + + var logEvent = new { telemetryEvent.PromptTokens, telemetryEvent.CompletionTokens, telemetryEvent.ModelName }; + Logger.LogDebug($"{nameof(TransformTextAsync)} complete; {JsonSerializer.Serialize(logEvent)}"); + + return response.Choices[0].Text; + } + catch (Exception ex) + { + Logger.LogError($"{nameof(TransformTextAsync)} failed", ex); + + AdvancedPasteGenerateCustomErrorEvent errorEvent = new(ex is PasteActionModeratedException ? PasteActionModeratedException.ErrorDescription : ex.Message); + PowerToysTelemetry.Log.WriteEvent(errorEvent); + + if (ex is PasteActionException) + { + throw; + } + else + { + throw new PasteActionException(ErrorHelpers.TranslateErrorText((ex as RequestFailedException)?.Status ?? -1), ex); + } + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs new file mode 100644 index 0000000000..b19a6d51cb --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/KernelService.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +using AdvancedPaste.Models; +using Azure.AI.OpenAI; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; + +namespace AdvancedPaste.Services.OpenAI; + +public sealed class KernelService(IKernelQueryCacheService queryCacheService, IAICredentialsProvider aiCredentialsProvider, IPromptModerationService promptModerationService, ICustomTextTransformService customTextTransformService) : + KernelServiceBase(queryCacheService, promptModerationService, customTextTransformService) +{ + private readonly IAICredentialsProvider _aiCredentialsProvider = aiCredentialsProvider; + + protected override string ModelName => "gpt-4o"; + + protected override PromptExecutionSettings PromptExecutionSettings => + new OpenAIPromptExecutionSettings() + { + ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions, + Temperature = 0.01, + }; + + protected override void AddChatCompletionService(IKernelBuilder kernelBuilder) => kernelBuilder.AddOpenAIChatCompletion(ModelName, _aiCredentialsProvider.Key); + + protected override AIServiceUsage GetAIServiceUsage(ChatMessageContent chatMessage) => + chatMessage.Metadata?.GetValueOrDefault("Usage") is CompletionsUsage completionsUsage + ? new(PromptTokens: completionsUsage.PromptTokens, CompletionTokens: completionsUsage.CompletionTokens) + : AIServiceUsage.None; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/PromptModerationService.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/PromptModerationService.cs new file mode 100644 index 0000000000..e78a44b533 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/PromptModerationService.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ClientModel; +using System.Threading.Tasks; + +using AdvancedPaste.Helpers; +using AdvancedPaste.Models; +using ManagedCommon; +using OpenAI.Moderations; + +namespace AdvancedPaste.Services.OpenAI; + +public sealed class PromptModerationService(IAICredentialsProvider aiCredentialsProvider) : IPromptModerationService +{ + private const string ModelName = "omni-moderation-latest"; + + private readonly IAICredentialsProvider _aiCredentialsProvider = aiCredentialsProvider; + + public async Task ValidateAsync(string fullPrompt) + { + try + { + ModerationClient moderationClient = new(ModelName, _aiCredentialsProvider.Key); + var moderationClientResult = await moderationClient.ClassifyTextAsync(fullPrompt); + var moderationResult = moderationClientResult.Value; + + Logger.LogDebug($"{nameof(PromptModerationService)}.{nameof(ValidateAsync)} complete; {nameof(moderationResult.Flagged)}={moderationResult.Flagged}"); + + if (moderationResult.Flagged) + { + throw new PasteActionModeratedException(); + } + } + catch (ClientResultException ex) + { + throw new PasteActionException(ErrorHelpers.TranslateErrorText(ex.Status), ex); + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/VaultCredentialsProvider.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/VaultCredentialsProvider.cs new file mode 100644 index 0000000000..169c1c2422 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/OpenAI/VaultCredentialsProvider.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +using Windows.Security.Credentials; + +namespace AdvancedPaste.Services.OpenAI; + +public sealed class VaultCredentialsProvider : IAICredentialsProvider +{ + public VaultCredentialsProvider() => Refresh(); + + public string Key { get; private set; } + + public bool IsConfigured => !string.IsNullOrEmpty(Key); + + public bool Refresh() + { + var oldKey = Key; + Key = LoadKey(); + return oldKey != Key; + } + + private static string LoadKey() + { + try + { + return new PasswordVault().Retrieve("https://platform.openai.com/api-keys", "PowerToys_AdvancedPaste_OpenAIKey")?.Password ?? string.Empty; + } + catch (Exception) + { + return string.Empty; + } + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs b/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs index f77cf8ef99..e7e7f9b4cf 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Services/PasteFormatExecutor.cs @@ -3,74 +3,41 @@ // See the LICENSE file in the project root for more information. using System; -using System.Globalization; -using System.IO; -using System.Net; using System.Threading.Tasks; + using AdvancedPaste.Helpers; using AdvancedPaste.Models; -using ManagedCommon; using Microsoft.PowerToys.Telemetry; using Windows.ApplicationModel.DataTransfer; -using Windows.Graphics.Imaging; -using Windows.Storage.Streams; namespace AdvancedPaste.Services; -public sealed class PasteFormatExecutor(AICompletionsHelper aiHelper) : IPasteFormatExecutor +public sealed class PasteFormatExecutor(IKernelService kernelService, ICustomTextTransformService customTextTransformService) : IPasteFormatExecutor { - private readonly AICompletionsHelper _aiHelper = aiHelper; + private readonly IKernelService _kernelService = kernelService; + private readonly ICustomTextTransformService _customTextTransformService = customTextTransformService; - public async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source) + public async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source) { if (!pasteFormat.IsEnabled) { return null; } - WriteTelemetry(pasteFormat.Format, source); + var format = pasteFormat.Format; - return await ExecutePasteFormatCoreAsync(pasteFormat, Clipboard.GetContent()); - } + WriteTelemetry(format, source); - private async Task ExecutePasteFormatCoreAsync(PasteFormat pasteFormat, DataPackageView clipboardData) - { - switch (pasteFormat.Format) - { - case PasteFormats.PlainText: - ToPlainText(clipboardData); - return null; + var clipboardData = Clipboard.GetContent(); - case PasteFormats.Markdown: - ToMarkdown(clipboardData); - return null; - - case PasteFormats.Json: - ToJson(clipboardData); - return null; - - case PasteFormats.ImageToText: - await ImageToTextAsync(clipboardData); - return null; - - case PasteFormats.PasteAsTxtFile: - await ToTxtFileAsync(clipboardData); - return null; - - case PasteFormats.PasteAsPngFile: - await ToPngFileAsync(clipboardData); - return null; - - case PasteFormats.PasteAsHtmlFile: - await ToHtmlFileAsync(clipboardData); - return null; - - case PasteFormats.Custom: - return await ToCustomAsync(pasteFormat.Prompt, clipboardData); - - default: - throw new ArgumentException($"Unknown paste format {pasteFormat.Format}", nameof(pasteFormat)); - } + // Run on thread-pool; although we use Async routines consistently, some actions still occasionally take a long time without yielding. + return await Task.Run(async () => + pasteFormat.Format switch + { + PasteFormats.KernelQuery => await _kernelService.TransformClipboardAsync(pasteFormat.Prompt, clipboardData, pasteFormat.IsSavedQuery), + PasteFormats.CustomTextTransformation => DataPackageHelpers.CreateFromText(await _customTextTransformService.TransformTextAsync(pasteFormat.Prompt, await clipboardData.GetTextAsync())), + _ => await TransformHelpers.TransformAsync(format, clipboardData), + }); } private static void WriteTelemetry(PasteFormats format, PasteActionSource source) @@ -93,161 +60,4 @@ public sealed class PasteFormatExecutor(AICompletionsHelper aiHelper) : IPasteFo throw new ArgumentOutOfRangeException(nameof(format)); } } - - private void ToPlainText(DataPackageView clipboardData) - { - Logger.LogTrace(); - SetClipboardTextContent(MarkdownHelper.PasteAsPlainTextFromClipboard(clipboardData)); - } - - private void ToMarkdown(DataPackageView clipboardData) - { - Logger.LogTrace(); - SetClipboardTextContent(MarkdownHelper.ToMarkdown(clipboardData)); - } - - private void ToJson(DataPackageView clipboardData) - { - Logger.LogTrace(); - SetClipboardTextContent(JsonHelper.ToJsonFromXmlOrCsv(clipboardData)); - } - - private async Task ImageToTextAsync(DataPackageView clipboardData) - { - Logger.LogTrace(); - - var bitmap = await ClipboardHelper.GetClipboardImageContentAsync(clipboardData); - var text = await OcrHelpers.ExtractTextAsync(bitmap); - SetClipboardTextContent(text); - } - - private async Task ToPngFileAsync(DataPackageView clipboardData) - { - Logger.LogTrace(); - - var clipboardBitmap = await ClipboardHelper.GetClipboardImageContentAsync(clipboardData); - - using var pngStream = new InMemoryRandomAccessStream(); - var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, pngStream); - encoder.SetSoftwareBitmap(clipboardBitmap); - await encoder.FlushAsync(); - - await SetClipboardFileContentAsync(pngStream.AsStreamForRead(), "png"); - } - - private async Task ToTxtFileAsync(DataPackageView clipboardData) - { - Logger.LogTrace(); - - var text = await ClipboardHelper.GetClipboardTextOrHtmlTextAsync(clipboardData); - await SetClipboardFileContentAsync(text, "txt"); - } - - private async Task ToHtmlFileAsync(DataPackageView clipboardData) - { - Logger.LogTrace(); - - var cfHtml = await ClipboardHelper.GetClipboardHtmlContentAsync(clipboardData); - var html = RemoveHtmlMetadata(cfHtml); - - await SetClipboardFileContentAsync(html, "html"); - } - - /// - /// Removes leading CF_HTML metadata from HTML clipboard data. - /// See: https://learn.microsoft.com/en-us/windows/win32/dataxchg/html-clipboard-format - /// - private static string RemoveHtmlMetadata(string cfHtml) - { - int? GetIntTagValue(string tagName) - { - var tagNameWithColon = tagName + ":"; - int tagStartPos = cfHtml.IndexOf(tagNameWithColon, StringComparison.InvariantCulture); - - const int tagValueLength = 10; - return tagStartPos != -1 && int.TryParse(cfHtml.AsSpan(tagStartPos + tagNameWithColon.Length, tagValueLength), CultureInfo.InvariantCulture, out int result) ? result : null; - } - - var startFragmentIndex = GetIntTagValue("StartFragment"); - var endFragmentIndex = GetIntTagValue("EndFragment"); - - return (startFragmentIndex == null || endFragmentIndex == null) ? cfHtml : cfHtml[startFragmentIndex.Value..endFragmentIndex.Value]; - } - - private static async Task SetClipboardFileContentAsync(string data, string fileExtension) - { - if (string.IsNullOrEmpty(data)) - { - throw new ArgumentException($"Empty value in {nameof(SetClipboardFileContentAsync)}", nameof(data)); - } - - var path = GetPasteAsFileTempFilePath(fileExtension); - - await File.WriteAllTextAsync(path, data); - await ClipboardHelper.SetClipboardFileContentAsync(path); - } - - private static async Task SetClipboardFileContentAsync(Stream stream, string fileExtension) - { - var path = GetPasteAsFileTempFilePath(fileExtension); - - using var fileStream = File.Create(path); - await stream.CopyToAsync(fileStream); - - await ClipboardHelper.SetClipboardFileContentAsync(path); - } - - private static string GetPasteAsFileTempFilePath(string fileExtension) - { - var prefix = ResourceLoaderInstance.ResourceLoader.GetString("PasteAsFile_FilePrefix"); - var timestamp = DateTime.Now.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture); - - return Path.Combine(Path.GetTempPath(), $"{prefix}{timestamp}.{fileExtension}"); - } - - private async Task ToCustomAsync(string prompt, DataPackageView clipboardData) - { - Logger.LogTrace(); - - if (string.IsNullOrWhiteSpace(prompt)) - { - return string.Empty; - } - - if (!clipboardData.Contains(StandardDataFormats.Text)) - { - Logger.LogWarning("Clipboard does not contain text data"); - return string.Empty; - } - - var currentClipboardText = await clipboardData.GetTextAsync(); - - if (string.IsNullOrWhiteSpace(currentClipboardText)) - { - Logger.LogWarning("Clipboard has no usable text data"); - return string.Empty; - } - - var aiResponse = await Task.Run(() => _aiHelper.AIFormatString(prompt, currentClipboardText)); - - return aiResponse.ApiRequestStatus == (int)HttpStatusCode.OK - ? aiResponse.Response - : throw new PasteActionException(TranslateErrorText(aiResponse.ApiRequestStatus)); - } - - private void SetClipboardTextContent(string content) - { - if (!string.IsNullOrEmpty(content)) - { - ClipboardHelper.SetClipboardTextContent(content); - } - } - - private static string TranslateErrorText(int apiRequestStatus) => (HttpStatusCode)apiRequestStatus switch - { - HttpStatusCode.TooManyRequests => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyTooManyRequests"), - HttpStatusCode.Unauthorized => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyUnauthorized"), - HttpStatusCode.OK => string.Empty, - _ => ResourceLoaderInstance.ResourceLoader.GetString("OpenAIApiKeyError") + apiRequestStatus.ToString(CultureInfo.InvariantCulture), - }; } diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw b/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw index 0ec81547e2..bbd63e916d 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw +++ b/src/modules/AdvancedPaste/AdvancedPaste/Strings/en-us/Resources.resw @@ -123,9 +123,6 @@ Clipboard does not contain any usable formats - - Clipboard data is not text - To custom with AI is not enabled @@ -140,7 +137,10 @@ An error occurred during the paste operation - + + + The paste operation was moderated due to sensitive content. Please try another query. + Clipboard history @@ -213,6 +213,9 @@ Open settings + + The AI assistant provided the following message: + Thumbs down feedback @@ -248,5 +251,5 @@ PowerToys_Paste_ - + \ No newline at end of file diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelErrorEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelErrorEvent.cs new file mode 100644 index 0000000000..425cf5dd4e --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelErrorEvent.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.Tracing; + +using Microsoft.PowerToys.Telemetry; +using Microsoft.PowerToys.Telemetry.Events; + +namespace AdvancedPaste.Telemetry; + +[EventData] +public class AdvancedPasteSemanticKernelErrorEvent(string error) : EventBase, IEvent +{ + public string Error { get; set; } = error; + + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs new file mode 100644 index 0000000000..75467dae70 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteSemanticKernelFormatEvent.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Linq; + +using AdvancedPaste.Models; +using Microsoft.PowerToys.Telemetry; +using Microsoft.PowerToys.Telemetry.Events; + +namespace AdvancedPaste.Telemetry; + +[EventData] +public class AdvancedPasteSemanticKernelFormatEvent(bool cacheUsed, bool isSavedQuery, int promptTokens, int completionTokens, string modelName, string actionChain) : EventBase, IEvent +{ + public static string FormatActionChain(IEnumerable actionChain) => FormatActionChain(actionChain.Select(item => item.Format)); + + public static string FormatActionChain(IEnumerable actionChain) => string.Join(", ", actionChain); + + public bool IsSavedQuery { get; set; } = isSavedQuery; + + public bool CacheUsed { get; set; } = cacheUsed; + + public int PromptTokens { get; set; } = promptTokens; + + public int CompletionTokens { get; set; } = completionTokens; + + public string ModelName { get; set; } = modelName; + + /// + /// Gets or sets a comma-separated list of paste formats used - in the same order they were executed. + /// Conceptually an array but formatted this way to work around https://github.com/dotnet/runtime/issues/10428 + /// + public string ActionChain { get; set; } = actionChain; + + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs b/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs index f2e461e47f..30e4a6f359 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/ViewModels/OptionsViewModel.cs @@ -5,6 +5,8 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO.Abstractions; using System.Linq; using System.Threading.Tasks; @@ -33,28 +35,29 @@ namespace AdvancedPaste.ViewModels private readonly DispatcherTimer _clipboardTimer; private readonly IUserSettings _userSettings; private readonly IPasteFormatExecutor _pasteFormatExecutor; - private readonly AICompletionsHelper _aiHelper; + private readonly IAICredentialsProvider _aiCredentialsProvider; public DataPackageView ClipboardData { get; set; } [ObservableProperty] - [NotifyPropertyChangedFor(nameof(IsCustomAIEnabled))] + [NotifyPropertyChangedFor(nameof(IsCustomAIAvailable))] [NotifyPropertyChangedFor(nameof(ClipboardHasData))] + [NotifyPropertyChangedFor(nameof(ClipboardHasDataForCustomAI))] [NotifyPropertyChangedFor(nameof(InputTxtBoxPlaceholderText))] - [NotifyPropertyChangedFor(nameof(AIDisabledErrorText))] + [NotifyPropertyChangedFor(nameof(CustomAIUnavailableErrorText))] private ClipboardFormat _availableClipboardFormats; [ObservableProperty] private bool _clipboardHistoryEnabled; [ObservableProperty] - [NotifyPropertyChangedFor(nameof(AIDisabledErrorText))] - [NotifyPropertyChangedFor(nameof(IsAIServiceEnabled))] - [NotifyPropertyChangedFor(nameof(IsCustomAIEnabled))] + [NotifyPropertyChangedFor(nameof(CustomAIUnavailableErrorText))] + [NotifyPropertyChangedFor(nameof(IsCustomAIServiceEnabled))] + [NotifyPropertyChangedFor(nameof(IsCustomAIAvailable))] private bool _isAllowedByGPO; [ObservableProperty] - private string _pasteOperationErrorText; + private PasteActionError _pasteActionError = PasteActionError.None; [ObservableProperty] private string _query = string.Empty; @@ -68,21 +71,25 @@ namespace AdvancedPaste.ViewModels public ObservableCollection CustomActionPasteFormats { get; } = []; - public bool IsAIServiceEnabled => IsAllowedByGPO && _aiHelper.IsAIEnabled; + public bool IsCustomAIServiceEnabled => IsAllowedByGPO && _aiCredentialsProvider.IsConfigured; - public bool IsCustomAIEnabled => IsAIServiceEnabled && ClipboardHasText; + public bool IsCustomAIAvailable => IsCustomAIServiceEnabled && ClipboardHasDataForCustomAI; + + public bool IsAdvancedAIEnabled => IsCustomAIServiceEnabled && _userSettings.IsAdvancedAIEnabled; public bool ClipboardHasData => AvailableClipboardFormats != ClipboardFormat.None; - private bool ClipboardHasText => AvailableClipboardFormats.HasFlag(ClipboardFormat.Text); + public bool ClipboardHasDataForCustomAI => PasteFormat.SupportsClipboardFormats(CustomAIFormat, AvailableClipboardFormats); + + private PasteFormats CustomAIFormat => _userSettings.IsAdvancedAIEnabled ? PasteFormats.KernelQuery : PasteFormats.CustomTextTransformation; private bool Visible => GetMainWindow()?.Visible is true; - public event EventHandler CustomActionActivated; + public event EventHandler PreviewRequested; - public OptionsViewModel(AICompletionsHelper aiHelper, IUserSettings userSettings, IPasteFormatExecutor pasteFormatExecutor) + public OptionsViewModel(IFileSystem fileSystem, IAICredentialsProvider aiCredentialsProvider, IUserSettings userSettings, IPasteFormatExecutor pasteFormatExecutor) { - _aiHelper = aiHelper; + _aiCredentialsProvider = aiCredentialsProvider; _userSettings = userSettings; _pasteFormatExecutor = pasteFormatExecutor; @@ -100,16 +107,25 @@ namespace AdvancedPaste.ViewModels _clipboardTimer.Start(); RefreshPasteFormats(); - _userSettings.Changed += (_, _) => EnqueueRefreshPasteFormats(); + _userSettings.Changed += UserSettings_Changed; PropertyChanged += (_, e) => { - string[] dirtyingProperties = [nameof(Query), nameof(IsAIServiceEnabled), nameof(IsCustomAIEnabled), nameof(AvailableClipboardFormats)]; + string[] dirtyingProperties = [nameof(Query), nameof(IsCustomAIServiceEnabled), nameof(IsCustomAIAvailable), nameof(AvailableClipboardFormats)]; if (dirtyingProperties.Contains(e.PropertyName)) { EnqueueRefreshPasteFormats(); } }; + + try + { + // Delete file that is no longer needed but might have been written by previous version and contain sensitive information. + fileSystem.File.Delete(new SettingsUtils(fileSystem).GetSettingsFilePath(Constants.AdvancedPasteModuleName, "lastQuery.json")); + } + catch + { + } } private static MainWindow GetMainWindow() => (App.Current as App)?.GetMainWindow(); @@ -123,6 +139,15 @@ namespace AdvancedPaste.ViewModels } } + private void UserSettings_Changed(object sender, EventArgs e) + { + OnPropertyChanged(nameof(ClipboardHasDataForCustomAI)); + OnPropertyChanged(nameof(IsCustomAIAvailable)); + OnPropertyChanged(nameof(IsAdvancedAIEnabled)); + + EnqueueRefreshPasteFormats(); + } + private void EnqueueRefreshPasteFormats() { if (_pasteFormatsDirty) @@ -138,9 +163,11 @@ namespace AdvancedPaste.ViewModels }); } - private PasteFormat CreatePasteFormat(PasteFormats format) => new(format, AvailableClipboardFormats, IsAIServiceEnabled, ResourceLoaderInstance.ResourceLoader.GetString); + private PasteFormat CreateStandardPasteFormat(PasteFormats format) => + PasteFormat.CreateStandardFormat(format, AvailableClipboardFormats, IsCustomAIServiceEnabled, ResourceLoaderInstance.ResourceLoader.GetString); - private PasteFormat CreatePasteFormat(AdvancedPasteCustomAction customAction) => new(customAction, AvailableClipboardFormats, IsAIServiceEnabled); + private PasteFormat CreateCustomAIPasteFormat(string name, string prompt, bool isSavedQuery) => + PasteFormat.CreateCustomAIFormat(CustomAIFormat, name, prompt, isSavedQuery, AvailableClipboardFormats, IsCustomAIServiceEnabled); private void RefreshPasteFormats() { @@ -177,9 +204,11 @@ namespace AdvancedPaste.ViewModels UpdateFormats(StandardPasteFormats, Enum.GetValues() .Where(format => PasteFormat.MetadataDict[format].IsCoreAction || _userSettings.AdditionalActions.Contains(format)) - .Select(CreatePasteFormat)); + .Select(CreateStandardPasteFormat)); - UpdateFormats(CustomActionPasteFormats, IsAIServiceEnabled ? _userSettings.CustomActions.Select(CreatePasteFormat) : []); + UpdateFormats( + CustomActionPasteFormats, + IsCustomAIServiceEnabled ? _userSettings.CustomActions.Select(customAction => CreateCustomAIPasteFormat(customAction.Name, customAction.Prompt, isSavedQuery: true)) : []); } public void Dispose() @@ -196,12 +225,12 @@ namespace AdvancedPaste.ViewModels } ClipboardData = Clipboard.GetContent(); - AvailableClipboardFormats = await ClipboardHelper.GetAvailableClipboardFormatsAsync(ClipboardData); + AvailableClipboardFormats = await ClipboardData.GetAvailableFormatsAsync(); } public async Task OnShowAsync() { - PasteOperationErrorText = string.Empty; + PasteActionError = PasteActionError.None; Query = string.Empty; await ReadClipboardAsync(); @@ -212,11 +241,12 @@ namespace AdvancedPaste.ViewModels _dispatcherQueue.TryEnqueue(() => { - GetMainWindow()?.FinishLoading(_aiHelper.IsAIEnabled); + GetMainWindow()?.FinishLoading(_aiCredentialsProvider.IsConfigured); OnPropertyChanged(nameof(InputTxtBoxPlaceholderText)); - OnPropertyChanged(nameof(AIDisabledErrorText)); - OnPropertyChanged(nameof(IsAIServiceEnabled)); - OnPropertyChanged(nameof(IsCustomAIEnabled)); + OnPropertyChanged(nameof(CustomAIUnavailableErrorText)); + OnPropertyChanged(nameof(IsCustomAIServiceEnabled)); + OnPropertyChanged(nameof(IsAdvancedAIEnabled)); + OnPropertyChanged(nameof(IsCustomAIAvailable)); }); } @@ -251,24 +281,24 @@ namespace AdvancedPaste.ViewModels public string InputTxtBoxPlaceholderText => ResourceLoaderInstance.ResourceLoader.GetString(ClipboardHasData ? "CustomFormatTextBox/PlaceholderText" : "ClipboardEmptyWarning"); - public string AIDisabledErrorText + public string CustomAIUnavailableErrorText { get { - if (!ClipboardHasText) - { - return ResourceLoaderInstance.ResourceLoader.GetString("ClipboardDataNotTextWarning"); - } - if (!IsAllowedByGPO) { return ResourceLoaderInstance.ResourceLoader.GetString("OpenAIGpoDisabled"); } - if (!_aiHelper.IsAIEnabled) + if (!_aiCredentialsProvider.IsConfigured) { return ResourceLoaderInstance.ResourceLoader.GetString("OpenAINotConfigured"); } + + if (!ClipboardHasDataForCustomAI) + { + return ResourceLoaderInstance.ResourceLoader.GetString("ClipboardEmptyWarning"); + } else { return string.Empty; @@ -280,24 +310,22 @@ namespace AdvancedPaste.ViewModels private string _customFormatResult; [RelayCommand] - public void PasteCustom() + public async Task PasteCustomAsync() { var text = GeneratedResponses.ElementAtOrDefault(CurrentResponseIndex); if (!string.IsNullOrEmpty(text)) { - ClipboardHelper.SetClipboardTextContent(text); - HideWindow(); - - if (_userSettings.SendPasteKeyCombination) - { - ClipboardHelper.SendPasteKeyCombination(); - } - - Query = string.Empty; + await CopyPasteAndHideAsync(DataPackageHelpers.CreateFromText(text)); } } + private async Task CopyPasteAndHideAsync(DataPackage package) + { + await ClipboardHelper.TryCopyPasteAsync(package, HideWindow); + Query = string.Empty; + } + // Command to select the previous custom format [RelayCommand] public void PreviousCustomFormat() @@ -329,7 +357,7 @@ namespace AdvancedPaste.ViewModels internal async Task ExecutePasteFormatAsync(PasteFormats format, PasteActionSource source) { await ReadClipboardAsync(); - await ExecutePasteFormatAsync(CreatePasteFormat(format), source); + await ExecutePasteFormatAsync(CreateStandardPasteFormat(format), source); } internal async Task ExecutePasteFormatAsync(PasteFormat pasteFormat, PasteActionSource source) @@ -342,59 +370,49 @@ namespace AdvancedPaste.ViewModels if (!pasteFormat.IsEnabled) { - var resourceId = pasteFormat.SupportsClipboardFormats(AvailableClipboardFormats) ? "PasteError" : "ClipboardEmptyWarning"; - PasteOperationErrorText = ResourceLoaderInstance.ResourceLoader.GetString(resourceId); + PasteActionError = PasteActionError.FromResourceId(pasteFormat.SupportsClipboardFormats(AvailableClipboardFormats) ? "PasteError" : "ClipboardEmptyWarning"); return; } - Busy = true; - PasteOperationErrorText = string.Empty; - Query = pasteFormat.Query; + var elapsedWatch = Stopwatch.StartNew(); + Logger.LogDebug($"Started executing {pasteFormat.Format} from source {source}"); - if (pasteFormat.Format == PasteFormats.Custom) - { - SaveQuery(Query); - } + Busy = true; + PasteActionError = PasteActionError.None; + Query = pasteFormat.Query; try { // Minimum time to show busy spinner for AI actions when triggered by global keyboard shortcut. var aiActionMinTaskTime = TimeSpan.FromSeconds(2); var delayTask = (Visible && source == PasteActionSource.GlobalKeyboardShortcut) ? Task.Delay(aiActionMinTaskTime) : Task.CompletedTask; - var aiOutput = await _pasteFormatExecutor.ExecutePasteFormatAsync(pasteFormat, source); + var dataPackage = await _pasteFormatExecutor.ExecutePasteFormatAsync(pasteFormat, source); await delayTask; - if (pasteFormat.Format != PasteFormats.Custom) - { - HideWindow(); + var outputText = await dataPackage.GetView().GetTextOrEmptyAsync(); + bool shouldPreview = pasteFormat.Metadata.CanPreview && _userSettings.ShowCustomPreview && !string.IsNullOrEmpty(outputText) && source != PasteActionSource.GlobalKeyboardShortcut; - if (source == PasteActionSource.GlobalKeyboardShortcut || _userSettings.SendPasteKeyCombination) - { - ClipboardHelper.SendPasteKeyCombination(); - } + if (shouldPreview) + { + GeneratedResponses.Add(outputText); + CurrentResponseIndex = GeneratedResponses.Count - 1; + PreviewRequested?.Invoke(this, EventArgs.Empty); } else { - var pasteResult = source == PasteActionSource.GlobalKeyboardShortcut || !_userSettings.ShowCustomPreview; - - GeneratedResponses.Add(aiOutput); - CurrentResponseIndex = GeneratedResponses.Count - 1; - CustomActionActivated?.Invoke(this, new CustomActionActivatedEventArgs(pasteFormat.Prompt, pasteResult)); - - if (pasteResult) - { - PasteCustom(); - } + await CopyPasteAndHideAsync(dataPackage); } } catch (Exception ex) { Logger.LogError("Error executing paste format", ex); - PasteOperationErrorText = ex is PasteActionException ? ex.Message : ResourceLoaderInstance.ResourceLoader.GetString("PasteError"); + PasteActionError = PasteActionError.FromException(ex); } Busy = false; + elapsedWatch.Stop(); + Logger.LogDebug($"Finished executing {pasteFormat.Format} from source {source}; timeTakenMs={elapsedWatch.ElapsedMilliseconds}"); } internal async Task ExecutePasteFormatAsync(VirtualKey key) @@ -413,20 +431,21 @@ namespace AdvancedPaste.ViewModels { Logger.LogTrace(); - await ReadClipboardAsync(); - var customAction = _userSettings.CustomActions.FirstOrDefault(customAction => customAction.Id == customActionId); if (customAction != null) { - await ExecutePasteFormatAsync(CreatePasteFormat(customAction), source); + await ReadClipboardAsync(); + await ExecutePasteFormatAsync(CreateCustomAIPasteFormat(customAction.Name, customAction.Prompt, isSavedQuery: true), source); } } - internal async Task GenerateCustomFunctionAsync(PasteActionSource triggerSource) + internal async Task ExecuteCustomAIFormatFromCurrentQueryAsync(PasteActionSource triggerSource) { - AdvancedPasteCustomAction customAction = new() { Name = "Default", Prompt = Query }; - await ExecutePasteFormatAsync(CreatePasteFormat(customAction), triggerSource); + var customAction = _userSettings.CustomActions + .FirstOrDefault(customAction => Models.KernelQueryCache.CacheKey.PromptComparer.Equals(customAction.Prompt, Query)); + + await ExecutePasteFormatAsync(CreateCustomAIPasteFormat(customAction?.Name ?? "Default", Query, isSavedQuery: customAction != null), triggerSource); } private void HideWindow() @@ -440,42 +459,6 @@ namespace AdvancedPaste.ViewModels } } - internal CustomQuery RecallPreviousCustomQuery() - { - return LoadPreviousQuery(); - } - - internal void SaveQuery(string inputQuery) - { - Logger.LogTrace(); - - DataPackageView clipboardData = Clipboard.GetContent(); - - if (clipboardData == null || !clipboardData.Contains(StandardDataFormats.Text)) - { - Logger.LogWarning("Clipboard does not contain text data"); - return; - } - - var currentClipboardText = Task.Run(async () => await clipboardData.GetTextAsync()).Result; - - var queryData = new CustomQuery - { - Query = inputQuery, - ClipboardData = currentClipboardText, - }; - - SettingsUtils utils = new(); - utils.SaveSettings(queryData.ToString(), Constants.AdvancedPasteModuleName, Constants.LastQueryJsonFileName); - } - - internal CustomQuery LoadPreviousQuery() - { - SettingsUtils utils = new(); - var query = utils.GetSettings(Constants.AdvancedPasteModuleName, Constants.LastQueryJsonFileName); - return query; - } - private bool IsClipboardHistoryEnabled() { string registryKey = @"HKEY_CURRENT_USER\Software\Microsoft\Clipboard\"; @@ -499,15 +482,7 @@ namespace AdvancedPaste.ViewModels { UpdateAllowedByGPO(); - if (IsAllowedByGPO) - { - var oldKey = _aiHelper.GetKey(); - var newKey = AICompletionsHelper.LoadOpenAIKey(); - _aiHelper.SetOpenAIKey(newKey); - return newKey != oldKey; - } - - return false; + return IsAllowedByGPO && _aiCredentialsProvider.Refresh(); } } } diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp index bef34c4cd4..9fd120dca0 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/dllmain.cpp @@ -55,6 +55,7 @@ namespace const wchar_t JSON_KEY_ADVANCED_PASTE_UI_HOTKEY[] = L"advanced-paste-ui-hotkey"; const wchar_t JSON_KEY_PASTE_AS_MARKDOWN_HOTKEY[] = L"paste-as-markdown-hotkey"; const wchar_t JSON_KEY_PASTE_AS_JSON_HOTKEY[] = L"paste-as-json-hotkey"; + const wchar_t JSON_KEY_IS_ADVANCED_AI_ENABLED[] = L"IsAdvancedAIEnabled"; const wchar_t JSON_KEY_SHOW_CUSTOM_PREVIEW[] = L"ShowCustomPreview"; const wchar_t JSON_KEY_VALUE[] = L"value"; @@ -99,6 +100,7 @@ private: using CustomAction = ActionData; std::vector m_custom_actions; + bool m_is_advanced_ai_enabled = false; bool m_preview_custom_format_output = true; Hotkey parse_single_hotkey(const wchar_t* keyName, const winrt::Windows::Data::Json::JsonObject& settingsObject) @@ -268,9 +270,9 @@ private: } } - void parse_hotkeys(PowerToysSettings::PowerToyValues& settings) + void read_settings(PowerToysSettings::PowerToyValues& settings) { - auto settingsObject = settings.get_raw_json(); + const auto settingsObject = settings.get_raw_json(); // Migrate Paste As Plain text shortcut Hotkey old_paste_as_plain_hotkey; @@ -352,6 +354,21 @@ private: } } } + + if (settingsObject.GetView().Size()) + { + const auto propertiesObject = settingsObject.GetNamedObject(JSON_KEY_PROPERTIES); + + if (propertiesObject.HasKey(JSON_KEY_IS_ADVANCED_AI_ENABLED)) + { + m_is_advanced_ai_enabled = propertiesObject.GetNamedObject(JSON_KEY_IS_ADVANCED_AI_ENABLED).GetNamedBoolean(JSON_KEY_VALUE); + } + + if (propertiesObject.HasKey(JSON_KEY_SHOW_CUSTOM_PREVIEW)) + { + m_preview_custom_format_output = propertiesObject.GetNamedObject(JSON_KEY_SHOW_CUSTOM_PREVIEW).GetNamedBoolean(JSON_KEY_VALUE); + } + } } bool is_process_running() const @@ -441,13 +458,7 @@ private: PowerToysSettings::PowerToyValues settings = PowerToysSettings::PowerToyValues::load_from_settings_file(get_key()); - parse_hotkeys(settings); - - auto settingsObject = settings.get_raw_json(); - if (settingsObject.GetView().Size() && settingsObject.GetNamedObject(JSON_KEY_PROPERTIES).HasKey(JSON_KEY_SHOW_CUSTOM_PREVIEW)) - { - m_preview_custom_format_output = settingsObject.GetNamedObject(JSON_KEY_PROPERTIES).GetNamedObject(JSON_KEY_SHOW_CUSTOM_PREVIEW).GetNamedBoolean(JSON_KEY_VALUE); - } + read_settings(settings); } catch (std::exception&) { @@ -809,13 +820,7 @@ public: PowerToysSettings::PowerToyValues values = PowerToysSettings::PowerToyValues::from_json_string(config, get_key()); - parse_hotkeys(values); - - const auto settingsObject = values.get_raw_json(); - if (settingsObject.GetView().Size() && settingsObject.GetNamedObject(JSON_KEY_PROPERTIES).HasKey(JSON_KEY_SHOW_CUSTOM_PREVIEW)) - { - m_preview_custom_format_output = settingsObject.GetNamedObject(JSON_KEY_PROPERTIES).GetNamedObject(JSON_KEY_SHOW_CUSTOM_PREVIEW).GetNamedBoolean(JSON_KEY_VALUE); - } + read_settings(values); std::unordered_map additionalActionMap; for (const auto& action : m_additional_actions) @@ -828,6 +833,7 @@ public: m_advanced_paste_ui_hotkey, m_paste_as_markdown_hotkey, m_paste_as_json_hotkey, + m_is_advanced_ai_enabled, m_preview_custom_format_output, additionalActionMap); diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/trace.cpp b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/trace.cpp index aa6162c465..c5af4f231e 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/trace.cpp +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/trace.cpp @@ -48,6 +48,7 @@ void Trace::AdvancedPaste_SettingsTelemetry(const PowertoyModuleIface::Hotkey& p const PowertoyModuleIface::Hotkey& advancedPasteUIHotkey, const PowertoyModuleIface::Hotkey& pasteMarkdownHotkey, const PowertoyModuleIface::Hotkey& pasteJsonHotkey, + const bool is_advanced_ai_enabled, const bool preview_custom_format_output, const std::unordered_map& additionalActionsHotkeys) noexcept { @@ -82,6 +83,7 @@ void Trace::AdvancedPaste_SettingsTelemetry(const PowertoyModuleIface::Hotkey& p TraceLoggingWideString(getHotkeyCStr(advancedPasteUIHotkey), "AdvancedPasteUIHotkey"), TraceLoggingWideString(getHotkeyCStr(pasteMarkdownHotkey), "PasteMarkdownHotkey"), TraceLoggingWideString(getHotkeyCStr(pasteJsonHotkey), "PasteJsonHotkey"), + TraceLoggingBoolean(is_advanced_ai_enabled, "IsAdvancedAIEnabled"), TraceLoggingBoolean(preview_custom_format_output, "ShowCustomPreview"), TraceLoggingWideString(getAdditionalActionHotkeyCStr(L"ImageToText"), "ImageToTextHotkey"), TraceLoggingWideString(getAdditionalActionHotkeyCStr(L"PasteAsTxtFile"), "PasteAsTxtFileHotkey"), diff --git a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/trace.h b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/trace.h index c7cee38877..7ef6250baa 100644 --- a/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/trace.h +++ b/src/modules/AdvancedPaste/AdvancedPasteModuleInterface/trace.h @@ -20,6 +20,7 @@ public: const PowertoyModuleIface::Hotkey& advancedPasteUIHotkey, const PowertoyModuleIface::Hotkey& pasteMarkdownHotkey, const PowertoyModuleIface::Hotkey& pasteJsonHotkey, + const bool is_advanced_ai_enabled, const bool preview_custom_format_output, const std::unordered_map& additionalActionsHotkeys) noexcept; }; diff --git a/src/modules/MouseWithoutBorders/App/Helper/MouseWithoutBordersHelper.csproj b/src/modules/MouseWithoutBorders/App/Helper/MouseWithoutBordersHelper.csproj index 705340f39f..f97c0c44f3 100644 --- a/src/modules/MouseWithoutBorders/App/Helper/MouseWithoutBordersHelper.csproj +++ b/src/modules/MouseWithoutBorders/App/Helper/MouseWithoutBordersHelper.csproj @@ -59,6 +59,8 @@ + + diff --git a/src/modules/MouseWithoutBorders/App/MouseWithoutBorders.csproj b/src/modules/MouseWithoutBorders/App/MouseWithoutBorders.csproj index 8bb21597d5..4c2e35c6ad 100644 --- a/src/modules/MouseWithoutBorders/App/MouseWithoutBorders.csproj +++ b/src/modules/MouseWithoutBorders/App/MouseWithoutBorders.csproj @@ -210,6 +210,8 @@ + + diff --git a/src/modules/MouseWithoutBorders/App/Service/MouseWithoutBordersService.csproj b/src/modules/MouseWithoutBorders/App/Service/MouseWithoutBordersService.csproj index 3925634618..1431445733 100644 --- a/src/modules/MouseWithoutBorders/App/Service/MouseWithoutBordersService.csproj +++ b/src/modules/MouseWithoutBorders/App/Service/MouseWithoutBordersService.csproj @@ -64,6 +64,8 @@ + + diff --git a/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs b/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs index 8322302ddf..d40bd686d3 100644 --- a/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs +++ b/src/settings-ui/Settings.UI.Library/AdvancedPasteProperties.cs @@ -23,17 +23,16 @@ namespace Microsoft.PowerToys.Settings.UI.Library PasteAsJsonShortcut = new(); CustomActions = new(); AdditionalActions = new(); + IsAdvancedAIEnabled = false; ShowCustomPreview = true; - SendPasteKeyCombination = true; CloseAfterLosingFocus = false; } [JsonConverter(typeof(BoolPropertyJsonConverter))] - public bool ShowCustomPreview { get; set; } + public bool IsAdvancedAIEnabled { get; set; } [JsonConverter(typeof(BoolPropertyJsonConverter))] - [CmdConfigureIgnore] - public bool SendPasteKeyCombination { get; set; } + public bool ShowCustomPreview { get; set; } [JsonConverter(typeof(BoolPropertyJsonConverter))] public bool CloseAfterLosingFocus { get; set; } diff --git a/src/settings-ui/Settings.UI.Library/Utilities/Helper.cs b/src/settings-ui/Settings.UI.Library/Utilities/Helper.cs index 58d1f5f469..862377f2d5 100644 --- a/src/settings-ui/Settings.UI.Library/Utilities/Helper.cs +++ b/src/settings-ui/Settings.UI.Library/Utilities/Helper.cs @@ -53,16 +53,18 @@ namespace Microsoft.PowerToys.Settings.UI.Library.Utilities return sendCustomAction.ToJsonString(); } - public static IFileSystemWatcher GetFileWatcher(string moduleName, string fileName, Action onChangedCallback) + public static IFileSystemWatcher GetFileWatcher(string moduleName, string fileName, Action onChangedCallback, IFileSystem fileSystem = null) { - var path = FileSystem.Path.Combine(LocalApplicationDataFolder(), $"Microsoft\\PowerToys\\{moduleName}"); + fileSystem ??= FileSystem; - if (!FileSystem.Directory.Exists(path)) + var path = fileSystem.Path.Combine(LocalApplicationDataFolder(), $"Microsoft\\PowerToys\\{moduleName}"); + + if (!fileSystem.Directory.Exists(path)) { - FileSystem.Directory.CreateDirectory(path); + fileSystem.Directory.CreateDirectory(path); } - var watcher = FileSystem.FileSystemWatcher.New(); + var watcher = fileSystem.FileSystemWatcher.New(); watcher.Path = path; watcher.Filter = fileName; watcher.NotifyFilter = NotifyFilters.LastWrite; diff --git a/src/settings-ui/Settings.UI/Assets/Settings/Icons/SemanticKernel.png b/src/settings-ui/Settings.UI/Assets/Settings/Icons/SemanticKernel.png new file mode 100644 index 0000000000000000000000000000000000000000..d74fb16d3f2e4d85f5c435907842f31e61eca5ab GIT binary patch literal 2202 zcmV;L2xa$)P)?4R7{$rlpH39$i@)4^9uE%OZ+qf9#la{8>IkI;>K3ST)h*CdTB#ZN zt?~Jz-#a_m{eJ;k_sH&{fuW_Fv}V`OhUlDACsbv7X3jS^QrvNCFGfET5D{qoV-p*= zmi%EZh}m zfJFV|?y8g?(ry6u(cy!y?mX8-xPK#Xv;K+d^Mik?c=k9T@&#O@C z&3fY%LNj|#1du|hQi%YnQ~?2gvNd|}#rHb{Lilf~FKwD1->1BYza)55D%_U5DQvS!rAiHjd$<2qto+b5{Yl zX%Nd6%mikFA=wU?g@ELBwiS?}3T(PhQmI}jvSS~;@XBQC)6tcJ0J0XE_QIu0!H!{_ zg=Mb7V`o^l1h#;6=B))4@?+uk)CPzFv<(EzutyFbIr_o#&qv|N$l`KUAp~Slw06CL zbm=*0tiBkAlPP6pgvtWUgvKb|obBJtUij`aXEft1JfCEOI1?nfAPI`ZE0UlHd-pf@ z{Nd>yH+ndU#UN5ftc*x3BDILbq8R=NBd=@*`(rcm$1;`bFu)dndi*|Q3jzUv&+_}@ zT#)3U1j+@87nt!04FBzgG%mzMY7r@y5NZ*tMI_U6Gtt74e{Mk!dl+jfb=oilQhK19 z@2)_1KecRIl=*+h9V(?>u>U_>f$=6Hv85_%5x135uAnM7Wwb^nz#nWyy~bdpQ!tkC z^6zfPjuQp~0P%WPTb(!$Fh`ph-LZSh*;I(sBJNP8+^gW!V&}{6Vcl|%$f1O?KoZkk zbNQVU=+384-_A|{c$@_!nP7bH0k~kMm6!tQRBC#!ts;BE#P(raxZHwCVOf9i>zlFb zq=5k7;6&^DNhae-E(qh3Qz7ZhpU#lA_hzd&x$h`WuM>s_FfmT%2lM$n?$~#>(Vkq* zXs!c;{YU~3!ITA3%PKA?3#KZji>CtG9#?yj5L}^H*>7;B-ATa8&jv&Xnm;>K6uqf2 z=;su18wjW@O@nD05D--;Of&t5DW zI1Moj;tFDB@4oiA%T}V!4qRvnl?zg?Z8|BShp%~KH3hjsLx2VWNkEae5I1x5#tC9F z)4c;Asvz1bHTrsR<#iW;3sf))mn*oy<-hvNq1D}dcg-PEc~nCHf&dbrA%KFy7daY+ zF;0q>`Z8yTI&un7Ev+@s8*jNR^h7b|xq!92P5_7!#PmD@;Oocxp7qA~(6ze~&JHxS z@K9d>MFA~5v?!p#BWQpngT$p6DJCK9(^NSF6ul|S4d1-V$BX)+!U2T^3MS+Z$k`xw zgu($~WN*50TpwI%_#y-H( zbP%WVDaci^;*vAMid8F|s)hFet3tJcQ9u-6M)hC@0pSY^0tFw=qcBSy5r`^8E0rW5 zAfV73+b2H6qPWlB^0n)2->D6&qCj5NB`I@Cm7r3BX$7VbL`26O|1Jv2C)xp{Rj5Qm zRf#?1PZ=* z(R7AN%OEKPK|H($c*zmRy#I<-mmXTuKXd_ELY@VT9BQIgbEx+?7$z76Y^9WNrA#$e z1u~!i{dhsTe7XMQ{4IZ0pWm#$fQUy>K_LY^3Zzl|;rVB-I@0JHyvSN(4HJwdkR`Oo zh7FR`B27!MU;)oS&OpIHVF@|Q<{Q4#_f?$*(6tW9`>)-y==?ALIBEt&3)&5V2^CMhI;a%}w2qqjb{ zcIS!sIboo7`^~zou8-U*%h9!D&DHGeT5HV^F~P9z3(z-%@vn{N - - - + + + @@ -111,6 +111,9 @@ + + + diff --git a/src/settings-ui/Settings.UI/Strings/en-us/Resources.resw b/src/settings-ui/Settings.UI/Strings/en-us/Resources.resw index c2a3b11269..2494e5f225 100644 --- a/src/settings-ui/Settings.UI/Strings/en-us/Resources.resw +++ b/src/settings-ui/Settings.UI/Strings/en-us/Resources.resw @@ -3959,8 +3959,14 @@ Activate by holding the key for the character you want to add an accent to, then Custom format preview - Preview the output of the custom format before pasting + Preview the output of AI formats and Image to text before pasting + + Enable advanced AI + + + Add advanced capabilities when using 'Paste with AI' including the power to 'chain' multiple transformations together and work with images and files. This feature may consume more API credits when used. + Advanced Paste is a tool to put your clipboard content into any format you need, focused towards developer workflows. It can paste as plain text, markdown, or json directly with the UX or with a direct keystroke invoke. These are fully locally executed. In addition, it has an AI powered option that is 100% opt-in and requires an Open AI key. Note: this will replace the formatted text in your clipboard with the selected format. diff --git a/src/settings-ui/Settings.UI/ViewModels/AdvancedPasteViewModel.cs b/src/settings-ui/Settings.UI/ViewModels/AdvancedPasteViewModel.cs index f5639dba14..ac25973ba2 100644 --- a/src/settings-ui/Settings.UI/ViewModels/AdvancedPasteViewModel.cs +++ b/src/settings-ui/Settings.UI/ViewModels/AdvancedPasteViewModel.cs @@ -319,6 +319,20 @@ namespace Microsoft.PowerToys.Settings.UI.ViewModels } } + public bool IsAdvancedAIEnabled + { + get => _advancedPasteSettings.Properties.IsAdvancedAIEnabled; + set + { + if (value != _advancedPasteSettings.Properties.IsAdvancedAIEnabled) + { + _advancedPasteSettings.Properties.IsAdvancedAIEnabled = value; + OnPropertyChanged(nameof(IsAdvancedAIEnabled)); + NotifySettingsChanged(); + } + } + } + public bool ShowCustomPreview { get => _advancedPasteSettings.Properties.ShowCustomPreview; @@ -422,10 +436,11 @@ namespace Microsoft.PowerToys.Settings.UI.ViewModels { try { - PasswordVault vault = new PasswordVault(); - PasswordCredential cred = new PasswordCredential("https://platform.openai.com/api-keys", "PowerToys_AdvancedPaste_OpenAIKey", password); + PasswordVault vault = new(); + PasswordCredential cred = new("https://platform.openai.com/api-keys", "PowerToys_AdvancedPaste_OpenAIKey", password); vault.Add(cred); OnPropertyChanged(nameof(IsOpenAIEnabled)); + IsAdvancedAIEnabled = true; // new users should get Semantic Kernel benefits immediately NotifySettingsChanged(); } catch (Exception) diff --git a/tools/BugReportTool/BugReportTool/Main.cpp b/tools/BugReportTool/BugReportTool/Main.cpp index 1675255f2c..6197e4b031 100644 --- a/tools/BugReportTool/BugReportTool/Main.cpp +++ b/tools/BugReportTool/BugReportTool/Main.cpp @@ -50,6 +50,7 @@ map> escapeInfo = { vector filesToDelete = { L"AdvancedPaste\\lastQuery.json", + L"AdvancedPaste\\kernelQueryCache.json", L"PowerToys Run\\Cache", L"PowerRename\\replace-mru.json", L"PowerRename\\search-mru.json",