diff --git a/libraries/Microsoft.Bot.Builder.AI.LUIS/LuisRecognizer.cs b/libraries/Microsoft.Bot.Builder.AI.LUIS/LuisRecognizer.cs index d212aa2de..792f021cb 100644 --- a/libraries/Microsoft.Bot.Builder.AI.LUIS/LuisRecognizer.cs +++ b/libraries/Microsoft.Bot.Builder.AI.LUIS/LuisRecognizer.cs @@ -38,6 +38,8 @@ namespace Microsoft.Bot.Builder.AI.Luis /// The context label for a LUIS trace activity. /// public const string LuisTraceLabel = "Luis Trace"; + + private readonly string _cacheKey; private readonly LuisRecognizerOptions _luisRecognizerOptions; @@ -67,6 +69,8 @@ namespace Microsoft.Bot.Builder.AI.Luis #pragma warning disable 618 // Reference to obsolete property, this is here only for backward compat and should be removed when DefaultHttpClient is removed. DefaultHttpClient = HttpClient; #pragma warning restore 618 + + _cacheKey = _luisRecognizerOptions.Application.Endpoint + _luisRecognizerOptions.Application.ApplicationId; } /// @@ -629,9 +633,19 @@ namespace Microsoft.Bot.Builder.AI.Luis private async Task RecognizeInternalAsync(ITurnContext turnContext, LuisRecognizerOptions predictionOptions, Dictionary telemetryProperties, Dictionary telemetryMetrics, CancellationToken cancellationToken) { var recognizer = predictionOptions ?? _luisRecognizerOptions; - var result = await recognizer.RecognizeInternalAsync(turnContext, HttpClient, cancellationToken).ConfigureAwait(false); - await OnRecognizerResultAsync(result, turnContext, telemetryProperties, telemetryMetrics, cancellationToken).ConfigureAwait(false); - return result; + var cached = turnContext.TurnState.Get(_cacheKey); + if (cached == null) + { + var result = await recognizer.RecognizeInternalAsync(turnContext, HttpClient, cancellationToken).ConfigureAwait(false); + await OnRecognizerResultAsync(result, turnContext, telemetryProperties, telemetryMetrics, cancellationToken).ConfigureAwait(false); + turnContext.TurnState.Set(_cacheKey, result); + _luisRecognizerOptions.TelemetryClient.TrackEvent("Luis result cached", telemetryProperties, telemetryMetrics); + + return result; + } + + _luisRecognizerOptions.TelemetryClient.TrackEvent("Read from cached Luis result", telemetryProperties, telemetryMetrics); + return cached; } /// @@ -647,9 +661,19 @@ namespace Microsoft.Bot.Builder.AI.Luis private async Task RecognizeInternalAsync(DialogContext dialogContext, Activity activity, LuisRecognizerOptions predictionOptions, Dictionary telemetryProperties, Dictionary telemetryMetrics, CancellationToken cancellationToken) { var recognizer = predictionOptions ?? _luisRecognizerOptions; - var result = await recognizer.RecognizeInternalAsync(dialogContext, activity, HttpClient, cancellationToken).ConfigureAwait(false); - await OnRecognizerResultAsync(result, dialogContext.Context, telemetryProperties, telemetryMetrics, cancellationToken).ConfigureAwait(false); - return result; + var turnContext = dialogContext.Context; + var cached = turnContext.TurnState.Get(_cacheKey); + if (cached == null) + { + var result = await recognizer.RecognizeInternalAsync(dialogContext, activity, HttpClient, cancellationToken).ConfigureAwait(false); + await OnRecognizerResultAsync(result, dialogContext.Context, telemetryProperties, telemetryMetrics, cancellationToken).ConfigureAwait(false); + turnContext.TurnState.Set(_cacheKey, result); + _luisRecognizerOptions.TelemetryClient.TrackEvent("Luis result cached", telemetryProperties, telemetryMetrics); + return result; + } + + _luisRecognizerOptions.TelemetryClient.TrackEvent("Read from cached Luis result", telemetryProperties, telemetryMetrics); + return cached; } /// diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/AdaptiveTestingBotComponent.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/AdaptiveTestingBotComponent.cs index ea88baa97..3f8a9571f 100644 --- a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/AdaptiveTestingBotComponent.cs +++ b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/AdaptiveTestingBotComponent.cs @@ -43,6 +43,7 @@ namespace Microsoft.Bot.Builder.Dialogs.Adaptive.Testing services.AddSingleton(sp => new DeclarativeType(AssertReplyActivity.Kind)); services.AddSingleton(sp => new DeclarativeType(AssertNoActivity.Kind)); services.AddSingleton(sp => new DeclarativeType(MemoryAssertions.Kind)); + services.AddSingleton(sp => new DeclarativeType(AssertTelemetryContains.Kind)); services.AddSingleton(sp => new DeclarativeType(HttpRequestSequenceMock.Kind)); services.AddSingleton(sp => new DeclarativeType(UserTokenBasicMock.Kind)); services.AddSingleton(sp => new DeclarativeType(SetProperties.Kind)); diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/Mocks/MockLuisRecognizer.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/Mocks/MockLuisRecognizer.cs index 98dfba4da..83150ddde 100644 --- a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/Mocks/MockLuisRecognizer.cs +++ b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/Mocks/MockLuisRecognizer.cs @@ -49,9 +49,17 @@ namespace Microsoft.Bot.Builder.AI.Luis.Testing public override async Task RecognizeAsync(DialogContext dialogContext, Activity activity, CancellationToken cancellationToken = default, Dictionary telemetryProperties = null, Dictionary telemetryMetrics = null) { HttpClientHandler newHandler = null, oldHandler = _recognizer.HttpClient; - + // Used for ResponsePath var recognizer = _recognizer.RecognizerOptions(dialogContext); + foreach (var middware in dialogContext.Context.Adapter.MiddlewareSet) + { + if (middware is TelemetryLoggerMiddleware telemetryMiddleware) + { + _recognizer.TelemetryClient = telemetryMiddleware.TelemetryClient; + } + } + recognizer.IncludeAPIResults = true; var middleware = dialogContext.Context.TurnState.Get(); diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/Schemas/Microsoft.Test.AssertTelemetryContains.schema b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/Schemas/Microsoft.Test.AssertTelemetryContains.schema new file mode 100644 index 000000000..6f092e1e5 --- /dev/null +++ b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/Schemas/Microsoft.Test.AssertTelemetryContains.schema @@ -0,0 +1,26 @@ +{ + "$schema": "https://schemas.botframework.com/schemas/component/v1.0/component.schema", + "$role": "implements(Microsoft.Test.ITestAction)", + "title": "Assert telemetry contains", + "description": "Checks whether telemetry log contsain specific events.", + "type": "object", + "required": [ + "events" + ], + "properties": { + "events": { + "type": "array", + "title": "Events name should be included in telemetry log", + "description": "A string array contains event names.", + "items": { + "type": "string" + } + }, + "description": { + "type": "string", + "title": "Description", + "description": "The description of which events should be included" + } + } +} + diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/TestActions/AssertTelemetryContains.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/TestActions/AssertTelemetryContains.cs new file mode 100644 index 000000000..31e7912b0 --- /dev/null +++ b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/TestActions/AssertTelemetryContains.cs @@ -0,0 +1,88 @@ +// Copyright(c) Microsoft Corporation.All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.Bot.Builder.Adapters; +using Moq; +using Newtonsoft.Json; + +namespace Microsoft.Bot.Builder.Dialogs.Adaptive.Testing.TestActions +{ + /// + /// Checks whether telemetry log contsain specific events. + /// + [DebuggerDisplay("AssertTelemetryContains")] + public class AssertTelemetryContains : TestAction + { + /// + /// Kind for the serialization. + /// + [JsonProperty("$kind")] + public const string Kind = "Microsoft.Test.AssertTelemetryContains"; + + /// + /// Initializes a new instance of the class. + /// + /// Path to source. + /// Line number in source. + [JsonConstructor] + public AssertTelemetryContains([CallerFilePath] string path = "", [CallerLineNumber] int line = 0) + { + RegisterSourcePath(path, line); + } + + /// + /// Gets or sets the description of this check. + /// + /// Description of what this check is. + [JsonProperty("description")] + public string Description { get; set; } + + /// + /// Gets the events should be contained. + /// + /// The events names. + [JsonProperty("events")] + public List Events { get; } = new List(); + + /// + public override Task ExecuteAsync(TestAdapter adapter, BotCallbackHandler callback, Inspector inspector = null) + { + var flag = true; + IBotTelemetryClient telemetryClient = null; + foreach (var middware in adapter.MiddlewareSet) + { + if (middware is TelemetryLoggerMiddleware telemetryMiddleware) + { + telemetryClient = telemetryMiddleware.TelemetryClient; + } + } + + var msgs = new List(); + foreach (var invocation in Mock.Get(telemetryClient).Invocations) + { + msgs.Add(invocation.Arguments[0].ToString()); + } + + foreach (var eve in Events) + { + if (!msgs.Contains(eve)) + { + flag = false; + break; + } + } + + if (flag == false) + { + throw new InvalidOperationException($"{Description} {string.Join(",", Events)} AssertTelemetryContains failed"); + } + + return Task.FromResult(flag); + } + } +} diff --git a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/TestScript.cs b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/TestScript.cs index 9973e6ecc..d4c731b7a 100644 --- a/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/TestScript.cs +++ b/libraries/Microsoft.Bot.Builder.Dialogs.Adaptive.Testing/TestScript.cs @@ -15,6 +15,7 @@ using Microsoft.Bot.Builder.Dialogs.Adaptive.Testing.UserTokenMocks; using Microsoft.Bot.Builder.Dialogs.Declarative.Resources; using Microsoft.Bot.Schema; using Microsoft.Extensions.Configuration; +using Moq; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; @@ -194,8 +195,10 @@ namespace Microsoft.Bot.Builder.Dialogs.Adaptive.Testing adapter.EnableTrace = EnableTrace; adapter.Locale = Locale; + var mockTelemetryClient = new Mock(); adapter.Use(new MockHttpRequestMiddleware(HttpRequestMocks)); adapter.Use(new MockSettingsMiddleware(SettingMocks)); + adapter.Use(new TelemetryLoggerMiddleware(mockTelemetryClient.Object, logPersonalInformation: true)); foreach (var userToken in UserTokenMocks) { diff --git a/tests/Microsoft.Bot.Builder.AI.LUIS.Tests/LuisOracleTests.cs b/tests/Microsoft.Bot.Builder.AI.LUIS.Tests/LuisOracleTests.cs index bd928c98b..7452373d4 100644 --- a/tests/Microsoft.Bot.Builder.AI.LUIS.Tests/LuisOracleTests.cs +++ b/tests/Microsoft.Bot.Builder.AI.LUIS.Tests/LuisOracleTests.cs @@ -400,7 +400,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("test")); Assert.Equal("testvalue", ((Dictionary)telemetryClient.Invocations[0].Arguments[1])["test"]); @@ -448,7 +448,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.Equal(8, ((Dictionary)telemetryClient.Invocations[0].Arguments[1]).Count); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("applicationId")); @@ -495,7 +495,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.Equal(7, ((Dictionary)telemetryClient.Invocations[0].Arguments[1]).Count); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("applicationId")); @@ -547,7 +547,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Equal(2, telemetryClient.Invocations.Count); + Assert.Equal(3, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("MyImportantProperty")); Assert.Equal("myImportantValue", ((Dictionary)telemetryClient.Invocations[0].Arguments[1])["MyImportantProperty"]); @@ -605,7 +605,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Equal(2, telemetryClient.Invocations.Count); + Assert.Equal(3, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("MyImportantProperty")); Assert.Equal("myImportantValue", ((Dictionary)telemetryClient.Invocations[0].Arguments[1])["MyImportantProperty"]); @@ -658,7 +658,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("applicationId")); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("intent")); @@ -702,7 +702,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("applicationId")); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("intent")); @@ -757,7 +757,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("test")); Assert.Equal("testvalue", ((Dictionary)telemetryClient.Invocations[0].Arguments[1])["test"]); diff --git a/tests/Microsoft.Bot.Builder.AI.LUIS.Tests/LuisV3OracleTests.cs b/tests/Microsoft.Bot.Builder.AI.LUIS.Tests/LuisV3OracleTests.cs index 2de8a8995..300553475 100644 --- a/tests/Microsoft.Bot.Builder.AI.LUIS.Tests/LuisV3OracleTests.cs +++ b/tests/Microsoft.Bot.Builder.AI.LUIS.Tests/LuisV3OracleTests.cs @@ -372,7 +372,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("test")); Assert.Equal("testvalue", ((Dictionary)telemetryClient.Invocations[0].Arguments[1])["test"]); @@ -419,7 +419,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); // Assert.IsTrue(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).Count == 8); @@ -467,7 +467,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).Count == 7); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("applicationId")); @@ -519,7 +519,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Equal(2, telemetryClient.Invocations.Count); + Assert.Equal(3, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("MyImportantProperty")); Assert.Equal("myImportantValue", ((Dictionary)telemetryClient.Invocations[0].Arguments[1])["MyImportantProperty"]); @@ -577,7 +577,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Equal(2, telemetryClient.Invocations.Count); + Assert.Equal(3, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("MyImportantProperty")); Assert.Equal("myImportantValue", ((Dictionary)telemetryClient.Invocations[0].Arguments[1])["MyImportantProperty"]); @@ -630,7 +630,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("applicationId")); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("intent")); @@ -674,7 +674,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("applicationId")); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("intent")); @@ -729,7 +729,7 @@ namespace Microsoft.Bot.Builder.AI.Luis.Tests // Assert Assert.NotNull(result); - Assert.Single(telemetryClient.Invocations); + Assert.Equal(2, telemetryClient.Invocations.Count); Assert.Equal("LuisResult", telemetryClient.Invocations[0].Arguments[0].ToString()); Assert.True(((Dictionary)telemetryClient.Invocations[0].Arguments[1]).ContainsKey("test")); Assert.Equal("testvalue", ((Dictionary)telemetryClient.Invocations[0].Arguments[1])["test"]); diff --git a/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/AskTests.cs b/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/AskTests.cs index 5a54f244b..742613f5a 100644 --- a/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/AskTests.cs +++ b/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/AskTests.cs @@ -44,5 +44,11 @@ namespace Microsoft.Bot.Builder.Dialogs.Adaptive.Tests { await TestUtils.RunTestScript(_resourceExplorerFixture.ResourceExplorer); } + + [Fact] + public async Task CacheLuisRecognizer() + { + await TestUtils.RunTestScript(_resourceExplorerFixture.ResourceExplorer); + } } } diff --git a/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/Tests/AskTests/AskName.dialog b/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/Tests/AskTests/AskName.dialog new file mode 100644 index 000000000..5d77d0960 --- /dev/null +++ b/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/Tests/AskTests/AskName.dialog @@ -0,0 +1,31 @@ +{ + "$schema": "../../../tests.schema", + "$kind": "Microsoft.AdaptiveDialog", + "recognizer": { + "$kind": "Microsoft.LuisRecognizer", + "applicationId": "00000000-0000-0000-0000-000000000000", + "endpointKey": "00000000000000000000000000000000", + "endpoint": "https://westus.api.cognitive.microsoft.com", + "predictionOptions": { + "IncludeAPIResults": true + } + }, + "triggers": [ + { + "$kind": "Microsoft.OnBeginDialog", + "actions": [ + { + "$kind": "Microsoft.TextInput", + "disabled": false, + "maxTurnCount": 3, + "alwaysPrompt": false, + "allowInterruptions": true, + "unrecognizedPrompt": "", + "invalidPrompt": "", + "prompt": "What is your name?" + } + ] + } + ], + "id": "AskName" +} diff --git a/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/Tests/AskTests/CacheLuisRecognizer.test.dialog b/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/Tests/AskTests/CacheLuisRecognizer.test.dialog new file mode 100644 index 000000000..80f43ae5d --- /dev/null +++ b/tests/Microsoft.Bot.Builder.Dialogs.Adaptive.Tests/Tests/AskTests/CacheLuisRecognizer.test.dialog @@ -0,0 +1,106 @@ +{ + "$schema": "../../../tests.schema", + "$kind": "Microsoft.Test.Script", + "description": "Test retries from Ask with DeleteProperties", + "httpRequestMocks": [ + "LuisNoEntities.mock", + "LuisBreadEntity.mock" + ], + "dialog": { + "$kind": "Microsoft.AdaptiveDialog", + "recognizer": { + "$kind": "Microsoft.LuisRecognizer", + "applicationId": "00000000-0000-0000-0000-000000000000", + "endpointKey": "00000000000000000000000000000000", + "endpoint": "https://westus.api.cognitive.microsoft.com", + "predictionOptions": { + "IncludeAPIResults": true + } + }, + "schema": "oneProperty.json", + "triggers": [ + { + "$kind": "Microsoft.OnBeginDialog", + "actions": [ + { + "$kind": "Microsoft.SendActivity", + "activity": "welcome" + } + ] + }, + { + "$kind": "Microsoft.OnEndOfActions", + "condition": "=!$Bread", + "priority": 0, + "actions": [ + { + "$kind": "Microsoft.Ask", + "activity": "Bread?", + "expectedProperties": [ + "Bread" + ] + } + ] + }, + { + "$kind": "Microsoft.OnAssignEntity", + "operation": "Add()", + "property": "Bread", + "value": "BreadEntity", + "actions": [ + { + "$kind": "Microsoft.BeginDialog", + "options": {}, + "dialog": "AskName" + } + ] + } + ] + }, + "script": [ + { + "$kind": "Microsoft.Test.UserConversationUpdate", + "membersAdded": [ + "Bot", + "User" + ], + "membersRemoved": [] + }, + { + "$kind": "Microsoft.Test.AssertReply", + "text": "welcome" + }, + { + "$kind": "Microsoft.Test.AssertReply", + "text": "Bread?" + }, + { + "$kind": "Microsoft.Test.UserSays", + "text": "no entities" + }, + { + "$kind": "Microsoft.Test.AssertReply", + "text": "Bread?" + }, + { + "$kind": "Microsoft.Test.UserSays", + "text": "rye" + }, + { + "$kind": "Microsoft.Test.AssertReply", + "text": "What is your name?" + }, + { + "$kind": "Microsoft.Test.UserSays", + "text": "rye" + }, + { + "$kind": "Microsoft.Test.AssertTelemetryContains", + "events": [ + "Luis result cached", + "Read from cached Luis result" + ], + "description": "Ensure telemetry log contains cached Luis info" + } + ] +} \ No newline at end of file diff --git a/tests/tests.schema b/tests/tests.schema index cd91abb9b..2e6ca55e2 100644 --- a/tests/tests.schema +++ b/tests/tests.schema @@ -349,6 +349,9 @@ { "$ref": "#/definitions/Microsoft.Test.AssertReplyOneOf" }, + { + "$ref": "#/definitions/Microsoft.Test.AssertTelemetryContains" + }, { "$ref": "#/definitions/Microsoft.Test.CustomEvent" }, @@ -9163,6 +9166,50 @@ } } }, + "Microsoft.Test.AssertTelemetryContains": { + "$role": "implements(Microsoft.Test.ITestAction)", + "title": "Assert telemetry contains", + "description": "Checks whether telemetry log contsain specific events.", + "type": "object", + "required": [ + "events", + "$kind" + ], + "additionalProperties": false, + "patternProperties": { + "^\\$": { + "title": "Tooling property", + "description": "Open ended property for tooling." + } + }, + "properties": { + "events": { + "type": "array", + "title": "Events name should be included in telemetry log", + "description": "A string array contains event names.", + "items": { + "type": "string" + } + }, + "description": { + "type": "string", + "title": "Description", + "description": "The description of which events should be included" + }, + "$kind": { + "title": "Kind of dialog object", + "description": "Defines the valid properties for the component you are configuring (from a dialog .schema file)", + "type": "string", + "pattern": "^[a-zA-Z][a-zA-Z0-9.]*$", + "const": "Microsoft.Test.AssertTelemetryContains" + }, + "$designer": { + "title": "Designer information", + "type": "object", + "description": "Extra information for the Bot Framework Composer." + } + } + }, "Microsoft.Test.CustomEvent": { "$role": "implements(Microsoft.Test.ITestAction)", "title": "User event", @@ -9463,6 +9510,9 @@ { "$ref": "#/definitions/Microsoft.Test.AssertReplyOneOf" }, + { + "$ref": "#/definitions/Microsoft.Test.AssertTelemetryContains" + }, { "$ref": "#/definitions/Microsoft.Test.CustomEvent" },