Added receivedServiceMessage in Connection.ts. The message can be any messages from service that are not being processed by SpeechRecognizer, Intent, Translator or Dialog. (#109)

* init changes, enable me to run test

* added USPEvent

* local speech.event works

* self cleanup

* renamed to serviceEvents, address CR comments, such as added serviceEvent for DialogServiceAdapter

* remove debug printout in DialogServiceConnectorTests.ts

* add comments for receivedServiceMessage and disable speech.event test case

BuildTestConfig.cmd

@@ -52,6 +52,6 @@ echo Writes any ^<ParamName^>:^<Value^> pair to the test settings file for JavaS
 echo.
 echo The file will be erased before new values are added.
 echo.
-echo Current settings available are: [SpeechSubscriptionKey:^<key^>] [SpeechRegion:^<region^>] [LuisSubscriptionKey:^<LuisKey^>]  [LuisRegion:^<region^>] [LuisAppId:^<LuisAppId^>]
+echo Current settings available are: [SpeechSubscriptionKey:^<key^>] [SpeechRegion:^<region^>] [LuisSubscriptionKey:^<LuisKey^>]  [LuisRegion:^<region^>] [LuisAppId:^<LuisAppId^>] [VoiceSignatureEnrollmentKey:^VoiceSignatureEnrollmentKey^>]
 echo.
 exit /b 1

src/common.speech/DialogServiceAdapter.ts

@@ -18,6 +18,7 @@ import {
     Promise,
     PromiseHelper,
     PromiseResult,
+    ServiceEvent,
 } from "../common/Exports";
 import { PullAudioOutputStreamImpl } from "../sdk/Audio/AudioOutputStream";
 import { AudioStreamFormatImpl } from "../sdk/Audio/AudioStreamFormat";
@@ -172,7 +173,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
     protected processTypeSpecificMessages(
         connectionMessage: SpeechConnectionMessage,
         successCallback?: (e: SpeechRecognitionResult) => void,
-        errorCallBack?: (e: string) => void): void {
+        errorCallBack?: (e: string) => void): boolean {
 
         const resultProps: PropertyCollection = new PropertyCollection();
         if (connectionMessage.messageType === MessageType.Text) {
@@ -180,6 +181,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
         }
 
         let result: SpeechRecognitionResult;
+        let processed: boolean;
 
         switch (connectionMessage.path.toLowerCase()) {
             case "speech.phrase":
@@ -201,6 +203,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
                         }
                     }
                 }
+                processed = true;
                 break;
             case "speech.hypothesis":
                 const hypothesis: SpeechHypothesis = SpeechHypothesis.fromJSON(connectionMessage.textBody);
@@ -229,6 +232,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
                         // trip things up.
                     }
                 }
+                processed = true;
                 break;
 
             case "audio":
@@ -247,6 +251,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
                         // trip things up.
                     }
                 }
+                processed = true;
                 break;
 
             case "response":
@@ -274,11 +279,13 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
                         }
                     }
                 }
+                processed = true;
                 break;
 
             default:
                 break;
         }
+        return processed;
     }
 
     // Cancels recognition.
@@ -666,10 +673,14 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
                                 break;
 
                             default:
-                                this.processTypeSpecificMessages(
+                                if (!this.processTypeSpecificMessages(
                                     connectionMessage,
                                     successCallback,
-                                    errorCallBack);
+                                    errorCallBack)) {
+                                        if (!!this.serviceEvents) {
+                                            this.serviceEvents.onEvent(new ServiceEvent(connectionMessage.path.toLowerCase(), connectionMessage.textBody));
+                                        }
+                                    }
                         }
 
                         return this.receiveDialogMessageOverride();

src/common.speech/IntentServiceRecognizer.ts

@@ -59,10 +59,11 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
     protected processTypeSpecificMessages(
         connectionMessage: SpeechConnectionMessage,
         successCallback?: (e: IntentRecognitionResult) => void,
-        errorCallBack?: (e: string) => void): void {
+        errorCallBack?: (e: string) => void): boolean {
 
         let result: IntentRecognitionResult;
         let ev: IntentRecognitionEventArgs;
+        let processed: boolean = false;
 
         const resultProps: PropertyCollection = new PropertyCollection();
         if (connectionMessage.messageType === MessageType.Text) {
@@ -97,7 +98,7 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
                         // trip things up.
                     }
                 }
-
+                processed = true;
                 break;
             case "speech.phrase":
                 const simple: SimpleSpeechPhrase = SimpleSpeechPhrase.fromJSON(connectionMessage.textBody);
@@ -153,7 +154,7 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
                     // and then return
                     this.privPendingIntentArgs = ev;
                 }
-
+                processed = true;
                 break;
             case "response":
                 // Response from LUIS
@@ -237,10 +238,12 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
                     successCallback = undefined;
                     errorCallBack = undefined;
                 }
+                processed = true;
                 break;
             default:
                 break;
         }
+        return processed;
     }
 
     // Cancels recognition.

src/common.speech/ServiceRecognizerBase.ts

@@ -22,6 +22,7 @@ import {
     Promise,
     PromiseHelper,
     PromiseResult,
+    ServiceEvent,
 } from "../common/Exports";
 import { AudioStreamFormatImpl } from "../sdk/Audio/AudioStreamFormat";
 import {
@@ -68,6 +69,7 @@ export abstract class ServiceRecognizerBase implements IDisposable {
     private privIsDisposed: boolean;
     private privMustReportEndOfStream: boolean;
     private privConnectionEvents: EventSource<ConnectionEvent>;
+    private privServiceEvents: EventSource<ServiceEvent>;
     private privSpeechContext: SpeechContext;
     private privDynamicGrammar: DynamicGrammarBuilder;
     private privAgentConfig: AgentConfig;
@@ -108,6 +110,7 @@ export abstract class ServiceRecognizerBase implements IDisposable {
         this.privRecognizer = recognizer;
         this.privRequestSession = new RequestSession(this.privAudioSource.id());
         this.privConnectionEvents = new EventSource<ConnectionEvent>();
+        this.privServiceEvents = new EventSource<ServiceEvent>();
         this.privDynamicGrammar = new DynamicGrammarBuilder();
         this.privSpeechContext = new SpeechContext(this.privDynamicGrammar);
         this.privAgentConfig = new AgentConfig();
@@ -146,6 +149,10 @@ export abstract class ServiceRecognizerBase implements IDisposable {
         return this.privConnectionEvents;
     }
 
+    public get serviceEvents(): EventSource<ServiceEvent> {
+        return this.privServiceEvents;
+    }
+
     public get recognitionMode(): RecognitionMode {
         return this.privRecognizerConfig.recognitionMode;
     }
@@ -274,7 +281,7 @@ export abstract class ServiceRecognizerBase implements IDisposable {
     protected abstract processTypeSpecificMessages(
         connectionMessage: SpeechConnectionMessage,
         successCallback?: (e: SpeechRecognitionResult) => void,
-        errorCallBack?: (e: string) => void): void;
+        errorCallBack?: (e: string) => void): boolean;
 
     protected sendTelemetryData = () => {
         const telemetryData = this.privRequestSession.getTelemetry();
@@ -422,11 +429,20 @@ export abstract class ServiceRecognizerBase implements IDisposable {
                                         this.sendSpeechContext(connection);
                                     });
                                 }
+                                break;
+
                             default:
-                                this.processTypeSpecificMessages(
+
+                                if (!this.processTypeSpecificMessages(
                                     connectionMessage,
                                     successCallback,
-                                    errorCallBack);
+                                    errorCallBack)) {
+                                        // here are some messages that the derived class has not processed, dispatch them to connect class
+                                        if (!!this.privServiceEvents) {
+                                            this.serviceEvents.onEvent(new ServiceEvent(connectionMessage.path.toLowerCase(), connectionMessage.textBody));
+                                        }
+                                    }
+
                         }
                     }
 
@@ -479,7 +495,7 @@ export abstract class ServiceRecognizerBase implements IDisposable {
         const authPromise = isUnAuthorized ? this.privAuthentication.fetchOnExpiry(this.privAuthFetchEventId) : this.privAuthentication.fetch(this.privAuthFetchEventId);
 
         this.privConnectionPromise = authPromise
-            .continueWithPromise((result: PromiseResult<AuthInfo>) => {
+                .continueWithPromise((result: PromiseResult<AuthInfo>) => {
                 if (result.isError) {
                     this.privRequestSession.onAuthCompleted(true, result.error);
                     throw new Error(result.error);

src/common.speech/SpeechServiceRecognizer.ts

@@ -48,11 +48,12 @@ export class SpeechServiceRecognizer extends ServiceRecognizerBase {
     protected processTypeSpecificMessages(
         connectionMessage: SpeechConnectionMessage,
         successCallback?: (e: SpeechRecognitionResult) => void,
-        errorCallBack?: (e: string) => void): void {
+        errorCallBack?: (e: string) => void): boolean {
 
         let result: SpeechRecognitionResult;
         const resultProps: PropertyCollection = new PropertyCollection();
         resultProps.setProperty(PropertyId.SpeechServiceResponse_JsonResult, connectionMessage.textBody);
+        let processed: boolean = false;
 
         switch (connectionMessage.path.toLowerCase()) {
             case "speech.hypothesis":
@@ -83,6 +84,7 @@ export class SpeechServiceRecognizer extends ServiceRecognizerBase {
                         // trip things up.
                     }
                 }
+                processed = true;
                 break;
             case "speech.phrase":
                 const simple: SimpleSpeechPhrase = SimpleSpeechPhrase.fromJSON(connectionMessage.textBody);
@@ -153,11 +155,12 @@ export class SpeechServiceRecognizer extends ServiceRecognizerBase {
                         errorCallBack = undefined;
                     }
                 }
-
+                processed = true;
                 break;
             default:
                 break;
         }
+        return processed;
     }
 
     // Cancels recognition.

src/common.speech/TranslationServiceRecognizer.ts

@@ -55,9 +55,11 @@ export class TranslationServiceRecognizer extends ServiceRecognizerBase {
     protected processTypeSpecificMessages(
         connectionMessage: SpeechConnectionMessage,
         successCallback?: (e: TranslationRecognitionResult) => void,
-        errorCallBack?: (e: string) => void): void {
+        errorCallBack?: (e: string) => void): boolean {
 
         const resultProps: PropertyCollection = new PropertyCollection();
+        let processed: boolean = false;
+
         if (connectionMessage.messageType === MessageType.Text) {
             resultProps.setProperty(PropertyId.SpeechServiceResponse_JsonResult, connectionMessage.textBody);
         }
@@ -77,7 +79,7 @@ export class TranslationServiceRecognizer extends ServiceRecognizerBase {
                         // trip things up.
                     }
                 }
-
+                processed = true;
                 break;
             case "translation.phrase":
                 const translatedPhrase: TranslationPhrase = TranslationPhrase.fromJSON(connectionMessage.textBody);
@@ -169,10 +171,12 @@ export class TranslationServiceRecognizer extends ServiceRecognizerBase {
                         }
                     }
                 }
+                processed = true;
                 break;
 
             case "translation.synthesis":
                 this.sendSynthesisAudio(connectionMessage.binaryBody, this.privRequestSession.sessionId);
+                processed = true;
                 break;
 
             case "translation.synthesis.end":
@@ -217,10 +221,12 @@ export class TranslationServiceRecognizer extends ServiceRecognizerBase {
                     default:
                         break;
                 }
+                processed = true;
                 break;
             default:
                 break;
         }
+        return processed;
     }
 
     // Cancels recognition.

src/common/ConnectionEvents.ts

@@ -5,6 +5,20 @@ import { ConnectionMessage } from "./ConnectionMessage";
 import { IStringDictionary } from "./IDictionary";
 import { EventType, PlatformEvent } from "./PlatformEvent";
 
+export class ServiceEvent extends PlatformEvent {
+    private privJsonResult: string;
+
+    constructor(eventName: string, jsonstring: string, eventType: EventType = EventType.Info) {
+      super(eventName, eventType);
+      this.privJsonResult = jsonstring;
+    }
+
+    public get jsonString(): string {
+        return this.privJsonResult;
+    }
+}
+
+// tslint:disable-next-line:max-classes-per-file
 export class ConnectionEvent extends PlatformEvent {
     private privConnectionId: string;
 

src/sdk/Connection.ts

@@ -9,10 +9,12 @@ import {
 import {
     ConnectionEvent,
     IDetachable,
+    ServiceEvent,
 } from "../common/Exports";
 import {
     ConnectionEventArgs,
     Recognizer,
+    ServiceEventArgs,
 } from "./Exports";
 
 /**
@@ -31,6 +33,7 @@ import {
 export class Connection {
     private privServiceRecognizer: ServiceRecognizerBase;
     private privEventListener: IDetachable;
+    private privServiceEventListener: IDetachable;
 
     /**
      * Gets the Connection instance from the specified recognizer.
@@ -55,6 +58,12 @@ export class Connection {
             }
         });
 
+        ret.privServiceEventListener = ret.privServiceRecognizer.serviceEvents.attach((e: ServiceEvent): void =>  {
+            if (!!ret.receivedServiceMessage) {
+                ret.receivedServiceMessage(new ServiceEventArgs(e.jsonString, e.name));
+            }
+        });
+
         return ret;
     }
 
@@ -74,12 +83,17 @@ export class Connection {
      * Closes the connection the service.
      * Users can optionally call closeConnection() to manually shutdown the connection of the associated Recognizer.
      *
-     * If closeConnection() is called during recognition, recognition will fail and cancel wtih an error.
+     * If closeConnection() is called during recognition, recognition will fail and cancel with an error.
      */
     public closeConnection(): void {
         this.privServiceRecognizer.disconnect();
     }
 
+    /**
+     * Any message from service that is not being processed by any other top level recognizers.
+     */
+    public receivedServiceMessage: (args: ServiceEventArgs) => void;
+
     /**
      * The Connected event to indicate that the recognizer is connected to service.
      */

src/sdk/Exports.ts

@@ -39,6 +39,7 @@ export { IntentRecognitionCanceledEventArgs } from "./IntentRecognitionCanceledE
 export { CancellationDetails } from "./CancellationDetails";
 export { CancellationErrorCode } from "./CancellationErrorCodes";
 export { ConnectionEventArgs } from "./ConnectionEventArgs";
+export { ServiceEventArgs } from "./ServiceEventArgs";
 export { Connection } from "./Connection";
 export { PhraseListGrammar } from "./PhraseListGrammar";
 export { DialogServiceConfig } from "./DialogServiceConfig";

src/sdk/ServiceEventArgs.ts

@@ -0,0 +1,35 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+//
+
+import { SessionEventArgs } from "./Exports";
+
+/**
+ * Defines payload for any Service message event
+ * Added in version 1.9.0
+ */
+
+export class ServiceEventArgs extends SessionEventArgs {
+    private privJsonResult: string;
+    private privEventName: string;
+
+    /**
+     * Creates and initializes an instance of this class.
+     * @constructor
+     * @param {string} json - json payload of the USP message.
+     */
+    public constructor(json: string,  name: string, sessionId?: string) {
+        super(sessionId);
+        this.privJsonResult = json;
+        this.privEventName = name;
+    }
+
+    public get jsonString(): string {
+        return this.privJsonResult;
+    }
+
+    public get eventName(): string {
+        return this.privEventName;
+    }
+}

tests/Settings.ts

@@ -8,6 +8,9 @@ export class Settings {
     public static SpeechRegion: string = "<<YOUR_REGION>>";
     public static SpeechEndpoint: string;
 
+    public static VoiceSignatureEnrollmentEndpoint: string = "wss://10.91.185.105:8446/speech/voicesignature?&language=en-us";
+    public static VoiceSignatureEnrollmentKey: string;
+
     public static SpeechTestEndpointId: string = "<<YOUR_TEST_ENDPOINT_ID>>";
 
     // Endpoint and key for timeout testing.
@@ -54,6 +57,7 @@ export class Settings {
     public static LuisAppId: string = "b687b851-56c5-4d31-816f-35a741a3f0be";
 
     public static WaveFile: string = Settings.InputDir + "whatstheweatherlike.wav";
+    public static VoiceSignatureWaveFile: string = Settings.InputDir + "StreamingEnrollment.wav";
     public static WaveFileLanguage: string = "en-US";
     public static WaveFileDuration: number = 12900000;
     public static WaveFileOffset: number = 1000000;

tests/SpeechRecognizerTests.ts

@@ -19,6 +19,13 @@ import { setTimeout } from "timers";
 import { ByteBufferAudioFile } from "./ByteBufferAudioFile";
 import WaitForCondition from "./Utilities";
 
+import {
+    DynamicGrammarBuilder,
+    IDynamicGrammar,
+    IDynamicGrammarGeneric,
+    IDynamicGrammarGroup,
+} from "../src/common.speech/Exports";
+
 const FIRST_EVENT_ID: number = 1;
 const Recognizing: string = "Recognizing";
 const Recognized: string = "Recognized";
@@ -92,7 +99,90 @@ const BuildSpeechConfig: () => sdk.SpeechConfig = (): sdk.SpeechConfig => {
     expect(s).not.toBeUndefined();
     return s;
 };
+/*
+test("speech.event from service", (done: jest.DoneCallback) => {
+    // tslint:disable-next-line:no-console
+    console.info("Name: speech.event from service");
 
+    expect(Settings.VoiceSignatureEnrollmentEndpoint && Settings.VoiceSignatureEnrollmentEndpoint.length > 0);
+    expect(Settings.VoiceSignatureEnrollmentKey && Settings.VoiceSignatureEnrollmentKey.length > 0);
+
+    const s: sdk.SpeechConfig = sdk.SpeechConfig.fromEndpoint(new URL(Settings.VoiceSignatureEnrollmentEndpoint), Settings.VoiceSignatureEnrollmentKey);
+    objsToClose.push(s);
+
+    const file: File = WaveFileAudioInput.LoadFile(Settings.VoiceSignatureWaveFile);
+    const config: sdk.AudioConfig = sdk.AudioConfig.fromWavFileInput(file);
+
+    const r: sdk.SpeechRecognizer = new sdk.SpeechRecognizer(s, config);
+    objsToClose.push(r);
+
+    expect(r).not.toBeUndefined();
+    expect(r instanceof sdk.Recognizer);
+
+    let sessionDone: boolean = false;
+
+    const phraseList: sdk.PhraseListGrammar = sdk.PhraseListGrammar.fromRecognizer(r);
+    let expectedText: string;
+    expectedText = "Hello, it's a good day for me to teach you the sound of my voice. You have learned what I look like, now you can hear what I sound like.";
+    expectedText += "The sound of my voice will help the transcription service to recognize my unique voice in the future.";
+    expectedText += "Training will provide a better experience with greater accuracy when talking or dictating.";
+    expectedText += "Thank you and goodbye.";
+
+    phraseList.addPhrase(expectedText);
+    let sessionId: string;
+    r.sessionStarted = (r: sdk.Recognizer, e: sdk.SessionEventArgs): void => {
+        sessionId = e.sessionId;
+    };
+
+    r.sessionStopped = (r: sdk.Recognizer, e: sdk.SessionEventArgs): void => {
+        sessionDone = true;
+    };
+
+    const connection: sdk.Connection = sdk.Connection.fromRecognizer(r);
+
+    let receivedSpeechEvent: boolean = false;
+    connection.receivedServiceMessage = (e: sdk.ServiceEventArgs): void => {
+       // tslint:disable-next-line:no-console
+       console.info("Receuved a Service message: '" + e.jsonString + "'");
+       if ( e.eventName === "speech.event" ) {
+           receivedSpeechEvent = true;
+       }
+    };
+
+    r.canceled = (r: sdk.Recognizer, e: sdk.SpeechRecognitionCanceledEventArgs): void => {
+        sessionDone = true;
+        try {
+            expect(e.errorDetails).toBeUndefined();
+            expect(e.reason).toEqual(sdk.CancellationReason.EndOfStream);
+        } catch (error) {
+            done.fail(error);
+        }
+    };
+
+    r.recognized = (r: sdk.Recognizer, e: sdk.SpeechRecognitionEventArgs): void => {
+        try {
+            const res: sdk.SpeechRecognitionResult = e.result;
+            expect(res).not.toBeUndefined();
+        } catch (error) {
+            done.fail(error);
+        }
+    };
+
+    r.startContinuousRecognitionAsync(
+        undefined,
+        (error: string) => {
+            done.fail(error);
+        });
+
+    WaitForCondition(() => (sessionDone), () => {
+        r.stopContinuousRecognitionAsync(() => {
+           expect(receivedSpeechEvent).toEqual(true);
+           done();
+        });
+    });
+
+}, 200000);
+*/
 test("testSpeechRecognizer1", () => {
     // tslint:disable-next-line:no-console
     console.info("Name: testSpeechRecognizer1");