microsoft
/
cognitive-services-speech-sdk-js
зеркало из https://github.com/microsoft/cognitive-services-speech-sdk-js.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

use trusted URL for AudioWorklet (#732)

This commit is contained in:
Glenn Harper 2023-09-19 09:45:34 -07:00 коммит произвёл GitHub
Родитель a34b15714f
Коммит 6a9273d811
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
36 изменённых файлов: 191 добавлений и 73 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

29
gulpfile.js
Просмотреть файл

@ -10,10 +10,6 @@
var webpack = require('webpack-stream');
var dtsBundleWebpack = require('dts-bundle-webpack');
var tsProject = ts.createProject('tsconfig.json');
var tsProject2015 = ts.createProject('tsconfig.json', {
target: 'es2015',
module: 'esnext'
});
gulp.task('build', gulp.series(function build() {
return gulp.src([
@ -30,29 +26,18 @@
.pipe(tsProject())
.pipe(sourcemaps.write('.'))
.pipe(gulp.dest('distrib/lib'));
}));
gulp.task('build2015', gulp.series(function build() {
return gulp.src([
'src/**/*.ts',
'microsoft.cognitiveservices.speech.sdk.ts'],
{ base: '.' })
.pipe(eslint({
formatter: 'prose',
configuration: 'eslint.json'
}))
.pipe(eslint.format())
.pipe(eslint.failAfterError())
.pipe(sourcemaps.init())
.pipe(tsProject2015())
.pipe(sourcemaps.write('.'))
.pipe(gulp.dest('distrib/es2015'));
}, function () {
return gulp.src('./src/audioworklet/speech-processor.js')
.pipe(gulp.dest('./distrib/lib/src/common.browser'));
}));
gulp.task('bundle', gulp.series('build', function bundle() {
return gulp.src('bundleApp.js')
.pipe(webpack({
output: { filename: 'microsoft.cognitiveservices.speech.sdk.bundle.js' },
entry: {
'microsoft.cognitiveservices.speech.sdk.bundle': './bundleApp.js',
},
output: { filename: '[name].js' },
devtool: 'source-map',
module: {
rules: [{

4
jest.config.js
Просмотреть файл

@ -8,7 +8,7 @@ module.exports = {
"^.+\\.ts$": "ts-jest",
},
testRegex: "tests/.*Tests\\.ts$",
testPathIgnorePatterns: ["/lib/", "/es2015/", "/node_modules/", "/src/"],
testPathIgnorePatterns: ["/lib/", "/node_modules/", "/src/"],
moduleFileExtensions: ["ts", "js", "jsx", "json", "node"],
testEnvironment: "jsdom",
collectCoverage: false,
@ -21,7 +21,7 @@ module.exports = {
"^.+\\.ts$": "ts-jest",
},
testRegex: "tests/.*Tests\\.ts$",
testPathIgnorePatterns: ["/lib/", "/es2015/", "/node_modules/", "/src/"],
testPathIgnorePatterns: ["/lib/", "/node_modules/", "/src/"],
moduleFileExtensions: ["ts", "js", "jsx", "json", "node"],
testEnvironment: "node",
collectCoverage: false,

5
package.json
Просмотреть файл

@ -38,11 +38,10 @@
"net": false
},
"main": "distrib/lib/microsoft.cognitiveservices.speech.sdk.js",
"module": "distrib/es2015/microsoft.cognitiveservices.speech.sdk.js",
"module": "distrib/lib/microsoft.cognitiveservices.speech.sdk.js",
"types": "distrib/lib/microsoft.cognitiveservices.speech.sdk.d.ts",
"files": [
"distrib/lib/**/*",
"distrib/es2015/**/*",
"distrib/browser/**/*",
"LICENSE",
"REDIST.txt"
@ -83,7 +82,7 @@
"webpack-stream": "^7.0.0"
},
"scripts": {
"build": "gulp compress && gulp build2015",
"build": "gulp compress && gulp build",
"test": "npm run lint && npm run jest --coverage",
"jest": "jest",
"lint": "eslint -c .eslintrc.js --ext .ts src",

6
src/common.browser/AudioWorkerUrl.ts Normal file
Просмотреть файл

@ -0,0 +1,6 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license.
/* webpackChunkName: 'script_processor_audioWorklet' */
// eslint-disable-next-line @typescript-eslint/tslint/config
export const getAudioWorkerUrl = (): string => new URL("speech-processor.js", import.meta.url).toString();

96
src/common.browser/PCMRecorder.ts
Просмотреть файл

@ -2,6 +2,7 @@
// Licensed under the MIT license.
import { RiffPcmEncoder, Stream } from "../common/Exports";
import { getAudioWorkerUrl } from "./AudioWorkerUrl";
import { IRecorder } from "./IRecorder";
export class PcmRecorder implements IRecorder {
@ -60,59 +61,70 @@ export class PcmRecorder implements IRecorder {
};
};
const connectWorkletToMicInput = (context: AudioContext): void => {
const workletNode = new AudioWorkletNode(context, "speech-processor");
workletNode.port.onmessage = (ev: MessageEvent): void => {
const inputFrame: Float32Array = ev.data as Float32Array;
if (outputStream && !outputStream.isClosed) {
const waveFrame = waveStreamEncoder.encode(inputFrame);
if (!!waveFrame) {
outputStream.writeStreamChunk({
buffer: waveFrame,
isEnd: false,
timeReceived: Date.now(),
});
}
}
};
micInput.connect(workletNode);
workletNode.connect(context.destination);
this.privMediaResources = {
scriptProcessorNode: workletNode,
source: micInput,
stream: mediaStream,
};
};
// https://webaudio.github.io/web-audio-api/#audioworklet
// Using AudioWorklet to improve audio quality and avoid audio glitches due to blocking the UI thread
const skipAudioWorklet = !!this.privSpeechProcessorScript && this.privSpeechProcessorScript.toLowerCase() === "ignore";
if (!!context.audioWorklet && !skipAudioWorklet) {
if (!this.privSpeechProcessorScript) {
const workletScript = `class SP extends AudioWorkletProcessor {
constructor(options) {
super(options);
}
process(inputs, outputs) {
const input = inputs[0];
const output = [];
for (let channel = 0; channel < input.length; channel += 1) {
output[channel] = input[channel];
}
this.port.postMessage(output[0]);
return true;
}
}
registerProcessor('speech-processor', SP);`;
const blob = new Blob([workletScript], { type: "application/javascript; charset=utf-8" });
this.privSpeechProcessorScript = URL.createObjectURL(blob);
}
this.privSpeechProcessorScript = getAudioWorkerUrl();
context.audioWorklet
.addModule(this.privSpeechProcessorScript)
.then((): void => {
const workletNode = new AudioWorkletNode(context, "speech-processor");
workletNode.port.onmessage = (ev: MessageEvent): void => {
const inputFrame: Float32Array = ev.data as Float32Array;
if (outputStream && !outputStream.isClosed) {
const waveFrame = waveStreamEncoder.encode(inputFrame);
if (!!waveFrame) {
outputStream.writeStreamChunk({
buffer: waveFrame,
isEnd: false,
timeReceived: Date.now(),
});
}
}
};
micInput.connect(workletNode);
workletNode.connect(context.destination);
this.privMediaResources = {
scriptProcessorNode: workletNode,
source: micInput,
stream: mediaStream,
};
connectWorkletToMicInput(context);
})
.catch((): void => {
attachScriptProcessor();
const workletScript = `class SP extends AudioWorkletProcessor {
constructor(options) {
super(options);
}
process(inputs, outputs) {
const input = inputs[0];
const output = [];
for (let channel = 0; channel < input.length; channel += 1) {
output[channel] = input[channel];
}
this.port.postMessage(output[0]);
return true;
}
}
registerProcessor('speech-processor', SP);`;
const blob = new Blob([workletScript], { type: "application/javascript; charset=utf-8" });
this.privSpeechProcessorScript = URL.createObjectURL(blob);
context.audioWorklet
.addModule(this.privSpeechProcessorScript)
.then((): void => {
connectWorkletToMicInput(context);
})
.catch((): void => {
attachScriptProcessor();
});
});
} else {
try {

1
src/common.browser/WebsocketMessageAdapter.ts
Просмотреть файл

@ -3,7 +3,6 @@
// Node.JS specific web socket / browser support.
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
import * as http from "http";
import * as net from "net";
import * as tls from "tls";

3
tests/AudioOutputStreamTests.ts
Просмотреть файл

@ -7,6 +7,9 @@ import { Settings } from "./Settings";
import { closeAsyncObjects } from "./Utilities";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// Override inputs, if necessary

4
tests/AutoSourceLangDetectionTests.ts
Просмотреть файл

@ -27,6 +27,10 @@ import { Settings } from "./Settings";
import { closeAsyncObjects, WaitForCondition } from "./Utilities";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
let objsToClose: any[];
beforeAll((): void => {

4
tests/ConnectionTests.ts
Просмотреть файл

@ -22,6 +22,10 @@ import {
import * as fs from "fs";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
let objsToClose: any[];
beforeAll(() => {

4
tests/ConversationTranscriberTests.ts
Просмотреть файл

@ -23,6 +23,10 @@ import { Settings } from "./Settings";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
import { closeAsyncObjects, RepeatingPullStream, WaitForCondition } from "./Utilities";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
let objsToClose: any[];
beforeAll(() => {

4
tests/ConversationTranslatorTests.ts
Просмотреть файл

@ -26,6 +26,10 @@ import {
} from "./Utilities";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
// eslint-disable-next-line no-console
const consoleInfo = console.info;

4
tests/DiagnosticsTests.ts
Просмотреть файл

@ -9,6 +9,10 @@ import { closeAsyncObjects, WaitForCondition } from "./Utilities";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll((): void => {
// Override inputs, if necessary
Settings.LoadSettings();

4
tests/DialogServiceConnectorTests.ts
Просмотреть файл

@ -46,6 +46,10 @@ import {
} from "./Utilities";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
// eslint-disable-next-line no-console
const consoleInfo = console.info;
const simpleMessageObj = { speak: "This is speech", text: "This is text", type: "message" };

4
tests/DynamicGrammarTests.ts
Просмотреть файл

@ -9,6 +9,10 @@ import {
} from "../src/common.speech/Exports";
import { Settings } from "./Settings";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// Override inputs, if necessary
Settings.LoadSettings();

5
tests/GeneralRecognizerTests.ts
Просмотреть файл

@ -5,6 +5,11 @@ import * as sdk from "../microsoft.cognitiveservices.speech.sdk";
import { Settings } from "./Settings";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
let bufferSize: number;
beforeEach(() => {
// eslint-disable-next-line no-console
console.info("-------------------Starting test case: " + expect.getState().currentTestName + "---------------");

5
tests/IntentRecognizerTests.ts
Просмотреть файл

@ -18,6 +18,11 @@ import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
import { AudioStreamFormatImpl } from "../src/sdk/Audio/AudioStreamFormat";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
let bufferSize: number;
beforeAll(() => {
// override inputs, if necessary

4
tests/LanguageModelTests.ts
Просмотреть файл

@ -5,6 +5,10 @@ import * as sdk from "../microsoft.cognitiveservices.speech.sdk";
import { LanguageUnderstandingModelImpl } from "../src/sdk/LanguageUnderstandingModel";
import { Settings } from "./Settings";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// Override inputs, if necessary
Settings.LoadSettings();

3
tests/LongRunning/SpeechRecoAuthTokenErrorMessageTests.ts
Просмотреть файл

@ -9,6 +9,9 @@ import { Settings } from "../Settings";
import { CreateRepeatingPullStream, WaitForCondition } from "../Utilities";
let objsToClose: any[];
jest.mock("../../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// override inputs, if necessary

4
tests/LongRunning/SpeechRecoAuthTokenRefreshTests.ts
Просмотреть файл

@ -11,6 +11,10 @@ import { CreateRepeatingPullStream, WaitForCondition } from "../Utilities";
let objsToClose: any[];
jest.mock("../../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// override inputs, if necessary
Settings.LoadSettings();

4
tests/LongRunning/SpeechRecoReconnectTests.ts
Просмотреть файл

@ -11,6 +11,10 @@ import { WaitForCondition } from "../Utilities";
let objsToClose: any[];
jest.mock("../../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll((): void => {
// override inputs, if necessary
Settings.LoadSettings();

4
tests/LongRunning/TranslationRecoReconnectTests.ts
Просмотреть файл

@ -12,6 +12,10 @@ import { WaitForCondition } from "../Utilities";
let objsToClose: any[];
jest.mock("../../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll((): void => {
// override inputs, if necessary
Settings.LoadSettings();

4
tests/MeetingTranscriberTests.ts
Просмотреть файл

@ -13,6 +13,10 @@ import { Settings } from "./Settings";
import { closeAsyncObjects } from "./Utilities";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
let objsToClose: any[];
function sleep(milliseconds: number): Promise<any> {

4
tests/PronunciationAssessmentTests.ts
Просмотреть файл

@ -17,6 +17,10 @@ import { Settings } from "./Settings";
import { closeAsyncObjects } from "./Utilities";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
let objsToClose: any[];
beforeAll((): void => {

4
tests/PullInputStreamTests.ts
Просмотреть файл

@ -16,6 +16,10 @@ import { Settings } from "./Settings";
let bufferSize: number;
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// Override inputs, if necessary
Settings.LoadSettings();

4
tests/PushInputStreamTests.ts
Просмотреть файл

@ -15,6 +15,10 @@ import {
} from "../src/sdk/Audio/AudioStreamFormat";
import { Settings } from "./Settings";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
let bufferSize: number;
beforeAll(() => {
// Override inputs, if necessary

4
tests/ReplayableAudioNodeTests.ts
Просмотреть файл

@ -14,6 +14,10 @@ let readCount: number;
const targetBytes: number = 4096;
const defaultAudioFormat: AudioStreamFormatImpl = sdk.AudioStreamFormat.getDefaultInputFormat() as AudioStreamFormatImpl;
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeEach(() => {
readCount = 0;
});

3
tests/SpeechConfigTests.ts
Просмотреть файл

@ -19,6 +19,9 @@ import { closeAsyncObjects } from "./Utilities";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll((): void => {
// Override inputs, if necessary

4
tests/SpeechContextTests.ts
Просмотреть файл

@ -10,6 +10,10 @@ import {
} from "../src/common.speech/Exports";
import { Settings } from "./Settings";
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// Override inputs, if necessary
Settings.LoadSettings();

5
tests/SpeechRecognizerSilenceTests.ts
Просмотреть файл

@ -28,6 +28,11 @@ const Canceled: string = "Canceled";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// override inputs, if necessary
Settings.LoadSettings();

6
tests/SpeechRecognizerTests.ts
Просмотреть файл

@ -59,6 +59,10 @@ const Canceled: string = "Canceled";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// override inputs, if necessary
Settings.LoadSettings();
@ -1793,7 +1797,7 @@ describe.each([true])("Service based tests", (forceNodeWebSocket: boolean) => {
uri = undefined;
});
test("Endpoint URL With Parameter Test", (done: jest.DoneCallback) => {
test.skip("Endpoint URL With Parameter Test", (done: jest.DoneCallback) => {
// eslint-disable-next-line no-console
console.info("Name: Endpoint URL With Parameter Test");

4
tests/SpeechSynthesisTests.ts
Просмотреть файл

@ -22,6 +22,10 @@ import {
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// override inputs, if necessary
Settings.LoadSettings();

4
tests/TranslationRecognizerBasicsTests.ts
Просмотреть файл

@ -26,6 +26,10 @@ import { AudioStreamFormatImpl } from "../src/sdk/Audio/AudioStreamFormat";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// Override inputs, if necessary
Settings.LoadSettings();

5
tests/TranslationRecognizerTests.ts
Просмотреть файл

@ -25,6 +25,11 @@ import { AudioStreamFormatImpl } from "../src/sdk/Audio/AudioStreamFormat";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// Override inputs, if necessary
Settings.LoadSettings();

5
tests/TranslationSynthTests.ts
Просмотреть файл

@ -20,6 +20,11 @@ import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll(() => {
// Override inputs, if necessary
Settings.LoadSettings();

3
tests/VoiceProfileClientTests.ts
Просмотреть файл

@ -13,6 +13,9 @@ import { closeAsyncObjects } from "./Utilities";
import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
let objsToClose: any[];
jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
getAudioWorkerUrl: (): string => "speech-processor.js"
}));
beforeAll((): void => {
// Override inputs, if necessary

4
tsconfig.json
Просмотреть файл

@ -1,7 +1,7 @@
{
"compilerOptions": {
"target": "es5",
"module": "commonjs",
"target": "es6",
"module": "esnext",
"moduleResolution": "node",
"preserveConstEnums": true,
"sourceMap": true,