voice - implement direct `MessagePort` communcation between audio worklet and shared process
This commit is contained in:
Benjamin Pasero
Родитель
7e3f23966d
Коммит
b96621b58c
|
|
@ -72,7 +72,7 @@ const vscodeResources = [
|
|||
'out-build/vs/workbench/contrib/terminal/browser/media/*.sh',
|
||||
'out-build/vs/workbench/contrib/terminal/browser/media/*.zsh',
|
||||
'out-build/vs/workbench/contrib/webview/browser/pre/*.js',
|
||||
'out-build/vs/workbench/services/voiceRecognition/electron-sandbox/bufferInputAudioProcessor.js',
|
||||
'out-build/vs/workbench/services/voiceRecognition/electron-sandbox/bufferedVoiceTranscriber.js',
|
||||
'out-build/vs/**/markdown.css',
|
||||
'out-build/vs/workbench/contrib/tasks/**/*.json',
|
||||
'!**/test/**'
|
||||
|
|
|
|||
|
|
@ -33,18 +33,35 @@ class Protocol implements IMessagePassingProtocol {
|
|||
}
|
||||
}
|
||||
|
||||
export interface IClientConnectionFilter {
|
||||
|
||||
/**
|
||||
* Allows to filter incoming messages to the
|
||||
* server to handle them differently.
|
||||
*
|
||||
* @param e the message event to handle
|
||||
* @returns `true` if the event was handled
|
||||
* and should not be processed by the server.
|
||||
*/
|
||||
handled(e: MessageEvent): boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* An implementation of a `IPCServer` on top of MessagePort style IPC communication.
|
||||
* The clients register themselves via Electron Utility Process IPC transfer.
|
||||
*/
|
||||
export class Server extends IPCServer {
|
||||
|
||||
private static getOnDidClientConnect(): Event<ClientConnectionEvent> {
|
||||
private static getOnDidClientConnect(filter?: IClientConnectionFilter): Event<ClientConnectionEvent> {
|
||||
assertType(isUtilityProcess(process), 'Electron Utility Process');
|
||||
|
||||
const onCreateMessageChannel = new Emitter<MessagePortMain>();
|
||||
|
||||
process.parentPort.on('message', (e: Electron.MessageEvent) => {
|
||||
process.parentPort.on('message', (e: MessageEvent) => {
|
||||
if (filter?.handled(e)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const port = firstOrDefault(e.ports);
|
||||
if (port) {
|
||||
onCreateMessageChannel.fire(port);
|
||||
|
|
@ -66,8 +83,8 @@ export class Server extends IPCServer {
|
|||
});
|
||||
}
|
||||
|
||||
constructor() {
|
||||
super(Server.getOnDidClientConnect());
|
||||
constructor(filter?: IClientConnectionFilter) {
|
||||
super(Server.getOnDidClientConnect(filter));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { Event } from 'vs/base/common/event';
|
||||
import { MessagePortMain, MessageEvent } from 'vs/base/parts/sandbox/node/electronTypes';
|
||||
import { Disposable, toDisposable } from 'vs/base/common/lifecycle';
|
||||
import { IVoiceRecognitionService } from 'vs/platform/voiceRecognition/common/voiceRecognitionService';
|
||||
|
||||
export class VoiceTranscriber extends Disposable {
|
||||
|
||||
constructor(
|
||||
private readonly onDidWindowConnectRaw: Event<MessagePortMain>,
|
||||
@IVoiceRecognitionService private readonly voiceRecognitionService: IVoiceRecognitionService,
|
||||
) {
|
||||
super();
|
||||
|
||||
this.registerListeners();
|
||||
}
|
||||
|
||||
private registerListeners(): void {
|
||||
this._register(this.onDidWindowConnectRaw(port => {
|
||||
const portHandler = async (e: MessageEvent) => {
|
||||
if (!(e.data instanceof Float32Array)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await this.voiceRecognitionService.transcribe(e.data);
|
||||
|
||||
port.postMessage(result);
|
||||
};
|
||||
|
||||
port.on('message', portHandler);
|
||||
this._register(toDisposable(() => port.off('message', portHandler)));
|
||||
|
||||
port.start();
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
|
@ -4,13 +4,16 @@
|
|||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { hostname, release } from 'os';
|
||||
import { MessagePortMain, MessageEvent } from 'vs/base/parts/sandbox/node/electronTypes';
|
||||
import { toErrorMessage } from 'vs/base/common/errorMessage';
|
||||
import { onUnexpectedError, setUnexpectedErrorHandler } from 'vs/base/common/errors';
|
||||
import { combinedDisposable, Disposable, toDisposable } from 'vs/base/common/lifecycle';
|
||||
import { Schemas } from 'vs/base/common/network';
|
||||
import { URI } from 'vs/base/common/uri';
|
||||
import { firstOrDefault } from 'vs/base/common/arrays';
|
||||
import { Emitter } from 'vs/base/common/event';
|
||||
import { ProxyChannel, StaticRouter } from 'vs/base/parts/ipc/common/ipc';
|
||||
import { Server as UtilityProcessMessagePortServer, once } from 'vs/base/parts/ipc/node/ipc.mp';
|
||||
import { IClientConnectionFilter, Server as UtilityProcessMessagePortServer, once } from 'vs/base/parts/ipc/node/ipc.mp';
|
||||
import { CodeCacheCleaner } from 'vs/code/node/sharedProcess/contrib/codeCacheCleaner';
|
||||
import { LanguagePackCachedDataCleaner } from 'vs/code/node/sharedProcess/contrib/languagePackCachedDataCleaner';
|
||||
import { LocalizationsUpdater } from 'vs/code/node/sharedProcess/contrib/localizationsUpdater';
|
||||
|
|
@ -113,13 +116,17 @@ import { nodeSocketFactory } from 'vs/platform/remote/node/nodeSocketFactory';
|
|||
import { NativeEnvironmentService } from 'vs/platform/environment/node/environmentService';
|
||||
import { IVoiceRecognitionService } from 'vs/platform/voiceRecognition/common/voiceRecognitionService';
|
||||
import { VoiceRecognitionService } from 'vs/platform/voiceRecognition/node/voiceRecognitionService';
|
||||
import { VoiceTranscriber } from 'vs/code/node/sharedProcess/contrib/voiceTranscriber';
|
||||
import { RawSharedProcessConnection, SharedProcessLifecycle } from 'vs/platform/sharedProcess/common/sharedProcess';
|
||||
|
||||
class SharedProcessMain extends Disposable {
|
||||
class SharedProcessMain extends Disposable implements IClientConnectionFilter {
|
||||
|
||||
private readonly server = this._register(new UtilityProcessMessagePortServer());
|
||||
private readonly server = this._register(new UtilityProcessMessagePortServer(this));
|
||||
|
||||
private lifecycleService: SharedProcessLifecycleService | undefined = undefined;
|
||||
|
||||
private readonly onDidWindowConnectRaw = this._register(new Emitter<MessagePortMain>());
|
||||
|
||||
constructor(private configuration: ISharedProcessConfiguration) {
|
||||
super();
|
||||
|
||||
|
|
@ -139,7 +146,7 @@ class SharedProcessMain extends Disposable {
|
|||
}
|
||||
};
|
||||
process.once('exit', onExit);
|
||||
once(process.parentPort, 'vscode:electron-main->shared-process=exit', onExit);
|
||||
once(process.parentPort, SharedProcessLifecycle.exit, onExit);
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
|
|
@ -171,7 +178,8 @@ class SharedProcessMain extends Disposable {
|
|||
instantiationService.createInstance(LogsDataCleaner),
|
||||
instantiationService.createInstance(LocalizationsUpdater),
|
||||
instantiationService.createInstance(ExtensionsContributions),
|
||||
instantiationService.createInstance(UserDataProfilesCleaner)
|
||||
instantiationService.createInstance(UserDataProfilesCleaner),
|
||||
instantiationService.createInstance(VoiceTranscriber, this.onDidWindowConnectRaw.event)
|
||||
));
|
||||
}
|
||||
|
||||
|
|
@ -353,7 +361,7 @@ class SharedProcessMain extends Disposable {
|
|||
services.set(IRemoteTunnelService, new SyncDescriptor(RemoteTunnelService));
|
||||
|
||||
// Voice Recognition
|
||||
services.set(IVoiceRecognitionService, new SyncDescriptor(VoiceRecognitionService, undefined, false /* proxied to other processes */));
|
||||
services.set(IVoiceRecognitionService, new SyncDescriptor(VoiceRecognitionService));
|
||||
|
||||
return new InstantiationService(services);
|
||||
}
|
||||
|
|
@ -412,10 +420,6 @@ class SharedProcessMain extends Disposable {
|
|||
// Remote Tunnel
|
||||
const remoteTunnelChannel = ProxyChannel.fromService(accessor.get(IRemoteTunnelService));
|
||||
this.server.registerChannel('remoteTunnel', remoteTunnelChannel);
|
||||
|
||||
// Voice Recognition
|
||||
const voiceRecognitionChannel = ProxyChannel.fromService(accessor.get(IVoiceRecognitionService));
|
||||
this.server.registerChannel('voiceRecognition', voiceRecognitionChannel);
|
||||
}
|
||||
|
||||
private registerErrorHandler(logService: ILogService): void {
|
||||
|
|
@ -434,6 +438,27 @@ class SharedProcessMain extends Disposable {
|
|||
logService.error(`[uncaught exception in sharedProcess]: ${message}`);
|
||||
});
|
||||
}
|
||||
|
||||
handled(e: MessageEvent): boolean {
|
||||
|
||||
// This filter on message port messages will look for
|
||||
// attempts of a window to connect raw to the shared
|
||||
// process to handle these connections separate from
|
||||
// our IPC based protocol.
|
||||
|
||||
if (e.data !== RawSharedProcessConnection.response) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const port = firstOrDefault(e.ports);
|
||||
if (port) {
|
||||
this.onDidWindowConnectRaw.fire(port);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function main(configuration: ISharedProcessConfiguration): Promise<void> {
|
||||
|
|
@ -442,12 +467,12 @@ export async function main(configuration: ISharedProcessConfiguration): Promise<
|
|||
// ready to accept message ports as client connections
|
||||
|
||||
const sharedProcess = new SharedProcessMain(configuration);
|
||||
process.parentPort.postMessage('vscode:shared-process->electron-main=ipc-ready');
|
||||
process.parentPort.postMessage(SharedProcessLifecycle.ipcReady);
|
||||
|
||||
// await initialization and signal this back to electron-main
|
||||
await sharedProcess.init();
|
||||
|
||||
process.parentPort.postMessage('vscode:shared-process->electron-main=init-done');
|
||||
process.parentPort.postMessage(SharedProcessLifecycle.initDone);
|
||||
}
|
||||
|
||||
process.parentPort.once('message', (e: Electron.MessageEvent) => {
|
||||
|
|
|
|||
|
|
@ -63,6 +63,20 @@ export function registerMainProcessRemoteService<T>(id: ServiceIdentifier<T>, ch
|
|||
export const ISharedProcessService = createDecorator<ISharedProcessService>('sharedProcessService');
|
||||
|
||||
export interface ISharedProcessService extends IRemoteService {
|
||||
|
||||
/**
|
||||
* Allows to create a `MessagePort` connection between the
|
||||
* shared process and the renderer process.
|
||||
*
|
||||
* Use this only when you need raw IPC to the shared process
|
||||
* via `postMessage` and `on('message')` of special data structures
|
||||
* like typed arrays.
|
||||
*
|
||||
* Callers have to call `port.start()` after having installed
|
||||
* listeners to enable the data flow.
|
||||
*/
|
||||
createRawConnection(): Promise<MessagePort>;
|
||||
|
||||
notifyRestored(): void;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
export const SharedProcessLifecycle = {
|
||||
exit: 'vscode:electron-main->shared-process=exit',
|
||||
ipcReady: 'vscode:shared-process->electron-main=ipc-ready',
|
||||
initDone: 'vscode:shared-process->electron-main=init-done'
|
||||
};
|
||||
|
||||
export const ChannelSharedProcessConnection = {
|
||||
request: 'vscode:createChannelSharedProcessConnection',
|
||||
response: 'vscode:createChannelSharedProcessConnectionResult'
|
||||
};
|
||||
|
||||
export const RawSharedProcessConnection = {
|
||||
request: 'vscode:createRawSharedProcessConnection',
|
||||
response: 'vscode:createRawSharedProcessConnectionResult'
|
||||
};
|
||||
|
|
@ -18,6 +18,7 @@ import { UtilityProcess } from 'vs/platform/utilityProcess/electron-main/utility
|
|||
import { NullTelemetryService } from 'vs/platform/telemetry/common/telemetryUtils';
|
||||
import { parseSharedProcessDebugPort } from 'vs/platform/environment/node/environmentService';
|
||||
import { assertIsDefined } from 'vs/base/common/types';
|
||||
import { ChannelSharedProcessConnection, RawSharedProcessConnection, SharedProcessLifecycle } from 'vs/platform/sharedProcess/common/sharedProcess';
|
||||
|
||||
export class SharedProcess extends Disposable {
|
||||
|
||||
|
|
@ -41,15 +42,18 @@ export class SharedProcess extends Disposable {
|
|||
|
||||
private registerListeners(): void {
|
||||
|
||||
// Shared process connections from workbench windows
|
||||
validatedIpcMain.on('vscode:createSharedProcessMessageChannel', (e, nonce: string) => this.onWindowConnection(e, nonce));
|
||||
// Shared process channel connections from workbench windows
|
||||
validatedIpcMain.on(ChannelSharedProcessConnection.request, (e, nonce: string) => this.onWindowConnection(e, nonce, ChannelSharedProcessConnection.response));
|
||||
|
||||
// Shared process raw connections from workbench windows
|
||||
validatedIpcMain.on(RawSharedProcessConnection.request, (e, nonce: string) => this.onWindowConnection(e, nonce, RawSharedProcessConnection.response));
|
||||
|
||||
// Lifecycle
|
||||
this._register(this.lifecycleMainService.onWillShutdown(() => this.onWillShutdown()));
|
||||
}
|
||||
|
||||
private async onWindowConnection(e: IpcMainEvent, nonce: string): Promise<void> {
|
||||
this.logService.trace('[SharedProcess] on vscode:createSharedProcessMessageChannel');
|
||||
private async onWindowConnection(e: IpcMainEvent, nonce: string, responseChannel: string): Promise<void> {
|
||||
this.logService.trace(`[SharedProcess] onWindowConnection for: ${responseChannel}`);
|
||||
|
||||
// release barrier if this is the first window connection
|
||||
if (!this.firstWindowConnectionBarrier.isOpen()) {
|
||||
|
|
@ -62,8 +66,10 @@ export class SharedProcess extends Disposable {
|
|||
|
||||
await this.whenReady();
|
||||
|
||||
// connect to the shared process
|
||||
const port = await this.connect();
|
||||
// connect to the shared process passing the responseChannel
|
||||
// as payload to give a hint what the connection is about
|
||||
|
||||
const port = await this.connect(responseChannel);
|
||||
|
||||
// Check back if the requesting window meanwhile closed
|
||||
// Since shared process is delayed on startup there is
|
||||
|
|
@ -75,13 +81,13 @@ export class SharedProcess extends Disposable {
|
|||
}
|
||||
|
||||
// send the port back to the requesting window
|
||||
e.sender.postMessage('vscode:createSharedProcessMessageChannelResult', nonce, [port]);
|
||||
e.sender.postMessage(responseChannel, nonce, [port]);
|
||||
}
|
||||
|
||||
private onWillShutdown(): void {
|
||||
this.logService.trace('[SharedProcess] onWillShutdown');
|
||||
|
||||
this.utilityProcess?.postMessage('vscode:electron-main->shared-process=exit');
|
||||
this.utilityProcess?.postMessage(SharedProcessLifecycle.exit);
|
||||
this.utilityProcess = undefined;
|
||||
}
|
||||
|
||||
|
|
@ -98,9 +104,9 @@ export class SharedProcess extends Disposable {
|
|||
|
||||
const whenReady = new DeferredPromise<void>();
|
||||
if (this.utilityProcess) {
|
||||
this.utilityProcess.once('vscode:shared-process->electron-main=init-done', () => whenReady.complete());
|
||||
this.utilityProcess.once(SharedProcessLifecycle.initDone, () => whenReady.complete());
|
||||
} else {
|
||||
validatedIpcMain.once('vscode:shared-process->electron-main=init-done', () => whenReady.complete());
|
||||
validatedIpcMain.once(SharedProcessLifecycle.initDone, () => whenReady.complete());
|
||||
}
|
||||
|
||||
await whenReady.p;
|
||||
|
|
@ -125,9 +131,9 @@ export class SharedProcess extends Disposable {
|
|||
// Wait for shared process indicating that IPC connections are accepted
|
||||
const sharedProcessIpcReady = new DeferredPromise<void>();
|
||||
if (this.utilityProcess) {
|
||||
this.utilityProcess.once('vscode:shared-process->electron-main=ipc-ready', () => sharedProcessIpcReady.complete());
|
||||
this.utilityProcess.once(SharedProcessLifecycle.ipcReady, () => sharedProcessIpcReady.complete());
|
||||
} else {
|
||||
validatedIpcMain.once('vscode:shared-process->electron-main=ipc-ready', () => sharedProcessIpcReady.complete());
|
||||
validatedIpcMain.once(SharedProcessLifecycle.ipcReady, () => sharedProcessIpcReady.complete());
|
||||
}
|
||||
|
||||
await sharedProcessIpcReady.p;
|
||||
|
|
@ -175,13 +181,13 @@ export class SharedProcess extends Disposable {
|
|||
};
|
||||
}
|
||||
|
||||
async connect(): Promise<MessagePortMain> {
|
||||
async connect(payload?: unknown): Promise<MessagePortMain> {
|
||||
|
||||
// Wait for shared process being ready to accept connection
|
||||
await this.whenIpcReady;
|
||||
|
||||
// Connect and return message port
|
||||
const utilityProcess = assertIsDefined(this.utilityProcess);
|
||||
return utilityProcess.connect();
|
||||
return utilityProcess.connect(payload);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { VSBuffer } from 'vs/base/common/buffer';
|
||||
import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
|
||||
|
||||
export const IVoiceRecognitionService = createDecorator<IVoiceRecognitionService>('voiceRecognitionService');
|
||||
|
|
@ -16,11 +15,11 @@ export interface IVoiceRecognitionService {
|
|||
* Given a buffer of audio data, attempts to
|
||||
* transcribe the spoken words into text.
|
||||
*
|
||||
* @param buffer the audio data obtained from
|
||||
* the microphone as uncompressed PCM data:
|
||||
* @param channelData the raw audio data obtained
|
||||
* from the microphone as uncompressed PCM data:
|
||||
* - 1 channel (mono)
|
||||
* - 16khz sampling rate
|
||||
* - 16bit sample size
|
||||
*/
|
||||
transcribe(buffer: VSBuffer): Promise<string>;
|
||||
transcribe(channelData: Float32Array): Promise<string>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { VSBuffer } from 'vs/base/common/buffer';
|
||||
import { ILogService } from 'vs/platform/log/common/log';
|
||||
import { IVoiceRecognitionService } from 'vs/platform/voiceRecognition/common/voiceRecognitionService';
|
||||
|
||||
|
|
@ -15,8 +14,8 @@ export class VoiceRecognitionService implements IVoiceRecognitionService {
|
|||
@ILogService private readonly logService: ILogService
|
||||
) { }
|
||||
|
||||
async transcribe(buffer: VSBuffer): Promise<string> {
|
||||
this.logService.info(`[voice] transcribe(${buffer.buffer.length / 4}): Begin`);
|
||||
async transcribe(channelData: Float32Array): Promise<string> {
|
||||
this.logService.info(`[voice] transcribe(${channelData.length}): Begin`);
|
||||
|
||||
const modulePath = process.env.VSCODE_VOICE_MODULE_PATH;
|
||||
if (!modulePath) {
|
||||
|
|
@ -24,7 +23,6 @@ export class VoiceRecognitionService implements IVoiceRecognitionService {
|
|||
}
|
||||
|
||||
const now = Date.now();
|
||||
const channelData = this.toFloat32Array(buffer);
|
||||
const conversionTime = Date.now() - now;
|
||||
|
||||
const voiceModule: { transcribe: (audioBuffer: { channelCount: 1; sampleRate: 16000; sampleSize: 16; channelData: Float32Array }, options: { language: string | 'auto'; suppressNonSpeechTokens: boolean }) => Promise<string> } = require.__$__nodeRequire(modulePath);
|
||||
|
|
@ -39,26 +37,8 @@ export class VoiceRecognitionService implements IVoiceRecognitionService {
|
|||
suppressNonSpeechTokens: true
|
||||
});
|
||||
|
||||
this.logService.info(`[voice] transcribe(${buffer.buffer.length / 4}): End (text: "${text}", took: ${Date.now() - now}ms total, ${conversionTime}ms uint8->float32 conversion)`);
|
||||
this.logService.info(`[voice] transcribe(${channelData.length}): End (text: "${text}", took: ${Date.now() - now}ms total, ${conversionTime}ms uint8->float32 conversion)`);
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
private toFloat32Array({ buffer: uint8Array }: VSBuffer): Float32Array {
|
||||
const float32Array = new Float32Array(uint8Array.length / 4);
|
||||
let offset = 0;
|
||||
|
||||
for (let i = 0; i < float32Array.length; i++) {
|
||||
const buffer = new ArrayBuffer(4);
|
||||
const view = new DataView(buffer);
|
||||
|
||||
for (let j = 0; j < 4; j++) {
|
||||
view.setUint8(j, uint8Array[offset++]);
|
||||
}
|
||||
|
||||
float32Array[i] = view.getFloat32(0, true);
|
||||
}
|
||||
|
||||
return float32Array;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { IChannel, IServerChannel, getDelayedChannel } from 'vs/base/parts/ipc/c
|
|||
import { ILogService } from 'vs/platform/log/common/log';
|
||||
import { Disposable } from 'vs/base/common/lifecycle';
|
||||
import { ISharedProcessService } from 'vs/platform/ipc/electron-sandbox/services';
|
||||
import { ChannelSharedProcessConnection, RawSharedProcessConnection } from 'vs/platform/sharedProcess/common/sharedProcess';
|
||||
import { mark } from 'vs/base/common/performance';
|
||||
import { Barrier, timeout } from 'vs/base/common/async';
|
||||
import { acquirePort } from 'vs/base/parts/ipc/electron-sandbox/ipc.mp';
|
||||
|
|
@ -44,7 +45,7 @@ export class SharedProcessService extends Disposable implements ISharedProcessSe
|
|||
// Acquire a message port connected to the shared process
|
||||
mark('code/willConnectSharedProcess');
|
||||
this.logService.trace('Renderer->SharedProcess#connect: before acquirePort');
|
||||
const port = await acquirePort('vscode:createSharedProcessMessageChannel', 'vscode:createSharedProcessMessageChannelResult');
|
||||
const port = await acquirePort(ChannelSharedProcessConnection.request, ChannelSharedProcessConnection.response);
|
||||
mark('code/didConnectSharedProcess');
|
||||
this.logService.trace('Renderer->SharedProcess#connect: connection established');
|
||||
|
||||
|
|
@ -64,4 +65,17 @@ export class SharedProcessService extends Disposable implements ISharedProcessSe
|
|||
registerChannel(channelName: string, channel: IServerChannel<string>): void {
|
||||
this.withSharedProcessConnection.then(connection => connection.registerChannel(channelName, channel));
|
||||
}
|
||||
|
||||
async createRawConnection(): Promise<MessagePort> {
|
||||
|
||||
// Await initialization of the shared process
|
||||
await this.connect();
|
||||
|
||||
// Create a new port to the shared process
|
||||
this.logService.trace('Renderer->SharedProcess#createRawConnection: before acquirePort');
|
||||
const port = await acquirePort(RawSharedProcessConnection.request, RawSharedProcessConnection.response);
|
||||
this.logService.trace('Renderer->SharedProcess#createRawConnection: connection established');
|
||||
|
||||
return port;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
//@ts-check
|
||||
'use strict';
|
||||
|
||||
// @ts-ignore
|
||||
class BufferInputAudioProcessor extends AudioWorkletProcessor {
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.channelCount = 1;
|
||||
this.bufferTimespan = 4000;
|
||||
this.startTime = undefined;
|
||||
|
||||
this.allInputUint8Array = undefined;
|
||||
this.currentInputUint8Arrays = []; // buffer over the duration of bufferTimespan
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {[[Float32Array]]} inputs
|
||||
*/
|
||||
process(inputs) {
|
||||
if (this.startTime === undefined) {
|
||||
this.startTime = Date.now();
|
||||
}
|
||||
|
||||
const inputChannelData = inputs[0][0];
|
||||
if ((!(inputChannelData instanceof Float32Array))) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.currentInputUint8Arrays.push(this.float32ArrayToUint8Array(inputChannelData.slice(0)));
|
||||
|
||||
if (Date.now() - this.startTime > this.bufferTimespan) {
|
||||
const currentInputUint8Arrays = this.currentInputUint8Arrays;
|
||||
this.currentInputUint8Arrays = [];
|
||||
|
||||
this.allInputUint8Array = this.joinUint8Arrays(this.allInputUint8Array ? [this.allInputUint8Array, ...currentInputUint8Arrays] : currentInputUint8Arrays);
|
||||
|
||||
// @ts-ignore
|
||||
this.port.postMessage(this.allInputUint8Array);
|
||||
|
||||
this.startTime = Date.now();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Uint8Array[]} uint8Arrays
|
||||
* @returns {Uint8Array}
|
||||
*/
|
||||
joinUint8Arrays(uint8Arrays) {
|
||||
const result = new Uint8Array(uint8Arrays.reduce((acc, curr) => acc + curr.length, 0));
|
||||
|
||||
let offset = 0;
|
||||
for (const uint8Array of uint8Arrays) {
|
||||
result.set(uint8Array, offset);
|
||||
offset += uint8Array.length;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Float32Array} float32Array
|
||||
* @returns {Uint8Array}
|
||||
*/
|
||||
float32ArrayToUint8Array(float32Array) {
|
||||
const uint8Array = new Uint8Array(float32Array.length * 4);
|
||||
let offset = 0;
|
||||
|
||||
for (let i = 0; i < float32Array.length; i++) {
|
||||
const buffer = new ArrayBuffer(4);
|
||||
const view = new DataView(buffer);
|
||||
view.setFloat32(0, float32Array[i], true);
|
||||
|
||||
for (let j = 0; j < 4; j++) {
|
||||
uint8Array[offset++] = view.getUint8(j);
|
||||
}
|
||||
}
|
||||
|
||||
return uint8Array;
|
||||
}
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
registerProcessor('buffer-input-audio-processor', BufferInputAudioProcessor);
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
//@ts-check
|
||||
'use strict';
|
||||
|
||||
// @ts-ignore
|
||||
class BufferedVoiceTranscriber extends AudioWorkletProcessor {
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.channelCount = 1;
|
||||
this.bufferTimespan = 4000;
|
||||
this.startTime = undefined;
|
||||
|
||||
this.allInputFloat32Array = undefined;
|
||||
this.currentInputFloat32Arrays = []; // buffer over the duration of bufferTimespan
|
||||
|
||||
this.registerListeners();
|
||||
}
|
||||
|
||||
registerListeners() {
|
||||
|
||||
// @ts-ignore
|
||||
const port = this.port;
|
||||
port.onmessage = event => {
|
||||
if (event.data === 'vscode:transferPortToAudioWorklet') {
|
||||
this.sharedProcessPort = event.ports[0];
|
||||
|
||||
this.sharedProcessPort.onmessage = event => {
|
||||
if (typeof event.data === 'string') {
|
||||
port.postMessage(event.data);
|
||||
}
|
||||
};
|
||||
|
||||
this.sharedProcessPort.start();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {[[Float32Array]]} inputs
|
||||
*/
|
||||
process(inputs) {
|
||||
if (this.startTime === undefined) {
|
||||
this.startTime = Date.now();
|
||||
}
|
||||
|
||||
const inputChannelData = inputs[0][0];
|
||||
if ((!(inputChannelData instanceof Float32Array))) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.currentInputFloat32Arrays.push(inputChannelData.slice(0));
|
||||
|
||||
if (Date.now() - this.startTime > this.bufferTimespan && this.sharedProcessPort) {
|
||||
const currentInputFloat32Arrays = this.currentInputFloat32Arrays;
|
||||
this.currentInputFloat32Arrays = [];
|
||||
|
||||
this.allInputFloat32Array = this.joinFloat32Arrays(this.allInputFloat32Array ? [this.allInputFloat32Array, ...currentInputFloat32Arrays] : currentInputFloat32Arrays);
|
||||
|
||||
// @ts-ignore
|
||||
this.sharedProcessPort.postMessage(this.allInputFloat32Array);
|
||||
|
||||
this.startTime = Date.now();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Float32Array[]} float32Arrays
|
||||
* @returns {Float32Array}
|
||||
*/
|
||||
joinFloat32Arrays(float32Arrays) {
|
||||
const result = new Float32Array(float32Arrays.reduce((acc, curr) => acc + curr.length, 0));
|
||||
|
||||
let offset = 0;
|
||||
for (const float32Array of float32Arrays) {
|
||||
result.set(float32Array, offset);
|
||||
offset += float32Array.length;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
registerProcessor('buffered-voice-transcriber', BufferedVoiceTranscriber);
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { registerSharedProcessRemoteService } from 'vs/platform/ipc/electron-sandbox/services';
|
||||
import { IVoiceRecognitionService } from 'vs/platform/voiceRecognition/common/voiceRecognitionService';
|
||||
|
||||
registerSharedProcessRemoteService(IVoiceRecognitionService, 'voiceRecognition');
|
||||
|
|
@ -4,15 +4,14 @@
|
|||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { localize } from 'vs/nls';
|
||||
import { VSBuffer } from 'vs/base/common/buffer';
|
||||
import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation';
|
||||
import { CancellationToken } from 'vs/base/common/cancellation';
|
||||
import { InstantiationType, registerSingleton } from 'vs/platform/instantiation/common/extensions';
|
||||
import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
|
||||
import { IVoiceRecognitionService } from 'vs/platform/voiceRecognition/common/voiceRecognitionService';
|
||||
import { Emitter, Event } from 'vs/base/common/event';
|
||||
import { IProgressService, ProgressLocation } from 'vs/platform/progress/common/progress';
|
||||
import { DeferredPromise } from 'vs/base/common/async';
|
||||
import { FileAccess } from 'vs/base/common/network';
|
||||
import { ISharedProcessService } from 'vs/platform/ipc/electron-sandbox/services';
|
||||
|
||||
export const IWorkbenchVoiceRecognitionService = createDecorator<IWorkbenchVoiceRecognitionService>('workbenchVoiceRecognitionService');
|
||||
|
||||
|
|
@ -29,9 +28,32 @@ export interface IWorkbenchVoiceRecognitionService {
|
|||
transcribe(cancellation: CancellationToken): Event<string>;
|
||||
}
|
||||
|
||||
class BufferInputAudioNode extends AudioWorkletNode {
|
||||
constructor(context: BaseAudioContext, options: AudioWorkletNodeOptions) {
|
||||
super(context, 'buffer-input-audio-processor', options);
|
||||
class BufferedVoiceTranscriber extends AudioWorkletNode {
|
||||
|
||||
constructor(
|
||||
context: BaseAudioContext,
|
||||
options: AudioWorkletNodeOptions,
|
||||
private readonly onDidTranscribe: Emitter<string>,
|
||||
private readonly sharedProcessService: ISharedProcessService
|
||||
) {
|
||||
super(context, 'buffered-voice-transcriber', options);
|
||||
|
||||
this.registerListeners();
|
||||
}
|
||||
|
||||
private registerListeners(): void {
|
||||
this.port.onmessage = e => {
|
||||
if (typeof e.data === 'string') {
|
||||
this.onDidTranscribe.fire(e.data);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async start(token: CancellationToken): Promise<void> {
|
||||
const rawSharedProcessConnection = await this.sharedProcessService.createRawConnection();
|
||||
token.onCancellationRequested(() => rawSharedProcessConnection.close());
|
||||
|
||||
this.port.postMessage('vscode:transferPortToAudioWorklet', [rawSharedProcessConnection]);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -39,8 +61,7 @@ class BufferInputAudioNode extends AudioWorkletNode {
|
|||
// - how to prevent data processing accumulation when processing is slow?
|
||||
// - how to make this a singleton service that enables ref-counting on multiple callers?
|
||||
// - cancellation should flow to the shared process
|
||||
// - voice module should directly transcribe the PCM32 data
|
||||
// - we should transfer the Float32Array directly without serialisation overhead maybe from AudioWorklet?
|
||||
// - voice module should directly transcribe the PCM32 data without wav+file conversion
|
||||
// - the audio worklet should be a TS file (try without any import/export?)
|
||||
|
||||
export class WorkbenchVoiceRecognitionService implements IWorkbenchVoiceRecognitionService {
|
||||
|
|
@ -52,21 +73,20 @@ export class WorkbenchVoiceRecognitionService implements IWorkbenchVoiceRecognit
|
|||
private static readonly AUDIO_CHANNELS = 1;
|
||||
|
||||
constructor(
|
||||
@IVoiceRecognitionService private readonly voiceRecognitionService: IVoiceRecognitionService,
|
||||
@IProgressService private readonly progressService: IProgressService
|
||||
@IProgressService private readonly progressService: IProgressService,
|
||||
@ISharedProcessService private readonly sharedProcessService: ISharedProcessService
|
||||
) { }
|
||||
|
||||
transcribe(cancellation: CancellationToken): Event<string> {
|
||||
const cts = new CancellationTokenSource(cancellation);
|
||||
const emitter = new Emitter<string>();
|
||||
cancellation.onCancellationRequested(() => emitter.dispose());
|
||||
const onDidTranscribe = new Emitter<string>();
|
||||
cancellation.onCancellationRequested(() => onDidTranscribe.dispose());
|
||||
|
||||
this.doTranscribe(emitter, cts.token);
|
||||
this.doTranscribe(onDidTranscribe, cancellation);
|
||||
|
||||
return emitter.event;
|
||||
return onDidTranscribe.event;
|
||||
}
|
||||
|
||||
private async doTranscribe(emitter: Emitter<string>, token: CancellationToken): Promise<void> {
|
||||
private async doTranscribe(onDidTranscribe: Emitter<string>, token: CancellationToken): Promise<void> {
|
||||
return this.progressService.withProgress({
|
||||
location: ProgressLocation.Window,
|
||||
title: localize('voiceTranscription', "Voice Transcription"),
|
||||
|
|
@ -103,39 +123,21 @@ export class WorkbenchVoiceRecognitionService implements IWorkbenchVoiceRecognit
|
|||
recordingDone.complete();
|
||||
});
|
||||
|
||||
await audioContext.audioWorklet.addModule(FileAccess.asBrowserUri('vs/workbench/services/voiceRecognition/electron-sandbox/bufferInputAudioProcessor.js').toString(true));
|
||||
await audioContext.audioWorklet.addModule(FileAccess.asBrowserUri('vs/workbench/services/voiceRecognition/electron-sandbox/bufferedVoiceTranscriber.js').toString(true));
|
||||
|
||||
const bufferInputAudioTarget = new BufferInputAudioNode(audioContext, {
|
||||
const bufferedVoiceTranscriberTarget = new BufferedVoiceTranscriber(audioContext, {
|
||||
channelCount: WorkbenchVoiceRecognitionService.AUDIO_CHANNELS,
|
||||
channelCountMode: 'explicit'
|
||||
});
|
||||
}, onDidTranscribe, this.sharedProcessService);
|
||||
await bufferedVoiceTranscriberTarget.start(token);
|
||||
|
||||
microphoneSource.connect(bufferInputAudioTarget);
|
||||
microphoneSource.connect(bufferedVoiceTranscriberTarget);
|
||||
|
||||
progress.report({ message: localize('voiceTranscriptionRecording', "Recording from microphone...") });
|
||||
|
||||
bufferInputAudioTarget.port.onmessage = async e => {
|
||||
if (e.data instanceof Uint8Array) {
|
||||
this.doTranscribeChunk(e.data, emitter, token);
|
||||
}
|
||||
};
|
||||
|
||||
return recordingDone.p;
|
||||
});
|
||||
}
|
||||
|
||||
private async doTranscribeChunk(data: Uint8Array, emitter: Emitter<string>, token: CancellationToken): Promise<void> {
|
||||
if (token.isCancellationRequested) {
|
||||
return;
|
||||
}
|
||||
|
||||
const text = await this.voiceRecognitionService.transcribe(VSBuffer.wrap(data));
|
||||
if (token.isCancellationRequested) {
|
||||
return;
|
||||
}
|
||||
|
||||
emitter.fire(text);
|
||||
}
|
||||
}
|
||||
|
||||
// Register Service
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ export class TestSharedProcessService implements ISharedProcessService {
|
|||
|
||||
declare readonly _serviceBrand: undefined;
|
||||
|
||||
createRawConnection(): never { throw new Error('Not Implemented'); }
|
||||
|
||||
getChannel(channelName: string): any { return undefined; }
|
||||
|
||||
registerChannel(channelName: string, channel: any): void { }
|
||||
|
|
|
|||
|
|
@ -77,7 +77,6 @@ import 'vs/workbench/services/environment/electron-sandbox/shellEnvironmentServi
|
|||
import 'vs/workbench/services/integrity/electron-sandbox/integrityService';
|
||||
import 'vs/workbench/services/workingCopy/electron-sandbox/workingCopyBackupService';
|
||||
import 'vs/workbench/services/checksum/electron-sandbox/checksumService';
|
||||
import 'vs/workbench/services/voiceRecognition/electron-sandbox/voiceRecognitionService';
|
||||
import 'vs/workbench/services/voiceRecognition/electron-sandbox/workbenchVoiceRecognitionService';
|
||||
import 'vs/platform/remote/electron-sandbox/sharedProcessTunnelService';
|
||||
import 'vs/workbench/services/tunnel/electron-sandbox/tunnelService';
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче