diff --git a/rollup.config.js b/rollup.config.js index 4688b8da02..ac3109885f 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -9,7 +9,6 @@ import copy from 'rollup-plugin-copy'; import pkg from './package.json'; const common = { - input: 'src/index.ts', plugins: [ commonjs({ extensions: ['.js', '.jsx', '.ts', '.tsx'], @@ -44,7 +43,7 @@ const getUmdConfig = (fileName, input) => ({ export default [ { ...common, - input: ['src/index.ts', 'src/createAssistantDevOrigin.ts'], + input: ['src/index.ts', 'src/createAssistantDevOrigin.ts','src/assistantSdk/voice/listener/worklet.js','src/assistantSdk/listenSdk/vps.worker.ts',], output: { ...common.output, dir: 'dist', @@ -76,6 +75,9 @@ export default [ 'src/assistantSdk/assistant.ts', 'src/mock.ts', 'src/index.ts', + 'src/assistantSdk/voice/listener/worklet.js', + 'src/assistantSdk/listenSdk/vps.worker.ts', + 'src/assistantSdk/listenSdk/listenSdk.ts', ], output: { ...common.output, diff --git a/src/assistantSdk/client/protocol.ts b/src/assistantSdk/client/protocol.ts index 57eb224dd0..b43ac4f971 100644 --- a/src/assistantSdk/client/protocol.ts +++ b/src/assistantSdk/client/protocol.ts @@ -280,11 +280,11 @@ export const createProtocol = ( status = 'connected'; - window.clearTimeout(clearReadyTimer); + clearTimeout(clearReadyTimer); /// считаем коннект = ready, если по истечении таймаута сокет не был разорван /// т.к бек может разрывать сокет, если с settings что-то не так - clearReadyTimer = window.setTimeout(() => { + clearReadyTimer = setTimeout(() => { if (status !== 'connected') { return; } @@ -349,7 +349,7 @@ export const createProtocol = ( }, init: () => { // в отличии от reconnect не обрывает коннект если он в порядке - if (status === 'ready' && window.navigator.onLine) { + if (status === 'ready' && (!window || window.navigator.onLine)) { return Promise.resolve(); } diff --git a/src/assistantSdk/client/transport.ts b/src/assistantSdk/client/transport.ts index 74d8d61fff..09e809fa64 100644 --- a/src/assistantSdk/client/transport.ts +++ b/src/assistantSdk/client/transport.ts @@ -46,20 +46,20 @@ export const createTransport = ({ createWS = defaultWSCreator, checkCertUrl }: C status = 'connecting'; emit('connecting'); - if (!hasCert && window.navigator.onLine) { - const okay = await checkCert(checkCertUrl!); + // if (!hasCert && window?.navigator.onLine) { + // const okay = await checkCert(checkCertUrl!); - if (!okay) { - status = 'closed'; - emit('close'); + // if (!okay) { + // status = 'closed'; + // emit('close'); - emit('error', new Error('Cert authority invalid')); + // emit('error', new Error('Cert authority invalid')); - return; - } + // return; + // } - hasCert = true; - } + // hasCert = true; + // } webSocket = createWS(url); @@ -70,7 +70,7 @@ export const createTransport = ({ createWS = defaultWSCreator, checkCertUrl }: C return; } - window.clearTimeout(retryTimeoutId); + clearTimeout(retryTimeoutId); retries = 0; @@ -90,10 +90,10 @@ export const createTransport = ({ createWS = defaultWSCreator, checkCertUrl }: C // пробуем переподключаться, если возникла ошибка при коннекте if (!webSocket || (webSocket.readyState === 3 && !stopped)) { - window.clearTimeout(retryTimeoutId); + clearTimeout(retryTimeoutId); if (retries < 2) { - retryTimeoutId = window.setTimeout(() => { + retryTimeoutId = setTimeout(() => { // eslint-disable-next-line @typescript-eslint/no-use-before-define open(url); @@ -129,17 +129,17 @@ export const createTransport = ({ createWS = defaultWSCreator, checkCertUrl }: C return; } - window.setTimeout(() => reconnect(url)); + setTimeout(() => reconnect(url)); close(); }; const send = (data: Uint8Array) => { - if (!window.navigator.onLine) { - close(); - emit('error'); - throw new Error('The client seems to be offline'); - } + // if (!window?.navigator.onLine) { + // close(); + // emit('error'); + // throw new Error('The client seems to be offline'); + // } webSocket.send(data); }; @@ -147,7 +147,7 @@ export const createTransport = ({ createWS = defaultWSCreator, checkCertUrl }: C return { close, get isOnline() { - return window.navigator.onLine; + return window?.navigator.onLine === true; }, on, open, diff --git a/src/assistantSdk/listenSdk/listenSdk.ts b/src/assistantSdk/listenSdk/listenSdk.ts new file mode 100644 index 0000000000..e98828b0c8 --- /dev/null +++ b/src/assistantSdk/listenSdk/listenSdk.ts @@ -0,0 +1,29 @@ +import type { VpsConfiguration } from '../../typings'; +import { createNavigatorAudioProvider } from '../voice/listener/navigatorAudioProvider'; +import { createVoiceListener } from '../voice/listener/voiceListener'; + +export type InifinteListenParams = Omit & { + token: string; + voiceMeta: Record; +}; + +const worker = new Worker(new URL('./vps.worker.js', import.meta.url), { type: 'module' }); + +export function createInifiniteListen(config: InifinteListenParams, restApiUrl: string) { + const listener = createVoiceListener( + (cb, ...args) => createNavigatorAudioProvider(cb, ...args), + (port: MessagePort) => { + worker.postMessage({ type: 'start' }, [port]); + }, + ); + worker.postMessage({ type: 'init', params: [config, restApiUrl] }); + + return { + start: () => { + listener.listen(); + }, + stop: () => { + listener.stop(); + }, + }; +} diff --git a/src/assistantSdk/listenSdk/vps.worker.ts b/src/assistantSdk/listenSdk/vps.worker.ts new file mode 100644 index 0000000000..977fa55491 --- /dev/null +++ b/src/assistantSdk/listenSdk/vps.worker.ts @@ -0,0 +1,100 @@ +import { v4 } from 'uuid'; + +import { createClient } from '../client/client'; +import { createProtocol } from '../client/protocol'; +import { createTransport } from '../client/transport'; + +function convertFieldValuesToString< + Obj extends Record, + ObjStringified = { [key in keyof Obj]: string }, +>(object: Obj): ObjStringified { + return Object.keys(object).reduce((acc: Record, key: string) => { + if (object[key]) { + acc[key] = + typeof object[key] === 'string' && (object[key] as string).startsWith('{') + ? (object[key] as string) + : JSON.stringify(object[key]); + } + return acc; + }, {}) as ObjStringified; +} + +let _client: ReturnType; +let _stream: (chunk: Uint8Array, last: boolean) => void; +let _push: (text: string) => void; + +function init({ token, voiceMeta, ...config }, restApiUrl: string) { + const sessionId = v4(); + const convertedVoiceMeta = convertFieldValuesToString(voiceMeta); + const transport = createTransport({}); + const protocol = createProtocol(transport, { ...config, getToken: () => token }); + _client = createClient(protocol, undefined, { + getVoiceMeta: () => convertedVoiceMeta, + }); + _push = (message: string) => + fetch(restApiUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json;charset=utf-8', + }, + body: JSON.stringify({ + sessionId, + message, + timestamp: new Date().toISOString(), + }), + }); + + _client.on('stt', ({ text, response }) => { + if (text?.data) { + _push(text.data); + return; + } + + if (response) { + const { decoderResultField } = response; + + if (decoderResultField?.hypothesis?.length) { + _push(decoderResultField.hypothesis[0].normalizedText || ''); + } + } + }); +} + +function start(port: MessagePort) { + _client + .init() + .then(() => { + _client.createVoiceStream(({ sendVoice }) => { + _stream = sendVoice; + + return Promise.resolve(); + }, {}); + }) + .catch((err) => { + console.error(err); + }); + + port.onmessage = (event) => { + _stream?.(event.data, false); + }; +} + +function handleChunk(chunk: Uint8Array, last: boolean) { + _stream?.(chunk, last); +} + +self.addEventListener('message', function (e) { + switch (e.data.type) { + case 'init': + init(e.data.params[0], e.data.params[1]); + break; + case 'start': + start(e.ports[0]); + break; + case 'chunk': + handleChunk(e.data.params[0], e.data.params[1]); + break; + case 'stop': + break; + } +}); diff --git a/src/assistantSdk/voice/listener/navigatorAudioProvider.ts b/src/assistantSdk/voice/listener/navigatorAudioProvider.ts index 0626d0b328..66abe42fbc 100644 --- a/src/assistantSdk/voice/listener/navigatorAudioProvider.ts +++ b/src/assistantSdk/voice/listener/navigatorAudioProvider.ts @@ -1,13 +1,7 @@ import { createAudioContext } from '../audioContext'; -import { worker } from './worker'; - async function initWorklet(context: AudioContext) { - const blob: Blob = new Blob([worker], { type: 'application/javascript; charset=utf-8' }); - const url: string = URL.createObjectURL(blob); - - await context.audioWorklet.addModule(url); - URL.revokeObjectURL(url); + await context.audioWorklet.addModule(new URL('./worklet.js', import.meta.url)); } const TARGET_SAMPLE_RATE = 16000; @@ -27,8 +21,9 @@ let analyser: AnalyserNode | null = null; */ const createAudioRecorder = ( stream: MediaStream, - cb: (buffer: ArrayBuffer, analyserArray: Uint8Array | null, last: boolean) => void, + cb?: (buffer: ArrayBuffer, analyserArray: Uint8Array | null, last: boolean) => void, useAnalyser?: boolean, + onGetPort?: (port: MessagePort) => void, ): Promise<() => void> => new Promise((resolve) => { let state: 'inactive' | 'recording' = 'inactive'; @@ -65,21 +60,28 @@ const createAudioRecorder = ( analyser = context.createAnalyser(); } - pcmProcessingNode = new AudioWorkletNode(context, 'pcm-processor', { sampleRate: context.sampleRate }); - pcmProcessingNode.port.onmessage = (e) => { - const { data } = e; - const last = state === 'inactive'; - - let analyserArray: Uint8Array | null = null; - if (analyser) { - analyserArray = new Uint8Array(analyser.frequencyBinCount); - - analyser?.getByteTimeDomainData(analyserArray); - } - - resolve(stop); - cb(data, analyserArray, last); - }; + pcmProcessingNode = new AudioWorkletNode(context, 'pcm-processor', { + processorOptions: { sampleRate: context.sampleRate }, + }); + if (onGetPort) { + setTimeout(() => resolve(stop)); + onGetPort(pcmProcessingNode.port); + } else { + pcmProcessingNode.port.onmessage = (e) => { + const { data } = e; + const last = state === 'inactive'; + + let analyserArray: Uint8Array | null = null; + if (analyser) { + analyserArray = new Uint8Array(analyser.frequencyBinCount); + + analyser?.getByteTimeDomainData(analyserArray); + } + + resolve(stop); + cb && cb(data, analyserArray, last); + }; + } source.connect(pcmProcessingNode); pcmProcessingNode.connect(context.destination); @@ -95,15 +97,16 @@ const createAudioRecorder = ( * @returns Promise, который содержит функцию прерывающую слушание */ export const createNavigatorAudioProvider = ( - cb: (buffer: ArrayBuffer, analyserArray: Uint8Array | null, last: boolean) => void, + cb?: (buffer: ArrayBuffer, analyserArray: Uint8Array | null, last: boolean) => void, useAnalyser?: boolean, + onGetPort?: (port: MessagePort) => void, ): Promise<() => void> => navigator.mediaDevices .getUserMedia({ audio: true, }) .then((stream) => { - return createAudioRecorder(stream, cb, useAnalyser); + return createAudioRecorder(stream, cb, useAnalyser, onGetPort); }) .catch((err) => { if (window.location.protocol === 'http:') { diff --git a/src/assistantSdk/voice/listener/voiceListener.ts b/src/assistantSdk/voice/listener/voiceListener.ts index fd6619dc5a..9dda7d8ef3 100644 --- a/src/assistantSdk/voice/listener/voiceListener.ts +++ b/src/assistantSdk/voice/listener/voiceListener.ts @@ -16,7 +16,10 @@ type VoiceStreamEvents = { * @param createAudioProvider Источник голоса * @returns Api для запуска и остановки слушания */ -export const createVoiceListener = (createAudioProvider = createNavigatorAudioProvider) => { +export const createVoiceListener = ( + createAudioProvider = createNavigatorAudioProvider, + onGetPort?: (port: MessagePort) => void, +) => { const { emit, on } = createNanoEvents(); let stopRecord: () => void; let status: VoiceListenerStatus = 'stopped'; @@ -30,13 +33,17 @@ export const createVoiceListener = (createAudioProvider = createNavigatorAudioPr emit('status', 'stopped'); }; - const listen = (handleVoice: VoiceHandler): Promise => { + const listen = (handleVoice?: VoiceHandler): Promise => { cancelableToken = { current: false }; let capturedToken = cancelableToken; status = 'started'; emit('status', 'started'); - return createAudioProvider((data, analyser, last) => handleVoice(new Uint8Array(data), analyser, last)) + return createAudioProvider( + handleVoice ? (data, analyser, last) => handleVoice(new Uint8Array(data), analyser, last) : undefined, + false, + onGetPort, + ) .then((recStop) => { stopRecord = recStop; }) diff --git a/src/assistantSdk/voice/listener/worker.js b/src/assistantSdk/voice/listener/worklet.js similarity index 57% rename from src/assistantSdk/voice/listener/worker.js rename to src/assistantSdk/voice/listener/worklet.js index c91dcf57a5..23238c91b2 100644 --- a/src/assistantSdk/voice/listener/worker.js +++ b/src/assistantSdk/voice/listener/worklet.js @@ -1,4 +1,3 @@ -export const worker = ` const DEFAULT_BUFFER_SIZE = 2048; const DEFAULT_SAMPLE_RATE = 16000; @@ -37,14 +36,30 @@ class PcmWorkletProcessor extends AudioWorkletProcessor { _buffer = new Float32Array(this._bufferSize); _sampleRate = DEFAULT_SAMPLE_RATE; _targetSampleRate = DEFAULT_SAMPLE_RATE; + _worker; + _port; constructor(options) { super(); - this._bufferSize = options.bufferSize || DEFAULT_BUFFER_SIZE; + this._bufferSize = options.processorOptions?.bufferSize || DEFAULT_BUFFER_SIZE; this._buffer = new Float32Array(this._bufferSize); - this._sampleRate = options.sampleRate || DEFAULT_SAMPLE_RATE; - this._targetSampleRate = options.targetSampleRate || DEFAULT_SAMPLE_RATE; + this._sampleRate = options.processorOptions?.sampleRate || DEFAULT_SAMPLE_RATE; + this._targetSampleRate = options.processorOptions?.targetSampleRate || DEFAULT_SAMPLE_RATE; + this._port = options.processorOptions?.port; + + // console.log('worker will init', options.processorOptions?.workerUrl, options.processorOptions?.workerParams) + + // try { + // this._worker = options.processorOptions?.workerUrl && new Worker(new URL(options.processorOptions?.workerUrl, import.meta.url), { type: "module" }); + // } catch (err) { + // console.error(err) + // } + + // this._worker?.postMessage({ type: 'init', params: options.processorOptions?.workerParams }); + // this._worker?.postMessage({ type: 'start' }); + + // console.log('worker started') } append(channelData) { @@ -60,23 +75,26 @@ class PcmWorkletProcessor extends AudioWorkletProcessor { } } - process (inputs, outputs, parameters) { + process(inputs, outputs, parameters) { this.append(inputs[0][0]); return true; } push() { - this.port.postMessage( - encode( - this._bytesWritten < this.bufferSize - ? this._buffer.slice(0, this._bytesWritten) - : this._buffer, - this._sampleRate, - this._targetSampleRate) + const chunk = encode( + this._bytesWritten < this.bufferSize ? this._buffer.slice(0, this._bytesWritten) : this._buffer, + this._sampleRate, + this._targetSampleRate, ); + + if (this._port) { + this._port.postMessage({ type: 'chunk', params: [chunk, false] }); + } else { + this.port.postMessage(chunk); + } + this._bytesWritten = 0; } } registerProcessor('pcm-processor', PcmWorkletProcessor); -`; diff --git a/src/index.ts b/src/index.ts index 108cf1300b..b679307775 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,3 +22,4 @@ export { createNavigatorAudioProvider } from './assistantSdk/voice/listener/navi export * from './typings'; export { GetHistoryResponse } from './proto'; export { PacketWrapperFromServer } from './assistantSdk/voice/recognizers/asr'; +export { createInifiniteListen } from './assistantSdk/listenSdk/listenSdk'; diff --git a/tsconfig.json b/tsconfig.json index 4293594204..254f9d747b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,7 @@ "rootDir": "src", "outDir": "dist", "target": "ES5", - "lib": ["ES2020.Promise", "esnext.asynciterable", "DOM"], + "lib": ["ES2020.Promise", "esnext.asynciterable", "DOM", "WebWorker.ImportScripts"], "declaration": true, "declarationMap": true, "composite": false,