feat: Переход слушания на worklet

salute-developers · Jul 16, 2024 · 36e72da · 36e72da
1 parent 206d948
commit 36e72da
Show file tree

Hide file tree

Showing 2 changed files with 103 additions and 59 deletions.
diff --git a/src/assistantSdk/voice/listener/navigatorAudioProvider.ts b/src/assistantSdk/voice/listener/navigatorAudioProvider.ts
@@ -1,46 +1,21 @@
 import { createAudioContext } from '../audioContext';
 
-/**
- * Понижает sample rate c inSampleRate до значения outSampleRate и преобразует Float32Array в ArrayBuffer
- * @param buffer Аудио
- * @param inSampleRate текущий sample rate
- * @param outSampleRate требуемый sample rate
- * @returns Аудио со значением sample rate = outSampleRate
- */
-const downsampleBuffer = (buffer: Float32Array, inSampleRate: number, outSampleRate: number): ArrayBuffer => {
-    if (outSampleRate > inSampleRate) {
-        throw new Error('downsampling rate show be smaller than original sample rate');
-    }
-    const sampleRateRatio = inSampleRate / outSampleRate;
-    const newLength = Math.round(buffer.length / sampleRateRatio);
-    const result = new Int16Array(newLength);
-
-    let offsetResult = 0;
-    let offsetBuffer = 0;
-
-    while (offsetResult < result.length) {
-        const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
-        let accum = 0;
-        let count = 0;
-        for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
-            accum += buffer[i];
-            count++;
-        }
-
-        result[offsetResult] = Math.min(1, accum / count) * 0x7fff;
-        offsetResult++;
-        offsetBuffer = nextOffsetBuffer;
-    }
-
-    return result.buffer;
-};
+import { worker } from './worker';
+
+async function initWorklet(context: AudioContext) {
+    const blob: Blob = new Blob([worker], { type: 'application/javascript; charset=utf-8' });
+    const url: string = URL.createObjectURL(blob);
+
+    await context.audioWorklet.addModule(url);
+    URL.revokeObjectURL(url);
+}
 
 const TARGET_SAMPLE_RATE = 16000;
 const IS_FIREFOX = typeof window !== 'undefined' && navigator.userAgent.toLowerCase().indexOf('firefox') > -1;
-const IS_SAFARI = typeof window !== 'undefined' && /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
 
 let context: AudioContext;
-let processor: ScriptProcessorNode;
+let pcmProcessingNode: AudioWorkletNode;
+let source: MediaStreamAudioSourceNode;
 
 /**
  * Преобразует stream в чанки (кусочки), и передает их в cb,
@@ -55,7 +30,6 @@ const createAudioRecorder = (
 ): Promise<() => void> =>
     new Promise((resolve) => {
         let state: 'inactive' | 'recording' = 'inactive';
-        let input: MediaStreamAudioSourceNode;
 
         const stop = () => {
             if (state === 'inactive') {
@@ -66,10 +40,10 @@ const createAudioRecorder = (
             stream.getTracks().forEach((track) => {
                 track.stop();
             });
-            input.disconnect();
+            source.disconnect(pcmProcessingNode);
         };
 
-        const start = () => {
+        const start = async () => {
             if (state !== 'inactive') {
                 throw new Error("Can't start not inactive recorder");
             }
@@ -81,34 +55,22 @@ const createAudioRecorder = (
                     // firefox не умеет выравнивать samplerate, будем делать это самостоятельно
                     sampleRate: IS_FIREFOX ? undefined : TARGET_SAMPLE_RATE,
                 });
+                await initWorklet(context);
             }
 
-            input = context.createMediaStreamSource(stream);
-
-            if (!processor) {
-                processor = context.createScriptProcessor(2048, 1, 1);
-            }
-
-            const listener = (e: AudioProcessingEvent) => {
-                const buffer = e.inputBuffer.getChannelData(0);
-                const data = downsampleBuffer(buffer, context.sampleRate, TARGET_SAMPLE_RATE);
+            source = context.createMediaStreamSource(stream);
 
+            pcmProcessingNode = new AudioWorkletNode(context, 'pcm-processor', { sampleRate: context.sampleRate });
+            pcmProcessingNode.port.onmessage = (e) => {
+                const { data } = e;
                 const last = state === 'inactive';
-                // // отсылаем только чанки где есть звук voiceData > 0, т.к.
-                // // в safari первые несколько чанков со звуком пустые
-                // const dataWithVoice = new Uint8Array(data).some((voiceData) => voiceData > 0);
+
                 resolve(stop);
                 cb(data, last);
-
-                if (last) {
-                    processor.removeEventListener('audioprocess', listener);
-                }
             };
 
-            processor.addEventListener('audioprocess', listener);
-
-            input.connect(processor);
-            processor.connect(context.destination);
+            source.connect(pcmProcessingNode);
+            pcmProcessingNode.connect(context.destination);
         };
 
         start();

diff --git a/src/assistantSdk/voice/listener/worker.js b/src/assistantSdk/voice/listener/worker.js
@@ -0,0 +1,82 @@
+export const worker = `
+const DEFAULT_BUFFER_SIZE = 2048;
+const DEFAULT_SAMPLE_RATE = 16000;
+
+function encode(buffer, sampleRate, targetSampleRate) {
+    if (sampleRate > targetSampleRate) {
+        throw new Error('downsampling rate show be smaller than original sample rate');
+    }
+
+    const sampleRateRatio = sampleRate / targetSampleRate;
+    const newLength = Math.round(buffer.length / sampleRateRatio);
+    const result = new Int16Array(newLength);
+
+    let offsetResult = 0;
+    let offsetBuffer = 0;
+
+    while (offsetResult < result.length) {
+        const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
+        let accum = 0;
+        let count = 0;
+        for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
+            accum += buffer[i];
+            count++;
+        }
+
+        result[offsetResult] = Math.min(1, accum / count) * 0x7fff;
+        offsetResult++;
+        offsetBuffer = nextOffsetBuffer;
+    }
+
+    return result.buffer;
+}
+
+class PcmWorkletProcessor extends AudioWorkletProcessor {
+    _bufferSize = DEFAULT_BUFFER_SIZE;
+    _bytesWritten = 0;
+    _buffer = new Float32Array(this._bufferSize);
+    _sampleRate = DEFAULT_SAMPLE_RATE;
+    _targetSampleRate = DEFAULT_SAMPLE_RATE;
+
+    constructor(options) {
+        super();
+
+        this._bufferSize = options.bufferSize || DEFAULT_BUFFER_SIZE;
+        this._buffer = new Float32Array(this._bufferSize);
+        this._sampleRate = options.sampleRate || DEFAULT_SAMPLE_RATE;
+        this._targetSampleRate = options.targetSampleRate || DEFAULT_SAMPLE_RATE;
+    }
+
+    append(channelData) {
+        if (!channelData) {
+            return;
+        }
+
+        for (let i = 0; i < channelData.length; i++) {
+            this._buffer[this._bytesWritten++] = channelData[i];
+            if (this._bytesWritten >= this._bufferSize) {
+                this.push();
+            }
+        }
+    }
+
+    process (inputs, outputs, parameters) {
+        this.append(inputs[0][0]);
+        return true;
+    }
+
+    push() {
+        this.port.postMessage(
+            encode(
+                this._bytesWritten < this.bufferSize
+                    ? this._buffer.slice(0, this._bytesWritten)
+                    : this._buffer,
+                this._sampleRate,
+                this._targetSampleRate)
+        );
+        this._bytesWritten = 0;
+    }
+}
+
+registerProcessor('pcm-processor', PcmWorkletProcessor);
+`;