feat: web-worker v2

salute-developers · Sep 11, 2024 · 6c3de01 · 6c3de01
1 parent cdf16a4
commit 6c3de01
Show file tree

Hide file tree

Showing 11 changed files with 6,579 additions and 15 deletions.
diff --git a/rollup.config.js b/rollup.config.js
@@ -43,7 +43,11 @@ const getUmdConfig = (fileName, input) => ({
 export default [
     {
         ...common,
-        input: ['src/index.ts', 'src/createAssistantDevOrigin.ts','src/assistantSdk/voice/listener/worklet.js','src/assistantSdk/listenSdk/vps.worker.ts',],
+        input: [
+            'src/index.ts',
+            'src/createAssistantDevOrigin.ts',
+            'src/assistantSdk/voice/listener/worklet.js' /*,'src/assistantSdk/listenSdk/vps.worker.ts','src/assistantSdk/listenSdk/sw.ts'*/,
+        ],
         output: {
             ...common.output,
             dir: 'dist',
@@ -76,8 +80,9 @@ export default [
             'src/mock.ts',
             'src/index.ts',
             'src/assistantSdk/voice/listener/worklet.js',
-            'src/assistantSdk/listenSdk/vps.worker.ts',
-            'src/assistantSdk/listenSdk/listenSdk.ts',
+            // 'src/assistantSdk/listenSdk/vps.worker.ts',
+            // 'src/assistantSdk/listenSdk/listenSdk.ts',
+            // 'src/assistantSdk/listenSdk/sw.ts',
         ],
         output: {
             ...common.output,

diff --git a/src/assistantSdk/assistant.ts b/src/assistantSdk/assistant.ts
@@ -281,17 +281,7 @@ export const createAssistant = ({
         getVoiceMeta: () => (getVoiceMeta ? convertFieldValuesToString(getVoiceMeta()) : {}),
     });
     const voice = createVoice(
-        (() => ({
-            init: () => Promise.resolve(),
-            createVoiceStream: (func) =>
-                func({
-                    sendVoice: (params) => {
-                        window.postMessage(JSON.stringify({ voice: params }));
-                    },
-                    messageId: '123',
-                }),
-            on: () => () => {},
-        }))(),
+        client,
         settings,
         (event) => {
             if (typeof event.tts !== 'undefined') {

diff --git a/src/index.ts b/src/index.ts
@@ -22,4 +22,8 @@ export { createNavigatorAudioProvider } from './assistantSdk/voice/listener/navi
 export * from './typings';
 export { GetHistoryResponse } from './proto';
 export { PacketWrapperFromServer } from './assistantSdk/voice/recognizers/asr';
-export { createInifiniteListen } from './assistantSdk/listenSdk/listenSdk';
+// export { createInifiniteListen } from './assistantSdk/listenSdk/listenSdk';
+export { createVoiceListener } from './assistantSdk/voice/listener/voiceListener';
+export { createClient } from './assistantSdk/client/client';
+export { createProtocol } from './assistantSdk/client/protocol';
+export { createTransport } from './assistantSdk/client/transport';
diff --git a/test/.proxyrc.js b/test/.proxyrc.js
@@ -0,0 +1,15 @@
+const { createProxyMiddleware } = require("http-proxy-middleware");
+
+module.exports = function (app) {
+  app.use(
+    createProxyMiddleware("/stt", {
+      target: "http://localhost:3333",
+      onError(err, req, res) {
+        res.writeHead(200, {
+            'Content-Type': 'text/plain'
+        });
+        res.end('ok');
+      },
+    })
+  );
+};
diff --git a/test/index.html b/test/index.html
@@ -0,0 +1,61 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8"/>
+    <title>My First Parcel App</title>
+    <style>
+      html {
+        position: fixed;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        font-size: 16px;
+      }
+
+      body {
+        background: transparent;
+        position: fixed;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+      }
+
+      input {
+        margin-top: 50%;
+        margin-left: 30%;
+        font-size: 3rem;
+      }
+
+      audio {
+        visibility: hidden;
+      }
+
+      .tick {
+        display: grid;
+        grid-template-columns: auto 1fr;
+        gap: 1em;
+        padding: 1em;
+        background-color: #fff;
+        color: #000;
+      }
+    </style>
+    <script type="module" src="index.ts"></script>
+
+  <body>
+    <audio
+      paused
+      loop
+      controls
+      src="https://upload.wikimedia.org/wikipedia/commons/0/07/Silence.ogg"></audio>
+    </head>
+    <!-- <input type="text" /> -->
+    <div class="tick">
+      <span>Set Timeout Ticks</span>
+      <span id="timeoutTicks">0</span>
+      <span>Seconds from start</span>
+      <span id="fromStart">0</span>
+    </div>
+  </body>
+</html>
diff --git a/test/index.ts b/test/index.ts
@@ -0,0 +1,109 @@
+import { createAssistantClient } from '@salutejs/client';
+import { createInifiniteListen } from './listenSdk'; 
+
+const token = '';
+const apiUrl = '/stt';
+
+const btn = document.createElement('button');
+btn.innerText = "start";
+document.body.appendChild(btn);
+
+function withoutWorker() {
+  const assistant = createAssistantClient({
+      url: "wss://nlp2vps.online.sberbank.ru:443/vps/",
+      userId: `webdbg_userid_${
+        Math.random().toString(36).substring(2, 13) +
+        Math.random().toString(36).substring(2, 13)
+      }`,
+      userChannel: "B2C",
+      locale: "ru",
+      device: {
+        platformType: "web",
+        surface: "SBERBOX",
+      },
+      settings: {
+        dubbing: -1,
+        asrAutoStop: -1,
+        authConnector: "developer_portal_jwt",
+      },
+      version: 5,
+      getToken: () => Promise.resolve(token),
+      getVoiceMeta: () => ({
+        SEND_EOU: true,
+        MAX_EOU_TIMEOUT_SEC: 4,
+      })
+  });
+
+  assistant.on('assistant', (event) => {
+    if (event.asr) {
+      fetch(apiUrl, {
+          method: 'POST',
+          headers: {
+              'Content-Type': 'application/json;charset=utf-8',
+          },
+          body: JSON.stringify({
+              sessionId: '29aa7fd7-ba0d-40c7-86e7-9f1ae115638b',
+              message: event.asr.text,
+              timestamp: new Date().toISOString(),
+          }),
+      });
+    }
+  });
+
+  btn.addEventListener('click', () => {
+    const audio = document.querySelector("audio");
+    audio?.play();
+    let counter = 0;
+    const started = Date.now();
+    setInterval(() => {
+      timeoutTicks.innerText = ++counter;
+      fromStart.innerText = Math.round((Date.now() - started)/1000);
+    }, 1000);
+    // start
+    assistant.listen();
+  });
+}
+
+function withWorker() {
+    const listener = createInifiniteListen(
+        {
+          voiceMeta: {
+            SEND_EOU: true,
+            MAX_EOU_TIMEOUT_SEC: 4,
+          },
+          url: "wss://nlp2vps.online.sberbank.ru:443/vps/",
+          userId: `webdbg_userid_${
+            Math.random().toString(36).substring(2, 13) +
+            Math.random().toString(36).substring(2, 13)
+          }`,
+          userChannel: "B2C",
+          locale: "ru",
+          device: {
+            platformType: "web",
+            surface: "SBERBOX",
+          },
+          settings: {
+            dubbing: -1,
+            asrAutoStop: -1,
+            authConnector: "developer_portal_jwt",
+          },
+          version: 5,
+          token,
+    }, apiUrl);
+
+
+    btn.addEventListener('click', () => {
+      const audio = document.querySelector("audio");
+      audio?.play();
+      let counter = 0;
+      const started = Date.now();
+      setInterval(() => {
+        timeoutTicks.innerText = ++counter;
+        fromStart.innerText = Math.round((Date.now() - started)/1000);
+      }, 1000);
+      // start
+      listener.start();
+    });
+}
+
+withoutWorker();
diff --git a/test/listenSdk.ts b/test/listenSdk.ts
@@ -0,0 +1,29 @@
+import type { VpsConfiguration } from '@salutejs/client';
+import { createNavigatorAudioProvider } from '@salutejs/client';
+import { createVoiceListener } from '@salutejs/client';
+
+export type InifinteListenParams = Omit<VpsConfiguration, 'fakeVps' | 'logger' | 'getToken' | 'vpsToken'> & {
+    token: string;
+    voiceMeta: Record<string, unknown>;
+};
+
+const worker = new Worker(new URL('./vps.worker.ts', import.meta.url), { type: 'module' });
+
+export function createInifiniteListen(config: InifinteListenParams, restApiUrl: string) {
+    const listener = createVoiceListener(
+        (cb, ...args) => createNavigatorAudioProvider(cb, ...args),
+        (port: MessagePort) => {
+            worker.postMessage({ type: 'start' }, [port]);
+        },
+    );
+    worker.postMessage({ type: 'init', params: [config, restApiUrl] });
+
+    return {
+        start: () => {
+            listener.listen();
+        },
+        stop: () => {
+            listener.stop();
+        },
+    };
+}