Merge branch 'master' into infinite-listen

CHANGELOG.md

@@ -1,3 +1,54 @@
+# v1.35.0 (Wed Jun 05 2024)
+
+#### 🚀 Enhancement
+
+- feat: Обновлён Scenario [#214](https://github.com/salute-developers/salutejs-client/pull/214) ([@evgeniysemin](https://github.com/evgeniysemin))
+
+#### ⚠️ Pushed to `master`
+
+- test: [autocommit] Save current bundle report ([@Salute-Eva](https://github.com/Salute-Eva))
+
+#### Authors: 2
+
+- [@evgeniysemin](https://github.com/evgeniysemin)
+- [@Salute-Eva](https://github.com/Salute-Eva)
+
+---
+
+# v1.34.1 (Sat Apr 27 2024)
+
+#### 🐛 Bug Fix
+
+- fix: window navigator on server [#213](https://github.com/salute-developers/salutejs-client/pull/213) ([@soulko](https://github.com/soulko))
+
+#### ⚠️ Pushed to `master`
+
+- test: [autocommit] Save current bundle report ([@Salute-Eva](https://github.com/Salute-Eva))
+
+#### Authors: 2
+
+- [@Salute-Eva](https://github.com/Salute-Eva)
+- Anton Kostenko ([@soulko](https://github.com/soulko))
+
+---
+
+# v1.34.0 (Thu Apr 25 2024)
+
+#### 🚀 Enhancement
+
+- feat: Добавлено событие с данными аналайзера [#212](https://github.com/salute-developers/salutejs-client/pull/212) ([@evgeniysemin](https://github.com/evgeniysemin))
+
+#### ⚠️ Pushed to `master`
+
+- test: [autocommit] Save current bundle report ([@Salute-Eva](https://github.com/Salute-Eva))
+
+#### Authors: 2
+
+- [@evgeniysemin](https://github.com/evgeniysemin)
+- [@Salute-Eva](https://github.com/Salute-Eva)
+
+---
+
 # v1.33.6 (Tue Apr 16 2024)
 
 #### 🐛 Bug Fix

bundle-report.json

@@ -1 +1 @@
-"{\"dist/assistant-hash1.js\":435929,\"dist/createAssistantDevOrigin.js\":122055,\"dist/index.js\":69234,\"esm/assistant-hash1.js\":141701,\"esm/assistant.js\":123,\"esm/common-hash1.js\":6287,\"esm/createAssistant.js\":24980,\"esm/createAssistantDev.js\":374,\"esm/createAssistantDevOrigin.js\":121929,\"esm/index.js\":1017,\"esm/mock.js\":5420,\"esm/record-hash1.js\":63740,\"esm/sdk-hash1.js\":261834,\"umd/assistant.development.min.js\":215522,\"umd/assistant.production.min.js\":12890}"
+"{\"dist/assistant-hash1.js\":436556,\"dist/createAssistantDevOrigin.js\":122055,\"dist/index.js\":69234,\"esm/assistant-hash1.js\":142308,\"esm/assistant.js\":123,\"esm/common-hash1.js\":6287,\"esm/createAssistant.js\":24980,\"esm/createAssistantDev.js\":374,\"esm/createAssistantDevOrigin.js\":121929,\"esm/index.js\":1017,\"esm/mock.js\":5420,\"esm/record-hash1.js\":63740,\"esm/sdk-hash1.js\":261834,\"umd/assistant.development.min.js\":215764,\"umd/assistant.production.min.js\":12890}"

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@salutejs/client",
-    "version": "1.33.6",
+    "version": "1.35.0",
     "description": "Модуль взаимодействия с виртуальным ассистентом",
     "author": "Salute Frontend Team <[email protected]>",
     "main": "dist/index.js",
@@ -34,7 +34,7 @@
         "smartapp"
     ],
     "dependencies": {
-        "@salutejs/scenario": "^0.38.0",
+        "@salutejs/scenario": "1.2.0",
         "lodash.clonedeep": "^4.5.0",
         "long": "^5.2.3",
         "protobufjs": "^7.2.6",

src/assistantSdk/assistant.ts

@@ -104,6 +104,7 @@ export type AssistantEvent = {
     emotion?: EmotionId;
     mtt?: { response: Music2TrackProtocol.MttResponse; mid: OriginalMessageType['messageId'] };
     listener?: { status: VoiceListenerStatus };
+    voiceAnalyser?: { data: Uint8Array };
 };
 
 export type VpsEvent =
@@ -137,6 +138,7 @@ export interface CreateAssistantDevOptions {
     getInitialMeta?: () => Promise<Record<string, unknown>>;
     /** Подставляет мету в первый чанк с голосом для управления рекогнайзером */
     getVoiceMeta?: () => Record<string, unknown>;
+    useVoiceAnalyser?: boolean;
 }
 
 type BackgroundAppOnCommand<T> = (
@@ -162,6 +164,7 @@ export const createAssistant = ({
     getInitialMeta,
     getVoiceMeta,
     checkCertUrl,
+    useVoiceAnalyser,
     ...configuration
 }: AssistantParams) => {
     const { on, emit } = createNanoEvents<AssistantEvents>();
@@ -301,6 +304,7 @@ export const createAssistant = ({
                 protocol.changeSettings({ dubbing: 1 });
             }
         },
+        useVoiceAnalyser,
     );
 
     /** завершает текущий апп */

src/assistantSdk/voice/listener/navigatorAudioProvider.ts

@@ -26,7 +26,8 @@ let source: MediaStreamAudioSourceNode;
  */
 const createAudioRecorder = (
     stream: MediaStream,
-    cb: (buffer: ArrayBuffer, last: boolean) => void,
+    cb: (buffer: ArrayBuffer, analyserArray: Uint8Array | null, last: boolean) => void,
+    useAnalyser?: boolean,
 ): Promise<() => void> =>
     new Promise((resolve) => {
         let state: 'inactive' | 'recording' = 'inactive';
@@ -82,13 +83,16 @@ const createAudioRecorder = (
  * @param cb Callback, куда будут передаваться чанки с голосом пользователя
  * @returns Promise, который содержит функцию прерывающую слушание
  */
-export const createNavigatorAudioProvider = (cb: (buffer: ArrayBuffer, last: boolean) => void): Promise<() => void> =>
+export const createNavigatorAudioProvider = (
+    cb: (buffer: ArrayBuffer, analyserArray: Uint8Array | null, last: boolean) => void,
+    useAnalyser?: boolean,
+): Promise<() => void> =>
     navigator.mediaDevices
         .getUserMedia({
             audio: true,
         })
         .then((stream) => {
-            return createAudioRecorder(stream, cb);
+            return createAudioRecorder(stream, cb, useAnalyser);
         })
         .catch((err) => {
             if (window.location.protocol === 'http:') {

src/assistantSdk/voice/listener/voiceListener.ts

@@ -4,7 +4,7 @@ import { createNavigatorAudioProvider } from './navigatorAudioProvider';
 
 export type VoiceListenerStatus = 'listen' | 'started' | 'stopped';
 
-export type VoiceHandler = (data: Uint8Array, last: boolean) => void;
+export type VoiceHandler = (data: Uint8Array, analyserArray: Uint8Array | null, last: boolean) => void;
 
 type VoiceStreamEvents = {
     status: (status: VoiceListenerStatus) => void;
@@ -16,11 +16,7 @@ type VoiceStreamEvents = {
  * @param createAudioProvider Источник голоса
  * @returns Api для запуска и остановки слушания
  */
-export const createVoiceListener = (
-    createAudioProvider: (
-        cb: (data: ArrayBuffer, last: boolean) => void,
-    ) => Promise<() => void> = createNavigatorAudioProvider,
-) => {
+export const createVoiceListener = (createAudioProvider = createNavigatorAudioProvider) => {
     const { emit, on } = createNanoEvents<VoiceStreamEvents>();
     let stopRecord: () => void;
     let status: VoiceListenerStatus = 'stopped';
@@ -40,7 +36,7 @@ export const createVoiceListener = (
         status = 'started';
         emit('status', 'started');
 
-        return createAudioProvider((data: ArrayBuffer, last: boolean) => handleVoice(new Uint8Array(data), last))
+        return createAudioProvider((data, analyser, last) => handleVoice(new Uint8Array(data), analyser, last))
             .then((recStop) => {
                 stopRecord = recStop;
             })

src/assistantSdk/voice/player/iosSilentModePatch.ts

@@ -1,8 +1,9 @@
-const ctx = typeof window !== 'undefined' ? window : self;
 const IS_APPLE_MOBILE =
-    (/iPad|iPhone|iPod/.test(navigator.platform) ||
-        (navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1)) &&
-    !(ctx as any).MSStream;
+    typeof window !== 'undefined'
+        ? (/iPad|iPhone|iPod/.test(navigator.platform) ||
+              (navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1)) &&
+          !(window as any).MSStream
+        : false;
 
 function createSilentAudioFile(sampleRate: number) {
     const arrayBuffer = new ArrayBuffer(10);

src/assistantSdk/voice/voice.ts

@@ -3,6 +3,7 @@ import { AppInfo, EmotionId, OriginalMessageType, Mid, SystemMessageDataType, Me
 import { AssistantSettings } from '../assistant';
 import { MutexedObject } from '../mutexedObject';
 
+import { createNavigatorAudioProvider } from './listener/navigatorAudioProvider';
 import { createVoiceListener, VoiceListenerStatus } from './listener/voiceListener';
 import { createVoicePlayer } from './player/voicePlayer';
 import { resolveAudioContext, isAudioSupported } from './audioContext';
@@ -36,13 +37,15 @@ export const createVoice = (
         listener?: { status: VoiceListenerStatus };
         mtt?: { response: Music2TrackProtocol.MttResponse; mid: OriginalMessageType['messageId'] };
         tts?: TtsEvent;
+        voiceAnalyser?: { data: Uint8Array };
     }) => void,
     /// пока onReady не вызван, треки не воспроизводятся
     /// когда случится onReady, очередь треков начнет проигрываться
     onReady?: () => void,
+    useAnalyser?: boolean,
 ) => {
     let voicePlayer: ReturnType<typeof createVoicePlayer>;
-    const listener = createVoiceListener();
+    const listener = createVoiceListener((cb) => createNavigatorAudioProvider(cb, useAnalyser));
     const subscriptions: Array<() => void> = [];
     const appInfoDict: Record<string, AppInfo> = {};
     const mesIdQueue: Array<string> = [];
@@ -128,8 +131,12 @@ export const createVoice = (
 
                     currentVoiceMessageId = messageId;
 
-                    return listener.listen((...args) => {
-                        sendVoice(...args, messageName);
+                    return listener.listen((chunk, analyser, last) => {
+                        if (analyser) {
+                            emit({ voiceAnalyser: { data: analyser } });
+                        }
+
+                        sendVoice(chunk, last, messageName);
                     });
                 },
                 {

src/typings.ts

@@ -11,6 +11,7 @@ import {
     UUID,
     Meta as ScenarioMeta,
     SystemMessagePayload,
+    VpsMessageReasonModel,
 } from '@salutejs/scenario';
 
 import {
@@ -465,14 +466,8 @@ export interface AssistantBackgroundAppInfo {
     state: Record<string, unknown>;
 }
 
-export interface SourceMeta {
-    sourceType?: string;
-    sourceName?: string;
-    suggestTitle?: string;
-}
 export interface Meta extends ScenarioMeta, AssistantMeta {
-    background_apps: AssistantBackgroundAppInfo[];
-    source?: SourceMeta;
+    source?: VpsMessageReasonModel;
     [key: string]: unknown;
 }