From 7b2d8b4e7b81d1b7b1a21bf9d6b4d9aa5ebd7dd4 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Fri, 2 Aug 2024 18:12:06 -0700 Subject: [PATCH] Include chat handling with minimal_assistant --- examples/speech-to-text/deepgram_stt.py | 8 ++------ examples/voice-assistant/minimal_assistant.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/examples/speech-to-text/deepgram_stt.py b/examples/speech-to-text/deepgram_stt.py index 95f45a402..6a8cc100e 100644 --- a/examples/speech-to-text/deepgram_stt.py +++ b/examples/speech-to-text/deepgram_stt.py @@ -32,7 +32,6 @@ async def _forward_transcription( async def entrypoint(ctx: JobContext): logger.info("starting speech-to-text example") stt = deepgram.STT() - tasks = [] async def transcribe_track(participant: rtc.RemoteParticipant, track: rtc.Track): audio_stream = rtc.AudioStream(track) @@ -40,10 +39,7 @@ async def transcribe_track(participant: rtc.RemoteParticipant, track: rtc.Track) room=ctx.room, participant=participant, track=track ) stt_stream = stt.stream() - stt_task = asyncio.create_task( - _forward_transcription(stt_stream, stt_forwarder) - ) - tasks.append(stt_task) + asyncio.create_task(_forward_transcription(stt_stream, stt_forwarder)) async for ev in audio_stream: stt_stream.push_frame(ev.frame) @@ -57,7 +53,7 @@ def on_track_subscribed( participant: rtc.RemoteParticipant, ): if track.kind == rtc.TrackKind.KIND_AUDIO: - tasks.append(asyncio.create_task(transcribe_track(participant, track))) + asyncio.create_task(transcribe_track(participant, track)) if __name__ == "__main__": diff --git a/examples/voice-assistant/minimal_assistant.py b/examples/voice-assistant/minimal_assistant.py index 976fb1243..2ae520663 100644 --- a/examples/voice-assistant/minimal_assistant.py +++ b/examples/voice-assistant/minimal_assistant.py @@ -1,6 +1,7 @@ import asyncio from dotenv import load_dotenv +from livekit import rtc from livekit.agents import AutoSubscribe, JobContext, WorkerOptions, cli, llm from livekit.agents.voice_assistant import VoiceAssistant from livekit.plugins import deepgram, openai, silero @@ -19,15 +20,31 @@ async def entrypoint(ctx: JobContext): await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY) + llm_plugin = openai.LLM() assistant = VoiceAssistant( vad=silero.VAD.load(), stt=deepgram.STT(), - llm=openai.LLM(), + llm=llm_plugin, tts=openai.TTS(), chat_ctx=initial_ctx, ) assistant.start(ctx.room) + # listen to incoming chat messages, only required if you'd like the agent to + # answer incoming messages from Chat + chat = rtc.ChatManager(ctx.room) + + async def answer_from_text(txt: str): + chat_ctx = assistant.chat_ctx.copy() + chat_ctx.append(role="user", text=txt) + stream = llm_plugin.chat(chat_ctx=chat_ctx) + await assistant.say(stream) + + @chat.on("message_received") + def on_chat_received(msg: rtc.ChatMessage): + if msg.message: + asyncio.create_task(answer_from_text(msg.message)) + await asyncio.sleep(1) await assistant.say("Hey, how can I help you today?", allow_interruptions=True)