syncing chat audio to s3

src/app/[uid]/layout.tsx

@@ -3,17 +3,29 @@ import { Header } from "@/components/header";
 export const dynamic = "force-dynamic",
   revalidate = 0;
 import useStore from "@/store";
+import { useEffect, useRef } from "react";
 
 export default function LoggedInLayout({
   children, // will be a page or nested layout
 }: {
   children: React.ReactNode;
 }) {
   const store = useStore();
+  const audioRef = useRef<HTMLAudioElement>(null);
+
+  useEffect(() => {
+    if (store.audioSrc && audioRef.current) {
+      audioRef.current.src = store.audioSrc;
+      audioRef.current.load();
+      audioRef.current.play();
+    }
+  }, [store.audioSrc]);
+
   return (
     <div className="relative">
       <Header>
         <audio
+          ref={audioRef}
           src={store.audioSrc}
           controls
           controlsList="nodownload noplaybackrate"

src/app/api/chatmodel/[chatid]/route.ts

@@ -1,4 +1,4 @@
-import { StreamingTextResponse, LangChainStream } from "ai";
+import { StreamingTextResponse, LangChainStream, nanoid } from "ai";
 import { eq } from "drizzle-orm";
 import { db } from "@/lib/db";
 import { chats } from "@/lib/db/schema";
@@ -68,7 +68,14 @@ export async function POST(
 
   const { stream, handlers } = LangChainStream({
     onCompletion: async (fullResponse: string) => {
-      const latestReponse = { role: "assistant", content: fullResponse };
+      const latestReponse = {
+        id: nanoid(),
+        role: "assistant",
+        content: fullResponse,
+        createdAt: new Date(),
+        audio: "",
+      };
+      console.log("latestReponse", latestReponse);
       if (orgId !== "") {
         // it means it is the first message in a specific chat id
         // Handling organization chat inputs

src/app/api/tts/route.ts

@@ -1,17 +1,31 @@
 import { env } from "@/app/env.mjs";
+import { db } from "@/lib/db";
+import { chats, Chat as ChatSchema } from "@/lib/db/schema";
+import { ChatLog } from "@/lib/types";
+import { saveAudio } from "@/utils/apiHelper";
+import { eq } from "drizzle-orm";
 import { NextResponse } from "next/server";
 import OpenAI from "openai";
 import * as z from "zod";
 export const maxDuration = 60;
 
 const bodyobj = z.object({
   text: z.string().min(1),
+  messageId: z.string().min(1),
+  orgId: z.string().min(1),
+  chatId: z.string().min(1),
 });
 
 export async function POST(request: Request) {
   const body = bodyobj.parse(await request.json());
 
   const text = body.text;
+  const messageId = body.messageId;
+  const orgId = body.orgId;
+  const chatId = body.chatId;
+  console.log("id of the message", body.messageId);
+  console.log("id of the message", body.orgId);
+  console.log("id of the message", body.chatId);
 
   const Openai = new OpenAI({
     apiKey: env.OPEN_AI_API_KEY,
@@ -24,10 +38,41 @@ export async function POST(request: Request) {
   });
 
   const buffer = Buffer.from(await mp3.arrayBuffer());
+  // fetching the chat
+  let chatlog: ChatLog = { log: [] };
+  let fetchedChat: ChatSchema[] = [];
+  fetchedChat = await db
+    .select()
+    .from(chats)
+    .where(eq(chats.id, Number(chatId)))
+    .all();
 
-  return new NextResponse(buffer, {
-    headers: {
-      "Content-Type": "audio/mpeg",
-    },
-  });
+  const msg = fetchedChat[0]?.messages;
+  if (fetchedChat.length === 1 && msg) {
+    chatlog = JSON.parse(msg as string) as ChatLog;
+  }
+
+  // finding the message with the given id if not found return last message from log
+  let message = chatlog.log.find((msg) => msg.id === messageId);
+  if (!message) {
+    message = chatlog.log[chatlog.log.length - 1];
+    message.id = messageId;
+  }
+  // adding the audio to the message
+  const audioUrl = await saveAudio({ buffer, chatId, messageId });
+  message.audio = audioUrl;
+
+  console.log("message", message);
+
+  await db
+    .update(chats)
+    .set({
+      messages: JSON.stringify({ log: chatlog.log } as ChatLog),
+      updatedAt: new Date(),
+    })
+    .where(eq(chats.id, Number(chatId)))
+    .run();
+  return new NextResponse(
+    JSON.stringify({ audioUrl: audioUrl, updatedMessages: chatlog.log }),
+  );
 }

src/components/chat.tsx

@@ -43,6 +43,8 @@ export default function Chat(props: ChatProps) {
     handleSubmit,
     append,
     setMessages,
+    isLoading,
+    data,
   } = useChat({
     api: `/api/chatmodel/${props.chatId}`,
     // initialMessages: props.chat.log as Message[],
@@ -60,6 +62,7 @@ export default function Chat(props: ChatProps) {
       updateRoomAsCompleted(error.message);
       setIsChatCompleted(true);
     },
+    sendExtraMessageFields: true,
   });
 
   useEffect(() => {

src/components/chatcard.tsx

@@ -40,7 +40,6 @@ const Chatcard = ({ chat, uid, org_id, org_slug }: Props) => {
     });
     const data = await res.json();
     setImageUrl(data.url);
-    console.log("this is image url", data.url);
     setIsGeneratingImage(() => false);
   };
 

src/components/chatmessage.tsx

@@ -29,7 +29,7 @@ const ChatMessage = (props: ChatMessageProps) => {
   const [isLoading, setIsLoading] = useState<boolean>(false);
   const [isRegenerating, setIsRegenerating] = useState<boolean>(false);
   const [isActionsOpen, setIsActionsOpen] = useState<boolean>(false);
-  const [audioSrc, setAudioSrc] = useState<string>("");
+  const [audioSrc, setAudioSrc] = useState<string>(props.chat.audio || "");
   const [isFetchingAudioBuffer, setIsFetchingAudioBuffer] =
     useState<boolean>(false);
   const store = useStore();
@@ -131,20 +131,30 @@ const ChatMessage = (props: ChatMessageProps) => {
     setIsActionsOpen(false);
   };
 
-  const textToSpeech = async () => {
+  const textToSpeech = async (id: string) => {
+    if (audioSrc !== "") {
+      store.setAudioSrc(audioSrc);
+      return;
+    }
     const text = props.chat.content;
     setIsFetchingAudioBuffer(true);
     try {
       const res = await fetch("/api/tts", {
         method: "post",
-        body: JSON.stringify({ text: text, voice: "en-US" }),
+        body: JSON.stringify({
+          text: text,
+          messageId: id,
+          orgId: props.orgId,
+          chatId: props.chatId,
+          voice: "en-US",
+        }),
       });
-      const arrayBuffer = await res.arrayBuffer();
-      const blob = new Blob([arrayBuffer], { type: "audio/mpeg" });
-      const url = URL.createObjectURL(blob);
+      const data = await res.json();
+      console.log("data", data);
+      const url = data.audioUrl;
+      props.setMessages(data.updatedMessages);
       store.setAudioSrc(url);
-      console.log("generated url", url);
-      // setAudioSrc(url);
+      setAudioSrc(url);
     } catch (err) {
       console.log(err);
     }
@@ -169,23 +179,18 @@ const ChatMessage = (props: ChatMessageProps) => {
             {userName}
           </p>
           {props.chat.role === "assistant" ? (
-            <Button size="xs" variant="ghost" onClick={textToSpeech}>
+            <Button
+              size="xs"
+              variant="ghost"
+              onClick={() => textToSpeech(props.chat.id)}
+            >
               {isFetchingAudioBuffer ? (
                 <CircleNotch className="animate-spin" />
               ) : (
                 <Play className="" />
               )}
             </Button>
           ) : null}
-          {props.chat.role === "assistant" && audioSrc !== "" ? (
-            <audio
-              src={audioSrc}
-              controls
-              controlsList="nodownload noplaybackrate"
-              className="ml-2 px-1 h-4 bg-green"
-            ></audio>
-          ) : // {/* <ReactAudioPlayer src={audioSrc} controls /> */}
-          null}
         </div>
         <ChatMessageActions
           isEditing={isEditing}

src/components/chatusersavatars.tsx

@@ -45,7 +45,6 @@ const Chatusersavatars = ({ chat, chatLive, allPresenceIds }: Props) => {
       const ids = getUserIdList(chatLive);
       // include ids of users who have not participated in the chat but viewing the chat
       const viewersIds = allPresenceIds?.filter((id) => !ids?.includes(id));
-      console.log("viewersIds", viewersIds);
       if (ids.length) {
         if (viewersIds) {
           getUsers([...ids, ...viewersIds]);

src/components/inputBar.tsx

@@ -67,6 +67,7 @@ const InputBar = (props: InputBarProps) => {
       role: "user",
       content: props.value,
       name: `${props.username},${props.userId}`,
+      audio: "",
     };
     if (props.choosenAI === "universal") {
       props.append(message as Message);

src/components/intermediatesteps.tsx

@@ -12,13 +12,13 @@ export function IntermediateStep(props: { message: Message }) {
 
   return (
     <div
-      className={`ml-auto bg-green-600 rounded px-4 py-2 max-w-[80%] mb-8 whitespace-pre-wrap flex flex-col cursor-pointer`}
+      className={`mx-auto bg-transparent text-xs rounded px-4 py-2 max-w-[80%] whitespace-pre-wrap flex flex-col cursor-pointer`}
     >
       <div
-        className={`text-right ${expanded ? "w-full" : ""}`}
+        className={`text-right mx-auto`}
         onClick={(e) => setExpanded(!expanded)}
       >
-        <code className="mr-2 bg-slate-600 px-2 py-1 rounded hover:text-blue-600">
+        <code className="mr-2 bg-slate-600 px-2 py-1 rounded">
           🛠️ <b>{parsedInput?.action.tool}</b>
         </code>
         <span className={expanded ? "hidden" : ""}>🔽</span>
@@ -46,7 +46,7 @@ export function IntermediateStep(props: { message: Message }) {
           </code>
         </div>
         <div
-          className={`bg-slate-600 rounded p-4 mt-1 max-w-0 ${
+          className={`bg-slate-600 overflow-auto rounded p-4 mt-1 max-w-0 ${
             expanded ? "max-w-full" : "transition-[max-width] delay-100"
           }`}
         >

src/lib/types/ai.ts

@@ -0,0 +1,7 @@
+// import ai from "ai"
+import { Message as Message2 } from "ai";
+declare module "ai" {
+  interface Message extends Message2 {
+    audio?: string;
+  }
+}

src/lib/types/index.ts

@@ -8,6 +8,7 @@ export type ChatEntry = {
   role: ChatRole;
   content: string;
   name: string;
+  audio?: string;
 };
 
 export type ChatLog = {

src/utils/apiHelper.ts

@@ -299,3 +299,36 @@ export const handleDBOperation = async (
       .run();
   }
 };
+
+export const saveAudio = async ({
+  buffer,
+  chatId,
+  messageId,
+}: {
+  buffer: Buffer;
+  chatId: string;
+  messageId: string;
+}): Promise<string> => {
+  const s3 = new S3Client({
+    region: env.AWS_REGION,
+    credentials: {
+      accessKeyId: env.AWS_ACCESS_KEY_ID,
+      secretAccessKey: env.AWS_SECRET_ACCESS_KEY,
+    },
+  });
+  const data = await s3.send(
+    new PutObjectCommand({
+      Bucket: env.BUCKET_NAME,
+      Body: buffer,
+      Key: `chataudio/${chatId}/${messageId}.mpeg`,
+      Metadata: {
+        "Content-Type": "audio/mpeg",
+        "chat-id": chatId,
+        "message-id": messageId,
+      },
+    }),
+  );
+
+  const audioUrl = `${env.IMAGE_PREFIX_URL}chataudio/${chatId}/${messageId}.mpeg`;
+  return audioUrl;
+};