update: image-chat

Technoculture · Sep 25, 2024 · 37196f6 · 37196f6
1 parent 203cb0b
commit 37196f6
Show file tree

Hide file tree

Showing 10 changed files with 303 additions and 71 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -19,6 +19,7 @@ jobs:
       - name: Lint and fix
         run: npm run lint:fix
         env:
+          ANTHROPIC_API_KEY: ${{secrets.ANTHROPIC_API_KEY}}
           TURSO_DB_URL: ${{secrets.TURSO_DB_URL}}
           TURSO_DB_AUTH_TOKEN: ${{secrets.TURSO_DB_AUTH_TOKEN}}
           OPEN_AI_API_KEY: ${{secrets.OPEN_AI_API_KEY}}

diff --git a/package.json b/package.json
@@ -120,6 +120,7 @@
     "terser": "^5.33.0",
     "tldraw": "2.0.2",
     "typescript": "5.0.3",
+    "use-stay-awake": "^0.1.7",
     "vaul": "0.8.0",
     "zod": "3.22.4",
     "zustand": "4.4.6"

diff --git a/src/app/api/imageInput/route.ts b/src/app/api/imageInput/route.ts
@@ -78,6 +78,7 @@ export async function POST(request: Request, response: NextApiResponse) {
       { status: 400 },
     );
   }
+  console.log("imageFile", imageFile);
   const parts = imageFile.name.split(".");
   const extension = parts[parts.length - 1];
   let awsImageUrl = "";

diff --git a/src/app/page.tsx b/src/app/page.tsx
@@ -59,6 +59,27 @@ export default function Home() {
     "model",
     parseAsString.withDefault("chat"),
   );
+  const [imageUrl, setImageUrl] = useQueryState(
+    "imageUrl",
+    parseAsString.withDefault(""),
+  );
+  const [imageName, setImageName] = useQueryState(
+    "imageName",
+    parseAsString.withDefault(""),
+  );
+  const [imageType, setImageType] = useQueryState(
+    "imageType",
+    parseAsString.withDefault(""),
+  );
+  const [imageSize, setImageSize] = useQueryState(
+    "imageSize",
+    parseAsString.withDefault(""),
+  );
+  const [imageExtension, setImageExtension] = useQueryState(
+    "imageExtension",
+    parseAsString.withDefault(""),
+  );
+  const [dropzoneActive, setDropzoneActive] = useState<boolean>(false);
 
   const { isSignedIn, orgId, orgSlug, userId } = useAuth();
   // if (isSignedIn) {
@@ -80,9 +101,27 @@ export default function Home() {
       });
       const data = await res.json();
 
-      router.push(
-        `/dashboard/chat/${data.newChatId}?new=true&clipboard=true&model=${chatType}&input=${input}`,
-      );
+      if (dropzoneActive) {
+        const queryParams = new URLSearchParams(window.location.search);
+        const params: { [key: string]: string } = {};
+        queryParams.forEach((value, key) => {
+          params[key] = value;
+        });
+        const params2 = {
+          ...params,
+          new: "true",
+          clipboard: "true",
+          model: chatType,
+          input: input,
+        };
+        const queryParamsString = new URLSearchParams(params2).toString();
+
+        router.push(`/dashboard/chat/${data.newChatId}?${queryParamsString}`);
+      } else {
+        router.push(
+          `/dashboard/chat/${data.newChatId}?new=true&clipboard=true&model=${chatType}&input=${input}`,
+        );
+      }
     } catch (error) {
       console.error("Error creating new chat:", error);
     }
@@ -120,6 +159,18 @@ export default function Home() {
                 {isSignedIn && orgId && orgSlug ? (
                   <div className="w-full md:min-w-[400px] lg:min-w-[600px] xl:min-w-[800px] ">
                     <InputBar
+                      imageExtension={imageExtension}
+                      setImageExtension={setImageExtension}
+                      dropzoneActive={dropzoneActive}
+                      setDropzoneActive={setDropzoneActive}
+                      imageUrl={imageUrl}
+                      setImageUrl={setImageUrl}
+                      imageName={imageName}
+                      setImageName={setImageName}
+                      imageType={imageType}
+                      setImageType={setImageType}
+                      imageSize={imageSize}
+                      setImageSize={setImageSize}
                       isHome={true}
                       value={input}
                       onChange={handleInputChange}
@@ -133,7 +184,7 @@ export default function Home() {
                     />
                     <div className="flex flex-col gap-y-4">
                       <OrgChatToggler orgId={orgId} orgSlug={orgSlug} />
-                      <div className="w-full md:w-[400px] lg:w-[600px] xl:w-[800px] h-[500px] overflow-y-scroll scrollbar-hide">
+                      <div className="w-full md:w-[400px] lg:w-[600px] xl:w-[800px] h-[500px] overflow-y-scroll scrollbar-hide self-center">
                         <ChatCardWrapper
                           isHome={true}
                           org_id={orgId}

diff --git a/src/components/VadAudio.tsx b/src/components/VadAudio.tsx
@@ -5,6 +5,7 @@ import { useMicVAD, utils } from "@ricky0123/vad-react";
 import { Microphone, StopCircle } from "@phosphor-icons/react";
 import { Button } from "@/components/button";
 import { cn } from "@/lib/utils";
+import useStayAwake from "use-stay-awake";
 
 interface VadAudioProps {
   onAudioCapture: (audioFile: File) => void;
@@ -24,6 +25,7 @@ export default function VadAudio({
   const audioChunks = useRef<Blob[]>([]);
   const timerRef = useRef<NodeJS.Timeout | null>(null);
   const startTimeRef = useRef<number | null>(null);
+  const device = useStayAwake();
 
   const vad = useMicVAD({
     onSpeechEnd: (audio: Float32Array) => {
@@ -51,6 +53,7 @@ export default function VadAudio({
   const handleStartListening = useCallback(() => {
     vad.start();
     startTimer();
+    device.preventSleeping();
     onStartListening();
     setIsListening(true);
     audioChunks.current = [];
@@ -62,6 +65,7 @@ export default function VadAudio({
     vad.pause();
     resetDuration();
     clearTimer();
+    device.allowSleeping();
   }, [vad]);
 
   const startTimer = () => {

diff --git a/src/components/chat.tsx b/src/components/chat.tsx
@@ -1,7 +1,7 @@
 "use client";
 import { useState, useEffect, useCallback } from "react";
 import { ChatType } from "@/lib/types";
-import InputBar from "@/components/inputBar";
+import InputBar, { Schema } from "@/components/inputBar";
 import { Message, useChat } from "ai/react";
 import Startnewchatbutton from "@/components/startnewchatbutton";
 import ChatMessageCombinator from "@/components/chatmessagecombinator";
@@ -164,25 +164,153 @@ export default function Chat(props: ChatProps) {
     },
     sendExtraMessageFields: true,
   });
-  console.log("messages", messages);
+
+  const handleFirstImageMessage = useCallback(async () => {
+    const params = new URLSearchParams(window.location.search);
+    if (
+      params.get("imageUrl") &&
+      params.get("imageName") &&
+      params.get("imageType") &&
+      params.get("imageSize")
+    ) {
+      const queryParams: { [key: string]: string } = {};
+      params.forEach((value, key) => {
+        queryParams[key] = value;
+      });
+      const ID = nanoid();
+      const imageMessasgeId = nanoid();
+      const message: Message = {
+        id: ID,
+        role: "user",
+        content: incomingInput || "",
+        name: `${props.username},${props.uid}`,
+        audio: "",
+      };
+      const createFileFromBlobUrl = async (
+        blobUrl: string,
+        fileName: string,
+      ) => {
+        const response = await fetch(blobUrl);
+        const blob = await response.blob();
+        return new File([blob], fileName, { type: blob.type });
+      };
+
+      const imageUrl = params.get("imageUrl")!;
+      const imageName = params.get("imageName")!;
+      const imageExtension = params.get("imageExtension")!;
+
+      const file = await createFileFromBlobUrl(
+        imageUrl,
+        `image.${imageExtension}`,
+      );
+      console.log("Created file from blob URL:", file);
+      const zodMessage: any = Schema.safeParse({
+        imageName: params.get("imageName"),
+        imageType: params.get("imageType"),
+        imageSize: Number(params.get("imageSize")),
+        file: file,
+        value: input,
+        userId: props.uid,
+        orgId: props.orgId,
+        chatId: props.chatId,
+        message: [message],
+        id: ID,
+        chattype: chattype,
+      });
+      console.log("zodMessageImage Extension:", imageExtension);
+      // console.log("zodmessage", zodMessage);
+      // console.log("dropzone", props.dropZoneActive);
+      console.log("zodMessage", zodMessage, imageExtension);
+      if (zodMessage.success) {
+        const zodMSG = JSON.stringify(zodMessage);
+        const formData = new FormData();
+        formData.append("zodMessage", zodMSG);
+        formData.append("file", file);
+        const response = await fetch("/api/imageInput", {
+          method: "POST",
+          body: formData,
+        });
+        if (response) {
+          console.log("responce", response);
+          let assistantMsg = "";
+          const reader = response.body?.getReader();
+          console.log("reader", reader);
+          const decoder = new TextDecoder();
+          let charsReceived = 0;
+          let content = "";
+          reader
+            ?.read()
+            .then(async function processText({ done, value }) {
+              if (done) {
+                console.log("Stream complete");
+                return;
+              }
+              charsReceived += value.length;
+              const chunk = decoder.decode(value, { stream: true });
+              assistantMsg += chunk === "" ? `${chunk} \n` : chunk;
+              content += chunk === "" ? `${chunk} \n` : chunk;
+              // console.log("assistMsg", assistantMsg);
+              setMessages([
+                ...messages,
+                awsImageMessage,
+                message,
+                {
+                  ...assistantMessage,
+                  content: assistantMsg,
+                },
+              ]);
+              reader.read().then(processText);
+            })
+            .then((e) => {
+              console.error("error", e);
+            });
+          const awsImageMessage = {
+            role: "user",
+            subRole: "input-image",
+            content: `${process.env.NEXT_PUBLIC_IMAGE_PREFIX_URL}imagefolder/${props.chatId}/${ID}.${imageExtension}`,
+            id: ID,
+          } as Message;
+          const assistantMessage: Message = {
+            id: ID,
+            role: "assistant",
+            content: content,
+          };
+
+          console.log("image chat", queryParams);
+          // image chat
+        }
+      }
+    }
+  }, []);
 
   //TODO: handle user incoming from dashboard when invoked a chat
   useEffect(() => {
     if (isNewChat === "true" && incomingInput) {
       //TODO: use types for useQueryState
       if (incomingInput && chattype !== "tldraw") {
-        const newMessage = {
-          id: nanoid(),
-          role: "user",
-          content: incomingInput,
-          name: `${props.username},${props.uid}`,
-          audio: "",
-        } as Message;
-        append(newMessage);
+        const params = new URLSearchParams(window.location.search);
+        if (
+          params.get("imageUrl") &&
+          params.get("imageName") &&
+          params.get("imageType") &&
+          params.get("imageSize")
+        ) {
+          console.log("zodMessage", "we made to here", params);
+          handleFirstImageMessage();
+        } else {
+          const newMessage = {
+            id: nanoid(),
+            role: "user",
+            content: incomingInput,
+            name: `${props.username},${props.uid}`,
+            audio: "",
+          } as Message;
+          append(newMessage);
+        }
       }
-      setIsFromClipboard("false");
-      setIsNewChat("false");
     }
+    setIsFromClipboard("false");
+    setIsNewChat("false");
   }, [isFromClipboard, isNewChat]);
 
   useEffect(() => {

diff --git a/src/components/chatcard.tsx b/src/components/chatcard.tsx
@@ -53,7 +53,6 @@ const Chatcard = ({
     const chatlog = JSON.parse(chat.messages as string) as ChatLog;
     console.log("chatlog", chatlog.log);
     const msgs = chatlog.log as ChatEntry[];
-    console.log("messages", msgs);
     const chats = msgs.slice(0, 2);
     const res = await fetch(`/api/generateTitle/${chat.id}/${org_id}`, {
       method: "POST",

diff --git a/src/components/inputBar.tsx b/src/components/inputBar.tsx
@@ -27,7 +27,7 @@ import VadAudio from "./VadAudio";
 const isValidImageType = (value: string) =>
   /^image\/(jpeg|png|jpg|webp)$/.test(value);
 
-const Schema = z.object({
+export const Schema = z.object({
   imageName: z.any(),
   imageType: z.string().refine(isValidImageType, {
     message: "File type must be JPEG, PNG, or WEBP image",