From d9c39de8e1de1a5c2df43005c9c5dfd1357ee2e8 Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Mon, 16 Sep 2024 09:39:35 -0600 Subject: [PATCH 01/18] init --- src/leapfrogai_ui/src/lib/constants/index.ts | 24 ++++ src/leapfrogai_ui/src/lib/mocks/openai.ts | 16 ++- src/leapfrogai_ui/src/lib/schemas/files.ts | 29 +++++ .../routes/api/audio/translation/+server.ts | 35 ++++++ .../api/audio/translation/server.test.ts | 109 ++++++++++++++++++ src/leapfrogai_ui/svelte.config.js | 1 + 6 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts create mode 100644 src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts diff --git a/src/leapfrogai_ui/src/lib/constants/index.ts b/src/leapfrogai_ui/src/lib/constants/index.ts index 5379d858a..32ca04cca 100644 --- a/src/leapfrogai_ui/src/lib/constants/index.ts +++ b/src/leapfrogai_ui/src/lib/constants/index.ts @@ -49,6 +49,29 @@ export const ACCEPTED_FILE_TYPES = [ '.docx', '.csv' ]; + +export const ACCEPTED_AUDIO_FILE_TYPES = [ + '.flac', + '.mp3', + '.mp4', + '.mpeg', + '.mpga', + '.m4a', + '.ogg', + '.wav', + '.webm' +]; + +export const ACCEPTED_AUDIO_FILE_MIME_TYPES = [ + 'audio/flac', + 'audio/mpeg', + 'audio/mp4', + 'audio/mpeg', + 'audio/ogg', + 'audio/wav', + 'audio/webm' +]; + export const ACCEPTED_MIME_TYPES = [ 'application/pdf', // .pdf 'text/plain', // .txt, .text @@ -78,6 +101,7 @@ export const NO_FILE_ERROR_TEXT = 'Please upload an image or select a pictogram' export const AVATAR_FILE_SIZE_ERROR_TEXT = `File must be less than ${MAX_AVATAR_SIZE / 1000000} MB`; export const FILE_SIZE_ERROR_TEXT = `File must be less than ${MAX_FILE_SIZE / 1000000} MB`; export const INVALID_FILE_TYPE_ERROR_TEXT = `Invalid file type, accepted types are: ${ACCEPTED_FILE_TYPES.join(', ')}`; +export const INVALID_AUDIO_FILE_TYPE_ERROR_TEXT = `Invalid file type, accepted types are: ${ACCEPTED_AUDIO_FILE_TYPES.join(', ')}`; export const NO_SELECTED_ASSISTANT_ID = 'noSelectedAssistantId'; export const FILE_UPLOAD_PROMPT = "The following are the user's files: "; diff --git a/src/leapfrogai_ui/src/lib/mocks/openai.ts b/src/leapfrogai_ui/src/lib/mocks/openai.ts index 25d397d72..1e674bb43 100644 --- a/src/leapfrogai_ui/src/lib/mocks/openai.ts +++ b/src/leapfrogai_ui/src/lib/mocks/openai.ts @@ -42,6 +42,7 @@ class OpenAI { listAssistants: boolean; deleteFile: boolean; fileContent: boolean; + translation: boolean; }; constructor({ apiKey, baseURL }: { apiKey: string; baseURL: string }) { @@ -71,7 +72,8 @@ class OpenAI { updateAssistant: false, listAssistants: false, deleteFile: false, - fileContent: false + fileContent: false, + translation: false }; } @@ -117,6 +119,18 @@ class OpenAI { this.errors[key] = false; } + audio = { + translations: { + create: vi.fn().mockImplementation(() => + { + if (this.errors.translation) this.throwError('translation'); + return Promise.resolve({ + text: 'Fake translation' + })} + ) + } + }; + files = { retrieve: vi.fn().mockImplementation((id) => { return Promise.resolve(this.uploadedFiles.find((file) => file.id === id)); diff --git a/src/leapfrogai_ui/src/lib/schemas/files.ts b/src/leapfrogai_ui/src/lib/schemas/files.ts index 5c475e5e1..14d1d85d6 100644 --- a/src/leapfrogai_ui/src/lib/schemas/files.ts +++ b/src/leapfrogai_ui/src/lib/schemas/files.ts @@ -1,7 +1,9 @@ import { array, mixed, object, string, ValidationError } from 'yup'; import { + ACCEPTED_AUDIO_FILE_MIME_TYPES, ACCEPTED_MIME_TYPES, FILE_SIZE_ERROR_TEXT, + INVALID_AUDIO_FILE_TYPE_ERROR_TEXT, INVALID_FILE_TYPE_ERROR_TEXT, MAX_FILE_SIZE } from '$constants'; @@ -63,3 +65,30 @@ export const fileSchema = object({ }) .noUnknown(true) .strict(); + +// TODO - max file size? +// TODO - validate lf api accepted file types +export const audioFileSchema = object({ + file: mixed() + .test('fileType', 'File is required.', (value) => value == null || value instanceof File) + .test('fileSize', FILE_SIZE_ERROR_TEXT, (value) => { + if (value == null) { + return true; + } + if (value.size > MAX_FILE_SIZE) { + return new ValidationError(FILE_SIZE_ERROR_TEXT); + } + return true; + }) + .test('type', INVALID_AUDIO_FILE_TYPE_ERROR_TEXT, (value) => { + if (value == null) { + return true; + } + if (!ACCEPTED_AUDIO_FILE_MIME_TYPES.includes(value.type)) { + return new ValidationError(INVALID_AUDIO_FILE_TYPE_ERROR_TEXT); + } + return true; + }) +}) + .noUnknown(true) + .strict(); diff --git a/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts b/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts new file mode 100644 index 000000000..637961fc3 --- /dev/null +++ b/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts @@ -0,0 +1,35 @@ +import { error, json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getOpenAiClient } from '$lib/server/constants'; +import { audioFileSchema } from '$schemas/files'; +import { env } from '$env/dynamic/private'; + +export const POST: RequestHandler = async ({ request, locals: { session } }) => { + if (!session) { + error(401, 'Unauthorized'); + } + + let file: File | null; + + // Validate request body + try { + const formData = await request.formData(); + file = formData.get('file') as File; + await audioFileSchema.validate({ file }, { abortEarly: false }); + } catch (e) { + console.error('Validation error:', e); + error(400, `Bad Request, File invalid: ${e}`); + } + + try { + const openai = getOpenAiClient(session.access_token); + const translation = await openai.audio.translations.create({ + file: file, + model: env.OPENAI_API_KEY ? 'whisper-1' : 'whisper' + }); + return json({ text: translation.text }); + } catch (e) { + console.error('file translation error', e); + error(500, 'Internal Error'); + } +}; diff --git a/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts b/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts new file mode 100644 index 000000000..473c58ea4 --- /dev/null +++ b/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts @@ -0,0 +1,109 @@ +import { getLocalsMock } from '$lib/mocks/misc'; +import type { RequestEvent } from '@sveltejs/kit'; +import type { RouteParams } from './$types'; +import { POST } from './+server'; +import { mockOpenAI } from '../../../../../vitest-setup'; + +describe('/api/audio/translation', () => { + it('returns a 401 when there is no session', async () => { + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST' + }); + + await expect( + POST({ + request, + params: {}, + locals: getLocalsMock({ nullSession: true }) + } as RequestEvent) + ).rejects.toMatchObject({ + status: 401 + }); + }); + + it('should return 400 if the form data is missing', async () => { + const request = new Request('http://thisurlhasnoeffect', { method: 'POST' }); + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ status: 400 }); + }); + + it('should return 400 if the file is missing from the form data', async () => { + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST', + body: new FormData() + }); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ + status: 400 + }); + }); + + it('should return 400 if the file in the form data is not of type File', async () => { + const formData = new FormData(); + formData.append('file', '123'); + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST', + body: formData + }); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ + status: 400 + }); + }); + + it('should return a 500 if there is an error translating the file', async () => { + mockOpenAI.setError('translation'); + + const fileContent = new Blob(['dummy content'], { type: 'audio/mp4' }); + const testFile = new File([fileContent], 'test.txt', { type: 'audio/mp4' }); + // In the test environment, formData.get('file') does not return a file of type File, so we mock it differently + // here + const request = { + formData: vi.fn().mockResolvedValue({ + get: vi.fn().mockReturnValue(testFile) + }) + } as unknown as Request; + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ + status: 500 + }); + }); + + it('should return translated text', async () => { + const fileContent = new Blob(['dummy content'], { type: 'audio/mp4' }); + const testFile = new File([fileContent], 'test.txt', { type: 'audio/mp4' }); + // In the test environment, formData.get('file') does not return a file of type File, so we mock it differently + // here + const request = { + formData: vi.fn().mockResolvedValue({ + get: vi.fn().mockReturnValue(testFile) + }) + } as unknown as Request; + + const res = await POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >); + const data = await res.json(); + expect(data).toEqual({ text: 'Fake translation' }); + }); +}); diff --git a/src/leapfrogai_ui/svelte.config.js b/src/leapfrogai_ui/svelte.config.js index 97f5b0c2f..f4934c1d1 100644 --- a/src/leapfrogai_ui/svelte.config.js +++ b/src/leapfrogai_ui/svelte.config.js @@ -13,6 +13,7 @@ const config = { // If your environment is not supported or you settled on a specific environment, switch out the adapter. // See https://kit.svelte.dev/docs/adapters for more information about adapters. adapter: adapter(), + csrf: { checkOrigin: process.env.NODE_ENV !== 'development' }, alias: { $components: 'src/lib/components', $webComponents: 'src/lib/web-components', From 3408a554b1da3724b7928830c7a23967708d66eb Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Mon, 16 Sep 2024 09:49:23 -0600 Subject: [PATCH 02/18] Add max file size --- src/leapfrogai_ui/src/lib/constants/index.ts | 2 ++ src/leapfrogai_ui/src/lib/schemas/files.ts | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/leapfrogai_ui/src/lib/constants/index.ts b/src/leapfrogai_ui/src/lib/constants/index.ts index 32ca04cca..d40a5c716 100644 --- a/src/leapfrogai_ui/src/lib/constants/index.ts +++ b/src/leapfrogai_ui/src/lib/constants/index.ts @@ -4,6 +4,7 @@ export const MAX_LABEL_SIZE = 100; export const DEFAULT_ASSISTANT_TEMP = 0.2; export const MAX_AVATAR_SIZE = 5000000; export const MAX_FILE_SIZE = 512000000; +export const MAX_AUDIO_FILE_SIZE = 25000000; export const MAX_FILE_NAME_SIZE = 27; export const MAX_NUM_FILES_UPLOAD = 50; // for chat completion @@ -100,6 +101,7 @@ export const FILE_TYPE_MAP = { export const NO_FILE_ERROR_TEXT = 'Please upload an image or select a pictogram'; export const AVATAR_FILE_SIZE_ERROR_TEXT = `File must be less than ${MAX_AVATAR_SIZE / 1000000} MB`; export const FILE_SIZE_ERROR_TEXT = `File must be less than ${MAX_FILE_SIZE / 1000000} MB`; +export const AUDIO_FILE_SIZE_ERROR_TEXT = `Audio file must be less than ${MAX_AUDIO_FILE_SIZE / 1000000} MB`; export const INVALID_FILE_TYPE_ERROR_TEXT = `Invalid file type, accepted types are: ${ACCEPTED_FILE_TYPES.join(', ')}`; export const INVALID_AUDIO_FILE_TYPE_ERROR_TEXT = `Invalid file type, accepted types are: ${ACCEPTED_AUDIO_FILE_TYPES.join(', ')}`; export const NO_SELECTED_ASSISTANT_ID = 'noSelectedAssistantId'; diff --git a/src/leapfrogai_ui/src/lib/schemas/files.ts b/src/leapfrogai_ui/src/lib/schemas/files.ts index 14d1d85d6..9b856ac27 100644 --- a/src/leapfrogai_ui/src/lib/schemas/files.ts +++ b/src/leapfrogai_ui/src/lib/schemas/files.ts @@ -2,9 +2,11 @@ import { array, mixed, object, string, ValidationError } from 'yup'; import { ACCEPTED_AUDIO_FILE_MIME_TYPES, ACCEPTED_MIME_TYPES, + AUDIO_FILE_SIZE_ERROR_TEXT, FILE_SIZE_ERROR_TEXT, INVALID_AUDIO_FILE_TYPE_ERROR_TEXT, INVALID_FILE_TYPE_ERROR_TEXT, + MAX_AUDIO_FILE_SIZE, MAX_FILE_SIZE } from '$constants'; @@ -71,12 +73,12 @@ export const fileSchema = object({ export const audioFileSchema = object({ file: mixed() .test('fileType', 'File is required.', (value) => value == null || value instanceof File) - .test('fileSize', FILE_SIZE_ERROR_TEXT, (value) => { + .test('fileSize', AUDIO_FILE_SIZE_ERROR_TEXT, (value) => { if (value == null) { return true; } - if (value.size > MAX_FILE_SIZE) { - return new ValidationError(FILE_SIZE_ERROR_TEXT); + if (value.size > MAX_AUDIO_FILE_SIZE) { + return new ValidationError(AUDIO_FILE_SIZE_ERROR_TEXT); } return true; }) From 419c74e05518f42b2bd1e5a1c057989ba019ccd9 Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Mon, 16 Sep 2024 15:37:47 -0600 Subject: [PATCH 03/18] before changing to use api instead of form action --- .../lib/components/ChatFileUploadForm.svelte | 33 ++++--- .../src/lib/components/FileChatActions.svelte | 87 +++++++++++++++++++ .../lib/components/UploadedFileCard.svelte | 8 +- .../lib/components/UploadedFileCards.svelte | 2 +- src/leapfrogai_ui/src/lib/constants/index.ts | 35 ++++---- .../src/lib/constants/toastMessages.ts | 7 ++ src/leapfrogai_ui/src/lib/schemas/files.ts | 16 ++-- src/leapfrogai_ui/src/lib/types/files.d.ts | 4 + .../routes/api/audio/translation/+server.ts | 1 + .../(dashboard)/[[thread_id]]/+page.server.ts | 17 ++++ .../(dashboard)/[[thread_id]]/+page.svelte | 18 ++-- 11 files changed, 187 insertions(+), 41 deletions(-) create mode 100644 src/leapfrogai_ui/src/lib/components/FileChatActions.svelte diff --git a/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte b/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte index 894d97eaf..3d94f92a6 100644 --- a/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte +++ b/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte @@ -11,10 +11,12 @@ ERROR_PROCESSING_FILE_MSG_TOAST, MAX_NUM_FILES_UPLOAD_MSG_TOAST } from '$constants/toastMessages'; + import type {LFFile} from "$lib/types/files"; export let form; export let uploadingFiles; export let attachedFileMetadata; + export let uploadedFiles: LFFile[]; const handleUploadError = (errorMsg) => { uploadingFiles = false; @@ -25,24 +27,20 @@ }); }; - const { enhance, submit } = superForm(form, { + const { + form: storeForm, + enhance, + submit + } = superForm(form, { validators: yup(filesSchema), invalidateAll: false, onResult({ result, cancel }) { uploadingFiles = false; if (result.type === 'success') { - attachedFileMetadata = attachedFileMetadata.filter((file) => file.status !== 'uploading'); - attachedFileMetadata = [ - ...attachedFileMetadata, - ...result.data.extractedFilesText.map((file) => ({ - id: uuidv4().substring(0, 8), - ...file - })) - ]; + attachedFileMetadata = result.data.extractedFilesText; } else { handleUploadError('Internal Error'); } - cancel(); // cancel the rest of the event chain and any form updates to prevent losing focus of chat input }, onError(e) { @@ -65,14 +63,23 @@ return; } uploadingFiles = true; - // Metadata is limited to 512 characters, we use a short id to save space for (const file of e.detail) { + // Metadata is limited to 512 characters, we use a short id to save space + const id = uuidv4().substring(0, 8); + file.id = id; attachedFileMetadata = [ ...attachedFileMetadata, - { id: uuidv4().substring(0, 8), name: file.name, type: file.type, status: 'uploading' } + { + id, + name: file.name, + type: file.type, + status: 'uploading' + } ]; } - + $storeForm.files = e.detail; + console.log("$storeForm.files", $storeForm.files) + uploadedFiles = [...e.detail]; submit(e.detail); }} accept={ACCEPTED_FILE_TYPES} diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte new file mode 100644 index 000000000..546d07d4f --- /dev/null +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte @@ -0,0 +1,87 @@ + + +
0 + ? 'ml-6 flex max-w-full gap-2 overflow-x-auto bg-gray-700' + : 'hidden'} +> + {#each audioFiles as file} + + {/each} +
diff --git a/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte b/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte index 6f4b491eb..951bb8c67 100644 --- a/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte +++ b/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte @@ -1,6 +1,6 @@
@@ -77,10 +156,9 @@ } ]; } - $storeForm.files = e.detail; - console.log("$storeForm.files", $storeForm.files) + uploadedFiles = [...e.detail]; - submit(e.detail); + convertFiles(e.detail); }} accept={ACCEPTED_FILE_TYPES} disabled={uploadingFiles} diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte index 546d07d4f..f429bef7a 100644 --- a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte @@ -1,5 +1,5 @@ @@ -80,8 +82,18 @@ : 'hidden'} > {#each audioFiles as file} - translateFile(file)} + disabled={translating} + > + {#if translating} + {`Translating ${shortenFileName(file.name)}`} + {:else} + {`Translate ${shortenFileName(file.name)}`}{/if} {/each} diff --git a/src/leapfrogai_ui/src/lib/constants/index.ts b/src/leapfrogai_ui/src/lib/constants/index.ts index 3736f4714..a020f7391 100644 --- a/src/leapfrogai_ui/src/lib/constants/index.ts +++ b/src/leapfrogai_ui/src/lib/constants/index.ts @@ -1,4 +1,5 @@ import type { LFAssistant } from '$lib/types/assistants'; +import {env} from "$env/dynamic/public"; export const MAX_LABEL_SIZE = 100; export const DEFAULT_ASSISTANT_TEMP = 0.2; @@ -13,10 +14,13 @@ export const ASSISTANTS_NAME_MAX_LENGTH = 256; export const ASSISTANTS_DESCRIPTION_MAX_LENGTH = 512; export const ASSISTANTS_INSTRUCTIONS_MAX_LENGTH = 256000; + + // 1 token is approx 4 characters, whenever our max context window changes, this value will need to change // leave a small buffer to prevent overflowing context (ex. 32k context window, set here to 31.750k) export const APPROX_MAX_CHARACTERS = 31750; + // TODO - once using API to save, these defaults should be returned by the POST call and would not need to be supplied // We only need to default the model and tools export const assistantDefaults: Omit = { @@ -112,3 +116,6 @@ export const INVALID_AUDIO_FILE_TYPE_ERROR_TEXT = `Invalid file type, accepted t export const NO_SELECTED_ASSISTANT_ID = 'noSelectedAssistantId'; export const FILE_UPLOAD_PROMPT = "The following are the user's files: "; + +export const ADJUSTED_MAX_CHARACTERS = + APPROX_MAX_CHARACTERS - Number(env.PUBLIC_MESSAGE_LENGTH_LIMIT) - FILE_UPLOAD_PROMPT.length - 2; \ No newline at end of file diff --git a/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts b/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts new file mode 100644 index 000000000..10e349196 --- /dev/null +++ b/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts @@ -0,0 +1,65 @@ +import type { RequestHandler } from './$types'; +import * as mupdf from 'mupdf'; +import { error, json } from '@sveltejs/kit'; +import { fileSchema } from '$schemas/files'; +import { shortenFileName } from '$helpers/stringHelpers'; +import type { LFFile } from '$lib/types/files'; + +export const POST: RequestHandler = async ({ request, fetch, locals: { session } }) => { + if (!session) { + error(401, 'Unauthorized'); + } + + let file: LFFile | null; + // Validate request body + try { + const formData = await request.formData(); + file = formData.get('file') as LFFile; + await fileSchema.validate({ file }, { abortEarly: false }); + } catch (e) { + console.error('Validation error:', e); + error(400, `Bad Request, File invalid: ${e}`); + } + + try { + let text = ''; + let buffer: ArrayBuffer; + const contentType = file.type; + if (contentType !== 'application/pdf') { + // Convert file to PDF + const formData = new FormData(); + formData.append('file', file); + const convertRes = await fetch('/api/files/convert', { + method: 'POST', + body: formData + }); + + if (!convertRes.ok) { + return error(500, { message: 'Error converting file', id: file.id }); + } + + const convertedFileBlob = await convertRes.blob(); + buffer = await convertedFileBlob.arrayBuffer(); + } else buffer = await file.arrayBuffer(); + + const document = mupdf.Document.openDocument(buffer, 'application/pdf'); + let i = 0; + while (i < document.countPages()) { + const page = document.loadPage(i); + const json = page.toStructuredText('preserve-whitespace').asJSON(); + for (const block of JSON.parse(json).blocks) { + for (const line of block.lines) { + text += line.text; + } + } + i++; + } + + return json({ + text + }); + } catch (e) { + console.error('file parse error', e); + error(500, { message: 'Internal Error', id: file.id }); + } +}; diff --git a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.server.ts b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.server.ts deleted file mode 100644 index 050b8f600..000000000 --- a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.server.ts +++ /dev/null @@ -1,131 +0,0 @@ -import * as mupdf from 'mupdf'; -import { fail, superValidate, withFiles } from 'sveltekit-superforms'; -import { yup } from 'sveltekit-superforms/adapters'; -import type { Actions } from './$types'; -import { filesSchema } from '$schemas/files'; -import type { FileMetadata } from '$lib/types/files'; -import { env } from '$env/dynamic/public'; -import { shortenFileName } from '$helpers/stringHelpers'; -import { APPROX_MAX_CHARACTERS, FILE_UPLOAD_PROMPT } from '$constants'; -import { ERROR_UPLOADING_FILE_MSG, FILE_CONTEXT_TOO_LARGE_ERROR_MSG } from '$constants/errors'; -import { v4 as uuidv4 } from 'uuid'; - -// Ensure length of file context message does not exceed total context window when including the -// file upload prompt, user's message, and string quotes -const ADJUSTED_MAX = - APPROX_MAX_CHARACTERS - Number(env.PUBLIC_MESSAGE_LENGTH_LIMIT) - FILE_UPLOAD_PROMPT.length - 2; - -export const actions: Actions = { - // Handles parsing text from files, will convert file to pdf if is not already - default: async ({ request, fetch, locals: { session } }) => { - if (!session) { - return fail(401, { message: 'Unauthorized' }); - } - - const form = await superValidate(request, yup(filesSchema)); - - if (!form.valid) { - console.log( - 'Files form action: Invalid form submission.', - 'id:', - form.id, - 'errors:', - form.errors - ); - return fail(400, { form }); - } - - const extractedFilesText: FileMetadata[] = []; - - if (form.data.files && form.data.files.length > 0) { - for (const file of form.data.files) { - console.log("Server side file", file) - let text = ''; - if (file) { - try { - let buffer: ArrayBuffer; - const contentType = file.type; - - // Skip audio files - if (contentType.startsWith('audio/')) { - extractedFilesText.push({ - id: file.id, - name: shortenFileName(file.name), - type: file.type, - text: 'Audio file contents were not processed', - status: 'complete' - }); - continue; - } - if (contentType !== 'application/pdf') { - // Convert file to PDF - const formData = new FormData(); - formData.append('file', file); - const convertRes = await fetch('/api/files/convert', { - method: 'POST', - body: formData - }); - - if (!convertRes.ok) { - throw new Error('Error converting file'); //caught locally - } - - const convertedFileBlob = await convertRes.blob(); - buffer = await convertedFileBlob.arrayBuffer(); - } else buffer = await file.arrayBuffer(); - - const document = mupdf.Document.openDocument(buffer, 'application/pdf'); - let i = 0; - while (i < document.countPages()) { - const page = document.loadPage(i); - const json = page.toStructuredText('preserve-whitespace').asJSON(); - for (const block of JSON.parse(json).blocks) { - for (const line of block.lines) { - text += line.text; - } - } - i++; - } - - extractedFilesText.push({ - id: file.id, - name: shortenFileName(file.name), - type: file.type, - text, - status: 'complete' - }); - - // If this file adds too much text (larger than allowed max), remove the text and set to error status - const totalTextLength = extractedFilesText.reduce( - (acc, fileMetadata) => acc + JSON.stringify(fileMetadata).length, - 0 - ); - if (totalTextLength > ADJUSTED_MAX) { - extractedFilesText[extractedFilesText.length - 1] = { - id: file.id, - name: shortenFileName(file.name), - type: file.type, - text: '', - status: 'error', - errorText: FILE_CONTEXT_TOO_LARGE_ERROR_MSG - }; - } - } catch (e) { - console.error(`Error uploading file: ${file}: ${e}`); - extractedFilesText.push({ - id: file.id, - name: shortenFileName(file.name), - type: file.type, - text: '', - status: 'error', - errorText: ERROR_UPLOADING_FILE_MSG - }); - } - } - } - - return withFiles({ extractedFilesText, form }); - } - return fail(400, { form }); - } -}; diff --git a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte index faff911ce..6cde7b37b 100644 --- a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte +++ b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte @@ -30,7 +30,7 @@ import FileChatActions from '$components/FileChatActions.svelte'; export let data; - $: console.log('attachedFileMetadata', attachedFileMetadata) + $: console.log('$chatMessages', $chatMessages) /** LOCAL VARS **/ let lengthInvalid: boolean; // bound to child LFTextArea @@ -408,7 +408,7 @@ > {/if} - + From 26cf95fcc2972541f083855d5d3db313514df72b Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Mon, 16 Sep 2024 17:05:53 -0600 Subject: [PATCH 05/18] style multiple --- .../lib/components/ChatFileUploadForm.svelte | 228 ++++++++---------- .../src/lib/components/FileChatActions.svelte | 44 ++-- .../routes/api/files/parse-text/+server.ts | 1 - .../(dashboard)/[[thread_id]]/+page.svelte | 3 +- .../chat/(dashboard)/[[thread_id]]/+page.ts | 6 +- 5 files changed, 131 insertions(+), 151 deletions(-) diff --git a/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte b/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte index 88ee52ea4..3162a10b2 100644 --- a/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte +++ b/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte @@ -3,9 +3,6 @@ import { PaperClipOutline } from 'flowbite-svelte-icons'; import { v4 as uuidv4 } from 'uuid'; import LFFileUploadBtn from '$components/LFFileUploadBtn.svelte'; - import { superForm } from 'sveltekit-superforms'; - import { yup } from 'sveltekit-superforms/adapters'; - import { filesSchema } from '$schemas/files'; import { toastStore } from '$stores'; import { ERROR_PROCESSING_FILE_MSG_TOAST, @@ -15,7 +12,6 @@ import { ERROR_UPLOADING_FILE_MSG, FILE_CONTEXT_TOO_LARGE_ERROR_MSG } from '$constants/errors'; import { shortenFileName } from '$helpers/stringHelpers'; - export let form; export let uploadingFiles; export let attachedFileMetadata; export let uploadedFiles: LFFile[]; @@ -29,47 +25,51 @@ }); }; - const { enhance, submit } = superForm(form, { - validators: yup(filesSchema), - invalidateAll: false, - onResult({ result, cancel }) { - uploadingFiles = false; - if (result.type === 'success') { - attachedFileMetadata = result.data.extractedFilesText; - } else { - handleUploadError('Internal Error'); - } - cancel(); // cancel the rest of the event chain and any form updates to prevent losing focus of chat input - }, - onError(e) { - handleUploadError(e.result.error.message); - uploadingFiles = false; - } - }); - const convertFiles = (files: LFFile[]) => { - const promises = []; - const parsedFiles = []; + uploadingFiles = true; + try { + const promises = []; + const parsedFiles = []; - for (const file of files) { - if (file.type.startsWith('audio/')) { - parsedFiles.push({ - id: file.id, - name: shortenFileName(file.name), - type: file.type, - text: 'Audio file contents were not processed', - status: 'complete' - }); - } else { - const formData = new FormData(); - formData.append('file', file); + for (const file of files) { + if (file.type.startsWith('audio/')) { + parsedFiles.push({ + id: file.id, + name: shortenFileName(file.name), + type: file.type, + text: 'Audio file contents were not processed', + status: 'complete' + }); + } else { + const formData = new FormData(); + formData.append('file', file); + + const promise = fetch('/api/files/parse-text', { + method: 'POST', + body: formData + }) + .then(async (response) => { + if (!response.ok) { + return { + id: file.id, + name: shortenFileName(file.name), + type: file.type, + text: '', + status: 'error', + errorText: ERROR_UPLOADING_FILE_MSG + }; + } - const promise = fetch('/api/files/parse-text', { - method: 'POST', - body: formData - }) - .then(async (response) => { - if (!response.ok) { + const result = await response.json(); + return { + id: file.id, + name: shortenFileName(file.name), + type: file.type, + text: result.text, + status: 'complete' + }; + }) + .catch(() => { return { id: file.id, name: shortenFileName(file.name), @@ -78,93 +78,75 @@ status: 'error', errorText: ERROR_UPLOADING_FILE_MSG }; - } + }); - const result = await response.json(); - return { - id: file.id, - name: shortenFileName(file.name), - type: file.type, - text: result.text, - status: 'complete' - }; - }) - .catch(() => { - return { - id: file.id, - name: shortenFileName(file.name), - type: file.type, - text: '', - status: 'error', - errorText: ERROR_UPLOADING_FILE_MSG - }; - }); - - promises.push(promise); + promises.push(promise); + } } - } - Promise.all(promises).then((results) => { - parsedFiles.push(...results); - const totalTextLength = parsedFiles.reduce( - (acc, fileMetadata) => acc + JSON.stringify(fileMetadata).length, - 0 - ); + Promise.all(promises).then((results) => { + parsedFiles.push(...results); + const totalTextLength = parsedFiles.reduce( + (acc, fileMetadata) => acc + JSON.stringify(fileMetadata).length, + 0 + ); - // If this file adds too much text (larger than allowed max), remove the text and set to error status - if (totalTextLength > ADJUSTED_MAX_CHARACTERS) { - let lastFile = parsedFiles[parsedFiles.length - 1]; - lastFile = { - id: lastFile.id, - name: shortenFileName(lastFile.name), - type: lastFile.type, - text: '', - status: 'error', - errorText: FILE_CONTEXT_TOO_LARGE_ERROR_MSG - }; - } + // If this file adds too much text (larger than allowed max), remove the text and set to error status + if (totalTextLength > ADJUSTED_MAX_CHARACTERS) { + let lastFile = parsedFiles[parsedFiles.length - 1]; + lastFile = { + id: lastFile.id, + name: shortenFileName(lastFile.name), + type: lastFile.type, + text: '', + status: 'error', + errorText: FILE_CONTEXT_TOO_LARGE_ERROR_MSG + }; + } - attachedFileMetadata = parsedFiles; - }); + attachedFileMetadata = parsedFiles; + }); + } catch { + handleUploadError('Internal Error'); + } + uploadingFiles = false; }; - - { - if (e.detail.length > MAX_NUM_FILES_UPLOAD) { - toastStore.addToast(MAX_NUM_FILES_UPLOAD_MSG_TOAST()); - return; - } - uploadingFiles = true; - for (const file of e.detail) { - // Metadata is limited to 512 characters, we use a short id to save space - const id = uuidv4().substring(0, 8); - file.id = id; - attachedFileMetadata = [ - ...attachedFileMetadata, - { - id, - name: file.name, - type: file.type, - status: 'uploading' - } - ]; - } + { + if (e.detail.length > MAX_NUM_FILES_UPLOAD) { + toastStore.addToast(MAX_NUM_FILES_UPLOAD_MSG_TOAST()); + return; + } + uploadingFiles = true; + for (const file of e.detail) { + // Metadata is limited to 512 characters, we use a short id to save space + const id = uuidv4().substring(0, 8); + file.id = id; + attachedFileMetadata = [ + ...attachedFileMetadata, + { + id, + name: file.name, + type: file.type, + status: 'uploading' + } + ]; + } - uploadedFiles = [...e.detail]; - convertFiles(e.detail); - }} - accept={ACCEPTED_FILE_TYPES} - disabled={uploadingFiles} - class="remove-btn-style flex rounded-lg p-1.5 text-gray-500 hover:bg-inherit dark:hover:bg-inherit" - > - - Attach file - - + uploadedFiles = [...e.detail]; + convertFiles(e.detail); + }} + accept={ACCEPTED_FILE_TYPES} + disabled={uploadingFiles} + class="remove-btn-style flex rounded-lg p-1.5 text-gray-500 hover:bg-inherit dark:hover:bg-inherit" +> + + Attach file + diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte index f429bef7a..fbb0de14f 100644 --- a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte @@ -1,5 +1,6 @@
0 - ? 'ml-6 flex max-w-full gap-2 overflow-x-auto bg-gray-700' + ? 'ml-6 flex max-w-full gap-2 overflow-x-auto bg-gray-700' : 'hidden'} > {#each audioFiles as file} - +
+ +
{/each}
diff --git a/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts b/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts index 10e349196..65f56db19 100644 --- a/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts +++ b/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts @@ -2,7 +2,6 @@ import type { RequestHandler } from './$types'; import * as mupdf from 'mupdf'; import { error, json } from '@sveltejs/kit'; import { fileSchema } from '$schemas/files'; -import { shortenFileName } from '$helpers/stringHelpers'; import type { LFFile } from '$lib/types/files'; export const POST: RequestHandler = async ({ request, fetch, locals: { session } }) => { diff --git a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte index 6cde7b37b..e3986d5b7 100644 --- a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte +++ b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte @@ -30,7 +30,6 @@ import FileChatActions from '$components/FileChatActions.svelte'; export let data; - $: console.log('$chatMessages', $chatMessages) /** LOCAL VARS **/ let lengthInvalid: boolean; // bound to child LFTextArea @@ -367,7 +366,7 @@
{#if !assistantMode} - + {/if} { const promises = [fetch('/api/assistants'), fetch('/api/files')]; - const form = await superValidate(yup(filesSchema)); if (params.thread_id) promises.push(fetch(`/api/threads/${params.thread_id}`)); @@ -29,5 +25,5 @@ export const load: PageLoad = async ({ params, fetch }) => { } } - return { thread, assistants, files, form }; + return { thread, assistants, files }; }; From 7bd9f56273b8d1dd0a72f16dac42e2b12889b7ee Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Tue, 17 Sep 2024 11:01:06 -0600 Subject: [PATCH 06/18] style fixes, fix same file double upload --- ...ploadForm.svelte => ChatFileUpload.svelte} | 85 ++++++++++--------- .../src/lib/components/FileChatActions.svelte | 20 +++-- .../lib/components/UploadedFileCard.svelte | 2 +- .../lib/components/UploadedFileCards.svelte | 4 +- src/leapfrogai_ui/src/lib/stores/threads.ts | 8 ++ .../(dashboard)/[[thread_id]]/+page.svelte | 17 ++-- 6 files changed, 78 insertions(+), 58 deletions(-) rename src/leapfrogai_ui/src/lib/components/{ChatFileUploadForm.svelte => ChatFileUpload.svelte} (72%) diff --git a/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte b/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte similarity index 72% rename from src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte rename to src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte index 3162a10b2..e63fc0461 100644 --- a/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte +++ b/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte @@ -14,7 +14,9 @@ export let uploadingFiles; export let attachedFileMetadata; - export let uploadedFiles: LFFile[]; + export let attachedFiles: LFFile[]; + + let fileUploadBtnRef: HTMLInputElement; const handleUploadError = (errorMsg) => { uploadingFiles = false; @@ -54,7 +56,7 @@ id: file.id, name: shortenFileName(file.name), type: file.type, - text: '', + text: ERROR_UPLOADING_FILE_MSG, status: 'error', errorText: ERROR_UPLOADING_FILE_MSG }; @@ -74,7 +76,7 @@ id: file.id, name: shortenFileName(file.name), type: file.type, - text: '', + text: ERROR_UPLOADING_FILE_MSG, status: 'error', errorText: ERROR_UPLOADING_FILE_MSG }; @@ -113,40 +115,45 @@ }; - { - if (e.detail.length > MAX_NUM_FILES_UPLOAD) { - toastStore.addToast(MAX_NUM_FILES_UPLOAD_MSG_TOAST()); - return; - } - uploadingFiles = true; - for (const file of e.detail) { - // Metadata is limited to 512 characters, we use a short id to save space - const id = uuidv4().substring(0, 8); - file.id = id; - attachedFileMetadata = [ - ...attachedFileMetadata, - { - id, - name: file.name, - type: file.type, - status: 'uploading' - } - ]; - } - uploadedFiles = [...e.detail]; - convertFiles(e.detail); - }} - accept={ACCEPTED_FILE_TYPES} - disabled={uploadingFiles} - class="remove-btn-style flex rounded-lg p-1.5 text-gray-500 hover:bg-inherit dark:hover:bg-inherit" -> - - Attach file - + { + if (e.detail.length > MAX_NUM_FILES_UPLOAD) { + e.detail.pop(); + toastStore.addToast(MAX_NUM_FILES_UPLOAD_MSG_TOAST()); + return; + } + uploadingFiles = true; + for (const file of e.detail) { + // Metadata is limited to 512 characters, we use a short id to save space + const id = uuidv4().substring(0, 8); + file.id = id; + attachedFileMetadata = [ + ...attachedFileMetadata, + { + id, + name: file.name, + type: file.type, + status: 'uploading' + } + ]; + } + + attachedFiles = [...e.detail]; + convertFiles(e.detail); + fileUploadBtnRef.value = ''; + }} + accept={ACCEPTED_FILE_TYPES} + disabled={uploadingFiles} + class="remove-btn-style flex rounded-lg p-1.5 text-gray-500 hover:bg-inherit dark:hover:bg-inherit" + > + + Attach file + + diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte index fbb0de14f..4dee72bbb 100644 --- a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte @@ -8,10 +8,13 @@ import { FILE_TRANSLATION_ERROR } from '$constants/toastMessages'; import { tick } from 'svelte'; import { page } from '$app/stores'; + import type { Message } from 'ai'; - export let uploadedFiles: LFFile[]; + export let attachedFiles: LFFile[]; export let attachedFileMetadata: FileMetadata[]; export let threadId: string; + export let originalMessages: Message[]; + export let setMessages: (messages: Message[]) => void; let translatingId: string; @@ -21,7 +24,11 @@ 'rounded text-xs px-2.5 py-0.5 text-gray-500 bg-gray-100 hover:bg-gray-400 dark:bg-gray-100 dark:text-gray-900 dark:hover:bg-gray-300 truncate'; const translateFile = async (fileMetadata: FileMetadata) => { - translatingId = fileMetadata.id || 'unknown'; + if (!fileMetadata.id) { + toastStore.addToast(FILE_TRANSLATION_ERROR()); + return; + } + translatingId = fileMetadata.id; await threadsStore.setSendingBlocked(true); try { if (!threadId) { @@ -43,10 +50,10 @@ } }); await threadsStore.addMessageToStore(newMessage); - // TODO - need to add to $chatMessages? + threadsStore.updateMessagesState(originalMessages, setMessages, newMessage); // translate - const file = uploadedFiles.find((f) => f.id === fileMetadata.id); + const file = attachedFiles.find((f) => f.id === fileMetadata.id); if (!file) throw Error('File not found'); const formData = new FormData(); @@ -64,13 +71,12 @@ role: 'assistant' }); await threadsStore.addMessageToStore(translationMessage); + threadsStore.updateMessagesState(originalMessages, setMessages, translationMessage); - uploadedFiles = uploadedFiles.filter((file) => file.id !== fileMetadata.id); + attachedFiles = attachedFiles.filter((file) => file.id !== fileMetadata.id); attachedFileMetadata = attachedFileMetadata.filter((file) => file.id !== fileMetadata.id); } catch { - translatingId = ''; toastStore.addToast(FILE_TRANSLATION_ERROR()); - await threadsStore.setSendingBlocked(false); } await threadsStore.setSendingBlocked(false); translatingId = ''; diff --git a/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte b/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte index 951bb8c67..3c57e0ce1 100644 --- a/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte +++ b/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte @@ -20,7 +20,7 @@ data-testid={`${fileMetadata.name}-file-uploaded-card`} horizontal padding="xs" - class="w-80 min-w-72" + class="w-80 min-w-72 border-none" on:mouseenter={() => (hovered = true)} on:mouseleave={() => (hovered = false)} > diff --git a/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte b/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte index 45ff54819..3ff26dc79 100644 --- a/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte +++ b/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte @@ -1,11 +1,13 @@ diff --git a/src/leapfrogai_ui/src/lib/stores/threads.ts b/src/leapfrogai_ui/src/lib/stores/threads.ts index 354f5e14c..273875b8a 100644 --- a/src/leapfrogai_ui/src/lib/stores/threads.ts +++ b/src/leapfrogai_ui/src/lib/stores/threads.ts @@ -8,6 +8,7 @@ import type { LFThread, NewThreadInput } from '$lib/types/threads'; import type { LFMessage } from '$lib/types/messages'; import { getMessageText } from '$helpers/threads'; import { saveMessage } from '$helpers/chatHelpers'; +import type { Message } from 'ai'; type ThreadsStore = { threads: LFThread[]; @@ -208,6 +209,13 @@ const createThreadsStore = () => { }); } }, + updateMessagesState: ( + originalMessages: Message[], + setMessages: (messages: Message[]) => void, + newMessage: LFMessage + ) => { + setMessages([...originalMessages, { ...newMessage, content: getMessageText(newMessage) }]); + }, deleteThread: async (id: string) => { try { await deleteThread(id); diff --git a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte index e3986d5b7..102c412f1 100644 --- a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte +++ b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte @@ -26,7 +26,7 @@ import { PaperPlaneOutline, StopOutline } from 'flowbite-svelte-icons'; import type {FileMetadata, LFFile} from '$lib/types/files'; import UploadedFileCards from '$components/UploadedFileCards.svelte'; - import ChatFileUploadForm from '$components/ChatFileUploadForm.svelte'; + import ChatFileUploadForm from '$components/ChatFileUpload.svelte'; import FileChatActions from '$components/FileChatActions.svelte'; export let data; @@ -35,8 +35,8 @@ let lengthInvalid: boolean; // bound to child LFTextArea let assistantsList: Array<{ id: string; text: string }>; let uploadingFiles = false; - let attachedFileMetadata: FileMetadata[] = []; - let uploadedFiles: LFFile[] = []; + let attachedFiles: LFFile[] = []; // the actual files uploaded + let attachedFileMetadata: FileMetadata[] = []; // metadata about the files uploaded, e.g. upload status, extracted text, etc... /** END LOCAL VARS **/ /** REACTIVE STATE **/ @@ -231,10 +231,7 @@ }, lengthOverride: true }); - setChatMessages([ - ...$chatMessages, - { ...contextMsg, content: getMessageText(contextMsg) } - ]); + threadsStore.updateMessagesState($chatMessages, setChatMessages, contextMsg) } // Save with API @@ -362,11 +359,11 @@ attachedFileMetadata.length > 0 && 'py-4' )} > - +
{#if !assistantMode} - + {/if} {/if}
- +
From 92b9f16dedb5ef843024ade508e464e326dea30d Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Tue, 17 Sep 2024 13:28:19 -0600 Subject: [PATCH 07/18] refactor unit test to not use form action --- .../src/lib/components/ChatFileUpload.svelte | 82 ++++----- .../lib/components/UploadedFileCards.svelte | 2 +- src/leapfrogai_ui/src/lib/constants/index.ts | 7 +- .../src/lib/helpers/apiHelpers.ts | 35 ++++ src/leapfrogai_ui/src/lib/mocks/openai.ts | 7 +- src/leapfrogai_ui/src/lib/schemas/files.ts | 2 +- src/leapfrogai_ui/src/lib/types/files.d.ts | 2 +- .../api/audio/translation/server.test.ts | 17 +- .../routes/api/files/convert/server.test.ts | 18 +- .../routes/api/files/parse-text/+server.ts | 10 +- .../api/files/parse-text/server.test.ts | 153 +++++++++++++++ .../(dashboard)/[[thread_id]]/+page.svelte | 14 +- .../[[thread_id]]/form-action.test.ts | 174 ------------------ 13 files changed, 259 insertions(+), 264 deletions(-) create mode 100644 src/leapfrogai_ui/src/lib/helpers/apiHelpers.ts create mode 100644 src/leapfrogai_ui/src/routes/api/files/parse-text/server.test.ts delete mode 100644 src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/form-action.test.ts diff --git a/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte b/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte index e63fc0461..d23ee221b 100644 --- a/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte +++ b/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte @@ -96,7 +96,7 @@ // If this file adds too much text (larger than allowed max), remove the text and set to error status if (totalTextLength > ADJUSTED_MAX_CHARACTERS) { let lastFile = parsedFiles[parsedFiles.length - 1]; - lastFile = { + parsedFiles[parsedFiles.length - 1] = { id: lastFile.id, name: shortenFileName(lastFile.name), type: lastFile.type, @@ -115,45 +115,43 @@ }; + { + if (e.detail.length > MAX_NUM_FILES_UPLOAD) { + e.detail.pop(); + toastStore.addToast(MAX_NUM_FILES_UPLOAD_MSG_TOAST()); + return; + } + uploadingFiles = true; + for (const file of e.detail) { + // Metadata is limited to 512 characters, we use a short id to save space + const id = uuidv4().substring(0, 8); + file.id = id; + attachedFileMetadata = [ + ...attachedFileMetadata, + { + id, + name: file.name, + type: file.type, + status: 'uploading' + } + ]; + } - { - if (e.detail.length > MAX_NUM_FILES_UPLOAD) { - e.detail.pop(); - toastStore.addToast(MAX_NUM_FILES_UPLOAD_MSG_TOAST()); - return; - } - uploadingFiles = true; - for (const file of e.detail) { - // Metadata is limited to 512 characters, we use a short id to save space - const id = uuidv4().substring(0, 8); - file.id = id; - attachedFileMetadata = [ - ...attachedFileMetadata, - { - id, - name: file.name, - type: file.type, - status: 'uploading' - } - ]; - } - - attachedFiles = [...e.detail]; - convertFiles(e.detail); - fileUploadBtnRef.value = ''; - }} - accept={ACCEPTED_FILE_TYPES} - disabled={uploadingFiles} - class="remove-btn-style flex rounded-lg p-1.5 text-gray-500 hover:bg-inherit dark:hover:bg-inherit" - > - - Attach file - - + attachedFiles = [...e.detail]; + convertFiles(e.detail); + fileUploadBtnRef.value = ''; + }} + accept={ACCEPTED_FILE_TYPES} + disabled={uploadingFiles} + class="remove-btn-style flex rounded-lg p-1.5 text-gray-500 hover:bg-inherit dark:hover:bg-inherit" +> + + Attach file + diff --git a/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte b/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte index 3ff26dc79..85ddfe7b3 100644 --- a/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte +++ b/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte @@ -1,6 +1,6 @@ diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.test.ts b/src/leapfrogai_ui/src/lib/components/FileChatActions.test.ts index 153743b2e..5927b7827 100644 --- a/src/leapfrogai_ui/src/lib/components/FileChatActions.test.ts +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.test.ts @@ -3,10 +3,15 @@ import { render, screen } from '@testing-library/svelte'; import type { FileMetadata, LFFile } from '$lib/types/files'; import userEvent from '@testing-library/user-event'; import { mockConvertFileNoId } from '$lib/mocks/file-mocks'; -import { mockNewMessage, mockTranslation, mockTranslationError } from '$lib/mocks/chat-mocks'; +import { + mockNewMessage, + mockTranslation, + mockTranslationError, + mockTranslationFileSizeError +} from '$lib/mocks/chat-mocks'; import { vi } from 'vitest'; import { toastStore } from '$stores'; -import { FILE_TRANSLATION_ERROR } from '$constants/toastMessages'; +import { AUDIO_FILE_SIZE_ERROR_TOAST, FILE_TRANSLATION_ERROR } from '$constants/toastMessages'; const mockFile1: LFFile = new File([], 'test1.mpeg', { type: 'audio/mpeg' }); const mockFile2: LFFile = new File([], 'test1.mp4', { type: 'audio/mp4' }); @@ -98,4 +103,23 @@ describe('FileChatActions', () => { await userEvent.click(screen.getByRole('button', { name: `Translate ${mockMetadata2.name}` })); expect(toastSpy).toHaveBeenCalledWith(FILE_TRANSLATION_ERROR()); }); + + it('dispatches a toast if the file is too big', async () => { + mockConvertFileNoId(''); + mockNewMessage(); + mockTranslationFileSizeError(); + + const toastSpy = vi.spyOn(toastStore, 'addToast'); + + render(FileChatActions, { + attachedFiles: [mockFile1, mockFile2], + attachedFileMetadata: [mockMetadata1, mockMetadata2], + threadId: '123', + originalMessages: [], + setMessages: vi.fn() + }); + + await userEvent.click(screen.getByRole('button', { name: `Translate ${mockMetadata2.name}` })); + expect(toastSpy).toHaveBeenCalledWith(AUDIO_FILE_SIZE_ERROR_TOAST()); + }); }); diff --git a/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte b/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte index 3c57e0ce1..f4ed7a802 100644 --- a/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte +++ b/src/leapfrogai_ui/src/lib/components/UploadedFileCard.svelte @@ -24,7 +24,7 @@ on:mouseenter={() => (hovered = true)} on:mouseleave={() => (hovered = false)} > -
+
{#if fileMetadata.status === 'uploading'} @@ -33,7 +33,7 @@ {:else if fileMetadata.type.startsWith('audio/')} = {}): ToastData => ({ kind: 'error', @@ -75,6 +69,13 @@ export const FILE_VECTOR_TIMEOUT_MSG_TOAST = (override: Partial = {}) export const FILE_TRANSLATION_ERROR = (override: Partial = {}): ToastData => ({ kind: 'error', title: 'Translation Error', - subtitle: 'There was an error translating your audio file', + subtitle: 'Error translating audio file', + ...override +}); + +export const AUDIO_FILE_SIZE_ERROR_TOAST = (override: Partial = {}): ToastData => ({ + kind: 'error', + title: 'File Too Large', + subtitle: AUDIO_FILE_SIZE_ERROR_TEXT, ...override }); diff --git a/src/leapfrogai_ui/src/lib/mocks/chat-mocks.ts b/src/leapfrogai_ui/src/lib/mocks/chat-mocks.ts index 4f876339d..e820f32d4 100644 --- a/src/leapfrogai_ui/src/lib/mocks/chat-mocks.ts +++ b/src/leapfrogai_ui/src/lib/mocks/chat-mocks.ts @@ -5,6 +5,7 @@ import type { LFMessage, NewMessageInput } from '$lib/types/messages'; import type { LFAssistant } from '$lib/types/assistants'; import { createStreamDataTransformer, StreamingTextResponse } from 'ai'; import type { LFThread } from '$lib/types/threads'; +import { AUDIO_FILE_SIZE_ERROR_TEXT } from '$constants'; type MockChatCompletionOptions = { responseMsg?: string[]; @@ -114,3 +115,14 @@ export const mockTranslationError = () => { }) ); }; + +export const mockTranslationFileSizeError = () => { + server.use( + http.post('/api/audio/translation', () => { + return HttpResponse.json( + { message: `ValidationError: ${AUDIO_FILE_SIZE_ERROR_TEXT}` }, + { status: 400 } + ); + }) + ); +}; diff --git a/src/leapfrogai_ui/src/lib/types/toast.d.ts b/src/leapfrogai_ui/src/lib/types/toast.d.ts index 924d59ae5..2d22d60b2 100644 --- a/src/leapfrogai_ui/src/lib/types/toast.d.ts +++ b/src/leapfrogai_ui/src/lib/types/toast.d.ts @@ -14,3 +14,9 @@ type ToastNotificationProps = { type ToastStore = { toasts: ToastNotificationProps[]; }; + +type ToastData = { + kind: ToastKind; + title: string; + subtitle?: string; +}; diff --git a/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts b/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts index 31e6cef52..175b410e5 100644 --- a/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts +++ b/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts @@ -3,6 +3,7 @@ import type { RequestHandler } from './$types'; import { getOpenAiClient } from '$lib/server/constants'; import { audioFileSchema } from '$schemas/files'; import { env } from '$env/dynamic/private'; +import { handleError } from '$helpers/apiHelpers'; export const POST: RequestHandler = async ({ request, locals: { session } }) => { if (!session) { @@ -18,8 +19,8 @@ export const POST: RequestHandler = async ({ request, locals: { session } }) => await audioFileSchema.validate({ file }, { abortEarly: false }); } catch (e) { - console.error('Validation error:', e); - error(400, `Bad Request, File invalid: ${e}`); + console.error(e); + error(400, { message: `${e}` }); } try { @@ -30,7 +31,6 @@ export const POST: RequestHandler = async ({ request, locals: { session } }) => }); return json({ text: translation.text }); } catch (e) { - console.error('file translation error', e); - error(500, 'Internal Error'); + return handleError(e); } }; diff --git a/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts b/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts index 65c444b47..b0aa3733a 100644 --- a/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts +++ b/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts @@ -4,6 +4,15 @@ import type { RouteParams } from './$types'; import { POST } from './+server'; import { mockOpenAI } from '../../../../../vitest-setup'; import { requestWithFormData } from '$helpers/apiHelpers'; +import * as constants from '$constants'; + +// Allows mocking important constants and only overriding values for specific tests +vi.mock('$constants', async () => { + const actualConstants = await vi.importActual('$constants'); + return { + ...actualConstants + }; +}); describe('/api/audio/translation', () => { it('returns a 401 when there is no session', async () => { @@ -66,6 +75,26 @@ describe('/api/audio/translation', () => { }); }); + it('should return 400 if the file is too big', async () => { + // @ts-expect-error - intentionally overriding a constant for testing + vi.spyOn(constants, 'MAX_AUDIO_FILE_SIZE', 'get').mockReturnValueOnce(1); + + const fileContent = new Blob(['dummy content'], { type: 'audio/mp4' }); + const testFile = new File([fileContent], 'test.txt', { type: 'audio/mp4' }); + const request = requestWithFormData(testFile); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ + status: 400 + }); + // Reset the mock after this test + vi.resetModules(); + }); + it('should return a 500 if there is an error translating the file', async () => { mockOpenAI.setError('translation'); diff --git a/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts b/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts index c09db53d8..73bd12a7e 100644 --- a/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts +++ b/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts @@ -39,7 +39,7 @@ export const POST: RequestHandler = async ({ request, fetch, locals: { session } }); if (!convertRes.ok) { - return error(500, { message: 'Error converting file', id: file.id }); + return error(500, { message: 'Error converting file' }); } const convertedFileBlob = await convertRes.blob(); @@ -63,6 +63,6 @@ export const POST: RequestHandler = async ({ request, fetch, locals: { session } text }); } catch (e) { - return handleError(e, { id: file.id }); + return handleError(e); } }; diff --git a/src/leapfrogai_ui/src/routes/api/files/parse-text/server.test.ts b/src/leapfrogai_ui/src/routes/api/files/parse-text/server.test.ts index 72ba15264..b23e233ea 100644 --- a/src/leapfrogai_ui/src/routes/api/files/parse-text/server.test.ts +++ b/src/leapfrogai_ui/src/routes/api/files/parse-text/server.test.ts @@ -95,7 +95,7 @@ describe('/api/files/parse-text', () => { >) ).rejects.toMatchObject({ status: 500, - body: { message: 'Error converting file', id: '1' } + body: { message: 'Error converting file' } }); }); From 991c6e436d4281cfd75b798795ad741fa05ad89f Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Thu, 19 Sep 2024 11:32:46 -0600 Subject: [PATCH 13/18] merge bug fixes from summarization branch --- .../src/lib/components/ChatFileUpload.svelte | 8 +-- .../components/LFSidebarDropdownItem.svelte | 2 +- .../src/lib/helpers/fileHelpers.test.ts | 66 +++++++++++++++---- .../src/lib/helpers/fileHelpers.ts | 23 +++++++ 4 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte b/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte index db0df6c46..e01575ce2 100644 --- a/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte +++ b/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte @@ -17,7 +17,7 @@ import type { LFFile } from '$lib/types/files'; import { ERROR_UPLOADING_FILE_MSG } from '$constants/errors'; import { shortenFileName } from '$helpers/stringHelpers'; - import { removeFilesUntilUnderLimit } from '$helpers/fileHelpers'; + import { removeFilesUntilUnderLimit, updateFileMetadata } from '$helpers/fileHelpers'; export let uploadingFiles; export let attachedFileMetadata; @@ -102,7 +102,7 @@ // If this file adds too much text (larger than allowed max), remove the text and set to error status removeFilesUntilUnderLimit(parsedFiles, ADJUSTED_MAX_CHARACTERS); - attachedFileMetadata = parsedFiles; + attachedFileMetadata = updateFileMetadata(attachedFileMetadata, parsedFiles); }); } catch { handleUploadError('Internal Error'); @@ -135,12 +135,12 @@ id, name: file.name, type: file.type, - status: 'uploading' + status: file.type.startsWith('audio/') ? 'complete' : 'uploading' } ]; } - attachedFiles = [...e.detail]; + attachedFiles = [...attachedFiles, ...e.detail]; convertFiles(e.detail); fileUploadBtnRef.value = ''; }} diff --git a/src/leapfrogai_ui/src/lib/components/LFSidebarDropdownItem.svelte b/src/leapfrogai_ui/src/lib/components/LFSidebarDropdownItem.svelte index 9282826d0..7f0206739 100644 --- a/src/leapfrogai_ui/src/lib/components/LFSidebarDropdownItem.svelte +++ b/src/leapfrogai_ui/src/lib/components/LFSidebarDropdownItem.svelte @@ -116,7 +116,7 @@ It adds a "three-dot" menu button with Popover, and delete confirmation Modal $$props.class )} > -

+

{label}

diff --git a/src/leapfrogai_ui/src/lib/helpers/fileHelpers.test.ts b/src/leapfrogai_ui/src/lib/helpers/fileHelpers.test.ts index 612f56d1b..73f63f452 100644 --- a/src/leapfrogai_ui/src/lib/helpers/fileHelpers.test.ts +++ b/src/leapfrogai_ui/src/lib/helpers/fileHelpers.test.ts @@ -1,15 +1,8 @@ import { faker } from '@faker-js/faker'; import type { FileMetadata } from '$lib/types/files'; -import { removeFilesUntilUnderLimit } from '$helpers/fileHelpers'; +import { removeFilesUntilUnderLimit, updateFileMetadata } from '$helpers/fileHelpers'; import { FILE_CONTEXT_TOO_LARGE_ERROR_MSG } from '$constants/errors'; - -const getMockFileMetadata = (text = faker.word.noun()): FileMetadata => ({ - id: faker.string.uuid().substring(0, 8), - name: 'fake-file.pdf', - type: 'application/pdf', - text, - status: 'complete' -}); +import { getMockFileMetadata } from '$testUtils/fakeData'; describe('removeFilesUntilUnderLimit', () => { test('removeFilesUntilUnderLimit should remove the largest file until total size is under the max limit', () => { @@ -19,9 +12,9 @@ describe('removeFilesUntilUnderLimit', () => { const text2 = faker.word.words(150); const text3 = faker.word.words(200); const files = [ - getMockFileMetadata(text1), - getMockFileMetadata(text2), - getMockFileMetadata(text3) + getMockFileMetadata({ text: text1 }), + getMockFileMetadata({ text: text2 }), + getMockFileMetadata({ text: text3 }) ]; // Files 2 and 3 will have different length once their text is removed and they are set to error status @@ -74,7 +67,7 @@ describe('removeFilesUntilUnderLimit', () => { const text1 = faker.word.words(10); const text2 = faker.word.words(20); - const files = [getMockFileMetadata(text1), getMockFileMetadata(text2)]; + const files = [getMockFileMetadata({ text: text1 }), getMockFileMetadata({ text: text2 })]; // Assume a limit of 50 characters const maxLimit = 10000000000000; @@ -91,7 +84,7 @@ describe('removeFilesUntilUnderLimit', () => { const text1 = faker.word.words(10); const text2 = faker.word.words(20); - const files = [getMockFileMetadata(text1), getMockFileMetadata(text2)]; + const files = [getMockFileMetadata({ text: text1 }), getMockFileMetadata({ text: text2 })]; // Assume a limit of 50 characters const maxLimit = 5; @@ -106,3 +99,48 @@ describe('removeFilesUntilUnderLimit', () => { expect(files[1].status).toBe('error'); }); }); + +describe('updateFileMetadata with order preservation', () => { + it('should update existing files, add new files, and preserve the original order of old metadata', () => { + const file1 = getMockFileMetadata({ status: 'complete' }); + const file2 = getMockFileMetadata({ status: 'uploading' }); + const file3 = getMockFileMetadata({ status: 'complete' }); + const file4 = getMockFileMetadata({ status: 'complete' }); + + // 3 files + const oldMetadata: FileMetadata[] = [file1, file2, file3]; + + // updated file, and new file + const newMetadata: FileMetadata[] = [file1, { ...file2, status: 'complete' }, file3, file4]; + + const result = updateFileMetadata(oldMetadata, newMetadata); + + expect(result).toHaveLength(4); + + // Check if the order is preserved and one file was updated and new one added + + expect(result[0]).toEqual(file1); + expect(result[1]).toEqual({ + ...file2, + status: 'complete' + }); + + expect(result[2]).toEqual(file3); + expect(result[3]).toEqual(file4); + }); + + it('should return new metadata if old metadata is empty', () => { + const oldMetadata: FileMetadata[] = []; + const newMetadata: FileMetadata[] = [getMockFileMetadata()]; + const result = updateFileMetadata(oldMetadata, newMetadata); + expect(result).toHaveLength(1); + expect(result).toEqual(newMetadata); + }); + + it('should keep old metadata if no new metadata is provided', () => { + const oldMetadata: FileMetadata[] = [getMockFileMetadata()]; + const newMetadata: FileMetadata[] = []; + const result = updateFileMetadata(oldMetadata, newMetadata); + expect(result).toEqual(oldMetadata); + }); +}); diff --git a/src/leapfrogai_ui/src/lib/helpers/fileHelpers.ts b/src/leapfrogai_ui/src/lib/helpers/fileHelpers.ts index 504ea13f6..a0cd0fc5b 100644 --- a/src/leapfrogai_ui/src/lib/helpers/fileHelpers.ts +++ b/src/leapfrogai_ui/src/lib/helpers/fileHelpers.ts @@ -37,3 +37,26 @@ export const removeFilesUntilUnderLimit = (parsedFiles: FileMetadata[], max: num totalTextLength = parsedFiles.reduce((total, file) => total + JSON.stringify(file).length, 0); //recalculate total size } }; + +// Combines old and new file metadata, updating the old metadata with new metadata +export const updateFileMetadata = ( + oldMetadata: FileMetadata[], + newMetadata: FileMetadata[] +): FileMetadata[] => { + // Create a map of new metadata + const newMetadataMap = new Map(newMetadata.map((file) => [file.id, file])); + + // Update and keep the original order from old metadata + const updatedMetadata = oldMetadata.map((oldFile) => { + const newFile = newMetadataMap.get(oldFile.id); + return newFile ? { ...oldFile, ...newFile } : oldFile; + }); + + // Filter out new files that aren't already in the old metadata + const newFilesToAdd = newMetadata.filter( + (newFile) => !oldMetadata.some((oldFile) => oldFile.id === newFile.id) + ); + + // Append new files at the end while keeping the original order of oldMetadata + return [...updatedMetadata, ...newFilesToAdd]; +}; From 3a4c6a3d90b280d05634fc1f22be43eebd1ae828 Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Thu, 19 Sep 2024 12:12:36 -0600 Subject: [PATCH 14/18] add skeleton response --- .../src/lib/components/FileChatActions.svelte | 14 +++++++++++ src/leapfrogai_ui/src/lib/schemas/chat.ts | 4 ++-- src/leapfrogai_ui/src/lib/types/messages.d.ts | 2 +- .../src/routes/api/messages/new/+server.ts | 1 - .../messages/[message_id]/+server.ts | 23 +++++++++++++++++++ 5 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 src/leapfrogai_ui/src/routes/api/threads/[thread_id]/messages/[message_id]/+server.ts diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte index 3001a4672..dbfabe86b 100644 --- a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte @@ -71,6 +71,18 @@ await threadsStore.addMessageToStore(newMessage); threadsStore.updateMessagesState(originalMessages, setMessages, newMessage); + + // This is used to create a "skeleton" message while the file is being translated + // It is deleted once the translation is complete + const emptyResponse = await saveMessage({ + thread_id: threadId, + content: '', + role: 'assistant', + }); + await threadsStore.addMessageToStore(emptyResponse); + threadsStore.updateMessagesState(originalMessages, setMessages, emptyResponse); + + // translate const file = attachedFiles.find((f) => f.id === fileMetadata.id); if (!file) { @@ -95,6 +107,8 @@ return; } + await threadsStore.deleteMessage(threadId, emptyResponse.id); + // save translation response const translationMessage = await saveMessage({ thread_id: threadId, diff --git a/src/leapfrogai_ui/src/lib/schemas/chat.ts b/src/leapfrogai_ui/src/lib/schemas/chat.ts index efa1db8f6..8835e44c9 100644 --- a/src/leapfrogai_ui/src/lib/schemas/chat.ts +++ b/src/leapfrogai_ui/src/lib/schemas/chat.ts @@ -15,8 +15,8 @@ export const stringIdArraySchema = object({ .noUnknown(true) .strict(); -const contentInputSchema = string().max(Number(env.PUBLIC_MESSAGE_LENGTH_LIMIT)).required(); -const contentInputSchemaNoLength = string().required(); +const contentInputSchema = string().max(Number(env.PUBLIC_MESSAGE_LENGTH_LIMIT)); +const contentInputSchemaNoLength = string(); export const messageInputSchema: ObjectSchema = object({ thread_id: string().required(), diff --git a/src/leapfrogai_ui/src/lib/types/messages.d.ts b/src/leapfrogai_ui/src/lib/types/messages.d.ts index 96108edd6..47693ea4a 100644 --- a/src/leapfrogai_ui/src/lib/types/messages.d.ts +++ b/src/leapfrogai_ui/src/lib/types/messages.d.ts @@ -8,7 +8,7 @@ import type { ChatRequestOptions, CreateMessage } from 'ai'; export type NewMessageInput = { thread_id: string; - content: string; + content?: string; role: 'user' | 'assistant'; assistantId?: string; metadata?: unknown; diff --git a/src/leapfrogai_ui/src/routes/api/messages/new/+server.ts b/src/leapfrogai_ui/src/routes/api/messages/new/+server.ts index f12a16123..01ec01a72 100644 --- a/src/leapfrogai_ui/src/routes/api/messages/new/+server.ts +++ b/src/leapfrogai_ui/src/routes/api/messages/new/+server.ts @@ -16,7 +16,6 @@ export const POST: RequestHandler = async ({ request, locals: { session } }) => try { requestData = (await request.json()) as NewMessageInput; if (requestData.lengthOverride) { - // TODO await messageInputSchema.validate(requestData, { abortEarly: false }); } else { await messageInputSchema.validate(requestData, { abortEarly: false }); diff --git a/src/leapfrogai_ui/src/routes/api/threads/[thread_id]/messages/[message_id]/+server.ts b/src/leapfrogai_ui/src/routes/api/threads/[thread_id]/messages/[message_id]/+server.ts new file mode 100644 index 000000000..70f0bd9fc --- /dev/null +++ b/src/leapfrogai_ui/src/routes/api/threads/[thread_id]/messages/[message_id]/+server.ts @@ -0,0 +1,23 @@ +import type { RequestHandler } from './$types'; +import { error } from '@sveltejs/kit'; +import { handleError } from '$helpers/apiHelpers'; +import { getOpenAiClient } from '$lib/server/constants'; + +export const POST: RequestHandler = async ({ params, locals: { session } }) => { + if (!session) { + error(401, 'Unauthorized'); + } + + if (!params?.thread_id || !params?.message_id) { + error(400, 'Bad Request'); + } + + try { + const openai = getOpenAiClient(session.access_token); + const message = await openai.beta.threads.messages.update(params.thread_id, params.message_id, { + + }); + } catch (e) { + return handleError(e); + } +}; From 26dac1c32689dcb16ec4a90a78943a3c4d5833ab Mon Sep 17 00:00:00 2001 From: Andrew Risse Date: Thu, 19 Sep 2024 12:56:49 -0600 Subject: [PATCH 15/18] use placeholder message --- .../src/lib/components/FileChatActions.svelte | 42 +++++++------------ src/leapfrogai_ui/src/lib/stores/threads.ts | 41 ++++++++++++++++++ 2 files changed, 56 insertions(+), 27 deletions(-) diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte index dbfabe86b..a51dfdc29 100644 --- a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte @@ -1,5 +1,6 @@