diff --git a/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte b/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte new file mode 100644 index 000000000..e01575ce2 --- /dev/null +++ b/src/leapfrogai_ui/src/lib/components/ChatFileUpload.svelte @@ -0,0 +1,153 @@ + + + { + if (e.detail.length > MAX_NUM_FILES_UPLOAD) { + e.detail.pop(); + toastStore.addToast(MAX_NUM_FILES_UPLOAD_MSG_TOAST()); + return; + } + uploadingFiles = true; + for (const file of e.detail) { + // Metadata is limited to 512 characters, we use a short id to save space + const id = uuidv4().substring(0, 8); + file.id = id; + attachedFileMetadata = [ + ...attachedFileMetadata, + { + id, + name: file.name, + type: file.type, + status: file.type.startsWith('audio/') ? 'complete' : 'uploading' + } + ]; + } + + attachedFiles = [...attachedFiles, ...e.detail]; + convertFiles(e.detail); + fileUploadBtnRef.value = ''; + }} + accept={ACCEPTED_FILE_TYPES} + disabled={uploadingFiles} + class="remove-btn-style flex rounded-lg p-1.5 text-gray-500 hover:bg-inherit dark:hover:bg-inherit" +> + + Attach file + diff --git a/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte b/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte deleted file mode 100644 index 894d97eaf..000000000 --- a/src/leapfrogai_ui/src/lib/components/ChatFileUploadForm.svelte +++ /dev/null @@ -1,85 +0,0 @@ - - -
- { - if (e.detail.length > MAX_NUM_FILES_UPLOAD) { - toastStore.addToast(MAX_NUM_FILES_UPLOAD_MSG_TOAST()); - return; - } - uploadingFiles = true; - // Metadata is limited to 512 characters, we use a short id to save space - for (const file of e.detail) { - attachedFileMetadata = [ - ...attachedFileMetadata, - { id: uuidv4().substring(0, 8), name: file.name, type: file.type, status: 'uploading' } - ]; - } - - submit(e.detail); - }} - accept={ACCEPTED_FILE_TYPES} - disabled={uploadingFiles} - class="remove-btn-style flex rounded-lg p-1.5 text-gray-500 hover:bg-inherit dark:hover:bg-inherit" - > - - Attach file - -
diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte new file mode 100644 index 000000000..f162407b6 --- /dev/null +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.svelte @@ -0,0 +1,164 @@ + + +
0 + ? 'ml-6 flex max-w-full gap-2 overflow-x-auto bg-gray-700' + : 'hidden'} +> + {#each audioFiles as file} +
+ +
+ {/each} +
diff --git a/src/leapfrogai_ui/src/lib/components/FileChatActions.test.ts b/src/leapfrogai_ui/src/lib/components/FileChatActions.test.ts new file mode 100644 index 000000000..03914e7a4 --- /dev/null +++ b/src/leapfrogai_ui/src/lib/components/FileChatActions.test.ts @@ -0,0 +1,139 @@ +import FileChatActions from '$components/FileChatActions.svelte'; +import { render, screen } from '@testing-library/svelte'; +import type { FileMetadata, LFFile } from '$lib/types/files'; +import userEvent from '@testing-library/user-event'; +import { mockConvertFileNoId } from '$lib/mocks/file-mocks'; +import { + mockNewMessage, + mockTranslation, + mockTranslationError, + mockTranslationFileSizeError +} from '$lib/mocks/chat-mocks'; +import { vi } from 'vitest'; +import { threadsStore, toastStore } from '$stores'; +import { AUDIO_FILE_SIZE_ERROR_TOAST, FILE_TRANSLATION_ERROR } from '$constants/toastMessages'; +import { getFakeThread } from '$testUtils/fakeData'; +import { NO_SELECTED_ASSISTANT_ID } from '$constants'; + +const thread = getFakeThread(); + +const mockFile1: LFFile = new File([], 'test1.mpeg', { type: 'audio/mpeg' }); +const mockFile2: LFFile = new File([], 'test1.mp4', { type: 'audio/mp4' }); + +mockFile1.id = '1'; +mockFile2.id = '2'; + +const mockMetadata1: FileMetadata = { + id: mockFile1.id, + name: mockFile1.name, + type: 'audio/mpeg', + status: 'complete', + text: '' +}; + +const mockMetadata2: FileMetadata = { + id: mockFile2.id, + name: mockFile2.name, + type: 'audio/mp4', + status: 'complete', + text: '' +}; + +describe('FileChatActions', () => { + beforeEach(() => { + threadsStore.set({ + threads: [thread], // uses date override starting in March + sendingBlocked: false, + selectedAssistantId: NO_SELECTED_ASSISTANT_ID, + lastVisitedThreadId: '', + streamingMessage: null + }); + }); + + it('should render a translate button for each audio file', () => { + render(FileChatActions, { + attachedFiles: [mockFile1, mockFile2], + attachedFileMetadata: [mockMetadata1, mockMetadata2], + threadId: thread.id, + originalMessages: [], + setMessages: vi.fn() + }); + + expect(screen.getByText(`Translate ${mockMetadata1.name}`)); + expect(screen.getByText(`Translate ${mockMetadata2.name}`)); + }); + + // Tests that correct endpoints are called when clicked + // This is testing implementation rather than behavior, but is the best we can do for this component without + // going up a level for a complicated integration test (behavior is tested in e2e) + it('creates a message and requests a translation for the user requesting translation', async () => { + const fetchSpy = vi.spyOn(global, 'fetch'); + + mockConvertFileNoId(''); + mockNewMessage(); + mockTranslation(); + mockNewMessage(); + render(FileChatActions, { + attachedFiles: [mockFile1, mockFile2], + attachedFileMetadata: [mockMetadata1, mockMetadata2], + threadId: thread.id, + originalMessages: [], + setMessages: vi.fn() + }); + + await userEvent.click(screen.getByRole('button', { name: `Translate ${mockMetadata2.name}` })); + expect(fetchSpy).toHaveBeenNthCalledWith( + 1, + expect.stringContaining('/api/messages/new'), + expect.any(Object) + ); + expect(fetchSpy).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('/api/audio/translation'), + expect.any(Object) + ); + expect(fetchSpy).toHaveBeenNthCalledWith( + 3, + expect.stringContaining('/api/messages/new'), + expect.any(Object) + ); + }); + + it('dispatches a toast if there is an error translating a file', async () => { + mockConvertFileNoId(''); + mockNewMessage(); + mockTranslationError(); + + const toastSpy = vi.spyOn(toastStore, 'addToast'); + + render(FileChatActions, { + attachedFiles: [mockFile1, mockFile2], + attachedFileMetadata: [mockMetadata1, mockMetadata2], + threadId: thread.id, + originalMessages: [], + setMessages: vi.fn() + }); + + await userEvent.click(screen.getByRole('button', { name: `Translate ${mockMetadata2.name}` })); + expect(toastSpy).toHaveBeenCalledWith(FILE_TRANSLATION_ERROR()); + }); + + it('dispatches a toast if the file is too big', async () => { + mockConvertFileNoId(''); + mockNewMessage(); + mockTranslationFileSizeError(); + + const toastSpy = vi.spyOn(toastStore, 'addToast'); + + render(FileChatActions, { + attachedFiles: [mockFile1, mockFile2], + attachedFileMetadata: [mockMetadata1, mockMetadata2], + threadId: thread.id, + originalMessages: [], + setMessages: vi.fn() + }); + + await userEvent.click(screen.getByRole('button', { name: `Translate ${mockMetadata2.name}` })); + expect(toastSpy).toHaveBeenCalledWith(AUDIO_FILE_SIZE_ERROR_TOAST()); + }); +}); diff --git a/src/leapfrogai_ui/src/lib/components/LFSidebarDropdownItem.svelte b/src/leapfrogai_ui/src/lib/components/LFSidebarDropdownItem.svelte index 9282826d0..7f0206739 100644 --- a/src/leapfrogai_ui/src/lib/components/LFSidebarDropdownItem.svelte +++ b/src/leapfrogai_ui/src/lib/components/LFSidebarDropdownItem.svelte @@ -116,7 +116,7 @@ It adds a "three-dot" menu button with Popover, and delete confirmation Modal $$props.class )} > -

+

{label}

diff --git a/src/leapfrogai_ui/src/lib/components/Message.svelte b/src/leapfrogai_ui/src/lib/components/Message.svelte index 5496a2b0f..0af165b1f 100644 --- a/src/leapfrogai_ui/src/lib/components/Message.svelte +++ b/src/leapfrogai_ui/src/lib/components/Message.svelte @@ -197,7 +197,7 @@ {/if}
- {#if message.role === 'user' && !editMode} + {#if message.role === 'user' && !editMode && !message.metadata?.wasTranscriptionOrTranslation} (editMode = true)} @@ -219,7 +219,7 @@ {/if} - {#if message.role !== 'user' && isLastMessage && !$threadsStore.sendingBlocked} + {#if message.role !== 'user' && isLastMessage && !$threadsStore.sendingBlocked && !message.metadata?.wasTranscriptionOrTranslation} import { fade } from 'svelte/transition'; - import { CloseOutline, FileOutline } from 'flowbite-svelte-icons'; + import { CloseOutline, FileMusicOutline, FileOutline } from 'flowbite-svelte-icons'; import { getFileType } from '$lib/utils/files.js'; import { Card, Spinner, ToolbarButton } from 'flowbite-svelte'; import { createEventDispatcher } from 'svelte'; @@ -20,11 +20,11 @@ data-testid={`${fileMetadata.name}-file-uploaded-card`} horizontal padding="xs" - class="w-80 min-w-72" + class="w-80 min-w-72 border-none" on:mouseenter={() => (hovered = true)} on:mouseleave={() => (hovered = false)} > -
+
{#if fileMetadata.status === 'uploading'} @@ -33,16 +33,20 @@ + {:else if fileMetadata.type.startsWith('audio/')} + {:else} {/if}
-
+
{fileMetadata.name}
diff --git a/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte b/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte index 1c6fef92b..85ddfe7b3 100644 --- a/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte +++ b/src/leapfrogai_ui/src/lib/components/UploadedFileCards.svelte @@ -1,18 +1,20 @@
0 - ? 'ml-6 flex max-w-full gap-2 overflow-x-auto bg-gray-700 ' + ? 'ml-6 flex max-w-full gap-2 overflow-x-auto bg-gray-700' : 'hidden'} > {#each attachedFileMetadata as fileMetadata} diff --git a/src/leapfrogai_ui/src/lib/constants/index.ts b/src/leapfrogai_ui/src/lib/constants/index.ts index 5379d858a..3736f4714 100644 --- a/src/leapfrogai_ui/src/lib/constants/index.ts +++ b/src/leapfrogai_ui/src/lib/constants/index.ts @@ -4,8 +4,9 @@ export const MAX_LABEL_SIZE = 100; export const DEFAULT_ASSISTANT_TEMP = 0.2; export const MAX_AVATAR_SIZE = 5000000; export const MAX_FILE_SIZE = 512000000; +export const MAX_AUDIO_FILE_SIZE = 25000000; export const MAX_FILE_NAME_SIZE = 27; -export const MAX_NUM_FILES_UPLOAD = 50; // for chat completion +export const MAX_NUM_FILES_UPLOAD = 10; // for chat completion // PER OPENAI SPEC export const ASSISTANTS_NAME_MAX_LENGTH = 256; @@ -37,6 +38,19 @@ export const assistantDefaults: Omit = { temperature: 0.2, response_format: 'auto' }; + +export const ACCEPTED_AUDIO_FILE_TYPES = [ + '.flac', + '.mp3', + '.mp4', + '.mpeg', + '.mpga', + '.m4a', + '.ogg', + '.wav', + '.webm' +]; + export const ACCEPTED_FILE_TYPES = [ '.pdf', '.txt', @@ -47,8 +61,21 @@ export const ACCEPTED_FILE_TYPES = [ '.pptx', '.doc', '.docx', - '.csv' + '.csv', + ...ACCEPTED_AUDIO_FILE_TYPES ]; + +export const ACCEPTED_AUDIO_FILE_MIME_TYPES = [ + 'audio/flac', + 'audio/mpeg', + 'audio/mp4', + 'audio/x-m4a', + 'audio/mpeg', + 'audio/ogg', + 'audio/wav', + 'audio/webm' +]; + export const ACCEPTED_MIME_TYPES = [ 'application/pdf', // .pdf 'text/plain', // .txt, .text @@ -58,7 +85,8 @@ export const ACCEPTED_MIME_TYPES = [ 'application/vnd.openxmlformats-officedocument.presentationml.presentation', // .pptx 'application/msword', // .doc 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', //.docx, - 'text/csv' + 'text/csv', + ...ACCEPTED_AUDIO_FILE_MIME_TYPES ]; export const FILE_TYPE_MAP = { @@ -71,13 +99,16 @@ export const FILE_TYPE_MAP = { 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'PPTX', 'application/msword': 'DOC', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'DOCX', - 'text/csv': 'CSV' + 'text/csv': 'CSV', + ...ACCEPTED_AUDIO_FILE_MIME_TYPES.reduce((acc, type) => ({ ...acc, [type]: 'AUDIO' }), {}) }; export const NO_FILE_ERROR_TEXT = 'Please upload an image or select a pictogram'; export const AVATAR_FILE_SIZE_ERROR_TEXT = `File must be less than ${MAX_AVATAR_SIZE / 1000000} MB`; export const FILE_SIZE_ERROR_TEXT = `File must be less than ${MAX_FILE_SIZE / 1000000} MB`; +export const AUDIO_FILE_SIZE_ERROR_TEXT = `Audio file must be less than ${MAX_AUDIO_FILE_SIZE / 1000000} MB`; export const INVALID_FILE_TYPE_ERROR_TEXT = `Invalid file type, accepted types are: ${ACCEPTED_FILE_TYPES.join(', ')}`; +export const INVALID_AUDIO_FILE_TYPE_ERROR_TEXT = `Invalid file type, accepted types are: ${ACCEPTED_AUDIO_FILE_TYPES.join(', ')}`; export const NO_SELECTED_ASSISTANT_ID = 'noSelectedAssistantId'; export const FILE_UPLOAD_PROMPT = "The following are the user's files: "; diff --git a/src/leapfrogai_ui/src/lib/constants/toastMessages.ts b/src/leapfrogai_ui/src/lib/constants/toastMessages.ts index a04760c2d..5a356ab06 100644 --- a/src/leapfrogai_ui/src/lib/constants/toastMessages.ts +++ b/src/leapfrogai_ui/src/lib/constants/toastMessages.ts @@ -1,10 +1,4 @@ -import { MAX_NUM_FILES_UPLOAD } from '$constants/index'; - -type ToastData = { - kind: ToastKind; - title: string; - subtitle?: string; -}; +import { AUDIO_FILE_SIZE_ERROR_TEXT, MAX_NUM_FILES_UPLOAD } from '$constants/index'; export const ERROR_SAVING_MSG_TOAST = (override: Partial = {}): ToastData => ({ kind: 'error', @@ -71,3 +65,17 @@ export const FILE_VECTOR_TIMEOUT_MSG_TOAST = (override: Partial = {}) subtitle: 'There was an error processing assistant files', ...override }); + +export const FILE_TRANSLATION_ERROR = (override: Partial = {}): ToastData => ({ + kind: 'error', + title: 'Translation Error', + subtitle: 'Error translating audio file', + ...override +}); + +export const AUDIO_FILE_SIZE_ERROR_TOAST = (override: Partial = {}): ToastData => ({ + kind: 'error', + title: 'File Too Large', + subtitle: AUDIO_FILE_SIZE_ERROR_TEXT, + ...override +}); diff --git a/src/leapfrogai_ui/src/lib/helpers/apiHelpers.ts b/src/leapfrogai_ui/src/lib/helpers/apiHelpers.ts new file mode 100644 index 000000000..923098af0 --- /dev/null +++ b/src/leapfrogai_ui/src/lib/helpers/apiHelpers.ts @@ -0,0 +1,35 @@ +import { error } from '@sveltejs/kit'; + +/* + * A generic error handler to log and structure error responses + * Try/catch catch blocks can pass their error to this function + * ex. + * catch (e) { + handleError(e, { id: file.id }); + } + */ +export const handleError = (e: unknown, additionalErrorInfo?: object) => { + console.error('Caught Error:', e); + + let status = 500; + let message = 'Internal Server Error'; + + if (e instanceof Error) { + message = e.message; + } else if (typeof e === 'object' && e !== null && 'status' in e) { + status = (e as { status: number }).status || 500; + message = + (e as unknown as { body: { message: string } }).body.message || 'Internal Server Error'; + } + error(status, { message, ...additionalErrorInfo }); +}; + +// In the test environment, formData.get('file') does not return a file of type File, so we mock it differently +// with this helper +export const requestWithFormData = (mockFile: unknown) => { + return { + formData: vi.fn().mockResolvedValue({ + get: vi.fn().mockReturnValue(mockFile) + }) + } as unknown as Request; +}; diff --git a/src/leapfrogai_ui/src/lib/helpers/fileHelpers.test.ts b/src/leapfrogai_ui/src/lib/helpers/fileHelpers.test.ts new file mode 100644 index 000000000..73f63f452 --- /dev/null +++ b/src/leapfrogai_ui/src/lib/helpers/fileHelpers.test.ts @@ -0,0 +1,146 @@ +import { faker } from '@faker-js/faker'; +import type { FileMetadata } from '$lib/types/files'; +import { removeFilesUntilUnderLimit, updateFileMetadata } from '$helpers/fileHelpers'; +import { FILE_CONTEXT_TOO_LARGE_ERROR_MSG } from '$constants/errors'; +import { getMockFileMetadata } from '$testUtils/fakeData'; + +describe('removeFilesUntilUnderLimit', () => { + test('removeFilesUntilUnderLimit should remove the largest file until total size is under the max limit', () => { + // Metadata stringified without text is 95 characters, so the text added below will increase the size from that + // baseline + const text1 = faker.word.words(50); + const text2 = faker.word.words(150); + const text3 = faker.word.words(200); + const files = [ + getMockFileMetadata({ text: text1 }), + getMockFileMetadata({ text: text2 }), + getMockFileMetadata({ text: text3 }) + ]; + + // Files 2 and 3 will have different length once their text is removed and they are set to error status + const file2WhenError = JSON.stringify({ + ...files[1], + text: '', + status: 'error', + errorText: FILE_CONTEXT_TOO_LARGE_ERROR_MSG + }); + const file3WhenError = JSON.stringify({ + ...files[2], + text: '', + status: 'error', + errorText: FILE_CONTEXT_TOO_LARGE_ERROR_MSG + }); + + const totalSize = files.reduce((total, file) => total + JSON.stringify(file).length, 0); + + // Expected length when it removes the text of the last two files, but replaces them with error status + const maxLimit = + totalSize - + JSON.stringify(files[1]).length - + JSON.stringify(files[2]).length + + file2WhenError.length + + file3WhenError.length; + + removeFilesUntilUnderLimit(files, maxLimit); + + // file 1 remains + expect(files[0].text).toBe(text1); + + // file 2 and 3 are removed and set to error status + expect(files[1].text).toBe(''); + expect(files[1].status).toBe('error'); + expect(files[1].errorText).toBe(FILE_CONTEXT_TOO_LARGE_ERROR_MSG); + expect(files[2].text).toBe(''); + expect(files[2].status).toBe('error'); + expect(files[2].errorText).toBe(FILE_CONTEXT_TOO_LARGE_ERROR_MSG); + + // Also check that the total size is under the limit + const totalSizeRecalculated = files.reduce( + (total, file) => total + JSON.stringify(file).length, + 0 + ); + + expect(totalSizeRecalculated).toBeLessThanOrEqual(maxLimit); + }); + + it('should not modify files if total size is already under the max limit', () => { + const text1 = faker.word.words(10); + const text2 = faker.word.words(20); + + const files = [getMockFileMetadata({ text: text1 }), getMockFileMetadata({ text: text2 })]; + + // Assume a limit of 50 characters + const maxLimit = 10000000000000; + + // Call the function to test + removeFilesUntilUnderLimit(files, maxLimit); + + // Expect no modifications to the files + expect(files[0].text).toBe(text1); + expect(files[1].text).toBe(text2); + }); + + it('can remove all files text and avoiding hanging (breaks out of while loop)', () => { + const text1 = faker.word.words(10); + const text2 = faker.word.words(20); + + const files = [getMockFileMetadata({ text: text1 }), getMockFileMetadata({ text: text2 })]; + + // Assume a limit of 50 characters + const maxLimit = 5; + + // Call the function to test + removeFilesUntilUnderLimit(files, maxLimit); + + // Expect no modifications to the files + expect(files[0].text).toBe(''); + expect(files[1].text).toBe(''); + expect(files[0].status).toBe('error'); + expect(files[1].status).toBe('error'); + }); +}); + +describe('updateFileMetadata with order preservation', () => { + it('should update existing files, add new files, and preserve the original order of old metadata', () => { + const file1 = getMockFileMetadata({ status: 'complete' }); + const file2 = getMockFileMetadata({ status: 'uploading' }); + const file3 = getMockFileMetadata({ status: 'complete' }); + const file4 = getMockFileMetadata({ status: 'complete' }); + + // 3 files + const oldMetadata: FileMetadata[] = [file1, file2, file3]; + + // updated file, and new file + const newMetadata: FileMetadata[] = [file1, { ...file2, status: 'complete' }, file3, file4]; + + const result = updateFileMetadata(oldMetadata, newMetadata); + + expect(result).toHaveLength(4); + + // Check if the order is preserved and one file was updated and new one added + + expect(result[0]).toEqual(file1); + expect(result[1]).toEqual({ + ...file2, + status: 'complete' + }); + + expect(result[2]).toEqual(file3); + expect(result[3]).toEqual(file4); + }); + + it('should return new metadata if old metadata is empty', () => { + const oldMetadata: FileMetadata[] = []; + const newMetadata: FileMetadata[] = [getMockFileMetadata()]; + const result = updateFileMetadata(oldMetadata, newMetadata); + expect(result).toHaveLength(1); + expect(result).toEqual(newMetadata); + }); + + it('should keep old metadata if no new metadata is provided', () => { + const oldMetadata: FileMetadata[] = [getMockFileMetadata()]; + const newMetadata: FileMetadata[] = []; + const result = updateFileMetadata(oldMetadata, newMetadata); + expect(result).toEqual(oldMetadata); + }); +}); diff --git a/src/leapfrogai_ui/src/lib/helpers/fileHelpers.ts b/src/leapfrogai_ui/src/lib/helpers/fileHelpers.ts index bc459624f..a0cd0fc5b 100644 --- a/src/leapfrogai_ui/src/lib/helpers/fileHelpers.ts +++ b/src/leapfrogai_ui/src/lib/helpers/fileHelpers.ts @@ -1,5 +1,6 @@ -import type { FileRow } from '$lib/types/files'; +import type { FileMetadata, FileRow } from '$lib/types/files'; import type { FileObject } from 'openai/resources/files'; +import { FILE_CONTEXT_TOO_LARGE_ERROR_MSG } from '$constants/errors'; export const convertFileObjectToFileRows = (files: FileObject[]): FileRow[] => files.map((file) => ({ @@ -8,3 +9,54 @@ export const convertFileObjectToFileRows = (files: FileObject[]): FileRow[] => created_at: file.created_at * 1000, status: 'hide' })); + +export const removeFilesUntilUnderLimit = (parsedFiles: FileMetadata[], max: number) => { + const numFiles = parsedFiles.length; + let numFilesReset = 0; + let totalTextLength = parsedFiles.reduce((total, file) => total + JSON.stringify(file).length, 0); + // Remove the largest files until the total size is within the allowed limit + while (totalTextLength > max) { + if (numFilesReset === numFiles) break; + let largestIndex = 0; + for (let i = 1; i < numFiles; i++) { + const item = JSON.stringify(parsedFiles[i]); + const largestItem = JSON.stringify(parsedFiles[largestIndex]); + if (item.length > largestItem.length) { + largestIndex = i; + } + } + + // remove the text and set to error status + parsedFiles[largestIndex] = { + ...parsedFiles[largestIndex], + text: '', + status: 'error', + errorText: FILE_CONTEXT_TOO_LARGE_ERROR_MSG + }; + numFilesReset += 1; + totalTextLength = parsedFiles.reduce((total, file) => total + JSON.stringify(file).length, 0); //recalculate total size + } +}; + +// Combines old and new file metadata, updating the old metadata with new metadata +export const updateFileMetadata = ( + oldMetadata: FileMetadata[], + newMetadata: FileMetadata[] +): FileMetadata[] => { + // Create a map of new metadata + const newMetadataMap = new Map(newMetadata.map((file) => [file.id, file])); + + // Update and keep the original order from old metadata + const updatedMetadata = oldMetadata.map((oldFile) => { + const newFile = newMetadataMap.get(oldFile.id); + return newFile ? { ...oldFile, ...newFile } : oldFile; + }); + + // Filter out new files that aren't already in the old metadata + const newFilesToAdd = newMetadata.filter( + (newFile) => !oldMetadata.some((oldFile) => oldFile.id === newFile.id) + ); + + // Append new files at the end while keeping the original order of oldMetadata + return [...updatedMetadata, ...newFilesToAdd]; +}; diff --git a/src/leapfrogai_ui/src/lib/mocks/chat-mocks.ts b/src/leapfrogai_ui/src/lib/mocks/chat-mocks.ts index 714f56dc9..e820f32d4 100644 --- a/src/leapfrogai_ui/src/lib/mocks/chat-mocks.ts +++ b/src/leapfrogai_ui/src/lib/mocks/chat-mocks.ts @@ -5,6 +5,7 @@ import type { LFMessage, NewMessageInput } from '$lib/types/messages'; import type { LFAssistant } from '$lib/types/assistants'; import { createStreamDataTransformer, StreamingTextResponse } from 'ai'; import type { LFThread } from '$lib/types/threads'; +import { AUDIO_FILE_SIZE_ERROR_TEXT } from '$constants'; type MockChatCompletionOptions = { responseMsg?: string[]; @@ -98,3 +99,30 @@ export const mockGetThread = (thread: LFThread) => { }) ); }; + +export const mockTranslation = () => { + server.use( + http.post('/api/audio/translation', () => { + return HttpResponse.json({ text: 'fake translation' }); + }) + ); +}; + +export const mockTranslationError = () => { + server.use( + http.post('/api/audio/translation', () => { + return new HttpResponse(null, { status: 500 }); + }) + ); +}; + +export const mockTranslationFileSizeError = () => { + server.use( + http.post('/api/audio/translation', () => { + return HttpResponse.json( + { message: `ValidationError: ${AUDIO_FILE_SIZE_ERROR_TEXT}` }, + { status: 400 } + ); + }) + ); +}; diff --git a/src/leapfrogai_ui/src/lib/mocks/openai.ts b/src/leapfrogai_ui/src/lib/mocks/openai.ts index 25d397d72..704dfa6cd 100644 --- a/src/leapfrogai_ui/src/lib/mocks/openai.ts +++ b/src/leapfrogai_ui/src/lib/mocks/openai.ts @@ -42,6 +42,7 @@ class OpenAI { listAssistants: boolean; deleteFile: boolean; fileContent: boolean; + translation: boolean; }; constructor({ apiKey, baseURL }: { apiKey: string; baseURL: string }) { @@ -71,7 +72,8 @@ class OpenAI { updateAssistant: false, listAssistants: false, deleteFile: false, - fileContent: false + fileContent: false, + translation: false }; } @@ -117,6 +119,17 @@ class OpenAI { this.errors[key] = false; } + audio = { + translations: { + create: vi.fn().mockImplementation(() => { + if (this.errors.translation) this.throwError('translation'); + return Promise.resolve({ + text: 'Fake translation' + }); + }) + } + }; + files = { retrieve: vi.fn().mockImplementation((id) => { return Promise.resolve(this.uploadedFiles.find((file) => file.id === id)); diff --git a/src/leapfrogai_ui/src/lib/schemas/chat.ts b/src/leapfrogai_ui/src/lib/schemas/chat.ts index efa1db8f6..8835e44c9 100644 --- a/src/leapfrogai_ui/src/lib/schemas/chat.ts +++ b/src/leapfrogai_ui/src/lib/schemas/chat.ts @@ -15,8 +15,8 @@ export const stringIdArraySchema = object({ .noUnknown(true) .strict(); -const contentInputSchema = string().max(Number(env.PUBLIC_MESSAGE_LENGTH_LIMIT)).required(); -const contentInputSchemaNoLength = string().required(); +const contentInputSchema = string().max(Number(env.PUBLIC_MESSAGE_LENGTH_LIMIT)); +const contentInputSchemaNoLength = string(); export const messageInputSchema: ObjectSchema = object({ thread_id: string().required(), diff --git a/src/leapfrogai_ui/src/lib/schemas/files.ts b/src/leapfrogai_ui/src/lib/schemas/files.ts index 5c475e5e1..522f4972b 100644 --- a/src/leapfrogai_ui/src/lib/schemas/files.ts +++ b/src/leapfrogai_ui/src/lib/schemas/files.ts @@ -1,20 +1,29 @@ import { array, mixed, object, string, ValidationError } from 'yup'; import { + ACCEPTED_AUDIO_FILE_MIME_TYPES, ACCEPTED_MIME_TYPES, + AUDIO_FILE_SIZE_ERROR_TEXT, FILE_SIZE_ERROR_TEXT, + INVALID_AUDIO_FILE_TYPE_ERROR_TEXT, INVALID_FILE_TYPE_ERROR_TEXT, + MAX_AUDIO_FILE_SIZE, MAX_FILE_SIZE } from '$constants'; +import type { LFFile } from '$lib/types/files'; export const filesSchema = object({ files: array().of( - mixed() + mixed() .test('fileType', 'Please upload a file.', (value) => value == null || value instanceof File) .test('fileSize', FILE_SIZE_ERROR_TEXT, (value) => { if (value == null) { return true; } - if (value.size > MAX_FILE_SIZE) { + if (ACCEPTED_AUDIO_FILE_MIME_TYPES.includes(value.type)) { + if (value.size > MAX_AUDIO_FILE_SIZE) { + return new ValidationError(AUDIO_FILE_SIZE_ERROR_TEXT); + } + } else if (value.size > MAX_FILE_SIZE) { return new ValidationError(FILE_SIZE_ERROR_TEXT); } return true; @@ -40,7 +49,7 @@ export const filesCheckSchema = object({ .strict(); export const fileSchema = object({ - file: mixed() + file: mixed() .test('fileType', 'File is required.', (value) => value == null || value instanceof File) .test('fileSize', FILE_SIZE_ERROR_TEXT, (value) => { if (value == null) { @@ -63,3 +72,28 @@ export const fileSchema = object({ }) .noUnknown(true) .strict(); + +export const audioFileSchema = object({ + file: mixed() + .test('fileType', 'File is required.', (value) => value == null || value instanceof File) + .test('fileSize', AUDIO_FILE_SIZE_ERROR_TEXT, (value) => { + if (value == null) { + return true; + } + if (value.size > MAX_AUDIO_FILE_SIZE) { + return new ValidationError(AUDIO_FILE_SIZE_ERROR_TEXT); + } + return true; + }) + .test('type', INVALID_AUDIO_FILE_TYPE_ERROR_TEXT, (value) => { + if (value == null) { + return true; + } + if (!ACCEPTED_AUDIO_FILE_MIME_TYPES.includes(value.type)) { + return new ValidationError(INVALID_AUDIO_FILE_TYPE_ERROR_TEXT); + } + return true; + }) +}) + .noUnknown(true) + .strict(); diff --git a/src/leapfrogai_ui/src/lib/stores/threads.ts b/src/leapfrogai_ui/src/lib/stores/threads.ts index 354f5e14c..0b9738fbb 100644 --- a/src/leapfrogai_ui/src/lib/stores/threads.ts +++ b/src/leapfrogai_ui/src/lib/stores/threads.ts @@ -8,6 +8,7 @@ import type { LFThread, NewThreadInput } from '$lib/types/threads'; import type { LFMessage } from '$lib/types/messages'; import { getMessageText } from '$helpers/threads'; import { saveMessage } from '$helpers/chatHelpers'; +import type { Message } from 'ai'; type ThreadsStore = { threads: LFThread[]; @@ -208,6 +209,63 @@ const createThreadsStore = () => { }); } }, + // The following temp empty message methods are for showing a message skeleton in the messages window + // while waiting for a full response (e.g. waiting for translation response) + addTempEmptyMessage: (threadId: string, tempId: string) => { + update((old) => { + const updatedThreads = [...old.threads]; + const threadIndex = old.threads.findIndex((c) => c.id === threadId); + const oldThread = old.threads[threadIndex]; + updatedThreads[threadIndex] = { + ...oldThread, + messages: [ + ...(oldThread.messages || []), + { id: tempId, role: 'assistant', content: '', createdAt: new Date() } + ] + }; + return { + ...old, + threads: updatedThreads + }; + }); + }, + replaceTempMessageWithActual: (threadId: string, tempId: string, newMessage: LFMessage) => { + update((old) => { + const updatedThreads = [...old.threads]; + const threadIndex = old.threads.findIndex((c) => c.id === threadId); + const tempMessageIndex = old.threads[threadIndex].messages?.findIndex( + (m) => m.id === tempId + ); + if (!tempMessageIndex || tempMessageIndex === -1) return { ...old }; + const oldThread = old.threads[threadIndex]; + const messagesCopy = [...(oldThread.messages || [])]; + messagesCopy[tempMessageIndex] = newMessage; + updatedThreads[threadIndex] = { + ...oldThread, + messages: messagesCopy + }; + return { + ...old, + threads: updatedThreads + }; + }); + }, + removeMessageFromStore: (threadId: string, messageId: string) => { + update((old) => { + const updatedThreads = [...old.threads]; + const threadIndex = old.threads.findIndex((c) => c.id === threadId); + updatedThreads[threadIndex].messages = + old.threads[threadIndex].messages?.filter((m) => m.id !== messageId) || []; + return { ...old, thread: updatedThreads }; + }); + }, + updateMessagesState: ( + originalMessages: Message[], + setMessages: (messages: Message[]) => void, + newMessage: LFMessage + ) => { + setMessages([...originalMessages, { ...newMessage, content: getMessageText(newMessage) }]); + }, deleteThread: async (id: string) => { try { await deleteThread(id); diff --git a/src/leapfrogai_ui/src/lib/types/files.d.ts b/src/leapfrogai_ui/src/lib/types/files.d.ts index 37c8db5f0..599260041 100644 --- a/src/leapfrogai_ui/src/lib/types/files.d.ts +++ b/src/leapfrogai_ui/src/lib/types/files.d.ts @@ -27,3 +27,7 @@ export type FileMetadata = { text: string; errorText?: string; }; + +export type LFFile = File & { + id?: string; +}; diff --git a/src/leapfrogai_ui/src/lib/types/messages.d.ts b/src/leapfrogai_ui/src/lib/types/messages.d.ts index 96108edd6..47693ea4a 100644 --- a/src/leapfrogai_ui/src/lib/types/messages.d.ts +++ b/src/leapfrogai_ui/src/lib/types/messages.d.ts @@ -8,7 +8,7 @@ import type { ChatRequestOptions, CreateMessage } from 'ai'; export type NewMessageInput = { thread_id: string; - content: string; + content?: string; role: 'user' | 'assistant'; assistantId?: string; metadata?: unknown; diff --git a/src/leapfrogai_ui/src/lib/types/toast.d.ts b/src/leapfrogai_ui/src/lib/types/toast.d.ts index 924d59ae5..2d22d60b2 100644 --- a/src/leapfrogai_ui/src/lib/types/toast.d.ts +++ b/src/leapfrogai_ui/src/lib/types/toast.d.ts @@ -14,3 +14,9 @@ type ToastNotificationProps = { type ToastStore = { toasts: ToastNotificationProps[]; }; + +type ToastData = { + kind: ToastKind; + title: string; + subtitle?: string; +}; diff --git a/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts b/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts new file mode 100644 index 000000000..175b410e5 --- /dev/null +++ b/src/leapfrogai_ui/src/routes/api/audio/translation/+server.ts @@ -0,0 +1,36 @@ +import { error, json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getOpenAiClient } from '$lib/server/constants'; +import { audioFileSchema } from '$schemas/files'; +import { env } from '$env/dynamic/private'; +import { handleError } from '$helpers/apiHelpers'; + +export const POST: RequestHandler = async ({ request, locals: { session } }) => { + if (!session) { + error(401, 'Unauthorized'); + } + + let file: File | null; + + // Validate request body + try { + const formData = await request.formData(); + file = formData.get('file') as File; + + await audioFileSchema.validate({ file }, { abortEarly: false }); + } catch (e) { + console.error(e); + error(400, { message: `${e}` }); + } + + try { + const openai = getOpenAiClient(session.access_token); + const translation = await openai.audio.translations.create({ + file: file, + model: env.OPENAI_API_KEY ? 'whisper-1' : 'whisper' + }); + return json({ text: translation.text }); + } catch (e) { + return handleError(e); + } +}; diff --git a/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts b/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts new file mode 100644 index 000000000..b0aa3733a --- /dev/null +++ b/src/leapfrogai_ui/src/routes/api/audio/translation/server.test.ts @@ -0,0 +1,127 @@ +import { getLocalsMock } from '$lib/mocks/misc'; +import type { RequestEvent } from '@sveltejs/kit'; +import type { RouteParams } from './$types'; +import { POST } from './+server'; +import { mockOpenAI } from '../../../../../vitest-setup'; +import { requestWithFormData } from '$helpers/apiHelpers'; +import * as constants from '$constants'; + +// Allows mocking important constants and only overriding values for specific tests +vi.mock('$constants', async () => { + const actualConstants = await vi.importActual('$constants'); + return { + ...actualConstants + }; +}); + +describe('/api/audio/translation', () => { + it('returns a 401 when there is no session', async () => { + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST' + }); + + await expect( + POST({ + request, + params: {}, + locals: getLocalsMock({ nullSession: true }) + } as RequestEvent) + ).rejects.toMatchObject({ + status: 401 + }); + }); + + it('should return 400 if the form data is missing', async () => { + const request = new Request('http://thisurlhasnoeffect', { method: 'POST' }); + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ status: 400 }); + }); + + it('should return 400 if the file is missing from the form data', async () => { + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST', + body: new FormData() + }); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ + status: 400 + }); + }); + + it('should return 400 if the file in the form data is not of type File', async () => { + const formData = new FormData(); + formData.append('file', '123'); + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST', + body: formData + }); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ + status: 400 + }); + }); + + it('should return 400 if the file is too big', async () => { + // @ts-expect-error - intentionally overriding a constant for testing + vi.spyOn(constants, 'MAX_AUDIO_FILE_SIZE', 'get').mockReturnValueOnce(1); + + const fileContent = new Blob(['dummy content'], { type: 'audio/mp4' }); + const testFile = new File([fileContent], 'test.txt', { type: 'audio/mp4' }); + const request = requestWithFormData(testFile); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ + status: 400 + }); + // Reset the mock after this test + vi.resetModules(); + }); + + it('should return a 500 if there is an error translating the file', async () => { + mockOpenAI.setError('translation'); + + const fileContent = new Blob(['dummy content'], { type: 'audio/mp4' }); + const testFile = new File([fileContent], 'test.txt', { type: 'audio/mp4' }); + const request = requestWithFormData(testFile); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >) + ).rejects.toMatchObject({ + status: 500 + }); + }); + + it('should return translated text', async () => { + const fileContent = new Blob(['dummy content'], { type: 'audio/mp4' }); + const testFile = new File([fileContent], 'test.txt', { type: 'audio/mp4' }); + const request = requestWithFormData(testFile); + + const res = await POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/audio/translation' + >); + const data = await res.json(); + expect(data).toEqual({ text: 'Fake translation' }); + }); +}); diff --git a/src/leapfrogai_ui/src/routes/api/files/convert/server.test.ts b/src/leapfrogai_ui/src/routes/api/files/convert/server.test.ts index 9a18626a6..3081925f6 100644 --- a/src/leapfrogai_ui/src/routes/api/files/convert/server.test.ts +++ b/src/leapfrogai_ui/src/routes/api/files/convert/server.test.ts @@ -3,6 +3,7 @@ import { getLocalsMock } from '$lib/mocks/misc'; import type { RequestEvent } from '@sveltejs/kit'; import type { RouteParams } from './$types'; import { afterAll } from 'vitest'; +import { requestWithFormData } from '$helpers/apiHelpers'; // Allows swapping out the mock per test const mocks = vi.hoisted(() => { @@ -93,14 +94,7 @@ describe('/api/files/convert', () => { const fileContent = new Blob(['dummy content'], { type: 'text/plain' }); const testFile = new File([fileContent], 'test.txt', { type: 'text/plain' }); - - // In the test environment, formData.get('file') does not return a file of type File, so we mock it differently - // here - const request = { - formData: vi.fn().mockResolvedValue({ - get: vi.fn().mockReturnValue(testFile) - }) - } as unknown as Request; + const request = requestWithFormData(testFile); await expect( POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< @@ -125,13 +119,7 @@ describe('/api/files/convert', () => { const testFile = new File([fileContent], 'test.txt', { type: 'text/plain' }); testFile.arrayBuffer = async () => fileContent.buffer; - // In the test environment, formData.get('file') does not return a file of type File, so we mock it differently - // here - const request = { - formData: vi.fn().mockResolvedValue({ - get: vi.fn().mockReturnValue(testFile) - }) - } as unknown as Request; + const request = requestWithFormData(testFile); const res = await POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< RouteParams, diff --git a/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts b/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts new file mode 100644 index 000000000..73bd12a7e --- /dev/null +++ b/src/leapfrogai_ui/src/routes/api/files/parse-text/+server.ts @@ -0,0 +1,68 @@ +import type { RequestHandler } from './$types'; +import * as mupdf from 'mupdf'; +import { error, json } from '@sveltejs/kit'; +import { fileSchema } from '$schemas/files'; +import type { LFFile } from '$lib/types/files'; +import { handleError } from '$helpers/apiHelpers'; + +/** + * Parses text from a file. If the file is not a PDF, it will first convert the file to a + * PDF before parsing the text. + */ +export const POST: RequestHandler = async ({ request, fetch, locals: { session } }) => { + if (!session) { + error(401, 'Unauthorized'); + } + + let file: LFFile | null; + // Validate request body + try { + const formData = await request.formData(); + file = formData.get('file') as LFFile; + await fileSchema.validate({ file }, { abortEarly: false }); + } catch (e) { + console.error('Validation error:', e); + error(400, `Bad Request, File invalid: ${e}`); + } + + try { + let text = ''; + let buffer: ArrayBuffer; + const contentType = file.type; + if (contentType !== 'application/pdf') { + // Convert file to PDF + const formData = new FormData(); + formData.append('file', file); + const convertRes = await fetch('/api/files/convert', { + method: 'POST', + body: formData + }); + + if (!convertRes.ok) { + return error(500, { message: 'Error converting file' }); + } + + const convertedFileBlob = await convertRes.blob(); + buffer = await convertedFileBlob.arrayBuffer(); + } else buffer = await file.arrayBuffer(); + + const document = mupdf.Document.openDocument(buffer, 'application/pdf'); + let i = 0; + while (i < document.countPages()) { + const page = document.loadPage(i); + const json = page.toStructuredText('preserve-whitespace').asJSON(); + for (const block of JSON.parse(json).blocks) { + for (const line of block.lines) { + text += line.text; + } + } + i++; + } + + return json({ + text + }); + } catch (e) { + return handleError(e); + } +}; diff --git a/src/leapfrogai_ui/src/routes/api/files/parse-text/server.test.ts b/src/leapfrogai_ui/src/routes/api/files/parse-text/server.test.ts new file mode 100644 index 000000000..b23e233ea --- /dev/null +++ b/src/leapfrogai_ui/src/routes/api/files/parse-text/server.test.ts @@ -0,0 +1,153 @@ +import { POST } from './+server'; +import { getLocalsMock } from '$lib/mocks/misc'; +import type { RequestEvent } from '@sveltejs/kit'; +import type { RouteParams } from '../../../../../.svelte-kit/types/src/routes/api/files/[file_id]/$types'; +import { mockConvertFileErrorNoId, mockConvertFileNoId } from '$lib/mocks/file-mocks'; +import type { LFFile } from '$lib/types/files'; +import { requestWithFormData } from '$helpers/apiHelpers'; +import * as mupdf from 'mupdf'; + +vi.mock('mupdf', () => ({ + Document: { + openDocument: vi.fn() + } +})); + +describe('/api/files/parse-text', () => { + afterEach(() => { + vi.resetAllMocks(); + }); + + it('returns a 401 when there is no session', async () => { + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST' + }); + + await expect( + POST({ request, params: {}, locals: getLocalsMock({ nullSession: true }) } as RequestEvent< + RouteParams, + '/api/files/parse-text' + >) + ).rejects.toMatchObject({ + status: 401 + }); + }); + + it('should return 400 if the form data is missing', async () => { + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST' + }); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/files/parse-text' + >) + ).rejects.toMatchObject({ + status: 400 + }); + }); + + it('should return 400 if the file is missing from the form data', async () => { + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST', + body: new FormData() + }); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/files/parse-text' + >) + ).rejects.toMatchObject({ + status: 400 + }); + }); + + it('should return 400 if the file in the form data is not of type File', async () => { + const formData = new FormData(); + formData.append('file', '123'); + const request = new Request('http://thisurlhasnoeffect', { + method: 'POST', + body: formData + }); + + await expect( + POST({ request, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/files/parse-text' + >) + ).rejects.toMatchObject({ + status: 400 + }); + }); + + it('returns 500 if there is an error converting a file to PDF', async () => { + mockConvertFileErrorNoId(); + const mockFile1: LFFile = new File(['content1'], 'test1.txt', { type: 'text/plain' }); + mockFile1.id = '1'; + const request = requestWithFormData(mockFile1); + + await expect( + POST({ request, fetch: global.fetch, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/files/parse-text' + >) + ).rejects.toMatchObject({ + status: 500, + body: { message: 'Error converting file' } + }); + }); + + it("returns 500 if there is an error parsing a PDF's text", async () => { + mupdf.Document.openDocument.mockImplementationOnce(() => { + throw new Error('Mocked error'); + }); + + mockConvertFileNoId('this is ignored'); + const mockFile1: LFFile = new File(['content1'], 'test1.txt', { type: 'text/plain' }); + mockFile1.id = '1'; + const request = requestWithFormData(mockFile1); + + await expect( + POST({ request, fetch: global.fetch, params: {}, locals: getLocalsMock() } as RequestEvent< + RouteParams, + '/api/files/parse-text' + >) + ).rejects.toMatchObject({ + status: 500, + body: { message: 'Mocked error' } + }); + }); + + it('parses text for a PDF file', async () => { + mupdf.Document.openDocument.mockReturnValue({ + countPages: vi.fn().mockReturnValue(1), + loadPage: vi.fn().mockReturnValue({ + toStructuredText: vi.fn().mockReturnValue({ + asJSON: vi.fn().mockReturnValue( + JSON.stringify({ + blocks: [{ lines: [{ text: 'Mocked PDF content' }] }] + }) + ) + }) + }) + }); + + mockConvertFileNoId('this is ignored'); + const mockFile1: LFFile = new File(['this is ignored'], 'test1.txt', { type: 'text/plain' }); + mockFile1.id = '1'; + + const request = requestWithFormData(mockFile1); + const res = await POST({ + request, + fetch: global.fetch, + params: {}, + locals: getLocalsMock() + } as RequestEvent); + expect(res.status).toEqual(200); + const resJson = await res.json(); + + expect(resJson.text).toEqual('Mocked PDF content'); + }); +}); diff --git a/src/leapfrogai_ui/src/routes/api/messages/new/+server.ts b/src/leapfrogai_ui/src/routes/api/messages/new/+server.ts index f12a16123..01ec01a72 100644 --- a/src/leapfrogai_ui/src/routes/api/messages/new/+server.ts +++ b/src/leapfrogai_ui/src/routes/api/messages/new/+server.ts @@ -16,7 +16,6 @@ export const POST: RequestHandler = async ({ request, locals: { session } }) => try { requestData = (await request.json()) as NewMessageInput; if (requestData.lengthOverride) { - // TODO await messageInputSchema.validate(requestData, { abortEarly: false }); } else { await messageInputSchema.validate(requestData, { abortEarly: false }); diff --git a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.server.ts b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.server.ts deleted file mode 100644 index c5c3c35c7..000000000 --- a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.server.ts +++ /dev/null @@ -1,114 +0,0 @@ -import * as mupdf from 'mupdf'; -import { fail, superValidate, withFiles } from 'sveltekit-superforms'; -import { yup } from 'sveltekit-superforms/adapters'; -import type { Actions } from './$types'; -import { filesSchema } from '$schemas/files'; -import type { FileMetadata } from '$lib/types/files'; -import { env } from '$env/dynamic/public'; -import { shortenFileName } from '$helpers/stringHelpers'; -import { APPROX_MAX_CHARACTERS, FILE_UPLOAD_PROMPT } from '$constants'; -import { ERROR_UPLOADING_FILE_MSG, FILE_CONTEXT_TOO_LARGE_ERROR_MSG } from '$constants/errors'; - -// Ensure length of file context message does not exceed total context window when including the -// file upload prompt, user's message, and string quotes -const ADJUSTED_MAX = - APPROX_MAX_CHARACTERS - Number(env.PUBLIC_MESSAGE_LENGTH_LIMIT) - FILE_UPLOAD_PROMPT.length - 2; - -export const actions: Actions = { - // Handles parsing text from files, will convert file to pdf if is not already - default: async ({ request, fetch, locals: { session } }) => { - if (!session) { - return fail(401, { message: 'Unauthorized' }); - } - - const form = await superValidate(request, yup(filesSchema)); - - if (!form.valid) { - console.log( - 'Files form action: Invalid form submission.', - 'id:', - form.id, - 'errors:', - form.errors - ); - return fail(400, { form }); - } - - const extractedFilesText: FileMetadata[] = []; - - if (form.data.files && form.data.files.length > 0) { - for (const file of form.data.files) { - let text = ''; - if (file) { - try { - let buffer: ArrayBuffer; - const contentType = file.type; - if (contentType !== 'application/pdf') { - // Convert file to PDF - const formData = new FormData(); - formData.append('file', file); - const convertRes = await fetch('/api/files/convert', { - method: 'POST', - body: formData - }); - - if (!convertRes.ok) { - throw new Error('Error converting file'); //caught locally - } - - const convertedFileBlob = await convertRes.blob(); - buffer = await convertedFileBlob.arrayBuffer(); - } else buffer = await file.arrayBuffer(); - - const document = mupdf.Document.openDocument(buffer, 'application/pdf'); - let i = 0; - while (i < document.countPages()) { - const page = document.loadPage(i); - const json = page.toStructuredText('preserve-whitespace').asJSON(); - for (const block of JSON.parse(json).blocks) { - for (const line of block.lines) { - text += line.text; - } - } - i++; - } - - extractedFilesText.push({ - name: shortenFileName(file.name), - type: file.type, - text, - status: 'complete' - }); - - // If this file adds too much text (larger than allowed max), remove the text and set to error status - const totalTextLength = extractedFilesText.reduce( - (acc, fileMetadata) => acc + JSON.stringify(fileMetadata).length, - 0 - ); - if (totalTextLength > ADJUSTED_MAX) { - extractedFilesText[extractedFilesText.length - 1] = { - name: shortenFileName(file.name), - type: file.type, - text: '', - status: 'error', - errorText: FILE_CONTEXT_TOO_LARGE_ERROR_MSG - }; - } - } catch (e) { - console.error(`Error uploading file: ${file}: ${e}`); - extractedFilesText.push({ - name: shortenFileName(file.name), - type: file.type, - text: '', - status: 'error', - errorText: ERROR_UPLOADING_FILE_MSG - }); - } - } - } - - return withFiles({ extractedFilesText, form }); - } - return fail(400, { form }); - } -}; diff --git a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte index 4613c9141..5c5179b99 100644 --- a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte +++ b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.svelte @@ -24,9 +24,10 @@ } from '$constants/toastMessages'; import SelectAssistantDropdown from '$components/SelectAssistantDropdown.svelte'; import { PaperPlaneOutline, StopOutline } from 'flowbite-svelte-icons'; - import type { FileMetadata } from '$lib/types/files'; + import type { FileMetadata, LFFile } from '$lib/types/files'; import UploadedFileCards from '$components/UploadedFileCards.svelte'; - import ChatFileUploadForm from '$components/ChatFileUploadForm.svelte'; + import ChatFileUploadForm from '$components/ChatFileUpload.svelte'; + import FileChatActions from '$components/FileChatActions.svelte'; export let data; @@ -34,7 +35,8 @@ let lengthInvalid: boolean; // bound to child LFTextArea let assistantsList: Array<{ id: string; text: string }>; let uploadingFiles = false; - let attachedFileMetadata: FileMetadata[] = []; + let attachedFiles: LFFile[] = []; // the actual files uploaded + let attachedFileMetadata: FileMetadata[] = []; // metadata about the files uploaded, e.g. upload status, extracted text, etc... /** END LOCAL VARS **/ /** REACTIVE STATE **/ @@ -229,14 +231,10 @@ }, lengthOverride: true }); - setChatMessages([ - ...$chatMessages, - { ...contextMsg, content: getMessageText(contextMsg) } - ]); + threadsStore.updateMessagesState($chatMessages, setChatMessages, contextMsg); } // Save with API - const newMessage = await saveMessage({ thread_id: data.thread.id, content: $chatInput, @@ -331,7 +329,7 @@
-
+
{#each activeThreadMessages as message, index (message.id)} {#if message.metadata?.hideMessage !== 'true'}

-
+
-
- +
0 && 'py-4' + )} + > +
{#if !assistantMode} - + {/if} + /> {#if !$isLoading && $status !== 'in_progress'} {/if}
+
diff --git a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.ts b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.ts index 064331c1c..b3ab7fbec 100644 --- a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.ts +++ b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/+page.ts @@ -2,13 +2,9 @@ import type { PageLoad } from './$types'; import { browser } from '$app/environment'; import type { LFThread } from '$lib/types/threads'; import { threadsStore } from '$stores'; -import { superValidate } from 'sveltekit-superforms'; -import { yup } from 'sveltekit-superforms/adapters'; -import { filesSchema } from '$schemas/files'; export const load: PageLoad = async ({ params, fetch }) => { const promises = [fetch('/api/assistants'), fetch('/api/files')]; - const form = await superValidate(yup(filesSchema)); if (params.thread_id) promises.push(fetch(`/api/threads/${params.thread_id}`)); @@ -29,5 +25,5 @@ export const load: PageLoad = async ({ params, fetch }) => { } } - return { thread, assistants, files, form }; + return { thread, assistants, files }; }; diff --git a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/form-action.test.ts b/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/form-action.test.ts deleted file mode 100644 index 9f428553d..000000000 --- a/src/leapfrogai_ui/src/routes/chat/(dashboard)/[[thread_id]]/form-action.test.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { getLocalsMock } from '$lib/mocks/misc'; -import type { ActionFailure, RequestEvent } from '@sveltejs/kit'; -import type { RouteParams } from './$types'; -import { actions } from './+page.server'; -import { mockConvertFileErrorNoId, mockConvertFileNoId } from '$lib/mocks/file-mocks'; -import * as superforms from 'sveltekit-superforms'; -import { afterAll } from 'vitest'; - -vi.mock('mupdf', () => ({ - Document: { - openDocument: vi.fn().mockReturnValue({ - countPages: vi.fn().mockReturnValue(1), - loadPage: vi.fn().mockReturnValue({ - toStructuredText: vi.fn().mockReturnValue({ - asJSON: vi.fn().mockReturnValue( - JSON.stringify({ - blocks: [{ lines: [{ text: 'Mocked PDF content' }] }] - }) - ) - }) - }) - }) - } -})); - -describe('chat page form action', () => { - afterAll(() => { - vi.restoreAllMocks(); - }); - it('returns a 401 if the request is unauthenticated', async () => { - const request = new Request('https://thisurldoesntmatter', { - method: 'POST' - }); - const res = await actions.default({ - request, - locals: getLocalsMock({ nullSession: true }) - } as RequestEvent); - - expect(res?.status).toEqual(401); - }); - - it('returns a 400 if the request data is invalid', async () => { - const request = new Request('https://thisurldoesntmatter', { - method: 'POST' - }); - - const res = await actions.default({ request, locals: getLocalsMock() } as RequestEvent< - RouteParams, - '/chat/(dashboard)/[[thread_id]]' - >); - expect(res?.status).toEqual(400); - }); - - it('returns a 400 if after validation there are no files', async () => { - vi.spyOn(superforms, 'superValidate').mockResolvedValue({ - valid: true, - data: { - files: [] - }, - id: '', - posted: false, - errors: {} - }); - - const request = new Request('https://thisurldoesntmatter', { - method: 'POST', - headers: { - 'Content-Type': 'multipart/form-data' - } - }); - - const res = (await actions.default({ - request, - fetch: global.fetch, - locals: getLocalsMock() - } as RequestEvent)) as ActionFailure; - - expect(res?.status).toEqual(400); - }); - - it('sets a file to error status if there is an error converting it', async () => { - mockConvertFileErrorNoId(); - - const mockFile1 = new File(['content1'], 'test1.txt', { type: 'text/plain' }); - - vi.spyOn(superforms, 'superValidate').mockResolvedValue({ - valid: true, - data: { - files: [mockFile1] - }, - id: '', - posted: false, - errors: {} - }); - - const request = new Request('https://thisurldoesntmatter', { - method: 'POST', - headers: { - 'Content-Type': 'multipart/form-data' - } - }); - - const res = (await actions.default({ - request, - fetch: global.fetch, - locals: getLocalsMock() - } as RequestEvent)) as ActionFailure; - - expect(res.extractedFilesText[0].status).toEqual('error'); - }); - - it('sets a file to error status if there is an error converting it', async () => { - mockConvertFileErrorNoId(); - - const mockFile1 = new File(['content1'], 'test1.txt', { type: 'text/plain' }); - - vi.spyOn(superforms, 'superValidate').mockResolvedValue({ - valid: true, - data: { - files: [mockFile1] - }, - id: '', - posted: false, - errors: {} - }); - - const request = new Request('https://thisurldoesntmatter', { - method: 'POST', - headers: { - 'Content-Type': 'multipart/form-data' - } - }); - - const res = (await actions.default({ - request, - fetch: global.fetch, - locals: getLocalsMock() - } as RequestEvent)) as ActionFailure; - - expect(res.extractedFilesText[0].status).toEqual('error'); - }); - - it('returns files with their text content', async () => { - mockConvertFileNoId('this is ignored'); - - const mockFile1 = new File(['this is ignored'], 'test1.txt', { type: 'text/plain' }); - - vi.spyOn(superforms, 'superValidate').mockResolvedValue({ - valid: true, - data: { - files: [mockFile1] - }, - id: '', - posted: false, - errors: {} - }); - - const request = new Request('https://thisurldoesntmatter', { - method: 'POST', - headers: { - 'Content-Type': 'multipart/form-data' - } - }); - - const res = (await actions.default({ - request, - fetch: global.fetch, - locals: getLocalsMock() - } as RequestEvent)) as ActionFailure; - - expect(res.extractedFilesText[0].status).toEqual('complete'); - expect(res.extractedFilesText[0].text).toEqual('Mocked PDF content'); - }); -}); diff --git a/src/leapfrogai_ui/svelte.config.js b/src/leapfrogai_ui/svelte.config.js index 97f5b0c2f..f4934c1d1 100644 --- a/src/leapfrogai_ui/svelte.config.js +++ b/src/leapfrogai_ui/svelte.config.js @@ -13,6 +13,7 @@ const config = { // If your environment is not supported or you settled on a specific environment, switch out the adapter. // See https://kit.svelte.dev/docs/adapters for more information about adapters. adapter: adapter(), + csrf: { checkOrigin: process.env.NODE_ENV !== 'development' }, alias: { $components: 'src/lib/components', $webComponents: 'src/lib/web-components', diff --git a/src/leapfrogai_ui/testUtils/fakeData/index.ts b/src/leapfrogai_ui/testUtils/fakeData/index.ts index 98cc60645..7ead385af 100644 --- a/src/leapfrogai_ui/testUtils/fakeData/index.ts +++ b/src/leapfrogai_ui/testUtils/fakeData/index.ts @@ -16,6 +16,7 @@ import type { Assistant } from 'openai/resources/beta/assistants'; import type { VectorStore } from 'openai/resources/beta/vector-stores/index'; import type { VectorStoreFile } from 'openai/resources/beta/vector-stores/files'; import { type APIKeyRow, PERMISSIONS } from '../../src/lib/types/apiKeys'; +import type { FileMetadata, FileUploadStatus } from '$lib/types/files'; const todayOverride = new Date('2024-03-20T00:00'); @@ -346,3 +347,18 @@ export const getFakeApiKeys = (options: GetFakeApiKeysOptions = {}): APIKeyRow[] }; export const fakeKeys = getFakeApiKeys({ numKeys: 4 }); + +type GetMockFileMetadataOptions = { + text?: string; + status?: FileUploadStatus; +}; +export const getMockFileMetadata = (options: GetMockFileMetadataOptions = {}): FileMetadata => { + const { text = faker.word.noun(), status = 'complete' } = options; + return { + id: faker.string.uuid().substring(0, 8), + name: 'fake-file.pdf', + type: 'application/pdf', + text, + status + }; +}; diff --git a/src/leapfrogai_ui/tests/fixtures/spanish.m4a b/src/leapfrogai_ui/tests/fixtures/spanish.m4a new file mode 100644 index 000000000..9f57e6baa Binary files /dev/null and b/src/leapfrogai_ui/tests/fixtures/spanish.m4a differ diff --git a/src/leapfrogai_ui/tests/translation.test.ts b/src/leapfrogai_ui/tests/translation.test.ts new file mode 100644 index 000000000..364f8491e --- /dev/null +++ b/src/leapfrogai_ui/tests/translation.test.ts @@ -0,0 +1,72 @@ +import { expect, test } from './fixtures'; +import { loadChatPage } from './helpers/navigationHelpers'; +import { createPDF, deleteFixtureFile, uploadFiles } from './helpers/fileHelpers'; +import { deleteActiveThread } from './helpers/threadHelpers'; +import { faker } from '@faker-js/faker'; + +test('it can translate an audio file', async ({ page, openAIClient }) => { + await loadChatPage(page); + + await uploadFiles({ + page, + filenames: ['spanish.m4a'], + testId: 'upload-file-btn' + }); + + const chatTools = page.getByTestId('chat-tools'); + await chatTools.getByRole('button', { name: 'Translate spanish.m4a' }).click(); + + await expect(page.getByTestId('loading-msg')).toHaveCount(1); // loading skeleton + await expect(page.getByTestId('loading-msg')).not.toBeVisible(); + await expect(page.getByTestId('message')).toHaveCount(2); + // Edit and regen disabled for translated messages + await expect(page.getByTestId('edit-message')).not.toBeVisible(); + await expect(page.getByTestId('regenerate btn')).not.toBeVisible(); + const messages = await page.getByTestId('message').all(); + const responseText = await messages[1].innerText(); + expect(responseText).toContain('unicorn'); + + await deleteActiveThread(page, openAIClient); +}); + +test('it can removes the audio file but keeps other files after translating', async ({ + page, + openAIClient +}) => { + await loadChatPage(page); + const fakeContent = faker.word.words(3); + const pdfFilename = await createPDF({ content: fakeContent, filename: 'shortname.pdf' }); + + await uploadFiles({ + page, + filenames: ['spanish.m4a', pdfFilename], + testId: 'upload-file-btn' + }); + + await page.getByTestId('spanish.m4a-uploaded'); + await page.getByTestId(`${pdfFilename}-uploaded`); + + const messagesContainer = page.getByTestId('messages-container'); + const chatToolsContainer = page.getByTestId('chat-tools'); + + const chatToolsPDFFileCard = chatToolsContainer.getByTestId(`${pdfFilename}-file-uploaded-card`); + const chatToolsAudioCard = chatToolsContainer.getByTestId('spanish.m4a-file-uploaded-card'); + + await expect(chatToolsPDFFileCard).toBeVisible(); + await expect(chatToolsAudioCard).toBeVisible(); + + const translateBtn = chatToolsContainer.getByRole('button', { name: 'Translate spanish.m4a' }); + await translateBtn.click(); + + await expect(page.getByTestId('message')).toHaveCount(2); + + await expect(chatToolsAudioCard).not.toBeVisible(); + await expect(translateBtn).not.toBeVisible(); + + await expect(messagesContainer.getByTestId('spanish.m4a-file-uploaded-card')).toBeVisible(); + await expect(chatToolsPDFFileCard).toBeVisible(); + + // cleanup + deleteFixtureFile(pdfFilename); + await deleteActiveThread(page, openAIClient); +});