diff --git a/selfie-ui/package.json b/selfie-ui/package.json
index 2126428..d1f8590 100644
--- a/selfie-ui/package.json
+++ b/selfie-ui/package.json
@@ -15,8 +15,11 @@
"@rjsf/utils": "^5.17.1",
"@rjsf/validator-ajv8": "^5.17.1",
"@tailwindcss/typography": "^0.5.10",
+ "@tanstack/match-sorter-utils": "^8.11.8",
+ "@tanstack/react-table": "^8.13.2",
"daisyui": "^4.6.1",
"deep-chat-react": "^1.4.11",
+ "filesize": "^10.1.0",
"json-schema": "^0.4.0",
"next": "14.1.0",
"next-themes": "^0.2.1",
diff --git a/selfie-ui/src/app/components/AddData/DocumentSourceSelector.tsx b/selfie-ui/src/app/components/AddData/DocumentSourceSelector.tsx
index c91005b..2f2c60f 100644
--- a/selfie-ui/src/app/components/AddData/DocumentSourceSelector.tsx
+++ b/selfie-ui/src/app/components/AddData/DocumentSourceSelector.tsx
@@ -37,14 +37,19 @@ const DocumentSourceSelector = ({ onSelect }: DocumentSourceSelectorProps) => {
};
return (
-
- Select a data source...
- {sources.map((source) => (
-
- {source.label}
-
- ))}
-
+ <>
+
+ Choose a data source and follow the instructions to upload new documents. Choose the method that most closely matches your data for the best results.
+
+
+ Select a data source...
+ {sources.map((source) => (
+
+ {source.label}
+
+ ))}
+
+ >
);
};
diff --git a/selfie-ui/src/app/components/Chat.tsx b/selfie-ui/src/app/components/Chat.tsx
index 78fa1cd..09680f6 100644
--- a/selfie-ui/src/app/components/Chat.tsx
+++ b/selfie-ui/src/app/components/Chat.tsx
@@ -1,8 +1,9 @@
"use client";
-import React, { useState } from "react";
+import React, { useEffect, useState } from "react";
import dynamic from "next/dynamic";
import { RequestDetails } from "deep-chat/dist/types/interceptors";
+import { useTheme } from "next-themes";
import { apiBaseUrl } from "../config";
const DeepChat = dynamic(
@@ -18,85 +19,78 @@ export const Chat = ({
shouldClear = false, // TODO: figure out how to use this
instruction = ''
}) => {
+ const { theme } = useTheme();
+
const [showIntroPanel, setShowIntroPanel] = useState(!!instruction);
- return {}}
- key={`${assistantName}-${assistantBio}-${userName}-${disableAugmentation}-${showIntroPanel}`} // Force re-render when props are changed
- htmlClassUtilities={{
- 'close-button': {
- events: {
- click: () => {
- setShowIntroPanel(false);
- },
- },
- styles: {
- default: {
- cursor: 'pointer',
- textAlign: 'center',
- backgroundColor: '#555',
- color: 'white',
- padding: '4px 8px',
- border: '1px solid #666',
- borderRadius: '10px',
- fontSize: '16px',
- marginBottom: '10px',
- },
- },
- },
- 'custom-button-text': { styles: { default: { pointerEvents: 'none' } } },
- }}
- style={{
- borderRadius: '10px',
- border: 'unset',
- backgroundColor: '#292929',
- width: '100%',
- maxWidth: 'inherit',
- display: 'block'
- }}
- messageStyles={{
- "default": {
- "ai": {"bubble": {"backgroundColor": "#545454", "color": "white"}}
- },
- "loading": {
- "bubble": {"backgroundColor": "#545454", "color": "white"}
+ const chatStyle = {
+ borderRadius: '10px',
+ border: 'unset solid 1px oklch(var(--b2)*0.2)', // The 0.2 is not working, can't match textarea-bordered so using --b2 below instead.
+ backgroundColor: 'oklch(var(--b2))',
+ // backgroundColor: 'oklch(var(--b1))',
+ width: '100%',
+ maxWidth: 'inherit',
+ display: 'block'
+ };
+
+ const chatMessageStyle = {
+ default: {
+ ai: { bubble: { backgroundColor: 'oklch(var(--b2))', color: 'oklch(var(--bc))' } }, // Slightly darker base color for AI bubble
+ },
+ loading: {
+ bubble: { backgroundColor: 'oklch(var(--b2))', color: 'oklch(var(--bc))' },
+ }
+ };
+
+ const chatInputStyle = {
+ styles: {
+ container: {
+ backgroundColor: 'oklch(var(--b3))', // Even more darker base color for input container
+ border: 'unset',
+ color: 'oklch(var(--bc))' // Base content color
}
- }}
- textInput={{
- "styles": {
- "container": {
- "backgroundColor": "#666666",
- "border": "unset",
- "color": "#e8e8e8"
- }
+ },
+ placeholder: { text: "Say anything here...", style: { color: 'oklch(var(--bc))' } } // Use base-200 color for placeholder
+ };
+
+ const chatSubmitButtonStyles = {
+ submit: {
+ container: {
+ default: { bottom: '0.7rem' }
},
- "placeholder": {"text": "Say anything here...", "style": {"color": "#bcbcbc"}}
- }}
- submitButtonStyles={{
- "submit": {
- "container": {
- "default": {"bottom": "0.7rem"}
- },
- "svg": {
- "styles": {
- "default": {
- "filter": "brightness(0) saturate(100%) invert(70%) sepia(52%) saturate(5617%) hue-rotate(185deg) brightness(101%) contrast(101%)"
- }
+ svg: {
+ styles: {
+ default: {
+ filter: "brightness(0) saturate(100%) invert(70%) sepia(52%) saturate(5617%) hue-rotate(185deg) brightness(101%) contrast(101%)"
}
}
}
- }}
- auxiliaryStyle="::-webkit-scrollbar {
- width: 10px;
- height: 10px;
- }
- ::-webkit-scrollbar-thumb {
- background-color: grey;
- border-radius: 5px;
- }
- ::-webkit-scrollbar-track {
- background-color: unset;
- }"
+ }
+ };
+
+ const auxiliaryStyle=`::-webkit-scrollbar {
+ width: 10px;
+ height: 10px;
+ }
+ ::-webkit-scrollbar-thumb {
+ background-color: oklch(var(--n));
+ border-radius: 5px;
+ }
+ ::-webkit-scrollbar-track {
+ background-color: unset;
+ }`
+
+ useEffect(() => {
+ setShowIntroPanel(!!instruction);
+ }, [instruction]);
+
+ return {
- const [documentConnections, setDocumentConnections] = useState([]);
- const [documents, setDocuments] = useState({});
- const [selectedDocuments, setSelectedDocuments] = useState>(new Set());
-
- const { isTaskRunning, taskMessage, executeTask } = useAsyncTask();
-
- const [stats, setStats] = useState({
- totalDocuments: 0,
- numDocumentsIndexed: 0,
- numEmbeddingIndexDocuments: 0
- });
-
- const fetchDocumentConnections = useCallback(async () => {
- executeTask(async () => {
- const response = await fetch(`${apiBaseUrl}/v1/data-sources`);
- const data = await response.json();
- setDocumentConnections(data);
- await Promise.all(data.map((connection: any) => fetchDocuments(connection.id)));
- }, {
- start: 'Loading data sources',
- success: 'Data sources loaded',
- error: 'Failed to load data sources',
- })
- }, [executeTask]);
-
- useEffect(() => {
- fetchDocumentConnections();
- }, [fetchDocumentConnections]);
-
- const fetchDocuments = async (sourceId: string) => {
- try {
- const response = await fetch(`${apiBaseUrl}/v1/documents?source_id=${sourceId}`);
- const docs = await response.json();
- setDocuments(prevDocs => ({ ...prevDocs, [sourceId]: docs }));
- } catch (error) {
- console.error('Error fetching documents:', error);
- }
- };
-
- useEffect(() => {
- if (selectedDocuments.size === 0) {
- setStats({
- totalDocuments: Object.values(documents).flat().length,
- numDocumentsIndexed: Object.values(documents).flat().filter((doc) => doc.is_indexed).length,
- numEmbeddingIndexDocuments: Object.values(documents).flat().filter((doc) => doc.is_indexed).reduce((acc, doc) => acc + (doc?.num_index_documents ?? 0), 0)
- });
- } else {
- setStats({
- totalDocuments: selectedDocuments.size,
- numDocumentsIndexed: Array.from(selectedDocuments).map((docId) => Object.values(documents).flat().find((doc) => doc.id === docId)).filter((doc) => doc?.is_indexed).length,
- numEmbeddingIndexDocuments: Array.from(selectedDocuments).map((docId) => Object.values(documents).flat().find((doc) => doc.id === docId)).filter((doc) => doc?.is_indexed).reduce((acc, doc) => acc + (doc?.num_index_documents ?? 0), 0)
- });
- }
- console.log(documents)
- }, [documents, documentConnections, selectedDocuments]);
-
- const toggleDocumentSelection = (docId: string) => {
- setSelectedDocuments(prevSelectedDocuments => {
- const newSelection = new Set(prevSelectedDocuments);
- if (newSelection.has(docId)) {
- newSelection.delete(docId);
- } else {
- newSelection.add(docId);
- }
- return newSelection;
- });
- };
-
- const columnNames = useMemo(() => {
- const firstDoc = documents[documentConnections[0]?.id]?.[0];
- return firstDoc ? Object.keys(firstDoc) : [];
- }, [documents, documentConnections]);
-
-
- return (
- <>
- {/* TODO: this is necessary until useAsyncTask is refactored to use global state */}
- {taskMessage && }
-
-
- Index documents to add them to the knowledge bank. Once indexed, data will be used automatically by the AI.
-
-
-
- handleIndexSelected([doc.id])}
- // onUnindexDocument={(doc) => handleIndexDocument(doc.id, true)}
- // onIndexDocuments={() => handleIndexSelected()}
- // onUnindexDocuments={() => handleIndexSelected(undefined, true)}
- stats={stats}
- />
-
- >
- );
-};
-
-DataManager.displayName = 'DataManager';
-
-export default DataManager;
diff --git a/selfie-ui/src/app/components/DocumentTable/DocumentTable.tsx b/selfie-ui/src/app/components/DocumentTable/DocumentTable.tsx
index abd64bb..8cdddc9 100644
--- a/selfie-ui/src/app/components/DocumentTable/DocumentTable.tsx
+++ b/selfie-ui/src/app/components/DocumentTable/DocumentTable.tsx
@@ -1,148 +1,214 @@
-import React, { useMemo, useState } from 'react';
+import React, { useEffect, useMemo, useState } from 'react';
+import { FaRegTrashAlt } from 'react-icons/fa';
import { ChevronDownIcon, ChevronUpIcon } from "@heroicons/react/20/solid";
-import { DataSource, Document, DocumentStats } from "@/app/types";
-import { isDateString, isNumeric } from "@/app/utils";
-import { DocumentTableActionBar } from "./DocumentTableActionBar";
-import DocumentTableRow from './DocumentTableRow';
+import { Document } from "@/app/types";
+import { filesize } from 'filesize';
+import {
+ createColumnHelper,
+ flexRender,
+ getCoreRowModel,
+ getFilteredRowModel,
+ getSortedRowModel,
+ SortingState,
+ useReactTable,
+ ColumnDef,
+} from '@tanstack/react-table';
+import { rankItem } from '@tanstack/match-sorter-utils'
+
+
+const fuzzyFilter = (row: any, columnId: string, value: any, addMeta: any) => {
+ const itemRank = rankItem(row.getValue(columnId), value)
+ addMeta({ itemRank })
+ return itemRank.passed
+}
+
+const columnHelper = createColumnHelper();
+const customColumnDefinitions: Partial JSX.Element | string }>> = {
+ id: {
+ header: 'ID',
+ },
+ created_at: {
+ header: 'Created At',
+ cell: (value: string) => new Date(value).toLocaleString(),
+ },
+ updated_at: {
+ header: 'Updated At',
+ cell: (value: string) => new Date(value).toLocaleString(),
+ },
+ size: {
+ header: 'Size',
+ cell: (value: number) => filesize(value),
+ },
+ connector_name: {
+ header: 'Connector',
+ },
+};
+
+const generateColumns = (data: Document[]) => {
+ const sample = data[0] || {};
+ return Object.keys(sample).map((key) => {
+ const id = key as keyof Document;
+ const custom = customColumnDefinitions[id];
+
+ return columnHelper.accessor(id, {
+ header: () => {custom?.header || id.toString().replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase())} ,
+ cell: custom?.cell ? (info) => custom?.cell?.(info.getValue()) : (info) => info.getValue(),
+ });
+ });
+};
interface DocumentTableProps {
- dataSources: DataSource[];
- documents: { [sourceId: string]: Document[] };
- columnNames: string[];
- selectedDocuments: Set;
- setSelectedDocuments: (selected: Set) => void;
- onToggleDocumentSelection: (docId: string) => void;
- // onIndexDocument: (doc: Document) => void | Promise;
- // onUnindexDocument: (doc: Document) => void | Promise;
- // onIndexDocuments: () => void | Promise;
- // onUnindexDocuments: () => void | Promise;
- disabled?: boolean;
- stats?: DocumentStats;
+ data: Document[];
+ onDeleteDocuments: (docIds: string[]) => void;
+ onSelectionChange?: (selectedIds: string[]) => void;
}
-const DocumentTable = ({
- dataSources,
- documents,
- columnNames,
- selectedDocuments,
- setSelectedDocuments,
- onToggleDocumentSelection,
- // onIndexDocument,
- // onUnindexDocument,
- // onIndexDocuments,
- // onUnindexDocuments,
- disabled = false,
- stats,
- }: DocumentTableProps) => {
- const [sortField, setSortField] = useState();
- const [sortDirection, setSortDirection] = useState<'asc'|'desc'>('asc');
- const [selectAll, setSelectAll] = useState(false);
-
- const handleToggleSelectAll = () => {
- setSelectAll(!selectAll);
- if (!selectAll) {
- const allDocIds = Object.values(documents).flat().map(doc => doc.id);
- setSelectedDocuments(new Set(allDocIds));
- } else {
- setSelectedDocuments(new Set());
- }
- };
+const DocumentTable: React.FC = ({ data, onDeleteDocuments, onSelectionChange = () => {} }) => {
+ const [sorting, setSorting] = useState([]);
+ const [selectedRows, setSelectedRows] = useState>({});
+ const [globalFilter, setGlobalFilter] = useState('');
- const handleHeaderClick = (fieldName: string) => {
- if (sortField === fieldName) {
- setSortDirection(sortDirection === 'asc' ? 'desc' : 'asc');
+ const allRowsSelected = data.length > 0 && data.every(({ id }) => selectedRows[id]);
+
+ useEffect(() => {
+ const newDataIds = new Set(data.map(item => item.id));
+ setSelectedRows(prevSelectedRows => {
+ return Object.keys(prevSelectedRows).reduce((acc: Record, cur: string) => {
+ if (newDataIds.has(cur)) {
+ acc[cur] = prevSelectedRows[cur];
+ }
+ return acc;
+ }, {});
+ });
+ }, [data]);;
+
+ useEffect(() => {
+ onSelectionChange(Object.keys(selectedRows).filter((id) => selectedRows[id]));
+ }, [selectedRows, onSelectionChange]);
+
+ const toggleAllRowsSelected = () => {
+ if (allRowsSelected) {
+ setSelectedRows({});
} else {
- setSortField(fieldName);
- setSortDirection('asc');
+ const newSelectedRows: Record = {};
+ data.forEach(({ id }) => {
+ newSelectedRows[id] = true;
+ });
+ setSelectedRows(newSelectedRows);
}
};
- const getSortableValue = (value: any) => {
- if (isDateString(value)) {
- return new Date(value).getTime();
- } else if (isNumeric(value)) {
- return parseFloat(value);
- }
- return value.toString();
+ const handleSelectRow = (id: string) => {
+ setSelectedRows((prev) => ({
+ ...prev,
+ [id]: !prev[id],
+ }));
};
- const sortedDocuments = useMemo(() => {
- if (!sortField) return documents;
+ const columns = useMemo(() => [
+ // Checkbox column
+ columnHelper.display({
+ id: 'selection',
+ header: () => (
+ e.stopPropagation()}
+ />
+ ),
+ cell: ({ row }) => (
+ handleSelectRow(row.original.id)}
+ />
+ ),
+ }),
+ ...generateColumns(data),
+ ...[onDeleteDocuments ? columnHelper.display({
+ id: 'delete',
+ header: () => Delete ,
+ cell: ({ row }) => (
+ onDeleteDocuments([row.original.id])} className="text-red-500 hover:text-red-700">
+
+
+ ),
+ }) : null].filter(Boolean),
+ ].filter((column): column is ColumnDef => column !== null), [data, selectedRows, onDeleteDocuments, allRowsSelected, toggleAllRowsSelected]);
- return Object.keys(documents).reduce((sortedDocs: any, key: string) => {
- sortedDocs[key] = [...documents[key]].sort((a, b) => {
- const aValue = getSortableValue(a.metadata[sortField]);
- const bValue = getSortableValue(b.metadata[sortField]);
+ const table = useReactTable({
+ data: data ?? [],
+ columns: columns,
+ state: {
+ sorting,
+ globalFilter
+ },
+ initialState: {
+ columnOrder: ['selection', 'id', 'name', 'content_type', 'connector_name', 'document_connection_id', 'size', 'created_at', 'updated_at', 'delete'],
+ },
+ onSortingChange: setSorting,
+ onGlobalFilterChange: setGlobalFilter,
+ globalFilterFn: fuzzyFilter,
+ getCoreRowModel: getCoreRowModel(),
+ getSortedRowModel: getSortedRowModel(),
+ getFilteredRowModel: getFilteredRowModel(),
+ });
- if (sortDirection === 'asc') {
- return aValue > bValue ? 1 : -1;
- } else {
- return aValue < bValue ? 1 : -1;
- }
- });
- return sortedDocs;
- }, {});
- }, [documents, sortField, sortDirection]);
-
- const indexableDocuments = Array.from(Object.keys(documents).map((coll) => documents[coll].filter(doc => !doc.is_indexed)).flat().filter(doc => selectedDocuments.has(doc.id)));
- const unindexableDocuments = Array.from(Object.keys(documents).map((coll) => documents[coll].filter(doc => doc.is_indexed)).flat().filter(doc => selectedDocuments.has(doc.id)));
+ const selectedDocuments = Object.keys(selectedRows).filter((id) => selectedRows[id]);
return (
- <>
- 0}
- disabled={disabled}
- stats={stats}
- />
-
+
+
+ onDeleteDocuments(selectedDocuments)}
+ >
+ Delete {selectedDocuments.length}
+
+
+ setGlobalFilter(e.target.value)}
+ placeholder="Search all columns..."
+ className="input input-sm input-bordered"
+ />
+
+
+
- >
+
);
};
diff --git a/selfie-ui/src/app/components/DocumentTable/DocumentTableActionBar.tsx b/selfie-ui/src/app/components/DocumentTable/DocumentTableActionBar.tsx
deleted file mode 100644
index d1ea30b..0000000
--- a/selfie-ui/src/app/components/DocumentTable/DocumentTableActionBar.tsx
+++ /dev/null
@@ -1,56 +0,0 @@
-import React from 'react';
-
-import { Document, DocumentStats } from "@/app/types";
-
-interface IndexDocumentsFormProps {
- // onIndexDocuments: () => void | Promise;
- // onUnindexDocuments: () => void | Promise;
- indexableDocuments: Document[];
- unindexableDocuments: Document[];
- hasSelectedDocuments: boolean;
- disabled?: boolean;
- stats?: DocumentStats;
-}
-
-export const DocumentTableActionBar: React.FC = ({
- // onIndexDocuments,
- // onUnindexDocuments,
- indexableDocuments,
- unindexableDocuments,
- hasSelectedDocuments,
- disabled = false,
- stats,
- }) => {
- // const handleSubmit = async (event: React.FormEvent, isIndex: boolean) => {
- // event.preventDefault();
- // await (isIndex ? onIndexDocuments() : onUnindexDocuments());
- // };
-
- return (
-
-
-
- {stats && Object.keys(stats).length &&
- Total: {stats.totalDocuments} | Indexed: {stats.numDocumentsIndexed} | Indexed Chunks: {stats.numEmbeddingIndexDocuments}
- }
-
- );
-};
diff --git a/selfie-ui/src/app/components/DocumentTable/DocumentTableRow.tsx b/selfie-ui/src/app/components/DocumentTable/DocumentTableRow.tsx
deleted file mode 100644
index 19a09c9..0000000
--- a/selfie-ui/src/app/components/DocumentTable/DocumentTableRow.tsx
+++ /dev/null
@@ -1,86 +0,0 @@
-import React, {useEffect, useState} from "react";
-import { formatDate, isDateString } from "@/app/utils";
-import { DataSource, Document } from "@/app/types";
-
-interface DocumentTableRowProps {
- doc: Document;
- dataSource: DataSource;
- columnNames: string[];
- onToggle: (docId: string) => void;
- // onIndexDocument: (doc: Document) => void | Promise;
- // onUnindexDocument: (doc: Document) => void | Promise;
- isSelected?: boolean;
- disabled?: boolean;
-}
-
-const DocumentTableRow = React.memo(({
- doc,
- dataSource,
- columnNames,
- onToggle,
- // onIndexDocument,
- // onUnindexDocument,
- isSelected = false,
- disabled = false,
-}) => {
- const [selected, setSelected] = useState(isSelected);
-
- useEffect(() => {
- setSelected(isSelected);
- }, [isSelected]);
-
- const handleCheckboxChange = () => {
- setSelected(!selected);
- onToggle(doc.id);
- };
-
- return (
-
-
-
-
-
-
- {dataSource.name}
-
-
-
-
- {doc.is_indexed ? '✅' : ''}
-
-
- {columnNames.map((colName) => (
-
-
- {isDateString(doc[colName]) ? formatDate(doc[colName]) : String(doc[colName])}
-
-
- ))}
-
- {!doc.is_indexed && onIndexDocument(doc)}
- className="btn btn-xs"
- disabled={disabled}
- >
- Index
- }
- {doc.is_indexed && onUnindexDocument(doc)}
- className="btn btn-xs btn-error btn-outline"
- disabled={disabled}
- >
- Unindex
- }
-
-
- );
-});
-
-DocumentTableRow.displayName = 'DocumentTableRow';
-
-export default DocumentTableRow;
diff --git a/selfie-ui/src/app/components/ManageData.tsx b/selfie-ui/src/app/components/ManageData.tsx
new file mode 100644
index 0000000..bd0a8f8
--- /dev/null
+++ b/selfie-ui/src/app/components/ManageData.tsx
@@ -0,0 +1,75 @@
+"use client";
+
+import React, {useCallback, useEffect, useState} from 'react';
+import {Document} from "@/app/types";
+import {apiBaseUrl} from "@/app/config";
+import TaskToast from "@/app/components/TaskToast";
+import useAsyncTask from "@/app/hooks/useAsyncTask";
+import {DocumentTable} from "@/app/components/DocumentTable";
+
+
+const ManageData = () => {
+ const [documents, setDocuments] = useState([]);
+
+ const { isTaskRunning, taskMessage, executeTask } = useAsyncTask();
+
+ const fetchDocuments = useCallback(async () => {
+ executeTask(async () => {
+ const response = await fetch(`${apiBaseUrl}/v1/documents`);
+ setDocuments(await response.json());
+ }, {
+ start: 'Loading documents',
+ success: 'Documents loaded',
+ error: 'Failed to load documents',
+ });
+ }, [executeTask]);
+
+ useEffect(() => {
+ fetchDocuments();
+ }, [fetchDocuments]);
+
+ const deleteDocuments = (docIds: string[]) => {
+ const plural = docIds.length > 1 ? 's' : '';
+ executeTask(async () => {
+ await fetch(`${apiBaseUrl}/v1/documents`, {
+ method: 'DELETE',
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({ document_ids: docIds }),
+ });
+ await fetchDocuments();
+ }, {
+ start: `Deleting document${plural}`,
+ success: `Document${plural} deleted`,
+ error: `Failed to delete document${plural}`,
+ });
+ }
+
+ return (
+ <>
+ {/* TODO: this is necessary until useAsyncTask is refactored to use global state */}
+ {taskMessage && }
+
+
+ {/*Index documents to add them to the knowledge bank. Once indexed, data will be used automatically by the AI.*/}
+ Documents that have been added to the knowledge bank are shown here. You can add more on the Add Data page .
+
+
+
+
+
+
+ >
+ );
+};
+
+ManageData.displayName = 'Manage Data';
+
+export default ManageData;
diff --git a/selfie-ui/src/app/components/Playground/PlaygroundChat.tsx b/selfie-ui/src/app/components/Playground/PlaygroundChat.tsx
index 1be97f8..3af6adc 100644
--- a/selfie-ui/src/app/components/Playground/PlaygroundChat.tsx
+++ b/selfie-ui/src/app/components/Playground/PlaygroundChat.tsx
@@ -42,26 +42,28 @@ const PlaygroundChat = ({ disabled = false, hasIndexedDocuments = true }: { disa
Chat
-
- {!!score &&
+ {!!summary &&
{/*
*/}
{summary}
-
Result Score: {score.toFixed(2)}
+ {documents.length ?
Result Score: {score.toFixed(2)}
: null }
}
{!!score &&
diff --git a/selfie-ui/src/app/components/ThemeChanger.tsx b/selfie-ui/src/app/components/ThemeChanger.tsx
index f4a9575..c138693 100644
--- a/selfie-ui/src/app/components/ThemeChanger.tsx
+++ b/selfie-ui/src/app/components/ThemeChanger.tsx
@@ -9,9 +9,9 @@ export const ThemeChanger = () => {
if (theme === 'system') {
const systemThemeIsDark = window.matchMedia &&
window.matchMedia('(prefers-color-scheme: dark)').matches;
- setTheme(systemThemeIsDark ? 'cupcake' : 'dark')
+ setTheme(systemThemeIsDark ? 'light' : 'dark')
} else {
- setTheme(theme === 'dark' ? 'cupcake' : 'dark')
+ setTheme(theme === 'dark' ? 'light' : 'dark')
}
}
diff --git a/selfie-ui/src/app/page.tsx b/selfie-ui/src/app/page.tsx
index 76fdbe8..68d077b 100644
--- a/selfie-ui/src/app/page.tsx
+++ b/selfie-ui/src/app/page.tsx
@@ -6,13 +6,13 @@ import { ThemeChanger } from "@/app/components/ThemeChanger";
import { AddData } from "@/app/components/AddData";
import useAsyncTask from "@/app/hooks/useAsyncTask";
import TaskToast from "@/app/components/TaskToast";
-import DataManager from "@/app/components/DataManager";
+import ManageData from "@/app/components/ManageData";
import { Playground } from "@/app/components/Playground";
const pages = [
{ component: Playground, id: 'playground' },
{ component: AddData, id: 'addData' },
- //{ component: DataManager, id: 'dataManager' },
+ { component: ManageData, id: 'manageData' },
];
const App = () => {
@@ -64,8 +64,8 @@ const App = () => {
{pages.map(({ component: Component, id }) => (
-
-
{renderComponentName(Component)}
+
+
{renderComponentName(Component)}
@@ -75,7 +75,7 @@ const App = () => {
{/* Sidebar content here */}
-
+
Selfie
{/* Sidebar content here */}
diff --git a/selfie-ui/src/app/types/index.ts b/selfie-ui/src/app/types/index.ts
index c92fd36..44d54ae 100644
--- a/selfie-ui/src/app/types/index.ts
+++ b/selfie-ui/src/app/types/index.ts
@@ -1,13 +1,14 @@
// TODO: define this type
-export type Document = any
-// export interface Document {
-// id: string
-// metadata: {
-// [key: string]: any
-// }
-// is_indexed: boolean
-// num_index_documents?: number
-// }
+
+export interface Document {
+ id: string;
+ created_at: string;
+ updated_at: string;
+ content_type: string;
+ name: string;
+ size: number;
+ connector_name: string;
+}
export interface Documents {
[sourceId: string]: Document[]
diff --git a/selfie-ui/tailwind.config.ts b/selfie-ui/tailwind.config.ts
index 05a6fec..6551e24 100644
--- a/selfie-ui/tailwind.config.ts
+++ b/selfie-ui/tailwind.config.ts
@@ -20,8 +20,7 @@ const config: Config = {
},
plugins: [require("@tailwindcss/typography"), require("daisyui")],
daisyui: {
- themes: ["dark", "cupcake"],
- // themes: ["light", "dark", "cupcake"],
+ themes: ["light", "dark"],
},
};
export default config;
diff --git a/selfie-ui/yarn.lock b/selfie-ui/yarn.lock
index 334a96b..4e1296b 100644
--- a/selfie-ui/yarn.lock
+++ b/selfie-ui/yarn.lock
@@ -271,6 +271,25 @@
lodash.merge "^4.6.2"
postcss-selector-parser "6.0.10"
+"@tanstack/match-sorter-utils@^8.11.8":
+ version "8.11.8"
+ resolved "https://registry.yarnpkg.com/@tanstack/match-sorter-utils/-/match-sorter-utils-8.11.8.tgz#9132c2a21cf18ca2f0071b604ddadb7a66e73367"
+ integrity sha512-3VPh0SYMGCa5dWQEqNab87UpCMk+ANWHDP4ALs5PeEW9EpfTAbrezzaOk/OiM52IESViefkoAOYuxdoa04p6aA==
+ dependencies:
+ remove-accents "0.4.2"
+
+"@tanstack/react-table@^8.13.2":
+ version "8.13.2"
+ resolved "https://registry.yarnpkg.com/@tanstack/react-table/-/react-table-8.13.2.tgz#a3aa737ae464abc651f68daa7e82dca17813606c"
+ integrity sha512-b6mR3mYkjRtJ443QZh9sc7CvGTce81J35F/XMr0OoWbx0KIM7TTTdyNP2XKObvkLpYnLpCrYDwI3CZnLezWvpg==
+ dependencies:
+ "@tanstack/table-core" "8.13.2"
+
+"@tanstack/table-core@8.13.2":
+ version "8.13.2"
+ resolved "https://registry.yarnpkg.com/@tanstack/table-core/-/table-core-8.13.2.tgz#2512574dd3d20dc94b7db1f9f48090f0c18b5c85"
+ integrity sha512-/2saD1lWBUV6/uNAwrsg2tw58uvMJ07bO2F1IWMxjFRkJiXKQRuc3Oq2aufeobD3873+4oIM/DRySIw7+QsPPw==
+
"@types/debug@^4.0.0":
version "4.1.12"
resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.12.tgz#a155f21690871953410df4b6b6f53187f0500917"
@@ -1363,6 +1382,11 @@ file-entry-cache@^6.0.1:
dependencies:
flat-cache "^3.0.4"
+filesize@^10.1.0:
+ version "10.1.0"
+ resolved "https://registry.yarnpkg.com/filesize/-/filesize-10.1.0.tgz#846f5cd8d16e073c5d6767651a8264f6149183cd"
+ integrity sha512-GTLKYyBSDz3nPhlLVPjPWZCnhkd9TrrRArNcy8Z+J2cqScB7h2McAzR6NBX6nYOoWafql0roY8hrocxnZBv9CQ==
+
fill-range@^7.0.1:
version "7.0.1"
resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
@@ -2994,6 +3018,11 @@ remarkable@^2.0.1:
argparse "^1.0.10"
autolinker "^3.11.0"
+remove-accents@0.4.2:
+ version "0.4.2"
+ resolved "https://registry.yarnpkg.com/remove-accents/-/remove-accents-0.4.2.tgz#0a43d3aaae1e80db919e07ae254b285d9e1c7bb5"
+ integrity sha512-7pXIJqJOq5tFgG1A2Zxti3Ht8jJF337m4sowbuHsW30ZnkQFnDzy9qBNhgzX8ZLW4+UBcXiiR7SwR6pokHsxiA==
+
require-from-string@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909"
diff --git a/selfie/api/documents.py b/selfie/api/documents.py
index c7675ca..474de6a 100644
--- a/selfie/api/documents.py
+++ b/selfie/api/documents.py
@@ -19,11 +19,21 @@ class IndexDocumentsRequest(BaseModel):
document_ids: List[str] = []
+class DeleteDocumentsRequest(BaseModel):
+ document_ids: List[str] = []
+
+
@router.get("/documents")
async def get_documents():
return DataManager().get_documents()
+@router.delete("/documents")
+async def index_documents(request: DeleteDocumentsRequest):
+ await DataManager().remove_documents([int(document_id) for document_id in request.document_ids])
+ return {"message": "Documents removed successfully"}
+
+
@router.delete("/documents/{document_id}")
async def delete_data_source(document_id: int, delete_indexed_data: bool = True):
await DataManager().remove_document(document_id, delete_indexed_data)
diff --git a/selfie/connectors/chatgpt/connector.py b/selfie/connectors/chatgpt/connector.py
index d77c988..ce1d088 100644
--- a/selfie/connectors/chatgpt/connector.py
+++ b/selfie/connectors/chatgpt/connector.py
@@ -6,7 +6,7 @@
from selfie.embeddings import EmbeddingDocumentModel, DataIndex
from selfie.parsers.chat import ChatFileParser # TODO Replace this with ChatGPTParser
from selfie.types.documents import DocumentDTO
-from selfie.utils import data_uri_to_string
+from selfie.utils import data_uri_to_dict
class ChatGPTConfiguration(BaseModel):
@@ -24,10 +24,10 @@ def load_document(self, configuration: dict[str, Any]) -> List[DocumentDTO]:
return [
DocumentDTO(
- content=(content := data_uri_to_string(data_uri)),
- content_type="application/json",
- name="todo",
- size=len(content.encode('utf-8'))
+ content=(parsed := data_uri_to_dict(data_uri))['content'],
+ content_type=parsed['content_type'],
+ name=parsed['name'],
+ size=len(parsed['content'].encode('utf-8'))
)
for data_uri in config.files
]
diff --git a/selfie/connectors/google_messages/connector.py b/selfie/connectors/google_messages/connector.py
index f8718c1..d077655 100644
--- a/selfie/connectors/google_messages/connector.py
+++ b/selfie/connectors/google_messages/connector.py
@@ -6,7 +6,7 @@
from selfie.embeddings import EmbeddingDocumentModel, DataIndex
from selfie.parsers.chat import ChatFileParser
from selfie.types.documents import DocumentDTO
-from selfie.utils import data_uri_to_string
+from selfie.utils import data_uri_to_dict
class GoogleMessagesConfiguration(BaseModel):
@@ -24,10 +24,10 @@ def load_document(self, configuration: dict[str, Any]) -> List[DocumentDTO]:
return [
DocumentDTO(
- content=data_uri_to_string(data_uri),
- content_type="application/json",
- name="todo",
- size=len(data_uri_to_string(data_uri).encode('utf-8'))
+ content=(parsed := data_uri_to_dict(data_uri))['content'],
+ content_type=parsed['content_type'],
+ name=parsed['name'],
+ size=len(parsed['content'].encode('utf-8'))
)
for data_uri in config.files
]
diff --git a/selfie/connectors/telegram/connector.py b/selfie/connectors/telegram/connector.py
index 99bd6ba..23335da 100644
--- a/selfie/connectors/telegram/connector.py
+++ b/selfie/connectors/telegram/connector.py
@@ -6,7 +6,7 @@
from selfie.embeddings import EmbeddingDocumentModel, DataIndex
from selfie.parsers.chat import ChatFileParser
from selfie.types.documents import DocumentDTO
-from selfie.utils import data_uri_to_string
+from selfie.utils import data_uri_to_dict
class TelegramConfiguration(BaseModel):
@@ -24,10 +24,10 @@ def load_document(self, configuration: dict[str, Any]) -> List[DocumentDTO]:
return [
DocumentDTO(
- content=data_uri_to_string(data_uri),
- content_type="text/html",
- name="todo",
- size=len(data_uri_to_string(data_uri).encode('utf-8'))
+ content=(parsed := data_uri_to_dict(data_uri))['content'],
+ content_type=parsed['content_type'],
+ name=parsed['name'],
+ size=len(parsed['content'].encode('utf-8'))
)
for data_uri in config.files
]
diff --git a/selfie/connectors/text_files/connector.py b/selfie/connectors/text_files/connector.py
index d54c8fd..b8fdbb5 100644
--- a/selfie/connectors/text_files/connector.py
+++ b/selfie/connectors/text_files/connector.py
@@ -7,7 +7,7 @@
from selfie.database import BaseModel, DataManager
from selfie.embeddings import EmbeddingDocumentModel
from selfie.types.documents import DocumentDTO
-from selfie.utils import data_uri_to_string
+from selfie.utils import data_uri_to_dict
class TextFilesConfiguration(BaseModel):
@@ -25,10 +25,10 @@ def load_document(self, configuration: dict[str, Any]) -> List[DocumentDTO]:
return [
DocumentDTO(
- content=data_uri_to_string(data_uri),
- content_type="text/plain",
- name="todo",
- size=len(data_uri_to_string(data_uri).encode('utf-8'))
+ content=(parsed := data_uri_to_dict(data_uri))['content'],
+ content_type=parsed['content_type'],
+ name=parsed['name'],
+ size=len(parsed['content'].encode('utf-8'))
)
for data_uri in config.files
]
diff --git a/selfie/connectors/whatsapp/connector.py b/selfie/connectors/whatsapp/connector.py
index 84f5246..9946214 100644
--- a/selfie/connectors/whatsapp/connector.py
+++ b/selfie/connectors/whatsapp/connector.py
@@ -6,7 +6,7 @@
from selfie.embeddings import EmbeddingDocumentModel, DataIndex
from selfie.parsers.chat import ChatFileParser
from selfie.types.documents import DocumentDTO
-from selfie.utils import data_uri_to_string
+from selfie.utils import data_uri_to_dict
class WhatsAppConfiguration(BaseModel):
@@ -24,10 +24,10 @@ def load_document(self, configuration: dict[str, Any]) -> List[DocumentDTO]:
return [
DocumentDTO(
- content=(content := data_uri_to_string(data_uri)),
- content_type="text/plain",
- name="todo",
- size=len(content.encode('utf-8'))
+ content=(parsed := data_uri_to_dict(data_uri))['content'],
+ content_type=parsed['content_type'],
+ name=parsed['name'],
+ size=len(parsed['content'].encode('utf-8'))
)
for data_uri in config.files
]
diff --git a/selfie/database/__init__.py b/selfie/database/__init__.py
index a2ed78e..433b4aa 100644
--- a/selfie/database/__init__.py
+++ b/selfie/database/__init__.py
@@ -116,17 +116,19 @@ def add_document_connection(
# def remove_data_source(self, source_id: int):
# DataSource.get_by_id(source_id).delete_instance()
- async def remove_document(self, document_id: int, delete_indexed_data: bool = True):
- with self.db.atomic():
- document = DocumentModel.get_by_id(document_id)
- if document is None:
- raise ValueError(f"No document found with ID {document_id}")
+ async def remove_documents(self, document_ids: List[int], delete_indexed_data: bool = True):
+ if delete_indexed_data:
+ await DataIndex("n/a").delete_documents_with_source_documents(document_ids)
- if delete_indexed_data:
- index = DataIndex("n/a")
- await index.delete_documents_with_source_documents([document.document_connection.id])
+ try:
+ with self.db.atomic():
+ DocumentModel.delete().where(DocumentModel.id.in_(document_ids)).execute()
+ except Exception as e:
+ logger.error(f"Error removing documents, but indexed data was removed: {e}")
+ raise e
- document.delete_instance()
+ async def remove_document(self, document_id: int, delete_indexed_data: bool = True):
+ return await self.remove_documents([document_id], delete_indexed_data)
async def remove_document_connection(self, document_connection_id: int, delete_documents: bool = True,
delete_indexed_data: bool = True):
diff --git a/selfie/parsers/chat/chatgpt.py b/selfie/parsers/chat/chatgpt.py
index 2357faa..5cc54ae 100644
--- a/selfie/parsers/chat/chatgpt.py
+++ b/selfie/parsers/chat/chatgpt.py
@@ -65,7 +65,7 @@ def extract_conversations(self, data: List[ChatGPTConversation]) -> ShareGPTConv
data (List[ChatGPTConversation]): The list of parsed JSON data
Returns:
- List[dict]: A list of conversation dictionaries
+ ShareGPTConversation: A conversation model containing the parsed chat data.
"""
conversations = []
for conversation in data:
diff --git a/selfie/utils.py b/selfie/utils.py
index e7ceb2e..179952e 100644
--- a/selfie/utils.py
+++ b/selfie/utils.py
@@ -2,13 +2,23 @@
from io import BytesIO
-def data_uri_to_string(data_uri):
+def data_uri_to_dict(data_uri):
metadata, encoded = data_uri.split(',', 1)
- data = base64.b64decode(encoded)
- mime_type = metadata.split(';')[0].split(':')[1]
- with BytesIO(data) as buffer:
- content = buffer.read()
- return content.decode('utf-8')
+ metadata = metadata.split(';')
+ mime_type = metadata[0].split(':')[1]
+ attributes = {}
+ for attr in metadata[1:]:
+ if "=" in attr:
+ key, value = attr.split('=')
+ attributes[key] = value
+ else:
+ attributes[attr] = None
+
+ return {
+ "content": base64.b64decode(encoded).decode('utf-8'),
+ "content_type": mime_type,
+ **attributes
+ }
def check_nested(obj, *keys):