From 2741f7be114b00a2d1fcc3475f4c77f12424d620 Mon Sep 17 00:00:00 2001 From: Daishan Peng Date: Mon, 21 Oct 2024 13:56:07 -0700 Subject: [PATCH] Address comments Signed-off-by: Daishan Peng --- pkg/api/handlers/agent.go | 8 +- .../handlers/knowledge/knowledge.go | 11 +- .../handlers/uploads/remoteknowledgesource.go | 1 - pkg/controller/routes.go | 1 - pkg/knowledge/knowledge.go | 4 +- .../knowledge/AgentKnowledgePanel.tsx | 152 ++++++++++++------ .../app/components/knowledge/FileItem.tsx | 8 +- .../components/knowledge/FileStatusIcon.tsx | 2 +- .../knowledge/RemoteFileItemChip.tsx | 55 +++---- .../knowledge/RemoteSourceSettingModal.tsx | 56 +++---- .../components/knowledge/file/FileModal.tsx | 10 +- .../knowledge/notion/NotionModal.tsx | 16 +- .../knowledge/onedrive/AddLinkModal.tsx | 4 +- .../knowledge/onedrive/OneDriveModal.tsx | 16 +- .../knowledge/website/AddWebsiteModal.tsx | 2 +- .../knowledge/website/WebsiteModal.tsx | 6 +- ui/admin/app/lib/model/knowledge.ts | 10 +- 17 files changed, 210 insertions(+), 152 deletions(-) diff --git a/pkg/api/handlers/agent.go b/pkg/api/handlers/agent.go index 6cdcbaff..d79aac92 100644 --- a/pkg/api/handlers/agent.go +++ b/pkg/api/handlers/agent.go @@ -200,9 +200,11 @@ func (a *AgentHandler) ApproveKnowledgeFile(req api.Context) error { return err } - file.Spec.Approved = &[]bool{body.Approve}[0] - - return req.Storage.Update(req.Context(), &file) + if file.Spec.Approved == nil || *file.Spec.Approved != body.Approve { + file.Spec.Approved = &body.Approve + return req.Storage.Update(req.Context(), &file) + } + return nil } func (a *AgentHandler) DeleteKnowledge(req api.Context) error { diff --git a/pkg/controller/handlers/knowledge/knowledge.go b/pkg/controller/handlers/knowledge/knowledge.go index 90333083..2bf38f0a 100644 --- a/pkg/controller/handlers/knowledge/knowledge.go +++ b/pkg/controller/handlers/knowledge/knowledge.go @@ -303,15 +303,6 @@ func compileFileStatuses(ctx context.Context, client kclient.Client, ws *v1.Work return final, errors.Join(errs...) } -func (a *Handler) BindWorkspace(req router.Request, resp router.Response) error { - kFile := req.Object.(*v1.KnowledgeFile) - // force to associate with workspace so that it can reenqueue workspace when file are changed - if err := req.Get(&v1.Workspace{}, kFile.Namespace, kFile.Spec.WorkspaceName); err != nil { - return err - } - return nil -} - func (a *Handler) CleanupFile(req router.Request, resp router.Response) error { kFile := req.Object.(*v1.KnowledgeFile) @@ -327,7 +318,7 @@ func (a *Handler) CleanupFile(req router.Request, resp router.Response) error { return err } - if _, err := a.ingester.DeleteKnowledgeFiles(req.Ctx, kFile.Namespace, filepath.Join(workspace.GetDir(ws.Status.WorkspaceID), kFile.Spec.FileName)); err != nil { + if _, err := a.ingester.DeleteKnowledgeFiles(req.Ctx, kFile.Namespace, filepath.Join(workspace.GetDir(ws.Status.WorkspaceID), kFile.Spec.FileName), ws.Spec.KnowledgeSetName); err != nil { return err } diff --git a/pkg/controller/handlers/uploads/remoteknowledgesource.go b/pkg/controller/handlers/uploads/remoteknowledgesource.go index f00ab68e..9d33ff57 100644 --- a/pkg/controller/handlers/uploads/remoteknowledgesource.go +++ b/pkg/controller/handlers/uploads/remoteknowledgesource.go @@ -367,7 +367,6 @@ func compileKnowledgeFiles(ctx context.Context, c client.Client, } else if err != nil { errs = append(errs, err) } - } if len(errs) > 0 { diff --git a/pkg/controller/routes.go b/pkg/controller/routes.go index 8739291e..7aa5ca64 100644 --- a/pkg/controller/routes.go +++ b/pkg/controller/routes.go @@ -80,7 +80,6 @@ func (c *Controller) setupRoutes() error { // Knowledge files root.Type(&v1.KnowledgeFile{}).HandlerFunc(cleanup.Cleanup) - root.Type(&v1.KnowledgeFile{}).HandlerFunc(knowledge.BindWorkspace) root.Type(&v1.KnowledgeFile{}).FinalizeFunc(v1.KnowledgeFileFinalizer, knowledge.CleanupFile) // Workspaces diff --git a/pkg/knowledge/knowledge.go b/pkg/knowledge/knowledge.go index cf5b36d5..5774f861 100644 --- a/pkg/knowledge/knowledge.go +++ b/pkg/knowledge/knowledge.go @@ -31,13 +31,15 @@ func (i *Ingester) IngestKnowledge(ctx context.Context, namespace, knowledgeSetN ) } -func (i *Ingester) DeleteKnowledgeFiles(ctx context.Context, namespace, knowledgeFilePath string) (*invoke.Response, error) { +func (i *Ingester) DeleteKnowledgeFiles(ctx context.Context, namespace, knowledgeFilePath string, knowledgeSetName string) (*invoke.Response, error) { return i.invoker.SystemAction( ctx, "ingest-delete-file-", namespace, system.KnowledgeDeleteFileTool, knowledgeFilePath, + "GPTSCRIPT_DATASET="+knowledgeSetName, + "KNOW_JSON=true", ) } diff --git a/ui/admin/app/components/knowledge/AgentKnowledgePanel.tsx b/ui/admin/app/components/knowledge/AgentKnowledgePanel.tsx index 677b3c85..c874d51e 100644 --- a/ui/admin/app/components/knowledge/AgentKnowledgePanel.tsx +++ b/ui/admin/app/components/knowledge/AgentKnowledgePanel.tsx @@ -1,15 +1,17 @@ import { Globe, SettingsIcon, UploadIcon } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; -import useSWR from "swr"; +import useSWR, { SWRResponse } from "swr"; import { IngestionStatus, KnowledgeFile, RemoteKnowledgeSourceType, + getIngestedFilesCount, getIngestionStatus, } from "~/lib/model/knowledge"; import { ApiRoutes } from "~/lib/routers/apiRoutes"; import { KnowledgeService } from "~/lib/service/api/knowledgeService"; +import { assetUrl } from "~/lib/utils"; import { Button } from "~/components/ui/button"; @@ -26,7 +28,7 @@ export function AgentKnowledgePanel({ agentId }: { agentId: string }) { const [isNotionModalOpen, setIsNotionModalOpen] = useState(false); const [isWebsiteModalOpen, setIsWebsiteModalOpen] = useState(false); - const getKnowledgeFiles = useSWR( + const getKnowledgeFiles: SWRResponse = useSWR( KnowledgeService.getKnowledgeForAgent.key(agentId), ({ agentId }) => KnowledgeService.getKnowledgeForAgent(agentId).then((items) => @@ -221,72 +223,132 @@ export function AgentKnowledgePanel({ agentId }: { agentId: string }) { } }; + const notionFiles = knowledge.filter( + (item) => item.remoteKnowledgeSourceType === "notion" + ); + const onedriveFiles = knowledge.filter( + (item) => item.remoteKnowledgeSourceType === "onedrive" + ); + const websiteFiles = knowledge.filter( + (item) => item.remoteKnowledgeSourceType === "website" + ); + const localFiles = knowledge.filter( + (item) => !item.remoteKnowledgeSourceType + ); + return (
-
+
Files
- +
+
+ {getIngestedFilesCount(localFiles) > 0 && ( + + {getIngestedFilesCount(localFiles)}{" "} + {getIngestedFilesCount(localFiles) === 1 + ? "file" + : "files"}{" "} + ingested + + )} +
+ +
-
+
- Notion logo + Notion logo Notion
- +
+ {getIngestedFilesCount(notionFiles) > 0 && ( + + {getIngestedFilesCount(notionFiles)}{" "} + {getIngestedFilesCount(notionFiles) === 1 + ? "file" + : "files"}{" "} + ingested + + )} + +
-
+
- OneDrive logo + OneDrive logo OneDrive
- +
+ {getIngestedFilesCount(onedriveFiles) > 0 && ( + + {getIngestedFilesCount(onedriveFiles)}{" "} + {getIngestedFilesCount(onedriveFiles) === 1 + ? "file" + : "files"}{" "} + ingested + + )} + +
-
+
Website
- +
+ {getIngestedFilesCount(websiteFiles) > 0 && ( + + {getIngestedFilesCount(websiteFiles)}{" "} + {getIngestedFilesCount(websiteFiles) === 1 + ? "file" + : "files"}{" "} + ingested + + )} + +
!item.remoteKnowledgeSourceType - )} + knowledge={localFiles} getKnowledgeFiles={getKnowledgeFiles} /> item.remoteKnowledgeSourceType === "notion" - )} + knowledgeFiles={notionFiles} handleRemoteKnowledgeSourceSync={ handleRemoteKnowledgeSourceSync } @@ -308,9 +368,7 @@ export function AgentKnowledgePanel({ agentId }: { agentId: string }) { onOpenChange={setIsOnedriveModalOpen} remoteKnowledgeSources={remoteKnowledgeSources} startPolling={startPolling} - knowledgeFiles={knowledge.filter( - (item) => item.remoteKnowledgeSourceType === "onedrive" - )} + knowledgeFiles={onedriveFiles} handleRemoteKnowledgeSourceSync={ handleRemoteKnowledgeSourceSync } @@ -321,9 +379,7 @@ export function AgentKnowledgePanel({ agentId }: { agentId: string }) { onOpenChange={setIsWebsiteModalOpen} remoteKnowledgeSources={remoteKnowledgeSources} startPolling={startPolling} - knowledgeFiles={knowledge.filter( - (item) => item.remoteKnowledgeSourceType === "website" - )} + knowledgeFiles={websiteFiles} handleRemoteKnowledgeSourceSync={ handleRemoteKnowledgeSourceSync } diff --git a/ui/admin/app/components/knowledge/FileItem.tsx b/ui/admin/app/components/knowledge/FileItem.tsx index 64a1f3f3..f8aad584 100644 --- a/ui/admin/app/components/knowledge/FileItem.tsx +++ b/ui/admin/app/components/knowledge/FileItem.tsx @@ -64,16 +64,16 @@ function FileItem({
{isApproved ? ( - // eslint-disable-next-line -
{ setIsApproved(false); approveFile(file, false); }} > -
+ ) : ( - )} +
+ {file.approved ? ( + + ) : ( + + )} +
diff --git a/ui/admin/app/components/knowledge/RemoteSourceSettingModal.tsx b/ui/admin/app/components/knowledge/RemoteSourceSettingModal.tsx index 4078b403..59a37d31 100644 --- a/ui/admin/app/components/knowledge/RemoteSourceSettingModal.tsx +++ b/ui/admin/app/components/knowledge/RemoteSourceSettingModal.tsx @@ -13,6 +13,7 @@ import { } from "~/components/ui/dialog"; import { Input } from "~/components/ui/input"; import { Switch } from "~/components/ui/switch"; +import { useAsync } from "~/hooks/useAsync"; type RemoteSourceSettingModalProps = { agentId: string; @@ -27,38 +28,36 @@ const RemoteSourceSettingModal: React.FC = ({ onOpenChange, remoteKnowledgeSource, }) => { - const [autoApprove, setAutoApprove] = useState( - remoteKnowledgeSource.autoApprove || false - ); + const [autoApprove, setAutoApprove] = useState(false); - const [syncSchedule, setSyncSchedule] = useState( - remoteKnowledgeSource.syncSchedule || "" - ); + useEffect(() => { + setAutoApprove(remoteKnowledgeSource?.autoApprove || false); + }, [remoteKnowledgeSource]); + + const [syncSchedule, setSyncSchedule] = useState(""); useEffect(() => { - setSyncSchedule(remoteKnowledgeSource.syncSchedule || ""); + setSyncSchedule(remoteKnowledgeSource?.syncSchedule || ""); }, [remoteKnowledgeSource]); - const handleSave = async () => { - try { - await KnowledgeService.updateRemoteKnowledgeSource( - agentId, - remoteKnowledgeSource.id, - { - ...remoteKnowledgeSource, - syncSchedule, - autoApprove, - } - ); - onOpenChange(false); - } catch (error) { - console.error("Failed to update sync schedule:", error); - } + const updateRemoteKnowledgeSource = async () => { + await KnowledgeService.updateRemoteKnowledgeSource( + agentId, + remoteKnowledgeSource.id, + { + ...remoteKnowledgeSource, + syncSchedule, + autoApprove, + } + ); + onOpenChange(false); }; + const handleSave = useAsync(updateRemoteKnowledgeSource); + return ( - + Update Source Settings @@ -91,14 +90,9 @@ const RemoteSourceSettingModal: React.FC = ({ id="autoApprove" className="mr-2" checked={autoApprove} - onChange={() => setAutoApprove(!autoApprove)} + onClick={() => setAutoApprove((prev) => !prev)} /> - + Include new pages

If enabled, new pages will be added to the knowledge @@ -106,7 +100,7 @@ const RemoteSourceSettingModal: React.FC = ({

- + diff --git a/ui/admin/app/components/knowledge/file/FileModal.tsx b/ui/admin/app/components/knowledge/file/FileModal.tsx index 96b25990..d55bfe5e 100644 --- a/ui/admin/app/components/knowledge/file/FileModal.tsx +++ b/ui/admin/app/components/knowledge/file/FileModal.tsx @@ -1,5 +1,6 @@ import { UploadIcon } from "lucide-react"; import { useCallback, useRef } from "react"; +import { SWRResponse } from "swr"; import { IngestionStatus, KnowledgeFile } from "~/lib/model/knowledge"; import { KnowledgeService } from "~/lib/service/api/knowledgeService"; @@ -23,8 +24,7 @@ import IngestionStatusComponent from "../IngestionStatus"; interface FileModalProps { agentId: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - getKnowledgeFiles: any; + getKnowledgeFiles: SWRResponse; isOpen: boolean; onOpenChange: (open: boolean) => void; startPolling: () => void; @@ -51,11 +51,13 @@ function FileModal({ // Revalidating here would cause knowledge to be refreshed // for each file being uploaded, which is not desirable. const newItem: KnowledgeFile = { + id: "", fileName: file.name, agentID: agentId, // set ingestion status to starting to ensure polling is enabled ingestionStatus: { status: IngestionStatus.Queued }, fileDetails: {}, + approved: true, }; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -100,7 +102,7 @@ function FileModal({ await KnowledgeService.deleteKnowledgeFromAgent(agentId, item.fileName); // optomistic update without cache revalidation - getKnowledgeFiles.mutate((prev: KnowledgeFile[]) => + getKnowledgeFiles.mutate((prev: KnowledgeFile[] | undefined) => prev?.filter((prevItem) => prevItem.fileName !== item.fileName) ); }); @@ -119,7 +121,7 @@ function FileModal({ className="mr-2" onClick={() => fileInputRef.current?.click()} > - + diff --git a/ui/admin/app/components/knowledge/notion/NotionModal.tsx b/ui/admin/app/components/knowledge/notion/NotionModal.tsx index 154e7831..8ff83524 100644 --- a/ui/admin/app/components/knowledge/notion/NotionModal.tsx +++ b/ui/admin/app/components/knowledge/notion/NotionModal.tsx @@ -7,6 +7,7 @@ import { RemoteKnowledgeSourceType, } from "~/lib/model/knowledge"; import { KnowledgeService } from "~/lib/service/api/knowledgeService"; +import { assetUrl } from "~/lib/utils"; import RemoteFileItemChip from "~/components/knowledge/RemoteFileItemChip"; import RemoteKnowledgeSourceStatus from "~/components/knowledge/RemoteKnowledgeSourceStatus"; @@ -54,11 +55,7 @@ export const NotionModal: FC = ({ const handleApproveAll = async () => { for (const file of knowledgeFiles) { - await KnowledgeService.approveKnowledgeFile( - agentId, - file.id!, - true - ); + await KnowledgeService.approveKnowledgeFile(agentId, file.id, true); } startPolling(); }; @@ -73,7 +70,10 @@ export const NotionModal: FC = ({
- Notion logo + Notion logo Notion
@@ -100,7 +100,7 @@ export const NotionModal: FC = ({
- +
{knowledgeFiles.map((item) => ( = ({ {loading ? ( ) : ( - "Add All" + "Ingest All" )}