Skip to content

Commit

Permalink
feat: add KnowledgeFile CR
Browse files Browse the repository at this point in the history 
Signed-off-by: Donnie Adams <[email protected]>
  • Loading branch information
@thedadams
thedadams committed Sep 23, 2024
1 parent a32885f commit 6ae38dc
Show file tree
Hide file tree
Showing 24 changed files with 1,123 additions and 217 deletions.
27 changes: 11 additions & 16 deletions pkg/api/handlers/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ func (a *AgentHandler) Create(req api.Context) error {
ObjectMeta: metav1.ObjectMeta{
GenerateName: system.AgentPrefix,
Namespace: req.Namespace(),
Finalizers: []string{v1.AgentFinalizer},
},
Spec: v1.AgentSpec{
Manifest: manifest,
Expand Down Expand Up @@ -146,41 +145,37 @@ func (a *AgentHandler) UploadFile(req api.Context) error {
return fmt.Errorf("failed to get agent with id %s: %w", id, err)
}

return uploadFile(req.Context(), req, a.workspaceClient, agent.Status.Workspace.WorkspaceID)
if err := uploadFile(req.Context(), req, a.workspaceClient, agent.Status.Workspace.WorkspaceID); err != nil {
return err
}

req.WriteHeader(http.StatusCreated)
return nil
}

func (a *AgentHandler) DeleteFile(req api.Context) error {
var (
id = req.PathValue("id")
filename = req.PathValue("file")
agent v1.Agent
id = req.PathValue("id")
agent v1.Agent
)

if err := req.Get(&agent, id); err != nil {
return fmt.Errorf("failed to get agent with id %s: %w", id, err)
}

return deleteFile(req.Context(), req, a.workspaceClient, agent.Status.Workspace.WorkspaceID, filename)
return deleteFile(req.Context(), req, a.workspaceClient, agent.Status.Workspace.WorkspaceID)
}

func (a *AgentHandler) Knowledge(req api.Context) error {
var (
id = req.PathValue("id")
agent v1.Agent
)
if err := req.Get(&agent, id); err != nil {
return fmt.Errorf("failed to get agent with id %s: %w", id, err)
}

return listFiles(req.Context(), req, a.workspaceClient, agent.Status.KnowledgeWorkspace.KnowledgeWorkspaceID)
return listKnowledgeFiles(req, new(v1.Agent))
}

func (a *AgentHandler) UploadKnowledge(req api.Context) error {
return uploadKnowledge(req, a.workspaceClient, req.PathValue("id"), new(v1.Agent))
}

func (a *AgentHandler) DeleteKnowledge(req api.Context) error {
return deleteKnowledge(req, a.workspaceClient, req.PathValue("file"), req.PathValue("id"), new(v1.Agent))
return deleteKnowledge(req, req.PathValue("file"), req.PathValue("id"), new(v1.Agent))
}

func (a *AgentHandler) IngestKnowledge(req api.Context) error {
Expand Down
95 changes: 78 additions & 17 deletions pkg/api/handlers/files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,24 @@ package handlers

import (
"context"
"errors"
"fmt"
"io"
"net/http"
"path/filepath"
"strings"

"github.com/gptscript-ai/otto/pkg/api"
"github.com/gptscript-ai/otto/pkg/api/types"
"github.com/gptscript-ai/otto/pkg/knowledge"
v1 "github.com/gptscript-ai/otto/pkg/storage/apis/otto.gptscript.ai/v1"
"github.com/gptscript-ai/otto/pkg/storage/selectors"
"github.com/gptscript-ai/otto/pkg/workspace"
wclient "github.com/thedadams/workspace-provider/pkg/client"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"sigs.k8s.io/controller-runtime/pkg/client"
)

func listFiles(ctx context.Context, req api.Context, wc *wclient.Client, workspaceID string) error {
Expand All @@ -21,19 +31,64 @@ func listFiles(ctx context.Context, req api.Context, wc *wclient.Client, workspa
return req.Write(types.FileList{Items: files})
}

func listKnowledgeFiles(req api.Context, parentObj knowledge.Knowledgeable) error {
if err := req.Get(parentObj, req.PathValue("id")); err != nil {
return fmt.Errorf("failed to get the parent object: %w", err)
}

var files v1.KnowledgeFileList
if err := req.Storage.List(req.Context(), &files, &client.ListOptions{
FieldSelector: fields.SelectorFromSet(selectors.RemoveEmpty(map[string]string{
"spec.agentName": parentObj.AgentName(),
"spec.workflowName": parentObj.WorkflowName(),
"spec.threadName": parentObj.ThreadName(),
})),
Namespace: parentObj.GetNamespace(),
}); err != nil {
return err
}

return req.Write(files)
}

func uploadKnowledge(req api.Context, workspaceClient *wclient.Client, parentName string, toUpdate knowledge.Knowledgeable) error {
if err := req.Get(toUpdate, parentName); err != nil {
return fmt.Errorf("failed to get parent with id %s: %w", req.PathValue("id"), err)
}

status := toUpdate.GetKnowledgeWorkspaceStatus()
status := toUpdate.KnowledgeWorkspaceStatus()
if err := uploadFile(req.Context(), req, workspaceClient, status.KnowledgeWorkspaceID); err != nil {
return err
}

filename := req.PathValue("file")
file := &v1.KnowledgeFile{
ObjectMeta: metav1.ObjectMeta{
Name: v1.ObjectNameFromAbsolutePath(
filepath.Join(workspace.GetDir(status.KnowledgeWorkspaceID), filename),
),
Namespace: toUpdate.GetNamespace(),
},
Spec: v1.KnowledgeFileSpec{
FileName: filename,
AgentName: toUpdate.AgentName(),
WorkflowName: toUpdate.WorkflowName(),
ThreadName: toUpdate.ThreadName(),
},
}

if err := req.Storage.Create(req.Context(), file); err != nil && !apierrors.IsAlreadyExists(err) {
_ = deleteFile(req.Context(), req, workspaceClient, status.KnowledgeWorkspaceID)
return err
}

status.KnowledgeGeneration++
status.HasKnowledge = true
return req.Storage.Status().Update(req.Context(), toUpdate)
if err := req.Storage.Status().Update(req.Context(), toUpdate); err != nil {
return err
}

return req.Write(file)
}

func uploadFile(ctx context.Context, req api.Context, wc *wclient.Client, workspaceID string) error {
Expand All @@ -52,32 +107,38 @@ func uploadFile(ctx context.Context, req api.Context, wc *wclient.Client, worksp
return fmt.Errorf("failed to write file %q to workspace %q: %w", file, workspaceID, err)
}

req.WriteHeader(http.StatusCreated)

return nil
}

func deleteKnowledge(req api.Context, workspaceClient *wclient.Client, filename, parentName string, toUpdate knowledge.Knowledgeable) error {
func deleteKnowledge(req api.Context, filename, parentName string, toUpdate knowledge.Knowledgeable) error {
if err := req.Get(toUpdate, parentName); err != nil {
return fmt.Errorf("failed to get parent with id %s: %w", parentName, err)
}

status := toUpdate.GetKnowledgeWorkspaceStatus()
if err := deleteFile(req.Context(), req, workspaceClient, status.KnowledgeWorkspaceID, filename); err != nil {
fileObjectName := v1.ObjectNameFromAbsolutePath(
filepath.Join(workspace.GetDir(toUpdate.KnowledgeWorkspaceStatus().KnowledgeWorkspaceID), filename),
)

if err := req.Storage.Delete(req.Context(), &v1.KnowledgeFile{
ObjectMeta: metav1.ObjectMeta{
Namespace: toUpdate.GetNamespace(),
Name: fileObjectName,
},
}); err != nil {
var apiErr *apierrors.StatusError
if errors.As(err, &apiErr) {
apiErr.ErrStatus.Details.Name = filename
apiErr.ErrStatus.Message = strings.ReplaceAll(apiErr.ErrStatus.Message, fileObjectName, filename)
}
return err
}

files, err := workspaceClient.Ls(req.Context(), status.KnowledgeWorkspaceID)
if err != nil {
return fmt.Errorf("failed to list files in workspace %s: %w", status.KnowledgeWorkspaceID, err)
}

status.KnowledgeGeneration++
status.HasKnowledge = len(files) > 0
return req.Storage.Status().Update(req.Context(), toUpdate)
req.WriteHeader(http.StatusNoContent)
return nil
}

func deleteFile(ctx context.Context, req api.Context, wc *wclient.Client, workspaceID, filename string) error {
func deleteFile(ctx context.Context, req api.Context, wc *wclient.Client, workspaceID string) error {
filename := req.PathValue("file")
if err := wc.DeleteFile(ctx, workspaceID, filename); err != nil {
return fmt.Errorf("failed to delete file %q from workspace %q: %w", filename, workspaceID, err)
}
Expand All @@ -92,7 +153,7 @@ func ingestKnowledge(req api.Context, workspaceClient *wclient.Client, parentNam
return fmt.Errorf("failed to get parent with id %s: %w", req.PathValue("id"), err)
}

status := toUpdate.GetKnowledgeWorkspaceStatus()
status := toUpdate.KnowledgeWorkspaceStatus()
files, err := workspaceClient.Ls(req.Context(), status.KnowledgeWorkspaceID)
if err != nil {
return err
Expand Down
27 changes: 12 additions & 15 deletions pkg/api/handlers/threads.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package handlers

import (
"fmt"
"net/http"

"github.com/gptscript-ai/otto/pkg/api"
"github.com/gptscript-ai/otto/pkg/api/types"
Expand Down Expand Up @@ -136,41 +137,37 @@ func (a *ThreadHandler) UploadFile(req api.Context) error {
return fmt.Errorf("failed to get thread with id %s: %w", id, err)
}

return uploadFile(req.Context(), req, a.workspaceClient, thread.Spec.WorkspaceID)
if err := uploadFile(req.Context(), req, a.workspaceClient, thread.Spec.WorkspaceID); err != nil {
return err
}

req.WriteHeader(http.StatusCreated)
return nil
}

func (a *ThreadHandler) DeleteFile(req api.Context) error {
var (
id = req.PathValue("id")
filename = req.PathValue("file")
thread v1.Thread
id = req.PathValue("id")
thread v1.Thread
)

if err := req.Get(&thread, id); err != nil {
return fmt.Errorf("failed to get thread with id %s: %w", id, err)
}

return deleteFile(req.Context(), req, a.workspaceClient, thread.Spec.WorkspaceID, filename)
return deleteFile(req.Context(), req, a.workspaceClient, thread.Spec.WorkspaceID)
}

func (a *ThreadHandler) Knowledge(req api.Context) error {
var (
id = req.PathValue("id")
thread v1.Thread
)
if err := req.Get(&thread, id); err != nil {
return fmt.Errorf("failed to get thread with id %s: %w", id, err)
}

return listFiles(req.Context(), req, a.workspaceClient, thread.Spec.KnowledgeWorkspaceID)
return listKnowledgeFiles(req, new(v1.Thread))
}

func (a *ThreadHandler) UploadKnowledge(req api.Context) error {
return uploadKnowledge(req, a.workspaceClient, req.PathValue("id"), new(v1.Thread))
}

func (a *ThreadHandler) DeleteKnowledge(req api.Context) error {
return deleteKnowledge(req, a.workspaceClient, req.PathValue("file"), req.PathValue("id"), new(v1.Thread))
return deleteKnowledge(req, req.PathValue("file"), req.PathValue("id"), new(v1.Thread))
}

func (a *ThreadHandler) IngestKnowledge(req api.Context) error {
Expand Down
23 changes: 9 additions & 14 deletions pkg/api/handlers/workflows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ func (a *WorkflowHandler) Create(req api.Context) error {
ObjectMeta: metav1.ObjectMeta{
GenerateName: system.WorkflowPrefix,
Namespace: req.Namespace(),
Finalizers: []string{v1.WorkflowFinalizer},
},
Spec: v1.WorkflowSpec{
Manifest: manifest,
Expand Down Expand Up @@ -150,41 +149,37 @@ func (a *WorkflowHandler) UploadFile(req api.Context) error {
return fmt.Errorf("failed to get workflow with id %s: %w", id, err)
}

return uploadFile(req.Context(), req, a.workspaceClient, workflow.Status.Workspace.WorkspaceID)
if err := uploadFile(req.Context(), req, a.workspaceClient, workflow.Status.Workspace.WorkspaceID); err != nil {
return err
}

req.WriteHeader(http.StatusCreated)
return nil
}

func (a *WorkflowHandler) DeleteFile(req api.Context) error {
var (
id = req.PathValue("id")
filename = req.PathValue("file")
workflow v1.Workflow
)

if err := req.Get(&workflow, id); err != nil {
return fmt.Errorf("failed to get workflow with id %s: %w", id, err)
}

return deleteFile(req.Context(), req, a.workspaceClient, workflow.Status.Workspace.WorkspaceID, filename)
return deleteFile(req.Context(), req, a.workspaceClient, workflow.Status.Workspace.WorkspaceID)
}

func (a *WorkflowHandler) Knowledge(req api.Context) error {
var (
id = req.PathValue("id")
workflow v1.Workflow
)
if err := req.Get(&workflow, id); err != nil {
return fmt.Errorf("failed to get workflow with id %s: %w", id, err)
}

return listFiles(req.Context(), req, a.workspaceClient, workflow.Status.KnowledgeWorkspace.KnowledgeWorkspaceID)
return listKnowledgeFiles(req, new(v1.Workflow))
}

func (a *WorkflowHandler) UploadKnowledge(req api.Context) error {
return uploadKnowledge(req, a.workspaceClient, req.PathValue("id"), new(v1.Workflow))
}

func (a *WorkflowHandler) DeleteKnowledge(req api.Context) error {
return deleteKnowledge(req, a.workspaceClient, req.PathValue("file"), req.PathValue("id"), new(v1.Workflow))
return deleteKnowledge(req, req.PathValue("file"), req.PathValue("id"), new(v1.Workflow))
}

func (a *WorkflowHandler) IngestKnowledge(req api.Context) error {
Expand Down
6 changes: 3 additions & 3 deletions pkg/api/router/router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func Router(services *services.Services) (http.Handler, error) {
mux.Handle("GET /agents/{id}/knowledge", w(agents.Knowledge))
mux.Handle("POST /agents/{id}/knowledge", w(agents.IngestKnowledge))
mux.Handle("POST /agents/{id}/knowledge/{file}", w(agents.UploadKnowledge))
mux.Handle("DELETE /agents/{id}/knowledge/{file}", w(agents.DeleteKnowledge))
mux.Handle("DELETE /agents/{id}/knowledge/{file...}", w(agents.DeleteKnowledge))

mux.Handle("POST /agents/{agent_id}/onedrive-links", w(agents.CreateOnedriveLinks))
mux.Handle("GET /agents/{agent_id}/onedrive-links", w(agents.GetOnedriveLinks))
Expand All @@ -59,7 +59,7 @@ func Router(services *services.Services) (http.Handler, error) {
mux.Handle("GET /workflows/{id}/knowledge", w(workflows.Knowledge))
mux.Handle("POST /workflows/{id}/knowledge", w(workflows.IngestKnowledge))
mux.Handle("POST /workflows/{id}/knowledge/{file}", w(workflows.UploadKnowledge))
mux.Handle("DELETE /workflows/{id}/knowledge/{file}", w(workflows.DeleteKnowledge))
mux.Handle("DELETE /workflows/{id}/knowledge/{file...}", w(workflows.DeleteKnowledge))

mux.Handle("POST /workflows/{workflow_id}/onedrive-links", w(workflows.CreateOnedriveLinks))
mux.Handle("GET /workflows/{workflow_id}/onedrive-links", w(workflows.GetOnedriveLinks))
Expand Down Expand Up @@ -87,7 +87,7 @@ func Router(services *services.Services) (http.Handler, error) {
mux.Handle("GET /threads/{id}/knowledge", w(threads.Knowledge))
mux.Handle("POST /threads/{id}/knowledge", w(threads.IngestKnowledge))
mux.Handle("POST /threads/{id}/knowledge/{file}", w(threads.UploadKnowledge))
mux.Handle("DELETE /threads/{id}/knowledge/{file}", w(threads.DeleteKnowledge))
mux.Handle("DELETE /threads/{id}/knowledge/{file...}", w(threads.DeleteKnowledge))

// Runs
mux.Handle("GET /runs", w(runs.List))
Expand Down
Loading

0 comments on commit 6ae38dc

Please sign in to comment.