Enhance: add more states to integration, control ingestion (#163)

Signed-off-by: Daishan Peng <[email protected]>

apiclient/types/knowledge.go

@@ -17,6 +17,7 @@ type FileDetails struct {
 	FilePath  string `json:"filePath,omitempty"`
 	URL       string `json:"url,omitempty"`
 	UpdatedAt string `json:"updatedAt,omitempty"`
+	Checksum  string `json:"checksum,omitempty"`
 }
 
 type IngestionStatus struct {

apiclient/types/remoteKnowledgeSource.go

@@ -28,11 +28,12 @@ type RemoteKnowledgeSourceManifest struct {
 type RemoteKnowledgeSourceList List[RemoteKnowledgeSource]
 
 type RemoteKnowledgeSourceInput struct {
-	SourceType            RemoteKnowledgeSourceType `json:"sourceType,omitempty"`
-	Exclude               []string                  `json:"exclude,omitempty"`
-	OneDriveConfig        *OneDriveConfig           `json:"onedriveConfig,omitempty"`
-	NotionConfig          *NotionConfig             `json:"notionConfig,omitempty"`
-	WebsiteCrawlingConfig *WebsiteCrawlingConfig    `json:"websiteCrawlingConfig,omitempty"`
+	DisableIngestionAfterSync bool                      `json:"disableIngestionAfterSync,omitempty"`
+	SourceType                RemoteKnowledgeSourceType `json:"sourceType,omitempty"`
+	Exclude                   []string                  `json:"exclude,omitempty"`
+	OneDriveConfig            *OneDriveConfig           `json:"onedriveConfig,omitempty"`
+	NotionConfig              *NotionConfig             `json:"notionConfig,omitempty"`
+	WebsiteCrawlingConfig     *WebsiteCrawlingConfig    `json:"websiteCrawlingConfig,omitempty"`
 }
 
 type OneDriveConfig struct {
@@ -54,19 +55,28 @@ type RemoteKnowledgeSourceState struct {
 }
 
 type OneDriveLinksConnectorState struct {
-	Folders FolderSet `json:"folders,omitempty"`
+	Folders FolderSet            `json:"folders,omitempty"`
+	Files   map[string]FileState `json:"files,omitempty"`
+}
+
+type FileState struct {
+	FileName   string `json:"fileName,omitempty"`
+	FolderPath string `json:"folderPath,omitempty"`
+	URL        string `json:"url,omitempty"`
 }
 
 type NotionConnectorState struct {
 	Pages map[string]NotionPage `json:"pages,omitempty"`
 }
 
 type NotionPage struct {
-	URL   string `json:"url,omitempty"`
-	Title string `json:"title,omitempty"`
+	URL        string `json:"url,omitempty"`
+	Title      string `json:"title,omitempty"`
+	FolderPath string `json:"folderPath,omitempty"`
 }
 
 type WebsiteCrawlingConnectorState struct {
 	ScrapeJobIds map[string]string `json:"scrapeJobIds"`
 	Folders      FolderSet         `json:"folders"`
+	Pages        map[string]Item   `json:"pages"`
 }

pkg/api/handlers/remoteknowledgesource.go

@@ -3,6 +3,7 @@ package handlers
 import (
 	"fmt"
 	"net/http"
+	"reflect"
 
 	"github.com/otto8-ai/otto8/apiclient/types"
 	"github.com/otto8-ai/otto8/pkg/api"
@@ -71,14 +72,17 @@ func updateRemoteKnowledgeSource(req api.Context, linksID, parentName string, pa
 		return fmt.Errorf("failed to decode request body: %w", err)
 	}
 
+	configChanged := checkConfigChanged(input, remoteKnowledgeSource)
 	remoteKnowledgeSource.Spec.RemoteKnowledgeSourceInput = input
 
 	if err := req.Update(&remoteKnowledgeSource); err != nil {
 		return fmt.Errorf("failed to update RemoteKnowledgeSource: %w", err)
 	}
 
-	if err := createSyncRequest(req, remoteKnowledgeSource); err != nil {
-		return fmt.Errorf("failed to create sync request: %w", err)
+	if configChanged {
+		if err := createSyncRequest(req, remoteKnowledgeSource); err != nil {
+			return fmt.Errorf("failed to create sync request: %w", err)
+		}
 	}
 
 	req.WriteHeader(http.StatusNoContent)
@@ -179,3 +183,19 @@ func convertRemoteKnowledgeSource(remoteKnowledgeSource v1.RemoteKnowledgeSource
 		Error:                         remoteKnowledgeSource.Status.Error,
 	}
 }
+
+func checkConfigChanged(input types.RemoteKnowledgeSourceInput, remoteKnowledgeSource v1.RemoteKnowledgeSource) bool {
+	if input.OneDriveConfig != nil && remoteKnowledgeSource.Spec.OneDriveConfig != nil {
+		return !reflect.DeepEqual(*input.OneDriveConfig, *remoteKnowledgeSource.Spec.OneDriveConfig)
+	}
+
+	if input.NotionConfig != nil && remoteKnowledgeSource.Spec.NotionConfig != nil {
+		return !reflect.DeepEqual(*input.NotionConfig, *remoteKnowledgeSource.Spec.NotionConfig)
+	}
+
+	if input.WebsiteCrawlingConfig != nil && remoteKnowledgeSource.Spec.WebsiteCrawlingConfig != nil {
+		return !reflect.DeepEqual(*input.WebsiteCrawlingConfig, *remoteKnowledgeSource.Spec.WebsiteCrawlingConfig)
+	}
+
+	return true
+}

pkg/controller/handlers/uploads/remoteknowledgesource.go

@@ -307,22 +307,24 @@ func (u *UploadHandler) HandleUploadRun(req router.Request, resp router.Response
 	remoteKnowledgeSource.Status.RunName = ""
 
 	// Create object to re-ingest knowledge
-	resp.Objects(
-		&v1.IngestKnowledgeRequest{
-			ObjectMeta: metav1.ObjectMeta{
-				GenerateName: system.IngestRequestPrefix,
-				Namespace:    req.Namespace,
-				Annotations: map[string]string{
-					// Don't prune because the cleanup handler will do that.
-					apply.AnnotationPrune: "false",
+	if !remoteKnowledgeSource.Spec.DisableIngestionAfterSync {
+		resp.Objects(
+			&v1.IngestKnowledgeRequest{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: system.IngestRequestPrefix,
+					Namespace:    req.Namespace,
+					Annotations: map[string]string{
+						// Don't prune because the cleanup handler will do that.
+						apply.AnnotationPrune: "false",
+					},
+				},
+				Spec: v1.IngestKnowledgeRequestSpec{
+					WorkspaceName: ws.Name,
+					HasKnowledge:  len(fileMetadata) > 0,
 				},
 			},
-			Spec: v1.IngestKnowledgeRequestSpec{
-				WorkspaceName: ws.Name,
-				HasKnowledge:  len(fileMetadata) > 0,
-			},
-		},
-	)
+		)
+	}
 
 	return nil
 }