add: otto blueprint

ingest.gpt

@@ -1,5 +1,7 @@
 Name: Knowledge Ingestion
 Description: Ingest content into a dataset.
 Credential: github.com/gptscript-ai/credentials/model-provider
+Params: Input: Input File
+Params: Dataset: Dataset ID
 
-#!${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool ingest --dataset ${GPTSCRIPT_DATASET} "${GPTSCRIPT_INPUT}"
+#!${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool ingest --flows-file=blueprint:otto --dataset ${DATASET} "${INPUT}"

load.gpt

@@ -2,5 +2,6 @@ Name: Knowledge Loading
 Description: Load a document, convert it to Knowledge JSON Format and store it to a file.
 Params: Input: Input File
 Params: Output: Output File
+Params: know_load_metadata: Comma-delimited key=value pairs to be added to the metadata of the loaded document.
 
 #!${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool load "${INPUT}" "${OUTPUT}"

pkg/cmd/load.go

@@ -20,7 +20,7 @@ import (
 type ClientLoad struct {
 	Loader       string            `usage:"Choose a document loader to use"`
 	OutputFormat string            `name:"format" usage:"Choose an output format" default:"structured"`
-	Metadata     map[string]string `usage:"Metadata to attach to the loaded files" env:"KNOW_LOAD_METADATA"`
+	Metadata     map[string]string `usage:"Metadata to attach to the loaded files" env:"METADATA"`
 }
 
 func (s *ClientLoad) Customize(cmd *cobra.Command) {

pkg/datastore/documentloader/documentloaders.go

@@ -155,10 +155,13 @@ func GetDocumentLoaderFunc(name string, config any) (LoaderFunc, error) {
 			return FromLangchain(lcgodocloaders.NewText(strings.NewReader(text))).Load(ctx)
 		}, nil
 	case "structured":
+		var structuredCfg structured.Structured
 		if config != nil {
-			return nil, fmt.Errorf("structured document loader does not accept configuration")
+			if err := mapstructure.Decode(config, &structuredCfg); err != nil {
+				return nil, fmt.Errorf("failed to decode structured document loader configuration: %w", err)
+			}
 		}
-		return new(structured.Structured).Load, nil
+		return structuredCfg.Load, nil
 	default:
 		return nil, fmt.Errorf("unknown document loader %q", name)
 	}

pkg/flows/config/blueprints.go

@@ -11,9 +11,13 @@ var BlueprintDefault []byte
 //go:embed blueprints/context.yaml
 var BlueprintContext []byte
 
+//go:embed blueprints/otto.yaml
+var BlueprintOtto []byte
+
 var Blueprints = map[string][]byte{
 	"default": BlueprintDefault,
 	"context": BlueprintContext,
+	"otto":    BlueprintOtto,
 }
 
 func GetBlueprint(name string) ([]byte, error) {

pkg/flows/config/blueprints/otto.yaml

@@ -0,0 +1,20 @@
+flows:
+  otto:
+    default: true
+    globals:
+      ingestion:
+        textsplitter:
+          chunkSize: 2048
+          chunkOverlap: 256
+    ingestion:
+      - filetypes: [".json"]
+        documentloader:
+          name: structured
+    retrieval:
+      retriever:
+        name: basic
+        options:
+          topK: 20
+
+
+

tool.gpt

@@ -4,4 +4,4 @@ Credential: github.com/gptscript-ai/credentials/model-provider
 Params: Datasets: The datasets to query
 Params: Query: The query to search for
 
-#!${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool retrieve --dataset="${DATASETS}" "${QUERY}"
+#!${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool retrieve --flows-file=blueprint:otto --dataset="${DATASETS}" "${QUERY}"