Skip to content

Commit

Permalink
chore: add default index options getter (#147)
Browse files Browse the repository at this point in the history
  • Loading branch information
henomis authored Nov 5, 2023
1 parent 940fca7 commit c1a816a
Show file tree
Hide file tree
Showing 8 changed files with 39 additions and 5 deletions.
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,17 +58,18 @@ import (
"context"

openaiembedder "github.com/henomis/lingoose/embedder/openai"
"github.com/henomis/lingoose/index"
"github.com/henomis/lingoose/index/option"
simplevectorindex "github.com/henomis/lingoose/index/simpleVectorIndex"
"github.com/henomis/lingoose/index/vectordb/jsondb"
"github.com/henomis/lingoose/llm/openai"
"github.com/henomis/lingoose/loader"
qapipeline "github.com/henomis/lingoose/pipeline/qa"
"github.com/henomis/lingoose/textsplitter"
)

func main() {
docs, _ := loader.NewPDFToTextLoader("./kb").WithPDFToTextPath("/opt/homebrew/bin/pdftotext").WithTextSplitter(textsplitter.NewRecursiveCharacterTextSplitter(2000, 200)).Load(context.Background())
index := simplevectorindex.New("db", ".", openaiembedder.New(openaiembedder.AdaEmbeddingV2))
docs, _ := loader.NewPDFToTextLoader("./kb").WithTextSplitter(textsplitter.NewRecursiveCharacterTextSplitter(2000, 200)).Load(context.Background())
index := index.New(jsondb.New().WithPersist("db.json"), openaiembedder.New(openaiembedder.AdaEmbeddingV2)).WithIncludeContents(true)
index.LoadFromDocuments(context.Background(), docs)
qapipeline.New(openai.NewChat().WithVerbose(true)).WithIndex(index).Query(context.Background(), "What is the NATO purpose?", option.WithTopK(1))
}
Expand Down
6 changes: 6 additions & 0 deletions index/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,3 +232,9 @@ func DeepCopyMetadata(metadata types.Meta) types.Meta {
}
return metadataCopy
}

func GetDefaultOptions() *option.Options {
return &option.Options{
TopK: defaultTopK,
}
}
7 changes: 5 additions & 2 deletions index/vectordb/jsondb/jsondb.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,14 @@ func (i *DB) Search(ctx context.Context, values []float64, options *option.Optio
}

func (i *DB) similaritySearch(
ctx context.Context,
_ context.Context,
embedding embedder.Embedding,
opts *option.Options,
) (index.SearchResults, error) {
_ = ctx
if opts == nil {
opts = index.GetDefaultOptions()
}

scores, err := i.cosineSimilarityBatch(embedding)
if err != nil {
return nil, fmt.Errorf("%w: %w", index.ErrInternal, err)
Expand Down
4 changes: 4 additions & 0 deletions index/vectordb/milvus/milvus.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@ func (d *DB) similaritySearch(
values []float64,
opts *option.Options,
) ([]milvusgoresponse.VectorData, error) {
if opts == nil {
opts = index.GetDefaultOptions()
}

if opts.Filter == nil {
opts.Filter = ""
}
Expand Down
8 changes: 8 additions & 0 deletions index/vectordb/pinecone/pinecone.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,14 @@ func (d *DB) similaritySearch(
values []float64,
opts *option.Options,
) ([]pineconegoresponse.QueryMatch, error) {
if opts == nil {
opts = index.GetDefaultOptions()
}

if opts.Filter == nil {
opts.Filter = map[string]string{}
}

err := d.getProjectID(ctx)
if err != nil {
return nil, fmt.Errorf("%w: %w", index.ErrInternal, err)
Expand Down
4 changes: 4 additions & 0 deletions index/vectordb/postgres/postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,10 @@ func (d *DB) similaritySearch(
values []float64,
opts *option.Options,
) (index.SearchResults, error) {
if opts == nil {
opts = index.GetDefaultOptions()
}

if opts.Filter == nil {
opts.Filter = ""
}
Expand Down
4 changes: 4 additions & 0 deletions index/vectordb/qdrant/qdrant.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ func (d *DB) similaritySearch(
values []float64,
opts *option.Options,
) ([]qdrantresponse.PointSearchResult, error) {
if opts == nil {
opts = index.GetDefaultOptions()
}

if opts.Filter == nil {
opts.Filter = qdrantrequest.Filter{}
}
Expand Down
4 changes: 4 additions & 0 deletions index/vectordb/redis/redis.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,10 @@ func (d *DB) similaritySearch(
values []float64,
opts *option.Options,
) ([]redisearch.Document, error) {
if opts == nil {
opts = index.GetDefaultOptions()
}

if opts.Filter == nil {
opts.Filter = redisearch.Filter{}
}
Expand Down

0 comments on commit c1a816a

Please sign in to comment.