Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Community: Adding bulk_size as a setable param for OpenSearchVectorSearch #28325

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Community: Adding bulk_size as a setable param for OpenSearchVectorSearch #28325

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
51843d4
let OpenSearchVectorSearch class function can accept bulk_size param …
manukychen Nov 24, 2024
9fbccec
fix classmethod self.bulk_size bug
manukychen Nov 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 10 additions & 18 deletions libs/community/langchain_community/vectorstores/opensearch_vector_search.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ def __init__(
self.client = _get_opensearch_client(opensearch_url, **kwargs)
self.async_client = _get_async_opensearch_client(opensearch_url, **kwargs)
self.engine = kwargs.get("engine", "nmslib")
self.bulk_size = kwargs.get("bulk_size", 500)

@property
def embeddings(self) -> Embeddings:
Expand All @@ -413,10 +414,9 @@ def __add(
embeddings: List[List[float]],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
**kwargs: Any,
) -> List[str]:
_validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
_validate_embeddings_and_bulk_size(len(embeddings), self.bulk_size)
index_name = kwargs.get("index_name", self.index_name)
text_field = kwargs.get("text_field", "text")
dim = len(embeddings[0])
Expand Down Expand Up @@ -454,10 +454,9 @@ async def __aadd(
embeddings: List[List[float]],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
**kwargs: Any,
) -> List[str]:
_validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
_validate_embeddings_and_bulk_size(len(embeddings), self.bulk_size)
index_name = kwargs.get("index_name", self.index_name)
text_field = kwargs.get("text_field", "text")
dim = len(embeddings[0])
Expand Down Expand Up @@ -560,7 +559,6 @@ def add_texts(
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
**kwargs: Any,
) -> List[str]:
"""Run more texts through the embeddings and add to the vectorstore.
Expand All @@ -587,7 +585,7 @@ def add_texts(
embeddings,
metadatas=metadatas,
ids=ids,
bulk_size=bulk_size,
bulk_size=self.bulk_size,
**kwargs,
)

Expand All @@ -596,7 +594,6 @@ async def aadd_texts(
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
**kwargs: Any,
) -> List[str]:
"""
Expand All @@ -609,7 +606,7 @@ async def aadd_texts(
embeddings,
metadatas=metadatas,
ids=ids,
bulk_size=bulk_size,
bulk_size=self.bulk_size,
**kwargs,
)

Expand All @@ -618,7 +615,6 @@ def add_embeddings(
text_embeddings: Iterable[Tuple[str, List[float]]],
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
bulk_size: int = 500,
**kwargs: Any,
) -> List[str]:
"""Add the given texts and embeddings to the vectorstore.
Expand Down Expand Up @@ -646,7 +642,7 @@ def add_embeddings(
list(embeddings),
metadatas=metadatas,
ids=ids,
bulk_size=bulk_size,
bulk_size=self.bulk_size,
**kwargs,
)

Expand Down Expand Up @@ -1085,7 +1081,6 @@ def from_texts(
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
bulk_size: int = 500,
ids: Optional[List[str]] = None,
**kwargs: Any,
) -> OpenSearchVectorSearch:
Expand Down Expand Up @@ -1139,7 +1134,7 @@ def from_texts(
texts,
embedding,
metadatas=metadatas,
bulk_size=bulk_size,
bulk_size=cls.bulk_size,
ids=ids,
**kwargs,
)
Expand All @@ -1150,7 +1145,6 @@ async def afrom_texts(
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
bulk_size: int = 500,
ids: Optional[List[str]] = None,
**kwargs: Any,
) -> OpenSearchVectorSearch:
Expand Down Expand Up @@ -1204,7 +1198,7 @@ async def afrom_texts(
texts,
embedding,
metadatas=metadatas,
bulk_size=bulk_size,
bulk_size=cls.bulk_size,
ids=ids,
**kwargs,
)
Expand All @@ -1216,7 +1210,6 @@ def from_embeddings(
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
bulk_size: int = 500,
ids: Optional[List[str]] = None,
**kwargs: Any,
) -> OpenSearchVectorSearch:
Expand Down Expand Up @@ -1285,7 +1278,7 @@ def from_embeddings(
"max_chunk_bytes",
"is_aoss",
]
_validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
_validate_embeddings_and_bulk_size(len(embeddings), cls.bulk_size)
dim = len(embeddings[0])
# Get the index name from either from kwargs or ENV Variable
# before falling back to random generation
Expand Down Expand Up @@ -1346,7 +1339,6 @@ async def afrom_embeddings(
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
bulk_size: int = 500,
ids: Optional[List[str]] = None,
**kwargs: Any,
) -> OpenSearchVectorSearch:
Expand Down Expand Up @@ -1417,7 +1409,7 @@ async def afrom_embeddings(
"max_chunk_bytes",
"is_aoss",
]
_validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
_validate_embeddings_and_bulk_size(len(embeddings), cls.bulk_size)
dim = len(embeddings[0])
# Get the index name from either from kwargs or ENV Variable
# before falling back to random generation
Expand Down
Loading