Skip to content

Commit

Permalink
Fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
benbrandt committed Dec 27, 2023
1 parent d9f460c commit 73152f6
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions bindings/python/tests/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,14 @@ def test_hugging_face():
tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=False)
text = "123\n123"
assert splitter.chunks(text, 1) == ["123\n", "123"]
assert splitter.chunks(text, 1) == ["123", "\n123"]


def test_hugging_face_range():
tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=False)
text = "123\n123"
assert splitter.chunks(text=text, chunk_capacity=(1, 2)) == ["123\n", "123"]
assert splitter.chunks(text=text, chunk_capacity=(1, 2)) == ["123", "\n123"]


def test_hugging_face_trim():
Expand Down

0 comments on commit 73152f6

Please sign in to comment.