Skip to content

Commit

Permalink
added break for recursive summary when number of partial summary does…
Browse files Browse the repository at this point in the history
…n't change
  • Loading branch information
JohannesWesch authored and NickyHavoc committed Feb 6, 2024
1 parent 4ef90a8 commit 8d842d2
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 5 deletions.
4 changes: 1 addition & 3 deletions src/intelligence_layer/core/chunk.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,7 @@ def __init__(
overlap_length_tokens, max_tokens_per_chunk
)
)
self.chunk_task = ChunkTask(
client, model, overlap_length_tokens // 2
)
self.chunk_task = ChunkTask(client, model, overlap_length_tokens // 2)
self.tokenizer = client.tokenizer(model)
self.max_tokens_per_chunk = max_tokens_per_chunk
self.overlap_length_tokens = overlap_length_tokens
Expand Down
5 changes: 3 additions & 2 deletions tests/core/test_chunk.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,15 @@ def test_overlapped_chunking(
print(first)

assert (
len(first) <= MAX_TOKENS + 2
len(first)
<= MAX_TOKENS + 2
# `+2` because re-tokenizing the chunk can add a few extra tokens at
# the beginning or end of each chunk. This is a hack.
)
next = output_tokenized[chunk_index + 1].tokens

found = False
for offset in range(len(next)-OVERLAP//2):
for offset in range(len(next) - OVERLAP // 2):
if first[-OVERLAP // 2 :] != next[offset : offset + OVERLAP // 2]:
continue
found = True
Expand Down

0 comments on commit 8d842d2

Please sign in to comment.