change chunk limit check in boht algorithms

Signed-off-by: yuye-aws <[email protected]>
opensearch-project · Apr 30, 2024 · 84cd362 · 84cd362
1 parent 3490a92
commit 84cd362
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunker.java b/src/main/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunker.java
@@ -61,7 +61,7 @@ public List<String> chunk(final String content, final Map<String, Object> runtim
         int nextDelimiterPosition = content.indexOf(delimiter);
 
         while (nextDelimiterPosition != -1) {
-            if (ChunkerUtil.checkRunTimeMaxChunkLimit(chunkResult.size(), runtimeMaxChunkLimit, stringTobeChunkedCount)) {
+            if (Chunker.checkRunTimeMaxChunkLimit(chunkResult.size(), runtimeMaxChunkLimit, stringTobeChunkedCount)) {
                 break;
             }
             end = nextDelimiterPosition + delimiter.length();

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/chunker/FixedTokenLengthChunker.java b/src/main/java/org/opensearch/neuralsearch/processor/chunker/FixedTokenLengthChunker.java
@@ -139,7 +139,7 @@ public List<String> chunk(final String content, final Map<String, Object> runtim
             } else {
                 startContentPosition = tokens.get(startTokenIndex).getStartOffset();
             }
-            if (ChunkerUtil.checkRunTimeMaxChunkLimit(chunkResult.size(), runtimeMaxChunkLimit, stringTobeChunkedCount)) {
+            if (Chunker.checkRunTimeMaxChunkLimit(chunkResult.size(), runtimeMaxChunkLimit, stringTobeChunkedCount)) {
                 // include all characters till the end if exceeds max chunk limit
                 chunkResult.add(content.substring(startContentPosition));
                 break;