assign positive default value for max chunk limit

Signed-off-by: yuye-aws <[email protected]>
yuye-aws · Mar 15, 2024 · 89c465c · 89c465c
1 parent 3b8a3af
commit 89c465c
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 6 deletions.
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextChunkingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextChunkingProcessor.java
@@ -45,7 +45,7 @@ public final class TextChunkingProcessor extends AbstractProcessor {
     @VisibleForTesting
     static final String MAX_CHUNK_LIMIT_FIELD = "max_chunk_limit";
 
-    private static final int DEFAULT_MAX_CHUNK_LIMIT = -1;
+    private static final int DEFAULT_MAX_CHUNK_LIMIT = 100;
     private static final String DEFAULT_ALGORITHM = FixedTokenLengthChunker.ALGORITHM_NAME;
 
     private int maxChunkLimit;
@@ -114,8 +114,10 @@ private void parseAlgorithmMap(final Map<String, Object> algorithmMap) {
         }
 
         Map<String, Object> chunkerParameters = (Map<String, Object>) algorithmValue;
-        // fixed token length algorithm needs analysis registry for tokenization
-        chunkerParameters.put(FixedTokenLengthChunker.ANALYSIS_REGISTRY_FIELD, analysisRegistry);
+        if (algorithmKey.equals(FixedTokenLengthChunker.ALGORITHM_NAME)) {
+            // fixed token length algorithm needs analysis registry for tokenization
+            chunkerParameters.put(FixedTokenLengthChunker.ANALYSIS_REGISTRY_FIELD, analysisRegistry);
+        }
         this.chunker = ChunkerFactory.create(algorithmKey, chunkerParameters);
         this.maxChunkLimit = parsePositiveIntegerParameter(chunkerParameters, MAX_CHUNK_LIMIT_FIELD, DEFAULT_MAX_CHUNK_LIMIT);
     }
@@ -269,7 +271,7 @@ private int chunkString(final String content, List<String> result, final Map<Str
         int updatedChunkCount = chunkCount;
         List<String> contentResult = chunker.chunk(content, runTimeParameters);
         updatedChunkCount += contentResult.size();
-        if (maxChunkLimit != DEFAULT_MAX_CHUNK_LIMIT && updatedChunkCount > maxChunkLimit) {
+        if (updatedChunkCount > maxChunkLimit) {
             throw new IllegalArgumentException(
                 String.format(
                     Locale.ROOT,

diff --git a/src/main/java/org/opensearch/neuralsearch/processor/chunker/ChunkerParameterParser.java b/src/main/java/org/opensearch/neuralsearch/processor/chunker/ChunkerParameterParser.java
@@ -55,8 +55,7 @@ public static int parsePositiveIntegerParameter(final Map<String, Object> parame
                 String.format(Locale.ROOT, "Parameter [%s] must be of %s type", fieldName, Integer.class.getName())
             );
         }
-        // some parameter has negative default value, indicating that this parameter is not effective
-        if (fieldValueInt != defaultValue && fieldValueInt <= 0) {
+        if (fieldValueInt <= 0) {
             throw new IllegalArgumentException(String.format(Locale.ROOT, "Parameter [%s] must be positive.", fieldName));
         }
         return fieldValueInt;