Skip to content

Commit

Permalink
add unit test for overlap rate too small in fixed token length algorithm
Browse files Browse the repository at this point in the history
Signed-off-by: yuye-aws <[email protected]>
  • Loading branch information
yuye-aws committed Mar 15, 2024
1 parent e4bdabc commit 9e37171
Showing 1 changed file with 14 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ public void testParseParameters_whenIllegalOverlapRateType_thenFail() {
);
}

public void testParseParameters_whenIllegalOverlapRateValue_thenFail() {
public void testParseParameters_whenTooLargeOverlapRate_thenFail() {
Map<String, Object> parameters = new HashMap<>();
parameters.put(OVERLAP_RATE_FIELD, 0.6);
IllegalArgumentException illegalArgumentException = assertThrows(
Expand All @@ -118,6 +118,19 @@ public void testParseParameters_whenIllegalOverlapRateValue_thenFail() {
);
}

public void testParseParameters_whenTooSmallOverlapRateValue_thenFail() {
Map<String, Object> parameters = new HashMap<>();
parameters.put(OVERLAP_RATE_FIELD, -1);
IllegalArgumentException illegalArgumentException = assertThrows(
IllegalArgumentException.class,
() -> fixedTokenLengthChunker.parseParameters(parameters)
);
assertEquals(
String.format(Locale.ROOT, "Parameter [%s] must be between %s and %s", OVERLAP_RATE_FIELD, 0.0, 0.5),
illegalArgumentException.getMessage()
);
}

public void testParseParameters_whenIllegalTokenizerType_thenFail() {
Map<String, Object> parameters = new HashMap<>();
parameters.put(TOKENIZER_FIELD, 111);
Expand Down

0 comments on commit 9e37171

Please sign in to comment.