Skip to content

Commit

Permalink
tune code
Browse files Browse the repository at this point in the history
Signed-off-by: yuye-aws <[email protected]>
  • Loading branch information
yuye-aws committed Mar 15, 2024
1 parent 18ba1b1 commit 2ce9840
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
import static org.opensearch.neuralsearch.processor.chunker.ChunkerParameterParser.parseIntegerParameter;

/**
* This processor is used for user input data text chunking.
* The chunking results could be fed to downstream embedding processor.
* This processor is used for text chunking.
* The text chunking results could be fed to downstream embedding processor.
* The processor needs two fields: algorithm and field_map,
* where algorithm defines chunking algorithm and parameters,
* and field_map specifies which fields needs chunking and the corresponding keys for the chunking results.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,18 @@ public final class ChunkerFactory {

private ChunkerFactory() {} // no instance of this factory class

private static final ImmutableMap<String, Function<Map<String, Object>, Chunker>> chunkers = ImmutableMap.of(
private static final Map<String, Function<Map<String, Object>, Chunker>> chunkerConstructors = ImmutableMap.of(
FixedTokenLengthChunker.ALGORITHM_NAME,
FixedTokenLengthChunker::new,
DelimiterChunker.ALGORITHM_NAME,
DelimiterChunker::new
);

@Getter
public static Set<String> allChunkerAlgorithms = chunkers.keySet();
public static Set<String> allChunkerAlgorithms = chunkerConstructors.keySet();

public static Chunker create(final String type, final Map<String, Object> parameters) {
Function<Map<String, Object>, Chunker> chunkerConstructionFunction = chunkers.get(type);
Function<Map<String, Object>, Chunker> chunkerConstructionFunction = chunkerConstructors.get(type);
// chunkerConstructionFunction is not null because we have validated the type in text chunking processor
Objects.requireNonNull(chunkerConstructionFunction);
return chunkerConstructionFunction.apply(parameters);
Expand Down

0 comments on commit 2ce9840

Please sign in to comment.