-
Notifications
You must be signed in to change notification settings - Fork 126
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactors FieldMapper logic. It removes the LegacyFieldMapper and replaces it with a FlatFieldMapper. The FlatFieldMapper's role is to create fields that do not build ANN indices. Additionally, it puts dimension, model_id, and knn_method_context in a new ANNConfig class and adds some safety checks around accessing them. This should make calling logic easier to handle. Lastly, it cleans up the parsing so that there isnt encoder parsing directly in the KNNVectorFieldMapper. Signed-off-by: John Mazanec <[email protected]>
- Loading branch information
1 parent
56698f7
commit 048f6b5
Showing
35 changed files
with
1,356 additions
and
864 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
119 changes: 119 additions & 0 deletions
119
src/main/java/org/opensearch/knn/index/mapper/ANNConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.mapper; | ||
|
|
||
| import lombok.Getter; | ||
| import org.opensearch.knn.index.engine.KNNMethodContext; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * Class holds information about how the ANN indices are created. The design of this class ensures that we do not | ||
| * accidentally configure an index that has multiple ways it can be created. This class is immutable. | ||
| */ | ||
| public final class ANNConfig { | ||
|
|
||
| @Getter | ||
| private final ANNConfigType annConfigType; | ||
| private final KNNMethodContext knnMethodContext; | ||
| private final String modelId; | ||
| private final Integer dimension; | ||
|
|
||
| /** | ||
| * Constructor | ||
| * | ||
| * @param annConfigType Configurational context index was built. Cannot be null | ||
| * @param knnMethodContext Method context used to create the index; null if not created from method | ||
| * @param modelId Model id used to create the index; null if not created from model | ||
| * @param dimension Dimension used to create the index; needs to be null for model-based indices | ||
| */ | ||
| public ANNConfig(ANNConfigType annConfigType, KNNMethodContext knnMethodContext, String modelId, Integer dimension) { | ||
| if (annConfigType == null) { | ||
| throw new IllegalArgumentException("ANNConfiguration cannot be null"); | ||
| } | ||
|
|
||
| this.annConfigType = annConfigType; | ||
| this.knnMethodContext = knnMethodContext; | ||
| this.modelId = modelId; | ||
| this.dimension = dimension; | ||
|
|
||
| if (ANNConfigType.FROM_METHOD == annConfigType) { | ||
| validateFromMethod(); | ||
| return; | ||
| } | ||
|
|
||
| if (ANNConfigType.FROM_MODEL == annConfigType) { | ||
| validateFromModel(); | ||
| return; | ||
| } | ||
|
|
||
| if (ANNConfigType.SKIP == annConfigType) { | ||
| validateSkip(); | ||
| } | ||
| } | ||
|
|
||
| private void validateFromMethod() { | ||
| if (knnMethodContext == null) { | ||
| throw new IllegalArgumentException("knnMethodContext cannot be null when created from method"); | ||
| } | ||
|
|
||
| if (modelId != null) { | ||
| throw new IllegalArgumentException("modelId cannot be specified when created from method"); | ||
| } | ||
|
|
||
| if (dimension == null) { | ||
| throw new IllegalArgumentException("dimension must be specified when created from method"); | ||
| } | ||
| } | ||
|
|
||
| private void validateFromModel() { | ||
| if (modelId == null) { | ||
| throw new IllegalArgumentException("modelId cannot be null when created from method"); | ||
| } | ||
|
|
||
| if (knnMethodContext != null) { | ||
| throw new IllegalArgumentException("knnMethodContext cannot be specified when created from method"); | ||
| } | ||
|
|
||
| if (dimension != null) { | ||
| throw new IllegalArgumentException("dimension must be null when created from model"); | ||
| } | ||
| } | ||
|
|
||
| private void validateSkip() { | ||
| if (knnMethodContext != null || modelId != null) { | ||
| throw new IllegalArgumentException("knnMethodContext or modelId cannot be specified when skipping"); | ||
| } | ||
|
|
||
| if (dimension == null) { | ||
| throw new IllegalArgumentException("dimension must be specified when created from model"); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @return Optional containing the modelId if created from model, otherwise empty | ||
| */ | ||
| public Optional<String> getModelId() { | ||
| return Optional.ofNullable(modelId); | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @return Optional containing the KNNMethodContext if created from method, otherwise empty | ||
| */ | ||
| public Optional<KNNMethodContext> getKnnMethodContext() { | ||
| return Optional.ofNullable(knnMethodContext); | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @return the dimension of the index; for model based indices, it will be null | ||
| */ | ||
| public Optional<Integer> getDimension() { | ||
| return Optional.ofNullable(dimension); | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
src/main/java/org/opensearch/knn/index/mapper/ANNConfigType.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.mapper; | ||
|
|
||
| /** | ||
| * Types of configurations to build ANN indices | ||
| */ | ||
| public enum ANNConfigType { | ||
| FROM_METHOD, | ||
| FROM_MODEL, | ||
| SKIP | ||
| } |
63 changes: 63 additions & 0 deletions
63
src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.mapper; | ||
|
|
||
| import org.apache.lucene.document.FieldType; | ||
| import org.opensearch.Version; | ||
| import org.opensearch.common.Explicit; | ||
| import org.opensearch.knn.index.VectorDataType; | ||
|
|
||
| /** | ||
| * Mapper used when you dont want to build an underlying KNN struct - you just want to | ||
| * store vectors as doc values | ||
| */ | ||
| public class FlatVectorFieldMapper extends KNNVectorFieldMapper { | ||
|
|
||
| private final PerDimensionValidator perDimensionValidator; | ||
|
|
||
| public FlatVectorFieldMapper( | ||
| String simpleName, | ||
| KNNVectorFieldType mappedFieldType, | ||
| MultiFields multiFields, | ||
| CopyTo copyTo, | ||
| Explicit<Boolean> ignoreMalformed, | ||
| boolean stored, | ||
| boolean hasDocValues, | ||
| Version indexCreatedVersion | ||
| ) { | ||
| super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion, null); | ||
| this.perDimensionValidator = selectPerDimensionValidator(vectorDataType); | ||
| this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); | ||
| this.fieldType.freeze(); | ||
| } | ||
|
|
||
| private PerDimensionValidator selectPerDimensionValidator(VectorDataType vectorDataType) { | ||
| if (VectorDataType.BINARY == vectorDataType) { | ||
| return PerDimensionValidator.DEFAULT_BIT_VALIDATOR; | ||
| } | ||
|
|
||
| if (VectorDataType.BYTE == vectorDataType) { | ||
| return PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; | ||
| } | ||
|
|
||
| return PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; | ||
| } | ||
|
|
||
| @Override | ||
| protected VectorValidator getVectorValidator() { | ||
| return VectorValidator.NOOP_VECTOR_VALIDATOR; | ||
| } | ||
|
|
||
| @Override | ||
| protected PerDimensionValidator getPerDimensionValidator() { | ||
| return perDimensionValidator; | ||
| } | ||
|
|
||
| @Override | ||
| protected PerDimensionProcessor getPerDimensionProcessor() { | ||
| return PerDimensionProcessor.NOOP_PROCESSOR; | ||
| } | ||
| } |
Oops, something went wrong.