From 3598302c808166c6592e488ec9ff3d70966bfc04 Mon Sep 17 00:00:00 2001 From: Jack Mazanec Date: Tue, 1 Sep 2020 15:36:48 -0700 Subject: [PATCH 1/2] Add default index settings when parsing index (#205) --- .../knn/index/KNNSettings.java | 22 ++++-- .../knn/index/KNNVectorFieldMapper.java | 52 ++++++++++++-- .../knn/index/KNNVectorFieldMapperTests.java | 70 +++++++++++++++++++ 3 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapperTests.java diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNSettings.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNSettings.java index b838910b..335073f6 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNSettings.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNSettings.java @@ -80,12 +80,22 @@ public class KNNSettings { public static final String KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE = "knn.circuit_breaker.unset.percentage"; public static final String KNN_INDEX = "index.knn"; + /** + * Default setting values + */ + public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; + public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING = 16; + public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH_SETTING = 512; + public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING = 512; + public static final Integer KNN_DEFAULT_ALGO_PARAM_INDEX_THREAD_QTY_SETTING = 1; + public static final Integer KNN_DEFAULT_CIRCUIT_BREAKER_UNSET_PERCENTAGE_SETTING = 75; + /** * Settings Definition */ public static final Setting INDEX_KNN_SPACE_TYPE = Setting.simpleString(KNN_SPACE_TYPE, - "l2", + INDEX_KNN_DEFAULT_SPACE_TYPE, new SpaceTypeValidator(), IndexScope); @@ -96,7 +106,7 @@ public class KNNSettings { * The parameter also determines the algorithm's memory consumption, which is roughly M * 8-10 bytes per stored element. */ public static final Setting INDEX_KNN_ALGO_PARAM_M_SETTING = Setting.intSetting(KNN_ALGO_PARAM_M, - 16, + INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING, 2, IndexScope); @@ -106,7 +116,7 @@ public class KNNSettings { * The value ef can be anything between k and the size of the dataset. */ public static final Setting INDEX_KNN_ALGO_PARAM_EF_SEARCH_SETTING = Setting.intSetting(KNN_ALGO_PARAM_EF_SEARCH, - 512, + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH_SETTING, 2, IndexScope, Dynamic); @@ -116,7 +126,7 @@ public class KNNSettings { * Bigger ef_construction leads to longer construction(more indexing time), but better index quality. */ public static final Setting INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING = Setting.intSetting(KNN_ALGO_PARAM_EF_CONSTRUCTION, - 512, + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING, 2, IndexScope); @@ -134,7 +144,7 @@ public class KNNSettings { * configure number of threads for graph construction. */ public static final Setting KNN_ALGO_PARAM_INDEX_THREAD_QTY_SETTING = Setting.intSetting(KNN_ALGO_PARAM_INDEX_THREAD_QTY, - 1, + KNN_DEFAULT_ALGO_PARAM_INDEX_THREAD_QTY_SETTING, 1, INDEX_THREAD_QTY_MAX, NodeScope, @@ -147,7 +157,7 @@ public class KNNSettings { public static final Setting KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE_SETTING = Setting.doubleSetting( KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE, - 75, + KNN_DEFAULT_CIRCUIT_BREAKER_UNSET_PERCENTAGE_SETTING, 0, 100, NodeScope, diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapper.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapper.java index f05c44e4..bb960b8a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapper.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapper.java @@ -17,6 +17,8 @@ import com.amazon.opendistroforelasticsearch.knn.index.util.KNNConstants; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; @@ -46,10 +48,18 @@ import java.util.List; import java.util.Map; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_SPACE_TYPE; + /** * Field Mapper for KNN vector type. */ public class KNNVectorFieldMapper extends FieldMapper { + + private static Logger logger = LogManager.getLogger(KNNVectorFieldMapper.class); + public static final String CONTENT_TYPE = "knn_vector"; public static final String KNN_FIELD = "knn_field"; @@ -144,12 +154,7 @@ public static class TypeParser implements Mapper.TypeParser { public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new KNNVectorFieldMapper.Builder(name); - builder.spaceTypeParam(KNNConstants.SPACE_TYPE, parserContext.mapperService().getIndexSettings().getValue( - KNNSettings.INDEX_KNN_SPACE_TYPE)); - builder.algoParams(KNNConstants.HNSW_ALGO_M, parserContext.mapperService().getIndexSettings().getValue( - KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING)); - builder.algoParams(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, parserContext.mapperService().getIndexSettings() - .getValue(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING)); + builder = buildKNNIndexSettings(builder, parserContext); /** * If dimension not provided. Throw Exception @@ -194,6 +199,41 @@ public Mapper.Builder parse(String name, Map node, ParserCont } return builder; } + + Builder buildKNNIndexSettings(Builder builder, ParserContext parserContext) { + try { + builder.spaceTypeParam(KNNConstants.SPACE_TYPE, parserContext.mapperService().getIndexSettings().getValue( + INDEX_KNN_SPACE_TYPE)); + } catch(IllegalArgumentException ex) { + logger.debug("[KNN] The setting \"" + KNNConstants.SPACE_TYPE + "\" was not set for the index. " + + "Likely caused by recent version upgrade. Setting the setting to the default value=" + + INDEX_KNN_DEFAULT_SPACE_TYPE); + builder.spaceTypeParam(KNNConstants.SPACE_TYPE, INDEX_KNN_DEFAULT_SPACE_TYPE); + } + + try { + builder.algoParams(KNNConstants.HNSW_ALGO_M, parserContext.mapperService().getIndexSettings().getValue( + KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING)); + } catch(IllegalArgumentException ex) { + logger.debug("[KNN] The setting \"" + KNNConstants.HNSW_ALGO_M + "\" was not set for the index. " + + "Likely caused by recent version upgrade. Setting the setting to the default value=" + + INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING); + builder.algoParams(KNNConstants.HNSW_ALGO_M, INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING); + } + + try { + builder.algoParams(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, parserContext.mapperService().getIndexSettings() + .getValue(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING)); + } catch(IllegalArgumentException ex) { + logger.debug("[KNN] The setting \"" + KNNConstants.HNSW_ALGO_EF_CONSTRUCTION + "\" was not set for" + + " the index. Likely caused by recent version upgrade. Setting the setting to the default value=" + + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING); + builder.algoParams(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING); + } + + return builder; + } } @Override diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapperTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapperTests.java new file mode 100644 index 00000000..2bc53350 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapperTests.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.knn.index; + +import com.amazon.opendistroforelasticsearch.knn.KNNTestCase; +import com.amazon.opendistroforelasticsearch.knn.index.util.KNNConstants; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; + +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; + +import static org.elasticsearch.Version.V_7_1_0; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class KNNVectorFieldMapperTests extends KNNTestCase { + + public void testBuildKNNIndexSettings_emptySettings_checkDefaultsSet() { + String indexName = "test-index"; + String fieldName = "test-fieldname"; + + Mapper.TypeParser.ParserContext parserContext = mock(Mapper.TypeParser.ParserContext.class); + MapperService mapperService = mock(MapperService.class); + IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(indexName).settings(settings(V_7_1_0)) + .numberOfShards(1) + .numberOfReplicas(0) + .version(7) + .mappingVersion(0) + .settingsVersion(0) + .aliasesVersion(0) + .creationDate(0) + .build(), + settings(V_7_1_0).build()); + when(parserContext.mapperService()).thenReturn(mapperService); + when(mapperService.getIndexSettings()).thenReturn(indexSettings); + + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName); + + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(); + typeParser.buildKNNIndexSettings(builder, parserContext); + + assertEquals(KNNVectorFieldMapper.Defaults.FIELD_TYPE.getAttributes().get(KNNConstants.SPACE_TYPE), + INDEX_KNN_DEFAULT_SPACE_TYPE); + + assertEquals(KNNVectorFieldMapper.Defaults.FIELD_TYPE.getAttributes().get(KNNConstants.HNSW_ALGO_M), + String.valueOf(INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING)); + + assertEquals(KNNVectorFieldMapper.Defaults.FIELD_TYPE.getAttributes().get( + KNNConstants.HNSW_ALGO_EF_CONSTRUCTION), String.valueOf( + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING)); + } +} \ No newline at end of file From 4dc3909329e967d0ae855dd9d4288d0800938283 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Tue, 1 Sep 2020 17:31:15 -0700 Subject: [PATCH 2/2] update default settings name --- .../knn/index/KNNSettings.java | 20 +++++++++---------- .../knn/index/KNNVectorFieldMapper.java | 12 +++++------ .../knn/index/KNNVectorFieldMapperTests.java | 8 ++++---- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNSettings.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNSettings.java index 335073f6..e71ec717 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNSettings.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNSettings.java @@ -84,11 +84,11 @@ public class KNNSettings { * Default setting values */ public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; - public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING = 16; - public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH_SETTING = 512; - public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING = 512; - public static final Integer KNN_DEFAULT_ALGO_PARAM_INDEX_THREAD_QTY_SETTING = 1; - public static final Integer KNN_DEFAULT_CIRCUIT_BREAKER_UNSET_PERCENTAGE_SETTING = 75; + public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M = 16; + public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH = 512; + public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION = 512; + public static final Integer KNN_DEFAULT_ALGO_PARAM_INDEX_THREAD_QTY = 1; + public static final Integer KNN_DEFAULT_CIRCUIT_BREAKER_UNSET_PERCENTAGE = 75; /** * Settings Definition @@ -106,7 +106,7 @@ public class KNNSettings { * The parameter also determines the algorithm's memory consumption, which is roughly M * 8-10 bytes per stored element. */ public static final Setting INDEX_KNN_ALGO_PARAM_M_SETTING = Setting.intSetting(KNN_ALGO_PARAM_M, - INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING, + INDEX_KNN_DEFAULT_ALGO_PARAM_M, 2, IndexScope); @@ -116,7 +116,7 @@ public class KNNSettings { * The value ef can be anything between k and the size of the dataset. */ public static final Setting INDEX_KNN_ALGO_PARAM_EF_SEARCH_SETTING = Setting.intSetting(KNN_ALGO_PARAM_EF_SEARCH, - INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH_SETTING, + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, 2, IndexScope, Dynamic); @@ -126,7 +126,7 @@ public class KNNSettings { * Bigger ef_construction leads to longer construction(more indexing time), but better index quality. */ public static final Setting INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING = Setting.intSetting(KNN_ALGO_PARAM_EF_CONSTRUCTION, - INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING, + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, 2, IndexScope); @@ -144,7 +144,7 @@ public class KNNSettings { * configure number of threads for graph construction. */ public static final Setting KNN_ALGO_PARAM_INDEX_THREAD_QTY_SETTING = Setting.intSetting(KNN_ALGO_PARAM_INDEX_THREAD_QTY, - KNN_DEFAULT_ALGO_PARAM_INDEX_THREAD_QTY_SETTING, + KNN_DEFAULT_ALGO_PARAM_INDEX_THREAD_QTY, 1, INDEX_THREAD_QTY_MAX, NodeScope, @@ -157,7 +157,7 @@ public class KNNSettings { public static final Setting KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE_SETTING = Setting.doubleSetting( KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE, - KNN_DEFAULT_CIRCUIT_BREAKER_UNSET_PERCENTAGE_SETTING, + KNN_DEFAULT_CIRCUIT_BREAKER_UNSET_PERCENTAGE, 0, 100, NodeScope, diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapper.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapper.java index bb960b8a..bd6d282c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapper.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapper.java @@ -48,8 +48,8 @@ import java.util.List; import java.util.Map; -import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING; -import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M; import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_SPACE_TYPE; @@ -217,8 +217,8 @@ Builder buildKNNIndexSettings(Builder builder, ParserContext parserContext) { } catch(IllegalArgumentException ex) { logger.debug("[KNN] The setting \"" + KNNConstants.HNSW_ALGO_M + "\" was not set for the index. " + "Likely caused by recent version upgrade. Setting the setting to the default value=" - + INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING); - builder.algoParams(KNNConstants.HNSW_ALGO_M, INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING); + + INDEX_KNN_DEFAULT_ALGO_PARAM_M); + builder.algoParams(KNNConstants.HNSW_ALGO_M, INDEX_KNN_DEFAULT_ALGO_PARAM_M); } try { @@ -227,9 +227,9 @@ Builder buildKNNIndexSettings(Builder builder, ParserContext parserContext) { } catch(IllegalArgumentException ex) { logger.debug("[KNN] The setting \"" + KNNConstants.HNSW_ALGO_EF_CONSTRUCTION + "\" was not set for" + " the index. Likely caused by recent version upgrade. Setting the setting to the default value=" - + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING); + + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION); builder.algoParams(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, - INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING); + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION); } return builder; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapperTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapperTests.java index 2bc53350..5493d20b 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapperTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNVectorFieldMapperTests.java @@ -22,8 +22,8 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; -import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING; -import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION; +import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M; import static com.amazon.opendistroforelasticsearch.knn.index.KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; import static org.elasticsearch.Version.V_7_1_0; @@ -61,10 +61,10 @@ public void testBuildKNNIndexSettings_emptySettings_checkDefaultsSet() { INDEX_KNN_DEFAULT_SPACE_TYPE); assertEquals(KNNVectorFieldMapper.Defaults.FIELD_TYPE.getAttributes().get(KNNConstants.HNSW_ALGO_M), - String.valueOf(INDEX_KNN_DEFAULT_ALGO_PARAM_M_SETTING)); + String.valueOf(INDEX_KNN_DEFAULT_ALGO_PARAM_M)); assertEquals(KNNVectorFieldMapper.Defaults.FIELD_TYPE.getAttributes().get( KNNConstants.HNSW_ALGO_EF_CONSTRUCTION), String.valueOf( - INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_SETTING)); + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION)); } } \ No newline at end of file