Skip to content

Commit

Permalink
Rebase upstream main into this branch
Browse files Browse the repository at this point in the history 
Fixed parameter m->r for specific describe

Signed-off-by: luyuncheng <[email protected]>
  • Loading branch information
@luyuncheng
luyuncheng committed Aug 4, 2023
1 parent 53c71bc commit 04333c8
Show file tree
Hide file tree
Showing 19 changed files with 518 additions and 466 deletions.
2 changes: 1 addition & 1 deletion benchmarks/osb/indices/faiss-nsg-index.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"space_type": "{{ target_index_space_type }}",
"engine": "faiss",
"parameters": {
"m": {{ hnsw_m }}
"r": {{ hnsw_m }}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/perf-tool/release-configs/faiss-nsg/index-spec.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"space_type": "l2",
"engine": "faiss",
"parameters": {
"m": 32
"r": 32
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/perf-tool/release-configs/faiss-nsg/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ steps:
index_name: target_index
- name: force_merge
index_name: target_index
max_num_segments: 10
max_num_segments: 1
- name: query
k: 100
r: 1
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/perf-tool/sample-configs/faiss-nsg/index-spec.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"space_type": "l2",
"engine": "faiss",
"parameters": {
"m": 32
"r": 32
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion jni/src/faiss_wrapper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "faiss/index_factory.h"
#include "faiss/index_io.h"
#include "faiss/IndexHNSW.h"
#include "faiss/IndexNSG.h"
#include "faiss/IndexIVFFlat.h"
#include "faiss/MetaIndexes.h"
#include "faiss/Index.h"
Expand Down Expand Up @@ -212,7 +213,7 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter
throw std::runtime_error("Invalid pointer to index");
}

auto nsgReader = dynamic_cast<const faiss::IndexNSG*>(indexReader->index);
auto nsgReader = dynamic_cast<const faiss::IndexNSG *>(indexReader->index);
if(nsgReader) {
// Search params not supported for the NSG index
throw std::runtime_error("NSG Index Type do not support for Filtered Search on Faiss");
Expand Down
1 change: 1 addition & 0 deletions src/main/java/org/opensearch/knn/common/KNNConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public class KNNConstants {
public static final String FAISS_IVF_DESCRIPTION = "IVF";
public static final String FAISS_FLAT_DESCRIPTION = "Flat";
public static final String FAISS_PQ_DESCRIPTION = "PQ";
public static final Integer FAISS_NSG_MIN_DOC_TO_FLAT = 256;

// Parameter defaults/limits
public static final Integer ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT = 1;
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/opensearch/knn/index/KNNSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.ByteSizeUnit;
import org.opensearch.common.unit.ByteSizeValue;
import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.index.IndexModule;
import org.opensearch.knn.index.memory.NativeMemoryCacheManager;
Expand All @@ -37,7 +37,7 @@
import static org.opensearch.common.settings.Setting.Property.Dynamic;
import static org.opensearch.common.settings.Setting.Property.IndexScope;
import static org.opensearch.common.settings.Setting.Property.NodeScope;
import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue;
import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue;
import static org.opensearch.common.unit.MemorySizeValue.parseBytesSizeValueOrHeapRatio;

/**
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import org.opensearch.common.util.BigArrays;
import org.opensearch.index.fielddata.IndexFieldData;
import org.opensearch.index.fielddata.IndexFieldDataCache;
import org.opensearch.indices.breaker.CircuitBreakerService;
import org.opensearch.core.indices.breaker.CircuitBreakerService;
import org.opensearch.search.DocValueFormat;
import org.opensearch.search.MultiValueMode;
import org.opensearch.search.aggregations.support.ValuesSourceType;
Expand Down
21 changes: 15 additions & 6 deletions src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@
import lombok.NonNull;
import lombok.extern.log4j.Log4j2;
import org.apache.lucene.store.ChecksumIndexInput;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.XContentHelper;
import org.opensearch.core.common.bytes.BytesArray;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.knn.index.KNNSettings;
Expand Down Expand Up @@ -50,6 +51,7 @@

import static org.apache.lucene.codecs.CodecUtil.FOOTER_MAGIC;
import static org.opensearch.knn.common.KNNConstants.FAISS_FLAT_DESCRIPTION;
import static org.opensearch.knn.common.KNNConstants.FAISS_NSG_MIN_DOC_TO_FLAT;
import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER;
import static org.opensearch.knn.common.KNNConstants.METHOD_NSG;
import static org.opensearch.knn.common.KNNConstants.NAME;
Expand Down Expand Up @@ -180,24 +182,31 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa
parameters.put(PARAMETERS, algoParams);
} else {
parameters.putAll(
XContentFactory.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, parametersString)
.map()
XContentHelper.createParser(
NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
new BytesArray(parametersString),
MediaTypeRegistry.getDefaultMediaType()
).map()
);
}

// Used to determine how many threads to use when indexing
parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY));
Object name = parameters.get(NAME);
if (name != null && name.equals(METHOD_NSG)) {
/**
* TODO:
* Search params not supported for the NSG index
*/
if (parameters.containsKey(PARAMETERS)) {
parameters.remove(PARAMETERS);
}
/** TODO:
* when numIds is too small, NSG graph would core/throw exception
* because There are too much invalid entries in the knn graph.
*/
if (pair.docs.length < 256) {
if (pair.docs.length < FAISS_NSG_MIN_DOC_TO_FLAT) {
parameters.put(INDEX_DESCRIPTION_PARAMETER, FAISS_FLAT_DESCRIPTION);
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/org/opensearch/knn/index/FaissIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ public void testEndToEnd_fromNSGMethod() throws Exception {
.field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue())
.field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName())
.startObject(KNNConstants.PARAMETERS)
.field(KNNConstants.METHOD_PARAMETER_M, 64)
.field(KNNConstants.METHOD_PARAMETER_R, 64)
.endObject()
.endObject()
.endObject()
Expand Down Expand Up @@ -254,7 +254,7 @@ public void testEndToEnd_fromNSGMethodIndexOne() throws Exception {
.field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue())
.field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName())
.startObject(KNNConstants.PARAMETERS)
.field(KNNConstants.METHOD_PARAMETER_M, 64)
.field(KNNConstants.METHOD_PARAMETER_R, 64)
.endObject()
.endObject()
.endObject()
Expand Down
24 changes: 12 additions & 12 deletions src/test/java/org/opensearch/knn/index/KNNSettingsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.network.NetworkModule;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.ByteSizeValue;
import org.opensearch.core.common.unit.ByteSizeValue;
import org.opensearch.env.Environment;
import org.opensearch.knn.KNNTestCase;
import org.opensearch.knn.plugin.KNNPlugin;
Expand All @@ -37,13 +37,13 @@ public class KNNSettingsTests extends KNNTestCase {
public void testGetSettingValueFromConfig() {
long expectedKNNCircuitBreakerLimit = 13;
Node mockNode = createMockNode(
Map.of(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT, "\"" + expectedKNNCircuitBreakerLimit + "kb\"")
Map.of(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT, "\"" + expectedKNNCircuitBreakerLimit + "kb\"")
);
mockNode.start();
ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class);
KNNSettings.state().setClusterService(clusterService);
long actualKNNCircuitBreakerLimit = ((ByteSizeValue) KNNSettings.state()
.getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb();
.getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb();
mockNode.close();
assertEquals(expectedKNNCircuitBreakerLimit, actualKNNCircuitBreakerLimit);
}
Expand All @@ -55,12 +55,12 @@ public void testGetSettingValueDefault() {
ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class);
KNNSettings.state().setClusterService(clusterService);
long actualKNNCircuitBreakerLimit = ((ByteSizeValue) KNNSettings.state()
.getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb();
.getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb();
mockNode.close();
assertEquals(
((ByteSizeValue) KNNSettings.dynamicCacheSettings.get(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT).getDefault(Settings.EMPTY))
.getKb(),
actualKNNCircuitBreakerLimit
((ByteSizeValue) KNNSettings.dynamicCacheSettings.get(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT).getDefault(Settings.EMPTY))
.getKb(),
actualKNNCircuitBreakerLimit

);
}
Expand Down Expand Up @@ -88,9 +88,9 @@ private List<Class<? extends Plugin>> basePlugins() {
private static Settings.Builder baseSettings() {
final Path tempDir = createTempDir();
return Settings.builder()
.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
.put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType())
.put(dataNode());
.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
.put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
.put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType())
.put(dataNode());
}
}
}
Loading

0 comments on commit 04333c8

Please sign in to comment.