Merge branch 'main' into poc/rewrite

Signed-off-by: Tejas Shah <[email protected]>

CHANGELOG.md

@@ -22,3 +22,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Maintenance
 ### Refactoring
 * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824)
+* Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913)

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java

@@ -20,10 +20,10 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.knn.KNNResult;
 import org.opensearch.knn.common.KNNConstants;
-import org.opensearch.knn.index.KNNMethod;
+import org.opensearch.knn.index.engine.KNNMethod;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.query.KNNQueryBuilder;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.io.IOException;
 import java.util.Arrays;

src/main/java/org/opensearch/knn/index/KNNIndexShard.java

@@ -23,7 +23,7 @@
 import org.opensearch.knn.index.memory.NativeMemoryCacheManager;
 import org.opensearch.knn.index.memory.NativeMemoryEntryContext;
 import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -37,7 +37,7 @@
 import static org.opensearch.knn.common.KNNConstants.MODEL_ID;
 import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE;
 import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD;
-import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading;
+import static org.opensearch.knn.index.util.IndexUtil.getParametersAtLoading;
 import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFilePrefix;
 import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileSuffix;
 

src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java

@@ -13,7 +13,7 @@
 import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams;
 import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams;
 import org.opensearch.knn.index.mapper.KNNVectorFieldMapper;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.util.Optional;
 import java.util.function.Function;

src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java

@@ -12,7 +12,7 @@
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.io.IOException;
 import java.util.HashSet;

src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java

@@ -14,7 +14,7 @@
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.knn.index.IndexUtil;
+import org.opensearch.knn.index.util.IndexUtil;
 import org.opensearch.knn.index.KNNSettings;
 import org.opensearch.knn.index.VectorDataType;
 import org.opensearch.knn.index.codec.transfer.VectorTransfer;
@@ -23,7 +23,7 @@
 import org.opensearch.knn.jni.JNIService;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.codec.util.KNNCodecUtil;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 import org.opensearch.knn.indices.Model;
 import org.opensearch.knn.indices.ModelCache;
 import org.opensearch.knn.plugin.stats.KNNCounter;
@@ -60,7 +60,7 @@
 import static org.opensearch.knn.common.KNNConstants.PARAMETERS;
 import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileName;
 import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize;
-import static org.opensearch.knn.index.util.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX;
+import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX;
 
 /**
  * This class writes the KNN docvalues to the segments

src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java

@@ -8,7 +8,7 @@
 import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.util.Optional;
 

src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java

@@ -9,7 +9,7 @@
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.util.Optional;
 

src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java

@@ -6,7 +6,7 @@
 package org.opensearch.knn.index.codec.params;
 
 import lombok.Getter;
-import org.opensearch.knn.index.MethodComponentContext;
+import org.opensearch.knn.index.engine.MethodComponentContext;
 
 import java.util.Map;
 

src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java → src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java

@@ -3,14 +3,12 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import org.opensearch.common.ValidationException;
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
 import org.opensearch.knn.training.VectorSpaceInfo;
 
 import java.util.Map;

src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java → src/main/java/org/opensearch/knn/index/engine/DefaultHnswContext.java

@@ -1,18 +1,12 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import com.google.common.collect.ImmutableMap;
-import org.opensearch.knn.index.Parameter;
+import org.opensearch.knn.index.engine.model.QueryContext;
 import org.opensearch.knn.index.query.request.MethodParameter;
 
 import java.util.Map;

src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java → src/main/java/org/opensearch/knn/index/engine/DefaultIVFContext.java

@@ -1,18 +1,12 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import com.google.common.collect.ImmutableMap;
-import org.opensearch.knn.index.Parameter;
+import org.opensearch.knn.index.engine.model.QueryContext;
 import org.opensearch.knn.index.query.request.MethodParameter;
 
 import java.util.Map;

src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java → src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java

@@ -1,17 +1,11 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
-import org.opensearch.knn.index.Parameter;
+import org.opensearch.knn.index.engine.model.QueryContext;
 
 import java.util.Collections;
 import java.util.Map;

src/main/java/org/opensearch/knn/index/util/JVMLibrary.java → src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java

@@ -3,10 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.knn.index.util;
-
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
+package org.opensearch.knn.index.engine;
 
 import java.util.Map;
 
@@ -23,7 +20,7 @@ public abstract class JVMLibrary extends AbstractKNNLibrary {
      * @param methods Map of k-NN methods that the library supports
      * @param version String representing version of library
      */
-    JVMLibrary(Map<String, KNNMethod> methods, Map<String, EngineSpecificMethodContext> engineMethodMetadataMap, String version) {
+    public JVMLibrary(Map<String, KNNMethod> methods, Map<String, EngineSpecificMethodContext> engineMethodMetadataMap, String version) {
         super(methods, engineMethodMetadataMap, version);
     }
 

src/main/java/org/opensearch/knn/index/util/KNNEngine.java → src/main/java/org/opensearch/knn/index/engine/KNNEngine.java

@@ -3,13 +3,14 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import com.google.common.collect.ImmutableSet;
 import org.opensearch.common.ValidationException;
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
 import org.opensearch.knn.index.SpaceType;
+import org.opensearch.knn.index.engine.faiss.Faiss;
+import org.opensearch.knn.index.engine.lucene.Lucene;
+import org.opensearch.knn.index.engine.nmslib.Nmslib;
 import org.opensearch.knn.training.VectorSpaceInfo;
 
 import java.util.List;

src/main/java/org/opensearch/knn/index/util/KNNLibrary.java → src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java

@@ -1,19 +1,11 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import org.opensearch.common.ValidationException;
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.training.VectorSpaceInfo;
 

src/main/java/org/opensearch/knn/index/KNNMethod.java → src/main/java/org/opensearch/knn/index/engine/KNNMethod.java

@@ -1,20 +1,15 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import org.opensearch.common.ValidationException;
 import org.opensearch.knn.common.KNNConstants;
+import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.training.VectorSpaceInfo;
 
 import java.util.ArrayList;

src/main/java/org/opensearch/knn/index/KNNMethodContext.java → src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java

@@ -1,15 +1,9 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -20,7 +14,7 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.SpaceType;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.mapper.MapperParsingException;

src/main/java/org/opensearch/knn/index/MethodComponent.java → src/main/java/org/opensearch/knn/index/engine/MethodComponent.java

@@ -1,15 +1,9 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import lombok.Getter;
 import org.opensearch.Version;
@@ -25,7 +19,7 @@
 import java.util.List;
 import java.util.ArrayList;
 
-import static org.opensearch.knn.validation.ParameterValidator.validateParameters;
+import static org.opensearch.knn.index.engine.validation.ParameterValidator.validateParameters;
 
 /**
  * MethodComponent defines the structure of an individual component that can make up an index

src/main/java/org/opensearch/knn/index/MethodComponentContext.java → src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java

@@ -1,15 +1,9 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;

src/main/java/org/opensearch/knn/index/util/NativeLibrary.java → src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java

@@ -3,12 +3,10 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import lombok.Getter;
 import org.opensearch.knn.common.KNNConstants;
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
 import org.opensearch.knn.index.SpaceType;
 
 import java.util.Map;
@@ -20,7 +18,7 @@
  * Abstract implementation of KNNLibrary. It contains several default methods and fields that
  * are common across different underlying libraries.
  */
-abstract class NativeLibrary extends AbstractKNNLibrary {
+public abstract class NativeLibrary extends AbstractKNNLibrary {
     private final Map<SpaceType, Function<Float, Float>> scoreTranslation;
     @Getter
     private final String extension;
@@ -34,7 +32,7 @@ abstract class NativeLibrary extends AbstractKNNLibrary {
      * @param version String representation of version of the library
      * @param extension String representing the extension that library files should use
      */
-    NativeLibrary(
+    public NativeLibrary(
         Map<String, KNNMethod> methods,
         Map<String, EngineSpecificMethodContext> engineMethods,
         Map<SpaceType, Function<Float, Float>> scoreTranslation,

src/main/java/org/opensearch/knn/index/Parameter.java → src/main/java/org/opensearch/knn/index/engine/Parameter.java

@@ -1,15 +1,9 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import org.opensearch.common.ValidationException;
 import org.opensearch.knn.training.VectorSpaceInfo;