Fixing the core breaking changes in various classes.

Signed-off-by: Navneet Verma <[email protected]>
opensearch-project · Aug 1, 2023 · 1453987 · 1453987
1 parent 84509fb
commit 1453987
Show file tree

Hide file tree

Showing 18 changed files with 76 additions and 72 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.9...2.x)
 ### Features
 ### Enhancements
+* Enabled the IVF algorithm to work with Filters of K-NN Query. [#1013](https://github.com/opensearch-project/k-NN/pull/1013)
 ### Bug Fixes
 ### Infrastructure
 ### Documentation

diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java
@@ -15,7 +15,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.core.xcontent.MediaTypeParserRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.util.KNNEngine;
 import org.opensearch.knn.indices.ModelMetadata;
@@ -186,7 +186,7 @@ public void searchKNNModel(String testModelID) throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
             assertEquals(EXP_NUM_OF_MODELS, searchResponse.getHits().getHits().length);
@@ -203,7 +203,7 @@ public void validateModelCreated(String modelId) throws Exception {
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
         assertEquals(modelId, responseMap.get(MODEL_ID));
         assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC);
     }

diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java
@@ -15,8 +15,8 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeUnit;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.index.IndexModule;
 import org.opensearch.knn.index.memory.NativeMemoryCacheManager;
@@ -37,7 +37,7 @@
 import static org.opensearch.common.settings.Setting.Property.Dynamic;
 import static org.opensearch.common.settings.Setting.Property.IndexScope;
 import static org.opensearch.common.settings.Setting.Property.NodeScope;
-import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue;
+import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue;
 import static org.opensearch.common.unit.MemorySizeValue.parseBytesSizeValueOrHeapRatio;
 
 /**

diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java
@@ -10,7 +10,7 @@
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.index.fielddata.IndexFieldData;
 import org.opensearch.index.fielddata.IndexFieldDataCache;
-import org.opensearch.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.MultiValueMode;
 import org.opensearch.search.aggregations.support.ValuesSourceType;

diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java
@@ -9,8 +9,9 @@
 import lombok.NonNull;
 import lombok.extern.log4j.Log4j2;
 import org.apache.lucene.store.ChecksumIndexInput;
-import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.xcontent.MediaTypeParserRegistry;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.knn.index.KNNSettings;
@@ -176,9 +177,12 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa
             parameters.put(PARAMETERS, algoParams);
         } else {
             parameters.putAll(
-                XContentFactory.xContent(MediaTypeParserRegistry.getDefaultMediaType())
-                    .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, parametersString)
-                    .map()
+                XContentHelper.createParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                    new BytesArray(parametersString),
+                    MediaTypeRegistry.getDefaultMediaType()
+                ).map()
             );
         }
 

diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java
@@ -10,7 +10,7 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.env.Environment;
 import org.opensearch.knn.KNNTestCase;
 import org.opensearch.knn.plugin.KNNPlugin;

diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java
@@ -29,7 +29,7 @@
 import org.junit.BeforeClass;
 import org.mockito.MockedStatic;
 import org.opensearch.common.io.PathUtils;
-import org.opensearch.common.unit.ByteSizeValue;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.knn.KNNTestCase;
 import org.opensearch.knn.index.KNNSettings;

diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java
@@ -17,7 +17,7 @@
 import org.opensearch.client.ResponseException;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.xcontent.MediaTypeParserRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.core.rest.RestStatus;
@@ -62,7 +62,7 @@ public void testDeleteModelExists() throws Exception {
         String responseBody = EntityUtils.toString(getModelResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 
@@ -99,7 +99,7 @@ public void testDeleteTrainingModel() throws Exception {
         String responseBody = EntityUtils.toString(getModelResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 
@@ -205,7 +205,7 @@ private void trainModel(String modelId, String trainingIndexName, String trainin
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 

diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java
@@ -16,7 +16,7 @@
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
-import org.opensearch.core.xcontent.MediaTypeParserRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.core.rest.RestStatus;
@@ -74,7 +74,7 @@ public void testGetModelExists() throws Exception {
         String responseBody = EntityUtils.toString(response.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
         assertEquals(modelId, responseMap.get(MODEL_ID));
         assertEquals(modelDescription, responseMap.get(MODEL_DESCRIPTION));
         assertEquals(FAISS.getName(), responseMap.get(KNN_ENGINE));
@@ -106,7 +106,7 @@ public void testGetModelExistsWithFilter() throws Exception {
         String responseBody = EntityUtils.toString(response.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertTrue(responseMap.size() == filteredPath.size());
         assertEquals(modelId, responseMap.get(MODEL_ID));

diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java
@@ -19,7 +19,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.xcontent.MediaTypeParserRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.index.query.MatchAllQueryBuilder;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
@@ -347,7 +347,7 @@ public void testModelIndexHealthMetricsStats() throws Exception {
 
             final Response response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName));
             final String responseBody = EntityUtils.toString(response.getEntity());
-            final Map<String, Object> statsMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+            final Map<String, Object> statsMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
             // Check that model health status is null since model index is not created to system yet
             assertNull(statsMap.get(StatNames.MODEL_INDEX_STATUS.getName()));
@@ -358,7 +358,7 @@ public void testModelIndexHealthMetricsStats() throws Exception {
         Response response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName));
 
         final String responseBody = EntityUtils.toString(response.getEntity());
-        final Map<String, Object> statsMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        final Map<String, Object> statsMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         // Check that model health status is not null
         assertNotNull(statsMap.get(modelIndexStatusName));
@@ -452,7 +452,7 @@ public void validateModelCreated(String modelId) throws Exception {
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
         assertEquals(modelId, responseMap.get(MODEL_ID));
         assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC);
     }

diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java
@@ -17,7 +17,7 @@
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
 import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.core.xcontent.MediaTypeParserRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.util.KNNEngine;
@@ -73,7 +73,7 @@ public void testNoModelExists() throws Exception {
         String responseBody = EntityUtils.toString(response.getEntity());
         assertNotNull(responseBody);
 
-        XContentParser parser = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody);
+        XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
         SearchResponse searchResponse = SearchResponse.fromXContent(parser);
         assertNotNull(searchResponse);
         assertEquals(searchResponse.getHits().getHits().length, 0);
@@ -133,7 +133,7 @@ public void testSearchModelExists() throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
 
@@ -177,7 +177,7 @@ public void testSearchModelWithoutSource() throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
 
@@ -225,7 +225,7 @@ public void testSearchModelWithSourceFilteringIncludes() throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
 
@@ -277,7 +277,7 @@ public void testSearchModelWithSourceFilteringExcludes() throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
 

diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java
@@ -15,7 +15,7 @@
 import org.opensearch.client.Response;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.core.xcontent.MediaTypeParserRegistry;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.core.rest.RestStatus;
 
@@ -96,8 +96,7 @@ public void testTrainModel_fail_notEnoughData() throws Exception {
         String trainResponseBody = EntityUtils.toString(trainResponse.getEntity());
         assertNotNull(trainResponseBody);
 
-        Map<String, Object> trainResponseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), trainResponseBody)
-            .map();
+        Map<String, Object> trainResponseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), trainResponseBody).map();
         String modelId = (String) trainResponseMap.get(MODEL_ID);
         assertNotNull(modelId);
 
@@ -106,7 +105,7 @@ public void testTrainModel_fail_notEnoughData() throws Exception {
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 
@@ -177,8 +176,7 @@ public void testTrainModel_fail_tooMuchData() throws Exception {
         String trainResponseBody = EntityUtils.toString(trainResponse.getEntity());
         assertNotNull(trainResponseBody);
 
-        Map<String, Object> trainResponseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), trainResponseBody)
-            .map();
+        Map<String, Object> trainResponseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), trainResponseBody).map();
         String modelId = (String) trainResponseMap.get(MODEL_ID);
         assertNotNull(modelId);
 
@@ -187,7 +185,7 @@ public void testTrainModel_fail_tooMuchData() throws Exception {
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 
@@ -258,7 +256,7 @@ public void testTrainModel_success_withId() throws Exception {
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 
@@ -328,8 +326,7 @@ public void testTrainModel_success_noId() throws Exception {
         String trainResponseBody = EntityUtils.toString(trainResponse.getEntity());
         assertNotNull(trainResponseBody);
 
-        Map<String, Object> trainResponseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), trainResponseBody)
-            .map();
+        Map<String, Object> trainResponseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), trainResponseBody).map();
         String modelId = (String) trainResponseMap.get(MODEL_ID);
         assertNotNull(modelId);
 
@@ -338,7 +335,7 @@ public void testTrainModel_success_noId() throws Exception {
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(MediaTypeParserRegistry.getDefaultMediaType().xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));