Merge branch 'main' into add_nsg_graph

# Conflicts:
#	src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java
#	src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java
#	src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java
#	src/testFixtures/java/org/opensearch/knn/TestUtils.java

.github/workflows/backwards_compatibility_tests_workflow.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         java: [ 11, 17 ]
         os: [ubuntu-latest]
-        bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0-SNAPSHOT" ]
+        bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0-SNAPSHOT"]
         opensearch_version : [ "3.0.0-SNAPSHOT" ]
         exclude:
           - os: windows-latest
@@ -88,7 +88,7 @@ jobs:
       matrix:
         java: [ 11, 17 ]
         os: [ubuntu-latest]
-        bwc_version: [ "2.9.0-SNAPSHOT" ]
+        bwc_version: [ "2.10.0-SNAPSHOT" ]
         opensearch_version: [ "3.0.0-SNAPSHOT" ]
 
     name: k-NN Rolling-Upgrade BWC Tests

CHANGELOG.md

@@ -16,8 +16,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.9...2.x)
 ### Features
 ### Enhancements
+* Enabled the IVF algorithm to work with Filters of K-NN Query. [#1013](https://github.com/opensearch-project/k-NN/pull/1013)
 ### Bug Fixes
 ### Infrastructure
 ### Documentation
 ### Maintenance
+* Update Guava Version to 32.0.1 [#1019](https://github.com/opensearch-project/k-NN/pull/1019)
 ### Refactoring
+* Fix TransportAddress Refactoring Changes in Core [#1020](https://github.com/opensearch-project/k-NN/pull/1020)

benchmarks/osb/requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile
 #
-aiohttp==3.8.1
+aiohttp==3.8.5
     # via opensearch-py
 aiosignal==1.2.0
     # via aiohttp
@@ -16,7 +16,7 @@ attrs==21.4.0
     #   jsonschema
 cachetools==4.2.4
     # via google-auth
-certifi==2022.12.7
+certifi==2023.7.22
     # via
     #   opensearch-benchmark
     #   opensearch-py
@@ -95,4 +95,4 @@ zipp==3.7.0
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools
+# setuptools

benchmarks/perf-tool/requirements.txt

@@ -8,7 +8,7 @@ cached-property==1.5.2
     # via h5py
 cerberus==1.3.4
     # via -r requirements.in
-certifi==2022.12.7
+certifi==2023.7.22
     # via
     #   opensearch-py
     #   requests

build.gradle

@@ -172,7 +172,7 @@ dependencies {
     api "org.opensearch:opensearch:${opensearch_version}"
     compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}"
     api group: 'com.google.guava', name: 'failureaccess', version:'1.0.1'
-    api group: 'com.google.guava', name: 'guava', version:'30.0-jre'
+    api group: 'com.google.guava', name: 'guava', version:'32.0.1-jre'
     api group: 'commons-lang', name: 'commons-lang', version: '2.6'
     testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}"
     testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.3'

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java

@@ -15,7 +15,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.util.KNNEngine;
 import org.opensearch.knn.indices.ModelMetadata;
@@ -186,7 +186,7 @@ public void searchKNNModel(String testModelID) throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
             assertEquals(EXP_NUM_OF_MODELS, searchResponse.getHits().getHits().length);
@@ -203,7 +203,7 @@ public void validateModelCreated(String modelId) throws Exception {
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
         assertEquals(modelId, responseMap.get(MODEL_ID));
         assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC);
     }

scripts/build.sh

@@ -105,26 +105,16 @@ if [ "$JAVA_HOME" = "" ]; then
     echo "SET JAVA_HOME=$JAVA_HOME"
 fi
 
-# Ensure gcc version is above 4.9.0 for faiss 1.7.4+ compilation
+# Ensure gcc version is above 4.9.0 and is not 8.3.1 for faiss 1.7.4+ compilation
 # https://github.com/opensearch-project/k-NN/issues/975
 GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3`
 GCC_REQUIRED_VERSION=4.9.0
 COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1`
-if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ]; then
-    echo "gcc version on this env is older than $GCC_REQUIRED_VERSION, exit 1"
+if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ] || [ "$GCC_VERSION" = "8.3.1" ]; then
+    echo "gcc version on this env is either older than $GCC_REQUIRED_VERSION, or equals 8.3.1, exit 1"
     exit 1
 fi
 
-# Ensure gcc version is below 8.0.0 for faiss 1.7.4+ compilation so it will not crash on arm64 CentOS7
-# https://github.com/opensearch-project/k-NN/issues/975
-GCC_REQUIRED_VERSION_CEILING=8.0.0
-COMPARE_VERSION_CEILING=`echo $GCC_REQUIRED_VERSION_CEILING $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | tail -n 1`
-if [ "$COMPARE_VERSION_CEILING" != "$GCC_REQUIRED_VERSION_CEILING" ] && (echo "$OSTYPE" | grep -i linux); then
-    echo "gcc version on this env is newer than $GCC_REQUIRED_VERSION_CEILING, exit 1"
-    exit 1
-fi
-
-
 # Build k-NN lib and plugin through gradle tasks
 cd $work_dir
 # Gradle build is used here to replace gradle assemble due to build will also call cmake and make before generating jars

src/main/java/org/opensearch/knn/index/query/KNNWeight.java

@@ -6,6 +6,7 @@
 package org.opensearch.knn.index.query;
 
 import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang.StringUtils;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.search.FilteredDocIdSetIterator;
@@ -49,6 +50,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
@@ -290,7 +292,7 @@ private Map<Integer, Float> doExactSearch(final LeafReaderContext leafReaderCont
         float[] queryVector = this.knnQuery.getQueryVector();
         try {
             final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName());
-            final SpaceType spaceType = SpaceType.getSpace(fieldInfo.getAttribute(SPACE_TYPE));
+            final SpaceType spaceType = getSpaceType(fieldInfo);
             // Creating min heap and init with MAX DocID and Score as -INF.
             final HitQueue queue = new HitQueue(this.knnQuery.getK(), true);
             ScoreDoc topDoc = queue.top();
@@ -351,4 +353,20 @@ public static float normalizeScore(float score) {
         if (score >= 0) return 1 / (1 + score);
         return -score + 1;
     }
+
+    private SpaceType getSpaceType(final FieldInfo fieldInfo) {
+        final String spaceTypeString = fieldInfo.getAttribute(SPACE_TYPE);
+        if (StringUtils.isNotEmpty(spaceTypeString)) {
+            return SpaceType.getSpace(spaceTypeString);
+        }
+
+        final String modelId = fieldInfo.getAttribute(MODEL_ID);
+        if (StringUtils.isNotEmpty(modelId)) {
+            ModelMetadata modelMetadata = modelDao.getMetadata(modelId);
+            return modelMetadata.getSpaceType();
+        }
+        throw new IllegalArgumentException(
+            String.format(Locale.ROOT, "Unable to find the Space Type from Field Info attribute for field %s", fieldInfo.getName())
+        );
+    }
 }

src/test/java/org/opensearch/knn/index/FaissIT.java

@@ -386,7 +386,9 @@ public void testDocDeletion() throws IOException {
         deleteKnnDoc(INDEX_NAME, "1");
     }
 
-    public void testEndToEnd_fromModel() throws Exception {
+    @SneakyThrows
+    public void testKNNQuery_withModelDifferentCombination_thenSuccess() {
+
         String modelId = "test-model";
         int dimension = 128;
 
@@ -435,10 +437,9 @@ public void testEndToEnd_fromModel() throws Exception {
         // Index some documents
         int numDocs = 100;
         for (int i = 0; i < numDocs; i++) {
-            Float[] indexVector = new Float[dimension];
+            float[] indexVector = new float[dimension];
             Arrays.fill(indexVector, (float) i);
-
-            addKnnDoc(indexName, Integer.toString(i), fieldName, indexVector);
+            addKnnDocWithAttributes(indexName, Integer.toString(i), fieldName, indexVector, ImmutableMap.of("rating", String.valueOf(i)));
         }
 
         // Run search and ensure that the values returned are expected
@@ -452,6 +453,34 @@ public void testEndToEnd_fromModel() throws Exception {
         for (int i = 0; i < k; i++) {
             assertEquals(numDocs - i - 1, Integer.parseInt(results.get(i).getDocId()));
         }
+
+        // doing exact search with filters
+        Response exactSearchFilteredResponse = searchKNNIndex(
+            indexName,
+            new KNNQueryBuilder(fieldName, queryVector, k, QueryBuilders.rangeQuery("rating").gte("90").lte("99")),
+            k
+        );
+        List<KNNResult> exactSearchFilteredResults = parseSearchResponse(
+            EntityUtils.toString(exactSearchFilteredResponse.getEntity()),
+            fieldName
+        );
+        for (int i = 0; i < k; i++) {
+            assertEquals(numDocs - i - 1, Integer.parseInt(exactSearchFilteredResults.get(i).getDocId()));
+        }
+
+        // doing exact search with filters
+        Response aNNSearchFilteredResponse = searchKNNIndex(
+            indexName,
+            new KNNQueryBuilder(fieldName, queryVector, k, QueryBuilders.rangeQuery("rating").gte("80").lte("99")),
+            k
+        );
+        List<KNNResult> aNNSearchFilteredResults = parseSearchResponse(
+            EntityUtils.toString(aNNSearchFilteredResponse.getEntity()),
+            fieldName
+        );
+        for (int i = 0; i < k; i++) {
+            assertEquals(numDocs - i - 1, Integer.parseInt(aNNSearchFilteredResults.get(i).getDocId()));
+        }
     }
 
     @SneakyThrows

src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java

@@ -17,7 +17,7 @@
 import org.opensearch.client.ResponseException;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.core.rest.RestStatus;
@@ -62,7 +62,7 @@ public void testDeleteModelExists() throws Exception {
         String responseBody = EntityUtils.toString(getModelResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 
@@ -99,7 +99,7 @@ public void testDeleteTrainingModel() throws Exception {
         String responseBody = EntityUtils.toString(getModelResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 
@@ -205,7 +205,7 @@ private void trainModel(String modelId, String trainingIndexName, String trainin
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertEquals(modelId, responseMap.get(MODEL_ID));
 

src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java

@@ -16,7 +16,7 @@
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.core.rest.RestStatus;
@@ -74,7 +74,7 @@ public void testGetModelExists() throws Exception {
         String responseBody = EntityUtils.toString(response.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
         assertEquals(modelId, responseMap.get(MODEL_ID));
         assertEquals(modelDescription, responseMap.get(MODEL_DESCRIPTION));
         assertEquals(FAISS.getName(), responseMap.get(KNN_ENGINE));
@@ -106,7 +106,7 @@ public void testGetModelExistsWithFilter() throws Exception {
         String responseBody = EntityUtils.toString(response.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         assertTrue(responseMap.size() == filteredPath.size());
         assertEquals(modelId, responseMap.get(MODEL_ID));

src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java

@@ -19,7 +19,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.index.query.MatchAllQueryBuilder;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
@@ -347,7 +347,7 @@ public void testModelIndexHealthMetricsStats() throws Exception {
 
             final Response response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName));
             final String responseBody = EntityUtils.toString(response.getEntity());
-            final Map<String, Object> statsMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+            final Map<String, Object> statsMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
             // Check that model health status is null since model index is not created to system yet
             assertNull(statsMap.get(StatNames.MODEL_INDEX_STATUS.getName()));
@@ -358,7 +358,7 @@ public void testModelIndexHealthMetricsStats() throws Exception {
         Response response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName));
 
         final String responseBody = EntityUtils.toString(response.getEntity());
-        final Map<String, Object> statsMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+        final Map<String, Object> statsMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
 
         // Check that model health status is not null
         assertNotNull(statsMap.get(modelIndexStatusName));
@@ -452,7 +452,7 @@ public void validateModelCreated(String modelId) throws Exception {
         String responseBody = EntityUtils.toString(getResponse.getEntity());
         assertNotNull(responseBody);
 
-        Map<String, Object> responseMap = createParser(XContentType.JSON.xContent(), responseBody).map();
+        Map<String, Object> responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map();
         assertEquals(modelId, responseMap.get(MODEL_ID));
         assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC);
     }

src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java

@@ -17,7 +17,7 @@
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
 import org.opensearch.core.xcontent.XContentParser;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.util.KNNEngine;
@@ -73,7 +73,7 @@ public void testNoModelExists() throws Exception {
         String responseBody = EntityUtils.toString(response.getEntity());
         assertNotNull(responseBody);
 
-        XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody);
+        XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
         SearchResponse searchResponse = SearchResponse.fromXContent(parser);
         assertNotNull(searchResponse);
         assertEquals(searchResponse.getHits().getHits().length, 0);
@@ -133,7 +133,7 @@ public void testSearchModelExists() throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
 
@@ -177,7 +177,7 @@ public void testSearchModelWithoutSource() throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
 
@@ -225,7 +225,7 @@ public void testSearchModelWithSourceFilteringIncludes() throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
 
@@ -277,7 +277,7 @@ public void testSearchModelWithSourceFilteringExcludes() throws Exception {
             String responseBody = EntityUtils.toString(response.getEntity());
             assertNotNull(responseBody);
 
-            XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody);
+            XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);