Allow validation for non knn index only after 2.17.0

CHANGELOG.md

@@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduced a writing layer in native engines where relies on the writing interface to process IO. (#2241)[https://github.com/opensearch-project/k-NN/pull/2241]
 ### Bug Fixes
 * Fixing the bug when a segment has no vector field present for disk based vector search (#2282)[https://github.com/opensearch-project/k-NN/pull/2282]
+* Allow validation for non knn index only after 2.17.0 (#2315)[https://github.com/opensearch-project/k-NN/pull/2315]
 ### Infrastructure
 * Updated C++ version in JNI from c++11 to c++17 [#2259](https://github.com/opensearch-project/k-NN/pull/2259)
 * Upgrade bytebuddy and objenesis version to match OpenSearch core and, update github ci runner for macos [#2279](https://github.com/opensearch-project/k-NN/pull/2279)

qa/restart-upgrade/build.gradle

@@ -96,6 +96,20 @@ testClusters {
                 excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneForceMerge"
             }
         }
+        if (!(knn_bwc_version.startsWith("2.13.") ||
+                knn_bwc_version.startsWith("2.14.") ||
+                knn_bwc_version.startsWith("2.15.") ||
+                knn_bwc_version.startsWith("2.16."))) {
+            filter {
+                excludeTestsMatching "org.opensearch.knn.bwc.ModelIT.testNonKNNIndex_withModelId"
+                excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withFaissEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withNmslibEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withFaissEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withNmslibEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
+            }
+        }
 
         nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
         nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
@@ -170,6 +184,21 @@ testClusters {
             }
         }
 
+        if (!(knn_bwc_version.startsWith("2.13.") ||
+                knn_bwc_version.startsWith("2.14.") ||
+                knn_bwc_version.startsWith("2.15.") ||
+                knn_bwc_version.startsWith("2.16."))) {
+            filter {
+                excludeTestsMatching "org.opensearch.knn.bwc.ModelIT.testNonKNNIndex_withModelId"
+                excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withFaissEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withNmslibEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withFaissEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withNmslibEngine"
+                excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
+            }
+        }
+
         nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
         nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
         systemProperty 'tests.security.manager', 'false'

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java

@@ -20,9 +20,12 @@ public abstract class AbstractRestartUpgradeTestCase extends KNNRestTestCase {
 
     @Before
     protected void setIndex() {
-        // Creating index name by concatenating "knn-bwc-" prefix with test method name
-        // for all the tests in this sub-project
-        testIndex = KNN_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT);
+        // Creating index name by concatenating "knn-bwc-" prefix with test class name and then with method name
+        // for all the tests in this sub-project to generate unique index name
+        testIndex = new StringBuilder().append(KNN_BWC_PREFIX)
+            .append(getTestClass().getName().toLowerCase(Locale.ROOT))
+            .append(getTestName().toLowerCase(Locale.ROOT))
+            .toString();
     }
 
     @Override

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java

@@ -43,24 +43,27 @@ public class ModelIT extends AbstractRestartUpgradeTestCase {
     private static final String TEST_MODEL_INDEX_DEFAULT = KNN_BWC_PREFIX + "test-model-index-default";
     private static final String TRAINING_INDEX = KNN_BWC_PREFIX + "train-index";
     private static final String TRAINING_INDEX_DEFAULT = KNN_BWC_PREFIX + "train-index-default";
+    private static final String TRAINING_INDEX_FOR_NON_KNN_INDEX = KNN_BWC_PREFIX + "train-index-for-non-knn-index";
     private static final String TRAINING_FIELD = "train-field";
     private static final String TEST_FIELD = "test-field";
     private static final int DIMENSIONS = 5;
     private static int DOC_ID = 0;
     private static int DOC_ID_TEST_MODEL_INDEX = 0;
     private static int DOC_ID_TEST_MODEL_INDEX_DEFAULT = 0;
     private static final int DELAY_MILLI_SEC = 1000;
-    private static final int EXP_NUM_OF_MODELS = 2;
+    private static final int MIN_NUM_OF_MODELS = 2;
     private static final int K = 5;
     private static final int NUM_DOCS = 10;
     private static final int NUM_DOCS_TEST_MODEL_INDEX = 100;
     private static final int NUM_DOCS_TEST_MODEL_INDEX_DEFAULT = 100;
+    private static final int NUM_DOCS_TEST_MODEL_INDEX_FOR_NON_KNN_INDEX = 100;
     private static final int NUM_OF_ATTEMPTS = 30;
     private static int QUERY_COUNT = 0;
     private static int QUERY_COUNT_TEST_MODEL_INDEX = 0;
     private static int QUERY_COUNT_TEST_MODEL_INDEX_DEFAULT = 0;
     private static final String TEST_MODEL_ID = "test-model-id";
     private static final String TEST_MODEL_ID_DEFAULT = "test-model-id-default";
+    private static final String TEST_MODEL_ID_FOR_NON_KNN_INDEX = "test-model-id-for-non-knn-index";
     private static final String MODEL_DESCRIPTION = "Description for train model test";
 
     // KNN model test
@@ -135,6 +138,32 @@ public void testKNNModelDefault() throws Exception {
         }
     }
 
+    public void testNonKNNIndex_withModelId() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+
+            // Create a training index and randomly ingest data into it
+            createBasicKnnIndex(TRAINING_INDEX_FOR_NON_KNN_INDEX, TRAINING_FIELD, DIMENSIONS);
+            bulkIngestRandomVectors(TRAINING_INDEX_FOR_NON_KNN_INDEX, TRAINING_FIELD, NUM_DOCS, DIMENSIONS);
+
+            trainKNNModel(TEST_MODEL_ID_FOR_NON_KNN_INDEX, TRAINING_INDEX_FOR_NON_KNN_INDEX, TRAINING_FIELD, DIMENSIONS, MODEL_DESCRIPTION);
+            validateModelCreated(TEST_MODEL_ID_FOR_NON_KNN_INDEX);
+
+            createKnnIndex(
+                testIndex,
+                createKNNDefaultScriptScoreSettings(),
+                modelIndexMapping(TEST_FIELD, TEST_MODEL_ID_FOR_NON_KNN_INDEX)
+            );
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+        } else {
+            Thread.sleep(1000);
+            DOC_ID = NUM_DOCS;
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+            deleteKNNIndex(testIndex);
+            deleteKNNIndex(TRAINING_INDEX_FOR_NON_KNN_INDEX);
+            deleteKNNModel(TEST_MODEL_ID_FOR_NON_KNN_INDEX);
+        }
+    }
+
     // Delete Models and ".opensearch-knn-models" index to clear cluster metadata
     @AfterClass
     public static void wipeAllModels() throws IOException {
@@ -168,7 +197,7 @@ public void searchKNNModel(String testModelID) throws Exception {
             XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
             SearchResponse searchResponse = SearchResponse.fromXContent(parser);
             assertNotNull(searchResponse);
-            assertEquals(EXP_NUM_OF_MODELS, searchResponse.getHits().getHits().length);
+            assertTrue(MIN_NUM_OF_MODELS <= searchResponse.getHits().getHits().length);
 
             for (SearchHit hit : searchResponse.getHits().getHits()) {
                 assertTrue(hit.getId().startsWith(testModelID));

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/PainlessScriptScoringIT.java

@@ -5,6 +5,9 @@
 
 package org.opensearch.knn.bwc;
 
+import org.opensearch.knn.index.SpaceType;
+import org.opensearch.knn.index.engine.KNNEngine;
+
 public class PainlessScriptScoringIT extends AbstractRestartUpgradeTestCase {
     private static final String TEST_FIELD = "test-field";
     private static final int DIMENSIONS = 5;
@@ -53,4 +56,72 @@ public void testKNNL1PainlessScriptScore() throws Exception {
         }
     }
 
+    public void testNonKNNIndex_withMethodParams_withFaissEngine() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createKnnIndex(
+                testIndex,
+                createKNNDefaultScriptScoreSettings(),
+                createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.FAISS.getName(), SpaceType.DEFAULT.getValue(), false)
+            );
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+        } else {
+            DOC_ID = NUM_DOCS;
+            QUERY_COUNT = NUM_DOCS;
+            String source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+            QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
+            source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            forceMergeKnnIndex(testIndex, 1);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            deleteKNNIndex(testIndex);
+        }
+    }
+
+    public void testNonKNNIndex_withMethodParams_withNmslibEngine() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createKnnIndex(
+                testIndex,
+                createKNNDefaultScriptScoreSettings(),
+                createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.NMSLIB.getName(), SpaceType.DEFAULT.getValue(), false)
+            );
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+        } else {
+            DOC_ID = NUM_DOCS;
+            QUERY_COUNT = NUM_DOCS;
+            String source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+            QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
+            source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            forceMergeKnnIndex(testIndex, 1);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            deleteKNNIndex(testIndex);
+        }
+    }
+
+    public void testNonKNNIndex_withMethodParams_withLuceneEngine() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createKnnIndex(
+                testIndex,
+                createKNNDefaultScriptScoreSettings(),
+                createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.LUCENE.getName(), SpaceType.DEFAULT.getValue(), false)
+            );
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+        } else {
+            DOC_ID = NUM_DOCS;
+            QUERY_COUNT = NUM_DOCS;
+            String source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+            QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
+            source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            forceMergeKnnIndex(testIndex, 1);
+            validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
+            deleteKNNIndex(testIndex);
+        }
+    }
 }

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java

@@ -14,6 +14,7 @@
 import org.opensearch.knn.KNNResult;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.core.rest.RestStatus;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.util.Collections;
 import java.util.HashMap;
@@ -96,6 +97,63 @@ public void testKNNInnerProductScriptScore() throws Exception {
         }
     }
 
+    public void testNonKNNIndex_withMethodParams_withFaissEngine() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createKnnIndex(
+                testIndex,
+                createKNNDefaultScriptScoreSettings(),
+                createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.FAISS.getName(), SpaceType.DEFAULT.getValue(), false)
+            );
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+        } else {
+            QUERY_COUNT = NUM_DOCS;
+            DOC_ID = NUM_DOCS;
+            validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+            QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
+            validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
+            deleteKNNIndex(testIndex);
+        }
+    }
+
+    public void testNonKNNIndex_withMethodParams_withNmslibEngine() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createKnnIndex(
+                testIndex,
+                createKNNDefaultScriptScoreSettings(),
+                createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.NMSLIB.getName(), SpaceType.DEFAULT.getValue(), false)
+            );
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+        } else {
+            QUERY_COUNT = NUM_DOCS;
+            DOC_ID = NUM_DOCS;
+            validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+            QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
+            validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
+            deleteKNNIndex(testIndex);
+        }
+    }
+
+    public void testNonKNNIndex_withMethodParams_withLuceneEngine() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            createKnnIndex(
+                testIndex,
+                createKNNDefaultScriptScoreSettings(),
+                createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.LUCENE.getName(), SpaceType.DEFAULT.getValue(), false)
+            );
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+        } else {
+            QUERY_COUNT = NUM_DOCS;
+            DOC_ID = NUM_DOCS;
+            validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
+            addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
+            QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
+            validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
+            deleteKNNIndex(testIndex);
+        }
+    }
+
     // Validate Script score search for space_type : "inner_product"
     private void validateKNNInnerProductScriptScoreSearch(String testIndex, String testField, int dimension, int numDocs, int k)
         throws Exception {
@@ -121,5 +179,4 @@ private void validateKNNInnerProductScriptScoreSearch(String testIndex, String t
             assertEquals(expDocID, actualDocID);
         }
     }
-
 }

src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java

@@ -262,7 +262,10 @@ public KNNVectorFieldMapper build(BuilderContext context) {
                 );
             }
 
-            if (originalParameters.getResolvedKnnMethodContext() == null) {
+            // return FlatVectorFieldMapper only for indices that are created on or after 2.17.0, for others, use either LuceneFieldMapper
+            // or
+            // MethodFieldMapper to maintain backwards compatibility
+            if (originalParameters.getResolvedKnnMethodContext() == null && context.indexCreatedVersion().onOrAfter(Version.V_2_17_0)) {
                 return FlatVectorFieldMapper.createFieldMapper(
                     buildFullName(context),
                     name,
@@ -375,8 +378,8 @@ public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserCont
                 );
             }
 
-            // Check for flat configuration
-            if (isKNNDisabled(parserContext.getSettings())) {
+            // Check for flat configuration and validate only if index is created after 2.17
+            if (isKNNDisabled(parserContext.getSettings()) && parserContext.indexVersionCreated().onOrAfter(Version.V_2_17_0)) {
                 validateFromFlat(builder);
             } else if (builder.modelId.get() != null) {
                 validateFromModel(builder);

src/test/java/org/opensearch/knn/index/OpenSearchIT.java

@@ -814,6 +814,30 @@ public void testKNNIndex_whenBuildVectorDataStructureIsLessThanDocCount_thenBuil
         deleteKNNIndex(indexName);
     }
 
+    public void testCreateNonKNNIndex_withKNNModelID_throwsException() throws Exception {
+        Settings settings = Settings.builder().put(createKNNDefaultScriptScoreSettings()).build();
+        ResponseException ex = expectThrows(
+            ResponseException.class,
+            () -> createKnnIndex(INDEX_NAME, settings, createKnnIndexMapping(FIELD_NAME, "random-model-id"))
+        );
+        String expMessage = "Cannot set modelId or method parameters when index.knn setting is false";
+        assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString(expMessage));
+    }
+
+    public void testCreateNonKNNIndex_withKNNMethodParams_throwsException() throws Exception {
+        Settings settings = Settings.builder().put(createKNNDefaultScriptScoreSettings()).build();
+        ResponseException ex = expectThrows(
+            ResponseException.class,
+            () -> createKnnIndex(
+                INDEX_NAME,
+                settings,
+                createKnnIndexMapping(FIELD_NAME, 2, "hnsw", KNNEngine.FAISS.getName(), SpaceType.DEFAULT.getValue(), false)
+            )
+        );
+        String expMessage = "Cannot set modelId or method parameters when index.knn setting is false";
+        assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString(expMessage));
+    }
+
     /*
       For this testcase, we will create index with setting build_vector_data_structure_threshold as -1, then index few documents, perform knn search,
       then, confirm hits because of exact search though there are no graph. In next step, update setting to 0, force merge segment to 1, perform knn search and confirm expected