From e371d583c2ebb5733f580cf80f6c7dcc21ea57c6 Mon Sep 17 00:00:00 2001 From: Varun Jain Date: Mon, 1 Jan 2024 21:26:31 -0800 Subject: [PATCH] Addressing comments of Navneet Signed-off-by: Varun Jain --- ...backwards_compatibility_tests_workflow.yml | 23 ++++--------------- CHANGELOG.md | 1 + build.gradle | 3 +-- gradle.properties | 4 ++++ gradle/formatting.gradle | 2 +- .../AbstractRestartUpgradeRestTestCase.java | 1 - .../neuralsearch/bwc/SemanticSearchIT.java | 2 +- .../resources/processor/IndexMappings.json | 4 ++-- .../bwc/AbstractRollingUpgradeTestCase.java | 1 - .../neuralsearch/bwc/SemanticSearchIT.java | 2 +- .../resources/processor/IndexMappings.json | 4 ++-- .../processor/TextEmbeddingProcessorIT.java | 2 +- .../TextImageEmbeddingProcessorIT.java | 2 +- .../CreateModelGroupRequestBody.json | 2 +- .../processor/UploadModelRequestBody.json | 2 +- .../neuralsearch/BaseNeuralSearchIT.java | 19 +++++++-------- .../neuralsearch/BaseSparseEncodingIT.java | 2 +- 17 files changed, 32 insertions(+), 44 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 699cfcec0..200f08501 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -32,18 +32,10 @@ jobs: with: java-version: ${{ matrix.java }} - - if: startsWith(matrix.os,'windows') - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows + - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-$BWC_VERSION_RESTART_UPGRADE to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}' - - - if: startsWith(matrix.os,'ubuntu') - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu - run: | - echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testAgainstNewCluster -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE - + ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=$BWC_VERSION_RESTART_UPGRADE' Rolling-Upgrade-BWCTests-NeuralSearch: strategy: @@ -67,14 +59,7 @@ jobs: with: java-version: ${{ matrix.java }} - - if: startsWith(matrix.os,'windows') - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows - run: | - echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' - - - if: startsWith(matrix.os,'ubuntu') - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu + - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-$BWC_VERSION_ROLLING_UPGRADE to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} run: | echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE' diff --git a/CHANGELOG.md b/CHANGELOG.md index f6862064f..3ae6baab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,3 +26,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Documentation ### Maintenance ### Refactoring +- Added spotless check in the build ([#515](https://github.com/opensearch-project/neural-search/pull/515)) diff --git a/build.gradle b/build.gradle index 4b1b8e340..ac0ebaa9d 100644 --- a/build.gradle +++ b/build.gradle @@ -135,7 +135,7 @@ allprojects { project.dependencyLicenses.enabled = false project.thirdPartyAudit.enabled = false project.loggerUsageCheck.enabled = false - project.forbiddenApis.ignoreFailures = true + project.forbiddenApis.ignoreFailures = false project.forbiddenPatterns { setEnabled(false) } @@ -143,7 +143,6 @@ allprojects { project.validateNebulaPom.enabled = false project.licenseFile = rootProject.file('LICENSE.txt') project.noticeFile = rootProject.file('NOTICE.txt') - project.forbiddenApis.ignoreFailures = true } } diff --git a/gradle.properties b/gradle.properties index b4dc3433d..98663c787 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 # +# The BWC version here should always be the latest opensearch version set in +# https://github.com/opensearch-project/OpenSearch/blob/main/libs/core/src/main/java/org/opensearch/Version.java . +# Wired compatibility of OpenSearch works like 3.x version is compatible with 2.(latest-major) version. +# Therefore, to run rolling-upgrade BWC Test on local machine the BWC version here should be set 2.(latest-major). systemProp.bwc.version=2.12.0-SNAPSHOT # For fixing Spotless check with Java 17 diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle index 60cace641..197b848c9 100644 --- a/gradle/formatting.gradle +++ b/gradle/formatting.gradle @@ -26,7 +26,7 @@ allprojects { } format("license", { licenseHeaderFile("${rootProject.file("formatter/license-header.txt")}", "package "); - target("src/*/java/**/*.java") + target("src/*/java/**/*.java","qa/*/java/**/*.java") }) } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java index d82f523ed..cf985d759 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -16,7 +16,6 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearchIT { - // protected static String testIndex; @Before protected String getIndexNameForTest() { diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index f2b6088ee..10abdb7f5 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -76,7 +76,7 @@ private String registerModelGroupAndGetModelId(String requestBody) throws Except String modelGroupId = registerModelGroup( String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) ); - return uploadModelId(String.format(LOCALE, requestBody, modelGroupId)); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { diff --git a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json index cf8a5c317..fc4c13646 100644 --- a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,8 +6,8 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 6, - "number_of_replicas": 0 + "number_of_shards": 3, + "number_of_replicas": 1 }, "mappings": { "properties": { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java index 314b73433..98ce95b72 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -19,7 +19,6 @@ import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT { - // protected String testIndex; @Before protected String getIndexNameForTest() { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java index 3a9c6a9a6..87fde97cf 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -89,7 +89,7 @@ private String registerModelGroupAndGetModelId(String requestBody) throws Except String modelGroupId = registerModelGroup( String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) ); - return uploadModelId(String.format(LOCALE, requestBody, modelGroupId)); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json index cf8a5c317..fc4c13646 100644 --- a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -6,8 +6,8 @@ "refresh_interval": "30s", "default_pipeline": "%s" }, - "number_of_shards": 6, - "number_of_replicas": 0 + "number_of_shards": 3, + "number_of_replicas": 1 }, "mappings": { "properties": { diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java index cfdb6ebaf..97d1c857c 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java @@ -49,7 +49,7 @@ public void testTextEmbeddingProcessor() throws Exception { private String uploadTextEmbeddingModel() throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - return uploadModel(requestBody); + return registerModelGroupAndUploadModel(requestBody); } private void createTextEmbeddingIndex() throws Exception { diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java index fa83ae4e0..f3cd65694 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java @@ -57,7 +57,7 @@ public void testEmbeddingProcessor_whenIngestingDocumentWithSourceWithoutMatchin private String uploadModel() throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - return uploadModel(requestBody); + return registerModelGroupAndUploadModel(requestBody); } private void createTextImageEmbeddingIndex() throws Exception { diff --git a/src/test/resources/processor/CreateModelGroupRequestBody.json b/src/test/resources/processor/CreateModelGroupRequestBody.json index 51511222c..2dd0852a4 100644 --- a/src/test/resources/processor/CreateModelGroupRequestBody.json +++ b/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -1,4 +1,4 @@ { - "name": "", + "name": "%s", "description": "This is a public model group" } diff --git a/src/test/resources/processor/UploadModelRequestBody.json b/src/test/resources/processor/UploadModelRequestBody.json index 95f9c9cb5..17fab7588 100644 --- a/src/test/resources/processor/UploadModelRequestBody.json +++ b/src/test/resources/processor/UploadModelRequestBody.json @@ -4,7 +4,7 @@ "model_format": "TORCH_SCRIPT", "model_task_type": "text_embedding", "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", - "model_group_id": "", + "model_group_id": "%s", "model_config": { "model_type": "bert", "embedding_dimension": 768, diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 8580c5d64..680d90b65 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -134,14 +134,13 @@ protected void updateClusterSettings(String settingKey, Object value) { assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } - protected String uploadModel(String requestBody) throws Exception { - String modelGroupId = registerModelGroup(); + protected String registerModelGroupAndUploadModel(String requestBody) throws Exception { + String modelGroupId = getModelGroupId(); // model group id is dynamically generated, we need to update model update request body after group is registered - requestBody = requestBody.replace("", modelGroupId); - return uploadModelId(requestBody); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); } - protected String uploadModelId(String requestBody) throws Exception { + protected String uploadModel(String requestBody) throws Exception { Response uploadResponse = makeRequest( client(), "POST", @@ -204,7 +203,7 @@ protected void loadModel(String modelId) throws Exception { @SneakyThrows protected String prepareModel() { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - String modelId = uploadModel(requestBody); + String modelId = registerModelGroupAndUploadModel(requestBody); loadModel(modelId); return modelId; } @@ -791,11 +790,13 @@ protected String getDeployedModelId() { } @SneakyThrows - private String registerModelGroup() { + private String getModelGroupId() { String modelGroupRegisterRequestBody = Files.readString( Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) - ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); - return registerModelGroup(modelGroupRegisterRequestBody); + ); + return registerModelGroup( + String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) + ); } protected String registerModelGroup(String modelGroupRegisterRequestBody) throws IOException, ParseException { diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java index 4e547e090..3072bc767 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java @@ -34,7 +34,7 @@ protected String prepareModel() { String requestBody = Files.readString( Path.of(classLoader.getResource("processor/UploadSparseEncodingModelRequestBody.json").toURI()) ); - String modelId = uploadModel(requestBody); + String modelId = registerModelGroupAndUploadModel(requestBody); loadModel(modelId); return modelId; }