diff --git a/.github/draft-release-notes-config.yml b/.github/draft-release-notes-config.yml index d168727b4..3fe43c8d4 100644 --- a/.github/draft-release-notes-config.yml +++ b/.github/draft-release-notes-config.yml @@ -37,4 +37,4 @@ categories: - 'Maintenance' - title: 'Refactoring' labels: - - 'Refactoring' \ No newline at end of file + - 'Refactoring' diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml index 9dcc7020d..15b9a5565 100644 --- a/.github/workflows/add-untriaged.yml +++ b/.github/workflows/add-untriaged.yml @@ -1,19 +1,19 @@ -name: Apply 'untriaged' label during issue lifecycle - -on: - issues: - types: [opened, reopened, transferred] - -jobs: - apply-label: - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['untriaged'] - }) +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml new file mode 100644 index 000000000..f0490155c --- /dev/null +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -0,0 +1,65 @@ +name: Backwards Compatibility Tests NeuralSearch +on: + push: + branches: + - "*" + - "feature/**" + pull_request: + branches: + - "*" + - "feature/**" + +jobs: + Restart-Upgrade-BWCTests-NeuralSearch: + strategy: + matrix: + java: [ 11, 17, 21 ] + os: [ubuntu-latest,windows-latest] + bwc_version : ["2.9.0","2.10.0","2.11.0","2.12.0-SNAPSHOT"] + opensearch_version : [ "3.0.0-SNAPSHOT" ] + + name: NeuralSearch Restart-Upgrade BWC Tests + runs-on: ${{ matrix.os }} + env: + BWC_VERSION_RESTART_UPGRADE: ${{ matrix.bwc_version }} + + steps: + - name: Checkout NeuralSearch + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} + run: | + echo "Running restart-upgrade backwards compatibility tests ..." + ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}' + + Rolling-Upgrade-BWCTests-NeuralSearch: + strategy: + matrix: + java: [ 11, 17, 21 ] + os: [ubuntu-latest,windows-latest] + bwc_version: [ "2.12.0-SNAPSHOT" ] + opensearch_version: [ "3.0.0-SNAPSHOT" ] + + name: NeuralSearch Rolling-Upgrade BWC Tests + runs-on: ${{ matrix.os }} + env: + BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }} + + steps: + - name: Checkout NeuralSearch + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}} + run: | + echo "Running rolling-upgrade backwards compatibility tests ..." + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index d654df6b4..f24f022b0 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -1,9 +1,9 @@ name: Delete merged branch of the backport PRs -on: +on: pull_request: types: - closed - + jobs: delete-branch: runs-on: ubuntu-latest @@ -12,4 +12,4 @@ jobs: - name: Delete merged branch uses: SvanBoxel/delete-merged-branch@main env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 3d0b81ad5..f41cc14bc 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -20,4 +20,4 @@ jobs: env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - name: Fail if there were link errors - run: exit ${{ steps.lychee.outputs.exit_code }} \ No newline at end of file + run: exit ${{ steps.lychee.outputs.exit_code }} diff --git a/CHANGELOG.md b/CHANGELOG.md index c9896ff13..3ae6baab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes - Fixing multiple issues reported in #497 ([#524](https://github.com/opensearch-project/neural-search/pull/524)) ### Infrastructure +- BWC tests for Neural Search ([#515](https://github.com/opensearch-project/neural-search/pull/515)) ### Documentation ### Maintenance ### Refactoring +- Added spotless check in the build ([#515](https://github.com/opensearch-project/neural-search/pull/515)) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0ce29d811..55678007e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -33,7 +33,7 @@ git clone https://github.com/[your username]/neural-search.git #### JDK 11 -OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable +OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. One easy way to get Java 11 on *nix is to use [sdkman](https://sdkman.io/). @@ -83,10 +83,10 @@ Please follow these formatting guidelines: ## Build -OpenSearch neural-search uses a [Gradle](https://docs.gradle.org/6.6.1/userguide/userguide.html) wrapper for its build. +OpenSearch neural-search uses a [Gradle](https://docs.gradle.org/6.6.1/userguide/userguide.html) wrapper for its build. Run `gradlew` on Unix systems. -Build OpenSearch neural-search using `gradlew build` +Build OpenSearch neural-search using `gradlew build` ``` ./gradlew build @@ -221,8 +221,8 @@ See [CONTRIBUTING](CONTRIBUTING.md). ## Backports -The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the -original PR with an appropriate label `backport ` is merged to main with the backport workflow -run successfully on the PR. For example, if a PR on main needs to be backported to `2.x` branch, add a label -`backport 2.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is +The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the +original PR with an appropriate label `backport ` is merged to main with the backport workflow +run successfully on the PR. For example, if a PR on main needs to be backported to `2.x` branch, add a label +`backport 2.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `2.x` branch. diff --git a/README.md b/README.md index b40318f01..979a0c51d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) ## OpenSearch Neural Search -**OpenSearch Neural Search** is an OpenSearch plugin that adds dense neural retrieval into the OpenSearch ecosystem. +**OpenSearch Neural Search** is an OpenSearch plugin that adds dense neural retrieval into the OpenSearch ecosystem. The plugin provides the capability for indexing documents and doing neural search on the indexed documents. ## Project Resources diff --git a/build.gradle b/build.gradle index a9368ed02..ac0ebaa9d 100644 --- a/build.gradle +++ b/build.gradle @@ -11,12 +11,14 @@ import org.opensearch.gradle.test.RestIntegTestTask import java.util.concurrent.Callable apply plugin: 'java' +apply plugin: 'java-test-fixtures' apply plugin: 'idea' apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.pluginzip' apply plugin: 'jacoco' apply plugin: "com.diffplug.spotless" apply plugin: 'io.freefair.lombok' +apply from: 'gradle/formatting.gradle' def pluginName = 'opensearch-neural-search' def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem' @@ -123,6 +125,25 @@ allprojects { version = "${opensearch_build}" targetCompatibility = JavaVersion.VERSION_11 sourceCompatibility = JavaVersion.VERSION_11 + + apply from: rootProject.file('repositories.gradle').absoluteFile + plugins.withId('java') { + sourceCompatibility = targetCompatibility = "11" + } + + afterEvaluate { + project.dependencyLicenses.enabled = false + project.thirdPartyAudit.enabled = false + project.loggerUsageCheck.enabled = false + project.forbiddenApis.ignoreFailures = false + project.forbiddenPatterns { + setEnabled(false) + } + project.testingConventions.enabled = false + project.validateNebulaPom.enabled = false + project.licenseFile = rootProject.file('LICENSE.txt') + project.noticeFile = rootProject.file('NOTICE.txt') + } } repositories { @@ -144,6 +165,7 @@ dependencies { zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" + testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0' // ml-common excluded reflection for runtime so we need to add it by ourselves. // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9 @@ -154,6 +176,10 @@ dependencies { runtimeOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0' runtimeOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1' runtimeOnly group: 'org.json', name: 'json', version: '20231013' + testFixturesImplementation "org.opensearch:common-utils:${version}" + testFixturesImplementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0' + testFixturesCompileOnly group: 'com.google.guava', name: 'guava', version:'32.0.1-jre' + testFixturesCompileOnly fileTree(dir: knnJarDirectory, include: '*.jar') } // In order to add the jar to the classpath, we need to unzip the @@ -173,6 +199,9 @@ compileJava { compileTestJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } +compileTestFixturesJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) +} def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() @@ -284,16 +313,6 @@ run { useCluster testClusters.integTest } -spotless { - java { - removeUnusedImports() - importOrder 'java', 'javax', 'org', 'com' - eclipse().configFile rootProject.file('formatterConfig.xml') - trimTrailingWhitespace() - endWithNewline() - } -} - jacocoTestReport { dependsOn integTest, test reports { @@ -302,6 +321,7 @@ jacocoTestReport { } } +check.dependsOn spotlessCheck check.dependsOn jacocoTestCoverageVerification jacocoTestCoverageVerification.dependsOn jacocoTestReport diff --git a/formatterConfig.xml b/formatter/formatterConfig.xml similarity index 100% rename from formatterConfig.xml rename to formatter/formatterConfig.xml diff --git a/formatter/license-header.txt b/formatter/license-header.txt new file mode 100644 index 000000000..1ce60bda7 --- /dev/null +++ b/formatter/license-header.txt @@ -0,0 +1,4 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 5e5cd9ced..98663c787 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,6 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 # +# The BWC version here should always be the latest opensearch version set in +# https://github.com/opensearch-project/OpenSearch/blob/main/libs/core/src/main/java/org/opensearch/Version.java . +# Wired compatibility of OpenSearch works like 3.x version is compatible with 2.(latest-major) version. +# Therefore, to run rolling-upgrade BWC Test on local machine the BWC version here should be set 2.(latest-major). +systemProp.bwc.version=2.12.0-SNAPSHOT + # For fixing Spotless check with Java 17 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle new file mode 100644 index 000000000..197b848c9 --- /dev/null +++ b/gradle/formatting.gradle @@ -0,0 +1,32 @@ +allprojects { + project.apply plugin: "com.diffplug.spotless" + spotless { + java { + // Normally this isn't necessary, but we have Java sources in + // non-standard places + target '**/*.java' + + removeUnusedImports() + eclipse().configFile rootProject.file('formatter/formatterConfig.xml') + trimTrailingWhitespace() + endWithNewline(); + + custom 'Refuse wildcard imports', { + // Wildcard imports can't be resolved; fail the build + if (it =~ /\s+import .*\*;/) { + throw new AssertionError("Do not use wildcard imports. 'spotlessApply' cannot resolve this issue.") + } + } + } + format 'misc', { + target '*.md', '**/*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg' + + trimTrailingWhitespace() + endWithNewline() + } + format("license", { + licenseHeaderFile("${rootProject.file("formatter/license-header.txt")}", "package "); + target("src/*/java/**/*.java","qa/*/java/**/*.java") + }) + } +} diff --git a/qa/build.gradle b/qa/build.gradle new file mode 100644 index 000000000..511dcc442 --- /dev/null +++ b/qa/build.gradle @@ -0,0 +1,199 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.apache.tools.ant.taskdefs.condition.Os + +import java.nio.file.Files +import java.util.concurrent.Callable +import java.nio.file.Path + +apply plugin: 'opensearch.testclusters' +apply plugin: 'opensearch.build' +apply plugin: 'opensearch.rest-test' +apply plugin: 'io.freefair.lombok' + +// Disable a few tasks that come with build +build.enabled = false +integTest.enabled = false +test.enabled = false +assemble.enabled = false +dependenciesInfo.enabled = false + +configurations { + zipArchive +} + +def knnJarDirectory = "$rootDir/build/dependencies/opensearch-knn" + +dependencies { + api "org.opensearch:opensearch:${opensearch_version}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" + compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') + api "org.apache.logging.log4j:log4j-api:${versions.log4j}" + api "org.apache.logging.log4j:log4j-core:${versions.log4j}" + api "junit:junit:${versions.junit}" + testImplementation "org.opensearch.test:framework:${opensearch_version}" + testImplementation(testFixtures(rootProject)) +} + +ext { + licenseFile = rootProject.file('LICENSE.txt') + noticeFile = rootProject.file('NOTICE.txt') +} + +def tmp_dir = project.file('build/private/artifact_tmp').absoluteFile +tmp_dir.mkdirs() +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT") +String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version + +String os_platform = "linux" +String artifact_type = "tar" +String file_ext = "tar.gz" + +if (Os.isFamily(Os.FAMILY_WINDOWS)) { + os_platform = "windows" + artifact_type = "zip" + file_ext = "zip" +} + +ext{ + plugins = [provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-ml-plugin-${opensearch_build}.zip"}.getSingleFile() + } + } + } + }), provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.zipArchive.asFileTree.matching{include "**/opensearch-knn-${opensearch_build}.zip"}.getSingleFile() + } + } + } + }), rootProject.tasks.bundlePlugin.archiveFile] +} + +task deleteTempDirectories { + doFirst { + File[] tempFiles = tmp_dir.listFiles() + for (File child : tempFiles) { + if (child.exists() && child.toString().contains("opensearch-")) { + Files.delete(child.toPath()); + } + } + } +} + +// Task to pull opensearch artifact from archive +task pullOpensearchArtifact { + dependsOn "deleteTempDirectories" + + doLast{ + ext{ + if (isSnapshot) { + srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}" + } else { + srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}" + } + } + ant.get( + src: srcUrl, + dest: tmp_dir.absolutePath, + httpusecaches: false + ) + copy { + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + from zipTree(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } else { + from tarTree(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")) + } + into tmp_dir.absolutePath + } + } +} + +// Task to pull ml plugin from archive +task pullMlCommonsBwcPlugin { + doLast { + copy { + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml")) + into Path.of(tmp_dir.absolutePath, "opensearch-ml") + } + } +} + +// Task to pull KNN plugin from archive +task pullKnnBwcPlugin { + dependsOn "pullOpensearchArtifact" + + doLast { + copy { + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) + into Path.of(tmp_dir.absolutePath, "opensearch-knn") + } + } +} + +// Task to pull neural search plugin from archive +task pullBwcPlugin { + doLast { + copy { + from(Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search")) + into Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + } + delete Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}") + } +} + +// Task to unzip ml-commons plugin from archive +task unZipBwcMlCommonsPlugin(type: Zip) { + dependsOn "pullMlCommonsBwcPlugin" + from(Path.of(tmp_dir.absolutePath, "opensearch-ml")) + destinationDirectory = tmp_dir + archiveFileName = "opensearch-ml-${neural_search_bwc_version_no_qualifier}.zip" + doLast { + delete Path.of(tmp_dir.absolutePath, "opensearch-ml") + } +} + +// Task to unzip knn plugin from archive +task unZipBwcKnnPlugin(type: Zip) { + dependsOn "pullKnnBwcPlugin" + dependsOn "unZipBwcMlCommonsPlugin" + from(Path.of(tmp_dir.absolutePath, "opensearch-knn")) + destinationDirectory = tmp_dir + archiveFileName = "opensearch-knn-${neural_search_bwc_version_no_qualifier}.zip" + doLast { + delete Path.of(tmp_dir.absolutePath, "opensearch-knn") + } +} + +// Task to unzip neural search plugin from archive +task unZipBwcPlugin(type: Zip) { + dependsOn "unZipBwcKnnPlugin" + dependsOn "pullBwcPlugin" + from(Path.of(tmp_dir.absolutePath, "opensearch-neural-search")) + destinationDirectory = tmp_dir + archiveFileName = "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}.zip" + doLast { + delete Path.of(tmp_dir.absolutePath, "opensearch-neural-search") + } +} + + +task bwcTestSuite { + dependsOn ":qa:restart-upgrade:testAgainstNewCluster" + dependsOn ":qa:rolling-upgrade:testRollingUpgrade" +} diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle new file mode 100644 index 000000000..b817c1395 --- /dev/null +++ b/qa/restart-upgrade/build.gradle @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask + +apply from : "$rootDir/qa/build.gradle" + +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +String baseName = "neuralSearchBwcCluster-restart" + +// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion +testClusters { + "${baseName}" { + testDistribution = "ARCHIVE" + versions = [neural_search_bwc_version, opensearch_version] + numberOfNodes = 3 + jvmArgs("-Xms1g", "-Xmx4g") + plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile) + plugin(project.tasks.unZipBwcKnnPlugin.archiveFile) + plugin(project.tasks.unZipBwcPlugin.archiveFile) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' + } +} + +// Task to run BWC tests against the old cluster +task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { + dependsOn "unZipBwcPlugin" + useCluster testClusters."${baseName}" + systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' + systemProperty 'tests.is_old_cluster', 'true' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// All nodes are upgraded to latest version and run the tests +task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstOldCluster" + dependsOn rootProject.tasks.assemble + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".upgradeAllNodesAndPluginsToNextVersion(project.ext.plugins) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.is_old_cluster', 'false' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java new file mode 100644 index 000000000..cf985d759 --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRestartUpgradeRestTestCase.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.neuralsearch.bwc; + +import java.util.Locale; +import java.util.Optional; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; +import static org.opensearch.neuralsearch.TestUtils.CLIENT_TIMEOUT_VALUE; +import static org.opensearch.neuralsearch.TestUtils.RESTART_UPGRADE_OLD_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.BWC_VERSION; +import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +public abstract class AbstractRestartUpgradeRestTestCase extends BaseNeuralSearchIT { + + @Before + protected String getIndexNameForTest() { + // Creating index name by concatenating "neural-bwc-" prefix with test method name + // for all the tests in this sub-project + return NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); + } + + @Override + protected final boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected final boolean preserveReposUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected final Settings restClientSettings() { + return Settings.builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, CLIENT_TIMEOUT_VALUE) + .build(); + } + + protected static final boolean isRunningAgainstOldCluster() { + return Boolean.parseBoolean(System.getProperty(RESTART_UPGRADE_OLD_CLUSTER)); + } + + protected final Optional getBWCVersion() { + return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); + } +} diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java new file mode 100644 index 000000000..dc8a94236 --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -0,0 +1,87 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.neuralsearch.bwc; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; + +import static org.opensearch.neuralsearch.TestUtils.getModelId; +import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; +import org.opensearch.neuralsearch.query.NeuralQueryBuilder; + +public class SemanticSearchIT extends AbstractRestartUpgradeRestTestCase { + + private static final String PIPELINE_NAME = "nlp-pipeline"; + private static final String TEST_FIELD = "passage_text"; + private static final String TEXT = "Hello world"; + private static final String TEXT_1 = "Hello world a"; + + // Test restart-upgrade Semantic Search + // Create Text Embedding Processor, Ingestion Pipeline and add document + // Validate process , pipeline and document count in restart-upgrade scenario + public void testTextEmbeddingProcessor_E2EFlow() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()) { + String modelId = uploadTextEmbeddingModel(); + loadModel(modelId); + createPipelineProcessor(modelId, PIPELINE_NAME); + createIndexWithConfiguration( + getIndexNameForTest(), + Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), + PIPELINE_NAME + ); + addDocument(getIndexNameForTest(), "0", TEST_FIELD, TEXT); + } else { + Map pipeline = getIngestionPipeline(PIPELINE_NAME); + assertNotNull(pipeline); + String modelId = getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); + loadModel(modelId); + addDocument(getIndexNameForTest(), "1", TEST_FIELD, TEXT_1); + validateTestIndex(modelId); + deletePipeline(PIPELINE_NAME); + deleteModel(modelId); + deleteIndex(getIndexNameForTest()); + } + } + + private void validateTestIndex(String modelId) throws Exception { + int docCount = getDocCount(getIndexNameForTest()); + assertEquals(2, docCount); + loadModel(modelId); + NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); + neuralQueryBuilder.fieldName("passage_embedding"); + neuralQueryBuilder.modelId(modelId); + neuralQueryBuilder.queryText(TEXT); + neuralQueryBuilder.k(1); + Map response = search(getIndexNameForTest(), neuralQueryBuilder, 1); + assertNotNull(response); + } + + private String uploadTextEmbeddingModel() throws Exception { + String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); + return registerModelGroupAndGetModelId(requestBody); + } + + private String registerModelGroupAndGetModelId(String requestBody) throws Exception { + String modelGroupRegisterRequestBody = Files.readString( + Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) + ); + String modelGroupId = registerModelGroup( + String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) + ); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); + } + + protected void createPipelineProcessor(String modelId, String pipelineName) throws Exception { + String requestBody = Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); + createPipelineProcessor(requestBody, pipelineName, modelId); + } + +} diff --git a/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json b/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json new file mode 100644 index 000000000..2dd0852a4 --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -0,0 +1,4 @@ +{ + "name": "%s", + "description": "This is a public model group" +} diff --git a/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json new file mode 100644 index 000000000..fc4c13646 --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/processor/IndexMappings.json @@ -0,0 +1,32 @@ +{ + "settings": { + "index": { + "knn": true, + "knn.algo_param.ef_search": 100, + "refresh_interval": "30s", + "default_pipeline": "%s" + }, + "number_of_shards": 3, + "number_of_replicas": 1 + }, + "mappings": { + "properties": { + "passage_embedding": { + "type": "knn_vector", + "dimension": 768, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 128, + "m": 24 + } + } + }, + "passage_text": { + "type": "text" + } + } + } +} diff --git a/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json b/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json new file mode 100644 index 000000000..ee4e253a6 --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/processor/PipelineConfiguration.json @@ -0,0 +1,13 @@ +{ + "description": "text embedding pipeline for hybrid", + "processors": [ + { + "text_embedding": { + "model_id": "%s", + "field_map": { + "passage_text": "passage_embedding" + } + } + } + ] +} diff --git a/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json b/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json new file mode 100644 index 000000000..17fab7588 --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/processor/UploadModelRequestBody.json @@ -0,0 +1,15 @@ +{ + "name": "traced_small_model", + "version": "1.0.0", + "model_format": "TORCH_SCRIPT", + "model_task_type": "text_embedding", + "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", + "model_group_id": "%s", + "model_config": { + "model_type": "bert", + "embedding_dimension": 768, + "framework_type": "sentence_transformers", + "all_config": "{\"architectures\":[\"BertModel\"],\"max_position_embeddings\":512,\"model_type\":\"bert\",\"num_attention_heads\":12,\"num_hidden_layers\":6}" + }, + "url": "https://github.com/opensearch-project/ml-commons/blob/2.x/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/text_embedding/traced_small_model.zip?raw=true" +} diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle new file mode 100644 index 000000000..51f03695c --- /dev/null +++ b/qa/rolling-upgrade/build.gradle @@ -0,0 +1,92 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask + +apply from : "$rootDir/qa/build.gradle" + +String default_bwc_version = System.getProperty("bwc.version") +String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +String baseName = "neuralSearchBwcCluster-rolling" + +// Creates a test cluster of previous version and loads k-NN plugin of bwcVersion +testClusters { + "${baseName}" { + testDistribution = "ARCHIVE" + versions = [neural_search_bwc_version, opensearch_version] + numberOfNodes = 3 + plugin(project.tasks.unZipBwcMlCommonsPlugin.archiveFile) + plugin(project.tasks.unZipBwcKnnPlugin.archiveFile) + plugin(project.tasks.unZipBwcPlugin.archiveFile) + jvmArgs("-Xms1g", "-Xmx4g") + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' + } +} + + +// Task to run BWC tests against the old cluster +task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { + dependsOn "unZipBwcPlugin" + useCluster testClusters."${baseName}" + systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + systemProperty 'tests.skip_delete_model_index', 'true' + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// Part of rolling upgrade. Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version +// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. +task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}" + dependsOn rootProject.tasks.assemble + dependsOn "testAgainstOldCluster" + doFirst { + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' + systemProperty 'tests.rest.first_round', 'true' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// Part of rolling upgrade. Upgrades the second node to new OpenSearch version with upgraded plugin version after the +// first node is upgraded. This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. +task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstOneThirdUpgradedCluster" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) + } + systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' + systemProperty 'tests.rest.first_round', 'false' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} + +// Part of rolling upgrade. Upgrades the third node to new OpenSearch version with upgraded plugin version after the +// second node is upgraded. This results in a fully upgraded cluster. +task testRollingUpgrade(type: StandaloneRestIntegTestTask) { + dependsOn "testAgainstTwoThirdsUpgradedCluster" + useCluster testClusters."${baseName}" + doFirst { + testClusters."${baseName}".upgradeNodeAndPluginToNextVersion(project.ext.plugins) + } + mustRunAfter "testAgainstOneThirdUpgradedCluster" + systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' + systemProperty 'tests.skip_delete_model_index', 'true' + systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + systemProperty 'tests.security.manager', 'false' +} diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java new file mode 100644 index 000000000..98ce95b72 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/AbstractRollingUpgradeTestCase.java @@ -0,0 +1,87 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.neuralsearch.bwc; + +import java.util.Locale; +import java.util.Optional; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; +import org.opensearch.test.rest.OpenSearchRestTestCase; +import static org.opensearch.neuralsearch.TestUtils.OLD_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.MIXED_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.UPGRADED_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.BWC_VERSION; +import static org.opensearch.neuralsearch.TestUtils.ROLLING_UPGRADE_FIRST_ROUND; +import static org.opensearch.neuralsearch.TestUtils.BWCSUITE_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; + +public abstract class AbstractRollingUpgradeTestCase extends BaseNeuralSearchIT { + + @Before + protected String getIndexNameForTest() { + // Creating index name by concatenating "neural-bwc-" prefix with test method name + // for all the tests in this sub-project + return NEURAL_SEARCH_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT); + } + + @Override + protected final boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected final boolean preserveReposUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected final Settings restClientSettings() { + return Settings.builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, "120s") + .build(); + } + + protected enum ClusterType { + OLD, + MIXED, + UPGRADED; + + public static ClusterType instance(String value) { + switch (value) { + case OLD_CLUSTER: + return OLD; + case MIXED_CLUSTER: + return MIXED; + case UPGRADED_CLUSTER: + return UPGRADED; + default: + throw new IllegalArgumentException("unknown cluster type: " + value); + } + } + } + + protected final ClusterType getClusterType() { + return ClusterType.instance(System.getProperty(BWCSUITE_CLUSTER)); + } + + protected final boolean isFirstMixedRound() { + return Boolean.parseBoolean(System.getProperty(ROLLING_UPGRADE_FIRST_ROUND, "false")); + } + + protected final Optional getBWCVersion() { + return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); + } + +} diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java new file mode 100644 index 000000000..41d21aa3f --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/SemanticSearchIT.java @@ -0,0 +1,106 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.neuralsearch.bwc; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import org.opensearch.neuralsearch.TestUtils; +import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; +import org.opensearch.neuralsearch.query.NeuralQueryBuilder; + +public class SemanticSearchIT extends AbstractRollingUpgradeTestCase { + private static final String PIPELINE_NAME = "nlp-pipeline"; + private static final String TEST_FIELD = "passage_text"; + private static final String TEXT = "Hello world"; + private static final String TEXT_MIXED = "Hello world mixed"; + private static final String TEXT_UPGRADED = "Hello world upgraded"; + private static final int NUM_DOCS_PER_ROUND = 1; + + // Test rolling-upgrade Semantic Search + // Create Text Embedding Processor, Ingestion Pipeline and add document + // Validate process , pipeline and document count in rolling-upgrade scenario + public void testTextEmbeddingProcessor_E2EFlow() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + switch (getClusterType()) { + case OLD: + String modelId = uploadTextEmbeddingModel(); + loadModel(modelId); + createPipelineProcessor(modelId, PIPELINE_NAME); + createIndexWithConfiguration( + getIndexNameForTest(), + Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), + PIPELINE_NAME + ); + addDocument(getIndexNameForTest(), "0", TEST_FIELD, TEXT); + break; + case MIXED: + modelId = getModelId(PIPELINE_NAME); + int totalDocsCountMixed; + if (isFirstMixedRound()) { + totalDocsCountMixed = NUM_DOCS_PER_ROUND; + validateTestIndexOnUpgrade(totalDocsCountMixed, modelId, TEXT); + addDocument(getIndexNameForTest(), "1", TEST_FIELD, TEXT_MIXED); + + } else { + totalDocsCountMixed = 2 * NUM_DOCS_PER_ROUND; + validateTestIndexOnUpgrade(totalDocsCountMixed, modelId, TEXT_MIXED); + } + break; + case UPGRADED: + modelId = getModelId(PIPELINE_NAME); + int totalDocsCountUpgraded = 3 * NUM_DOCS_PER_ROUND; + loadModel(modelId); + addDocument(getIndexNameForTest(), "2", TEST_FIELD, TEXT_UPGRADED); + validateTestIndexOnUpgrade(totalDocsCountUpgraded, modelId, TEXT_UPGRADED); + deletePipeline(PIPELINE_NAME); + deleteModel(modelId); + deleteIndex(getIndexNameForTest()); + break; + } + + } + + private void validateTestIndexOnUpgrade(int numberOfDocs, String modelId, String text) throws Exception { + int docCount = getDocCount(getIndexNameForTest()); + assertEquals(numberOfDocs, docCount); + loadModel(modelId); + NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); + neuralQueryBuilder.fieldName("passage_embedding"); + neuralQueryBuilder.modelId(modelId); + neuralQueryBuilder.queryText(text); + neuralQueryBuilder.k(1); + Map response = search(getIndexNameForTest(), neuralQueryBuilder, 1); + assertNotNull(response); + } + + private String uploadTextEmbeddingModel() throws Exception { + String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); + return registerModelGroupAndGetModelId(requestBody); + } + + private String registerModelGroupAndGetModelId(String requestBody) throws Exception { + String modelGroupRegisterRequestBody = Files.readString( + Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) + ); + String modelGroupId = registerModelGroup( + String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) + ); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); + } + + protected void createPipelineProcessor(String modelId, String pipelineName) throws Exception { + String requestBody = Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI())); + createPipelineProcessor(requestBody, pipelineName, modelId); + } + + private String getModelId(String pipelineName) { + Map pipeline = getIngestionPipeline(pipelineName); + assertNotNull(pipeline); + return TestUtils.getModelId(pipeline, TEXT_EMBEDDING_PROCESSOR); + } +} diff --git a/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json b/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json new file mode 100644 index 000000000..2dd0852a4 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -0,0 +1,4 @@ +{ + "name": "%s", + "description": "This is a public model group" +} diff --git a/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json new file mode 100644 index 000000000..fc4c13646 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/IndexMappings.json @@ -0,0 +1,32 @@ +{ + "settings": { + "index": { + "knn": true, + "knn.algo_param.ef_search": 100, + "refresh_interval": "30s", + "default_pipeline": "%s" + }, + "number_of_shards": 3, + "number_of_replicas": 1 + }, + "mappings": { + "properties": { + "passage_embedding": { + "type": "knn_vector", + "dimension": 768, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 128, + "m": 24 + } + } + }, + "passage_text": { + "type": "text" + } + } + } +} diff --git a/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json b/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json new file mode 100644 index 000000000..ee4e253a6 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/PipelineConfiguration.json @@ -0,0 +1,13 @@ +{ + "description": "text embedding pipeline for hybrid", + "processors": [ + { + "text_embedding": { + "model_id": "%s", + "field_map": { + "passage_text": "passage_embedding" + } + } + } + ] +} diff --git a/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json b/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json new file mode 100644 index 000000000..17fab7588 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/UploadModelRequestBody.json @@ -0,0 +1,15 @@ +{ + "name": "traced_small_model", + "version": "1.0.0", + "model_format": "TORCH_SCRIPT", + "model_task_type": "text_embedding", + "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", + "model_group_id": "%s", + "model_config": { + "model_type": "bert", + "embedding_dimension": 768, + "framework_type": "sentence_transformers", + "all_config": "{\"architectures\":[\"BertModel\"],\"max_position_embeddings\":512,\"model_type\":\"bert\",\"num_attention_heads\":12,\"num_hidden_layers\":6}" + }, + "url": "https://github.com/opensearch-project/ml-commons/blob/2.x/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/text_embedding/traced_small_model.zip?raw=true" +} diff --git a/repositories.gradle b/repositories.gradle new file mode 100644 index 000000000..16f39204b --- /dev/null +++ b/repositories.gradle @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} diff --git a/settings.gradle b/settings.gradle index 57fb25f3c..232fd316c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -9,3 +9,7 @@ */ rootProject.name = 'neural-search' + +include ":qa" +include ":qa:rolling-upgrade" +include ":qa:restart-upgrade" diff --git a/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java b/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java index 2198f7e9e..bfbb2e6d9 100644 --- a/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/common/VectorUtil.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.common; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java index b9a95dea8..e12211d28 100644 --- a/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java +++ b/src/main/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessor.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.ml; import static org.opensearch.neuralsearch.processor.TextImageEmbeddingProcessor.INPUT_IMAGE; diff --git a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java index f244419b3..aacb8d2e6 100644 --- a/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java +++ b/src/main/java/org/opensearch/neuralsearch/plugin/NeuralSearch.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.plugin; import static org.opensearch.neuralsearch.settings.NeuralSearchSettings.NEURAL_SEARCH_HYBRID_SEARCH_DISABLED; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java b/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java index b7aa43c64..9d3c3adb5 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.search.util.HybridSearchResultFormatUtil.isHybridQueryDelimiterElement; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java index e66c3f7f3..c34cd0b42 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.util.ArrayList; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java index c95e0f526..b20b889cb 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessor.java @@ -2,11 +2,9 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; -import static org.opensearch.ingest.ConfigurationUtils.*; -import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.TYPE; +import static org.opensearch.ingest.ConfigurationUtils.readOptionalStringProperty; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java index 0d6742dbe..0563c92a0 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessor.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.search.util.HybridSearchResultFormatUtil.isHybridQueryStartStopElement; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java index c322102d5..a20b52517 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflow.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.util.Arrays; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java index d1433e983..8acf95bf7 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessor.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java index 3e87188e8..c1b8f92a6 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessor.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java index 9a252bace..09fcf3d97 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessor.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.util.HashMap; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java index e656beca3..001f1670d 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java index 2a78d5ac6..c4b6dfb3f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java index 0b45fb616..f5195f79f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java index f05d24823..23d8e01be 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactory.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java index 6e0a5db65..dbeabe94b 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; public interface ScoreCombinationTechnique { diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java index c92abba4d..a915057df 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtil.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import java.util.Arrays; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java index 22745c50a..c9e0551e2 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombiner.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import java.util.ArrayList; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java index 92ab2b1df..0af46a4a4 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.factory; import static org.opensearch.ingest.ConfigurationUtils.readOptionalMap; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java index 609ee07be..75e00e85a 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/SparseEncodingProcessorFactory.java @@ -2,12 +2,13 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.factory; import static org.opensearch.ingest.ConfigurationUtils.readMap; import static org.opensearch.ingest.ConfigurationUtils.readStringProperty; -import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.*; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.TYPE; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.MODEL_ID_FIELD; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.FIELD_MAP_FIELD; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java index adf6f6d21..7802cb1f6 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextEmbeddingProcessorFactory.java @@ -2,12 +2,13 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.factory; import static org.opensearch.ingest.ConfigurationUtils.readMap; import static org.opensearch.ingest.ConfigurationUtils.readStringProperty; -import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.*; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.TYPE; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.MODEL_ID_FIELD; +import static org.opensearch.neuralsearch.processor.TextEmbeddingProcessor.FIELD_MAP_FIELD; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java index 3820d84c7..b98f4fcc0 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactory.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.factory; import static org.opensearch.ingest.ConfigurationUtils.readMap; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java index 7f7d18546..2bb6bbed7 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.ArrayList; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java index bd1112840..4fdf3c0a6 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.Arrays; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java index 667c237c7..ca6ad20d6 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactory.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java index 8dd124804..0b784c678 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java index e2e3385eb..263115f8f 100644 --- a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java +++ b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizer.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java index 2c79c56e5..8846f6977 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQuery.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java index a0019c826..b0104639b 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java index e3e6a0862..5abfd0b5e 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryScorer.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java index 605892ea0..69ee5015f 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java +++ b/src/main/java/org/opensearch/neuralsearch/query/HybridQueryWeight.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java index 2e0e06adc..cda01767e 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.opensearch.knn.index.query.KNNQueryBuilder.FILTER_FIELD; diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java index 039d6a319..226594a87 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java index 54b478bdc..9dab0a695 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java +++ b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query.visitor; import java.util.Map; diff --git a/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java b/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java index dea9c6bae..1299537bb 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java +++ b/src/main/java/org/opensearch/neuralsearch/search/HitsThresholdChecker.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.search; import java.util.Locale; diff --git a/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java b/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java index 556f09bfd..8b7a12d29 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java +++ b/src/main/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollector.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.search; import java.io.IOException; diff --git a/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java b/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java index 539e0e22e..bf05fdc9d 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java +++ b/src/main/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcher.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.search.query; import static org.opensearch.neuralsearch.search.util.HybridSearchResultFormatUtil.createDelimiterElementForHybridSearchResults; diff --git a/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java b/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java index b345a105a..162647b11 100644 --- a/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtil.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.search.util; import java.util.Objects; diff --git a/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java b/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java index 4ac88f75f..bf887830d 100644 --- a/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java +++ b/src/main/java/org/opensearch/neuralsearch/settings/NeuralSearchSettings.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.settings; import org.opensearch.common.settings.Setting; diff --git a/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java b/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java index 1f7dd9d4a..d7d77b823 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtil.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.util; import org.opensearch.Version; diff --git a/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java b/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java index 5c1d486d4..d638fb9c8 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/RetryUtil.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.util; import java.util.List; diff --git a/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java b/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java index 76ce0fa16..e36b42cd6 100644 --- a/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java +++ b/src/main/java/org/opensearch/neuralsearch/util/TokenWeightUtil.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.util; import java.util.ArrayList; diff --git a/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java b/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java index 38915e4b5..58f05fd07 100644 --- a/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java +++ b/src/test/java/org/opensearch/neuralsearch/NeuralSearchIT.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch; import java.io.IOException; diff --git a/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java b/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java index 905b32aaa..3220ce8c6 100644 --- a/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java +++ b/src/test/java/org/opensearch/neuralsearch/NeuralSearchTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch; import org.opensearch.knn.common.KNNConstants; diff --git a/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java b/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java index b277f9a8f..ba9521303 100644 --- a/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/ValidateDependentPluginInstallationIT.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch; import java.io.BufferedReader; diff --git a/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java b/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java index 1eba51e4f..a06e8f84d 100644 --- a/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/common/VectorUtilTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.common; import java.util.Collections; diff --git a/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java b/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java index 185934b07..3cf04bb05 100644 --- a/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java +++ b/src/test/java/org/opensearch/neuralsearch/constants/TestCommonConstants.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.constants; import java.util.List; diff --git a/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java b/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java index 68d5f79eb..d9b816597 100644 --- a/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/ml/MLCommonsClientAccessorTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.ml; import static org.mockito.Mockito.mock; diff --git a/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java b/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java index 32901cf12..2a66f6992 100644 --- a/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java +++ b/src/test/java/org/opensearch/neuralsearch/plugin/NeuralSearchTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.plugin; import static org.mockito.Mockito.mock; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java b/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java index a5bdda1e3..0096f7f94 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/CompoundTopDocsTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java index f3db15754..3c8adf5b3 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java @@ -2,9 +2,10 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import java.util.Collections; @@ -13,8 +14,7 @@ import org.junit.After; import org.junit.Before; import org.opensearch.common.settings.Settings; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; import com.google.common.primitives.Floats; @@ -27,8 +27,6 @@ public class NeuralQueryEnricherProcessorIT extends BaseNeuralSearchIT { private static final String search_pipeline = "search-pipeline"; private static final String ingest_pipeline = "nlp-pipeline"; private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private final float[] testVector = createRandomVector(TEST_DIMENSION); @Before diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java index f6de3e58d..df5c4ad60 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.util.Collections; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java index a1cc58447..5a48b9938 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorIT.java @@ -2,9 +2,11 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; +import static org.opensearch.neuralsearch.TestUtils.RELATION_EQUAL_TO; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import java.io.IOException; @@ -21,8 +23,7 @@ import org.junit.Before; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.HybridQueryBuilder; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -47,14 +48,11 @@ public class NormalizationProcessorIT extends BaseNeuralSearchIT { private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field-1"; private static final String TEST_TEXT_FIELD_NAME_2 = "test-text-field-2"; - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private static final String SEARCH_PIPELINE = "phase-results-pipeline"; private final float[] testVector1 = createRandomVector(TEST_DIMENSION); private final float[] testVector2 = createRandomVector(TEST_DIMENSION); private final float[] testVector3 = createRandomVector(TEST_DIMENSION); private final float[] testVector4 = createRandomVector(TEST_DIMENSION); - private final static String RELATION_EQUAL_TO = "eq"; @Before public void setUp() throws Exception { diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java index 26d9fc808..57698cd7e 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import static org.hamcrest.Matchers.startsWith; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java index 95c2ba0c2..f34f8f59b 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NormalizationProcessorWorkflowTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import static org.mockito.Mockito.spy; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java index ae146311b..d6467b065 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationIT.java @@ -2,11 +2,12 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.assertHybridSearchResults; import static org.opensearch.neuralsearch.TestUtils.assertWeightedScores; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; @@ -20,8 +21,7 @@ import org.junit.Before; import org.opensearch.client.ResponseException; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.HybridQueryBuilder; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -42,8 +42,6 @@ public class ScoreCombinationIT extends BaseNeuralSearchIT { private static final String TEST_DOC_TEXT5 = "Say hello and enter my friend"; private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field-1"; - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private static final String SEARCH_PIPELINE = "phase-results-pipeline"; private final float[] testVector1 = createRandomVector(TEST_DIMENSION); private final float[] testVector2 = createRandomVector(TEST_DIMENSION); diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java index 78c4e1139..d2c1ddb4f 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreCombinationTechniqueTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import static org.opensearch.neuralsearch.TestUtils.DELTA_FOR_SCORE_ASSERTION; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java index 60f5518f2..9f83a3608 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java @@ -2,9 +2,10 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.assertHybridSearchResults; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; @@ -16,8 +17,7 @@ import org.junit.After; import org.junit.Before; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.HybridQueryBuilder; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -34,8 +34,6 @@ public class ScoreNormalizationIT extends BaseNeuralSearchIT { private static final String TEST_DOC_TEXT4 = "Hello, I'm glad to you see you pal"; private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field-1"; - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private static final String SEARCH_PIPELINE = "phase-results-pipeline"; private final float[] testVector1 = createRandomVector(TEST_DIMENSION); private final float[] testVector2 = createRandomVector(TEST_DIMENSION); diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java index 123e57e74..34f0af48c 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationTechniqueTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.util.List; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java index 78207c6e4..416707956 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessIT.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.nio.file.Files; @@ -16,7 +15,7 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.neuralsearch.common.BaseSparseEncodingIT; +import org.opensearch.neuralsearch.BaseSparseEncodingIT; import com.google.common.collect.ImmutableList; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java index 7ddbfcc21..815ea851b 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/SparseEncodingProcessorTests.java @@ -2,14 +2,27 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.any; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import static org.mockito.Mockito.verify; -import java.util.*; +import java.util.Map; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + import java.util.function.BiConsumer; import java.util.stream.IntStream; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java index 7aad49b03..97d1c857c 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorIT.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.nio.file.Files; @@ -16,7 +15,7 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import com.google.common.collect.ImmutableList; @@ -50,7 +49,7 @@ public void testTextEmbeddingProcessor() throws Exception { private String uploadTextEmbeddingModel() throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - return uploadModel(requestBody); + return registerModelGroupAndUploadModel(requestBody); } private void createTextEmbeddingIndex() throws Exception { diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java index f1936d812..8c2f1c1be 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java @@ -2,12 +2,18 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.isNull; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.isA; import java.util.ArrayList; import java.util.HashMap; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java index 77b0abbc6..f3cd65694 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorIT.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import java.nio.file.Files; @@ -16,7 +15,7 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import com.google.common.collect.ImmutableList; @@ -58,7 +57,7 @@ public void testEmbeddingProcessor_whenIngestingDocumentWithSourceWithoutMatchin private String uploadModel() throws Exception { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - return uploadModel(requestBody); + return registerModelGroupAndUploadModel(requestBody); } private void createTextImageEmbeddingIndex() throws Exception { diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java index 604e4d075..431d6a440 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/TextImageEmbeddingProcessorTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor; import static org.mockito.ArgumentMatchers.anyMap; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java index 125930007..7d3b3fb61 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechniqueTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import static org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique.PARAM_NAME_WEIGHTS; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java index 6697ccd3e..ed06b0876 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/BaseScoreCombinationTechniqueTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java index 3f70c229f..495e2f4cd 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechniqueTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import static org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique.PARAM_NAME_WEIGHTS; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java index 7b1b07f64..0c6e1f81d 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechniqueTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import static org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique.PARAM_NAME_WEIGHTS; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java index ee46bf0a1..b36a6b492 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationFactoryTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java index ca13cb2eb..9e00e3833 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationUtilTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.combination; import static org.hamcrest.Matchers.allOf; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java index efd1cce4f..9895d5b97 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactoryTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.factory; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java index 564373315..fa91d61a5 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/factory/TextImageEmbeddingProcessorFactoryTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.factory; import static org.hamcrest.Matchers.allOf; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java index 5732687ca..bccc0820a 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechniqueTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java index 1b38b7bb4..1fe1edf25 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechniqueTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.normalization; import java.util.List; diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java index bf1489fe3..d9dcd5540 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationFactoryTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.processor.normalization; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java index 5e53dd69d..7bae96780 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java index 864ebdc68..7e80d7fda 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryIT.java @@ -2,13 +2,15 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.opensearch.index.query.QueryBuilders.matchQuery; import static org.opensearch.neuralsearch.TestUtils.DELTA_FOR_SCORE_ASSERTION; +import static org.opensearch.neuralsearch.TestUtils.RELATION_EQUAL_TO; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import java.io.IOException; @@ -29,8 +31,7 @@ import org.opensearch.index.query.NestedQueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import com.google.common.primitives.Floats; @@ -55,9 +56,6 @@ public class HybridQueryIT extends BaseNeuralSearchIT { private static final String TEST_KNN_VECTOR_FIELD_NAME_2 = "test-knn-vector-2"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field-1"; private static final String TEST_NESTED_TYPE_FIELD_NAME_1 = "user"; - - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private static final String NESTED_FIELD_1 = "firstname"; private static final String NESTED_FIELD_2 = "lastname"; private static final String NESTED_FIELD_1_VALUE = "john"; @@ -65,7 +63,6 @@ public class HybridQueryIT extends BaseNeuralSearchIT { private final float[] testVector1 = createRandomVector(TEST_DIMENSION); private final float[] testVector2 = createRandomVector(TEST_DIMENSION); private final float[] testVector3 = createRandomVector(TEST_DIMENSION); - private final static String RELATION_EQUAL_TO = "eq"; private static final String SEARCH_PIPELINE = "phase-results-pipeline"; @Before diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java index 3777c2d14..1a2e3f26e 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryScorerTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java index a1d632755..dfbc97b69 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java index 0b9af2bcd..10d480475 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/HybridQueryWeightTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.mockito.ArgumentMatchers.eq; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java index 068be51ef..dd63abbea 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryBuilderTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.mockito.ArgumentMatchers.any; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java index 5e9f5834a..47f5cbb06 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralQueryIT.java @@ -2,10 +2,11 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.opensearch.neuralsearch.TestUtils.DELTA_FOR_SCORE_ASSERTION; +import static org.opensearch.neuralsearch.TestUtils.TEST_DIMENSION; +import static org.opensearch.neuralsearch.TestUtils.TEST_SPACE_TYPE; import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import static org.opensearch.neuralsearch.TestUtils.objectToFloat; @@ -18,8 +19,7 @@ import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.common.BaseNeuralSearchIT; +import org.opensearch.neuralsearch.BaseNeuralSearchIT; import com.google.common.primitives.Floats; @@ -37,9 +37,6 @@ public class NeuralQueryIT extends BaseNeuralSearchIT { private static final String TEST_KNN_VECTOR_FIELD_NAME_2 = "test-knn-vector-2"; private static final String TEST_TEXT_FIELD_NAME_1 = "test-text-field"; private static final String TEST_KNN_VECTOR_FIELD_NAME_NESTED = "nested.knn.field"; - - private static final int TEST_DIMENSION = 768; - private static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; private final float[] testVector = createRandomVector(TEST_DIMENSION); @Before diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java index e95794b1c..2220ce326 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.mockito.ArgumentMatchers.any; diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java index 3f95d5638..afabd46c9 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryIT.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static org.opensearch.neuralsearch.TestUtils.objectToFloat; @@ -16,8 +15,8 @@ import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.neuralsearch.BaseSparseEncodingIT; import org.opensearch.neuralsearch.TestUtils; -import org.opensearch.neuralsearch.common.BaseSparseEncodingIT; import lombok.SneakyThrows; diff --git a/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java b/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java index 5b73e12ec..97e1f7c5d 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java +++ b/src/test/java/org/opensearch/neuralsearch/query/OpenSearchQueryTestCase.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query; import static java.util.Collections.emptyMap; diff --git a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java index 7570ece54..e513ab035 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.query.visitor; import java.util.HashMap; diff --git a/src/test/java/org/opensearch/neuralsearch/search/HitsThresholdCheckerTests.java b/src/test/java/org/opensearch/neuralsearch/search/HitsThresholdCheckerTests.java index 3ce9e3dfe..4b4938cc4 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/HitsThresholdCheckerTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/HitsThresholdCheckerTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.search; import java.util.stream.IntStream; diff --git a/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java b/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java index 72cf8be49..b67a1ee05 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/HybridTopScoreDocCollectorTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.search; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; diff --git a/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java b/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java index 55a16c4bb..e609eec05 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/query/HybridQueryPhaseSearcherTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.search.query; import static org.hamcrest.Matchers.containsString; diff --git a/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java b/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java index 65971b6d6..16f2f10ce 100644 --- a/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/search/util/HybridSearchResultFormatUtilTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.search.util; import static org.junit.Assert.assertEquals; diff --git a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java index 30399cfea..6219a26b5 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java +++ b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterTestUtils.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.util; import static org.mockito.Mockito.mock; diff --git a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java index f85be25d5..07975bd7c 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/util/NeuralSearchClusterUtilTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.util; import static org.opensearch.neuralsearch.util.NeuralSearchClusterTestUtils.mockClusterService; diff --git a/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java b/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java index a4bc2c495..887d8fc17 100644 --- a/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java +++ b/src/test/java/org/opensearch/neuralsearch/util/TokenWeightUtilTests.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.util; import java.util.List; diff --git a/src/test/resources/processor/CreateModelGroupRequestBody.json b/src/test/resources/processor/CreateModelGroupRequestBody.json index 91f68e222..2dd0852a4 100644 --- a/src/test/resources/processor/CreateModelGroupRequestBody.json +++ b/src/test/resources/processor/CreateModelGroupRequestBody.json @@ -1,4 +1,4 @@ { - "name": "", + "name": "%s", "description": "This is a public model group" -} \ No newline at end of file +} diff --git a/src/test/resources/processor/IndexMappings.json b/src/test/resources/processor/IndexMappings.json index 02de6c6af..13faad6c4 100644 --- a/src/test/resources/processor/IndexMappings.json +++ b/src/test/resources/processor/IndexMappings.json @@ -87,4 +87,3 @@ } } } - diff --git a/src/test/resources/processor/SearchRequestPipelineConfiguration.json b/src/test/resources/processor/SearchRequestPipelineConfiguration.json index 44d3b3ef0..5cd9988cb 100644 --- a/src/test/resources/processor/SearchRequestPipelineConfiguration.json +++ b/src/test/resources/processor/SearchRequestPipelineConfiguration.json @@ -8,4 +8,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/test/resources/processor/SparseEncodingIndexMappings.json b/src/test/resources/processor/SparseEncodingIndexMappings.json index 87dee278e..9748e8f3d 100644 --- a/src/test/resources/processor/SparseEncodingIndexMappings.json +++ b/src/test/resources/processor/SparseEncodingIndexMappings.json @@ -23,4 +23,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/test/resources/processor/SparseEncodingPipelineConfiguration.json b/src/test/resources/processor/SparseEncodingPipelineConfiguration.json index 82d13c8fe..04a4baf80 100644 --- a/src/test/resources/processor/SparseEncodingPipelineConfiguration.json +++ b/src/test/resources/processor/SparseEncodingPipelineConfiguration.json @@ -15,4 +15,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/test/resources/processor/UploadModelRequestBody.json b/src/test/resources/processor/UploadModelRequestBody.json index 95f9c9cb5..17fab7588 100644 --- a/src/test/resources/processor/UploadModelRequestBody.json +++ b/src/test/resources/processor/UploadModelRequestBody.json @@ -4,7 +4,7 @@ "model_format": "TORCH_SCRIPT", "model_task_type": "text_embedding", "model_content_hash_value": "e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021", - "model_group_id": "", + "model_group_id": "%s", "model_config": { "model_type": "bert", "embedding_dimension": 768, diff --git a/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json b/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json index c45334bae..5c2a73f6e 100644 --- a/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json +++ b/src/test/resources/processor/UploadSparseEncodingModelRequestBody.json @@ -4,7 +4,7 @@ "function_name": "SPARSE_TOKENIZE", "description": "test model", "model_format": "TORCH_SCRIPT", - "model_group_id": "", + "model_group_id": "%s", "model_content_hash_value": "b345e9e943b62c405a8dd227ef2c46c84c5ff0a0b71b584be9132b37bce91a9a", "url": "https://github.com/opensearch-project/ml-commons/raw/main/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/sparse_encoding/sparse_demo.zip" -} \ No newline at end of file +} diff --git a/src/test/java/org/opensearch/neuralsearch/common/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java similarity index 89% rename from src/test/java/org/opensearch/neuralsearch/common/BaseNeuralSearchIT.java rename to src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 1e8164dee..680d90b65 100644 --- a/src/test/java/org/opensearch/neuralsearch/common/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -2,8 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - -package org.opensearch.neuralsearch.common; +package org.opensearch.neuralsearch; import static org.opensearch.neuralsearch.common.VectorUtil.vectorAsListToArray; @@ -26,6 +25,7 @@ import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.ParseException; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; @@ -45,7 +45,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.index.SpaceType; -import org.opensearch.neuralsearch.OpenSearchSecureRestTestCase; import org.opensearch.neuralsearch.util.NeuralSearchClusterUtil; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.threadpool.TestThreadPool; @@ -135,11 +134,13 @@ protected void updateClusterSettings(String settingKey, Object value) { assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } - protected String uploadModel(String requestBody) throws Exception { - String modelGroupId = registerModelGroup(); + protected String registerModelGroupAndUploadModel(String requestBody) throws Exception { + String modelGroupId = getModelGroupId(); // model group id is dynamically generated, we need to update model update request body after group is registered - requestBody = requestBody.replace("", modelGroupId); + return uploadModel(String.format(LOCALE, requestBody, modelGroupId)); + } + protected String uploadModel(String requestBody) throws Exception { Response uploadResponse = makeRequest( client(), "POST", @@ -202,7 +203,7 @@ protected void loadModel(String modelId) throws Exception { @SneakyThrows protected String prepareModel() { String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI())); - String modelId = uploadModel(requestBody); + String modelId = registerModelGroupAndUploadModel(requestBody); loadModel(modelId); return modelId; } @@ -270,18 +271,17 @@ protected void createPipelineProcessor(String modelId, String pipelineName) thro } protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception { + String requestBody = Files.readString(Path.of(classLoader.getResource(PIPELINE_CONFIGS_BY_TYPE.get(processorType)).toURI())); + createPipelineProcessor(requestBody, pipelineName, modelId); + } + + protected void createPipelineProcessor(String requestBody, String pipelineName, String modelId) throws Exception { Response pipelineCreateResponse = makeRequest( client(), "PUT", "/_ingest/pipeline/" + pipelineName, null, - toHttpEntity( - String.format( - LOCALE, - Files.readString(Path.of(classLoader.getResource(PIPELINE_CONFIGS_BY_TYPE.get(processorType)).toURI())), - modelId - ) - ), + toHttpEntity(String.format(LOCALE, requestBody, modelId)), ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, DEFAULT_USER_AGENT)) ); Map node = XContentHelper.convertToMap( @@ -790,10 +790,16 @@ protected String getDeployedModelId() { } @SneakyThrows - private String registerModelGroup() { + private String getModelGroupId() { String modelGroupRegisterRequestBody = Files.readString( Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI()) - ).replace("", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)); + ); + return registerModelGroup( + String.format(LOCALE, modelGroupRegisterRequestBody, "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8)) + ); + } + + protected String registerModelGroup(String modelGroupRegisterRequestBody) throws IOException, ParseException { Response modelGroupResponse = makeRequest( client(), "POST", @@ -812,6 +818,63 @@ private String registerModelGroup() { return modelGroupId; } + // Method that waits till the health of nodes in the cluster goes green + protected void waitForClusterHealthGreen(String numOfNodes) throws IOException { + Request waitForGreen = new Request("GET", "/_cluster/health"); + waitForGreen.addParameter("wait_for_nodes", numOfNodes); + waitForGreen.addParameter("wait_for_status", "green"); + client().performRequest(waitForGreen); + } + + /** + * Add a single Doc to an index + * + * @param index name of the index + * @param docId + * @param fieldName name of the field + * @param text to be added + */ + protected void addDocument(String index, String docId, String fieldName, String text) throws IOException { + Request request = new Request("PUT", "/" + index + "/_doc/" + docId + "?refresh=true"); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, text).endObject(); + request.setJsonEntity(builder.toString()); + client().performRequest(request); + } + + /** + * Get ingest pipeline + * @param pipelineName of the ingest pipeline + * + * @return get pipeline response as a map object + */ + @SneakyThrows + protected Map getIngestionPipeline(String pipelineName) { + Request request = new Request("GET", "/_ingest/pipeline/" + pipelineName); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + String responseBody = EntityUtils.toString(response.getEntity()); + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + return (Map) responseMap.get(pipelineName); + } + + /** + * Delete pipeline + * + * @param pipelineName of the pipeline + * + * @return delete pipeline response as a map object + */ + @SneakyThrows + protected Map deletePipeline(String pipelineName) { + Request request = new Request("DELETE", "/_ingest/pipeline/" + pipelineName); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + String responseBody = EntityUtils.toString(response.getEntity()); + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + return responseMap; + } + /** * Enumeration for types of pipeline processors, used to lookup resources like create * processor request as those are type specific diff --git a/src/test/java/org/opensearch/neuralsearch/common/BaseSparseEncodingIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java similarity index 98% rename from src/test/java/org/opensearch/neuralsearch/common/BaseSparseEncodingIT.java rename to src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java index 46fb807ce..3072bc767 100644 --- a/src/test/java/org/opensearch/neuralsearch/common/BaseSparseEncodingIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseSparseEncodingIT.java @@ -2,8 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - -package org.opensearch.neuralsearch.common; +package org.opensearch.neuralsearch; import java.nio.file.Files; import java.nio.file.Path; @@ -35,7 +34,7 @@ protected String prepareModel() { String requestBody = Files.readString( Path.of(classLoader.getResource("processor/UploadSparseEncodingModelRequestBody.json").toURI()) ); - String modelId = uploadModel(requestBody); + String modelId = registerModelGroupAndUploadModel(requestBody); loadModel(modelId); return modelId; } diff --git a/src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java similarity index 83% rename from src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java rename to src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java index 07f9e53d7..133f42daf 100644 --- a/src/test/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/OpenSearchSecureRestTestCase.java @@ -2,17 +2,23 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; +import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; +import static org.opensearch.neuralsearch.TestUtils.NEURAL_SEARCH_BWC_PREFIX; +import static org.opensearch.neuralsearch.TestUtils.OPENDISTRO_SECURITY; +import static org.opensearch.neuralsearch.TestUtils.OPENSEARCH_SYSTEM_INDEX_PREFIX; +import static org.opensearch.neuralsearch.TestUtils.SECURITY_AUDITLOG_PREFIX; +import static org.opensearch.neuralsearch.TestUtils.SKIP_DELETE_MODEL_INDEX; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import org.apache.hc.client5.http.auth.AuthScope; @@ -24,6 +30,7 @@ import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.ParseException; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.ssl.SSLContextBuilder; @@ -58,6 +65,12 @@ public abstract class OpenSearchSecureRestTestCase extends OpenSearchRestTestCas private static final String INTERNAL_INDICES_PREFIX = "."; private static String protocol; + private final Set IMMUTABLE_INDEX_PREFIXES = Set.of( + NEURAL_SEARCH_BWC_PREFIX, + SECURITY_AUDITLOG_PREFIX, + OPENSEARCH_SYSTEM_INDEX_PREFIX + ); + @Override protected String getProtocol() { if (protocol == null) { @@ -148,7 +161,7 @@ protected boolean preserveIndicesUponCompletion() { } @After - public void deleteExternalIndices() throws IOException { + public void deleteExternalIndices() throws IOException, ParseException { final Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json" + "&expand_wildcards=all")); final MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType()); try ( @@ -174,8 +187,29 @@ public void deleteExternalIndices() throws IOException { .collect(Collectors.toList()); for (final String indexName : externalIndices) { - adminClient().performRequest(new Request("DELETE", "/" + indexName)); + if (!skipDeleteIndex(indexName)) { + adminClient().performRequest(new Request("DELETE", "/" + indexName)); + } } } } + + private boolean getSkipDeleteModelIndexFlag() { + return Boolean.parseBoolean(System.getProperty(SKIP_DELETE_MODEL_INDEX, "false")); + } + + private boolean skipDeleteModelIndex(String indexName) { + return (MODEL_INDEX_NAME.equals(indexName) && getSkipDeleteModelIndexFlag()); + } + + private boolean skipDeleteIndex(String indexName) { + if (indexName != null + && !OPENDISTRO_SECURITY.equals(indexName) + && IMMUTABLE_INDEX_PREFIXES.stream().noneMatch(indexName::startsWith) + && !skipDeleteModelIndex(indexName)) { + return false; + } + + return true; + } } diff --git a/src/test/java/org/opensearch/neuralsearch/TestUtils.java b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java similarity index 88% rename from src/test/java/org/opensearch/neuralsearch/TestUtils.java rename to src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java index 385855a2e..99e396395 100644 --- a/src/test/java/org/opensearch/neuralsearch/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/TestUtils.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch; import static org.junit.Assert.assertEquals; @@ -27,6 +26,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.index.SpaceType; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.fetch.FetchSearchResult; @@ -34,8 +34,25 @@ public class TestUtils { - private static final String RELATION_EQUAL_TO = "eq"; + public static final String RELATION_EQUAL_TO = "eq"; public static final float DELTA_FOR_SCORE_ASSERTION = 0.001f; + public static final String RESTART_UPGRADE_OLD_CLUSTER = "tests.is_old_cluster"; + public static final String BWC_VERSION = "tests.plugin_bwc_version"; + public static final String NEURAL_SEARCH_BWC_PREFIX = "neuralsearch-bwc-"; + public static final String OLD_CLUSTER = "old_cluster"; + public static final String MIXED_CLUSTER = "mixed_cluster"; + public static final String UPGRADED_CLUSTER = "upgraded_cluster"; + public static final String ROLLING_UPGRADE_FIRST_ROUND = "tests.rest.first_round"; + public static final String BWCSUITE_CLUSTER = "tests.rest.bwcsuite_cluster"; + public static final String NODES_BWC_CLUSTER = "3"; + public static final int TEST_DIMENSION = 768; + public static final SpaceType TEST_SPACE_TYPE = SpaceType.L2; + public static final String CLIENT_TIMEOUT_VALUE = "90s"; + public static final String OPENDISTRO_SECURITY = ".opendistro_security"; + public static final String SKIP_DELETE_MODEL_INDEX = "tests.skip_delete_model_index"; + public static final String SECURITY_AUDITLOG_PREFIX = "security-auditlog"; + public static final String OPENSEARCH_SYSTEM_INDEX_PREFIX = ".opensearch"; + public static final String TEXT_EMBEDDING_PROCESSOR = "text_embedding"; /** * Convert an xContentBuilder to a map @@ -285,4 +302,12 @@ private static Optional getMaxScore(Map searchResponseAsM Map hitsMap = (Map) searchResponseAsMap.get("hits"); return hitsMap.get("max_score") == null ? Optional.empty() : Optional.of(((Double) hitsMap.get("max_score")).floatValue()); } + + public static String getModelId(Map pipeline, String processor) { + ArrayList> processors = (ArrayList>) pipeline.get("processors"); + + Map textEmbeddingProcessor = (Map) processors.get(0).get(processor); + + return (String) textEmbeddingProcessor.get("model_id"); + } }